From 862a4f2e3b83d5ed5e50567d401a1c9895247fb1 Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Fri, 29 Aug 2025 10:49:30 +0200 Subject: [PATCH] Hallucinate tags and automatically generating tags via ollama Holy shit what a fat change !!!! --- src/pages/Index.tsx | 357 ++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 345 insertions(+), 12 deletions(-) diff --git a/src/pages/Index.tsx b/src/pages/Index.tsx index 2deac22..9b336ba 100644 --- a/src/pages/Index.tsx +++ b/src/pages/Index.tsx @@ -31,6 +31,7 @@ interface Note { topLetter: string; topLetterFrequency: number; letterCount: number; + tags?: string[]; snippet?: string; isProblematic?: boolean; problemReason?: string; @@ -47,6 +48,10 @@ const MEILISEARCH_API_KEY = '31qXgGbQ3lT4DYHQ0TOKpMzh7wcigs7agHyxP5Fz6T6D61xsvNr const NOTE_INDEX = 'notes'; const SCRATCH_INDEX = 'scratch'; const SETTINGS_INDEX = 'settings'; + +// Ollama configuration +const OLLAMA_ENDPOINT = 'http://localhost:11434'; +const OLLAMA_MODEL = 'gemma3:4b-it-qat'; const meiliHeaders = { 'Content-Type': 'application/json', 'Authorization': `Bearer ${MEILISEARCH_API_KEY}`, @@ -60,6 +65,7 @@ const mapHitToNote = (hit: any): Note => ({ topLetter: hit.topLetter, topLetterFrequency: hit.topLetterFrequency, letterCount: hit.letterCount || 0, + tags: hit.tags || [], }); const Index = () => { @@ -88,6 +94,8 @@ const Index = () => { const [cacheMode, setCacheMode] = useState<'global' | 'scoped'>('global'); const [debugInfo, setDebugInfo] = useState([]); const [showDebugPanel, setShowDebugPanel] = useState(false); + const [autoGenerateTags, setAutoGenerateTags] = useState(true); + const [ollamaStatus, setOllamaStatus] = useState<'unknown' | 'online' | 'offline'>('unknown'); const { resolvedTheme, setTheme } = useTheme(); @@ -207,6 +215,113 @@ const Index = () => { return { mostFrequentLetter, mostFrequentLetterFrequency, letterCount }; }; + // Check Ollama status + const checkOllamaStatus = async () => { + try { + const response = await fetch(`${OLLAMA_ENDPOINT}/api/tags`, { + method: 'GET', + headers: { + 'Content-Type': 'application/json', + }, + }); + + if (response.ok) { + setOllamaStatus('online'); + return true; + } else { + setOllamaStatus('offline'); + return false; + } + } catch (error) { + setOllamaStatus('offline'); + return false; + } + }; + + // Generate tags using Ollama + const generateTags = async (content: string): Promise => { + try { + const systemPrompt = `You are a helpful assistant that generates searchable tags for journal entries. + +Your task is to analyze the content and generate 3-8 relevant tags that would help the author find this note later. +Focus on: +- Programming languages, frameworks, libraries mentioned +- Technical concepts and methodologies +- Tools and utilities discussed +- Problem domains or contexts +- Key technical terms that someone might search for + +Return ONLY a JSON array of strings, no other text. Example: ["golang", "testing", "array-comparison", "cmp-library"] + +Keep tags concise, use lowercase, and separate words with hyphens if needed.`; + + const userPrompt = `Generate tags for this journal entry: + +${content}`; + + const response = await fetchWithTiming(`${OLLAMA_ENDPOINT}/api/generate`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + model: OLLAMA_MODEL, + system: systemPrompt, + prompt: userPrompt, + stream: false, + }), + }, 'Generate Tags'); + + if (!response.ok) { + throw new Error('Failed to generate tags'); + } + + const data = await response.json(); + const responseText = data.response?.trim(); + + if (!responseText) { + throw new Error('Empty response from Ollama'); + } + + // Try to parse JSON from the response + try { + const tags = JSON.parse(responseText); + if (Array.isArray(tags) && tags.every(tag => typeof tag === 'string')) { + addDebugInfo(`Generated ${tags.length} tags: ${tags.join(', ')}`); + return tags; + } + } catch (parseError) { + console.warn('Failed to parse tags as JSON, trying to extract from text:', responseText); + } + + // Fallback: try to extract tags from text response + const tagMatch = responseText.match(/\[(.*?)\]/); + if (tagMatch) { + const tags = tagMatch[1] + .split(',') + .map((tag: string) => tag.trim().replace(/"/g, '')) + .filter((tag: string) => tag.length > 0); + addDebugInfo(`Extracted ${tags.length} tags from text: ${tags.join(', ')}`); + return tags; + } + + // Last resort: return empty array + addDebugInfo('Could not extract tags from Ollama response'); + return []; + } catch (error) { + console.error('Error generating tags:', error); + const errorMessage = error instanceof Error ? error.message : 'Unknown error'; + addDebugInfo(`Tag generation failed: ${errorMessage}`); + + // Check if it's a connection error + if (errorMessage.includes('fetch') || errorMessage.includes('network') || errorMessage.includes('ECONNREFUSED')) { + addDebugInfo('Ollama appears to be offline. Please ensure Ollama is running on localhost:11434'); + } + + return []; + } + }; + // Clean content function const cleanContent = (content: string) => { return content.trim(); @@ -319,6 +434,13 @@ const Index = () => { const { mostFrequentLetter, mostFrequentLetterFrequency, letterCount } = calculateLetterFrequency(trimmedContent); const now = new Date(); + // Generate tags using Ollama if enabled + let tags: string[] = []; + if (autoGenerateTags) { + addDebugInfo('Generating tags for new note...'); + tags = await generateTags(trimmedContent); + } + const document = { id: generateRandomString(32), date: now.getTime(), @@ -327,6 +449,7 @@ const Index = () => { topLetter: mostFrequentLetter, topLetterFrequency: mostFrequentLetterFrequency, letterCount: letterCount, + tags: tags, }; const response = await fetchWithTiming(`${MEILISEARCH_ENDPOINT}/indexes/${NOTE_INDEX}/documents`, { @@ -350,6 +473,7 @@ const Index = () => { topLetter: document.topLetter, topLetterFrequency: document.topLetterFrequency, letterCount: document.letterCount, + tags: document.tags, }; // Update cache and set as previous note @@ -359,7 +483,7 @@ const Index = () => { toast({ title: "Note saved", - description: "Your note has been saved successfully.", + description: `Your note has been saved successfully${tags.length > 0 ? ` with ${tags.length} tags` : ''}.`, }); return newNote; @@ -380,6 +504,13 @@ const Index = () => { const trimmedContent = cleanContent(note.content); const { mostFrequentLetter, mostFrequentLetterFrequency, letterCount } = calculateLetterFrequency(trimmedContent); + // Regenerate tags if content has changed significantly and auto-generation is enabled + let tags = note.tags || []; + if (autoGenerateTags && trimmedContent !== note.content) { + addDebugInfo('Content changed, regenerating tags...'); + tags = await generateTags(trimmedContent); + } + const document = { id: note.id, content: trimmedContent, @@ -388,6 +519,7 @@ const Index = () => { topLetter: mostFrequentLetter, topLetterFrequency: mostFrequentLetterFrequency, letterCount: letterCount, + tags: tags, }; const response = await fetchWithTiming(`${MEILISEARCH_ENDPOINT}/indexes/${NOTE_INDEX}/documents`, { @@ -404,11 +536,11 @@ const Index = () => { } // Update cache - setNoteCache(prev => prev.map(n => n.id === note.id ? { ...note, content: trimmedContent, letterCount } : n)); + setNoteCache(prev => prev.map(n => n.id === note.id ? { ...note, content: trimmedContent, letterCount, tags } : n)); toast({ title: "Note updated", - description: "Your changes have been saved.", + description: `Your changes have been saved${tags.length > 0 ? ` with ${tags.length} tags` : ''}.`, }); } catch (error) { console.error('Error updating note:', error); @@ -637,7 +769,7 @@ const Index = () => { q: query, matchingStrategy: 'all', limit: 50, - attributesToHighlight: ['content'], + attributesToHighlight: ['content', 'tags'], showRankingScore: true, highlightPreTag: '', highlightPostTag: '', @@ -863,6 +995,65 @@ const Index = () => { setIsCleanupOpen(true); }; + // Regenerate tags for all notes + const regenerateAllTags = async () => { + try { + setIsLoading(true); + addDebugInfo('Regenerating tags for all notes...'); + + // Load all notes + const response = await fetchWithTiming(`${MEILISEARCH_ENDPOINT}/indexes/${NOTE_INDEX}/search`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${MEILISEARCH_API_KEY}`, + }, + body: JSON.stringify({ + q: '', + limit: 1000, + sort: ['date:desc'], + }), + }, 'Load All Notes for Tag Regeneration'); + + if (!response.ok) { + throw new Error('Failed to load notes for tag regeneration'); + } + + const data = await response.json(); + const notes: Note[] = data.hits.map((hit: any) => mapHitToNote(hit)); + + let updatedCount = 0; + for (const note of notes) { + try { + const newTags = await generateTags(note.content); + if (JSON.stringify(newTags) !== JSON.stringify(note.tags || [])) { + const updatedNote = { ...note, tags: newTags }; + await updateNote(updatedNote); + updatedCount++; + addDebugInfo(`Updated tags for note ${note.id}: ${newTags.join(', ')}`); + } + } catch (error) { + console.error(`Error updating tags for note ${note.id}:`, error); + } + } + + addDebugInfo(`Tag regeneration complete: ${updatedCount} notes updated`); + toast({ + title: "Tag regeneration complete", + description: `Updated tags for ${updatedCount} notes.`, + }); + } catch (error) { + console.error('Error regenerating tags:', error); + toast({ + title: "Error", + description: "Failed to regenerate tags. Please try again.", + variant: "destructive", + }); + } finally { + setIsLoading(false); + } + }; + // Update the keyboard shortcuts effect useEffect(() => { const handleKeyDown = async (e: KeyboardEvent) => { @@ -1154,7 +1345,7 @@ const Index = () => { 'Content-Type': 'application/json', 'Authorization': `Bearer ${MEILISEARCH_API_KEY}`, }, - body: JSON.stringify(['date', 'topLetter', 'letterCount', 'topLetterFrequency']), + body: JSON.stringify(['date', 'topLetter', 'letterCount', 'topLetterFrequency', 'tags']), }, 'Configure Notes Filterable Attributes'), // Scratch index configurations @@ -1247,6 +1438,8 @@ const Index = () => { await loadNotes(); await loadLatestScratch(); await loadFontSizeSetting(); + await loadAutoGenerateTagsSetting(); + await checkOllamaStatus(); const totalTime = Date.now() - dataLoadStartTime; debugTiming('Total Data Loading', dataLoadStartTime); @@ -1302,6 +1495,33 @@ const Index = () => { } }; + // Load auto-generate tags setting + const loadAutoGenerateTagsSetting = async () => { + try { + const response = await fetchWithTiming(`${MEILISEARCH_ENDPOINT}/indexes/${SETTINGS_INDEX}/search`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${MEILISEARCH_API_KEY}`, + }, + body: JSON.stringify({ + q: 'autoGenerateTags', + filter: 'key = "autoGenerateTags"', + limit: 1, + }), + }, 'Load Auto Generate Tags Setting'); + + if (response.ok) { + const data = await response.json(); + if (data.hits.length > 0) { + setAutoGenerateTags(data.hits[0].value === 'true'); + } + } + } catch (error) { + console.error('Error loading auto-generate tags setting:', error); + } + }; + // Save font size setting const saveFontSizeSetting = async (newFontSize: string) => { try { @@ -1339,6 +1559,43 @@ const Index = () => { } }; + // Save auto-generate tags setting + const saveAutoGenerateTagsSetting = async (enabled: boolean) => { + try { + const document = { + key: 'autoGenerateTags', + value: enabled.toString(), + updatedAt: new Date().getTime(), + }; + + const response = await fetchWithTiming(`${MEILISEARCH_ENDPOINT}/indexes/${SETTINGS_INDEX}/documents`, { + method: 'POST', + headers: { + 'Content-Type': 'application/json', + 'Authorization': `Bearer ${MEILISEARCH_API_KEY}`, + }, + body: JSON.stringify(document), + }, 'Save Auto Generate Tags Setting'); + + if (response.status !== 202) { + throw new Error('Failed to save auto-generate tags setting'); + } + + setAutoGenerateTags(enabled); + toast({ + title: "Auto-tagging updated", + description: `Automatic tag generation ${enabled ? 'enabled' : 'disabled'}.`, + }); + } catch (error) { + console.error('Error saving auto-generate tags setting:', error); + toast({ + title: "Error", + description: "Failed to save auto-generate tags setting.", + variant: "destructive", + }); + } + }; + return (
{/* Header */} @@ -1366,6 +1623,14 @@ const Index = () => { Extra Large +
+ Auto-tags + +
+ )}
{previousNote ? (
{formatDate(previousNote.epochTime)}
+ {previousNote.tags && previousNote.tags.length > 0 && ( +
+ {previousNote.tags.map((tag, index) => ( + + {tag} + + ))} +
+ )}