Sanitize out "-" and remove the arbitrary 100 query limit

We don't want to display utc

lib/db.dart

@@ -6,10 +6,23 @@ import 'package:sqflite_common_ffi/sqflite_ffi.dart';
 const settingsDir = '.journaler';
 const dbFileName = 'journaler.db';
 
+// Add this at the top level
+typedef ShowMessageCallback = void Function(String message);
+
 class DB {
   static late Database db;
+  static bool _hasIcuSupport = false;
+  static ShowMessageCallback? _showMessage;
 
-  static const String _schema = '''
+  // Add this to register the callback
+  static void registerMessageCallback(ShowMessageCallback callback) {
+    _showMessage = callback;
+  }
+
+  // Add this method to check ICU status
+  static bool get hasIcuSupport => _hasIcuSupport;
+
+  static const String _baseSchema = '''
 CREATE TABLE IF NOT EXISTS notes (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   date TEXT DEFAULT CURRENT_TIMESTAMP,
@@ -17,10 +30,7 @@ CREATE TABLE IF NOT EXISTS notes (
 );
 CREATE INDEX IF NOT EXISTS idx_notes_date ON notes (date);
 CREATE UNIQUE INDEX IF NOT EXISTS idx_notes_date_unique ON notes (date);
--- Todos are "static", we are only interested in the latest entry
+
--- ie. they're not really a list but instead a string
--- But we will also keep a history of all todos
--- Because we might as well
 CREATE TABLE IF NOT EXISTS scratches (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   date TEXT DEFAULT CURRENT_TIMESTAMP,
@@ -33,15 +43,31 @@ CREATE TABLE IF NOT EXISTS settings (
   key TEXT PRIMARY KEY NOT NULL,
   value TEXT NOT NULL
 );
+''';
 
--- Create virtual FTS5 table for searching notes content
+  static const String _ftsSchemaWithIcu = '''
+-- Create virtual FTS5 table with Unicode tokenizer
 CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
   content,
   date,
   content='notes',
-  content_rowid='id'
+  content_rowid='id',
+  tokenize='unicode61 remove_diacritics 2 tokenchars "\u0401\u0451\u0410-\u044f"'
 );
+''';
 
+  static const String _ftsSchemaBasic = '''
+-- Create virtual FTS5 table with basic tokenizer
+CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
+  content,
+  date,
+  content='notes',
+  content_rowid='id',
+  tokenize='unicode61'
+);
+''';
+
+  static const String _ftsTriggers = '''
 -- Trigger to keep FTS table in sync with notes table when inserting
 CREATE TRIGGER IF NOT EXISTS notes_ai AFTER INSERT ON notes BEGIN
   INSERT INTO notes_fts(rowid, content, date) VALUES (new.id, new.content, new.date);
@@ -58,6 +84,124 @@ CREATE TRIGGER IF NOT EXISTS notes_au AFTER UPDATE ON notes BEGIN
 END;
 ''';
 
+  // Add this method to check ICU status with visible feedback
+  static Future<bool> checkAndShowIcuStatus() async {
+    final status = _hasIcuSupport;
+    final message =
+        status
+            ? 'ICU support is enabled - Full Unicode search available'
+            : 'ICU support is not available - Unicode search will be limited\n'
+                'Looking for sqlite3_icu.dll in the application directory';
+    debugPrint(message);
+    _showMessage?.call(message);
+    return status;
+  }
+
+  static Future<bool> _checkIcuSupport(Database db) async {
+    try {
+      debugPrint(
+        '\n================== CHECKING UNICODE SUPPORT ==================',
+      );
+
+      // Test Unicode support with a simple query
+      try {
+        debugPrint('Testing Unicode support...');
+        // Test with some Cyrillic characters
+        await db.rawQuery("SELECT 'тест' LIKE '%ест%'");
+        await db.rawQuery("SELECT 'ТЕСТ' LIKE '%ЕСТ%'");
+
+        final message = 'Unicode support is available';
+        debugPrint(message);
+        _showMessage?.call(message);
+        return true;
+      } catch (e, stackTrace) {
+        final message = 'Unicode support test failed';
+        debugPrint('$message:');
+        debugPrint('Error: $e');
+        debugPrint('Stack trace: $stackTrace');
+
+        // Try to get SQLite version and compile options for debugging
+        try {
+          final version = await db.rawQuery('SELECT sqlite_version()');
+          final compileOpts = await db.rawQuery('PRAGMA compile_options');
+          debugPrint('SQLite version: $version');
+          debugPrint('SQLite compile options: $compileOpts');
+        } catch (e) {
+          debugPrint('Could not get SQLite info: $e');
+        }
+
+        _showMessage?.call(message);
+        return false;
+      }
+    } catch (e, stackTrace) {
+      final message = 'Failed to test Unicode support';
+      debugPrint('$message:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+      _showMessage?.call(message);
+      return false;
+    } finally {
+      debugPrint('=====================================================\n');
+    }
+  }
+
+  static Future<void> _recreateFtsTable(Database db, bool useIcu) async {
+    debugPrint('Updating FTS table configuration...');
+
+    try {
+      // Start a transaction to ensure data safety
+      await db.transaction((txn) async {
+        // First, create a temporary table with the new configuration
+        final tempTableName = 'notes_fts_temp';
+        final schema = useIcu ? _ftsSchemaWithIcu : _ftsSchemaBasic;
+
+        // Create temp table with new configuration
+        await txn.execute(schema.replaceAll('notes_fts', tempTableName));
+
+        // Copy data from old FTS table if it exists
+        try {
+          debugPrint('Copying existing FTS data to temporary table...');
+          await txn.execute('''
+            INSERT INTO $tempTableName(rowid, content, date)
+            SELECT rowid, content, date FROM notes_fts
+          ''');
+          debugPrint('Data copied successfully');
+        } catch (e) {
+          debugPrint('No existing FTS data to copy: $e');
+        }
+
+        // Drop old triggers
+        debugPrint('Updating triggers...');
+        await txn.execute('DROP TRIGGER IF EXISTS notes_ai');
+        await txn.execute('DROP TRIGGER IF EXISTS notes_ad');
+        await txn.execute('DROP TRIGGER IF EXISTS notes_au');
+
+        // Drop old FTS table
+        await txn.execute('DROP TABLE IF EXISTS notes_fts');
+
+        // Rename temp table to final name
+        await txn.execute('ALTER TABLE $tempTableName RENAME TO notes_fts');
+
+        // Create new triggers
+        await txn.execute(_ftsTriggers);
+
+        // Rebuild FTS index from notes table to ensure consistency
+        debugPrint('Rebuilding FTS index from notes table...');
+        await txn.execute('''
+          INSERT OR REPLACE INTO notes_fts(rowid, content, date)
+          SELECT id, content, date FROM notes
+        ''');
+
+        debugPrint('FTS table update completed successfully');
+      });
+    } catch (e, stackTrace) {
+      debugPrint('Error updating FTS table:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+      rethrow;
+    }
+  }
+
   static Future<String> _getDatabasePath() async {
     debugPrint('Attempting to get database path...');
     if (Platform.isWindows || Platform.isLinux) {
@@ -88,26 +232,100 @@ END;
 
   static Future<void> init() async {
     debugPrint('Starting database initialization...');
+
+    // Initialize SQLite FFI
     sqfliteFfiInit();
+    final databaseFactory = databaseFactoryFfi;
+
+    // Create a temporary database to check version
+    try {
+      debugPrint(
+        '\n================== SQLITE VERSION CHECK ==================',
+      );
+      final tempDb = await databaseFactory.openDatabase(
+        ':memory:',
+        options: OpenDatabaseOptions(
+          version: 1,
+          onCreate: (db, version) async {
+            final results = await db.rawQuery('SELECT sqlite_version()');
+            debugPrint('SQLite version: ${results.first.values.first}');
+
+            final compileOpts = await db.rawQuery('PRAGMA compile_options');
+            debugPrint('SQLite compile options:');
+            for (var opt in compileOpts) {
+              debugPrint('  ${opt.values.first}');
+            }
+          },
+        ),
+      );
+      await tempDb.close();
+      debugPrint('=====================================================\n');
+    } catch (e, stackTrace) {
+      debugPrint('Error checking SQLite version:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+    }
+
+    await databaseFactory.setDatabasesPath(
+      await databaseFactory.getDatabasesPath(),
+    );
 
     final dbPath = await _getDatabasePath();
     debugPrint('Database path: $dbPath');
 
     try {
-      db = await databaseFactoryFfi.openDatabase(
+      db = await databaseFactory.openDatabase(
         dbPath,
         options: OpenDatabaseOptions(
-          version: 1,
+          version: 2,
+          onConfigure: (db) async {
+            debugPrint('Configuring database...');
+            await db.execute('PRAGMA foreign_keys = ON');
+            debugPrint('Database configured');
+          },
           onCreate: (db, version) async {
             debugPrint('Creating database schema...');
-            await db.execute(_schema);
+            await db.execute(_baseSchema);
+
+            // Check for Unicode support on first creation
+            _hasIcuSupport = await _checkIcuSupport(db);
+            await _recreateFtsTable(db, _hasIcuSupport);
+
             debugPrint('Database schema created successfully');
           },
+          onOpen: (db) async {
+            debugPrint('Database opened, checking Unicode support...');
+            try {
+              _hasIcuSupport = await _checkIcuSupport(db);
+              debugPrint('Unicode support check completed: $_hasIcuSupport');
+            } catch (e, stackTrace) {
+              debugPrint('Error during Unicode support check:');
+              debugPrint('Error: $e');
+              debugPrint('Stack trace: $stackTrace');
+              rethrow;
+            }
+          },
+          onUpgrade: (db, oldVersion, newVersion) async {
+            debugPrint('Upgrading database from $oldVersion to $newVersion');
+
+            if (oldVersion < 2) {
+              // Check for Unicode support during upgrade
+              _hasIcuSupport = await _checkIcuSupport(db);
+              await _recreateFtsTable(db, _hasIcuSupport);
+            }
+          },
         ),
       );
-      debugPrint('Database opened and initialized');
+
-    } catch (e) {
+      // Store Unicode support status in settings for future reference
-      debugPrint('Failed to initialize database: $e');
+      await setSetting('has_icu_support', _hasIcuSupport.toString());
+      debugPrint(
+        'Database opened and initialized (Unicode support: $_hasIcuSupport)',
+      );
+    } catch (e, stackTrace) {
+      debugPrint('Failed to initialize database:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
       rethrow;
     }
   }
@@ -141,7 +359,6 @@ END;
         return [];
       }
 
-      // Process the query for partial word matching
       // Split into individual terms, filter empty ones
       List<String> terms =
           query
@@ -154,16 +371,16 @@ END;
         return [];
       }
 
-      // Add wildcards to each term for prefix matching (e.g., "fuck*" will match "fucked")
+      // Process terms for FTS5 query using proper tokenization
-      // Join terms with AND for all-term matching (results must contain ALL terms)
       String ftsQuery = terms
           .map((term) {
-            // Remove any special characters that might break the query
+            // Remove dangerous characters but preserve Unicode
-            String sanitizedTerm = term.replaceAll(RegExp(r'[^\w]'), '');
+            String sanitizedTerm = term.replaceAll(RegExp(r'''['"]'''), '');
             if (sanitizedTerm.isEmpty) return '';
 
-            // Add wildcard for stemming/prefix matching
+            // Use proper FTS5 syntax: each word becomes a separate token with prefix matching
-            return '$sanitizedTerm*';
+            List<String> words = sanitizedTerm.split(RegExp(r'\s+'));
+            return words.map((word) => '$word*').join(' OR ');
           })
           .where((term) => term.isNotEmpty)
           .join(' AND ');
@@ -173,26 +390,27 @@ END;
         return [];
       }
 
+      ftsQuery = ftsQuery.replaceAll('-', ' ');
       debugPrint('FTS query: "$ftsQuery"');
 
-      // Execute the FTS query with AND logic
+      // Execute the FTS query
       final List<Map<String, dynamic>> results = await db.rawQuery(
         '''
-        SELECT n.id, n.date, n.content, snippet(notes_fts, 0, '<b>', '</b>', '...', 20) as snippet
+        SELECT n.id, n.date, n.content,
+               snippet(notes_fts, -1, '<b>', '</b>', '...', 64) as snippet
         FROM notes_fts
         JOIN notes n ON notes_fts.rowid = n.id
         WHERE notes_fts MATCH ?
-        ORDER BY n.date DESC
+        ORDER BY rank
-        LIMIT 100
         ''',
         [ftsQuery],
       );
 
-      debugPrint('Search query "$ftsQuery" returned ${results.length} results');
+      debugPrint('Search returned ${results.length} results');
       return results;
-    } catch (e) {
+    } catch (e, stackTrace) {
       debugPrint('Search failed: $e');
-      // Return empty results rather than crashing on malformed queries
+      debugPrint('Stack trace: $stackTrace');
       return [];
     }
   }

lib/main.dart

@@ -550,11 +550,10 @@ class MainPageState extends State<MainPage> with WindowListener {
 
     final scratch = await getLatestScratch();
     _scratchController.text = scratch?.content ?? "";
-    _currentEntryController.text = "";
 
     await _checkNavigation();
 
-    debugPrint("Data loaded.");
+    debugPrint("Data loaded");
   }
 
   // Load volume setting from database
@@ -586,6 +585,7 @@ class MainPageState extends State<MainPage> with WindowListener {
     // Handle current entry
     if (currentEntry.isNotEmpty) {
       await createNote(currentEntry);
+      _currentEntryController.clear(); // Clear the input field after saving
     }
 
     // Handle scratch pad
@@ -755,6 +755,11 @@ class MainPageState extends State<MainPage> with WindowListener {
                                       .where((note) => note.content.isNotEmpty)
                                       .toList();
 
+                              // Sort by date, newest first
+                              filteredResults.sort(
+                                (a, b) => b.date.compareTo(a.date),
+                              );
+
                               // Important: update the dialog state after search completes
                               dialogSetState(() {
                                 _searchResults = filteredResults;
@@ -800,7 +805,7 @@ class MainPageState extends State<MainPage> with WindowListener {
                                                 vertical: 2,
                                               ), // Tighter padding
                                           title: Text(
-                                            note.date,
+                                            note.displayDate,
                                             style: const TextStyle(
                                               fontWeight: FontWeight.bold,
                                               fontSize:
@@ -839,7 +844,7 @@ class MainPageState extends State<MainPage> with WindowListener {
 
                                             // Navigate to the selected note
                                             Navigator.of(context).pop();
-                                            this.setState(() {
+                                            setState(() {
                                               _currentlyDisplayedNote = note;
                                               _previousEntryController.text =
                                                   note.content;
@@ -905,6 +910,207 @@ class MainPageState extends State<MainPage> with WindowListener {
     );
   }
 
+  // Show cleanup dialog
+  void _showCleanupDialog() async {
+    double sensitivity = 0.7; // Default 70%
+    final problematicEntries = await findProblematicEntries(
+      maxCharPercentage: sensitivity,
+    );
+
+    if (!mounted) return;
+
+    showDialog(
+      context: context,
+      builder: (BuildContext context) {
+        return StatefulBuilder(
+          builder: (context, dialogSetState) {
+            return AlertDialog(
+              title: const Text('Cleanup Problematic Entries'),
+              content: SizedBox(
+                width: MediaQuery.of(context).size.width * 0.7,
+                height: MediaQuery.of(context).size.height * 0.7,
+                child: Column(
+                  children: [
+                    Row(
+                      children: [
+                        const Text('Sensitivity: '),
+                        Expanded(
+                          child: Slider(
+                            value: sensitivity,
+                            min: 0.3,
+                            max: 0.9,
+                            divisions: 12,
+                            label: '${(sensitivity * 100).toInt()}%',
+                            onChanged: (value) async {
+                              dialogSetState(() {
+                                sensitivity =
+                                    (value * 100).round() /
+                                    100; // Round to 2 decimal places
+                              });
+                              // Refresh results with new sensitivity
+                              final newResults = await findProblematicEntries(
+                                maxCharPercentage: sensitivity,
+                              );
+                              dialogSetState(() {
+                                problematicEntries.clear();
+                                problematicEntries.addAll(newResults);
+                              });
+                            },
+                          ),
+                        ),
+                        Text('${(sensitivity * 100).toInt()}%'),
+                      ],
+                    ),
+                    Text(
+                      'Found ${problematicEntries.length} potentially problematic entries',
+                      style: const TextStyle(fontWeight: FontWeight.bold),
+                    ),
+                    const SizedBox(height: 16),
+                    Expanded(
+                      child:
+                          problematicEntries.isEmpty
+                              ? const Center(
+                                child: Text('No problematic entries found!'),
+                              )
+                              : ListView.builder(
+                                itemCount: problematicEntries.length,
+                                itemBuilder: (context, index) {
+                                  final note = problematicEntries[index];
+                                  return Card(
+                                    margin: const EdgeInsets.only(bottom: 8),
+                                    child: ListTile(
+                                      title: Text(
+                                        note.displayDate,
+                                        style: const TextStyle(
+                                          fontWeight: FontWeight.bold,
+                                        ),
+                                      ),
+                                      subtitle: Column(
+                                        crossAxisAlignment:
+                                            CrossAxisAlignment.start,
+                                        children: [
+                                          Text(
+                                            'Reason: ${note.problemReason}',
+                                            style: TextStyle(
+                                              color: Colors.red[700],
+                                              fontWeight: FontWeight.bold,
+                                            ),
+                                          ),
+                                          const SizedBox(height: 4),
+                                          Text(note.content),
+                                        ],
+                                      ),
+                                      trailing: Row(
+                                        mainAxisSize: MainAxisSize.min,
+                                        children: [
+                                          IconButton(
+                                            icon: const Icon(Icons.visibility),
+                                            tooltip: 'View in main window',
+                                            onPressed: () {
+                                              // Close the dialog
+                                              Navigator.of(context).pop();
+                                              // Navigate to the note in main window
+                                              setState(() {
+                                                _currentlyDisplayedNote = note;
+                                                _previousEntryController.text =
+                                                    note.content;
+                                              });
+                                              _checkNavigation();
+                                            },
+                                          ),
+                                          IconButton(
+                                            icon: const Icon(Icons.delete),
+                                            color: Colors.red,
+                                            tooltip:
+                                                'Delete (hold Shift to skip confirmation)',
+                                            onPressed: () async {
+                                              // Check if shift is pressed
+                                              final isShiftPressed =
+                                                  HardwareKeyboard
+                                                      .instance
+                                                      .isShiftPressed;
+
+                                              bool shouldDelete =
+                                                  isShiftPressed;
+                                              if (!isShiftPressed) {
+                                                // Show confirmation dialog
+                                                shouldDelete =
+                                                    await showDialog<bool>(
+                                                      context: context,
+                                                      builder:
+                                                          (
+                                                            context,
+                                                          ) => AlertDialog(
+                                                            title: const Text(
+                                                              'Delete Entry?',
+                                                            ),
+                                                            content: const Text(
+                                                              'Are you sure you want to delete this entry? This action cannot be undone.',
+                                                            ),
+                                                            actions: [
+                                                              TextButton(
+                                                                onPressed:
+                                                                    () => Navigator.of(
+                                                                      context,
+                                                                    ).pop(
+                                                                      false,
+                                                                    ),
+                                                                child:
+                                                                    const Text(
+                                                                      'Cancel',
+                                                                    ),
+                                                              ),
+                                                              TextButton(
+                                                                onPressed:
+                                                                    () => Navigator.of(
+                                                                      context,
+                                                                    ).pop(true),
+                                                                child:
+                                                                    const Text(
+                                                                      'Delete',
+                                                                    ),
+                                                              ),
+                                                            ],
+                                                          ),
+                                                    ) ??
+                                                    false;
+                                              }
+
+                                              if (shouldDelete) {
+                                                await deleteNote(note.date);
+                                                dialogSetState(() {
+                                                  problematicEntries.removeAt(
+                                                    index,
+                                                  );
+                                                });
+                                              }
+                                            },
+                                          ),
+                                        ],
+                                      ),
+                                    ),
+                                  );
+                                },
+                              ),
+                    ),
+                  ],
+                ),
+              ),
+              actions: [
+                TextButton(
+                  onPressed: () {
+                    Navigator.of(context).pop();
+                  },
+                  child: const Text('Close'),
+                ),
+              ],
+            );
+          },
+        );
+      },
+    );
+  }
+
   @override
   Widget build(BuildContext context) {
     // Wrap Scaffold with RawKeyboardListener as workaround for Escape key
@@ -936,6 +1142,12 @@ class MainPageState extends State<MainPage> with WindowListener {
         appBar: AppBar(
           title: const Text('Journaler'),
           actions: <Widget>[
+            // Add cleanup button
+            IconButton(
+              icon: const Icon(Icons.cleaning_services),
+              tooltip: 'Cleanup Problematic Entries',
+              onPressed: _showCleanupDialog,
+            ),
             // Add search button
             IconButton(
               icon: const Icon(Icons.search),
@@ -1051,10 +1263,10 @@ class MainPageState extends State<MainPage> with WindowListener {
                         children: [
                           Expanded(
                             child: Text(
-                              _currentlyDisplayedNote?.date ==
+                              _currentlyDisplayedNote?.displayDate ==
-                                      previousNote?.date
+                                      previousNote?.displayDate
-                                  ? 'Previous Entry (Latest)'
+                                  ? 'Previous Entry: ${_currentlyDisplayedNote?.displayDate ?? 'N/A'}'
-                                  : 'Entry: ${_currentlyDisplayedNote?.date ?? 'N/A'}',
+                                  : 'Entry: ${_currentlyDisplayedNote?.displayDate ?? 'N/A'}',
                               style: TextStyle(
                                 fontSize: 18,
                                 color: Colors.grey,
@@ -1083,17 +1295,17 @@ class MainPageState extends State<MainPage> with WindowListener {
                           style: Theme.of(context).textTheme.bodyMedium,
                           decoration: InputDecoration(
                             hintText:
-                                _currentlyDisplayedNote?.date !=
+                                _currentlyDisplayedNote?.displayDate !=
-                                        previousNote?.date
+                                        previousNote?.displayDate
-                                    ? 'Viewing note from ${_currentlyDisplayedNote?.date} (Editable)'
+                                    ? 'Viewing note from ${_currentlyDisplayedNote?.displayDate} (Editable)'
-                                    : 'Latest Note',
+                                    : 'Latest Note: ${_currentlyDisplayedNote?.displayDate ?? 'N/A'}',
                             border: const OutlineInputBorder(),
                             filled:
-                                _currentlyDisplayedNote?.date !=
+                                _currentlyDisplayedNote?.displayDate !=
-                                previousNote?.date,
+                                previousNote?.displayDate,
                             fillColor:
-                                _currentlyDisplayedNote?.date !=
+                                _currentlyDisplayedNote?.displayDate !=
-                                        previousNote?.date
+                                        previousNote?.displayDate
                                     ? Colors.grey.withOpacity(0.1)
                                     : null,
                           ),

lib/notes.dart

@@ -1,12 +1,25 @@
 import 'package:journaler/db.dart';
+import 'package:intl/intl.dart';
 
 class Note {
   final String date;
+  late final String displayDate;
   String content;
-  String?
+  String? snippet;
-  snippet; // Optional field to hold highlighted snippets for search results
+  bool isProblematic;
+  String problemReason;
 
-  Note({required this.date, required this.content, this.snippet});
+  Note({
+    required this.date,
+    required this.content,
+    this.snippet,
+    this.isProblematic = false,
+    this.problemReason = '',
+  }) {
+    final dtUtc = DateFormat('yyyy-MM-dd HH:mm:ss').parse(date, true);
+    final dtLocal = dtUtc.toLocal();
+    displayDate = DateFormat('yyyy-MM-dd HH:mm:ss').format(dtLocal);
+  }
 }
 
 class Scratch {
@@ -30,11 +43,18 @@ Future<Note?> getLatestNote() async {
 }
 
 Future<void> createNote(String content) async {
-  // Trim the content to avoid saving just whitespace
+  // Trim each line, sometimes we fuck up by doing a lil "foobar            "
-  final trimmedContent = content.trim();
+  // Maybe I should also look for \s{2,}...
+  final lines = content.split('\n');
+  final trimmedLines = <String>[];
+  for (final line in lines) {
+    final trimmedContent = line.trim().replaceAll(RegExp(r'\s{2,}'), ' ');
     if (trimmedContent.isEmpty) {
-    return;
+      continue;
     }
+    trimmedLines.add(trimmedContent);
+  }
+  final trimmedContent = trimmedLines.join('\n');
   await DB.db.insert('notes', {'content': trimmedContent});
 }
 
@@ -147,3 +167,42 @@ Future<List<Note>> searchNotes(String query) async {
       )
       .toList();
 }
+
+// Find potentially problematic entries based on character distribution
+Future<List<Note>> findProblematicEntries({
+  double maxCharPercentage = 0.7,
+}) async {
+  // Simple SQLite query that counts character occurrences using replace
+  final List<Map<String, dynamic>> results = await DB.db.rawQuery(
+    '''
+    WITH char_counts AS (
+      SELECT
+        id,
+        date,
+        content,
+        substr(content, 1, 1) as char,
+        (length(content) - length(replace(content, substr(content, 1, 1), ''))) as char_count,
+        length(content) as total_length,
+        cast(length(content) - length(replace(content, substr(content, 1, 1), '')) as float) / length(content) as percentage
+      FROM notes
+    )
+    SELECT *
+    FROM char_counts
+    WHERE percentage > ?
+    ORDER BY date DESC
+  ''',
+    [maxCharPercentage],
+  );
+
+  return results
+      .map(
+        (row) => Note(
+          date: row['date'] as String,
+          content: row['content'] as String,
+          isProblematic: true,
+          problemReason:
+              'Character "${row['char']}" makes up ${(row['percentage'] * 100).toStringAsFixed(1)}% of the content',
+        ),
+      )
+      .toList();
+}

pubspec.lock

@@ -224,6 +224,14 @@ packages:
       url: "https://pub.dev"
     source: hosted
     version: "4.5.4"
+  intl:
+    dependency: "direct main"
+    description:
+      name: intl
+      sha256: "3df61194eb431efc39c4ceba583b95633a403f46c9fd341e550ce0bfa50e9aa5"
+      url: "https://pub.dev"
+    source: hosted
+    version: "0.20.2"
   json_annotation:
     dependency: transitive
     description:

pubspec.yaml

@@ -40,6 +40,7 @@ dependencies:
   sqflite_common_ffi: ^2.3.5
   path: ^1.8.0
   ps_list: ^0.0.5
+  intl: ^0.20.2
 
 dev_dependencies:
   flutter_test:
@@ -68,6 +69,7 @@ flutter:
   assets:
     - assets/ # Include the main assets directory for the icon
     - assets/sounds/
+    - assets/windows/sqlite3_icu.dll
   #   - images/a_dot_burr.jpeg
   #   - images/a_dot_ham.jpeg