Add icu support for cyrillic fts

lib/db.dart

@@ -1,15 +1,29 @@
-import 'dart:io' show Platform, Directory;
+import 'dart:io' show Platform, Directory, File;
 import 'package:flutter/material.dart';
+import 'package:flutter/services.dart';
 import 'package:path/path.dart' as path;
 import 'package:sqflite_common_ffi/sqflite_ffi.dart';
 
 const settingsDir = '.journaler';
 const dbFileName = 'journaler.db';
 
+// Add this at the top level
+typedef ShowMessageCallback = void Function(String message);
+
 class DB {
   static late Database db;
+  static bool _hasIcuSupport = false;
+  static ShowMessageCallback? _showMessage;
 
-  static const String _schema = '''
+  // Add this to register the callback
+  static void registerMessageCallback(ShowMessageCallback callback) {
+    _showMessage = callback;
+  }
+
+  // Add this method to check ICU status
+  static bool get hasIcuSupport => _hasIcuSupport;
+
+  static const String _baseSchema = '''
 CREATE TABLE IF NOT EXISTS notes (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   date TEXT DEFAULT CURRENT_TIMESTAMP,
@@ -17,10 +31,7 @@ CREATE TABLE IF NOT EXISTS notes (
 );
 CREATE INDEX IF NOT EXISTS idx_notes_date ON notes (date);
 CREATE UNIQUE INDEX IF NOT EXISTS idx_notes_date_unique ON notes (date);
--- Todos are "static", we are only interested in the latest entry
+
--- ie. they're not really a list but instead a string
--- But we will also keep a history of all todos
--- Because we might as well
 CREATE TABLE IF NOT EXISTS scratches (
   id INTEGER PRIMARY KEY AUTOINCREMENT,
   date TEXT DEFAULT CURRENT_TIMESTAMP,
@@ -33,15 +44,31 @@ CREATE TABLE IF NOT EXISTS settings (
   key TEXT PRIMARY KEY NOT NULL,
   value TEXT NOT NULL
 );
+''';
 
--- Create virtual FTS5 table for searching notes content
+  static const String _ftsSchemaWithIcu = '''
+-- Create virtual FTS5 table with Unicode tokenizer
 CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
   content,
   date,
   content='notes',
-  content_rowid='id'
+  content_rowid='id',
+  tokenize='unicode61 remove_diacritics 2 tokenchars "\u0401\u0451\u0410-\u044f"'
 );
+''';
 
+  static const String _ftsSchemaBasic = '''
+-- Create virtual FTS5 table with basic tokenizer
+CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
+  content,
+  date,
+  content='notes',
+  content_rowid='id',
+  tokenize='unicode61'
+);
+''';
+
+  static const String _ftsTriggers = '''
 -- Trigger to keep FTS table in sync with notes table when inserting
 CREATE TRIGGER IF NOT EXISTS notes_ai AFTER INSERT ON notes BEGIN
   INSERT INTO notes_fts(rowid, content, date) VALUES (new.id, new.content, new.date);
@@ -58,6 +85,205 @@ CREATE TRIGGER IF NOT EXISTS notes_au AFTER UPDATE ON notes BEGIN
 END;
 ''';
 
+  static Future<String?> _getIcuExtensionPath() async {
+    if (!Platform.isWindows) {
+      debugPrint('Not on Windows, skipping ICU');
+      return null;
+    }
+
+    try {
+      // Get the executable's directory
+      final exePath = Platform.resolvedExecutable;
+      final exeDir = path.dirname(exePath);
+      final currentDir = Directory.current.path;
+
+      debugPrint('\n================== ICU EXTENSION SETUP ==================');
+      debugPrint('Current working directory: $currentDir');
+      debugPrint('Executable path: $exePath');
+      debugPrint('Executable directory: $exeDir');
+
+      // In release mode, we want the DLL next to the exe
+      final targetPath = path.join(exeDir, 'sqlite3_icu.dll');
+      final targetFile = File(targetPath);
+
+      // If the DLL doesn't exist in the target location, copy it from assets
+      if (!await targetFile.exists()) {
+        debugPrint(
+          'DLL not found at target location, attempting to copy from assets...',
+        );
+        try {
+          // Load from assets and write to target location
+          final assetPath = 'assets/windows/sqlite3_icu.dll';
+          debugPrint('Trying to load from asset path: $assetPath');
+
+          final data = await rootBundle.load(assetPath);
+          debugPrint(
+            'Asset loaded successfully, size: ${data.lengthInBytes} bytes',
+          );
+
+          debugPrint('Writing to target path: $targetPath');
+          await targetFile.writeAsBytes(
+            data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes),
+          );
+          debugPrint('Successfully copied DLL to: $targetPath');
+        } catch (e, stackTrace) {
+          debugPrint('Failed to copy DLL from assets:');
+          debugPrint('Error: $e');
+          debugPrint('Stack trace: $stackTrace');
+
+          // Try to list available assets for debugging
+          try {
+            final manifestContent = await rootBundle.loadString(
+              'AssetManifest.json',
+            );
+            debugPrint('Available assets in manifest:');
+            debugPrint(manifestContent);
+          } catch (e) {
+            debugPrint('Could not load asset manifest: $e');
+          }
+          return null;
+        }
+      } else {
+        debugPrint('Found existing DLL at: $targetPath');
+      }
+
+      debugPrint('Verifying DLL exists at: $targetPath');
+      if (await targetFile.exists()) {
+        final fileSize = await targetFile.length();
+        debugPrint('DLL exists, size: $fileSize bytes');
+      } else {
+        debugPrint('DLL does not exist at target path!');
+      }
+
+      debugPrint('=====================================================\n');
+      return targetPath;
+    } catch (e, stackTrace) {
+      debugPrint('Error setting up ICU extension:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+      debugPrint('=====================================================\n');
+      return null;
+    }
+  }
+
+  // Add this method to check ICU status with visible feedback
+  static Future<bool> checkAndShowIcuStatus() async {
+    final status = _hasIcuSupport;
+    final message =
+        status
+            ? 'ICU support is enabled - Full Unicode search available'
+            : 'ICU support is not available - Unicode search will be limited\n'
+                'Looking for sqlite3_icu.dll in the application directory';
+    debugPrint(message);
+    _showMessage?.call(message);
+    return status;
+  }
+
+  static Future<bool> _checkIcuSupport(Database db) async {
+    try {
+      debugPrint(
+        '\n================== CHECKING UNICODE SUPPORT ==================',
+      );
+
+      // Test Unicode support with a simple query
+      try {
+        debugPrint('Testing Unicode support...');
+        // Test with some Cyrillic characters
+        await db.rawQuery("SELECT 'тест' LIKE '%ест%'");
+        await db.rawQuery("SELECT 'ТЕСТ' LIKE '%ЕСТ%'");
+
+        final message = 'Unicode support is available';
+        debugPrint(message);
+        _showMessage?.call(message);
+        return true;
+      } catch (e, stackTrace) {
+        final message = 'Unicode support test failed';
+        debugPrint('$message:');
+        debugPrint('Error: $e');
+        debugPrint('Stack trace: $stackTrace');
+
+        // Try to get SQLite version and compile options for debugging
+        try {
+          final version = await db.rawQuery('SELECT sqlite_version()');
+          final compileOpts = await db.rawQuery('PRAGMA compile_options');
+          debugPrint('SQLite version: $version');
+          debugPrint('SQLite compile options: $compileOpts');
+        } catch (e) {
+          debugPrint('Could not get SQLite info: $e');
+        }
+
+        _showMessage?.call(message);
+        return false;
+      }
+    } catch (e, stackTrace) {
+      final message = 'Failed to test Unicode support';
+      debugPrint('$message:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+      _showMessage?.call(message);
+      return false;
+    } finally {
+      debugPrint('=====================================================\n');
+    }
+  }
+
+  static Future<void> _recreateFtsTable(Database db, bool useIcu) async {
+    debugPrint('Updating FTS table configuration...');
+
+    try {
+      // Start a transaction to ensure data safety
+      await db.transaction((txn) async {
+        // First, create a temporary table with the new configuration
+        final tempTableName = 'notes_fts_temp';
+        final schema = useIcu ? _ftsSchemaWithIcu : _ftsSchemaBasic;
+
+        // Create temp table with new configuration
+        await txn.execute(schema.replaceAll('notes_fts', tempTableName));
+
+        // Copy data from old FTS table if it exists
+        try {
+          debugPrint('Copying existing FTS data to temporary table...');
+          await txn.execute('''
+            INSERT INTO $tempTableName(rowid, content, date)
+            SELECT rowid, content, date FROM notes_fts
+          ''');
+          debugPrint('Data copied successfully');
+        } catch (e) {
+          debugPrint('No existing FTS data to copy: $e');
+        }
+
+        // Drop old triggers
+        debugPrint('Updating triggers...');
+        await txn.execute('DROP TRIGGER IF EXISTS notes_ai');
+        await txn.execute('DROP TRIGGER IF EXISTS notes_ad');
+        await txn.execute('DROP TRIGGER IF EXISTS notes_au');
+
+        // Drop old FTS table
+        await txn.execute('DROP TABLE IF EXISTS notes_fts');
+
+        // Rename temp table to final name
+        await txn.execute('ALTER TABLE $tempTableName RENAME TO notes_fts');
+
+        // Create new triggers
+        await txn.execute(_ftsTriggers);
+
+        // Rebuild FTS index from notes table to ensure consistency
+        debugPrint('Rebuilding FTS index from notes table...');
+        await txn.execute('''
+          INSERT OR REPLACE INTO notes_fts(rowid, content, date)
+          SELECT id, content, date FROM notes
+        ''');
+
+        debugPrint('FTS table update completed successfully');
+      });
+    } catch (e, stackTrace) {
+      debugPrint('Error updating FTS table:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+      rethrow;
+    }
+  }
+
   static Future<String> _getDatabasePath() async {
     debugPrint('Attempting to get database path...');
     if (Platform.isWindows || Platform.isLinux) {
@@ -88,26 +314,100 @@ END;
 
   static Future<void> init() async {
     debugPrint('Starting database initialization...');
+
+    // Initialize SQLite FFI
     sqfliteFfiInit();
+    final databaseFactory = databaseFactoryFfi;
+
+    // Create a temporary database to check version
+    try {
+      debugPrint(
+        '\n================== SQLITE VERSION CHECK ==================',
+      );
+      final tempDb = await databaseFactory.openDatabase(
+        ':memory:',
+        options: OpenDatabaseOptions(
+          version: 1,
+          onCreate: (db, version) async {
+            final results = await db.rawQuery('SELECT sqlite_version()');
+            debugPrint('SQLite version: ${results.first.values.first}');
+
+            final compileOpts = await db.rawQuery('PRAGMA compile_options');
+            debugPrint('SQLite compile options:');
+            for (var opt in compileOpts) {
+              debugPrint('  ${opt.values.first}');
+            }
+          },
+        ),
+      );
+      await tempDb.close();
+      debugPrint('=====================================================\n');
+    } catch (e, stackTrace) {
+      debugPrint('Error checking SQLite version:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
+    }
+
+    await databaseFactory.setDatabasesPath(
+      await databaseFactory.getDatabasesPath(),
+    );
 
     final dbPath = await _getDatabasePath();
     debugPrint('Database path: $dbPath');
 
     try {
-      db = await databaseFactoryFfi.openDatabase(
+      db = await databaseFactory.openDatabase(
         dbPath,
         options: OpenDatabaseOptions(
-          version: 1,
+          version: 2,
+          onConfigure: (db) async {
+            debugPrint('Configuring database...');
+            await db.execute('PRAGMA foreign_keys = ON');
+            debugPrint('Database configured');
+          },
           onCreate: (db, version) async {
             debugPrint('Creating database schema...');
-            await db.execute(_schema);
+            await db.execute(_baseSchema);
+
+            // Check for Unicode support on first creation
+            _hasIcuSupport = await _checkIcuSupport(db);
+            await _recreateFtsTable(db, _hasIcuSupport);
+
             debugPrint('Database schema created successfully');
           },
+          onOpen: (db) async {
+            debugPrint('Database opened, checking Unicode support...');
+            try {
+              _hasIcuSupport = await _checkIcuSupport(db);
+              debugPrint('Unicode support check completed: $_hasIcuSupport');
+            } catch (e, stackTrace) {
+              debugPrint('Error during Unicode support check:');
+              debugPrint('Error: $e');
+              debugPrint('Stack trace: $stackTrace');
+              rethrow;
+            }
+          },
+          onUpgrade: (db, oldVersion, newVersion) async {
+            debugPrint('Upgrading database from $oldVersion to $newVersion');
+
+            if (oldVersion < 2) {
+              // Check for Unicode support during upgrade
+              _hasIcuSupport = await _checkIcuSupport(db);
+              await _recreateFtsTable(db, _hasIcuSupport);
+            }
+          },
         ),
       );
-      debugPrint('Database opened and initialized');
+
-    } catch (e) {
+      // Store Unicode support status in settings for future reference
-      debugPrint('Failed to initialize database: $e');
+      await setSetting('has_icu_support', _hasIcuSupport.toString());
+      debugPrint(
+        'Database opened and initialized (Unicode support: $_hasIcuSupport)',
+      );
+    } catch (e, stackTrace) {
+      debugPrint('Failed to initialize database:');
+      debugPrint('Error: $e');
+      debugPrint('Stack trace: $stackTrace');
       rethrow;
     }
   }
@@ -154,16 +454,21 @@ END;
         return [];
       }
 
-      // Add wildcards to each term for prefix matching (e.g., "fuck*" will match "fucked")
+      // Add wildcards to each term for prefix matching
       // Join terms with AND for all-term matching (results must contain ALL terms)
       String ftsQuery = terms
           .map((term) {
-            // Remove any special characters that might break the query
+            // Only remove dangerous characters but preserve Unicode
-            String sanitizedTerm = term.replaceAll(RegExp(r'[^\w]'), '');
+            String sanitizedTerm = term.replaceAll(RegExp(r'''['\\]'''), '');
             if (sanitizedTerm.isEmpty) return '';
 
-            // Add wildcard for stemming/prefix matching
+            if (_hasIcuSupport) {
-            return '$sanitizedTerm*';
+              // With ICU support, we can use wildcards with Unicode
+              return '*$sanitizedTerm*';
+            } else {
+              // Without ICU, just use the term as is for basic matching
+              return sanitizedTerm;
+            }
           })
           .where((term) => term.isNotEmpty)
           .join(' AND ');
@@ -173,7 +478,7 @@ END;
         return [];
       }
 
-      debugPrint('FTS query: "$ftsQuery"');
+      debugPrint('FTS query: "$ftsQuery" (ICU: $_hasIcuSupport)');
 
       // Execute the FTS query with AND logic
       final List<Map<String, dynamic>> results = await db.rawQuery(
@@ -192,8 +497,7 @@ END;
       return results;
     } catch (e) {
       debugPrint('Search failed: $e');
-      // Return empty results rather than crashing on malformed queries
+      rethrow;
-      return [];
     }
   }
 }

pubspec.yaml

@@ -69,6 +69,7 @@ flutter:
   assets:
     - assets/ # Include the main assets directory for the icon
     - assets/sounds/
+    - assets/windows/sqlite3_icu.dll
   #   - images/a_dot_burr.jpeg
   #   - images/a_dot_ham.jpeg