Add icu support for cyrillic fts
This commit is contained in:
BIN
assets/windows/sqlite3_icu.dll
Normal file
BIN
assets/windows/sqlite3_icu.dll
Normal file
Binary file not shown.
348
lib/db.dart
348
lib/db.dart
@@ -1,15 +1,29 @@
|
||||
import 'dart:io' show Platform, Directory;
|
||||
import 'dart:io' show Platform, Directory, File;
|
||||
import 'package:flutter/material.dart';
|
||||
import 'package:flutter/services.dart';
|
||||
import 'package:path/path.dart' as path;
|
||||
import 'package:sqflite_common_ffi/sqflite_ffi.dart';
|
||||
|
||||
const settingsDir = '.journaler';
|
||||
const dbFileName = 'journaler.db';
|
||||
|
||||
// Add this at the top level
|
||||
typedef ShowMessageCallback = void Function(String message);
|
||||
|
||||
class DB {
|
||||
static late Database db;
|
||||
static bool _hasIcuSupport = false;
|
||||
static ShowMessageCallback? _showMessage;
|
||||
|
||||
static const String _schema = '''
|
||||
// Add this to register the callback
|
||||
static void registerMessageCallback(ShowMessageCallback callback) {
|
||||
_showMessage = callback;
|
||||
}
|
||||
|
||||
// Add this method to check ICU status
|
||||
static bool get hasIcuSupport => _hasIcuSupport;
|
||||
|
||||
static const String _baseSchema = '''
|
||||
CREATE TABLE IF NOT EXISTS notes (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
date TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
@@ -17,10 +31,7 @@ CREATE TABLE IF NOT EXISTS notes (
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS idx_notes_date ON notes (date);
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS idx_notes_date_unique ON notes (date);
|
||||
-- Todos are "static", we are only interested in the latest entry
|
||||
-- ie. they're not really a list but instead a string
|
||||
-- But we will also keep a history of all todos
|
||||
-- Because we might as well
|
||||
|
||||
CREATE TABLE IF NOT EXISTS scratches (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
date TEXT DEFAULT CURRENT_TIMESTAMP,
|
||||
@@ -33,15 +44,31 @@ CREATE TABLE IF NOT EXISTS settings (
|
||||
key TEXT PRIMARY KEY NOT NULL,
|
||||
value TEXT NOT NULL
|
||||
);
|
||||
''';
|
||||
|
||||
-- Create virtual FTS5 table for searching notes content
|
||||
static const String _ftsSchemaWithIcu = '''
|
||||
-- Create virtual FTS5 table with Unicode tokenizer
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
||||
content,
|
||||
date,
|
||||
content='notes',
|
||||
content_rowid='id'
|
||||
content_rowid='id',
|
||||
tokenize='unicode61 remove_diacritics 2 tokenchars "\u0401\u0451\u0410-\u044f"'
|
||||
);
|
||||
''';
|
||||
|
||||
static const String _ftsSchemaBasic = '''
|
||||
-- Create virtual FTS5 table with basic tokenizer
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS notes_fts USING fts5(
|
||||
content,
|
||||
date,
|
||||
content='notes',
|
||||
content_rowid='id',
|
||||
tokenize='unicode61'
|
||||
);
|
||||
''';
|
||||
|
||||
static const String _ftsTriggers = '''
|
||||
-- Trigger to keep FTS table in sync with notes table when inserting
|
||||
CREATE TRIGGER IF NOT EXISTS notes_ai AFTER INSERT ON notes BEGIN
|
||||
INSERT INTO notes_fts(rowid, content, date) VALUES (new.id, new.content, new.date);
|
||||
@@ -58,6 +85,205 @@ CREATE TRIGGER IF NOT EXISTS notes_au AFTER UPDATE ON notes BEGIN
|
||||
END;
|
||||
''';
|
||||
|
||||
static Future<String?> _getIcuExtensionPath() async {
|
||||
if (!Platform.isWindows) {
|
||||
debugPrint('Not on Windows, skipping ICU');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
// Get the executable's directory
|
||||
final exePath = Platform.resolvedExecutable;
|
||||
final exeDir = path.dirname(exePath);
|
||||
final currentDir = Directory.current.path;
|
||||
|
||||
debugPrint('\n================== ICU EXTENSION SETUP ==================');
|
||||
debugPrint('Current working directory: $currentDir');
|
||||
debugPrint('Executable path: $exePath');
|
||||
debugPrint('Executable directory: $exeDir');
|
||||
|
||||
// In release mode, we want the DLL next to the exe
|
||||
final targetPath = path.join(exeDir, 'sqlite3_icu.dll');
|
||||
final targetFile = File(targetPath);
|
||||
|
||||
// If the DLL doesn't exist in the target location, copy it from assets
|
||||
if (!await targetFile.exists()) {
|
||||
debugPrint(
|
||||
'DLL not found at target location, attempting to copy from assets...',
|
||||
);
|
||||
try {
|
||||
// Load from assets and write to target location
|
||||
final assetPath = 'assets/windows/sqlite3_icu.dll';
|
||||
debugPrint('Trying to load from asset path: $assetPath');
|
||||
|
||||
final data = await rootBundle.load(assetPath);
|
||||
debugPrint(
|
||||
'Asset loaded successfully, size: ${data.lengthInBytes} bytes',
|
||||
);
|
||||
|
||||
debugPrint('Writing to target path: $targetPath');
|
||||
await targetFile.writeAsBytes(
|
||||
data.buffer.asUint8List(data.offsetInBytes, data.lengthInBytes),
|
||||
);
|
||||
debugPrint('Successfully copied DLL to: $targetPath');
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('Failed to copy DLL from assets:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
|
||||
// Try to list available assets for debugging
|
||||
try {
|
||||
final manifestContent = await rootBundle.loadString(
|
||||
'AssetManifest.json',
|
||||
);
|
||||
debugPrint('Available assets in manifest:');
|
||||
debugPrint(manifestContent);
|
||||
} catch (e) {
|
||||
debugPrint('Could not load asset manifest: $e');
|
||||
}
|
||||
return null;
|
||||
}
|
||||
} else {
|
||||
debugPrint('Found existing DLL at: $targetPath');
|
||||
}
|
||||
|
||||
debugPrint('Verifying DLL exists at: $targetPath');
|
||||
if (await targetFile.exists()) {
|
||||
final fileSize = await targetFile.length();
|
||||
debugPrint('DLL exists, size: $fileSize bytes');
|
||||
} else {
|
||||
debugPrint('DLL does not exist at target path!');
|
||||
}
|
||||
|
||||
debugPrint('=====================================================\n');
|
||||
return targetPath;
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('Error setting up ICU extension:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
debugPrint('=====================================================\n');
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Add this method to check ICU status with visible feedback
|
||||
static Future<bool> checkAndShowIcuStatus() async {
|
||||
final status = _hasIcuSupport;
|
||||
final message =
|
||||
status
|
||||
? 'ICU support is enabled - Full Unicode search available'
|
||||
: 'ICU support is not available - Unicode search will be limited\n'
|
||||
'Looking for sqlite3_icu.dll in the application directory';
|
||||
debugPrint(message);
|
||||
_showMessage?.call(message);
|
||||
return status;
|
||||
}
|
||||
|
||||
static Future<bool> _checkIcuSupport(Database db) async {
|
||||
try {
|
||||
debugPrint(
|
||||
'\n================== CHECKING UNICODE SUPPORT ==================',
|
||||
);
|
||||
|
||||
// Test Unicode support with a simple query
|
||||
try {
|
||||
debugPrint('Testing Unicode support...');
|
||||
// Test with some Cyrillic characters
|
||||
await db.rawQuery("SELECT 'тест' LIKE '%ест%'");
|
||||
await db.rawQuery("SELECT 'ТЕСТ' LIKE '%ЕСТ%'");
|
||||
|
||||
final message = 'Unicode support is available';
|
||||
debugPrint(message);
|
||||
_showMessage?.call(message);
|
||||
return true;
|
||||
} catch (e, stackTrace) {
|
||||
final message = 'Unicode support test failed';
|
||||
debugPrint('$message:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
|
||||
// Try to get SQLite version and compile options for debugging
|
||||
try {
|
||||
final version = await db.rawQuery('SELECT sqlite_version()');
|
||||
final compileOpts = await db.rawQuery('PRAGMA compile_options');
|
||||
debugPrint('SQLite version: $version');
|
||||
debugPrint('SQLite compile options: $compileOpts');
|
||||
} catch (e) {
|
||||
debugPrint('Could not get SQLite info: $e');
|
||||
}
|
||||
|
||||
_showMessage?.call(message);
|
||||
return false;
|
||||
}
|
||||
} catch (e, stackTrace) {
|
||||
final message = 'Failed to test Unicode support';
|
||||
debugPrint('$message:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
_showMessage?.call(message);
|
||||
return false;
|
||||
} finally {
|
||||
debugPrint('=====================================================\n');
|
||||
}
|
||||
}
|
||||
|
||||
static Future<void> _recreateFtsTable(Database db, bool useIcu) async {
|
||||
debugPrint('Updating FTS table configuration...');
|
||||
|
||||
try {
|
||||
// Start a transaction to ensure data safety
|
||||
await db.transaction((txn) async {
|
||||
// First, create a temporary table with the new configuration
|
||||
final tempTableName = 'notes_fts_temp';
|
||||
final schema = useIcu ? _ftsSchemaWithIcu : _ftsSchemaBasic;
|
||||
|
||||
// Create temp table with new configuration
|
||||
await txn.execute(schema.replaceAll('notes_fts', tempTableName));
|
||||
|
||||
// Copy data from old FTS table if it exists
|
||||
try {
|
||||
debugPrint('Copying existing FTS data to temporary table...');
|
||||
await txn.execute('''
|
||||
INSERT INTO $tempTableName(rowid, content, date)
|
||||
SELECT rowid, content, date FROM notes_fts
|
||||
''');
|
||||
debugPrint('Data copied successfully');
|
||||
} catch (e) {
|
||||
debugPrint('No existing FTS data to copy: $e');
|
||||
}
|
||||
|
||||
// Drop old triggers
|
||||
debugPrint('Updating triggers...');
|
||||
await txn.execute('DROP TRIGGER IF EXISTS notes_ai');
|
||||
await txn.execute('DROP TRIGGER IF EXISTS notes_ad');
|
||||
await txn.execute('DROP TRIGGER IF EXISTS notes_au');
|
||||
|
||||
// Drop old FTS table
|
||||
await txn.execute('DROP TABLE IF EXISTS notes_fts');
|
||||
|
||||
// Rename temp table to final name
|
||||
await txn.execute('ALTER TABLE $tempTableName RENAME TO notes_fts');
|
||||
|
||||
// Create new triggers
|
||||
await txn.execute(_ftsTriggers);
|
||||
|
||||
// Rebuild FTS index from notes table to ensure consistency
|
||||
debugPrint('Rebuilding FTS index from notes table...');
|
||||
await txn.execute('''
|
||||
INSERT OR REPLACE INTO notes_fts(rowid, content, date)
|
||||
SELECT id, content, date FROM notes
|
||||
''');
|
||||
|
||||
debugPrint('FTS table update completed successfully');
|
||||
});
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('Error updating FTS table:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
|
||||
static Future<String> _getDatabasePath() async {
|
||||
debugPrint('Attempting to get database path...');
|
||||
if (Platform.isWindows || Platform.isLinux) {
|
||||
@@ -88,26 +314,100 @@ END;
|
||||
|
||||
static Future<void> init() async {
|
||||
debugPrint('Starting database initialization...');
|
||||
|
||||
// Initialize SQLite FFI
|
||||
sqfliteFfiInit();
|
||||
final databaseFactory = databaseFactoryFfi;
|
||||
|
||||
// Create a temporary database to check version
|
||||
try {
|
||||
debugPrint(
|
||||
'\n================== SQLITE VERSION CHECK ==================',
|
||||
);
|
||||
final tempDb = await databaseFactory.openDatabase(
|
||||
':memory:',
|
||||
options: OpenDatabaseOptions(
|
||||
version: 1,
|
||||
onCreate: (db, version) async {
|
||||
final results = await db.rawQuery('SELECT sqlite_version()');
|
||||
debugPrint('SQLite version: ${results.first.values.first}');
|
||||
|
||||
final compileOpts = await db.rawQuery('PRAGMA compile_options');
|
||||
debugPrint('SQLite compile options:');
|
||||
for (var opt in compileOpts) {
|
||||
debugPrint(' ${opt.values.first}');
|
||||
}
|
||||
},
|
||||
),
|
||||
);
|
||||
await tempDb.close();
|
||||
debugPrint('=====================================================\n');
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('Error checking SQLite version:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
}
|
||||
|
||||
await databaseFactory.setDatabasesPath(
|
||||
await databaseFactory.getDatabasesPath(),
|
||||
);
|
||||
|
||||
final dbPath = await _getDatabasePath();
|
||||
debugPrint('Database path: $dbPath');
|
||||
|
||||
try {
|
||||
db = await databaseFactoryFfi.openDatabase(
|
||||
db = await databaseFactory.openDatabase(
|
||||
dbPath,
|
||||
options: OpenDatabaseOptions(
|
||||
version: 1,
|
||||
version: 2,
|
||||
onConfigure: (db) async {
|
||||
debugPrint('Configuring database...');
|
||||
await db.execute('PRAGMA foreign_keys = ON');
|
||||
debugPrint('Database configured');
|
||||
},
|
||||
onCreate: (db, version) async {
|
||||
debugPrint('Creating database schema...');
|
||||
await db.execute(_schema);
|
||||
await db.execute(_baseSchema);
|
||||
|
||||
// Check for Unicode support on first creation
|
||||
_hasIcuSupport = await _checkIcuSupport(db);
|
||||
await _recreateFtsTable(db, _hasIcuSupport);
|
||||
|
||||
debugPrint('Database schema created successfully');
|
||||
},
|
||||
onOpen: (db) async {
|
||||
debugPrint('Database opened, checking Unicode support...');
|
||||
try {
|
||||
_hasIcuSupport = await _checkIcuSupport(db);
|
||||
debugPrint('Unicode support check completed: $_hasIcuSupport');
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('Error during Unicode support check:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
rethrow;
|
||||
}
|
||||
},
|
||||
onUpgrade: (db, oldVersion, newVersion) async {
|
||||
debugPrint('Upgrading database from $oldVersion to $newVersion');
|
||||
|
||||
if (oldVersion < 2) {
|
||||
// Check for Unicode support during upgrade
|
||||
_hasIcuSupport = await _checkIcuSupport(db);
|
||||
await _recreateFtsTable(db, _hasIcuSupport);
|
||||
}
|
||||
},
|
||||
),
|
||||
);
|
||||
debugPrint('Database opened and initialized');
|
||||
} catch (e) {
|
||||
debugPrint('Failed to initialize database: $e');
|
||||
|
||||
// Store Unicode support status in settings for future reference
|
||||
await setSetting('has_icu_support', _hasIcuSupport.toString());
|
||||
debugPrint(
|
||||
'Database opened and initialized (Unicode support: $_hasIcuSupport)',
|
||||
);
|
||||
} catch (e, stackTrace) {
|
||||
debugPrint('Failed to initialize database:');
|
||||
debugPrint('Error: $e');
|
||||
debugPrint('Stack trace: $stackTrace');
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
@@ -154,16 +454,21 @@ END;
|
||||
return [];
|
||||
}
|
||||
|
||||
// Add wildcards to each term for prefix matching (e.g., "fuck*" will match "fucked")
|
||||
// Add wildcards to each term for prefix matching
|
||||
// Join terms with AND for all-term matching (results must contain ALL terms)
|
||||
String ftsQuery = terms
|
||||
.map((term) {
|
||||
// Remove any special characters that might break the query
|
||||
String sanitizedTerm = term.replaceAll(RegExp(r'[^\w]'), '');
|
||||
// Only remove dangerous characters but preserve Unicode
|
||||
String sanitizedTerm = term.replaceAll(RegExp(r'''['\\]'''), '');
|
||||
if (sanitizedTerm.isEmpty) return '';
|
||||
|
||||
// Add wildcard for stemming/prefix matching
|
||||
return '$sanitizedTerm*';
|
||||
if (_hasIcuSupport) {
|
||||
// With ICU support, we can use wildcards with Unicode
|
||||
return '*$sanitizedTerm*';
|
||||
} else {
|
||||
// Without ICU, just use the term as is for basic matching
|
||||
return sanitizedTerm;
|
||||
}
|
||||
})
|
||||
.where((term) => term.isNotEmpty)
|
||||
.join(' AND ');
|
||||
@@ -173,7 +478,7 @@ END;
|
||||
return [];
|
||||
}
|
||||
|
||||
debugPrint('FTS query: "$ftsQuery"');
|
||||
debugPrint('FTS query: "$ftsQuery" (ICU: $_hasIcuSupport)');
|
||||
|
||||
// Execute the FTS query with AND logic
|
||||
final List<Map<String, dynamic>> results = await db.rawQuery(
|
||||
@@ -192,8 +497,7 @@ END;
|
||||
return results;
|
||||
} catch (e) {
|
||||
debugPrint('Search failed: $e');
|
||||
// Return empty results rather than crashing on malformed queries
|
||||
return [];
|
||||
rethrow;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -69,6 +69,7 @@ flutter:
|
||||
assets:
|
||||
- assets/ # Include the main assets directory for the icon
|
||||
- assets/sounds/
|
||||
- assets/windows/sqlite3_icu.dll
|
||||
# - images/a_dot_burr.jpeg
|
||||
# - images/a_dot_ham.jpeg
|
||||
|
||||
|
||||
Reference in New Issue
Block a user