From 89f8889f1e2f6b018f6debb1bcf34924098c081a Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Fri, 23 May 2025 21:36:36 +0200 Subject: [PATCH] Rework everything to use meilisearch --- lib/main.dart | 28 ++-- lib/meilisearch.dart | 343 +++++++++++++++++++++++++++++++++++++++++++ lib/notes.dart | 179 ---------------------- pubspec.lock | 6 +- pubspec.yaml | 1 + 5 files changed, 362 insertions(+), 195 deletions(-) create mode 100644 lib/meilisearch.dart diff --git a/lib/main.dart b/lib/main.dart index 94aa687..1ce2bd4 100644 --- a/lib/main.dart +++ b/lib/main.dart @@ -11,6 +11,7 @@ import 'package:flutter/gestures.dart'; import 'dart:math'; import 'package:path/path.dart' as path; import 'package:ps_list/ps_list.dart'; +import 'package:journaler/meilisearch.dart'; // TODO: Sound does not play when ran from a different workdir? Weird // TODO: Fix saving the same scratch over and over again @@ -481,7 +482,7 @@ class MainPageState extends State with WindowListener { return; } - final prev = await getPreviousNote(_currentlyDisplayedNote!.date); + final prev = await getPreviousTo(_currentlyDisplayedNote!.date); final bool isLatest = _currentlyDisplayedNote!.date == previousNote?.date; setState(() { @@ -496,10 +497,10 @@ class MainPageState extends State with WindowListener { // Save the current note content before navigating away if (_currentlyDisplayedNote != null) { _currentlyDisplayedNote!.content = _previousEntryController.text; - await updateNote(_currentlyDisplayedNote!); + await updateNote(_currentlyDisplayedNote!.date, _currentlyDisplayedNote!.content); } - final prevNote = await getPreviousNote(_currentlyDisplayedNote!.date); + final prevNote = await getPreviousTo(_currentlyDisplayedNote!.date); if (prevNote != null) { setState(() { _currentlyDisplayedNote = prevNote; @@ -515,10 +516,10 @@ class MainPageState extends State with WindowListener { // Save the current note content before navigating away if (_currentlyDisplayedNote != null) { _currentlyDisplayedNote!.content = _previousEntryController.text; - await updateNote(_currentlyDisplayedNote!); + await updateNote(_currentlyDisplayedNote!.date, _currentlyDisplayedNote!.content); } - final nextNote = await getNextNote(_currentlyDisplayedNote!.date); + final nextNote = await getNextTo(_currentlyDisplayedNote!.date); if (nextNote != null) { setState(() { _currentlyDisplayedNote = nextNote; @@ -543,7 +544,7 @@ class MainPageState extends State with WindowListener { _startPopupTimer(); - final note = await getLatestNote(); + final note = await getLatest(); previousNote = note; _currentlyDisplayedNote = note; _previousEntryController.text = _currentlyDisplayedNote?.content ?? ""; @@ -594,12 +595,12 @@ class MainPageState extends State with WindowListener { // Handle previous/currently displayed note if (_currentlyDisplayedNote != null) { _currentlyDisplayedNote!.content = previousEntry; - await updateNote(_currentlyDisplayedNote!); + await updateNote(_currentlyDisplayedNote!.date, _currentlyDisplayedNote!.content); // If the note was deleted (due to being empty), update the UI state if (previousEntry.isEmpty) { // Check if we need to navigate to another note - Note? nextNote = await getLatestNote(); + Note? nextNote = await getLatest(); setState(() { _currentlyDisplayedNote = nextNote; if (nextNote != null) { @@ -838,7 +839,8 @@ class MainPageState extends State with WindowListener { _currentlyDisplayedNote!.content = _previousEntryController.text; await updateNote( - _currentlyDisplayedNote!, + _currentlyDisplayedNote!.date, + _currentlyDisplayedNote!.content, ); } @@ -913,8 +915,8 @@ class MainPageState extends State with WindowListener { // Show cleanup dialog void _showCleanupDialog() async { double sensitivity = 0.7; // Default 70% - final problematicEntries = await findProblematicEntries( - maxCharPercentage: sensitivity, + final problematicEntries = await getProblematic( + threshold: sensitivity, ); if (!mounted) return; @@ -948,8 +950,8 @@ class MainPageState extends State with WindowListener { 100; // Round to 2 decimal places }); // Refresh results with new sensitivity - final newResults = await findProblematicEntries( - maxCharPercentage: sensitivity, + final newResults = await getProblematic( + threshold: sensitivity, ); dialogSetState(() { problematicEntries.clear(); diff --git a/lib/meilisearch.dart b/lib/meilisearch.dart new file mode 100644 index 0000000..1ace5c8 --- /dev/null +++ b/lib/meilisearch.dart @@ -0,0 +1,343 @@ +import 'dart:convert'; +import 'dart:core'; +import 'package:http/http.dart' as http; +import 'package:journaler/notes.dart'; + +const endpoint = 'https://meili.site.quack-lab.dev/'; +const noteIndex = 'notes'; +const scratchIndex = 'scratch'; +const settingsIndex = 'settings'; +const apiKey = String.fromEnvironment('MEILISEARCH_API_KEY', defaultValue: ''); + +class MeilisearchQuery { + String q; + String? filter; + int? limit; + int? offset; + bool? showRankingScore; + double? rankingScoreThreshold; + String? highlightPreTag; + String? highlightPostTag; + List? attributesToHighlight; + List? sort; + + MeilisearchQuery({ + required this.q, + this.filter, + this.sort, + this.limit, + this.offset, + this.showRankingScore, + this.rankingScoreThreshold, + this.highlightPreTag, + this.highlightPostTag, + this.attributesToHighlight, + }); + + Map toJson() { + final Map json = {'q': q}; + if (filter != null) json['filter'] = filter; + if (sort != null) json['sort'] = sort; + if (limit != null) json['limit'] = limit; + if (offset != null) json['offset'] = offset; + if (showRankingScore != null) json['showRankingScore'] = showRankingScore; + if (rankingScoreThreshold != null) { + json['rankingScoreThreshold'] = rankingScoreThreshold; + } + if (highlightPreTag != null) json['highlightPreTag'] = highlightPreTag; + if (highlightPostTag != null) json['highlightPostTag'] = highlightPostTag; + if (attributesToHighlight != null) { + json['attributesToHighlight'] = attributesToHighlight; + } + return json; + } +} + +class MeilisearchResponse { + final List> hits; + final String query; + final int processingTimeMs; + final int limit; + final int offset; + final double estimatedTotalHits; + + MeilisearchResponse({ + required this.hits, + required this.query, + required this.processingTimeMs, + required this.limit, + required this.offset, + required this.estimatedTotalHits, + }); + + static MeilisearchResponse fromJson(Map json) { + return MeilisearchResponse( + hits: json['hits'], + query: json['query'], + processingTimeMs: json['processingTimeMs'], + limit: json['limit'], + offset: json['offset'], + estimatedTotalHits: json['estimatedTotalHits'], + ); + } +} + +// Settings Management +Future getSetting(String key) async { + final searchCondition = MeilisearchQuery(q: '', filter: 'key = $key'); + final response = await http.post( + Uri.parse('$endpoint/indexes/$settingsIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get settings'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + if (responseJson.hits.isEmpty) { + return null; + } + return responseJson.hits.first['value'] as String?; +} + +Future setSetting(String key, String value) async { + final document = {'key': key, 'value': value}; + final response = await http.post( + Uri.parse('$endpoint/indexes/$settingsIndex/documents'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(document), + ); + if (response.statusCode != 200) { + throw Exception('Failed to set settings'); + } +} + +// Maybe we could factor a lot of this out into a separate function +// But we don't care for now... +Future> searchNotes(String query) async { + final searchCondition = MeilisearchQuery( + q: query, + limit: 1000, + attributesToHighlight: ['content'], + showRankingScore: true, + highlightPreTag: '', + highlightPostTag: '', + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to search notes'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + return responseJson.hits + .map( + (hit) => Note( + date: hit['date'] as String, + content: hit['content'] as String, + ), + ) + .toList(); +} + +Future getPreviousTo(String date) async { + final searchCondition = MeilisearchQuery( + q: '', + filter: 'date < $date', + sort: ['date DESC'], + limit: 1, + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get previous note'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + if (responseJson.hits.isEmpty) { + return null; + } + return Note( + date: responseJson.hits.first['date'] as String, + content: responseJson.hits.first['content'] as String, + ); +} + +Future getNextTo(String date) async { + final searchCondition = MeilisearchQuery( + q: '', + filter: 'date > $date', + sort: ['date ASC'], + limit: 1, + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get next note'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + if (responseJson.hits.isEmpty) { + return null; + } + return Note( + date: responseJson.hits.first['date'] as String, + content: responseJson.hits.first['content'] as String, + ); +} + +Future getLatest() async { + final searchCondition = MeilisearchQuery( + q: '', + sort: ['date DESC'], + limit: 1, + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get latest note'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + if (responseJson.hits.isEmpty) { + return null; + } + return Note( + date: responseJson.hits.first['date'] as String, + content: responseJson.hits.first['content'] as String, + ); +} + +Future createNote(String content) async { + final lines = content.split('\n'); + final trimmedLines = []; + for (final line in lines) { + final trimmedContent = line.trim().replaceAll(RegExp(r'\s{2,}'), ' '); + if (trimmedContent.isEmpty) { + continue; + } + trimmedLines.add(trimmedContent); + } + final trimmedContent = trimmedLines.join('\n'); + + final letterFrequency = {}; + for (final char in trimmedContent.split('')) { + letterFrequency[char] = (letterFrequency[char] ?? 0) + 1; + } + + final mostFrequentLetter = + letterFrequency.entries.reduce((a, b) => a.value > b.value ? a : b).key; + final mostFrequentLetterCount = letterFrequency[mostFrequentLetter]; + + final document = { + 'id': DateTime.now().toIso8601String(), + 'content': content, + 'topLetter': mostFrequentLetter, + 'topLetterFrequency': mostFrequentLetterCount, + }; + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/documents'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(document), + ); + if (response.statusCode != 200) { + throw Exception('Failed to create note'); + } + return Note( + date: document['id'] as String, + content: document['content'] as String, + ); +} + +Future> getProblematic({double threshold = 0.7}) async { + final searchCondition = MeilisearchQuery( + q: '', + filter: 'topLetterFrequency > $threshold', + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get problematic notes'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + return responseJson.hits + .map( + (hit) => Note( + date: hit['date'] as String, + content: hit['content'] as String, + isProblematic: true, + problemReason: + 'Character "${hit['topLetter']}" makes up ${(hit['topLetterFrequency'] * 100).toStringAsFixed(1)}% of the content', + ), + ) + .toList(); +} + +Future updateNote(String id, String content) async { + final document = {'id': id, 'content': content}; + final response = await http.post( + Uri.parse('$endpoint/indexes/$noteIndex/documents'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(document), + ); + if (response.statusCode != 200) { + throw Exception('Failed to update note'); + } +} + +Future deleteNote(String id) async { + final response = await http.delete( + Uri.parse('$endpoint/indexes/$noteIndex/documents/$id'), + headers: {'Authorization': 'Bearer $apiKey'}, + ); + if (response.statusCode != 200) { + throw Exception('Failed to delete note'); + } +} + +Future getLatestScratch() async { + final searchCondition = MeilisearchQuery( + q: '', + sort: ['date DESC'], + limit: 1, + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$scratchIndex/search'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get latest scratch'); + } + final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body)); + if (responseJson.hits.isEmpty) { + return null; + } + return Scratch( + date: responseJson.hits.first['date'] as String, + content: responseJson.hits.first['content'] as String, + ); +} + +Future createScratch(String content) async { + final document = {'id': DateTime.now().toIso8601String(), 'content': content}; + final response = await http.post( + Uri.parse('$endpoint/indexes/$scratchIndex/documents'), + headers: {'Authorization': 'Bearer $apiKey'}, + body: jsonEncode(document), + ); + if (response.statusCode != 200) { + throw Exception('Failed to create scratch'); + } +} diff --git a/lib/notes.dart b/lib/notes.dart index 08def38..bd52d60 100644 --- a/lib/notes.dart +++ b/lib/notes.dart @@ -1,4 +1,3 @@ -import 'package:journaler/db.dart'; import 'package:intl/intl.dart'; class Note { @@ -28,181 +27,3 @@ class Scratch { Scratch({required this.date, required this.content}); } - -Future getLatestNote() async { - final note = await DB.db.rawQuery( - 'SELECT content, date FROM notes ORDER BY date DESC LIMIT 1', - ); - if (note.isEmpty) { - return null; - } - return Note( - date: note[0]['date'] as String, - content: note[0]['content'] as String, - ); -} - -Future createNote(String content) async { - // Trim each line, sometimes we fuck up by doing a lil "foobar " - // Maybe I should also look for \s{2,}... - final lines = content.split('\n'); - final trimmedLines = []; - for (final line in lines) { - final trimmedContent = line.trim().replaceAll(RegExp(r'\s{2,}'), ' '); - if (trimmedContent.isEmpty) { - continue; - } - trimmedLines.add(trimmedContent); - } - final trimmedContent = trimmedLines.join('\n'); - await DB.db.insert('notes', {'content': trimmedContent}); -} - -Future updateNote(Note note) async { - // Trim the content to avoid saving just whitespace - final trimmedContent = note.content.trim(); - if (trimmedContent.isEmpty) { - // Delete the note if content is empty - await DB.db.delete('notes', where: 'date = ?', whereArgs: [note.date]); - return; - } - - await DB.db.update( - 'notes', - {'content': trimmedContent}, - where: 'date = ?', - whereArgs: [note.date], - ); -} - -Future getLatestScratch() async { - final scratch = await DB.db.rawQuery( - 'SELECT content, date FROM scratches ORDER BY date DESC LIMIT 1', - ); - - if (scratch.isEmpty) { - return null; - } else { - return Scratch( - date: scratch[0]['date'] as String, - content: scratch[0]['content'] as String, - ); - } -} - -Future createScratch(String content) async { - // Trim content but allow empty scratch notes (they might be intentionally cleared) - final trimmedContent = content.trim(); - await DB.db.insert('scratches', {'content': trimmedContent}); -} - -// Get the note immediately older than the given date -Future getPreviousNote(String currentDate) async { - final List> notes = await DB.db.query( - 'notes', - where: 'date < ?', - whereArgs: [currentDate], - orderBy: 'date DESC', - limit: 1, - ); - if (notes.isNotEmpty) { - return Note( - date: notes.first['date'] as String, - content: notes.first['content'] as String, - ); - } - return null; -} - -// Get the note immediately newer than the given date -Future getNextNote(String currentDate) async { - final List> notes = await DB.db.query( - 'notes', - where: 'date > ?', - whereArgs: [currentDate], - orderBy: 'date ASC', - limit: 1, - ); - if (notes.isNotEmpty) { - return Note( - date: notes.first['date'] as String, - content: notes.first['content'] as String, - ); - } - // If there's no newer note, it means we might be at the latest - // but let's double-check by explicitly getting the latest again. - // This handles the case where the `currentDate` might not be the absolute latest. - return getLatestNote(); -} - -/// Delete a note by its date -Future deleteNote(String date) async { - final result = await DB.db.delete( - 'notes', - where: 'date = ?', - whereArgs: [date], - ); - - return result > 0; // Return true if a note was deleted -} - -// Search notes using full-text search -Future> searchNotes(String query) async { - if (query.isEmpty) { - return []; - } - - // Call DB search function - final List> results = await DB.searchNotes(query); - - // Convert results to Note objects - return results - .map( - (result) => Note( - date: result['date'] as String, - content: result['content'] as String, - snippet: - result['snippet'] as String?, // Highlighted snippets from FTS - ), - ) - .toList(); -} - -// Find potentially problematic entries based on character distribution -Future> findProblematicEntries({ - double maxCharPercentage = 0.7, -}) async { - // Simple SQLite query that counts character occurrences using replace - final List> results = await DB.db.rawQuery( - ''' - WITH char_counts AS ( - SELECT - id, - date, - content, - substr(content, 1, 1) as char, - (length(content) - length(replace(content, substr(content, 1, 1), ''))) as char_count, - length(content) as total_length, - cast(length(content) - length(replace(content, substr(content, 1, 1), '')) as float) / length(content) as percentage - FROM notes - ) - SELECT * - FROM char_counts - WHERE percentage > ? - ORDER BY date DESC - ''', - [maxCharPercentage], - ); - - return results - .map( - (row) => Note( - date: row['date'] as String, - content: row['content'] as String, - isProblematic: true, - problemReason: - 'Character "${row['char']}" makes up ${(row['percentage'] * 100).toStringAsFixed(1)}% of the content', - ), - ) - .toList(); -} diff --git a/pubspec.lock b/pubspec.lock index 275ffe3..b2a2bd3 100644 --- a/pubspec.lock +++ b/pubspec.lock @@ -201,13 +201,13 @@ packages: source: sdk version: "0.0.0" http: - dependency: transitive + dependency: "direct main" description: name: http - sha256: fe7ab022b76f3034adc518fb6ea04a82387620e19977665ea18d30a1cf43442f + sha256: "2c11f3f94c687ee9bad77c171151672986360b2b001d109814ee7140b2cf261b" url: "https://pub.dev" source: hosted - version: "1.3.0" + version: "1.4.0" http_parser: dependency: transitive description: diff --git a/pubspec.yaml b/pubspec.yaml index 877cefc..2d9f0b8 100644 --- a/pubspec.yaml +++ b/pubspec.yaml @@ -41,6 +41,7 @@ dependencies: path: ^1.8.0 ps_list: ^0.0.5 intl: ^0.20.2 + http: ^1.4.0 dev_dependencies: flutter_test: