dave/journaler
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 29 Wiki Activity

Rework everything to use meilisearch

Browse Source
This commit is contained in:
PhatPhuckDave
2025-05-23 21:36:36 +02:00
parent 597ce8c9cf
commit 89f8889f1e
5 changed files with 362 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
lib/main.dart
View File

@@ -11,6 +11,7 @@ import 'package:flutter/gestures.dart';
import 'dart:math';
import 'package:path/path.dart' as path;
import 'package:ps_list/ps_list.dart';
import 'package:journaler/meilisearch.dart';
// TODO: Sound does not play when ran from a different workdir? Weird
// TODO: Fix saving the same scratch over and over again
@@ -481,7 +482,7 @@ class MainPageState extends State<MainPage> with WindowListener {
return;
}
final prev = await getPreviousNote(_currentlyDisplayedNote!.date);
final prev = await getPreviousTo(_currentlyDisplayedNote!.date);
final bool isLatest = _currentlyDisplayedNote!.date == previousNote?.date;
setState(() {
@@ -496,10 +497,10 @@ class MainPageState extends State<MainPage> with WindowListener {
// Save the current note content before navigating away
if (_currentlyDisplayedNote != null) {
_currentlyDisplayedNote!.content = _previousEntryController.text;
await updateNote(_currentlyDisplayedNote!);
await updateNote(_currentlyDisplayedNote!.date, _currentlyDisplayedNote!.content);
}
final prevNote = await getPreviousNote(_currentlyDisplayedNote!.date);
final prevNote = await getPreviousTo(_currentlyDisplayedNote!.date);
if (prevNote != null) {
setState(() {
_currentlyDisplayedNote = prevNote;
@@ -515,10 +516,10 @@ class MainPageState extends State<MainPage> with WindowListener {
// Save the current note content before navigating away
if (_currentlyDisplayedNote != null) {
_currentlyDisplayedNote!.content = _previousEntryController.text;
await updateNote(_currentlyDisplayedNote!);
await updateNote(_currentlyDisplayedNote!.date, _currentlyDisplayedNote!.content);
}
final nextNote = await getNextNote(_currentlyDisplayedNote!.date);
final nextNote = await getNextTo(_currentlyDisplayedNote!.date);
if (nextNote != null) {
setState(() {
_currentlyDisplayedNote = nextNote;
@@ -543,7 +544,7 @@ class MainPageState extends State<MainPage> with WindowListener {
_startPopupTimer();
final note = await getLatestNote();
final note = await getLatest();
previousNote = note;
_currentlyDisplayedNote = note;
_previousEntryController.text = _currentlyDisplayedNote?.content ?? "";
@@ -594,12 +595,12 @@ class MainPageState extends State<MainPage> with WindowListener {
// Handle previous/currently displayed note
if (_currentlyDisplayedNote != null) {
_currentlyDisplayedNote!.content = previousEntry;
await updateNote(_currentlyDisplayedNote!);
await updateNote(_currentlyDisplayedNote!.date, _currentlyDisplayedNote!.content);
// If the note was deleted (due to being empty), update the UI state
if (previousEntry.isEmpty) {
// Check if we need to navigate to another note
Note? nextNote = await getLatestNote();
Note? nextNote = await getLatest();
setState(() {
_currentlyDisplayedNote = nextNote;
if (nextNote != null) {
@@ -838,7 +839,8 @@ class MainPageState extends State<MainPage> with WindowListener {
_currentlyDisplayedNote!.content =
_previousEntryController.text;
await updateNote(
_currentlyDisplayedNote!,
_currentlyDisplayedNote!.date,
_currentlyDisplayedNote!.content,
);
}
@@ -913,8 +915,8 @@ class MainPageState extends State<MainPage> with WindowListener {
// Show cleanup dialog
void _showCleanupDialog() async {
double sensitivity = 0.7; // Default 70%
final problematicEntries = await findProblematicEntries(
maxCharPercentage: sensitivity,
final problematicEntries = await getProblematic(
threshold: sensitivity,
);
if (!mounted) return;
@@ -948,8 +950,8 @@ class MainPageState extends State<MainPage> with WindowListener {
100; // Round to 2 decimal places
});
// Refresh results with new sensitivity
final newResults = await findProblematicEntries(
maxCharPercentage: sensitivity,
final newResults = await getProblematic(
threshold: sensitivity,
);
dialogSetState(() {
problematicEntries.clear();

343
lib/meilisearch.dart Normal file
View File

@@ -0,0 +1,343 @@
import 'dart:convert';
import 'dart:core';
import 'package:http/http.dart' as http;
import 'package:journaler/notes.dart';
const endpoint = 'https://meili.site.quack-lab.dev/';
const noteIndex = 'notes';
const scratchIndex = 'scratch';
const settingsIndex = 'settings';
const apiKey = String.fromEnvironment('MEILISEARCH_API_KEY', defaultValue: '');
class MeilisearchQuery {
String q;
String? filter;
int? limit;
int? offset;
bool? showRankingScore;
double? rankingScoreThreshold;
String? highlightPreTag;
String? highlightPostTag;
List<String>? attributesToHighlight;
List<String>? sort;
MeilisearchQuery({
required this.q,
this.filter,
this.sort,
this.limit,
this.offset,
this.showRankingScore,
this.rankingScoreThreshold,
this.highlightPreTag,
this.highlightPostTag,
this.attributesToHighlight,
});
Map<String, dynamic> toJson() {
final Map<String, dynamic> json = {'q': q};
if (filter != null) json['filter'] = filter;
if (sort != null) json['sort'] = sort;
if (limit != null) json['limit'] = limit;
if (offset != null) json['offset'] = offset;
if (showRankingScore != null) json['showRankingScore'] = showRankingScore;
if (rankingScoreThreshold != null) {
json['rankingScoreThreshold'] = rankingScoreThreshold;
}
if (highlightPreTag != null) json['highlightPreTag'] = highlightPreTag;
if (highlightPostTag != null) json['highlightPostTag'] = highlightPostTag;
if (attributesToHighlight != null) {
json['attributesToHighlight'] = attributesToHighlight;
}
return json;
}
}
class MeilisearchResponse {
final List<Map<String, dynamic>> hits;
final String query;
final int processingTimeMs;
final int limit;
final int offset;
final double estimatedTotalHits;
MeilisearchResponse({
required this.hits,
required this.query,
required this.processingTimeMs,
required this.limit,
required this.offset,
required this.estimatedTotalHits,
});
static MeilisearchResponse fromJson(Map<String, dynamic> json) {
return MeilisearchResponse(
hits: json['hits'],
query: json['query'],
processingTimeMs: json['processingTimeMs'],
limit: json['limit'],
offset: json['offset'],
estimatedTotalHits: json['estimatedTotalHits'],
);
}
}
// Settings Management
Future<String?> getSetting(String key) async {
final searchCondition = MeilisearchQuery(q: '', filter: 'key = $key');
final response = await http.post(
Uri.parse('$endpoint/indexes/$settingsIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to get settings');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
if (responseJson.hits.isEmpty) {
return null;
}
return responseJson.hits.first['value'] as String?;
}
Future<void> setSetting(String key, String value) async {
final document = {'key': key, 'value': value};
final response = await http.post(
Uri.parse('$endpoint/indexes/$settingsIndex/documents'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(document),
);
if (response.statusCode != 200) {
throw Exception('Failed to set settings');
}
}
// Maybe we could factor a lot of this out into a separate function
// But we don't care for now...
Future<List<Note>> searchNotes(String query) async {
final searchCondition = MeilisearchQuery(
q: query,
limit: 1000,
attributesToHighlight: ['content'],
showRankingScore: true,
highlightPreTag: '<b>',
highlightPostTag: '</b>',
);
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to search notes');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
return responseJson.hits
.map(
(hit) => Note(
date: hit['date'] as String,
content: hit['content'] as String,
),
)
.toList();
}
Future<Note?> getPreviousTo(String date) async {
final searchCondition = MeilisearchQuery(
q: '',
filter: 'date < $date',
sort: ['date DESC'],
limit: 1,
);
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to get previous note');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
if (responseJson.hits.isEmpty) {
return null;
}
return Note(
date: responseJson.hits.first['date'] as String,
content: responseJson.hits.first['content'] as String,
);
}
Future<Note?> getNextTo(String date) async {
final searchCondition = MeilisearchQuery(
q: '',
filter: 'date > $date',
sort: ['date ASC'],
limit: 1,
);
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to get next note');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
if (responseJson.hits.isEmpty) {
return null;
}
return Note(
date: responseJson.hits.first['date'] as String,
content: responseJson.hits.first['content'] as String,
);
}
Future<Note?> getLatest() async {
final searchCondition = MeilisearchQuery(
q: '',
sort: ['date DESC'],
limit: 1,
);
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to get latest note');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
if (responseJson.hits.isEmpty) {
return null;
}
return Note(
date: responseJson.hits.first['date'] as String,
content: responseJson.hits.first['content'] as String,
);
}
Future<Note> createNote(String content) async {
final lines = content.split('\n');
final trimmedLines = <String>[];
for (final line in lines) {
final trimmedContent = line.trim().replaceAll(RegExp(r'\s{2,}'), ' ');
if (trimmedContent.isEmpty) {
continue;
}
trimmedLines.add(trimmedContent);
}
final trimmedContent = trimmedLines.join('\n');
final letterFrequency = <String, int>{};
for (final char in trimmedContent.split('')) {
letterFrequency[char] = (letterFrequency[char] ?? 0) + 1;
}
final mostFrequentLetter =
letterFrequency.entries.reduce((a, b) => a.value > b.value ? a : b).key;
final mostFrequentLetterCount = letterFrequency[mostFrequentLetter];
final document = {
'id': DateTime.now().toIso8601String(),
'content': content,
'topLetter': mostFrequentLetter,
'topLetterFrequency': mostFrequentLetterCount,
};
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/documents'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(document),
);
if (response.statusCode != 200) {
throw Exception('Failed to create note');
}
return Note(
date: document['id'] as String,
content: document['content'] as String,
);
}
Future<List<Note>> getProblematic({double threshold = 0.7}) async {
final searchCondition = MeilisearchQuery(
q: '',
filter: 'topLetterFrequency > $threshold',
);
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to get problematic notes');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
return responseJson.hits
.map(
(hit) => Note(
date: hit['date'] as String,
content: hit['content'] as String,
isProblematic: true,
problemReason:
'Character "${hit['topLetter']}" makes up ${(hit['topLetterFrequency'] * 100).toStringAsFixed(1)}% of the content',
),
)
.toList();
}
Future<void> updateNote(String id, String content) async {
final document = {'id': id, 'content': content};
final response = await http.post(
Uri.parse('$endpoint/indexes/$noteIndex/documents'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(document),
);
if (response.statusCode != 200) {
throw Exception('Failed to update note');
}
}
Future<void> deleteNote(String id) async {
final response = await http.delete(
Uri.parse('$endpoint/indexes/$noteIndex/documents/$id'),
headers: {'Authorization': 'Bearer $apiKey'},
);
if (response.statusCode != 200) {
throw Exception('Failed to delete note');
}
}
Future<Scratch?> getLatestScratch() async {
final searchCondition = MeilisearchQuery(
q: '',
sort: ['date DESC'],
limit: 1,
);
final response = await http.post(
Uri.parse('$endpoint/indexes/$scratchIndex/search'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(searchCondition.toJson()),
);
if (response.statusCode != 200) {
throw Exception('Failed to get latest scratch');
}
final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
if (responseJson.hits.isEmpty) {
return null;
}
return Scratch(
date: responseJson.hits.first['date'] as String,
content: responseJson.hits.first['content'] as String,
);
}
Future<void> createScratch(String content) async {
final document = {'id': DateTime.now().toIso8601String(), 'content': content};
final response = await http.post(
Uri.parse('$endpoint/indexes/$scratchIndex/documents'),
headers: {'Authorization': 'Bearer $apiKey'},
body: jsonEncode(document),
);
if (response.statusCode != 200) {
throw Exception('Failed to create scratch');
}
}

179
lib/notes.dart
View File

@@ -1,4 +1,3 @@
import 'package:journaler/db.dart';
import 'package:intl/intl.dart';
class Note {
@@ -28,181 +27,3 @@ class Scratch {
Scratch({required this.date, required this.content});
}
Future<Note?> getLatestNote() async {
final note = await DB.db.rawQuery(
'SELECT content, date FROM notes ORDER BY date DESC LIMIT 1',
);
if (note.isEmpty) {
return null;
}
return Note(
date: note[0]['date'] as String,
content: note[0]['content'] as String,
);
}
Future<void> createNote(String content) async {
// Trim each line, sometimes we fuck up by doing a lil "foobar "
// Maybe I should also look for \s{2,}...
final lines = content.split('\n');
final trimmedLines = <String>[];
for (final line in lines) {
final trimmedContent = line.trim().replaceAll(RegExp(r'\s{2,}'), ' ');
if (trimmedContent.isEmpty) {
continue;
}
trimmedLines.add(trimmedContent);
}
final trimmedContent = trimmedLines.join('\n');
await DB.db.insert('notes', {'content': trimmedContent});
}
Future<void> updateNote(Note note) async {
// Trim the content to avoid saving just whitespace
final trimmedContent = note.content.trim();
if (trimmedContent.isEmpty) {
// Delete the note if content is empty
await DB.db.delete('notes', where: 'date = ?', whereArgs: [note.date]);
return;
}
await DB.db.update(
'notes',
{'content': trimmedContent},
where: 'date = ?',
whereArgs: [note.date],
);
}
Future<Scratch?> getLatestScratch() async {
final scratch = await DB.db.rawQuery(
'SELECT content, date FROM scratches ORDER BY date DESC LIMIT 1',
);
if (scratch.isEmpty) {
return null;
} else {
return Scratch(
date: scratch[0]['date'] as String,
content: scratch[0]['content'] as String,
);
}
}
Future<void> createScratch(String content) async {
// Trim content but allow empty scratch notes (they might be intentionally cleared)
final trimmedContent = content.trim();
await DB.db.insert('scratches', {'content': trimmedContent});
}
// Get the note immediately older than the given date
Future<Note?> getPreviousNote(String currentDate) async {
final List<Map<String, dynamic>> notes = await DB.db.query(
'notes',
where: 'date < ?',
whereArgs: [currentDate],
orderBy: 'date DESC',
limit: 1,
);
if (notes.isNotEmpty) {
return Note(
date: notes.first['date'] as String,
content: notes.first['content'] as String,
);
}
return null;
}
// Get the note immediately newer than the given date
Future<Note?> getNextNote(String currentDate) async {
final List<Map<String, dynamic>> notes = await DB.db.query(
'notes',
where: 'date > ?',
whereArgs: [currentDate],
orderBy: 'date ASC',
limit: 1,
);
if (notes.isNotEmpty) {
return Note(
date: notes.first['date'] as String,
content: notes.first['content'] as String,
);
}
// If there's no newer note, it means we might be at the latest
// but let's double-check by explicitly getting the latest again.
// This handles the case where the `currentDate` might not be the absolute latest.
return getLatestNote();
}
/// Delete a note by its date
Future<bool> deleteNote(String date) async {
final result = await DB.db.delete(
'notes',
where: 'date = ?',
whereArgs: [date],
);
return result > 0; // Return true if a note was deleted
}
// Search notes using full-text search
Future<List<Note>> searchNotes(String query) async {
if (query.isEmpty) {
return [];
}
// Call DB search function
final List<Map<String, dynamic>> results = await DB.searchNotes(query);
// Convert results to Note objects
return results
.map(
(result) => Note(
date: result['date'] as String,
content: result['content'] as String,
snippet:
result['snippet'] as String?, // Highlighted snippets from FTS
),
)
.toList();
}
// Find potentially problematic entries based on character distribution
Future<List<Note>> findProblematicEntries({
double maxCharPercentage = 0.7,
}) async {
// Simple SQLite query that counts character occurrences using replace
final List<Map<String, dynamic>> results = await DB.db.rawQuery(
'''
WITH char_counts AS (
SELECT
id,
date,
content,
substr(content, 1, 1) as char,
(length(content) - length(replace(content, substr(content, 1, 1), ''))) as char_count,
length(content) as total_length,
cast(length(content) - length(replace(content, substr(content, 1, 1), '')) as float) / length(content) as percentage
FROM notes
)
SELECT *
FROM char_counts
WHERE percentage > ?
ORDER BY date DESC
''',
[maxCharPercentage],
);
return results
.map(
(row) => Note(
date: row['date'] as String,
content: row['content'] as String,
isProblematic: true,
problemReason:
'Character "${row['char']}" makes up ${(row['percentage'] * 100).toStringAsFixed(1)}% of the content',
),
)
.toList();
}

6
pubspec.lock
View File

@@ -201,13 +201,13 @@ packages:
source: sdk
version: "0.0.0"
http:
dependency: transitive
dependency: "direct main"
description:
name: http
sha256: fe7ab022b76f3034adc518fb6ea04a82387620e19977665ea18d30a1cf43442f
sha256: "2c11f3f94c687ee9bad77c171151672986360b2b001d109814ee7140b2cf261b"
url: "https://pub.dev"
source: hosted
version: "1.3.0"
version: "1.4.0"
http_parser:
dependency: transitive
description:

1
pubspec.yaml
View File

@@ -41,6 +41,7 @@ dependencies:
path: ^1.8.0
ps_list: ^0.0.5
intl: ^0.20.2
http: ^1.4.0
dev_dependencies:
flutter_test: