Fix loading notes when approaching newest

lib/meilifix.dart

@@ -0,0 +1,220 @@
+import 'dart:convert';
+import 'dart:core';
+import 'package:http/http.dart' as http;
+import 'package:journaler/meilisearch_config.dart';
+
+const noteIndex = 'notes';
+const scratchIndex = 'scratch';
+final alphanum = RegExp(r'[a-zA-Z0-9]', caseSensitive: false);
+
+class MeilisearchQuery {
+  String q;
+  String? filter;
+  int? limit;
+  int? offset;
+  bool? showRankingScore;
+  double? rankingScoreThreshold;
+  String? highlightPreTag;
+  String? highlightPostTag;
+  List<String>? attributesToHighlight;
+  List<String>? sort;
+
+  MeilisearchQuery({
+    required this.q,
+    this.filter,
+    this.sort,
+    this.limit,
+    this.offset,
+    this.showRankingScore,
+    this.rankingScoreThreshold,
+    this.highlightPreTag,
+    this.highlightPostTag,
+    this.attributesToHighlight,
+  });
+
+  Map<String, dynamic> toJson() {
+    final Map<String, dynamic> json = {'q': q};
+    if (filter != null) json['filter'] = filter;
+    if (sort != null) json['sort'] = sort;
+    if (limit != null) json['limit'] = limit;
+    if (offset != null) json['offset'] = offset;
+    if (showRankingScore != null) json['showRankingScore'] = showRankingScore;
+    if (rankingScoreThreshold != null) {
+      json['rankingScoreThreshold'] = rankingScoreThreshold;
+    }
+    if (highlightPreTag != null) json['highlightPreTag'] = highlightPreTag;
+    if (highlightPostTag != null) json['highlightPostTag'] = highlightPostTag;
+    if (attributesToHighlight != null) {
+      json['attributesToHighlight'] = attributesToHighlight;
+    }
+    return json;
+  }
+}
+
+class MeilisearchResponse {
+  final List<dynamic> hits;
+  final String query;
+  final int processingTimeMs;
+  final int limit;
+  final int offset;
+  final int estimatedTotalHits;
+
+  MeilisearchResponse({
+    required this.hits,
+    required this.query,
+    required this.processingTimeMs,
+    required this.limit,
+    required this.offset,
+    required this.estimatedTotalHits,
+  });
+
+  static MeilisearchResponse fromJson(Map<String, dynamic> json) {
+    return MeilisearchResponse(
+      hits: json['hits'],
+      query: json['query'],
+      processingTimeMs: json['processingTimeMs'],
+      limit: json['limit'],
+      offset: json['offset'],
+      estimatedTotalHits: json['estimatedTotalHits'],
+    );
+  }
+}
+
+Future<Map<String, String>> _getHeaders() async {
+  final apiKey = await getMeilisearchApiKey();
+  return {
+    'Authorization': 'Bearer $apiKey',
+    'Content-Type': 'application/json',
+  };
+}
+
+Future<String> _getEndpoint() async {
+  return await getMeilisearchEndpoint();
+}
+
+Future<List<Map<String, dynamic>>> getAllDocuments(String index) async {
+  final endpoint = await _getEndpoint();
+  final headers = await _getHeaders();
+  final allDocuments = <Map<String, dynamic>>[];
+  var offset = 0;
+  const limit = 100;
+
+  while (true) {
+    final response = await http.get(
+      Uri.parse('$endpoint/indexes/$index/documents?limit=$limit&offset=$offset'),
+      headers: headers,
+    );
+    if (response.statusCode != 200) {
+      throw Exception('Failed to get documents: ${response.statusCode}');
+    }
+    final responseJson = jsonDecode(response.body);
+    final documents = List<Map<String, dynamic>>.from(responseJson['results']);
+    
+    if (documents.isEmpty) {
+      break;
+    }
+    
+    allDocuments.addAll(documents);
+    print('Found ${allDocuments.length} documents so far in $index');
+    
+    if (documents.length < limit) {
+      break;
+    }
+    
+    offset += limit;
+  }
+  
+  print('Total documents found in $index: ${allDocuments.length}');
+  return allDocuments;
+}
+
+Map<String, dynamic> fixDocument(Map<String, dynamic> doc) {
+  final content = doc['content'] as String;
+  final date = doc['date'] as int;
+
+  // Calculate letter frequency
+  final letterFrequency = <String, int>{};
+  for (final char in content.split('')) {
+    if (alphanum.hasMatch(char)) {
+      letterFrequency[char] = (letterFrequency[char] ?? 0) + 1;
+    }
+  }
+
+  final mostFrequentLetter =
+      letterFrequency.entries.isEmpty
+          ? 'a'
+          : letterFrequency.entries
+              .reduce((a, b) => a.value > b.value ? a : b)
+              .key;
+
+  final mostFrequentLetterCount =
+      letterFrequency.isEmpty
+          ? 0.0
+          : letterFrequency[mostFrequentLetter]! / content.length;
+
+  return {
+    ...doc,
+    'dateISO':
+        DateTime.fromMillisecondsSinceEpoch(date).toUtc().toIso8601String(),
+    'topLetter': mostFrequentLetter,
+    'topLetterFrequency': mostFrequentLetterCount,
+  };
+}
+
+Future<void> updateDocument(String index, Map<String, dynamic> doc) async {
+  final endpoint = await _getEndpoint();
+  final headers = await _getHeaders();
+  final response = await http.post(
+    Uri.parse('$endpoint/indexes/$index/documents'),
+    headers: headers,
+    body: jsonEncode(doc),
+  );
+  if (response.statusCode != 202) {
+    throw Exception('Failed to update document: ${response.statusCode}');
+  }
+}
+
+Future<void> fixAllDocuments() async {
+  print('Fixing notes...');
+  final notes = await getAllDocuments(noteIndex);
+  final noteBatches = <List<Map<String, dynamic>>>[];
+  for (var i = 0; i < notes.length; i += 10) {
+    noteBatches.add(notes.skip(i).take(10).toList());
+  }
+
+  for (final batch in noteBatches) {
+    await Future.wait(
+      batch.map((note) async {
+        final fixed = fixDocument(note);
+        await updateDocument(noteIndex, fixed);
+        print('Fixed note: ${note['id']}');
+      }),
+    );
+  }
+
+  print('Fixing scratches...');
+  final scratches = await getAllDocuments(scratchIndex);
+  final scratchBatches = <List<Map<String, dynamic>>>[];
+  for (var i = 0; i < scratches.length; i += 10) {
+    scratchBatches.add(scratches.skip(i).take(10).toList());
+  }
+
+  for (final batch in scratchBatches) {
+    await Future.wait(
+      batch.map((scratch) async {
+        final fixed = fixDocument(scratch);
+        await updateDocument(scratchIndex, fixed);
+        print('Fixed scratch: ${scratch['id']}');
+      }),
+    );
+  }
+}
+
+void main() async {
+  try {
+    await fixAllDocuments();
+    print('All documents fixed successfully!');
+  } catch (e) {
+    print('Error fixing documents: $e');
+  }
+}

lib/meilisearch.dart

@@ -142,6 +142,11 @@ Future<void> init() async {
       body: jsonEncode({'uid': settingsIndex, 'primaryKey': 'key'}),
     );
   }
+  await http.put(
+    Uri.parse('$endpoint/indexes/$settingsIndex/settings/filterable-attributes'),
+    headers: headers,
+    body: jsonEncode(['key', 'value']),
+  );
 }
 
 Future<bool> indexExists(String index) async {
@@ -343,10 +348,14 @@ Future<Note> createNote(String content) async {
     }
   }
 
-  final mostFrequentLetter =
+  // Handle the case where there are no alphanumeric characters
-      letterFrequency.entries.reduce((a, b) => a.value > b.value ? a : b).key;
+  String mostFrequentLetter = 'a'; // Default value
-  final mostFrequentLetterCount =
+  double mostFrequentLetterCount = 0.0; // Default value
-      letterFrequency[mostFrequentLetter]! / trimmedContent.length;
+  
+  if (letterFrequency.isNotEmpty) {
+    mostFrequentLetter = letterFrequency.entries.reduce((a, b) => a.value > b.value ? a : b).key;
+    mostFrequentLetterCount = letterFrequency[mostFrequentLetter]! / (trimmedContent.length > 0 ? trimmedContent.length : 1);
+  }
 
   final document = {
     'id': generateRandomString(32),
@@ -426,10 +435,14 @@ Future<void> updateNote(Note note) async {
     }
   }
 
-  final mostFrequentLetter =
+  // Handle the case where there are no alphanumeric characters
-      letterFrequency.entries.reduce((a, b) => a.value > b.value ? a : b).key;
+  String mostFrequentLetter = 'a'; // Default value
-  final mostFrequentLetterRatio =
+  double mostFrequentLetterRatio = 0.0; // Default value
-      letterFrequency[mostFrequentLetter]! / trimmedContent.length;
+  
+  if (letterFrequency.isNotEmpty) {
+    mostFrequentLetter = letterFrequency.entries.reduce((a, b) => a.value > b.value ? a : b).key;
+    mostFrequentLetterRatio = letterFrequency[mostFrequentLetter]! / (trimmedContent.length > 0 ? trimmedContent.length : 1);
+  }
 
   final document = {
     'id': note.id,
@@ -522,3 +535,101 @@ Future<Scratch> createScratch(String content) async {
     content: document['content'] as String,
   );
 }
+
+Future<List<Note>> getNotesBefore(int epochTime, {int limit = 50}) async {
+  final endpoint = await _getEndpoint();
+  final headers = await _getHeaders();
+  final searchCondition = MeilisearchQuery(
+    q: '',
+    filter: 'date < $epochTime',
+    sort: ['date:desc'],
+    limit: limit,
+  );
+  final response = await http.post(
+    Uri.parse('$endpoint/indexes/$noteIndex/search'),
+    headers: headers,
+    body: jsonEncode(searchCondition.toJson()),
+  );
+  if (response.statusCode != 200) {
+    throw Exception(
+      'Failed to get notes before timestamp, backend responded with ${response.statusCode}',
+    );
+  }
+  final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
+  return responseJson.hits
+      .map(
+        (hit) => Note(
+          id: hit['id'] as String,
+          epochTime: hit['date'] as int,
+          content: hit['content'] as String,
+        ),
+      )
+      .toList();
+}
+
+Future<List<Note>> getNotesAfter(int epochTime, {int limit = 50}) async {
+  final endpoint = await _getEndpoint();
+  final headers = await _getHeaders();
+  final searchCondition = MeilisearchQuery(
+    q: '',
+    filter: 'date > $epochTime',
+    sort: ['date:asc'],
+    limit: limit,
+  );
+  final response = await http.post(
+    Uri.parse('$endpoint/indexes/$noteIndex/search'),
+    headers: headers,
+    body: jsonEncode(searchCondition.toJson()),
+  );
+  if (response.statusCode != 200) {
+    throw Exception(
+      'Failed to get notes after timestamp, backend responded with ${response.statusCode}',
+    );
+  }
+  final responseJson = MeilisearchResponse.fromJson(jsonDecode(response.body));
+  return responseJson.hits
+      .map(
+        (hit) => Note(
+          id: hit['id'] as String,
+          epochTime: hit['date'] as int,
+          content: hit['content'] as String,
+        ),
+      )
+      .toList();
+}
+
+Future<Duration> getPopupInterval() async {
+  final value = await getSetting('popupInterval');
+  if (value == null) {
+    return const Duration(minutes: 20);
+  }
+  return Duration(minutes: int.parse(value));
+}
+
+Future<void> setPopupInterval(Duration interval) async {
+  await setSetting('popupInterval', interval.inMinutes.toString());
+}
+
+Future<int> getCacheSizeBefore() async {
+  final value = await getSetting('cacheSizeBefore');
+  if (value == null) {
+    return 50; // Default value
+  }
+  return int.parse(value);
+}
+
+Future<void> setCacheSizeBefore(int size) async {
+  await setSetting('cacheSizeBefore', size.toString());
+}
+
+Future<int> getCacheSizeAfter() async {
+  final value = await getSetting('cacheSizeAfter');
+  if (value == null) {
+    return 50; // Default value
+  }
+  return int.parse(value);
+}
+
+Future<void> setCacheSizeAfter(int size) async {
+  await setSetting('cacheSizeAfter', size.toString());
+}