From d78db03d5d610493faf77ef14cc1e7b594058ddf Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Sun, 25 May 2025 02:24:54 +0200 Subject: [PATCH] Add Meilisearch document handling and fix utilities --- lib/meilifix.dart | 225 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 225 insertions(+) create mode 100644 lib/meilifix.dart diff --git a/lib/meilifix.dart b/lib/meilifix.dart new file mode 100644 index 0000000..b73f08c --- /dev/null +++ b/lib/meilifix.dart @@ -0,0 +1,225 @@ +import 'dart:convert'; +import 'dart:core'; +import 'package:http/http.dart' as http; +import 'package:journaler/meilisearch_config.dart'; + +const noteIndex = 'notes'; +const scratchIndex = 'scratch'; +final alphanum = RegExp(r'[a-zA-Z0-9]', caseSensitive: false); + +class MeilisearchQuery { + String q; + String? filter; + int? limit; + int? offset; + bool? showRankingScore; + double? rankingScoreThreshold; + String? highlightPreTag; + String? highlightPostTag; + List? attributesToHighlight; + List? sort; + + MeilisearchQuery({ + required this.q, + this.filter, + this.sort, + this.limit, + this.offset, + this.showRankingScore, + this.rankingScoreThreshold, + this.highlightPreTag, + this.highlightPostTag, + this.attributesToHighlight, + }); + + Map toJson() { + final Map json = {'q': q}; + if (filter != null) json['filter'] = filter; + if (sort != null) json['sort'] = sort; + if (limit != null) json['limit'] = limit; + if (offset != null) json['offset'] = offset; + if (showRankingScore != null) json['showRankingScore'] = showRankingScore; + if (rankingScoreThreshold != null) { + json['rankingScoreThreshold'] = rankingScoreThreshold; + } + if (highlightPreTag != null) json['highlightPreTag'] = highlightPreTag; + if (highlightPostTag != null) json['highlightPostTag'] = highlightPostTag; + if (attributesToHighlight != null) { + json['attributesToHighlight'] = attributesToHighlight; + } + return json; + } +} + +class MeilisearchResponse { + final List hits; + final String query; + final int processingTimeMs; + final int limit; + final int offset; + final int estimatedTotalHits; + + MeilisearchResponse({ + required this.hits, + required this.query, + required this.processingTimeMs, + required this.limit, + required this.offset, + required this.estimatedTotalHits, + }); + + static MeilisearchResponse fromJson(Map json) { + return MeilisearchResponse( + hits: json['hits'], + query: json['query'], + processingTimeMs: json['processingTimeMs'], + limit: json['limit'], + offset: json['offset'], + estimatedTotalHits: json['estimatedTotalHits'], + ); + } +} + +Future> _getHeaders() async { + final apiKey = await getMeilisearchApiKey(); + return { + 'Authorization': 'Bearer $apiKey', + 'Content-Type': 'application/json', + }; +} + +Future _getEndpoint() async { + return await getMeilisearchEndpoint(); +} + +Future>> getAllDocuments(String index) async { + final endpoint = await _getEndpoint(); + final headers = await _getHeaders(); + final allDocuments = >[]; + var offset = 0; + const limit = 100; // Process in batches of 100 + + while (true) { + final searchCondition = MeilisearchQuery( + q: '', + limit: limit, + offset: offset, + ); + final response = await http.post( + Uri.parse('$endpoint/indexes/$index/search'), + headers: headers, + body: jsonEncode(searchCondition.toJson()), + ); + if (response.statusCode != 200) { + throw Exception('Failed to get documents: ${response.statusCode}'); + } + final responseJson = MeilisearchResponse.fromJson( + jsonDecode(response.body), + ); + final hits = List>.from(responseJson.hits); + + if (hits.isEmpty) { + break; + } + + allDocuments.addAll(hits); + offset += limit; + + if (hits.length < limit) { + break; + } + } + + return allDocuments; +} + +Map fixDocument(Map doc) { + final content = doc['content'] as String; + final date = doc['date'] as int; + + // Calculate letter frequency + final letterFrequency = {}; + for (final char in content.split('')) { + if (alphanum.hasMatch(char)) { + letterFrequency[char] = (letterFrequency[char] ?? 0) + 1; + } + } + + final mostFrequentLetter = + letterFrequency.entries.isEmpty + ? 'a' + : letterFrequency.entries + .reduce((a, b) => a.value > b.value ? a : b) + .key; + + final mostFrequentLetterCount = + letterFrequency.isEmpty + ? 0.0 + : letterFrequency[mostFrequentLetter]! / content.length; + + return { + ...doc, + 'dateISO': + DateTime.fromMillisecondsSinceEpoch(date).toUtc().toIso8601String(), + 'topLetter': mostFrequentLetter, + 'topLetterFrequency': mostFrequentLetterCount, + }; +} + +Future updateDocument(String index, Map doc) async { + final endpoint = await _getEndpoint(); + final headers = await _getHeaders(); + final response = await http.post( + Uri.parse('$endpoint/indexes/$index/documents'), + headers: headers, + body: jsonEncode(doc), + ); + if (response.statusCode != 202) { + throw Exception('Failed to update document: ${response.statusCode}'); + } +} + +Future fixAllDocuments() async { + print('Fixing notes...'); + final notes = await getAllDocuments(noteIndex); + final noteBatches = >>[]; + for (var i = 0; i < notes.length; i += 10) { + noteBatches.add(notes.skip(i).take(10).toList()); + } + + for (final batch in noteBatches) { + await Future.wait( + batch.map((note) async { + final fixed = fixDocument(note); + await updateDocument(noteIndex, fixed); + print('Fixed note: ${note['id']}'); + }), + ); + } + + print('Fixing scratches...'); + final scratches = await getAllDocuments(scratchIndex); + final scratchBatches = >>[]; + for (var i = 0; i < scratches.length; i += 10) { + scratchBatches.add(scratches.skip(i).take(10).toList()); + } + + for (final batch in scratchBatches) { + await Future.wait( + batch.map((scratch) async { + final fixed = fixDocument(scratch); + await updateDocument(scratchIndex, fixed); + print('Fixed scratch: ${scratch['id']}'); + }), + ); + } +} + +void main() async { + try { + await fixAllDocuments(); + print('All documents fixed successfully!'); + } catch (e) { + print('Error fixing documents: $e'); + } +}