Add Meilisearch document handling and fix utilities
This commit is contained in:
225
lib/meilifix.dart
Normal file
225
lib/meilifix.dart
Normal file
@@ -0,0 +1,225 @@
|
||||
import 'dart:convert';
|
||||
import 'dart:core';
|
||||
import 'package:http/http.dart' as http;
|
||||
import 'package:journaler/meilisearch_config.dart';
|
||||
|
||||
const noteIndex = 'notes';
|
||||
const scratchIndex = 'scratch';
|
||||
final alphanum = RegExp(r'[a-zA-Z0-9]', caseSensitive: false);
|
||||
|
||||
class MeilisearchQuery {
|
||||
String q;
|
||||
String? filter;
|
||||
int? limit;
|
||||
int? offset;
|
||||
bool? showRankingScore;
|
||||
double? rankingScoreThreshold;
|
||||
String? highlightPreTag;
|
||||
String? highlightPostTag;
|
||||
List<String>? attributesToHighlight;
|
||||
List<String>? sort;
|
||||
|
||||
MeilisearchQuery({
|
||||
required this.q,
|
||||
this.filter,
|
||||
this.sort,
|
||||
this.limit,
|
||||
this.offset,
|
||||
this.showRankingScore,
|
||||
this.rankingScoreThreshold,
|
||||
this.highlightPreTag,
|
||||
this.highlightPostTag,
|
||||
this.attributesToHighlight,
|
||||
});
|
||||
|
||||
Map<String, dynamic> toJson() {
|
||||
final Map<String, dynamic> json = {'q': q};
|
||||
if (filter != null) json['filter'] = filter;
|
||||
if (sort != null) json['sort'] = sort;
|
||||
if (limit != null) json['limit'] = limit;
|
||||
if (offset != null) json['offset'] = offset;
|
||||
if (showRankingScore != null) json['showRankingScore'] = showRankingScore;
|
||||
if (rankingScoreThreshold != null) {
|
||||
json['rankingScoreThreshold'] = rankingScoreThreshold;
|
||||
}
|
||||
if (highlightPreTag != null) json['highlightPreTag'] = highlightPreTag;
|
||||
if (highlightPostTag != null) json['highlightPostTag'] = highlightPostTag;
|
||||
if (attributesToHighlight != null) {
|
||||
json['attributesToHighlight'] = attributesToHighlight;
|
||||
}
|
||||
return json;
|
||||
}
|
||||
}
|
||||
|
||||
class MeilisearchResponse {
|
||||
final List<dynamic> hits;
|
||||
final String query;
|
||||
final int processingTimeMs;
|
||||
final int limit;
|
||||
final int offset;
|
||||
final int estimatedTotalHits;
|
||||
|
||||
MeilisearchResponse({
|
||||
required this.hits,
|
||||
required this.query,
|
||||
required this.processingTimeMs,
|
||||
required this.limit,
|
||||
required this.offset,
|
||||
required this.estimatedTotalHits,
|
||||
});
|
||||
|
||||
static MeilisearchResponse fromJson(Map<String, dynamic> json) {
|
||||
return MeilisearchResponse(
|
||||
hits: json['hits'],
|
||||
query: json['query'],
|
||||
processingTimeMs: json['processingTimeMs'],
|
||||
limit: json['limit'],
|
||||
offset: json['offset'],
|
||||
estimatedTotalHits: json['estimatedTotalHits'],
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
Future<Map<String, String>> _getHeaders() async {
|
||||
final apiKey = await getMeilisearchApiKey();
|
||||
return {
|
||||
'Authorization': 'Bearer $apiKey',
|
||||
'Content-Type': 'application/json',
|
||||
};
|
||||
}
|
||||
|
||||
Future<String> _getEndpoint() async {
|
||||
return await getMeilisearchEndpoint();
|
||||
}
|
||||
|
||||
Future<List<Map<String, dynamic>>> getAllDocuments(String index) async {
|
||||
final endpoint = await _getEndpoint();
|
||||
final headers = await _getHeaders();
|
||||
final allDocuments = <Map<String, dynamic>>[];
|
||||
var offset = 0;
|
||||
const limit = 100; // Process in batches of 100
|
||||
|
||||
while (true) {
|
||||
final searchCondition = MeilisearchQuery(
|
||||
q: '',
|
||||
limit: limit,
|
||||
offset: offset,
|
||||
);
|
||||
final response = await http.post(
|
||||
Uri.parse('$endpoint/indexes/$index/search'),
|
||||
headers: headers,
|
||||
body: jsonEncode(searchCondition.toJson()),
|
||||
);
|
||||
if (response.statusCode != 200) {
|
||||
throw Exception('Failed to get documents: ${response.statusCode}');
|
||||
}
|
||||
final responseJson = MeilisearchResponse.fromJson(
|
||||
jsonDecode(response.body),
|
||||
);
|
||||
final hits = List<Map<String, dynamic>>.from(responseJson.hits);
|
||||
|
||||
if (hits.isEmpty) {
|
||||
break;
|
||||
}
|
||||
|
||||
allDocuments.addAll(hits);
|
||||
offset += limit;
|
||||
|
||||
if (hits.length < limit) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
return allDocuments;
|
||||
}
|
||||
|
||||
Map<String, dynamic> fixDocument(Map<String, dynamic> doc) {
|
||||
final content = doc['content'] as String;
|
||||
final date = doc['date'] as int;
|
||||
|
||||
// Calculate letter frequency
|
||||
final letterFrequency = <String, int>{};
|
||||
for (final char in content.split('')) {
|
||||
if (alphanum.hasMatch(char)) {
|
||||
letterFrequency[char] = (letterFrequency[char] ?? 0) + 1;
|
||||
}
|
||||
}
|
||||
|
||||
final mostFrequentLetter =
|
||||
letterFrequency.entries.isEmpty
|
||||
? 'a'
|
||||
: letterFrequency.entries
|
||||
.reduce((a, b) => a.value > b.value ? a : b)
|
||||
.key;
|
||||
|
||||
final mostFrequentLetterCount =
|
||||
letterFrequency.isEmpty
|
||||
? 0.0
|
||||
: letterFrequency[mostFrequentLetter]! / content.length;
|
||||
|
||||
return {
|
||||
...doc,
|
||||
'dateISO':
|
||||
DateTime.fromMillisecondsSinceEpoch(date).toUtc().toIso8601String(),
|
||||
'topLetter': mostFrequentLetter,
|
||||
'topLetterFrequency': mostFrequentLetterCount,
|
||||
};
|
||||
}
|
||||
|
||||
Future<void> updateDocument(String index, Map<String, dynamic> doc) async {
|
||||
final endpoint = await _getEndpoint();
|
||||
final headers = await _getHeaders();
|
||||
final response = await http.post(
|
||||
Uri.parse('$endpoint/indexes/$index/documents'),
|
||||
headers: headers,
|
||||
body: jsonEncode(doc),
|
||||
);
|
||||
if (response.statusCode != 202) {
|
||||
throw Exception('Failed to update document: ${response.statusCode}');
|
||||
}
|
||||
}
|
||||
|
||||
Future<void> fixAllDocuments() async {
|
||||
print('Fixing notes...');
|
||||
final notes = await getAllDocuments(noteIndex);
|
||||
final noteBatches = <List<Map<String, dynamic>>>[];
|
||||
for (var i = 0; i < notes.length; i += 10) {
|
||||
noteBatches.add(notes.skip(i).take(10).toList());
|
||||
}
|
||||
|
||||
for (final batch in noteBatches) {
|
||||
await Future.wait(
|
||||
batch.map((note) async {
|
||||
final fixed = fixDocument(note);
|
||||
await updateDocument(noteIndex, fixed);
|
||||
print('Fixed note: ${note['id']}');
|
||||
}),
|
||||
);
|
||||
}
|
||||
|
||||
print('Fixing scratches...');
|
||||
final scratches = await getAllDocuments(scratchIndex);
|
||||
final scratchBatches = <List<Map<String, dynamic>>>[];
|
||||
for (var i = 0; i < scratches.length; i += 10) {
|
||||
scratchBatches.add(scratches.skip(i).take(10).toList());
|
||||
}
|
||||
|
||||
for (final batch in scratchBatches) {
|
||||
await Future.wait(
|
||||
batch.map((scratch) async {
|
||||
final fixed = fixDocument(scratch);
|
||||
await updateDocument(scratchIndex, fixed);
|
||||
print('Fixed scratch: ${scratch['id']}');
|
||||
}),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
void main() async {
|
||||
try {
|
||||
await fixAllDocuments();
|
||||
print('All documents fixed successfully!');
|
||||
} catch (e) {
|
||||
print('Error fixing documents: $e');
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user