journaler/lib/meilifix.dart

import 'dart:convert';
import 'dart:core';
import 'package:http/http.dart' as http;
import 'package:journaler/meilisearch_config.dart';

const noteIndex = 'notes';
const scratchIndex = 'scratch';
final alphanum = RegExp(r'[a-zA-Z0-9]', caseSensitive: false);

class MeilisearchQuery {
  String q;
  String? filter;
  int? limit;
  int? offset;
  bool? showRankingScore;
  double? rankingScoreThreshold;
  String? highlightPreTag;
  String? highlightPostTag;
  List<String>? attributesToHighlight;
  List<String>? sort;

  MeilisearchQuery({
    required this.q,
    this.filter,
    this.sort,
    this.limit,
    this.offset,
    this.showRankingScore,
    this.rankingScoreThreshold,
    this.highlightPreTag,
    this.highlightPostTag,
    this.attributesToHighlight,
  });

  Map<String, dynamic> toJson() {
    final Map<String, dynamic> json = {'q': q};
    if (filter != null) json['filter'] = filter;
    if (sort != null) json['sort'] = sort;
    if (limit != null) json['limit'] = limit;
    if (offset != null) json['offset'] = offset;
    if (showRankingScore != null) json['showRankingScore'] = showRankingScore;
    if (rankingScoreThreshold != null) {
      json['rankingScoreThreshold'] = rankingScoreThreshold;
    }
    if (highlightPreTag != null) json['highlightPreTag'] = highlightPreTag;
    if (highlightPostTag != null) json['highlightPostTag'] = highlightPostTag;
    if (attributesToHighlight != null) {
      json['attributesToHighlight'] = attributesToHighlight;
    }
    return json;
  }
}

class MeilisearchResponse {
  final List<dynamic> hits;
  final String query;
  final int processingTimeMs;
  final int limit;
  final int offset;
  final int estimatedTotalHits;

  MeilisearchResponse({
    required this.hits,
    required this.query,
    required this.processingTimeMs,
    required this.limit,
    required this.offset,
    required this.estimatedTotalHits,
  });

  static MeilisearchResponse fromJson(Map<String, dynamic> json) {
    return MeilisearchResponse(
      hits: json['hits'],
      query: json['query'],
      processingTimeMs: json['processingTimeMs'],
      limit: json['limit'],
      offset: json['offset'],
      estimatedTotalHits: json['estimatedTotalHits'],
    );
  }
}

Future<Map<String, String>> _getHeaders() async {
  final apiKey = await getMeilisearchApiKey();
  return {
    'Authorization': 'Bearer $apiKey',
    'Content-Type': 'application/json',
  };
}

Future<String> _getEndpoint() async {
  return await getMeilisearchEndpoint();
}

Future<List<Map<String, dynamic>>> getAllDocuments(String index) async {
  final endpoint = await _getEndpoint();
  final headers = await _getHeaders();
  final allDocuments = <Map<String, dynamic>>[];
  var offset = 0;
  const limit = 100;

  while (true) {
    final response = await http.get(
      Uri.parse('$endpoint/indexes/$index/documents?limit=$limit&offset=$offset'),
      headers: headers,
    );
    if (response.statusCode != 200) {
      throw Exception('Failed to get documents: ${response.statusCode}');
    }
    final responseJson = jsonDecode(response.body);
    final documents = List<Map<String, dynamic>>.from(responseJson['results']);
    
    if (documents.isEmpty) {
      break;
    }
    
    allDocuments.addAll(documents);
    print('Found ${allDocuments.length} documents so far in $index');
    
    if (documents.length < limit) {
      break;
    }
    
    offset += limit;
  }
  
  print('Total documents found in $index: ${allDocuments.length}');
  return allDocuments;
}

Map<String, dynamic> fixDocument(Map<String, dynamic> doc) {
  final content = doc['content'] as String;
  final date = doc['date'] as int;

  // Calculate letter frequency
  final letterFrequency = <String, int>{};
  for (final char in content.split('')) {
    if (alphanum.hasMatch(char)) {
      letterFrequency[char] = (letterFrequency[char] ?? 0) + 1;
    }
  }

  final mostFrequentLetter =
      letterFrequency.entries.isEmpty
          ? 'a'
          : letterFrequency.entries
              .reduce((a, b) => a.value > b.value ? a : b)
              .key;

  final mostFrequentLetterCount =
      letterFrequency.isEmpty
          ? 0.0
          : letterFrequency[mostFrequentLetter]! / content.length;

  return {
    ...doc,
    'dateISO':
        DateTime.fromMillisecondsSinceEpoch(date).toUtc().toIso8601String(),
    'topLetter': mostFrequentLetter,
    'topLetterFrequency': mostFrequentLetterCount,
  };
}

Future<void> updateDocument(String index, Map<String, dynamic> doc) async {
  final endpoint = await _getEndpoint();
  final headers = await _getHeaders();
  final response = await http.post(
    Uri.parse('$endpoint/indexes/$index/documents'),
    headers: headers,
    body: jsonEncode(doc),
  );
  if (response.statusCode != 202) {
    throw Exception('Failed to update document: ${response.statusCode}');
  }
}

Future<void> fixAllDocuments() async {
  print('Fixing notes...');
  final notes = await getAllDocuments(noteIndex);
  final noteBatches = <List<Map<String, dynamic>>>[];
  for (var i = 0; i < notes.length; i += 10) {
    noteBatches.add(notes.skip(i).take(10).toList());
  }

  for (final batch in noteBatches) {
    await Future.wait(
      batch.map((note) async {
        final fixed = fixDocument(note);
        await updateDocument(noteIndex, fixed);
        print('Fixed note: ${note['id']}');
      }),
    );
  }

  print('Fixing scratches...');
  final scratches = await getAllDocuments(scratchIndex);
  final scratchBatches = <List<Map<String, dynamic>>>[];
  for (var i = 0; i < scratches.length; i += 10) {
    scratchBatches.add(scratches.skip(i).take(10).toList());
  }

  for (final batch in scratchBatches) {
    await Future.wait(
      batch.map((scratch) async {
        final fixed = fixDocument(scratch);
        await updateDocument(scratchIndex, fixed);
        print('Fixed scratch: ${scratch['id']}');
      }),
    );
  }
}

void main() async {
  try {
    await fixAllDocuments();
    print('All documents fixed successfully!');
  } catch (e) {
    print('Error fixing documents: $e');
  }
}