Sanitize out "-" and remove the arbitrary 100 query limit

lib/db.dart

@@ -360,11 +360,12 @@ END;
       }
 
       // Split into individual terms, filter empty ones
-      List<String> terms = query
+      List<String> terms =
-          .trim()
+          query
-          .split(RegExp(r'\s+'))
+              .trim()
-          .where((term) => term.isNotEmpty)
+              .split(RegExp(r'\s+'))
-          .toList();
+              .where((term) => term.isNotEmpty)
+              .toList();
 
       if (terms.isEmpty) {
         return [];
@@ -376,7 +377,7 @@ END;
             // Remove dangerous characters but preserve Unicode
             String sanitizedTerm = term.replaceAll(RegExp(r'''['"]'''), '');
             if (sanitizedTerm.isEmpty) return '';
-            
+
             // Use proper FTS5 syntax: each word becomes a separate token with prefix matching
             List<String> words = sanitizedTerm.split(RegExp(r'\s+'));
             return words.map((word) => '$word*').join(' OR ');
@@ -389,6 +390,7 @@ END;
         return [];
       }
 
+      ftsQuery = ftsQuery.replaceAll('-', ' ');
       debugPrint('FTS query: "$ftsQuery"');
 
       // Execute the FTS query
@@ -400,7 +402,6 @@ END;
         JOIN notes n ON notes_fts.rowid = n.id
         WHERE notes_fts MATCH ?
         ORDER BY rank
-        LIMIT 100
         ''',
         [ftsQuery],
       );

lib/main.dart

@@ -756,7 +756,9 @@ class MainPageState extends State<MainPage> with WindowListener {
                                       .toList();
 
                               // Sort by date, newest first
-                              filteredResults.sort((a, b) => b.date.compareTo(a.date));
+                              filteredResults.sort(
+                                (a, b) => b.date.compareTo(a.date),
+                              );
 
                               // Important: update the dialog state after search completes
                               dialogSetState(() {
@@ -908,6 +910,207 @@ class MainPageState extends State<MainPage> with WindowListener {
     );
   }
 
+  // Show cleanup dialog
+  void _showCleanupDialog() async {
+    double sensitivity = 0.7; // Default 70%
+    final problematicEntries = await findProblematicEntries(
+      maxCharPercentage: sensitivity,
+    );
+
+    if (!mounted) return;
+
+    showDialog(
+      context: context,
+      builder: (BuildContext context) {
+        return StatefulBuilder(
+          builder: (context, dialogSetState) {
+            return AlertDialog(
+              title: const Text('Cleanup Problematic Entries'),
+              content: SizedBox(
+                width: MediaQuery.of(context).size.width * 0.7,
+                height: MediaQuery.of(context).size.height * 0.7,
+                child: Column(
+                  children: [
+                    Row(
+                      children: [
+                        const Text('Sensitivity: '),
+                        Expanded(
+                          child: Slider(
+                            value: sensitivity,
+                            min: 0.3,
+                            max: 0.9,
+                            divisions: 12,
+                            label: '${(sensitivity * 100).toInt()}%',
+                            onChanged: (value) async {
+                              dialogSetState(() {
+                                sensitivity =
+                                    (value * 100).round() /
+                                    100; // Round to 2 decimal places
+                              });
+                              // Refresh results with new sensitivity
+                              final newResults = await findProblematicEntries(
+                                maxCharPercentage: sensitivity,
+                              );
+                              dialogSetState(() {
+                                problematicEntries.clear();
+                                problematicEntries.addAll(newResults);
+                              });
+                            },
+                          ),
+                        ),
+                        Text('${(sensitivity * 100).toInt()}%'),
+                      ],
+                    ),
+                    Text(
+                      'Found ${problematicEntries.length} potentially problematic entries',
+                      style: const TextStyle(fontWeight: FontWeight.bold),
+                    ),
+                    const SizedBox(height: 16),
+                    Expanded(
+                      child:
+                          problematicEntries.isEmpty
+                              ? const Center(
+                                child: Text('No problematic entries found!'),
+                              )
+                              : ListView.builder(
+                                itemCount: problematicEntries.length,
+                                itemBuilder: (context, index) {
+                                  final note = problematicEntries[index];
+                                  return Card(
+                                    margin: const EdgeInsets.only(bottom: 8),
+                                    child: ListTile(
+                                      title: Text(
+                                        note.displayDate,
+                                        style: const TextStyle(
+                                          fontWeight: FontWeight.bold,
+                                        ),
+                                      ),
+                                      subtitle: Column(
+                                        crossAxisAlignment:
+                                            CrossAxisAlignment.start,
+                                        children: [
+                                          Text(
+                                            'Reason: ${note.problemReason}',
+                                            style: TextStyle(
+                                              color: Colors.red[700],
+                                              fontWeight: FontWeight.bold,
+                                            ),
+                                          ),
+                                          const SizedBox(height: 4),
+                                          Text(note.content),
+                                        ],
+                                      ),
+                                      trailing: Row(
+                                        mainAxisSize: MainAxisSize.min,
+                                        children: [
+                                          IconButton(
+                                            icon: const Icon(Icons.visibility),
+                                            tooltip: 'View in main window',
+                                            onPressed: () {
+                                              // Close the dialog
+                                              Navigator.of(context).pop();
+                                              // Navigate to the note in main window
+                                              setState(() {
+                                                _currentlyDisplayedNote = note;
+                                                _previousEntryController.text =
+                                                    note.content;
+                                              });
+                                              _checkNavigation();
+                                            },
+                                          ),
+                                          IconButton(
+                                            icon: const Icon(Icons.delete),
+                                            color: Colors.red,
+                                            tooltip:
+                                                'Delete (hold Shift to skip confirmation)',
+                                            onPressed: () async {
+                                              // Check if shift is pressed
+                                              final isShiftPressed =
+                                                  HardwareKeyboard
+                                                      .instance
+                                                      .isShiftPressed;
+
+                                              bool shouldDelete =
+                                                  isShiftPressed;
+                                              if (!isShiftPressed) {
+                                                // Show confirmation dialog
+                                                shouldDelete =
+                                                    await showDialog<bool>(
+                                                      context: context,
+                                                      builder:
+                                                          (
+                                                            context,
+                                                          ) => AlertDialog(
+                                                            title: const Text(
+                                                              'Delete Entry?',
+                                                            ),
+                                                            content: const Text(
+                                                              'Are you sure you want to delete this entry? This action cannot be undone.',
+                                                            ),
+                                                            actions: [
+                                                              TextButton(
+                                                                onPressed:
+                                                                    () => Navigator.of(
+                                                                      context,
+                                                                    ).pop(
+                                                                      false,
+                                                                    ),
+                                                                child:
+                                                                    const Text(
+                                                                      'Cancel',
+                                                                    ),
+                                                              ),
+                                                              TextButton(
+                                                                onPressed:
+                                                                    () => Navigator.of(
+                                                                      context,
+                                                                    ).pop(true),
+                                                                child:
+                                                                    const Text(
+                                                                      'Delete',
+                                                                    ),
+                                                              ),
+                                                            ],
+                                                          ),
+                                                    ) ??
+                                                    false;
+                                              }
+
+                                              if (shouldDelete) {
+                                                await deleteNote(note.date);
+                                                dialogSetState(() {
+                                                  problematicEntries.removeAt(
+                                                    index,
+                                                  );
+                                                });
+                                              }
+                                            },
+                                          ),
+                                        ],
+                                      ),
+                                    ),
+                                  );
+                                },
+                              ),
+                    ),
+                  ],
+                ),
+              ),
+              actions: [
+                TextButton(
+                  onPressed: () {
+                    Navigator.of(context).pop();
+                  },
+                  child: const Text('Close'),
+                ),
+              ],
+            );
+          },
+        );
+      },
+    );
+  }
+
   @override
   Widget build(BuildContext context) {
     // Wrap Scaffold with RawKeyboardListener as workaround for Escape key
@@ -939,6 +1142,12 @@ class MainPageState extends State<MainPage> with WindowListener {
         appBar: AppBar(
           title: const Text('Journaler'),
           actions: <Widget>[
+            // Add cleanup button
+            IconButton(
+              icon: const Icon(Icons.cleaning_services),
+              tooltip: 'Cleanup Problematic Entries',
+              onPressed: _showCleanupDialog,
+            ),
             // Add search button
             IconButton(
               icon: const Icon(Icons.search),

lib/notes.dart

@@ -6,8 +6,16 @@ class Note {
   late final String displayDate;
   String content;
   String? snippet;
+  bool isProblematic;
+  String problemReason;
 
-  Note({required this.date, required this.content, this.snippet}) {
+  Note({
+    required this.date,
+    required this.content,
+    this.snippet,
+    this.isProblematic = false,
+    this.problemReason = '',
+  }) {
     final dtUtc = DateFormat('yyyy-MM-dd HH:mm:ss').parse(date, true);
     final dtLocal = dtUtc.toLocal();
     displayDate = DateFormat('yyyy-MM-dd HH:mm:ss').format(dtLocal);
@@ -35,11 +43,18 @@ Future<Note?> getLatestNote() async {
 }
 
 Future<void> createNote(String content) async {
-  // Trim the content to avoid saving just whitespace
+  // Trim each line, sometimes we fuck up by doing a lil "foobar            "
-  final trimmedContent = content.trim();
+  // Maybe I should also look for \s{2,}...
-  if (trimmedContent.isEmpty) {
+  final lines = content.split('\n');
-    return;
+  final trimmedLines = <String>[];
+  for (final line in lines) {
+    final trimmedContent = line.trim().replaceAll(RegExp(r'\s{2,}'), ' ');
+    if (trimmedContent.isEmpty) {
+      continue;
+    }
+    trimmedLines.add(trimmedContent);
   }
+  final trimmedContent = trimmedLines.join('\n');
   await DB.db.insert('notes', {'content': trimmedContent});
 }
 
@@ -152,3 +167,42 @@ Future<List<Note>> searchNotes(String query) async {
       )
       .toList();
 }
+
+// Find potentially problematic entries based on character distribution
+Future<List<Note>> findProblematicEntries({
+  double maxCharPercentage = 0.7,
+}) async {
+  // Simple SQLite query that counts character occurrences using replace
+  final List<Map<String, dynamic>> results = await DB.db.rawQuery(
+    '''
+    WITH char_counts AS (
+      SELECT
+        id,
+        date,
+        content,
+        substr(content, 1, 1) as char,
+        (length(content) - length(replace(content, substr(content, 1, 1), ''))) as char_count,
+        length(content) as total_length,
+        cast(length(content) - length(replace(content, substr(content, 1, 1), '')) as float) / length(content) as percentage
+      FROM notes
+    )
+    SELECT *
+    FROM char_counts
+    WHERE percentage > ?
+    ORDER BY date DESC
+  ''',
+    [maxCharPercentage],
+  );
+
+  return results
+      .map(
+        (row) => Note(
+          date: row['date'] as String,
+          content: row['content'] as String,
+          isProblematic: true,
+          problemReason:
+              'Character "${row['char']}" makes up ${(row['percentage'] * 100).toStringAsFixed(1)}% of the content',
+        ),
+      )
+      .toList();
+}