From 11beac50ebfe22cc37f75c749a080fe921d3a8f2 Mon Sep 17 00:00:00 2001
From: PhatPhuckDave <david.majdandzic@hotmail.com>
Date: Sun, 25 May 2025 03:09:36 +0200
Subject: [PATCH] Refactor AddMessages to use structured payload and improve
 error handling in chatsniffer

---
 service/chatsniffer/init_meili.go | 12 ++++-
 service/chatsniffer/main.go       | 81 ++++++++++++++++++++++++++++++-
 2 files changed, 90 insertions(+), 3 deletions(-)

diff --git a/service/chatsniffer/init_meili.go b/service/chatsniffer/init_meili.go
index 0eddaeb..b13662a 100644
--- a/service/chatsniffer/init_meili.go
+++ b/service/chatsniffer/init_meili.go
@@ -22,7 +22,8 @@ type IndexConfig struct {
 
 type ChatMessage struct {
 	MessageHash string `json:"message_hash"`
-	Timestamp   string `json:"timestamp"`
+	Timestamp   string `json:"timestamp"`  // ISO timestamp
+	EpochTime   int64  `json:"epoch_time"` // Unix epoch timestamp for filtering
 	Event       string `json:"event"`
 	Sender      string `json:"sender"`
 	Msg         string `json:"msg"`
@@ -103,8 +104,14 @@ func AddMessages(messages []ChatMessage) error {
 	}
 	defer resp.Body.Close()
 
+	// Read response body for better error messages
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error reading response body: %v", err)
+	}
+
 	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
-		return fmt.Errorf("failed to add messages. Status: %d", resp.StatusCode)
+		return fmt.Errorf("failed to add messages. Status: %d, Response: %s", resp.StatusCode, string(body))
 	}
 
 	return nil
@@ -196,6 +203,7 @@ func setIndexSettings() error {
 	// Then set filterable attributes
 	filterableAttributes := []string{
 		"timestamp",
+		"epoch_time", // Add epoch_time for numeric filtering
 		"event",
 		"sender",
 		"language",
diff --git a/service/chatsniffer/main.go b/service/chatsniffer/main.go
index 87634a4..8557a96 100644
--- a/service/chatsniffer/main.go
+++ b/service/chatsniffer/main.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"time"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
 	"github.com/bmatcuk/doublestar/v4"
@@ -67,6 +68,9 @@ func main() {
 		return
 	}
 	logger.Info("Successfully saved %d messages", len(chatMessages))
+
+	// Clear chat tables in source files
+	clearChatTables(matches)
 }
 
 func loadLuaStates(matches []string) *sync.Map {
@@ -163,10 +167,15 @@ func loadStateChatMessages(L *lua.LState, log *logger.Logger) []ChatMessage {
 }
 
 func parseChatMessage(chatStr string) (ChatMessage, error) {
+	// Debug: Print the raw string
+	logger.Debug("Raw chat string: %q", chatStr)
+
 	// Split by pipe - we expect 6 parts (5 pipes)
 	parts := strings.Split(chatStr, "|")
+	logger.Debug("Split into %d parts: %v", len(parts), parts)
+
 	if len(parts) != 6 {
-		return ChatMessage{}, fmt.Errorf("invalid message format: %s", chatStr)
+		return ChatMessage{}, fmt.Errorf("invalid message format: expected 6 parts, got %d: %s", len(parts), chatStr)
 	}
 
 	timestamp := parts[0]
@@ -176,9 +185,16 @@ func parseChatMessage(chatStr string) (ChatMessage, error) {
 	language := parts[4]
 	channel := parts[5]
 
+	// Parse ISO timestamp to epoch
+	epochTime, err := parseISOTimestamp(timestamp)
+	if err != nil {
+		return ChatMessage{}, fmt.Errorf("invalid timestamp format: %v", err)
+	}
+
 	return ChatMessage{
 		MessageHash: GenerateMessageHash(timestamp, event, sender, msg, language, channel),
 		Timestamp:   timestamp,
+		EpochTime:   epochTime,
 		Event:       event,
 		Sender:      sender,
 		Msg:         msg,
@@ -186,3 +202,66 @@ func parseChatMessage(chatStr string) (ChatMessage, error) {
 		Channel:     channel,
 	}, nil
 }
+
+func parseISOTimestamp(isoTime string) (int64, error) {
+	// Debug: Print the timestamp we're trying to parse
+	logger.Debug("Parsing timestamp: %q", isoTime)
+
+	// Parse the timestamp format used in chat messages (e.g., "2025-05-25 02:51:05")
+	t, err := time.Parse("2006-01-02 15:04:05", isoTime)
+	if err != nil {
+		logger.Debug("Failed to parse timestamp: %v", err)
+		return 0, err
+	}
+	return t.Unix(), nil
+}
+
+func clearChatTables(matches []string) {
+	for _, path := range matches {
+		log := logger.Default.WithPrefix(path)
+		log.Info("Reading source file")
+		fileContent, err := os.ReadFile(path)
+		if err != nil {
+			logger.Error("Failed to read file: %v", err)
+			continue
+		}
+		strContent := string(fileContent)
+		log.Info("Read %d bytes", len(strContent))
+
+		strContent = clearChatTable(strContent)
+		log.Info("Cleared chat table, now %d bytes", len(strContent))
+
+		log.Info("Writing file")
+		err = os.WriteFile(path, []byte(strContent), 0644)
+		if err != nil {
+			logger.Error("Failed to write file: %v", err)
+			continue
+		}
+		log.Info("Done")
+	}
+}
+
+func clearChatTable(sourceContent string) string {
+	lines := strings.Split(sourceContent, "\n")
+	writeIndex := 0
+	isInChat := false
+	for _, line := range lines {
+		if strings.HasPrefix(line, "Heimdall_Chat = {") {
+			isInChat = true
+			lines[writeIndex] = "Heimdall_Chat = {"
+			writeIndex++
+			continue
+		}
+		if isInChat && strings.HasPrefix(line, "}") {
+			isInChat = false
+			lines[writeIndex] = "}"
+			writeIndex++
+			continue
+		}
+		if !isInChat {
+			lines[writeIndex] = line
+			writeIndex++
+		}
+	}
+	return strings.Join(lines[:writeIndex], "\n")
+}