From 5e6a5e830e91a12411c1fb1849bdcdb4af2d0f04 Mon Sep 17 00:00:00 2001
From: PhatPhuckDave <david.majdandzic@hotmail.com>
Date: Sun, 25 May 2025 03:04:11 +0200
Subject: [PATCH] Add initial implementation of chatsniffer with Meilisearch
 integration and required dependencies

---
 service/chatsniffer/go.mod        |   9 ++
 service/chatsniffer/go.sum        |   6 +
 service/chatsniffer/init_meili.go | 239 ++++++++++++++++++++++++++++++
 service/chatsniffer/main.go       | 188 +++++++++++++++++++++++
 service/chatsniffer/succ.sh       |   1 +
 5 files changed, 443 insertions(+)
 create mode 100644 service/chatsniffer/go.mod
 create mode 100644 service/chatsniffer/go.sum
 create mode 100644 service/chatsniffer/init_meili.go
 create mode 100644 service/chatsniffer/main.go
 create mode 100644 service/chatsniffer/succ.sh

diff --git a/service/chatsniffer/go.mod b/service/chatsniffer/go.mod
new file mode 100644
index 0000000..4bc0f49
--- /dev/null
+++ b/service/chatsniffer/go.mod
@@ -0,0 +1,9 @@
+module chatsniffer
+
+go 1.24.3
+
+require (
+	git.site.quack-lab.dev/dave/cylogger v1.2.2
+	github.com/bmatcuk/doublestar/v4 v4.8.1
+	github.com/yuin/gopher-lua v1.1.1
+)
diff --git a/service/chatsniffer/go.sum b/service/chatsniffer/go.sum
new file mode 100644
index 0000000..80d70c6
--- /dev/null
+++ b/service/chatsniffer/go.sum
@@ -0,0 +1,6 @@
+git.site.quack-lab.dev/dave/cylogger v1.2.2 h1:4xUXASEBlG9NiGxh7f57xHh9imW4unHzakIEpQoKC5E=
+git.site.quack-lab.dev/dave/cylogger v1.2.2/go.mod h1:VS9MI4Y/cwjCBZgel7dSfCQlwtAgHmfvixOoBgBhtKg=
+github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
+github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
+github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
diff --git a/service/chatsniffer/init_meili.go b/service/chatsniffer/init_meili.go
new file mode 100644
index 0000000..0eddaeb
--- /dev/null
+++ b/service/chatsniffer/init_meili.go
@@ -0,0 +1,239 @@
+package main
+
+import (
+	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+
+	logger "git.site.quack-lab.dev/dave/cylogger"
+)
+
+var meiliToken string
+
+type IndexConfig struct {
+	Uid        string `json:"uid"`
+	PrimaryKey string `json:"primaryKey"`
+}
+
+type ChatMessage struct {
+	MessageHash string `json:"message_hash"`
+	Timestamp   string `json:"timestamp"`
+	Event       string `json:"event"`
+	Sender      string `json:"sender"`
+	Msg         string `json:"msg"`
+	Language    string `json:"language"`
+	Channel     string `json:"channel"`
+}
+
+func Init() error {
+	// Load Meilisearch token
+	meiliToken = os.Getenv("MEILI_TOKEN")
+	if meiliToken == "" {
+		return fmt.Errorf("MEILI_TOKEN environment variable not set")
+	}
+	logger.Info("Meilisearch token loaded")
+
+	config := IndexConfig{
+		Uid:        meiliIndex,
+		PrimaryKey: "message_hash", // Meilisearch will use this for deduplication
+	}
+
+	// Create index
+	err := createIndex(config)
+	if err != nil {
+		return fmt.Errorf("error creating index: %v", err)
+	}
+
+	// Set up index settings
+	err = setIndexSettings()
+	if err != nil {
+		return fmt.Errorf("error setting index settings: %v", err)
+	}
+
+	return nil
+}
+
+// GenerateMessageHash creates a unique hash for a message that will be identical
+// for identical messages, ensuring perfect deduplication
+func GenerateMessageHash(timestamp, event, sender, msg, language, channel string) string {
+	// Combine all fields that make a message unique
+	content := fmt.Sprintf("%s|%s|%s|%s|%s|%s",
+		timestamp,
+		event,
+		sender,
+		msg,
+		language,
+		channel,
+	)
+
+	// Create SHA-256 hash of the combined content
+	hash := sha256.Sum256([]byte(content))
+	return hex.EncodeToString(hash[:])
+}
+
+// AddMessages adds multiple messages to the index in a single batch request
+// Meilisearch will handle deduplication based on the message_hash
+func AddMessages(messages []ChatMessage) error {
+	jsonData, err := json.Marshal(messages)
+	if err != nil {
+		return fmt.Errorf("error marshaling messages: %v", err)
+	}
+
+	req, err := http.NewRequest(
+		http.MethodPost,
+		meiliEndpoint+"indexes/"+meiliIndex+"/documents",
+		bytes.NewBuffer(jsonData),
+	)
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+meiliToken)
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error adding messages: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+		return fmt.Errorf("failed to add messages. Status: %d", resp.StatusCode)
+	}
+
+	return nil
+}
+
+func createIndex(config IndexConfig) error {
+	jsonData, err := json.Marshal(config)
+	if err != nil {
+		return fmt.Errorf("error marshaling config: %v", err)
+	}
+
+	req, err := http.NewRequest(
+		http.MethodPost,
+		meiliEndpoint+"indexes",
+		bytes.NewBuffer(jsonData),
+	)
+	if err != nil {
+		return fmt.Errorf("error creating request: %v", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+meiliToken)
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error creating index: %v", err)
+	}
+	defer resp.Body.Close()
+
+	// Read response body for better error messages
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error reading response body: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted {
+		return fmt.Errorf("failed to create index. Status: %d, Response: %s", resp.StatusCode, string(body))
+	}
+
+	logger.Info("Index created successfully")
+	return nil
+}
+
+func setIndexSettings() error {
+	// First set searchable attributes
+	searchableAttributes := []string{
+		"timestamp",
+		"event",
+		"sender",
+		"msg",
+		"language",
+		"channel",
+	}
+
+	jsonData, err := json.Marshal(searchableAttributes)
+	if err != nil {
+		return fmt.Errorf("error marshaling searchable settings: %v", err)
+	}
+
+	req, err := http.NewRequest(
+		http.MethodPut,
+		meiliEndpoint+"indexes/"+meiliIndex+"/settings/searchable-attributes",
+		bytes.NewBuffer(jsonData),
+	)
+	if err != nil {
+		return fmt.Errorf("error creating searchable settings request: %v", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+meiliToken)
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error setting searchable attributes: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error reading searchable settings response: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+		return fmt.Errorf("failed to set searchable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
+	}
+
+	// Then set filterable attributes
+	filterableAttributes := []string{
+		"timestamp",
+		"event",
+		"sender",
+		"language",
+		"channel",
+	}
+
+	jsonData, err = json.Marshal(filterableAttributes)
+	if err != nil {
+		return fmt.Errorf("error marshaling filterable settings: %v", err)
+	}
+
+	req, err = http.NewRequest(
+		http.MethodPut,
+		meiliEndpoint+"indexes/"+meiliIndex+"/settings/filterable-attributes",
+		bytes.NewBuffer(jsonData),
+	)
+	if err != nil {
+		return fmt.Errorf("error creating filterable settings request: %v", err)
+	}
+
+	req.Header.Set("Authorization", "Bearer "+meiliToken)
+	req.Header.Set("Content-Type", "application/json")
+
+	resp, err = client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error setting filterable attributes: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err = io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error reading filterable settings response: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
+		return fmt.Errorf("failed to set filterable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
+	}
+
+	logger.Info("Index settings set successfully")
+	return nil
+}
diff --git a/service/chatsniffer/main.go b/service/chatsniffer/main.go
new file mode 100644
index 0000000..87634a4
--- /dev/null
+++ b/service/chatsniffer/main.go
@@ -0,0 +1,188 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	logger "git.site.quack-lab.dev/dave/cylogger"
+	"github.com/bmatcuk/doublestar/v4"
+	lua "github.com/yuin/gopher-lua"
+)
+
+const meiliEndpoint = "https://meili.site.quack-lab.dev/"
+const meiliIndex = "chatlog"
+
+var debug *bool
+
+func main() {
+	root := flag.String("root", ".", "Root workdir")
+	debug = flag.Bool("d", false, "Debug")
+	flag.Parse()
+	logger.InitFlag()
+	if *debug {
+		logger.SetLevel(logger.LevelDebug)
+	}
+
+	err := Init()
+	if err != nil {
+		logger.Error("Failed to initialize Meilisearch: %v", err)
+		return
+	}
+
+	logger.Info("Root: %q", *root)
+	cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
+	cleanedRoot, err = filepath.Abs(cleanedRoot)
+	if err != nil {
+		logger.Error("error getting absolute path: %v", err)
+		return
+	}
+	cleanedRoot = filepath.Clean(cleanedRoot)
+	cleanedRoot = strings.TrimSuffix(cleanedRoot, "/")
+
+	logger.Info("Looking for Heimdall.lua in %q", cleanedRoot)
+	matches, err := doublestar.Glob(os.DirFS(cleanedRoot), "**/Heimdall.lua")
+	if err != nil {
+		logger.Error("error matching Heimdall.lua: %v", err)
+		return
+	}
+	logger.Info("Found %d Heimdall.lua files.", len(matches))
+	if len(matches) == 0 {
+		logger.Info("No Heimdall.lua files found. Exiting.")
+		return
+	}
+	for i, match := range matches {
+		matches[i] = filepath.Join(cleanedRoot, match)
+	}
+
+	luaStates := loadLuaStates(matches)
+	chatMessages := loadChatMessages(luaStates)
+
+	// Save messages to Meilisearch
+	if err := AddMessages(chatMessages); err != nil {
+		logger.Error("Failed to save messages: %v", err)
+		return
+	}
+	logger.Info("Successfully saved %d messages", len(chatMessages))
+}
+
+func loadLuaStates(matches []string) *sync.Map {
+	wg := sync.WaitGroup{}
+	fileLuaStates := &sync.Map{}
+	for _, match := range matches {
+		wg.Add(1)
+		go func(path string) {
+			defer wg.Done()
+			log := logger.Default.WithPrefix(path)
+			L := lua.NewState()
+
+			filestat, err := os.Stat(path)
+			if err != nil {
+				log.Error("error getting file stats: %v", err)
+				return
+			}
+			log.Info("File size: %.2f MB", float64(filestat.Size())/1024/1024)
+
+			log.Info("Running Lua file")
+			if err := L.DoFile(path); err != nil {
+				log.Error("error executing Lua file %q: %v", path, err)
+				return
+			}
+			log.Info("Lua file loaded")
+			fileLuaStates.Store(path, L)
+		}(match)
+	}
+	wg.Wait()
+	return fileLuaStates
+}
+
+func loadChatMessages(luaStates *sync.Map) []ChatMessage {
+	var messages []ChatMessage
+	wg := sync.WaitGroup{}
+	messageChan := make(chan []ChatMessage, 100) // Buffer for concurrent processing
+
+	luaStates.Range(func(path, state any) bool {
+		wg.Add(1)
+		go func(path string, state *lua.LState) {
+			defer wg.Done()
+			log := logger.Default.WithPrefix(path)
+			fileMessages := loadStateChatMessages(state, log)
+			messageChan <- fileMessages
+		}(path.(string), state.(*lua.LState))
+		return true
+	})
+
+	// Close channel when all goroutines are done
+	go func() {
+		wg.Wait()
+		close(messageChan)
+	}()
+
+	// Collect all messages
+	for fileMessages := range messageChan {
+		messages = append(messages, fileMessages...)
+	}
+
+	return messages
+}
+
+func loadStateChatMessages(L *lua.LState, log *logger.Logger) []ChatMessage {
+	log.Info("Getting Heimdall_Chat")
+	heimdallChat := L.GetGlobal("Heimdall_Chat")
+	if heimdallChat.Type() == lua.LTNil {
+		log.Warning("Heimdall_Chat not found. Skipping file.")
+		return nil
+	}
+
+	chatTable, ok := heimdallChat.(*lua.LTable)
+	if !ok {
+		log.Warning("Heimdall_Chat is not a table. Skipping file.")
+		return nil
+	}
+
+	var messages []ChatMessage
+	chatTable.ForEach(func(_, value lua.LValue) {
+		chatStr := value.String()
+		// Remove quotes and trailing comma if present
+		chatStr = strings.Trim(chatStr, "\", ")
+
+		// Parse the chat message
+		message, err := parseChatMessage(chatStr)
+		if err != nil {
+			log.Warning("Invalid chat format: %s", chatStr)
+			return
+		}
+		messages = append(messages, message)
+	})
+
+	log.Info("Loaded %d chat messages", len(messages))
+	return messages
+}
+
+func parseChatMessage(chatStr string) (ChatMessage, error) {
+	// Split by pipe - we expect 6 parts (5 pipes)
+	parts := strings.Split(chatStr, "|")
+	if len(parts) != 6 {
+		return ChatMessage{}, fmt.Errorf("invalid message format: %s", chatStr)
+	}
+
+	timestamp := parts[0]
+	event := parts[1]
+	sender := parts[2]
+	msg := parts[3]
+	language := parts[4]
+	channel := parts[5]
+
+	return ChatMessage{
+		MessageHash: GenerateMessageHash(timestamp, event, sender, msg, language, channel),
+		Timestamp:   timestamp,
+		Event:       event,
+		Sender:      sender,
+		Msg:         msg,
+		Language:    language,
+		Channel:     channel,
+	}, nil
+}
diff --git a/service/chatsniffer/succ.sh b/service/chatsniffer/succ.sh
new file mode 100644
index 0000000..7a8e3c1
--- /dev/null
+++ b/service/chatsniffer/succ.sh
@@ -0,0 +1 @@
+./chatsniffer.exe -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"
\ No newline at end of file