dave/wow-AchievementSniffer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add initial implementation of chatsniffer with Meilisearch integration and required dependencies

Browse Source
This commit is contained in:
PhatPhuckDave
2025-05-25 03:04:11 +02:00
parent 9035a8284c
commit 5e6a5e830e
5 changed files with 443 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
service/chatsniffer/go.mod Normal file
View File

@@ -0,0 +1,9 @@
module chatsniffer
go 1.24.3
require (
git.site.quack-lab.dev/dave/cylogger v1.2.2
github.com/bmatcuk/doublestar/v4 v4.8.1
github.com/yuin/gopher-lua v1.1.1
)

6
service/chatsniffer/go.sum Normal file
View File

@@ -0,0 +1,6 @@
git.site.quack-lab.dev/dave/cylogger v1.2.2 h1:4xUXASEBlG9NiGxh7f57xHh9imW4unHzakIEpQoKC5E=
git.site.quack-lab.dev/dave/cylogger v1.2.2/go.mod h1:VS9MI4Y/cwjCBZgel7dSfCQlwtAgHmfvixOoBgBhtKg=
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=

239
service/chatsniffer/init_meili.go Normal file
View File

@@ -0,0 +1,239 @@
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
logger "git.site.quack-lab.dev/dave/cylogger"
)
var meiliToken string
type IndexConfig struct {
Uid string `json:"uid"`
PrimaryKey string `json:"primaryKey"`
}
type ChatMessage struct {
MessageHash string `json:"message_hash"`
Timestamp string `json:"timestamp"`
Event string `json:"event"`
Sender string `json:"sender"`
Msg string `json:"msg"`
Language string `json:"language"`
Channel string `json:"channel"`
}
func Init() error {
// Load Meilisearch token
meiliToken = os.Getenv("MEILI_TOKEN")
if meiliToken == "" {
return fmt.Errorf("MEILI_TOKEN environment variable not set")
}
logger.Info("Meilisearch token loaded")
config := IndexConfig{
Uid: meiliIndex,
PrimaryKey: "message_hash", // Meilisearch will use this for deduplication
}
// Create index
err := createIndex(config)
if err != nil {
return fmt.Errorf("error creating index: %v", err)
}
// Set up index settings
err = setIndexSettings()
if err != nil {
return fmt.Errorf("error setting index settings: %v", err)
}
return nil
}
// GenerateMessageHash creates a unique hash for a message that will be identical
// for identical messages, ensuring perfect deduplication
func GenerateMessageHash(timestamp, event, sender, msg, language, channel string) string {
// Combine all fields that make a message unique
content := fmt.Sprintf("%s|%s|%s|%s|%s|%s",
timestamp,
event,
sender,
msg,
language,
channel,
)
// Create SHA-256 hash of the combined content
hash := sha256.Sum256([]byte(content))
return hex.EncodeToString(hash[:])
}
// AddMessages adds multiple messages to the index in a single batch request
// Meilisearch will handle deduplication based on the message_hash
func AddMessages(messages []ChatMessage) error {
jsonData, err := json.Marshal(messages)
if err != nil {
return fmt.Errorf("error marshaling messages: %v", err)
}
req, err := http.NewRequest(
http.MethodPost,
meiliEndpoint+"indexes/"+meiliIndex+"/documents",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error adding messages: %v", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to add messages. Status: %d", resp.StatusCode)
}
return nil
}
func createIndex(config IndexConfig) error {
jsonData, err := json.Marshal(config)
if err != nil {
return fmt.Errorf("error marshaling config: %v", err)
}
req, err := http.NewRequest(
http.MethodPost,
meiliEndpoint+"indexes",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error creating index: %v", err)
}
defer resp.Body.Close()
// Read response body for better error messages
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading response body: %v", err)
}
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to create index. Status: %d, Response: %s", resp.StatusCode, string(body))
}
logger.Info("Index created successfully")
return nil
}
func setIndexSettings() error {
// First set searchable attributes
searchableAttributes := []string{
"timestamp",
"event",
"sender",
"msg",
"language",
"channel",
}
jsonData, err := json.Marshal(searchableAttributes)
if err != nil {
return fmt.Errorf("error marshaling searchable settings: %v", err)
}
req, err := http.NewRequest(
http.MethodPut,
meiliEndpoint+"indexes/"+meiliIndex+"/settings/searchable-attributes",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating searchable settings request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error setting searchable attributes: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading searchable settings response: %v", err)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to set searchable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
}
// Then set filterable attributes
filterableAttributes := []string{
"timestamp",
"event",
"sender",
"language",
"channel",
}
jsonData, err = json.Marshal(filterableAttributes)
if err != nil {
return fmt.Errorf("error marshaling filterable settings: %v", err)
}
req, err = http.NewRequest(
http.MethodPut,
meiliEndpoint+"indexes/"+meiliIndex+"/settings/filterable-attributes",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating filterable settings request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
resp, err = client.Do(req)
if err != nil {
return fmt.Errorf("error setting filterable attributes: %v", err)
}
defer resp.Body.Close()
body, err = io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading filterable settings response: %v", err)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to set filterable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
}
logger.Info("Index settings set successfully")
return nil
}

188
service/chatsniffer/main.go Normal file
View File

@@ -0,0 +1,188 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/bmatcuk/doublestar/v4"
lua "github.com/yuin/gopher-lua"
)
const meiliEndpoint = "https://meili.site.quack-lab.dev/"
const meiliIndex = "chatlog"
var debug *bool
func main() {
root := flag.String("root", ".", "Root workdir")
debug = flag.Bool("d", false, "Debug")
flag.Parse()
logger.InitFlag()
if *debug {
logger.SetLevel(logger.LevelDebug)
}
err := Init()
if err != nil {
logger.Error("Failed to initialize Meilisearch: %v", err)
return
}
logger.Info("Root: %q", *root)
cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
cleanedRoot, err = filepath.Abs(cleanedRoot)
if err != nil {
logger.Error("error getting absolute path: %v", err)
return
}
cleanedRoot = filepath.Clean(cleanedRoot)
cleanedRoot = strings.TrimSuffix(cleanedRoot, "/")
logger.Info("Looking for Heimdall.lua in %q", cleanedRoot)
matches, err := doublestar.Glob(os.DirFS(cleanedRoot), "**/Heimdall.lua")
if err != nil {
logger.Error("error matching Heimdall.lua: %v", err)
return
}
logger.Info("Found %d Heimdall.lua files.", len(matches))
if len(matches) == 0 {
logger.Info("No Heimdall.lua files found. Exiting.")
return
}
for i, match := range matches {
matches[i] = filepath.Join(cleanedRoot, match)
}
luaStates := loadLuaStates(matches)
chatMessages := loadChatMessages(luaStates)
// Save messages to Meilisearch
if err := AddMessages(chatMessages); err != nil {
logger.Error("Failed to save messages: %v", err)
return
}
logger.Info("Successfully saved %d messages", len(chatMessages))
}
func loadLuaStates(matches []string) *sync.Map {
wg := sync.WaitGroup{}
fileLuaStates := &sync.Map{}
for _, match := range matches {
wg.Add(1)
go func(path string) {
defer wg.Done()
log := logger.Default.WithPrefix(path)
L := lua.NewState()
filestat, err := os.Stat(path)
if err != nil {
log.Error("error getting file stats: %v", err)
return
}
log.Info("File size: %.2f MB", float64(filestat.Size())/1024/1024)
log.Info("Running Lua file")
if err := L.DoFile(path); err != nil {
log.Error("error executing Lua file %q: %v", path, err)
return
}
log.Info("Lua file loaded")
fileLuaStates.Store(path, L)
}(match)
}
wg.Wait()
return fileLuaStates
}
func loadChatMessages(luaStates *sync.Map) []ChatMessage {
var messages []ChatMessage
wg := sync.WaitGroup{}
messageChan := make(chan []ChatMessage, 100) // Buffer for concurrent processing
luaStates.Range(func(path, state any) bool {
wg.Add(1)
go func(path string, state *lua.LState) {
defer wg.Done()
log := logger.Default.WithPrefix(path)
fileMessages := loadStateChatMessages(state, log)
messageChan <- fileMessages
}(path.(string), state.(*lua.LState))
return true
})
// Close channel when all goroutines are done
go func() {
wg.Wait()
close(messageChan)
}()
// Collect all messages
for fileMessages := range messageChan {
messages = append(messages, fileMessages...)
}
return messages
}
func loadStateChatMessages(L *lua.LState, log *logger.Logger) []ChatMessage {
log.Info("Getting Heimdall_Chat")
heimdallChat := L.GetGlobal("Heimdall_Chat")
if heimdallChat.Type() == lua.LTNil {
log.Warning("Heimdall_Chat not found. Skipping file.")
return nil
}
chatTable, ok := heimdallChat.(*lua.LTable)
if !ok {
log.Warning("Heimdall_Chat is not a table. Skipping file.")
return nil
}
var messages []ChatMessage
chatTable.ForEach(func(_, value lua.LValue) {
chatStr := value.String()
// Remove quotes and trailing comma if present
chatStr = strings.Trim(chatStr, "\", ")
// Parse the chat message
message, err := parseChatMessage(chatStr)
if err != nil {
log.Warning("Invalid chat format: %s", chatStr)
return
}
messages = append(messages, message)
})
log.Info("Loaded %d chat messages", len(messages))
return messages
}
func parseChatMessage(chatStr string) (ChatMessage, error) {
// Split by pipe - we expect 6 parts (5 pipes)
parts := strings.Split(chatStr, "|")
if len(parts) != 6 {
return ChatMessage{}, fmt.Errorf("invalid message format: %s", chatStr)
}
timestamp := parts[0]
event := parts[1]
sender := parts[2]
msg := parts[3]
language := parts[4]
channel := parts[5]
return ChatMessage{
MessageHash: GenerateMessageHash(timestamp, event, sender, msg, language, channel),
Timestamp: timestamp,
Event: event,
Sender: sender,
Msg: msg,
Language: language,
Channel: channel,
}, nil
}

1
service/chatsniffer/succ.sh Normal file
View File

@@ -0,0 +1 @@
./chatsniffer.exe -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"