Compare commits

...

19 Commits

Author SHA1 Message Date
e841dd5ecb Fix saveAchievementsToSourceFiles call in main.go and update succ.sh to include directory change before script execution 2025-05-25 03:19:09 +02:00
11beac50eb Refactor AddMessages to use structured payload and improve error handling in chatsniffer 2025-05-25 03:16:23 +02:00
5e6a5e830e Add initial implementation of chatsniffer with Meilisearch integration and required dependencies 2025-05-25 03:04:11 +02:00
9035a8284c Add Flush function to send remaining achievements and call it in saveAchievements 2025-05-24 23:05:37 +02:00
4fbc49c52f Implement batch processing for achievement saving with HTTP request handling 2025-05-24 22:47:11 +02:00
cdefc1735f Refactor SQL queries to use parameterized values for player names and similarity percentage 2025-05-24 22:14:30 +02:00
54cd559abe Implement loadUniquePlayers function to retrieve and store unique player names from the database 2025-05-21 18:16:11 +02:00
69459d8822 Update readme 2025-05-21 13:21:53 +02:00
259801c699 Add README.md to document current workflow 2025-05-21 13:20:33 +02:00
72d8c4052c Update file paths for matches and change logging level for achievement saving 2025-05-21 13:17:43 +02:00
b44e24e732 Enhance achievement saving by adding concurrency and logging 2025-05-21 13:12:46 +02:00
70c417e7f4 Implement writing "already seen" to source 2025-05-21 13:10:33 +02:00
4301294b66 Implement removing read achievements from source files 2025-05-21 13:01:10 +02:00
5a1bff50d3 Update debug file 2025-05-21 12:53:14 +02:00
c1cc508dcf Clean up shop 2025-05-21 12:45:55 +02:00
d3efd378f2 Implement alreadyseen and writingtofileTM 2025-05-21 12:41:39 +02:00
3f659d351d Implement writing to database with the new structure 2025-05-21 12:26:22 +02:00
f55a6de8c0 Add debug lua data file temporarily 2025-05-21 12:23:01 +02:00
028aa4e80b Refactor a large part of main to be less retarded 2025-05-21 12:22:48 +02:00
12 changed files with 3586 additions and 171 deletions

2760
Heimdall.lua Normal file

File diff suppressed because it is too large Load Diff

12
README.md Normal file
View File

@@ -0,0 +1,12 @@
Currently the only services used are the main one (see succ.sh)<br>
And the data
The NSQ service was meant to be used for pulling achievements from other players<br>
But currently there are no other players so it simply is not necessary
The cacher service is also obsolete because we switched our approach to `service/data/cache.sql`<br>
A single file, much more better<br>
Currently it runs for about 20 seconds for half a million rows in achievements<br>
Could be better, could be worse
The querying is then done via `service/data/groupPlayers.sql` with the query inserted into line 48

View File

@@ -1,9 +1,27 @@
package main
import (
"bytes"
"encoding/json"
"fmt"
"io"
"net/http"
"sync"
"time"
logger "git.site.quack-lab.dev/dave/cylogger"
)
const (
batchSize = 1000
timeout = 5 * time.Second
)
var (
batchBuffer = make([]NSQMessage, 0, batchSize)
batchMutex sync.Mutex
)
var whitelistedAchievements = map[string]bool{
"15": true,
"958": true,
@@ -65,19 +83,90 @@ var whitelistedAchievements = map[string]bool{
"12448": true,
}
func Save(message *NSQMessage, db *DB) error {
func Save(message NSQMessage) error {
_, ok := whitelistedAchievements[message.ID]
if !ok {
logger.Debug("Received message for non-whitelisted achievement %s", message.ID)
return nil
}
_, err := db.writeConn.Exec("INSERT OR IGNORE INTO achievements (name, id, date, completed) VALUES (?, ?, ?, ?)",
message.Name, message.ID, message.Date, message.Completed)
if err != nil {
logger.Error("Error inserting into database: %v", err)
return err
batchMutex.Lock()
batchBuffer = append(batchBuffer, message)
currentBatchSize := len(batchBuffer)
batchMutex.Unlock()
logger.Debug("Added achievement to batch. Current batch size: %d/%d", currentBatchSize, batchSize)
if currentBatchSize >= batchSize {
batchMutex.Lock()
batch := make([]NSQMessage, len(batchBuffer))
copy(batch, batchBuffer)
batchBuffer = batchBuffer[:0]
batchMutex.Unlock()
logger.Info("Batch size reached %d, sending batch to %s", len(batch), backendEndpoint)
return sendBatch(batch)
}
return nil
}
// Flush sends any remaining achievements in the buffer
func Flush() error {
batchMutex.Lock()
if len(batchBuffer) == 0 {
batchMutex.Unlock()
return nil
}
batch := make([]NSQMessage, len(batchBuffer))
copy(batch, batchBuffer)
batchBuffer = batchBuffer[:0]
batchMutex.Unlock()
logger.Info("Flushing final batch of %d achievements to %s", len(batch), backendEndpoint)
return sendBatch(batch)
}
func sendBatch(batch []NSQMessage) error {
logger.Debug("Preparing to send batch of %d achievements", len(batch))
data, err := json.Marshal(batch)
if err != nil {
logger.Error("Failed to marshal batch: %v", err)
return fmt.Errorf("error marshaling batch: %v", err)
}
logger.Debug("Successfully marshaled batch to JSON, size: %d bytes", len(data))
client := &http.Client{
Timeout: timeout,
}
req, err := http.NewRequest("POST", backendEndpoint, bytes.NewBuffer(data))
if err != nil {
logger.Error("Failed to create HTTP request: %v", err)
return fmt.Errorf("error creating request: %v", err)
}
req.Header.Set("Content-Type", "application/json")
logger.Debug("Created HTTP request to %s", backendEndpoint)
resp, err := client.Do(req)
if err != nil {
logger.Error("Failed to send HTTP request: %v", err)
return fmt.Errorf("error sending batch: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
logger.Error("Failed to read response body: %v", err)
}
if resp.StatusCode != http.StatusOK {
logger.Error("Received non-OK status code: %d, body: %s", resp.StatusCode, string(body))
return fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
}
logger.Info("Successfully sent batch of %d achievements", len(batch))
return nil
}

336
main.go
View File

@@ -20,14 +20,11 @@ import (
)
const nsqEndpoint = "https://nsq.site.quack-lab.dev/pub?topic=wowspy"
const backendEndpoint = "https://sniffer-be.site.quack-lab.dev/achievements"
var debug *bool
var nsqWorkers = 32
var allPlayersAchievementsGlobal = make(map[string][]NSQMessage) // PlayerName -> list of all their achievements
var allPlayerNamesGlobal = make(map[string]bool) // Set of all player names
var globalDataMutex = &sync.Mutex{}
func main() {
root := flag.String("root", ".", "Root workdir")
debug = flag.Bool("d", false, "Debug")
@@ -37,19 +34,9 @@ func main() {
logger.SetLevel(logger.LevelDebug) // Assuming LevelDebug is the correct constant for cylogger
}
db := DB{
path: "service/data/db.db",
}
err := db.Open()
if err != nil {
logger.Error("error opening database: %v", err)
return
}
defer db.Close()
logger.Info("Root: %q", *root)
cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
cleanedRoot, err = filepath.Abs(cleanedRoot)
cleanedRoot, err := filepath.Abs(cleanedRoot)
if err != nil {
logger.Error("error getting absolute path: %v", err)
return
@@ -68,136 +55,216 @@ func main() {
logger.Info("No Heimdall.lua files found. Exiting.")
return
}
// matches = matches[:1]
// --- Pass 1: Extract all data ---
logger.Info("Starting Pass 1: Extracting data from all Heimdall.lua files...")
var wgPass1 sync.WaitGroup
for _, match := range matches {
wgPass1.Add(1)
go loadAchievements(filepath.Join(cleanedRoot, match), &wgPass1)
}
wgPass1.Wait()
logger.Info("Finished Pass 1: Loaded %d unique players from %d files.", len(allPlayerNamesGlobal), len(matches))
if *debug {
globalDataMutex.Lock()
logger.Debug("Total achievements loaded globally: %d", countTotalAchievements(allPlayersAchievementsGlobal))
globalDataMutex.Unlock()
for i, match := range matches {
matches[i] = filepath.Join(cleanedRoot, match)
}
wgSave := sync.WaitGroup{}
wgSave.Add(1)
luaStates := loadLuaStates(matches)
achievements := loadAchievements(luaStates)
wg := sync.WaitGroup{}
wg.Add(1)
// We can save the achievements to the database while doing something else unrelated
go func() {
logger.Info("Saving achievements to database...")
for playerName, achList := range allPlayersAchievementsGlobal {
logger.Debug("Saving %d achievements for player %s", len(achList), playerName)
for _, ach := range achList {
Save(&ach, &db)
}
}
wgSave.Done()
defer wg.Done()
saveAchievements(achievements)
}()
saveAchievementsToSourceFiles(luaStates, achievements)
wg.Wait()
}
// --- Process and Send to NSQ ---
// logger.Info("Starting NSQ message publishing...")
// nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size
// var wgNsqWorkers sync.WaitGroup
// for i := 0; i < nsqWorkers; i++ {
// wgNsqWorkers.Add(1)
// go NsqWorker(&wgNsqWorkers, nsqMessagesChan)
// }
func saveAchievementsToSourceFiles(luaStates *sync.Map, achievements *sync.Map) {
wg := sync.WaitGroup{}
luaStates.Range(func(k, v any) bool {
path := k.(string)
state := v.(*lua.LState)
log := logger.Default.WithPrefix(path)
// go func() {
// globalDataMutex.Lock()
// defer globalDataMutex.Unlock()
// for playerName, achList := range allPlayersAchievementsGlobal {
// for _, ach := range achList {
// // ach.Name is already correctly set during extraction
// nsqMessagesChan <- ach
// logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID)
// }
// }
// close(nsqMessagesChan) // Close channel when all messages are sent
// logger.Info("All NSQ messages queued.")
// }()
// --- Pass 2: Update Lua file states (in memory) ---
logger.Info("Starting Pass 2: Updating Lua states (setting alreadySeen and clearing players)...")
var wgPass2 sync.WaitGroup
if len(allPlayerNamesGlobal) > 0 { // Only run pass 2 if there are players to report
for _, match := range matches {
wgPass2.Add(1)
go updateLuaFileState(filepath.Join(cleanedRoot, match), &wgPass2, allPlayerNamesGlobal)
log.Info("Clearing existing achievements")
achievementTable := state.GetGlobal("Heimdall_Achievements")
if achievementTable.Type() != lua.LTTable {
achievementTable = &lua.LTable{}
state.SetGlobal("Heimdall_Achievements", achievementTable)
}
wgPass2.Wait()
logger.Info("Finished Pass 2: Lua states updated where applicable.")
} else {
logger.Info("Skipping Pass 2 as no players were found globally.")
}
log.Info("Clearing existing players table")
emptyTable := &lua.LTable{}
state.SetField(achievementTable, "players", emptyTable)
// wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed
wgSave.Wait()
logger.Info("All NSQ workers finished. Program complete.")
log.Info("Updating seen table")
seenTable := state.GetField(achievementTable, "alreadySeen")
if seenTable.Type() != lua.LTTable {
seenTable = &lua.LTable{}
state.SetField(achievementTable, "alreadySeen", seenTable)
}
fixSource(path, achievements)
return true
})
wg.Wait()
}
// Helper function to count total achievements for debugging
func countTotalAchievements(achMap map[string][]NSQMessage) int {
count := 0
for _, achList := range achMap {
count += len(achList)
}
return count
}
func loadAchievements(path string, wg *sync.WaitGroup) {
func fixSource(path string, achievements *sync.Map) {
log := logger.Default.WithPrefix(path)
log.Info("Extracting achievements")
defer wg.Done()
L := lua.NewState()
defer L.Close()
filestat, err := os.Stat(path)
log.Info("Reading source file")
fileContent, err := os.ReadFile(path)
if err != nil {
log.Error("error getting file stats: %v", err)
logger.Error("Failed to read file: %v", err)
return
}
log.Info("File size: %.2f MB", float64(filestat.Size())/1024/1024)
strContent := string(fileContent)
log.Info("Read %d bytes", len(strContent))
log.Info("Running Lua file")
if err := L.DoFile(path); err != nil {
log.Error("error executing Lua file %q: %v", path, err)
strContent = removeAchievements(strContent)
log.Info("Removed achievements, now %d bytes", len(strContent))
strContent = addAlreadySeen(strContent, achievements)
log.Info("Added alreadySeen, now %d bytes", len(strContent))
log.Info("Writing file")
err = os.WriteFile(path, []byte(strContent), 0644)
if err != nil {
logger.Error("Failed to write file: %v", err)
return
}
log.Info("Done")
}
func removeAchievements(sourceContent string) string {
lines := strings.Split(sourceContent, "\n")
writeIndex := 0
isInPlayers := false
for _, line := range lines {
if strings.HasPrefix(line, "\t[\"players\"] = {") {
isInPlayers = true
lines[writeIndex] = line
writeIndex++
continue
}
if isInPlayers && strings.HasPrefix(line, "\t}") {
isInPlayers = false
lines[writeIndex] = line
writeIndex++
continue
}
if !isInPlayers {
lines[writeIndex] = line
writeIndex++
}
}
return strings.Join(lines[:writeIndex], "\n")
}
func addAlreadySeen(strContent string, achievements *sync.Map) string {
lines := strings.Split(strContent, "\n")
modifiedLines := make([]string, 0, len(lines))
for _, line := range lines {
if strings.HasPrefix(line, "\t[\"alreadySeen\"] = {") {
modifiedLines = append(modifiedLines, line)
achievements.Range(func(k, v any) bool {
logger.Trace("Adding alreadySeen for %s", k)
playerName := k.(string)
modifiedLines = append(modifiedLines, fmt.Sprintf("\t\t[\"%s\"] = true,", playerName))
return true
})
continue
}
modifiedLines = append(modifiedLines, line)
}
return strings.Join(modifiedLines, "\n")
}
func saveAchievements(achievements *sync.Map) {
count := 0
achievements.Range(func(k, v any) bool {
playerName := k.(string)
playerAchievements := v.(*[]NSQMessage)
logger.Debug("Saving %d achievements for player %s", len(*playerAchievements), playerName)
for _, ach := range *playerAchievements {
Save(ach)
}
count++
if count%1000 == 0 {
logger.Info("Saved %d achievements", count)
}
return true
})
Flush()
logger.Info("Saved %d achievements", count)
}
func loadLuaStates(matches []string) *sync.Map {
wg := sync.WaitGroup{}
fileLuaStates := &sync.Map{}
for _, match := range matches {
wg.Add(1)
go func(path string) {
defer wg.Done()
log := logger.Default.WithPrefix(path)
L := lua.NewState()
filestat, err := os.Stat(match)
if err != nil {
log.Error("error getting file stats: %v", err)
return
}
log.Info("File size: %.2f MB", float64(filestat.Size())/1024/1024)
log.Info("Running Lua file")
if err := L.DoFile(path); err != nil {
log.Error("error executing Lua file %q: %v", path, err)
return
}
log.Info("Lua file loaded")
fileLuaStates.Store(match, L)
}(match)
}
wg.Wait()
return fileLuaStates
}
func loadAchievements(luaStates *sync.Map) *sync.Map {
achievements := &sync.Map{}
wg := sync.WaitGroup{}
luaStates.Range(func(path, state any) bool {
wg.Add(1)
go func(path string, state *lua.LState) {
log := logger.Default.WithPrefix(path)
defer wg.Done()
// We directly mutate achievements to avoid reducing and mapping later on
// Removing 1 off of the x of the O(xn)
loadStateAchievements(state, log, achievements)
}(path.(string), state.(*lua.LState))
return true
})
wg.Wait()
return achievements
}
func loadStateAchievements(L *lua.LState, log *logger.Logger, achievements *sync.Map) {
log.Info("Getting Heimdall_Achievements")
heimdallAchievements := L.GetGlobal("Heimdall_Achievements")
if heimdallAchievements.Type() == lua.LTNil {
log.Warning("Heimdall_Achievements not found in %q. Skipping file.", path)
log.Warning("Heimdall_Achievements not found. Skipping file.")
return
}
log.Info("Getting players table")
playersTableLua := L.GetField(heimdallAchievements, "players")
if playersTableLua.Type() == lua.LTNil {
log.Info("'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path)
log.Info("'players' table is nil in Heimdall_Achievements. No player data to extract.")
return
}
log.Info("Casting players table")
playersTable, ok := playersTableLua.(*lua.LTable)
if !ok {
log.Warning("'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String())
log.Warning("'players' field in Heimdall_Achievements is not a table. Skipping.")
return
}
var filePlayerAchievements []NSQMessage
var filePlayerNames = make(map[string]bool)
log.Info("Iterating over players")
counter := 0
playersTable.ForEach(func(playerNameLua lua.LValue, playerAchievementsLua lua.LValue) {
currentPlayerName := playerNameLua.String()
filePlayerNames[currentPlayerName] = true // Track name
playerAchievements, _ := achievements.LoadOrStore(currentPlayerName, &[]NSQMessage{})
playerAchievementsSlice := playerAchievements.(*[]NSQMessage)
achievementsTableLua, ok := playerAchievementsLua.(*lua.LTable)
if !ok {
@@ -238,7 +305,8 @@ func loadAchievements(path string, wg *sync.WaitGroup) {
}
if currentAchievement.ID != "" { // Ensure we have at least an ID before adding
filePlayerAchievements = append(filePlayerAchievements, currentAchievement)
// Will this change be reflected in the map...?
*playerAchievementsSlice = append(*playerAchievementsSlice, currentAchievement)
}
counter++
@@ -248,59 +316,13 @@ func loadAchievements(path string, wg *sync.WaitGroup) {
})
})
log.Info("Processed %d achievements", counter)
if len(filePlayerAchievements) > 0 || len(filePlayerNames) > 0 {
globalDataMutex.Lock()
for _, ach := range filePlayerAchievements {
allPlayersAchievementsGlobal[ach.Name] = append(allPlayersAchievementsGlobal[ach.Name], ach)
}
for name := range filePlayerNames {
allPlayerNamesGlobal[name] = true
}
globalDataMutex.Unlock()
log.Info("Players in file: %d. Achievements in file: %d.", len(filePlayerNames), len(filePlayerAchievements))
} else {
log.Info("No player data or names extracted")
}
achievements.Range(func(key, value any) bool {
log.Trace("Player: %s, Achievements: %d", key, len(*value.(*[]NSQMessage)))
return true
})
}
// updateLuaFileState is for Pass 2
func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map[string]bool) {
log := logger.Default.WithPrefix(filepath.Base(path))
log.Info("Updating Lua state")
defer wg.Done()
L := lua.NewState()
defer L.Close()
if err := L.DoFile(path); err != nil {
log.Error("error executing Lua file %q: %v. Cannot update its state.", path, err)
return
}
heimdallAchievementsVal := L.GetGlobal("Heimdall_Achievements")
if heimdallAchievementsVal.Type() == lua.LTNil {
log.Warning("Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path)
return
}
heimdallAchievementsTable, ok := heimdallAchievementsVal.(*lua.LTable)
if !ok {
log.Warning("Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String())
return
}
luaAlreadySeen := L.NewTable()
for name := range allKnownPlayerNames {
luaAlreadySeen.RawSetString(name, lua.LTrue)
}
L.SetField(heimdallAchievementsTable, "alreadySeen", luaAlreadySeen)
log.Debug("Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames))
L.SetField(heimdallAchievementsTable, "players", L.NewTable())
log.Debug("Cleared Heimdall_Achievements.players for %q.", path)
}
func NsqWorker(wg *sync.WaitGroup, messages <-chan NSQMessage) { // Changed to read-only channel
defer wg.Done()
for msg := range messages {

View File

@@ -0,0 +1,9 @@
module chatsniffer
go 1.24.3
require (
git.site.quack-lab.dev/dave/cylogger v1.2.2
github.com/bmatcuk/doublestar/v4 v4.8.1
github.com/yuin/gopher-lua v1.1.1
)

View File

@@ -0,0 +1,6 @@
git.site.quack-lab.dev/dave/cylogger v1.2.2 h1:4xUXASEBlG9NiGxh7f57xHh9imW4unHzakIEpQoKC5E=
git.site.quack-lab.dev/dave/cylogger v1.2.2/go.mod h1:VS9MI4Y/cwjCBZgel7dSfCQlwtAgHmfvixOoBgBhtKg=
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=

View File

@@ -0,0 +1,247 @@
package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
logger "git.site.quack-lab.dev/dave/cylogger"
)
var meiliToken string
type IndexConfig struct {
Uid string `json:"uid"`
PrimaryKey string `json:"primaryKey"`
}
type ChatMessage struct {
MessageHash string `json:"message_hash"`
Timestamp string `json:"timestamp"` // ISO timestamp
EpochTime int64 `json:"epoch_time"` // Unix epoch timestamp for filtering
Event string `json:"event"`
Sender string `json:"sender"`
Msg string `json:"msg"`
Language string `json:"language"`
Channel string `json:"channel"`
}
func Init() error {
// Load Meilisearch token
meiliToken = os.Getenv("MEILI_TOKEN")
if meiliToken == "" {
return fmt.Errorf("MEILI_TOKEN environment variable not set")
}
logger.Info("Meilisearch token loaded")
config := IndexConfig{
Uid: meiliIndex,
PrimaryKey: "message_hash", // Meilisearch will use this for deduplication
}
// Create index
err := createIndex(config)
if err != nil {
return fmt.Errorf("error creating index: %v", err)
}
// Set up index settings
err = setIndexSettings()
if err != nil {
return fmt.Errorf("error setting index settings: %v", err)
}
return nil
}
// GenerateMessageHash creates a unique hash for a message that will be identical
// for identical messages, ensuring perfect deduplication
func GenerateMessageHash(timestamp, event, sender, msg, language, channel string) string {
// Combine all fields that make a message unique
content := fmt.Sprintf("%s|%s|%s|%s|%s|%s",
timestamp,
event,
sender,
msg,
language,
channel,
)
// Create SHA-256 hash of the combined content
hash := sha256.Sum256([]byte(content))
return hex.EncodeToString(hash[:])
}
// AddMessages adds multiple messages to the index in a single batch request
// Meilisearch will handle deduplication based on the message_hash
func AddMessages(messages []ChatMessage) error {
jsonData, err := json.Marshal(messages)
if err != nil {
return fmt.Errorf("error marshaling messages: %v", err)
}
req, err := http.NewRequest(
http.MethodPost,
meiliEndpoint+"indexes/"+meiliIndex+"/documents",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error adding messages: %v", err)
}
defer resp.Body.Close()
// Read response body for better error messages
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading response body: %v", err)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to add messages. Status: %d, Response: %s", resp.StatusCode, string(body))
}
return nil
}
func createIndex(config IndexConfig) error {
jsonData, err := json.Marshal(config)
if err != nil {
return fmt.Errorf("error marshaling config: %v", err)
}
req, err := http.NewRequest(
http.MethodPost,
meiliEndpoint+"indexes",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error creating index: %v", err)
}
defer resp.Body.Close()
// Read response body for better error messages
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading response body: %v", err)
}
if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to create index. Status: %d, Response: %s", resp.StatusCode, string(body))
}
logger.Info("Index created successfully")
return nil
}
func setIndexSettings() error {
// First set searchable attributes
searchableAttributes := []string{
"timestamp",
"event",
"sender",
"msg",
"language",
"channel",
}
jsonData, err := json.Marshal(searchableAttributes)
if err != nil {
return fmt.Errorf("error marshaling searchable settings: %v", err)
}
req, err := http.NewRequest(
http.MethodPut,
meiliEndpoint+"indexes/"+meiliIndex+"/settings/searchable-attributes",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating searchable settings request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("error setting searchable attributes: %v", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading searchable settings response: %v", err)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to set searchable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
}
// Then set filterable attributes
filterableAttributes := []string{
"timestamp",
"epoch_time", // Add epoch_time for numeric filtering
"event",
"sender",
"language",
"channel",
}
jsonData, err = json.Marshal(filterableAttributes)
if err != nil {
return fmt.Errorf("error marshaling filterable settings: %v", err)
}
req, err = http.NewRequest(
http.MethodPut,
meiliEndpoint+"indexes/"+meiliIndex+"/settings/filterable-attributes",
bytes.NewBuffer(jsonData),
)
if err != nil {
return fmt.Errorf("error creating filterable settings request: %v", err)
}
req.Header.Set("Authorization", "Bearer "+meiliToken)
req.Header.Set("Content-Type", "application/json")
resp, err = client.Do(req)
if err != nil {
return fmt.Errorf("error setting filterable attributes: %v", err)
}
defer resp.Body.Close()
body, err = io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("error reading filterable settings response: %v", err)
}
if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
return fmt.Errorf("failed to set filterable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
}
logger.Info("Index settings set successfully")
return nil
}

267
service/chatsniffer/main.go Normal file
View File

@@ -0,0 +1,267 @@
package main
import (
"flag"
"fmt"
"os"
"path/filepath"
"strings"
"sync"
"time"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/bmatcuk/doublestar/v4"
lua "github.com/yuin/gopher-lua"
)
const meiliEndpoint = "https://meili.site.quack-lab.dev/"
const meiliIndex = "chatlog"
var debug *bool
func main() {
root := flag.String("root", ".", "Root workdir")
debug = flag.Bool("d", false, "Debug")
flag.Parse()
logger.InitFlag()
if *debug {
logger.SetLevel(logger.LevelDebug)
}
err := Init()
if err != nil {
logger.Error("Failed to initialize Meilisearch: %v", err)
return
}
logger.Info("Root: %q", *root)
cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
cleanedRoot, err = filepath.Abs(cleanedRoot)
if err != nil {
logger.Error("error getting absolute path: %v", err)
return
}
cleanedRoot = filepath.Clean(cleanedRoot)
cleanedRoot = strings.TrimSuffix(cleanedRoot, "/")
logger.Info("Looking for Heimdall.lua in %q", cleanedRoot)
matches, err := doublestar.Glob(os.DirFS(cleanedRoot), "**/Heimdall.lua")
if err != nil {
logger.Error("error matching Heimdall.lua: %v", err)
return
}
logger.Info("Found %d Heimdall.lua files.", len(matches))
if len(matches) == 0 {
logger.Info("No Heimdall.lua files found. Exiting.")
return
}
for i, match := range matches {
matches[i] = filepath.Join(cleanedRoot, match)
}
luaStates := loadLuaStates(matches)
chatMessages := loadChatMessages(luaStates)
// Save messages to Meilisearch
if err := AddMessages(chatMessages); err != nil {
logger.Error("Failed to save messages: %v", err)
return
}
logger.Info("Successfully saved %d messages", len(chatMessages))
// Clear chat tables in source files
clearChatTables(matches)
}
func loadLuaStates(matches []string) *sync.Map {
wg := sync.WaitGroup{}
fileLuaStates := &sync.Map{}
for _, match := range matches {
wg.Add(1)
go func(path string) {
defer wg.Done()
log := logger.Default.WithPrefix(path)
L := lua.NewState()
filestat, err := os.Stat(path)
if err != nil {
log.Error("error getting file stats: %v", err)
return
}
log.Info("File size: %.2f MB", float64(filestat.Size())/1024/1024)
log.Info("Running Lua file")
if err := L.DoFile(path); err != nil {
log.Error("error executing Lua file %q: %v", path, err)
return
}
log.Info("Lua file loaded")
fileLuaStates.Store(path, L)
}(match)
}
wg.Wait()
return fileLuaStates
}
func loadChatMessages(luaStates *sync.Map) []ChatMessage {
var messages []ChatMessage
wg := sync.WaitGroup{}
messageChan := make(chan []ChatMessage, 100) // Buffer for concurrent processing
luaStates.Range(func(path, state any) bool {
wg.Add(1)
go func(path string, state *lua.LState) {
defer wg.Done()
log := logger.Default.WithPrefix(path)
fileMessages := loadStateChatMessages(state, log)
messageChan <- fileMessages
}(path.(string), state.(*lua.LState))
return true
})
// Close channel when all goroutines are done
go func() {
wg.Wait()
close(messageChan)
}()
// Collect all messages
for fileMessages := range messageChan {
messages = append(messages, fileMessages...)
}
return messages
}
func loadStateChatMessages(L *lua.LState, log *logger.Logger) []ChatMessage {
log.Info("Getting Heimdall_Chat")
heimdallChat := L.GetGlobal("Heimdall_Chat")
if heimdallChat.Type() == lua.LTNil {
log.Warning("Heimdall_Chat not found. Skipping file.")
return nil
}
chatTable, ok := heimdallChat.(*lua.LTable)
if !ok {
log.Warning("Heimdall_Chat is not a table. Skipping file.")
return nil
}
var messages []ChatMessage
chatTable.ForEach(func(_, value lua.LValue) {
chatStr := value.String()
// Remove quotes and trailing comma if present
chatStr = strings.Trim(chatStr, "\", ")
// Parse the chat message
message, err := parseChatMessage(chatStr)
if err != nil {
log.Warning("Invalid chat format: %s", chatStr)
return
}
messages = append(messages, message)
})
log.Info("Loaded %d chat messages", len(messages))
return messages
}
func parseChatMessage(chatStr string) (ChatMessage, error) {
// Debug: Print the raw string
logger.Debug("Raw chat string: %q", chatStr)
// Split by pipe - we expect 6 parts (5 pipes)
parts := strings.Split(chatStr, "|")
logger.Debug("Split into %d parts: %v", len(parts), parts)
if len(parts) != 6 {
return ChatMessage{}, fmt.Errorf("invalid message format: expected 6 parts, got %d: %s", len(parts), chatStr)
}
timestamp := parts[0]
event := parts[1]
sender := parts[2]
msg := parts[3]
language := parts[4]
channel := parts[5]
// Parse ISO timestamp to epoch
epochTime, err := parseISOTimestamp(timestamp)
if err != nil {
return ChatMessage{}, fmt.Errorf("invalid timestamp format: %v", err)
}
return ChatMessage{
MessageHash: GenerateMessageHash(timestamp, event, sender, msg, language, channel),
Timestamp: timestamp,
EpochTime: epochTime,
Event: event,
Sender: sender,
Msg: msg,
Language: language,
Channel: channel,
}, nil
}
func parseISOTimestamp(isoTime string) (int64, error) {
// Debug: Print the timestamp we're trying to parse
logger.Debug("Parsing timestamp: %q", isoTime)
// Parse the timestamp format used in chat messages (e.g., "2025-05-25 02:51:05")
t, err := time.Parse("2006-01-02 15:04:05", isoTime)
if err != nil {
logger.Debug("Failed to parse timestamp: %v", err)
return 0, err
}
return t.Unix(), nil
}
func clearChatTables(matches []string) {
for _, path := range matches {
log := logger.Default.WithPrefix(path)
log.Info("Reading source file")
fileContent, err := os.ReadFile(path)
if err != nil {
logger.Error("Failed to read file: %v", err)
continue
}
strContent := string(fileContent)
log.Info("Read %d bytes", len(strContent))
strContent = clearChatTable(strContent)
log.Info("Cleared chat table, now %d bytes", len(strContent))
log.Info("Writing file")
err = os.WriteFile(path, []byte(strContent), 0644)
if err != nil {
logger.Error("Failed to write file: %v", err)
continue
}
log.Info("Done")
}
}
func clearChatTable(sourceContent string) string {
lines := strings.Split(sourceContent, "\n")
writeIndex := 0
isInChat := false
for _, line := range lines {
if strings.HasPrefix(line, "Heimdall_Chat = {") {
isInChat = true
lines[writeIndex] = "Heimdall_Chat = {"
writeIndex++
continue
}
if isInChat && strings.HasPrefix(line, "}") {
isInChat = false
lines[writeIndex] = "}"
writeIndex++
continue
}
if !isInChat {
lines[writeIndex] = line
writeIndex++
}
}
return strings.Join(lines[:writeIndex], "\n")
}

View File

@@ -0,0 +1 @@
./chatsniffer.exe -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"

View File

@@ -4,14 +4,14 @@ with char1_achievements as (
date,
completed
from achievements
where name = 'Extazyk'
where name = $1
),
char2_achievements as (
select id,
date,
completed
from achievements
where name = 'Smokemantra'
where name = $2
),
all_achievements as (
select id

View File

@@ -5,15 +5,15 @@ with RECURSIVE connected_players as (
total_achievements1 as achievements,
similarity_percentage
from similar_pairs
where similarity_percentage >= 70
where similarity_percentage >= $1
union
select name2,
name1,
matching_count,
total_achievements2,
similarity_percentage
from similar_pairs
where similarity_percentage >= 70
from similar_pairs
where similarity_percentage >= $1
union
select case
when sp.name1 = cp.player_name then sp.name2
@@ -31,7 +31,7 @@ with RECURSIVE connected_players as (
sp.name1 = cp.player_name
or sp.name2 = cp.player_name
)
and sp.similarity_percentage >= 70
and sp.similarity_percentage >= $1
where case
when sp.name1 = cp.player_name then sp.name2
else sp.name2
@@ -45,7 +45,7 @@ select group_root,
MIN(matching_count) as min_matching,
AVG(matching_count) as avg_matching
from connected_players
where group_root in ('Paskoo')
where group_root in ($2)
group by group_root
having count(*) > 1
order by count(*) desc,

View File

@@ -1 +1,3 @@
go run . -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"
./main.exe -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"
cd service/chatsniffer
sh succ.sh