From 114c0b909b8880be39059d6471d4191581ffc7bc Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Wed, 21 May 2025 11:34:42 +0200 Subject: [PATCH] Make main.go write into database instead of nsq, there's just no need for it like I thought there would be --- db.go | 84 +++++++++++++++++++++++++++++++++++++++++++ dbwriter.go | 83 +++++++++++++++++++++++++++++++++++++++++++ main.go | 100 +++++++++++++++++++++++++++++++++------------------- succ.sh | 1 + 4 files changed, 231 insertions(+), 37 deletions(-) create mode 100644 db.go create mode 100644 dbwriter.go create mode 100644 succ.sh diff --git a/db.go b/db.go new file mode 100644 index 0000000..f8fe04c --- /dev/null +++ b/db.go @@ -0,0 +1,84 @@ +package main + +import ( + "database/sql" + "fmt" + "os" + "time" + + _ "github.com/mattn/go-sqlite3" + logger "git.site.quack-lab.dev/dave/cylogger" +) + +type DB struct { + Ready bool + path string + readConn *sql.DB + writeConn *sql.DB +} + +func (db *DB) Open() error { + if db.path == "" { + return fmt.Errorf("database path not set") + } + + file, err := os.Open(db.path) + if err != nil { + if os.IsNotExist(err) { + logger.Info("Database file does not exist at %s, creating", db.path) + file, err := os.Create(db.path) + if err != nil { + return fmt.Errorf("failed to create database file: %v", err) + } + logger.Info("Database created at %s", db.path) + file.Close() + } else { + return fmt.Errorf("failed to open database file: %v", err) + } + } + file.Close() + + writeConn, err := sql.Open("sqlite3", db.path+"?_journal=WAL&_synchronous=NORMAL") + if err != nil { + logger.Error("%++v", err) + return err + } + writeConn.SetMaxOpenConns(1) + writeConn.SetConnMaxIdleTime(30 * time.Second) + writeConn.SetConnMaxLifetime(30 * time.Second) + db.writeConn = writeConn + + readConn, err := sql.Open("sqlite3", db.path+"?mode=ro&_journal=WAL&_synchronous=NORMAL&_mode=ro") + if err != nil { + logger.Error("%++v", err) + return err + } + readConn.SetMaxOpenConns(4) + readConn.SetConnMaxIdleTime(30 * time.Second) + readConn.SetConnMaxLifetime(30 * time.Second) + db.readConn = readConn + + db.Ready = true + return nil +} + +func (db *DB) Init(ddl string) error { + if !db.Ready { + return fmt.Errorf("database not ready") + } + return nil +} + +func (db *DB) Close() error { + err := db.writeConn.Close() + if err != nil { + return err + } + + err = db.readConn.Close() + if err != nil { + return err + } + + return nil +} \ No newline at end of file diff --git a/dbwriter.go b/dbwriter.go new file mode 100644 index 0000000..e35a596 --- /dev/null +++ b/dbwriter.go @@ -0,0 +1,83 @@ +package main + +import ( + logger "git.site.quack-lab.dev/dave/cylogger" +) + +var whitelistedAchievements = map[string]bool{ + "15": true, + "958": true, + "1276": true, + "2088": true, + "2151": true, + "5466": true, + "5759": true, + "6470": true, + "6763": true, + "7392": true, + "7393": true, + "7394": true, + "7958": true, + "8939": true, + "8992": true, + "9048": true, + "94103": true, + "10059": true, + "10079": true, + "10278": true, + "10657": true, + "10672": true, + "10684": true, + "10688": true, + "10689": true, + "10692": true, + "10693": true, + "10698": true, + "10790": true, + "10875": true, + "11124": true, + "11126": true, + "11127": true, + "11128": true, + "11157": true, + "11164": true, + "11188": true, + "11189": true, + "11190": true, + "11446": true, + "11473": true, + "11610": true, + "11674": true, + "11992": true, + "11993": true, + "11994": true, + "11995": true, + "11996": true, + "11997": true, + "11998": true, + "11999": true, + "12000": true, + "12001": true, + "12026": true, + "12074": true, + "12445": true, + "12447": true, + "12448": true, +} + +func Save(message *NSQMessage, db *DB) error { + _, ok := whitelistedAchievements[message.ID] + if !ok { + logger.Warning("Received message for non-whitelisted achievement %s", message.ID) + return nil + } + + _, err := db.writeConn.Exec("INSERT OR IGNORE INTO achievements (name, id, date, completed) VALUES (?, ?, ?, ?)", + message.Name, message.ID, message.Date, message.Completed) + if err != nil { + logger.Error("Error inserting into database: %v", err) + return err + } + + return nil +} diff --git a/main.go b/main.go index a4b7108..efa44ae 100644 --- a/main.go +++ b/main.go @@ -37,9 +37,19 @@ func main() { logger.SetLevel(logger.LevelDebug) // Assuming LevelDebug is the correct constant for cylogger } + db := DB{ + path: "service/data/db.db", + } + err := db.Open() + if err != nil { + logger.Error("error opening database: %v", err) + return + } + defer db.Close() + logger.Info("Root: %q", *root) cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1) - cleanedRoot, err := filepath.Abs(cleanedRoot) + cleanedRoot, err = filepath.Abs(cleanedRoot) if err != nil { logger.Error("error getting absolute path: %v", err) return @@ -59,8 +69,7 @@ func main() { return } - // Debugging - matches = matches[:2] + // matches = matches[:1] // --- Pass 1: Extract all data --- logger.Info("Starting Pass 1: Extracting data from all Heimdall.lua files...") @@ -77,29 +86,42 @@ func main() { globalDataMutex.Unlock() } - // --- Process and Send to NSQ --- - logger.Info("Starting NSQ message publishing...") - nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size - var wgNsqWorkers sync.WaitGroup - for i := 0; i < nsqWorkers; i++ { - wgNsqWorkers.Add(1) - go NsqWorker(&wgNsqWorkers, nsqMessagesChan) - } - + wgSave := sync.WaitGroup{} + wgSave.Add(1) go func() { - globalDataMutex.Lock() - defer globalDataMutex.Unlock() + logger.Info("Saving achievements to database...") for playerName, achList := range allPlayersAchievementsGlobal { + logger.Debug("Saving %d achievements for player %s", len(achList), playerName) for _, ach := range achList { - // ach.Name is already correctly set during extraction - nsqMessagesChan <- ach - logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID) + Save(&ach, &db) } } - close(nsqMessagesChan) // Close channel when all messages are sent - logger.Info("All NSQ messages queued.") + wgSave.Done() }() + // --- Process and Send to NSQ --- + // logger.Info("Starting NSQ message publishing...") + // nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size + // var wgNsqWorkers sync.WaitGroup + // for i := 0; i < nsqWorkers; i++ { + // wgNsqWorkers.Add(1) + // go NsqWorker(&wgNsqWorkers, nsqMessagesChan) + // } + + // go func() { + // globalDataMutex.Lock() + // defer globalDataMutex.Unlock() + // for playerName, achList := range allPlayersAchievementsGlobal { + // for _, ach := range achList { + // // ach.Name is already correctly set during extraction + // nsqMessagesChan <- ach + // logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID) + // } + // } + // close(nsqMessagesChan) // Close channel when all messages are sent + // logger.Info("All NSQ messages queued.") + // }() + // --- Pass 2: Update Lua file states (in memory) --- logger.Info("Starting Pass 2: Updating Lua states (setting alreadySeen and clearing players)...") var wgPass2 sync.WaitGroup @@ -114,7 +136,8 @@ func main() { logger.Info("Skipping Pass 2 as no players were found globally.") } - wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed + // wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed + wgSave.Wait() logger.Info("All NSQ workers finished. Program complete.") } @@ -129,30 +152,31 @@ func countTotalAchievements(achMap map[string][]NSQMessage) int { // extractPlayerAchievementsFromFile is for Pass 1 func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) { - logger.Info("Extracting achievements from %q", path) + log := logger.Default.WithPrefix(filepath.Base(path)) + log.Info("Extracting achievements") defer wg.Done() L := lua.NewState() defer L.Close() if err := L.DoFile(path); err != nil { - logger.Error("Pass 1: error executing Lua file %q: %v", path, err) + log.Error("error executing Lua file %q: %v", path, err) return } heimdallAchievements := L.GetGlobal("Heimdall_Achievements") if heimdallAchievements.Type() == lua.LTNil { - logger.Warning("Pass 1: Heimdall_Achievements not found in %q. Skipping file.", path) + log.Warning("Heimdall_Achievements not found in %q. Skipping file.", path) return } playersTableLua := L.GetField(heimdallAchievements, "players") if playersTableLua.Type() == lua.LTNil { - logger.Info("Pass 1: 'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path) + log.Info("'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path) return } playersTable, ok := playersTableLua.(*lua.LTable) if !ok { - logger.Warning("Pass 1: 'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String()) + log.Warning("'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String()) return } @@ -165,14 +189,14 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) { achievementsTableLua, ok := playerAchievementsLua.(*lua.LTable) if !ok { - logger.Error("Pass 1: Achievements for player %s is not a table in %q. Skipping achievements for this player.", currentPlayerName, path) + log.Error("Achievements for player %s is not a table. Skipping achievements for this player.", currentPlayerName) return } achievementsTableLua.ForEach(func(_ lua.LValue, achievementDataLua lua.LValue) { achievementTable, ok := achievementDataLua.(*lua.LTable) if !ok { - logger.Error("Pass 1: Achievement data for player %s is not a table in %q. Skipping this achievement.", currentPlayerName, path) + log.Error("Achievement data for player %s is not a table. Skipping this achievement.", currentPlayerName) return } @@ -184,21 +208,21 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) { } else if idVal.Type() == lua.LTString { currentAchievement.ID = idVal.String() } else { - logger.Warning("Pass 1: Missing or invalid 'id' (expected number or string) for achievement for player %s in %q.", currentPlayerName, path) + log.Warning("Missing or invalid 'id' (expected number or string) for achievement for player %s.", currentPlayerName) } dateVal := achievementTable.RawGetString("date") if dateVal.Type() == lua.LTString { currentAchievement.Date = dateVal.String() } else { - logger.Warning("Pass 1: Missing or invalid 'date' (expected string) for achievement for player %s in %q.", currentPlayerName, path) + log.Warning("Missing or invalid 'date' (expected string) for achievement for player %s.", currentPlayerName) } completedVal := achievementTable.RawGetString("completed") if completedVal.Type() == lua.LTBool { currentAchievement.Completed = lua.LVAsBool(completedVal) } else { - logger.Warning("Pass 1: Missing or invalid 'completed' (expected boolean) for achievement for player %s in %q.", currentPlayerName, path) + log.Warning("Missing or invalid 'completed' (expected boolean) for achievement for player %s.", currentPlayerName) } if currentAchievement.ID != "" { // Ensure we have at least an ID before adding @@ -216,32 +240,34 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) { allPlayerNamesGlobal[name] = true } globalDataMutex.Unlock() - logger.Info("Pass 1: Extracted from %q. Players in file: %d. Achievements in file: %d.", path, len(filePlayerNames), len(filePlayerAchievements)) + log.Info("Extracted from %q. Players in file: %d. Achievements in file: %d.", path, len(filePlayerNames), len(filePlayerAchievements)) } else { - logger.Info("Pass 1: No player data or names extracted from %q.", path) + log.Info("No player data or names extracted from %q.", path) } } // updateLuaFileState is for Pass 2 func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map[string]bool) { + log := logger.Default.WithPrefix(filepath.Base(path)) + log.Info("Updating Lua state") defer wg.Done() L := lua.NewState() defer L.Close() if err := L.DoFile(path); err != nil { - logger.Error("Pass 2: error executing Lua file %q: %v. Cannot update its state.", path, err) + log.Error("error executing Lua file %q: %v. Cannot update its state.", path, err) return } heimdallAchievementsVal := L.GetGlobal("Heimdall_Achievements") if heimdallAchievementsVal.Type() == lua.LTNil { - logger.Warning("Pass 2: Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path) + log.Warning("Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path) return } heimdallAchievementsTable, ok := heimdallAchievementsVal.(*lua.LTable) if !ok { - logger.Warning("Pass 2: Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String()) + log.Warning("Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String()) return } @@ -251,10 +277,10 @@ func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map } L.SetField(heimdallAchievementsTable, "alreadySeen", luaAlreadySeen) - logger.Debug("Pass 2: Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames)) + log.Debug("Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames)) L.SetField(heimdallAchievementsTable, "players", L.NewTable()) - logger.Debug("Pass 2: Cleared Heimdall_Achievements.players for %q.", path) + log.Debug("Cleared Heimdall_Achievements.players for %q.", path) } func NsqWorker(wg *sync.WaitGroup, messages <-chan NSQMessage) { // Changed to read-only channel diff --git a/succ.sh b/succ.sh new file mode 100644 index 0000000..baaffe0 --- /dev/null +++ b/succ.sh @@ -0,0 +1 @@ +go run . -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/" \ No newline at end of file