dave/wow-AchievementSniffer
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Make main.go write into database instead of nsq, there's just no need for it like I thought there would be

Browse Source
This commit is contained in:
PhatPhuckDave
2025-05-21 11:34:42 +02:00
parent 39116eb74e
commit 114c0b909b
4 changed files with 231 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files

84
db.go Normal file
View File

@@ -0,0 +1,84 @@
package main
import (
"database/sql"
"fmt"
"os"
"time"
_ "github.com/mattn/go-sqlite3"
logger "git.site.quack-lab.dev/dave/cylogger"
)
type DB struct {
Ready bool
path string
readConn *sql.DB
writeConn *sql.DB
}
func (db *DB) Open() error {
if db.path == "" {
return fmt.Errorf("database path not set")
}
file, err := os.Open(db.path)
if err != nil {
if os.IsNotExist(err) {
logger.Info("Database file does not exist at %s, creating", db.path)
file, err := os.Create(db.path)
if err != nil {
return fmt.Errorf("failed to create database file: %v", err)
}
logger.Info("Database created at %s", db.path)
file.Close()
} else {
return fmt.Errorf("failed to open database file: %v", err)
}
}
file.Close()
writeConn, err := sql.Open("sqlite3", db.path+"?_journal=WAL&_synchronous=NORMAL")
if err != nil {
logger.Error("%++v", err)
return err
}
writeConn.SetMaxOpenConns(1)
writeConn.SetConnMaxIdleTime(30 * time.Second)
writeConn.SetConnMaxLifetime(30 * time.Second)
db.writeConn = writeConn
readConn, err := sql.Open("sqlite3", db.path+"?mode=ro&_journal=WAL&_synchronous=NORMAL&_mode=ro")
if err != nil {
logger.Error("%++v", err)
return err
}
readConn.SetMaxOpenConns(4)
readConn.SetConnMaxIdleTime(30 * time.Second)
readConn.SetConnMaxLifetime(30 * time.Second)
db.readConn = readConn
db.Ready = true
return nil
}
func (db *DB) Init(ddl string) error {
if !db.Ready {
return fmt.Errorf("database not ready")
}
return nil
}
func (db *DB) Close() error {
err := db.writeConn.Close()
if err != nil {
return err
}
err = db.readConn.Close()
if err != nil {
return err
}
return nil
}

83
dbwriter.go Normal file
View File

@@ -0,0 +1,83 @@
package main
import (
logger "git.site.quack-lab.dev/dave/cylogger"
)
var whitelistedAchievements = map[string]bool{
"15": true,
"958": true,
"1276": true,
"2088": true,
"2151": true,
"5466": true,
"5759": true,
"6470": true,
"6763": true,
"7392": true,
"7393": true,
"7394": true,
"7958": true,
"8939": true,
"8992": true,
"9048": true,
"94103": true,
"10059": true,
"10079": true,
"10278": true,
"10657": true,
"10672": true,
"10684": true,
"10688": true,
"10689": true,
"10692": true,
"10693": true,
"10698": true,
"10790": true,
"10875": true,
"11124": true,
"11126": true,
"11127": true,
"11128": true,
"11157": true,
"11164": true,
"11188": true,
"11189": true,
"11190": true,
"11446": true,
"11473": true,
"11610": true,
"11674": true,
"11992": true,
"11993": true,
"11994": true,
"11995": true,
"11996": true,
"11997": true,
"11998": true,
"11999": true,
"12000": true,
"12001": true,
"12026": true,
"12074": true,
"12445": true,
"12447": true,
"12448": true,
}
func Save(message *NSQMessage, db *DB) error {
_, ok := whitelistedAchievements[message.ID]
if !ok {
logger.Warning("Received message for non-whitelisted achievement %s", message.ID)
return nil
}
_, err := db.writeConn.Exec("INSERT OR IGNORE INTO achievements (name, id, date, completed) VALUES (?, ?, ?, ?)",
message.Name, message.ID, message.Date, message.Completed)
if err != nil {
logger.Error("Error inserting into database: %v", err)
return err
}
return nil
}

100
main.go
View File

@@ -37,9 +37,19 @@ func main() {
logger.SetLevel(logger.LevelDebug) // Assuming LevelDebug is the correct constant for cylogger
}
db := DB{
path: "service/data/db.db",
}
err := db.Open()
if err != nil {
logger.Error("error opening database: %v", err)
return
}
defer db.Close()
logger.Info("Root: %q", *root)
cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
cleanedRoot, err := filepath.Abs(cleanedRoot)
cleanedRoot, err = filepath.Abs(cleanedRoot)
if err != nil {
logger.Error("error getting absolute path: %v", err)
return
@@ -59,8 +69,7 @@ func main() {
return
}
// Debugging
matches = matches[:2]
// matches = matches[:1]
// --- Pass 1: Extract all data ---
logger.Info("Starting Pass 1: Extracting data from all Heimdall.lua files...")
@@ -77,29 +86,42 @@ func main() {
globalDataMutex.Unlock()
}
// --- Process and Send to NSQ ---
logger.Info("Starting NSQ message publishing...")
nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size
var wgNsqWorkers sync.WaitGroup
for i := 0; i < nsqWorkers; i++ {
wgNsqWorkers.Add(1)
go NsqWorker(&wgNsqWorkers, nsqMessagesChan)
}
wgSave := sync.WaitGroup{}
wgSave.Add(1)
go func() {
globalDataMutex.Lock()
defer globalDataMutex.Unlock()
logger.Info("Saving achievements to database...")
for playerName, achList := range allPlayersAchievementsGlobal {
logger.Debug("Saving %d achievements for player %s", len(achList), playerName)
for _, ach := range achList {
// ach.Name is already correctly set during extraction
nsqMessagesChan <- ach
logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID)
Save(&ach, &db)
}
}
close(nsqMessagesChan) // Close channel when all messages are sent
logger.Info("All NSQ messages queued.")
wgSave.Done()
}()
// --- Process and Send to NSQ ---
// logger.Info("Starting NSQ message publishing...")
// nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size
// var wgNsqWorkers sync.WaitGroup
// for i := 0; i < nsqWorkers; i++ {
// wgNsqWorkers.Add(1)
// go NsqWorker(&wgNsqWorkers, nsqMessagesChan)
// }
// go func() {
// globalDataMutex.Lock()
// defer globalDataMutex.Unlock()
// for playerName, achList := range allPlayersAchievementsGlobal {
// for _, ach := range achList {
// // ach.Name is already correctly set during extraction
// nsqMessagesChan <- ach
// logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID)
// }
// }
// close(nsqMessagesChan) // Close channel when all messages are sent
// logger.Info("All NSQ messages queued.")
// }()
// --- Pass 2: Update Lua file states (in memory) ---
logger.Info("Starting Pass 2: Updating Lua states (setting alreadySeen and clearing players)...")
var wgPass2 sync.WaitGroup
@@ -114,7 +136,8 @@ func main() {
logger.Info("Skipping Pass 2 as no players were found globally.")
}
wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed
// wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed
wgSave.Wait()
logger.Info("All NSQ workers finished. Program complete.")
}
@@ -129,30 +152,31 @@ func countTotalAchievements(achMap map[string][]NSQMessage) int {
// extractPlayerAchievementsFromFile is for Pass 1
func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
logger.Info("Extracting achievements from %q", path)
log := logger.Default.WithPrefix(filepath.Base(path))
log.Info("Extracting achievements")
defer wg.Done()
L := lua.NewState()
defer L.Close()
if err := L.DoFile(path); err != nil {
logger.Error("Pass 1: error executing Lua file %q: %v", path, err)
log.Error("error executing Lua file %q: %v", path, err)
return
}
heimdallAchievements := L.GetGlobal("Heimdall_Achievements")
if heimdallAchievements.Type() == lua.LTNil {
logger.Warning("Pass 1: Heimdall_Achievements not found in %q. Skipping file.", path)
log.Warning("Heimdall_Achievements not found in %q. Skipping file.", path)
return
}
playersTableLua := L.GetField(heimdallAchievements, "players")
if playersTableLua.Type() == lua.LTNil {
logger.Info("Pass 1: 'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path)
log.Info("'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path)
return
}
playersTable, ok := playersTableLua.(*lua.LTable)
if !ok {
logger.Warning("Pass 1: 'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String())
log.Warning("'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String())
return
}
@@ -165,14 +189,14 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
achievementsTableLua, ok := playerAchievementsLua.(*lua.LTable)
if !ok {
logger.Error("Pass 1: Achievements for player %s is not a table in %q. Skipping achievements for this player.", currentPlayerName, path)
log.Error("Achievements for player %s is not a table. Skipping achievements for this player.", currentPlayerName)
return
}
achievementsTableLua.ForEach(func(_ lua.LValue, achievementDataLua lua.LValue) {
achievementTable, ok := achievementDataLua.(*lua.LTable)
if !ok {
logger.Error("Pass 1: Achievement data for player %s is not a table in %q. Skipping this achievement.", currentPlayerName, path)
log.Error("Achievement data for player %s is not a table. Skipping this achievement.", currentPlayerName)
return
}
@@ -184,21 +208,21 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
} else if idVal.Type() == lua.LTString {
currentAchievement.ID = idVal.String()
} else {
logger.Warning("Pass 1: Missing or invalid 'id' (expected number or string) for achievement for player %s in %q.", currentPlayerName, path)
log.Warning("Missing or invalid 'id' (expected number or string) for achievement for player %s.", currentPlayerName)
}
dateVal := achievementTable.RawGetString("date")
if dateVal.Type() == lua.LTString {
currentAchievement.Date = dateVal.String()
} else {
logger.Warning("Pass 1: Missing or invalid 'date' (expected string) for achievement for player %s in %q.", currentPlayerName, path)
log.Warning("Missing or invalid 'date' (expected string) for achievement for player %s.", currentPlayerName)
}
completedVal := achievementTable.RawGetString("completed")
if completedVal.Type() == lua.LTBool {
currentAchievement.Completed = lua.LVAsBool(completedVal)
} else {
logger.Warning("Pass 1: Missing or invalid 'completed' (expected boolean) for achievement for player %s in %q.", currentPlayerName, path)
log.Warning("Missing or invalid 'completed' (expected boolean) for achievement for player %s.", currentPlayerName)
}
if currentAchievement.ID != "" { // Ensure we have at least an ID before adding
@@ -216,32 +240,34 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
allPlayerNamesGlobal[name] = true
}
globalDataMutex.Unlock()
logger.Info("Pass 1: Extracted from %q. Players in file: %d. Achievements in file: %d.", path, len(filePlayerNames), len(filePlayerAchievements))
log.Info("Extracted from %q. Players in file: %d. Achievements in file: %d.", path, len(filePlayerNames), len(filePlayerAchievements))
} else {
logger.Info("Pass 1: No player data or names extracted from %q.", path)
log.Info("No player data or names extracted from %q.", path)
}
}
// updateLuaFileState is for Pass 2
func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map[string]bool) {
log := logger.Default.WithPrefix(filepath.Base(path))
log.Info("Updating Lua state")
defer wg.Done()
L := lua.NewState()
defer L.Close()
if err := L.DoFile(path); err != nil {
logger.Error("Pass 2: error executing Lua file %q: %v. Cannot update its state.", path, err)
log.Error("error executing Lua file %q: %v. Cannot update its state.", path, err)
return
}
heimdallAchievementsVal := L.GetGlobal("Heimdall_Achievements")
if heimdallAchievementsVal.Type() == lua.LTNil {
logger.Warning("Pass 2: Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path)
log.Warning("Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path)
return
}
heimdallAchievementsTable, ok := heimdallAchievementsVal.(*lua.LTable)
if !ok {
logger.Warning("Pass 2: Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String())
log.Warning("Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String())
return
}
@@ -251,10 +277,10 @@ func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map
}
L.SetField(heimdallAchievementsTable, "alreadySeen", luaAlreadySeen)
logger.Debug("Pass 2: Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames))
log.Debug("Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames))
L.SetField(heimdallAchievementsTable, "players", L.NewTable())
logger.Debug("Pass 2: Cleared Heimdall_Achievements.players for %q.", path)
log.Debug("Cleared Heimdall_Achievements.players for %q.", path)
}
func NsqWorker(wg *sync.WaitGroup, messages <-chan NSQMessage) { // Changed to read-only channel

1
succ.sh Normal file
View File

@@ -0,0 +1 @@
go run . -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"