Make main.go write into database instead of nsq, there's just no need for it like I thought there would be

db.go

@@ -0,0 +1,84 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"os"
+	"time"
+
+	_ "github.com/mattn/go-sqlite3"
+	logger "git.site.quack-lab.dev/dave/cylogger"
+)
+
+type DB struct {
+	Ready     bool
+	path      string
+	readConn  *sql.DB
+	writeConn *sql.DB
+}
+
+func (db *DB) Open() error {
+	if db.path == "" {
+		return fmt.Errorf("database path not set")
+	}
+
+	file, err := os.Open(db.path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			logger.Info("Database file does not exist at %s, creating", db.path)
+			file, err := os.Create(db.path)
+			if err != nil {
+				return fmt.Errorf("failed to create database file: %v", err)
+			}
+			logger.Info("Database created at %s", db.path)
+			file.Close()
+		} else {
+			return fmt.Errorf("failed to open database file: %v", err)
+		}
+	}
+	file.Close()
+
+	writeConn, err := sql.Open("sqlite3", db.path+"?_journal=WAL&_synchronous=NORMAL")
+	if err != nil {
+		logger.Error("%++v", err)
+		return err
+	}
+	writeConn.SetMaxOpenConns(1)
+	writeConn.SetConnMaxIdleTime(30 * time.Second)
+	writeConn.SetConnMaxLifetime(30 * time.Second)
+	db.writeConn = writeConn
+
+	readConn, err := sql.Open("sqlite3", db.path+"?mode=ro&_journal=WAL&_synchronous=NORMAL&_mode=ro")
+	if err != nil {
+		logger.Error("%++v", err)
+		return err
+	}
+	readConn.SetMaxOpenConns(4)
+	readConn.SetConnMaxIdleTime(30 * time.Second)
+	readConn.SetConnMaxLifetime(30 * time.Second)
+	db.readConn = readConn
+
+	db.Ready = true
+	return nil
+}
+
+func (db *DB) Init(ddl string) error {
+	if !db.Ready {
+		return fmt.Errorf("database not ready")
+	}
+	return nil
+}
+
+func (db *DB) Close() error {
+	err := db.writeConn.Close()
+	if err != nil {
+		return err
+	}
+
+	err = db.readConn.Close()
+	if err != nil {
+		return err
+	}
+
+	return nil
+}

dbwriter.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	logger "git.site.quack-lab.dev/dave/cylogger"
+)
+
+var whitelistedAchievements = map[string]bool{
+	"15":    true,
+	"958":   true,
+	"1276":  true,
+	"2088":  true,
+	"2151":  true,
+	"5466":  true,
+	"5759":  true,
+	"6470":  true,
+	"6763":  true,
+	"7392":  true,
+	"7393":  true,
+	"7394":  true,
+	"7958":  true,
+	"8939":  true,
+	"8992":  true,
+	"9048":  true,
+	"94103": true,
+	"10059": true,
+	"10079": true,
+	"10278": true,
+	"10657": true,
+	"10672": true,
+	"10684": true,
+	"10688": true,
+	"10689": true,
+	"10692": true,
+	"10693": true,
+	"10698": true,
+	"10790": true,
+	"10875": true,
+	"11124": true,
+	"11126": true,
+	"11127": true,
+	"11128": true,
+	"11157": true,
+	"11164": true,
+	"11188": true,
+	"11189": true,
+	"11190": true,
+	"11446": true,
+	"11473": true,
+	"11610": true,
+	"11674": true,
+	"11992": true,
+	"11993": true,
+	"11994": true,
+	"11995": true,
+	"11996": true,
+	"11997": true,
+	"11998": true,
+	"11999": true,
+	"12000": true,
+	"12001": true,
+	"12026": true,
+	"12074": true,
+	"12445": true,
+	"12447": true,
+	"12448": true,
+}
+
+func Save(message *NSQMessage, db *DB) error {
+	_, ok := whitelistedAchievements[message.ID]
+	if !ok {
+		logger.Warning("Received message for non-whitelisted achievement %s", message.ID)
+		return nil
+	}
+
+	_, err := db.writeConn.Exec("INSERT OR IGNORE INTO achievements (name, id, date, completed) VALUES (?, ?, ?, ?)",
+		message.Name, message.ID, message.Date, message.Completed)
+	if err != nil {
+		logger.Error("Error inserting into database: %v", err)
+		return err
+	}
+
+	return nil
+}

main.go

@@ -37,9 +37,19 @@ func main() {
 		logger.SetLevel(logger.LevelDebug) // Assuming LevelDebug is the correct constant for cylogger
 	}
 
+	db := DB{
+		path: "service/data/db.db",
+	}
+	err := db.Open()
+	if err != nil {
+		logger.Error("error opening database: %v", err)
+		return
+	}
+	defer db.Close()
+
 	logger.Info("Root: %q", *root)
 	cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
-	cleanedRoot, err := filepath.Abs(cleanedRoot)
+	cleanedRoot, err = filepath.Abs(cleanedRoot)
 	if err != nil {
 		logger.Error("error getting absolute path: %v", err)
 		return
@@ -59,8 +69,7 @@ func main() {
 		return
 	}
 
-	// Debugging
+	// matches = matches[:1]
-	matches = matches[:2]
 
 	// --- Pass 1: Extract all data ---
 	logger.Info("Starting Pass 1: Extracting data from all Heimdall.lua files...")
@@ -77,29 +86,42 @@ func main() {
 		globalDataMutex.Unlock()
 	}
 
-	// --- Process and Send to NSQ ---
+	wgSave := sync.WaitGroup{}
-	logger.Info("Starting NSQ message publishing...")
+	wgSave.Add(1)
-	nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size
-	var wgNsqWorkers sync.WaitGroup
-	for i := 0; i < nsqWorkers; i++ {
-		wgNsqWorkers.Add(1)
-		go NsqWorker(&wgNsqWorkers, nsqMessagesChan)
-	}
-
 	go func() {
-		globalDataMutex.Lock()
+		logger.Info("Saving achievements to database...")
-		defer globalDataMutex.Unlock()
 		for playerName, achList := range allPlayersAchievementsGlobal {
+			logger.Debug("Saving %d achievements for player %s", len(achList), playerName)
 			for _, ach := range achList {
-				// ach.Name is already correctly set during extraction
+				Save(&ach, &db)
-				nsqMessagesChan <- ach
-				logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID)
 			}
 		}
-		close(nsqMessagesChan) // Close channel when all messages are sent
+		wgSave.Done()
-		logger.Info("All NSQ messages queued.")
 	}()
 
+	// --- Process and Send to NSQ ---
+	// logger.Info("Starting NSQ message publishing...")
+	// nsqMessagesChan := make(chan NSQMessage, 10000) // Increased buffer size
+	// var wgNsqWorkers sync.WaitGroup
+	// for i := 0; i < nsqWorkers; i++ {
+	// 	wgNsqWorkers.Add(1)
+	// 	go NsqWorker(&wgNsqWorkers, nsqMessagesChan)
+	// }
+
+	// go func() {
+	// 	globalDataMutex.Lock()
+	// 	defer globalDataMutex.Unlock()
+	// 	for playerName, achList := range allPlayersAchievementsGlobal {
+	// 		for _, ach := range achList {
+	// 			// ach.Name is already correctly set during extraction
+	// 			nsqMessagesChan <- ach
+	// 			logger.Debug("Queued NSQ message for Player: %s, AchID: %s", playerName, ach.ID)
+	// 		}
+	// 	}
+	// 	close(nsqMessagesChan) // Close channel when all messages are sent
+	// 	logger.Info("All NSQ messages queued.")
+	// }()
+
 	// --- Pass 2: Update Lua file states (in memory) ---
 	logger.Info("Starting Pass 2: Updating Lua states (setting alreadySeen and clearing players)...")
 	var wgPass2 sync.WaitGroup
@@ -114,7 +136,8 @@ func main() {
 		logger.Info("Skipping Pass 2 as no players were found globally.")
 	}
 
-	wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed
+	// wgNsqWorkers.Wait() // Wait for all NSQ messages to be processed
+	wgSave.Wait()
 	logger.Info("All NSQ workers finished. Program complete.")
 }
 
@@ -129,30 +152,31 @@ func countTotalAchievements(achMap map[string][]NSQMessage) int {
 
 // extractPlayerAchievementsFromFile is for Pass 1
 func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
-	logger.Info("Extracting achievements from %q", path)
+	log := logger.Default.WithPrefix(filepath.Base(path))
+	log.Info("Extracting achievements")
 	defer wg.Done()
 	L := lua.NewState()
 	defer L.Close()
 
 	if err := L.DoFile(path); err != nil {
-		logger.Error("Pass 1: error executing Lua file %q: %v", path, err)
+		log.Error("error executing Lua file %q: %v", path, err)
 		return
 	}
 
 	heimdallAchievements := L.GetGlobal("Heimdall_Achievements")
 	if heimdallAchievements.Type() == lua.LTNil {
-		logger.Warning("Pass 1: Heimdall_Achievements not found in %q. Skipping file.", path)
+		log.Warning("Heimdall_Achievements not found in %q. Skipping file.", path)
 		return
 	}
 
 	playersTableLua := L.GetField(heimdallAchievements, "players")
 	if playersTableLua.Type() == lua.LTNil {
-		logger.Info("Pass 1: 'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path)
+		log.Info("'players' table is nil in Heimdall_Achievements in %q. No player data to extract.", path)
 		return
 	}
 	playersTable, ok := playersTableLua.(*lua.LTable)
 	if !ok {
-		logger.Warning("Pass 1: 'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String())
+		log.Warning("'players' field in Heimdall_Achievements is not a table in %q (type: %s). Skipping.", path, playersTableLua.Type().String())
 		return
 	}
 
@@ -165,14 +189,14 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
 
 		achievementsTableLua, ok := playerAchievementsLua.(*lua.LTable)
 		if !ok {
-			logger.Error("Pass 1: Achievements for player %s is not a table in %q. Skipping achievements for this player.", currentPlayerName, path)
+			log.Error("Achievements for player %s is not a table. Skipping achievements for this player.", currentPlayerName)
 			return
 		}
 
 		achievementsTableLua.ForEach(func(_ lua.LValue, achievementDataLua lua.LValue) {
 			achievementTable, ok := achievementDataLua.(*lua.LTable)
 			if !ok {
-				logger.Error("Pass 1: Achievement data for player %s is not a table in %q. Skipping this achievement.", currentPlayerName, path)
+				log.Error("Achievement data for player %s is not a table. Skipping this achievement.", currentPlayerName)
 				return
 			}
 
@@ -184,21 +208,21 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
 			} else if idVal.Type() == lua.LTString {
 				currentAchievement.ID = idVal.String()
 			} else {
-				logger.Warning("Pass 1: Missing or invalid 'id' (expected number or string) for achievement for player %s in %q.", currentPlayerName, path)
+				log.Warning("Missing or invalid 'id' (expected number or string) for achievement for player %s.", currentPlayerName)
 			}
 
 			dateVal := achievementTable.RawGetString("date")
 			if dateVal.Type() == lua.LTString {
 				currentAchievement.Date = dateVal.String()
 			} else {
-				logger.Warning("Pass 1: Missing or invalid 'date' (expected string) for achievement for player %s in %q.", currentPlayerName, path)
+				log.Warning("Missing or invalid 'date' (expected string) for achievement for player %s.", currentPlayerName)
 			}
 
 			completedVal := achievementTable.RawGetString("completed")
 			if completedVal.Type() == lua.LTBool {
 				currentAchievement.Completed = lua.LVAsBool(completedVal)
 			} else {
-				logger.Warning("Pass 1: Missing or invalid 'completed' (expected boolean) for achievement for player %s in %q.", currentPlayerName, path)
+				log.Warning("Missing or invalid 'completed' (expected boolean) for achievement for player %s.", currentPlayerName)
 			}
 
 			if currentAchievement.ID != "" { // Ensure we have at least an ID before adding
@@ -216,32 +240,34 @@ func extractPlayerAchievementsFromFile(path string, wg *sync.WaitGroup) {
 			allPlayerNamesGlobal[name] = true
 		}
 		globalDataMutex.Unlock()
-		logger.Info("Pass 1: Extracted from %q. Players in file: %d. Achievements in file: %d.", path, len(filePlayerNames), len(filePlayerAchievements))
+		log.Info("Extracted from %q. Players in file: %d. Achievements in file: %d.", path, len(filePlayerNames), len(filePlayerAchievements))
 	} else {
-		logger.Info("Pass 1: No player data or names extracted from %q.", path)
+		log.Info("No player data or names extracted from %q.", path)
 	}
 }
 
 // updateLuaFileState is for Pass 2
 func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map[string]bool) {
+	log := logger.Default.WithPrefix(filepath.Base(path))
+	log.Info("Updating Lua state")
 	defer wg.Done()
 	L := lua.NewState()
 	defer L.Close()
 
 	if err := L.DoFile(path); err != nil {
-		logger.Error("Pass 2: error executing Lua file %q: %v. Cannot update its state.", path, err)
+		log.Error("error executing Lua file %q: %v. Cannot update its state.", path, err)
 		return
 	}
 
 	heimdallAchievementsVal := L.GetGlobal("Heimdall_Achievements")
 	if heimdallAchievementsVal.Type() == lua.LTNil {
-		logger.Warning("Pass 2: Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path)
+		log.Warning("Heimdall_Achievements not found in %q after script execution. Cannot set 'alreadySeen' or clear 'players'.", path)
 		return
 	}
 
 	heimdallAchievementsTable, ok := heimdallAchievementsVal.(*lua.LTable)
 	if !ok {
-		logger.Warning("Pass 2: Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String())
+		log.Warning("Heimdall_Achievements in %q is not a table (type: %s). Cannot update.", path, heimdallAchievementsVal.Type().String())
 		return
 	}
 
@@ -251,10 +277,10 @@ func updateLuaFileState(path string, wg *sync.WaitGroup, allKnownPlayerNames map
 	}
 
 	L.SetField(heimdallAchievementsTable, "alreadySeen", luaAlreadySeen)
-	logger.Debug("Pass 2: Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames))
+	log.Debug("Set Heimdall_Achievements.alreadySeen for %q with %d total player names.", path, len(allKnownPlayerNames))
 
 	L.SetField(heimdallAchievementsTable, "players", L.NewTable())
-	logger.Debug("Pass 2: Cleared Heimdall_Achievements.players for %q.", path)
+	log.Debug("Cleared Heimdall_Achievements.players for %q.", path)
 }
 
 func NsqWorker(wg *sync.WaitGroup, messages <-chan NSQMessage) { // Changed to read-only channel

succ.sh

@@ -0,0 +1 @@
+go run . -root "C:/Users/Administrator/Seafile/Games-WoW/Ruski/WTF/"