Implement batch processing for achievement saving with HTTP request handling

main.go

@@ -20,6 +20,7 @@ import (
 )
 
 const nsqEndpoint = "https://nsq.site.quack-lab.dev/pub?topic=wowspy"
+const backendEndpoint = "https://sniffer-be.site.quack-lab.dev/achievements"
 
 var debug *bool
 var nsqWorkers = 32
@@ -33,19 +34,9 @@ func main() {
 		logger.SetLevel(logger.LevelDebug) // Assuming LevelDebug is the correct constant for cylogger
 	}
 
-	db := DB{
-		path: "service/data/db.db",
-	}
-	err := db.Open()
-	if err != nil {
-		logger.Error("error opening database: %v", err)
-		return
-	}
-	defer db.Close()
-
 	logger.Info("Root: %q", *root)
 	cleanedRoot := strings.Replace(*root, "~", os.Getenv("HOME"), 1)
-	cleanedRoot, err = filepath.Abs(cleanedRoot)
+	cleanedRoot, err := filepath.Abs(cleanedRoot)
 	if err != nil {
 		logger.Error("error getting absolute path: %v", err)
 		return
@@ -75,33 +66,12 @@ func main() {
 	// We can save the achievements to the database while doing something else unrelated
 	go func() {
 		defer wg.Done()
-		saveAchievementsToDB(&db, achievements)
+		saveAchievements(achievements)
 	}()
-	loadUniquePlayers(&db, achievements)
-	saveAchievementsToSourceFiles(luaStates, achievements)
+	// saveAchievementsToSourceFiles(luaStates, achievements)
 	wg.Wait()
 }
 
-// loadUniquePlayers loads all unique players from the database and stores them in the achievements map
-// So that we can populate our "alreadySeen" with the database values too
-func loadUniquePlayers(db *DB, achievements *sync.Map) {
-	rows, err := db.readConn.Query("SELECT DISTINCT name FROM achievements")
-	if err != nil {
-		logger.Error("error querying unique players: %v", err)
-		return
-	}
-	defer rows.Close()
-	for rows.Next() {
-		var playerName string
-		err = rows.Scan(&playerName)
-		if err != nil {
-			logger.Error("error scanning player name: %v", err)
-		}
-		achievements.Store(playerName, &[]NSQMessage{})
-	}
-	return
-}
-
 func saveAchievementsToSourceFiles(luaStates *sync.Map, achievements *sync.Map) {
 	wg := sync.WaitGroup{}
 	luaStates.Range(func(k, v any) bool {
@@ -202,14 +172,14 @@ func addAlreadySeen(strContent string, achievements *sync.Map) string {
 	return strings.Join(modifiedLines, "\n")
 }
 
-func saveAchievementsToDB(db *DB, achievements *sync.Map) {
+func saveAchievements(achievements *sync.Map) {
 	count := 0
 	achievements.Range(func(k, v any) bool {
 		playerName := k.(string)
 		playerAchievements := v.(*[]NSQMessage)
 		logger.Debug("Saving %d achievements for player %s", len(*playerAchievements), playerName)
 		for _, ach := range *playerAchievements {
-			Save(ach, db)
+			Save(ach)
 		}
 		count++
 		if count%1000 == 0 {