Implement batch processing for achievement saving with HTTP request handling

dbwriter.go

@@ -1,9 +1,27 @@
 package main
 
 import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"sync"
+	"time"
+
 	logger "git.site.quack-lab.dev/dave/cylogger"
 )
 
+const (
+	batchSize = 1000
+	timeout   = 5 * time.Second
+)
+
+var (
+	batchBuffer = make([]NSQMessage, 0, batchSize)
+	batchMutex  sync.Mutex
+)
+
 var whitelistedAchievements = map[string]bool{
 	"15":    true,
 	"958":   true,
@@ -65,19 +83,73 @@ var whitelistedAchievements = map[string]bool{
 	"12448": true,
 }
 
-func Save(message NSQMessage, db *DB) error {
+func Save(message NSQMessage) error {
 	_, ok := whitelistedAchievements[message.ID]
 	if !ok {
 		logger.Debug("Received message for non-whitelisted achievement %s", message.ID)
 		return nil
 	}
 
-	_, err := db.writeConn.Exec("INSERT OR IGNORE INTO achievements (name, id, date, completed) VALUES (?, ?, ?, ?)",
-		message.Name, message.ID, message.Date, message.Completed)
-	if err != nil {
-		logger.Error("Error inserting into database: %v", err)
-		return err
+	batchMutex.Lock()
+	batchBuffer = append(batchBuffer, message)
+	currentBatchSize := len(batchBuffer)
+	batchMutex.Unlock()
+
+	logger.Debug("Added achievement to batch. Current batch size: %d/%d", currentBatchSize, batchSize)
+
+	if currentBatchSize >= batchSize {
+		batchMutex.Lock()
+		batch := make([]NSQMessage, len(batchBuffer))
+		copy(batch, batchBuffer)
+		batchBuffer = batchBuffer[:0]
+		batchMutex.Unlock()
+
+		logger.Info("Batch size reached %d, sending batch to %s", len(batch), backendEndpoint)
+		return sendBatch(batch)
 	}
 
 	return nil
 }
+
+func sendBatch(batch []NSQMessage) error {
+	logger.Debug("Preparing to send batch of %d achievements", len(batch))
+
+	data, err := json.Marshal(batch)
+	if err != nil {
+		logger.Error("Failed to marshal batch: %v", err)
+		return fmt.Errorf("error marshaling batch: %v", err)
+	}
+	logger.Debug("Successfully marshaled batch to JSON, size: %d bytes", len(data))
+
+	client := &http.Client{
+		Timeout: timeout,
+	}
+
+	req, err := http.NewRequest("POST", backendEndpoint, bytes.NewBuffer(data))
+	if err != nil {
+		logger.Error("Failed to create HTTP request: %v", err)
+		return fmt.Errorf("error creating request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+	logger.Debug("Created HTTP request to %s", backendEndpoint)
+
+	resp, err := client.Do(req)
+	if err != nil {
+		logger.Error("Failed to send HTTP request: %v", err)
+		return fmt.Errorf("error sending batch: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		logger.Error("Failed to read response body: %v", err)
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		logger.Error("Received non-OK status code: %d, body: %s", resp.StatusCode, string(body))
+		return fmt.Errorf("unexpected status code: %d, body: %s", resp.StatusCode, string(body))
+	}
+
+	logger.Info("Successfully sent batch of %d achievements", len(batch))
+	return nil
+}