Wire up the new flatten functions

main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"strings"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
 	utils "git.site.quack-lab.dev/dave/cyutils"
@@ -51,23 +52,42 @@ func DoIngest() {
 		logger.Error("Failed to read data directory: %v", err)
 		return
 	}
-	totalFiles := len(killmailFiles)
-	utils.WithWorkers(6, killmailFiles, func(worker int, index int, file os.DirEntry) {
+	filteredKillmailFiles := []os.DirEntry{}
+	for _, file := range killmailFiles {
+		if strings.HasSuffix(file.Name(), ".bz2") {
+			filteredKillmailFiles = append(filteredKillmailFiles, file)
+		}
+	}
+	filteredKillmailFiles = filteredKillmailFiles[:1] // DEBUG
+
+	totalFiles := len(filteredKillmailFiles)
+	utils.WithWorkers(6, filteredKillmailFiles, func(worker int, index int, file os.DirEntry) {
 		flog := logger.Default.WithPrefix(fmt.Sprintf("worker %d", worker)).WithPrefix(fmt.Sprintf("file %d of %d", index, totalFiles)).WithPrefix(file.Name())
 		flog.Info("Loading killmails")
 
-		killmails, err := LoadBz2Killmails(filepath.Join("data", file.Name()))
+		killmails, err := LoadBz2Killmails(filepath.Join("data", filteredKillmailFiles[index].Name()))
 		if err != nil {
 			flog.Error("Failed to load killmails: %v", err)
 			return
 		}
 		flog.Info("Loaded %d killmails", len(killmails))
+		killmails = killmails[:1] // DEBUG
+		killmail := killmails[0] // DEBUG
 
-		err = db.SaveKillmails(killmails)
+		flatKillmails, flatKillmailAttackers, flatKillmailItems, err := FlattenKillmail(db, killmail)
 		if err != nil {
-			flog.Error("Failed to save killmails: %v", err)
+			flog.Error("Failed to flatten killmail: %v", err)
 			return
 		}
-		flog.Info("Saved %d killmails", len(killmails))
+		flog.Info("Flattened killmail: %+v", flatKillmails)
+		flog.Info("Flattened killmail attackers: %+v", flatKillmailAttackers)
+		flog.Info("Flattened killmail items: %+v", flatKillmailItems)
+
+		// err = db.SaveKillmails(killmails)
+		// if err != nil {
+		// 	flog.Error("Failed to save killmails: %v", err)
+		// 	return
+		// }
+		// flog.Info("Saved %d killmails", len(killmails))
 	})
 }