101 lines
3.0 KiB
Go
101 lines
3.0 KiB
Go
package main
|
|
|
|
import (
|
|
"flag"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"strings"
|
|
|
|
logger "git.site.quack-lab.dev/dave/cylogger"
|
|
utils "git.site.quack-lab.dev/dave/cyutils"
|
|
)
|
|
|
|
func main() {
|
|
ingest := flag.Bool("ingest", false, "ingest killmails from data directory")
|
|
server := flag.Bool("server", false, "start API server")
|
|
port := flag.String("port", "3000", "API server port")
|
|
flag.Parse()
|
|
logger.InitFlag()
|
|
logger.Default = logger.Default.ToFile("zkill.log")
|
|
logger.Info("Starting")
|
|
|
|
db, err := GetDB()
|
|
if err != nil {
|
|
logger.Error("Failed to get database: %v", err)
|
|
return
|
|
}
|
|
|
|
if *ingest {
|
|
DoIngest()
|
|
return
|
|
}
|
|
|
|
if *server {
|
|
StartAPIServer(*port)
|
|
return
|
|
}
|
|
|
|
logger.Info("Querying fits")
|
|
params := QueryParams{}
|
|
stats, err := db.QueryFits(params)
|
|
if err != nil {
|
|
logger.Error("Failed to query fits: %v", err)
|
|
return
|
|
}
|
|
logger.Info("Total killmails: %d", stats.TotalKillmails)
|
|
}
|
|
|
|
func DoIngest() {
|
|
killmailFiles, err := os.ReadDir("data")
|
|
if err != nil {
|
|
logger.Error("Failed to read data directory: %v", err)
|
|
return
|
|
}
|
|
filteredKillmailFiles := []os.DirEntry{}
|
|
for _, file := range killmailFiles {
|
|
if strings.HasSuffix(file.Name(), ".bz2") {
|
|
filteredKillmailFiles = append(filteredKillmailFiles, file)
|
|
}
|
|
}
|
|
filteredKillmailFiles = filteredKillmailFiles[:1] // DEBUG
|
|
|
|
totalFiles := len(filteredKillmailFiles)
|
|
utils.WithWorkers(6, filteredKillmailFiles, func(worker int, index int, file os.DirEntry) {
|
|
flog := logger.Default.WithPrefix(fmt.Sprintf("worker %d", worker)).WithPrefix(fmt.Sprintf("file %d of %d", index, totalFiles)).WithPrefix(file.Name())
|
|
flog.Info("Loading killmails")
|
|
|
|
killmails, err := LoadBz2Killmails(filepath.Join("data", filteredKillmailFiles[index].Name()))
|
|
if err != nil {
|
|
flog.Error("Failed to load killmails: %v", err)
|
|
return
|
|
}
|
|
flog.Info("Loaded %d killmails", len(killmails))
|
|
killmails = killmails[:1000] // DEBUG
|
|
// logger.Default.SetLevel(logger.LevelDebug)
|
|
|
|
FlatKillmails := []*FlatKillmail{}
|
|
FlatKillmailAttackers := []FlatKillmailAttacker{}
|
|
FlatKillmailItems := []FlatKillmailItem{}
|
|
utils.WithWorkers(4, killmails, func(worker int, index int, killmail Killmail) {
|
|
flog.WithPrefix(fmt.Sprintf("worker %d", worker)).WithPrefix(fmt.Sprintf("killmail %d", index)).WithPrefix(fmt.Sprintf("killmail_%d", killmail.KillmailID))
|
|
flog.Info("Flattening killmail")
|
|
flatKillmails, flatKillmailAttackers, flatKillmailItems, err := FlattenKillmail(db, killmail)
|
|
if err != nil {
|
|
flog.Error("Failed to flatten killmail: %v", err)
|
|
return
|
|
}
|
|
FlatKillmails = append(FlatKillmails, flatKillmails)
|
|
FlatKillmailAttackers = append(FlatKillmailAttackers, flatKillmailAttackers...)
|
|
FlatKillmailItems = append(FlatKillmailItems, flatKillmailItems...)
|
|
})
|
|
|
|
err = db.SaveFlatKillmails(FlatKillmails, FlatKillmailAttackers, FlatKillmailItems)
|
|
if err != nil {
|
|
flog.Error("Failed to save killmails: %v", err)
|
|
return
|
|
}
|
|
flog.Info("Saved %d killmails with %d attackers and %d items", len(FlatKillmails), len(FlatKillmailAttackers), len(FlatKillmailItems))
|
|
})
|
|
}
|