Hallucinate an "enrich" script/function to pad out the killmail metadata
This commit is contained in:
189
clickhouse.go
189
clickhouse.go
@@ -9,40 +9,41 @@ import (
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
)
|
||||
|
||||
// Legacy implementation - commented out
|
||||
// Values returns all fields in the order expected by ClickHouse INSERT
|
||||
func (fk *FlatKillmail) Values() []interface{} {
|
||||
// Convert attackers to the tuple format expected by ClickHouse
|
||||
attackers := make([][]interface{}, 0, len(fk.Attackers))
|
||||
for _, attacker := range fk.Attackers {
|
||||
attackers = append(attackers, attacker.Values())
|
||||
}
|
||||
|
||||
// Convert items to tuple format
|
||||
items := make([][]interface{}, 0, len(fk.Items))
|
||||
for _, item := range fk.Items {
|
||||
items = append(items, item.Values())
|
||||
}
|
||||
|
||||
return []interface{}{
|
||||
fk.KillmailID,
|
||||
fk.KillmailTime,
|
||||
fk.SolarSystemID,
|
||||
fk.KillmailHash,
|
||||
fk.VictimShipTypeID,
|
||||
fk.VictimCharacterID,
|
||||
fk.VictimCorporationID,
|
||||
fk.VictimAllianceID,
|
||||
fk.VictimDamageTaken,
|
||||
fk.VictimPosX,
|
||||
fk.VictimPosY,
|
||||
fk.VictimPosZ,
|
||||
fk.AttackerCount,
|
||||
fk.TotalDamageDone,
|
||||
fk.FinalBlowShipType,
|
||||
attackers, // Converted attackers
|
||||
items, // Converted items
|
||||
}
|
||||
}
|
||||
// func (fk *FlatKillmail) Values() []interface{} {
|
||||
// // Convert attackers to the tuple format expected by ClickHouse
|
||||
// attackers := make([][]interface{}, 0, len(fk.Attackers))
|
||||
// for _, attacker := range fk.Attackers {
|
||||
// attackers = append(attackers, attacker.Values())
|
||||
// }
|
||||
//
|
||||
// // Convert items to tuple format
|
||||
// items := make([][]interface{}, 0, len(fk.Items))
|
||||
// for _, item := range fk.Items {
|
||||
// items = append(items, item.Values())
|
||||
// }
|
||||
//
|
||||
// return []interface{}{
|
||||
// fk.KillmailID,
|
||||
// fk.KillmailTime,
|
||||
// fk.SolarSystemID,
|
||||
// fk.KillmailHash,
|
||||
// fk.VictimShipTypeID,
|
||||
// fk.VictimCharacterID,
|
||||
// fk.VictimCorporationID,
|
||||
// fk.VictimAllianceID,
|
||||
// fk.VictimDamageTaken,
|
||||
// fk.VictimPosX,
|
||||
// fk.VictimPosY,
|
||||
// fk.VictimPosZ,
|
||||
// fk.AttackerCount,
|
||||
// fk.TotalDamageDone,
|
||||
// fk.FinalBlowShipType,
|
||||
// attackers, // Converted attackers
|
||||
// items, // Converted items
|
||||
// }
|
||||
// }
|
||||
|
||||
func (a Attacker) Values() []interface{} {
|
||||
finalBlow := uint8(0)
|
||||
@@ -71,51 +72,52 @@ func (i Item) Values() []interface{} {
|
||||
}
|
||||
}
|
||||
|
||||
func (fm *FlatModule) Values() []interface{} {
|
||||
return []interface{}{
|
||||
fm.KillmailID,
|
||||
fm.ItemTypeID,
|
||||
string(fm.Slot),
|
||||
}
|
||||
}
|
||||
|
||||
// FlattenKillmail converts the nested JSON structure to ClickHouse format
|
||||
func (k *Killmail) FlattenKillmail() (*FlatKillmail, []*FlatModule) {
|
||||
flat := &FlatKillmail{
|
||||
KillmailID: k.KillmailID,
|
||||
KillmailTime: k.KillmailTime,
|
||||
SolarSystemID: k.SolarSystemID,
|
||||
KillmailHash: k.KillmailHash,
|
||||
VictimShipTypeID: k.Victim.ShipTypeID,
|
||||
VictimCharacterID: k.Victim.CharacterID,
|
||||
VictimCorporationID: k.Victim.CorporationID,
|
||||
VictimAllianceID: k.Victim.AllianceID,
|
||||
VictimDamageTaken: k.Victim.DamageTaken,
|
||||
VictimPosX: k.Victim.Position.X,
|
||||
VictimPosY: k.Victim.Position.Y,
|
||||
VictimPosZ: k.Victim.Position.Z,
|
||||
AttackerCount: uint16(len(k.Attackers)),
|
||||
Attackers: k.Attackers,
|
||||
Items: k.Victim.Items,
|
||||
}
|
||||
|
||||
var modules []*FlatModule
|
||||
for _, item := range k.Victim.Items {
|
||||
moduleSlot := GetModuleSlotByKillmailFlag(item.Flag)
|
||||
// We only care about fitted items
|
||||
if moduleSlot == ModuleSlotOther {
|
||||
continue
|
||||
}
|
||||
|
||||
modules = append(modules, &FlatModule{
|
||||
KillmailID: k.KillmailID,
|
||||
ItemTypeID: item.ItemTypeID,
|
||||
Slot: moduleSlot,
|
||||
})
|
||||
}
|
||||
|
||||
return flat, modules
|
||||
}
|
||||
// Legacy implementation - commented out
|
||||
// func (fm *FlatModule) Values() []interface{} {
|
||||
// return []interface{}{
|
||||
// fm.KillmailID,
|
||||
// fm.ItemTypeID,
|
||||
// string(fm.Slot),
|
||||
// }
|
||||
// }
|
||||
//
|
||||
// // FlattenKillmail converts the nested JSON structure to ClickHouse format
|
||||
// func (k *Killmail) FlattenKillmail() (*FlatKillmail, []*FlatModule) {
|
||||
// flat := &FlatKillmail{
|
||||
// KillmailID: k.KillmailID,
|
||||
// KillmailTime: k.KillmailTime,
|
||||
// SolarSystemID: k.SolarSystemID,
|
||||
// KillmailHash: k.KillmailHash,
|
||||
// VictimShipTypeID: k.Victim.ShipTypeID,
|
||||
// VictimCharacterID: k.Victim.CharacterID,
|
||||
// VictimCorporationID: k.Victim.CorporationID,
|
||||
// VictimAllianceID: k.Victim.AllianceID,
|
||||
// VictimDamageTaken: k.Victim.DamageTaken,
|
||||
// VictimPosX: k.Victim.Position.X,
|
||||
// VictimPosY: k.Victim.Position.Y,
|
||||
// VictimPosZ: k.Victim.Position.Z,
|
||||
// AttackerCount: uint16(len(k.Attackers)),
|
||||
// Attackers: k.Attackers,
|
||||
// Items: k.Victim.Items,
|
||||
// }
|
||||
//
|
||||
// var modules []*FlatModule
|
||||
// for _, item := range k.Victim.Items {
|
||||
// moduleSlot := GetModuleSlotByKillmailFlag(item.Flag)
|
||||
// // We only care about fitted items
|
||||
// if moduleSlot == ModuleSlotOther {
|
||||
// continue
|
||||
// }
|
||||
//
|
||||
// modules = append(modules, &FlatModule{
|
||||
// KillmailID: k.KillmailID,
|
||||
// ItemTypeID: item.ItemTypeID,
|
||||
// Slot: moduleSlot,
|
||||
// })
|
||||
// }
|
||||
//
|
||||
// return flat, modules
|
||||
// }
|
||||
|
||||
func (db *DBWrapper) SaveKillmails(killmails []Killmail) error {
|
||||
ctx := context.Background()
|
||||
@@ -175,21 +177,22 @@ func (db *DBWrapper) processBatch(ctx context.Context, batch []Killmail, flatBat
|
||||
continue
|
||||
}
|
||||
|
||||
flatKM, modules := km.FlattenKillmail()
|
||||
|
||||
// Append to killmails batch
|
||||
err := flatBatch.Append(flatKM.Values()...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Append fitted modules to modules batch
|
||||
for _, mod := range modules {
|
||||
err = moduleBatch.Append(mod.Values()...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// Legacy implementation commented out
|
||||
// flatKM, modules := km.FlattenKillmail()
|
||||
//
|
||||
// // Append to killmails batch
|
||||
// err := flatBatch.Append(flatKM.Values()...)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
//
|
||||
// // Append fitted modules to modules batch
|
||||
// for _, mod := range modules {
|
||||
// err = moduleBatch.Append(mod.Values()...)
|
||||
// if err != nil {
|
||||
// return err
|
||||
// }
|
||||
// }
|
||||
}
|
||||
|
||||
return nil
|
||||
|
||||
750
enrich.go
Normal file
750
enrich.go
Normal file
@@ -0,0 +1,750 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"zkillsusser/models"
|
||||
|
||||
logger "git.site.quack-lab.dev/dave/cylogger"
|
||||
"git.site.quack-lab.dev/dave/cyutils"
|
||||
"golang.org/x/sync/errgroup"
|
||||
"gorm.io/gorm"
|
||||
)
|
||||
|
||||
type FlatKillmail struct {
|
||||
KillmailID int64
|
||||
KillmailHash string
|
||||
KillmailTime string
|
||||
|
||||
SolarSystemID int32
|
||||
SolarSystemName string
|
||||
ConstellationName string
|
||||
RegionName string
|
||||
Security float32
|
||||
|
||||
VictimCharacterID int64
|
||||
VictimCharacterName string
|
||||
VictimCorporationID int64
|
||||
VictimCorporationName string
|
||||
VictimAllianceID *int64
|
||||
VictimAllianceName string
|
||||
VictimShipTypeID int32
|
||||
VictimShipTypeName string
|
||||
VictimShipGroupName string
|
||||
VictimShipCategoryName string
|
||||
VictimDamageTaken int64
|
||||
|
||||
AttackerCount uint16
|
||||
HTTPLastModified string
|
||||
}
|
||||
|
||||
type FlatKillmailAttacker struct {
|
||||
KillmailID int64
|
||||
|
||||
CharacterID int64
|
||||
CharacterName string
|
||||
CorporationID int64
|
||||
CorporationName string
|
||||
AllianceID *int64
|
||||
AllianceName string
|
||||
ShipTypeID int32
|
||||
ShipTypeName string
|
||||
ShipGroupName string
|
||||
WeaponTypeID int32
|
||||
WeaponTypeName string
|
||||
|
||||
DamageDone int64
|
||||
FinalBlow bool
|
||||
SecurityStatus float32
|
||||
}
|
||||
|
||||
type FlatKillmailItem struct {
|
||||
KillmailID int64
|
||||
|
||||
ItemTypeID int32
|
||||
ItemTypeName string
|
||||
ItemGroupName string
|
||||
ItemCategoryName string
|
||||
ItemMarketGroupName string
|
||||
|
||||
Flag int32
|
||||
QuantityDestroyed int64
|
||||
QuantityDropped int64
|
||||
Singleton int32
|
||||
}
|
||||
|
||||
type Cache[T any, K comparable] struct {
|
||||
m sync.Map
|
||||
getter func(ctx context.Context, db *gorm.DB, key K) (T, error)
|
||||
logger func(key K) *logger.Logger
|
||||
}
|
||||
|
||||
func NewCache[T any, K comparable](getter func(ctx context.Context, db *gorm.DB, key K) (T, error), logger func(key K) *logger.Logger) *Cache[T, K] {
|
||||
return &Cache[T, K]{
|
||||
getter: getter,
|
||||
logger: logger,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *Cache[T, K]) Get(ctx context.Context, db *gorm.DB, key K) (T, error) {
|
||||
var zero T
|
||||
val, found := c.m.Load(key)
|
||||
if found {
|
||||
return val.(T), nil
|
||||
}
|
||||
|
||||
flog := c.logger(key)
|
||||
flog.Debug("Querying database")
|
||||
result, err := c.getter(ctx, db, key)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get: %v", err)
|
||||
return zero, err
|
||||
}
|
||||
|
||||
c.m.Store(key, result)
|
||||
flog.Debug("Cached")
|
||||
return result, nil
|
||||
}
|
||||
|
||||
type FlatCache struct {
|
||||
types *Cache[*models.InvType, int32]
|
||||
groups *Cache[*models.InvGroup, int32]
|
||||
categories *Cache[*models.InvCategory, int32]
|
||||
marketGroups *Cache[*models.InvMarketGroup, int32]
|
||||
systems *Cache[*models.MapSolarSystem, int32]
|
||||
constellations *Cache[*models.MapConstellation, int32]
|
||||
regions *Cache[*models.MapRegion, int32]
|
||||
}
|
||||
|
||||
func getTypeFromDB(ctx context.Context, db *gorm.DB, typeID int32) (*models.InvType, error) {
|
||||
var t models.InvType
|
||||
if err := db.Where("typeID = ?", typeID).First(&t).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get type %d: %w", typeID, err)
|
||||
}
|
||||
return &t, nil
|
||||
}
|
||||
|
||||
func getGroupFromDB(ctx context.Context, db *gorm.DB, groupID int32) (*models.InvGroup, error) {
|
||||
var g models.InvGroup
|
||||
if err := db.Where("groupID = ?", groupID).First(&g).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get group %d: %w", groupID, err)
|
||||
}
|
||||
return &g, nil
|
||||
}
|
||||
|
||||
func getCategoryFromDB(ctx context.Context, db *gorm.DB, categoryID int32) (*models.InvCategory, error) {
|
||||
var c models.InvCategory
|
||||
if err := db.Where("categoryID = ?", categoryID).First(&c).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get category %d: %w", categoryID, err)
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func getMarketGroupFromDB(ctx context.Context, db *gorm.DB, marketGroupID int32) (*models.InvMarketGroup, error) {
|
||||
var mg models.InvMarketGroup
|
||||
if err := db.Where("marketGroupID = ?", marketGroupID).First(&mg).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get market group %d: %w", marketGroupID, err)
|
||||
}
|
||||
return &mg, nil
|
||||
}
|
||||
|
||||
func getSolarSystemFromDB(ctx context.Context, db *gorm.DB, systemID int32) (*models.MapSolarSystem, error) {
|
||||
var s models.MapSolarSystem
|
||||
if err := db.Where("solarSystemID = ?", systemID).First(&s).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get solar system %d: %w", systemID, err)
|
||||
}
|
||||
return &s, nil
|
||||
}
|
||||
|
||||
func getConstellationFromDB(ctx context.Context, db *gorm.DB, constellationID int32) (*models.MapConstellation, error) {
|
||||
var c models.MapConstellation
|
||||
if err := db.Where("constellationID = ?", constellationID).First(&c).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get constellation %d: %w", constellationID, err)
|
||||
}
|
||||
return &c, nil
|
||||
}
|
||||
|
||||
func getRegionFromDB(ctx context.Context, db *gorm.DB, regionID int32) (*models.MapRegion, error) {
|
||||
var r models.MapRegion
|
||||
if err := db.Where("regionID = ?", regionID).First(&r).Error; err != nil {
|
||||
return nil, fmt.Errorf("failed to get region %d: %w", regionID, err)
|
||||
}
|
||||
return &r, nil
|
||||
}
|
||||
|
||||
func NewFlatCache() *FlatCache {
|
||||
return &FlatCache{
|
||||
types: NewCache(getTypeFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getType").WithPrefix(fmt.Sprintf("type_%d", key))
|
||||
}),
|
||||
groups: NewCache(getGroupFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getGroup").WithPrefix(fmt.Sprintf("group_%d", key))
|
||||
}),
|
||||
categories: NewCache(getCategoryFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getCategory").WithPrefix(fmt.Sprintf("category_%d", key))
|
||||
}),
|
||||
marketGroups: NewCache(getMarketGroupFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getMarketGroup").WithPrefix(fmt.Sprintf("marketgroup_%d", key))
|
||||
}),
|
||||
systems: NewCache(getSolarSystemFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getSolarSystem").WithPrefix(fmt.Sprintf("system_%d", key))
|
||||
}),
|
||||
constellations: NewCache(getConstellationFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getConstellation").WithPrefix(fmt.Sprintf("constellation_%d", key))
|
||||
}),
|
||||
regions: NewCache(getRegionFromDB, func(key int32) *logger.Logger {
|
||||
return logger.Default.WithPrefix("getRegion").WithPrefix(fmt.Sprintf("region_%d", key))
|
||||
}),
|
||||
}
|
||||
}
|
||||
|
||||
func FlattenKillmail(db *gorm.DB, killmail Killmail) (*FlatKillmail, []FlatKillmailAttacker, []FlatKillmailItem, error) {
|
||||
flog := logger.Default.WithPrefix("FlattenKillmail").WithPrefix(fmt.Sprintf("killmail_%d", killmail.KillmailID))
|
||||
flog.Info("Starting flattening killmail")
|
||||
cache := NewFlatCache()
|
||||
client := cyutils.LimitedHTTP(10.0, 20)
|
||||
|
||||
flat := &FlatKillmail{
|
||||
KillmailID: killmail.KillmailID,
|
||||
KillmailHash: killmail.KillmailHash,
|
||||
KillmailTime: killmail.KillmailTime.Format("2006-01-02 15:04:05"),
|
||||
HTTPLastModified: killmail.HTTPLastModified.Format("2006-01-02 15:04:05"),
|
||||
AttackerCount: uint16(len(killmail.Attackers)),
|
||||
}
|
||||
|
||||
g, ctx := errgroup.WithContext(context.Background())
|
||||
|
||||
g.Go(func() error {
|
||||
return flattenSolarSystem(ctx, db, cache, int32(killmail.SolarSystemID), flat)
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
return flattenVictim(ctx, db, cache, client, killmail.Victim, flat)
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
flog.Error("Failed to flatten killmail: %v", err)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
|
||||
flog.Debug("Flattening %d attackers", len(killmail.Attackers))
|
||||
attackers := make([]FlatKillmailAttacker, 0, len(killmail.Attackers))
|
||||
for i, attacker := range killmail.Attackers {
|
||||
attackerLog := flog.WithPrefix(fmt.Sprintf("attacker_%d", i))
|
||||
flatAttacker, err := flattenAttacker(ctx, db, cache, client, killmail.KillmailID, attacker)
|
||||
if err != nil {
|
||||
attackerLog.Error("Failed to flatten attacker: %v", err)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
attackers = append(attackers, *flatAttacker)
|
||||
}
|
||||
|
||||
flog.Debug("Flattening %d items", len(killmail.Victim.Items))
|
||||
items := make([]FlatKillmailItem, 0, len(killmail.Victim.Items))
|
||||
for i, item := range killmail.Victim.Items {
|
||||
itemLog := flog.WithPrefix(fmt.Sprintf("item_%d", i))
|
||||
flatItem, err := flattenItemType(ctx, db, cache, killmail.KillmailID, item)
|
||||
if err != nil {
|
||||
itemLog.Error("Failed to flatten item: %v", err)
|
||||
return nil, nil, nil, err
|
||||
}
|
||||
items = append(items, *flatItem)
|
||||
}
|
||||
|
||||
flog.Info("Successfully flattened killmail with %d attackers and %d items", len(attackers), len(items))
|
||||
return flat, attackers, items, nil
|
||||
}
|
||||
|
||||
func flattenSolarSystem(ctx context.Context, db *gorm.DB, cache *FlatCache, systemID int32, flat *FlatKillmail) error {
|
||||
flog := logger.Default.WithPrefix("flattenSolarSystem").WithPrefix(fmt.Sprintf("system_%d", systemID))
|
||||
flog.Debug("Fetching solar system")
|
||||
|
||||
system, err := cache.systems.Get(ctx, db, systemID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
flat.SolarSystemID = system.SolarSystemID
|
||||
flat.SolarSystemName = system.SolarSystemName
|
||||
flat.Security = system.Security
|
||||
|
||||
flog.Debug("Fetching constellation %d", system.ConstellationID)
|
||||
constellation, err := cache.constellations.Get(ctx, db, system.ConstellationID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
flat.ConstellationName = constellation.ConstellationName
|
||||
|
||||
flog.Debug("Fetching region %d", constellation.RegionID)
|
||||
region, err := cache.regions.Get(ctx, db, constellation.RegionID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
flat.RegionName = region.RegionName
|
||||
return nil
|
||||
}
|
||||
|
||||
func flattenVictim(ctx context.Context, db *gorm.DB, cache *FlatCache, client *http.Client, victim Victim, flat *FlatKillmail) error {
|
||||
flog := logger.Default.WithPrefix("flattenVictim")
|
||||
flog.Debug("Starting victim flattening")
|
||||
|
||||
flat.VictimCharacterID = victim.CharacterID
|
||||
flat.VictimCorporationID = victim.CorporationID
|
||||
if victim.AllianceID != 0 {
|
||||
flat.VictimAllianceID = &victim.AllianceID
|
||||
}
|
||||
flat.VictimShipTypeID = int32(victim.ShipTypeID)
|
||||
flat.VictimDamageTaken = victim.DamageTaken
|
||||
|
||||
g, _ := errgroup.WithContext(ctx)
|
||||
|
||||
if victim.CharacterID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching character name for ID %d", victim.CharacterID)
|
||||
name, err := getCharacterName(client, victim.CharacterID)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get character name: %v", err)
|
||||
return err
|
||||
}
|
||||
flat.VictimCharacterName = name
|
||||
flog.Debug("Got character name: %s", name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if victim.CorporationID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching corporation name for ID %d", victim.CorporationID)
|
||||
name, err := getCorporationName(client, victim.CorporationID)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get corporation name: %v", err)
|
||||
return err
|
||||
}
|
||||
flat.VictimCorporationName = name
|
||||
flog.Debug("Got corporation name: %s", name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if victim.AllianceID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching alliance name for ID %d", victim.AllianceID)
|
||||
name, err := getAllianceName(client, victim.AllianceID)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get alliance name: %v", err)
|
||||
return err
|
||||
}
|
||||
flat.VictimAllianceName = name
|
||||
flog.Debug("Got alliance name: %s", name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching ship type name for ID %d", victim.ShipTypeID)
|
||||
typeName, err := flattenTypeName(ctx, db, cache, int32(victim.ShipTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.VictimShipTypeName = typeName
|
||||
flog.Debug("Got ship type name: %s", typeName)
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching ship group name for type ID %d", victim.ShipTypeID)
|
||||
groupName, err := flattenGroupName(ctx, db, cache, int32(victim.ShipTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.VictimShipGroupName = groupName
|
||||
flog.Debug("Got ship group name: %s", groupName)
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching ship category name for type ID %d", victim.ShipTypeID)
|
||||
categoryName, err := flattenCategoryName(ctx, db, cache, int32(victim.ShipTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.VictimShipCategoryName = categoryName
|
||||
flog.Debug("Got ship category name: %s", categoryName)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
flog.Error("Failed to flatten victim: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
flog.Info("Successfully flattened victim")
|
||||
return nil
|
||||
}
|
||||
|
||||
func flattenAttacker(ctx context.Context, db *gorm.DB, cache *FlatCache, client *http.Client, killmailID int64, attacker Attacker) (*FlatKillmailAttacker, error) {
|
||||
flog := logger.Default.WithPrefix("flattenAttacker").WithPrefix(fmt.Sprintf("character_%d", attacker.CharacterID))
|
||||
flog.Debug("Starting attacker flattening")
|
||||
|
||||
flat := &FlatKillmailAttacker{
|
||||
KillmailID: killmailID,
|
||||
CharacterID: attacker.CharacterID,
|
||||
CorporationID: attacker.CorporationID,
|
||||
ShipTypeID: int32(attacker.ShipTypeID),
|
||||
WeaponTypeID: int32(attacker.WeaponTypeID),
|
||||
DamageDone: attacker.DamageDone,
|
||||
FinalBlow: attacker.FinalBlow,
|
||||
SecurityStatus: float32(attacker.SecurityStatus),
|
||||
}
|
||||
|
||||
if attacker.AllianceID != 0 {
|
||||
flat.AllianceID = &attacker.AllianceID
|
||||
}
|
||||
|
||||
g, _ := errgroup.WithContext(ctx)
|
||||
|
||||
if attacker.CharacterID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching character name")
|
||||
name, err := getCharacterName(client, attacker.CharacterID)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get character name: %v", err)
|
||||
return err
|
||||
}
|
||||
flat.CharacterName = name
|
||||
flog.Debug("Got character name: %s", name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if attacker.CorporationID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching corporation name")
|
||||
name, err := getCorporationName(client, attacker.CorporationID)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get corporation name: %v", err)
|
||||
return err
|
||||
}
|
||||
flat.CorporationName = name
|
||||
flog.Debug("Got corporation name: %s", name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if attacker.AllianceID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching alliance name")
|
||||
name, err := getAllianceName(client, attacker.AllianceID)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get alliance name: %v", err)
|
||||
return err
|
||||
}
|
||||
flat.AllianceName = name
|
||||
flog.Debug("Got alliance name: %s", name)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching ship type name for ID %d", attacker.ShipTypeID)
|
||||
typeName, err := flattenTypeName(ctx, db, cache, int32(attacker.ShipTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.ShipTypeName = typeName
|
||||
flog.Debug("Got ship type name: %s", typeName)
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching ship group name for type ID %d", attacker.ShipTypeID)
|
||||
groupName, err := flattenGroupName(ctx, db, cache, int32(attacker.ShipTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.ShipGroupName = groupName
|
||||
flog.Debug("Got ship group name: %s", groupName)
|
||||
return nil
|
||||
})
|
||||
|
||||
if attacker.WeaponTypeID != 0 {
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching weapon type name for ID %d", attacker.WeaponTypeID)
|
||||
typeName, err := flattenTypeName(ctx, db, cache, int32(attacker.WeaponTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.WeaponTypeName = typeName
|
||||
flog.Debug("Got weapon type name: %s", typeName)
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
flog.Error("Failed to flatten attacker: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
flog.Info("Successfully flattened attacker")
|
||||
return flat, nil
|
||||
}
|
||||
|
||||
func flattenItemType(ctx context.Context, db *gorm.DB, cache *FlatCache, killmailID int64, item Item) (*FlatKillmailItem, error) {
|
||||
flog := logger.Default.WithPrefix("flattenItemType").WithPrefix(fmt.Sprintf("item_%d", item.ItemTypeID))
|
||||
flog.Debug("Starting item flattening")
|
||||
|
||||
flat := &FlatKillmailItem{
|
||||
KillmailID: killmailID,
|
||||
ItemTypeID: int32(item.ItemTypeID),
|
||||
Flag: int32(item.Flag),
|
||||
QuantityDestroyed: derefInt64(item.QuantityDestroyed),
|
||||
QuantityDropped: derefInt64(item.QuantityDropped),
|
||||
Singleton: int32(item.Singleton),
|
||||
}
|
||||
|
||||
g, _ := errgroup.WithContext(ctx)
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching item type name")
|
||||
typeName, err := flattenTypeName(ctx, db, cache, int32(item.ItemTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.ItemTypeName = typeName
|
||||
flog.Debug("Got item type name: %s", typeName)
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching item group name")
|
||||
groupName, err := flattenGroupName(ctx, db, cache, int32(item.ItemTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.ItemGroupName = groupName
|
||||
flog.Debug("Got item group name: %s", groupName)
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching item category name")
|
||||
categoryName, err := flattenCategoryName(ctx, db, cache, int32(item.ItemTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.ItemCategoryName = categoryName
|
||||
flog.Debug("Got item category name: %s", categoryName)
|
||||
return nil
|
||||
})
|
||||
|
||||
g.Go(func() error {
|
||||
flog.Debug("Fetching item market group name")
|
||||
marketGroupName, err := flattenMarketGroupName(ctx, db, cache, int32(item.ItemTypeID))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
flat.ItemMarketGroupName = marketGroupName
|
||||
flog.Debug("Got item market group name: %s", marketGroupName)
|
||||
return nil
|
||||
})
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
flog.Error("Failed to flatten item: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
flog.Info("Successfully flattened item")
|
||||
return flat, nil
|
||||
}
|
||||
|
||||
func flattenTypeName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) {
|
||||
flog := logger.Default.WithPrefix("flattenTypeName").WithPrefix(fmt.Sprintf("type_%d", typeID))
|
||||
flog.Debug("Fetching type name")
|
||||
|
||||
itemType, err := cache.types.Get(ctx, db, typeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
flog.Debug("Got type name: %s", itemType.TypeName)
|
||||
return itemType.TypeName, nil
|
||||
}
|
||||
|
||||
func flattenGroupName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) {
|
||||
flog := logger.Default.WithPrefix("flattenGroupName").WithPrefix(fmt.Sprintf("type_%d", typeID))
|
||||
flog.Debug("Fetching group name")
|
||||
|
||||
itemType, err := cache.types.Get(ctx, db, typeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
group, err := cache.groups.Get(ctx, db, itemType.GroupID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
flog.Debug("Got group name: %s", group.GroupName)
|
||||
return group.GroupName, nil
|
||||
}
|
||||
|
||||
func flattenCategoryName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) {
|
||||
flog := logger.Default.WithPrefix("flattenCategoryName").WithPrefix(fmt.Sprintf("type_%d", typeID))
|
||||
flog.Debug("Fetching category name")
|
||||
|
||||
itemType, err := cache.types.Get(ctx, db, typeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
group, err := cache.groups.Get(ctx, db, itemType.GroupID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
category, err := cache.categories.Get(ctx, db, group.CategoryID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
flog.Debug("Got category name: %s", category.CategoryName)
|
||||
return category.CategoryName, nil
|
||||
}
|
||||
|
||||
func flattenMarketGroupName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) {
|
||||
flog := logger.Default.WithPrefix("flattenMarketGroupName").WithPrefix(fmt.Sprintf("type_%d", typeID))
|
||||
flog.Debug("Fetching market group name")
|
||||
|
||||
itemType, err := cache.types.Get(ctx, db, typeID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
if itemType.MarketGroupID == 0 {
|
||||
flog.Debug("Type has no market group")
|
||||
return "", nil
|
||||
}
|
||||
|
||||
marketGroup, err := cache.marketGroups.Get(ctx, db, itemType.MarketGroupID)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
|
||||
flog.Debug("Got market group name: %s", marketGroup.MarketGroupName)
|
||||
return marketGroup.MarketGroupName, nil
|
||||
}
|
||||
|
||||
func getCharacterName(client *http.Client, characterID int64) (string, error) {
|
||||
flog := logger.Default.WithPrefix("getCharacterName").WithPrefix(fmt.Sprintf("character_%d", characterID))
|
||||
flog.Debug("Fetching character name from ESI")
|
||||
|
||||
esiURL := fmt.Sprintf("https://esi.evetech.net/characters/%d", characterID)
|
||||
proxyURL := fmt.Sprintf("https://proxy.site.quack-lab.dev?url=%s", url.QueryEscape(esiURL))
|
||||
|
||||
req, err := http.NewRequest("GET", proxyURL, nil)
|
||||
if err != nil {
|
||||
flog.Error("Failed to create request: %v", err)
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
flog.Debug("Sending HTTP request")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
flog.Error("Failed to fetch character: %v", err)
|
||||
return "", fmt.Errorf("failed to fetch character %d: %w", characterID, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
flog.Error("Character returned status %d", resp.StatusCode)
|
||||
return "", fmt.Errorf("character %d returned status %d", characterID, resp.StatusCode)
|
||||
}
|
||||
|
||||
var char Character
|
||||
if err := json.NewDecoder(resp.Body).Decode(&char); err != nil {
|
||||
flog.Error("Failed to decode character: %v", err)
|
||||
return "", fmt.Errorf("failed to decode character %d: %w", characterID, err)
|
||||
}
|
||||
|
||||
flog.Debug("Got character name: %s", char.Name)
|
||||
return char.Name, nil
|
||||
}
|
||||
|
||||
func getCorporationName(client *http.Client, corporationID int64) (string, error) {
|
||||
flog := logger.Default.WithPrefix("getCorporationName").WithPrefix(fmt.Sprintf("corporation_%d", corporationID))
|
||||
flog.Debug("Fetching corporation name from ESI")
|
||||
|
||||
esiURL := fmt.Sprintf("https://esi.evetech.net/corporations/%d", corporationID)
|
||||
proxyURL := fmt.Sprintf("https://proxy.site.quack-lab.dev?url=%s", url.QueryEscape(esiURL))
|
||||
|
||||
req, err := http.NewRequest("GET", proxyURL, nil)
|
||||
if err != nil {
|
||||
flog.Error("Failed to create request: %v", err)
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
flog.Debug("Sending HTTP request")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
flog.Error("Failed to fetch corporation: %v", err)
|
||||
return "", fmt.Errorf("failed to fetch corporation %d: %w", corporationID, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
flog.Error("Corporation returned status %d", resp.StatusCode)
|
||||
return "", fmt.Errorf("corporation %d returned status %d", corporationID, resp.StatusCode)
|
||||
}
|
||||
|
||||
var corp Corporation
|
||||
if err := json.NewDecoder(resp.Body).Decode(&corp); err != nil {
|
||||
flog.Error("Failed to decode corporation: %v", err)
|
||||
return "", fmt.Errorf("failed to decode corporation %d: %w", corporationID, err)
|
||||
}
|
||||
|
||||
flog.Debug("Got corporation name: %s", corp.Name)
|
||||
return corp.Name, nil
|
||||
}
|
||||
|
||||
func getAllianceName(client *http.Client, allianceID int64) (string, error) {
|
||||
flog := logger.Default.WithPrefix("getAllianceName").WithPrefix(fmt.Sprintf("alliance_%d", allianceID))
|
||||
flog.Debug("Fetching alliance name from ESI")
|
||||
|
||||
esiURL := fmt.Sprintf("https://esi.evetech.net/alliances/%d", allianceID)
|
||||
proxyURL := fmt.Sprintf("https://proxy.site.quack-lab.dev?url=%s", url.QueryEscape(esiURL))
|
||||
|
||||
req, err := http.NewRequest("GET", proxyURL, nil)
|
||||
if err != nil {
|
||||
flog.Error("Failed to create request: %v", err)
|
||||
return "", fmt.Errorf("failed to create request: %w", err)
|
||||
}
|
||||
|
||||
flog.Debug("Sending HTTP request")
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
flog.Error("Failed to fetch alliance: %v", err)
|
||||
return "", fmt.Errorf("failed to fetch alliance %d: %w", allianceID, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode != http.StatusOK {
|
||||
flog.Error("Alliance returned status %d", resp.StatusCode)
|
||||
return "", fmt.Errorf("alliance %d returned status %d", allianceID, resp.StatusCode)
|
||||
}
|
||||
|
||||
var alliance Alliance
|
||||
if err := json.NewDecoder(resp.Body).Decode(&alliance); err != nil {
|
||||
flog.Error("Failed to decode alliance: %v", err)
|
||||
return "", fmt.Errorf("failed to decode alliance %d: %w", allianceID, err)
|
||||
}
|
||||
|
||||
flog.Debug("Got alliance name: %s", alliance.Name)
|
||||
return alliance.Name, nil
|
||||
}
|
||||
4
go.mod
4
go.mod
@@ -8,7 +8,9 @@ require (
|
||||
git.site.quack-lab.dev/dave/cylogger v1.5.0
|
||||
git.site.quack-lab.dev/dave/cyutils v1.5.0
|
||||
github.com/ClickHouse/clickhouse-go/v2 v2.42.0
|
||||
github.com/hexops/valast v1.5.0
|
||||
github.com/stretchr/testify v1.11.1
|
||||
golang.org/x/sync v0.19.0
|
||||
gorm.io/driver/sqlite v1.6.0
|
||||
gorm.io/gorm v1.31.1
|
||||
)
|
||||
@@ -22,7 +24,6 @@ require (
|
||||
github.com/go-faster/errors v0.7.1 // indirect
|
||||
github.com/google/go-cmp v0.7.0 // indirect
|
||||
github.com/google/uuid v1.6.0 // indirect
|
||||
github.com/hexops/valast v1.5.0 // indirect
|
||||
github.com/jinzhu/inflection v1.0.0 // indirect
|
||||
github.com/jinzhu/now v1.1.5 // indirect
|
||||
github.com/klauspost/compress v1.18.0 // indirect
|
||||
@@ -36,7 +37,6 @@ require (
|
||||
go.opentelemetry.io/otel/trace v1.39.0 // indirect
|
||||
go.yaml.in/yaml/v3 v3.0.4 // indirect
|
||||
golang.org/x/mod v0.30.0 // indirect
|
||||
golang.org/x/sync v0.19.0 // indirect
|
||||
golang.org/x/sys v0.39.0 // indirect
|
||||
golang.org/x/text v0.32.0 // indirect
|
||||
golang.org/x/time v0.12.0 // indirect
|
||||
|
||||
39
types.go
39
types.go
@@ -47,45 +47,6 @@ type Position struct {
|
||||
Z float64 `json:"z"`
|
||||
}
|
||||
|
||||
// region Clickhouse killmails
|
||||
// FlatKillmail - Main analytical table
|
||||
// Denormalized for fast aggregations
|
||||
|
||||
type FlatKillmail struct {
|
||||
// Core killmail data
|
||||
KillmailID int64
|
||||
KillmailTime time.Time
|
||||
SolarSystemID int64
|
||||
KillmailHash string
|
||||
|
||||
// Victim data (flattened)
|
||||
VictimShipTypeID int64
|
||||
VictimCharacterID int64
|
||||
VictimCorporationID int64
|
||||
VictimAllianceID int64
|
||||
VictimDamageTaken int64
|
||||
|
||||
// Victim position
|
||||
VictimPosX float64
|
||||
VictimPosY float64
|
||||
VictimPosZ float64
|
||||
|
||||
// Attacker summary stats
|
||||
AttackerCount uint16
|
||||
TotalDamageDone int64
|
||||
FinalBlowShipType int64
|
||||
|
||||
Attackers []Attacker
|
||||
Items []Item
|
||||
}
|
||||
|
||||
// FlatModule - Separate table optimized for module co-occurrence queries
|
||||
type FlatModule struct {
|
||||
KillmailID int64
|
||||
ItemTypeID int64
|
||||
Slot ModuleSlot
|
||||
}
|
||||
|
||||
// Helper functions
|
||||
func boolToUint8(b bool) uint8 {
|
||||
if b {
|
||||
|
||||
Reference in New Issue
Block a user