From 560b1dd346ebb9350168ef0e2da248cf3da76d53 Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Sat, 24 Jan 2026 21:26:30 +0100 Subject: [PATCH] Hallucinate an "enrich" script/function to pad out the killmail metadata --- clickhouse.go | 189 ++++++------- enrich.go | 750 ++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 4 +- types.go | 39 --- 4 files changed, 848 insertions(+), 134 deletions(-) create mode 100644 enrich.go diff --git a/clickhouse.go b/clickhouse.go index 4ed7281..7512be3 100644 --- a/clickhouse.go +++ b/clickhouse.go @@ -9,40 +9,41 @@ import ( "github.com/ClickHouse/clickhouse-go/v2/lib/driver" ) +// Legacy implementation - commented out // Values returns all fields in the order expected by ClickHouse INSERT -func (fk *FlatKillmail) Values() []interface{} { - // Convert attackers to the tuple format expected by ClickHouse - attackers := make([][]interface{}, 0, len(fk.Attackers)) - for _, attacker := range fk.Attackers { - attackers = append(attackers, attacker.Values()) - } - - // Convert items to tuple format - items := make([][]interface{}, 0, len(fk.Items)) - for _, item := range fk.Items { - items = append(items, item.Values()) - } - - return []interface{}{ - fk.KillmailID, - fk.KillmailTime, - fk.SolarSystemID, - fk.KillmailHash, - fk.VictimShipTypeID, - fk.VictimCharacterID, - fk.VictimCorporationID, - fk.VictimAllianceID, - fk.VictimDamageTaken, - fk.VictimPosX, - fk.VictimPosY, - fk.VictimPosZ, - fk.AttackerCount, - fk.TotalDamageDone, - fk.FinalBlowShipType, - attackers, // Converted attackers - items, // Converted items - } -} +// func (fk *FlatKillmail) Values() []interface{} { +// // Convert attackers to the tuple format expected by ClickHouse +// attackers := make([][]interface{}, 0, len(fk.Attackers)) +// for _, attacker := range fk.Attackers { +// attackers = append(attackers, attacker.Values()) +// } +// +// // Convert items to tuple format +// items := make([][]interface{}, 0, len(fk.Items)) +// for _, item := range fk.Items { +// items = append(items, item.Values()) +// } +// +// return []interface{}{ +// fk.KillmailID, +// fk.KillmailTime, +// fk.SolarSystemID, +// fk.KillmailHash, +// fk.VictimShipTypeID, +// fk.VictimCharacterID, +// fk.VictimCorporationID, +// fk.VictimAllianceID, +// fk.VictimDamageTaken, +// fk.VictimPosX, +// fk.VictimPosY, +// fk.VictimPosZ, +// fk.AttackerCount, +// fk.TotalDamageDone, +// fk.FinalBlowShipType, +// attackers, // Converted attackers +// items, // Converted items +// } +// } func (a Attacker) Values() []interface{} { finalBlow := uint8(0) @@ -71,51 +72,52 @@ func (i Item) Values() []interface{} { } } -func (fm *FlatModule) Values() []interface{} { - return []interface{}{ - fm.KillmailID, - fm.ItemTypeID, - string(fm.Slot), - } -} - -// FlattenKillmail converts the nested JSON structure to ClickHouse format -func (k *Killmail) FlattenKillmail() (*FlatKillmail, []*FlatModule) { - flat := &FlatKillmail{ - KillmailID: k.KillmailID, - KillmailTime: k.KillmailTime, - SolarSystemID: k.SolarSystemID, - KillmailHash: k.KillmailHash, - VictimShipTypeID: k.Victim.ShipTypeID, - VictimCharacterID: k.Victim.CharacterID, - VictimCorporationID: k.Victim.CorporationID, - VictimAllianceID: k.Victim.AllianceID, - VictimDamageTaken: k.Victim.DamageTaken, - VictimPosX: k.Victim.Position.X, - VictimPosY: k.Victim.Position.Y, - VictimPosZ: k.Victim.Position.Z, - AttackerCount: uint16(len(k.Attackers)), - Attackers: k.Attackers, - Items: k.Victim.Items, - } - - var modules []*FlatModule - for _, item := range k.Victim.Items { - moduleSlot := GetModuleSlotByKillmailFlag(item.Flag) - // We only care about fitted items - if moduleSlot == ModuleSlotOther { - continue - } - - modules = append(modules, &FlatModule{ - KillmailID: k.KillmailID, - ItemTypeID: item.ItemTypeID, - Slot: moduleSlot, - }) - } - - return flat, modules -} +// Legacy implementation - commented out +// func (fm *FlatModule) Values() []interface{} { +// return []interface{}{ +// fm.KillmailID, +// fm.ItemTypeID, +// string(fm.Slot), +// } +// } +// +// // FlattenKillmail converts the nested JSON structure to ClickHouse format +// func (k *Killmail) FlattenKillmail() (*FlatKillmail, []*FlatModule) { +// flat := &FlatKillmail{ +// KillmailID: k.KillmailID, +// KillmailTime: k.KillmailTime, +// SolarSystemID: k.SolarSystemID, +// KillmailHash: k.KillmailHash, +// VictimShipTypeID: k.Victim.ShipTypeID, +// VictimCharacterID: k.Victim.CharacterID, +// VictimCorporationID: k.Victim.CorporationID, +// VictimAllianceID: k.Victim.AllianceID, +// VictimDamageTaken: k.Victim.DamageTaken, +// VictimPosX: k.Victim.Position.X, +// VictimPosY: k.Victim.Position.Y, +// VictimPosZ: k.Victim.Position.Z, +// AttackerCount: uint16(len(k.Attackers)), +// Attackers: k.Attackers, +// Items: k.Victim.Items, +// } +// +// var modules []*FlatModule +// for _, item := range k.Victim.Items { +// moduleSlot := GetModuleSlotByKillmailFlag(item.Flag) +// // We only care about fitted items +// if moduleSlot == ModuleSlotOther { +// continue +// } +// +// modules = append(modules, &FlatModule{ +// KillmailID: k.KillmailID, +// ItemTypeID: item.ItemTypeID, +// Slot: moduleSlot, +// }) +// } +// +// return flat, modules +// } func (db *DBWrapper) SaveKillmails(killmails []Killmail) error { ctx := context.Background() @@ -175,21 +177,22 @@ func (db *DBWrapper) processBatch(ctx context.Context, batch []Killmail, flatBat continue } - flatKM, modules := km.FlattenKillmail() - - // Append to killmails batch - err := flatBatch.Append(flatKM.Values()...) - if err != nil { - return err - } - - // Append fitted modules to modules batch - for _, mod := range modules { - err = moduleBatch.Append(mod.Values()...) - if err != nil { - return err - } - } + // Legacy implementation commented out + // flatKM, modules := km.FlattenKillmail() + // + // // Append to killmails batch + // err := flatBatch.Append(flatKM.Values()...) + // if err != nil { + // return err + // } + // + // // Append fitted modules to modules batch + // for _, mod := range modules { + // err = moduleBatch.Append(mod.Values()...) + // if err != nil { + // return err + // } + // } } return nil diff --git a/enrich.go b/enrich.go new file mode 100644 index 0000000..7194b3d --- /dev/null +++ b/enrich.go @@ -0,0 +1,750 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "sync" + + "zkillsusser/models" + + logger "git.site.quack-lab.dev/dave/cylogger" + "git.site.quack-lab.dev/dave/cyutils" + "golang.org/x/sync/errgroup" + "gorm.io/gorm" +) + +type FlatKillmail struct { + KillmailID int64 + KillmailHash string + KillmailTime string + + SolarSystemID int32 + SolarSystemName string + ConstellationName string + RegionName string + Security float32 + + VictimCharacterID int64 + VictimCharacterName string + VictimCorporationID int64 + VictimCorporationName string + VictimAllianceID *int64 + VictimAllianceName string + VictimShipTypeID int32 + VictimShipTypeName string + VictimShipGroupName string + VictimShipCategoryName string + VictimDamageTaken int64 + + AttackerCount uint16 + HTTPLastModified string +} + +type FlatKillmailAttacker struct { + KillmailID int64 + + CharacterID int64 + CharacterName string + CorporationID int64 + CorporationName string + AllianceID *int64 + AllianceName string + ShipTypeID int32 + ShipTypeName string + ShipGroupName string + WeaponTypeID int32 + WeaponTypeName string + + DamageDone int64 + FinalBlow bool + SecurityStatus float32 +} + +type FlatKillmailItem struct { + KillmailID int64 + + ItemTypeID int32 + ItemTypeName string + ItemGroupName string + ItemCategoryName string + ItemMarketGroupName string + + Flag int32 + QuantityDestroyed int64 + QuantityDropped int64 + Singleton int32 +} + +type Cache[T any, K comparable] struct { + m sync.Map + getter func(ctx context.Context, db *gorm.DB, key K) (T, error) + logger func(key K) *logger.Logger +} + +func NewCache[T any, K comparable](getter func(ctx context.Context, db *gorm.DB, key K) (T, error), logger func(key K) *logger.Logger) *Cache[T, K] { + return &Cache[T, K]{ + getter: getter, + logger: logger, + } +} + +func (c *Cache[T, K]) Get(ctx context.Context, db *gorm.DB, key K) (T, error) { + var zero T + val, found := c.m.Load(key) + if found { + return val.(T), nil + } + + flog := c.logger(key) + flog.Debug("Querying database") + result, err := c.getter(ctx, db, key) + if err != nil { + flog.Error("Failed to get: %v", err) + return zero, err + } + + c.m.Store(key, result) + flog.Debug("Cached") + return result, nil +} + +type FlatCache struct { + types *Cache[*models.InvType, int32] + groups *Cache[*models.InvGroup, int32] + categories *Cache[*models.InvCategory, int32] + marketGroups *Cache[*models.InvMarketGroup, int32] + systems *Cache[*models.MapSolarSystem, int32] + constellations *Cache[*models.MapConstellation, int32] + regions *Cache[*models.MapRegion, int32] +} + +func getTypeFromDB(ctx context.Context, db *gorm.DB, typeID int32) (*models.InvType, error) { + var t models.InvType + if err := db.Where("typeID = ?", typeID).First(&t).Error; err != nil { + return nil, fmt.Errorf("failed to get type %d: %w", typeID, err) + } + return &t, nil +} + +func getGroupFromDB(ctx context.Context, db *gorm.DB, groupID int32) (*models.InvGroup, error) { + var g models.InvGroup + if err := db.Where("groupID = ?", groupID).First(&g).Error; err != nil { + return nil, fmt.Errorf("failed to get group %d: %w", groupID, err) + } + return &g, nil +} + +func getCategoryFromDB(ctx context.Context, db *gorm.DB, categoryID int32) (*models.InvCategory, error) { + var c models.InvCategory + if err := db.Where("categoryID = ?", categoryID).First(&c).Error; err != nil { + return nil, fmt.Errorf("failed to get category %d: %w", categoryID, err) + } + return &c, nil +} + +func getMarketGroupFromDB(ctx context.Context, db *gorm.DB, marketGroupID int32) (*models.InvMarketGroup, error) { + var mg models.InvMarketGroup + if err := db.Where("marketGroupID = ?", marketGroupID).First(&mg).Error; err != nil { + return nil, fmt.Errorf("failed to get market group %d: %w", marketGroupID, err) + } + return &mg, nil +} + +func getSolarSystemFromDB(ctx context.Context, db *gorm.DB, systemID int32) (*models.MapSolarSystem, error) { + var s models.MapSolarSystem + if err := db.Where("solarSystemID = ?", systemID).First(&s).Error; err != nil { + return nil, fmt.Errorf("failed to get solar system %d: %w", systemID, err) + } + return &s, nil +} + +func getConstellationFromDB(ctx context.Context, db *gorm.DB, constellationID int32) (*models.MapConstellation, error) { + var c models.MapConstellation + if err := db.Where("constellationID = ?", constellationID).First(&c).Error; err != nil { + return nil, fmt.Errorf("failed to get constellation %d: %w", constellationID, err) + } + return &c, nil +} + +func getRegionFromDB(ctx context.Context, db *gorm.DB, regionID int32) (*models.MapRegion, error) { + var r models.MapRegion + if err := db.Where("regionID = ?", regionID).First(&r).Error; err != nil { + return nil, fmt.Errorf("failed to get region %d: %w", regionID, err) + } + return &r, nil +} + +func NewFlatCache() *FlatCache { + return &FlatCache{ + types: NewCache(getTypeFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getType").WithPrefix(fmt.Sprintf("type_%d", key)) + }), + groups: NewCache(getGroupFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getGroup").WithPrefix(fmt.Sprintf("group_%d", key)) + }), + categories: NewCache(getCategoryFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getCategory").WithPrefix(fmt.Sprintf("category_%d", key)) + }), + marketGroups: NewCache(getMarketGroupFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getMarketGroup").WithPrefix(fmt.Sprintf("marketgroup_%d", key)) + }), + systems: NewCache(getSolarSystemFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getSolarSystem").WithPrefix(fmt.Sprintf("system_%d", key)) + }), + constellations: NewCache(getConstellationFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getConstellation").WithPrefix(fmt.Sprintf("constellation_%d", key)) + }), + regions: NewCache(getRegionFromDB, func(key int32) *logger.Logger { + return logger.Default.WithPrefix("getRegion").WithPrefix(fmt.Sprintf("region_%d", key)) + }), + } +} + +func FlattenKillmail(db *gorm.DB, killmail Killmail) (*FlatKillmail, []FlatKillmailAttacker, []FlatKillmailItem, error) { + flog := logger.Default.WithPrefix("FlattenKillmail").WithPrefix(fmt.Sprintf("killmail_%d", killmail.KillmailID)) + flog.Info("Starting flattening killmail") + cache := NewFlatCache() + client := cyutils.LimitedHTTP(10.0, 20) + + flat := &FlatKillmail{ + KillmailID: killmail.KillmailID, + KillmailHash: killmail.KillmailHash, + KillmailTime: killmail.KillmailTime.Format("2006-01-02 15:04:05"), + HTTPLastModified: killmail.HTTPLastModified.Format("2006-01-02 15:04:05"), + AttackerCount: uint16(len(killmail.Attackers)), + } + + g, ctx := errgroup.WithContext(context.Background()) + + g.Go(func() error { + return flattenSolarSystem(ctx, db, cache, int32(killmail.SolarSystemID), flat) + }) + + g.Go(func() error { + return flattenVictim(ctx, db, cache, client, killmail.Victim, flat) + }) + + if err := g.Wait(); err != nil { + flog.Error("Failed to flatten killmail: %v", err) + return nil, nil, nil, err + } + + flog.Debug("Flattening %d attackers", len(killmail.Attackers)) + attackers := make([]FlatKillmailAttacker, 0, len(killmail.Attackers)) + for i, attacker := range killmail.Attackers { + attackerLog := flog.WithPrefix(fmt.Sprintf("attacker_%d", i)) + flatAttacker, err := flattenAttacker(ctx, db, cache, client, killmail.KillmailID, attacker) + if err != nil { + attackerLog.Error("Failed to flatten attacker: %v", err) + return nil, nil, nil, err + } + attackers = append(attackers, *flatAttacker) + } + + flog.Debug("Flattening %d items", len(killmail.Victim.Items)) + items := make([]FlatKillmailItem, 0, len(killmail.Victim.Items)) + for i, item := range killmail.Victim.Items { + itemLog := flog.WithPrefix(fmt.Sprintf("item_%d", i)) + flatItem, err := flattenItemType(ctx, db, cache, killmail.KillmailID, item) + if err != nil { + itemLog.Error("Failed to flatten item: %v", err) + return nil, nil, nil, err + } + items = append(items, *flatItem) + } + + flog.Info("Successfully flattened killmail with %d attackers and %d items", len(attackers), len(items)) + return flat, attackers, items, nil +} + +func flattenSolarSystem(ctx context.Context, db *gorm.DB, cache *FlatCache, systemID int32, flat *FlatKillmail) error { + flog := logger.Default.WithPrefix("flattenSolarSystem").WithPrefix(fmt.Sprintf("system_%d", systemID)) + flog.Debug("Fetching solar system") + + system, err := cache.systems.Get(ctx, db, systemID) + if err != nil { + return err + } + + flat.SolarSystemID = system.SolarSystemID + flat.SolarSystemName = system.SolarSystemName + flat.Security = system.Security + + flog.Debug("Fetching constellation %d", system.ConstellationID) + constellation, err := cache.constellations.Get(ctx, db, system.ConstellationID) + if err != nil { + return err + } + + flat.ConstellationName = constellation.ConstellationName + + flog.Debug("Fetching region %d", constellation.RegionID) + region, err := cache.regions.Get(ctx, db, constellation.RegionID) + if err != nil { + return err + } + + flat.RegionName = region.RegionName + return nil +} + +func flattenVictim(ctx context.Context, db *gorm.DB, cache *FlatCache, client *http.Client, victim Victim, flat *FlatKillmail) error { + flog := logger.Default.WithPrefix("flattenVictim") + flog.Debug("Starting victim flattening") + + flat.VictimCharacterID = victim.CharacterID + flat.VictimCorporationID = victim.CorporationID + if victim.AllianceID != 0 { + flat.VictimAllianceID = &victim.AllianceID + } + flat.VictimShipTypeID = int32(victim.ShipTypeID) + flat.VictimDamageTaken = victim.DamageTaken + + g, _ := errgroup.WithContext(ctx) + + if victim.CharacterID != 0 { + g.Go(func() error { + flog.Debug("Fetching character name for ID %d", victim.CharacterID) + name, err := getCharacterName(client, victim.CharacterID) + if err != nil { + flog.Error("Failed to get character name: %v", err) + return err + } + flat.VictimCharacterName = name + flog.Debug("Got character name: %s", name) + return nil + }) + } + + if victim.CorporationID != 0 { + g.Go(func() error { + flog.Debug("Fetching corporation name for ID %d", victim.CorporationID) + name, err := getCorporationName(client, victim.CorporationID) + if err != nil { + flog.Error("Failed to get corporation name: %v", err) + return err + } + flat.VictimCorporationName = name + flog.Debug("Got corporation name: %s", name) + return nil + }) + } + + if victim.AllianceID != 0 { + g.Go(func() error { + flog.Debug("Fetching alliance name for ID %d", victim.AllianceID) + name, err := getAllianceName(client, victim.AllianceID) + if err != nil { + flog.Error("Failed to get alliance name: %v", err) + return err + } + flat.VictimAllianceName = name + flog.Debug("Got alliance name: %s", name) + return nil + }) + } + + g.Go(func() error { + flog.Debug("Fetching ship type name for ID %d", victim.ShipTypeID) + typeName, err := flattenTypeName(ctx, db, cache, int32(victim.ShipTypeID)) + if err != nil { + return err + } + flat.VictimShipTypeName = typeName + flog.Debug("Got ship type name: %s", typeName) + return nil + }) + + g.Go(func() error { + flog.Debug("Fetching ship group name for type ID %d", victim.ShipTypeID) + groupName, err := flattenGroupName(ctx, db, cache, int32(victim.ShipTypeID)) + if err != nil { + return err + } + flat.VictimShipGroupName = groupName + flog.Debug("Got ship group name: %s", groupName) + return nil + }) + + g.Go(func() error { + flog.Debug("Fetching ship category name for type ID %d", victim.ShipTypeID) + categoryName, err := flattenCategoryName(ctx, db, cache, int32(victim.ShipTypeID)) + if err != nil { + return err + } + flat.VictimShipCategoryName = categoryName + flog.Debug("Got ship category name: %s", categoryName) + return nil + }) + + if err := g.Wait(); err != nil { + flog.Error("Failed to flatten victim: %v", err) + return err + } + + flog.Info("Successfully flattened victim") + return nil +} + +func flattenAttacker(ctx context.Context, db *gorm.DB, cache *FlatCache, client *http.Client, killmailID int64, attacker Attacker) (*FlatKillmailAttacker, error) { + flog := logger.Default.WithPrefix("flattenAttacker").WithPrefix(fmt.Sprintf("character_%d", attacker.CharacterID)) + flog.Debug("Starting attacker flattening") + + flat := &FlatKillmailAttacker{ + KillmailID: killmailID, + CharacterID: attacker.CharacterID, + CorporationID: attacker.CorporationID, + ShipTypeID: int32(attacker.ShipTypeID), + WeaponTypeID: int32(attacker.WeaponTypeID), + DamageDone: attacker.DamageDone, + FinalBlow: attacker.FinalBlow, + SecurityStatus: float32(attacker.SecurityStatus), + } + + if attacker.AllianceID != 0 { + flat.AllianceID = &attacker.AllianceID + } + + g, _ := errgroup.WithContext(ctx) + + if attacker.CharacterID != 0 { + g.Go(func() error { + flog.Debug("Fetching character name") + name, err := getCharacterName(client, attacker.CharacterID) + if err != nil { + flog.Error("Failed to get character name: %v", err) + return err + } + flat.CharacterName = name + flog.Debug("Got character name: %s", name) + return nil + }) + } + + if attacker.CorporationID != 0 { + g.Go(func() error { + flog.Debug("Fetching corporation name") + name, err := getCorporationName(client, attacker.CorporationID) + if err != nil { + flog.Error("Failed to get corporation name: %v", err) + return err + } + flat.CorporationName = name + flog.Debug("Got corporation name: %s", name) + return nil + }) + } + + if attacker.AllianceID != 0 { + g.Go(func() error { + flog.Debug("Fetching alliance name") + name, err := getAllianceName(client, attacker.AllianceID) + if err != nil { + flog.Error("Failed to get alliance name: %v", err) + return err + } + flat.AllianceName = name + flog.Debug("Got alliance name: %s", name) + return nil + }) + } + + g.Go(func() error { + flog.Debug("Fetching ship type name for ID %d", attacker.ShipTypeID) + typeName, err := flattenTypeName(ctx, db, cache, int32(attacker.ShipTypeID)) + if err != nil { + return err + } + flat.ShipTypeName = typeName + flog.Debug("Got ship type name: %s", typeName) + return nil + }) + + g.Go(func() error { + flog.Debug("Fetching ship group name for type ID %d", attacker.ShipTypeID) + groupName, err := flattenGroupName(ctx, db, cache, int32(attacker.ShipTypeID)) + if err != nil { + return err + } + flat.ShipGroupName = groupName + flog.Debug("Got ship group name: %s", groupName) + return nil + }) + + if attacker.WeaponTypeID != 0 { + g.Go(func() error { + flog.Debug("Fetching weapon type name for ID %d", attacker.WeaponTypeID) + typeName, err := flattenTypeName(ctx, db, cache, int32(attacker.WeaponTypeID)) + if err != nil { + return err + } + flat.WeaponTypeName = typeName + flog.Debug("Got weapon type name: %s", typeName) + return nil + }) + } + + if err := g.Wait(); err != nil { + flog.Error("Failed to flatten attacker: %v", err) + return nil, err + } + + flog.Info("Successfully flattened attacker") + return flat, nil +} + +func flattenItemType(ctx context.Context, db *gorm.DB, cache *FlatCache, killmailID int64, item Item) (*FlatKillmailItem, error) { + flog := logger.Default.WithPrefix("flattenItemType").WithPrefix(fmt.Sprintf("item_%d", item.ItemTypeID)) + flog.Debug("Starting item flattening") + + flat := &FlatKillmailItem{ + KillmailID: killmailID, + ItemTypeID: int32(item.ItemTypeID), + Flag: int32(item.Flag), + QuantityDestroyed: derefInt64(item.QuantityDestroyed), + QuantityDropped: derefInt64(item.QuantityDropped), + Singleton: int32(item.Singleton), + } + + g, _ := errgroup.WithContext(ctx) + + g.Go(func() error { + flog.Debug("Fetching item type name") + typeName, err := flattenTypeName(ctx, db, cache, int32(item.ItemTypeID)) + if err != nil { + return err + } + flat.ItemTypeName = typeName + flog.Debug("Got item type name: %s", typeName) + return nil + }) + + g.Go(func() error { + flog.Debug("Fetching item group name") + groupName, err := flattenGroupName(ctx, db, cache, int32(item.ItemTypeID)) + if err != nil { + return err + } + flat.ItemGroupName = groupName + flog.Debug("Got item group name: %s", groupName) + return nil + }) + + g.Go(func() error { + flog.Debug("Fetching item category name") + categoryName, err := flattenCategoryName(ctx, db, cache, int32(item.ItemTypeID)) + if err != nil { + return err + } + flat.ItemCategoryName = categoryName + flog.Debug("Got item category name: %s", categoryName) + return nil + }) + + g.Go(func() error { + flog.Debug("Fetching item market group name") + marketGroupName, err := flattenMarketGroupName(ctx, db, cache, int32(item.ItemTypeID)) + if err != nil { + return err + } + flat.ItemMarketGroupName = marketGroupName + flog.Debug("Got item market group name: %s", marketGroupName) + return nil + }) + + if err := g.Wait(); err != nil { + flog.Error("Failed to flatten item: %v", err) + return nil, err + } + + flog.Info("Successfully flattened item") + return flat, nil +} + +func flattenTypeName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) { + flog := logger.Default.WithPrefix("flattenTypeName").WithPrefix(fmt.Sprintf("type_%d", typeID)) + flog.Debug("Fetching type name") + + itemType, err := cache.types.Get(ctx, db, typeID) + if err != nil { + return "", err + } + + flog.Debug("Got type name: %s", itemType.TypeName) + return itemType.TypeName, nil +} + +func flattenGroupName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) { + flog := logger.Default.WithPrefix("flattenGroupName").WithPrefix(fmt.Sprintf("type_%d", typeID)) + flog.Debug("Fetching group name") + + itemType, err := cache.types.Get(ctx, db, typeID) + if err != nil { + return "", err + } + + group, err := cache.groups.Get(ctx, db, itemType.GroupID) + if err != nil { + return "", err + } + + flog.Debug("Got group name: %s", group.GroupName) + return group.GroupName, nil +} + +func flattenCategoryName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) { + flog := logger.Default.WithPrefix("flattenCategoryName").WithPrefix(fmt.Sprintf("type_%d", typeID)) + flog.Debug("Fetching category name") + + itemType, err := cache.types.Get(ctx, db, typeID) + if err != nil { + return "", err + } + + group, err := cache.groups.Get(ctx, db, itemType.GroupID) + if err != nil { + return "", err + } + + category, err := cache.categories.Get(ctx, db, group.CategoryID) + if err != nil { + return "", err + } + + flog.Debug("Got category name: %s", category.CategoryName) + return category.CategoryName, nil +} + +func flattenMarketGroupName(ctx context.Context, db *gorm.DB, cache *FlatCache, typeID int32) (string, error) { + flog := logger.Default.WithPrefix("flattenMarketGroupName").WithPrefix(fmt.Sprintf("type_%d", typeID)) + flog.Debug("Fetching market group name") + + itemType, err := cache.types.Get(ctx, db, typeID) + if err != nil { + return "", err + } + + if itemType.MarketGroupID == 0 { + flog.Debug("Type has no market group") + return "", nil + } + + marketGroup, err := cache.marketGroups.Get(ctx, db, itemType.MarketGroupID) + if err != nil { + return "", err + } + + flog.Debug("Got market group name: %s", marketGroup.MarketGroupName) + return marketGroup.MarketGroupName, nil +} + +func getCharacterName(client *http.Client, characterID int64) (string, error) { + flog := logger.Default.WithPrefix("getCharacterName").WithPrefix(fmt.Sprintf("character_%d", characterID)) + flog.Debug("Fetching character name from ESI") + + esiURL := fmt.Sprintf("https://esi.evetech.net/characters/%d", characterID) + proxyURL := fmt.Sprintf("https://proxy.site.quack-lab.dev?url=%s", url.QueryEscape(esiURL)) + + req, err := http.NewRequest("GET", proxyURL, nil) + if err != nil { + flog.Error("Failed to create request: %v", err) + return "", fmt.Errorf("failed to create request: %w", err) + } + + flog.Debug("Sending HTTP request") + resp, err := client.Do(req) + if err != nil { + flog.Error("Failed to fetch character: %v", err) + return "", fmt.Errorf("failed to fetch character %d: %w", characterID, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + flog.Error("Character returned status %d", resp.StatusCode) + return "", fmt.Errorf("character %d returned status %d", characterID, resp.StatusCode) + } + + var char Character + if err := json.NewDecoder(resp.Body).Decode(&char); err != nil { + flog.Error("Failed to decode character: %v", err) + return "", fmt.Errorf("failed to decode character %d: %w", characterID, err) + } + + flog.Debug("Got character name: %s", char.Name) + return char.Name, nil +} + +func getCorporationName(client *http.Client, corporationID int64) (string, error) { + flog := logger.Default.WithPrefix("getCorporationName").WithPrefix(fmt.Sprintf("corporation_%d", corporationID)) + flog.Debug("Fetching corporation name from ESI") + + esiURL := fmt.Sprintf("https://esi.evetech.net/corporations/%d", corporationID) + proxyURL := fmt.Sprintf("https://proxy.site.quack-lab.dev?url=%s", url.QueryEscape(esiURL)) + + req, err := http.NewRequest("GET", proxyURL, nil) + if err != nil { + flog.Error("Failed to create request: %v", err) + return "", fmt.Errorf("failed to create request: %w", err) + } + + flog.Debug("Sending HTTP request") + resp, err := client.Do(req) + if err != nil { + flog.Error("Failed to fetch corporation: %v", err) + return "", fmt.Errorf("failed to fetch corporation %d: %w", corporationID, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + flog.Error("Corporation returned status %d", resp.StatusCode) + return "", fmt.Errorf("corporation %d returned status %d", corporationID, resp.StatusCode) + } + + var corp Corporation + if err := json.NewDecoder(resp.Body).Decode(&corp); err != nil { + flog.Error("Failed to decode corporation: %v", err) + return "", fmt.Errorf("failed to decode corporation %d: %w", corporationID, err) + } + + flog.Debug("Got corporation name: %s", corp.Name) + return corp.Name, nil +} + +func getAllianceName(client *http.Client, allianceID int64) (string, error) { + flog := logger.Default.WithPrefix("getAllianceName").WithPrefix(fmt.Sprintf("alliance_%d", allianceID)) + flog.Debug("Fetching alliance name from ESI") + + esiURL := fmt.Sprintf("https://esi.evetech.net/alliances/%d", allianceID) + proxyURL := fmt.Sprintf("https://proxy.site.quack-lab.dev?url=%s", url.QueryEscape(esiURL)) + + req, err := http.NewRequest("GET", proxyURL, nil) + if err != nil { + flog.Error("Failed to create request: %v", err) + return "", fmt.Errorf("failed to create request: %w", err) + } + + flog.Debug("Sending HTTP request") + resp, err := client.Do(req) + if err != nil { + flog.Error("Failed to fetch alliance: %v", err) + return "", fmt.Errorf("failed to fetch alliance %d: %w", allianceID, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + flog.Error("Alliance returned status %d", resp.StatusCode) + return "", fmt.Errorf("alliance %d returned status %d", allianceID, resp.StatusCode) + } + + var alliance Alliance + if err := json.NewDecoder(resp.Body).Decode(&alliance); err != nil { + flog.Error("Failed to decode alliance: %v", err) + return "", fmt.Errorf("failed to decode alliance %d: %w", allianceID, err) + } + + flog.Debug("Got alliance name: %s", alliance.Name) + return alliance.Name, nil +} diff --git a/go.mod b/go.mod index 6db48d0..6a8816d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,9 @@ require ( git.site.quack-lab.dev/dave/cylogger v1.5.0 git.site.quack-lab.dev/dave/cyutils v1.5.0 github.com/ClickHouse/clickhouse-go/v2 v2.42.0 + github.com/hexops/valast v1.5.0 github.com/stretchr/testify v1.11.1 + golang.org/x/sync v0.19.0 gorm.io/driver/sqlite v1.6.0 gorm.io/gorm v1.31.1 ) @@ -22,7 +24,6 @@ require ( github.com/go-faster/errors v0.7.1 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/uuid v1.6.0 // indirect - github.com/hexops/valast v1.5.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect github.com/klauspost/compress v1.18.0 // indirect @@ -36,7 +37,6 @@ require ( go.opentelemetry.io/otel/trace v1.39.0 // indirect go.yaml.in/yaml/v3 v3.0.4 // indirect golang.org/x/mod v0.30.0 // indirect - golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.39.0 // indirect golang.org/x/text v0.32.0 // indirect golang.org/x/time v0.12.0 // indirect diff --git a/types.go b/types.go index a2e7d8f..dbbc32c 100644 --- a/types.go +++ b/types.go @@ -47,45 +47,6 @@ type Position struct { Z float64 `json:"z"` } -// region Clickhouse killmails -// FlatKillmail - Main analytical table -// Denormalized for fast aggregations - -type FlatKillmail struct { - // Core killmail data - KillmailID int64 - KillmailTime time.Time - SolarSystemID int64 - KillmailHash string - - // Victim data (flattened) - VictimShipTypeID int64 - VictimCharacterID int64 - VictimCorporationID int64 - VictimAllianceID int64 - VictimDamageTaken int64 - - // Victim position - VictimPosX float64 - VictimPosY float64 - VictimPosZ float64 - - // Attacker summary stats - AttackerCount uint16 - TotalDamageDone int64 - FinalBlowShipType int64 - - Attackers []Attacker - Items []Item -} - -// FlatModule - Separate table optimized for module co-occurrence queries -type FlatModule struct { - KillmailID int64 - ItemTypeID int64 - Slot ModuleSlot -} - // Helper functions func boolToUint8(b bool) uint8 { if b {