package main import "time" type Killmail struct { KillmailID int64 `json:"killmail_id" gorm:"primaryKey;column:killmail_id"` KillmailTime time.Time `json:"killmail_time" gorm:"column:killmail_time;index"` SolarSystemID int64 `json:"solar_system_id" gorm:"column:solar_system_id;index"` KillmailHash string `json:"killmail_hash" gorm:"column:killmail_hash"` HTTPLastModified time.Time `json:"http_last_modified" gorm:"column:http_last_modified"` Attackers []Attacker `json:"attackers" gorm:"foreignKey:KillmailID;references:KillmailID;constraint:OnDelete:CASCADE"` Victim Victim `json:"victim" gorm:"foreignKey:KillmailID;references:KillmailID;constraint:OnDelete:CASCADE"` } func (k *Killmail) TableName() string { return "zkill_killmails" } type Attacker struct { ID int64 `gorm:"primaryKey;autoIncrement;column:id"` KillmailID int64 `json:"killmail_id" gorm:"column:killmail_id;index"` AllianceID int64 `json:"alliance_id" gorm:"column:alliance_id"` CharacterID int64 `json:"character_id" gorm:"column:character_id"` CorporationID int64 `json:"corporation_id" gorm:"column:corporation_id"` DamageDone int64 `json:"damage_done" gorm:"column:damage_done"` FinalBlow bool `json:"final_blow" gorm:"column:final_blow"` SecurityStatus float64 `json:"security_status" gorm:"column:security_status"` ShipTypeID int64 `json:"ship_type_id" gorm:"column:ship_type_id"` WeaponTypeID int64 `json:"weapon_type_id" gorm:"column:weapon_type_id"` } func (a *Attacker) TableName() string { return "zkill_attackers" } type Victim struct { ID int64 `gorm:"primaryKey;autoIncrement;column:id"` KillmailID int64 `json:"killmail_id" gorm:"column:killmail_id;index"` AllianceID int64 `json:"alliance_id" gorm:"column:alliance_id"` CharacterID int64 `json:"character_id" gorm:"column:character_id"` CorporationID int64 `json:"corporation_id" gorm:"column:corporation_id"` DamageTaken int64 `json:"damage_taken" gorm:"column:damage_taken"` Items []Item `json:"items" gorm:"foreignKey:VictimID;references:ID;constraint:OnDelete:CASCADE"` Position Position `json:"position" gorm:"foreignKey:VictimID;references:ID;constraint:OnDelete:CASCADE"` ShipTypeID int64 `json:"ship_type_id" gorm:"column:ship_type_id;index"` } func (v *Victim) TableName() string { return "zkill_victims" } type Item struct { ID int64 `gorm:"primaryKey;autoIncrement;column:id"` VictimID int64 `json:"victim_id" gorm:"column:victim_id;index:idx_items_victim_item_flag"` Flag int64 `json:"flag" gorm:"column:flag;index:idx_items_victim_item_flag"` ItemTypeID int64 `json:"item_type_id" gorm:"column:item_type_id;index:idx_items_victim_item_flag"` QuantityDestroyed *int64 `json:"quantity_destroyed,omitempty" gorm:"column:quantity_destroyed"` Singleton int64 `json:"singleton" gorm:"column:singleton"` QuantityDropped *int64 `json:"quantity_dropped,omitempty" gorm:"column:quantity_dropped"` } func (i *Item) TableName() string { return "zkill_items" } type Position struct { ID int64 `gorm:"primaryKey;autoIncrement;column:id"` VictimID int64 `json:"victim_id" gorm:"column:victim_id;index"` X float64 `json:"x" gorm:"column:x"` Y float64 `json:"y" gorm:"column:y"` Z float64 `json:"z" gorm:"column:z"` } func (p *Position) TableName() string { return "zkill_positions" } // =============================================== // CLICKHOUSE FLATTENED STRUCTURES // =============================================== // FlatKillmail - Main analytical table // Denormalized for fast aggregations type FlatKillmail struct { // Core killmail data KillmailID int64 `ch:"killmail_id"` KillmailTime time.Time `ch:"killmail_time"` SolarSystemID int64 `ch:"solar_system_id"` KillmailHash string `ch:"killmail_hash"` // Victim data (flattened) VictimShipTypeID int64 `ch:"victim_ship_type_id"` VictimCharacterID int64 `ch:"victim_character_id"` VictimCorporationID int64 `ch:"victim_corporation_id"` VictimAllianceID int64 `ch:"victim_alliance_id"` VictimDamageTaken int64 `ch:"victim_damage_taken"` // Victim position VictimPosX float64 `ch:"victim_pos_x"` VictimPosY float64 `ch:"victim_pos_y"` VictimPosZ float64 `ch:"victim_pos_z"` // Attacker summary stats AttackerCount uint16 `ch:"attacker_count"` TotalDamageDone int64 `ch:"total_damage_done"` FinalBlowShipType int64 `ch:"final_blow_ship_type"` // Attackers as array (for when you need details) - stored as [][]interface{} for ClickHouse Attackers [][]interface{} `ch:"attackers"` // Items as array (for when you need the full fit) - stored as [][]interface{} for ClickHouse Items [][]interface{} `ch:"items"` } // AttackerFlat - Nested in array column type AttackerFlat struct { CharacterID int64 `ch:"character_id"` CorporationID int64 `ch:"corporation_id"` AllianceID int64 `ch:"alliance_id"` ShipTypeID int64 `ch:"ship_type_id"` WeaponTypeID int64 `ch:"weapon_type_id"` DamageDone int64 `ch:"damage_done"` FinalBlow uint8 `ch:"final_blow"` // ClickHouse uses uint8 for bool SecurityStatus float64 `ch:"security_status"` } type ItemFlat struct { Flag int64 `ch:"flag"` ItemTypeID int64 `ch:"item_type_id"` QuantityDestroyed int64 `ch:"quantity_destroyed"` // Default to 0 instead of nullable QuantityDropped int64 `ch:"quantity_dropped"` // Default to 0 instead of nullable Singleton int64 `ch:"singleton"` } // FittedModule - Separate table optimized for module co-occurrence queries // This is your secret weapon for "what else was fitted with X" type FittedModule struct { KillmailID int64 `ch:"killmail_id"` KillmailTime time.Time `ch:"killmail_time"` SolarSystemID int64 `ch:"solar_system_id"` VictimShipTypeID int64 `ch:"victim_ship_type_id"` ItemTypeID int64 `ch:"item_type_id"` Flag int64 `ch:"flag"` // Slot type QuantityDestroyed int64 `ch:"quantity_destroyed"` QuantityDropped int64 `ch:"quantity_dropped"` } // =============================================== // CONVERSION FUNCTIONS // =============================================== // FlattenKillmail converts the nested JSON structure to ClickHouse format func (k *Killmail) FlattenKillmail() *FlatKillmail { flat := &FlatKillmail{ KillmailID: k.KillmailID, KillmailTime: k.KillmailTime, SolarSystemID: k.SolarSystemID, KillmailHash: k.KillmailHash, VictimShipTypeID: k.Victim.ShipTypeID, VictimCharacterID: k.Victim.CharacterID, VictimCorporationID: k.Victim.CorporationID, VictimAllianceID: k.Victim.AllianceID, VictimDamageTaken: k.Victim.DamageTaken, VictimPosX: k.Victim.Position.X, VictimPosY: k.Victim.Position.Y, VictimPosZ: k.Victim.Position.Z, AttackerCount: uint16(len(k.Attackers)), } // Convert attackers to slice of slices for ClickHouse Array(Tuple(...)) flat.Attackers = make([][]interface{}, len(k.Attackers)) for i, a := range k.Attackers { flat.Attackers[i] = []interface{}{ a.CharacterID, a.CorporationID, a.AllianceID, a.ShipTypeID, a.WeaponTypeID, a.DamageDone, boolToUint8(a.FinalBlow), a.SecurityStatus, } flat.TotalDamageDone += a.DamageDone if a.FinalBlow { flat.FinalBlowShipType = a.ShipTypeID } } // Convert items to slice of slices flat.Items = make([][]interface{}, len(k.Victim.Items)) for i, item := range k.Victim.Items { flat.Items[i] = []interface{}{ item.Flag, item.ItemTypeID, derefInt64(item.QuantityDestroyed), derefInt64(item.QuantityDropped), item.Singleton, } } return flat } // ExtractFittedModules creates the denormalized module records func (k *Killmail) ExtractFittedModules() []FittedModule { modules := make([]FittedModule, 0, len(k.Victim.Items)) for _, item := range k.Victim.Items { modules = append(modules, FittedModule{ KillmailID: k.KillmailID, KillmailTime: k.KillmailTime, SolarSystemID: k.SolarSystemID, VictimShipTypeID: k.Victim.ShipTypeID, ItemTypeID: item.ItemTypeID, Flag: item.Flag, QuantityDestroyed: derefInt64(item.QuantityDestroyed), QuantityDropped: derefInt64(item.QuantityDropped), }) } return modules } // Helper functions func boolToUint8(b bool) uint8 { if b { return 1 } return 0 } func derefInt64(ptr *int64) int64 { if ptr == nil { return 0 } return *ptr }