238 lines
8.5 KiB
Go
238 lines
8.5 KiB
Go
package main
|
|
|
|
import "time"
|
|
|
|
type Killmail struct {
|
|
KillmailID int64 `json:"killmail_id" gorm:"primaryKey;column:killmail_id"`
|
|
KillmailTime time.Time `json:"killmail_time" gorm:"column:killmail_time;index"`
|
|
SolarSystemID int64 `json:"solar_system_id" gorm:"column:solar_system_id;index"`
|
|
KillmailHash string `json:"killmail_hash" gorm:"column:killmail_hash"`
|
|
HTTPLastModified time.Time `json:"http_last_modified" gorm:"column:http_last_modified"`
|
|
Attackers []Attacker `json:"attackers" gorm:"foreignKey:KillmailID;references:KillmailID;constraint:OnDelete:CASCADE"`
|
|
Victim Victim `json:"victim" gorm:"foreignKey:KillmailID;references:KillmailID;constraint:OnDelete:CASCADE"`
|
|
}
|
|
|
|
func (k *Killmail) TableName() string {
|
|
return "zkill_killmails"
|
|
}
|
|
|
|
type Attacker struct {
|
|
ID int64 `gorm:"primaryKey;autoIncrement;column:id"`
|
|
KillmailID int64 `json:"killmail_id" gorm:"column:killmail_id;index"`
|
|
AllianceID int64 `json:"alliance_id" gorm:"column:alliance_id"`
|
|
CharacterID int64 `json:"character_id" gorm:"column:character_id"`
|
|
CorporationID int64 `json:"corporation_id" gorm:"column:corporation_id"`
|
|
DamageDone int64 `json:"damage_done" gorm:"column:damage_done"`
|
|
FinalBlow bool `json:"final_blow" gorm:"column:final_blow"`
|
|
SecurityStatus float64 `json:"security_status" gorm:"column:security_status"`
|
|
ShipTypeID int64 `json:"ship_type_id" gorm:"column:ship_type_id"`
|
|
WeaponTypeID int64 `json:"weapon_type_id" gorm:"column:weapon_type_id"`
|
|
}
|
|
|
|
func (a *Attacker) TableName() string {
|
|
return "zkill_attackers"
|
|
}
|
|
|
|
type Victim struct {
|
|
ID int64 `gorm:"primaryKey;autoIncrement;column:id"`
|
|
KillmailID int64 `json:"killmail_id" gorm:"column:killmail_id;index"`
|
|
AllianceID int64 `json:"alliance_id" gorm:"column:alliance_id"`
|
|
CharacterID int64 `json:"character_id" gorm:"column:character_id"`
|
|
CorporationID int64 `json:"corporation_id" gorm:"column:corporation_id"`
|
|
DamageTaken int64 `json:"damage_taken" gorm:"column:damage_taken"`
|
|
Items []Item `json:"items" gorm:"foreignKey:VictimID;references:ID;constraint:OnDelete:CASCADE"`
|
|
Position Position `json:"position" gorm:"foreignKey:VictimID;references:ID;constraint:OnDelete:CASCADE"`
|
|
ShipTypeID int64 `json:"ship_type_id" gorm:"column:ship_type_id;index"`
|
|
}
|
|
|
|
func (v *Victim) TableName() string {
|
|
return "zkill_victims"
|
|
}
|
|
|
|
type Item struct {
|
|
ID int64 `gorm:"primaryKey;autoIncrement;column:id"`
|
|
VictimID int64 `json:"victim_id" gorm:"column:victim_id;index:idx_items_victim_item_flag"`
|
|
Flag int64 `json:"flag" gorm:"column:flag;index:idx_items_victim_item_flag"`
|
|
ItemTypeID int64 `json:"item_type_id" gorm:"column:item_type_id;index:idx_items_victim_item_flag"`
|
|
QuantityDestroyed *int64 `json:"quantity_destroyed,omitempty" gorm:"column:quantity_destroyed"`
|
|
Singleton int64 `json:"singleton" gorm:"column:singleton"`
|
|
QuantityDropped *int64 `json:"quantity_dropped,omitempty" gorm:"column:quantity_dropped"`
|
|
}
|
|
|
|
func (i *Item) TableName() string {
|
|
return "zkill_items"
|
|
}
|
|
|
|
type Position struct {
|
|
ID int64 `gorm:"primaryKey;autoIncrement;column:id"`
|
|
VictimID int64 `json:"victim_id" gorm:"column:victim_id;index"`
|
|
X float64 `json:"x" gorm:"column:x"`
|
|
Y float64 `json:"y" gorm:"column:y"`
|
|
Z float64 `json:"z" gorm:"column:z"`
|
|
}
|
|
|
|
func (p *Position) TableName() string {
|
|
return "zkill_positions"
|
|
}
|
|
|
|
// ===============================================
|
|
// CLICKHOUSE FLATTENED STRUCTURES
|
|
// ===============================================
|
|
|
|
// FlatKillmail - Main analytical table
|
|
// Denormalized for fast aggregations
|
|
type FlatKillmail struct {
|
|
// Core killmail data
|
|
KillmailID int64 `ch:"killmail_id"`
|
|
KillmailTime time.Time `ch:"killmail_time"`
|
|
SolarSystemID int64 `ch:"solar_system_id"`
|
|
KillmailHash string `ch:"killmail_hash"`
|
|
|
|
// Victim data (flattened)
|
|
VictimShipTypeID int64 `ch:"victim_ship_type_id"`
|
|
VictimCharacterID int64 `ch:"victim_character_id"`
|
|
VictimCorporationID int64 `ch:"victim_corporation_id"`
|
|
VictimAllianceID int64 `ch:"victim_alliance_id"`
|
|
VictimDamageTaken int64 `ch:"victim_damage_taken"`
|
|
|
|
// Victim position
|
|
VictimPosX float64 `ch:"victim_pos_x"`
|
|
VictimPosY float64 `ch:"victim_pos_y"`
|
|
VictimPosZ float64 `ch:"victim_pos_z"`
|
|
|
|
// Attacker summary stats
|
|
AttackerCount uint16 `ch:"attacker_count"`
|
|
TotalDamageDone int64 `ch:"total_damage_done"`
|
|
FinalBlowShipType int64 `ch:"final_blow_ship_type"`
|
|
|
|
// Attackers as array (for when you need details) - stored as [][]interface{} for ClickHouse
|
|
Attackers [][]interface{} `ch:"attackers"`
|
|
|
|
// Items as array (for when you need the full fit) - stored as [][]interface{} for ClickHouse
|
|
Items [][]interface{} `ch:"items"`
|
|
}
|
|
|
|
// AttackerFlat - Nested in array column
|
|
type AttackerFlat struct {
|
|
CharacterID int64 `ch:"character_id"`
|
|
CorporationID int64 `ch:"corporation_id"`
|
|
AllianceID int64 `ch:"alliance_id"`
|
|
ShipTypeID int64 `ch:"ship_type_id"`
|
|
WeaponTypeID int64 `ch:"weapon_type_id"`
|
|
DamageDone int64 `ch:"damage_done"`
|
|
FinalBlow uint8 `ch:"final_blow"` // ClickHouse uses uint8 for bool
|
|
SecurityStatus float64 `ch:"security_status"`
|
|
}
|
|
|
|
type ItemFlat struct {
|
|
Flag int64 `ch:"flag"`
|
|
ItemTypeID int64 `ch:"item_type_id"`
|
|
QuantityDestroyed int64 `ch:"quantity_destroyed"` // Default to 0 instead of nullable
|
|
QuantityDropped int64 `ch:"quantity_dropped"` // Default to 0 instead of nullable
|
|
Singleton int64 `ch:"singleton"`
|
|
}
|
|
|
|
// FittedModule - Separate table optimized for module co-occurrence queries
|
|
// This is your secret weapon for "what else was fitted with X"
|
|
type FittedModule struct {
|
|
KillmailID int64 `ch:"killmail_id"`
|
|
KillmailTime time.Time `ch:"killmail_time"`
|
|
SolarSystemID int64 `ch:"solar_system_id"`
|
|
VictimShipTypeID int64 `ch:"victim_ship_type_id"`
|
|
ItemTypeID int64 `ch:"item_type_id"`
|
|
Flag int64 `ch:"flag"` // Slot type
|
|
QuantityDestroyed int64 `ch:"quantity_destroyed"`
|
|
QuantityDropped int64 `ch:"quantity_dropped"`
|
|
}
|
|
|
|
// ===============================================
|
|
// CONVERSION FUNCTIONS
|
|
// ===============================================
|
|
|
|
// FlattenKillmail converts the nested JSON structure to ClickHouse format
|
|
func (k *Killmail) FlattenKillmail() *FlatKillmail {
|
|
flat := &FlatKillmail{
|
|
KillmailID: k.KillmailID,
|
|
KillmailTime: k.KillmailTime,
|
|
SolarSystemID: k.SolarSystemID,
|
|
KillmailHash: k.KillmailHash,
|
|
VictimShipTypeID: k.Victim.ShipTypeID,
|
|
VictimCharacterID: k.Victim.CharacterID,
|
|
VictimCorporationID: k.Victim.CorporationID,
|
|
VictimAllianceID: k.Victim.AllianceID,
|
|
VictimDamageTaken: k.Victim.DamageTaken,
|
|
VictimPosX: k.Victim.Position.X,
|
|
VictimPosY: k.Victim.Position.Y,
|
|
VictimPosZ: k.Victim.Position.Z,
|
|
AttackerCount: uint16(len(k.Attackers)),
|
|
}
|
|
|
|
// Convert attackers to slice of slices for ClickHouse Array(Tuple(...))
|
|
flat.Attackers = make([][]interface{}, len(k.Attackers))
|
|
for i, a := range k.Attackers {
|
|
flat.Attackers[i] = []interface{}{
|
|
a.CharacterID,
|
|
a.CorporationID,
|
|
a.AllianceID,
|
|
a.ShipTypeID,
|
|
a.WeaponTypeID,
|
|
a.DamageDone,
|
|
boolToUint8(a.FinalBlow),
|
|
a.SecurityStatus,
|
|
}
|
|
flat.TotalDamageDone += a.DamageDone
|
|
if a.FinalBlow {
|
|
flat.FinalBlowShipType = a.ShipTypeID
|
|
}
|
|
}
|
|
|
|
// Convert items to slice of slices
|
|
flat.Items = make([][]interface{}, len(k.Victim.Items))
|
|
for i, item := range k.Victim.Items {
|
|
flat.Items[i] = []interface{}{
|
|
item.Flag,
|
|
item.ItemTypeID,
|
|
derefInt64(item.QuantityDestroyed),
|
|
derefInt64(item.QuantityDropped),
|
|
item.Singleton,
|
|
}
|
|
}
|
|
|
|
return flat
|
|
}
|
|
|
|
// ExtractFittedModules creates the denormalized module records
|
|
func (k *Killmail) ExtractFittedModules() []FittedModule {
|
|
modules := make([]FittedModule, 0, len(k.Victim.Items))
|
|
|
|
for _, item := range k.Victim.Items {
|
|
modules = append(modules, FittedModule{
|
|
KillmailID: k.KillmailID,
|
|
KillmailTime: k.KillmailTime,
|
|
SolarSystemID: k.SolarSystemID,
|
|
VictimShipTypeID: k.Victim.ShipTypeID,
|
|
ItemTypeID: item.ItemTypeID,
|
|
Flag: item.Flag,
|
|
QuantityDestroyed: derefInt64(item.QuantityDestroyed),
|
|
QuantityDropped: derefInt64(item.QuantityDropped),
|
|
})
|
|
}
|
|
|
|
return modules
|
|
}
|
|
|
|
// Helper functions
|
|
func boolToUint8(b bool) uint8 {
|
|
if b {
|
|
return 1
|
|
}
|
|
return 0
|
|
}
|
|
|
|
func derefInt64(ptr *int64) int64 {
|
|
if ptr == nil {
|
|
return 0
|
|
}
|
|
return *ptr
|
|
}
|