Deretardify the killmail flattening

clickhouse.go

@@ -20,37 +20,8 @@ func (k *Killmail) FlattenKillmail() (*FlatKillmail, []*FlatModule) {
 		VictimPosY:          k.Victim.Position.Y,
 		VictimPosZ:          k.Victim.Position.Z,
 		AttackerCount:       uint16(len(k.Attackers)),
-	}
-
-	// Convert attackers to slice of slices for ClickHouse Array(Tuple(...))
-	flat.Attackers = make([]FlatAttacker, len(k.Attackers))
-	for i, a := range k.Attackers {
-		flat.Attackers[i] = FlatAttacker{
-			a.CharacterID,
-			a.CorporationID,
-			a.AllianceID,
-			a.ShipTypeID,
-			a.WeaponTypeID,
-			a.DamageDone,
-			boolToUint8(a.FinalBlow),
-			a.SecurityStatus,
-		}
-		flat.TotalDamageDone += a.DamageDone
-		if a.FinalBlow {
-			flat.FinalBlowShipType = a.ShipTypeID
-		}
-	}
-
-	// Convert items to slice of slices
-	flat.Items = make([]FlatItem, len(k.Victim.Items))
-	for i, item := range k.Victim.Items {
-		flat.Items[i] = FlatItem{
-			item.Flag,
-			item.ItemTypeID,
-			derefInt64(item.QuantityDestroyed),
-			derefInt64(item.QuantityDropped),
-			item.Singleton,
-		}
+		Attackers:           k.Attackers,
+		Items:               k.Victim.Items,
 	}
 
 	var modules []*FlatModule
@@ -62,15 +33,9 @@ func (k *Killmail) FlattenKillmail() (*FlatKillmail, []*FlatModule) {
 		}
 
 		modules = append(modules, &FlatModule{
-			KillmailID:        k.KillmailID,
-			KillmailTime:      k.KillmailTime,
-			SolarSystemID:     k.SolarSystemID,
-			VictimShipTypeID:  k.Victim.ShipTypeID,
-			ItemTypeID:        item.ItemTypeID,
-			Flag:              item.Flag,
-			QuantityDestroyed: derefInt64(item.QuantityDestroyed),
-			QuantityDropped:   derefInt64(item.QuantityDropped),
-			Slot:              moduleSlot,
+			KillmailID: k.KillmailID,
+			ItemTypeID: item.ItemTypeID,
+			Slot:       moduleSlot,
 		})
 	}
 

clickhouse_test.go

@@ -1,2 +1,48 @@
 package main
 
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestFlattenKillmail(t *testing.T) {
+	t.Run("Basic example", func(t *testing.T) {
+		expectedKM := &FlatKillmail{
+			KillmailID:    123649064,
+			KillmailTime:  time.Date(2025, 1, 1, 0, 0, 2, 0, time.UTC),
+			SolarSystemID: 30003773,
+			KillmailHash:  "15ff5b893371ebe1d725257c13518e7a3396b1ab",
+			AttackerCount: 2,
+			Attackers:     DummyKM.Attackers,
+			Items:         DummyKM.Victim.Items,
+
+			VictimAllianceID:    1354830081,
+			VictimCharacterID:   2117292634,
+			VictimCorporationID: 310281086,
+			VictimDamageTaken:   1503,
+			VictimPosX:          5.475880002962112e+12,
+			VictimPosY:          -3.928311111477058e+12,
+			VictimPosZ:          -2.7851583306225946e+11,
+			VictimShipTypeID:    11198,
+		}
+		expectedModules := []*FlatModule{
+			{KillmailID: 123649064, ItemTypeID: 31165, Slot: ModuleSlotRig},
+			{KillmailID: 123649064, ItemTypeID: 5973, Slot: ModuleSlotMid},
+			{KillmailID: 123649064, ItemTypeID: 3244, Slot: ModuleSlotMid},
+			{KillmailID: 123649064, ItemTypeID: 2605, Slot: ModuleSlotLow},
+			{KillmailID: 123649064, ItemTypeID: 58972, Slot: ModuleSlotHigh},
+			{KillmailID: 123649064, ItemTypeID: 5443, Slot: ModuleSlotMid},
+			{KillmailID: 123649064, ItemTypeID: 1405, Slot: ModuleSlotLow},
+			{KillmailID: 123649064, ItemTypeID: 11370, Slot: ModuleSlotHigh},
+			{KillmailID: 123649064, ItemTypeID: 1405, Slot: ModuleSlotLow},
+			{KillmailID: 123649064, ItemTypeID: 31165, Slot: ModuleSlotRig},
+			{KillmailID: 123649064, ItemTypeID: 6001, Slot: ModuleSlotMid},
+		}
+
+		km, modules := DummyKM.FlattenKillmail()
+		assert.Equal(t, expectedKM, km)
+		assert.Equal(t, expectedModules, modules)
+	})
+}

types.go

@@ -75,41 +75,15 @@ type FlatKillmail struct {
 	TotalDamageDone   int64
 	FinalBlowShipType int64
 
-	Attackers []FlatAttacker
-	Items     []FlatItem
-}
-
-// FlatAttacker - Nested in array column
-type FlatAttacker struct {
-	CharacterID    int64
-	CorporationID  int64
-	AllianceID     int64
-	ShipTypeID     int64
-	WeaponTypeID   int64
-	DamageDone     int64
-	FinalBlow      uint8
-	SecurityStatus float64
-}
-
-type FlatItem struct {
-	Flag              int64
-	ItemTypeID        int64
-	QuantityDestroyed int64 // Default to 0 instead of nullable
-	QuantityDropped   int64 // Default to 0 instead of nullable
-	Singleton         int64
+	Attackers []Attacker
+	Items     []Item
 }
 
 // FlatModule - Separate table optimized for module co-occurrence queries
 type FlatModule struct {
-	KillmailID        int64
-	KillmailTime      time.Time
-	SolarSystemID     int64
-	VictimShipTypeID  int64
-	ItemTypeID        int64
-	Flag              int64 // Slot type
-	Slot              ModuleSlot
-	QuantityDestroyed int64
-	QuantityDropped   int64
+	KillmailID int64
+	ItemTypeID int64
+	Slot       ModuleSlot
 }
 
 // Helper functions