From b5f1d4cf3a2e812f2b7aec4c30757fe6575605e1 Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Tue, 6 Jan 2026 17:25:28 +0100 Subject: [PATCH] Rework the statistics to be calculated on the database --- db.go | 145 +++++++++++++++++++++++++++++++++++-------------------- main.go | 1 - types.go | 19 +++----- 3 files changed, 100 insertions(+), 65 deletions(-) diff --git a/db.go b/db.go index 18e385e..d9abaf7 100644 --- a/db.go +++ b/db.go @@ -257,45 +257,29 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { if totalKillmails == 0 { return &FitStatistics{ TotalKillmails: 0, - ShipBreakdown: make(map[int64]Stats), - SystemBreakdown: make(map[int64]Stats), - HighSlotModules: make(map[int32]Stats), - MidSlotModules: make(map[int32]Stats), - LowSlotModules: make(map[int32]Stats), - Rigs: make(map[int32]Stats), - Drones: make(map[int32]Stats), + ShipBreakdown: make(map[int64]int64), + SystemBreakdown: make(map[int64]int64), + HighSlotModules: make(map[int32]int64), + MidSlotModules: make(map[int32]int64), + LowSlotModules: make(map[int32]int64), + Rigs: make(map[int32]int64), + Drones: make(map[int32]int64), KillmailIDs: []int64{}, }, nil } - // Calculate ship breakdown - shipCounts := make(map[int64]int64) - for _, shipTypeID := range shipTypeIDs { - shipCounts[shipTypeID]++ + // Calculate ship breakdown from database + shipBreakdown, err := db.calculateShipBreakdown(query, args, totalKillmails, flog) + if err != nil { + flog.Error("Failed to calculate ship breakdown: %v", err) + return nil, err } - shipBreakdown := make(map[int64]Stats) - for shipTypeID, count := range shipCounts { - percentage := float64(count) / float64(totalKillmails) * 100.0 - shipBreakdown[shipTypeID] = Stats{ - Count: count, - Percentage: percentage, - } - } - - // Calculate system breakdown - systemCounts := make(map[int64]int64) - for _, systemID := range systemIDs { - systemCounts[systemID]++ - } - - systemBreakdown := make(map[int64]Stats) - for systemID, count := range systemCounts { - percentage := float64(count) / float64(totalKillmails) * 100.0 - systemBreakdown[systemID] = Stats{ - Count: count, - Percentage: percentage, - } + // Calculate system breakdown from database + systemBreakdown, err := db.calculateSystemBreakdown(query, args, totalKillmails, flog) + if err != nil { + flog.Error("Failed to calculate system breakdown: %v", err) + return nil, err } // Calculate module statistics @@ -303,11 +287,11 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { TotalKillmails: totalKillmails, ShipBreakdown: shipBreakdown, SystemBreakdown: systemBreakdown, - HighSlotModules: make(map[int32]Stats), - MidSlotModules: make(map[int32]Stats), - LowSlotModules: make(map[int32]Stats), - Rigs: make(map[int32]Stats), - Drones: make(map[int32]Stats), + HighSlotModules: make(map[int32]int64), + MidSlotModules: make(map[int32]int64), + LowSlotModules: make(map[int32]int64), + Rigs: make(map[int32]int64), + Drones: make(map[int32]int64), KillmailIDs: limitKillmails(killmailIDs, params.KillmailLimit), } @@ -638,6 +622,66 @@ func limitKillmails(killmailIDs []int64, limit int) []int64 { return killmailIDs[:limit] } +func (db *DBWrapper) calculateShipBreakdown(baseQuery string, args []interface{}, totalKillmails int64, flog *cylogger.Logger) (map[int64]int64, error) { + ctx := context.Background() + + // Modify the base query to group by ship type + query := strings.Replace(baseQuery, "SELECT\n\t\t\tfk.killmail_id,\n\t\t\tfk.solar_system_id,\n\t\t\tfk.victim_ship_type_id\n\t\tFROM killmails fk", + "SELECT\n\t\t\tfk.victim_ship_type_id,\n\t\t\tCOUNT(*) as count\n\t\tFROM killmails fk", 1) + + // Add GROUP BY + query += " GROUP BY fk.victim_ship_type_id ORDER BY count DESC" + + flog.Debug("Ship breakdown query: %s", query) + rows, err := db.ch.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("failed to execute ship breakdown query: %w", err) + } + defer rows.Close() + + shipBreakdown := make(map[int64]int64) + for rows.Next() { + var shipTypeID int64 + var count uint64 + if err := rows.Scan(&shipTypeID, &count); err != nil { + return nil, fmt.Errorf("failed to scan ship breakdown row: %w", err) + } + shipBreakdown[shipTypeID] = int64(count) + } + + return shipBreakdown, nil +} + +func (db *DBWrapper) calculateSystemBreakdown(baseQuery string, args []interface{}, totalKillmails int64, flog *cylogger.Logger) (map[int64]int64, error) { + ctx := context.Background() + + // Modify the base query to group by solar system + query := strings.Replace(baseQuery, "SELECT\n\t\t\tfk.killmail_id,\n\t\t\tfk.solar_system_id,\n\t\t\tfk.victim_ship_type_id\n\t\tFROM killmails fk", + "SELECT\n\t\t\tfk.solar_system_id,\n\t\t\tCOUNT(*) as count\n\t\tFROM killmails fk", 1) + + // Add GROUP BY + query += " GROUP BY fk.solar_system_id ORDER BY count DESC" + + flog.Debug("System breakdown query: %s", query) + rows, err := db.ch.Query(ctx, query, args...) + if err != nil { + return nil, fmt.Errorf("failed to execute system breakdown query: %w", err) + } + defer rows.Close() + + systemBreakdown := make(map[int64]int64) + for rows.Next() { + var systemID int64 + var count uint64 + if err := rows.Scan(&systemID, &count); err != nil { + return nil, fmt.Errorf("failed to scan system breakdown row: %w", err) + } + systemBreakdown[systemID] = int64(count) + } + + return systemBreakdown, nil +} + func (db *DBWrapper) calculateModuleStats(killmailIDs []int64, stats *FitStatistics, flog *cylogger.Logger) error { if len(killmailIDs) == 0 { return nil @@ -650,10 +694,10 @@ func (db *DBWrapper) calculateModuleStats(killmailIDs []int64, stats *FitStatist args := make([]interface{}, len(killmailIDs)) for i, id := range killmailIDs { placeholders[i] = "?" - args = append(args, id) + args[i] = id } - // Query module statistics - count distinct killmails per (item_type_id, slot) combination + // Single query to get all module stats grouped by slot query := fmt.Sprintf(` SELECT fm.item_type_id, @@ -664,38 +708,35 @@ func (db *DBWrapper) calculateModuleStats(killmailIDs []int64, stats *FitStatist GROUP BY fm.item_type_id, fm.slot ORDER BY count DESC`, strings.Join(placeholders, ",")) + flog.Debug("Module stats query: %s", query) rows, err := db.ch.Query(ctx, query, args...) if err != nil { return fmt.Errorf("failed to query module stats: %w", err) } defer rows.Close() - totalKillmails := float64(stats.TotalKillmails) - + // Map results to appropriate slot maps for rows.Next() { - var itemTypeID int32 + var itemTypeID int64 var slot string - var count int64 + var count uint64 if err := rows.Scan(&itemTypeID, &slot, &count); err != nil { return fmt.Errorf("failed to scan module row: %w", err) } - percentage := float64(count) / totalKillmails * 100.0 - stat := Stats{Count: count, Percentage: percentage} - // Map slot to the appropriate map switch slot { case "Low": - stats.LowSlotModules[itemTypeID] = stat + stats.LowSlotModules[int32(itemTypeID)] = int64(count) case "Mid": - stats.MidSlotModules[itemTypeID] = stat + stats.MidSlotModules[int32(itemTypeID)] = int64(count) case "High": - stats.HighSlotModules[itemTypeID] = stat + stats.HighSlotModules[int32(itemTypeID)] = int64(count) case "Rig": - stats.Rigs[itemTypeID] = stat + stats.Rigs[int32(itemTypeID)] = int64(count) case "Drone": - stats.Drones[itemTypeID] = stat + stats.Drones[int32(itemTypeID)] = int64(count) } } diff --git a/main.go b/main.go index 847fb1c..40479d2 100644 --- a/main.go +++ b/main.go @@ -38,7 +38,6 @@ func main() { logger.Info("Querying fits") params := QueryParams{ Ship: 32872, - Modules: []int64{16433}, } stats, err := db.QueryFits(params) if err != nil { diff --git a/types.go b/types.go index 47d019a..8ef17f7 100644 --- a/types.go +++ b/types.go @@ -139,17 +139,12 @@ func (CacheEntry) TableName() string { type FitStatistics struct { TotalKillmails int64 - ShipBreakdown map[int64]Stats // shipTypeID -> {Count, Percentage} - SystemBreakdown map[int64]Stats // systemID -> {Count, Percentage} - HighSlotModules map[int32]Stats // typeID -> {Count, Percentage} - MidSlotModules map[int32]Stats // typeID -> {Count, Percentage} - LowSlotModules map[int32]Stats // typeID -> {Count, Percentage} - Rigs map[int32]Stats // typeID -> {Count, Percentage} - Drones map[int32]Stats // typeID -> {Count, Percentage} + ShipBreakdown map[int64]int64 + SystemBreakdown map[int64]int64 + HighSlotModules map[int32]int64 + MidSlotModules map[int32]int64 + LowSlotModules map[int32]int64 + Rigs map[int32]int64 + Drones map[int32]int64 KillmailIDs []int64 } - -type Stats struct { - Count int64 - Percentage float64 -}