From 810eaad04a73449f471662da30a57fbe77eb95a1 Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Tue, 6 Jan 2026 16:17:14 +0100 Subject: [PATCH] Rework the main query --- db.go | 397 +++++++++++++++++++++++++++++++++------------------------- 1 file changed, 229 insertions(+), 168 deletions(-) diff --git a/db.go b/db.go index c189596..028fd80 100644 --- a/db.go +++ b/db.go @@ -8,6 +8,7 @@ import ( "zkillsusser/models" + "git.site.quack-lab.dev/dave/cylogger" logger "git.site.quack-lab.dev/dave/cylogger" "github.com/ClickHouse/clickhouse-go/v2" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" @@ -167,204 +168,160 @@ func (db *DBWrapper) Init() error { } func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { + ctx := context.Background() flog := logger.Default.WithPrefix("QueryFits").WithPrefix(fmt.Sprintf("%+v", params)) flog.Info("Starting query") + // Expand groups into item type IDs newItemTypes, err := db.ExpandGroupsIntoItemTypeIds(params.Groups) if err != nil { flog.Error("Failed to expand groups: %v", err) return nil, err } params.Modules = append(params.Modules, newItemTypes...) - modules := deduplicateInt64(params.Modules) flog.Debug("Deduplicated modules: %d -> %d", len(params.Modules), len(modules)) - // ctx := context.Background() + // Build the base query - start with all killmails + baseQuery := ` + SELECT + fk.killmail_id, + fk.solar_system_id, + fk.victim_ship_type_id + FROM flat_killmails fk` - // var killmailIDs []int64 - // var systemIDs []int64 - // var shipTypeIDsFromResults []int64 + args := []interface{}{} + whereClauses := []string{} - // moduleFilterIDs := deduplicateInt64(append(modules, groupModuleTypeIDs...)) - // if len(moduleFilterIDs) > 0 { - // modules = moduleFilterIDs - // placeholders := make([]string, len(moduleFilterIDs)) - // moduleArgs := make([]interface{}, len(moduleFilterIDs)) - // for i, moduleID := range moduleFilterIDs { - // placeholders[i] = "?" - // moduleArgs[i] = moduleID - // } + // Apply filters + if params.Ship > 0 { + whereClauses = append(whereClauses, "fk.victim_ship_type_id = ?") + args = append(args, params.Ship) + } - // var shipPlaceholders []string - // var shipArgs []interface{} - // if len(shipTypeIDs) > 0 { - // shipPlaceholders = make([]string, len(shipTypeIDs)) - // for i, shipID := range shipTypeIDs { - // shipPlaceholders[i] = "?" - // shipArgs = append(shipArgs, shipID) - // } - // } else if !isEmpty { - // shipPlaceholders = []string{"?"} - // shipArgs = []interface{}{params.Ship} - // } + if len(params.Systems) > 0 { + placeholders := make([]string, len(params.Systems)) + for i := range params.Systems { + placeholders[i] = "?" + args = append(args, params.Systems[i]) + } + whereClauses = append(whereClauses, "fk.solar_system_id IN ("+strings.Join(placeholders, ",")+")") + } - // var moduleQuery string - // var args []interface{} - // if len(shipPlaceholders) > 0 { - // moduleQuery = "SELECT DISTINCT killmail_id, solar_system_id, victim_ship_type_id FROM fitted_modules WHERE victim_ship_type_id IN (" + strings.Join(shipPlaceholders, ",") + ") AND item_type_id IN (" + strings.Join(placeholders, ",") + ")" - // args = shipArgs - // args = append(args, moduleArgs...) - // } else { - // moduleQuery = "SELECT DISTINCT killmail_id, solar_system_id, victim_ship_type_id FROM fitted_modules WHERE item_type_id IN (" + strings.Join(placeholders, ",") + ")" - // args = moduleArgs - // } + // For module filters, we need to join with fitted_modules + var moduleJoin string + if len(modules) > 0 { + placeholders := make([]string, len(modules)) + for i := range modules { + placeholders[i] = "?" + args = append(args, modules[i]) + } + moduleJoin = ` + INNER JOIN fitted_modules fm ON fk.killmail_id = fm.killmail_id` + whereClauses = append(whereClauses, "fm.item_type_id IN ("+strings.Join(placeholders, ",")+")") + } - // if len(params.Systems) > 0 { - // sysPlaceholders := make([]string, len(params.Systems)) - // for i := range params.Systems { - // sysPlaceholders[i] = "?" - // args = append(args, params.Systems[i]) - // } - // moduleQuery += " AND solar_system_id IN (" + strings.Join(sysPlaceholders, ",") + ")" - // } + // Build final query + query := baseQuery + moduleJoin + if len(whereClauses) > 0 { + query += " WHERE " + strings.Join(whereClauses, " AND ") + } - // rows, err := db.ch.Query(ctx, moduleQuery, args...) - // if err != nil { - // flog.Error("Failed to query filtered killmails: %v", err) - // return nil, err - // } - // for rows.Next() { - // var id, systemID, shipTypeID int64 - // if err := rows.Scan(&id, &systemID, &shipTypeID); err != nil { - // rows.Close() - // return nil, err - // } - // killmailIDs = append(killmailIDs, id) - // systemIDs = append(systemIDs, systemID) - // shipTypeIDsFromResults = append(shipTypeIDsFromResults, shipTypeID) - // } - // rows.Close() - // } else { - // // No module filter - query flat_killmails directly - // var query string - // var args []interface{} - // if len(shipTypeIDs) > 0 { - // shipPlaceholders := make([]string, len(shipTypeIDs)) - // for i, shipID := range shipTypeIDs { - // shipPlaceholders[i] = "?" - // args = append(args, shipID) - // } - // query = "SELECT killmail_id, solar_system_id, victim_ship_type_id FROM flat_killmails WHERE victim_ship_type_id IN (" + strings.Join(shipPlaceholders, ",") + ")" - // } else if !isEmpty { - // query = "SELECT killmail_id, solar_system_id, victim_ship_type_id FROM flat_killmails WHERE victim_ship_type_id = ?" - // args = []interface{}{params.Ship} - // } else { - // query = "SELECT killmail_id, solar_system_id, victim_ship_type_id FROM flat_killmails" - // } + flog.Debug("Executing query: %s", query) + rows, err := db.ch.Query(ctx, query, args...) + if err != nil { + flog.Error("Failed to execute query: %v", err) + return nil, err + } + defer rows.Close() - // if len(params.Systems) > 0 { - // placeholders := make([]string, len(params.Systems)) - // for i := range params.Systems { - // placeholders[i] = "?" - // args = append(args, params.Systems[i]) - // } - // if strings.Contains(query, "WHERE") { - // query += " AND solar_system_id IN (" + strings.Join(placeholders, ",") + ")" - // } else { - // query += " WHERE solar_system_id IN (" + strings.Join(placeholders, ",") + ")" - // } - // } + // Collect results + var killmailIDs []int64 + var systemIDs []int64 + var shipTypeIDs []int64 - // rows, err := db.ch.Query(ctx, query, args...) - // if err != nil { - // flog.Error("Failed to execute query: %v", err) - // return nil, err - // } - // defer rows.Close() + for rows.Next() { + var killmailID, systemID, shipTypeID int64 + if err := rows.Scan(&killmailID, &systemID, &shipTypeID); err != nil { + flog.Error("Failed to scan row: %v", err) + return nil, err + } + killmailIDs = append(killmailIDs, killmailID) + systemIDs = append(systemIDs, systemID) + shipTypeIDs = append(shipTypeIDs, shipTypeID) + } - // for rows.Next() { - // var killmailID, systemID, shipTypeID int64 - // if err := rows.Scan(&killmailID, &systemID, &shipTypeID); err != nil { - // flog.Error("Failed to scan row: %v", err) - // return nil, err - // } - // killmailIDs = append(killmailIDs, killmailID) - // systemIDs = append(systemIDs, systemID) - // shipTypeIDsFromResults = append(shipTypeIDsFromResults, shipTypeID) - // } - // } + totalKillmails := int64(len(killmailIDs)) + flog.Info("Found %d killmails after filtering", totalKillmails) - // totalKillmails := int64(len(killmailIDs)) - // flog.Info("Found %d killmails after filtering", totalKillmails) - // if totalKillmails > 0 { - // flog.Debug("Sample killmail IDs: %v", killmailIDs[:min(5, len(killmailIDs))]) - // } + if totalKillmails == 0 { + return &FitStatistics{ + TotalKillmails: 0, + ShipBreakdown: make(map[int64]Stats), + SystemBreakdown: make(map[int64]Stats), + HighSlotModules: make(map[int32]Stats), + MidSlotModules: make(map[int32]Stats), + LowSlotModules: make(map[int32]Stats), + Rigs: make(map[int32]Stats), + Drones: make(map[int32]Stats), + KillmailIDs: []int64{}, + }, nil + } - // stats := &FitStatistics{ - // TotalKillmails: totalKillmails, - // ShipBreakdown: make(map[int64]Stats), - // SystemBreakdown: make(map[int64]Stats), - // HighSlotModules: make(map[int32]Stats), - // MidSlotModules: make(map[int32]Stats), - // LowSlotModules: make(map[int32]Stats), - // Rigs: make(map[int32]Stats), - // Drones: make(map[int32]Stats), - // KillmailIDs: limitKillmails(killmailIDs, params.KillmailLimit), - // } + // Calculate ship breakdown + shipCounts := make(map[int64]int64) + for _, shipTypeID := range shipTypeIDs { + shipCounts[shipTypeID]++ + } - // if totalKillmails == 0 { - // flog.Info("No killmails found, returning empty statistics") - // return stats, nil - // } + shipBreakdown := make(map[int64]Stats) + for shipTypeID, count := range shipCounts { + percentage := float64(count) / float64(totalKillmails) * 100.0 + shipBreakdown[shipTypeID] = Stats{ + Count: count, + Percentage: percentage, + } + } - // // Calculate ship breakdown if params are empty or we have ship data - // if isEmpty || len(shipTypeIDsFromResults) > 0 { - // flog.Debug("Calculating ship breakdown") - // shipCounts := make(map[int64]int64) - // for _, shipTypeID := range shipTypeIDsFromResults { - // shipCounts[shipTypeID]++ - // } + // Calculate system breakdown + systemCounts := make(map[int64]int64) + for _, systemID := range systemIDs { + systemCounts[systemID]++ + } - // for shipTypeID, count := range shipCounts { - // percentage := float64(count) / float64(totalKillmails) * 100.0 - // stats.ShipBreakdown[shipTypeID] = Stats{ - // Count: count, - // Percentage: percentage, - // } - // } - // flog.Debug("Ship breakdown: %d unique ships", len(stats.ShipBreakdown)) - // } + systemBreakdown := make(map[int64]Stats) + for systemID, count := range systemCounts { + percentage := float64(count) / float64(totalKillmails) * 100.0 + systemBreakdown[systemID] = Stats{ + Count: count, + Percentage: percentage, + } + } - // flog.Debug("Calculating system breakdown") - // systemCounts := make(map[int64]int64) - // for _, systemID := range systemIDs { - // systemCounts[systemID]++ - // } + // Calculate module statistics + stats := &FitStatistics{ + TotalKillmails: totalKillmails, + ShipBreakdown: shipBreakdown, + SystemBreakdown: systemBreakdown, + HighSlotModules: make(map[int32]Stats), + MidSlotModules: make(map[int32]Stats), + LowSlotModules: make(map[int32]Stats), + Rigs: make(map[int32]Stats), + Drones: make(map[int32]Stats), + KillmailIDs: limitKillmails(killmailIDs, params.KillmailLimit), + } - // // Calculate system percentages - // for systemID, count := range systemCounts { - // percentage := float64(count) / float64(totalKillmails) * 100.0 - // stats.SystemBreakdown[systemID] = Stats{ - // Count: count, - // Percentage: percentage, - // } - // } - // flog.Debug("System breakdown: %d unique systems", len(stats.SystemBreakdown)) + // Get module statistics for the filtered killmails + if err := db.calculateModuleStats(killmailIDs, stats, flog); err != nil { + flog.Error("Failed to calculate module stats: %v", err) + return nil, err + } - // flog.Debug("Calculating module statistics for %d killmails", len(killmailIDs)) + flog.Info("Statistics calculated: %d high, %d mid, %d low, %d rigs, %d drones", + len(stats.HighSlotModules), len(stats.MidSlotModules), len(stats.LowSlotModules), + len(stats.Rigs), len(stats.Drones)) - // if err := db.calculateStats(params, shipTypeIDs, killmailIDs, stats, totalKillmails, flog); err != nil { - // flog.Error("Failed to calculate module stats: %v", err) - // return nil, err - // } - - // flog.Info("Statistics calculated: %d high, %d mid, %d low, %d rigs, %d drones", - // len(stats.HighSlotModules), len(stats.MidSlotModules), len(stats.LowSlotModules), - // len(stats.Rigs), len(stats.Drones)) - - // return stats, nil - return nil, nil + return stats, nil } func (db *DBWrapper) ExpandGroupsIntoItemTypeIds(groups []int64) ([]int64, error) { @@ -640,3 +597,107 @@ func (db *DBWrapper) CacheClean() error { Where("created_at < ?", threshold). Delete(&CacheEntry{}).Error } + +func (db *DBWrapper) GetModuleSlots(moduleIDs []int64) (map[int64]ModuleSlot, error) { + if len(moduleIDs) == 0 { + return make(map[int64]ModuleSlot), nil + } + + var effects []models.DgmTypeEffect + qres := db.db.Model(&models.DgmTypeEffect{}). + Select("typeID, effectID"). + Where("typeID IN ? AND effectID IN (11, 12, 13, 2663)", moduleIDs). + Find(&effects) + if qres.Error != nil { + return nil, qres.Error + } + + result := make(map[int64]ModuleSlot) + for _, e := range effects { + var slot ModuleSlot + switch e.EffectID { + case 11: + slot = ModuleSlotLow + case 12: + slot = ModuleSlotHigh + case 13: + slot = ModuleSlotMid + case 2663: + slot = ModuleSlotRig + } + result[int64(e.TypeID)] = slot + } + + return result, nil +} + +func limitKillmails(killmailIDs []int64, limit int) []int64 { + if limit <= 0 || len(killmailIDs) <= limit { + return killmailIDs + } + return killmailIDs[:limit] +} + +func (db *DBWrapper) calculateModuleStats(killmailIDs []int64, stats *FitStatistics, flog *cylogger.Logger) error { + if len(killmailIDs) == 0 { + return nil + } + + ctx := context.Background() + + // Create placeholders for killmail IDs + placeholders := make([]string, len(killmailIDs)) + args := make([]interface{}, len(killmailIDs)) + for i, id := range killmailIDs { + placeholders[i] = "?" + args = append(args, id) + } + + // Query module statistics - count distinct killmails per (item_type_id, slot) combination + query := fmt.Sprintf(` + SELECT + fm.item_type_id, + fm.slot, + COUNT(DISTINCT fm.killmail_id) as count + FROM fitted_modules fm + WHERE fm.killmail_id IN (%s) + GROUP BY fm.item_type_id, fm.slot + ORDER BY count DESC`, strings.Join(placeholders, ",")) + + rows, err := db.ch.Query(ctx, query, args...) + if err != nil { + return fmt.Errorf("failed to query module stats: %w", err) + } + defer rows.Close() + + totalKillmails := float64(stats.TotalKillmails) + + for rows.Next() { + var itemTypeID int32 + var slot string + var count int64 + + if err := rows.Scan(&itemTypeID, &slot, &count); err != nil { + return fmt.Errorf("failed to scan module row: %w", err) + } + + percentage := float64(count) / totalKillmails * 100.0 + stat := Stats{Count: count, Percentage: percentage} + + // Map slot to the appropriate map + switch slot { + case "Low": + stats.LowSlotModules[itemTypeID] = stat + case "Mid": + stats.MidSlotModules[itemTypeID] = stat + case "High": + stats.HighSlotModules[itemTypeID] = stat + case "Rig": + stats.Rigs[itemTypeID] = stat + case "Drone": + stats.Drones[itemTypeID] = stat + } + } + + return nil +}