diff --git a/db.go b/db.go index 547cfe6..1cb3757 100644 --- a/db.go +++ b/db.go @@ -322,89 +322,74 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { ctx := context.Background() - // Build ClickHouse query - query := "SELECT killmail_id, solar_system_id FROM flat_killmails WHERE victim_ship_type_id = ?" - args := []interface{}{params.Ship} - - flog.Debug("Checking total killmails for ship type %d", params.Ship) - var totalCount uint64 - countQuery := "SELECT count() FROM flat_killmails WHERE victim_ship_type_id = ?" - if err := db.ch.QueryRow(ctx, countQuery, params.Ship).Scan(&totalCount); err != nil { - flog.Error("Failed to count total killmails: %v", err) - } else { - flog.Info("Total killmails for ship type %d: %d", params.Ship, totalCount) - } - flog.Debug("Base query: victim_ship_type_id = %d", params.Ship) - - if len(params.Systems) > 0 { - // Build IN clause with placeholders - placeholders := make([]string, len(params.Systems)) - for i := range params.Systems { - placeholders[i] = "?" - args = append(args, params.Systems[i]) - } - query += " AND solar_system_id IN (" + fmt.Sprintf("%s", placeholders) + ")" - flog.Debug("Added system filter: %d systems", len(params.Systems)) - } - - if len(modules) > 0 { - flog.Debug("Looking up module slots for %d modules", len(modules)) - moduleSlots, err := db.getModuleSlots(modules) - if err != nil { - flog.Error("Failed to get module slots: %v", err) - return nil, err - } - flog.Debug("Found slots for %d modules", len(moduleSlots)) - - for _, moduleID := range modules { - slot, exists := moduleSlots[moduleID] - if !exists { - flog.Debug("Module %d has no slot, skipping", moduleID) - continue - } - - var flagMin, flagMax int64 - switch slot { - case "low": - flagMin, flagMax = 11, 18 - case "mid": - flagMin, flagMax = 19, 26 - case "high": - flagMin, flagMax = 27, 34 - case "rig": - flagMin, flagMax = 92, 99 - case "drone": - flagMin, flagMax = 87, 87 - default: - flog.Debug("Unknown slot type %s for module %d", slot, moduleID) - continue - } - - query += " AND killmail_id IN (SELECT killmail_id FROM fitted_modules WHERE item_type_id = ? AND flag BETWEEN ? AND ?)" - args = append(args, moduleID, flagMin, flagMax) - flog.Debug("Added module filter: module %d in %s slot (flags %d-%d)", moduleID, slot, flagMin, flagMax) - } - } - var killmailIDs []int64 var systemIDs []int64 - flog.Debug("Executing filtered query to get killmail IDs") - rows, err := db.ch.Query(ctx, query, args...) - if err != nil { - flog.Error("Failed to execute query: %v", err) - return nil, err - } - defer rows.Close() + if len(modules) > 0 { + placeholders := make([]string, len(modules)) + moduleArgs := make([]interface{}, len(modules)) + for i, moduleID := range modules { + placeholders[i] = "?" + moduleArgs[i] = moduleID + } + moduleQuery := "SELECT DISTINCT killmail_id, solar_system_id FROM fitted_modules WHERE victim_ship_type_id = ? AND item_type_id IN (" + strings.Join(placeholders, ",") + ")" + args := []interface{}{params.Ship} + args = append(args, moduleArgs...) - for rows.Next() { - var killmailID, systemID int64 - if err := rows.Scan(&killmailID, &systemID); err != nil { - flog.Error("Failed to scan row: %v", err) + if len(params.Systems) > 0 { + sysPlaceholders := make([]string, len(params.Systems)) + for i := range params.Systems { + sysPlaceholders[i] = "?" + args = append(args, params.Systems[i]) + } + moduleQuery += " AND solar_system_id IN (" + strings.Join(sysPlaceholders, ",") + ")" + } + + rows, err := db.ch.Query(ctx, moduleQuery, args...) + if err != nil { + flog.Error("Failed to query filtered killmails: %v", err) return nil, err } - killmailIDs = append(killmailIDs, killmailID) - systemIDs = append(systemIDs, systemID) + for rows.Next() { + var id, systemID int64 + if err := rows.Scan(&id, &systemID); err != nil { + rows.Close() + return nil, err + } + killmailIDs = append(killmailIDs, id) + systemIDs = append(systemIDs, systemID) + } + rows.Close() + } else { + // No module filter - query flat_killmails directly + query := "SELECT killmail_id, solar_system_id FROM flat_killmails WHERE victim_ship_type_id = ?" + args := []interface{}{params.Ship} + + if len(params.Systems) > 0 { + placeholders := make([]string, len(params.Systems)) + for i := range params.Systems { + placeholders[i] = "?" + args = append(args, params.Systems[i]) + } + query += " AND solar_system_id IN (" + strings.Join(placeholders, ",") + ")" + } + + rows, err := db.ch.Query(ctx, query, args...) + if err != nil { + flog.Error("Failed to execute query: %v", err) + return nil, err + } + defer rows.Close() + + for rows.Next() { + var killmailID, systemID int64 + if err := rows.Scan(&killmailID, &systemID); err != nil { + flog.Error("Failed to scan row: %v", err) + return nil, err + } + killmailIDs = append(killmailIDs, killmailID) + systemIDs = append(systemIDs, systemID) + } } totalKillmails := int64(len(killmailIDs)) @@ -446,7 +431,7 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { flog.Debug("Calculating module statistics for %d killmails", len(killmailIDs)) - if err := db.calculateModuleStats(params.Ship, params.Systems, stats, totalKillmails, flog); err != nil { + if err := db.calculateModuleStats(params, killmailIDs, stats, totalKillmails, flog); err != nil { flog.Error("Failed to calculate module stats: %v", err) return nil, err } @@ -535,25 +520,22 @@ func (db *DBWrapper) getModuleSlots(moduleIDs []int64) (map[int64]string, error) return result, nil } -func (db *DBWrapper) calculateModuleStats(shipTypeID int64, systemIDs []int64, stats *FitStatistics, total int64, flog *logger.Logger) error { - flog.Debug("Querying module stats for ship type %d", shipTypeID) +func (db *DBWrapper) calculateModuleStats(params QueryParams, killmailIDs []int64, stats *FitStatistics, total int64, flog *logger.Logger) error { + if len(killmailIDs) == 0 { + return nil + } ctx := context.Background() - // Query fitted_modules directly with ship filter - avoids huge IN clause - query := "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM fitted_modules WHERE victim_ship_type_id = ?" - args := []interface{}{shipTypeID} - - if len(systemIDs) > 0 { - placeholders := make([]string, len(systemIDs)) - for i := range systemIDs { - placeholders[i] = "?" - args = append(args, systemIDs[i]) - } - query += " AND solar_system_id IN (" + strings.Join(placeholders, ",") + ")" + placeholders := make([]string, len(killmailIDs)) + args := make([]interface{}, len(killmailIDs)) + for i, id := range killmailIDs { + placeholders[i] = "?" + args[i] = id } - query += " GROUP BY item_type_id, flag" + // Count fits (killmails) that have each module, grouped by item_type_id and flag to determine slot + query := "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM fitted_modules WHERE killmail_id IN (" + strings.Join(placeholders, ",") + ") GROUP BY item_type_id, flag" rows, err := db.ch.Query(ctx, query, args...) if err != nil { @@ -562,52 +544,84 @@ func (db *DBWrapper) calculateModuleStats(shipTypeID int64, systemIDs []int64, s } defer rows.Close() - var items []struct { - ItemTypeID int64 - Flag int64 - Count uint64 - } + // Map to aggregate counts per item_type_id (not per flag) + itemCounts := make(map[int64]uint64) + itemFlags := make(map[int64]int64) for rows.Next() { - var item struct { - ItemTypeID int64 - Flag int64 - Count uint64 - } - if err := rows.Scan(&item.ItemTypeID, &item.Flag, &item.Count); err != nil { + var itemTypeID, flag int64 + var count uint64 + if err := rows.Scan(&itemTypeID, &flag, &count); err != nil { flog.Error("Failed to scan module stat: %v", err) return err } - items = append(items, item) + // Only count fitted modules - ignore cargo (flag 5) and other non-module flags + if flag < 11 || (flag > 34 && flag != 87 && (flag < 92 || flag > 99)) { + continue + } + // Aggregate: if we've seen this item_type_id before, use the max count (should be same, but just in case) + if existing, exists := itemCounts[itemTypeID]; !exists || count > existing { + itemCounts[itemTypeID] = count + itemFlags[itemTypeID] = flag + } } - flog.Debug("Found %d item type/flag combinations", len(items)) + // For filtered modules, they should be in 100% of fits - ADD THEM FIRST + filteredModules := make(map[int64]bool) + moduleSlots := make(map[int64]string) + if len(params.Modules) > 0 { + slots, err := db.getModuleSlots(params.Modules) + if err == nil { + moduleSlots = slots + } + for _, moduleID := range params.Modules { + filteredModules[moduleID] = true + // Add filtered modules immediately with 100% + moduleStats := ModuleStats{ + Count: total, + Percentage: 100.0, + } + slot, _ := moduleSlots[moduleID] + switch slot { + case "low": + stats.LowSlotModules[int32(moduleID)] = moduleStats + case "mid": + stats.MidSlotModules[int32(moduleID)] = moduleStats + case "high": + stats.HighSlotModules[int32(moduleID)] = moduleStats + case "rig": + stats.Rigs[int32(moduleID)] = moduleStats + case "drone": + stats.Drones[int32(moduleID)] = moduleStats + default: + stats.HighSlotModules[int32(moduleID)] = moduleStats + } + } + } - for _, item := range items { - percentage := float64(item.Count) / float64(total) * 100.0 + // Add all other modules from query results + for itemTypeID, count := range itemCounts { + if filteredModules[itemTypeID] { + continue // Already added above + } + percentage := float64(count) / float64(total) * 100.0 moduleStats := ModuleStats{ - Count: int64(item.Count), + Count: int64(count), Percentage: percentage, } + flag := itemFlags[itemTypeID] switch { - case item.Flag >= 11 && item.Flag <= 18: - stats.LowSlotModules[int32(item.ItemTypeID)] = moduleStats - flog.Trace("Low slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage) - case item.Flag >= 19 && item.Flag <= 26: - stats.MidSlotModules[int32(item.ItemTypeID)] = moduleStats - flog.Trace("Mid slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage) - case item.Flag >= 27 && item.Flag <= 34: - stats.HighSlotModules[int32(item.ItemTypeID)] = moduleStats - flog.Trace("High slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage) - case item.Flag >= 92 && item.Flag <= 99: - stats.Rigs[int32(item.ItemTypeID)] = moduleStats - flog.Trace("Rig %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage) - case item.Flag == 87: - stats.Drones[int32(item.ItemTypeID)] = moduleStats - flog.Trace("Drone %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage) - default: - flog.Trace("Ignoring item %d with flag %d (not a module slot)", item.ItemTypeID, item.Flag) + case flag >= 11 && flag <= 18: + stats.LowSlotModules[int32(itemTypeID)] = moduleStats + case flag >= 19 && flag <= 26: + stats.MidSlotModules[int32(itemTypeID)] = moduleStats + case flag >= 27 && flag <= 34: + stats.HighSlotModules[int32(itemTypeID)] = moduleStats + case flag >= 92 && flag <= 99: + stats.Rigs[int32(itemTypeID)] = moduleStats + case flag == 87: + stats.Drones[int32(itemTypeID)] = moduleStats } } diff --git a/main.go b/main.go index f35147c..5af7375 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ func main() { ingest := flag.Bool("ingest", false, "ingest killmails from data directory") flag.Parse() logger.InitFlag() + logger.Default = logger.Default.ToFile("zkill.log") logger.Info("Starting") db, err := GetDB() @@ -34,6 +35,7 @@ func main() { logger.Info("Querying fits") params := QueryParams{ Ship: 32872, // Algos typeID + Modules: []int64{11355, 18981}, } stats, err := db.QueryFits(params) if err != nil {