Actually make shit work
This commit is contained in:
264
db.go
264
db.go
@@ -322,89 +322,74 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) {
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Build ClickHouse query
|
||||
query := "SELECT killmail_id, solar_system_id FROM flat_killmails WHERE victim_ship_type_id = ?"
|
||||
args := []interface{}{params.Ship}
|
||||
|
||||
flog.Debug("Checking total killmails for ship type %d", params.Ship)
|
||||
var totalCount uint64
|
||||
countQuery := "SELECT count() FROM flat_killmails WHERE victim_ship_type_id = ?"
|
||||
if err := db.ch.QueryRow(ctx, countQuery, params.Ship).Scan(&totalCount); err != nil {
|
||||
flog.Error("Failed to count total killmails: %v", err)
|
||||
} else {
|
||||
flog.Info("Total killmails for ship type %d: %d", params.Ship, totalCount)
|
||||
}
|
||||
flog.Debug("Base query: victim_ship_type_id = %d", params.Ship)
|
||||
|
||||
if len(params.Systems) > 0 {
|
||||
// Build IN clause with placeholders
|
||||
placeholders := make([]string, len(params.Systems))
|
||||
for i := range params.Systems {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, params.Systems[i])
|
||||
}
|
||||
query += " AND solar_system_id IN (" + fmt.Sprintf("%s", placeholders) + ")"
|
||||
flog.Debug("Added system filter: %d systems", len(params.Systems))
|
||||
}
|
||||
|
||||
if len(modules) > 0 {
|
||||
flog.Debug("Looking up module slots for %d modules", len(modules))
|
||||
moduleSlots, err := db.getModuleSlots(modules)
|
||||
if err != nil {
|
||||
flog.Error("Failed to get module slots: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
flog.Debug("Found slots for %d modules", len(moduleSlots))
|
||||
|
||||
for _, moduleID := range modules {
|
||||
slot, exists := moduleSlots[moduleID]
|
||||
if !exists {
|
||||
flog.Debug("Module %d has no slot, skipping", moduleID)
|
||||
continue
|
||||
}
|
||||
|
||||
var flagMin, flagMax int64
|
||||
switch slot {
|
||||
case "low":
|
||||
flagMin, flagMax = 11, 18
|
||||
case "mid":
|
||||
flagMin, flagMax = 19, 26
|
||||
case "high":
|
||||
flagMin, flagMax = 27, 34
|
||||
case "rig":
|
||||
flagMin, flagMax = 92, 99
|
||||
case "drone":
|
||||
flagMin, flagMax = 87, 87
|
||||
default:
|
||||
flog.Debug("Unknown slot type %s for module %d", slot, moduleID)
|
||||
continue
|
||||
}
|
||||
|
||||
query += " AND killmail_id IN (SELECT killmail_id FROM fitted_modules WHERE item_type_id = ? AND flag BETWEEN ? AND ?)"
|
||||
args = append(args, moduleID, flagMin, flagMax)
|
||||
flog.Debug("Added module filter: module %d in %s slot (flags %d-%d)", moduleID, slot, flagMin, flagMax)
|
||||
}
|
||||
}
|
||||
|
||||
var killmailIDs []int64
|
||||
var systemIDs []int64
|
||||
|
||||
flog.Debug("Executing filtered query to get killmail IDs")
|
||||
rows, err := db.ch.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
flog.Error("Failed to execute query: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
if len(modules) > 0 {
|
||||
placeholders := make([]string, len(modules))
|
||||
moduleArgs := make([]interface{}, len(modules))
|
||||
for i, moduleID := range modules {
|
||||
placeholders[i] = "?"
|
||||
moduleArgs[i] = moduleID
|
||||
}
|
||||
moduleQuery := "SELECT DISTINCT killmail_id, solar_system_id FROM fitted_modules WHERE victim_ship_type_id = ? AND item_type_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
args := []interface{}{params.Ship}
|
||||
args = append(args, moduleArgs...)
|
||||
|
||||
for rows.Next() {
|
||||
var killmailID, systemID int64
|
||||
if err := rows.Scan(&killmailID, &systemID); err != nil {
|
||||
flog.Error("Failed to scan row: %v", err)
|
||||
if len(params.Systems) > 0 {
|
||||
sysPlaceholders := make([]string, len(params.Systems))
|
||||
for i := range params.Systems {
|
||||
sysPlaceholders[i] = "?"
|
||||
args = append(args, params.Systems[i])
|
||||
}
|
||||
moduleQuery += " AND solar_system_id IN (" + strings.Join(sysPlaceholders, ",") + ")"
|
||||
}
|
||||
|
||||
rows, err := db.ch.Query(ctx, moduleQuery, args...)
|
||||
if err != nil {
|
||||
flog.Error("Failed to query filtered killmails: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
killmailIDs = append(killmailIDs, killmailID)
|
||||
systemIDs = append(systemIDs, systemID)
|
||||
for rows.Next() {
|
||||
var id, systemID int64
|
||||
if err := rows.Scan(&id, &systemID); err != nil {
|
||||
rows.Close()
|
||||
return nil, err
|
||||
}
|
||||
killmailIDs = append(killmailIDs, id)
|
||||
systemIDs = append(systemIDs, systemID)
|
||||
}
|
||||
rows.Close()
|
||||
} else {
|
||||
// No module filter - query flat_killmails directly
|
||||
query := "SELECT killmail_id, solar_system_id FROM flat_killmails WHERE victim_ship_type_id = ?"
|
||||
args := []interface{}{params.Ship}
|
||||
|
||||
if len(params.Systems) > 0 {
|
||||
placeholders := make([]string, len(params.Systems))
|
||||
for i := range params.Systems {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, params.Systems[i])
|
||||
}
|
||||
query += " AND solar_system_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
}
|
||||
|
||||
rows, err := db.ch.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
flog.Error("Failed to execute query: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
for rows.Next() {
|
||||
var killmailID, systemID int64
|
||||
if err := rows.Scan(&killmailID, &systemID); err != nil {
|
||||
flog.Error("Failed to scan row: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
killmailIDs = append(killmailIDs, killmailID)
|
||||
systemIDs = append(systemIDs, systemID)
|
||||
}
|
||||
}
|
||||
|
||||
totalKillmails := int64(len(killmailIDs))
|
||||
@@ -446,7 +431,7 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) {
|
||||
|
||||
flog.Debug("Calculating module statistics for %d killmails", len(killmailIDs))
|
||||
|
||||
if err := db.calculateModuleStats(params.Ship, params.Systems, stats, totalKillmails, flog); err != nil {
|
||||
if err := db.calculateModuleStats(params, killmailIDs, stats, totalKillmails, flog); err != nil {
|
||||
flog.Error("Failed to calculate module stats: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
@@ -535,25 +520,22 @@ func (db *DBWrapper) getModuleSlots(moduleIDs []int64) (map[int64]string, error)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (db *DBWrapper) calculateModuleStats(shipTypeID int64, systemIDs []int64, stats *FitStatistics, total int64, flog *logger.Logger) error {
|
||||
flog.Debug("Querying module stats for ship type %d", shipTypeID)
|
||||
func (db *DBWrapper) calculateModuleStats(params QueryParams, killmailIDs []int64, stats *FitStatistics, total int64, flog *logger.Logger) error {
|
||||
if len(killmailIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Query fitted_modules directly with ship filter - avoids huge IN clause
|
||||
query := "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM fitted_modules WHERE victim_ship_type_id = ?"
|
||||
args := []interface{}{shipTypeID}
|
||||
|
||||
if len(systemIDs) > 0 {
|
||||
placeholders := make([]string, len(systemIDs))
|
||||
for i := range systemIDs {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, systemIDs[i])
|
||||
}
|
||||
query += " AND solar_system_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
placeholders := make([]string, len(killmailIDs))
|
||||
args := make([]interface{}, len(killmailIDs))
|
||||
for i, id := range killmailIDs {
|
||||
placeholders[i] = "?"
|
||||
args[i] = id
|
||||
}
|
||||
|
||||
query += " GROUP BY item_type_id, flag"
|
||||
// Count fits (killmails) that have each module, grouped by item_type_id and flag to determine slot
|
||||
query := "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM fitted_modules WHERE killmail_id IN (" + strings.Join(placeholders, ",") + ") GROUP BY item_type_id, flag"
|
||||
|
||||
rows, err := db.ch.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
@@ -562,52 +544,84 @@ func (db *DBWrapper) calculateModuleStats(shipTypeID int64, systemIDs []int64, s
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []struct {
|
||||
ItemTypeID int64
|
||||
Flag int64
|
||||
Count uint64
|
||||
}
|
||||
// Map to aggregate counts per item_type_id (not per flag)
|
||||
itemCounts := make(map[int64]uint64)
|
||||
itemFlags := make(map[int64]int64)
|
||||
|
||||
for rows.Next() {
|
||||
var item struct {
|
||||
ItemTypeID int64
|
||||
Flag int64
|
||||
Count uint64
|
||||
}
|
||||
if err := rows.Scan(&item.ItemTypeID, &item.Flag, &item.Count); err != nil {
|
||||
var itemTypeID, flag int64
|
||||
var count uint64
|
||||
if err := rows.Scan(&itemTypeID, &flag, &count); err != nil {
|
||||
flog.Error("Failed to scan module stat: %v", err)
|
||||
return err
|
||||
}
|
||||
items = append(items, item)
|
||||
// Only count fitted modules - ignore cargo (flag 5) and other non-module flags
|
||||
if flag < 11 || (flag > 34 && flag != 87 && (flag < 92 || flag > 99)) {
|
||||
continue
|
||||
}
|
||||
// Aggregate: if we've seen this item_type_id before, use the max count (should be same, but just in case)
|
||||
if existing, exists := itemCounts[itemTypeID]; !exists || count > existing {
|
||||
itemCounts[itemTypeID] = count
|
||||
itemFlags[itemTypeID] = flag
|
||||
}
|
||||
}
|
||||
|
||||
flog.Debug("Found %d item type/flag combinations", len(items))
|
||||
// For filtered modules, they should be in 100% of fits - ADD THEM FIRST
|
||||
filteredModules := make(map[int64]bool)
|
||||
moduleSlots := make(map[int64]string)
|
||||
if len(params.Modules) > 0 {
|
||||
slots, err := db.getModuleSlots(params.Modules)
|
||||
if err == nil {
|
||||
moduleSlots = slots
|
||||
}
|
||||
for _, moduleID := range params.Modules {
|
||||
filteredModules[moduleID] = true
|
||||
// Add filtered modules immediately with 100%
|
||||
moduleStats := ModuleStats{
|
||||
Count: total,
|
||||
Percentage: 100.0,
|
||||
}
|
||||
slot, _ := moduleSlots[moduleID]
|
||||
switch slot {
|
||||
case "low":
|
||||
stats.LowSlotModules[int32(moduleID)] = moduleStats
|
||||
case "mid":
|
||||
stats.MidSlotModules[int32(moduleID)] = moduleStats
|
||||
case "high":
|
||||
stats.HighSlotModules[int32(moduleID)] = moduleStats
|
||||
case "rig":
|
||||
stats.Rigs[int32(moduleID)] = moduleStats
|
||||
case "drone":
|
||||
stats.Drones[int32(moduleID)] = moduleStats
|
||||
default:
|
||||
stats.HighSlotModules[int32(moduleID)] = moduleStats
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, item := range items {
|
||||
percentage := float64(item.Count) / float64(total) * 100.0
|
||||
// Add all other modules from query results
|
||||
for itemTypeID, count := range itemCounts {
|
||||
if filteredModules[itemTypeID] {
|
||||
continue // Already added above
|
||||
}
|
||||
percentage := float64(count) / float64(total) * 100.0
|
||||
moduleStats := ModuleStats{
|
||||
Count: int64(item.Count),
|
||||
Count: int64(count),
|
||||
Percentage: percentage,
|
||||
}
|
||||
|
||||
flag := itemFlags[itemTypeID]
|
||||
switch {
|
||||
case item.Flag >= 11 && item.Flag <= 18:
|
||||
stats.LowSlotModules[int32(item.ItemTypeID)] = moduleStats
|
||||
flog.Trace("Low slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
|
||||
case item.Flag >= 19 && item.Flag <= 26:
|
||||
stats.MidSlotModules[int32(item.ItemTypeID)] = moduleStats
|
||||
flog.Trace("Mid slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
|
||||
case item.Flag >= 27 && item.Flag <= 34:
|
||||
stats.HighSlotModules[int32(item.ItemTypeID)] = moduleStats
|
||||
flog.Trace("High slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
|
||||
case item.Flag >= 92 && item.Flag <= 99:
|
||||
stats.Rigs[int32(item.ItemTypeID)] = moduleStats
|
||||
flog.Trace("Rig %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
|
||||
case item.Flag == 87:
|
||||
stats.Drones[int32(item.ItemTypeID)] = moduleStats
|
||||
flog.Trace("Drone %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
|
||||
default:
|
||||
flog.Trace("Ignoring item %d with flag %d (not a module slot)", item.ItemTypeID, item.Flag)
|
||||
case flag >= 11 && flag <= 18:
|
||||
stats.LowSlotModules[int32(itemTypeID)] = moduleStats
|
||||
case flag >= 19 && flag <= 26:
|
||||
stats.MidSlotModules[int32(itemTypeID)] = moduleStats
|
||||
case flag >= 27 && flag <= 34:
|
||||
stats.HighSlotModules[int32(itemTypeID)] = moduleStats
|
||||
case flag >= 92 && flag <= 99:
|
||||
stats.Rigs[int32(itemTypeID)] = moduleStats
|
||||
case flag == 87:
|
||||
stats.Drones[int32(itemTypeID)] = moduleStats
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
2
main.go
2
main.go
@@ -14,6 +14,7 @@ func main() {
|
||||
ingest := flag.Bool("ingest", false, "ingest killmails from data directory")
|
||||
flag.Parse()
|
||||
logger.InitFlag()
|
||||
logger.Default = logger.Default.ToFile("zkill.log")
|
||||
logger.Info("Starting")
|
||||
|
||||
db, err := GetDB()
|
||||
@@ -34,6 +35,7 @@ func main() {
|
||||
logger.Info("Querying fits")
|
||||
params := QueryParams{
|
||||
Ship: 32872, // Algos typeID
|
||||
Modules: []int64{11355, 18981},
|
||||
}
|
||||
stats, err := db.QueryFits(params)
|
||||
if err != nil {
|
||||
|
||||
Reference in New Issue
Block a user