Rework the main query
This commit is contained in:
397
db.go
397
db.go
@@ -8,6 +8,7 @@ import (
|
||||
|
||||
"zkillsusser/models"
|
||||
|
||||
"git.site.quack-lab.dev/dave/cylogger"
|
||||
logger "git.site.quack-lab.dev/dave/cylogger"
|
||||
"github.com/ClickHouse/clickhouse-go/v2"
|
||||
"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
|
||||
@@ -167,204 +168,160 @@ func (db *DBWrapper) Init() error {
|
||||
}
|
||||
|
||||
func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) {
|
||||
ctx := context.Background()
|
||||
flog := logger.Default.WithPrefix("QueryFits").WithPrefix(fmt.Sprintf("%+v", params))
|
||||
flog.Info("Starting query")
|
||||
|
||||
// Expand groups into item type IDs
|
||||
newItemTypes, err := db.ExpandGroupsIntoItemTypeIds(params.Groups)
|
||||
if err != nil {
|
||||
flog.Error("Failed to expand groups: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
params.Modules = append(params.Modules, newItemTypes...)
|
||||
|
||||
modules := deduplicateInt64(params.Modules)
|
||||
flog.Debug("Deduplicated modules: %d -> %d", len(params.Modules), len(modules))
|
||||
|
||||
// ctx := context.Background()
|
||||
// Build the base query - start with all killmails
|
||||
baseQuery := `
|
||||
SELECT
|
||||
fk.killmail_id,
|
||||
fk.solar_system_id,
|
||||
fk.victim_ship_type_id
|
||||
FROM flat_killmails fk`
|
||||
|
||||
// var killmailIDs []int64
|
||||
// var systemIDs []int64
|
||||
// var shipTypeIDsFromResults []int64
|
||||
args := []interface{}{}
|
||||
whereClauses := []string{}
|
||||
|
||||
// moduleFilterIDs := deduplicateInt64(append(modules, groupModuleTypeIDs...))
|
||||
// if len(moduleFilterIDs) > 0 {
|
||||
// modules = moduleFilterIDs
|
||||
// placeholders := make([]string, len(moduleFilterIDs))
|
||||
// moduleArgs := make([]interface{}, len(moduleFilterIDs))
|
||||
// for i, moduleID := range moduleFilterIDs {
|
||||
// placeholders[i] = "?"
|
||||
// moduleArgs[i] = moduleID
|
||||
// }
|
||||
// Apply filters
|
||||
if params.Ship > 0 {
|
||||
whereClauses = append(whereClauses, "fk.victim_ship_type_id = ?")
|
||||
args = append(args, params.Ship)
|
||||
}
|
||||
|
||||
// var shipPlaceholders []string
|
||||
// var shipArgs []interface{}
|
||||
// if len(shipTypeIDs) > 0 {
|
||||
// shipPlaceholders = make([]string, len(shipTypeIDs))
|
||||
// for i, shipID := range shipTypeIDs {
|
||||
// shipPlaceholders[i] = "?"
|
||||
// shipArgs = append(shipArgs, shipID)
|
||||
// }
|
||||
// } else if !isEmpty {
|
||||
// shipPlaceholders = []string{"?"}
|
||||
// shipArgs = []interface{}{params.Ship}
|
||||
// }
|
||||
if len(params.Systems) > 0 {
|
||||
placeholders := make([]string, len(params.Systems))
|
||||
for i := range params.Systems {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, params.Systems[i])
|
||||
}
|
||||
whereClauses = append(whereClauses, "fk.solar_system_id IN ("+strings.Join(placeholders, ",")+")")
|
||||
}
|
||||
|
||||
// var moduleQuery string
|
||||
// var args []interface{}
|
||||
// if len(shipPlaceholders) > 0 {
|
||||
// moduleQuery = "SELECT DISTINCT killmail_id, solar_system_id, victim_ship_type_id FROM fitted_modules WHERE victim_ship_type_id IN (" + strings.Join(shipPlaceholders, ",") + ") AND item_type_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
// args = shipArgs
|
||||
// args = append(args, moduleArgs...)
|
||||
// } else {
|
||||
// moduleQuery = "SELECT DISTINCT killmail_id, solar_system_id, victim_ship_type_id FROM fitted_modules WHERE item_type_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
// args = moduleArgs
|
||||
// }
|
||||
// For module filters, we need to join with fitted_modules
|
||||
var moduleJoin string
|
||||
if len(modules) > 0 {
|
||||
placeholders := make([]string, len(modules))
|
||||
for i := range modules {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, modules[i])
|
||||
}
|
||||
moduleJoin = `
|
||||
INNER JOIN fitted_modules fm ON fk.killmail_id = fm.killmail_id`
|
||||
whereClauses = append(whereClauses, "fm.item_type_id IN ("+strings.Join(placeholders, ",")+")")
|
||||
}
|
||||
|
||||
// if len(params.Systems) > 0 {
|
||||
// sysPlaceholders := make([]string, len(params.Systems))
|
||||
// for i := range params.Systems {
|
||||
// sysPlaceholders[i] = "?"
|
||||
// args = append(args, params.Systems[i])
|
||||
// }
|
||||
// moduleQuery += " AND solar_system_id IN (" + strings.Join(sysPlaceholders, ",") + ")"
|
||||
// }
|
||||
// Build final query
|
||||
query := baseQuery + moduleJoin
|
||||
if len(whereClauses) > 0 {
|
||||
query += " WHERE " + strings.Join(whereClauses, " AND ")
|
||||
}
|
||||
|
||||
// rows, err := db.ch.Query(ctx, moduleQuery, args...)
|
||||
// if err != nil {
|
||||
// flog.Error("Failed to query filtered killmails: %v", err)
|
||||
// return nil, err
|
||||
// }
|
||||
// for rows.Next() {
|
||||
// var id, systemID, shipTypeID int64
|
||||
// if err := rows.Scan(&id, &systemID, &shipTypeID); err != nil {
|
||||
// rows.Close()
|
||||
// return nil, err
|
||||
// }
|
||||
// killmailIDs = append(killmailIDs, id)
|
||||
// systemIDs = append(systemIDs, systemID)
|
||||
// shipTypeIDsFromResults = append(shipTypeIDsFromResults, shipTypeID)
|
||||
// }
|
||||
// rows.Close()
|
||||
// } else {
|
||||
// // No module filter - query flat_killmails directly
|
||||
// var query string
|
||||
// var args []interface{}
|
||||
// if len(shipTypeIDs) > 0 {
|
||||
// shipPlaceholders := make([]string, len(shipTypeIDs))
|
||||
// for i, shipID := range shipTypeIDs {
|
||||
// shipPlaceholders[i] = "?"
|
||||
// args = append(args, shipID)
|
||||
// }
|
||||
// query = "SELECT killmail_id, solar_system_id, victim_ship_type_id FROM flat_killmails WHERE victim_ship_type_id IN (" + strings.Join(shipPlaceholders, ",") + ")"
|
||||
// } else if !isEmpty {
|
||||
// query = "SELECT killmail_id, solar_system_id, victim_ship_type_id FROM flat_killmails WHERE victim_ship_type_id = ?"
|
||||
// args = []interface{}{params.Ship}
|
||||
// } else {
|
||||
// query = "SELECT killmail_id, solar_system_id, victim_ship_type_id FROM flat_killmails"
|
||||
// }
|
||||
flog.Debug("Executing query: %s", query)
|
||||
rows, err := db.ch.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
flog.Error("Failed to execute query: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
// if len(params.Systems) > 0 {
|
||||
// placeholders := make([]string, len(params.Systems))
|
||||
// for i := range params.Systems {
|
||||
// placeholders[i] = "?"
|
||||
// args = append(args, params.Systems[i])
|
||||
// }
|
||||
// if strings.Contains(query, "WHERE") {
|
||||
// query += " AND solar_system_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
// } else {
|
||||
// query += " WHERE solar_system_id IN (" + strings.Join(placeholders, ",") + ")"
|
||||
// }
|
||||
// }
|
||||
// Collect results
|
||||
var killmailIDs []int64
|
||||
var systemIDs []int64
|
||||
var shipTypeIDs []int64
|
||||
|
||||
// rows, err := db.ch.Query(ctx, query, args...)
|
||||
// if err != nil {
|
||||
// flog.Error("Failed to execute query: %v", err)
|
||||
// return nil, err
|
||||
// }
|
||||
// defer rows.Close()
|
||||
for rows.Next() {
|
||||
var killmailID, systemID, shipTypeID int64
|
||||
if err := rows.Scan(&killmailID, &systemID, &shipTypeID); err != nil {
|
||||
flog.Error("Failed to scan row: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
killmailIDs = append(killmailIDs, killmailID)
|
||||
systemIDs = append(systemIDs, systemID)
|
||||
shipTypeIDs = append(shipTypeIDs, shipTypeID)
|
||||
}
|
||||
|
||||
// for rows.Next() {
|
||||
// var killmailID, systemID, shipTypeID int64
|
||||
// if err := rows.Scan(&killmailID, &systemID, &shipTypeID); err != nil {
|
||||
// flog.Error("Failed to scan row: %v", err)
|
||||
// return nil, err
|
||||
// }
|
||||
// killmailIDs = append(killmailIDs, killmailID)
|
||||
// systemIDs = append(systemIDs, systemID)
|
||||
// shipTypeIDsFromResults = append(shipTypeIDsFromResults, shipTypeID)
|
||||
// }
|
||||
// }
|
||||
totalKillmails := int64(len(killmailIDs))
|
||||
flog.Info("Found %d killmails after filtering", totalKillmails)
|
||||
|
||||
// totalKillmails := int64(len(killmailIDs))
|
||||
// flog.Info("Found %d killmails after filtering", totalKillmails)
|
||||
// if totalKillmails > 0 {
|
||||
// flog.Debug("Sample killmail IDs: %v", killmailIDs[:min(5, len(killmailIDs))])
|
||||
// }
|
||||
if totalKillmails == 0 {
|
||||
return &FitStatistics{
|
||||
TotalKillmails: 0,
|
||||
ShipBreakdown: make(map[int64]Stats),
|
||||
SystemBreakdown: make(map[int64]Stats),
|
||||
HighSlotModules: make(map[int32]Stats),
|
||||
MidSlotModules: make(map[int32]Stats),
|
||||
LowSlotModules: make(map[int32]Stats),
|
||||
Rigs: make(map[int32]Stats),
|
||||
Drones: make(map[int32]Stats),
|
||||
KillmailIDs: []int64{},
|
||||
}, nil
|
||||
}
|
||||
|
||||
// stats := &FitStatistics{
|
||||
// TotalKillmails: totalKillmails,
|
||||
// ShipBreakdown: make(map[int64]Stats),
|
||||
// SystemBreakdown: make(map[int64]Stats),
|
||||
// HighSlotModules: make(map[int32]Stats),
|
||||
// MidSlotModules: make(map[int32]Stats),
|
||||
// LowSlotModules: make(map[int32]Stats),
|
||||
// Rigs: make(map[int32]Stats),
|
||||
// Drones: make(map[int32]Stats),
|
||||
// KillmailIDs: limitKillmails(killmailIDs, params.KillmailLimit),
|
||||
// }
|
||||
// Calculate ship breakdown
|
||||
shipCounts := make(map[int64]int64)
|
||||
for _, shipTypeID := range shipTypeIDs {
|
||||
shipCounts[shipTypeID]++
|
||||
}
|
||||
|
||||
// if totalKillmails == 0 {
|
||||
// flog.Info("No killmails found, returning empty statistics")
|
||||
// return stats, nil
|
||||
// }
|
||||
shipBreakdown := make(map[int64]Stats)
|
||||
for shipTypeID, count := range shipCounts {
|
||||
percentage := float64(count) / float64(totalKillmails) * 100.0
|
||||
shipBreakdown[shipTypeID] = Stats{
|
||||
Count: count,
|
||||
Percentage: percentage,
|
||||
}
|
||||
}
|
||||
|
||||
// // Calculate ship breakdown if params are empty or we have ship data
|
||||
// if isEmpty || len(shipTypeIDsFromResults) > 0 {
|
||||
// flog.Debug("Calculating ship breakdown")
|
||||
// shipCounts := make(map[int64]int64)
|
||||
// for _, shipTypeID := range shipTypeIDsFromResults {
|
||||
// shipCounts[shipTypeID]++
|
||||
// }
|
||||
// Calculate system breakdown
|
||||
systemCounts := make(map[int64]int64)
|
||||
for _, systemID := range systemIDs {
|
||||
systemCounts[systemID]++
|
||||
}
|
||||
|
||||
// for shipTypeID, count := range shipCounts {
|
||||
// percentage := float64(count) / float64(totalKillmails) * 100.0
|
||||
// stats.ShipBreakdown[shipTypeID] = Stats{
|
||||
// Count: count,
|
||||
// Percentage: percentage,
|
||||
// }
|
||||
// }
|
||||
// flog.Debug("Ship breakdown: %d unique ships", len(stats.ShipBreakdown))
|
||||
// }
|
||||
systemBreakdown := make(map[int64]Stats)
|
||||
for systemID, count := range systemCounts {
|
||||
percentage := float64(count) / float64(totalKillmails) * 100.0
|
||||
systemBreakdown[systemID] = Stats{
|
||||
Count: count,
|
||||
Percentage: percentage,
|
||||
}
|
||||
}
|
||||
|
||||
// flog.Debug("Calculating system breakdown")
|
||||
// systemCounts := make(map[int64]int64)
|
||||
// for _, systemID := range systemIDs {
|
||||
// systemCounts[systemID]++
|
||||
// }
|
||||
// Calculate module statistics
|
||||
stats := &FitStatistics{
|
||||
TotalKillmails: totalKillmails,
|
||||
ShipBreakdown: shipBreakdown,
|
||||
SystemBreakdown: systemBreakdown,
|
||||
HighSlotModules: make(map[int32]Stats),
|
||||
MidSlotModules: make(map[int32]Stats),
|
||||
LowSlotModules: make(map[int32]Stats),
|
||||
Rigs: make(map[int32]Stats),
|
||||
Drones: make(map[int32]Stats),
|
||||
KillmailIDs: limitKillmails(killmailIDs, params.KillmailLimit),
|
||||
}
|
||||
|
||||
// // Calculate system percentages
|
||||
// for systemID, count := range systemCounts {
|
||||
// percentage := float64(count) / float64(totalKillmails) * 100.0
|
||||
// stats.SystemBreakdown[systemID] = Stats{
|
||||
// Count: count,
|
||||
// Percentage: percentage,
|
||||
// }
|
||||
// }
|
||||
// flog.Debug("System breakdown: %d unique systems", len(stats.SystemBreakdown))
|
||||
// Get module statistics for the filtered killmails
|
||||
if err := db.calculateModuleStats(killmailIDs, stats, flog); err != nil {
|
||||
flog.Error("Failed to calculate module stats: %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// flog.Debug("Calculating module statistics for %d killmails", len(killmailIDs))
|
||||
flog.Info("Statistics calculated: %d high, %d mid, %d low, %d rigs, %d drones",
|
||||
len(stats.HighSlotModules), len(stats.MidSlotModules), len(stats.LowSlotModules),
|
||||
len(stats.Rigs), len(stats.Drones))
|
||||
|
||||
// if err := db.calculateStats(params, shipTypeIDs, killmailIDs, stats, totalKillmails, flog); err != nil {
|
||||
// flog.Error("Failed to calculate module stats: %v", err)
|
||||
// return nil, err
|
||||
// }
|
||||
|
||||
// flog.Info("Statistics calculated: %d high, %d mid, %d low, %d rigs, %d drones",
|
||||
// len(stats.HighSlotModules), len(stats.MidSlotModules), len(stats.LowSlotModules),
|
||||
// len(stats.Rigs), len(stats.Drones))
|
||||
|
||||
// return stats, nil
|
||||
return nil, nil
|
||||
return stats, nil
|
||||
}
|
||||
|
||||
func (db *DBWrapper) ExpandGroupsIntoItemTypeIds(groups []int64) ([]int64, error) {
|
||||
@@ -640,3 +597,107 @@ func (db *DBWrapper) CacheClean() error {
|
||||
Where("created_at < ?", threshold).
|
||||
Delete(&CacheEntry{}).Error
|
||||
}
|
||||
|
||||
func (db *DBWrapper) GetModuleSlots(moduleIDs []int64) (map[int64]ModuleSlot, error) {
|
||||
if len(moduleIDs) == 0 {
|
||||
return make(map[int64]ModuleSlot), nil
|
||||
}
|
||||
|
||||
var effects []models.DgmTypeEffect
|
||||
qres := db.db.Model(&models.DgmTypeEffect{}).
|
||||
Select("typeID, effectID").
|
||||
Where("typeID IN ? AND effectID IN (11, 12, 13, 2663)", moduleIDs).
|
||||
Find(&effects)
|
||||
if qres.Error != nil {
|
||||
return nil, qres.Error
|
||||
}
|
||||
|
||||
result := make(map[int64]ModuleSlot)
|
||||
for _, e := range effects {
|
||||
var slot ModuleSlot
|
||||
switch e.EffectID {
|
||||
case 11:
|
||||
slot = ModuleSlotLow
|
||||
case 12:
|
||||
slot = ModuleSlotHigh
|
||||
case 13:
|
||||
slot = ModuleSlotMid
|
||||
case 2663:
|
||||
slot = ModuleSlotRig
|
||||
}
|
||||
result[int64(e.TypeID)] = slot
|
||||
}
|
||||
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func limitKillmails(killmailIDs []int64, limit int) []int64 {
|
||||
if limit <= 0 || len(killmailIDs) <= limit {
|
||||
return killmailIDs
|
||||
}
|
||||
return killmailIDs[:limit]
|
||||
}
|
||||
|
||||
func (db *DBWrapper) calculateModuleStats(killmailIDs []int64, stats *FitStatistics, flog *cylogger.Logger) error {
|
||||
if len(killmailIDs) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
|
||||
// Create placeholders for killmail IDs
|
||||
placeholders := make([]string, len(killmailIDs))
|
||||
args := make([]interface{}, len(killmailIDs))
|
||||
for i, id := range killmailIDs {
|
||||
placeholders[i] = "?"
|
||||
args = append(args, id)
|
||||
}
|
||||
|
||||
// Query module statistics - count distinct killmails per (item_type_id, slot) combination
|
||||
query := fmt.Sprintf(`
|
||||
SELECT
|
||||
fm.item_type_id,
|
||||
fm.slot,
|
||||
COUNT(DISTINCT fm.killmail_id) as count
|
||||
FROM fitted_modules fm
|
||||
WHERE fm.killmail_id IN (%s)
|
||||
GROUP BY fm.item_type_id, fm.slot
|
||||
ORDER BY count DESC`, strings.Join(placeholders, ","))
|
||||
|
||||
rows, err := db.ch.Query(ctx, query, args...)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to query module stats: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
totalKillmails := float64(stats.TotalKillmails)
|
||||
|
||||
for rows.Next() {
|
||||
var itemTypeID int32
|
||||
var slot string
|
||||
var count int64
|
||||
|
||||
if err := rows.Scan(&itemTypeID, &slot, &count); err != nil {
|
||||
return fmt.Errorf("failed to scan module row: %w", err)
|
||||
}
|
||||
|
||||
percentage := float64(count) / totalKillmails * 100.0
|
||||
stat := Stats{Count: count, Percentage: percentage}
|
||||
|
||||
// Map slot to the appropriate map
|
||||
switch slot {
|
||||
case "Low":
|
||||
stats.LowSlotModules[itemTypeID] = stat
|
||||
case "Mid":
|
||||
stats.MidSlotModules[itemTypeID] = stat
|
||||
case "High":
|
||||
stats.HighSlotModules[itemTypeID] = stat
|
||||
case "Rig":
|
||||
stats.Rigs[itemTypeID] = stat
|
||||
case "Drone":
|
||||
stats.Drones[itemTypeID] = stat
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user