zkill-susser/db.go

package main

import (
	"context"
	"fmt"
	"strings"

	logger "git.site.quack-lab.dev/dave/cylogger"
	"github.com/ClickHouse/clickhouse-go/v2"
	"github.com/ClickHouse/clickhouse-go/v2/lib/driver"
	"gorm.io/driver/sqlite"
	"gorm.io/gorm"
	"gorm.io/gorm/schema"
)

type QueryParams struct {
	Ship    int64
	Systems []int64
	Modules []int64
}

type FitStatistics struct {
	TotalKillmails  int64
	SystemBreakdown map[int64]int64
	HighSlotModules map[int32]ModuleStats
	MidSlotModules  map[int32]ModuleStats
	LowSlotModules  map[int32]ModuleStats
	Rigs            map[int32]ModuleStats
	Drones          map[int32]ModuleStats
}

type ModuleStats struct {
	Count      int64
	Percentage float64
}

type DB interface {
	DB() *gorm.DB
	Raw(sql string, args ...any) *gorm.DB
	SaveKillmails(killmails []Killmail) error
	InitTables() error
	QueryFits(params QueryParams) (*FitStatistics, error)
}

type DBWrapper struct {
	ch     driver.Conn
	gormDB *gorm.DB // For SQLite (EVE static data)
}

var db *DBWrapper

func GetDB() (DB, error) {
	if db != nil {
		return db, nil
	}

	// ClickHouse connection - use HTTP interface on port 8123
	// Change "localhost" to your Linux host IP or hostname
	options := &clickhouse.Options{
		Addr: []string{"clickhouse.site.quack-lab.dev"}, // TODO: Change to your Linux host, e.g., "192.168.1.100:8123" or "clickhouse.example.com:8123"
		Auth: clickhouse.Auth{
			Database: "zkill",
			Username: "default",
			Password: "", // Set if you configure a password on Linux host
		},
		Protocol: clickhouse.HTTP,
	}

	conn, err := clickhouse.Open(options)
	if err != nil {
		return nil, fmt.Errorf("failed to connect to ClickHouse: %w", err)
	}

	// SQLite connection for EVE static data
	sqliteDB, err := gorm.Open(sqlite.Open("sqlite-latest.sqlite"), &gorm.Config{
		NamingStrategy: schema.NamingStrategy{
			NoLowerCase: true,
		},
	})
	if err != nil {
		return nil, fmt.Errorf("failed to connect to SQLite: %w", err)
	}

	db = &DBWrapper{
		ch:     conn,
		gormDB: sqliteDB,
	}

	return db, nil
}

func (db *DBWrapper) InitTables() error {
	ctx := context.Background()

	// Create flat_killmails table
	createFlatKillmails := `
	CREATE TABLE IF NOT EXISTS flat_killmails (
		killmail_id Int64,
		killmail_time DateTime,
		solar_system_id Int64,
		killmail_hash String,
		victim_ship_type_id Int64,
		victim_character_id Int64,
		victim_corporation_id Int64,
		victim_alliance_id Int64,
		victim_damage_taken Int64,
		victim_pos_x Float64,
		victim_pos_y Float64,
		victim_pos_z Float64,
		attacker_count UInt16,
		total_damage_done Int64,
		final_blow_ship_type Int64,
		attackers Array(Tuple(
			Int64,  -- character_id
			Int64,  -- corporation_id
			Int64,  -- alliance_id
			Int64,  -- ship_type_id
			Int64,  -- weapon_type_id
			Int64,  -- damage_done
			UInt8,  -- final_blow
			Float64 -- security_status
		)),
		items Array(Tuple(
			Int64,  -- flag
			Int64,  -- item_type_id
			Int64,  -- quantity_destroyed
			Int64,  -- quantity_dropped
			Int64   -- singleton
		))
	) ENGINE = MergeTree()
	ORDER BY (solar_system_id, victim_ship_type_id, killmail_time)`

	if err := db.ch.Exec(ctx, createFlatKillmails); err != nil {
		return fmt.Errorf("failed to create flat_killmails table: %w", err)
	}

	// Create fitted_modules table
	createFittedModules := `
	CREATE TABLE IF NOT EXISTS fitted_modules (
		killmail_id Int64,
		killmail_time DateTime,
		solar_system_id Int64,
		victim_ship_type_id Int64,
		item_type_id Int64,
		flag Int64,
		quantity_destroyed Int64,
		quantity_dropped Int64
	) ENGINE = MergeTree()
	ORDER BY (item_type_id, victim_ship_type_id, solar_system_id)`

	if err := db.ch.Exec(ctx, createFittedModules); err != nil {
		return fmt.Errorf("failed to create fitted_modules table: %w", err)
	}

	return nil
}

func (db *DBWrapper) Raw(sql string, args ...any) *gorm.DB {
	return db.gormDB.Raw(sql, args...)
}

func (db *DBWrapper) DB() *gorm.DB {
	return db.gormDB
}

func (db *DBWrapper) SaveKillmails(killmails []Killmail) error {
	ctx := context.Background()

	// Prepare batch for flat_killmails
	flatBatch, err := db.ch.PrepareBatch(ctx, "INSERT INTO flat_killmails")
	if err != nil {
		return fmt.Errorf("failed to prepare flat_killmails batch: %w", err)
	}

	// Prepare batch for fitted_modules
	moduleBatch, err := db.ch.PrepareBatch(ctx, "INSERT INTO fitted_modules")
	if err != nil {
		return fmt.Errorf("failed to prepare fitted_modules batch: %w", err)
	}

	// Process in batches
	batchSize := 1000
	for i := 0; i < len(killmails); i += batchSize {
		end := i + batchSize
		if end > len(killmails) {
			end = len(killmails)
		}

		for _, km := range killmails[i:end] {
			flat := km.FlattenKillmail()
			modules := km.ExtractFittedModules()

			// Convert attackers to slice of slices for ClickHouse Array(Tuple(...))
			attackersSlice := make([][]interface{}, len(flat.Attackers))
			for j, a := range flat.Attackers {
				attackersSlice[j] = []interface{}{
					a.CharacterID,
					a.CorporationID,
					a.AllianceID,
					a.ShipTypeID,
					a.WeaponTypeID,
					a.DamageDone,
					a.FinalBlow,
					a.SecurityStatus,
				}
			}

			// Convert items to slice of slices
			itemsSlice := make([][]interface{}, len(flat.Items))
			for j, item := range flat.Items {
				itemsSlice[j] = []interface{}{
					item.Flag,
					item.ItemTypeID,
					item.QuantityDestroyed,
					item.QuantityDropped,
					item.Singleton,
				}
			}

			// Append to flat_killmails batch
			if err := flatBatch.Append(
				flat.KillmailID,
				flat.KillmailTime,
				flat.SolarSystemID,
				flat.KillmailHash,
				flat.VictimShipTypeID,
				flat.VictimCharacterID,
				flat.VictimCorporationID,
				flat.VictimAllianceID,
				flat.VictimDamageTaken,
				flat.VictimPosX,
				flat.VictimPosY,
				flat.VictimPosZ,
				flat.AttackerCount,
				flat.TotalDamageDone,
				flat.FinalBlowShipType,
				attackersSlice,
				itemsSlice,
			); err != nil {
				return fmt.Errorf("failed to append flat killmail: %w", err)
			}

			// Append modules to fitted_modules batch
			for _, mod := range modules {
				if err := moduleBatch.AppendStruct(&mod); err != nil {
					return fmt.Errorf("failed to append module: %w", err)
				}
			}
		}

		// Send batches every 1000 records
		if err := flatBatch.Send(); err != nil {
			return fmt.Errorf("failed to send flat_killmails batch: %w", err)
		}
		if err := moduleBatch.Send(); err != nil {
			return fmt.Errorf("failed to send fitted_modules batch: %w", err)
		}

		// Prepare new batches for next iteration
		if end < len(killmails) {
			flatBatch, err = db.ch.PrepareBatch(ctx, "INSERT INTO flat_killmails")
			if err != nil {
				return fmt.Errorf("failed to prepare flat_killmails batch: %w", err)
			}
			moduleBatch, err = db.ch.PrepareBatch(ctx, "INSERT INTO fitted_modules")
			if err != nil {
				return fmt.Errorf("failed to prepare fitted_modules batch: %w", err)
			}
		}
	}

	return nil
}

func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) {
	flog := logger.Default.WithPrefix("QueryFits").WithPrefix(fmt.Sprintf("%+v", params))
	flog.Info("Starting query")

	modules := deduplicateInt64(params.Modules)
	flog.Debug("Deduplicated modules: %d -> %d", len(params.Modules), len(modules))

	ctx := context.Background()

	// Build ClickHouse query
	query := "SELECT killmail_id, solar_system_id FROM flat_killmails WHERE victim_ship_type_id = ?"
	args := []interface{}{params.Ship}

	flog.Debug("Checking total killmails for ship type %d", params.Ship)
	var totalCount uint64
	countQuery := "SELECT count() FROM flat_killmails WHERE victim_ship_type_id = ?"
	if err := db.ch.QueryRow(ctx, countQuery, params.Ship).Scan(&totalCount); err != nil {
		flog.Error("Failed to count total killmails: %v", err)
	} else {
		flog.Info("Total killmails for ship type %d: %d", params.Ship, totalCount)
	}
	flog.Debug("Base query: victim_ship_type_id = %d", params.Ship)

	if len(params.Systems) > 0 {
		// Build IN clause with placeholders
		placeholders := make([]string, len(params.Systems))
		for i := range params.Systems {
			placeholders[i] = "?"
			args = append(args, params.Systems[i])
		}
		query += " AND solar_system_id IN (" + fmt.Sprintf("%s", placeholders) + ")"
		flog.Debug("Added system filter: %d systems", len(params.Systems))
	}

	if len(modules) > 0 {
		flog.Debug("Looking up module slots for %d modules", len(modules))
		moduleSlots, err := db.getModuleSlots(modules)
		if err != nil {
			flog.Error("Failed to get module slots: %v", err)
			return nil, err
		}
		flog.Debug("Found slots for %d modules", len(moduleSlots))

		for _, moduleID := range modules {
			slot, exists := moduleSlots[moduleID]
			if !exists {
				flog.Debug("Module %d has no slot, skipping", moduleID)
				continue
			}

			var flagMin, flagMax int64
			switch slot {
			case "low":
				flagMin, flagMax = 11, 18
			case "mid":
				flagMin, flagMax = 19, 26
			case "high":
				flagMin, flagMax = 27, 34
			case "rig":
				flagMin, flagMax = 92, 99
			case "drone":
				flagMin, flagMax = 87, 87
			default:
				flog.Debug("Unknown slot type %s for module %d", slot, moduleID)
				continue
			}

			query += " AND killmail_id IN (SELECT killmail_id FROM fitted_modules WHERE item_type_id = ? AND flag BETWEEN ? AND ?)"
			args = append(args, moduleID, flagMin, flagMax)
			flog.Debug("Added module filter: module %d in %s slot (flags %d-%d)", moduleID, slot, flagMin, flagMax)
		}
	}

	var killmailIDs []int64
	var systemIDs []int64

	flog.Debug("Executing filtered query to get killmail IDs")
	rows, err := db.ch.Query(ctx, query, args...)
	if err != nil {
		flog.Error("Failed to execute query: %v", err)
		return nil, err
	}
	defer rows.Close()

	for rows.Next() {
		var killmailID, systemID int64
		if err := rows.Scan(&killmailID, &systemID); err != nil {
			flog.Error("Failed to scan row: %v", err)
			return nil, err
		}
		killmailIDs = append(killmailIDs, killmailID)
		systemIDs = append(systemIDs, systemID)
	}

	totalKillmails := int64(len(killmailIDs))
	flog.Info("Found %d killmails after filtering", totalKillmails)
	if totalKillmails > 0 {
		flog.Debug("Sample killmail IDs: %v", killmailIDs[:min(5, len(killmailIDs))])
	}

	stats := &FitStatistics{
		TotalKillmails:  totalKillmails,
		SystemBreakdown: make(map[int64]int64),
		HighSlotModules: make(map[int32]ModuleStats),
		MidSlotModules:  make(map[int32]ModuleStats),
		LowSlotModules:  make(map[int32]ModuleStats),
		Rigs:            make(map[int32]ModuleStats),
		Drones:          make(map[int32]ModuleStats),
	}

	if totalKillmails == 0 {
		flog.Info("No killmails found, returning empty statistics")
		return stats, nil
	}

	flog.Debug("Calculating system breakdown")
	for _, systemID := range systemIDs {
		stats.SystemBreakdown[systemID]++
	}
	flog.Debug("System breakdown: %d unique systems", len(stats.SystemBreakdown))

	flog.Debug("Calculating module statistics for %d killmails", len(killmailIDs))

	if err := db.calculateModuleStats(killmailIDs, stats, totalKillmails, flog); err != nil {
		flog.Error("Failed to calculate module stats: %v", err)
		return nil, err
	}

	flog.Info("Statistics calculated: %d high, %d mid, %d low, %d rigs, %d drones",
		len(stats.HighSlotModules), len(stats.MidSlotModules), len(stats.LowSlotModules),
		len(stats.Rigs), len(stats.Drones))

	return stats, nil
}

func deduplicateInt64(slice []int64) []int64 {
	seen := make(map[int64]bool)
	result := make([]int64, 0, len(slice))
	for _, v := range slice {
		if !seen[v] {
			seen[v] = true
			result = append(result, v)
		}
	}
	return result
}

var moduleSlotCache = make(map[int64]string)

func (db *DBWrapper) getModuleSlots(moduleIDs []int64) (map[int64]string, error) {
	result := make(map[int64]string)
	uncached := make([]int64, 0)

	for _, id := range moduleIDs {
		if slot, cached := moduleSlotCache[id]; cached {
			result[id] = slot
		} else {
			uncached = append(uncached, id)
		}
	}

	if len(uncached) == 0 {
		return result, nil
	}

	var effects []struct {
		TypeID   int32
		EffectID int32
	}
	err := db.gormDB.Table("dgmTypeEffects").
		Select("typeID, effectID").
		Where("typeID IN ? AND effectID IN (11, 12, 13, 2663)", uncached).
		Find(&effects).Error
	if err != nil {
		return nil, err
	}

	for _, e := range effects {
		var slot string
		switch e.EffectID {
		case 11:
			slot = "low"
		case 12:
			slot = "high"
		case 13:
			slot = "mid"
		case 2663:
			slot = "rig"
		}
		result[int64(e.TypeID)] = slot
		moduleSlotCache[int64(e.TypeID)] = slot
	}

	droneCategoryID := int32(18)
	var droneTypeIDs []int32
	err = db.gormDB.Table("invTypes").
		Select("invTypes.typeID").
		Joins("INNER JOIN invGroups ON invTypes.groupID = invGroups.groupID").
		Where("invTypes.typeID IN ? AND invGroups.categoryID = ?", uncached, droneCategoryID).
		Pluck("invTypes.typeID", &droneTypeIDs).Error
	if err != nil {
		return nil, err
	}

	for _, id := range droneTypeIDs {
		result[int64(id)] = "drone"
		moduleSlotCache[int64(id)] = "drone"
	}

	return result, nil
}

func (db *DBWrapper) calculateModuleStats(killmailIDs []int64, stats *FitStatistics, total int64, flog *logger.Logger) error {
	flog.Debug("Querying module stats for %d killmails", len(killmailIDs))

	if len(killmailIDs) == 0 {
		return nil
	}

	ctx := context.Background()

	// Build IN clause with placeholders
	placeholders := make([]string, len(killmailIDs))
	args := make([]interface{}, len(killmailIDs))
	for i, id := range killmailIDs {
		placeholders[i] = "?"
		args[i] = id
	}
	query := fmt.Sprintf("SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM fitted_modules WHERE killmail_id IN (%s) GROUP BY item_type_id, flag",
		strings.Join(placeholders, ","))

	rows, err := db.ch.Query(ctx, query, args...)
	if err != nil {
		flog.Error("Failed to query module stats: %v", err)
		return err
	}
	defer rows.Close()

	var items []struct {
		ItemTypeID int32
		Flag       int64
		Count      uint64
	}

	for rows.Next() {
		var item struct {
			ItemTypeID int32
			Flag       int64
			Count      uint64
		}
		if err := rows.Scan(&item.ItemTypeID, &item.Flag, &item.Count); err != nil {
			flog.Error("Failed to scan module stat: %v", err)
			return err
		}
		items = append(items, item)
	}

	flog.Debug("Found %d item type/flag combinations", len(items))

	for _, item := range items {
		percentage := float64(item.Count) / float64(total) * 100.0
		moduleStats := ModuleStats{
			Count:      int64(item.Count),
			Percentage: percentage,
		}

		switch {
		case item.Flag >= 11 && item.Flag <= 18:
			stats.LowSlotModules[item.ItemTypeID] = moduleStats
			flog.Trace("Low slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
		case item.Flag >= 19 && item.Flag <= 26:
			stats.MidSlotModules[item.ItemTypeID] = moduleStats
			flog.Trace("Mid slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
		case item.Flag >= 27 && item.Flag <= 34:
			stats.HighSlotModules[item.ItemTypeID] = moduleStats
			flog.Trace("High slot module %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
		case item.Flag >= 92 && item.Flag <= 99:
			stats.Rigs[item.ItemTypeID] = moduleStats
			flog.Trace("Rig %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
		case item.Flag == 87:
			stats.Drones[item.ItemTypeID] = moduleStats
			flog.Trace("Drone %d: %d killmails (%.2f%%)", item.ItemTypeID, item.Count, percentage)
		default:
			flog.Trace("Ignoring item %d with flag %d (not a module slot)", item.ItemTypeID, item.Flag)
		}
	}

	return nil
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}