From 65a3cb587bf6bcba6adc480dfea57a16807859eb Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Tue, 6 Jan 2026 18:06:19 +0100 Subject: [PATCH] Rehallucinate the API --- api.go | 724 +++++++++++++++++++++++++------------------------- clickhouse.go | 12 +- db.go | 233 +++++++++++++--- main.go | 6 +- types.go | 4 + 5 files changed, 583 insertions(+), 396 deletions(-) diff --git a/api.go b/api.go index 315baeb..fee8c58 100644 --- a/api.go +++ b/api.go @@ -1,7 +1,13 @@ package main import ( + "encoding/json" + "net/http" + "strconv" + "strings" "time" + + logger "git.site.quack-lab.dev/dave/cylogger" ) type APIStatisticsRequest struct { @@ -62,387 +68,382 @@ type APIImageData struct { const cacheTTL = 24 * time.Hour -// func convertFitStatistics(stats *FitStatistics) *APIFitStatistics { -// api := &APIFitStatistics{ -// TotalKillmails: stats.TotalKillmails, -// Ships: convertMapToItemCounts(stats.ShipBreakdown), -// SystemBreakdown: convertMapToItemCounts(stats.SystemBreakdown), -// HighSlotModules: convertModuleMapToItemCounts(stats.HighSlotModules), -// MidSlotModules: convertModuleMapToItemCounts(stats.MidSlotModules), -// LowSlotModules: convertModuleMapToItemCounts(stats.LowSlotModules), -// Rigs: convertModuleMapToItemCounts(stats.Rigs), -// Drones: convertModuleMapToItemCounts(stats.Drones), -// KillmailIDs: stats.KillmailIDs, -// } -// return api -// } +func convertFitStatistics(stats *FitStatistics) *APIFitStatistics { + api := &APIFitStatistics{ + TotalKillmails: stats.TotalKillmails, + Ships: convertMapToItemCounts(stats.ShipBreakdown), + SystemBreakdown: convertMapToItemCounts(stats.SystemBreakdown), + HighSlotModules: convertModuleMapToItemCounts(stats.HighSlotModules), + MidSlotModules: convertModuleMapToItemCounts(stats.MidSlotModules), + LowSlotModules: convertModuleMapToItemCounts(stats.LowSlotModules), + Rigs: convertModuleMapToItemCounts(stats.Rigs), + Drones: convertModuleMapToItemCounts(stats.Drones), + KillmailIDs: stats.KillmailIDs, + } + return api +} -// func convertMapToItemCounts(m map[int64]Stats) []APIItemCount { -// result := make([]APIItemCount, 0, len(m)) -// for id, stats := range m { -// result = append(result, APIItemCount{ -// ItemId: id, -// Count: stats.Count, -// }) -// } -// return result -// } +func convertMapToItemCounts(m map[int64]int64) []APIItemCount { + result := make([]APIItemCount, 0, len(m)) + for id, count := range m { + result = append(result, APIItemCount{ + ItemId: id, + Count: count, + }) + } + return result +} -// func convertModuleMapToItemCounts(m map[int32]Stats) []APIItemCount { -// result := make([]APIItemCount, 0, len(m)) -// for id, stats := range m { -// result = append(result, APIItemCount{ -// ItemId: int64(id), -// Count: stats.Count, -// }) -// } -// return result -// } +func convertModuleMapToItemCounts(m map[int32]int64) []APIItemCount { + result := make([]APIItemCount, 0, len(m)) + for id, count := range m { + result = append(result, APIItemCount{ + ItemId: int64(id), + Count: count, + }) + } + return result +} -// func handleStatistics(w http.ResponseWriter, r *http.Request) { -// flog := logger.Default.WithPrefix("handleStatistics") -// flog.Trace("Request received: %s %s", r.Method, r.URL.Path) +func handleStatistics(w http.ResponseWriter, r *http.Request) { + flog := logger.Default.WithPrefix("handleStatistics") + flog.Trace("Request received: %s %s", r.Method, r.URL.Path) -// if r.Method != http.MethodPost { -// flog.Debug("Method not allowed: %s", r.Method) -// http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) -// return -// } + if r.Method != http.MethodPost { + flog.Debug("Method not allowed: %s", r.Method) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } -// flog.Debug("Decoding request body") -// var req APIStatisticsRequest -// if err := json.NewDecoder(r.Body).Decode(&req); err != nil { -// flog.Error("Failed to decode request body: %v", err) -// http.Error(w, "Invalid request body", http.StatusBadRequest) -// return -// } -// flog.Dump("Request", req) + flog.Debug("Decoding request body") + var req APIStatisticsRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + flog.Error("Failed to decode request body: %v", err) + http.Error(w, "Invalid request body", http.StatusBadRequest) + return + } + flog.Dump("Request", req) -// params := QueryParams{} -// if req.Ship != nil { -// params.Ship = *req.Ship -// flog.Debug("Ship filter: %d", params.Ship) -// } -// params.Systems = req.Systems -// if len(params.Systems) > 0 { -// flog.Debug("Systems filter: %d systems", len(params.Systems)) -// } -// params.Modules = req.Modules -// if len(params.Modules) > 0 { -// flog.Debug("Modules filter: %d modules", len(params.Modules)) -// } -// params.Groups = req.Groups -// if len(params.Groups) > 0 { -// flog.Debug("Groups filter: %d groups", len(params.Groups)) -// } -// // Killmail limit defaults to 20 when not provided or invalid -// if req.KillmailLimit != nil && *req.KillmailLimit > 0 { -// params.KillmailLimit = *req.KillmailLimit -// } else { -// params.KillmailLimit = 20 -// } -// flog.Debug("Killmail limit: %d", params.KillmailLimit) + params := QueryParams{} + if req.Ship != nil { + params.Ship = *req.Ship + flog.Debug("Ship filter: %d", params.Ship) + } + params.Systems = req.Systems + if len(params.Systems) > 0 { + flog.Debug("Systems filter: %d systems", len(params.Systems)) + } + params.Modules = req.Modules + if len(params.Modules) > 0 { + flog.Debug("Modules filter: %d modules", len(params.Modules)) + } + params.Groups = req.Groups + if len(params.Groups) > 0 { + flog.Debug("Groups filter: %d groups", len(params.Groups)) + } + // Killmail limit defaults to 20 when not provided or invalid + if req.KillmailLimit != nil && *req.KillmailLimit > 0 { + params.KillmailLimit = *req.KillmailLimit + } else { + params.KillmailLimit = 20 + } + flog.Debug("Killmail limit: %d", params.KillmailLimit) -// db, err := GetDB() -// if err != nil { -// flog.Error("Failed to get database: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Trace("Database connection obtained") + db, err := GetDB() + if err != nil { + flog.Error("Failed to get database: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + flog.Trace("Database connection obtained") -// cacheKeyBytes, _ := json.Marshal(req) -// cacheKey := "stats:" + string(cacheKeyBytes) -// flog.Debug("Checking cache for key: %s", cacheKey) + flog.Info("Querying database") + flog.Debug("Executing QueryFits with params: %+v", params) + stats, err := db.QueryFits(params) + if err != nil { + flog.Error("Failed to query fits: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + flog.Info("Query completed: %d total killmails", stats.TotalKillmails) + flog.Dump("Statistics", stats) -// cachedData, found := db.GetCacheEntry(cacheKey, 3*24*time.Hour) -// if found { -// flog.Info("Serving from cache") -// w.Header().Set("Content-Type", "application/json") -// w.Write(cachedData) -// return -// } + flog.Debug("Converting statistics to API format") + apiStats := convertFitStatistics(stats) + flog.Dump("API Statistics", apiStats) -// flog.Info("Cache miss, querying database") -// flog.Debug("Executing QueryFits with params: %+v", params) -// stats, err := db.QueryFits(params) -// if err != nil { -// flog.Error("Failed to query fits: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Info("Query completed: %d total killmails", stats.TotalKillmails) -// flog.Dump("Statistics", stats) + w.Header().Set("Content-Type", "application/json") + flog.Trace("Encoding response") + if err := json.NewEncoder(w).Encode(apiStats); err != nil { + flog.Error("Failed to encode response: %v", err) + return + } + flog.Info("Response sent successfully") +} -// flog.Debug("Converting statistics to API format") -// apiStats := convertFitStatistics(stats) -// flog.Dump("API Statistics", apiStats) +func handleSearch(w http.ResponseWriter, r *http.Request) { + flog := logger.Default.WithPrefix("handleSearch") + flog.Trace("Request received: %s %s", r.Method, r.URL.Path) -// responseData, err := json.Marshal(apiStats) -// if err != nil { -// flog.Error("Failed to marshal response: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } + if r.Method != http.MethodGet { + flog.Debug("Method not allowed: %s", r.Method) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } -// flog.Debug("Storing in cache") -// if err := db.CacheEntry(cacheKey, responseData); err != nil { -// flog.Error("Failed to cache statistics: %v", err) -// } + query := r.URL.Query().Get("q") + flog.Debug("Search query: %q", query) + if query == "" { + flog.Info("Empty query, returning empty results") + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode([]APISearchResult{}) + return + } -// w.Header().Set("Content-Type", "application/json") -// flog.Trace("Encoding response") -// w.Write(responseData) -// flog.Info("Response sent successfully") -// } + db, err := GetDB() + if err != nil { + flog.Error("Failed to get database: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + flog.Trace("Database connection obtained") -// func handleSearch(w http.ResponseWriter, r *http.Request) { -// flog := logger.Default.WithPrefix("handleSearch") -// flog.Trace("Request received: %s %s", r.Method, r.URL.Path) + results := []APISearchResult{} -// if r.Method != http.MethodGet { -// flog.Debug("Method not allowed: %s", r.Method) -// http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) -// return -// } + flog.Debug("Searching ships") + ships, err := db.SearchShips(query, 10) + if err != nil { + flog.Error("Failed to search ships: %v", err) + } else { + flog.Info("Found %d ships", len(ships)) + for _, ship := range ships { + results = append(results, APISearchResult{ + ID: int64(ship.TypeID), + Name: ship.TypeName, + Type: "ship", + }) + } + } -// query := r.URL.Query().Get("q") -// flog.Debug("Search query: %q", query) -// if query == "" { -// flog.Info("Empty query, returning empty results") -// w.Header().Set("Content-Type", "application/json") -// json.NewEncoder(w).Encode([]APISearchResult{}) -// return -// } + flog.Debug("Searching systems") + systems, err := db.SearchSystems(query, 10) + if err != nil { + flog.Error("Failed to search systems: %v", err) + } else { + flog.Info("Found %d systems", len(systems)) + for _, system := range systems { + results = append(results, APISearchResult{ + ID: int64(system.SolarSystemID), + Name: system.SolarSystemName, + Type: "system", + }) + } + } -// db, err := GetDB() -// if err != nil { -// flog.Error("Failed to get database: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Trace("Database connection obtained") + flog.Debug("Searching modules") + modules, err := db.SearchModules(query, 10) + if err != nil { + flog.Error("Failed to search modules: %v", err) + } else { + flog.Info("Found %d modules", len(modules)) + for _, module := range modules { + results = append(results, APISearchResult{ + ID: int64(module.TypeID), + Name: module.TypeName, + Type: "module", + }) + } + } -// results := []APISearchResult{} + flog.Debug("Searching groups") + groups, err := db.SearchGroups(query, 10) + if err != nil { + flog.Error("Failed to search groups: %v", err) + } else { + flog.Info("Found %d groups", len(groups)) + for _, group := range groups { + results = append(results, APISearchResult{ + ID: int64(group.GroupID), + Name: group.GroupName, + Type: "group", + }) + } + } -// flog.Debug("Searching ships") -// ships, err := db.SearchShips(query, 10) -// if err != nil { -// flog.Error("Failed to search ships: %v", err) -// } else { -// flog.Info("Found %d ships", len(ships)) -// for _, ship := range ships { -// results = append(results, APISearchResult{ -// ID: int64(ship.TypeID), -// Name: ship.TypeName, -// Type: "ship", -// }) -// } -// } + flog.Info("Total search results: %d", len(results)) + flog.Dump("Results", results) -// flog.Debug("Searching systems") -// systems, err := db.SearchSystems(query, 10) -// if err != nil { -// flog.Error("Failed to search systems: %v", err) -// } else { -// flog.Info("Found %d systems", len(systems)) -// for _, system := range systems { -// results = append(results, APISearchResult{ -// ID: int64(system.SolarSystemID), -// Name: system.SolarSystemName, -// Type: "system", -// }) -// } -// } + w.Header().Set("Content-Type", "application/json") + flog.Trace("Encoding response") + if err := json.NewEncoder(w).Encode(results); err != nil { + flog.Error("Failed to encode response: %v", err) + return + } + flog.Info("Response sent successfully") +} -// flog.Debug("Searching modules") -// modules, err := db.SearchModules(query, 10) -// if err != nil { -// flog.Error("Failed to search modules: %v", err) -// } else { -// flog.Info("Found %d modules", len(modules)) -// for _, module := range modules { -// results = append(results, APISearchResult{ -// ID: int64(module.TypeID), -// Name: module.TypeName, -// Type: "module", -// }) -// } -// } +func handleItemNames(w http.ResponseWriter, r *http.Request) { + flog := logger.Default.WithPrefix("handleItemNames") + flog.Trace("Request received: %s %s", r.Method, r.URL.Path) -// flog.Debug("Searching groups") -// groups, err := db.SearchGroups(query, 10) -// if err != nil { -// flog.Error("Failed to search groups: %v", err) -// } else { -// flog.Info("Found %d groups", len(groups)) -// for _, group := range groups { -// results = append(results, APISearchResult{ -// ID: int64(group.GroupID), -// Name: group.GroupName, -// Type: "group", -// }) -// } -// } + if r.Method != http.MethodGet { + flog.Debug("Method not allowed: %s", r.Method) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } -// flog.Info("Total search results: %d", len(results)) -// flog.Dump("Results", results) + idsParam := r.URL.Query().Get("ids") + flog.Debug("IDs parameter: %q", idsParam) + if idsParam == "" { + flog.Info("Empty IDs parameter, returning empty map") + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(APIItemNames{}) + return + } -// w.Header().Set("Content-Type", "application/json") -// flog.Trace("Encoding response") -// if err := json.NewEncoder(w).Encode(results); err != nil { -// flog.Error("Failed to encode response: %v", err) -// return -// } -// flog.Info("Response sent successfully") -// } + flog.Debug("Parsing IDs") + idStrs := strings.Split(idsParam, ",") + flog.Trace("ID strings: %v", idStrs) + ids := make([]int32, 0, len(idStrs)) + for _, idStr := range idStrs { + id, err := strconv.ParseInt(idStr, 10, 32) + if err != nil { + flog.Debug("Failed to parse ID %q: %v", idStr, err) + continue + } + ids = append(ids, int32(id)) + } + flog.Info("Parsed %d valid IDs from %d strings", len(ids), len(idStrs)) -// func handleItemNames(w http.ResponseWriter, r *http.Request) { -// flog := logger.Default.WithPrefix("handleItemNames") -// flog.Trace("Request received: %s %s", r.Method, r.URL.Path) + if len(ids) == 0 { + flog.Info("No valid IDs, returning empty map") + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(APIItemNames{}) + return + } -// if r.Method != http.MethodGet { -// flog.Debug("Method not allowed: %s", r.Method) -// http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) -// return -// } + db, err := GetDB() + if err != nil { + flog.Error("Failed to get database: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + flog.Trace("Database connection obtained") -// idsParam := r.URL.Query().Get("ids") -// flog.Debug("IDs parameter: %q", idsParam) -// if idsParam == "" { -// flog.Info("Empty IDs parameter, returning empty map") -// w.Header().Set("Content-Type", "application/json") -// json.NewEncoder(w).Encode(APIItemNames{}) -// return -// } + flog.Debug("Getting item types for %d IDs", len(ids)) + // Convert []int32 to []int64 for GetItemTypes + itemIDs := make([]int64, len(ids)) + for i, id := range ids { + itemIDs[i] = int64(id) + } -// flog.Debug("Parsing IDs") -// idStrs := strings.Split(idsParam, ",") -// flog.Trace("ID strings: %v", idStrs) -// ids := make([]int32, 0, len(idStrs)) -// for _, idStr := range idStrs { -// id, err := strconv.ParseInt(idStr, 10, 32) -// if err != nil { -// flog.Debug("Failed to parse ID %q: %v", idStr, err) -// continue -// } -// ids = append(ids, int32(id)) -// } -// flog.Info("Parsed %d valid IDs from %d strings", len(ids), len(idStrs)) + items, err := db.GetItemTypes(itemIDs) + if err != nil { + flog.Error("Failed to get item types: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } -// if len(ids) == 0 { -// flog.Info("No valid IDs, returning empty map") -// w.Header().Set("Content-Type", "application/json") -// json.NewEncoder(w).Encode(APIItemNames{}) -// return -// } + // Convert to APIItemNames format (map[string]string) + names := make(APIItemNames) + for _, item := range items { + names[strconv.FormatInt(int64(item.TypeID), 10)] = item.TypeName + } -// db, err := GetDB() -// if err != nil { -// flog.Error("Failed to get database: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Trace("Database connection obtained") + flog.Info("Total names: %d", len(names)) + flog.Dump("Names", names) -// flog.Debug("Getting item names for %d IDs", len(ids)) -// names, err := db.GetItemNames(ids) -// if err != nil { -// flog.Error("Failed to get item names: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Info("Total names: %d", len(names)) -// flog.Dump("Names", names) + w.Header().Set("Content-Type", "application/json") + flog.Trace("Encoding response") + if err := json.NewEncoder(w).Encode(names); err != nil { + flog.Error("Failed to encode response: %v", err) + return + } + flog.Info("Response sent successfully") +} -// w.Header().Set("Content-Type", "application/json") -// flog.Trace("Encoding response") -// if err := json.NewEncoder(w).Encode(names); err != nil { -// flog.Error("Failed to encode response: %v", err) -// return -// } -// flog.Info("Response sent successfully") -// } +func handleItemGroup(w http.ResponseWriter, r *http.Request) { + flog := logger.Default.WithPrefix("handleItemGroup") + flog.Trace("Request received: %s %s", r.Method, r.URL.Path) -// func handleItemGroup(w http.ResponseWriter, r *http.Request) { -// flog := logger.Default.WithPrefix("handleItemGroup") -// flog.Trace("Request received: %s %s", r.Method, r.URL.Path) + if r.Method != http.MethodGet { + flog.Debug("Method not allowed: %s", r.Method) + http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) + return + } -// if r.Method != http.MethodGet { -// flog.Debug("Method not allowed: %s", r.Method) -// http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) -// return -// } + flog.Debug("Parsing item ID from path: %s", r.URL.Path) + path := strings.TrimPrefix(r.URL.Path, "/api/items/") + path = strings.TrimSuffix(path, "/group") + flog.Trace("Trimmed path: %q", path) + parts := strings.Split(path, "/") + flog.Trace("Path parts: %v", parts) + if len(parts) == 0 { + flog.Error("Invalid path: no parts after splitting") + http.Error(w, "Invalid item ID", http.StatusBadRequest) + return + } + itemID, err := strconv.ParseInt(parts[0], 10, 64) + if err != nil { + flog.Error("Failed to parse item ID %q: %v", parts[0], err) + http.Error(w, "Invalid item ID", http.StatusBadRequest) + return + } + flog.Info("Item ID: %d", itemID) -// flog.Debug("Parsing item ID from path: %s", r.URL.Path) -// path := strings.TrimPrefix(r.URL.Path, "/api/items/") -// path = strings.TrimSuffix(path, "/group") -// flog.Trace("Trimmed path: %q", path) -// parts := strings.Split(path, "/") -// flog.Trace("Path parts: %v", parts) -// if len(parts) == 0 { -// flog.Error("Invalid path: no parts after splitting") -// http.Error(w, "Invalid item ID", http.StatusBadRequest) -// return -// } -// itemID, err := strconv.ParseInt(parts[0], 10, 64) -// if err != nil { -// flog.Error("Failed to parse item ID %q: %v", parts[0], err) -// http.Error(w, "Invalid item ID", http.StatusBadRequest) -// return -// } -// flog.Info("Item ID: %d", itemID) + db, err := GetDB() + if err != nil { + flog.Error("Failed to get database: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } + flog.Trace("Database connection obtained") -// db, err := GetDB() -// if err != nil { -// flog.Error("Failed to get database: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Trace("Database connection obtained") + flog.Debug("Getting item type for itemID: %d", itemID) + items, err := db.GetItemTypes([]int64{itemID}) + if err != nil { + flog.Error("Failed to get item type: %v", err) + http.Error(w, "Internal server error", http.StatusInternalServerError) + return + } -// flog.Debug("Getting group for itemID: %d", itemID) -// groupID, err := db.GetItemGroup(itemID) -// if err != nil { -// if strings.Contains(err.Error(), "record not found") { -// flog.Info("Item not found: typeID %d", itemID) -// http.Error(w, "Item not found", http.StatusNotFound) -// return -// } -// flog.Error("Failed to get item group: %v", err) -// http.Error(w, "Internal server error", http.StatusInternalServerError) -// return -// } -// flog.Info("Found groupID %d for itemID %d", groupID, itemID) + if len(items) == 0 { + flog.Info("Item not found: typeID %d", itemID) + http.Error(w, "Item not found", http.StatusNotFound) + return + } -// groupInfo := APIGroupInfo{ -// GroupID: groupID, -// } -// flog.Dump("Group Info", groupInfo) + item := items[0] + flog.Info("Found groupID %d for itemID %d", item.GroupID, itemID) -// w.Header().Set("Content-Type", "application/json") -// flog.Trace("Encoding response") -// if err := json.NewEncoder(w).Encode(groupInfo); err != nil { -// flog.Error("Failed to encode response: %v", err) -// return -// } -// flog.Info("Response sent successfully") -// } + groupInfo := APIGroupInfo{ + GroupID: int64(item.GroupID), + } + flog.Dump("Group Info", groupInfo) -// func corsMiddleware(next http.HandlerFunc) http.HandlerFunc { -// return func(w http.ResponseWriter, r *http.Request) { -// w.Header().Set("Access-Control-Allow-Origin", "*") -// w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") -// w.Header().Set("Access-Control-Allow-Headers", "Content-Type") + w.Header().Set("Content-Type", "application/json") + flog.Trace("Encoding response") + if err := json.NewEncoder(w).Encode(groupInfo); err != nil { + flog.Error("Failed to encode response: %v", err) + return + } + flog.Info("Response sent successfully") +} -// if r.Method == http.MethodOptions { -// w.WriteHeader(http.StatusNoContent) -// return -// } +func corsMiddleware(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS") + w.Header().Set("Access-Control-Allow-Headers", "Content-Type") -// next(w, r) -// } -// } + if r.Method == http.MethodOptions { + w.WriteHeader(http.StatusNoContent) + return + } + + next(w, r) + } +} // func handleImage(w http.ResponseWriter, r *http.Request) { // flog := logger.Default.WithPrefix("handleImage") @@ -698,29 +699,30 @@ const cacheTTL = 24 * time.Hour // json.NewEncoder(w).Encode(APIImageBatchResponse{Images: results}) // } -// func StartAPIServer(port string) { -// flog := logger.Default.WithPrefix("StartAPIServer") -// flog.Info("Initializing API server") +func StartAPIServer(port string) { + flog := logger.Default.WithPrefix("StartAPIServer") + flog.Info("Initializing API server") -// mux := http.NewServeMux() -// mux.HandleFunc("/api/statistics", corsMiddleware(handleStatistics)) -// mux.HandleFunc("/api/search", corsMiddleware(handleSearch)) -// mux.HandleFunc("/api/items/names", corsMiddleware(handleItemNames)) -// mux.HandleFunc("/api/items/", corsMiddleware(handleItemGroup)) -// mux.HandleFunc("/api/images/batch", corsMiddleware(handleImageBatch)) -// mux.HandleFunc("/api/images/", corsMiddleware(handleImage)) + mux := http.NewServeMux() + mux.HandleFunc("/api/statistics", corsMiddleware(handleStatistics)) + mux.HandleFunc("/api/search", corsMiddleware(handleSearch)) + mux.HandleFunc("/api/items/", corsMiddleware(func(w http.ResponseWriter, r *http.Request) { + if strings.HasSuffix(r.URL.Path, "/group") { + handleItemGroup(w, r) + } else { + handleItemNames(w, r) + } + })) -// flog.Debug("Registered routes:") -// flog.Debug(" POST /api/statistics") -// flog.Debug(" GET /api/search") -// flog.Debug(" GET /api/items/names") -// flog.Debug(" GET /api/items/{id}/group") -// flog.Debug(" POST /api/images/batch") -// flog.Debug(" GET /api/images/{typeId}/{size}") + flog.Debug("Registered routes:") + flog.Debug(" POST /api/statistics") + flog.Debug(" GET /api/search") + flog.Debug(" GET /api/items/ (names)") + flog.Debug(" GET /api/items/{id}/group") -// flog.Info("Starting API server on port %s", port) -// flog.Trace("Listening on :%s", port) -// if err := http.ListenAndServe(":"+port, mux); err != nil { -// flog.Error("Failed to start API server: %v", err) -// } -// } + flog.Info("Starting API server on port %s", port) + flog.Trace("Listening on :%s", port) + if err := http.ListenAndServe(":"+port, mux); err != nil { + flog.Error("Failed to start API server: %v", err) + } +} diff --git a/clickhouse.go b/clickhouse.go index 79e1c48..4ed7281 100644 --- a/clickhouse.go +++ b/clickhouse.go @@ -3,6 +3,7 @@ package main import ( "context" "fmt" + "strings" "git.site.quack-lab.dev/dave/cyutils" "github.com/ClickHouse/clickhouse-go/v2/lib/driver" @@ -200,7 +201,15 @@ func (db *DBWrapper) checkExistingKillmails(ctx context.Context, killmailIDs []i return make(map[int64]struct{}), nil } - rows, err := db.ch.Query(ctx, "SELECT killmail_id FROM killmails WHERE killmail_id IN ?", killmailIDs) + placeholders := make([]string, len(killmailIDs)) + args := make([]interface{}, len(killmailIDs)) + for j, id := range killmailIDs { + placeholders[j] = "?" + args[j] = id + } + + query := fmt.Sprintf("SELECT killmail_id FROM killmails WHERE killmail_id IN (%s)", strings.Join(placeholders, ",")) + rows, err := db.ch.Query(ctx, query, args...) if err != nil { return nil, err } @@ -213,5 +222,6 @@ func (db *DBWrapper) checkExistingKillmails(ctx context.Context, killmailIDs []i existing[id] = struct{}{} } } + return existing, nil } diff --git a/db.go b/db.go index f9199be..207acc7 100644 --- a/db.go +++ b/db.go @@ -17,15 +17,6 @@ import ( "gorm.io/gorm/schema" ) -// Add methods to QueryParams for template use -func (qp QueryParams) HasModules() bool { - return len(qp.Modules) > 0 -} - -type ModuleStatsData struct { - KillmailIDs []int64 -} - type DB interface { Init() error Get() *gorm.DB @@ -191,15 +182,49 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { modules := deduplicateInt64(params.Modules) flog.Debug("Deduplicated modules: %d -> %d", len(params.Modules), len(modules)) - // Create a copy of params with deduplicated modules for template use - templateParams := params - templateParams.Modules = modules + // Build the base query - start with all killmails + baseQuery := ` + SELECT + fk.killmail_id, + fk.solar_system_id, + fk.victim_ship_type_id + FROM killmails fk` - // Generate query using template - query, args, err := ExecuteTemplate(MainQueryTmpl, templateParams) - if err != nil { - flog.Error("Failed to execute main query template: %v", err) - return nil, err + args := []interface{}{} + whereClauses := []string{} + + // Apply filters + if params.Ship > 0 { + whereClauses = append(whereClauses, "fk.victim_ship_type_id = ?") + args = append(args, params.Ship) + } + + if len(params.Systems) > 0 { + placeholders := make([]string, len(params.Systems)) + for i := range params.Systems { + placeholders[i] = "?" + args = append(args, params.Systems[i]) + } + whereClauses = append(whereClauses, "fk.solar_system_id IN ("+strings.Join(placeholders, ",")+")") + } + + // For module filters, we need to join with modules + var moduleJoin string + if len(modules) > 0 { + placeholders := make([]string, len(modules)) + for i := range modules { + placeholders[i] = "?" + args = append(args, modules[i]) + } + moduleJoin = ` + INNER JOIN modules fm ON fk.killmail_id = fm.killmail_id` + whereClauses = append(whereClauses, "fm.item_type_id IN ("+strings.Join(placeholders, ",")+")") + } + + // Build final query + query := baseQuery + moduleJoin + if len(whereClauses) > 0 { + query += " WHERE " + strings.Join(whereClauses, " AND ") } flog.Debug("Executing query: %s", query) @@ -244,14 +269,14 @@ func (db *DBWrapper) QueryFits(params QueryParams) (*FitStatistics, error) { } // Calculate ship breakdown from database - shipBreakdown, err := db.calculateShipBreakdown(templateParams, totalKillmails, flog) + shipBreakdown, err := db.calculateShipBreakdown(query, args, totalKillmails, flog) if err != nil { flog.Error("Failed to calculate ship breakdown: %v", err) return nil, err } // Calculate system breakdown from database - systemBreakdown, err := db.calculateSystemBreakdown(templateParams, totalKillmails, flog) + systemBreakdown, err := db.calculateSystemBreakdown(query, args, totalKillmails, flog) if err != nil { flog.Error("Failed to calculate system breakdown: %v", err) return nil, err @@ -362,6 +387,150 @@ func deduplicateInt64(slice []int64) []int64 { return result } +func (db *DBWrapper) calculateStats(params QueryParams, shipTypeIDs []int64, killmailIDs []int64, stats *FitStatistics, total int64, flog *logger.Logger) error { + // if total == 0 { + // return nil + // } + + // ctx := context.Background() + + // var query string + // var args []interface{} + + // if len(killmailIDs) > 0 && len(killmailIDs) < 100000 { + // placeholders := make([]string, len(killmailIDs)) + // for i := range killmailIDs { + // placeholders[i] = "?" + // args = append(args, killmailIDs[i]) + // } + // query = "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM modules WHERE killmail_id IN (" + strings.Join(placeholders, ",") + ")" + // } else { + // var shipPlaceholders []string + // if len(shipTypeIDs) > 0 { + // shipPlaceholders = make([]string, len(shipTypeIDs)) + // for i, shipID := range shipTypeIDs { + // shipPlaceholders[i] = "?" + // args = append(args, shipID) + // } + // } else if params.Ship != 0 { + // shipPlaceholders = []string{"?"} + // args = []interface{}{params.Ship} + // } + + // if len(shipPlaceholders) > 0 { + // query = "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM modules WHERE victim_ship_type_id IN (" + strings.Join(shipPlaceholders, ",") + ")" + // } else { + // query = "SELECT item_type_id, flag, count(DISTINCT killmail_id) as count FROM modules" + // } + + // if len(params.Systems) > 0 { + // sysPlaceholders := make([]string, len(params.Systems)) + // for i := range params.Systems { + // sysPlaceholders[i] = "?" + // args = append(args, params.Systems[i]) + // } + // if len(shipPlaceholders) > 0 { + // query += " AND solar_system_id IN (" + strings.Join(sysPlaceholders, ",") + ")" + // } else { + // query += " WHERE solar_system_id IN (" + strings.Join(sysPlaceholders, ",") + ")" + // } + // } + // } + + // query += " GROUP BY item_type_id, flag" + + // rows, err := db.ch.Query(ctx, query, args...) + // if err != nil { + // flog.Error("Failed to query module stats: %v", err) + // return err + // } + // defer rows.Close() + + // // Map to aggregate counts per item_type_id (not per flag) + // itemCounts := make(map[int64]uint64) + // itemFlags := make(map[int64]int64) + + // for rows.Next() { + // var itemTypeID, flag int64 + // var count uint64 + // if err := rows.Scan(&itemTypeID, &flag, &count); err != nil { + // flog.Error("Failed to scan module stat: %v", err) + // return err + // } + // // Only count fitted modules - ignore cargo (flag 5) and other non-module flags + // if flag < 11 || (flag > 34 && flag != 87 && (flag < 92 || flag > 99)) { + // continue + // } + // // Aggregate: if we've seen this item_type_id before, use the max count (should be same, but just in case) + // if existing, exists := itemCounts[itemTypeID]; !exists || count > existing { + // itemCounts[itemTypeID] = count + // itemFlags[itemTypeID] = flag + // } + // } + + // // For filtered modules, they should be in 100% of fits - ADD THEM FIRST + // filteredModules := make(map[int64]bool) + // moduleSlots := make(map[int64]string) + // if len(params.Modules) > 0 { + // slots, err := db.getModuleSlots(params.Modules) + // if err == nil { + // moduleSlots = slots + // } + // for _, moduleID := range params.Modules { + // filteredModules[moduleID] = true + // // Add filtered modules immediately with 100% + // Stats := Stats{ + // Count: total, + // Percentage: 100.0, + // } + // slot, _ := moduleSlots[moduleID] + // switch slot { + // case "low": + // stats.LowSlotModules[int32(moduleID)] = Stats + // case "mid": + // stats.MidSlotModules[int32(moduleID)] = Stats + // case "high": + // stats.HighSlotModules[int32(moduleID)] = Stats + // case "rig": + // stats.Rigs[int32(moduleID)] = Stats + // case "drone": + // stats.Drones[int32(moduleID)] = Stats + // default: + // stats.HighSlotModules[int32(moduleID)] = Stats + // } + // } + // } + + // // Add all modules from query results (filtered ones already added with 100%) + // for itemTypeID, count := range itemCounts { + // if filteredModules[itemTypeID] { + // continue + // } + // percentage := float64(count) / float64(total) * 100.0 + // Stats := Stats{ + // Count: int64(count), + // Percentage: percentage, + // } + + // flag := itemFlags[itemTypeID] + // switch { + // case flag >= 11 && flag <= 18: + // stats.LowSlotModules[int32(itemTypeID)] = Stats + // case flag >= 19 && flag <= 26: + // stats.MidSlotModules[int32(itemTypeID)] = Stats + // case flag >= 27 && flag <= 34: + // stats.HighSlotModules[int32(itemTypeID)] = Stats + // case flag >= 92 && flag <= 99: + // stats.Rigs[int32(itemTypeID)] = Stats + // case flag == 87: + // stats.Drones[int32(itemTypeID)] = Stats + // } + // } + + // return nil + return nil +} + func (db *DBWrapper) CacheSet(key string, data []byte) error { flog := logger.Default.WithPrefix("CacheSet").WithPrefix(key) flog.Dump("data", data) @@ -453,14 +622,15 @@ func limitKillmails(killmailIDs []int64, limit int) []int64 { return killmailIDs[:limit] } -func (db *DBWrapper) calculateShipBreakdown(templateData QueryParams, totalKillmails int64, flog *cylogger.Logger) (map[int64]int64, error) { +func (db *DBWrapper) calculateShipBreakdown(baseQuery string, args []interface{}, totalKillmails int64, flog *cylogger.Logger) (map[int64]int64, error) { ctx := context.Background() - // Generate query using template - query, args, err := ExecuteTemplate(ShipBreakdownTmpl, templateData) - if err != nil { - return nil, fmt.Errorf("failed to execute ship breakdown template: %w", err) - } + // Modify the base query to group by ship type + query := strings.Replace(baseQuery, "SELECT\n\t\t\tfk.killmail_id,\n\t\t\tfk.solar_system_id,\n\t\t\tfk.victim_ship_type_id\n\t\tFROM killmails fk", + "SELECT\n\t\t\tfk.victim_ship_type_id,\n\t\t\tCOUNT(*) as count\n\t\tFROM killmails fk", 1) + + // Add GROUP BY + query += " GROUP BY fk.victim_ship_type_id ORDER BY count DESC" flog.Debug("Ship breakdown query: %s", query) rows, err := db.ch.Query(ctx, query, args...) @@ -482,14 +652,15 @@ func (db *DBWrapper) calculateShipBreakdown(templateData QueryParams, totalKillm return shipBreakdown, nil } -func (db *DBWrapper) calculateSystemBreakdown(templateData QueryParams, totalKillmails int64, flog *cylogger.Logger) (map[int64]int64, error) { +func (db *DBWrapper) calculateSystemBreakdown(baseQuery string, args []interface{}, totalKillmails int64, flog *cylogger.Logger) (map[int64]int64, error) { ctx := context.Background() - // Generate query using template - query, args, err := ExecuteTemplate(SystemBreakdownTmpl, templateData) - if err != nil { - return nil, fmt.Errorf("failed to execute system breakdown template: %w", err) - } + // Modify the base query to group by solar system + query := strings.Replace(baseQuery, "SELECT\n\t\t\tfk.killmail_id,\n\t\t\tfk.solar_system_id,\n\t\t\tfk.victim_ship_type_id\n\t\tFROM killmails fk", + "SELECT\n\t\t\tfk.solar_system_id,\n\t\t\tCOUNT(*) as count\n\t\tFROM killmails fk", 1) + + // Add GROUP BY + query += " GROUP BY fk.solar_system_id ORDER BY count DESC" flog.Debug("System breakdown query: %s", query) rows, err := db.ch.Query(ctx, query, args...) diff --git a/main.go b/main.go index 40479d2..84303a5 100644 --- a/main.go +++ b/main.go @@ -13,7 +13,7 @@ import ( func main() { ingest := flag.Bool("ingest", false, "ingest killmails from data directory") server := flag.Bool("server", false, "start API server") - // port := flag.String("port", "3000", "API server port") + port := flag.String("port", "3000", "API server port") flag.Parse() logger.InitFlag() logger.Default = logger.Default.ToFile("zkill.log") @@ -31,13 +31,13 @@ func main() { } if *server { - // StartAPIServer(*port) + StartAPIServer(*port) return } logger.Info("Querying fits") params := QueryParams{ - Ship: 32872, + Ship: 32872, } stats, err := db.QueryFits(params) if err != nil { diff --git a/types.go b/types.go index 8ef17f7..8a0e182 100644 --- a/types.go +++ b/types.go @@ -123,6 +123,10 @@ type QueryParams struct { KillmailLimit int } +type ModuleStatsData struct { + KillmailIDs []int64 +} + // CacheEntry stores both statistics (JSON) and images (blobs) in unified cache // For 404s, we store a special marker: []byte{0xFF, 0xFE, 0xFD} (NOT_FOUND_MARKER) type CacheEntry struct {