package main import ( "bytes" "crypto/sha256" "encoding/hex" "encoding/json" "fmt" "io" "net/http" "os" logger "git.site.quack-lab.dev/dave/cylogger" ) var meiliToken string type IndexConfig struct { Uid string `json:"uid"` PrimaryKey string `json:"primaryKey"` } type ChatMessage struct { MessageHash string `json:"message_hash"` Timestamp string `json:"timestamp"` // ISO timestamp EpochTime int64 `json:"epoch_time"` // Unix epoch timestamp for filtering Event string `json:"event"` Sender string `json:"sender"` Msg string `json:"msg"` Language string `json:"language"` Channel string `json:"channel"` } func Init() error { // Load Meilisearch token meiliToken = os.Getenv("MEILI_TOKEN") if meiliToken == "" { return fmt.Errorf("MEILI_TOKEN environment variable not set") } logger.Info("Meilisearch token loaded") config := IndexConfig{ Uid: meiliIndex, PrimaryKey: "message_hash", // Meilisearch will use this for deduplication } // Create index err := createIndex(config) if err != nil { return fmt.Errorf("error creating index: %v", err) } // Set up index settings err = setIndexSettings() if err != nil { return fmt.Errorf("error setting index settings: %v", err) } return nil } // GenerateMessageHash creates a unique hash for a message that will be identical // for identical messages, ensuring perfect deduplication func GenerateMessageHash(timestamp, event, sender, msg, language, channel string) string { // Combine all fields that make a message unique content := fmt.Sprintf("%s|%s|%s|%s|%s|%s", timestamp, event, sender, msg, language, channel, ) // Create SHA-256 hash of the combined content hash := sha256.Sum256([]byte(content)) return hex.EncodeToString(hash[:]) } // AddMessages adds multiple messages to the index in a single batch request // Meilisearch will handle deduplication based on the message_hash func AddMessages(messages []ChatMessage) error { jsonData, err := json.Marshal(messages) if err != nil { return fmt.Errorf("error marshaling messages: %v", err) } req, err := http.NewRequest( http.MethodPost, meiliEndpoint+"indexes/"+meiliIndex+"/documents", bytes.NewBuffer(jsonData), ) if err != nil { return fmt.Errorf("error creating request: %v", err) } req.Header.Set("Authorization", "Bearer "+meiliToken) req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { return fmt.Errorf("error adding messages: %v", err) } defer resp.Body.Close() // Read response body for better error messages body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("error reading response body: %v", err) } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { return fmt.Errorf("failed to add messages. Status: %d, Response: %s", resp.StatusCode, string(body)) } return nil } func createIndex(config IndexConfig) error { jsonData, err := json.Marshal(config) if err != nil { return fmt.Errorf("error marshaling config: %v", err) } req, err := http.NewRequest( http.MethodPost, meiliEndpoint+"indexes", bytes.NewBuffer(jsonData), ) if err != nil { return fmt.Errorf("error creating request: %v", err) } req.Header.Set("Authorization", "Bearer "+meiliToken) req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { return fmt.Errorf("error creating index: %v", err) } defer resp.Body.Close() // Read response body for better error messages body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("error reading response body: %v", err) } if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted { return fmt.Errorf("failed to create index. Status: %d, Response: %s", resp.StatusCode, string(body)) } logger.Info("Index created successfully") return nil } func setIndexSettings() error { // First set searchable attributes searchableAttributes := []string{ "timestamp", "event", "sender", "msg", "language", "channel", } jsonData, err := json.Marshal(searchableAttributes) if err != nil { return fmt.Errorf("error marshaling searchable settings: %v", err) } req, err := http.NewRequest( http.MethodPut, meiliEndpoint+"indexes/"+meiliIndex+"/settings/searchable-attributes", bytes.NewBuffer(jsonData), ) if err != nil { return fmt.Errorf("error creating searchable settings request: %v", err) } req.Header.Set("Authorization", "Bearer "+meiliToken) req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { return fmt.Errorf("error setting searchable attributes: %v", err) } defer resp.Body.Close() body, err := io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("error reading searchable settings response: %v", err) } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { return fmt.Errorf("failed to set searchable attributes. Status: %d, Response: %s", resp.StatusCode, string(body)) } // Then set filterable attributes filterableAttributes := []string{ "timestamp", "epoch_time", // Add epoch_time for numeric filtering "event", "sender", "language", "channel", } jsonData, err = json.Marshal(filterableAttributes) if err != nil { return fmt.Errorf("error marshaling filterable settings: %v", err) } req, err = http.NewRequest( http.MethodPut, meiliEndpoint+"indexes/"+meiliIndex+"/settings/filterable-attributes", bytes.NewBuffer(jsonData), ) if err != nil { return fmt.Errorf("error creating filterable settings request: %v", err) } req.Header.Set("Authorization", "Bearer "+meiliToken) req.Header.Set("Content-Type", "application/json") resp, err = client.Do(req) if err != nil { return fmt.Errorf("error setting filterable attributes: %v", err) } defer resp.Body.Close() body, err = io.ReadAll(resp.Body) if err != nil { return fmt.Errorf("error reading filterable settings response: %v", err) } if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted { return fmt.Errorf("failed to set filterable attributes. Status: %d, Response: %s", resp.StatusCode, string(body)) } logger.Info("Index settings set successfully") return nil }