wow-AchievementSniffer/service/chatsniffer/init_meili.go

package main

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"

	logger "git.site.quack-lab.dev/dave/cylogger"
)

var meiliToken string

type IndexConfig struct {
	Uid        string `json:"uid"`
	PrimaryKey string `json:"primaryKey"`
}

type ChatMessage struct {
	MessageHash string `json:"message_hash"`
	Timestamp   string `json:"timestamp"`  // ISO timestamp
	EpochTime   int64  `json:"epoch_time"` // Unix epoch timestamp for filtering
	Event       string `json:"event"`
	Sender      string `json:"sender"`
	Msg         string `json:"msg"`
	Language    string `json:"language"`
	Channel     string `json:"channel"`
}

func Init() error {
	// Load Meilisearch token
	meiliToken = os.Getenv("MEILI_TOKEN")
	if meiliToken == "" {
		return fmt.Errorf("MEILI_TOKEN environment variable not set")
	}
	logger.Info("Meilisearch token loaded")

	config := IndexConfig{
		Uid:        meiliIndex,
		PrimaryKey: "message_hash", // Meilisearch will use this for deduplication
	}

	// Create index
	err := createIndex(config)
	if err != nil {
		return fmt.Errorf("error creating index: %v", err)
	}

	// Set up index settings
	err = setIndexSettings()
	if err != nil {
		return fmt.Errorf("error setting index settings: %v", err)
	}

	return nil
}

// GenerateMessageHash creates a unique hash for a message that will be identical
// for identical messages, ensuring perfect deduplication
func GenerateMessageHash(timestamp, event, sender, msg, language, channel string) string {
	// Combine all fields that make a message unique
	content := fmt.Sprintf("%s|%s|%s|%s|%s|%s",
		timestamp,
		event,
		sender,
		msg,
		language,
		channel,
	)

	// Create SHA-256 hash of the combined content
	hash := sha256.Sum256([]byte(content))
	return hex.EncodeToString(hash[:])
}

// AddMessages adds multiple messages to the index in a single batch request
// Meilisearch will handle deduplication based on the message_hash
func AddMessages(messages []ChatMessage) error {
	jsonData, err := json.Marshal(messages)
	if err != nil {
		return fmt.Errorf("error marshaling messages: %v", err)
	}

	req, err := http.NewRequest(
		http.MethodPost,
		meiliEndpoint+"indexes/"+meiliIndex+"/documents",
		bytes.NewBuffer(jsonData),
	)
	if err != nil {
		return fmt.Errorf("error creating request: %v", err)
	}

	req.Header.Set("Authorization", "Bearer "+meiliToken)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("error adding messages: %v", err)
	}
	defer resp.Body.Close()

	// Read response body for better error messages
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("failed to add messages. Status: %d, Response: %s", resp.StatusCode, string(body))
	}

	return nil
}

func createIndex(config IndexConfig) error {
	jsonData, err := json.Marshal(config)
	if err != nil {
		return fmt.Errorf("error marshaling config: %v", err)
	}

	req, err := http.NewRequest(
		http.MethodPost,
		meiliEndpoint+"indexes",
		bytes.NewBuffer(jsonData),
	)
	if err != nil {
		return fmt.Errorf("error creating request: %v", err)
	}

	req.Header.Set("Authorization", "Bearer "+meiliToken)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("error creating index: %v", err)
	}
	defer resp.Body.Close()

	// Read response body for better error messages
	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("error reading response body: %v", err)
	}

	if resp.StatusCode != http.StatusCreated && resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("failed to create index. Status: %d, Response: %s", resp.StatusCode, string(body))
	}

	logger.Info("Index created successfully")
	return nil
}

func setIndexSettings() error {
	// First set searchable attributes
	searchableAttributes := []string{
		"timestamp",
		"event",
		"sender",
		"msg",
		"language",
		"channel",
	}

	jsonData, err := json.Marshal(searchableAttributes)
	if err != nil {
		return fmt.Errorf("error marshaling searchable settings: %v", err)
	}

	req, err := http.NewRequest(
		http.MethodPut,
		meiliEndpoint+"indexes/"+meiliIndex+"/settings/searchable-attributes",
		bytes.NewBuffer(jsonData),
	)
	if err != nil {
		return fmt.Errorf("error creating searchable settings request: %v", err)
	}

	req.Header.Set("Authorization", "Bearer "+meiliToken)
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return fmt.Errorf("error setting searchable attributes: %v", err)
	}
	defer resp.Body.Close()

	body, err := io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("error reading searchable settings response: %v", err)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("failed to set searchable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
	}

	// Then set filterable attributes
	filterableAttributes := []string{
		"timestamp",
		"epoch_time", // Add epoch_time for numeric filtering
		"event",
		"sender",
		"language",
		"channel",
	}

	jsonData, err = json.Marshal(filterableAttributes)
	if err != nil {
		return fmt.Errorf("error marshaling filterable settings: %v", err)
	}

	req, err = http.NewRequest(
		http.MethodPut,
		meiliEndpoint+"indexes/"+meiliIndex+"/settings/filterable-attributes",
		bytes.NewBuffer(jsonData),
	)
	if err != nil {
		return fmt.Errorf("error creating filterable settings request: %v", err)
	}

	req.Header.Set("Authorization", "Bearer "+meiliToken)
	req.Header.Set("Content-Type", "application/json")

	resp, err = client.Do(req)
	if err != nil {
		return fmt.Errorf("error setting filterable attributes: %v", err)
	}
	defer resp.Body.Close()

	body, err = io.ReadAll(resp.Body)
	if err != nil {
		return fmt.Errorf("error reading filterable settings response: %v", err)
	}

	if resp.StatusCode != http.StatusOK && resp.StatusCode != http.StatusAccepted {
		return fmt.Errorf("failed to set filterable attributes. Status: %d, Response: %s", resp.StatusCode, string(body))
	}

	logger.Info("Index settings set successfully")
	return nil
}