Great use of json you clanker...

mprocs.yaml

@@ -74,5 +74,29 @@ procs:
       CLICKHOUSE_USERNAME: "default"
       CLICKHOUSE_PASSWORD: ""
     autostart: false
+    stop:
+      send-keys: ["<C-c>"]
+      
+  missing-attacker-character-name-reader:
+    shell: "go run ."
+    cwd: "pipeline/missing-attacker-character-name-reader"
+    env:
+      CLICKHOUSE_HOST: "clickhouse-zkill.site.quack-lab.dev"
+      CLICKHOUSE_DATABASE: "zkill"
+      CLICKHOUSE_USERNAME: "default"
+      CLICKHOUSE_PASSWORD: ""
+    autostart: false
+    stop:
+      send-keys: ["<C-c>"]
+
+  missing-victim-character-name-reader:
+    shell: "go run ."
+    cwd: "pipeline/missing-victim-character-name-reader"
+    env:
+      CLICKHOUSE_HOST: "clickhouse-zkill.site.quack-lab.dev"
+      CLICKHOUSE_DATABASE: "zkill"
+      CLICKHOUSE_USERNAME: "default"
+      CLICKHOUSE_PASSWORD: ""
+    autostart: false
     stop:
       send-keys: ["<C-c>"]

pipeline/missing-attacker-character-name-reader/main.go

@@ -20,10 +20,6 @@ const (
 	PollInterval = 30 * time.Second
 )
 
-type AttackerCharacterIDMessage struct {
-	CharacterID int64 `json:"character_id"`
-}
-
 func main() {
 	if err := config.InitConfig(); err != nil {
 		logger.Error("Failed to initialize config: %v", err)
@@ -99,11 +95,7 @@ func processMissingAttackerCharacterNames(ctx context.Context, chClient *clickho
 				return ctx.Err()
 			}
 
-			message := AttackerCharacterIDMessage{
-				CharacterID: characterID,
-			}
-
-			messageBytes, err := json.Marshal(message)
+			messageBytes, err := json.Marshal(characterID)
 			if err != nil {
 				logger.Error("Failed to marshal attacker character ID message: %v", err)
 				continue
@@ -130,10 +122,15 @@ func processMissingAttackerCharacterNames(ctx context.Context, chClient *clickho
 
 func queryMissingAttackerCharacterNames(ctx context.Context, chClient *clickhouse.ClickhouseClient) ([]int64, error) {
 	query := `
-		SELECT DISTINCT character_id
-		FROM zkill.killmail_attackers
-		WHERE character_name = '' OR character_name IS NULL
-		ORDER BY character_id
+SELECT
+	DISTINCT character_id
+FROM
+	zkill.killmail_attackers
+WHERE
+	character_name = ''
+	OR character_name IS NULL
+ORDER BY
+	character_id
 	`
 
 	rows, err := chClient.Query(ctx, query)

pipeline/missing-victim-character-name-reader/main.go

@@ -20,10 +20,6 @@ const (
 	PollInterval = 30 * time.Second
 )
 
-type VictimCharacterIDMessage struct {
-	VictimCharacterID int64 `json:"victim_character_id"`
-}
-
 func main() {
 	if err := config.InitConfig(); err != nil {
 		logger.Error("Failed to initialize config: %v", err)
@@ -99,11 +95,7 @@ func processMissingVictimCharacterNames(ctx context.Context, chClient *clickhous
 				return ctx.Err()
 			}
 
-			message := VictimCharacterIDMessage{
-				VictimCharacterID: victimCharacterID,
-			}
-
-			messageBytes, err := json.Marshal(message)
+			messageBytes, err := json.Marshal(victimCharacterID)
 			if err != nil {
 				logger.Error("Failed to marshal victim character ID message: %v", err)
 				continue
@@ -130,10 +122,15 @@ func processMissingVictimCharacterNames(ctx context.Context, chClient *clickhous
 
 func queryMissingVictimCharacterNames(ctx context.Context, chClient *clickhouse.ClickhouseClient) ([]int64, error) {
 	query := `
-		SELECT DISTINCT victim_character_id
-		FROM zkill.killmails
-		WHERE victim_character_name = '' OR victim_character_name IS NULL
-		ORDER BY victim_character_id
+SELECT
+	DISTINCT victim_character_id
+FROM
+	zkill.killmails
+WHERE
+	victim_character_name = ''
+	OR victim_character_name IS NULL
+ORDER BY
+	victim_character_id
 	`
 
 	rows, err := chClient.Query(ctx, query)

pipeline/nsq-to-clickhouse-reader/main.go

@@ -173,5 +173,5 @@ func main() {
 
 	logger.Info("Connected to NSQ, consuming from topic: %s, channel: %s", Topic, Channel)
 
-	logger.Info("Stage 2 completed")
+	select {}
 }