diff --git a/clickhouse.go b/clickhouse.go index 842fbee..4ed7281 100644 --- a/clickhouse.go +++ b/clickhouse.go @@ -132,10 +132,26 @@ func (db *DBWrapper) SaveKillmails(killmails []Killmail) error { return fmt.Errorf("failed to prepare modules batch: %w", err) } + var batchErrors []error cyutils.Batched(killmails, 1000, func(batch []Killmail) { - db.processBatch(ctx, batch, flatBatch, moduleBatch) + if err := db.processBatch(ctx, batch, flatBatch, moduleBatch); err != nil { + batchErrors = append(batchErrors, err) + } }) + // Send batches + if err := flatBatch.Send(); err != nil { + return fmt.Errorf("failed to send killmails batch: %w", err) + } + if err := moduleBatch.Send(); err != nil { + return fmt.Errorf("failed to send modules batch: %w", err) + } + + // Check for any batch processing errors + if len(batchErrors) > 0 { + return fmt.Errorf("batch processing errors: %v", batchErrors) + } + return nil } @@ -176,16 +192,6 @@ func (db *DBWrapper) processBatch(ctx context.Context, batch []Killmail, flatBat } } - // Send batches - err = flatBatch.Send() - if err != nil { - return err - } - err = moduleBatch.Send() - if err != nil { - return err - } - return nil }