package main import ( "archive/tar" "compress/bzip2" "encoding/json" "io" "os" logger "git.site.quack-lab.dev/dave/cylogger" ) func IterBz2Killmails(path string, out chan<- Killmail) error { log := logger.Default.WithPrefix(path) log.Debug("Iterating killmails") f, err := os.Open(path) if err != nil { return err } defer f.Close() bz2r := bzip2.NewReader(f) tarReader := tar.NewReader(bz2r) for { hdr, err := tarReader.Next() if err != nil { if err == io.EOF { log.Debug("EOF reached") break } return err } // Only regular files; skip directories etc if hdr.Typeflag != tar.TypeReg { log.Debug("Skipping non-regular file: %s", hdr.Name) continue } flog := log.WithPrefix(hdr.Name) flog.Debug("Reading file") data, err := io.ReadAll(tarReader) if err != nil { flog.Error("Failed to read file: %v", err) return err } var killmail Killmail if err := json.Unmarshal(data, &killmail); err != nil { flog.Error("Failed to unmarshal JSON: %v", err) return err } flog.Debug("Sending killmail to channel: %+v", killmail.KillmailID) out <- killmail } log.Debug("Closing channel") close(out) return nil } func LoadBz2Killmails(path string) ([]Killmail, error) { log := logger.Default.WithPrefix(path) log.Debug("Loading killmails") out := make(chan Killmail) go func() { err := IterBz2Killmails(path, out) if err != nil { log.Error("Failed to iterate over tar.bz2 file: %v", err) return } }() log.Debug("Receiving killmails") var killmails []Killmail for killmail := range out { killmails = append(killmails, killmail) log.Debug("Received killmail: %+v", killmail.KillmailID) } log.Debug("Closing channel") return killmails, nil }