Implement parallel processing of instructions

main.go

@@ -7,8 +7,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"sync"
 	"sync/atomic"
+	utils "git.site.quack-lab.dev/dave/cyutils"
 )
 
 const deliminer = ","
@@ -130,28 +130,35 @@ func handleStatusErrors(status chan error) {
 	}
 }
 
-// processInstructions processes all instructions from the channel
+// processInstructions processes all instructions from the channel using parallel workers
 func processInstructions(instructions chan *LinkInstruction) int32 {
 	var instructionsDone int32 = 0
-	var wg sync.WaitGroup
+
+	// Collect all instructions first
+	var allInstructions []*LinkInstruction
 	for {
 		instruction, ok := <-instructions
 		if !ok {
 			LogInfo("No more instructions to process")
 			break
 		}
+		allInstructions = append(allInstructions, instruction)
+	}
+
+	// Process instructions in parallel using cyutils.WithWorkers
+	// Let the library handle worker count - use 4 workers as a reasonable default
+	utils.WithWorkers(4, allInstructions, func(workerID int, _ int, instruction *LinkInstruction) {
 		LogInfo("Processing: %s", instruction.String())
 		status := make(chan error)
 		go instruction.RunAsync(status)
-		wg.Add(1)
 		err := <-status
 		if err != nil {
 			LogError("Failed processing instruction: %v", err)
+		} else {
+			atomic.AddInt32(&instructionsDone, 1)
 		}
-		atomic.AddInt32(&instructionsDone, 1)
-		wg.Done()
-	}
-	wg.Wait()
+	})
+
 	return instructionsDone
 }