Implement parallel file processing

This commit is contained in:
2025-03-27 22:22:43 +01:00
parent 89eed3f847
commit 4e4b7bbd19

17
main.go
View File

@@ -3,7 +3,6 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"log"
"os" "os"
"sort" "sort"
"sync" "sync"
@@ -13,7 +12,6 @@ import (
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
"modify/logger" "modify/logger"
"modify/processor"
) )
type GlobalStats struct { type GlobalStats struct {
@@ -90,7 +88,15 @@ func main() {
// TODO: Utilize parallel workers for this // TODO: Utilize parallel workers for this
// Then for each file run all commands associated with the file // Then for each file run all commands associated with the file
workers := make(chan struct{}, *utils.ParallelFiles)
wg := sync.WaitGroup{}
for file, commands := range associations { for file, commands := range associations {
workers <- struct{}{}
wg.Add(1)
go func(file string, commands []utils.ModifyCommand) {
defer func() { <-workers }()
defer wg.Done()
fileData, err := os.ReadFile(file) fileData, err := os.ReadFile(file)
if err != nil { if err != nil {
logger.Error("Failed to read file %q: %v", file, err) logger.Error("Failed to read file %q: %v", file, err)
@@ -120,6 +126,7 @@ func main() {
logger.Error("Failed to write file %q: %v", file, err) logger.Error("Failed to write file %q: %v", file, err)
return return
} }
}(file, commands)
} }
// This will also relieve processor of some of the file loading // This will also relieve processor of some of the file loading
@@ -169,12 +176,6 @@ func main() {
// return // return
// } // }
// Create the processor based on mode
var proc processor.Processor = &processor.RegexProcessor{}
var wg sync.WaitGroup
log.Printf("%#v", proc)
log.Printf("%#v", wg)
// Process each file // Process each file
// for _, file := range files { // for _, file := range files {
// wg.Add(1) // wg.Add(1)