dave/BigChef
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Implement parallel file processing

Browse Source
This commit is contained in:
PhatPhuckDave
2025-03-27 22:22:43 +01:00
parent 89eed3f847
commit 4e4b7bbd19
1 changed files with 34 additions and 33 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
main.go
View File

@@ -3,7 +3,6 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"log"
"os" "os"
"sort" "sort"
"sync" "sync"
@@ -13,7 +12,6 @@ import (
"github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5"
"modify/logger" "modify/logger"
"modify/processor"
) )
type GlobalStats struct { type GlobalStats struct {
@@ -90,36 +88,45 @@ func main() {
// TODO: Utilize parallel workers for this // TODO: Utilize parallel workers for this
// Then for each file run all commands associated with the file // Then for each file run all commands associated with the file
workers := make(chan struct{}, *utils.ParallelFiles)
wg := sync.WaitGroup{}
for file, commands := range associations { for file, commands := range associations {
fileData, err := os.ReadFile(file) workers <- struct{}{}
if err != nil { wg.Add(1)
logger.Error("Failed to read file %q: %v", file, err) go func(file string, commands []utils.ModifyCommand) {
return defer func() { <-workers }()
} defer wg.Done()
logger.Trace("Loaded %d bytes of data for file %q", len(fileData), file)
fileDataStr := string(fileData)
// Aggregate all the modifications and execute them fileData, err := os.ReadFile(file)
modifications := []utils.ReplaceCommand{} if err != nil {
for _, command := range commands { logger.Error("Failed to read file %q: %v", file, err)
logger.Info("Processing file %q with command %q", file, command.Pattern) return
// TODO: Run processor and return modifications }
} logger.Trace("Loaded %d bytes of data for file %q", len(fileData), file)
fileDataStr := string(fileData)
// Sort commands in reverse order for safe replacements // Aggregate all the modifications and execute them
sort.Slice(modifications, func(i, j int) bool { modifications := []utils.ReplaceCommand{}
return modifications[i].From > modifications[j].From for _, command := range commands {
}) logger.Info("Processing file %q with command %q", file, command.Pattern)
logger.Trace("Applying %d replacement commands in reverse order", len(modifications)) // TODO: Run processor and return modifications
}
fileDataStr, count := utils.ExecuteModifications(modifications, fileDataStr) // Sort commands in reverse order for safe replacements
logger.Info("Executed %d modifications for file %q", count, file) sort.Slice(modifications, func(i, j int) bool {
return modifications[i].From > modifications[j].From
})
logger.Trace("Applying %d replacement commands in reverse order", len(modifications))
err = os.WriteFile(file, []byte(fileDataStr), 0644) fileDataStr, count := utils.ExecuteModifications(modifications, fileDataStr)
if err != nil { logger.Info("Executed %d modifications for file %q", count, file)
logger.Error("Failed to write file %q: %v", file, err)
return err = os.WriteFile(file, []byte(fileDataStr), 0644)
} if err != nil {
logger.Error("Failed to write file %q: %v", file, err)
return
}
}(file, commands)
} }
// This will also relieve processor of some of the file loading // This will also relieve processor of some of the file loading
@@ -169,12 +176,6 @@ func main() {
// return // return
// } // }
// Create the processor based on mode
var proc processor.Processor = &processor.RegexProcessor{}
var wg sync.WaitGroup
log.Printf("%#v", proc)
log.Printf("%#v", wg)
// Process each file // Process each file
// for _, file := range files { // for _, file := range files {
// wg.Add(1) // wg.Add(1)