Implement parallel file processing
This commit is contained in:
17
main.go
17
main.go
@@ -3,7 +3,6 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"sort"
|
||||
"sync"
|
||||
@@ -13,7 +12,6 @@ import (
|
||||
"github.com/go-git/go-git/v5"
|
||||
|
||||
"modify/logger"
|
||||
"modify/processor"
|
||||
)
|
||||
|
||||
type GlobalStats struct {
|
||||
@@ -90,7 +88,15 @@ func main() {
|
||||
|
||||
// TODO: Utilize parallel workers for this
|
||||
// Then for each file run all commands associated with the file
|
||||
workers := make(chan struct{}, *utils.ParallelFiles)
|
||||
wg := sync.WaitGroup{}
|
||||
for file, commands := range associations {
|
||||
workers <- struct{}{}
|
||||
wg.Add(1)
|
||||
go func(file string, commands []utils.ModifyCommand) {
|
||||
defer func() { <-workers }()
|
||||
defer wg.Done()
|
||||
|
||||
fileData, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
logger.Error("Failed to read file %q: %v", file, err)
|
||||
@@ -120,6 +126,7 @@ func main() {
|
||||
logger.Error("Failed to write file %q: %v", file, err)
|
||||
return
|
||||
}
|
||||
}(file, commands)
|
||||
}
|
||||
|
||||
// This will also relieve processor of some of the file loading
|
||||
@@ -169,12 +176,6 @@ func main() {
|
||||
// return
|
||||
// }
|
||||
|
||||
// Create the processor based on mode
|
||||
var proc processor.Processor = &processor.RegexProcessor{}
|
||||
|
||||
var wg sync.WaitGroup
|
||||
log.Printf("%#v", proc)
|
||||
log.Printf("%#v", wg)
|
||||
// Process each file
|
||||
// for _, file := range files {
|
||||
// wg.Add(1)
|
||||
|
Reference in New Issue
Block a user