Implement parallel file processing
This commit is contained in:
17
main.go
17
main.go
@@ -3,7 +3,6 @@ package main
|
|||||||
import (
|
import (
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
|
||||||
"os"
|
"os"
|
||||||
"sort"
|
"sort"
|
||||||
"sync"
|
"sync"
|
||||||
@@ -13,7 +12,6 @@ import (
|
|||||||
"github.com/go-git/go-git/v5"
|
"github.com/go-git/go-git/v5"
|
||||||
|
|
||||||
"modify/logger"
|
"modify/logger"
|
||||||
"modify/processor"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type GlobalStats struct {
|
type GlobalStats struct {
|
||||||
@@ -90,7 +88,15 @@ func main() {
|
|||||||
|
|
||||||
// TODO: Utilize parallel workers for this
|
// TODO: Utilize parallel workers for this
|
||||||
// Then for each file run all commands associated with the file
|
// Then for each file run all commands associated with the file
|
||||||
|
workers := make(chan struct{}, *utils.ParallelFiles)
|
||||||
|
wg := sync.WaitGroup{}
|
||||||
for file, commands := range associations {
|
for file, commands := range associations {
|
||||||
|
workers <- struct{}{}
|
||||||
|
wg.Add(1)
|
||||||
|
go func(file string, commands []utils.ModifyCommand) {
|
||||||
|
defer func() { <-workers }()
|
||||||
|
defer wg.Done()
|
||||||
|
|
||||||
fileData, err := os.ReadFile(file)
|
fileData, err := os.ReadFile(file)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("Failed to read file %q: %v", file, err)
|
logger.Error("Failed to read file %q: %v", file, err)
|
||||||
@@ -120,6 +126,7 @@ func main() {
|
|||||||
logger.Error("Failed to write file %q: %v", file, err)
|
logger.Error("Failed to write file %q: %v", file, err)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
}(file, commands)
|
||||||
}
|
}
|
||||||
|
|
||||||
// This will also relieve processor of some of the file loading
|
// This will also relieve processor of some of the file loading
|
||||||
@@ -169,12 +176,6 @@ func main() {
|
|||||||
// return
|
// return
|
||||||
// }
|
// }
|
||||||
|
|
||||||
// Create the processor based on mode
|
|
||||||
var proc processor.Processor = &processor.RegexProcessor{}
|
|
||||||
|
|
||||||
var wg sync.WaitGroup
|
|
||||||
log.Printf("%#v", proc)
|
|
||||||
log.Printf("%#v", wg)
|
|
||||||
// Process each file
|
// Process each file
|
||||||
// for _, file := range files {
|
// for _, file := range files {
|
||||||
// wg.Add(1)
|
// wg.Add(1)
|
||||||
|
Reference in New Issue
Block a user