Restore the parallel processing
This commit is contained in:
32
main.go
32
main.go
@@ -10,6 +10,7 @@ import (
|
||||
_ "embed"
|
||||
|
||||
logger "git.site.quack-lab.dev/dave/cylogger"
|
||||
utils "git.site.quack-lab.dev/dave/cyutils"
|
||||
"github.com/PuerkitoBio/goquery"
|
||||
)
|
||||
|
||||
@@ -34,26 +35,35 @@ func main() {
|
||||
classGroups[baseName] = append(classGroups[baseName], file)
|
||||
}
|
||||
|
||||
// Process each group
|
||||
// Convert groups to work items for parallel processing
|
||||
var workItems []WorkItem
|
||||
for baseName, groupFiles := range classGroups {
|
||||
if len(groupFiles) == 1 {
|
||||
workItems = append(workItems, WorkItem{
|
||||
BaseName: baseName,
|
||||
GroupFiles: groupFiles,
|
||||
})
|
||||
}
|
||||
|
||||
// Process each group in parallel
|
||||
utils.WithWorkers(100, workItems, func(worker int, item WorkItem) {
|
||||
if len(item.GroupFiles) == 1 {
|
||||
// Single file, process normally
|
||||
class, err := ParseClass(groupFiles[0])
|
||||
class, err := ParseClass(item.GroupFiles[0])
|
||||
if err != nil {
|
||||
logger.Error("Error parsing file: %v", err)
|
||||
continue
|
||||
return
|
||||
}
|
||||
class.Write(*outdir, classTemplate)
|
||||
} else {
|
||||
// Multiple files for same class, merge them
|
||||
mergedClass, err := MergeClasses(groupFiles)
|
||||
mergedClass, err := MergeClasses(item.GroupFiles)
|
||||
if err != nil {
|
||||
logger.Error("Error merging classes for %s: %v", baseName, err)
|
||||
continue
|
||||
logger.Error("Error merging classes for %s: %v", item.BaseName, err)
|
||||
return
|
||||
}
|
||||
mergedClass.Write(*outdir, classTemplate)
|
||||
}
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func MapType(t string) string {
|
||||
@@ -267,3 +277,9 @@ func getOriginalClassName(file string) string {
|
||||
|
||||
return strings.TrimSpace(class.Text())
|
||||
}
|
||||
|
||||
// WorkItem represents a group of files to be processed
|
||||
type WorkItem struct {
|
||||
BaseName string
|
||||
GroupFiles []string
|
||||
}
|
||||
|
Reference in New Issue
Block a user