Restore the parallel processing
This commit is contained in:
32
main.go
32
main.go
@@ -10,6 +10,7 @@ import (
|
|||||||
_ "embed"
|
_ "embed"
|
||||||
|
|
||||||
logger "git.site.quack-lab.dev/dave/cylogger"
|
logger "git.site.quack-lab.dev/dave/cylogger"
|
||||||
|
utils "git.site.quack-lab.dev/dave/cyutils"
|
||||||
"github.com/PuerkitoBio/goquery"
|
"github.com/PuerkitoBio/goquery"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -34,26 +35,35 @@ func main() {
|
|||||||
classGroups[baseName] = append(classGroups[baseName], file)
|
classGroups[baseName] = append(classGroups[baseName], file)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process each group
|
// Convert groups to work items for parallel processing
|
||||||
|
var workItems []WorkItem
|
||||||
for baseName, groupFiles := range classGroups {
|
for baseName, groupFiles := range classGroups {
|
||||||
if len(groupFiles) == 1 {
|
workItems = append(workItems, WorkItem{
|
||||||
|
BaseName: baseName,
|
||||||
|
GroupFiles: groupFiles,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// Process each group in parallel
|
||||||
|
utils.WithWorkers(100, workItems, func(worker int, item WorkItem) {
|
||||||
|
if len(item.GroupFiles) == 1 {
|
||||||
// Single file, process normally
|
// Single file, process normally
|
||||||
class, err := ParseClass(groupFiles[0])
|
class, err := ParseClass(item.GroupFiles[0])
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("Error parsing file: %v", err)
|
logger.Error("Error parsing file: %v", err)
|
||||||
continue
|
return
|
||||||
}
|
}
|
||||||
class.Write(*outdir, classTemplate)
|
class.Write(*outdir, classTemplate)
|
||||||
} else {
|
} else {
|
||||||
// Multiple files for same class, merge them
|
// Multiple files for same class, merge them
|
||||||
mergedClass, err := MergeClasses(groupFiles)
|
mergedClass, err := MergeClasses(item.GroupFiles)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error("Error merging classes for %s: %v", baseName, err)
|
logger.Error("Error merging classes for %s: %v", item.BaseName, err)
|
||||||
continue
|
return
|
||||||
}
|
}
|
||||||
mergedClass.Write(*outdir, classTemplate)
|
mergedClass.Write(*outdir, classTemplate)
|
||||||
}
|
}
|
||||||
}
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
func MapType(t string) string {
|
func MapType(t string) string {
|
||||||
@@ -267,3 +277,9 @@ func getOriginalClassName(file string) string {
|
|||||||
|
|
||||||
return strings.TrimSpace(class.Text())
|
return strings.TrimSpace(class.Text())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// WorkItem represents a group of files to be processed
|
||||||
|
type WorkItem struct {
|
||||||
|
BaseName string
|
||||||
|
GroupFiles []string
|
||||||
|
}
|
||||||
|
Reference in New Issue
Block a user