Make recursive much much faster

By virtue of parallelism

main.go

@@ -28,6 +28,7 @@ const DefaultColor = White
 const PathColor = Green
 
 var DirRegex, _ = regexp.Compile(`^(.+?)[/\\]sync$`)
+var FileRegex, _ = regexp.Compile(`^sync$`)
 var programName = os.Args[0]
 
 func main() {
@@ -131,6 +132,7 @@ func ReadFromFilesRecursively(input string, output chan LinkInstruction, status
 		wg.Add(1)
 		go func() {
 			defer wg.Done()
+			log.Println(file)
 			file = NormalizePath(file)
 			log.Printf("Processing file: %s%s%s", PathColor, file, DefaultColor)
 

util.go

@@ -2,10 +2,12 @@ package main
 
 import (
 	"fmt"
-	"io/fs"
+	"log"
 	"os"
 	"path/filepath"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 )
 
@@ -67,29 +69,62 @@ func GetSyncFilesRecursively(input string, output chan string, status chan error
 	defer close(status)
 
 	var filesProcessed int32
+	var foldersProcessed int32
 	progressTicker := time.NewTicker(200 * time.Millisecond)
 	defer progressTicker.Stop()
 	go func() {
 		for {
+			fmt.Printf("\rFiles processed: %d; Folders processed: %d;", filesProcessed, foldersProcessed)
 			<-progressTicker.C
-			fmt.Printf("\rFiles processed: %d", filesProcessed)
 		}
 	}()
 
-	err := filepath.WalkDir(input, func(path string, file fs.DirEntry, err error) error {
+	var wg sync.WaitGroup
+	wg.Add(1)
+	var initial sync.Once
+	directories := make(chan string, 10000)
+	workerPool := make(chan struct{}, 10000)
+	directories <- input
+
+	log.Printf("%+v", len(workerPool))
+	go func() {
+		for directory := range directories {
+			workerPool <- struct{}{}
+			wg.Add(1)
+			go func(directory string) {
+				atomic.AddInt32(&foldersProcessed, 1)
+				defer wg.Done()
+				defer func() { <-workerPool }()
+
+				// log.Printf("Reading directory %s", directory)
+				files, err := os.ReadDir(directory)
 				if err != nil {
-			return err
+					log.Printf("Error reading directory %s: %+v", directory, err)
+					return
 				}
 
-		// Effectively only find files named "sync" (with no extension!!)
+				for _, file := range files {
-		if !file.IsDir() && DirRegex.MatchString(path) {
+					// log.Printf("Processing file %s", file.Name())
-			output <- path
+					if file.IsDir() {
+						directories <- filepath.Join(directory, file.Name())
+					} else {
+						// log.Println(file.Name(), DirRegex.MatchString(file.Name()))
+						if FileRegex.MatchString(file.Name()) {
+							log.Printf("Writing")
+							output <- filepath.Join(directory, file.Name())
 						}
-		filesProcessed++
+						atomic.AddInt32(&filesProcessed, 1)
+					}
+				}
+				// log.Printf("Done reading directory %s", directory)
 			
-		return nil
+				initial.Do(func() {
+					wg.Done()
 				})
-	if err != nil {
+			}(directory)
-		status <- err
 		}
+	}()
+
+	wg.Wait()
+	log.Printf("Done")
 }