Maybe fix the directory crawler

README.md

@@ -4,27 +4,81 @@ A small Go tool for creating symbolic links
 
 Created out of infuriating difficulty of creating symbolic links on windows
 
-## Custom syntax
+## Instruction Formats
 
-The tool works with "instructions" that describe symbolic links
+The tool supports two formats for defining symbolic links:
 
-They are, in any form, \<source>,\<destination>,\<force?>
+### 1. CSV Format (Legacy)
+
+Simple comma-separated values with the format: `<source>,<destination>[,force][,hard][,delete]`
 
 For example:
-`sync this,that`
+```
+source_path,target_path
+source_path,target_path,true
+source_path,target_path,true,true
+source_path,target_path,true,true,true
+```
 
-It supports input of these instructions through:
+Or with named flags:
+```
+source_path,target_path,force=true,hard=true,delete=true
+source_path,target_path,f=true,h=true,d=true
+```
+
+### 2. YAML Format (Recommended)
+
+A more readable format using YAML:
+
+```yaml
+links:
+  - source: ~/Documents/config.ini
+    target: ~/.config/app/config.ini
+    force: true
+  
+  - source: ~/Pictures
+    target: ~/Documents/Pictures
+    hard: true
+    force: true
+  
+  - source: ~/Scripts/script.sh
+    target: ~/bin/script.sh
+    delete: true
+```
+
+Alternatively, you can use an array directly:
+
+```yaml
+- source: ~/Documents/config.ini
+  target: ~/.config/app/config.ini
+  force: true
+
+- source: ~/Pictures
+  target: ~/Documents/Pictures
+  hard: true
+```
+
+## Input Methods
+
+The tool supports input of these instructions through:
 
 - Stdin
-	- `echo "this,that" | sync`
+  - `echo "this,that" | sync`
 - Run arguments
-	- `sync this,that foo,bar "foo 2","C:/bar"`
+  - `sync this,that foo,bar "foo 2","C:/bar"`
 - Files
-	- `sync -f <file>`
-	- Where the file contains instructions, one instruction per line
+  - `sync -f <file>` (CSV format)
+  - `sync -f <file.yaml>` or `sync -f <file.yml>` (YAML format)
+  - Where the file contains instructions, one per line for CSV or structured YAML
 - Directories
-	- `sync -r <directory>`
-	- This mode will look for "sync" files recursively in directories and run their instructions
+  - `sync -r <directory>`
+  - This mode will look for "sync", "sync.yaml", or "sync.yml" files recursively in directories and run their instructions
+
+## Options
+
+- `force: true` - Overwrite an existing symbolic link at the target location
+- `hard: true` - Create a hard link instead of a symbolic link
+- `delete: true` - Delete a non-symlink file at the target location (implies `force: true`)
 
 ## Use case
 

go.mod

@@ -1,3 +1,5 @@
 module cln
 
 go 1.21.7
+
+require gopkg.in/yaml.v3 v3.0.1

go.sum

@@ -0,0 +1,4 @@
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

instruction.go

@@ -5,17 +5,21 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"regexp"
-	"strconv"
 	"strings"
+
+	"gopkg.in/yaml.v3"
 )
 
 type LinkInstruction struct {
-	Source string
-	Target string
-	Force  bool
-	Hard   bool
-	Delete bool
+	Source string `yaml:"source"`
+	Target string `yaml:"target"`
+	Force  bool   `yaml:"force,omitempty"`
+	Hard   bool   `yaml:"hard,omitempty"`
+	Delete bool   `yaml:"delete,omitempty"`
+}
+
+type YAMLConfig struct {
+	Links []LinkInstruction `yaml:"links"`
 }
 
 func (instruction *LinkInstruction) Tidy() {
@@ -29,11 +33,34 @@ func (instruction *LinkInstruction) Tidy() {
 }
 
 func (instruction *LinkInstruction) String() string {
-	return fmt.Sprintf("%s%s%s%s%s%s%s%s%s%s%s%s%s", SourceColor, instruction.Source, DefaultColor, deliminer, TargetColor, instruction.Target, DefaultColor, deliminer, strconv.FormatBool(instruction.Force), deliminer, strconv.FormatBool(instruction.Hard), deliminer, strconv.FormatBool(instruction.Delete))
+	var flags []string
+	if instruction.Force {
+		flags = append(flags, "force=true")
+	}
+	if instruction.Hard {
+		flags = append(flags, "hard=true")
+	}
+	if instruction.Delete {
+		flags = append(flags, "delete=true")
+	}
+
+	flagsStr := ""
+	if len(flags) > 0 {
+		flagsStr = " [" + strings.Join(flags, ", ") + "]"
+	}
+
+	return fmt.Sprintf("%s%s%s → %s%s%s%s",
+		SourceColor, instruction.Source, DefaultColor,
+		TargetColor, instruction.Target, DefaultColor,
+		flagsStr)
 }
 
 func ParseInstruction(line, workdir string) (LinkInstruction, error) {
 	line = strings.TrimSpace(line)
+	if strings.HasPrefix(line, "#") {
+		return LinkInstruction{}, fmt.Errorf("comment line")
+	}
+
 	parts := strings.Split(line, deliminer)
 	instruction := LinkInstruction{}
 
@@ -43,19 +70,48 @@ func ParseInstruction(line, workdir string) (LinkInstruction, error) {
 
 	instruction.Source = strings.TrimSpace(parts[0])
 	instruction.Target = strings.TrimSpace(parts[1])
-	instruction.Force = false
-	if len(parts) > 2 {
-		res, _ := regexp.MatchString(`^(?i)\s*T|TRUE\s*$`, parts[2])
-		instruction.Force = res
-	}
-	if len(parts) > 3 {
-		res, _ := regexp.MatchString(`^(?i)\s*T|TRUE\s*$`, parts[3])
-		instruction.Hard = res
-	}
-	if len(parts) > 4 {
-		res, _ := regexp.MatchString(`^(?i)\s*T|TRUE\s*$`, parts[4])
-		instruction.Delete = res
-		instruction.Force = res
+
+	for i := 2; i < len(parts); i++ {
+		flagPart := strings.TrimSpace(parts[i])
+
+		// Support for legacy format (backward compatibility)
+		if !strings.Contains(flagPart, "=") {
+			// Legacy format: positional boolean flags
+			switch i {
+			case 2: // Force flag (3rd position)
+				instruction.Force = isTrue(flagPart)
+			case 3: // Hard flag (4th position)
+				instruction.Hard = isTrue(flagPart)
+			case 4: // Delete flag (5th position)
+				instruction.Delete = isTrue(flagPart)
+				if instruction.Delete {
+					instruction.Force = true // Delete implies Force
+				}
+			}
+			continue
+		}
+
+		// New format: named flags (name=value)
+		nameValue := strings.SplitN(flagPart, "=", 2)
+		if len(nameValue) != 2 {
+			// Skip malformed flags
+			continue
+		}
+
+		flagName := strings.ToLower(strings.TrimSpace(nameValue[0]))
+		flagValue := strings.TrimSpace(nameValue[1])
+
+		switch flagName {
+		case "force", "f":
+			instruction.Force = isTrue(flagValue)
+		case "hard", "h":
+			instruction.Hard = isTrue(flagValue)
+		case "delete", "d":
+			instruction.Delete = isTrue(flagValue)
+			if instruction.Delete {
+				instruction.Force = true // Delete implies Force
+			}
+		}
 	}
 
 	instruction.Tidy()
@@ -68,6 +124,11 @@ func ParseInstruction(line, workdir string) (LinkInstruction, error) {
 	return instruction, nil
 }
 
+func isTrue(value string) bool {
+	value = strings.ToLower(strings.TrimSpace(value))
+	return value == "true" || value == "t" || value == "yes" || value == "y" || value == "1"
+}
+
 func (instruction *LinkInstruction) RunAsync(status chan (error)) {
 	defer close(status)
 	if !FileExists(instruction.Source) {
@@ -152,3 +213,43 @@ func (instruction *LinkInstruction) RunAsync(status chan (error)) {
 
 	status <- nil
 }
+
+func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error reading YAML file: %w", err)
+	}
+
+	// First try to parse as a list of link instructions
+	var config YAMLConfig
+	err = yaml.Unmarshal(data, &config)
+	if err != nil || len(config.Links) == 0 {
+		// If that fails, try parsing as a direct list of instructions
+		var instructions []LinkInstruction
+		err = yaml.Unmarshal(data, &instructions)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing YAML: %w", err)
+		}
+		config.Links = instructions
+	}
+
+	for i := range config.Links {
+		config.Links[i].Tidy()
+		config.Links[i].Source, _ = ConvertHome(config.Links[i].Source)
+		config.Links[i].Target, _ = ConvertHome(config.Links[i].Target)
+		config.Links[i].Source = NormalizePath(config.Links[i].Source, workdir)
+		config.Links[i].Target = NormalizePath(config.Links[i].Target, workdir)
+
+		// If Delete is true, Force must also be true
+		if config.Links[i].Delete {
+			config.Links[i].Force = true
+		}
+	}
+
+	return config.Links, nil
+}
+
+func IsYAMLFile(filename string) bool {
+	ext := strings.ToLower(filepath.Ext(filename))
+	return ext == ".yaml" || ext == ".yml"
+}

main.go

@@ -20,8 +20,8 @@ const ImportantColor = BRed
 const DefaultColor = White
 const PathColor = Green
 
-var DirRegex, _ = regexp.Compile(`^(.+?)[/\\]sync$`)
-var FileRegex, _ = regexp.Compile(`^sync$`)
+var DirRegex, _ = regexp.Compile(`^(.+?)[/\\]sync(?:\.ya?ml)?$`)
+var FileRegex, _ = regexp.Compile(`^sync(?:\.ya?ml)?$`)
 var programName = os.Args[0]
 
 func main() {
@@ -51,28 +51,35 @@ func main() {
 	case *recurse != "":
 		log.Printf("Recurse: %s", *recurse)
 		go ReadFromFilesRecursively(*recurse, instructions, status)
-	
+
 	case *file != "":
 		log.Printf("File: %s", *file)
 		go ReadFromFile(*file, instructions, status, true)
-	
+
 	case len(flag.Args()) > 0:
 		log.Printf("Reading from command line arguments")
 		go ReadFromArgs(instructions, status)
-	
+
 	case IsPipeInput():
 		log.Printf("Reading from stdin pipe")
 		go ReadFromStdin(instructions, status)
-	
+
 	default:
 		if _, err := os.Stat("sync"); err == nil {
 			log.Printf("Using default sync file")
 			go ReadFromFile("sync", instructions, status, true)
+		} else if _, err := os.Stat("sync.yaml"); err == nil {
+			log.Printf("Using default sync.yaml file")
+			go ReadFromFile("sync.yaml", instructions, status, true)
+		} else if _, err := os.Stat("sync.yml"); err == nil {
+			log.Printf("Using default sync.yml file")
+			go ReadFromFile("sync.yml", instructions, status, true)
 		} else {
 			log.Printf("No input provided")
 			log.Printf("Provide input as: ")
 			log.Printf("Arguments - %s <source>,<target>,<force?>", programName)
 			log.Printf("File - %s -f <file>", programName)
+			log.Printf("YAML File - %s -f <file.yaml>", programName)
 			log.Printf("Folder (finding sync files in folder recursively) - %s -r <folder>", programName)
 			log.Printf("stdin - (cat <file> | %s)", programName)
 			os.Exit(1)
@@ -191,9 +198,31 @@ func ReadFromFile(input string, output chan *LinkInstruction, status chan error,
 
 	input = NormalizePath(input, filepath.Dir(input))
 	log.Printf("Reading input from file: %s%s%s", PathColor, input, DefaultColor)
+
+	// Check if this is a YAML file
+	if IsYAMLFile(input) {
+		log.Printf("Parsing as YAML file")
+		instructions, err := ParseYAMLFile(input, filepath.Dir(input))
+		if err != nil {
+			log.Printf("Failed to parse YAML file %s%s%s: %s%+v%s",
+				SourceColor, input, DefaultColor, ErrorColor, err, DefaultColor)
+			status <- err
+			return
+		}
+
+		for _, instruction := range instructions {
+			instr := instruction // Create a copy to avoid reference issues
+			log.Printf("Read YAML instruction: %s", instr.String())
+			output <- &instr
+		}
+		return
+	}
+
+	// Handle CSV format (legacy)
 	file, err := os.Open(input)
 	if err != nil {
-		log.Fatalf("Failed to open file %s%s%s: %s%+v%s", SourceColor, input, DefaultColor, ErrorColor, err, DefaultColor)
+		log.Fatalf("Failed to open file %s%s%s: %s%+v%s",
+			SourceColor, input, DefaultColor, ErrorColor, err, DefaultColor)
 		return
 	}
 	defer file.Close()
@@ -203,7 +232,8 @@ func ReadFromFile(input string, output chan *LinkInstruction, status chan error,
 		line := scanner.Text()
 		instruction, err := ParseInstruction(line, filepath.Dir(input))
 		if err != nil {
-			log.Printf("Error parsing line: %s'%s'%s, error: %s%+v%s", SourceColor, line, DefaultColor, ErrorColor, err, DefaultColor)
+			log.Printf("Error parsing line: %s'%s'%s, error: %s%+v%s",
+				SourceColor, line, DefaultColor, ErrorColor, err, DefaultColor)
 			continue
 		}
 		log.Printf("Read instruction: %s", instruction.String())

sync

@@ -1 +0,0 @@
-foo,"~/bar"

sync.yaml

@@ -0,0 +1,10 @@
+- source: main.go
+  target: test/main.go
+
+- source: README.md
+  target: test/README.md
+
+- source: sync.yaml
+  target: test/sync.yaml
+
+

sync.yaml.example

@@ -0,0 +1,26 @@
+# Example sync.yaml file
+# You can use this format to define symbolic links
+# Each link specifies source, target, and optional flags
+links:
+  - source: ~/Documents/config.ini
+    target: ~/.config/app/config.ini
+    # This will create a symbolic link, overwriting any existing symlink
+    force: true
+  
+  - source: ~/Pictures
+    target: ~/Documents/Pictures
+    # This will create a hard link instead of a symbolic link
+    hard: true
+    force: true
+  
+  - source: ~/Scripts/script.sh
+    target: ~/bin/script.sh
+    # This will delete a non-symlink file at the target location
+    # 'delete: true' implies 'force: true'
+    delete: true
+
+# Alternative format:
+# Instead of using the 'links' property, you can define an array directly:
+# - source: ~/Documents/config.ini
+#   target: ~/.config/app/config.ini
+#   force: true 

util.go

@@ -79,74 +79,106 @@ func GetSyncFilesRecursively(input string, output chan string, status chan error
 
 	var filesProcessed int32
 	var foldersProcessed int32
+	var activeWorkers int32
+
 	progressTicker := time.NewTicker(200 * time.Millisecond)
 	defer progressTicker.Stop()
 
-	var wg sync.WaitGroup
-	var initial sync.Once
-	var done bool
-	wg.Add(1)
+	done := make(chan struct{})
+	defer close(done)
+
 	directories := make(chan string, 100000)
 	workerPool := make(chan struct{}, 4000)
 	directories <- input
 
 	go func() {
 		for {
-			fmt.Printf("\rFiles processed: %d; Folders processed: %d; Workers: %d; Directory Stack Size: %d;", atomic.LoadInt32((&filesProcessed)), atomic.LoadInt32(&foldersProcessed), len(workerPool), len(directories))
-			<-progressTicker.C
+			select {
+			case <-progressTicker.C:
+				dirCount := len(directories)
+				workers := atomic.LoadInt32(&activeWorkers)
+				fmt.Printf("\rFiles processed: %d; Folders processed: %d; Active workers: %d; Directory queue: %d",
+					atomic.LoadInt32(&filesProcessed),
+					atomic.LoadInt32(&foldersProcessed),
+					workers,
+					dirCount)
+			case <-done:
+				// Final progress update
+				fmt.Printf("\rFiles processed: %d; Folders processed: %d; Completed successfully\n",
+					atomic.LoadInt32(&filesProcessed),
+					atomic.LoadInt32(&foldersProcessed))
+				return
+			}
 		}
 	}()
 
-	// log.Printf("%+v", len(workerPool))
-	go func() {
-		for directory := range directories {
-			workerPool <- struct{}{}
-			wg.Add(1)
-			go func(directory string) {
-				atomic.AddInt32(&foldersProcessed, 1)
-				defer wg.Done()
-				defer func() { <-workerPool }()
+	allDone := make(chan struct{})
 
-				files, err := os.ReadDir(directory)
-				if err != nil {
-					log.Printf("Error reading directory %s: %+v", directory, err)
+	go func() {
+		var wg sync.WaitGroup
+
+		go func() {
+			for {
+				if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
+					time.Sleep(10 * time.Millisecond)
+					if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
+						close(allDone)
+						return
+					}
+				}
+				time.Sleep(50 * time.Millisecond) 
+			}
+		}()
+
+		for {
+			select {
+			case directory, ok := <-directories:
+				if !ok {
+					wg.Wait()
 					return
 				}
 
-				for _, file := range files {
-					// log.Printf("Processing file %s", file.Name())
-					if file.IsDir() {
-						directories <- filepath.Join(directory, file.Name())
-					} else {
-						// log.Println(file.Name(), DirRegex.MatchString(file.Name()))
-						if FileRegex.MatchString(file.Name()) {
-							// log.Printf("Writing")
-							output <- filepath.Join(directory, file.Name())
-						}
-						atomic.AddInt32(&filesProcessed, 1)
-					}
-				}
-				// log.Printf("Done reading directory %s", directory)
-				done = len(directories) == 0
-				if done {
-					initial.Do(func() {
-						wg.Done()
-					})
-				}
-			}(directory)
+				atomic.AddInt32(&activeWorkers, 1)
+
+				go func(dir string) {
+					workerPool <- struct{}{}
+
+					atomic.AddInt32(&foldersProcessed, 1)
+					processDirectory(dir, directories, output, &filesProcessed)
+
+					<-workerPool
+					atomic.AddInt32(&activeWorkers, -1)
+				}(directory)
+			}
 		}
 	}()
 
-	// This actually does not go through ALL files sadly...
-	// It so happens (very often) that we manage to quit between one iteration ending
-	// And another beginning
-	// In such a state workgroup is decreased and, before it has a chance to increase, we are done
-	// What I should do here is only terminate if directories is empty
-	// ...but how do I do that?
-	// I might be wrong... Fuck knows...
-	// It also sometimes happens that wg.Wait triggers after wg.Done on line 97 but before the next (what would be!) wg.Add on line 94
-	// This happens much more often with a small number of workers
-	// Such is the nature of race conditions...
-	wg.Wait()
-	log.Printf("Files processed: %d; Folders processed: %d", filesProcessed, foldersProcessed)
+	<-allDone
+
+	log.Printf("Files processed: %d; Folders processed: %d",
+		atomic.LoadInt32(&filesProcessed),
+		atomic.LoadInt32(&foldersProcessed))
+}
+
+func processDirectory(directory string, directories chan<- string, output chan<- string, filesProcessed *int32) {
+	files, err := os.ReadDir(directory)
+	if err != nil {
+		log.Printf("Error reading directory %s: %+v", directory, err)
+		return
+	}
+
+	for _, file := range files {
+		if file.IsDir() {
+			directories <- filepath.Join(directory, file.Name())
+		} else {
+			if FileRegex.MatchString(file.Name()) || IsYAMLSyncFile(file.Name()) {
+				output <- filepath.Join(directory, file.Name())
+			}
+			atomic.AddInt32(filesProcessed, 1)
+		}
+	}
+}
+
+func IsYAMLSyncFile(filename string) bool {
+	return filename == "sync.yaml" || filename == "sync.yml"
 }