Ensure we're cleaning up after our tests

oopsie

.gitignore

@@ -4,3 +4,5 @@ cln
 cln.log
 .qodo
 *.log
+*.out
+test_temp

go.mod

@@ -1,7 +1,17 @@
 module cln
 
-go 1.21.7
+go 1.23.6
 
 require gopkg.in/yaml.v3 v3.0.1
 
-require github.com/bmatcuk/doublestar/v4 v4.8.1
+require (
+	git.site.quack-lab.dev/dave/cyutils v1.4.0
+	github.com/bmatcuk/doublestar/v4 v4.8.1
+	github.com/stretchr/testify v1.11.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
+)

go.sum

@@ -1,5 +1,15 @@
+git.site.quack-lab.dev/dave/cyutils v1.4.0 h1:/Xo3QfLIFNab+axHneWmUK4MyfuObl+qq8whF9vTQpk=
+git.site.quack-lab.dev/dave/cyutils v1.4.0/go.mod h1:fBjALu2Cp2u2bDr+E4zbGVMBeIgFzROg+4TCcTNAiQU=
 github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
 github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

instruction.go

@@ -20,6 +20,7 @@ type LinkInstruction struct {
 
 type YAMLConfig struct {
 	Links []LinkInstruction `yaml:"links"`
+	From  []string          `yaml:"from,omitempty"`
 }
 
 func (instruction *LinkInstruction) Tidy() {
@@ -269,40 +270,24 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
 		return nil, fmt.Errorf("error reading YAML file: %w", err)
 	}
 
-	// First try to parse as a list of link instructions
-	var config YAMLConfig
-	err = yaml.Unmarshal(data, &config)
-	if err != nil || len(config.Links) == 0 {
-		// If that fails, try parsing as a direct list of instructions
-		var instructions []LinkInstruction
-		err = yaml.Unmarshal(data, &instructions)
-		if err != nil {
-			return nil, fmt.Errorf("error parsing YAML: %w", err)
-		}
-		config.Links = instructions
+	// Parse as a direct list of instructions
+	var instructions []LinkInstruction
+	err = yaml.Unmarshal(data, &instructions)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing YAML: %w", err)
 	}
 
-	expanded := []LinkInstruction{}
-	for _, link := range config.Links {
-		LogSource("Expanding pattern source %s in YAML file %s", link.Source, filename)
-		newlinks, err := ExpandPattern(link.Source, workdir, link.Target)
-		if err != nil {
-			return nil, fmt.Errorf("error expanding pattern: %w", err)
-		}
-		// "Clone" the original link instruction for each expanded link
-		for i := range newlinks {
-			newlinks[i].Delete = link.Delete
-			newlinks[i].Hard = link.Hard
-			newlinks[i].Force = link.Force
-		}
-		LogInfo("Expanded pattern %s in YAML file %s to %d links",
-			FormatSourcePath(link.Source), FormatSourcePath(filename), len(newlinks))
-		expanded = append(expanded, newlinks...)
+	// Preprocess instructions: expand globs and from references
+	// Create a new visited map for this file
+	visited := make(map[string]bool)
+	processedInstructions, err := preprocessInstructions(instructions, filename, workdir, visited)
+	if err != nil {
+		return nil, err
 	}
 
-	for i := range expanded {
-		link := &expanded[i]
-
+	// Final processing: normalize paths and set defaults
+	for i := range processedInstructions {
+		link := &processedInstructions[i]
 		link.Tidy()
 		link.Source, _ = ConvertHome(link.Source)
 		link.Target, _ = ConvertHome(link.Target)
@@ -315,12 +300,137 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
 		}
 	}
 
-	return expanded, nil
+	return processedInstructions, nil
+}
+
+// preprocessInstructions handles glob expansion and from references
+func preprocessInstructions(instructions []LinkInstruction, filename, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	var result []LinkInstruction
+
+	for _, instr := range instructions {
+		if instr.Source == "" {
+			continue // Skip invalid instructions
+		}
+
+		if instr.Target == "" {
+			// This is a from reference - load the referenced file
+			fromInstructions, err := loadFromReference(instr.Source, filename, workdir, visited)
+			if err != nil {
+				return nil, fmt.Errorf("error loading from reference %s: %w", instr.Source, err)
+			}
+			result = append(result, fromInstructions...)
+		} else {
+			// This is a regular instruction - expand globs if needed
+			expandedInstructions, err := expandGlobs(instr, filename, workdir)
+			if err != nil {
+				return nil, fmt.Errorf("error expanding globs for %s: %w", instr.Source, err)
+			}
+			result = append(result, expandedInstructions...)
+		}
+	}
+
+	return result, nil
+}
+
+// loadFromReference loads instructions from a referenced file
+func loadFromReference(fromFile, currentFile, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	// Convert relative paths to absolute paths based on the current file's directory
+	fromPath := fromFile
+	if !filepath.IsAbs(fromPath) {
+		currentDir := filepath.Dir(currentFile)
+		fromPath = filepath.Join(currentDir, fromPath)
+	}
+
+	// Normalize the path
+	fromPath = filepath.Clean(fromPath)
+
+	// Recursively parse the referenced file with cycle detection
+	fromWorkdir := filepath.Dir(fromPath)
+	return parseYAMLFileRecursive(fromPath, fromWorkdir, visited)
+}
+
+// expandGlobs expands glob patterns in a single instruction
+func expandGlobs(instr LinkInstruction, filename, workdir string) ([]LinkInstruction, error) {
+	LogSource("Expanding pattern source %s in YAML file %s", instr.Source, filename)
+	newlinks, err := ExpandPattern(instr.Source, workdir, instr.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	// Clone the original instruction properties for each expanded link
+	for i := range newlinks {
+		newlinks[i].Delete = instr.Delete
+		newlinks[i].Hard = instr.Hard
+		newlinks[i].Force = instr.Force
+	}
+
+	LogInfo("Expanded pattern %s in YAML file %s to %d links",
+		FormatSourcePath(instr.Source), FormatSourcePath(filename), len(newlinks))
+
+	return newlinks, nil
+}
+
+// ParseYAMLFileRecursive parses a YAML file and recursively processes any "From" references
+func ParseYAMLFileRecursive(filename, workdir string) ([]LinkInstruction, error) {
+	visited := make(map[string]bool)
+	return parseYAMLFileRecursive(filename, workdir, visited)
+}
+
+// parseYAMLFileRecursive is the internal recursive function that tracks visited files to prevent cycles
+func parseYAMLFileRecursive(filename, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	// Normalize the filename to prevent cycles with different path representations
+	normalizedFilename, err := filepath.Abs(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error normalizing filename: %w", err)
+	}
+
+	// Check for cycles
+	if visited[normalizedFilename] {
+		return nil, fmt.Errorf("circular reference detected: %s", filename)
+	}
+	visited[normalizedFilename] = true
+	defer delete(visited, normalizedFilename)
+
+	// Parse the current file and preprocess it with cycle detection
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error reading YAML file: %w", err)
+	}
+
+	// Parse as a direct list of instructions
+	var instructions []LinkInstruction
+	err = yaml.Unmarshal(data, &instructions)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing YAML: %w", err)
+	}
+
+	// Preprocess instructions: expand globs and from references
+	processedInstructions, err := preprocessInstructions(instructions, filename, workdir, visited)
+	if err != nil {
+		return nil, err
+	}
+
+	// Final processing: normalize paths and set defaults
+	for i := range processedInstructions {
+		link := &processedInstructions[i]
+		link.Tidy()
+		link.Source, _ = ConvertHome(link.Source)
+		link.Target, _ = ConvertHome(link.Target)
+		link.Source = NormalizePath(link.Source, workdir)
+		link.Target = NormalizePath(link.Target, workdir)
+
+		// If Delete is true, Force must also be true
+		if link.Delete {
+			link.Force = true
+		}
+	}
+
+	return processedInstructions, nil
 }
 
 func ExpandPattern(source, workdir, target string) (links []LinkInstruction, err error) {
 	static, pattern := doublestar.SplitPattern(source)
-	if static == "" {
+	if static == "" || static == "." {
 		static = workdir
 	}
 	LogInfo("Static part: %s", static)

main.go

@@ -7,8 +7,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"sync"
 	"sync/atomic"
+	utils "git.site.quack-lab.dev/dave/cyutils"
 )
 
 const deliminer = ","
@@ -30,7 +30,27 @@ func main() {
 	flag.Parse()
 	undo = *undoF
 
-	if *debug {
+	setupLogging(*debug)
+
+	instructions := make(chan *LinkInstruction, 1000)
+	status := make(chan error)
+
+	startInputSource(*recurse, *file, instructions, status)
+
+	go handleStatusErrors(status)
+
+	instructionsDone := processInstructions(instructions)
+
+	if instructionsDone == 0 {
+		LogInfo("No instructions were processed")
+		os.Exit(1)
+	}
+	LogInfo("All done")
+}
+
+// setupLogging configures logging based on debug flag
+func setupLogging(debug bool) {
+	if debug {
 		log.SetFlags(log.Lmicroseconds | log.Lshortfile)
 		logFile, err := os.Create(programName + ".log")
 		if err != nil {
@@ -42,19 +62,19 @@ func main() {
 	} else {
 		log.SetFlags(log.Lmicroseconds)
 	}
+}
 
-	instructions := make(chan *LinkInstruction, 1000)
-	status := make(chan error)
-
+// startInputSource determines and starts the appropriate input source
+func startInputSource(recurse, file string, instructions chan *LinkInstruction, status chan error) {
 	// Check input sources in priority order
 	switch {
-	case *recurse != "":
-		LogInfo("Recurse: %s", *recurse)
-		go ReadFromFilesRecursively(*recurse, instructions, status)
+	case recurse != "":
+		LogInfo("Recurse: %s", recurse)
+		go ReadFromFilesRecursively(recurse, instructions, status)
 
-	case *file != "":
-		LogInfo("File: %s", *file)
-		go ReadFromFile(*file, instructions, status, true)
+	case file != "":
+		LogInfo("File: %s", file)
+		go ReadFromFile(file, instructions, status, true)
 
 	case len(flag.Args()) > 0:
 		LogInfo("Reading from command line arguments")
@@ -65,64 +85,81 @@ func main() {
 	// 	go ReadFromStdin(instructions, status)
 
 	default:
-		if _, err := os.Stat("sync"); err == nil {
-			LogInfo("Using default sync file")
-			go ReadFromFile("sync", instructions, status, true)
-		} else if _, err := os.Stat("sync.yaml"); err == nil {
-			LogInfo("Using default sync.yaml file")
-			go ReadFromFile("sync.yaml", instructions, status, true)
-		} else if _, err := os.Stat("sync.yml"); err == nil {
-			LogInfo("Using default sync.yml file")
-			go ReadFromFile("sync.yml", instructions, status, true)
-		} else {
-			LogInfo("No input provided")
-			LogInfo("Provide input as: ")
-			LogInfo("Arguments - %s <source>,<target>,<force?>", programName)
-			LogInfo("File - %s -f <file>", programName)
-			LogInfo("YAML File - %s -f <file.yaml>", programName)
-			LogInfo("Folder (finding sync files in folder recursively) - %s -r <folder>", programName)
-			LogInfo("stdin - (cat <file> | %s)", programName)
-			os.Exit(1)
+		startDefaultInputSource(instructions, status)
+	}
+}
+
+// startDefaultInputSource tries to find default sync files
+func startDefaultInputSource(instructions chan *LinkInstruction, status chan error) {
+	if _, err := os.Stat("sync"); err == nil {
+		LogInfo("Using default sync file")
+		go ReadFromFile("sync", instructions, status, true)
+	} else if _, err := os.Stat("sync.yaml"); err == nil {
+		LogInfo("Using default sync.yaml file")
+		go ReadFromFile("sync.yaml", instructions, status, true)
+	} else if _, err := os.Stat("sync.yml"); err == nil {
+		LogInfo("Using default sync.yml file")
+		go ReadFromFile("sync.yml", instructions, status, true)
+	} else {
+		showUsageAndExit()
+	}
+}
+
+// showUsageAndExit displays usage information and exits
+func showUsageAndExit() {
+	LogInfo("No input provided")
+	LogInfo("Provide input as: ")
+	LogInfo("Arguments - %s <source>,<target>,<force?>", programName)
+	LogInfo("File - %s -f <file>", programName)
+	LogInfo("YAML File - %s -f <file.yaml>", programName)
+	LogInfo("Folder (finding sync files in folder recursively) - %s -r <folder>", programName)
+	LogInfo("stdin - (cat <file> | %s)", programName)
+	os.Exit(1)
+}
+
+// handleStatusErrors processes status channel errors
+func handleStatusErrors(status chan error) {
+	for {
+		err, ok := <-status
+		if !ok {
+			break
+		}
+		if err != nil {
+			LogError("%v", err)
 		}
 	}
+}
 
-	go func() {
-		for {
-			err, ok := <-status
-			if !ok {
-				break
-			}
-			if err != nil {
-				LogError("%v", err)
-			}
-		}
-	}()
-
+// processInstructions processes all instructions from the channel using parallel workers
+func processInstructions(instructions chan *LinkInstruction) int32 {
 	var instructionsDone int32 = 0
-	var wg sync.WaitGroup
+
+	// Collect all instructions first
+	var allInstructions []*LinkInstruction
 	for {
 		instruction, ok := <-instructions
 		if !ok {
 			LogInfo("No more instructions to process")
 			break
 		}
+		allInstructions = append(allInstructions, instruction)
+	}
+
+	// Process instructions in parallel using cyutils.WithWorkers
+	// Let the library handle worker count - use 4 workers as a reasonable default
+	utils.WithWorkers(4, allInstructions, func(workerID int, _ int, instruction *LinkInstruction) {
 		LogInfo("Processing: %s", instruction.String())
 		status := make(chan error)
 		go instruction.RunAsync(status)
-		wg.Add(1)
 		err := <-status
 		if err != nil {
 			LogError("Failed processing instruction: %v", err)
+		} else {
+			atomic.AddInt32(&instructionsDone, 1)
 		}
-		atomic.AddInt32(&instructionsDone, 1)
-		wg.Done()
-	}
-	wg.Wait()
-	if instructionsDone == 0 {
-		LogInfo("No instructions were processed")
-		os.Exit(1)
-	}
-	LogInfo("All done")
+	})
+
+	return instructionsDone
 }
 
 func IsPipeInput() bool {
@@ -201,7 +238,7 @@ func ReadFromFile(input string, output chan *LinkInstruction, status chan error,
 	// Check if this is a YAML file
 	if IsYAMLFile(input) {
 		LogInfo("Parsing as YAML file")
-		instructions, err := ParseYAMLFile(input, filepath.Dir(input))
+		instructions, err := ParseYAMLFileRecursive(input, filepath.Dir(input))
 		if err != nil {
 			LogError("Failed to parse YAML file %s: %v",
 				FormatSourcePath(input), err)

release.sh

@@ -45,4 +45,8 @@ echo "Uploading the things..."
 curl -X POST \
   -H "Authorization: token $TOKEN" \
   -F "attachment=@cln.exe" \
-  "$GITEA/api/v1/repos/$REPO/releases/${RELEASE_ID}/assets?name=cln.exe"
+  "$GITEA/api/v1/repos/$REPO/releases/${RELEASE_ID}/assets?name=cln.exe"
+curl -X POST \
+  -H "Authorization: token $TOKEN" \
+  -F "attachment=@cln" \
+  "$GITEA/api/v1/repos/$REPO/releases/${RELEASE_ID}/assets?name=cln"