Fix home ~ resolution

.gitignore

@@ -5,3 +5,4 @@ cln.log
 .qodo
 *.log
 *.out
+test_temp

build.sh

@@ -1,2 +1,2 @@
-GOOS=windows GOARCH=amd64 go build -o cln.exe .
+CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -o cln.exe .
-GOOS=linux GOARCH=amd64 go build -o cln .
+CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o cln .

colors.go

@@ -83,6 +83,7 @@ const (
 // The acceptable range is [16, 231] but here we remove some very dark colors
 // That make text unreadable on a dark terminal
 // See https://www.hackitu.de/termcolor256/
+// Wait - why are we hardcoding this? lol do for loops not exist in our universe?
 var colors = []int{22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 57, 62, 63, 64, 65, 67, 68, 69, 70, 71, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 96, 97, 98, 99, 100, 101, 103, 104, 105, 106, 107, 108, 109, 110, 111, 112, 113, 114, 115, 116, 117, 118, 119, 120, 121, 122, 123, 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, 141, 142, 143, 144, 148, 149, 150, 151, 152, 154, 155, 156, 157, 158, 159, 160, 161, 162, 163, 164, 165, 166, 167, 168, 169, 170, 171, 172, 173, 174, 175, 176, 177, 178, 179, 180, 181, 182, 184, 185, 186, 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, 219, 220, 221, 222, 223, 226, 227, 228, 229, 230}
 var colorsIndex int = -1
 var shuffled bool

go.mod

@@ -1,10 +1,11 @@
 module cln
 
-go 1.21.7
+go 1.23.6
 
 require gopkg.in/yaml.v3 v3.0.1
 
 require (
+	git.site.quack-lab.dev/dave/cyutils v1.4.0
 	github.com/bmatcuk/doublestar/v4 v4.8.1
 	github.com/stretchr/testify v1.11.1
 )
@@ -12,4 +13,5 @@ require (
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/time v0.12.0 // indirect
 )

go.sum

@@ -1,3 +1,5 @@
+git.site.quack-lab.dev/dave/cyutils v1.4.0 h1:/Xo3QfLIFNab+axHneWmUK4MyfuObl+qq8whF9vTQpk=
+git.site.quack-lab.dev/dave/cyutils v1.4.0/go.mod h1:fBjALu2Cp2u2bDr+E4zbGVMBeIgFzROg+4TCcTNAiQU=
 github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
 github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
@@ -6,6 +8,8 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
 github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
+golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

home_test.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHomeDirectoryPatternExpansion(t *testing.T) {
+	testDir := getTestSubDir(t)
+
+	// Ensure we're working within the project directory
+	ensureInProjectDir(t, testDir)
+
+	// Change to test directory
+	originalDir, _ := os.Getwd()
+	defer os.Chdir(originalDir)
+	os.Chdir(testDir)
+
+	// Get the actual home directory
+	homeDir, err := os.UserHomeDir()
+	assert.NoError(t, err)
+
+	// Create a test directory in the home folder
+	testHomeDir := filepath.Join(homeDir, "synclib_test")
+	err = os.MkdirAll(testHomeDir, 0755)
+	assert.NoError(t, err)
+	defer os.RemoveAll(testHomeDir) // Cleanup
+
+	// Create a test file in the home directory
+	testFile := filepath.Join(testHomeDir, "testhome.csv")
+	err = os.WriteFile(testFile, []byte("test content"), 0644)
+	assert.NoError(t, err)
+
+	// Test the pattern with ~/ that should match the file
+	pattern := "~/synclib_test/testhome.csv"
+	links, err := ExpandPattern(pattern, testDir, "target.csv")
+
+	// This should work but currently fails due to the bug
+	assert.NoError(t, err)
+	assert.Equal(t, 1, len(links), "Pattern should match exactly 1 file")
+
+	if len(links) > 0 {
+		assert.Contains(t, links[0].Source, "testhome.csv")
+		assert.Equal(t, "target.csv", links[0].Target)
+	}
+}

instruction.go

@@ -270,40 +270,24 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
 		return nil, fmt.Errorf("error reading YAML file: %w", err)
 	}
 
-	// First try to parse as a list of link instructions
+	// Parse as a direct list of instructions
-	var config YAMLConfig
-	err = yaml.Unmarshal(data, &config)
-	if err != nil || len(config.Links) == 0 {
-		// If that fails, try parsing as a direct list of instructions
 	var instructions []LinkInstruction
 	err = yaml.Unmarshal(data, &instructions)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing YAML: %w", err)
 	}
-		config.Links = instructions
-	}
 
-	expanded := []LinkInstruction{}
+	// Preprocess instructions: expand globs and from references
-	for _, link := range config.Links {
+	// Create a new visited map for this file
-		LogSource("Expanding pattern source %s in YAML file %s", link.Source, filename)
+	visited := make(map[string]bool)
-		newlinks, err := ExpandPattern(link.Source, workdir, link.Target)
+	processedInstructions, err := preprocessInstructions(instructions, filename, workdir, visited)
 	if err != nil {
-			return nil, fmt.Errorf("error expanding pattern: %w", err)
+		return nil, err
-		}
-		// "Clone" the original link instruction for each expanded link
-		for i := range newlinks {
-			newlinks[i].Delete = link.Delete
-			newlinks[i].Hard = link.Hard
-			newlinks[i].Force = link.Force
-		}
-		LogInfo("Expanded pattern %s in YAML file %s to %d links",
-			FormatSourcePath(link.Source), FormatSourcePath(filename), len(newlinks))
-		expanded = append(expanded, newlinks...)
 	}
 
-	for i := range expanded {
+	// Final processing: normalize paths and set defaults
-		link := &expanded[i]
+	for i := range processedInstructions {
-
+		link := &processedInstructions[i]
 		link.Tidy()
 		link.Source, _ = ConvertHome(link.Source)
 		link.Target, _ = ConvertHome(link.Target)
@@ -316,7 +300,85 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
 		}
 	}
 
-	return expanded, nil
+	return processedInstructions, nil
+}
+
+// preprocessInstructions handles glob expansion and from references
+func preprocessInstructions(instructions []LinkInstruction, filename, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	var result []LinkInstruction
+
+	for _, instr := range instructions {
+		if instr.Source == "" {
+			continue // Skip invalid instructions
+		}
+
+		if instr.Target == "" {
+			// This is a from reference - load the referenced file
+			fromInstructions, err := loadFromReference(instr.Source, filename, workdir, visited)
+			if err != nil {
+				return nil, fmt.Errorf("error loading from reference %s: %w", instr.Source, err)
+			}
+			result = append(result, fromInstructions...)
+		} else {
+			// This is a regular instruction - expand globs if needed
+			expandedInstructions, err := expandGlobs(instr, filename, workdir)
+			if err != nil {
+				return nil, fmt.Errorf("error expanding globs for %s: %w", instr.Source, err)
+			}
+			result = append(result, expandedInstructions...)
+		}
+	}
+
+	return result, nil
+}
+
+// loadFromReference loads instructions from a referenced file
+func loadFromReference(fromFile, currentFile, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	// First convert home directory if it starts with ~
+	fromPath, err := ConvertHome(fromFile)
+	if err != nil {
+		return nil, fmt.Errorf("error converting home directory: %w", err)
+	}
+
+	// Convert relative paths to absolute paths based on the current file's directory
+	if !filepath.IsAbs(fromPath) {
+		currentDir := filepath.Dir(currentFile)
+		fromPath = filepath.Join(currentDir, fromPath)
+	}
+
+	// Normalize the path
+	fromPath = filepath.Clean(fromPath)
+
+	// Recursively parse the referenced file with cycle detection
+	fromWorkdir := filepath.Dir(fromPath)
+	return parseYAMLFileRecursive(fromPath, fromWorkdir, visited)
+}
+
+// expandGlobs expands glob patterns in a single instruction
+func expandGlobs(instr LinkInstruction, filename, workdir string) ([]LinkInstruction, error) {
+// Convert home directory (~) before expanding pattern
+	convertedSource, err := ConvertHome(instr.Source)
+	if err != nil {
+		return nil, fmt.Errorf("error converting home directory in source %s: %w", instr.Source, err)
+	}
+
+	LogSource("Expanding pattern source %s in YAML file %s", convertedSource, filename)
+	newlinks, err := ExpandPattern(convertedSource, workdir, instr.Target)
+	if err != nil {
+		return nil, err
+	}
+
+	// Clone the original instruction properties for each expanded link
+	for i := range newlinks {
+		newlinks[i].Delete = instr.Delete
+		newlinks[i].Hard = instr.Hard
+		newlinks[i].Force = instr.Force
+	}
+
+	LogInfo("Expanded pattern %s in YAML file %s to %d links",
+		FormatSourcePath(instr.Source), FormatSourcePath(filename), len(newlinks))
+
+	return newlinks, nil
 }
 
 // ParseYAMLFileRecursive parses a YAML file and recursively processes any "From" references
@@ -340,53 +402,49 @@ func parseYAMLFileRecursive(filename, workdir string, visited map[string]bool) (
 	visited[normalizedFilename] = true
 	defer delete(visited, normalizedFilename)
 
-	// Parse the current file
+	// Parse the current file and preprocess it with cycle detection
-	instructions, err := ParseYAMLFile(filename, workdir)
-	if err != nil {
-		return nil, err
-	}
-
-	// Read the file to check for "From" references
 	data, err := os.ReadFile(filename)
 	if err != nil {
 		return nil, fmt.Errorf("error reading YAML file: %w", err)
 	}
 
-	var config YAMLConfig
+	// Parse as a direct list of instructions
-	err = yaml.Unmarshal(data, &config)
+	var instructions []LinkInstruction
+	err = yaml.Unmarshal(data, &instructions)
 	if err != nil {
-		// If parsing as YAMLConfig fails, there are no "From" references to process
+		return nil, fmt.Errorf("error parsing YAML: %w", err)
-		return instructions, nil
 	}
 
-	// Process "From" references
+	// Preprocess instructions: expand globs and from references
-	for _, fromFile := range config.From {
+	processedInstructions, err := preprocessInstructions(instructions, filename, workdir, visited)
-		// Convert relative paths to absolute paths based on the current file's directory
-		fromPath := fromFile
-		if !filepath.IsAbs(fromPath) {
-			currentDir := filepath.Dir(filename)
-			fromPath = filepath.Join(currentDir, fromPath)
-		}
-
-		// Normalize the path
-		fromPath = filepath.Clean(fromPath)
-
-		// Recursively parse the referenced file
-		// Use the directory of the referenced file as the workdir for pattern expansion
-		fromWorkdir := filepath.Dir(fromPath)
-		fromInstructions, err := parseYAMLFileRecursive(fromPath, fromWorkdir, visited)
 	if err != nil {
-			return nil, fmt.Errorf("error parsing referenced file %s: %w", fromFile, err)
+		return nil, err
 	}
 
-		// Append the instructions from the referenced file
+	// Final processing: normalize paths and set defaults
-		instructions = append(instructions, fromInstructions...)
+	for i := range processedInstructions {
+		link := &processedInstructions[i]
+		link.Tidy()
+		link.Source, _ = ConvertHome(link.Source)
+		link.Target, _ = ConvertHome(link.Target)
+		link.Source = NormalizePath(link.Source, workdir)
+		link.Target = NormalizePath(link.Target, workdir)
+
+		// If Delete is true, Force must also be true
+		if link.Delete {
+			link.Force = true
+		}
 	}
 
-	return instructions, nil
+	return processedInstructions, nil
 }
 
 func ExpandPattern(source, workdir, target string) (links []LinkInstruction, err error) {
+	// Convert home directory (~) before splitting pattern
+	source, err = ConvertHome(source)
+	if err != nil {
+		return nil, fmt.Errorf("error converting home directory in source %s: %w", source, err)
+	}
 	static, pattern := doublestar.SplitPattern(source)
 	if static == "" || static == "." {
 		static = workdir

main.go

@@ -7,8 +7,8 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"sync"
 	"sync/atomic"
+	utils "git.site.quack-lab.dev/dave/cyutils"
 )
 
 const deliminer = ","
@@ -130,28 +130,35 @@ func handleStatusErrors(status chan error) {
 	}
 }
 
-// processInstructions processes all instructions from the channel
+// processInstructions processes all instructions from the channel using parallel workers
 func processInstructions(instructions chan *LinkInstruction) int32 {
 	var instructionsDone int32 = 0
-	var wg sync.WaitGroup
+
+	// Collect all instructions first
+	var allInstructions []*LinkInstruction
 	for {
 		instruction, ok := <-instructions
 		if !ok {
 			LogInfo("No more instructions to process")
 			break
 		}
+		allInstructions = append(allInstructions, instruction)
+	}
+
+	// Process instructions in parallel using cyutils.WithWorkers
+	// Let the library handle worker count - use 4 workers as a reasonable default
+	utils.WithWorkers(4, allInstructions, func(workerID int, _ int, instruction *LinkInstruction) {
 		LogInfo("Processing: %s", instruction.String())
 		status := make(chan error)
 		go instruction.RunAsync(status)
-		wg.Add(1)
 		err := <-status
 		if err != nil {
 			LogError("Failed processing instruction: %v", err)
-		}
+		} else {
 			atomic.AddInt32(&instructionsDone, 1)
-		wg.Done()
 		}
-	wg.Wait()
+	})
+
 	return instructionsDone
 }