Implement a "from" to config files that loads other files

oopsie

.gitignore

@@ -3,3 +3,5 @@
 cln
 cln.log
 .qodo
+*.log
+*.out

build.sh

@@ -0,0 +1,2 @@
+GOOS=windows GOARCH=amd64 go build -o cln.exe .
+GOOS=linux GOARCH=amd64 go build -o cln .

go.mod

@@ -4,4 +4,12 @@ go 1.21.7
 
 require gopkg.in/yaml.v3 v3.0.1
 
-require github.com/bmatcuk/doublestar/v4 v4.8.1
+require (
+	github.com/bmatcuk/doublestar/v4 v4.8.1
+	github.com/stretchr/testify v1.11.1
+)
+
+require (
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+)

go.sum

@@ -1,5 +1,11 @@
 github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
 github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

instruction.go

@@ -20,6 +20,7 @@ type LinkInstruction struct {
 
 type YAMLConfig struct {
 	Links []LinkInstruction `yaml:"links"`
+	From  []string          `yaml:"from,omitempty"`
 }
 
 func (instruction *LinkInstruction) Tidy() {
@@ -318,9 +319,76 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
 	return expanded, nil
 }
 
+// ParseYAMLFileRecursive parses a YAML file and recursively processes any "From" references
+func ParseYAMLFileRecursive(filename, workdir string) ([]LinkInstruction, error) {
+	visited := make(map[string]bool)
+	return parseYAMLFileRecursive(filename, workdir, visited)
+}
+
+// parseYAMLFileRecursive is the internal recursive function that tracks visited files to prevent cycles
+func parseYAMLFileRecursive(filename, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	// Normalize the filename to prevent cycles with different path representations
+	normalizedFilename, err := filepath.Abs(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error normalizing filename: %w", err)
+	}
+
+	// Check for cycles
+	if visited[normalizedFilename] {
+		return nil, fmt.Errorf("circular reference detected: %s", filename)
+	}
+	visited[normalizedFilename] = true
+	defer delete(visited, normalizedFilename)
+
+	// Parse the current file
+	instructions, err := ParseYAMLFile(filename, workdir)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read the file to check for "From" references
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error reading YAML file: %w", err)
+	}
+
+	var config YAMLConfig
+	err = yaml.Unmarshal(data, &config)
+	if err != nil {
+		// If parsing as YAMLConfig fails, there are no "From" references to process
+		return instructions, nil
+	}
+
+	// Process "From" references
+	for _, fromFile := range config.From {
+		// Convert relative paths to absolute paths based on the current file's directory
+		fromPath := fromFile
+		if !filepath.IsAbs(fromPath) {
+			currentDir := filepath.Dir(filename)
+			fromPath = filepath.Join(currentDir, fromPath)
+		}
+
+		// Normalize the path
+		fromPath = filepath.Clean(fromPath)
+
+		// Recursively parse the referenced file
+		// Use the directory of the referenced file as the workdir for pattern expansion
+		fromWorkdir := filepath.Dir(fromPath)
+		fromInstructions, err := parseYAMLFileRecursive(fromPath, fromWorkdir, visited)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing referenced file %s: %w", fromFile, err)
+		}
+
+		// Append the instructions from the referenced file
+		instructions = append(instructions, fromInstructions...)
+	}
+
+	return instructions, nil
+}
+
 func ExpandPattern(source, workdir, target string) (links []LinkInstruction, err error) {
 	static, pattern := doublestar.SplitPattern(source)
-	if static == "" {
+	if static == "" || static == "." {
 		static = workdir
 	}
 	LogInfo("Static part: %s", static)

main.go

@@ -7,7 +7,6 @@ import (
 	"log"
 	"os"
 	"path/filepath"
-	"regexp"
 	"sync"
 	"sync/atomic"
 )
@@ -20,7 +19,6 @@ const ImportantColor = BRed
 const DefaultColor = Reset
 const PathColor = Green
 
-var FileRegex, _ = regexp.Compile(`sync\.ya?ml$`)
 var programName = os.Args[0]
 var undo = false
 
@@ -32,7 +30,27 @@ func main() {
 	flag.Parse()
 	undo = *undoF
 
-	if *debug {
+	setupLogging(*debug)
+
+	instructions := make(chan *LinkInstruction, 1000)
+	status := make(chan error)
+
+	startInputSource(*recurse, *file, instructions, status)
+
+	go handleStatusErrors(status)
+
+	instructionsDone := processInstructions(instructions)
+
+	if instructionsDone == 0 {
+		LogInfo("No instructions were processed")
+		os.Exit(1)
+	}
+	LogInfo("All done")
+}
+
+// setupLogging configures logging based on debug flag
+func setupLogging(debug bool) {
+	if debug {
 		log.SetFlags(log.Lmicroseconds | log.Lshortfile)
 		logFile, err := os.Create(programName + ".log")
 		if err != nil {
@@ -44,19 +62,19 @@ func main() {
 	} else {
 		log.SetFlags(log.Lmicroseconds)
 	}
+}
 
-	instructions := make(chan *LinkInstruction, 1000)
-	status := make(chan error)
-
+// startInputSource determines and starts the appropriate input source
+func startInputSource(recurse, file string, instructions chan *LinkInstruction, status chan error) {
 	// Check input sources in priority order
 	switch {
-	case *recurse != "":
-		LogInfo("Recurse: %s", *recurse)
-		go ReadFromFilesRecursively(*recurse, instructions, status)
+	case recurse != "":
+		LogInfo("Recurse: %s", recurse)
+		go ReadFromFilesRecursively(recurse, instructions, status)
 
-	case *file != "":
-		LogInfo("File: %s", *file)
-		go ReadFromFile(*file, instructions, status, true)
+	case file != "":
+		LogInfo("File: %s", file)
+		go ReadFromFile(file, instructions, status, true)
 
 	case len(flag.Args()) > 0:
 		LogInfo("Reading from command line arguments")
@@ -67,39 +85,53 @@ func main() {
 	// 	go ReadFromStdin(instructions, status)
 
 	default:
-		if _, err := os.Stat("sync"); err == nil {
-			LogInfo("Using default sync file")
-			go ReadFromFile("sync", instructions, status, true)
-		} else if _, err := os.Stat("sync.yaml"); err == nil {
-			LogInfo("Using default sync.yaml file")
-			go ReadFromFile("sync.yaml", instructions, status, true)
-		} else if _, err := os.Stat("sync.yml"); err == nil {
-			LogInfo("Using default sync.yml file")
-			go ReadFromFile("sync.yml", instructions, status, true)
-		} else {
-			LogInfo("No input provided")
-			LogInfo("Provide input as: ")
-			LogInfo("Arguments - %s <source>,<target>,<force?>", programName)
-			LogInfo("File - %s -f <file>", programName)
-			LogInfo("YAML File - %s -f <file.yaml>", programName)
-			LogInfo("Folder (finding sync files in folder recursively) - %s -r <folder>", programName)
-			LogInfo("stdin - (cat <file> | %s)", programName)
-			os.Exit(1)
+		startDefaultInputSource(instructions, status)
+	}
+}
+
+// startDefaultInputSource tries to find default sync files
+func startDefaultInputSource(instructions chan *LinkInstruction, status chan error) {
+	if _, err := os.Stat("sync"); err == nil {
+		LogInfo("Using default sync file")
+		go ReadFromFile("sync", instructions, status, true)
+	} else if _, err := os.Stat("sync.yaml"); err == nil {
+		LogInfo("Using default sync.yaml file")
+		go ReadFromFile("sync.yaml", instructions, status, true)
+	} else if _, err := os.Stat("sync.yml"); err == nil {
+		LogInfo("Using default sync.yml file")
+		go ReadFromFile("sync.yml", instructions, status, true)
+	} else {
+		showUsageAndExit()
+	}
+}
+
+// showUsageAndExit displays usage information and exits
+func showUsageAndExit() {
+	LogInfo("No input provided")
+	LogInfo("Provide input as: ")
+	LogInfo("Arguments - %s <source>,<target>,<force?>", programName)
+	LogInfo("File - %s -f <file>", programName)
+	LogInfo("YAML File - %s -f <file.yaml>", programName)
+	LogInfo("Folder (finding sync files in folder recursively) - %s -r <folder>", programName)
+	LogInfo("stdin - (cat <file> | %s)", programName)
+	os.Exit(1)
+}
+
+// handleStatusErrors processes status channel errors
+func handleStatusErrors(status chan error) {
+	for {
+		err, ok := <-status
+		if !ok {
+			break
+		}
+		if err != nil {
+			LogError("%v", err)
 		}
 	}
+}
 
-	go func() {
-		for {
-			err, ok := <-status
-			if !ok {
-				break
-			}
-			if err != nil {
-				LogError("%v", err)
-			}
-		}
-	}()
-
+// processInstructions processes all instructions from the channel
+func processInstructions(instructions chan *LinkInstruction) int32 {
 	var instructionsDone int32 = 0
 	var wg sync.WaitGroup
 	for {
@@ -120,11 +152,7 @@ func main() {
 		wg.Done()
 	}
 	wg.Wait()
-	if instructionsDone == 0 {
-		LogInfo("No instructions were processed")
-		os.Exit(1)
-	}
-	LogInfo("All done")
+	return instructionsDone
 }
 
 func IsPipeInput() bool {
@@ -145,58 +173,49 @@ func ReadFromFilesRecursively(input string, output chan *LinkInstruction, status
 
 	files := make(chan string, 128)
 	fileStatus := make(chan error)
-	var wg sync.WaitGroup
 	go GetSyncFilesRecursively(input, files, fileStatus)
-	go func() {
-		wg.Wait()
-		close(files)
-	}()
-
-	go func() {
-		for {
-			err, ok := <-fileStatus
-			if !ok {
-				break
-			}
-			if err != nil {
-				LogError("Failed to get sync files recursively: %v", err)
-				status <- err
-			}
-		}
-	}()
 
+	// Collect all files first
+	var syncFiles []string
 	for {
 		file, ok := <-files
 		if !ok {
-			LogInfo("No more files to process")
 			break
 		}
-		wg.Add(1)
-		go func() {
-			defer wg.Done()
-			LogInfo(file)
-			file = NormalizePath(file, workdir)
-			LogInfo("Processing file: %s", FormatPathValue(file))
+		syncFiles = append(syncFiles, file)
+	}
 
-			// This "has" to be done because instructions are resolved in relation to cwd
-			fileDir := FileRegex.FindStringSubmatch(file)
-			if fileDir == nil {
-				LogError("Failed to extract directory from %s", FormatSourcePath(file))
-				return
-			}
-			LogInfo("Changing directory to %s (for %s)",
-				FormatPathValue(fileDir[1]),
-				FormatPathValue(file))
-			err := os.Chdir(fileDir[1])
-			if err != nil {
-				LogError("Failed to change directory to %s: %v",
-					FormatSourcePath(fileDir[1]), err)
-				return
-			}
-			ReadFromFile(file, output, status, false)
-			// Don't return directory, stay where we are
-			os.Chdir(workdir)
-		}()
+	// Check for errors from file search
+	for {
+		err, ok := <-fileStatus
+		if !ok {
+			break
+		}
+		if err != nil {
+			LogError("Failed to get sync files recursively: %v", err)
+			status <- err
+		}
+	}
+
+	// Process each file
+	for _, file := range syncFiles {
+		file = NormalizePath(file, workdir)
+		LogInfo("Processing file: %s", FormatPathValue(file))
+
+		// Change to the directory containing the sync file
+		fileDir := filepath.Dir(file)
+		originalDir, _ := os.Getwd()
+		err := os.Chdir(fileDir)
+		if err != nil {
+			LogError("Failed to change directory to %s: %v", FormatSourcePath(fileDir), err)
+			continue
+		}
+
+		// Read and process the file
+		ReadFromFile(file, output, status, false)
+
+		// Return to original directory
+		os.Chdir(originalDir)
 	}
 }
 
@@ -212,7 +231,7 @@ func ReadFromFile(input string, output chan *LinkInstruction, status chan error,
 	// Check if this is a YAML file
 	if IsYAMLFile(input) {
 		LogInfo("Parsing as YAML file")
-		instructions, err := ParseYAMLFile(input, filepath.Dir(input))
+		instructions, err := ParseYAMLFileRecursive(input, filepath.Dir(input))
 		if err != nil {
 			LogError("Failed to parse YAML file %s: %v",
 				FormatSourcePath(input), err)

release.sh

@@ -16,8 +16,8 @@ fi
 echo "Tag: $TAG"
 
 echo "Building the thing..."
-go build .
-go install .
+sh build.sh
+sh install.sh
 
 echo "Creating a release..."
 TOKEN="$GITEA_API_KEY"
@@ -46,3 +46,7 @@ curl -X POST \
   -H "Authorization: token $TOKEN" \
   -F "attachment=@cln.exe" \
   "$GITEA/api/v1/repos/$REPO/releases/${RELEASE_ID}/assets?name=cln.exe"
+curl -X POST \
+  -H "Authorization: token $TOKEN" \
+  -F "attachment=@cln" \
+  "$GITEA/api/v1/repos/$REPO/releases/${RELEASE_ID}/assets?name=cln"

synclib.log

@@ -1,26 +0,0 @@
-18:24:37.967093 Using default sync.yaml file
-18:24:37.967093 Input 'sync.yaml' is not absolute, prepending work dir '.'
-18:24:37.967593 Reading input from file: C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml
-18:24:37.967593 Parsing as YAML file
-18:24:37.967593 Failed to parse YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml: error unmarshaling YAML: yaml: unmarshal errors:
-  line 1: cannot unmarshal !!seq into main.YAMLConfig
-18:24:37.967593 No more instructions to process
-18:24:37.968092 No instructions were processed
-18:27:59.691333 Using default sync.yaml file
-18:27:59.691333 Input 'sync.yaml' is not absolute, prepending work dir '.'
-18:27:59.691834 Reading input from file: C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml
-18:27:59.691834 Parsing as YAML file
-18:27:59.692335 Expanding wildcard source \* in YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml
-18:27:59.692335 Expanded wildcard source \* to 0 links
-18:27:59.692836 Expanded wildcard source \* in YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml to 0 links
-18:27:59.692836 No more instructions to process
-18:27:59.692836 No instructions were processed
-18:28:04.075821 Using default sync.yaml file
-18:28:04.076320 Input 'sync.yaml' is not absolute, prepending work dir '.'
-18:28:04.076320 Reading input from file: C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml
-18:28:04.076320 Parsing as YAML file
-18:28:04.076320 Expanding wildcard source \* in YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml
-18:28:04.076821 Expanded wildcard source \* to 0 links
-18:28:04.076821 Expanded wildcard source \* in YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml to 0 links
-18:28:04.076821 No more instructions to process
-18:28:04.076821 No instructions were processed

util.go

@@ -5,9 +5,8 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
-	"sync"
-	"sync/atomic"
-	"time"
+
+	"github.com/bmatcuk/doublestar/v4"
 )
 
 func IsSymlink(path string) (bool, error) {
@@ -76,112 +75,24 @@ func GetSyncFilesRecursively(input string, output chan string, status chan error
 	defer close(output)
 	defer close(status)
 
-	var filesProcessed int32
-	var foldersProcessed int32
-	var activeWorkers int32
+	workdir, _ := os.Getwd()
+	input = NormalizePath(input, workdir)
+	LogInfo("Searching for sync files recursively starting in %s", FormatPathValue(input))
 
-	progressTicker := time.NewTicker(200 * time.Millisecond)
-	defer progressTicker.Stop()
-
-	done := make(chan struct{})
-	defer close(done)
-
-	directories := make(chan string, 100000)
-	workerPool := make(chan struct{}, 4000)
-	directories <- input
-
-	go func() {
-		for {
-			select {
-			case <-progressTicker.C:
-				dirCount := len(directories)
-				workers := atomic.LoadInt32(&activeWorkers)
-				fmt.Printf("\rFiles processed: %d; Folders processed: %d; Active workers: %d; Directory queue: %d",
-					atomic.LoadInt32(&filesProcessed),
-					atomic.LoadInt32(&foldersProcessed),
-					workers,
-					dirCount)
-			case <-done:
-				// Final progress update
-				fmt.Printf("\nFiles processed: %d; Folders processed: %d; Completed successfully\n",
-					atomic.LoadInt32(&filesProcessed),
-					atomic.LoadInt32(&foldersProcessed))
-				return
-			}
-		}
-	}()
-
-	allDone := make(chan struct{})
-
-	go func() {
-		// WTF is this waitgroup?
-		// Nowhere is it added...
-		var wg sync.WaitGroup
-
-		go func() {
-			for {
-				if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
-					time.Sleep(10 * time.Millisecond)
-					if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
-						close(allDone)
-						return
-					}
-				}
-				time.Sleep(50 * time.Millisecond)
-			}
-		}()
-
-		for {
-			select {
-			case directory, ok := <-directories:
-				if !ok {
-					wg.Wait()
-					return
-				}
-
-				atomic.AddInt32(&activeWorkers, 1)
-
-				go func(dir string) {
-					workerPool <- struct{}{}
-
-					atomic.AddInt32(&foldersProcessed, 1)
-					processDirectory(dir, directories, output, &filesProcessed)
-
-					<-workerPool
-					atomic.AddInt32(&activeWorkers, -1)
-				}(directory)
-			}
-		}
-	}()
-
-	<-allDone
-
-	if atomic.LoadInt32(&filesProcessed) > 0 {
-		LogInfo("Files processed: %d; Folders processed: %d",
-			atomic.LoadInt32(&filesProcessed),
-			atomic.LoadInt32(&foldersProcessed))
-	}
-}
-
-func processDirectory(directory string, directories chan<- string, output chan<- string, filesProcessed *int32) {
-	files, err := os.ReadDir(directory)
+	// Use doublestar to find all sync.yml and sync.yaml files recursively
+	pattern := "**/sync.y*ml"
+	files, err := doublestar.Glob(os.DirFS(input), pattern)
 	if err != nil {
-		LogError("Error reading directory %s: %v", directory, err)
+		LogError("Failed to search for pattern %s: %v", pattern, err)
+		status <- err
 		return
 	}
 
 	for _, file := range files {
-		if file.IsDir() {
-			directories <- filepath.Join(directory, file.Name())
-		} else {
-			if IsYAMLSyncFile(file.Name()) {
-				output <- filepath.Join(directory, file.Name())
-			}
-			atomic.AddInt32(filesProcessed, 1)
-		}
+		fullPath := filepath.Join(input, file)
+		LogInfo("Found sync file: %s", FormatPathValue(fullPath))
+		output <- fullPath
 	}
-}
 
-func IsYAMLSyncFile(filename string) bool {
-	return filename == "sync.yaml" || filename == "sync.yml"
+	LogInfo("Completed recursive search for sync files")
 }