Implement a "from" to config files that loads other files

go.mod

@@ -4,10 +4,12 @@ go 1.21.7
 
 require gopkg.in/yaml.v3 v3.0.1
 
-require github.com/bmatcuk/doublestar/v4 v4.8.1
+require (
+	github.com/bmatcuk/doublestar/v4 v4.8.1
+	github.com/stretchr/testify v1.11.1
+)
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/stretchr/testify v1.11.1 // indirect
 )

instruction.go

@@ -20,6 +20,7 @@ type LinkInstruction struct {
 
 type YAMLConfig struct {
 	Links []LinkInstruction `yaml:"links"`
+	From  []string          `yaml:"from,omitempty"`
 }
 
 func (instruction *LinkInstruction) Tidy() {
@@ -318,9 +319,76 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
 	return expanded, nil
 }
 
+// ParseYAMLFileRecursive parses a YAML file and recursively processes any "From" references
+func ParseYAMLFileRecursive(filename, workdir string) ([]LinkInstruction, error) {
+	visited := make(map[string]bool)
+	return parseYAMLFileRecursive(filename, workdir, visited)
+}
+
+// parseYAMLFileRecursive is the internal recursive function that tracks visited files to prevent cycles
+func parseYAMLFileRecursive(filename, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
+	// Normalize the filename to prevent cycles with different path representations
+	normalizedFilename, err := filepath.Abs(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error normalizing filename: %w", err)
+	}
+
+	// Check for cycles
+	if visited[normalizedFilename] {
+		return nil, fmt.Errorf("circular reference detected: %s", filename)
+	}
+	visited[normalizedFilename] = true
+	defer delete(visited, normalizedFilename)
+
+	// Parse the current file
+	instructions, err := ParseYAMLFile(filename, workdir)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read the file to check for "From" references
+	data, err := os.ReadFile(filename)
+	if err != nil {
+		return nil, fmt.Errorf("error reading YAML file: %w", err)
+	}
+
+	var config YAMLConfig
+	err = yaml.Unmarshal(data, &config)
+	if err != nil {
+		// If parsing as YAMLConfig fails, there are no "From" references to process
+		return instructions, nil
+	}
+
+	// Process "From" references
+	for _, fromFile := range config.From {
+		// Convert relative paths to absolute paths based on the current file's directory
+		fromPath := fromFile
+		if !filepath.IsAbs(fromPath) {
+			currentDir := filepath.Dir(filename)
+			fromPath = filepath.Join(currentDir, fromPath)
+		}
+
+		// Normalize the path
+		fromPath = filepath.Clean(fromPath)
+
+		// Recursively parse the referenced file
+		// Use the directory of the referenced file as the workdir for pattern expansion
+		fromWorkdir := filepath.Dir(fromPath)
+		fromInstructions, err := parseYAMLFileRecursive(fromPath, fromWorkdir, visited)
+		if err != nil {
+			return nil, fmt.Errorf("error parsing referenced file %s: %w", fromFile, err)
+		}
+
+		// Append the instructions from the referenced file
+		instructions = append(instructions, fromInstructions...)
+	}
+
+	return instructions, nil
+}
+
 func ExpandPattern(source, workdir, target string) (links []LinkInstruction, err error) {
 	static, pattern := doublestar.SplitPattern(source)
-	if static == "" {
+	if static == "" || static == "." {
 		static = workdir
 	}
 	LogInfo("Static part: %s", static)

instruction_test.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -1450,18 +1451,6 @@ func TestErrorPaths(t *testing.T) {
 		assert.Contains(t, err.Error(), "invalid format")
 	})
 
-	t.Run("ReadFromFile_nonexistent_file", func(t *testing.T) {
-		// Skip this test as ReadFromFile calls os.Exit(1) when file doesn't exist
-		// which terminates the test process
-		t.Skip("Skipping test that calls os.Exit(1) when file doesn't exist")
-	})
-
-	t.Run("ReadFromFile_invalid_yaml", func(t *testing.T) {
-		// Skip this test as ReadFromFile calls os.Exit(1) when YAML parsing fails
-		// which terminates the test process
-		t.Skip("Skipping test that calls os.Exit(1) when YAML parsing fails")
-	})
-
 	t.Run("ReadFromArgs_no_args", func(t *testing.T) {
 		// Test ReadFromArgs with no command line arguments
 		instructions := make(chan *LinkInstruction, 10)
@@ -1665,18 +1654,6 @@ func TestErrorPaths(t *testing.T) {
 		// FormatErrorMessage just formats the message, no "ERROR" prefix
 	})
 
-	t.Run("GenerateRandomAnsiColor", func(t *testing.T) {
-		// Test GenerateRandomAnsiColor function
-		color1 := GenerateRandomAnsiColor()
-		color2 := GenerateRandomAnsiColor()
-
-		// Colors should be different (though there's a small chance they could be the same)
-		// At least they should be valid ANSI color codes
-		assert.NotEmpty(t, color1)
-		assert.NotEmpty(t, color2)
-		assert.Contains(t, color1, "\x1b[")
-		assert.Contains(t, color2, "\x1b[")
-	})
 
 	t.Run("RunAsync_error_handling", func(t *testing.T) {
 		// Test RunAsync error handling with invalid source
@@ -1738,7 +1715,7 @@ func TestErrorPaths(t *testing.T) {
 		assert.Empty(t, links)
 
 		// Invalid pattern - this actually returns an error
-		links, err = ExpandPattern("invalid[", testDir, "target.txt")
+		_, err = ExpandPattern("invalid[", testDir, "target.txt")
 		assert.Error(t, err)
 		assert.Contains(t, err.Error(), "syntax error in pattern")
 	})
@@ -1850,12 +1827,6 @@ func TestMainFunctionsCoverage(t *testing.T) {
 		// Don't close channels - let the goroutine handle its own cleanup
 	})
 
-	t.Run("startInputSource_with_args", func(t *testing.T) {
-		// Skip this test as it calls showUsageAndExit() which calls os.Exit(1)
-		// and would terminate the test process
-		t.Skip("Skipping test that calls showUsageAndExit() as it terminates the process")
-	})
-
 	t.Run("startDefaultInputSource_with_sync_file", func(t *testing.T) {
 		// Create sync file
 		syncFile := filepath.Join(testDir, "sync")
@@ -1939,7 +1910,6 @@ func TestMainFunctionsCoverage(t *testing.T) {
 
 		// This confirms that startDefaultInputSource would call showUsageAndExit()
 		// We can't actually call it because os.Exit(1) terminates the process
-		t.Skip("Skipping test that calls showUsageAndExit() as it terminates the process")
 	})
 
 	t.Run("showUsageAndExit", func(t *testing.T) {
@@ -2079,3 +2049,295 @@ func TestMainFunctionsCoverage(t *testing.T) {
 		// Don't close channels - let the goroutine handle its own cleanup
 	})
 }
+
+// Test YAMLConfig "From" functionality
+func TestYAMLConfigFrom(t *testing.T) {
+	testDir := createTestDir(t)
+	defer cleanupTestDir(t, testDir)
+
+	// Ensure we're working within the project directory
+	ensureInProjectDir(t, testDir)
+
+	// Change to test directory
+	originalDir, _ := os.Getwd()
+	defer os.Chdir(originalDir)
+	os.Chdir(testDir)
+
+	// Create source files
+	srcFile1 := filepath.Join(testDir, "src1.txt")
+	srcFile2 := filepath.Join(testDir, "src2.txt")
+	srcFile3 := filepath.Join(testDir, "src3.txt")
+
+	err := os.WriteFile(srcFile1, []byte("content1"), 0644)
+	assert.NoError(t, err)
+	err = os.WriteFile(srcFile2, []byte("content2"), 0644)
+	assert.NoError(t, err)
+	err = os.WriteFile(srcFile3, []byte("content3"), 0644)
+	assert.NoError(t, err)
+
+	t.Run("ParseYAMLFileRecursive_simple_from", func(t *testing.T) {
+		// Create a referenced config file
+		referencedConfig := filepath.Join(testDir, "referenced.yaml")
+		referencedYAML := `links:
+  - source: src2.txt
+    target: dst2.txt
+  - source: src3.txt
+    target: dst3.txt`
+		err := os.WriteFile(referencedConfig, []byte(referencedYAML), 0644)
+		assert.NoError(t, err)
+
+		// Create main config file that references the other
+		mainConfig := filepath.Join(testDir, "main.yaml")
+		mainYAML := `links:
+  - source: src1.txt
+    target: dst1.txt
+from:
+  - referenced.yaml`
+		err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse recursively
+		instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.NoError(t, err)
+		assert.Equal(t, 3, len(instructions))
+
+		// Check that all instructions are present
+		sources := make([]string, len(instructions))
+		targets := make([]string, len(instructions))
+		for i, inst := range instructions {
+			sources[i] = filepath.Base(inst.Source)
+			targets[i] = filepath.Base(inst.Target)
+		}
+
+		assert.Contains(t, sources, "src1.txt")
+		assert.Contains(t, sources, "src2.txt")
+		assert.Contains(t, sources, "src3.txt")
+		assert.Contains(t, targets, "dst1.txt")
+		assert.Contains(t, targets, "dst2.txt")
+		assert.Contains(t, targets, "dst3.txt")
+	})
+
+	t.Run("ParseYAMLFileRecursive_multiple_from", func(t *testing.T) {
+		// Create multiple referenced config files
+		config1 := filepath.Join(testDir, "config1.yaml")
+		config1YAML := `links:
+  - source: src1.txt
+    target: dst1.txt`
+		err := os.WriteFile(config1, []byte(config1YAML), 0644)
+		assert.NoError(t, err)
+
+		config2 := filepath.Join(testDir, "config2.yaml")
+		config2YAML := `links:
+  - source: src2.txt
+    target: dst2.txt`
+		err = os.WriteFile(config2, []byte(config2YAML), 0644)
+		assert.NoError(t, err)
+
+		// Create main config that references both
+		mainConfig := filepath.Join(testDir, "main2.yaml")
+		mainYAML := `links:
+  - source: src3.txt
+    target: dst3.txt
+from:
+  - config1.yaml
+  - config2.yaml`
+		err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse recursively
+		instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.NoError(t, err)
+		assert.Equal(t, 3, len(instructions))
+
+		// Check that all instructions are present
+		sources := make([]string, len(instructions))
+		for i, inst := range instructions {
+			sources[i] = filepath.Base(inst.Source)
+		}
+		assert.Contains(t, sources, "src1.txt")
+		assert.Contains(t, sources, "src2.txt")
+		assert.Contains(t, sources, "src3.txt")
+	})
+
+	t.Run("ParseYAMLFileRecursive_nested_from", func(t *testing.T) {
+		// Create a deeply nested config structure
+		nestedDir := filepath.Join(testDir, "nested")
+		err := os.MkdirAll(nestedDir, 0755)
+		assert.NoError(t, err)
+
+		// Create source files in the nested directory
+		nestedSrcFile := filepath.Join(nestedDir, "nested_src.txt")
+		err = os.WriteFile(nestedSrcFile, []byte("nested content"), 0644)
+		assert.NoError(t, err)
+
+		// Create a config file in the nested directory with relative paths
+		nestedConfig := filepath.Join(nestedDir, "nested.yaml")
+		nestedYAML := `links:
+  - source: nested_src.txt
+    target: nested_dst.txt`
+		err = os.WriteFile(nestedConfig, []byte(nestedYAML), 0644)
+		assert.NoError(t, err)
+
+		// Create a config that references the nested one
+		intermediateConfig := filepath.Join(testDir, "intermediate.yaml")
+		intermediateYAML := `links:
+  - source: src2.txt
+    target: dst2.txt
+from:
+  - nested/nested.yaml`
+		err = os.WriteFile(intermediateConfig, []byte(intermediateYAML), 0644)
+		assert.NoError(t, err)
+
+		// Create main config that references the intermediate one
+		mainConfig := filepath.Join(testDir, "main3.yaml")
+		mainYAML := `links:
+  - source: src3.txt
+    target: dst3.txt
+from:
+  - intermediate.yaml`
+		err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse recursively
+		instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.NoError(t, err)
+		assert.Equal(t, 3, len(instructions))
+
+		// Check that all instructions are present
+		sources := make([]string, len(instructions))
+		for i, inst := range instructions {
+			sources[i] = filepath.Base(inst.Source)
+		}
+		assert.Contains(t, sources, "nested_src.txt")
+		assert.Contains(t, sources, "src2.txt")
+		assert.Contains(t, sources, "src3.txt")
+	})
+
+	t.Run("ParseYAMLFileRecursive_absolute_paths", func(t *testing.T) {
+		// Create a config file with absolute path reference
+		absoluteConfig := filepath.Join(testDir, "absolute.yaml")
+		absoluteYAML := `links:
+  - source: src1.txt
+    target: dst1.txt`
+		err := os.WriteFile(absoluteConfig, []byte(absoluteYAML), 0644)
+		assert.NoError(t, err)
+
+		// Create main config with absolute path reference
+		mainConfig := filepath.Join(testDir, "main4.yaml")
+		mainYAML := fmt.Sprintf(`links:
+  - source: src2.txt
+    target: dst2.txt
+from:
+  - %s`, absoluteConfig)
+		err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse recursively
+		instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.NoError(t, err)
+		assert.Equal(t, 2, len(instructions))
+
+		// Check that both instructions are present
+		sources := make([]string, len(instructions))
+		for i, inst := range instructions {
+			sources[i] = filepath.Base(inst.Source)
+		}
+		assert.Contains(t, sources, "src1.txt")
+		assert.Contains(t, sources, "src2.txt")
+	})
+
+	t.Run("ParseYAMLFileRecursive_circular_reference", func(t *testing.T) {
+		// Create config files that reference each other (circular)
+		config1 := filepath.Join(testDir, "circular1.yaml")
+		config1YAML := `links:
+  - source: src1.txt
+    target: dst1.txt
+from:
+  - circular2.yaml`
+		err := os.WriteFile(config1, []byte(config1YAML), 0644)
+		assert.NoError(t, err)
+
+		config2 := filepath.Join(testDir, "circular2.yaml")
+		config2YAML := `links:
+  - source: src2.txt
+    target: dst2.txt
+from:
+  - circular1.yaml`
+		err = os.WriteFile(config2, []byte(config2YAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse should detect circular reference
+		_, err = ParseYAMLFileRecursive(config1, testDir)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "circular reference detected")
+	})
+
+	t.Run("ParseYAMLFileRecursive_nonexistent_file", func(t *testing.T) {
+		// Create config that references a non-existent file
+		mainConfig := filepath.Join(testDir, "main5.yaml")
+		mainYAML := `links:
+  - source: src1.txt
+    target: dst1.txt
+from:
+  - nonexistent.yaml`
+		err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse should return error for non-existent file
+		_, err = ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "error parsing referenced file")
+	})
+
+	t.Run("ParseYAMLFileRecursive_no_from", func(t *testing.T) {
+		// Create config without "from" field (should work like regular ParseYAMLFile)
+		mainConfig := filepath.Join(testDir, "main6.yaml")
+		mainYAML := `links:
+  - source: src1.txt
+    target: dst1.txt`
+		err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse recursively
+		instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(instructions))
+		assert.Contains(t, instructions[0].Source, "src1.txt")
+		assert.Contains(t, instructions[0].Target, "dst1.txt")
+	})
+
+	t.Run("ParseYAMLFileRecursive_empty_from", func(t *testing.T) {
+		// Create config with empty "from" field
+		mainConfig := filepath.Join(testDir, "main7.yaml")
+		mainYAML := `links:
+  - source: src1.txt
+    target: dst1.txt
+from: []`
+		err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse recursively
+		instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.NoError(t, err)
+		assert.Equal(t, 1, len(instructions))
+		assert.Contains(t, instructions[0].Source, "src1.txt")
+		assert.Contains(t, instructions[0].Target, "dst1.txt")
+	})
+
+	t.Run("ParseYAMLFileRecursive_self_reference", func(t *testing.T) {
+		// Create config that references itself
+		mainConfig := filepath.Join(testDir, "main8.yaml")
+		mainYAML := `links:
+  - source: src1.txt
+    target: dst1.txt
+from:
+  - main8.yaml`
+		err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
+		assert.NoError(t, err)
+
+		// Parse should detect self-reference
+		_, err = ParseYAMLFileRecursive(mainConfig, testDir)
+		assert.Error(t, err)
+		assert.Contains(t, err.Error(), "circular reference detected")
+	})
+}

main.go

@@ -231,7 +231,7 @@ func ReadFromFile(input string, output chan *LinkInstruction, status chan error,
 	// Check if this is a YAML file
 	if IsYAMLFile(input) {
 		LogInfo("Parsing as YAML file")
-		instructions, err := ParseYAMLFile(input, filepath.Dir(input))
+		instructions, err := ParseYAMLFileRecursive(input, filepath.Dir(input))
 		if err != nil {
 			LogError("Failed to parse YAML file %s: %v",
 				FormatSourcePath(input), err)