dave/synclib
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 4 Wiki Activity

Implement a "from" to config files that loads other files

Browse Source
This commit is contained in:
PhatPhuckDave
2025-10-06 22:17:34 +02:00
parent 21d7f56ccf
commit eb81ec4162
4 changed files with 368 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
go.mod
View File

@@ -4,10 +4,12 @@ go 1.21.7
require gopkg.in/yaml.v3 v3.0.1
require github.com/bmatcuk/doublestar/v4 v4.8.1
require (
github.com/bmatcuk/doublestar/v4 v4.8.1
github.com/stretchr/testify v1.11.1
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/stretchr/testify v1.11.1 // indirect
)

70
instruction.go
View File

@@ -20,6 +20,7 @@ type LinkInstruction struct {
type YAMLConfig struct {
Links []LinkInstruction `yaml:"links"`
From []string `yaml:"from,omitempty"`
}
func (instruction *LinkInstruction) Tidy() {
@@ -318,9 +319,76 @@ func ParseYAMLFile(filename, workdir string) ([]LinkInstruction, error) {
return expanded, nil
}
// ParseYAMLFileRecursive parses a YAML file and recursively processes any "From" references
func ParseYAMLFileRecursive(filename, workdir string) ([]LinkInstruction, error) {
visited := make(map[string]bool)
return parseYAMLFileRecursive(filename, workdir, visited)
}
// parseYAMLFileRecursive is the internal recursive function that tracks visited files to prevent cycles
func parseYAMLFileRecursive(filename, workdir string, visited map[string]bool) ([]LinkInstruction, error) {
// Normalize the filename to prevent cycles with different path representations
normalizedFilename, err := filepath.Abs(filename)
if err != nil {
return nil, fmt.Errorf("error normalizing filename: %w", err)
}
// Check for cycles
if visited[normalizedFilename] {
return nil, fmt.Errorf("circular reference detected: %s", filename)
}
visited[normalizedFilename] = true
defer delete(visited, normalizedFilename)
// Parse the current file
instructions, err := ParseYAMLFile(filename, workdir)
if err != nil {
return nil, err
}
// Read the file to check for "From" references
data, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("error reading YAML file: %w", err)
}
var config YAMLConfig
err = yaml.Unmarshal(data, &config)
if err != nil {
// If parsing as YAMLConfig fails, there are no "From" references to process
return instructions, nil
}
// Process "From" references
for _, fromFile := range config.From {
// Convert relative paths to absolute paths based on the current file's directory
fromPath := fromFile
if !filepath.IsAbs(fromPath) {
currentDir := filepath.Dir(filename)
fromPath = filepath.Join(currentDir, fromPath)
}
// Normalize the path
fromPath = filepath.Clean(fromPath)
// Recursively parse the referenced file
// Use the directory of the referenced file as the workdir for pattern expansion
fromWorkdir := filepath.Dir(fromPath)
fromInstructions, err := parseYAMLFileRecursive(fromPath, fromWorkdir, visited)
if err != nil {
return nil, fmt.Errorf("error parsing referenced file %s: %w", fromFile, err)
}
// Append the instructions from the referenced file
instructions = append(instructions, fromInstructions...)
}
return instructions, nil
}
func ExpandPattern(source, workdir, target string) (links []LinkInstruction, err error) {
static, pattern := doublestar.SplitPattern(source)
if static == "" {
if static == "" || static == "." {
static = workdir
}
LogInfo("Static part: %s", static)

326
instruction_test.go
View File

@@ -2,6 +2,7 @@ package main
import (
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -1450,18 +1451,6 @@ func TestErrorPaths(t *testing.T) {
assert.Contains(t, err.Error(), "invalid format")
})
t.Run("ReadFromFile_nonexistent_file", func(t *testing.T) {
// Skip this test as ReadFromFile calls os.Exit(1) when file doesn't exist
// which terminates the test process
t.Skip("Skipping test that calls os.Exit(1) when file doesn't exist")
})
t.Run("ReadFromFile_invalid_yaml", func(t *testing.T) {
// Skip this test as ReadFromFile calls os.Exit(1) when YAML parsing fails
// which terminates the test process
t.Skip("Skipping test that calls os.Exit(1) when YAML parsing fails")
})
t.Run("ReadFromArgs_no_args", func(t *testing.T) {
// Test ReadFromArgs with no command line arguments
instructions := make(chan *LinkInstruction, 10)
@@ -1665,18 +1654,6 @@ func TestErrorPaths(t *testing.T) {
// FormatErrorMessage just formats the message, no "ERROR" prefix
})
t.Run("GenerateRandomAnsiColor", func(t *testing.T) {
// Test GenerateRandomAnsiColor function
color1 := GenerateRandomAnsiColor()
color2 := GenerateRandomAnsiColor()
// Colors should be different (though there's a small chance they could be the same)
// At least they should be valid ANSI color codes
assert.NotEmpty(t, color1)
assert.NotEmpty(t, color2)
assert.Contains(t, color1, "\x1b[")
assert.Contains(t, color2, "\x1b[")
})
t.Run("RunAsync_error_handling", func(t *testing.T) {
// Test RunAsync error handling with invalid source
@@ -1738,7 +1715,7 @@ func TestErrorPaths(t *testing.T) {
assert.Empty(t, links)
// Invalid pattern - this actually returns an error
links, err = ExpandPattern("invalid[", testDir, "target.txt")
_, err = ExpandPattern("invalid[", testDir, "target.txt")
assert.Error(t, err)
assert.Contains(t, err.Error(), "syntax error in pattern")
})
@@ -1850,12 +1827,6 @@ func TestMainFunctionsCoverage(t *testing.T) {
// Don't close channels - let the goroutine handle its own cleanup
})
t.Run("startInputSource_with_args", func(t *testing.T) {
// Skip this test as it calls showUsageAndExit() which calls os.Exit(1)
// and would terminate the test process
t.Skip("Skipping test that calls showUsageAndExit() as it terminates the process")
})
t.Run("startDefaultInputSource_with_sync_file", func(t *testing.T) {
// Create sync file
syncFile := filepath.Join(testDir, "sync")
@@ -1939,7 +1910,6 @@ func TestMainFunctionsCoverage(t *testing.T) {
// This confirms that startDefaultInputSource would call showUsageAndExit()
// We can't actually call it because os.Exit(1) terminates the process
t.Skip("Skipping test that calls showUsageAndExit() as it terminates the process")
})
t.Run("showUsageAndExit", func(t *testing.T) {
@@ -2079,3 +2049,295 @@ func TestMainFunctionsCoverage(t *testing.T) {
// Don't close channels - let the goroutine handle its own cleanup
})
}
// Test YAMLConfig "From" functionality
func TestYAMLConfigFrom(t *testing.T) {
testDir := createTestDir(t)
defer cleanupTestDir(t, testDir)
// Ensure we're working within the project directory
ensureInProjectDir(t, testDir)
// Change to test directory
originalDir, _ := os.Getwd()
defer os.Chdir(originalDir)
os.Chdir(testDir)
// Create source files
srcFile1 := filepath.Join(testDir, "src1.txt")
srcFile2 := filepath.Join(testDir, "src2.txt")
srcFile3 := filepath.Join(testDir, "src3.txt")
err := os.WriteFile(srcFile1, []byte("content1"), 0644)
assert.NoError(t, err)
err = os.WriteFile(srcFile2, []byte("content2"), 0644)
assert.NoError(t, err)
err = os.WriteFile(srcFile3, []byte("content3"), 0644)
assert.NoError(t, err)
t.Run("ParseYAMLFileRecursive_simple_from", func(t *testing.T) {
// Create a referenced config file
referencedConfig := filepath.Join(testDir, "referenced.yaml")
referencedYAML := `links:
- source: src2.txt
target: dst2.txt
- source: src3.txt
target: dst3.txt`
err := os.WriteFile(referencedConfig, []byte(referencedYAML), 0644)
assert.NoError(t, err)
// Create main config file that references the other
mainConfig := filepath.Join(testDir, "main.yaml")
mainYAML := `links:
- source: src1.txt
target: dst1.txt
from:
- referenced.yaml`
err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse recursively
instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
assert.NoError(t, err)
assert.Equal(t, 3, len(instructions))
// Check that all instructions are present
sources := make([]string, len(instructions))
targets := make([]string, len(instructions))
for i, inst := range instructions {
sources[i] = filepath.Base(inst.Source)
targets[i] = filepath.Base(inst.Target)
}
assert.Contains(t, sources, "src1.txt")
assert.Contains(t, sources, "src2.txt")
assert.Contains(t, sources, "src3.txt")
assert.Contains(t, targets, "dst1.txt")
assert.Contains(t, targets, "dst2.txt")
assert.Contains(t, targets, "dst3.txt")
})
t.Run("ParseYAMLFileRecursive_multiple_from", func(t *testing.T) {
// Create multiple referenced config files
config1 := filepath.Join(testDir, "config1.yaml")
config1YAML := `links:
- source: src1.txt
target: dst1.txt`
err := os.WriteFile(config1, []byte(config1YAML), 0644)
assert.NoError(t, err)
config2 := filepath.Join(testDir, "config2.yaml")
config2YAML := `links:
- source: src2.txt
target: dst2.txt`
err = os.WriteFile(config2, []byte(config2YAML), 0644)
assert.NoError(t, err)
// Create main config that references both
mainConfig := filepath.Join(testDir, "main2.yaml")
mainYAML := `links:
- source: src3.txt
target: dst3.txt
from:
- config1.yaml
- config2.yaml`
err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse recursively
instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
assert.NoError(t, err)
assert.Equal(t, 3, len(instructions))
// Check that all instructions are present
sources := make([]string, len(instructions))
for i, inst := range instructions {
sources[i] = filepath.Base(inst.Source)
}
assert.Contains(t, sources, "src1.txt")
assert.Contains(t, sources, "src2.txt")
assert.Contains(t, sources, "src3.txt")
})
t.Run("ParseYAMLFileRecursive_nested_from", func(t *testing.T) {
// Create a deeply nested config structure
nestedDir := filepath.Join(testDir, "nested")
err := os.MkdirAll(nestedDir, 0755)
assert.NoError(t, err)
// Create source files in the nested directory
nestedSrcFile := filepath.Join(nestedDir, "nested_src.txt")
err = os.WriteFile(nestedSrcFile, []byte("nested content"), 0644)
assert.NoError(t, err)
// Create a config file in the nested directory with relative paths
nestedConfig := filepath.Join(nestedDir, "nested.yaml")
nestedYAML := `links:
- source: nested_src.txt
target: nested_dst.txt`
err = os.WriteFile(nestedConfig, []byte(nestedYAML), 0644)
assert.NoError(t, err)
// Create a config that references the nested one
intermediateConfig := filepath.Join(testDir, "intermediate.yaml")
intermediateYAML := `links:
- source: src2.txt
target: dst2.txt
from:
- nested/nested.yaml`
err = os.WriteFile(intermediateConfig, []byte(intermediateYAML), 0644)
assert.NoError(t, err)
// Create main config that references the intermediate one
mainConfig := filepath.Join(testDir, "main3.yaml")
mainYAML := `links:
- source: src3.txt
target: dst3.txt
from:
- intermediate.yaml`
err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse recursively
instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
assert.NoError(t, err)
assert.Equal(t, 3, len(instructions))
// Check that all instructions are present
sources := make([]string, len(instructions))
for i, inst := range instructions {
sources[i] = filepath.Base(inst.Source)
}
assert.Contains(t, sources, "nested_src.txt")
assert.Contains(t, sources, "src2.txt")
assert.Contains(t, sources, "src3.txt")
})
t.Run("ParseYAMLFileRecursive_absolute_paths", func(t *testing.T) {
// Create a config file with absolute path reference
absoluteConfig := filepath.Join(testDir, "absolute.yaml")
absoluteYAML := `links:
- source: src1.txt
target: dst1.txt`
err := os.WriteFile(absoluteConfig, []byte(absoluteYAML), 0644)
assert.NoError(t, err)
// Create main config with absolute path reference
mainConfig := filepath.Join(testDir, "main4.yaml")
mainYAML := fmt.Sprintf(`links:
- source: src2.txt
target: dst2.txt
from:
- %s`, absoluteConfig)
err = os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse recursively
instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
assert.NoError(t, err)
assert.Equal(t, 2, len(instructions))
// Check that both instructions are present
sources := make([]string, len(instructions))
for i, inst := range instructions {
sources[i] = filepath.Base(inst.Source)
}
assert.Contains(t, sources, "src1.txt")
assert.Contains(t, sources, "src2.txt")
})
t.Run("ParseYAMLFileRecursive_circular_reference", func(t *testing.T) {
// Create config files that reference each other (circular)
config1 := filepath.Join(testDir, "circular1.yaml")
config1YAML := `links:
- source: src1.txt
target: dst1.txt
from:
- circular2.yaml`
err := os.WriteFile(config1, []byte(config1YAML), 0644)
assert.NoError(t, err)
config2 := filepath.Join(testDir, "circular2.yaml")
config2YAML := `links:
- source: src2.txt
target: dst2.txt
from:
- circular1.yaml`
err = os.WriteFile(config2, []byte(config2YAML), 0644)
assert.NoError(t, err)
// Parse should detect circular reference
_, err = ParseYAMLFileRecursive(config1, testDir)
assert.Error(t, err)
assert.Contains(t, err.Error(), "circular reference detected")
})
t.Run("ParseYAMLFileRecursive_nonexistent_file", func(t *testing.T) {
// Create config that references a non-existent file
mainConfig := filepath.Join(testDir, "main5.yaml")
mainYAML := `links:
- source: src1.txt
target: dst1.txt
from:
- nonexistent.yaml`
err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse should return error for non-existent file
_, err = ParseYAMLFileRecursive(mainConfig, testDir)
assert.Error(t, err)
assert.Contains(t, err.Error(), "error parsing referenced file")
})
t.Run("ParseYAMLFileRecursive_no_from", func(t *testing.T) {
// Create config without "from" field (should work like regular ParseYAMLFile)
mainConfig := filepath.Join(testDir, "main6.yaml")
mainYAML := `links:
- source: src1.txt
target: dst1.txt`
err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse recursively
instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
assert.NoError(t, err)
assert.Equal(t, 1, len(instructions))
assert.Contains(t, instructions[0].Source, "src1.txt")
assert.Contains(t, instructions[0].Target, "dst1.txt")
})
t.Run("ParseYAMLFileRecursive_empty_from", func(t *testing.T) {
// Create config with empty "from" field
mainConfig := filepath.Join(testDir, "main7.yaml")
mainYAML := `links:
- source: src1.txt
target: dst1.txt
from: []`
err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse recursively
instructions, err := ParseYAMLFileRecursive(mainConfig, testDir)
assert.NoError(t, err)
assert.Equal(t, 1, len(instructions))
assert.Contains(t, instructions[0].Source, "src1.txt")
assert.Contains(t, instructions[0].Target, "dst1.txt")
})
t.Run("ParseYAMLFileRecursive_self_reference", func(t *testing.T) {
// Create config that references itself
mainConfig := filepath.Join(testDir, "main8.yaml")
mainYAML := `links:
- source: src1.txt
target: dst1.txt
from:
- main8.yaml`
err := os.WriteFile(mainConfig, []byte(mainYAML), 0644)
assert.NoError(t, err)
// Parse should detect self-reference
_, err = ParseYAMLFileRecursive(mainConfig, testDir)
assert.Error(t, err)
assert.Contains(t, err.Error(), "circular reference detected")
})
}

2
main.go
View File

@@ -231,7 +231,7 @@ func ReadFromFile(input string, output chan *LinkInstruction, status chan error,
// Check if this is a YAML file
if IsYAMLFile(input) {
LogInfo("Parsing as YAML file")
instructions, err := ParseYAMLFile(input, filepath.Dir(input))
instructions, err := ParseYAMLFileRecursive(input, filepath.Dir(input))
if err != nil {
LogError("Failed to parse YAML file %s: %v",
FormatSourcePath(input), err)