Add support (and preference) to toml from yml

go.mod

@@ -12,6 +12,7 @@ require (
 )
 
 require (
+	github.com/BurntSushi/toml v1.5.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/hexops/valast v1.5.0 // indirect
 	github.com/jinzhu/inflection v1.0.0 // indirect

go.sum

@@ -1,5 +1,7 @@
 git.site.quack-lab.dev/dave/cylogger v1.3.0 h1:eTWPUD+ThVi8kGIsRcE0XDeoH3yFb5miFEODyKUdWJw=
 git.site.quack-lab.dev/dave/cylogger v1.3.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk=
+github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
+github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
 github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
 github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=

main.go

@@ -13,7 +13,7 @@ import (
 	"cook/processor"
 	"cook/utils"
 
-	"gopkg.in/yaml.v3"
+	"github.com/BurntSushi/toml"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
 )
@@ -402,132 +402,150 @@ func HandleSpecialArgs(args []string, db utils.DB) (bool, error) {
 func CreateExampleConfig() {
 	createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig")
 	createExampleConfigLogger.Debug("Creating example configuration file")
-	commands := []utils.ModifyCommand{
+
-		// Global modifiers only entry (no name/regex/lua/files)
+	// TOML structure with commands array
-		{
+	tomlData := struct {
-			Modifiers: map[string]interface{}{
+		Modifiers map[string]interface{} `toml:"modifiers"`
-				"foobar":   4,
+		Commands  []utils.ModifyCommand  `toml:"commands"`
-				"multiply": 1.5,
+	}{
-				"prefix":   "NEW_",
+		Modifiers: map[string]interface{}{
-				"enabled":  true,
+			"foobar":   4,
+			"multiply": 1.5,
+			"prefix":   "NEW_",
+			"enabled":  true,
+		},
+		Commands: []utils.ModifyCommand{
+			// Multi-regex example using variable in Lua
+			{
+				Name:    "RFToolsMultiply",
+				Regexes: []string{"generatePerTick = !num", "ticksPer\\w+ = !num", "generatorRFPerTick = !num"},
+				Lua:     "* foobar",
+				Files:   []string{"polymc/instances/**/rftools*.toml", `polymc\instances\**\rftools*.toml`},
+				Reset:   true,
 			},
-		},
+			// Named capture groups with arithmetic and string ops
-		// Multi-regex example using $variable in Lua
+			{
-		{
+				Name:  "UpdateAmountsAndItems",
-			Name:    "RFToolsMultiply",
+				Regex: `(?P<amount>!num)\s+units\s+of\s+(?P<item>[A-Za-z_\-]+)`,
-			Regexes: []string{"generatePerTick = !num", "ticksPer\\w+ = !num", "generatorRFPerTick = !num"},
+				Lua:   `amount = amount * multiply; item = upper(item); return true`,
-			Lua:     "* $foobar",
+				Files: []string{"data/**/*.txt"},
-			Files:   []string{"polymc/instances/**/rftools*.toml", `polymc\\instances\\**\\rftools*.toml`},
+			},
-			Reset:   true,
+			// Full replacement via Lua 'replacement' variable
-			// LogLevel defaults to INFO
+			{
-		},
+				Name:  "BumpMinorVersion",
-		// Named capture groups with arithmetic and string ops
+				Regex: `version\s*=\s*"(?P<major>!num)\.(?P<minor>!num)\.(?P<patch>!num)"`,
-		{
+				Lua:   `replacement = format("version=\"%s.%s.%s\"", major, num(minor)+1, 0); return true`,
-			Name:  "UpdateAmountsAndItems",
+				Files: []string{"config/*.ini", "config/*.cfg"},
-			Regex: `(?P<amount>!num)\s+units\s+of\s+(?P<item>[A-Za-z_\-]+)`,
+			},
-			Lua:   `amount = amount * $multiply; item = upper(item); return true`,
+			// TOML multiline regex example - this is the key feature!
-			Files: []string{"data/**/*.txt"},
+			{
-			// INFO log level
+				Name: "StressValues",
-		},
+				Regex: `
-		// Full replacement via Lua 'replacement' variable
+\[kinetics\.stressValues\.v2\.capacity\]
-		{
+
-			Name:  "BumpMinorVersion",
+    steam_engine = !num
-			Regex: `version\s*=\s*"(?P<major>!num)\.(?P<minor>!num)\.(?P<patch>!num)"`,
+
-			Lua:   `replacement = format("version=\"%s.%s.%s\"", major, num(minor)+1, 0); return true`,
+    water_wheel = !num
-			Files: []string{"config/*.ini", "config/*.cfg"},
+
-		},
+    copper_valve_handle = !num
-		// Multiline regex example (DOTALL is auto-enabled). Captures numeric in nested XML.
+
-		{
+    hand_crank = !num
-			Name:  "XMLNestedValueMultiply",
+
-			Regex: `<item>\s*\s*<name>!any<\/name>\s*\s*<value>(!num)<\/value>\s*\s*<\/item>`,
+    creative_motor = !num`,
-			Lua:   `* $multiply`,
+				Lua:     "v1 * multiply",
-			Files: []string{"data/**/*.xml"},
+				Files:   []string{"*.txt"},
-			// Demonstrates multiline regex in YAML
+				Isolate: true,
-		},
+			},
-		// Multiline regexES array, with different patterns handled by same Lua
+			// Network configuration with complex multiline regex
-		{
+			{
-			Name: "MultiLinePatterns",
+				Name: "NetworkConfig",
-			Regexes: []string{
+				Regex: `
-				`<entry>\s*\n\s*<id>(?P<id>!num)</id>\s*\n\s*<score>(?P<score>!num)</score>\s*\n\s*</entry>`,
+networking\.firewall\.allowPing = true
-				`\[block\]\nkey=(?P<key>[A-Za-z_]+)\nvalue=(?P<val>!num)`,
+
+networking\.firewall\.allowedTCPPorts = \[ 47984 47989 47990 \]
+
+networking\.firewall\.allowedUDPPortRanges = \[
+    \{ from = \d+; to = \d+; \}
+    \{ from = 8000; to = 8010; \}
+\]`,
+				Lua:     `replacement = string.gsub(block[1], 'true', 'false')`,
+				Files:   []string{"*.conf"},
+				Isolate: true,
+			},
+			// Use equals operator shorthand and boolean variable
+			{
+				Name:  "EnableFlags",
+				Regex: `enabled\s*=\s*(true|false)`,
+				Lua:   `= enabled`,
+				Files: []string{"**/*.toml"},
+			},
+			// Demonstrate NoDedup to allow overlapping replacements
+			{
+				Name:    "OverlappingGroups",
+				Regex:   `(?P<a>!num)(?P<b>!num)`,
+				Lua:     `a = num(a) + 1; b = num(b) + 1; return true`,
+				Files:   []string{"overlap/**/*.txt"},
+				NoDedup: true,
+			},
+			// Isolate command example operating on entire matched block
+			{
+				Name:     "IsolateUppercaseBlock",
+				Regex:    `BEGIN\n(?P<block>!any)\nEND`,
+				Lua:      `block = upper(block); return true`,
+				Files:    []string{"logs/**/*.log"},
+				Isolate:  true,
+				LogLevel: "TRACE",
+			},
+			// Using !rep placeholder and arrays of files
+			{
+				Name:  "RepeatPlaceholderExample",
+				Regex: `name: (.*) !rep(, .* , 2)`,
+				Lua:   `-- no-op, just demonstrate placeholder; return false`,
+				Files: []string{"lists/**/*.yml", "lists/**/*.yaml"},
+			},
+			// Using string variable in Lua expression
+			{
+				Name:  "PrefixKeys",
+				Regex: `(?P<key>[A-Za-z0-9_]+)\s*=`,
+				Lua:   `key = prefix .. key; return true`,
+				Files: []string{"**/*.properties"},
+			},
+			// JSON mode examples
+			{
+				Name:  "JSONArrayMultiply",
+				JSON:  true,
+				Lua:   `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`,
+				Files: []string{"data/**/*.json"},
+			},
+			{
+				Name:  "JSONObjectUpdate",
+				JSON:  true,
+				Lua:   `data.version = "2.0.0"; data.enabled = true; return true`,
+				Files: []string{"config/**/*.json"},
+			},
+			{
+				Name:  "JSONNestedModify",
+				JSON:  true,
+				Lua:   `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`,
+				Files: []string{"settings/**/*.json"},
 			},
-			Lua:      `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`,
-			Files:    []string{"examples/**/*.*"},
-			LogLevel: "DEBUG",
-		},
-		// Use equals operator shorthand and boolean variable
-		{
-			Name:  "EnableFlags",
-			Regex: `enabled\s*=\s*(true|false)`,
-			Lua:   `= $enabled`,
-			Files: []string{"**/*.toml"},
-		},
-		// Demonstrate NoDedup to allow overlapping replacements
-		{
-			Name:    "OverlappingGroups",
-			Regex:   `(?P<a>!num)(?P<b>!num)`,
-			Lua:     `a = num(a) + 1; b = num(b) + 1; return true`,
-			Files:   []string{"overlap/**/*.txt"},
-			NoDedup: true,
-		},
-		// Isolate command example operating on entire matched block
-		{
-			Name:     "IsolateUppercaseBlock",
-			Regex:    `BEGIN\n(?P<block>!any)\nEND`,
-			Lua:      `block = upper(block); return true`,
-			Files:    []string{"logs/**/*.log"},
-			Isolate:  true,
-			LogLevel: "TRACE",
-		},
-		// Using !rep placeholder and arrays of files
-		{
-			Name:  "RepeatPlaceholderExample",
-			Regex: `name: (.*) !rep(, .* , 2)`,
-			Lua:   `-- no-op, just demonstrate placeholder; return false`,
-			Files: []string{"lists/**/*.yml", "lists/**/*.yaml"},
-		},
-		// Using string variable in Lua expression
-		{
-			Name:  "PrefixKeys",
-			Regex: `(?P<key>[A-Za-z0-9_]+)\s*=`,
-			Lua:   `key = $prefix .. key; return true`,
-			Files: []string{"**/*.properties"},
-		},
-		// JSON mode examples
-		{
-			Name:  "JSONArrayMultiply",
-			JSON:  true,
-			Lua:   `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`,
-			Files: []string{"data/**/*.json"},
-		},
-		{
-			Name:  "JSONObjectUpdate",
-			JSON:  true,
-			Lua:   `data.version = "2.0.0"; data.enabled = true; return true`,
-			Files: []string{"config/**/*.json"},
-		},
-		{
-			Name:  "JSONNestedModify",
-			JSON:  true,
-			Lua:   `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`,
-			Files: []string{"settings/**/*.json"},
 		},
 	}
 
-	data, err := yaml.Marshal(commands)
+	data, err := toml.Marshal(tomlData)
 	if err != nil {
 		createExampleConfigLogger.Error("Failed to marshal example config: %v", err)
 		return
 	}
 
-	createExampleConfigLogger.Debug("Writing example_cook.yml")
+	createExampleConfigLogger.Debug("Writing example_cook.toml")
-	err = os.WriteFile("example_cook.yml", data, 0644)
+	err = os.WriteFile("example_cook.toml", data, 0644)
 	if err != nil {
-		createExampleConfigLogger.Error("Failed to write example_cook.yml: %v", err)
+		createExampleConfigLogger.Error("Failed to write example_cook.toml: %v", err)
 		return
 	}
 
-	createExampleConfigLogger.Info("Wrote example_cook.yml")
+	createExampleConfigLogger.Info("Wrote example_cook.toml")
 }
 
 var NothingToDo = errors.New("nothing to do")

utils/modifycommand.go

@@ -8,6 +8,7 @@ import (
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
 	"github.com/bmatcuk/doublestar/v4"
+	"github.com/BurntSushi/toml"
 	"gopkg.in/yaml.v3"
 )
 
@@ -15,18 +16,18 @@ import (
 var modifyCommandLogger = logger.Default.WithPrefix("utils/modifycommand")
 
 type ModifyCommand struct {
-	Name      string                 `yaml:"name,omitempty"`
+	Name      string                 `yaml:"name,omitempty" toml:"name,omitempty"`
-	Regex     string                 `yaml:"regex,omitempty"`
+	Regex     string                 `yaml:"regex,omitempty" toml:"regex,omitempty"`
-	Regexes   []string               `yaml:"regexes,omitempty"`
+	Regexes   []string               `yaml:"regexes,omitempty" toml:"regexes,omitempty"`
-	Lua       string                 `yaml:"lua,omitempty"`
+	Lua       string                 `yaml:"lua,omitempty" toml:"lua,omitempty"`
-	Files     []string               `yaml:"files,omitempty"`
+	Files     []string               `yaml:"files,omitempty" toml:"files,omitempty"`
-	Reset     bool                   `yaml:"reset,omitempty"`
+	Reset     bool                   `yaml:"reset,omitempty" toml:"reset,omitempty"`
-	LogLevel  string                 `yaml:"loglevel,omitempty"`
+	LogLevel  string                 `yaml:"loglevel,omitempty" toml:"loglevel,omitempty"`
-	Isolate   bool                   `yaml:"isolate,omitempty"`
+	Isolate   bool                   `yaml:"isolate,omitempty" toml:"isolate,omitempty"`
-	NoDedup   bool                   `yaml:"nodedup,omitempty"`
+	NoDedup   bool                   `yaml:"nodedup,omitempty" toml:"nodedup,omitempty"`
-	Disabled  bool                   `yaml:"disable,omitempty"`
+	Disabled  bool                   `yaml:"disable,omitempty" toml:"disable,omitempty"`
-	JSON      bool                   `yaml:"json,omitempty"`
+	JSON      bool                   `yaml:"json,omitempty" toml:"json,omitempty"`
-	Modifiers map[string]interface{} `yaml:"modifiers,omitempty"`
+	Modifiers map[string]interface{} `yaml:"modifiers,omitempty" toml:"modifiers,omitempty"`
 }
 
 type CookFile []ModifyCommand
@@ -265,11 +266,27 @@ func LoadCommands(args []string) ([]ModifyCommand, error) {
 
 	for _, arg := range args {
 		loadCommandsLogger.Debug("Processing argument for commands: %q", arg)
-		newCommands, err := LoadCommandsFromCookFiles(arg)
+		var newCommands []ModifyCommand
-		if err != nil {
+		var err error
-			loadCommandsLogger.Error("Failed to load commands from argument %q: %v", arg, err)
+
-			return nil, fmt.Errorf("failed to load commands from cook files: %w", err)
+		// Check file extension to determine format
+		if strings.HasSuffix(arg, ".toml") {
+			loadCommandsLogger.Debug("Loading TOML commands from %q", arg)
+			newCommands, err = LoadCommandsFromTomlFiles(arg)
+			if err != nil {
+				loadCommandsLogger.Error("Failed to load TOML commands from argument %q: %v", arg, err)
+				return nil, fmt.Errorf("failed to load commands from TOML files: %w", err)
+			}
+		} else {
+			// Default to YAML for .yml, .yaml, or any other extension
+			loadCommandsLogger.Debug("Loading YAML commands from %q", arg)
+			newCommands, err = LoadCommandsFromCookFiles(arg)
+			if err != nil {
+				loadCommandsLogger.Error("Failed to load YAML commands from argument %q: %v", arg, err)
+				return nil, fmt.Errorf("failed to load commands from cook files: %w", err)
+			}
 		}
+
 		loadCommandsLogger.Debug("Successfully loaded %d commands from %q", len(newCommands), arg)
 		for _, cmd := range newCommands {
 			if cmd.Disabled {
@@ -373,3 +390,83 @@ func FilterCommands(commands []ModifyCommand, filter string) []ModifyCommand {
 	filterCommandsLogger.Trace("Filtered commands: %v", filteredCommands)
 	return filteredCommands
 }
+
+func LoadCommandsFromTomlFiles(pattern string) ([]ModifyCommand, error) {
+	loadTomlFilesLogger := modifyCommandLogger.WithPrefix("LoadCommandsFromTomlFiles").WithField("pattern", pattern)
+	loadTomlFilesLogger.Debug("Loading commands from TOML files based on pattern")
+	loadTomlFilesLogger.Trace("Input pattern: %q", pattern)
+	static, pattern := SplitPattern(pattern)
+	commands := []ModifyCommand{}
+	tomlFiles, err := doublestar.Glob(os.DirFS(static), pattern)
+	if err != nil {
+		loadTomlFilesLogger.Error("Failed to glob TOML files for pattern %q: %v", pattern, err)
+		return nil, fmt.Errorf("failed to glob TOML files: %w", err)
+	}
+	loadTomlFilesLogger.Debug("Found %d TOML files for pattern %q", len(tomlFiles), pattern)
+	loadTomlFilesLogger.Trace("TOML files found: %v", tomlFiles)
+
+	for _, tomlFile := range tomlFiles {
+		tomlFile = filepath.Join(static, tomlFile)
+		tomlFile = filepath.Clean(tomlFile)
+		tomlFile = strings.ReplaceAll(tomlFile, "\\", "/")
+		loadTomlFilesLogger.Debug("Loading commands from individual TOML file: %q", tomlFile)
+
+		tomlFileData, err := os.ReadFile(tomlFile)
+		if err != nil {
+			loadTomlFilesLogger.Error("Failed to read TOML file %q: %v", tomlFile, err)
+			return nil, fmt.Errorf("failed to read TOML file: %w", err)
+		}
+		loadTomlFilesLogger.Trace("Read %d bytes from TOML file %q", len(tomlFileData), tomlFile)
+		newCommands, err := LoadCommandsFromTomlFile(tomlFileData)
+		if err != nil {
+			loadTomlFilesLogger.Error("Failed to load commands from TOML file data for %q: %v", tomlFile, err)
+			return nil, fmt.Errorf("failed to load commands from TOML file: %w", err)
+		}
+		commands = append(commands, newCommands...)
+		loadTomlFilesLogger.Debug("Added %d commands from TOML file %q. Total commands now: %d", len(newCommands), tomlFile, len(commands))
+	}
+
+	loadTomlFilesLogger.Debug("Finished loading commands from TOML files. Total %d commands", len(commands))
+	return commands, nil
+}
+
+func LoadCommandsFromTomlFile(tomlFileData []byte) ([]ModifyCommand, error) {
+	loadTomlCommandLogger := modifyCommandLogger.WithPrefix("LoadCommandsFromTomlFile")
+	loadTomlCommandLogger.Debug("Unmarshaling commands from TOML file data")
+	loadTomlCommandLogger.Trace("TOML file data length: %d", len(tomlFileData))
+
+	// TOML structure for commands array
+	var tomlData struct {
+		Commands []ModifyCommand `toml:"commands"`
+		// Also support direct array without wrapper
+		DirectCommands []ModifyCommand `toml:"-"`
+	}
+
+	// First try to parse as wrapped structure
+	err := toml.Unmarshal(tomlFileData, &tomlData)
+	if err != nil {
+		loadTomlCommandLogger.Error("Failed to unmarshal TOML file data: %v", err)
+		return nil, fmt.Errorf("failed to unmarshal TOML file: %w", err)
+	}
+
+	var commands []ModifyCommand
+
+	// If we found commands in the wrapped structure, use those
+	if len(tomlData.Commands) > 0 {
+		commands = tomlData.Commands
+		loadTomlCommandLogger.Debug("Found %d commands in wrapped TOML structure", len(commands))
+	} else {
+		// Try to parse as direct array (similar to YAML format)
+		commands = []ModifyCommand{}
+		err = toml.Unmarshal(tomlFileData, &commands)
+		if err != nil {
+			loadTomlCommandLogger.Error("Failed to unmarshal TOML file data as direct array: %v", err)
+			return nil, fmt.Errorf("failed to unmarshal TOML file as direct array: %w", err)
+		}
+		loadTomlCommandLogger.Debug("Found %d commands in direct TOML array", len(commands))
+	}
+
+	loadTomlCommandLogger.Debug("Successfully unmarshaled %d commands", len(commands))
+	loadTomlCommandLogger.Trace("Unmarshaled commands: %v", commands)
+	return commands, nil
+}