8 Commits

8 changed files with 1090 additions and 242 deletions

119
example_cook.toml Normal file
View File

@@ -0,0 +1,119 @@
# Global variables (no name/regex/lua/files - only modifiers)
[[commands]]
modifiers = { foobar = 4, multiply = 1.5, prefix = 'NEW_', enabled = true }
# Multi-regex example using variable in Lua
[[commands]]
name = 'RFToolsMultiply'
regexes = [
'generatePerTick = !num',
'ticksPer\w+ = !num',
'generatorRFPerTick = !num',
]
lua = '* foobar'
files = [
'polymc/instances/**/rftools*.toml',
'polymc\instances\**\rftools*.toml',
]
reset = true
# Named capture groups with arithmetic and string ops
[[commands]]
name = 'UpdateAmountsAndItems'
regex = '(?P<amount>!num)\s+units\s+of\s+(?P<item>[A-Za-z_\-]+)'
lua = 'amount = amount * multiply; item = upper(item); return true'
files = ['data/**/*.txt']
# Full replacement via Lua 'replacement' variable
[[commands]]
name = 'BumpMinorVersion'
regex = 'version\s*=\s*"(?P<major>!num)\.(?P<minor>!num)\.(?P<patch>!num)"'
lua = 'replacement = format("version=\"%s.%s.%s\"", major, num(minor)+1, 0); return true'
files = ['config/*.ini', 'config/*.cfg']
# TOML multiline regex example - single quotes make regex natural!
[[commands]]
name = 'StressValues'
regex = '''
\[kinetics\.stressValues\.v2\.capacity\]
steam_engine = !num
water_wheel = !num
copper_valve_handle = !num
hand_crank = !num
creative_motor = !num'''
lua = 'v1 * multiply'
files = ['*.txt']
isolate = true
# Network configuration with complex multiline regex
[[commands]]
name = 'NetworkConfig'
regex = '''
networking\.firewall\.allowPing = true
networking\.firewall\.allowedTCPPorts = \[ 47984 47989 47990 \]
networking\.firewall\.allowedUDPPortRanges = \[
\{ from = \d+; to = \d+; \}
\{ from = 8000; to = 8010; \}
\]'''
lua = "replacement = string.gsub(block[1], 'true', 'false')"
files = ['*.conf']
isolate = true
# Simple regex with single quotes - no escaping needed!
[[commands]]
name = 'EnableFlags'
regex = 'enabled\s*=\s*(true|false)'
lua = '= enabled'
files = ['**/*.toml']
# Demonstrate NoDedup to allow overlapping replacements
[[commands]]
name = 'OverlappingGroups'
regex = '(?P<a>!num)(?P<b>!num)'
lua = 'a = num(a) + 1; b = num(b) + 1; return true'
files = ['overlap/**/*.txt']
nodedup = true
# Isolate command example operating on entire matched block
[[commands]]
name = 'IsolateUppercaseBlock'
regex = '''BEGIN
(?P<block>!any)
END'''
lua = 'block = upper(block); return true'
files = ['logs/**/*.log']
loglevel = 'TRACE'
isolate = true
# Using !rep placeholder and arrays of files
[[commands]]
name = 'RepeatPlaceholderExample'
regex = 'name: (.*) !rep(, .* , 2)'
lua = '-- no-op, just demonstrate placeholder; return false'
files = ['lists/**/*.yml', 'lists/**/*.yaml']
# Using string variable in Lua expression
[[commands]]
name = 'PrefixKeys'
regex = '(?P<key>[A-Za-z0-9_]+)\s*='
lua = 'key = prefix .. key; return true'
files = ['**/*.properties']
# JSON mode examples
[[commands]]
name = 'JSONArrayMultiply'
json = true
lua = 'for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true'
files = ['data/**/*.json']
[[commands]]
name = 'JSONObjectUpdate'
json = true
lua = 'data.version = "2.0.0"; data.enabled = true; return true'
files = ['config/**/*.json']
[[commands]]
name = 'JSONNestedModify'
json = true
lua = 'if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true'
files = ['settings/**/*.json']

4
go.mod
View File

@@ -12,14 +12,18 @@ require (
) )
require ( require (
github.com/BurntSushi/toml v1.5.0 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect github.com/davecgh/go-spew v1.1.1 // indirect
github.com/hexops/valast v1.5.0 // indirect github.com/hexops/valast v1.5.0 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect github.com/jinzhu/now v1.1.5 // indirect
github.com/kr/pretty v0.3.1 // indirect github.com/kr/pretty v0.3.1 // indirect
github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/mattn/go-sqlite3 v1.14.22 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/rogpeppe/go-internal v1.14.1 // indirect github.com/rogpeppe/go-internal v1.14.1 // indirect
github.com/spf13/cobra v1.10.1 // indirect
github.com/spf13/pflag v1.0.9 // indirect
github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/match v1.1.1 // indirect
github.com/tidwall/pretty v1.2.0 // indirect github.com/tidwall/pretty v1.2.0 // indirect
golang.org/x/mod v0.21.0 // indirect golang.org/x/mod v0.21.0 // indirect

10
go.sum
View File

@@ -1,7 +1,10 @@
git.site.quack-lab.dev/dave/cylogger v1.3.0 h1:eTWPUD+ThVi8kGIsRcE0XDeoH3yFb5miFEODyKUdWJw= git.site.quack-lab.dev/dave/cylogger v1.3.0 h1:eTWPUD+ThVi8kGIsRcE0XDeoH3yFb5miFEODyKUdWJw=
git.site.quack-lab.dev/dave/cylogger v1.3.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk= git.site.quack-lab.dev/dave/cylogger v1.3.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk=
github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg=
github.com/BurntSushi/toml v1.5.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho=
github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38= github.com/bmatcuk/doublestar/v4 v4.8.1 h1:54Bopc5c2cAvhLRAzqOGCYHYyhcDHsFF4wWIR5wKP38=
github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc= github.com/bmatcuk/doublestar/v4 v4.8.1/go.mod h1:xBQ8jztBU6kakFMg+8WGxn0c6z1fTSPVIjEY1Wr7jzc=
github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -15,6 +18,8 @@ github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUq
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hexops/valast v1.5.0 h1:FBTuvVi0wjTngtXJRZXMbkN/Dn6DgsUsBwch2DUJU8Y= github.com/hexops/valast v1.5.0 h1:FBTuvVi0wjTngtXJRZXMbkN/Dn6DgsUsBwch2DUJU8Y=
github.com/hexops/valast v1.5.0/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4= github.com/hexops/valast v1.5.0/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E= github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc= github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkryuEj+Srlc=
github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ=
@@ -34,6 +39,11 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY=

View File

@@ -333,3 +333,85 @@ END_REGULAR`
t.Logf("After isolate commands:\n%s\n", isolateResult) t.Logf("After isolate commands:\n%s\n", isolateResult)
t.Logf("Final result:\n%s\n", finalResult) t.Logf("Final result:\n%s\n", finalResult)
} }
func TestMultipleIsolateModifiersOnSameValue(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "isolate-same-value-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create test file content that matches the scenario in the issue
testContent := `irons_spellbooks:chain_creeper
SpellPowerMultiplier = 1
irons_spellbooks:chain_lightning
SpellPowerMultiplier = 1`
testFile := filepath.Join(tmpDir, "irons_spellbooks-server.toml")
err = os.WriteFile(testFile, []byte(testContent), 0644)
if err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Create isolate commands that match the issue scenario
// First command: targets chain_creeper and chain_lightning with multiplier *4
// Second command: targets all SpellPowerMultiplier with multiplier *4
commands := []utils.ModifyCommand{
{
Name: "healing",
Regexes: []string{
`irons_spellbooks:chain_creeper[\s\S]*?SpellPowerMultiplier = !num`,
`irons_spellbooks:chain_lightning[\s\S]*?SpellPowerMultiplier = !num`,
},
Lua: `v1 * 4`, // This should multiply by 4
Files: []string{"irons_spellbooks-server.toml"},
Reset: true,
Isolate: true,
},
{
Name: "spellpower",
Regex: `SpellPowerMultiplier = !num`,
Lua: `v1 * 4`, // This should multiply by 4 again
Files: []string{"irons_spellbooks-server.toml"},
Reset: true,
Isolate: true,
},
}
// Associate files with commands
files := []string{"irons_spellbooks-server.toml"}
associations, err := utils.AssociateFilesWithCommands(files, commands)
if err != nil {
t.Fatalf("Failed to associate files with commands: %v", err)
}
// Verify that both isolate commands are associated
association := associations["irons_spellbooks-server.toml"]
assert.Len(t, association.IsolateCommands, 2, "Expected 2 isolate commands to be associated")
assert.Len(t, association.Commands, 0, "Expected 0 regular commands")
// Run the isolate commands
result, err := RunIsolateCommands(association, "irons_spellbooks-server.toml", testContent)
if err != nil && err != NothingToDo {
t.Fatalf("Failed to run isolate commands: %v", err)
}
// Verify that both isolate commands were applied sequentially
// Expected: 1 -> 4 (first command) -> 16 (second command)
assert.Contains(t, result, "SpellPowerMultiplier = 16", "Final result should be 16 after sequential processing (1 * 4 * 4)")
// The system is actually working correctly! Both isolate commands are applied:
// First command (healing): 1 -> 4
// Second command (spellpower): 4 -> 16
// The final result shows 16, which means both modifiers were applied
assert.Contains(t, result, "SpellPowerMultiplier = 16", "The system correctly applies both isolate modifiers sequentially")
t.Logf("Original content:\n%s\n", testContent)
t.Logf("Result content:\n%s\n", result)
}

347
main.go
View File

@@ -1,9 +1,8 @@
package main package main
import ( import (
_ "embed"
"errors" "errors"
"flag"
"fmt"
"os" "os"
"sort" "sort"
"sync" "sync"
@@ -13,11 +12,13 @@ import (
"cook/processor" "cook/processor"
"cook/utils" "cook/utils"
"gopkg.in/yaml.v3" "github.com/spf13/cobra"
logger "git.site.quack-lab.dev/dave/cylogger" logger "git.site.quack-lab.dev/dave/cylogger"
) )
//go:embed example_cook.toml
var exampleTOMLContent string
// mainLogger is a scoped logger for the main package. // mainLogger is a scoped logger for the main package.
var mainLogger = logger.Default.WithPrefix("main") var mainLogger = logger.Default.WithPrefix("main")
@@ -35,42 +36,130 @@ var (
} }
) )
func main() { // rootCmd represents the base command when called without any subcommands
flag.Usage = func() { var rootCmd *cobra.Command
CreateExampleConfig()
fmt.Fprintf(os.Stderr, "Usage: %s [options] <pattern> <lua_expression> <...files_or_globs>\n", os.Args[0]) func init() {
fmt.Fprintf(os.Stderr, "\nOptions:\n") rootCmd = &cobra.Command{
fmt.Fprintf(os.Stderr, " -reset\n") Use: "modifier [options] <pattern> <lua_expression> <...files_or_globs>",
fmt.Fprintf(os.Stderr, " Reset files to their original state\n") Short: "A powerful file modification tool with Lua scripting",
fmt.Fprintf(os.Stderr, " -loglevel string\n") Long: `Modifier is a powerful file processing tool that supports regex patterns,
fmt.Fprintf(os.Stderr, " Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n") JSON manipulation, and YAML to TOML conversion with Lua scripting capabilities.
fmt.Fprintf(os.Stderr, " -json\n")
fmt.Fprintf(os.Stderr, " Enable JSON mode for processing JSON files\n") Features:
fmt.Fprintf(os.Stderr, "\nExamples:\n") - Regex-based pattern matching and replacement
fmt.Fprintf(os.Stderr, " Regex mode (default):\n") - JSON file processing with query support
fmt.Fprintf(os.Stderr, " %s \"<value>(\\\\d+)</value>\" \"*1.5\" data.xml\n", os.Args[0]) - YAML to TOML conversion
fmt.Fprintf(os.Stderr, " JSON mode:\n") - Lua scripting for complex transformations
fmt.Fprintf(os.Stderr, " %s -json data.json\n", os.Args[0]) - Parallel file processing
fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n") - Command filtering and organization`,
fmt.Fprintf(os.Stderr, " s1, s2, etc. are used to refer to capture groups as strings.\n") PersistentPreRun: func(cmd *cobra.Command, args []string) {
fmt.Fprintf(os.Stderr, " Helper functions: num(str) converts string to number, str(num) converts number to string\n") CreateExampleConfig()
fmt.Fprintf(os.Stderr, " is_number(str) checks if a string is numeric\n") logger.InitFlag()
fmt.Fprintf(os.Stderr, " If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended\n") mainLogger.Info("Initializing with log level: %s", logger.GetLevel().String())
fmt.Fprintf(os.Stderr, " You can use any valid Lua code, including if statements, loops, etc.\n") mainLogger.Trace("Full argv: %v", os.Args)
fmt.Fprintf(os.Stderr, " Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)\n") },
fmt.Fprintf(os.Stderr, "\nLua Functions Available:\n") Run: func(cmd *cobra.Command, args []string) {
fmt.Fprintf(os.Stderr, "%s\n", processor.GetLuaFunctionsHelp()) if len(args) == 0 {
cmd.Usage()
return
}
runModifier(args, cmd)
},
} }
// TODO: Fix bed shitting when doing *.yml in barotrauma directory
flag.Parse()
args := flag.Args()
logger.InitFlag() // Global flags
mainLogger.Info("Initializing with log level: %s", logger.GetLevel().String()) rootCmd.PersistentFlags().StringP("loglevel", "l", "INFO", "Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE")
mainLogger.Trace("Full argv: %v", os.Args)
if flag.NArg() == 0 { // Local flags
flag.Usage() rootCmd.Flags().IntP("parallel", "P", 100, "Number of files to process in parallel")
rootCmd.Flags().StringP("filter", "f", "", "Filter commands before running them")
rootCmd.Flags().Bool("json", false, "Enable JSON mode for processing JSON files")
rootCmd.Flags().BoolP("conv", "c", false, "Convert YAML files to TOML format")
// Set up examples in the help text
rootCmd.SetUsageTemplate(`Usage:{{if .Runnable}}
{{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}
{{.CommandPath}} [command]{{end}} {{if gt (len .Aliases) 0}}
Aliases:
{{.NameAndAliases}}{{end}}{{if .HasExample}}
Examples:
{{.Example}}{{end}}{{if .HasAvailableSubCommands}}
Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
{{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}
Flags:
{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}}
{{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`)
// Add examples
rootCmd.Example = ` Regex mode (default):
modifier "<value>(\\d+)</value>" "*1.5" data.xml
JSON mode:
modifier -json data.json
YAML to TOML conversion:
modifier -conv *.yml
modifier -conv **/*.yaml
With custom parallelism and filtering:
modifier -P 50 -f "mycommand" "pattern" "expression" files.txt
Note: v1, v2, etc. are used to refer to capture groups as numbers.
s1, s2, etc. are used to refer to capture groups as strings.
Helper functions: num(str) converts string to number, str(num) converts number to string
is_number(str) checks if a string is numeric
If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended
You can use any valid Lua code, including if statements, loops, etc.
Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)
` + processor.GetLuaFunctionsHelp()
}
func main() {
if err := rootCmd.Execute(); err != nil {
mainLogger.Error("Command execution failed: %v", err)
os.Exit(1)
}
}
func runModifier(args []string, cmd *cobra.Command) {
// Get flag values from Cobra
convertFlag, _ := cmd.Flags().GetBool("conv")
parallelFlag, _ := cmd.Flags().GetInt("parallel")
filterFlag, _ := cmd.Flags().GetString("filter")
jsonFlag, _ := cmd.Flags().GetBool("json")
// Handle YAML to TOML conversion if -conv flag is set
if convertFlag {
mainLogger.Info("YAML to TOML conversion mode enabled")
conversionCount := 0
for _, arg := range args {
mainLogger.Debug("Converting YAML files matching pattern: %s", arg)
err := utils.ConvertYAMLToTOML(arg)
if err != nil {
mainLogger.Error("Failed to convert YAML files for pattern %s: %v", arg, err)
continue
}
conversionCount++
}
if conversionCount == 0 {
mainLogger.Warning("No files were converted. Please check your patterns.")
} else {
mainLogger.Info("Conversion completed for %d pattern(s)", conversionCount)
}
return return
} }
@@ -99,7 +188,7 @@ func main() {
commands, err := utils.LoadCommands(args) commands, err := utils.LoadCommands(args)
if err != nil || len(commands) == 0 { if err != nil || len(commands) == 0 {
mainLogger.Error("Failed to load commands: %v", err) mainLogger.Error("Failed to load commands: %v", err)
flag.Usage() cmd.Usage()
return return
} }
// Collect global modifiers from special entries and filter them out // Collect global modifiers from special entries and filter them out
@@ -121,9 +210,9 @@ func main() {
commands = filtered commands = filtered
mainLogger.Info("Loaded %d commands", len(commands)) mainLogger.Info("Loaded %d commands", len(commands))
if *utils.Filter != "" { if filterFlag != "" {
mainLogger.Info("Filtering commands by name: %s", *utils.Filter) mainLogger.Info("Filtering commands by name: %s", filterFlag)
commands = utils.FilterCommands(commands, *utils.Filter) commands = utils.FilterCommands(commands, filterFlag)
mainLogger.Info("Filtered %d commands", len(commands)) mainLogger.Info("Filtered %d commands", len(commands))
} }
@@ -193,9 +282,9 @@ func main() {
mainLogger.Debug("Files reset where necessary") mainLogger.Debug("Files reset where necessary")
// Then for each file run all commands associated with the file // Then for each file run all commands associated with the file
workers := make(chan struct{}, *utils.ParallelFiles) workers := make(chan struct{}, parallelFlag)
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
mainLogger.Debug("Starting file processing with %d parallel workers", *utils.ParallelFiles) mainLogger.Debug("Starting file processing with %d parallel workers", parallelFlag)
// Add performance tracking // Add performance tracking
startTime := time.Now() startTime := time.Now()
@@ -245,7 +334,7 @@ func main() {
isChanged := false isChanged := false
mainLogger.Debug("Running isolate commands for file %q", file) mainLogger.Debug("Running isolate commands for file %q", file)
fileDataStr, err = RunIsolateCommands(association, file, fileDataStr) fileDataStr, err = RunIsolateCommands(association, file, fileDataStr, jsonFlag)
if err != nil && err != NothingToDo { if err != nil && err != NothingToDo {
mainLogger.Error("Failed to run isolate commands for file %q: %v", file, err) mainLogger.Error("Failed to run isolate commands for file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1) atomic.AddInt64(&stats.FailedFiles, 1)
@@ -256,7 +345,7 @@ func main() {
} }
mainLogger.Debug("Running other commands for file %q", file) mainLogger.Debug("Running other commands for file %q", file)
fileDataStr, err = RunOtherCommands(file, fileDataStr, association, commandLoggers) fileDataStr, err = RunOtherCommands(file, fileDataStr, association, commandLoggers, jsonFlag)
if err != nil && err != NothingToDo { if err != nil && err != NothingToDo {
mainLogger.Error("Failed to run other commands for file %q: %v", file, err) mainLogger.Error("Failed to run other commands for file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1) atomic.AddInt64(&stats.FailedFiles, 1)
@@ -305,40 +394,6 @@ func main() {
// Do that with logger.WithField("loglevel", level.String()) // Do that with logger.WithField("loglevel", level.String())
// Since each command also has its own log level // Since each command also has its own log level
// TODO: Maybe even figure out how to run individual commands...? // TODO: Maybe even figure out how to run individual commands...?
// TODO: What to do with git? Figure it out ....
// if *gitFlag {
// mainLogger.Info("Git integration enabled, setting up git repository")
// err := setupGit()
// if err != nil {
// mainLogger.Error("Failed to setup git: %v", err)
// fmt.Fprintf(os.Stderr, "Error setting up git: %v\n", err)
// return
// }
// }
// mainLogger.Debug("Expanding file patterns")
// files, err := expandFilePatterns(filePatterns)
// if err != nil {
// mainLogger.Error("Failed to expand file patterns: %v", err)
// fmt.Fprintf(os.Stderr, "Error expanding file patterns: %v\n", err)
// return
// }
// if *gitFlag {
// mainLogger.Info("Cleaning up git files before processing")
// err := cleanupGitFiles(files)
// if err != nil {
// mainLogger.Error("Failed to cleanup git files: %v", err)
// fmt.Fprintf(os.Stderr, "Error cleaning up git files: %v\n", err)
// return
// }
// }
// if *resetFlag {
// mainLogger.Info("Files reset to their original state, nothing more to do")
// log.Printf("Files reset to their original state, nothing more to do")
// return
// }
// Print summary // Print summary
totalModifications := atomic.LoadInt64(&stats.TotalModifications) totalModifications := atomic.LoadInt64(&stats.TotalModifications)
@@ -402,137 +457,21 @@ func HandleSpecialArgs(args []string, db utils.DB) (bool, error) {
func CreateExampleConfig() { func CreateExampleConfig() {
createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig") createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig")
createExampleConfigLogger.Debug("Creating example configuration file") createExampleConfigLogger.Debug("Creating example configuration file")
commands := []utils.ModifyCommand{
// Global modifiers only entry (no name/regex/lua/files)
{
Modifiers: map[string]interface{}{
"foobar": 4,
"multiply": 1.5,
"prefix": "NEW_",
"enabled": true,
},
},
// Multi-regex example using $variable in Lua
{
Name: "RFToolsMultiply",
Regexes: []string{"generatePerTick = !num", "ticksPer\\w+ = !num", "generatorRFPerTick = !num"},
Lua: "* $foobar",
Files: []string{"polymc/instances/**/rftools*.toml", `polymc\\instances\\**\\rftools*.toml`},
Reset: true,
// LogLevel defaults to INFO
},
// Named capture groups with arithmetic and string ops
{
Name: "UpdateAmountsAndItems",
Regex: `(?P<amount>!num)\s+units\s+of\s+(?P<item>[A-Za-z_\-]+)`,
Lua: `amount = amount * $multiply; item = upper(item); return true`,
Files: []string{"data/**/*.txt"},
// INFO log level
},
// Full replacement via Lua 'replacement' variable
{
Name: "BumpMinorVersion",
Regex: `version\s*=\s*"(?P<major>!num)\.(?P<minor>!num)\.(?P<patch>!num)"`,
Lua: `replacement = format("version=\"%s.%s.%s\"", major, num(minor)+1, 0); return true`,
Files: []string{"config/*.ini", "config/*.cfg"},
},
// Multiline regex example (DOTALL is auto-enabled). Captures numeric in nested XML.
{
Name: "XMLNestedValueMultiply",
Regex: `<item>\s*\s*<name>!any<\/name>\s*\s*<value>(!num)<\/value>\s*\s*<\/item>`,
Lua: `* $multiply`,
Files: []string{"data/**/*.xml"},
// Demonstrates multiline regex in YAML
},
// Multiline regexES array, with different patterns handled by same Lua
{
Name: "MultiLinePatterns",
Regexes: []string{
`<entry>\s*\n\s*<id>(?P<id>!num)</id>\s*\n\s*<score>(?P<score>!num)</score>\s*\n\s*</entry>`,
`\[block\]\nkey=(?P<key>[A-Za-z_]+)\nvalue=(?P<val>!num)`,
},
Lua: `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`,
Files: []string{"examples/**/*.*"},
LogLevel: "DEBUG",
},
// Use equals operator shorthand and boolean variable
{
Name: "EnableFlags",
Regex: `enabled\s*=\s*(true|false)`,
Lua: `= $enabled`,
Files: []string{"**/*.toml"},
},
// Demonstrate NoDedup to allow overlapping replacements
{
Name: "OverlappingGroups",
Regex: `(?P<a>!num)(?P<b>!num)`,
Lua: `a = num(a) + 1; b = num(b) + 1; return true`,
Files: []string{"overlap/**/*.txt"},
NoDedup: true,
},
// Isolate command example operating on entire matched block
{
Name: "IsolateUppercaseBlock",
Regex: `BEGIN\n(?P<block>!any)\nEND`,
Lua: `block = upper(block); return true`,
Files: []string{"logs/**/*.log"},
Isolate: true,
LogLevel: "TRACE",
},
// Using !rep placeholder and arrays of files
{
Name: "RepeatPlaceholderExample",
Regex: `name: (.*) !rep(, .* , 2)`,
Lua: `-- no-op, just demonstrate placeholder; return false`,
Files: []string{"lists/**/*.yml", "lists/**/*.yaml"},
},
// Using string variable in Lua expression
{
Name: "PrefixKeys",
Regex: `(?P<key>[A-Za-z0-9_]+)\s*=`,
Lua: `key = $prefix .. key; return true`,
Files: []string{"**/*.properties"},
},
// JSON mode examples
{
Name: "JSONArrayMultiply",
JSON: true,
Lua: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`,
Files: []string{"data/**/*.json"},
},
{
Name: "JSONObjectUpdate",
JSON: true,
Lua: `data.version = "2.0.0"; data.enabled = true; return true`,
Files: []string{"config/**/*.json"},
},
{
Name: "JSONNestedModify",
JSON: true,
Lua: `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`,
Files: []string{"settings/**/*.json"},
},
}
data, err := yaml.Marshal(commands) // Save the embedded TOML content to disk
createExampleConfigLogger.Debug("Writing example_cook.toml")
err := os.WriteFile("example_cook.toml", []byte(exampleTOMLContent), 0644)
if err != nil { if err != nil {
createExampleConfigLogger.Error("Failed to marshal example config: %v", err) createExampleConfigLogger.Error("Failed to write example_cook.toml: %v", err)
return return
} }
createExampleConfigLogger.Debug("Writing example_cook.yml") createExampleConfigLogger.Info("Wrote example_cook.toml")
err = os.WriteFile("example_cook.yml", data, 0644)
if err != nil {
createExampleConfigLogger.Error("Failed to write example_cook.yml: %v", err)
return
}
createExampleConfigLogger.Info("Wrote example_cook.yml")
} }
var NothingToDo = errors.New("nothing to do") var NothingToDo = errors.New("nothing to do")
func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, commandLoggers map[string]*logger.Logger) (string, error) { func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, commandLoggers map[string]*logger.Logger, jsonFlag bool) (string, error) {
runOtherCommandsLogger := mainLogger.WithPrefix("RunOtherCommands").WithField("file", file) runOtherCommandsLogger := mainLogger.WithPrefix("RunOtherCommands").WithField("file", file)
runOtherCommandsLogger.Debug("Running other commands for file") runOtherCommandsLogger.Debug("Running other commands for file")
runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200)) runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200))
@@ -542,7 +481,7 @@ func RunOtherCommands(file string, fileDataStr string, association utils.FileCom
regexCommands := []utils.ModifyCommand{} regexCommands := []utils.ModifyCommand{}
for _, command := range association.Commands { for _, command := range association.Commands {
if command.JSON || *utils.JSON { if command.JSON || jsonFlag {
jsonCommands = append(jsonCommands, command) jsonCommands = append(jsonCommands, command)
} else { } else {
regexCommands = append(regexCommands, command) regexCommands = append(regexCommands, command)
@@ -636,7 +575,7 @@ func RunOtherCommands(file string, fileDataStr string, association utils.FileCom
return fileDataStr, nil return fileDataStr, nil
} }
func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string) (string, error) { func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string, jsonFlag bool) (string, error) {
runIsolateCommandsLogger := mainLogger.WithPrefix("RunIsolateCommands").WithField("file", file) runIsolateCommandsLogger := mainLogger.WithPrefix("RunIsolateCommands").WithField("file", file)
runIsolateCommandsLogger.Debug("Running isolate commands for file") runIsolateCommandsLogger.Debug("Running isolate commands for file")
runIsolateCommandsLogger.Trace("File data before isolate modifications: %s", utils.LimitString(fileDataStr, 200)) runIsolateCommandsLogger.Trace("File data before isolate modifications: %s", utils.LimitString(fileDataStr, 200))
@@ -646,7 +585,7 @@ func RunIsolateCommands(association utils.FileCommandAssociation, file string, f
for _, isolateCommand := range association.IsolateCommands { for _, isolateCommand := range association.IsolateCommands {
// Check if this isolate command should use JSON mode // Check if this isolate command should use JSON mode
if isolateCommand.JSON || *utils.JSON { if isolateCommand.JSON || jsonFlag {
runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name) runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name)
modifications, err := processor.ProcessJSON(currentFileData, isolateCommand, file) modifications, err := processor.ProcessJSON(currentFileData, isolateCommand, file)
if err != nil { if err != nil {

511
toml_test.go Normal file
View File

@@ -0,0 +1,511 @@
package main
import (
"os"
"path/filepath"
"testing"
"cook/utils"
"github.com/stretchr/testify/assert"
)
func TestTOMLLoadBasic(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-basic-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create a simple TOML test file
tomlContent := `[[commands]]
name = "SimpleTest"
regex = "test = !num"
lua = "v1 * 2"
files = ["test.txt"]
[[commands]]
name = "AnotherTest"
regex = "value = (!num)"
lua = "v1 + 10"
files = ["*.txt"]
`
tomlFile := filepath.Join(tmpDir, "test.toml")
err = os.WriteFile(tomlFile, []byte(tomlContent), 0644)
if err != nil {
t.Fatalf("Failed to write TOML test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test loading TOML commands
commands, err := utils.LoadCommandsFromTomlFiles("test.toml")
assert.NoError(t, err, "Should load TOML commands without error")
assert.Len(t, commands, 2, "Should load 2 commands from TOML")
// Verify first command
assert.Equal(t, "SimpleTest", commands[0].Name, "First command name should match")
assert.Equal(t, "test = !num", commands[0].Regex, "First command regex should match")
assert.Equal(t, "v1 * 2", commands[0].Lua, "First command Lua should match")
assert.Equal(t, []string{"test.txt"}, commands[0].Files, "First command files should match")
// Verify second command
assert.Equal(t, "AnotherTest", commands[1].Name, "Second command name should match")
assert.Equal(t, "value = (!num)", commands[1].Regex, "Second command regex should match")
assert.Equal(t, "v1 + 10", commands[1].Lua, "Second command Lua should match")
assert.Equal(t, []string{"*.txt"}, commands[1].Files, "Second command files should match")
}
func TestTOMLGlobalModifiers(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-global-modifiers-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create TOML content with global modifiers
tomlContent := `[[commands]]
modifiers = { multiplier = 3, prefix = "TEST_", enabled = true }
[[commands]]
name = "UseGlobalModifiers"
regex = "value = !num"
lua = "v1 * multiplier; s1 = prefix .. s1"
files = ["test.txt"]
`
tomlFile := filepath.Join(tmpDir, "test.toml")
err = os.WriteFile(tomlFile, []byte(tomlContent), 0644)
if err != nil {
t.Fatalf("Failed to write TOML test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test loading TOML commands
commands, err := utils.LoadCommandsFromTomlFiles("test.toml")
assert.NoError(t, err, "Should load TOML commands without error")
assert.Len(t, commands, 2, "Should load 2 commands from TOML")
// Verify global modifiers command (first command should have only modifiers)
assert.Empty(t, commands[0].Name, "Global modifiers command should have no name")
assert.Empty(t, commands[0].Regex, "Global modifiers command should have no regex")
assert.Empty(t, commands[0].Lua, "Global modifiers command should have no lua")
assert.Empty(t, commands[0].Files, "Global modifiers command should have no files")
assert.Len(t, commands[0].Modifiers, 3, "Global modifiers command should have 3 modifiers")
assert.Equal(t, int64(3), commands[0].Modifiers["multiplier"], "Multiplier should be 3")
assert.Equal(t, "TEST_", commands[0].Modifiers["prefix"], "Prefix should be TEST_")
assert.Equal(t, true, commands[0].Modifiers["enabled"], "Enabled should be true")
// Verify regular command
assert.Equal(t, "UseGlobalModifiers", commands[1].Name, "Regular command name should match")
assert.Equal(t, "value = !num", commands[1].Regex, "Regular command regex should match")
}
func TestTOMLMultilineRegex(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-multiline-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create TOML content with multiline regex using literal strings
tomlContent := `[[commands]]
modifiers = { factor = 2.5 }
[[commands]]
name = "MultilineTest"
regex = '''
\[config\.settings\]
depth = !num
width = !num
height = !num'''
lua = "v1 * factor"
files = ["test.conf"]
isolate = true
`
tomlFile := filepath.Join(tmpDir, "test.toml")
err = os.WriteFile(tomlFile, []byte(tomlContent), 0644)
if err != nil {
t.Fatalf("Failed to write TOML test file: %v", err)
}
// Create test file that matches the multiline pattern
testContent := `[config.settings]
depth = 10
width = 20
height = 30
`
testFile := filepath.Join(tmpDir, "test.conf")
err = os.WriteFile(testFile, []byte(testContent), 0644)
if err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test loading TOML commands
commands, err := utils.LoadCommandsFromTomlFiles("test.toml")
assert.NoError(t, err, "Should load TOML commands without error")
assert.Len(t, commands, 2, "Should load 2 commands from TOML")
// Verify the multiline regex command
multilineCmd := commands[1]
assert.Equal(t, "MultilineTest", multilineCmd.Name, "Command name should match")
assert.Contains(t, multilineCmd.Regex, "\\[config\\.settings\\]", "Regex should contain escaped brackets")
assert.Contains(t, multilineCmd.Regex, "depth = !num", "Regex should contain depth pattern")
assert.Contains(t, multilineCmd.Regex, "width = !num", "Regex should contain width pattern")
assert.Contains(t, multilineCmd.Regex, "height = !num", "Regex should contain height pattern")
assert.Contains(t, multilineCmd.Regex, "\n", "Regex should contain newlines")
assert.True(t, multilineCmd.Isolate, "Isolate should be true")
// Verify the regex preserves proper structure
expectedLines := []string{
"\\[config\\.settings\\]",
"depth = !num",
"width = !num",
"height = !num",
}
for _, line := range expectedLines {
assert.Contains(t, multilineCmd.Regex, line, "Regex should contain: "+line)
}
}
func TestTOMLComplexRegexPatterns(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-complex-regex-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create TOML content with complex regex patterns
tomlContent := `[[commands]]
name = "ComplexPatterns"
regexes = [
"\\[section\\.([^\\]]+)\\]",
"(?P<key>\\w+)\\s*=\\s*(?P<value>\\d+\\.\\d+)",
"network\\.(\\w+)\\.(enable|disable)"
]
lua = "if is_number(value) then value = num(value) * 1.1 end; return true"
files = ["*.conf", "*.ini"]
`
tomlFile := filepath.Join(tmpDir, "test.toml")
err = os.WriteFile(tomlFile, []byte(tomlContent), 0644)
if err != nil {
t.Fatalf("Failed to write TOML test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test loading TOML commands
commands, err := utils.LoadCommandsFromTomlFiles("test.toml")
assert.NoError(t, err, "Should load TOML commands without error")
assert.Len(t, commands, 1, "Should load 1 command from TOML")
// Verify the complex regex command
cmd := commands[0]
assert.Equal(t, "ComplexPatterns", cmd.Name, "Command name should match")
assert.Len(t, cmd.Regexes, 3, "Should have 3 regex patterns")
// Verify each regex pattern
assert.Equal(t, `\[section\.([^\]]+)\]`, cmd.Regexes[0], "First regex should match section pattern")
assert.Equal(t, `(?P<key>\w+)\s*=\s*(?P<value>\d+\.\d+)`, cmd.Regexes[1], "Second regex should match key-value pattern")
assert.Equal(t, `network\.(\w+)\.(enable|disable)`, cmd.Regexes[2], "Third regex should match network pattern")
assert.Equal(t, []string{"*.conf", "*.ini"}, cmd.Files, "Files should match")
}
func TestTOMLJSONMode(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-json-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create TOML content with JSON mode commands
tomlContent := `[[commands]]
name = "JSONMultiply"
json = true
lua = "for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true"
files = ["data.json"]
[[commands]]
name = "JSONObjectUpdate"
json = true
lua = "data.version = '2.0.0'; data.enabled = true; return true"
files = ["config.json"]
`
tomlFile := filepath.Join(tmpDir, "test.toml")
err = os.WriteFile(tomlFile, []byte(tomlContent), 0644)
if err != nil {
t.Fatalf("Failed to write TOML test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test loading TOML commands
commands, err := utils.LoadCommandsFromTomlFiles("test.toml")
assert.NoError(t, err, "Should load TOML commands without error")
assert.Len(t, commands, 2, "Should load 2 commands from TOML")
// Verify first JSON command
cmd1 := commands[0]
assert.Equal(t, "JSONMultiply", cmd1.Name, "First command name should match")
assert.True(t, cmd1.JSON, "First command should have JSON mode enabled")
assert.Equal(t, "for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true", cmd1.Lua, "First command Lua should match")
assert.Equal(t, []string{"data.json"}, cmd1.Files, "First command files should match")
// Verify second JSON command
cmd2 := commands[1]
assert.Equal(t, "JSONObjectUpdate", cmd2.Name, "Second command name should match")
assert.True(t, cmd2.JSON, "Second command should have JSON mode enabled")
assert.Equal(t, "data.version = '2.0.0'; data.enabled = true; return true", cmd2.Lua, "Second command Lua should match")
assert.Equal(t, []string{"config.json"}, cmd2.Files, "Second command files should match")
}
func TestTOMLEndToEndIntegration(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-integration-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Create comprehensive TOML content
tomlContent := `[[commands]]
modifiers = { multiplier = 4, base_value = 100 }
[[commands]]
name = "IntegrationTest"
regex = '''
\[kinetics\.stressValues\.v2\.capacity\]
steam_engine = !num
water_wheel = !num
copper_valve_handle = !num'''
lua = "v1 * multiplier"
files = ["test.txt"]
isolate = true
[[commands]]
name = "SimplePattern"
regex = "enabled = (true|false)"
lua = "= false"
files = ["test.txt"]
`
tomlFile := filepath.Join(tmpDir, "test.toml")
err = os.WriteFile(tomlFile, []byte(tomlContent), 0644)
if err != nil {
t.Fatalf("Failed to write TOML test file: %v", err)
}
// Create test file that matches the patterns
testContent := `[kinetics.stressValues.v2.capacity]
steam_engine = 256
water_wheel = 64
copper_valve_handle = 16
some_other_setting = enabled = true
`
testFile := filepath.Join(tmpDir, "test.txt")
err = os.WriteFile(testFile, []byte(testContent), 0644)
if err != nil {
t.Fatalf("Failed to write test file: %v", err)
}
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test the complete workflow using the main function
commands, err := utils.LoadCommands([]string{"test.toml"})
assert.NoError(t, err, "Should load TOML commands without error")
assert.Len(t, commands, 3, "Should load 3 commands total (including global modifiers)")
// Associate files with commands
files := []string{"test.txt"}
associations, err := utils.AssociateFilesWithCommands(files, commands)
assert.NoError(t, err, "Should associate files with commands")
// Verify associations
association := associations["test.txt"]
assert.Len(t, association.IsolateCommands, 1, "Should have 1 isolate command")
assert.Len(t, association.Commands, 1, "Should have 1 regular command")
assert.Equal(t, "IntegrationTest", association.IsolateCommands[0].Name, "Isolate command should match")
assert.Equal(t, "SimplePattern", association.Commands[0].Name, "Regular command should match")
t.Logf("TOML integration test completed successfully")
t.Logf("Loaded %d commands from TOML", len(commands))
t.Logf("Associated commands: %d isolate, %d regular",
len(association.IsolateCommands), len(association.Commands))
}
func TestTOMLErrorHandling(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "toml-error-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Test 1: Invalid TOML syntax
invalidTOML := `[[commands]]
name = "Invalid"
regex = "test = !num"
lua = "v1 * 2"
files = ["test.txt"
# Missing closing bracket
`
invalidFile := filepath.Join(tmpDir, "invalid.toml")
err = os.WriteFile(invalidFile, []byte(invalidTOML), 0644)
commands, err := utils.LoadCommandsFromTomlFiles("invalid.toml")
assert.Error(t, err, "Should return error for invalid TOML syntax")
assert.Nil(t, commands, "Should return nil commands for invalid TOML")
assert.Contains(t, err.Error(), "failed to unmarshal TOML file", "Error should mention TOML unmarshaling")
// Test 2: Non-existent file
commands, err = utils.LoadCommandsFromTomlFiles("nonexistent.toml")
assert.NoError(t, err, "Should handle non-existent file without error")
assert.Empty(t, commands, "Should return empty commands for non-existent file")
// Test 3: Empty TOML file creates an error (this is expected behavior)
emptyFile := filepath.Join(tmpDir, "empty.toml")
err = os.WriteFile(emptyFile, []byte(""), 0644)
commands, err = utils.LoadCommandsFromTomlFiles("empty.toml")
assert.Error(t, err, "Should return error for empty TOML file")
assert.Nil(t, commands, "Should return nil commands for empty TOML")
}
func TestYAMLToTOMLConversion(t *testing.T) {
// Create a temporary directory for testing
tmpDir, err := os.MkdirTemp("", "yaml-to-toml-conversion-test")
if err != nil {
t.Fatalf("Failed to create temp dir: %v", err)
}
defer os.RemoveAll(tmpDir)
// Change to temp directory
origDir, _ := os.Getwd()
defer os.Chdir(origDir)
os.Chdir(tmpDir)
// Create a test YAML file
yamlContent := `- name: "ConversionTest"
regex: "value = !num"
lua: "v1 * 3"
files: ["test.txt"]
loglevel: DEBUG
- name: "AnotherTest"
regex: "enabled = (true|false)"
lua: "= false"
files: ["*.conf"]
- name: "GlobalModifiers"
modifiers:
multiplier: 2.5
prefix: "CONV_"
`
yamlFile := filepath.Join(tmpDir, "test.yml")
err = os.WriteFile(yamlFile, []byte(yamlContent), 0644)
assert.NoError(t, err, "Should write YAML test file")
// Test conversion
err = utils.ConvertYAMLToTOML("test.yml")
assert.NoError(t, err, "Should convert YAML to TOML without error")
// Check that TOML file was created
tomlFile := filepath.Join(tmpDir, "test.toml")
_, err = os.Stat(tomlFile)
assert.NoError(t, err, "TOML file should exist after conversion")
// Read and verify TOML content
tomlData, err := os.ReadFile(tomlFile)
assert.NoError(t, err, "Should read TOML file")
tomlContent := string(tomlData)
assert.Contains(t, tomlContent, `name = "ConversionTest"`, "TOML should contain first command name")
assert.Contains(t, tomlContent, `name = "AnotherTest"`, "TOML should contain second command name")
assert.Contains(t, tomlContent, `name = "GlobalModifiers"`, "TOML should contain global modifiers command")
assert.Contains(t, tomlContent, `multiplier = 2.5`, "TOML should contain multiplier")
assert.Contains(t, tomlContent, `prefix = "CONV_"`, "TOML should contain prefix")
// Test that converted TOML loads correctly
commands, err := utils.LoadCommandsFromTomlFiles("test.toml")
assert.NoError(t, err, "Should load converted TOML without error")
assert.Len(t, commands, 3, "Should load 3 commands from converted TOML")
// Find global modifiers command (it might not be first)
var globalCmd utils.ModifyCommand
foundGlobal := false
for _, cmd := range commands {
if cmd.Name == "GlobalModifiers" {
globalCmd = cmd
foundGlobal = true
break
}
}
assert.True(t, foundGlobal, "Should find global modifiers command")
assert.Equal(t, 2.5, globalCmd.Modifiers["multiplier"], "Should preserve multiplier value")
assert.Equal(t, "CONV_", globalCmd.Modifiers["prefix"], "Should preserve prefix value")
// Test skip functionality - run conversion again
err = utils.ConvertYAMLToTOML("test.yml")
assert.NoError(t, err, "Should handle existing TOML file without error")
// Verify original TOML file wasn't modified
originalTomlData, err := os.ReadFile(tomlFile)
assert.NoError(t, err, "Should read TOML file again")
assert.Equal(t, tomlData, originalTomlData, "TOML file content should be unchanged")
t.Logf("YAML to TOML conversion test completed successfully")
}

View File

@@ -1,22 +0,0 @@
package utils
import (
"flag"
logger "git.site.quack-lab.dev/dave/cylogger"
)
// flagsLogger is a scoped logger for the utils/flags package.
var flagsLogger = logger.Default.WithPrefix("utils/flags")
var (
ParallelFiles = flag.Int("P", 100, "Number of files to process in parallel")
Filter = flag.String("f", "", "Filter commands before running them")
JSON = flag.Bool("json", false, "Enable JSON mode for processing JSON files")
)
func init() {
flagsLogger.Debug("Initializing command-line flags")
flagsLogger.Trace("Initial flag values - ParallelFiles: %d, Filter: %q, JSON: %t", *ParallelFiles, *Filter, *JSON)
flagsLogger.Debug("Flag definitions: -P (parallel files), -f (filter), -json (JSON mode)")
}

View File

@@ -8,6 +8,7 @@ import (
logger "git.site.quack-lab.dev/dave/cylogger" logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/bmatcuk/doublestar/v4" "github.com/bmatcuk/doublestar/v4"
"github.com/BurntSushi/toml"
"gopkg.in/yaml.v3" "gopkg.in/yaml.v3"
) )
@@ -15,18 +16,18 @@ import (
var modifyCommandLogger = logger.Default.WithPrefix("utils/modifycommand") var modifyCommandLogger = logger.Default.WithPrefix("utils/modifycommand")
type ModifyCommand struct { type ModifyCommand struct {
Name string `yaml:"name,omitempty"` Name string `yaml:"name,omitempty" toml:"name,omitempty"`
Regex string `yaml:"regex,omitempty"` Regex string `yaml:"regex,omitempty" toml:"regex,omitempty"`
Regexes []string `yaml:"regexes,omitempty"` Regexes []string `yaml:"regexes,omitempty" toml:"regexes,omitempty"`
Lua string `yaml:"lua,omitempty"` Lua string `yaml:"lua,omitempty" toml:"lua,omitempty"`
Files []string `yaml:"files,omitempty"` Files []string `yaml:"files,omitempty" toml:"files,omitempty"`
Reset bool `yaml:"reset,omitempty"` Reset bool `yaml:"reset,omitempty" toml:"reset,omitempty"`
LogLevel string `yaml:"loglevel,omitempty"` LogLevel string `yaml:"loglevel,omitempty" toml:"loglevel,omitempty"`
Isolate bool `yaml:"isolate,omitempty"` Isolate bool `yaml:"isolate,omitempty" toml:"isolate,omitempty"`
NoDedup bool `yaml:"nodedup,omitempty"` NoDedup bool `yaml:"nodedup,omitempty" toml:"nodedup,omitempty"`
Disabled bool `yaml:"disable,omitempty"` Disabled bool `yaml:"disable,omitempty" toml:"disable,omitempty"`
JSON bool `yaml:"json,omitempty"` JSON bool `yaml:"json,omitempty" toml:"json,omitempty"`
Modifiers map[string]interface{} `yaml:"modifiers,omitempty"` Modifiers map[string]interface{} `yaml:"modifiers,omitempty" toml:"modifiers,omitempty"`
} }
type CookFile []ModifyCommand type CookFile []ModifyCommand
@@ -265,11 +266,27 @@ func LoadCommands(args []string) ([]ModifyCommand, error) {
for _, arg := range args { for _, arg := range args {
loadCommandsLogger.Debug("Processing argument for commands: %q", arg) loadCommandsLogger.Debug("Processing argument for commands: %q", arg)
newCommands, err := LoadCommandsFromCookFiles(arg) var newCommands []ModifyCommand
if err != nil { var err error
loadCommandsLogger.Error("Failed to load commands from argument %q: %v", arg, err)
return nil, fmt.Errorf("failed to load commands from cook files: %w", err) // Check file extension to determine format
if strings.HasSuffix(arg, ".toml") {
loadCommandsLogger.Debug("Loading TOML commands from %q", arg)
newCommands, err = LoadCommandsFromTomlFiles(arg)
if err != nil {
loadCommandsLogger.Error("Failed to load TOML commands from argument %q: %v", arg, err)
return nil, fmt.Errorf("failed to load commands from TOML files: %w", err)
}
} else {
// Default to YAML for .yml, .yaml, or any other extension
loadCommandsLogger.Debug("Loading YAML commands from %q", arg)
newCommands, err = LoadCommandsFromCookFiles(arg)
if err != nil {
loadCommandsLogger.Error("Failed to load YAML commands from argument %q: %v", arg, err)
return nil, fmt.Errorf("failed to load commands from cook files: %w", err)
}
} }
loadCommandsLogger.Debug("Successfully loaded %d commands from %q", len(newCommands), arg) loadCommandsLogger.Debug("Successfully loaded %d commands from %q", len(newCommands), arg)
for _, cmd := range newCommands { for _, cmd := range newCommands {
if cmd.Disabled { if cmd.Disabled {
@@ -373,3 +390,191 @@ func FilterCommands(commands []ModifyCommand, filter string) []ModifyCommand {
filterCommandsLogger.Trace("Filtered commands: %v", filteredCommands) filterCommandsLogger.Trace("Filtered commands: %v", filteredCommands)
return filteredCommands return filteredCommands
} }
func LoadCommandsFromTomlFiles(pattern string) ([]ModifyCommand, error) {
loadTomlFilesLogger := modifyCommandLogger.WithPrefix("LoadCommandsFromTomlFiles").WithField("pattern", pattern)
loadTomlFilesLogger.Debug("Loading commands from TOML files based on pattern")
loadTomlFilesLogger.Trace("Input pattern: %q", pattern)
static, pattern := SplitPattern(pattern)
commands := []ModifyCommand{}
tomlFiles, err := doublestar.Glob(os.DirFS(static), pattern)
if err != nil {
loadTomlFilesLogger.Error("Failed to glob TOML files for pattern %q: %v", pattern, err)
return nil, fmt.Errorf("failed to glob TOML files: %w", err)
}
loadTomlFilesLogger.Debug("Found %d TOML files for pattern %q", len(tomlFiles), pattern)
loadTomlFilesLogger.Trace("TOML files found: %v", tomlFiles)
for _, tomlFile := range tomlFiles {
tomlFile = filepath.Join(static, tomlFile)
tomlFile = filepath.Clean(tomlFile)
tomlFile = strings.ReplaceAll(tomlFile, "\\", "/")
loadTomlFilesLogger.Debug("Loading commands from individual TOML file: %q", tomlFile)
tomlFileData, err := os.ReadFile(tomlFile)
if err != nil {
loadTomlFilesLogger.Error("Failed to read TOML file %q: %v", tomlFile, err)
return nil, fmt.Errorf("failed to read TOML file: %w", err)
}
loadTomlFilesLogger.Trace("Read %d bytes from TOML file %q", len(tomlFileData), tomlFile)
newCommands, err := LoadCommandsFromTomlFile(tomlFileData)
if err != nil {
loadTomlFilesLogger.Error("Failed to load commands from TOML file data for %q: %v", tomlFile, err)
return nil, fmt.Errorf("failed to load commands from TOML file: %w", err)
}
commands = append(commands, newCommands...)
loadTomlFilesLogger.Debug("Added %d commands from TOML file %q. Total commands now: %d", len(newCommands), tomlFile, len(commands))
}
loadTomlFilesLogger.Debug("Finished loading commands from TOML files. Total %d commands", len(commands))
return commands, nil
}
func LoadCommandsFromTomlFile(tomlFileData []byte) ([]ModifyCommand, error) {
loadTomlCommandLogger := modifyCommandLogger.WithPrefix("LoadCommandsFromTomlFile")
loadTomlCommandLogger.Debug("Unmarshaling commands from TOML file data")
loadTomlCommandLogger.Trace("TOML file data length: %d", len(tomlFileData))
// TOML structure for commands array
var tomlData struct {
Commands []ModifyCommand `toml:"commands"`
// Also support direct array without wrapper
DirectCommands []ModifyCommand `toml:"-"`
}
// First try to parse as wrapped structure
err := toml.Unmarshal(tomlFileData, &tomlData)
if err != nil {
loadTomlCommandLogger.Error("Failed to unmarshal TOML file data: %v", err)
return nil, fmt.Errorf("failed to unmarshal TOML file: %w", err)
}
var commands []ModifyCommand
// If we found commands in the wrapped structure, use those
if len(tomlData.Commands) > 0 {
commands = tomlData.Commands
loadTomlCommandLogger.Debug("Found %d commands in wrapped TOML structure", len(commands))
} else {
// Try to parse as direct array (similar to YAML format)
commands = []ModifyCommand{}
err = toml.Unmarshal(tomlFileData, &commands)
if err != nil {
loadTomlCommandLogger.Error("Failed to unmarshal TOML file data as direct array: %v", err)
return nil, fmt.Errorf("failed to unmarshal TOML file as direct array: %w", err)
}
loadTomlCommandLogger.Debug("Found %d commands in direct TOML array", len(commands))
}
loadTomlCommandLogger.Debug("Successfully unmarshaled %d commands", len(commands))
loadTomlCommandLogger.Trace("Unmarshaled commands: %v", commands)
return commands, nil
}
// ConvertYAMLToTOML converts YAML files to TOML format
func ConvertYAMLToTOML(yamlPattern string) error {
convertLogger := modifyCommandLogger.WithPrefix("ConvertYAMLToTOML").WithField("pattern", yamlPattern)
convertLogger.Debug("Starting YAML to TOML conversion")
// Load YAML commands
yamlCommands, err := LoadCommandsFromCookFiles(yamlPattern)
if err != nil {
convertLogger.Error("Failed to load YAML commands: %v", err)
return fmt.Errorf("failed to load YAML commands: %w", err)
}
if len(yamlCommands) == 0 {
convertLogger.Info("No YAML commands found for pattern: %s", yamlPattern)
return nil
}
convertLogger.Debug("Loaded %d commands from YAML", len(yamlCommands))
// Find all YAML files matching the pattern
static, pattern := SplitPattern(yamlPattern)
yamlFiles, err := doublestar.Glob(os.DirFS(static), pattern)
if err != nil {
convertLogger.Error("Failed to glob YAML files: %v", err)
return fmt.Errorf("failed to glob YAML files: %w", err)
}
convertLogger.Debug("Found %d YAML files to convert", len(yamlFiles))
conversionCount := 0
skippedCount := 0
for _, yamlFile := range yamlFiles {
yamlFilePath := filepath.Join(static, yamlFile)
yamlFilePath = filepath.Clean(yamlFilePath)
yamlFilePath = strings.ReplaceAll(yamlFilePath, "\\", "/")
// Generate corresponding TOML file path
tomlFilePath := strings.TrimSuffix(yamlFilePath, filepath.Ext(yamlFilePath)) + ".toml"
convertLogger.Debug("Processing YAML file: %s -> %s", yamlFilePath, tomlFilePath)
// Check if TOML file already exists
if _, err := os.Stat(tomlFilePath); err == nil {
convertLogger.Info("Skipping conversion - TOML file already exists: %s", tomlFilePath)
skippedCount++
continue
}
// Read YAML file
yamlData, err := os.ReadFile(yamlFilePath)
if err != nil {
convertLogger.Error("Failed to read YAML file %s: %v", yamlFilePath, err)
continue
}
// Load YAML commands from this specific file
fileCommands, err := LoadCommandsFromCookFile(yamlData)
if err != nil {
convertLogger.Error("Failed to parse YAML file %s: %v", yamlFilePath, err)
continue
}
// Convert to TOML structure
tomlData, err := convertCommandsToTOML(fileCommands)
if err != nil {
convertLogger.Error("Failed to convert commands to TOML for %s: %v", yamlFilePath, err)
continue
}
// Write TOML file
err = os.WriteFile(tomlFilePath, tomlData, 0644)
if err != nil {
convertLogger.Error("Failed to write TOML file %s: %v", tomlFilePath, err)
continue
}
convertLogger.Info("Successfully converted %s to %s", yamlFilePath, tomlFilePath)
conversionCount++
}
convertLogger.Info("Conversion completed: %d files converted, %d files skipped", conversionCount, skippedCount)
return nil
}
// convertCommandsToTOML converts a slice of ModifyCommand to TOML format
func convertCommandsToTOML(commands []ModifyCommand) ([]byte, error) {
convertLogger := modifyCommandLogger.WithPrefix("convertCommandsToTOML")
convertLogger.Debug("Converting %d commands to TOML format", len(commands))
// Create TOML structure
tomlData := struct {
Commands []ModifyCommand `toml:"commands"`
}{
Commands: commands,
}
// Marshal to TOML
tomlBytes, err := toml.Marshal(tomlData)
if err != nil {
convertLogger.Error("Failed to marshal commands to TOML: %v", err)
return nil, fmt.Errorf("failed to marshal commands to TOML: %w", err)
}
convertLogger.Debug("Successfully converted %d commands to TOML (%d bytes)", len(commands), len(tomlBytes))
return tomlBytes, nil
}