diff --git a/.gitignore b/.gitignore index 6640f57..0745d0d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.exe .qodo *.sqlite +testfiles diff --git a/main.go b/main.go index 18af82b..99d7cfe 100644 --- a/main.go +++ b/main.go @@ -44,9 +44,13 @@ func main() { fmt.Fprintf(os.Stderr, " Reset files to their original state\n") fmt.Fprintf(os.Stderr, " -loglevel string\n") fmt.Fprintf(os.Stderr, " Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n") + fmt.Fprintf(os.Stderr, " -json\n") + fmt.Fprintf(os.Stderr, " Enable JSON mode for processing JSON files\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, " Regex mode (default):\n") fmt.Fprintf(os.Stderr, " %s \"(\\\\d+)\" \"*1.5\" data.xml\n", os.Args[0]) + fmt.Fprintf(os.Stderr, " JSON mode:\n") + fmt.Fprintf(os.Stderr, " %s -json data.json\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n") fmt.Fprintf(os.Stderr, " s1, s2, etc. are used to refer to capture groups as strings.\n") fmt.Fprintf(os.Stderr, " Helper functions: num(str) converts string to number, str(num) converts number to string\n") @@ -426,7 +430,7 @@ func CreateExampleConfig() { }, // Multiline regex example (DOTALL is auto-enabled). Captures numeric in nested XML. { - Name: "XMLNestedValueMultiply", + Name: "XMLNestedValueMultiply", Regex: `\s*\s*!any<\/name>\s*\s*(!num)<\/value>\s*\s*<\/item>`, Lua: `* $multiply`, Files: []string{"data/**/*.xml"}, @@ -439,8 +443,8 @@ func CreateExampleConfig() { `\s*\n\s*(?P!num)\s*\n\s*(?P!num)\s*\n\s*`, `\[block\]\nkey=(?P[A-Za-z_]+)\nvalue=(?P!num)`, }, - Lua: `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`, - Files: []string{"examples/**/*.*"}, + Lua: `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`, + Files: []string{"examples/**/*.*"}, LogLevel: "DEBUG", }, // Use equals operator shorthand and boolean variable @@ -452,19 +456,19 @@ func CreateExampleConfig() { }, // Demonstrate NoDedup to allow overlapping replacements { - Name: "OverlappingGroups", - Regex: `(?P!num)(?P!num)`, - Lua: `a = num(a) + 1; b = num(b) + 1; return true`, - Files: []string{"overlap/**/*.txt"}, + Name: "OverlappingGroups", + Regex: `(?P!num)(?P!num)`, + Lua: `a = num(a) + 1; b = num(b) + 1; return true`, + Files: []string{"overlap/**/*.txt"}, NoDedup: true, }, // Isolate command example operating on entire matched block { - Name: "IsolateUppercaseBlock", - Regex: `BEGIN\n(?P!any)\nEND`, - Lua: `block = upper(block); return true`, - Files: []string{"logs/**/*.log"}, - Isolate: true, + Name: "IsolateUppercaseBlock", + Regex: `BEGIN\n(?P!any)\nEND`, + Lua: `block = upper(block); return true`, + Files: []string{"logs/**/*.log"}, + Isolate: true, LogLevel: "TRACE", }, // Using !rep placeholder and arrays of files @@ -481,6 +485,25 @@ func CreateExampleConfig() { Lua: `key = $prefix .. key; return true`, Files: []string{"**/*.properties"}, }, + // JSON mode examples + { + Name: "JSONArrayMultiply", + JSON: true, + Lua: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`, + Files: []string{"data/**/*.json"}, + }, + { + Name: "JSONObjectUpdate", + JSON: true, + Lua: `data.version = "2.0.0"; data.enabled = true; return true`, + Files: []string{"config/**/*.json"}, + }, + { + Name: "JSONNestedModify", + JSON: true, + Lua: `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`, + Files: []string{"settings/**/*.json"}, + }, } data, err := yaml.Marshal(commands) @@ -506,11 +529,51 @@ func RunOtherCommands(file string, fileDataStr string, association utils.FileCom runOtherCommandsLogger.Debug("Running other commands for file") runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200)) - // Aggregate all the modifications and execute them + // Separate JSON and regex commands for different processing approaches + jsonCommands := []utils.ModifyCommand{} + regexCommands := []utils.ModifyCommand{} + + for _, command := range association.Commands { + if command.JSON || *utils.JSON { + jsonCommands = append(jsonCommands, command) + } else { + regexCommands = append(regexCommands, command) + } + } + + // Process JSON commands sequentially (each operates on the entire file) + for _, command := range jsonCommands { + cmdLogger := logger.Default + if cmdLog, ok := commandLoggers[command.Name]; ok { + cmdLogger = cmdLog + } + + cmdLogger.Debug("Processing file with JSON mode for command %q", command.Name) + newModifications, err := processor.ProcessJSON(fileDataStr, command, file) + if err != nil { + runOtherCommandsLogger.Error("Failed to process file with JSON command %q: %v", command.Name, err) + continue + } + + // Apply JSON modifications immediately + if len(newModifications) > 0 { + var count int + fileDataStr, count = utils.ExecuteModifications(newModifications, fileDataStr) + atomic.AddInt64(&stats.TotalModifications, int64(count)) + cmdLogger.Debug("Applied %d JSON modifications for command %q", count, command.Name) + } + + count, ok := stats.ModificationsPerCommand.Load(command.Name) + if !ok { + count = 0 + } + stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications)) + } + + // Aggregate regex modifications and execute them modifications := []utils.ReplaceCommand{} numCommandsConsidered := 0 - for _, command := range association.Commands { - // Use command-specific logger if available, otherwise fall back to default logger + for _, command := range regexCommands { cmdLogger := logger.Default if cmdLog, ok := commandLoggers[command.Name]; ok { cmdLogger = cmdLog @@ -572,35 +635,62 @@ func RunIsolateCommands(association utils.FileCommandAssociation, file string, f anythingDone := false for _, isolateCommand := range association.IsolateCommands { - runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex) - patterns := isolateCommand.Regexes - if len(patterns) == 0 { - patterns = []string{isolateCommand.Regex} - } - for idx, pattern := range patterns { - tmpCmd := isolateCommand - tmpCmd.Regex = pattern - modifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file) + // Check if this isolate command should use JSON mode + if isolateCommand.JSON || *utils.JSON { + runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name) + modifications, err := processor.ProcessJSON(fileDataStr, isolateCommand, file) if err != nil { - runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err) + runIsolateCommandsLogger.Error("Failed to process file with JSON isolate command %q: %v", isolateCommand.Name, err) continue } if len(modifications) == 0 { - runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns)) + runIsolateCommandsLogger.Debug("JSON isolate command %q produced no modifications", isolateCommand.Name) continue } anythingDone = true - runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications)) - runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications) + runIsolateCommandsLogger.Debug("Executing %d JSON isolate modifications for file", len(modifications)) + runIsolateCommandsLogger.Trace("JSON isolate modifications: %v", modifications) var count int fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr) - runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(fileDataStr, 200)) + runIsolateCommandsLogger.Trace("File data after JSON isolate modifications: %s", utils.LimitString(fileDataStr, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) - runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count) + runIsolateCommandsLogger.Info("Executed %d JSON isolate modifications for file", count) + } else { + // Regular regex processing for isolate commands + runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex) + patterns := isolateCommand.Regexes + if len(patterns) == 0 { + patterns = []string{isolateCommand.Regex} + } + for idx, pattern := range patterns { + tmpCmd := isolateCommand + tmpCmd.Regex = pattern + modifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file) + if err != nil { + runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err) + continue + } + + if len(modifications) == 0 { + runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns)) + continue + } + anythingDone = true + + runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications)) + runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications) + var count int + fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr) + runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(fileDataStr, 200)) + + atomic.AddInt64(&stats.TotalModifications, int64(count)) + + runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count) + } } } if !anythingDone { diff --git a/processor/json.go b/processor/json.go new file mode 100644 index 0000000..f9a11a6 --- /dev/null +++ b/processor/json.go @@ -0,0 +1,216 @@ +package processor + +import ( + "cook/utils" + "encoding/json" + "fmt" + "time" + + logger "git.site.quack-lab.dev/dave/cylogger" + lua "github.com/yuin/gopher-lua" +) + +// jsonLogger is a scoped logger for the processor/json package. +var jsonLogger = logger.Default.WithPrefix("processor/json") + +// ProcessJSON applies Lua processing to JSON content +func ProcessJSON(content string, command utils.ModifyCommand, filename string) ([]utils.ReplaceCommand, error) { + processJsonLogger := jsonLogger.WithPrefix("ProcessJSON").WithField("commandName", command.Name).WithField("file", filename) + processJsonLogger.Debug("Starting JSON processing for file") + processJsonLogger.Trace("Initial file content length: %d", len(content)) + + var commands []utils.ReplaceCommand + startTime := time.Now() + + // Parse JSON content + var jsonData interface{} + err := json.Unmarshal([]byte(content), &jsonData) + if err != nil { + processJsonLogger.Error("Failed to parse JSON content: %v", err) + return commands, fmt.Errorf("failed to parse JSON: %v", err) + } + processJsonLogger.Debug("Successfully parsed JSON content") + + // Create Lua state + L, err := NewLuaState() + if err != nil { + processJsonLogger.Error("Error creating Lua state: %v", err) + return commands, fmt.Errorf("error creating Lua state: %v", err) + } + defer L.Close() + + // Set filename global + L.SetGlobal("file", lua.LString(filename)) + + // Convert JSON data to Lua table + luaTable, err := ToLuaTable(L, jsonData) + if err != nil { + processJsonLogger.Error("Failed to convert JSON to Lua table: %v", err) + return commands, fmt.Errorf("failed to convert JSON to Lua table: %v", err) + } + + // Set the JSON data as a global variable + L.SetGlobal("data", luaTable) + processJsonLogger.Debug("Set JSON data as Lua global 'data'") + + // Build and execute Lua script for JSON mode + luaExpr := BuildJSONLuaScript(command.Lua) + processJsonLogger.Debug("Built Lua script from expression: %q", command.Lua) + processJsonLogger.Trace("Full Lua script: %q", utils.LimitString(luaExpr, 200)) + + if err := L.DoString(luaExpr); err != nil { + processJsonLogger.Error("Lua script execution failed: %v\nScript: %s", err, utils.LimitString(luaExpr, 200)) + return commands, fmt.Errorf("lua script execution failed: %v", err) + } + processJsonLogger.Debug("Lua script executed successfully") + + // Check if modification flag is set + modifiedVal := L.GetGlobal("modified") + if modifiedVal.Type() != lua.LTBool || !lua.LVAsBool(modifiedVal) { + processJsonLogger.Debug("Skipping - no modifications indicated by Lua script") + return commands, nil + } + + // Get the modified data from Lua + modifiedData := L.GetGlobal("data") + if modifiedData.Type() != lua.LTTable { + processJsonLogger.Error("Expected 'data' to be a table after Lua processing, got %s", modifiedData.Type().String()) + return commands, fmt.Errorf("expected 'data' to be a table after Lua processing") + } + + // Convert back to Go interface + goData, err := FromLua(L, modifiedData) + if err != nil { + processJsonLogger.Error("Failed to convert Lua table back to Go: %v", err) + return commands, fmt.Errorf("failed to convert Lua table back to Go: %v", err) + } + + // Marshal back to JSON + modifiedJSON, err := json.MarshalIndent(goData, "", " ") + if err != nil { + processJsonLogger.Error("Failed to marshal modified data to JSON: %v", err) + return commands, fmt.Errorf("failed to marshal modified data to JSON: %v", err) + } + + // Create replacement command for the entire file + // For JSON mode, we always replace the entire content + commands = append(commands, utils.ReplaceCommand{ + From: 0, + To: len(content), + With: string(modifiedJSON), + }) + + processJsonLogger.Debug("Total JSON processing time: %v", time.Since(startTime)) + processJsonLogger.Debug("Generated %d total modifications", len(commands)) + return commands, nil +} + +// ToLuaTable converts a Go interface{} to a Lua table recursively +func ToLuaTable(L *lua.LState, data interface{}) (*lua.LTable, error) { + toLuaTableLogger := jsonLogger.WithPrefix("ToLuaTable") + toLuaTableLogger.Debug("Converting Go interface to Lua table") + toLuaTableLogger.Trace("Input data type: %T", data) + + switch v := data.(type) { + case map[string]interface{}: + toLuaTableLogger.Debug("Converting map to Lua table") + table := L.CreateTable(0, len(v)) + for key, value := range v { + luaValue, err := ToLuaValue(L, value) + if err != nil { + toLuaTableLogger.Error("Failed to convert map value for key %q: %v", key, err) + return nil, err + } + table.RawSetString(key, luaValue) + } + return table, nil + + case []interface{}: + toLuaTableLogger.Debug("Converting slice to Lua table") + table := L.CreateTable(len(v), 0) + for i, value := range v { + luaValue, err := ToLuaValue(L, value) + if err != nil { + toLuaTableLogger.Error("Failed to convert slice value at index %d: %v", i, err) + return nil, err + } + table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed + } + return table, nil + + case string: + toLuaTableLogger.Debug("Converting string to Lua string") + return nil, fmt.Errorf("expected table or array, got string") + + case float64: + toLuaTableLogger.Debug("Converting float64 to Lua number") + return nil, fmt.Errorf("expected table or array, got number") + + case bool: + toLuaTableLogger.Debug("Converting bool to Lua boolean") + return nil, fmt.Errorf("expected table or array, got boolean") + + case nil: + toLuaTableLogger.Debug("Converting nil to Lua nil") + return nil, fmt.Errorf("expected table or array, got nil") + + default: + toLuaTableLogger.Error("Unsupported type for Lua table conversion: %T", v) + return nil, fmt.Errorf("unsupported type for Lua table conversion: %T", v) + } +} + +// ToLuaValue converts a Go interface{} to a Lua value +func ToLuaValue(L *lua.LState, data interface{}) (lua.LValue, error) { + toLuaValueLogger := jsonLogger.WithPrefix("ToLuaValue") + toLuaValueLogger.Debug("Converting Go interface to Lua value") + toLuaValueLogger.Trace("Input data type: %T", data) + + switch v := data.(type) { + case map[string]interface{}: + toLuaValueLogger.Debug("Converting map to Lua table") + table := L.CreateTable(0, len(v)) + for key, value := range v { + luaValue, err := ToLuaValue(L, value) + if err != nil { + toLuaValueLogger.Error("Failed to convert map value for key %q: %v", key, err) + return lua.LNil, err + } + table.RawSetString(key, luaValue) + } + return table, nil + + case []interface{}: + toLuaValueLogger.Debug("Converting slice to Lua table") + table := L.CreateTable(len(v), 0) + for i, value := range v { + luaValue, err := ToLuaValue(L, value) + if err != nil { + toLuaValueLogger.Error("Failed to convert slice value at index %d: %v", i, err) + return lua.LNil, err + } + table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed + } + return table, nil + + case string: + toLuaValueLogger.Debug("Converting string to Lua string") + return lua.LString(v), nil + + case float64: + toLuaValueLogger.Debug("Converting float64 to Lua number") + return lua.LNumber(v), nil + + case bool: + toLuaValueLogger.Debug("Converting bool to Lua boolean") + return lua.LBool(v), nil + + case nil: + toLuaValueLogger.Debug("Converting nil to Lua nil") + return lua.LNil, nil + + default: + toLuaValueLogger.Error("Unsupported type for Lua value conversion: %T", v) + return lua.LNil, fmt.Errorf("unsupported type for Lua value conversion: %T", v) + } +} diff --git a/processor/json_test.go b/processor/json_test.go new file mode 100644 index 0000000..9d0a7bb --- /dev/null +++ b/processor/json_test.go @@ -0,0 +1,117 @@ +package processor + +import ( + "cook/utils" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestProcessJSON(t *testing.T) { + tests := []struct { + name string + input string + luaExpression string + expectedOutput string + expectedMods int + }{ + { + name: "Basic JSON object modification", + input: `{"name": "test", "value": 42}`, + luaExpression: `data.value = data.value * 2; return true`, + expectedOutput: `{ + "name": "test", + "value": 84 +}`, + expectedMods: 1, + }, + { + name: "JSON array modification", + input: `{"items": [{"id": 1, "value": 10}, {"id": 2, "value": 20}]}`, + luaExpression: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 1.5 end; return true`, + expectedOutput: `{ + "items": [ + { + "id": 1, + "value": 15 + }, + { + "id": 2, + "value": 30 + } + ] +}`, + expectedMods: 1, + }, + { + name: "JSON nested object modification", + input: `{"config": {"settings": {"enabled": false, "timeout": 30}}}`, + luaExpression: `data.config.settings.enabled = true; data.config.settings.timeout = 60; return true`, + expectedOutput: `{ + "config": { + "settings": { + "enabled": true, + "timeout": 60 + } + } +}`, + expectedMods: 1, + }, + { + name: "JSON no modification", + input: `{"name": "test", "value": 42}`, + luaExpression: `return false`, + expectedOutput: `{"name": "test", "value": 42}`, + expectedMods: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + command := utils.ModifyCommand{ + Name: tt.name, + JSON: true, + Lua: tt.luaExpression, + } + + modifications, err := ProcessJSON(tt.input, command, "test.json") + assert.NoError(t, err, "ProcessJSON failed: %v", err) + + if len(modifications) > 0 { + // Execute modifications + result, count := utils.ExecuteModifications(modifications, tt.input) + assert.Equal(t, tt.expectedMods, count, "Expected %d modifications, got %d", tt.expectedMods, count) + assert.Equal(t, tt.expectedOutput, result, "Expected output: %s, got: %s", tt.expectedOutput, result) + } else { + assert.Equal(t, 0, tt.expectedMods, "Expected no modifications but got some") + } + }) + } +} + +func TestToLuaValue(t *testing.T) { + L, err := NewLuaState() + if err != nil { + t.Fatalf("Failed to create Lua state: %v", err) + } + defer L.Close() + + tests := []struct { + name string + input interface{} + expected string + }{ + {"string", "hello", "hello"}, + {"number", 42.0, "42"}, + {"boolean", true, "true"}, + {"nil", nil, "nil"}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, err := ToLuaValue(L, tt.input) + assert.NoError(t, err) + assert.Equal(t, tt.expected, result.String()) + }) + } +} diff --git a/processor/processor.go b/processor/processor.go index d40b7d8..f85eb97 100644 --- a/processor/processor.go +++ b/processor/processor.go @@ -300,6 +300,26 @@ func BuildLuaScript(luaExpr string) string { return fullScript } +// BuildJSONLuaScript prepares a Lua expression for JSON mode +func BuildJSONLuaScript(luaExpr string) string { + buildJsonLuaScriptLogger := processorLogger.WithPrefix("BuildJSONLuaScript").WithField("inputLuaExpr", luaExpr) + buildJsonLuaScriptLogger.Debug("Building full Lua script for JSON mode from expression") + + // Perform $var substitutions from globalVariables + luaExpr = replaceVariables(luaExpr) + + fullScript := fmt.Sprintf(` + function run() + %s + end + local res = run() + modified = res == nil or res + `, luaExpr) + buildJsonLuaScriptLogger.Trace("Generated full JSON Lua script: %q", utils.LimitString(fullScript, 200)) + + return fullScript +} + func replaceVariables(expr string) string { // $varName -> literal value varNameRe := regexp.MustCompile(`\$(\w+)`) diff --git a/utils/flags.go b/utils/flags.go index d76d56f..d11f761 100644 --- a/utils/flags.go +++ b/utils/flags.go @@ -12,9 +12,10 @@ var flagsLogger = logger.Default.WithPrefix("utils/flags") var ( ParallelFiles = flag.Int("P", 100, "Number of files to process in parallel") Filter = flag.String("f", "", "Filter commands before running them") + JSON = flag.Bool("json", false, "Enable JSON mode for processing JSON files") ) func init() { flagsLogger.Debug("Initializing flags") - flagsLogger.Trace("ParallelFiles initial value: %d, Filter initial value: %q", *ParallelFiles, *Filter) + flagsLogger.Trace("ParallelFiles initial value: %d, Filter initial value: %q, JSON initial value: %t", *ParallelFiles, *Filter, *JSON) } diff --git a/utils/modifycommand.go b/utils/modifycommand.go index 9937055..3d7a560 100644 --- a/utils/modifycommand.go +++ b/utils/modifycommand.go @@ -25,6 +25,7 @@ type ModifyCommand struct { Isolate bool `yaml:"isolate,omitempty"` NoDedup bool `yaml:"nodedup,omitempty"` Disabled bool `yaml:"disable,omitempty"` + JSON bool `yaml:"json,omitempty"` Modifiers map[string]interface{} `yaml:"modifiers,omitempty"` } @@ -33,10 +34,15 @@ type CookFile []ModifyCommand func (c *ModifyCommand) Validate() error { validateLogger := modifyCommandLogger.WithPrefix("Validate").WithField("commandName", c.Name) validateLogger.Debug("Validating command") - if c.Regex == "" && len(c.Regexes) == 0 { - validateLogger.Error("Validation failed: Regex pattern is required") - return fmt.Errorf("pattern is required") + + // For JSON mode, regex patterns are not required + if !c.JSON { + if c.Regex == "" && len(c.Regexes) == 0 { + validateLogger.Error("Validation failed: Regex pattern is required for non-JSON mode") + return fmt.Errorf("pattern is required for non-JSON mode") + } } + if c.Lua == "" { validateLogger.Error("Validation failed: Lua expression is required") return fmt.Errorf("lua expression is required")