dave/BigChef
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Hallucinate a json mode implementation

Browse Source
This commit is contained in:
PhatPhuckDave
2025-08-21 20:39:35 +02:00
parent ff30b00e71
commit bff7cc2a27
7 changed files with 485 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@@ -1,3 +1,4 @@
*.exe
.qodo
*.sqlite
testfiles

150
main.go
View File

@@ -44,9 +44,13 @@ func main() {
fmt.Fprintf(os.Stderr, " Reset files to their original state\n")
fmt.Fprintf(os.Stderr, " -loglevel string\n")
fmt.Fprintf(os.Stderr, " Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n")
fmt.Fprintf(os.Stderr, " -json\n")
fmt.Fprintf(os.Stderr, " Enable JSON mode for processing JSON files\n")
fmt.Fprintf(os.Stderr, "\nExamples:\n")
fmt.Fprintf(os.Stderr, " Regex mode (default):\n")
fmt.Fprintf(os.Stderr, " %s \"<value>(\\\\d+)</value>\" \"*1.5\" data.xml\n", os.Args[0])
fmt.Fprintf(os.Stderr, " JSON mode:\n")
fmt.Fprintf(os.Stderr, " %s -json data.json\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n")
fmt.Fprintf(os.Stderr, " s1, s2, etc. are used to refer to capture groups as strings.\n")
fmt.Fprintf(os.Stderr, " Helper functions: num(str) converts string to number, str(num) converts number to string\n")
@@ -426,7 +430,7 @@ func CreateExampleConfig() {
},
// Multiline regex example (DOTALL is auto-enabled). Captures numeric in nested XML.
{
Name: "XMLNestedValueMultiply",
Name: "XMLNestedValueMultiply",
Regex: `<item>\s*\s*<name>!any<\/name>\s*\s*<value>(!num)<\/value>\s*\s*<\/item>`,
Lua: `* $multiply`,
Files: []string{"data/**/*.xml"},
@@ -439,8 +443,8 @@ func CreateExampleConfig() {
`<entry>\s*\n\s*<id>(?P<id>!num)</id>\s*\n\s*<score>(?P<score>!num)</score>\s*\n\s*</entry>`,
`\[block\]\nkey=(?P<key>[A-Za-z_]+)\nvalue=(?P<val>!num)`,
},
Lua: `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`,
Files: []string{"examples/**/*.*"},
Lua: `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`,
Files: []string{"examples/**/*.*"},
LogLevel: "DEBUG",
},
// Use equals operator shorthand and boolean variable
@@ -452,19 +456,19 @@ func CreateExampleConfig() {
},
// Demonstrate NoDedup to allow overlapping replacements
{
Name: "OverlappingGroups",
Regex: `(?P<a>!num)(?P<b>!num)`,
Lua: `a = num(a) + 1; b = num(b) + 1; return true`,
Files: []string{"overlap/**/*.txt"},
Name: "OverlappingGroups",
Regex: `(?P<a>!num)(?P<b>!num)`,
Lua: `a = num(a) + 1; b = num(b) + 1; return true`,
Files: []string{"overlap/**/*.txt"},
NoDedup: true,
},
// Isolate command example operating on entire matched block
{
Name: "IsolateUppercaseBlock",
Regex: `BEGIN\n(?P<block>!any)\nEND`,
Lua: `block = upper(block); return true`,
Files: []string{"logs/**/*.log"},
Isolate: true,
Name: "IsolateUppercaseBlock",
Regex: `BEGIN\n(?P<block>!any)\nEND`,
Lua: `block = upper(block); return true`,
Files: []string{"logs/**/*.log"},
Isolate: true,
LogLevel: "TRACE",
},
// Using !rep placeholder and arrays of files
@@ -481,6 +485,25 @@ func CreateExampleConfig() {
Lua: `key = $prefix .. key; return true`,
Files: []string{"**/*.properties"},
},
// JSON mode examples
{
Name: "JSONArrayMultiply",
JSON: true,
Lua: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`,
Files: []string{"data/**/*.json"},
},
{
Name: "JSONObjectUpdate",
JSON: true,
Lua: `data.version = "2.0.0"; data.enabled = true; return true`,
Files: []string{"config/**/*.json"},
},
{
Name: "JSONNestedModify",
JSON: true,
Lua: `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`,
Files: []string{"settings/**/*.json"},
},
}
data, err := yaml.Marshal(commands)
@@ -506,11 +529,51 @@ func RunOtherCommands(file string, fileDataStr string, association utils.FileCom
runOtherCommandsLogger.Debug("Running other commands for file")
runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200))
// Aggregate all the modifications and execute them
// Separate JSON and regex commands for different processing approaches
jsonCommands := []utils.ModifyCommand{}
regexCommands := []utils.ModifyCommand{}
for _, command := range association.Commands {
if command.JSON || *utils.JSON {
jsonCommands = append(jsonCommands, command)
} else {
regexCommands = append(regexCommands, command)
}
}
// Process JSON commands sequentially (each operates on the entire file)
for _, command := range jsonCommands {
cmdLogger := logger.Default
if cmdLog, ok := commandLoggers[command.Name]; ok {
cmdLogger = cmdLog
}
cmdLogger.Debug("Processing file with JSON mode for command %q", command.Name)
newModifications, err := processor.ProcessJSON(fileDataStr, command, file)
if err != nil {
runOtherCommandsLogger.Error("Failed to process file with JSON command %q: %v", command.Name, err)
continue
}
// Apply JSON modifications immediately
if len(newModifications) > 0 {
var count int
fileDataStr, count = utils.ExecuteModifications(newModifications, fileDataStr)
atomic.AddInt64(&stats.TotalModifications, int64(count))
cmdLogger.Debug("Applied %d JSON modifications for command %q", count, command.Name)
}
count, ok := stats.ModificationsPerCommand.Load(command.Name)
if !ok {
count = 0
}
stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
}
// Aggregate regex modifications and execute them
modifications := []utils.ReplaceCommand{}
numCommandsConsidered := 0
for _, command := range association.Commands {
// Use command-specific logger if available, otherwise fall back to default logger
for _, command := range regexCommands {
cmdLogger := logger.Default
if cmdLog, ok := commandLoggers[command.Name]; ok {
cmdLogger = cmdLog
@@ -572,35 +635,62 @@ func RunIsolateCommands(association utils.FileCommandAssociation, file string, f
anythingDone := false
for _, isolateCommand := range association.IsolateCommands {
runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex)
patterns := isolateCommand.Regexes
if len(patterns) == 0 {
patterns = []string{isolateCommand.Regex}
}
for idx, pattern := range patterns {
tmpCmd := isolateCommand
tmpCmd.Regex = pattern
modifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file)
// Check if this isolate command should use JSON mode
if isolateCommand.JSON || *utils.JSON {
runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name)
modifications, err := processor.ProcessJSON(fileDataStr, isolateCommand, file)
if err != nil {
runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err)
runIsolateCommandsLogger.Error("Failed to process file with JSON isolate command %q: %v", isolateCommand.Name, err)
continue
}
if len(modifications) == 0 {
runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns))
runIsolateCommandsLogger.Debug("JSON isolate command %q produced no modifications", isolateCommand.Name)
continue
}
anythingDone = true
runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications))
runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications)
runIsolateCommandsLogger.Debug("Executing %d JSON isolate modifications for file", len(modifications))
runIsolateCommandsLogger.Trace("JSON isolate modifications: %v", modifications)
var count int
fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(fileDataStr, 200))
runIsolateCommandsLogger.Trace("File data after JSON isolate modifications: %s", utils.LimitString(fileDataStr, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count)
runIsolateCommandsLogger.Info("Executed %d JSON isolate modifications for file", count)
} else {
// Regular regex processing for isolate commands
runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex)
patterns := isolateCommand.Regexes
if len(patterns) == 0 {
patterns = []string{isolateCommand.Regex}
}
for idx, pattern := range patterns {
tmpCmd := isolateCommand
tmpCmd.Regex = pattern
modifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file)
if err != nil {
runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err)
continue
}
if len(modifications) == 0 {
runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns))
continue
}
anythingDone = true
runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications))
runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications)
var count int
fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(fileDataStr, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count)
}
}
}
if !anythingDone {

216
processor/json.go Normal file
View File

@@ -0,0 +1,216 @@
package processor
import (
"cook/utils"
"encoding/json"
"fmt"
"time"
logger "git.site.quack-lab.dev/dave/cylogger"
lua "github.com/yuin/gopher-lua"
)
// jsonLogger is a scoped logger for the processor/json package.
var jsonLogger = logger.Default.WithPrefix("processor/json")
// ProcessJSON applies Lua processing to JSON content
func ProcessJSON(content string, command utils.ModifyCommand, filename string) ([]utils.ReplaceCommand, error) {
processJsonLogger := jsonLogger.WithPrefix("ProcessJSON").WithField("commandName", command.Name).WithField("file", filename)
processJsonLogger.Debug("Starting JSON processing for file")
processJsonLogger.Trace("Initial file content length: %d", len(content))
var commands []utils.ReplaceCommand
startTime := time.Now()
// Parse JSON content
var jsonData interface{}
err := json.Unmarshal([]byte(content), &jsonData)
if err != nil {
processJsonLogger.Error("Failed to parse JSON content: %v", err)
return commands, fmt.Errorf("failed to parse JSON: %v", err)
}
processJsonLogger.Debug("Successfully parsed JSON content")
// Create Lua state
L, err := NewLuaState()
if err != nil {
processJsonLogger.Error("Error creating Lua state: %v", err)
return commands, fmt.Errorf("error creating Lua state: %v", err)
}
defer L.Close()
// Set filename global
L.SetGlobal("file", lua.LString(filename))
// Convert JSON data to Lua table
luaTable, err := ToLuaTable(L, jsonData)
if err != nil {
processJsonLogger.Error("Failed to convert JSON to Lua table: %v", err)
return commands, fmt.Errorf("failed to convert JSON to Lua table: %v", err)
}
// Set the JSON data as a global variable
L.SetGlobal("data", luaTable)
processJsonLogger.Debug("Set JSON data as Lua global 'data'")
// Build and execute Lua script for JSON mode
luaExpr := BuildJSONLuaScript(command.Lua)
processJsonLogger.Debug("Built Lua script from expression: %q", command.Lua)
processJsonLogger.Trace("Full Lua script: %q", utils.LimitString(luaExpr, 200))
if err := L.DoString(luaExpr); err != nil {
processJsonLogger.Error("Lua script execution failed: %v\nScript: %s", err, utils.LimitString(luaExpr, 200))
return commands, fmt.Errorf("lua script execution failed: %v", err)
}
processJsonLogger.Debug("Lua script executed successfully")
// Check if modification flag is set
modifiedVal := L.GetGlobal("modified")
if modifiedVal.Type() != lua.LTBool || !lua.LVAsBool(modifiedVal) {
processJsonLogger.Debug("Skipping - no modifications indicated by Lua script")
return commands, nil
}
// Get the modified data from Lua
modifiedData := L.GetGlobal("data")
if modifiedData.Type() != lua.LTTable {
processJsonLogger.Error("Expected 'data' to be a table after Lua processing, got %s", modifiedData.Type().String())
return commands, fmt.Errorf("expected 'data' to be a table after Lua processing")
}
// Convert back to Go interface
goData, err := FromLua(L, modifiedData)
if err != nil {
processJsonLogger.Error("Failed to convert Lua table back to Go: %v", err)
return commands, fmt.Errorf("failed to convert Lua table back to Go: %v", err)
}
// Marshal back to JSON
modifiedJSON, err := json.MarshalIndent(goData, "", " ")
if err != nil {
processJsonLogger.Error("Failed to marshal modified data to JSON: %v", err)
return commands, fmt.Errorf("failed to marshal modified data to JSON: %v", err)
}
// Create replacement command for the entire file
// For JSON mode, we always replace the entire content
commands = append(commands, utils.ReplaceCommand{
From: 0,
To: len(content),
With: string(modifiedJSON),
})
processJsonLogger.Debug("Total JSON processing time: %v", time.Since(startTime))
processJsonLogger.Debug("Generated %d total modifications", len(commands))
return commands, nil
}
// ToLuaTable converts a Go interface{} to a Lua table recursively
func ToLuaTable(L *lua.LState, data interface{}) (*lua.LTable, error) {
toLuaTableLogger := jsonLogger.WithPrefix("ToLuaTable")
toLuaTableLogger.Debug("Converting Go interface to Lua table")
toLuaTableLogger.Trace("Input data type: %T", data)
switch v := data.(type) {
case map[string]interface{}:
toLuaTableLogger.Debug("Converting map to Lua table")
table := L.CreateTable(0, len(v))
for key, value := range v {
luaValue, err := ToLuaValue(L, value)
if err != nil {
toLuaTableLogger.Error("Failed to convert map value for key %q: %v", key, err)
return nil, err
}
table.RawSetString(key, luaValue)
}
return table, nil
case []interface{}:
toLuaTableLogger.Debug("Converting slice to Lua table")
table := L.CreateTable(len(v), 0)
for i, value := range v {
luaValue, err := ToLuaValue(L, value)
if err != nil {
toLuaTableLogger.Error("Failed to convert slice value at index %d: %v", i, err)
return nil, err
}
table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed
}
return table, nil
case string:
toLuaTableLogger.Debug("Converting string to Lua string")
return nil, fmt.Errorf("expected table or array, got string")
case float64:
toLuaTableLogger.Debug("Converting float64 to Lua number")
return nil, fmt.Errorf("expected table or array, got number")
case bool:
toLuaTableLogger.Debug("Converting bool to Lua boolean")
return nil, fmt.Errorf("expected table or array, got boolean")
case nil:
toLuaTableLogger.Debug("Converting nil to Lua nil")
return nil, fmt.Errorf("expected table or array, got nil")
default:
toLuaTableLogger.Error("Unsupported type for Lua table conversion: %T", v)
return nil, fmt.Errorf("unsupported type for Lua table conversion: %T", v)
}
}
// ToLuaValue converts a Go interface{} to a Lua value
func ToLuaValue(L *lua.LState, data interface{}) (lua.LValue, error) {
toLuaValueLogger := jsonLogger.WithPrefix("ToLuaValue")
toLuaValueLogger.Debug("Converting Go interface to Lua value")
toLuaValueLogger.Trace("Input data type: %T", data)
switch v := data.(type) {
case map[string]interface{}:
toLuaValueLogger.Debug("Converting map to Lua table")
table := L.CreateTable(0, len(v))
for key, value := range v {
luaValue, err := ToLuaValue(L, value)
if err != nil {
toLuaValueLogger.Error("Failed to convert map value for key %q: %v", key, err)
return lua.LNil, err
}
table.RawSetString(key, luaValue)
}
return table, nil
case []interface{}:
toLuaValueLogger.Debug("Converting slice to Lua table")
table := L.CreateTable(len(v), 0)
for i, value := range v {
luaValue, err := ToLuaValue(L, value)
if err != nil {
toLuaValueLogger.Error("Failed to convert slice value at index %d: %v", i, err)
return lua.LNil, err
}
table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed
}
return table, nil
case string:
toLuaValueLogger.Debug("Converting string to Lua string")
return lua.LString(v), nil
case float64:
toLuaValueLogger.Debug("Converting float64 to Lua number")
return lua.LNumber(v), nil
case bool:
toLuaValueLogger.Debug("Converting bool to Lua boolean")
return lua.LBool(v), nil
case nil:
toLuaValueLogger.Debug("Converting nil to Lua nil")
return lua.LNil, nil
default:
toLuaValueLogger.Error("Unsupported type for Lua value conversion: %T", v)
return lua.LNil, fmt.Errorf("unsupported type for Lua value conversion: %T", v)
}
}

117
processor/json_test.go Normal file
View File

@@ -0,0 +1,117 @@
package processor
import (
"cook/utils"
"testing"
"github.com/stretchr/testify/assert"
)
func TestProcessJSON(t *testing.T) {
tests := []struct {
name string
input string
luaExpression string
expectedOutput string
expectedMods int
}{
{
name: "Basic JSON object modification",
input: `{"name": "test", "value": 42}`,
luaExpression: `data.value = data.value * 2; return true`,
expectedOutput: `{
"name": "test",
"value": 84
}`,
expectedMods: 1,
},
{
name: "JSON array modification",
input: `{"items": [{"id": 1, "value": 10}, {"id": 2, "value": 20}]}`,
luaExpression: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 1.5 end; return true`,
expectedOutput: `{
"items": [
{
"id": 1,
"value": 15
},
{
"id": 2,
"value": 30
}
]
}`,
expectedMods: 1,
},
{
name: "JSON nested object modification",
input: `{"config": {"settings": {"enabled": false, "timeout": 30}}}`,
luaExpression: `data.config.settings.enabled = true; data.config.settings.timeout = 60; return true`,
expectedOutput: `{
"config": {
"settings": {
"enabled": true,
"timeout": 60
}
}
}`,
expectedMods: 1,
},
{
name: "JSON no modification",
input: `{"name": "test", "value": 42}`,
luaExpression: `return false`,
expectedOutput: `{"name": "test", "value": 42}`,
expectedMods: 0,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
command := utils.ModifyCommand{
Name: tt.name,
JSON: true,
Lua: tt.luaExpression,
}
modifications, err := ProcessJSON(tt.input, command, "test.json")
assert.NoError(t, err, "ProcessJSON failed: %v", err)
if len(modifications) > 0 {
// Execute modifications
result, count := utils.ExecuteModifications(modifications, tt.input)
assert.Equal(t, tt.expectedMods, count, "Expected %d modifications, got %d", tt.expectedMods, count)
assert.Equal(t, tt.expectedOutput, result, "Expected output: %s, got: %s", tt.expectedOutput, result)
} else {
assert.Equal(t, 0, tt.expectedMods, "Expected no modifications but got some")
}
})
}
}
func TestToLuaValue(t *testing.T) {
L, err := NewLuaState()
if err != nil {
t.Fatalf("Failed to create Lua state: %v", err)
}
defer L.Close()
tests := []struct {
name string
input interface{}
expected string
}{
{"string", "hello", "hello"},
{"number", 42.0, "42"},
{"boolean", true, "true"},
{"nil", nil, "nil"},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
result, err := ToLuaValue(L, tt.input)
assert.NoError(t, err)
assert.Equal(t, tt.expected, result.String())
})
}
}

20
processor/processor.go
View File

@@ -300,6 +300,26 @@ func BuildLuaScript(luaExpr string) string {
return fullScript
}
// BuildJSONLuaScript prepares a Lua expression for JSON mode
func BuildJSONLuaScript(luaExpr string) string {
buildJsonLuaScriptLogger := processorLogger.WithPrefix("BuildJSONLuaScript").WithField("inputLuaExpr", luaExpr)
buildJsonLuaScriptLogger.Debug("Building full Lua script for JSON mode from expression")
// Perform $var substitutions from globalVariables
luaExpr = replaceVariables(luaExpr)
fullScript := fmt.Sprintf(`
function run()
%s
end
local res = run()
modified = res == nil or res
`, luaExpr)
buildJsonLuaScriptLogger.Trace("Generated full JSON Lua script: %q", utils.LimitString(fullScript, 200))
return fullScript
}
func replaceVariables(expr string) string {
// $varName -> literal value
varNameRe := regexp.MustCompile(`\$(\w+)`)

3
utils/flags.go
View File

@@ -12,9 +12,10 @@ var flagsLogger = logger.Default.WithPrefix("utils/flags")
var (
ParallelFiles = flag.Int("P", 100, "Number of files to process in parallel")
Filter = flag.String("f", "", "Filter commands before running them")
JSON = flag.Bool("json", false, "Enable JSON mode for processing JSON files")
)
func init() {
flagsLogger.Debug("Initializing flags")
flagsLogger.Trace("ParallelFiles initial value: %d, Filter initial value: %q", *ParallelFiles, *Filter)
flagsLogger.Trace("ParallelFiles initial value: %d, Filter initial value: %q, JSON initial value: %t", *ParallelFiles, *Filter, *JSON)
}

12
utils/modifycommand.go
View File

@@ -25,6 +25,7 @@ type ModifyCommand struct {
Isolate bool `yaml:"isolate,omitempty"`
NoDedup bool `yaml:"nodedup,omitempty"`
Disabled bool `yaml:"disable,omitempty"`
JSON bool `yaml:"json,omitempty"`
Modifiers map[string]interface{} `yaml:"modifiers,omitempty"`
}
@@ -33,10 +34,15 @@ type CookFile []ModifyCommand
func (c *ModifyCommand) Validate() error {
validateLogger := modifyCommandLogger.WithPrefix("Validate").WithField("commandName", c.Name)
validateLogger.Debug("Validating command")
if c.Regex == "" && len(c.Regexes) == 0 {
validateLogger.Error("Validation failed: Regex pattern is required")
return fmt.Errorf("pattern is required")
// For JSON mode, regex patterns are not required
if !c.JSON {
if c.Regex == "" && len(c.Regexes) == 0 {
validateLogger.Error("Validation failed: Regex pattern is required for non-JSON mode")
return fmt.Errorf("pattern is required for non-JSON mode")
}
}
if c.Lua == "" {
validateLogger.Error("Validation failed: Lua expression is required")
return fmt.Errorf("lua expression is required")