Hallucinate a json mode implementation

.gitignore

@@ -1,3 +1,4 @@
 *.exe
 .qodo
 *.sqlite
+testfiles

main.go

@@ -44,9 +44,13 @@ func main() {
 		fmt.Fprintf(os.Stderr, "        Reset files to their original state\n")
 		fmt.Fprintf(os.Stderr, "  -loglevel string\n")
 		fmt.Fprintf(os.Stderr, "        Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n")
+		fmt.Fprintf(os.Stderr, "  -json\n")
+		fmt.Fprintf(os.Stderr, "        Enable JSON mode for processing JSON files\n")
 		fmt.Fprintf(os.Stderr, "\nExamples:\n")
 		fmt.Fprintf(os.Stderr, "  Regex mode (default):\n")
 		fmt.Fprintf(os.Stderr, "    %s \"<value>(\\\\d+)</value>\" \"*1.5\" data.xml\n", os.Args[0])
+		fmt.Fprintf(os.Stderr, "  JSON mode:\n")
+		fmt.Fprintf(os.Stderr, "    %s -json data.json\n", os.Args[0])
 		fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n")
 		fmt.Fprintf(os.Stderr, "      s1, s2, etc. are used to refer to capture groups as strings.\n")
 		fmt.Fprintf(os.Stderr, "      Helper functions: num(str) converts string to number, str(num) converts number to string\n")
@@ -481,6 +485,25 @@ func CreateExampleConfig() {
 			Lua:   `key = $prefix .. key; return true`,
 			Files: []string{"**/*.properties"},
 		},
+		// JSON mode examples
+		{
+			Name:  "JSONArrayMultiply",
+			JSON:  true,
+			Lua:   `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`,
+			Files: []string{"data/**/*.json"},
+		},
+		{
+			Name:  "JSONObjectUpdate",
+			JSON:  true,
+			Lua:   `data.version = "2.0.0"; data.enabled = true; return true`,
+			Files: []string{"config/**/*.json"},
+		},
+		{
+			Name:  "JSONNestedModify",
+			JSON:  true,
+			Lua:   `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`,
+			Files: []string{"settings/**/*.json"},
+		},
 	}
 
 	data, err := yaml.Marshal(commands)
@@ -506,11 +529,51 @@ func RunOtherCommands(file string, fileDataStr string, association utils.FileCom
 	runOtherCommandsLogger.Debug("Running other commands for file")
 	runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200))
 
-	// Aggregate all the modifications and execute them
+	// Separate JSON and regex commands for different processing approaches
+	jsonCommands := []utils.ModifyCommand{}
+	regexCommands := []utils.ModifyCommand{}
+
+	for _, command := range association.Commands {
+		if command.JSON || *utils.JSON {
+			jsonCommands = append(jsonCommands, command)
+		} else {
+			regexCommands = append(regexCommands, command)
+		}
+	}
+
+	// Process JSON commands sequentially (each operates on the entire file)
+	for _, command := range jsonCommands {
+		cmdLogger := logger.Default
+		if cmdLog, ok := commandLoggers[command.Name]; ok {
+			cmdLogger = cmdLog
+		}
+
+		cmdLogger.Debug("Processing file with JSON mode for command %q", command.Name)
+		newModifications, err := processor.ProcessJSON(fileDataStr, command, file)
+		if err != nil {
+			runOtherCommandsLogger.Error("Failed to process file with JSON command %q: %v", command.Name, err)
+			continue
+		}
+
+		// Apply JSON modifications immediately
+		if len(newModifications) > 0 {
+			var count int
+			fileDataStr, count = utils.ExecuteModifications(newModifications, fileDataStr)
+			atomic.AddInt64(&stats.TotalModifications, int64(count))
+			cmdLogger.Debug("Applied %d JSON modifications for command %q", count, command.Name)
+		}
+
+		count, ok := stats.ModificationsPerCommand.Load(command.Name)
+		if !ok {
+			count = 0
+		}
+		stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
+	}
+
+	// Aggregate regex modifications and execute them
 	modifications := []utils.ReplaceCommand{}
 	numCommandsConsidered := 0
-	for _, command := range association.Commands {
-		// Use command-specific logger if available, otherwise fall back to default logger
+	for _, command := range regexCommands {
 		cmdLogger := logger.Default
 		if cmdLog, ok := commandLoggers[command.Name]; ok {
 			cmdLogger = cmdLog
@@ -572,6 +635,32 @@ func RunIsolateCommands(association utils.FileCommandAssociation, file string, f
 
 	anythingDone := false
 	for _, isolateCommand := range association.IsolateCommands {
+		// Check if this isolate command should use JSON mode
+		if isolateCommand.JSON || *utils.JSON {
+			runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name)
+			modifications, err := processor.ProcessJSON(fileDataStr, isolateCommand, file)
+			if err != nil {
+				runIsolateCommandsLogger.Error("Failed to process file with JSON isolate command %q: %v", isolateCommand.Name, err)
+				continue
+			}
+
+			if len(modifications) == 0 {
+				runIsolateCommandsLogger.Debug("JSON isolate command %q produced no modifications", isolateCommand.Name)
+				continue
+			}
+			anythingDone = true
+
+			runIsolateCommandsLogger.Debug("Executing %d JSON isolate modifications for file", len(modifications))
+			runIsolateCommandsLogger.Trace("JSON isolate modifications: %v", modifications)
+			var count int
+			fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
+			runIsolateCommandsLogger.Trace("File data after JSON isolate modifications: %s", utils.LimitString(fileDataStr, 200))
+
+			atomic.AddInt64(&stats.TotalModifications, int64(count))
+
+			runIsolateCommandsLogger.Info("Executed %d JSON isolate modifications for file", count)
+		} else {
+			// Regular regex processing for isolate commands
 			runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex)
 			patterns := isolateCommand.Regexes
 			if len(patterns) == 0 {
@@ -603,6 +692,7 @@ func RunIsolateCommands(association utils.FileCommandAssociation, file string, f
 				runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count)
 			}
 		}
+	}
 	if !anythingDone {
 		runIsolateCommandsLogger.Debug("No isolate modifications were made for file")
 		return fileDataStr, NothingToDo

processor/json.go

@@ -0,0 +1,216 @@
+package processor
+
+import (
+	"cook/utils"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	logger "git.site.quack-lab.dev/dave/cylogger"
+	lua "github.com/yuin/gopher-lua"
+)
+
+// jsonLogger is a scoped logger for the processor/json package.
+var jsonLogger = logger.Default.WithPrefix("processor/json")
+
+// ProcessJSON applies Lua processing to JSON content
+func ProcessJSON(content string, command utils.ModifyCommand, filename string) ([]utils.ReplaceCommand, error) {
+	processJsonLogger := jsonLogger.WithPrefix("ProcessJSON").WithField("commandName", command.Name).WithField("file", filename)
+	processJsonLogger.Debug("Starting JSON processing for file")
+	processJsonLogger.Trace("Initial file content length: %d", len(content))
+
+	var commands []utils.ReplaceCommand
+	startTime := time.Now()
+
+	// Parse JSON content
+	var jsonData interface{}
+	err := json.Unmarshal([]byte(content), &jsonData)
+	if err != nil {
+		processJsonLogger.Error("Failed to parse JSON content: %v", err)
+		return commands, fmt.Errorf("failed to parse JSON: %v", err)
+	}
+	processJsonLogger.Debug("Successfully parsed JSON content")
+
+	// Create Lua state
+	L, err := NewLuaState()
+	if err != nil {
+		processJsonLogger.Error("Error creating Lua state: %v", err)
+		return commands, fmt.Errorf("error creating Lua state: %v", err)
+	}
+	defer L.Close()
+
+	// Set filename global
+	L.SetGlobal("file", lua.LString(filename))
+
+	// Convert JSON data to Lua table
+	luaTable, err := ToLuaTable(L, jsonData)
+	if err != nil {
+		processJsonLogger.Error("Failed to convert JSON to Lua table: %v", err)
+		return commands, fmt.Errorf("failed to convert JSON to Lua table: %v", err)
+	}
+
+	// Set the JSON data as a global variable
+	L.SetGlobal("data", luaTable)
+	processJsonLogger.Debug("Set JSON data as Lua global 'data'")
+
+	// Build and execute Lua script for JSON mode
+	luaExpr := BuildJSONLuaScript(command.Lua)
+	processJsonLogger.Debug("Built Lua script from expression: %q", command.Lua)
+	processJsonLogger.Trace("Full Lua script: %q", utils.LimitString(luaExpr, 200))
+
+	if err := L.DoString(luaExpr); err != nil {
+		processJsonLogger.Error("Lua script execution failed: %v\nScript: %s", err, utils.LimitString(luaExpr, 200))
+		return commands, fmt.Errorf("lua script execution failed: %v", err)
+	}
+	processJsonLogger.Debug("Lua script executed successfully")
+
+	// Check if modification flag is set
+	modifiedVal := L.GetGlobal("modified")
+	if modifiedVal.Type() != lua.LTBool || !lua.LVAsBool(modifiedVal) {
+		processJsonLogger.Debug("Skipping - no modifications indicated by Lua script")
+		return commands, nil
+	}
+
+	// Get the modified data from Lua
+	modifiedData := L.GetGlobal("data")
+	if modifiedData.Type() != lua.LTTable {
+		processJsonLogger.Error("Expected 'data' to be a table after Lua processing, got %s", modifiedData.Type().String())
+		return commands, fmt.Errorf("expected 'data' to be a table after Lua processing")
+	}
+
+	// Convert back to Go interface
+	goData, err := FromLua(L, modifiedData)
+	if err != nil {
+		processJsonLogger.Error("Failed to convert Lua table back to Go: %v", err)
+		return commands, fmt.Errorf("failed to convert Lua table back to Go: %v", err)
+	}
+
+	// Marshal back to JSON
+	modifiedJSON, err := json.MarshalIndent(goData, "", "  ")
+	if err != nil {
+		processJsonLogger.Error("Failed to marshal modified data to JSON: %v", err)
+		return commands, fmt.Errorf("failed to marshal modified data to JSON: %v", err)
+	}
+
+	// Create replacement command for the entire file
+	// For JSON mode, we always replace the entire content
+	commands = append(commands, utils.ReplaceCommand{
+		From: 0,
+		To:   len(content),
+		With: string(modifiedJSON),
+	})
+
+	processJsonLogger.Debug("Total JSON processing time: %v", time.Since(startTime))
+	processJsonLogger.Debug("Generated %d total modifications", len(commands))
+	return commands, nil
+}
+
+// ToLuaTable converts a Go interface{} to a Lua table recursively
+func ToLuaTable(L *lua.LState, data interface{}) (*lua.LTable, error) {
+	toLuaTableLogger := jsonLogger.WithPrefix("ToLuaTable")
+	toLuaTableLogger.Debug("Converting Go interface to Lua table")
+	toLuaTableLogger.Trace("Input data type: %T", data)
+
+	switch v := data.(type) {
+	case map[string]interface{}:
+		toLuaTableLogger.Debug("Converting map to Lua table")
+		table := L.CreateTable(0, len(v))
+		for key, value := range v {
+			luaValue, err := ToLuaValue(L, value)
+			if err != nil {
+				toLuaTableLogger.Error("Failed to convert map value for key %q: %v", key, err)
+				return nil, err
+			}
+			table.RawSetString(key, luaValue)
+		}
+		return table, nil
+
+	case []interface{}:
+		toLuaTableLogger.Debug("Converting slice to Lua table")
+		table := L.CreateTable(len(v), 0)
+		for i, value := range v {
+			luaValue, err := ToLuaValue(L, value)
+			if err != nil {
+				toLuaTableLogger.Error("Failed to convert slice value at index %d: %v", i, err)
+				return nil, err
+			}
+			table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed
+		}
+		return table, nil
+
+	case string:
+		toLuaTableLogger.Debug("Converting string to Lua string")
+		return nil, fmt.Errorf("expected table or array, got string")
+
+	case float64:
+		toLuaTableLogger.Debug("Converting float64 to Lua number")
+		return nil, fmt.Errorf("expected table or array, got number")
+
+	case bool:
+		toLuaTableLogger.Debug("Converting bool to Lua boolean")
+		return nil, fmt.Errorf("expected table or array, got boolean")
+
+	case nil:
+		toLuaTableLogger.Debug("Converting nil to Lua nil")
+		return nil, fmt.Errorf("expected table or array, got nil")
+
+	default:
+		toLuaTableLogger.Error("Unsupported type for Lua table conversion: %T", v)
+		return nil, fmt.Errorf("unsupported type for Lua table conversion: %T", v)
+	}
+}
+
+// ToLuaValue converts a Go interface{} to a Lua value
+func ToLuaValue(L *lua.LState, data interface{}) (lua.LValue, error) {
+	toLuaValueLogger := jsonLogger.WithPrefix("ToLuaValue")
+	toLuaValueLogger.Debug("Converting Go interface to Lua value")
+	toLuaValueLogger.Trace("Input data type: %T", data)
+
+	switch v := data.(type) {
+	case map[string]interface{}:
+		toLuaValueLogger.Debug("Converting map to Lua table")
+		table := L.CreateTable(0, len(v))
+		for key, value := range v {
+			luaValue, err := ToLuaValue(L, value)
+			if err != nil {
+				toLuaValueLogger.Error("Failed to convert map value for key %q: %v", key, err)
+				return lua.LNil, err
+			}
+			table.RawSetString(key, luaValue)
+		}
+		return table, nil
+
+	case []interface{}:
+		toLuaValueLogger.Debug("Converting slice to Lua table")
+		table := L.CreateTable(len(v), 0)
+		for i, value := range v {
+			luaValue, err := ToLuaValue(L, value)
+			if err != nil {
+				toLuaValueLogger.Error("Failed to convert slice value at index %d: %v", i, err)
+				return lua.LNil, err
+			}
+			table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed
+		}
+		return table, nil
+
+	case string:
+		toLuaValueLogger.Debug("Converting string to Lua string")
+		return lua.LString(v), nil
+
+	case float64:
+		toLuaValueLogger.Debug("Converting float64 to Lua number")
+		return lua.LNumber(v), nil
+
+	case bool:
+		toLuaValueLogger.Debug("Converting bool to Lua boolean")
+		return lua.LBool(v), nil
+
+	case nil:
+		toLuaValueLogger.Debug("Converting nil to Lua nil")
+		return lua.LNil, nil
+
+	default:
+		toLuaValueLogger.Error("Unsupported type for Lua value conversion: %T", v)
+		return lua.LNil, fmt.Errorf("unsupported type for Lua value conversion: %T", v)
+	}
+}

processor/json_test.go

@@ -0,0 +1,117 @@
+package processor
+
+import (
+	"cook/utils"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestProcessJSON(t *testing.T) {
+	tests := []struct {
+		name           string
+		input          string
+		luaExpression  string
+		expectedOutput string
+		expectedMods   int
+	}{
+		{
+			name:          "Basic JSON object modification",
+			input:         `{"name": "test", "value": 42}`,
+			luaExpression: `data.value = data.value * 2; return true`,
+			expectedOutput: `{
+  "name": "test",
+  "value": 84
+}`,
+			expectedMods: 1,
+		},
+		{
+			name:          "JSON array modification",
+			input:         `{"items": [{"id": 1, "value": 10}, {"id": 2, "value": 20}]}`,
+			luaExpression: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 1.5 end; return true`,
+			expectedOutput: `{
+  "items": [
+    {
+      "id": 1,
+      "value": 15
+    },
+    {
+      "id": 2,
+      "value": 30
+    }
+  ]
+}`,
+			expectedMods: 1,
+		},
+		{
+			name:          "JSON nested object modification",
+			input:         `{"config": {"settings": {"enabled": false, "timeout": 30}}}`,
+			luaExpression: `data.config.settings.enabled = true; data.config.settings.timeout = 60; return true`,
+			expectedOutput: `{
+  "config": {
+    "settings": {
+      "enabled": true,
+      "timeout": 60
+    }
+  }
+}`,
+			expectedMods: 1,
+		},
+		{
+			name:           "JSON no modification",
+			input:          `{"name": "test", "value": 42}`,
+			luaExpression:  `return false`,
+			expectedOutput: `{"name": "test", "value": 42}`,
+			expectedMods:   0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			command := utils.ModifyCommand{
+				Name: tt.name,
+				JSON: true,
+				Lua:  tt.luaExpression,
+			}
+
+			modifications, err := ProcessJSON(tt.input, command, "test.json")
+			assert.NoError(t, err, "ProcessJSON failed: %v", err)
+
+			if len(modifications) > 0 {
+				// Execute modifications
+				result, count := utils.ExecuteModifications(modifications, tt.input)
+				assert.Equal(t, tt.expectedMods, count, "Expected %d modifications, got %d", tt.expectedMods, count)
+				assert.Equal(t, tt.expectedOutput, result, "Expected output: %s, got: %s", tt.expectedOutput, result)
+			} else {
+				assert.Equal(t, 0, tt.expectedMods, "Expected no modifications but got some")
+			}
+		})
+	}
+}
+
+func TestToLuaValue(t *testing.T) {
+	L, err := NewLuaState()
+	if err != nil {
+		t.Fatalf("Failed to create Lua state: %v", err)
+	}
+	defer L.Close()
+
+	tests := []struct {
+		name     string
+		input    interface{}
+		expected string
+	}{
+		{"string", "hello", "hello"},
+		{"number", 42.0, "42"},
+		{"boolean", true, "true"},
+		{"nil", nil, "nil"},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			result, err := ToLuaValue(L, tt.input)
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expected, result.String())
+		})
+	}
+}

processor/processor.go

@@ -300,6 +300,26 @@ func BuildLuaScript(luaExpr string) string {
 	return fullScript
 }
 
+// BuildJSONLuaScript prepares a Lua expression for JSON mode
+func BuildJSONLuaScript(luaExpr string) string {
+	buildJsonLuaScriptLogger := processorLogger.WithPrefix("BuildJSONLuaScript").WithField("inputLuaExpr", luaExpr)
+	buildJsonLuaScriptLogger.Debug("Building full Lua script for JSON mode from expression")
+
+	// Perform $var substitutions from globalVariables
+	luaExpr = replaceVariables(luaExpr)
+
+	fullScript := fmt.Sprintf(`
+		function run()
+			%s
+		end
+		local res = run()
+		modified = res == nil or res
+	`, luaExpr)
+	buildJsonLuaScriptLogger.Trace("Generated full JSON Lua script: %q", utils.LimitString(fullScript, 200))
+
+	return fullScript
+}
+
 func replaceVariables(expr string) string {
 	// $varName -> literal value
 	varNameRe := regexp.MustCompile(`\$(\w+)`)

utils/db.go

@@ -7,6 +7,7 @@ import (
 	logger "git.site.quack-lab.dev/dave/cylogger"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
+	gormlogger "gorm.io/gorm/logger"
 )
 
 // dbLogger is a scoped logger for the utils/db package.
@@ -43,7 +44,7 @@ func GetDB() (DB, error) {
 	db, err := gorm.Open(sqlite.Open(dbFile), &gorm.Config{
 		// SkipDefaultTransaction: true,
 		PrepareStmt: true,
-		// Logger:      gormlogger.Default.LogMode(gormlogger.Silent),
+		Logger:      gormlogger.Default.LogMode(gormlogger.Silent),
 	})
 	if err != nil {
 		getDBLogger.Error("Failed to open database: %v", err)
@@ -120,7 +121,8 @@ func (db *DBWrapper) GetFile(filePath string) ([]byte, error) {
 	var fileSnapshot FileSnapshot
 	err := db.db.Model(&FileSnapshot{}).Where("file_path = ?", filePath).First(&fileSnapshot).Error
 	if err != nil {
-		getFileLogger.Error("Failed to get file from database: %v", err)
+		// Downgrade not-found to warning to avoid noisy errors during first run
+		getFileLogger.Warning("Failed to get file from database: %v", err)
 		return nil, err
 	}
 	getFileLogger.Debug("File found in database")

utils/file.go

@@ -102,8 +102,18 @@ func ResetWhereNecessary(associations map[string]FileCommandAssociation, db DB)
 		fileData, err := db.GetFile(file)
 		if err != nil {
 			resetWhereNecessaryLogger.Warning("Failed to get original content for file %q from database: %v", file, err)
+			// Seed the snapshot from current disk content if missing, then use it as fallback
+			currentData, readErr := os.ReadFile(file)
+			if readErr != nil {
+				resetWhereNecessaryLogger.Warning("Additionally failed to read current file content for %q: %v", file, readErr)
 				continue
 			}
+			// Best-effort attempt to save baseline; ignore errors to avoid blocking reset
+			if saveErr := db.SaveFile(file, currentData); saveErr != nil {
+				resetWhereNecessaryLogger.Warning("Failed to seed baseline snapshot for %q: %v", file, saveErr)
+			}
+			fileData = currentData
+		}
 		resetWhereNecessaryLogger.Trace("Retrieved original file data length for %q: %d", file, len(fileData))
 		resetWhereNecessaryLogger.Debug("Writing original content back to file %q", file)
 		err = os.WriteFile(file, fileData, 0644)

utils/flags.go

@@ -12,9 +12,10 @@ var flagsLogger = logger.Default.WithPrefix("utils/flags")
 var (
 	ParallelFiles = flag.Int("P", 100, "Number of files to process in parallel")
 	Filter        = flag.String("f", "", "Filter commands before running them")
+	JSON          = flag.Bool("json", false, "Enable JSON mode for processing JSON files")
 )
 
 func init() {
 	flagsLogger.Debug("Initializing flags")
-	flagsLogger.Trace("ParallelFiles initial value: %d, Filter initial value: %q", *ParallelFiles, *Filter)
+	flagsLogger.Trace("ParallelFiles initial value: %d, Filter initial value: %q, JSON initial value: %t", *ParallelFiles, *Filter, *JSON)
 }

utils/modifycommand.go

@@ -25,6 +25,7 @@ type ModifyCommand struct {
 	Isolate   bool                   `yaml:"isolate,omitempty"`
 	NoDedup   bool                   `yaml:"nodedup,omitempty"`
 	Disabled  bool                   `yaml:"disable,omitempty"`
+	JSON      bool                   `yaml:"json,omitempty"`
 	Modifiers map[string]interface{} `yaml:"modifiers,omitempty"`
 }
 
@@ -33,10 +34,15 @@ type CookFile []ModifyCommand
 func (c *ModifyCommand) Validate() error {
 	validateLogger := modifyCommandLogger.WithPrefix("Validate").WithField("commandName", c.Name)
 	validateLogger.Debug("Validating command")
+
+	// For JSON mode, regex patterns are not required
+	if !c.JSON {
 		if c.Regex == "" && len(c.Regexes) == 0 {
-		validateLogger.Error("Validation failed: Regex pattern is required")
-		return fmt.Errorf("pattern is required")
+			validateLogger.Error("Validation failed: Regex pattern is required for non-JSON mode")
+			return fmt.Errorf("pattern is required for non-JSON mode")
 		}
+	}
+
 	if c.Lua == "" {
 		validateLogger.Error("Validation failed: Lua expression is required")
 		return fmt.Errorf("lua expression is required")