package processor import ( "cook/utils" "encoding/json" "fmt" "sort" "strconv" "strings" "time" logger "git.site.quack-lab.dev/dave/cylogger" "github.com/tidwall/gjson" lua "github.com/yuin/gopher-lua" ) // jsonLogger is a scoped logger for the processor/json package. var jsonLogger = logger.Default.WithPrefix("processor/json") // ProcessJSON applies Lua processing to JSON content func ProcessJSON(content string, command utils.ModifyCommand, filename string) ([]utils.ReplaceCommand, error) { processJsonLogger := jsonLogger.WithPrefix("ProcessJSON").WithField("commandName", command.Name).WithField("file", filename) processJsonLogger.Debug("Starting JSON processing for file") processJsonLogger.Trace("Initial file content length: %d", len(content)) var commands []utils.ReplaceCommand startTime := time.Now() // Parse JSON content var jsonData interface{} err := json.Unmarshal([]byte(content), &jsonData) if err != nil { processJsonLogger.Error("Failed to parse JSON content: %v", err) return commands, fmt.Errorf("failed to parse JSON: %v", err) } processJsonLogger.Debug("Successfully parsed JSON content") // Create Lua state L, err := NewLuaState() if err != nil { processJsonLogger.Error("Error creating Lua state: %v", err) return commands, fmt.Errorf("error creating Lua state: %v", err) } defer L.Close() // Set filename global L.SetGlobal("file", lua.LString(filename)) // Convert JSON data to Lua table luaTable, err := ToLuaTable(L, jsonData) if err != nil { processJsonLogger.Error("Failed to convert JSON to Lua table: %v", err) return commands, fmt.Errorf("failed to convert JSON to Lua table: %v", err) } // Set the JSON data as a global variable L.SetGlobal("data", luaTable) processJsonLogger.Debug("Set JSON data as Lua global 'data'") // Build and execute Lua script for JSON mode luaExpr := BuildJSONLuaScript(command.Lua) processJsonLogger.Debug("Built Lua script from expression: %q", command.Lua) processJsonLogger.Trace("Full Lua script: %q", utils.LimitString(luaExpr, 200)) if err := L.DoString(luaExpr); err != nil { processJsonLogger.Error("Lua script execution failed: %v\nScript: %s", err, utils.LimitString(luaExpr, 200)) return commands, fmt.Errorf("lua script execution failed: %v", err) } processJsonLogger.Debug("Lua script executed successfully") // Check if modification flag is set modifiedVal := L.GetGlobal("modified") if modifiedVal.Type() != lua.LTBool || !lua.LVAsBool(modifiedVal) { processJsonLogger.Debug("Skipping - no modifications indicated by Lua script") return commands, nil } // Get the modified data from Lua modifiedData := L.GetGlobal("data") if modifiedData.Type() != lua.LTTable { processJsonLogger.Error("Expected 'data' to be a table after Lua processing, got %s", modifiedData.Type().String()) return commands, fmt.Errorf("expected 'data' to be a table after Lua processing") } // Convert back to Go interface goData, err := FromLua(L, modifiedData) if err != nil { processJsonLogger.Error("Failed to convert Lua table back to Go: %v", err) return commands, fmt.Errorf("failed to convert Lua table back to Go: %v", err) } commands, err = applyJSONChanges(content, jsonData, goData) if err != nil { processJsonLogger.Error("Failed to apply JSON changes: %v", err) return commands, fmt.Errorf("failed to apply JSON changes: %v", err) } processJsonLogger.Debug("Total JSON processing time: %v", time.Since(startTime)) processJsonLogger.Debug("Generated %d total modifications", len(commands)) return commands, nil } // applyJSONChanges compares original and modified data and applies changes surgically func applyJSONChanges(content string, originalData, modifiedData interface{}) ([]utils.ReplaceCommand, error) { var commands []utils.ReplaceCommand appliedCommands, err := applyChanges(content, originalData, modifiedData) if err == nil && len(appliedCommands) > 0 { return appliedCommands, nil } return commands, fmt.Errorf("failed to make any changes to the json") } // applyChanges attempts to make surgical changes while preserving exact formatting func applyChanges(content string, originalData, modifiedData interface{}) ([]utils.ReplaceCommand, error) { var commands []utils.ReplaceCommand // Find all changes between original and modified data changes := findDeepChanges("", originalData, modifiedData) jsonLogger.Debug("applyChanges: Found %d changes: %v", len(changes), changes) if len(changes) == 0 { return commands, nil } // Sort removal operations by index in descending order to avoid index shifting var removals []string var additions []string var valueChanges []string for path := range changes { if strings.HasSuffix(path, "@remove") { removals = append(removals, path) } else if strings.HasSuffix(path, "@add") { additions = append(additions, path) } else { valueChanges = append(valueChanges, path) } } jsonLogger.Debug("applyChanges: %d removals, %d additions, %d value changes", len(removals), len(additions), len(valueChanges)) // Apply removals first (from end to beginning to avoid index shifting) for _, removalPath := range removals { actualPath := strings.TrimSuffix(removalPath, "@remove") index := extractIndexFromRemovalPath(removalPath) arrayPath := getArrayPathFromElementPath(actualPath) // Get the array element to remove result := gjson.Get(content, actualPath) if !result.Exists() { continue } // Find the exact byte range to remove (including comma/formatting) startPos, endPos := findArrayElementRemovalRange(content, arrayPath, index) if startPos >= 0 && endPos > startPos { commands = append(commands, utils.ReplaceCommand{ From: startPos, To: endPos, With: "", // Remove the element }) } } // Apply additions (new fields) for _, additionPath := range additions { actualPath := strings.TrimSuffix(additionPath, "@add") newValue := changes[additionPath] jsonLogger.Debug("Processing addition: path=%s, value=%v", actualPath, newValue) // Find the parent object to add the field to parentPath := getParentPath(actualPath) fieldName := getFieldName(actualPath) jsonLogger.Debug("Parent path: %s, field name: %s", parentPath, fieldName) // Get the parent object var parentResult gjson.Result if parentPath == "" { // Adding to root object - get the entire JSON parentResult = gjson.Parse(content) } else { parentResult = gjson.Get(content, parentPath) } if !parentResult.Exists() { jsonLogger.Debug("Parent path %s does not exist, skipping", parentPath) continue } // Find where to insert the new field (at the end of the object) startPos := int(parentResult.Index + len(parentResult.Raw) - 1) // Before closing brace jsonLogger.Debug("Inserting at pos %d", startPos) // Convert the new value to JSON string newValueStr := convertValueToJSONString(newValue) // Insert the new field insertText := fmt.Sprintf(`,"%s":%s`, fieldName, newValueStr) jsonLogger.Debug("Inserting text: %q", insertText) commands = append(commands, utils.ReplaceCommand{ From: startPos, To: startPos, With: insertText, }) jsonLogger.Debug("Added addition command: From=%d, To=%d, With=%q", startPos, startPos, insertText) } // Apply value changes (in reverse order to avoid position shifting) sort.Slice(valueChanges, func(i, j int) bool { // Get positions for comparison resultI := gjson.Get(content, valueChanges[i]) resultJ := gjson.Get(content, valueChanges[j]) return resultI.Index > resultJ.Index // Descending order }) for _, path := range valueChanges { newValue := changes[path] jsonLogger.Debug("Processing value change: path=%s, value=%v", path, newValue) // Get the current value and its position in the original JSON result := gjson.Get(content, path) if !result.Exists() { jsonLogger.Debug("Path %s does not exist, skipping", path) continue // Skip if path doesn't exist } // Get the exact byte positions of this value startPos := result.Index endPos := startPos + len(result.Raw) jsonLogger.Debug("Found value at pos %d-%d: %q", startPos, endPos, result.Raw) // Convert the new value to JSON string newValueStr := convertValueToJSONString(newValue) jsonLogger.Debug("Converting to: %q", newValueStr) // Create a replacement command for this specific value commands = append(commands, utils.ReplaceCommand{ From: int(startPos), To: int(endPos), With: newValueStr, }) jsonLogger.Debug("Added command: From=%d, To=%d, With=%q", int(startPos), int(endPos), newValueStr) } return commands, nil } // extractIndexFromRemovalPath extracts the array index from a removal path like "Rows.0.Inputs.1@remove" func extractIndexFromRemovalPath(path string) int { parts := strings.Split(strings.TrimSuffix(path, "@remove"), ".") if len(parts) > 0 { lastPart := parts[len(parts)-1] if index, err := strconv.Atoi(lastPart); err == nil { return index } } return -1 } // getArrayPathFromElementPath converts "Rows.0.Inputs.1" to "Rows.0.Inputs" func getArrayPathFromElementPath(elementPath string) string { parts := strings.Split(elementPath, ".") if len(parts) > 0 { return strings.Join(parts[:len(parts)-1], ".") } return "" } // getParentPath extracts the parent path from a full path like "Rows.0.Inputs.1" func getParentPath(fullPath string) string { parts := strings.Split(fullPath, ".") if len(parts) > 0 { return strings.Join(parts[:len(parts)-1], ".") } return "" } // getFieldName extracts the field name from a full path like "Rows.0.Inputs.1" func getFieldName(fullPath string) string { parts := strings.Split(fullPath, ".") if len(parts) > 0 { return parts[len(parts)-1] } return "" } // convertValueToJSONString converts a Go interface{} to a JSON string representation func convertValueToJSONString(value interface{}) string { switch v := value.(type) { case string: return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` case float64: if v == float64(int64(v)) { return strconv.FormatInt(int64(v), 10) } return strconv.FormatFloat(v, 'f', -1, 64) case bool: return strconv.FormatBool(v) case nil: return "null" default: // For complex types, we need to avoid json.Marshal // This should not happen if we're doing true surgical edits return "" } } // findArrayElementRemovalRange finds the exact byte range to remove for an array element func findArrayElementRemovalRange(content, arrayPath string, elementIndex int) (int, int) { // Get the array using gjson arrayResult := gjson.Get(content, arrayPath) if !arrayResult.Exists() || !arrayResult.IsArray() { return -1, -1 } // Get all array elements elements := arrayResult.Array() if elementIndex >= len(elements) { return -1, -1 } // Get the target element elementResult := elements[elementIndex] startPos := int(elementResult.Index) endPos := int(elementResult.Index + len(elementResult.Raw)) // Handle comma removal properly if elementIndex == 0 && len(elements) > 1 { // First element but not the only one - remove comma after for i := endPos; i < len(content) && i < endPos+50; i++ { if content[i] == ',' { endPos = i + 1 break } } } else if elementIndex == len(elements)-1 && len(elements) > 1 { // Last element and not the only one - remove comma before prevElementEnd := int(elements[elementIndex-1].Index + len(elements[elementIndex-1].Raw)) for i := prevElementEnd; i < startPos && i < len(content); i++ { if content[i] == ',' { startPos = i break } } } // If it's the only element, don't remove any commas return startPos, endPos } // findDeepChanges recursively finds all paths that need to be changed func findDeepChanges(basePath string, original, modified interface{}) map[string]interface{} { changes := make(map[string]interface{}) switch orig := original.(type) { case map[string]interface{}: if mod, ok := modified.(map[string]interface{}); ok { // Check for new keys added in modified data for key, modValue := range mod { var currentPath string if basePath == "" { currentPath = key } else { currentPath = basePath + "." + key } if origValue, exists := orig[key]; exists { // Key exists in both, check if value changed switch modValue.(type) { case map[string]interface{}, []interface{}: // Recursively check nested structures nestedChanges := findDeepChanges(currentPath, origValue, modValue) for nestedPath, nestedValue := range nestedChanges { changes[nestedPath] = nestedValue } default: // Primitive value - check if changed if !deepEqual(origValue, modValue) { changes[currentPath] = modValue } } } else { // New key added - mark for addition changes[currentPath+"@add"] = modValue } } } case []interface{}: if mod, ok := modified.([]interface{}); ok { // Handle array changes by detecting specific element operations if len(orig) != len(mod) { // Array length changed - detect if it's element removal if len(orig) > len(mod) { // Element(s) removed - find which ones by comparing content removedIndices := findRemovedArrayElements(orig, mod) for _, removedIndex := range removedIndices { var currentPath string if basePath == "" { currentPath = fmt.Sprintf("%d@remove", removedIndex) } else { currentPath = fmt.Sprintf("%s.%d@remove", basePath, removedIndex) } changes[currentPath] = nil // Mark for removal } } else { // Elements added - more complex, skip for now } } else { // Same length - check individual elements for value changes for i, modValue := range mod { var currentPath string if basePath == "" { currentPath = strconv.Itoa(i) } else { currentPath = basePath + "." + strconv.Itoa(i) } if i < len(orig) { // Index exists in both, check if value changed switch modValue.(type) { case map[string]interface{}, []interface{}: // Recursively check nested structures nestedChanges := findDeepChanges(currentPath, orig[i], modValue) for nestedPath, nestedValue := range nestedChanges { changes[nestedPath] = nestedValue } default: // Primitive value - check if changed if !deepEqual(orig[i], modValue) { changes[currentPath] = modValue } } } } } } default: // For primitive types, compare directly if !deepEqual(original, modified) { if basePath == "" { changes[""] = modified } else { changes[basePath] = modified } } } return changes } // findRemovedArrayElements compares two arrays and returns indices of removed elements func findRemovedArrayElements(original, modified []interface{}) []int { var removedIndices []int // Simple approach: find elements in original that don't exist in modified for i, origElement := range original { found := false for _, modElement := range modified { if deepEqual(origElement, modElement) { found = true break } } if !found { removedIndices = append(removedIndices, i) } } return removedIndices } // deepEqual performs deep comparison of two values func deepEqual(a, b interface{}) bool { if a == nil && b == nil { return true } if a == nil || b == nil { return false } switch av := a.(type) { case map[string]interface{}: if bv, ok := b.(map[string]interface{}); ok { if len(av) != len(bv) { return false } for k, v := range av { if !deepEqual(v, bv[k]) { return false } } return true } return false case []interface{}: if bv, ok := b.([]interface{}); ok { if len(av) != len(bv) { return false } for i, v := range av { if !deepEqual(v, bv[i]) { return false } } return true } return false default: return a == b } } // ToLuaTable converts a Go interface{} to a Lua table recursively func ToLuaTable(L *lua.LState, data interface{}) (*lua.LTable, error) { toLuaTableLogger := jsonLogger.WithPrefix("ToLuaTable") toLuaTableLogger.Debug("Converting Go interface to Lua table") toLuaTableLogger.Trace("Input data type: %T", data) switch v := data.(type) { case map[string]interface{}: toLuaTableLogger.Debug("Converting map to Lua table") table := L.CreateTable(0, len(v)) for key, value := range v { luaValue, err := ToLuaValue(L, value) if err != nil { toLuaTableLogger.Error("Failed to convert map value for key %q: %v", key, err) return nil, err } table.RawSetString(key, luaValue) } return table, nil case []interface{}: toLuaTableLogger.Debug("Converting slice to Lua table") table := L.CreateTable(len(v), 0) for i, value := range v { luaValue, err := ToLuaValue(L, value) if err != nil { toLuaTableLogger.Error("Failed to convert slice value at index %d: %v", i, err) return nil, err } table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed } return table, nil case string: toLuaTableLogger.Debug("Converting string to Lua string") return nil, fmt.Errorf("expected table or array, got string") case float64: toLuaTableLogger.Debug("Converting float64 to Lua number") return nil, fmt.Errorf("expected table or array, got number") case bool: toLuaTableLogger.Debug("Converting bool to Lua boolean") return nil, fmt.Errorf("expected table or array, got boolean") case nil: toLuaTableLogger.Debug("Converting nil to Lua nil") return nil, fmt.Errorf("expected table or array, got nil") default: toLuaTableLogger.Error("Unsupported type for Lua table conversion: %T", v) return nil, fmt.Errorf("unsupported type for Lua table conversion: %T", v) } } // ToLuaValue converts a Go interface{} to a Lua value func ToLuaValue(L *lua.LState, data interface{}) (lua.LValue, error) { toLuaValueLogger := jsonLogger.WithPrefix("ToLuaValue") toLuaValueLogger.Debug("Converting Go interface to Lua value") toLuaValueLogger.Trace("Input data type: %T", data) switch v := data.(type) { case map[string]interface{}: toLuaValueLogger.Debug("Converting map to Lua table") table := L.CreateTable(0, len(v)) for key, value := range v { luaValue, err := ToLuaValue(L, value) if err != nil { toLuaValueLogger.Error("Failed to convert map value for key %q: %v", key, err) return lua.LNil, err } table.RawSetString(key, luaValue) } return table, nil case []interface{}: toLuaValueLogger.Debug("Converting slice to Lua table") table := L.CreateTable(len(v), 0) for i, value := range v { luaValue, err := ToLuaValue(L, value) if err != nil { toLuaValueLogger.Error("Failed to convert slice value at index %d: %v", i, err) return lua.LNil, err } table.RawSetInt(i+1, luaValue) // Lua arrays are 1-indexed } return table, nil case string: toLuaValueLogger.Debug("Converting string to Lua string") return lua.LString(v), nil case float64: toLuaValueLogger.Debug("Converting float64 to Lua number") return lua.LNumber(v), nil case bool: toLuaValueLogger.Debug("Converting bool to Lua boolean") return lua.LBool(v), nil case nil: toLuaValueLogger.Debug("Converting nil to Lua nil") return lua.LNil, nil default: toLuaValueLogger.Error("Unsupported type for Lua value conversion: %T", v) return lua.LNil, fmt.Errorf("unsupported type for Lua value conversion: %T", v) } }