// Package processor provides JSON processing and Lua script execution capabilities // for data transformation and manipulation. package processor import ( "cook/utils" "encoding/json" "fmt" "sort" "strconv" "strings" "time" logger "git.site.quack-lab.dev/dave/cylogger" "github.com/tidwall/gjson" lua "github.com/yuin/gopher-lua" ) // jsonLogger is a scoped logger for the processor/json package. var jsonLogger = logger.Default.WithPrefix("processor/json") // ProcessJSON applies Lua processing to JSON content func ProcessJSON(content string, command utils.ModifyCommand, filename string) ([]utils.ReplaceCommand, error) { processJSONLogger := jsonLogger.WithPrefix("ProcessJSON").WithField("commandName", command.Name).WithField("file", filename) processJSONLogger.Debug("Starting JSON processing for file") processJSONLogger.Trace("Initial file content length: %d", len(content)) var commands []utils.ReplaceCommand startTime := time.Now() // Parse JSON content var jsonData interface{} err := json.Unmarshal([]byte(content), &jsonData) if err != nil { processJSONLogger.Error("Failed to parse JSON content: %v", err) return commands, fmt.Errorf("failed to parse JSON: %v", err) } processJSONLogger.Debug("Successfully parsed JSON content") // Create Lua state L, err := NewLuaState() if err != nil { processJSONLogger.Error("Error creating Lua state: %v", err) return commands, fmt.Errorf("error creating Lua state: %v", err) } defer L.Close() // Set filename global L.SetGlobal("file", lua.LString(filename)) // Convert JSON data to Lua table luaTable, err := ToLuaTable(L, jsonData) if err != nil { processJSONLogger.Error("Failed to convert JSON to Lua table: %v", err) return commands, fmt.Errorf("failed to convert JSON to Lua table: %v", err) } // Set the JSON data as a global variable L.SetGlobal("data", luaTable) processJSONLogger.Debug("Set JSON data as Lua global 'data'") // Build and execute Lua script for JSON mode luaExpr := BuildJSONLuaScript(command.Lua, command.SourceDir) processJSONLogger.Debug("Built Lua script from expression: %q", command.Lua) processJSONLogger.Trace("Full Lua script: %q", utils.LimitString(luaExpr, 200)) if err := L.DoString(luaExpr); err != nil { processJSONLogger.Error("Lua script execution failed: %v\nScript: %s", err, utils.LimitString(luaExpr, 200)) return commands, fmt.Errorf("lua script execution failed: %v", err) } processJSONLogger.Debug("Lua script executed successfully") // Check if modification flag is set modifiedVal := L.GetGlobal("modified") if modifiedVal.Type() != lua.LTBool || !lua.LVAsBool(modifiedVal) { processJSONLogger.Debug("Skipping - no modifications indicated by Lua script") return commands, nil } // Get the modified data from Lua modifiedData := L.GetGlobal("data") if modifiedData.Type() != lua.LTTable { processJSONLogger.Error("Expected 'data' to be a table after Lua processing, got %s", modifiedData.Type().String()) return commands, fmt.Errorf("expected 'data' to be a table after Lua processing") } // Convert back to Go interface goData, err := FromLua(L, modifiedData) if err != nil { processJSONLogger.Error("Failed to convert Lua table back to Go: %v", err) return commands, fmt.Errorf("failed to convert Lua table back to Go: %v", err) } processJSONLogger.Debug("About to call applyChanges with original data and modified data") commands, err = applyChanges(content, jsonData, goData) if err != nil { processJSONLogger.Error("Failed to apply surgical JSON changes: %v", err) return commands, fmt.Errorf("failed to apply surgical JSON changes: %v", err) } processJSONLogger.Debug("Total JSON processing time: %v", time.Since(startTime)) processJSONLogger.Debug("Generated %d total modifications", len(commands)) return commands, nil } // applyChanges attempts to make surgical changes while preserving exact formatting func applyChanges(content string, originalData, modifiedData interface{}) ([]utils.ReplaceCommand, error) { var commands []utils.ReplaceCommand // Find all changes between original and modified data changes := findDeepChanges("", originalData, modifiedData) jsonLogger.Debug("applyChanges: Found %d changes: %v", len(changes), changes) if len(changes) == 0 { return commands, nil } // Sort removal operations by index in descending order to avoid index shifting var removals []string var additions []string var valueChanges []string for path := range changes { if strings.HasSuffix(path, "@remove") { removals = append(removals, path) } else if strings.HasSuffix(path, "@add") { additions = append(additions, path) } else { valueChanges = append(valueChanges, path) } } jsonLogger.Debug("applyChanges: %d removals, %d additions, %d value changes", len(removals), len(additions), len(valueChanges)) // Apply removals first (from end to beginning to avoid index shifting) for _, removalPath := range removals { actualPath := strings.TrimSuffix(removalPath, "@remove") elementIndex := extractIndexFromRemovalPath(actualPath) arrayPath := getArrayPathFromElementPath(actualPath) jsonLogger.Debug("Processing removal: path=%s, index=%d, arrayPath=%s", actualPath, elementIndex, arrayPath) // Find the exact byte range to remove from, to := findArrayElementRemovalRange(content, arrayPath, elementIndex) jsonLogger.Debug("Removing bytes %d-%d", from, to) commands = append(commands, utils.ReplaceCommand{ From: from, To: to, With: "", }) jsonLogger.Debug("Added removal command: From=%d, To=%d, With=\"\"", from, to) } // Apply additions (new fields) for _, additionPath := range additions { actualPath := strings.TrimSuffix(additionPath, "@add") newValue := changes[additionPath] jsonLogger.Debug("Processing addition: path=%s, value=%v", actualPath, newValue) // Find the parent object to add the field to parentPath := getParentPath(actualPath) fieldName := getFieldName(actualPath) jsonLogger.Debug("Parent path: %s, field name: %s", parentPath, fieldName) // Get the parent object var parentResult gjson.Result if parentPath == "" { // Adding to root object - get the entire JSON parentResult = gjson.Parse(content) } else { parentResult = gjson.Get(content, parentPath) } if !parentResult.Exists() { jsonLogger.Debug("Parent path %s does not exist, skipping", parentPath) continue } // Find where to insert the new field (at the end of the object) startPos := int(parentResult.Index + len(parentResult.Raw) - 1) // Before closing brace jsonLogger.Debug("Inserting at pos %d", startPos) // Convert the new value to JSON string newValueStr := convertValueToJSONString(newValue) // Insert the new field with pretty-printed formatting // Format: ,"fieldName": { ... } insertText := fmt.Sprintf(`,"%s": %s`, fieldName, newValueStr) commands = append(commands, utils.ReplaceCommand{ From: startPos, To: startPos, With: insertText, }) jsonLogger.Debug("Added addition command: From=%d, To=%d, With=%q", startPos, startPos, insertText) } // Apply value changes (in reverse order to avoid position shifting) sort.Slice(valueChanges, func(i, j int) bool { // Get positions for comparison resultI := gjson.Get(content, valueChanges[i]) resultJ := gjson.Get(content, valueChanges[j]) return resultI.Index > resultJ.Index // Descending order }) for _, path := range valueChanges { newValue := changes[path] jsonLogger.Debug("Processing value change: path=%s, value=%v", path, newValue) // Get the current value and its position in the original JSON result := gjson.Get(content, path) if !result.Exists() { jsonLogger.Debug("Path %s does not exist, skipping", path) continue // Skip if path doesn't exist } // Get the exact byte positions of this value startPos := result.Index endPos := startPos + len(result.Raw) jsonLogger.Debug("Found value at pos %d-%d: %q", startPos, endPos, result.Raw) // Convert the new value to JSON string newValueStr := convertValueToJSONString(newValue) jsonLogger.Debug("Converting to: %q", newValueStr) // Create a replacement command for this specific value commands = append(commands, utils.ReplaceCommand{ From: int(startPos), To: int(endPos), With: newValueStr, }) jsonLogger.Debug("Added command: From=%d, To=%d, With=%q", int(startPos), int(endPos), newValueStr) } return commands, nil } // extractIndexFromRemovalPath extracts the array index from a removal path like "Rows.0.Inputs.1@remove" func extractIndexFromRemovalPath(path string) int { parts := strings.Split(strings.TrimSuffix(path, "@remove"), ".") if len(parts) > 0 { lastPart := parts[len(parts)-1] if index, err := strconv.Atoi(lastPart); err == nil { return index } } return -1 } // getArrayPathFromElementPath converts "Rows.0.Inputs.1" to "Rows.0.Inputs" func getArrayPathFromElementPath(elementPath string) string { parts := strings.Split(elementPath, ".") if len(parts) > 0 { return strings.Join(parts[:len(parts)-1], ".") } return "" } // getParentPath extracts the parent path from a full path like "Rows.0.Inputs.1" func getParentPath(fullPath string) string { parts := strings.Split(fullPath, ".") if len(parts) > 0 { return strings.Join(parts[:len(parts)-1], ".") } return "" } // getFieldName extracts the field name from a full path like "Rows.0.Inputs.1" func getFieldName(fullPath string) string { parts := strings.Split(fullPath, ".") if len(parts) > 0 { return parts[len(parts)-1] } return "" } // convertValueToJSONString converts a Go interface{} to a JSON string representation func convertValueToJSONString(value interface{}) string { switch v := value.(type) { case string: return `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` case float64: if v == float64(int64(v)) { return strconv.FormatInt(int64(v), 10) } return strconv.FormatFloat(v, 'f', -1, 64) case bool: return strconv.FormatBool(v) case nil: return "null" case map[string]interface{}: // Handle maps specially to avoid double-escaping of keys var pairs []string for key, val := range v { // The key might already have escaped quotes from Lua, so we need to be careful // If the key already contains escaped quotes, we need to unescape them first keyStr := key if strings.Contains(key, `\"`) { // Key already has escaped quotes, use it as-is keyStr = `"` + key + `"` } else { // Normal key, escape quotes keyStr = `"` + strings.ReplaceAll(key, `"`, `\"`) + `"` } valStr := convertValueToJSONString(val) pairs = append(pairs, keyStr+":"+valStr) } return "{" + strings.Join(pairs, ",") + "}" default: // For other complex types (arrays), we need to use json.Marshal jsonBytes, err := json.Marshal(v) if err != nil { return "null" // Fallback to null if marshaling fails } return string(jsonBytes) } } // findArrayElementRemovalRange finds the exact byte range to remove for an array element func findArrayElementRemovalRange(content, arrayPath string, elementIndex int) (int, int) { // Get the array using gjson var arrayResult gjson.Result if arrayPath == "" { // Root-level array arrayResult = gjson.Parse(content) } else { arrayResult = gjson.Get(content, arrayPath) } if !arrayResult.Exists() || !arrayResult.IsArray() { return -1, -1 } // Get all array elements elements := arrayResult.Array() if elementIndex >= len(elements) { return -1, -1 } // Get the target element elementResult := elements[elementIndex] startPos := int(elementResult.Index) endPos := int(elementResult.Index + len(elementResult.Raw)) // Handle comma removal properly if elementIndex == 0 && len(elements) > 1 { // First element but not the only one - remove comma after for i := endPos; i < len(content) && i < endPos+50; i++ { if content[i] == ',' { endPos = i + 1 break } } } else if elementIndex == len(elements)-1 && len(elements) > 1 { // Last element and not the only one - remove comma before prevElementEnd := int(elements[elementIndex-1].Index + len(elements[elementIndex-1].Raw)) for i := prevElementEnd; i < startPos && i < len(content); i++ { if content[i] == ',' { startPos = i break } } } // If it's the only element, don't remove any commas return startPos, endPos } // findDeepChanges recursively finds all paths that need to be changed func findDeepChanges(basePath string, original, modified interface{}) map[string]interface{} { changes := make(map[string]interface{}) switch orig := original.(type) { case map[string]interface{}: if mod, ok := modified.(map[string]interface{}); ok { // Check for new keys added in modified data for key, modValue := range mod { var currentPath string if basePath == "" { currentPath = key } else { currentPath = basePath + "." + key } if origValue, exists := orig[key]; exists { // Key exists in both, check if value changed switch modValue.(type) { case map[string]interface{}, []interface{}: // Recursively check nested structures nestedChanges := findDeepChanges(currentPath, origValue, modValue) for nestedPath, nestedValue := range nestedChanges { changes[nestedPath] = nestedValue } default: // Primitive value - check if changed if !deepEqual(origValue, modValue) { changes[currentPath] = modValue } } } else { // New key added - mark for addition changes[currentPath+"@add"] = modValue } } } case []interface{}: if mod, ok := modified.([]interface{}); ok { // Handle array changes by detecting specific element operations if len(orig) != len(mod) { // Array length changed - detect if it's element removal if len(orig) > len(mod) { // Element(s) removed - find which ones by comparing content removedIndices := findRemovedArrayElements(orig, mod) for _, removedIndex := range removedIndices { var currentPath string if basePath == "" { currentPath = fmt.Sprintf("%d@remove", removedIndex) } else { currentPath = fmt.Sprintf("%s.%d@remove", basePath, removedIndex) } changes[currentPath] = nil // Mark for removal } } } else { // Same length - check individual elements for value changes for i, modValue := range mod { var currentPath string if basePath == "" { currentPath = strconv.Itoa(i) } else { currentPath = basePath + "." + strconv.Itoa(i) } if i < len(orig) { // Index exists in both, check if value changed switch modValue.(type) { case map[string]interface{}, []interface{}: // Recursively check nested structures nestedChanges := findDeepChanges(currentPath, orig[i], modValue) for nestedPath, nestedValue := range nestedChanges { changes[nestedPath] = nestedValue } default: // Primitive value - check if changed if !deepEqual(orig[i], modValue) { changes[currentPath] = modValue } } } } } } } // Note: No default case needed - JSON data from unmarshaling is always // map[string]interface{} or []interface{} at the top level return changes } // findRemovedArrayElements compares two arrays and returns indices of removed elements func findRemovedArrayElements(original, modified []interface{}) []int { var removedIndices []int // Simple approach: find elements in original that don't exist in modified for i, origElement := range original { found := false for _, modElement := range modified { if deepEqual(origElement, modElement) { found = true break } } if !found { removedIndices = append(removedIndices, i) } } return removedIndices } // deepEqual performs deep comparison of two values func deepEqual(a, b interface{}) bool { if a == nil && b == nil { return true } if a == nil || b == nil { return false } switch av := a.(type) { case map[string]interface{}: if bv, ok := b.(map[string]interface{}); ok { if len(av) != len(bv) { return false } for k, v := range av { if !deepEqual(v, bv[k]) { return false } } return true } return false case []interface{}: if bv, ok := b.([]interface{}); ok { if len(av) != len(bv) { return false } for i, v := range av { if !deepEqual(v, bv[i]) { return false } } return true } return false default: return a == b } } // ToLuaTable converts a Go interface{} (map or array) to a Lua table // This should only be called with map[string]interface{} or []interface{} from JSON unmarshaling func ToLuaTable(L *lua.LState, data interface{}) (*lua.LTable, error) { switch v := data.(type) { case map[string]interface{}: table := L.CreateTable(0, len(v)) for key, value := range v { table.RawSetString(key, ToLuaValue(L, value)) } return table, nil case []interface{}: table := L.CreateTable(len(v), 0) for i, value := range v { table.RawSetInt(i+1, ToLuaValue(L, value)) // Lua arrays are 1-indexed } return table, nil default: // This should only happen with invalid JSON (root-level primitives) return nil, fmt.Errorf("expected table or array, got %T", v) } } // ToLuaValue converts a Go interface{} to a Lua value func ToLuaValue(L *lua.LState, data interface{}) lua.LValue { switch v := data.(type) { case map[string]interface{}: table := L.CreateTable(0, len(v)) for key, value := range v { table.RawSetString(key, ToLuaValue(L, value)) } return table case []interface{}: table := L.CreateTable(len(v), 0) for i, value := range v { table.RawSetInt(i+1, ToLuaValue(L, value)) // Lua arrays are 1-indexed } return table case string: return lua.LString(v) case float64: return lua.LNumber(v) case bool: return lua.LBool(v) case nil: return lua.LNil default: // This should never happen with JSON-unmarshaled data return lua.LNil } }