From 3d01822e77fcf045ecc1aa3ae7a935315435d267 Mon Sep 17 00:00:00 2001 From: PhatPhuckDave Date: Thu, 21 Aug 2025 23:03:56 +0200 Subject: [PATCH] Fix failing test --- processor/json.go | 308 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 238 insertions(+), 70 deletions(-) diff --git a/processor/json.go b/processor/json.go index b205e90..b3b37dc 100644 --- a/processor/json.go +++ b/processor/json.go @@ -4,10 +4,13 @@ import ( "cook/utils" "encoding/json" "fmt" + "sort" + "strconv" + "strings" "time" logger "git.site.quack-lab.dev/dave/cylogger" - "github.com/tidwall/sjson" + "github.com/tidwall/gjson" lua "github.com/yuin/gopher-lua" ) @@ -101,85 +104,212 @@ func ProcessJSON(content string, command utils.ModifyCommand, filename string) ( // applySurgicalJSONChanges compares original and modified data and applies changes surgically func applySurgicalJSONChanges(content string, originalData, modifiedData interface{}) ([]utils.ReplaceCommand, error) { var commands []utils.ReplaceCommand - + // Convert both to JSON for comparison originalJSON, err := json.Marshal(originalData) if err != nil { return commands, fmt.Errorf("failed to marshal original data: %v", err) } - + modifiedJSON, err := json.Marshal(modifiedData) if err != nil { return commands, fmt.Errorf("failed to marshal modified data: %v", err) } - + // If no changes, return empty commands if string(originalJSON) == string(modifiedJSON) { return commands, nil } - - // Try true surgical approach that preserves formatting + + // ALWAYS try surgical approach first - never fall back to full replacement + // This ensures field order is always preserved surgicalCommands, err := applyTrueSurgicalChanges(content, originalData, modifiedData) if err == nil && len(surgicalCommands) > 0 { return surgicalCommands, nil } - - // Fall back to full replacement with proper formatting - modifiedJSONIndented, err := json.MarshalIndent(modifiedData, "", " ") - if err != nil { - return commands, fmt.Errorf("failed to marshal modified data with indentation: %v", err) - } - - commands = append(commands, utils.ReplaceCommand{ - From: 0, - To: len(content), - With: string(modifiedJSONIndented), - }) - - return commands, nil + + // If surgical approach fails or finds no changes, something is wrong + // This should not happen if there are actual changes + return commands, fmt.Errorf("surgical approach failed but changes detected") } // applyTrueSurgicalChanges attempts to make surgical changes while preserving exact formatting func applyTrueSurgicalChanges(content string, originalData, modifiedData interface{}) ([]utils.ReplaceCommand, error) { var commands []utils.ReplaceCommand - - // Find changes by comparing the data structures + + // Find all changes between original and modified data changes := findDeepChanges("", originalData, modifiedData) - + if len(changes) == 0 { return commands, nil } - - // Apply changes surgically using sjson.Set() to preserve formatting - modifiedContent := content - for path, newValue := range changes { - var err error - modifiedContent, err = sjson.Set(modifiedContent, path, newValue) - if err != nil { - return nil, fmt.Errorf("failed to apply surgical change at path %s: %v", path, err) + + // Sort removal operations by index in descending order to avoid index shifting + var removals []string + var valueChanges []string + + for path := range changes { + if strings.HasSuffix(path, "@remove") { + removals = append(removals, path) + } else { + valueChanges = append(valueChanges, path) } } - - // If we successfully made changes, create a replacement command - if modifiedContent != content { + + // Sort removals by index (descending) to process from end to beginning + sort.Slice(removals, func(i, j int) bool { + // Extract index from path like "Rows.0.Inputs.1@remove" + indexI := extractIndexFromRemovalPath(removals[i]) + indexJ := extractIndexFromRemovalPath(removals[j]) + return indexI > indexJ // Descending order + }) + + // Apply removals first (from end to beginning to avoid index shifting) + for _, removalPath := range removals { + actualPath := strings.TrimSuffix(removalPath, "@remove") + index := extractIndexFromRemovalPath(removalPath) + arrayPath := getArrayPathFromElementPath(actualPath) + + fmt.Printf("DEBUG: Removing element at path '%s' (index %d, array path '%s')\n", actualPath, index, arrayPath) + + // Get the array element to remove + result := gjson.Get(content, actualPath) + if !result.Exists() { + fmt.Printf("DEBUG: Element does not exist\n") + continue + } + + // Find the exact byte range to remove (including comma/formatting) + startPos, endPos := findArrayElementRemovalRange(content, arrayPath, index) + if startPos >= 0 && endPos > startPos { + fmt.Printf("DEBUG: Removing bytes %d-%d: %q\n", startPos, endPos, content[startPos:endPos]) + commands = append(commands, utils.ReplaceCommand{ + From: startPos, + To: endPos, + With: "", // Remove the element + }) + } else { + fmt.Printf("DEBUG: Invalid range %d-%d\n", startPos, endPos) + } + } + + // Apply value changes + for _, path := range valueChanges { + newValue := changes[path] + + // Get the current value and its position in the original JSON + result := gjson.Get(content, path) + if !result.Exists() { + continue // Skip if path doesn't exist + } + + // Get the exact byte positions of this value + startPos := result.Index + endPos := startPos + len(result.Raw) + + // Convert the new value to JSON string WITHOUT using json.Marshal + var newValueStr string + switch v := newValue.(type) { + case string: + newValueStr = `"` + strings.ReplaceAll(v, `"`, `\"`) + `"` + case float64: + if v == float64(int64(v)) { + newValueStr = strconv.FormatInt(int64(v), 10) + } else { + newValueStr = strconv.FormatFloat(v, 'f', -1, 64) + } + case bool: + newValueStr = strconv.FormatBool(v) + case nil: + newValueStr = "null" + default: + // For complex types, we need to avoid json.Marshal + // This should not happen if we're doing true surgical edits + continue + } + + // Create a replacement command for this specific value commands = append(commands, utils.ReplaceCommand{ - From: 0, - To: len(content), - With: modifiedContent, + From: int(startPos), + To: int(endPos), + With: newValueStr, }) } - + return commands, nil } +// extractIndexFromRemovalPath extracts the array index from a removal path like "Rows.0.Inputs.1@remove" +func extractIndexFromRemovalPath(path string) int { + parts := strings.Split(strings.TrimSuffix(path, "@remove"), ".") + if len(parts) > 0 { + lastPart := parts[len(parts)-1] + if index, err := strconv.Atoi(lastPart); err == nil { + return index + } + } + return -1 +} + +// getArrayPathFromElementPath converts "Rows.0.Inputs.1" to "Rows.0.Inputs" +func getArrayPathFromElementPath(elementPath string) string { + parts := strings.Split(elementPath, ".") + if len(parts) > 0 { + return strings.Join(parts[:len(parts)-1], ".") + } + return "" +} + +// findArrayElementRemovalRange finds the exact byte range to remove for an array element +func findArrayElementRemovalRange(content, arrayPath string, elementIndex int) (int, int) { + // Get the array using gjson + arrayResult := gjson.Get(content, arrayPath) + if !arrayResult.Exists() || !arrayResult.IsArray() { + return -1, -1 + } + + // Get all array elements + elements := arrayResult.Array() + if elementIndex >= len(elements) { + return -1, -1 + } + + // Get the target element + elementResult := elements[elementIndex] + startPos := int(elementResult.Index) + endPos := int(elementResult.Index + len(elementResult.Raw)) + + // Handle comma removal properly + if elementIndex == 0 && len(elements) > 1 { + // First element but not the only one - remove comma after + for i := endPos; i < len(content) && i < endPos+50; i++ { + if content[i] == ',' { + endPos = i + 1 + break + } + } + } else if elementIndex == len(elements)-1 && len(elements) > 1 { + // Last element and not the only one - remove comma before + prevElementEnd := int(elements[elementIndex-1].Index + len(elements[elementIndex-1].Raw)) + for i := prevElementEnd; i < startPos && i < len(content); i++ { + if content[i] == ',' { + startPos = i + break + } + } + } + // If it's the only element, don't remove any commas + + return startPos, endPos +} + // findDeepChanges recursively finds all paths that need to be changed func findDeepChanges(basePath string, original, modified interface{}) map[string]interface{} { changes := make(map[string]interface{}) - + switch orig := original.(type) { case map[string]interface{}: if mod, ok := modified.(map[string]interface{}); ok { - // Check each key in the modified data for key, modValue := range mod { var currentPath string if basePath == "" { @@ -187,57 +317,74 @@ func findDeepChanges(basePath string, original, modified interface{}) map[string } else { currentPath = basePath + "." + key } - + if origValue, exists := orig[key]; exists { // Key exists in both, check if value changed - if !deepEqual(origValue, modValue) { - // If it's a nested object/array, recurse - switch modValue.(type) { - case map[string]interface{}, []interface{}: - nestedChanges := findDeepChanges(currentPath, origValue, modValue) - for nestedPath, nestedValue := range nestedChanges { - changes[nestedPath] = nestedValue - } - default: - // Primitive value changed + switch modValue.(type) { + case map[string]interface{}, []interface{}: + // Recursively check nested structures + nestedChanges := findDeepChanges(currentPath, origValue, modValue) + for nestedPath, nestedValue := range nestedChanges { + changes[nestedPath] = nestedValue + } + default: + // Primitive value - check if changed + if !deepEqual(origValue, modValue) { changes[currentPath] = modValue } } } else { - // New key added - changes[currentPath] = modValue + // New key added - handle as structural change (skip for now) + // This is complex for surgical editing as it requires inserting into object } } } case []interface{}: if mod, ok := modified.([]interface{}); ok { - // For arrays, check each index - for i, modValue := range mod { - var currentPath string - if basePath == "" { - currentPath = fmt.Sprintf("%d", i) + // Handle array changes by detecting specific element operations + if len(orig) != len(mod) { + // Array length changed - detect if it's element removal + if len(orig) > len(mod) { + // Element(s) removed - find which ones by comparing content + removedIndices := findRemovedArrayElements(orig, mod) + for _, removedIndex := range removedIndices { + var currentPath string + if basePath == "" { + currentPath = fmt.Sprintf("%d@remove", removedIndex) + } else { + currentPath = fmt.Sprintf("%s.%d@remove", basePath, removedIndex) + } + changes[currentPath] = nil // Mark for removal + } } else { - currentPath = fmt.Sprintf("%s.%d", basePath, i) + // Elements added - more complex, skip for now } - - if i < len(orig) { - // Index exists in both, check if value changed - if !deepEqual(orig[i], modValue) { - // If it's a nested object/array, recurse + } else { + // Same length - check individual elements for value changes + for i, modValue := range mod { + var currentPath string + if basePath == "" { + currentPath = strconv.Itoa(i) + } else { + currentPath = basePath + "." + strconv.Itoa(i) + } + + if i < len(orig) { + // Index exists in both, check if value changed switch modValue.(type) { case map[string]interface{}, []interface{}: + // Recursively check nested structures nestedChanges := findDeepChanges(currentPath, orig[i], modValue) for nestedPath, nestedValue := range nestedChanges { changes[nestedPath] = nestedValue } default: - // Primitive value changed - changes[currentPath] = modValue + // Primitive value - check if changed + if !deepEqual(orig[i], modValue) { + changes[currentPath] = modValue + } } } - } else { - // New array element added - changes[currentPath] = modValue } } } @@ -251,10 +398,31 @@ func findDeepChanges(basePath string, original, modified interface{}) map[string } } } - + return changes } +// findRemovedArrayElements compares two arrays and returns indices of removed elements +func findRemovedArrayElements(original, modified []interface{}) []int { + var removedIndices []int + + // Simple approach: find elements in original that don't exist in modified + for i, origElement := range original { + found := false + for _, modElement := range modified { + if deepEqual(origElement, modElement) { + found = true + break + } + } + if !found { + removedIndices = append(removedIndices, i) + } + } + + return removedIndices +} + // deepEqual performs deep comparison of two values func deepEqual(a, b interface{}) bool { if a == nil && b == nil {