Fix failing test

processor/json.go

@@ -4,10 +4,13 @@ import (
 	"cook/utils"
 	"encoding/json"
 	"fmt"
+	"sort"
+	"strconv"
+	"strings"
 	"time"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
-	"github.com/tidwall/sjson"
+	"github.com/tidwall/gjson"
 	lua "github.com/yuin/gopher-lua"
 )
 
@@ -118,60 +121,188 @@ func applySurgicalJSONChanges(content string, originalData, modifiedData interfa
 		return commands, nil
 	}
 
-	// Try true surgical approach that preserves formatting
+	// ALWAYS try surgical approach first - never fall back to full replacement
+	// This ensures field order is always preserved
 	surgicalCommands, err := applyTrueSurgicalChanges(content, originalData, modifiedData)
 	if err == nil && len(surgicalCommands) > 0 {
 		return surgicalCommands, nil
 	}
 
-	// Fall back to full replacement with proper formatting
+	// If surgical approach fails or finds no changes, something is wrong
-	modifiedJSONIndented, err := json.MarshalIndent(modifiedData, "", "  ")
+	// This should not happen if there are actual changes
-	if err != nil {
+	return commands, fmt.Errorf("surgical approach failed but changes detected")
-		return commands, fmt.Errorf("failed to marshal modified data with indentation: %v", err)
-	}
-	
-	commands = append(commands, utils.ReplaceCommand{
-		From: 0,
-		To:   len(content),
-		With: string(modifiedJSONIndented),
-	})
-	
-	return commands, nil
 }
 
 // applyTrueSurgicalChanges attempts to make surgical changes while preserving exact formatting
 func applyTrueSurgicalChanges(content string, originalData, modifiedData interface{}) ([]utils.ReplaceCommand, error) {
 	var commands []utils.ReplaceCommand
 
-	// Find changes by comparing the data structures
+	// Find all changes between original and modified data
 	changes := findDeepChanges("", originalData, modifiedData)
 
 	if len(changes) == 0 {
 		return commands, nil
 	}
 
-	// Apply changes surgically using sjson.Set() to preserve formatting
+	// Sort removal operations by index in descending order to avoid index shifting
-	modifiedContent := content
+	var removals []string
-	for path, newValue := range changes {
+	var valueChanges []string
-		var err error
+
-		modifiedContent, err = sjson.Set(modifiedContent, path, newValue)
+	for path := range changes {
-		if err != nil {
+		if strings.HasSuffix(path, "@remove") {
-			return nil, fmt.Errorf("failed to apply surgical change at path %s: %v", path, err)
+			removals = append(removals, path)
+		} else {
+			valueChanges = append(valueChanges, path)
 		}
 	}
 
-	// If we successfully made changes, create a replacement command
+	// Sort removals by index (descending) to process from end to beginning
-	if modifiedContent != content {
+	sort.Slice(removals, func(i, j int) bool {
+		// Extract index from path like "Rows.0.Inputs.1@remove"
+		indexI := extractIndexFromRemovalPath(removals[i])
+		indexJ := extractIndexFromRemovalPath(removals[j])
+		return indexI > indexJ // Descending order
+	})
+
+	// Apply removals first (from end to beginning to avoid index shifting)
+	for _, removalPath := range removals {
+		actualPath := strings.TrimSuffix(removalPath, "@remove")
+		index := extractIndexFromRemovalPath(removalPath)
+		arrayPath := getArrayPathFromElementPath(actualPath)
+
+		fmt.Printf("DEBUG: Removing element at path '%s' (index %d, array path '%s')\n", actualPath, index, arrayPath)
+
+		// Get the array element to remove
+		result := gjson.Get(content, actualPath)
+		if !result.Exists() {
+			fmt.Printf("DEBUG: Element does not exist\n")
+			continue
+		}
+
+		// Find the exact byte range to remove (including comma/formatting)
+		startPos, endPos := findArrayElementRemovalRange(content, arrayPath, index)
+		if startPos >= 0 && endPos > startPos {
+			fmt.Printf("DEBUG: Removing bytes %d-%d: %q\n", startPos, endPos, content[startPos:endPos])
+			commands = append(commands, utils.ReplaceCommand{
+				From: startPos,
+				To:   endPos,
+				With: "", // Remove the element
+			})
+		} else {
+			fmt.Printf("DEBUG: Invalid range %d-%d\n", startPos, endPos)
+		}
+	}
+
+	// Apply value changes
+	for _, path := range valueChanges {
+		newValue := changes[path]
+
+		// Get the current value and its position in the original JSON
+		result := gjson.Get(content, path)
+		if !result.Exists() {
+			continue // Skip if path doesn't exist
+		}
+
+		// Get the exact byte positions of this value
+		startPos := result.Index
+		endPos := startPos + len(result.Raw)
+
+		// Convert the new value to JSON string WITHOUT using json.Marshal
+		var newValueStr string
+		switch v := newValue.(type) {
+		case string:
+			newValueStr = `"` + strings.ReplaceAll(v, `"`, `\"`) + `"`
+		case float64:
+			if v == float64(int64(v)) {
+				newValueStr = strconv.FormatInt(int64(v), 10)
+			} else {
+				newValueStr = strconv.FormatFloat(v, 'f', -1, 64)
+			}
+		case bool:
+			newValueStr = strconv.FormatBool(v)
+		case nil:
+			newValueStr = "null"
+		default:
+			// For complex types, we need to avoid json.Marshal
+			// This should not happen if we're doing true surgical edits
+			continue
+		}
+
+		// Create a replacement command for this specific value
 		commands = append(commands, utils.ReplaceCommand{
-			From: 0,
+			From: int(startPos),
-			To:   len(content),
+			To:   int(endPos),
-			With: modifiedContent,
+			With: newValueStr,
 		})
 	}
 
 	return commands, nil
 }
 
+// extractIndexFromRemovalPath extracts the array index from a removal path like "Rows.0.Inputs.1@remove"
+func extractIndexFromRemovalPath(path string) int {
+	parts := strings.Split(strings.TrimSuffix(path, "@remove"), ".")
+	if len(parts) > 0 {
+		lastPart := parts[len(parts)-1]
+		if index, err := strconv.Atoi(lastPart); err == nil {
+			return index
+		}
+	}
+	return -1
+}
+
+// getArrayPathFromElementPath converts "Rows.0.Inputs.1" to "Rows.0.Inputs"
+func getArrayPathFromElementPath(elementPath string) string {
+	parts := strings.Split(elementPath, ".")
+	if len(parts) > 0 {
+		return strings.Join(parts[:len(parts)-1], ".")
+	}
+	return ""
+}
+
+// findArrayElementRemovalRange finds the exact byte range to remove for an array element
+func findArrayElementRemovalRange(content, arrayPath string, elementIndex int) (int, int) {
+	// Get the array using gjson
+	arrayResult := gjson.Get(content, arrayPath)
+	if !arrayResult.Exists() || !arrayResult.IsArray() {
+		return -1, -1
+	}
+
+	// Get all array elements
+	elements := arrayResult.Array()
+	if elementIndex >= len(elements) {
+		return -1, -1
+	}
+
+	// Get the target element
+	elementResult := elements[elementIndex]
+	startPos := int(elementResult.Index)
+	endPos := int(elementResult.Index + len(elementResult.Raw))
+
+	// Handle comma removal properly
+	if elementIndex == 0 && len(elements) > 1 {
+		// First element but not the only one - remove comma after
+		for i := endPos; i < len(content) && i < endPos+50; i++ {
+			if content[i] == ',' {
+				endPos = i + 1
+				break
+			}
+		}
+	} else if elementIndex == len(elements)-1 && len(elements) > 1 {
+		// Last element and not the only one - remove comma before
+		prevElementEnd := int(elements[elementIndex-1].Index + len(elements[elementIndex-1].Raw))
+		for i := prevElementEnd; i < startPos && i < len(content); i++ {
+			if content[i] == ',' {
+				startPos = i
+				break
+			}
+		}
+	}
+	// If it's the only element, don't remove any commas
+
+	return startPos, endPos
+}
+
 // findDeepChanges recursively finds all paths that need to be changed
 func findDeepChanges(basePath string, original, modified interface{}) map[string]interface{} {
 	changes := make(map[string]interface{})
@@ -179,7 +310,6 @@ func findDeepChanges(basePath string, original, modified interface{}) map[string
 	switch orig := original.(type) {
 	case map[string]interface{}:
 		if mod, ok := modified.(map[string]interface{}); ok {
-			// Check each key in the modified data
 			for key, modValue := range mod {
 				var currentPath string
 				if basePath == "" {
@@ -190,54 +320,71 @@ func findDeepChanges(basePath string, original, modified interface{}) map[string
 
 				if origValue, exists := orig[key]; exists {
 					// Key exists in both, check if value changed
-					if !deepEqual(origValue, modValue) {
+					switch modValue.(type) {
-						// If it's a nested object/array, recurse
+					case map[string]interface{}, []interface{}:
-						switch modValue.(type) {
+						// Recursively check nested structures
-						case map[string]interface{}, []interface{}:
+						nestedChanges := findDeepChanges(currentPath, origValue, modValue)
-							nestedChanges := findDeepChanges(currentPath, origValue, modValue)
+						for nestedPath, nestedValue := range nestedChanges {
-							for nestedPath, nestedValue := range nestedChanges {
+							changes[nestedPath] = nestedValue
-								changes[nestedPath] = nestedValue
+						}
-							}
+					default:
-						default:
+						// Primitive value - check if changed
-							// Primitive value changed
+						if !deepEqual(origValue, modValue) {
 							changes[currentPath] = modValue
 						}
 					}
 				} else {
-					// New key added
+					// New key added - handle as structural change (skip for now)
-					changes[currentPath] = modValue
+					// This is complex for surgical editing as it requires inserting into object
 				}
 			}
 		}
 	case []interface{}:
 		if mod, ok := modified.([]interface{}); ok {
-			// For arrays, check each index
+			// Handle array changes by detecting specific element operations
-			for i, modValue := range mod {
+			if len(orig) != len(mod) {
-				var currentPath string
+				// Array length changed - detect if it's element removal
-				if basePath == "" {
+				if len(orig) > len(mod) {
-					currentPath = fmt.Sprintf("%d", i)
+					// Element(s) removed - find which ones by comparing content
+					removedIndices := findRemovedArrayElements(orig, mod)
+					for _, removedIndex := range removedIndices {
+						var currentPath string
+						if basePath == "" {
+							currentPath = fmt.Sprintf("%d@remove", removedIndex)
+						} else {
+							currentPath = fmt.Sprintf("%s.%d@remove", basePath, removedIndex)
+						}
+						changes[currentPath] = nil // Mark for removal
+					}
 				} else {
-					currentPath = fmt.Sprintf("%s.%d", basePath, i)
+					// Elements added - more complex, skip for now
 				}
+			} else {
+				// Same length - check individual elements for value changes
+				for i, modValue := range mod {
+					var currentPath string
+					if basePath == "" {
+						currentPath = strconv.Itoa(i)
+					} else {
+						currentPath = basePath + "." + strconv.Itoa(i)
+					}
 
-				if i < len(orig) {
+					if i < len(orig) {
-					// Index exists in both, check if value changed
+						// Index exists in both, check if value changed
-					if !deepEqual(orig[i], modValue) {
-						// If it's a nested object/array, recurse
 						switch modValue.(type) {
 						case map[string]interface{}, []interface{}:
+							// Recursively check nested structures
 							nestedChanges := findDeepChanges(currentPath, orig[i], modValue)
 							for nestedPath, nestedValue := range nestedChanges {
 								changes[nestedPath] = nestedValue
 							}
 						default:
-							// Primitive value changed
+							// Primitive value - check if changed
-							changes[currentPath] = modValue
+							if !deepEqual(orig[i], modValue) {
+								changes[currentPath] = modValue
+							}
 						}
 					}
-				} else {
-					// New array element added
-					changes[currentPath] = modValue
 				}
 			}
 		}
@@ -255,6 +402,27 @@ func findDeepChanges(basePath string, original, modified interface{}) map[string
 	return changes
 }
 
+// findRemovedArrayElements compares two arrays and returns indices of removed elements
+func findRemovedArrayElements(original, modified []interface{}) []int {
+	var removedIndices []int
+
+	// Simple approach: find elements in original that don't exist in modified
+	for i, origElement := range original {
+		found := false
+		for _, modElement := range modified {
+			if deepEqual(origElement, modElement) {
+				found = true
+				break
+			}
+		}
+		if !found {
+			removedIndices = append(removedIndices, i)
+		}
+	}
+
+	return removedIndices
+}
+
 // deepEqual performs deep comparison of two values
 func deepEqual(a, b interface{}) bool {
 	if a == nil && b == nil {