Refactor everything to processors and implement json and xml processors such as they are

2025-03-24 15:45:04 +01:00
parent a796e8cac5
commit 4f70eaa329
9 changed files with 3109 additions and 1477 deletions
--- a/processor/json_test.go
+++ b/processor/json_test.go
@@ -0,0 +1,511 @@
+package processor
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+// TestJSONProcessor_Process_NumericValues tests processing numeric JSON values
+func TestJSONProcessor_Process_NumericValues(t *testing.T) {
+	// Test JSON with numeric price values we want to modify
+	testJSON := `{
+  "catalog": {
+    "books": [
+      {
+        "id": "bk101",
+        "author": "Gambardella, Matthew",
+        "title": "JSON Developer's Guide",
+        "genre": "Computer",
+        "price": 44.95,
+        "publish_date": "2000-10-01"
+      },
+      {
+        "id": "bk102",
+        "author": "Ralls, Kim",
+        "title": "Midnight Rain",
+        "genre": "Fantasy",
+        "price": 5.95,
+        "publish_date": "2000-12-16"
+      }
+    ]
+  }
+}`
+
+	// Create a JSON processor
+	processor := NewJSONProcessor(&TestLogger{T: t})
+
+	// Process the JSON content directly to double all prices
+	jsonPathExpr := "$.catalog.books[*].price"
+	modifiedJSON, modCount, matchCount, err := processor.ProcessContent(testJSON, jsonPathExpr, "v1 = v1 * 2", "*2")
+	if err != nil {
+		t.Fatalf("Failed to process JSON content: %v", err)
+	}
+
+	// Check that we found and modified the correct number of nodes
+	if matchCount != 2 {
+		t.Errorf("Expected to match 2 nodes, got %d", matchCount)
+	}
+	if modCount != 2 {
+		t.Errorf("Expected to modify 2 nodes, got %d", modCount)
+	}
+
+	// Parse the JSON to check values more precisely
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(modifiedJSON), &result); err != nil {
+		t.Fatalf("Failed to parse modified JSON: %v", err)
+	}
+
+	// Navigate to the books array
+	catalog, ok := result["catalog"].(map[string]interface{})
+	if !ok {
+		t.Fatalf("No catalog object found in result")
+	}
+	books, ok := catalog["books"].([]interface{})
+	if !ok {
+		t.Fatalf("No books array found in catalog")
+	}
+
+	// Check that both books have their prices doubled
+	// Note: The JSON numbers might be parsed as float64
+	book1, ok := books[0].(map[string]interface{})
+	if !ok {
+		t.Fatalf("First book is not an object")
+	}
+	price1, ok := book1["price"].(float64)
+	if !ok {
+		t.Fatalf("Price of first book is not a number")
+	}
+	if price1 != 89.9 {
+		t.Errorf("Expected first book price to be 89.9, got %v", price1)
+	}
+
+	book2, ok := books[1].(map[string]interface{})
+	if !ok {
+		t.Fatalf("Second book is not an object")
+	}
+	price2, ok := book2["price"].(float64)
+	if !ok {
+		t.Fatalf("Price of second book is not a number")
+	}
+	if price2 != 11.9 {
+		t.Errorf("Expected second book price to be 11.9, got %v", price2)
+	}
+}
+
+// TestJSONProcessor_Process_StringValues tests processing string JSON values
+func TestJSONProcessor_Process_StringValues(t *testing.T) {
+	// Test JSON with string values we want to modify
+	testJSON := `{
+  "config": {
+    "settings": [
+      { "name": "maxUsers", "value": "100" },
+      { "name": "timeout", "value": "30" },
+      { "name": "retries", "value": "5" }
+    ]
+  }
+}`
+
+	// Create a JSON processor
+	processor := NewJSONProcessor(&TestLogger{T: t})
+
+	// Process the JSON content directly to double all numeric values
+	jsonPathExpr := "$.config.settings[*].value"
+	modifiedJSON, modCount, matchCount, err := processor.ProcessContent(testJSON, jsonPathExpr, "v1 = v1 * 2", "*2")
+	if err != nil {
+		t.Fatalf("Failed to process JSON content: %v", err)
+	}
+
+	// Check that we found and modified the correct number of nodes
+	if matchCount != 3 {
+		t.Errorf("Expected to match 3 nodes, got %d", matchCount)
+	}
+	if modCount != 3 {
+		t.Errorf("Expected to modify 3 nodes, got %d", modCount)
+	}
+
+	// Check that the string values were doubled
+	if !strings.Contains(modifiedJSON, `"value": "200"`) {
+		t.Errorf("Modified content does not contain updated value 200")
+	}
+	if !strings.Contains(modifiedJSON, `"value": "60"`) {
+		t.Errorf("Modified content does not contain updated value 60")
+	}
+	if !strings.Contains(modifiedJSON, `"value": "10"`) {
+		t.Errorf("Modified content does not contain updated value 10")
+	}
+
+	// Verify the JSON is valid after modification
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(modifiedJSON), &result); err != nil {
+		t.Fatalf("Modified JSON is not valid: %v", err)
+	}
+}
+
+// TestJSONProcessor_FindNodes tests the JSONPath implementation
+func TestJSONProcessor_FindNodes(t *testing.T) {
+	// Test simple JSONPath functionality
+	testCases := []struct {
+		name      string
+		jsonData  string
+		path      string
+		expectLen int
+		expectErr bool
+	}{
+		{
+			name:      "Root element",
+			jsonData:  `{"name": "root", "value": 100}`,
+			path:      "$",
+			expectLen: 1,
+			expectErr: false,
+		},
+		{
+			name:      "Direct property",
+			jsonData:  `{"name": "test", "value": 100}`,
+			path:      "$.value",
+			expectLen: 1,
+			expectErr: false,
+		},
+		{
+			name:      "Array access",
+			jsonData:  `{"items": [10, 20, 30]}`,
+			path:      "$.items[1]",
+			expectLen: 1,
+			expectErr: false,
+		},
+		{
+			name:      "All array elements",
+			jsonData:  `{"items": [10, 20, 30]}`,
+			path:      "$.items[*]",
+			expectLen: 3,
+			expectErr: false,
+		},
+		{
+			name:      "Nested property",
+			jsonData:  `{"user": {"name": "John", "age": 30}}`,
+			path:      "$.user.age",
+			expectLen: 1,
+			expectErr: false,
+		},
+		{
+			name:      "Array of objects",
+			jsonData:  `{"users": [{"name": "John"}, {"name": "Jane"}]}`,
+			path:      "$.users[*].name",
+			expectLen: 2,
+			expectErr: false,
+		},
+		{
+			name:      "Invalid path",
+			jsonData:  `{"name": "test"}`,
+			path:      "$.invalid[[", // Double bracket should cause an error
+			expectLen: 0,
+			expectErr: true,
+		},
+	}
+
+	processor := &JSONProcessor{Logger: &TestLogger{}}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Parse the JSON data
+			var jsonDoc interface{}
+			if err := json.Unmarshal([]byte(tc.jsonData), &jsonDoc); err != nil {
+				t.Fatalf("Failed to parse test JSON: %v", err)
+			}
+
+			// Find nodes with the given path
+			nodes, err := processor.findNodePaths(jsonDoc, tc.path)
+
+			// Check error expectation
+			if tc.expectErr && err == nil {
+				t.Errorf("Expected error but got none")
+			}
+			if !tc.expectErr && err != nil {
+				t.Errorf("Unexpected error: %v", err)
+			}
+
+			// Skip further checks if we expected an error
+			if tc.expectErr {
+				return
+			}
+
+			// Check the number of nodes found
+			if len(nodes) != tc.expectLen {
+				t.Errorf("Expected %d nodes, got %d", tc.expectLen, len(nodes))
+			}
+		})
+	}
+}
+
+// TestJSONProcessor_NestedModifications tests modifying nested JSON objects
+func TestJSONProcessor_NestedModifications(t *testing.T) {
+	testJSON := `{
+  "company": {
+    "name": "ABC Corp",
+    "departments": [
+      {
+        "id": "dev",
+        "name": "Development",
+        "employees": [
+          {"id": 1, "name": "John Doe", "salary": 75000},
+          {"id": 2, "name": "Jane Smith", "salary": 82000}
+        ]
+      },
+      {
+        "id": "sales",
+        "name": "Sales",
+        "employees": [
+          {"id": 3, "name": "Bob Johnson", "salary": 65000},
+          {"id": 4, "name": "Alice Brown", "salary": 68000}
+        ]
+      }
+    ]
+  }
+}`
+
+	// Create a JSON processor
+	processor := NewJSONProcessor(&TestLogger{T: t})
+
+	// Process the JSON to give everyone a 10% raise
+	jsonPathExpr := "$.company.departments[*].employees[*].salary"
+	modifiedJSON, modCount, matchCount, err := processor.ProcessContent(testJSON, jsonPathExpr, "v1 = v1 * 1.1", "10% raise")
+	if err != nil {
+		t.Fatalf("Failed to process JSON content: %v", err)
+	}
+
+	// Check counts
+	if matchCount != 4 {
+		t.Errorf("Expected to match 4 salary nodes, got %d", matchCount)
+	}
+	if modCount != 4 {
+		t.Errorf("Expected to modify 4 nodes, got %d", modCount)
+	}
+
+	// Parse the result to verify changes
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(modifiedJSON), &result); err != nil {
+		t.Fatalf("Failed to parse modified JSON: %v", err)
+	}
+
+	// Get company > departments
+	company := result["company"].(map[string]interface{})
+	departments := company["departments"].([]interface{})
+
+	// Check first department's first employee
+	dept1 := departments[0].(map[string]interface{})
+	employees1 := dept1["employees"].([]interface{})
+	emp1 := employees1[0].(map[string]interface{})
+	salary1 := emp1["salary"].(float64)
+
+	// Salary should be 75000 * 1.1 = 82500
+	if salary1 != 82500 {
+		t.Errorf("Expected first employee salary to be 82500, got %v", salary1)
+	}
+
+	// Check second department's second employee
+	dept2 := departments[1].(map[string]interface{})
+	employees2 := dept2["employees"].([]interface{})
+	emp4 := employees2[1].(map[string]interface{})
+	salary4 := emp4["salary"].(float64)
+
+	// Salary should be 68000 * 1.1 = 74800
+	if salary4 != 74800 {
+		t.Errorf("Expected fourth employee salary to be 74800, got %v", salary4)
+	}
+}
+
+// TestJSONProcessor_ArrayManipulation tests modifying JSON arrays
+func TestJSONProcessor_ArrayManipulation(t *testing.T) {
+	testJSON := `{
+  "dataPoints": [10, 20, 30, 40, 50]
+}`
+
+	// Create a JSON processor
+	processor := NewJSONProcessor(&TestLogger{T: t})
+
+	// Process the JSON to normalize values (divide by max value)
+	jsonPathExpr := "$.dataPoints[*]"
+	modifiedJSON, modCount, matchCount, err := processor.ProcessContent(testJSON, jsonPathExpr, "v1 = v1 / 50", "normalize")
+	if err != nil {
+		t.Fatalf("Failed to process JSON content: %v", err)
+	}
+
+	// Check counts
+	if matchCount != 5 {
+		t.Errorf("Expected to match 5 data points, got %d", matchCount)
+	}
+	if modCount != 5 {
+		t.Errorf("Expected to modify 5 nodes, got %d", modCount)
+	}
+
+	// Parse the result to verify changes
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(modifiedJSON), &result); err != nil {
+		t.Fatalf("Failed to parse modified JSON: %v", err)
+	}
+
+	// Get the data points array
+	dataPoints := result["dataPoints"].([]interface{})
+
+	// Check values (should be divided by 50)
+	expectedValues := []float64{0.2, 0.4, 0.6, 0.8, 1.0}
+	for i, val := range dataPoints {
+		if val.(float64) != expectedValues[i] {
+			t.Errorf("Expected dataPoints[%d] to be %v, got %v", i, expectedValues[i], val)
+		}
+	}
+}
+
+// TestJSONProcessor_ConditionalModification tests applying changes only to certain elements
+func TestJSONProcessor_ConditionalModification(t *testing.T) {
+	testJSON := `{
+  "products": [
+    {"id": "p1", "name": "Laptop", "price": 999.99, "discount": 0},
+    {"id": "p2", "name": "Headphones", "price": 59.99, "discount": 0},
+    {"id": "p3", "name": "Mouse", "price": 29.99, "discount": 0}
+  ]
+}`
+
+	// Create a JSON processor
+	processor := NewJSONProcessor(&TestLogger{T: t})
+
+	// Process: apply 10% discount to items over $50, 5% to others
+	luaScript := `
+	-- Get the path to find the parent (product)
+	local path = string.gsub(_PATH, ".discount$", "")
+	
+	-- Custom logic based on price
+	local price = _PARENT.price
+	if price > 50 then
+		v1 = 0.1  -- 10% discount
+	else
+		v1 = 0.05 -- 5% discount
+	end
+	`
+
+	jsonPathExpr := "$.products[*].discount"
+	modifiedJSON, modCount, matchCount, err := processor.ProcessContent(testJSON, jsonPathExpr, luaScript, "apply discounts")
+	if err != nil {
+		t.Fatalf("Failed to process JSON content: %v", err)
+	}
+
+	// Check counts
+	if matchCount != 3 {
+		t.Errorf("Expected to match 3 discount nodes, got %d", matchCount)
+	}
+	if modCount != 3 {
+		t.Errorf("Expected to modify 3 nodes, got %d", modCount)
+	}
+
+	// Parse the result to verify changes
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(modifiedJSON), &result); err != nil {
+		t.Fatalf("Failed to parse modified JSON: %v", err)
+	}
+
+	// Get products array
+	products := result["products"].([]interface{})
+
+	// Laptop and Headphones should have 10% discount
+	laptop := products[0].(map[string]interface{})
+	headphones := products[1].(map[string]interface{})
+	mouse := products[2].(map[string]interface{})
+
+	if laptop["discount"].(float64) != 0.1 {
+		t.Errorf("Expected laptop discount to be 0.1, got %v", laptop["discount"])
+	}
+
+	if headphones["discount"].(float64) != 0.1 {
+		t.Errorf("Expected headphones discount to be 0.1, got %v", headphones["discount"])
+	}
+
+	if mouse["discount"].(float64) != 0.05 {
+		t.Errorf("Expected mouse discount to be 0.05, got %v", mouse["discount"])
+	}
+}
+
+// TestJSONProcessor_ComplexScripts tests using more complex Lua scripts
+func TestJSONProcessor_ComplexScripts(t *testing.T) {
+	testJSON := `{
+  "metrics": [
+    {"name": "CPU", "values": [45, 60, 75, 90, 80]},
+    {"name": "Memory", "values": [30, 40, 45, 50, 60]},
+    {"name": "Disk", "values": [20, 25, 30, 40, 50]}
+  ]
+}`
+
+	// Create a JSON processor
+	processor := NewJSONProcessor(&TestLogger{T: t})
+
+	// Apply a moving average transformation
+	luaScript := `
+	-- This script transforms an array using a moving average
+	local values = {}
+	local window = 3 -- window size
+	
+	-- Get all the values as a table
+	for i = 1, 5 do
+		local element = _VALUE[i]
+		if element then
+			values[i] = element
+		end
+	end
+	
+	-- Calculate moving averages
+	local result = {}
+	for i = 1, #values do
+		local sum = 0
+		local count = 0
+		
+		-- Sum the window
+		for j = math.max(1, i-(window-1)/2), math.min(#values, i+(window-1)/2) do
+			sum = sum + values[j]
+			count = count + 1
+		end
+		
+		-- Set the average
+		result[i] = sum / count
+	end
+	
+	-- Update all values
+	for i = 1, #result do
+		_VALUE[i] = result[i]
+	end
+	`
+
+	jsonPathExpr := "$.metrics[*].values"
+	modifiedJSON, modCount, matchCount, err := processor.ProcessContent(testJSON, jsonPathExpr, luaScript, "moving average")
+	if err != nil {
+		t.Fatalf("Failed to process JSON content: %v", err)
+	}
+
+	// Check counts
+	if matchCount != 3 {
+		t.Errorf("Expected to match 3 value arrays, got %d", matchCount)
+	}
+	if modCount != 3 {
+		t.Errorf("Expected to modify 3 nodes, got %d", modCount)
+	}
+
+	// Parse and verify the values were smoothed
+	var result map[string]interface{}
+	if err := json.Unmarshal([]byte(modifiedJSON), &result); err != nil {
+		t.Fatalf("Failed to parse modified JSON: %v", err)
+	}
+
+	// The modification logic would smooth out the values
+	// We'll check that the JSON is valid at least
+	metrics := result["metrics"].([]interface{})
+	if len(metrics) != 3 {
+		t.Errorf("Expected 3 metrics, got %d", len(metrics))
+	}
+
+	// Each metrics should have 5 values
+	for i, metric := range metrics {
+		m := metric.(map[string]interface{})
+		values := m["values"].([]interface{})
+		if len(values) != 5 {
+			t.Errorf("Metric %d should have 5 values, got %d", i, len(values))
+		}
+	}
+}