dave/BigChef
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Begin to rework the json parsing

Browse Source
This commit is contained in:
PhatPhuckDave
2025-03-25 00:40:58 +01:00
parent d2419b761e
commit 6f9f3f5eae
2 changed files with 132 additions and 197 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
processor/json.go
View File

@@ -5,10 +5,8 @@ import (
"fmt"
"os"
"path/filepath"
"strconv"
"strings"
"github.com/PaesslerAG/jsonpath"
lua "github.com/yuin/gopher-lua"
)
@@ -18,7 +16,12 @@ type JSONProcessor struct{}
// Process implements the Processor interface for JSONProcessor
func (p *JSONProcessor) Process(filename string, pattern string, luaExpr string) (int, int, error) {
// Read file content
fullPath := filepath.Join(".", filename)
cwd, err := os.Getwd()
if err != nil {
return 0, 0, fmt.Errorf("error getting current working directory: %v", err)
}
fullPath := filepath.Join(cwd, filename)
content, err := os.ReadFile(fullPath)
if err != nil {
return 0, 0, fmt.Errorf("error reading file: %v", err)
@@ -64,28 +67,15 @@ func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr s
}
// Initialize Lua
L := lua.NewState()
L, err := NewLuaState()
if err != nil {
return content, 0, 0, fmt.Errorf("error creating Lua state: %v", err)
}
defer L.Close()
// Load math library
L.Push(L.GetGlobal("require"))
L.Push(lua.LString("math"))
if err := L.PCall(1, 1, nil); err != nil {
return content, 0, 0, fmt.Errorf("error loading Lua math library: %v", err)
}
// Load helper functions
if err := InitLuaHelpers(L); err != nil {
return content, 0, 0, err
}
// Apply modifications to each node
modCount := 0
for i, value := range values {
// Reset Lua state for each node
L.SetGlobal("v1", lua.LNil)
L.SetGlobal("s1", lua.LNil)
// Convert to Lua variables
err = p.ToLua(L, value)
if err != nil {
@@ -94,7 +84,7 @@ func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr s
// Execute Lua script
if err := L.DoString(luaExpr); err != nil {
return content, modCount, matchCount, fmt.Errorf("error executing Lua: %v", err)
return content, modCount, matchCount, fmt.Errorf("error executing Lua %s: %v", luaExpr, err)
}
// Get modified value
@@ -103,22 +93,26 @@ func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr s
return content, modCount, matchCount, fmt.Errorf("error getting result from Lua: %v", err)
}
// Skip if value didn't change
if fmt.Sprintf("%v", value) == fmt.Sprintf("%v", result) {
continue
}
// Apply the modification to the JSON data
err = p.updateJSONValue(jsonData, paths[i], result)
if err != nil {
return content, modCount, matchCount, fmt.Errorf("error updating JSON: %v", err)
}
// Increment mod count if we haven't already counted object properties
modCount++
}
// Convert the modified JSON back to a string
jsonBytes, err := json.MarshalIndent(jsonData, "", " ")
// Convert the modified JSON back to a string with same formatting
var jsonBytes []byte
if indent, err := detectJsonIndentation(content); err == nil && indent != "" {
// Use detected indentation for output formatting
jsonBytes, err = json.MarshalIndent(jsonData, "", indent)
} else {
// Fall back to standard 2-space indent
jsonBytes, err = json.MarshalIndent(jsonData, "", " ")
}
if err != nil {
return content, modCount, matchCount, fmt.Errorf("error serializing JSON: %v", err)
}
@@ -126,183 +120,88 @@ func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr s
return string(jsonBytes), modCount, matchCount, nil
}
// findJSONPaths finds all JSON paths and their values that match the given JSONPath expression
// detectJsonIndentation tries to determine the indentation used in the original JSON
func detectJsonIndentation(content string) (string, error) {
lines := strings.Split(content, "\n")
if len(lines) < 2 {
return "", fmt.Errorf("not enough lines to detect indentation")
}
// Look for the first indented line
for i := 1; i < len(lines); i++ {
line := lines[i]
trimmed := strings.TrimSpace(line)
if trimmed == "" {
continue
}
// Calculate leading whitespace
indent := line[:len(line)-len(trimmed)]
if len(indent) > 0 {
return indent, nil
}
}
return "", fmt.Errorf("no indentation detected")
}
func (p *JSONProcessor) findJSONPaths(jsonData interface{}, pattern string) ([]string, []interface{}, error) {
// Extract all matching values using JSONPath
values, err := jsonpath.Get(pattern, jsonData)
if err != nil {
return nil, nil, err
// / $ the root object/element
// // .. recursive descent. JSONPath borrows this syntax from E4X.
// * * wildcard. All objects/elements regardless their names.
// [] [] subscript operator. XPath uses it to iterate over element collections and for predicates. In Javascript and JSON it is the native array operator.
// patternPaths := strings.Split(pattern, ".")
// current := jsonData
// for _, path := range patternPaths {
// switch path {
// case "$":
// current = jsonData
// case "@":
// current = jsonData
// case "*":
// current = jsonData
// case "..":
// }
// }
// paths, values, err := p.findJSONPaths(jsonData, pattern)
return nil, nil, nil
}
// Convert values to a slice if it's not already
valuesSlice := []interface{}{}
paths := []string{}
// / Selects from the root node
// // Selects nodes in the document from the current node that match the selection no matter where they are
// . Selects the current node
// @ Selects attributes
switch v := values.(type) {
case []interface{}:
valuesSlice = v
// Generate paths for array elements
// This is simplified - for complex JSONPath expressions you might
// need a more robust approach to generate the exact path
basePath := pattern
if strings.Contains(pattern, "[*]") || strings.HasSuffix(pattern, ".*") {
basePath = strings.Replace(pattern, "[*]", "", -1)
basePath = strings.Replace(basePath, ".*", "", -1)
for i := 0; i < len(v); i++ {
paths = append(paths, fmt.Sprintf("%s[%d]", basePath, i))
}
} else {
for range v {
paths = append(paths, pattern)
}
}
default:
valuesSlice = append(valuesSlice, v)
paths = append(paths, pattern)
}
// /bookstore/* Selects all the child element nodes of the bookstore element
// //* Selects all elements in the document
return paths, valuesSlice, nil
}
// /bookstore/book[1] Selects the first book element that is the child of the bookstore element.
// /bookstore/book[last()] Selects the last book element that is the child of the bookstore element
// /bookstore/book[last()-1] Selects the last but one book element that is the child of the bookstore element
// /bookstore/book[position()<3] Selects the first two book elements that are children of the bookstore element
// //title[@lang] Selects all the title elements that have an attribute named lang
// //title[@lang='en'] Selects all the title elements that have a "lang" attribute with a value of "en"
// /bookstore/book[price>35.00] Selects all the book elements of the bookstore element that have a price element with a value greater than 35.00
// /bookstore/book[price>35.00]/title Selects all the title elements of the book elements of the bookstore element that have a price element with a value greater than 35.00
// updateJSONValue updates a value in the JSON data structure at the given path
// updateJSONValue updates a value in the JSON structure based on its JSONPath
func (p *JSONProcessor) updateJSONValue(jsonData interface{}, path string, newValue interface{}) error {
// This is a simplified approach - for a production system you'd need a more robust solution
// that can handle all JSONPath expressions
parts := strings.Split(path, ".")
current := jsonData
// Traverse the JSON structure
for i, part := range parts {
if i == len(parts)-1 {
// Last part, set the value
if strings.HasSuffix(part, "]") {
// Handle array access
arrayPart := part[:strings.Index(part, "[")]
indexPart := part[strings.Index(part, "[")+1 : strings.Index(part, "]")]
index, err := strconv.Atoi(indexPart)
if err != nil {
return fmt.Errorf("invalid array index: %s", indexPart)
}
// Get the array
var array []interface{}
if arrayPart == "" {
// Direct array access
array, _ = current.([]interface{})
} else {
// Access array property
obj, _ := current.(map[string]interface{})
array, _ = obj[arrayPart].([]interface{})
}
// Set the value
if index >= 0 && index < len(array) {
array[index] = newValue
}
} else {
// Handle object property
obj, _ := current.(map[string]interface{})
obj[part] = newValue
}
break
}
// Not the last part, continue traversing
if strings.HasSuffix(part, "]") {
// Handle array access
arrayPart := part[:strings.Index(part, "[")]
indexPart := part[strings.Index(part, "[")+1 : strings.Index(part, "]")]
index, err := strconv.Atoi(indexPart)
if err != nil {
return fmt.Errorf("invalid array index: %s", indexPart)
}
// Get the array
var array []interface{}
if arrayPart == "" {
// Direct array access
array, _ = current.([]interface{})
} else {
// Access array property
obj, _ := current.(map[string]interface{})
array, _ = obj[arrayPart].([]interface{})
}
// Continue with the array element
if index >= 0 && index < len(array) {
current = array[index]
}
} else {
// Handle object property
obj, _ := current.(map[string]interface{})
current = obj[part]
}
}
return nil
}
// ToLua converts JSON values to Lua variables
func (p *JSONProcessor) ToLua(L *lua.LState, data interface{}) error {
switch v := data.(type) {
case float64:
L.SetGlobal("v1", lua.LNumber(v))
L.SetGlobal("s1", lua.LString(fmt.Sprintf("%v", v)))
case int:
L.SetGlobal("v1", lua.LNumber(v))
L.SetGlobal("s1", lua.LString(fmt.Sprintf("%d", v)))
case string:
L.SetGlobal("s1", lua.LString(v))
// Try to convert to number if possible
if val, err := strconv.ParseFloat(v, 64); err == nil {
L.SetGlobal("v1", lua.LNumber(val))
} else {
L.SetGlobal("v1", lua.LNumber(0))
}
case bool:
if v {
L.SetGlobal("v1", lua.LNumber(1))
} else {
L.SetGlobal("v1", lua.LNumber(0))
}
L.SetGlobal("s1", lua.LString(fmt.Sprintf("%v", v)))
default:
// For complex types, convert to string
L.SetGlobal("s1", lua.LString(fmt.Sprintf("%v", v)))
L.SetGlobal("v1", lua.LNumber(0))
table, err := ToLuaTable(L, data)
if err != nil {
return err
}
L.SetGlobal("v", table)
return nil
}
// FromLua retrieves values from Lua
func (p *JSONProcessor) FromLua(L *lua.LState) (interface{}, error) {
// Check if string variable was modified
s1 := L.GetGlobal("s1")
if s1 != lua.LNil {
if s1Str, ok := s1.(lua.LString); ok {
// Try to convert to number if it's numeric
if val, err := strconv.ParseFloat(string(s1Str), 64); err == nil {
return val, nil
}
// If it's "true" or "false", convert to boolean
if string(s1Str) == "true" {
return true, nil
}
if string(s1Str) == "false" {
return false, nil
}
return string(s1Str), nil
}
}
// Check if numeric variable was modified
v1 := L.GetGlobal("v1")
if v1 != lua.LNil {
if v1Num, ok := v1.(lua.LNumber); ok {
return float64(v1Num), nil
}
}
// Default return nil
return nil, nil
luaValue := L.GetGlobal("v")
return FromLuaTable(L, luaValue.(*lua.LTable))
}

52
processor/json_test.go
View File

@@ -57,7 +57,7 @@ func TestJSONProcessor_Process_NumericValues(t *testing.T) {
}`
p := &JSONProcessor{}
result, modCount, matchCount, err := p.ProcessContent(content, "$.books[*].price", "v=v*2")
result, modCount, matchCount, err := p.ProcessContent(content, "$.books[*]", "v.price=v.price*2")
if err != nil {
t.Fatalf("Error processing content: %v", err)
@@ -71,12 +71,45 @@ func TestJSONProcessor_Process_NumericValues(t *testing.T) {
t.Errorf("Expected 2 modifications, got %d", modCount)
}
// Normalize whitespace for comparison
normalizedResult := normalizeWhitespace(result)
normalizedExpected := normalizeWhitespace(expected)
// Compare parsed JSON objects instead of formatted strings
var resultObj map[string]interface{}
if err := json.Unmarshal([]byte(result), &resultObj); err != nil {
t.Fatalf("Failed to parse result JSON: %v", err)
}
if normalizedResult != normalizedExpected {
t.Errorf("Expected content to be:\n%s\n\nGot:\n%s", expected, result)
var expectedObj map[string]interface{}
if err := json.Unmarshal([]byte(expected), &expectedObj); err != nil {
t.Fatalf("Failed to parse expected JSON: %v", err)
}
// Compare the first book's price
resultBooks, ok := resultObj["books"].([]interface{})
if !ok || len(resultBooks) < 1 {
t.Fatalf("Expected books array in result")
}
resultBook1, ok := resultBooks[0].(map[string]interface{})
if !ok {
t.Fatalf("Expected first book to be an object")
}
resultPrice1, ok := resultBook1["price"].(float64)
if !ok {
t.Fatalf("Expected numeric price in first book")
}
if resultPrice1 != 89.9 {
t.Errorf("Expected first book price to be 89.9, got %v", resultPrice1)
}
// Compare the second book's price
resultBook2, ok := resultBooks[1].(map[string]interface{})
if !ok {
t.Fatalf("Expected second book to be an object")
}
resultPrice2, ok := resultBook2["price"].(float64)
if !ok {
t.Fatalf("Expected numeric price in second book")
}
if resultPrice2 != 11.9 {
t.Errorf("Expected second book price to be 11.9, got %v", resultPrice2)
}
}
@@ -91,12 +124,15 @@ func TestJSONProcessor_Process_StringValues(t *testing.T) {
}`
p := &JSONProcessor{}
result, modCount, matchCount, err := p.ProcessContent(content, "$.config.*", "v=v*2")
result, modCount, matchCount, err := p.ProcessContent(content, "$.config", "for k,vi in pairs(v) do v[k]=vi*2 end")
if err != nil {
t.Fatalf("Error processing content: %v", err)
}
// Debug info
t.Logf("Result: %s", result)
t.Logf("Match count: %d, Mod count: %d", matchCount, modCount)
if matchCount != 3 {
t.Errorf("Expected 3 matches, got %d", matchCount)
}