dave/BigChef
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 34 Wiki Activity

Remove xml and json

Browse Source 
They are simply not as useful as regex at all
There is nothing they can do regex cannot
And they have one massive penalty: the encoding
Which often results in MASSIVE diffs
This commit is contained in:
PhatPhuckDave
2025-03-27 21:28:20 +01:00
parent df212b7fcc
commit 25326ea11b
4 changed files with 0 additions and 4171 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
processor/json.go
View File

@@ -1,194 +0,0 @@
package processor
import (
"encoding/json"
"fmt"
"modify/logger"
"modify/processor/jsonpath"
lua "github.com/yuin/gopher-lua"
)
// JSONProcessor implements the Processor interface for JSON documents
type JSONProcessor struct{}
// ProcessContent implements the Processor interface for JSONProcessor
func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr string) (string, int, int, error) {
logger.Debug("Processing JSON content with JSONPath: %s", pattern)
// Parse JSON document
logger.Trace("Parsing JSON document")
var jsonData interface{}
err := json.Unmarshal([]byte(content), &jsonData)
if err != nil {
logger.Error("Failed to parse JSON: %v", err)
return content, 0, 0, fmt.Errorf("error parsing JSON: %v", err)
}
// Find nodes matching the JSONPath pattern
logger.Debug("Executing JSONPath query: %s", pattern)
nodes, err := jsonpath.Get(jsonData, pattern)
if err != nil {
logger.Error("Failed to execute JSONPath: %v", err)
return content, 0, 0, fmt.Errorf("error getting nodes: %v", err)
}
matchCount := len(nodes)
logger.Debug("Found %d nodes matching JSONPath", matchCount)
if matchCount == 0 {
logger.Warning("No nodes matched the JSONPath pattern: %s", pattern)
return content, 0, 0, nil
}
modCount := 0
for i, node := range nodes {
logger.Trace("Processing node #%d at path: %s with value: %v", i+1, node.Path, node.Value)
// Initialize Lua
L, err := NewLuaState()
if err != nil {
logger.Error("Failed to create Lua state: %v", err)
return content, len(nodes), 0, fmt.Errorf("error creating Lua state: %v", err)
}
defer L.Close()
logger.Trace("Lua state initialized successfully")
err = p.ToLua(L, node.Value)
if err != nil {
logger.Error("Failed to convert value to Lua: %v", err)
return content, len(nodes), 0, fmt.Errorf("error converting to Lua: %v", err)
}
logger.Trace("Converted node value to Lua: %v", node.Value)
originalScript := luaExpr
fullScript := BuildLuaScript(luaExpr)
logger.Debug("Original script: %q, Full script: %q", originalScript, fullScript)
// Execute Lua script
logger.Trace("Executing Lua script: %q", fullScript)
if err := L.DoString(fullScript); err != nil {
logger.Error("Failed to execute Lua script: %v", err)
return content, len(nodes), 0, fmt.Errorf("error executing Lua %q: %v", fullScript, err)
}
logger.Trace("Lua script executed successfully")
// Get modified value
result, err := p.FromLua(L)
if err != nil {
logger.Error("Failed to get result from Lua: %v", err)
return content, len(nodes), 0, fmt.Errorf("error getting result from Lua: %v", err)
}
logger.Trace("Retrieved modified value from Lua: %v", result)
modified := false
modified = L.GetGlobal("modified").String() == "true"
if !modified {
logger.Debug("No changes made to node at path: %s", node.Path)
continue
}
// Apply the modification to the JSON data
logger.Debug("Updating JSON at path: %s with new value: %v", node.Path, result)
err = p.updateJSONValue(jsonData, node.Path, result)
if err != nil {
logger.Error("Failed to update JSON at path %s: %v", node.Path, err)
return content, len(nodes), 0, fmt.Errorf("error updating JSON: %v", err)
}
logger.Debug("Updated JSON at path: %s successfully", node.Path)
modCount++
}
logger.Info("JSON processing complete: %d modifications from %d matches", modCount, matchCount)
// Convert the modified JSON back to a string with same formatting
logger.Trace("Marshalling JSON data back to string")
var jsonBytes []byte
jsonBytes, err = json.MarshalIndent(jsonData, "", " ")
if err != nil {
logger.Error("Failed to marshal JSON: %v", err)
return content, modCount, matchCount, fmt.Errorf("error marshalling JSON: %v", err)
}
return string(jsonBytes), modCount, matchCount, nil
}
// updateJSONValue updates a value in the JSON structure based on its JSONPath
func (p *JSONProcessor) updateJSONValue(jsonData interface{}, path string, newValue interface{}) error {
logger.Trace("Updating JSON value at path: %s", path)
// Special handling for root node
if path == "$" {
logger.Debug("Handling special case for root node update")
// For the root node, we'll copy the value to the jsonData reference
// This is a special case since we can't directly replace the interface{} variable
// We need to handle different types of root elements
switch rootValue := newValue.(type) {
case map[string]interface{}:
// For objects, we need to copy over all keys
rootMap, ok := jsonData.(map[string]interface{})
if !ok {
// If the original wasn't a map, completely replace it with the new map
// This is handled by the jsonpath.Set function
logger.Debug("Root was not a map, replacing entire root")
return jsonpath.Set(jsonData, path, newValue)
}
// Clear the original map
logger.Trace("Clearing original root map")
for k := range rootMap {
delete(rootMap, k)
}
// Copy all keys from the new map
logger.Trace("Copying keys to root map")
for k, v := range rootValue {
rootMap[k] = v
}
return nil
case []interface{}:
// For arrays, we need to handle similarly
rootArray, ok := jsonData.([]interface{})
if !ok {
// If the original wasn't an array, use jsonpath.Set
logger.Debug("Root was not an array, replacing entire root")
return jsonpath.Set(jsonData, path, newValue)
}
// Clear and recreate the array
logger.Trace("Replacing root array")
*&rootArray = rootValue
return nil
default:
// For other types, use jsonpath.Set
logger.Debug("Replacing root with primitive value")
return jsonpath.Set(jsonData, path, newValue)
}
}
// For non-root paths, use the regular Set method
logger.Trace("Using regular Set method for non-root path")
err := jsonpath.Set(jsonData, path, newValue)
if err != nil {
logger.Error("Failed to set JSON value at path %s: %v", path, err)
return fmt.Errorf("failed to update JSON value at path '%s': %w", path, err)
}
return nil
}
// ToLua converts JSON values to Lua variables
func (p *JSONProcessor) ToLua(L *lua.LState, data interface{}) error {
table, err := ToLua(L, data)
if err != nil {
return err
}
L.SetGlobal("v", table)
return nil
}
// FromLua retrieves values from Lua
func (p *JSONProcessor) FromLua(L *lua.LState) (interface{}, error) {
luaValue := L.GetGlobal("v")
return FromLua(L, luaValue)
}

1771
processor/json_test.go
View File

File diff suppressed because it is too large Load Diff

434
processor/xml.go
View File

@@ -1,434 +0,0 @@
package processor
import (
"fmt"
"modify/logger"
"modify/processor/xpath"
"strings"
"github.com/antchfx/xmlquery"
lua "github.com/yuin/gopher-lua"
)
// XMLProcessor implements the Processor interface for XML documents
type XMLProcessor struct{}
// ProcessContent implements the Processor interface for XMLProcessor
func (p *XMLProcessor) ProcessContent(content string, path string, luaExpr string) (string, int, int, error) {
logger.Debug("Processing XML content with XPath: %s", path)
// Parse XML document
// We can't really use encoding/xml here because it requires a pre defined struct
// And we HAVE TO parse dynamic unknown XML
logger.Trace("Parsing XML document")
doc, err := xmlquery.Parse(strings.NewReader(content))
if err != nil {
logger.Error("Failed to parse XML: %v", err)
return content, 0, 0, fmt.Errorf("error parsing XML: %v", err)
}
// Find nodes matching the XPath pattern
logger.Debug("Executing XPath query: %s", path)
nodes, err := xpath.Get(doc, path)
if err != nil {
logger.Error("Failed to execute XPath: %v", err)
return content, 0, 0, fmt.Errorf("error executing XPath: %v", err)
}
matchCount := len(nodes)
logger.Debug("Found %d nodes matching XPath", matchCount)
if matchCount == 0 {
logger.Warning("No nodes matched the XPath pattern: %s", path)
return content, 0, 0, nil
}
// Apply modifications to each node
modCount := 0
for i, node := range nodes {
logger.Trace("Processing node #%d: %s", i+1, node.Data)
L, err := NewLuaState()
if err != nil {
logger.Error("Failed to create Lua state: %v", err)
return content, 0, 0, fmt.Errorf("error creating Lua state: %v", err)
}
defer L.Close()
logger.Trace("Converting XML node to Lua")
err = p.ToLua(L, node)
if err != nil {
logger.Error("Failed to convert XML node to Lua: %v", err)
return content, modCount, matchCount, fmt.Errorf("error converting to Lua: %v", err)
}
luaScript := BuildLuaScript(luaExpr)
logger.Trace("Executing Lua script: %s", luaScript)
err = L.DoString(luaScript)
if err != nil {
logger.Error("Failed to execute Lua script: %v", err)
return content, modCount, matchCount, fmt.Errorf("error executing Lua: %v", err)
}
result, err := p.FromLua(L)
if err != nil {
logger.Error("Failed to get result from Lua: %v", err)
return content, modCount, matchCount, fmt.Errorf("error getting result from Lua: %v", err)
}
logger.Trace("Lua returned result: %#v", result)
modified := false
modified = L.GetGlobal("modified").String() == "true"
if !modified {
logger.Debug("No changes made to node at path: %s", node.Data)
continue
}
// Apply modification based on the result
if updatedValue, ok := result.(string); ok {
// If the result is a simple string, update the node value directly
logger.Debug("Updating node with string value: %s", updatedValue)
xpath.Set(doc, path, updatedValue)
} else if nodeData, ok := result.(map[string]interface{}); ok {
// If the result is a map, apply more complex updates
logger.Debug("Updating node with complex data structure")
updateNodeFromMap(node, nodeData)
}
modCount++
logger.Debug("Successfully modified node #%d", i+1)
}
logger.Info("XML processing complete: %d modifications from %d matches", modCount, matchCount)
// Serialize the modified XML document to string
if doc.FirstChild != nil && doc.FirstChild.Type == xmlquery.DeclarationNode {
// If we have an XML declaration, start with it
declaration := doc.FirstChild.OutputXML(true)
// Remove the firstChild (declaration) before serializing the rest of the document
doc.FirstChild = doc.FirstChild.NextSibling
return ConvertToNamedEntities(declaration + doc.OutputXML(true)), modCount, matchCount, nil
}
// Convert numeric entities to named entities for better readability
return ConvertToNamedEntities(doc.OutputXML(true)), modCount, matchCount, nil
}
func (p *XMLProcessor) ToLua(L *lua.LState, data interface{}) error {
table, err := p.ToLuaTable(L, data)
if err != nil {
return err
}
L.SetGlobal("v", table)
return nil
}
// ToLua converts XML node values to Lua variables
func (p *XMLProcessor) ToLuaTable(L *lua.LState, data interface{}) (lua.LValue, error) {
// Check if data is an xmlquery.Node
node, ok := data.(*xmlquery.Node)
if !ok {
return nil, fmt.Errorf("expected xmlquery.Node, got %T", data)
}
// Create a simple table with essential data
table := L.NewTable()
// For element nodes, just provide basic info
L.SetField(table, "type", lua.LString(nodeTypeToString(node.Type)))
L.SetField(table, "name", lua.LString(node.Data))
L.SetField(table, "value", lua.LString(node.InnerText()))
// Add children if any
children := L.NewTable()
for child := node.FirstChild; child != nil; child = child.NextSibling {
childTable, err := p.ToLuaTable(L, child)
if err == nil {
children.Append(childTable)
}
}
L.SetField(table, "children", children)
attrs := L.NewTable()
if len(node.Attr) > 0 {
for _, attr := range node.Attr {
L.SetField(attrs, attr.Name.Local, lua.LString(attr.Value))
}
}
L.SetField(table, "attr", attrs)
return table, nil
}
// FromLua gets modified values from Lua
func (p *XMLProcessor) FromLua(L *lua.LState) (interface{}, error) {
luaValue := L.GetGlobal("v")
// Handle string values directly
if luaValue.Type() == lua.LTString {
return luaValue.String(), nil
}
// Handle tables (for attributes and more complex updates)
if luaValue.Type() == lua.LTTable {
return luaTableToMap(L, luaValue.(*lua.LTable)), nil
}
return luaValue.String(), nil
}
// Simple helper to convert a Lua table to a Go map
func luaTableToMap(L *lua.LState, table *lua.LTable) map[string]interface{} {
result := make(map[string]interface{})
table.ForEach(func(k, v lua.LValue) {
if k.Type() == lua.LTString {
key := k.String()
if v.Type() == lua.LTTable {
result[key] = luaTableToMap(L, v.(*lua.LTable))
} else {
result[key] = v.String()
}
}
})
return result
}
// Simple helper to convert node type to string
func nodeTypeToString(nodeType xmlquery.NodeType) string {
switch nodeType {
case xmlquery.ElementNode:
return "element"
case xmlquery.TextNode:
return "text"
case xmlquery.AttributeNode:
return "attribute"
default:
return "other"
}
}
// Helper function to update an XML node from a map
func updateNodeFromMap(node *xmlquery.Node, data map[string]interface{}) {
// Update node value if present
if value, ok := data["value"]; ok {
if strValue, ok := value.(string); ok {
// For element nodes, replace text content
if node.Type == xmlquery.ElementNode {
// Find the first text child if it exists
var textNode *xmlquery.Node
for child := node.FirstChild; child != nil; child = child.NextSibling {
if child.Type == xmlquery.TextNode {
textNode = child
break
}
}
if textNode != nil {
// Update existing text node
textNode.Data = strValue
} else {
// Create new text node
newText := &xmlquery.Node{
Type: xmlquery.TextNode,
Data: strValue,
Parent: node,
}
// Insert at beginning of children
if node.FirstChild != nil {
newText.NextSibling = node.FirstChild
node.FirstChild.PrevSibling = newText
node.FirstChild = newText
} else {
node.FirstChild = newText
node.LastChild = newText
}
}
} else if node.Type == xmlquery.TextNode {
// Directly update text node
node.Data = strValue
} else if node.Type == xmlquery.AttributeNode {
// Update attribute value
if node.Parent != nil {
for i, attr := range node.Parent.Attr {
if attr.Name.Local == node.Data {
node.Parent.Attr[i].Value = strValue
break
}
}
}
}
}
}
// Update attributes if present
if attrs, ok := data["attr"].(map[string]interface{}); ok && node.Type == xmlquery.ElementNode {
for name, value := range attrs {
if strValue, ok := value.(string); ok {
// Look for existing attribute
found := false
for i, attr := range node.Attr {
if attr.Name.Local == name {
node.Attr[i].Value = strValue
found = true
break
}
}
// Add new attribute if not found
if !found {
node.Attr = append(node.Attr, xmlquery.Attr{
Name: struct {
Space, Local string
}{Local: name},
Value: strValue,
})
}
}
}
}
}
// Helper function to get a string representation of node type
func nodeTypeName(nodeType xmlquery.NodeType) string {
switch nodeType {
case xmlquery.ElementNode:
return "element"
case xmlquery.TextNode:
return "text"
case xmlquery.AttributeNode:
return "attribute"
case xmlquery.CommentNode:
return "comment"
case xmlquery.DeclarationNode:
return "declaration"
default:
return "unknown"
}
}
// ConvertToNamedEntities replaces numeric XML entities with their named counterparts
func ConvertToNamedEntities(xml string) string {
// Basic XML entities
replacements := map[string]string{
// Basic XML entities
""": """, // double quote
"'": "'", // single quote
"<": "<", // less than
">": ">", // greater than
"&": "&", // ampersand
// Common symbols
" ": " ", // non-breaking space
"©": "©", // copyright
"®": "®", // registered trademark
"€": "€", // euro
"£": "£", // pound
"¥": "¥", // yen
"¢": "¢", // cent
"§": "§", // section
"™": "™", // trademark
"♠": "♠", // spade
"♣": "♣", // club
"♥": "♥", // heart
"♦": "♦", // diamond
// Special characters
"¡": "¡", // inverted exclamation
"¿": "¿", // inverted question
"«": "«", // left angle quotes
"»": "»", // right angle quotes
"·": "·", // middle dot
"•": "•", // bullet
"…": "…", // horizontal ellipsis
"′": "′", // prime
"″": "″", // double prime
"‾": "‾", // overline
"⁄": "⁄", // fraction slash
// Math symbols
"±": "±", // plus-minus
"×": "×", // multiplication
"÷": "÷", // division
"∞": "∞", // infinity
"≈": "≈", // almost equal
"≠": "≠", // not equal
"≤": "≤", // less than or equal
"≥": "≥", // greater than or equal
"∑": "∑", // summation
"√": "√", // square root
"∫": "∫", // integral
// Accented characters
"À": "À", // A grave
"Á": "Á", // A acute
"Â": "Â", // A circumflex
"Ã": "Ã", // A tilde
"Ä": "Ä", // A umlaut
"Å": "Å", // A ring
"Æ": "Æ", // AE ligature
"Ç": "Ç", // C cedilla
"È": "È", // E grave
"É": "É", // E acute
"Ê": "Ê", // E circumflex
"Ë": "Ë", // E umlaut
"Ì": "Ì", // I grave
"Í": "Í", // I acute
"Î": "Î", // I circumflex
"Ï": "Ï", // I umlaut
"Ð": "Ð", // Eth
"Ñ": "Ñ", // N tilde
"Ò": "Ò", // O grave
"Ó": "Ó", // O acute
"Ô": "Ô", // O circumflex
"Õ": "Õ", // O tilde
"Ö": "Ö", // O umlaut
"Ø": "Ø", // O slash
"Ù": "Ù", // U grave
"Ú": "Ú", // U acute
"Û": "Û", // U circumflex
"Ü": "Ü", // U umlaut
"Ý": "Ý", // Y acute
"Þ": "Þ", // Thorn
"ß": "ß", // Sharp s
"à": "à", // a grave
"á": "á", // a acute
"â": "â", // a circumflex
"ã": "ã", // a tilde
"ä": "ä", // a umlaut
"å": "å", // a ring
"æ": "æ", // ae ligature
"ç": "ç", // c cedilla
"è": "è", // e grave
"é": "é", // e acute
"ê": "ê", // e circumflex
"ë": "ë", // e umlaut
"ì": "ì", // i grave
"í": "í", // i acute
"î": "î", // i circumflex
"ï": "ï", // i umlaut
"ð": "ð", // eth
"ñ": "ñ", // n tilde
"ò": "ò", // o grave
"ó": "ó", // o acute
"ô": "ô", // o circumflex
"õ": "õ", // o tilde
"ö": "ö", // o umlaut
"ø": "ø", // o slash
"ù": "ù", // u grave
"ú": "ú", // u acute
"û": "û", // u circumflex
"ü": "ü", // u umlaut
"ý": "ý", // y acute
"þ": "þ", // thorn
"ÿ": "ÿ", // y umlaut
}
result := xml
for numeric, named := range replacements {
result = strings.ReplaceAll(result, numeric, named)
}
return result
}

1772
processor/xml_test.go
View File

File diff suppressed because it is too large Load Diff