Implement a more better logging solution

This commit is contained in:
2025-03-27 17:47:39 +01:00
parent 81d8259dfc
commit 9cea103042
7 changed files with 579 additions and 78 deletions

View File

@@ -3,7 +3,7 @@ package processor
import (
"encoding/json"
"fmt"
"log"
"modify/logger"
"modify/processor/jsonpath"
lua "github.com/yuin/gopher-lua"
@@ -14,80 +14,98 @@ type JSONProcessor struct{}
// ProcessContent implements the Processor interface for JSONProcessor
func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr string) (string, int, int, error) {
logger.Debug("Processing JSON content with JSONPath: %s", pattern)
// Parse JSON document
logger.Trace("Parsing JSON document")
var jsonData interface{}
err := json.Unmarshal([]byte(content), &jsonData)
if err != nil {
logger.Error("Failed to parse JSON: %v", err)
return content, 0, 0, fmt.Errorf("error parsing JSON: %v", err)
}
// Find nodes matching the JSONPath pattern
logger.Debug("Executing JSONPath query: %s", pattern)
nodes, err := jsonpath.Get(jsonData, pattern)
if err != nil {
logger.Error("Failed to execute JSONPath: %v", err)
return content, 0, 0, fmt.Errorf("error getting nodes: %v", err)
}
matchCount := len(nodes)
logger.Debug("Found %d nodes matching JSONPath", matchCount)
if matchCount == 0 {
logger.Warning("No nodes matched the JSONPath pattern: %s", pattern)
return content, 0, 0, nil
}
modCount := 0
for _, node := range nodes {
log.Printf("Processing node at path: %s with value: %v", node.Path, node.Value)
for i, node := range nodes {
logger.Trace("Processing node #%d at path: %s with value: %v", i+1, node.Path, node.Value)
// Initialize Lua
L, err := NewLuaState()
if err != nil {
logger.Error("Failed to create Lua state: %v", err)
return content, len(nodes), 0, fmt.Errorf("error creating Lua state: %v", err)
}
defer L.Close()
log.Println("Lua state initialized successfully.")
logger.Trace("Lua state initialized successfully")
err = p.ToLua(L, node.Value)
if err != nil {
logger.Error("Failed to convert value to Lua: %v", err)
return content, len(nodes), 0, fmt.Errorf("error converting to Lua: %v", err)
}
log.Printf("Converted node value to Lua: %v", node.Value)
logger.Trace("Converted node value to Lua: %v", node.Value)
originalScript := luaExpr
fullScript := BuildLuaScript(luaExpr)
log.Printf("Original script: %q, Full script: %q", originalScript, fullScript)
logger.Debug("Original script: %q, Full script: %q", originalScript, fullScript)
// Execute Lua script
log.Printf("Executing Lua script: %q", fullScript)
logger.Trace("Executing Lua script: %q", fullScript)
if err := L.DoString(fullScript); err != nil {
logger.Error("Failed to execute Lua script: %v", err)
return content, len(nodes), 0, fmt.Errorf("error executing Lua %q: %v", fullScript, err)
}
log.Println("Lua script executed successfully.")
logger.Trace("Lua script executed successfully")
// Get modified value
result, err := p.FromLua(L)
if err != nil {
logger.Error("Failed to get result from Lua: %v", err)
return content, len(nodes), 0, fmt.Errorf("error getting result from Lua: %v", err)
}
log.Printf("Retrieved modified value from Lua: %v", result)
logger.Trace("Retrieved modified value from Lua: %v", result)
modified := false
modified = L.GetGlobal("modified").String() == "true"
if !modified {
log.Printf("No changes made to node at path: %s", node.Path)
logger.Debug("No changes made to node at path: %s", node.Path)
continue
}
// Apply the modification to the JSON data
logger.Debug("Updating JSON at path: %s with new value: %v", node.Path, result)
err = p.updateJSONValue(jsonData, node.Path, result)
if err != nil {
logger.Error("Failed to update JSON at path %s: %v", node.Path, err)
return content, len(nodes), 0, fmt.Errorf("error updating JSON: %v", err)
}
log.Printf("Updated JSON at path: %s with new value: %v", node.Path, result)
logger.Debug("Updated JSON at path: %s successfully", node.Path)
modCount++
}
logger.Info("JSON processing complete: %d modifications from %d matches", modCount, matchCount)
// Convert the modified JSON back to a string with same formatting
logger.Trace("Marshalling JSON data back to string")
var jsonBytes []byte
jsonBytes, err = json.MarshalIndent(jsonData, "", " ")
if err != nil {
logger.Error("Failed to marshal JSON: %v", err)
return content, modCount, matchCount, fmt.Errorf("error marshalling JSON: %v", err)
}
return string(jsonBytes), modCount, matchCount, nil
@@ -95,8 +113,11 @@ func (p *JSONProcessor) ProcessContent(content string, pattern string, luaExpr s
// updateJSONValue updates a value in the JSON structure based on its JSONPath
func (p *JSONProcessor) updateJSONValue(jsonData interface{}, path string, newValue interface{}) error {
logger.Trace("Updating JSON value at path: %s", path)
// Special handling for root node
if path == "$" {
logger.Debug("Handling special case for root node update")
// For the root node, we'll copy the value to the jsonData reference
// This is a special case since we can't directly replace the interface{} variable
@@ -108,15 +129,18 @@ func (p *JSONProcessor) updateJSONValue(jsonData interface{}, path string, newVa
if !ok {
// If the original wasn't a map, completely replace it with the new map
// This is handled by the jsonpath.Set function
logger.Debug("Root was not a map, replacing entire root")
return jsonpath.Set(jsonData, path, newValue)
}
// Clear the original map
logger.Trace("Clearing original root map")
for k := range rootMap {
delete(rootMap, k)
}
// Copy all keys from the new map
logger.Trace("Copying keys to root map")
for k, v := range rootValue {
rootMap[k] = v
}
@@ -127,22 +151,27 @@ func (p *JSONProcessor) updateJSONValue(jsonData interface{}, path string, newVa
rootArray, ok := jsonData.([]interface{})
if !ok {
// If the original wasn't an array, use jsonpath.Set
logger.Debug("Root was not an array, replacing entire root")
return jsonpath.Set(jsonData, path, newValue)
}
// Clear and recreate the array
logger.Trace("Replacing root array")
*&rootArray = rootValue
return nil
default:
// For other types, use jsonpath.Set
logger.Debug("Replacing root with primitive value")
return jsonpath.Set(jsonData, path, newValue)
}
}
// For non-root paths, use the regular Set method
logger.Trace("Using regular Set method for non-root path")
err := jsonpath.Set(jsonData, path, newValue)
if err != nil {
logger.Error("Failed to set JSON value at path %s: %v", path, err)
return fmt.Errorf("failed to update JSON value at path '%s': %w", path, err)
}
return nil

View File

@@ -2,13 +2,14 @@ package processor
import (
"fmt"
"log"
"os"
"path/filepath"
"strings"
"github.com/antchfx/xmlquery"
lua "github.com/yuin/gopher-lua"
"modify/logger"
)
// Processor defines the interface for all file processors
@@ -42,14 +43,18 @@ func NewLuaState() (*lua.LState, error) {
// defer L.Close()
// Load math library
logger.Debug("Loading Lua math library")
L.Push(L.GetGlobal("require"))
L.Push(lua.LString("math"))
if err := L.PCall(1, 1, nil); err != nil {
logger.Error("Failed to load Lua math library: %v", err)
return nil, fmt.Errorf("error loading Lua math library: %v", err)
}
// Initialize helper functions
logger.Debug("Initializing Lua helper functions")
if err := InitLuaHelpers(L); err != nil {
logger.Error("Failed to initialize Lua helper functions: %v", err)
return nil, err
}
@@ -60,29 +65,40 @@ func Process(p Processor, filename string, pattern string, luaExpr string) (int,
// Read file content
cwd, err := os.Getwd()
if err != nil {
logger.Error("Failed to get current working directory: %v", err)
return 0, 0, fmt.Errorf("error getting current working directory: %v", err)
}
fullPath := filepath.Join(cwd, filename)
logger.Trace("Reading file content from: %s", fullPath)
content, err := os.ReadFile(fullPath)
if err != nil {
logger.Error("Failed to read file %s: %v", fullPath, err)
return 0, 0, fmt.Errorf("error reading file: %v", err)
}
fileContent := string(content)
logger.Trace("File %s read successfully, size: %d bytes", fullPath, len(content))
// Process the content
logger.Debug("Processing content for file: %s", filename)
modifiedContent, modCount, matchCount, err := p.ProcessContent(fileContent, pattern, luaExpr)
if err != nil {
logger.Error("Error processing content for file %s: %v", filename, err)
return 0, 0, err
}
// If we made modifications, save the file
if modCount > 0 {
logger.Info("Writing %d modifications to file: %s", modCount, filename)
err = os.WriteFile(fullPath, []byte(modifiedContent), 0644)
if err != nil {
logger.Error("Failed to write to file %s: %v", fullPath, err)
return 0, 0, fmt.Errorf("error writing file: %v", err)
}
logger.Debug("File %s written successfully", filename)
} else {
logger.Debug("No modifications to write for file: %s", filename)
}
return modCount, matchCount, nil
@@ -174,25 +190,28 @@ func FromLua(L *lua.LState, luaValue lua.LValue) (interface{}, error) {
}
func IsLuaTableArray(L *lua.LState, v *lua.LTable) (bool, error) {
logger.Trace("Checking if Lua table is an array")
L.SetGlobal("table_to_check", v)
// Use our predefined helper function from InitLuaHelpers
err := L.DoString(`is_array = isArray(table_to_check)`)
if err != nil {
logger.Error("Error determining if table is an array: %v", err)
return false, fmt.Errorf("error determining if table is array: %w", err)
}
// Check the result of our Lua function
isArray := L.GetGlobal("is_array")
// LVIsFalse returns true if a given LValue is a nil or false otherwise false.
if !lua.LVIsFalse(isArray) {
return true, nil
}
return false, nil
result := !lua.LVIsFalse(isArray)
logger.Trace("Lua table is array: %v", result)
return result, nil
}
// InitLuaHelpers initializes common Lua helper functions
func InitLuaHelpers(L *lua.LState) error {
logger.Debug("Loading Lua helper functions")
helperScript := `
-- Custom Lua helpers for math operations
function min(a, b) return math.min(a, b) end
@@ -239,9 +258,11 @@ end
modified = false
`
if err := L.DoString(helperScript); err != nil {
logger.Error("Failed to load Lua helper functions: %v", err)
return fmt.Errorf("error loading helper functions: %v", err)
}
logger.Debug("Setting up Lua print function to Go")
L.SetGlobal("print", L.NewFunction(printToGo))
return nil
}
@@ -280,6 +301,8 @@ func PrependLuaAssignment(luaExpr string) string {
// BuildLuaScript prepares a Lua expression from shorthand notation
func BuildLuaScript(luaExpr string) string {
logger.Debug("Building Lua script from expression: %s", luaExpr)
luaExpr = PrependLuaAssignment(luaExpr)
// This allows the user to specify whether or not they modified a value
@@ -300,17 +323,14 @@ func BuildLuaScript(luaExpr string) string {
}
func printToGo(L *lua.LState) int {
// Get the number of arguments passed to the Lua print function
n := L.GetTop()
// Create a slice to hold the arguments
args := make([]interface{}, n)
for i := 1; i <= n; i++ {
args[i-1] = L.Get(i) // Get the argument from Lua stack
top := L.GetTop()
args := make([]interface{}, top)
for i := 1; i <= top; i++ {
args[i-1] = L.Get(i)
}
// Print the arguments to Go's stdout
log.Print("Lua: ")
log.Println(args...)
return 0 // No return values
message := fmt.Sprint(args...)
logger.Info("[Lua] %s", message)
return 0
}
// Max returns the maximum of two integers

View File

@@ -2,13 +2,14 @@ package processor
import (
"fmt"
"log"
"regexp"
"sort"
"strconv"
"strings"
lua "github.com/yuin/gopher-lua"
"modify/logger"
)
// RegexProcessor implements the Processor interface using regex patterns
@@ -98,16 +99,17 @@ type ReplaceCommand struct {
// ProcessContent applies regex replacement with Lua processing
func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr string) (string, int, int, error) {
pattern = ResolveRegexPlaceholders(pattern)
logger.Debug("Compiling regex pattern: %s", pattern)
compiledPattern, err := regexp.Compile(pattern)
if err != nil {
log.Printf("Error compiling pattern: %v", err)
logger.Error("Error compiling pattern: %v", err)
return "", 0, 0, fmt.Errorf("error compiling pattern: %v", err)
}
log.Printf("Compiled pattern successfully: %s", pattern)
logger.Debug("Compiled pattern successfully: %s", pattern)
previous := luaExpr
luaExpr = BuildLuaScript(luaExpr)
log.Printf("Changing Lua expression from: %s to: %s", previous, luaExpr)
logger.Debug("Changing Lua expression from: %s to: %s", previous, luaExpr)
// Initialize Lua environment
modificationCount := 0
@@ -115,7 +117,7 @@ func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr
// Process all regex matches
result := content
indices := compiledPattern.FindAllStringSubmatchIndex(content, -1)
log.Printf("Found %d matches in the content", len(indices))
logger.Debug("Found %d matches in the content", len(indices))
// We walk backwards because we're replacing something with something else that might be longer
// And in the case it is longer than the original all indicces past that change will be fucked up
@@ -124,17 +126,17 @@ func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr
for i := len(indices) - 1; i >= 0; i-- {
L, err := NewLuaState()
if err != nil {
log.Printf("Error creating Lua state: %v", err)
logger.Error("Error creating Lua state: %v", err)
return "", 0, 0, fmt.Errorf("error creating Lua state: %v", err)
}
// Hmm... Maybe we don't want to defer this..
// Maybe we want to close them every iteration
// We'll leave it as is for now
defer L.Close()
log.Printf("Lua state created successfully")
logger.Trace("Lua state created successfully")
matchIndices := indices[i]
log.Printf("Processing match indices: %v", matchIndices)
logger.Trace("Processing match indices: %v", matchIndices)
// Why we're doing this whole song and dance of indices is to properly handle empty matches
// Plus it's a little cleaner to surgically replace our matches
@@ -144,21 +146,21 @@ func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr
// As if concatenating in the middle of the array
// Plus it supports lookarounds
match := content[matchIndices[0]:matchIndices[1]]
log.Printf("Matched content: %s", match)
logger.Trace("Matched content: %s", match)
groups := matchIndices[2:]
if len(groups) <= 0 {
log.Println("No capture groups for lua to chew on")
logger.Warning("No capture groups for lua to chew on")
continue
}
if len(groups)%2 == 1 {
log.Println("Odd number of indices of groups, what the fuck?")
logger.Warning("Odd number of indices of groups, what the fuck?")
continue
}
for _, index := range groups {
if index == -1 {
// return "", 0, 0, fmt.Errorf("negative indices encountered: %v. This indicates that there was an issue with the match indices, possibly due to an empty match or an unexpected pattern. Please check the regex pattern and input content.", matchIndices)
log.Printf("Negative indices encountered: %v. This indicates that there was an issue with the match indices, possibly due to an empty match or an unexpected pattern. This is not an error but it's possibly not what you want.", matchIndices)
logger.Warning("Negative indices encountered: %v. This indicates that there was an issue with the match indices, possibly due to an empty match or an unexpected pattern. This is not an error but it's possibly not what you want.", matchIndices)
continue
}
}
@@ -187,25 +189,25 @@ func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr
}
for _, capture := range captureGroups {
log.Printf("Capture group: %+v", *capture)
logger.Trace("Capture group: %+v", *capture)
}
if err := p.ToLua(L, captureGroups); err != nil {
log.Printf("Error setting Lua variables: %v", err)
logger.Error("Error setting Lua variables: %v", err)
continue
}
log.Println("Lua variables set successfully")
logger.Trace("Lua variables set successfully")
if err := L.DoString(luaExpr); err != nil {
log.Printf("Error executing Lua code %s for groups %+v: %v", luaExpr, captureGroups, err)
logger.Error("Error executing Lua code %s for groups %+v: %v", luaExpr, captureGroups, err)
continue
}
log.Println("Lua code executed successfully")
logger.Trace("Lua code executed successfully")
// Get modifications from Lua
captureGroups, err = p.FromLuaCustom(L, captureGroups)
if err != nil {
log.Printf("Error getting modifications: %v", err)
logger.Error("Error getting modifications: %v", err)
continue
}
@@ -214,12 +216,20 @@ func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr
if replacementVar.Type() != lua.LTNil {
replacement = replacementVar.String()
}
// Check if modification flag is set
modifiedVal := L.GetGlobal("modified")
if modifiedVal.Type() != lua.LTBool || !lua.LVAsBool(modifiedVal) {
logger.Debug("No modifications made by Lua script")
continue
}
if replacement == "" {
commands := make([]ReplaceCommand, 0, len(captureGroups))
// Apply the modifications to the original match
replacement = match
for _, capture := range captureGroups {
log.Printf("Applying modification: %s", capture.Updated)
logger.Debug("Applying modification: %s", capture.Updated)
// Indices of the group are relative to content
// To relate them to match we have to subtract the match start index
// replacement = replacement[:groupStart] + newVal + replacement[groupEnd:]
@@ -238,12 +248,13 @@ func (p *RegexProcessor) ProcessContent(content string, pattern string, luaExpr
replacement = replacement[:command.From] + command.With + replacement[command.To:]
}
}
modificationCount++
result = result[:matchIndices[0]] + replacement + result[matchIndices[1]:]
log.Printf("Modification count updated: %d", modificationCount)
logger.Debug("Modification count updated: %d", modificationCount)
}
log.Printf("Process completed with %d modifications", modificationCount)
logger.Debug("Process completed with %d modifications", modificationCount)
return result, modificationCount, len(indices), nil
}
@@ -256,7 +267,8 @@ func ResolveRegexPlaceholders(pattern string) string {
// Handle special pattern modifications
if !strings.HasPrefix(pattern, "(?s)") {
pattern = "(?s)" + pattern
log.Printf("Pattern modified to include (?s): %s", pattern)
// Use fmt.Printf for test compatibility
fmt.Printf("Pattern modified to include (?s): %s\n", pattern)
}
namedGroupNum := regexp.MustCompile(`(?:(\?<[^>]+>)(!num))`)

22
processor/test_helper.go Normal file
View File

@@ -0,0 +1,22 @@
package processor
import (
"io/ioutil"
"modify/logger"
"os"
)
func init() {
// Initialize logger with ERROR level for tests
// to minimize noise in test output
logger.Init(logger.LevelError)
// Optionally redirect logger output to discard
// This prevents logger output from interfering with test output
disableTestLogs := os.Getenv("ENABLE_TEST_LOGS") != "1"
if disableTestLogs {
// Create a new logger that writes to nowhere
silentLogger := logger.New(ioutil.Discard, "", 0)
logger.DefaultLogger = silentLogger
}
}

View File

@@ -2,7 +2,7 @@ package processor
import (
"fmt"
"log"
"modify/logger"
"modify/processor/xpath"
"strings"
@@ -15,69 +15,91 @@ type XMLProcessor struct{}
// ProcessContent implements the Processor interface for XMLProcessor
func (p *XMLProcessor) ProcessContent(content string, path string, luaExpr string) (string, int, int, error) {
logger.Debug("Processing XML content with XPath: %s", path)
// Parse XML document
// We can't really use encoding/xml here because it requires a pre defined struct
// And we HAVE TO parse dynamic unknown XML
logger.Trace("Parsing XML document")
doc, err := xmlquery.Parse(strings.NewReader(content))
if err != nil {
logger.Error("Failed to parse XML: %v", err)
return content, 0, 0, fmt.Errorf("error parsing XML: %v", err)
}
// Find nodes matching the XPath pattern
logger.Debug("Executing XPath query: %s", path)
nodes, err := xpath.Get(doc, path)
if err != nil {
logger.Error("Failed to execute XPath: %v", err)
return content, 0, 0, fmt.Errorf("error executing XPath: %v", err)
}
matchCount := len(nodes)
logger.Debug("Found %d nodes matching XPath", matchCount)
if matchCount == 0 {
logger.Warning("No nodes matched the XPath pattern: %s", path)
return content, 0, 0, nil
}
// Apply modifications to each node
modCount := 0
for _, node := range nodes {
for i, node := range nodes {
logger.Trace("Processing node #%d: %s", i+1, node.Data)
L, err := NewLuaState()
if err != nil {
logger.Error("Failed to create Lua state: %v", err)
return content, 0, 0, fmt.Errorf("error creating Lua state: %v", err)
}
defer L.Close()
logger.Trace("Converting XML node to Lua")
err = p.ToLua(L, node)
if err != nil {
logger.Error("Failed to convert XML node to Lua: %v", err)
return content, modCount, matchCount, fmt.Errorf("error converting to Lua: %v", err)
}
err = L.DoString(BuildLuaScript(luaExpr))
luaScript := BuildLuaScript(luaExpr)
logger.Trace("Executing Lua script: %s", luaScript)
err = L.DoString(luaScript)
if err != nil {
logger.Error("Failed to execute Lua script: %v", err)
return content, modCount, matchCount, fmt.Errorf("error executing Lua: %v", err)
}
result, err := p.FromLua(L)
if err != nil {
logger.Error("Failed to get result from Lua: %v", err)
return content, modCount, matchCount, fmt.Errorf("error getting result from Lua: %v", err)
}
log.Printf("%#v", result)
logger.Trace("Lua returned result: %#v", result)
modified := false
modified = L.GetGlobal("modified").String() == "true"
if !modified {
log.Printf("No changes made to node at path: %s", node.Data)
logger.Debug("No changes made to node at path: %s", node.Data)
continue
}
// Apply modification based on the result
if updatedValue, ok := result.(string); ok {
// If the result is a simple string, update the node value directly
logger.Debug("Updating node with string value: %s", updatedValue)
xpath.Set(doc, path, updatedValue)
} else if nodeData, ok := result.(map[string]interface{}); ok {
// If the result is a map, apply more complex updates
logger.Debug("Updating node with complex data structure")
updateNodeFromMap(node, nodeData)
}
modCount++
logger.Debug("Successfully modified node #%d", i+1)
}
logger.Info("XML processing complete: %d modifications from %d matches", modCount, matchCount)
// Serialize the modified XML document to string
if doc.FirstChild != nil && doc.FirstChild.Type == xmlquery.DeclarationNode {
// If we have an XML declaration, start with it