Refactor lua helper script to separate file

And write a few tests for it
This commit is contained in:
2025-11-15 15:42:57 +01:00
parent 3e552428a5
commit 917063db0c
3 changed files with 336 additions and 247 deletions

View File

@@ -0,0 +1,94 @@
-- Load the helper script
dofile("luahelper.lua")
-- Test helper function
local function assert(condition, message)
if not condition then error("ASSERTION FAILED: " .. (message or "unknown error")) end
end
local function test(name, fn)
local ok, err = pcall(fn)
if ok then
print("PASS: " .. name)
else
print("FAIL: " .. name .. " - " .. tostring(err))
end
end
-- Test fromCSV basic parsing
test("fromCSV basic", function()
local csv = "a,b,c\n1,2,3\n4,5,6"
local rows = fromCSV(csv)
assert(#rows == 3, "Should have 3 rows")
assert(rows[1][1] == "a", "First row first field should be 'a'")
assert(rows[2][2] == "2", "Second row second field should be '2'")
end)
-- Test fromCSV with headers
test("fromCSV with headers", function()
local csv = "foo,bar,baz\n1,2,3\n4,5,6"
local rows = fromCSV(csv, ",", true)
assert(#rows == 2, "Should have 2 data rows")
assert(rows[1][1] == "1", "First row first field should be '1'")
assert(rows[1].foo == "1", "First row foo should be '1'")
assert(rows[1].bar == "2", "First row bar should be '2'")
assert(rows[1].baz == "3", "First row baz should be '3'")
end)
-- Test fromCSV with custom delimiter
test("fromCSV with tab delimiter", function()
local csv = "a\tb\tc\n1\t2\t3"
local rows = fromCSV(csv, "\t")
assert(#rows == 2, "Should have 2 rows")
assert(rows[1][1] == "a", "First row first field should be 'a'")
assert(rows[2][2] == "2", "Second row second field should be '2'")
end)
-- Test fromCSV with quoted fields
test("fromCSV with quoted fields", function()
local csv = '"hello,world","test"\n"foo","bar"'
local rows = fromCSV(csv)
assert(#rows == 2, "Should have 2 rows")
assert(rows[1][1] == "hello,world", "Quoted field with comma should be preserved")
assert(rows[1][2] == "test", "Second field should be 'test'")
end)
-- Test toCSV basic
test("toCSV basic", function()
local rows = { { "a", "b", "c" }, { "1", "2", "3" } }
local csv = toCSV(rows)
assert(csv == "a,b,c\n1,2,3", "CSV output should match expected")
end)
-- Test toCSV with custom delimiter
test("toCSV with tab delimiter", function()
local rows = { { "a", "b", "c" }, { "1", "2", "3" } }
local csv = toCSV(rows, "\t")
assert(csv == "a\tb\tc\n1\t2\t3", "TSV output should match expected")
end)
-- Test toCSV with fields needing quoting
test("toCSV with quoted fields", function()
local rows = { { "hello,world", "test" }, { "foo", "bar" } }
local csv = toCSV(rows)
assert(csv == '"hello,world",test\nfoo,bar', "Fields with commas should be quoted")
end)
-- Test round trip
test("fromCSV toCSV round trip", function()
local original = "a,b,c\n1,2,3\n4,5,6"
local rows = fromCSV(original)
local csv = toCSV(rows)
assert(csv == original, "Round trip should preserve original")
end)
-- Test round trip with headers
test("fromCSV toCSV round trip with headers", function()
local original = "foo,bar,baz\n1,2,3\n4,5,6"
local rows = fromCSV(original, ",", true)
local csv = toCSV(rows)
local expected = "1,2,3\n4,5,6"
assert(csv == expected, "Round trip with headers should preserve data rows")
end)
print("\nAll tests completed!")

235
processor/luahelper.lua Normal file
View File

@@ -0,0 +1,235 @@
-- Custom Lua helpers for math operations
function min(a, b) return math.min(a, b) end
function max(a, b) return math.max(a, b) end
function round(x, n)
if n == nil then n = 0 end
return math.floor(x * 10 ^ n + 0.5) / 10 ^ n
end
function floor(x) return math.floor(x) end
function ceil(x) return math.ceil(x) end
function upper(s) return string.upper(s) end
function lower(s) return string.lower(s) end
function format(s, ...) return string.format(s, ...) end
function trim(s) return string.gsub(s, "^%s*(.-)%s*$", "%1") end
-- String split helper
function strsplit(inputstr, sep)
if sep == nil then sep = "%s" end
local t = {}
for str in string.gmatch(inputstr, "([^" .. sep .. "]+)") do
table.insert(t, str)
end
return t
end
---@param table table
---@param depth number?
function dump(table, depth)
if depth == nil then depth = 0 end
if depth > 200 then
print("Error: Depth > 200 in dump()")
return
end
for k, v in pairs(table) do
if type(v) == "table" then
print(string.rep(" ", depth) .. k .. ":")
dump(v, depth + 1)
else
print(string.rep(" ", depth) .. k .. ": ", v)
end
end
end
--- Parses CSV text into rows and fields using a minimal RFC 4180 state machine.
---
--- Requirements/assumptions:
--- - Input is a single string containing the entire CSV content.
--- - Field separators are specified by delimiter parameter (default: comma).
--- - Newlines between rows may be "\n" or "\r\n". "\r\n" is treated as one line break.
--- - Fields may be quoted with double quotes (").
--- - Inside quoted fields, doubled quotes ("") represent a literal quote character.
--- - No backslash escaping is supported (not part of RFC 4180).
--- - Newlines inside quoted fields are preserved as part of the field.
--- - Leading/trailing spaces are preserved; no trimming is performed.
--- - Empty fields and empty rows are preserved.
--- - The final row is emitted even if the text does not end with a newline.
---
--- @param csv string The CSV text to parse.
--- @param delimiter string? The field delimiter (default: ",").
--- @param hasHeaders boolean? If true, first row is treated as headers and rows can be accessed by header name (default: false).
--- @return table A table (array) of rows; each row is a table with numeric indices and optionally header-named keys.
function fromCSV(csv, delimiter, hasHeaders)
if delimiter == nil then delimiter = "," end
if hasHeaders == nil then hasHeaders = false end
local allRows = {}
local fields = {}
local field = {}
local STATE_DEFAULT = 1
local STATE_IN_QUOTES = 2
local STATE_QUOTE_IN_QUOTES = 3
local state = STATE_DEFAULT
local i = 1
local len = #csv
while i <= len do
local c = csv:sub(i, i)
if state == STATE_DEFAULT then
if c == '"' then
state = STATE_IN_QUOTES
i = i + 1
elseif c == delimiter then
table.insert(fields, table.concat(field))
field = {}
i = i + 1
elseif c == "\r" or c == "\n" then
table.insert(fields, table.concat(field))
field = {}
table.insert(allRows, fields)
fields = {}
if c == "\r" and i < len and csv:sub(i + 1, i + 1) == "\n" then
i = i + 2
else
i = i + 1
end
else
table.insert(field, c)
i = i + 1
end
elseif state == STATE_IN_QUOTES then
if c == '"' then
state = STATE_QUOTE_IN_QUOTES
i = i + 1
else
table.insert(field, c)
i = i + 1
end
else -- STATE_QUOTE_IN_QUOTES
if c == '"' then
table.insert(field, '"')
state = STATE_IN_QUOTES
i = i + 1
elseif c == delimiter then
table.insert(fields, table.concat(field))
field = {}
state = STATE_DEFAULT
i = i + 1
elseif c == "\r" or c == "\n" then
table.insert(fields, table.concat(field))
field = {}
table.insert(allRows, fields)
fields = {}
state = STATE_DEFAULT
if c == "\r" and i < len and csv:sub(i + 1, i + 1) == "\n" then
i = i + 2
else
i = i + 1
end
else
state = STATE_DEFAULT
-- Don't increment i, reprocess character in DEFAULT state
end
end
end
if #field > 0 or #fields > 0 then
table.insert(fields, table.concat(field))
table.insert(allRows, fields)
end
if hasHeaders and #allRows > 0 then
local headers = allRows[1]
local rows = {}
for i = 2, #allRows do
local row = {}
local dataRow = allRows[i]
for j = 1, #dataRow do
row[j] = dataRow[j]
if headers[j] ~= nil and headers[j] ~= "" then row[headers[j]] = dataRow[j] end
end
table.insert(rows, row)
end
return rows
end
return allRows
end
--- Converts a table of rows back to CSV text format (RFC 4180 compliant).
---
--- Requirements:
--- - Input is a table (array) of rows, where each row is a table (array) of field values.
--- - Field values are converted to strings using tostring().
--- - Fields are quoted if they contain the delimiter, newlines, or double quotes.
--- - Double quotes inside quoted fields are doubled ("").
--- - Fields are joined with the specified delimiter; rows are joined with newlines.
---
--- @param rows table Array of rows, where each row is an array of field values.
--- @param delimiter string? The field delimiter (default: ",").
--- @return string CSV-formatted text.
function toCSV(rows, delimiter)
if delimiter == nil then delimiter = "," end
local rowStrings = {}
for _, row in ipairs(rows) do
local fieldStrings = {}
for _, field in ipairs(row) do
local fieldStr = tostring(field)
local needsQuoting = false
if
fieldStr:find(delimiter)
or fieldStr:find("\n")
or fieldStr:find("\r")
or fieldStr:find('"')
then
needsQuoting = true
end
if needsQuoting then
fieldStr = fieldStr:gsub('"', '""')
fieldStr = '"' .. fieldStr .. '"'
end
table.insert(fieldStrings, fieldStr)
end
table.insert(rowStrings, table.concat(fieldStrings, delimiter))
end
return table.concat(rowStrings, "\n")
end
-- String to number conversion helper
function num(str) return tonumber(str) or 0 end
-- Number to string conversion
function str(num) return tostring(num) end
-- Check if string is numeric
function is_number(str) return tonumber(str) ~= nil end
function isArray(t)
if type(t) ~= "table" then return false end
local max = 0
local count = 0
for k, _ in pairs(t) do
if type(k) ~= "number" or k < 1 or math.floor(k) ~= k then return false end
max = math.max(max, k)
count = count + 1
end
return max == count
end
modified = false

View File

@@ -1,6 +1,7 @@
package processor
import (
_ "embed"
"fmt"
"io"
"net/http"
@@ -13,6 +14,9 @@ import (
lua "github.com/yuin/gopher-lua"
)
//go:embed luahelper.lua
var helperScript string
// processorLogger is a scoped logger for the processor package.
var processorLogger = logger.Default.WithPrefix("processor")
@@ -160,250 +164,6 @@ func InitLuaHelpers(L *lua.LState) error {
initLuaHelpersLogger := processorLogger.WithPrefix("InitLuaHelpers")
initLuaHelpersLogger.Debug("Loading Lua helper functions")
helperScript := `
-- Custom Lua helpers for math operations
function min(a, b) return math.min(a, b) end
function max(a, b) return math.max(a, b) end
function round(x, n)
if n == nil then n = 0 end
return math.floor(x * 10^n + 0.5) / 10^n
end
function floor(x) return math.floor(x) end
function ceil(x) return math.ceil(x) end
function upper(s) return string.upper(s) end
function lower(s) return string.lower(s) end
function format(s, ...) return string.format(s, ...) end
function trim(s) return string.gsub(s, "^%s*(.-)%s*$", "%1") end
-- String split helper
function strsplit(inputstr, sep)
if sep == nil then
sep = "%s"
end
local t = {}
for str in string.gmatch(inputstr, "([^"..sep.."]+)") do
table.insert(t, str)
end
return t
end
---@param table table
---@param depth number?
function dump(table, depth)
if depth == nil then
depth = 0
end
if (depth > 200) then
print("Error: Depth > 200 in dump()")
return
end
for k, v in pairs(table) do
if (type(v) == "table") then
print(string.rep(" ", depth) .. k .. ":")
dump(v, depth + 1)
else
print(string.rep(" ", depth) .. k .. ": ", v)
end
end
end
--- Parses CSV text into rows and fields using a minimal RFC 4180 state machine.
---
--- Requirements/assumptions:
--- - Input is a single string containing the entire CSV content.
--- - Field separators are specified by delimiter parameter (default: comma).
--- - Newlines between rows may be "\n" or "\r\n". "\r\n" is treated as one line break.
--- - Fields may be quoted with double quotes (").
--- - Inside quoted fields, doubled quotes ("") represent a literal quote character.
--- - No backslash escaping is supported (not part of RFC 4180).
--- - Newlines inside quoted fields are preserved as part of the field.
--- - Leading/trailing spaces are preserved; no trimming is performed.
--- - Empty fields and empty rows are preserved.
--- - The final row is emitted even if the text does not end with a newline.
---
--- @param csv string The CSV text to parse.
--- @param delimiter string? The field delimiter (default: ",").
--- @param hasHeaders boolean? If true, first row is treated as headers and rows can be accessed by header name (default: false).
--- @return table A table (array) of rows; each row is a table with numeric indices and optionally header-named keys.
function fromCSV(csv, delimiter, hasHeaders)
if delimiter == nil then
delimiter = ","
end
if hasHeaders == nil then
hasHeaders = false
end
local allRows = {}
local fields = {}
local field = {}
local STATE_DEFAULT = 1
local STATE_IN_QUOTES = 2
local STATE_QUOTE_IN_QUOTES = 3
local state = STATE_DEFAULT
local i = 1
local len = #csv
while i <= len do
local c = csv:sub(i, i)
if state == STATE_DEFAULT then
if c == '"' then
state = STATE_IN_QUOTES
i = i + 1
elseif c == delimiter then
table.insert(fields, table.concat(field))
field = {}
i = i + 1
elseif c == '\r' or c == '\n' then
table.insert(fields, table.concat(field))
field = {}
table.insert(allRows, fields)
fields = {}
if c == '\r' and i < len and csv:sub(i + 1, i + 1) == '\n' then
i = i + 2
else
i = i + 1
end
else
table.insert(field, c)
i = i + 1
end
elseif state == STATE_IN_QUOTES then
if c == '"' then
state = STATE_QUOTE_IN_QUOTES
i = i + 1
else
table.insert(field, c)
i = i + 1
end
else -- STATE_QUOTE_IN_QUOTES
if c == '"' then
table.insert(field, '"')
state = STATE_IN_QUOTES
i = i + 1
elseif c == delimiter then
table.insert(fields, table.concat(field))
field = {}
state = STATE_DEFAULT
i = i + 1
elseif c == '\r' or c == '\n' then
table.insert(fields, table.concat(field))
field = {}
table.insert(allRows, fields)
fields = {}
state = STATE_DEFAULT
if c == '\r' and i < len and csv:sub(i + 1, i + 1) == '\n' then
i = i + 2
else
i = i + 1
end
else
state = STATE_DEFAULT
-- Don't increment i, reprocess character in DEFAULT state
end
end
end
if #field > 0 or #fields > 0 then
table.insert(fields, table.concat(field))
table.insert(allRows, fields)
end
if hasHeaders and #allRows > 0 then
local headers = allRows[1]
local rows = {}
for i = 2, #allRows do
local row = {}
local dataRow = allRows[i]
for j = 1, #dataRow do
row[j] = dataRow[j]
if headers[j] ~= nil and headers[j] ~= "" then
row[headers[j]] = dataRow[j]
end
end
table.insert(rows, row)
end
return rows
end
return allRows
end
--- Converts a table of rows back to CSV text format (RFC 4180 compliant).
---
--- Requirements:
--- - Input is a table (array) of rows, where each row is a table (array) of field values.
--- - Field values are converted to strings using tostring().
--- - Fields are quoted if they contain the delimiter, newlines, or double quotes.
--- - Double quotes inside quoted fields are doubled ("").
--- - Fields are joined with the specified delimiter; rows are joined with newlines.
---
--- @param rows table Array of rows, where each row is an array of field values.
--- @param delimiter string? The field delimiter (default: ",").
--- @return string CSV-formatted text.
function toCSV(rows, delimiter)
if delimiter == nil then
delimiter = ","
end
local rowStrings = {}
for _, row in ipairs(rows) do
local fieldStrings = {}
for _, field in ipairs(row) do
local fieldStr = tostring(field)
local needsQuoting = false
if fieldStr:find(delimiter) or fieldStr:find('\n') or fieldStr:find('\r') or fieldStr:find('"') then
needsQuoting = true
end
if needsQuoting then
fieldStr = fieldStr:gsub('"', '""')
fieldStr = '"' .. fieldStr .. '"'
end
table.insert(fieldStrings, fieldStr)
end
table.insert(rowStrings, table.concat(fieldStrings, delimiter))
end
return table.concat(rowStrings, '\n')
end
-- String to number conversion helper
function num(str)
return tonumber(str) or 0
end
-- Number to string conversion
function str(num)
return tostring(num)
end
-- Check if string is numeric
function is_number(str)
return tonumber(str) ~= nil
end
function isArray(t)
if type(t) ~= "table" then return false end
local max = 0
local count = 0
for k, _ in pairs(t) do
if type(k) ~= "number" or k < 1 or math.floor(k) ~= k then
return false
end
max = math.max(max, k)
count = count + 1
end
return max == count
end
modified = false
`
if err := L.DoString(helperScript); err != nil {
initLuaHelpersLogger.Error("Failed to load Lua helper functions: %v", err)
return fmt.Errorf("error loading helper functions: %v", err)
@@ -551,9 +311,9 @@ func fetch(L *lua.LState) int {
fetchLogger.Debug("Fetching URL: %q", url)
// Get options from second argument if provided
var method string = "GET"
var headers map[string]string = make(map[string]string)
var body string = ""
var method = "GET"
var headers = make(map[string]string)
var body = ""
if L.GetTop() > 1 {
options := L.ToTable(2)