diff --git a/processor/luahelper-test.lua b/processor/luahelper-test.lua index a2fc6df..eb38523 100644 --- a/processor/luahelper-test.lua +++ b/processor/luahelper-test.lua @@ -15,10 +15,29 @@ local function test(name, fn) end end +-- Test fromCSV option validation +test("fromCSV invalid option", function() + local csv = "a,b,c\n1,2,3" + local rows, err = fromCSV(csv, { invalidOption = true }) + assert(rows ~= nil and #rows == 0, "Should return empty table on error") + assert(err ~= nil, "Should return error message") + assert(string.find(err, "unknown option"), "Error should mention unknown option") +end) + +-- Test toCSV error handling +test("toCSV invalid delimiter", function() + local rows = { { "a", "b", "c" } } + local csv, err = toCSV(rows, { delimiter = 123 }) + -- toCSV converts delimiter to string, so 123 becomes "123" + assert(csv == "a123b123c", "Should convert delimiter to string") + assert(err == nil, "Should not return error") +end) + -- Test fromCSV basic parsing test("fromCSV basic", function() local csv = "a,b,c\n1,2,3\n4,5,6" - local rows = fromCSV(csv) + local rows, err = fromCSV(csv) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows") assert(rows[1][1] == "a", "First row first field should be 'a'") assert(rows[2][2] == "2", "Second row second field should be '2'") @@ -27,7 +46,8 @@ end) -- Test fromCSV with headers test("fromCSV with headers", function() local csv = "foo,bar,baz\n1,2,3\n4,5,6" - local rows = fromCSV(csv, { hasHeaders = true }) + local rows, err = fromCSV(csv, { hasheader = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 data rows") assert(rows[1][1] == "1", "First row first field should be '1'") assert(rows[1].foo == "1", "First row foo should be '1'") @@ -38,7 +58,8 @@ end) -- Test fromCSV with custom delimiter test("fromCSV with tab delimiter", function() local csv = "a\tb\tc\n1\t2\t3" - local rows = fromCSV(csv, { delimiter = "\t" }) + local rows, err = fromCSV(csv, { delimiter = "\t" }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows") assert(rows[1][1] == "a", "First row first field should be 'a'") assert(rows[2][2] == "2", "Second row second field should be '2'") @@ -47,7 +68,8 @@ end) -- Test fromCSV with quoted fields test("fromCSV with quoted fields", function() local csv = '"hello,world","test"\n"foo","bar"' - local rows = fromCSV(csv) + local rows, err = fromCSV(csv) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows") assert(rows[1][1] == "hello,world", "Quoted field with comma should be preserved") assert(rows[1][2] == "test", "Second field should be 'test'") @@ -56,37 +78,44 @@ end) -- Test toCSV basic test("toCSV basic", function() local rows = { { "a", "b", "c" }, { "1", "2", "3" } } - local csv = toCSV(rows) + local csv, err = toCSV(rows) + if err then error("toCSV error: " .. err) end assert(csv == "a,b,c\n1,2,3", "CSV output should match expected") end) -- Test toCSV with custom delimiter test("toCSV with tab delimiter", function() local rows = { { "a", "b", "c" }, { "1", "2", "3" } } - local csv = toCSV(rows, "\t") + local csv, err = toCSV(rows, { delimiter = "\t" }) + if err then error("toCSV error: " .. err) end assert(csv == "a\tb\tc\n1\t2\t3", "TSV output should match expected") end) -- Test toCSV with fields needing quoting test("toCSV with quoted fields", function() local rows = { { "hello,world", "test" }, { "foo", "bar" } } - local csv = toCSV(rows) + local csv, err = toCSV(rows) + if err then error("toCSV error: " .. err) end assert(csv == '"hello,world",test\nfoo,bar', "Fields with commas should be quoted") end) -- Test round trip test("fromCSV toCSV round trip", function() local original = "a,b,c\n1,2,3\n4,5,6" - local rows = fromCSV(original) - local csv = toCSV(rows) + local rows, err = fromCSV(original) + if err then error("fromCSV error: " .. err) end + local csv, err = toCSV(rows) + if err then error("toCSV error: " .. err) end assert(csv == original, "Round trip should preserve original") end) -- Test round trip with headers test("fromCSV toCSV round trip with headers", function() local original = "foo,bar,baz\n1,2,3\n4,5,6" - local rows = fromCSV(original, { hasHeaders = true }) - local csv = toCSV(rows) + local rows, err = fromCSV(original, { hasheader = true }) + if err then error("fromCSV error: " .. err) end + local csv, err = toCSV(rows) + if err then error("toCSV error: " .. err) end local expected = "1,2,3\n4,5,6" assert(csv == expected, "Round trip with headers should preserve data rows") end) @@ -94,7 +123,8 @@ end) -- Test fromCSV with comments test("fromCSV with comments", function() local csv = "# This is a comment\nfoo,bar,baz\n1,2,3\n# Another comment\n4,5,6" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows (comments filtered, header + 2 data rows)") assert(rows[1][1] == "foo", "First row should be header row") assert(rows[2][1] == "1", "Second row first field should be '1'") @@ -104,7 +134,8 @@ end) -- Test fromCSV with comments and headers test("fromCSV with comments and headers", function() local csv = "#mercenary_profiles\nId,Name,Value\n1,Test,100\n# End of data\n2,Test2,200" - local rows = fromCSV(csv, { hasHeaders = true, hasComments = true }) + local rows, err = fromCSV(csv, { hasheader = true, hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 data rows") assert(rows[1].Id == "1", "First row Id should be '1'") assert(rows[1].Name == "Test", "First row Name should be 'Test'") @@ -115,7 +146,8 @@ end) -- Test fromCSV with comments disabled test("fromCSV without comments", function() local csv = "# This should not be filtered\nfoo,bar\n1,2" - local rows = fromCSV(csv, { hasComments = false }) + local rows, err = fromCSV(csv, { hascomments = false }) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows (including comment)") assert(rows[1][1] == "# This should not be filtered", "Comment line should be preserved") end) @@ -123,7 +155,8 @@ end) -- Test fromCSV with comment at start test("fromCSV comment at start", function() local csv = "# Header comment\nId,Name\n1,Test" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows (comment filtered)") assert(rows[1][1] == "Id", "First row should be header") end) @@ -131,7 +164,8 @@ end) -- Test fromCSV with comment with leading whitespace test("fromCSV comment with whitespace", function() local csv = " # Comment with spaces\nId,Name\n1,Test" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows (comment with spaces filtered)") assert(rows[1][1] == "Id", "First row should be header") end) @@ -139,7 +173,8 @@ end) -- Test fromCSV with comment with tabs test("fromCSV comment with tabs", function() local csv = "\t# Comment with tab\nId,Name\n1,Test" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows (comment with tab filtered)") assert(rows[1][1] == "Id", "First row should be header") end) @@ -147,7 +182,8 @@ end) -- Test fromCSV with multiple consecutive comments test("fromCSV multiple consecutive comments", function() local csv = "# First comment\n# Second comment\n# Third comment\nId,Name\n1,Test" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows (all comments filtered)") assert(rows[1][1] == "Id", "First row should be header") end) @@ -155,7 +191,8 @@ end) -- Test fromCSV with comment in middle of data test("fromCSV comment in middle", function() local csv = "Id,Name\n1,Test\n# Middle comment\n2,Test2" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows (comment filtered)") assert(rows[1][1] == "Id", "First row should be header") assert(rows[2][1] == "1", "Second row should be first data") @@ -165,7 +202,8 @@ end) -- Test fromCSV with comment at end test("fromCSV comment at end", function() local csv = "Id,Name\n1,Test\n# End comment" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows (end comment filtered)") assert(rows[1][1] == "Id", "First row should be header") assert(rows[2][1] == "1", "Second row should be data") @@ -174,7 +212,8 @@ end) -- Test fromCSV with empty comment line test("fromCSV empty comment", function() local csv = "#\nId,Name\n1,Test" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows (empty comment filtered)") assert(rows[1][1] == "Id", "First row should be header") end) @@ -182,7 +221,8 @@ end) -- Test fromCSV with comment and headers test("fromCSV comment with headers enabled", function() local csv = "#mercenary_profiles\nId,Name,Value\n1,Test,100\n2,Test2,200" - local rows = fromCSV(csv, { hasHeaders = true, hasComments = true }) + local rows, err = fromCSV(csv, { hasheader = true, hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 data rows") assert(rows[1].Id == "1", "First row Id should be '1'") assert(rows[1].Name == "Test", "First row Name should be 'Test'") @@ -192,7 +232,8 @@ end) -- Test fromCSV with comment and TSV delimiter test("fromCSV comment with tab delimiter", function() local csv = "# Comment\nId\tName\n1\tTest" - local rows = fromCSV(csv, { delimiter = "\t", hasComments = true }) + local rows, err = fromCSV(csv, { delimiter = "\t", hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 2, "Should have 2 rows") assert(rows[1][1] == "Id", "First row should be header") assert(rows[2][1] == "1", "Second row first field should be '1'") @@ -201,7 +242,8 @@ end) -- Test fromCSV with comment and headers and TSV test("fromCSV comment with headers and TSV", function() local csv = "#mercenary_profiles\nId\tName\tValue\n1\tTest\t100" - local rows = fromCSV(csv, { delimiter = "\t", hasHeaders = true, hasComments = true }) + local rows, err = fromCSV(csv, { delimiter = "\t", hasheader = true, hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 1, "Should have 1 data row") assert(rows[1].Id == "1", "Row Id should be '1'") assert(rows[1].Name == "Test", "Row Name should be 'Test'") @@ -211,7 +253,8 @@ end) -- Test fromCSV with data field starting with # (not a comment) test("fromCSV data field starting with hash", function() local csv = "Id,Name\n1,#NotAComment\n2,Test" - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows (data with # not filtered)") assert(rows[1][1] == "Id", "First row should be header") assert(rows[2][2] == "#NotAComment", "Second row should have #NotAComment as data") @@ -220,7 +263,8 @@ end) -- Test fromCSV with quoted field starting with # test("fromCSV quoted field with hash", function() local csv = 'Id,Name\n1,"#NotAComment"\n2,Test' - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows (quoted # not filtered)") assert(rows[2][2] == "#NotAComment", "Quoted field with # should be preserved") end) @@ -228,7 +272,8 @@ end) -- Test fromCSV with comment after quoted field test("fromCSV comment after quoted field", function() local csv = 'Id,Name\n1,"Test"\n# This is a comment\n2,Test2' - local rows = fromCSV(csv, { hasComments = true }) + local rows, err = fromCSV(csv, { hascomments = true }) + if err then error("fromCSV error: " .. err) end assert(#rows == 3, "Should have 3 rows (comment filtered)") assert(rows[2][2] == "Test", "Quoted field should be preserved") assert(rows[3][1] == "2", "Third row should be second data row") @@ -362,13 +407,85 @@ test("isArray function", function() assert(isArray(123) == false, "isArray should return false for number") end) -test("toCSV assigns header keys correctly", function() +test("fromCSV assigns header keys correctly", function() local teststr = [[ #mercenary_profiles Id ModifyStartCost ModifyStep ModifyLevelLimit Health ResistSheet WoundSlots MeleeDamage MeleeAccuracy RangeAccuracy ReceiveAmputationChance ReceiveWoundChanceMult AttackWoundChanceMult Dodge Los StarvationLimit PainThresholdLimit PainThresholdRegen TalentPerkId ActorId SkinIndex HairType HairColorHex VoiceBank Immunity CreatureClass john_hawkwood_boss 20 0.1 140 blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0 HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee blunt 8 16 crit 1.60 critchance 0.05 0.5 0.5 0.03 0.5 1.2 0.3 8 2200 16 2 talent_the_man_who_sold_the_world human_male 0 hair1 #633D08 player Human francis_reid_daly 20 0.1 130 blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0 HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee blunt 7 14 crit 1.70 critchance 0.05 0.5 0.4 0.04 0.9 1 0.3 8 2000 10 1 talent_weapon_durability human_male 0 player Human ]] + local rows, err = fromCSV(teststr, { delimiter = "\t", hasheader = true, hascomments = true }) + if err then error("fromCSV error: " .. err) end + assert(#rows == 2, "Should have 2 data rows") + + -- Test first row + assert(rows[1].Id == "john_hawkwood_boss", "First row Id should be 'john_hawkwood_boss'") + assert(rows[1].ModifyStartCost == "20", "First row ModifyStartCost should be '20'") + assert(rows[1].ModifyStep == "0.1", "First row ModifyStep should be '0.1'") + assert(rows[1].Health == "140", "First row Health should be '140'") + assert(rows[1].ActorId == "human_male", "First row ActorId should be 'human_male'") + assert(rows[1].HairColorHex == "#633D08", "First row HairColorHex should be '#633D08'") + + -- Test second row + assert(rows[2].Id == "francis_reid_daly", "Second row Id should be 'francis_reid_daly'") + assert(rows[2].ModifyStartCost == "20", "Second row ModifyStartCost should be '20'") + assert(rows[2].ModifyStep == "0.1", "Second row ModifyStep should be '0.1'") + assert(rows[2].Health == "130", "Second row Health should be '130'") + assert(rows[2].ActorId == "human_male", "Second row ActorId should be 'human_male'") + + -- Test that numeric indices still work + assert(rows[1][1] == "john_hawkwood_boss", "First row first field by index should work") + assert(rows[1][2] == "20", "First row second field by index should work") + + -- Debug: Print first row keys to verify headers are assigned + print("DEBUG: First row keys:") + for k, v in pairs(rows[1]) do + if type(k) == "string" then print(" " .. k .. " = " .. tostring(v)) end + end +end) + +test("fromCSV debug header assignment", function() + local csv = "Id Name Value\n1 Test 100\n2 Test2 200" + local rows, err = fromCSV(csv, { delimiter = "\t", hasheader = true }) + if err then error("fromCSV error: " .. err) end + print("DEBUG: Number of rows: " .. #rows) + if #rows > 0 then + print("DEBUG: First row Id = " .. tostring(rows[1].Id)) + print("DEBUG: First row Name = " .. tostring(rows[1].Name)) + print("DEBUG: First row Value = " .. tostring(rows[1].Value)) + print("DEBUG: First row keys:") + for k, v in pairs(rows[1]) do + print(" " .. tostring(k) .. " (" .. type(k) .. ") = " .. tostring(v)) + end + end + assert(rows[1].Id == "1", "Id should be '1'") + assert(rows[1].Name == "Test", "Name should be 'Test'") + assert(rows[1].Value == "100", "Value should be '100'") +end) + +test("fromCSV real world mercenary file format", function() + local csv = [[#mercenary_profiles +Id ModifyStartCost ModifyStep ModifyLevelLimit Health ResistSheet WoundSlots MeleeDamage MeleeAccuracy RangeAccuracy ReceiveAmputationChance ReceiveWoundChanceMult AttackWoundChanceMult Dodge Los StarvationLimit PainThresholdLimit PainThresholdRegen TalentPerkId ActorId SkinIndex HairType HairColorHex VoiceBank Immunity CreatureClass +john_hawkwood_boss 20 0.1 140 blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0 HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee blunt 8 16 crit 1.60 critchance 0.05 0.5 0.5 0.03 0.5 1.2 0.3 8 2200 16 2 talent_the_man_who_sold_the_world human_male 0 hair1 #633D08 player Human +francis_reid_daly 20 0.1 130 blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0 HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee blunt 7 14 crit 1.70 critchance 0.05 0.5 0.4 0.04 0.9 1 0.3 8 2000 10 1 talent_weapon_durability human_male 0 player Human +]] + local rows, err = fromCSV(csv, { delimiter = "\t", hasheader = true, hascomments = true }) + if err then error("fromCSV error: " .. err) end + print("DEBUG: Number of rows: " .. #rows) + assert(#rows == 2, "Should have 2 data rows") + + if #rows > 0 then + print("DEBUG: First row Id = " .. tostring(rows[1].Id)) + print("DEBUG: First row ModifyStartCost = " .. tostring(rows[1].ModifyStartCost)) + print("DEBUG: First row all string keys:") + for k, v in pairs(rows[1]) do + if type(k) == "string" then print(" " .. tostring(k) .. " = " .. tostring(v)) end + end + end + + assert(rows[1].Id == "john_hawkwood_boss", "First row Id should be 'john_hawkwood_boss'") + assert(rows[1].ModifyStartCost == "20", "First row ModifyStartCost should be '20'") + assert(rows[2].Id == "francis_reid_daly", "Second row Id should be 'francis_reid_daly'") end) print("\nAll tests completed!") diff --git a/processor/luahelper.lua b/processor/luahelper.lua index 0047da3..412bf6e 100644 --- a/processor/luahelper.lua +++ b/processor/luahelper.lua @@ -53,6 +53,34 @@ end --- @field hasheader boolean? If true, first non-comment row is treated as headers (default: false). --- @field hascomments boolean? If true, lines starting with '#' are skipped (default: false). +--- @type ParserOptions +parserDefaultOptions = { delimiter = ",", hasheader = false, hascomments = false } + +--- Validates options against a set of valid option keys. +--- @param options ParserOptions? The options table to validate +--- @return boolean #True if options are valid +--- @return string? #Error message if invalid, nil if valid +function areOptionsValid(options) + if options == nil then return true, nil end + + if type(options) ~= "table" then return false, "options must be a table" end + + -- Build valid options list from validOptions table + local validOptionsStr = "" + for k, _ in pairs(parserDefaultOptions) do + validOptionsStr = validOptionsStr .. k .. ", " + end + + for k, _ in pairs(options) do + if parserDefaultOptions[k] == nil then + return false, + "unknown option: " .. tostring(k) .. " (valid options: " .. validOptionsStr .. ")" + end + end + + return true, nil +end + --- Parses CSV text into rows and fields using a minimal RFC 4180 state machine. --- --- Requirements/assumptions: @@ -66,16 +94,22 @@ end --- - Leading/trailing spaces are preserved; no trimming is performed. --- - Empty fields and empty rows are preserved. --- - The final row is emitted even if the text does not end with a newline. ---- - Lines starting with '#' (after optional leading whitespace) are treated as comments and skipped if hasComments is true. +--- - Lines starting with '#' (after optional leading whitespace) are treated as comments and skipped if hascomments is true. --- --- @param csv string The CSV text to parse. --- @param options ParserOptions? Options for the parser ---- @return table A table (array) of rows; each row is a table with numeric indices and optionally header-named keys. +--- @return table #A table (array) of rows; each row is a table with numeric indices and optionally header-named keys. +--- @return string? #Error message if parsing fails. function fromCSV(csv, options) if options == nil then options = {} end - local delimiter = options.delimiter or "," - local hasheader = options.hasheader or false - local hascomments = options.hascomments or false + + -- Validate options + local isValid, err = areOptionsValid(options) + if not isValid then return {}, err end + + local delimiter = options.delimiter or parserDefaultOptions.delimiter + local hasheader = options.hasheader or parserDefaultOptions.hasheader + local hascomments = options.hascomments or parserDefaultOptions.hascomments local allRows = {} local fields = {} @@ -186,12 +220,13 @@ function fromCSV(csv, options) row[headerName] = dataRow[j] end end + row.Headers = headers table.insert(rows, row) end - return rows + return rows, nil end - return allRows + return allRows, nil end --- Converts a table of rows back to CSV text format (RFC 4180 compliant). @@ -202,14 +237,46 @@ end --- - Fields are quoted if they contain the delimiter, newlines, or double quotes. --- - Double quotes inside quoted fields are doubled (""). --- - Fields are joined with the specified delimiter; rows are joined with newlines. +--- - If includeHeaders is true and rows have a Headers field, headers are included as the first row. --- --- @param rows table Array of rows, where each row is an array of field values. ---- @param delimiter string? The field delimiter (default: ","). ---- @return string CSV-formatted text. -function toCSV(rows, delimiter) - if delimiter == nil then delimiter = "," end +--- @param options ParserOptions? Options for the parser +--- @return string? #CSV-formatted text, error string? +--- @return string? #Error message if conversion fails. +function toCSV(rows, options) + if options == nil then options = {} end + + -- Validate options + local isValid, err = areOptionsValid(options) + if not isValid then return nil, err end + + local delimiter = options.delimiter or parserDefaultOptions.delimiter + local includeHeaders = options.hasheader or parserDefaultOptions.hasheader local rowStrings = {} + -- Include headers row if requested and available + if includeHeaders and #rows > 0 and rows[1].Headers ~= nil then + local headerStrings = {} + for _, header in ipairs(rows[1].Headers) do + local headerStr = tostring(header) + local needsQuoting = false + if + headerStr:find(delimiter) + or headerStr:find("\n") + or headerStr:find("\r") + or headerStr:find('"') + then + needsQuoting = true + end + if needsQuoting then + headerStr = headerStr:gsub('"', '""') + headerStr = '"' .. headerStr .. '"' + end + table.insert(headerStrings, headerStr) + end + table.insert(rowStrings, table.concat(headerStrings, delimiter)) + end + for _, row in ipairs(rows) do local fieldStrings = {} @@ -237,7 +304,7 @@ function toCSV(rows, delimiter) table.insert(rowStrings, table.concat(fieldStrings, delimiter)) end - return table.concat(rowStrings, "\n") + return table.concat(rowStrings, "\n"), nil end -- String to number conversion helper