Add a complex test for csv parser

README.md

@@ -16,6 +16,7 @@ A Go-based tool for modifying XML, JSON, and text documents using XPath/JSONPath
   - String manipulations
   - Date conversions
   - Structural changes
+  - CSV/TSV parsing with comments and headers
   - Whole ass Lua environment
 - **Error Handling**: Comprehensive error detection for:
   - Invalid XML/JSON
@@ -101,6 +102,87 @@ chef -xml "//item" "if tonumber(v.stock) > 0 then v.price = v.price * 0.8 end" i
 <item stock="5" price="8.00"/>
 ```
 
+### 6. CSV/TSV Processing
+The Lua environment includes CSV parsing functions that support comments, headers, and custom delimiters.
+
+```lua
+-- Basic CSV parsing
+local rows = fromCSV(csvText)
+
+-- With options
+local rows = fromCSV(csvText, {
+    delimiter = "\t",        -- Tab delimiter for TSV (default: ",")
+    hasHeaders = true,       -- First row is headers (default: false)
+    hasComments = true       -- Filter lines starting with # (default: false)
+})
+
+-- Access by index
+local value = rows[1][2]
+
+-- Access by header name (when hasHeaders = true)
+local value = rows[1].Name
+
+-- Convert back to CSV
+local csv = toCSV(rows, "\t")  -- Optional delimiter parameter
+```
+
+**Example with commented TSV file:**
+```lua
+-- Input file:
+-- #mercenary_profiles
+-- Id	Name	Value
+-- 1	Test	100
+-- 2	Test2	200
+
+local csv = readFile("mercenaries.tsv")
+local rows = fromCSV(csv, {
+    delimiter = "\t",
+    hasHeaders = true,
+    hasComments = true
+})
+
+-- Access data
+rows[1].Name  -- "Test"
+rows[2].Value -- "200"
+```
+
+## Lua Helper Functions
+
+The Lua environment includes many helper functions:
+
+### Math Functions
+- `min(a, b)`, `max(a, b)` - Min/max of two numbers
+- `round(x, n)` - Round to n decimal places
+- `floor(x)`, `ceil(x)` - Floor/ceiling functions
+
+### String Functions
+- `upper(s)`, `lower(s)` - Case conversion
+- `trim(s)` - Remove leading/trailing whitespace
+- `format(s, ...)` - String formatting
+- `strsplit(inputstr, sep)` - Split string by separator
+
+### CSV Functions
+- `fromCSV(csv, options)` - Parse CSV/TSV text into table of rows
+  - Options: `delimiter` (default: ","), `hasHeaders` (default: false), `hasComments` (default: false)
+- `toCSV(rows, delimiter)` - Convert table of rows back to CSV text
+
+### Conversion Functions
+- `num(str)` - Convert string to number (returns 0 if invalid)
+- `str(num)` - Convert number to string
+- `is_number(str)` - Check if string is numeric
+
+### Table Functions
+- `isArray(t)` - Check if table is a sequential array
+- `dump(table, depth)` - Print table structure recursively
+
+### HTTP Functions
+- `fetch(url, options)` - Make HTTP request, returns response table
+  - Options: `method`, `headers`, `body`
+  - Returns: `{status, statusText, ok, body, headers}`
+
+### Regex Functions
+- `re(pattern, input)` - Apply regex pattern, returns table with matches
+
 ## Installation
 
 ```bash

processor/luahelper-test.lua

@@ -15,10 +15,29 @@ local function test(name, fn)
 	end
 end
 
+-- Test fromCSV option validation
+test("fromCSV invalid option", function()
+	local csv = "a,b,c\n1,2,3"
+	local rows, err = fromCSV(csv, { invalidOption = true })
+	assert(rows ~= nil and #rows == 0, "Should return empty table on error")
+	assert(err ~= nil, "Should return error message")
+	assert(string.find(err, "unknown option"), "Error should mention unknown option")
+end)
+
+-- Test toCSV error handling
+test("toCSV invalid delimiter", function()
+	local rows = { { "a", "b", "c" } }
+	local csv, err = toCSV(rows, { delimiter = 123 })
+	-- toCSV converts delimiter to string, so 123 becomes "123"
+	assert(csv == "a123b123c", "Should convert delimiter to string")
+	assert(err == nil, "Should not return error")
+end)
+
 -- Test fromCSV basic parsing
 test("fromCSV basic", function()
 	local csv = "a,b,c\n1,2,3\n4,5,6"
-	local rows = fromCSV(csv)
+	local rows, err = fromCSV(csv)
+	if err then error("fromCSV error: " .. err) end
 	assert(#rows == 3, "Should have 3 rows")
 	assert(rows[1][1] == "a", "First row first field should be 'a'")
 	assert(rows[2][2] == "2", "Second row second field should be '2'")
@@ -27,7 +46,8 @@ end)
 -- Test fromCSV with headers
 test("fromCSV with headers", function()
 	local csv = "foo,bar,baz\n1,2,3\n4,5,6"
-	local rows = fromCSV(csv, ",", true)
+	local rows, err = fromCSV(csv, { hasheader = true })
+	if err then error("fromCSV error: " .. err) end
 	assert(#rows == 2, "Should have 2 data rows")
 	assert(rows[1][1] == "1", "First row first field should be '1'")
 	assert(rows[1].foo == "1", "First row foo should be '1'")
@@ -38,7 +58,8 @@ end)
 -- Test fromCSV with custom delimiter
 test("fromCSV with tab delimiter", function()
 	local csv = "a\tb\tc\n1\t2\t3"
-	local rows = fromCSV(csv, "\t")
+	local rows, err = fromCSV(csv, { delimiter = "\t" })
+	if err then error("fromCSV error: " .. err) end
 	assert(#rows == 2, "Should have 2 rows")
 	assert(rows[1][1] == "a", "First row first field should be 'a'")
 	assert(rows[2][2] == "2", "Second row second field should be '2'")
@@ -47,7 +68,8 @@ end)
 -- Test fromCSV with quoted fields
 test("fromCSV with quoted fields", function()
 	local csv = '"hello,world","test"\n"foo","bar"'
-	local rows = fromCSV(csv)
+	local rows, err = fromCSV(csv)
+	if err then error("fromCSV error: " .. err) end
 	assert(#rows == 2, "Should have 2 rows")
 	assert(rows[1][1] == "hello,world", "Quoted field with comma should be preserved")
 	assert(rows[1][2] == "test", "Second field should be 'test'")
@@ -56,41 +78,207 @@ end)
 -- Test toCSV basic
 test("toCSV basic", function()
 	local rows = { { "a", "b", "c" }, { "1", "2", "3" } }
-	local csv = toCSV(rows)
+	local csv, err = toCSV(rows)
+	if err then error("toCSV error: " .. err) end
 	assert(csv == "a,b,c\n1,2,3", "CSV output should match expected")
 end)
 
 -- Test toCSV with custom delimiter
 test("toCSV with tab delimiter", function()
 	local rows = { { "a", "b", "c" }, { "1", "2", "3" } }
-	local csv = toCSV(rows, "\t")
+	local csv, err = toCSV(rows, { delimiter = "\t" })
+	if err then error("toCSV error: " .. err) end
 	assert(csv == "a\tb\tc\n1\t2\t3", "TSV output should match expected")
 end)
 
 -- Test toCSV with fields needing quoting
 test("toCSV with quoted fields", function()
 	local rows = { { "hello,world", "test" }, { "foo", "bar" } }
-	local csv = toCSV(rows)
+	local csv, err = toCSV(rows)
+	if err then error("toCSV error: " .. err) end
 	assert(csv == '"hello,world",test\nfoo,bar', "Fields with commas should be quoted")
 end)
 
 -- Test round trip
 test("fromCSV toCSV round trip", function()
 	local original = "a,b,c\n1,2,3\n4,5,6"
-	local rows = fromCSV(original)
-	local csv = toCSV(rows)
+	local rows, err = fromCSV(original)
+	if err then error("fromCSV error: " .. err) end
+	local csv, err = toCSV(rows)
+	if err then error("toCSV error: " .. err) end
 	assert(csv == original, "Round trip should preserve original")
 end)
 
 -- Test round trip with headers
 test("fromCSV toCSV round trip with headers", function()
 	local original = "foo,bar,baz\n1,2,3\n4,5,6"
-	local rows = fromCSV(original, ",", true)
-	local csv = toCSV(rows)
+	local rows, err = fromCSV(original, { hasheader = true })
+	if err then error("fromCSV error: " .. err) end
+	local csv, err = toCSV(rows)
+	if err then error("toCSV error: " .. err) end
 	local expected = "1,2,3\n4,5,6"
 	assert(csv == expected, "Round trip with headers should preserve data rows")
 end)
 
+-- Test fromCSV with comments
+test("fromCSV with comments", function()
+	local csv = "# This is a comment\nfoo,bar,baz\n1,2,3\n# Another comment\n4,5,6"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 3, "Should have 3 rows (comments filtered, header + 2 data rows)")
+	assert(rows[1][1] == "foo", "First row should be header row")
+	assert(rows[2][1] == "1", "Second row first field should be '1'")
+	assert(rows[3][1] == "4", "Third row first field should be '4'")
+end)
+
+-- Test fromCSV with comments and headers
+test("fromCSV with comments and headers", function()
+	local csv = "#mercenary_profiles\nId,Name,Value\n1,Test,100\n# End of data\n2,Test2,200"
+	local rows, err = fromCSV(csv, { hasheader = true, hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 data rows")
+	assert(rows[1].Id == "1", "First row Id should be '1'")
+	assert(rows[1].Name == "Test", "First row Name should be 'Test'")
+	assert(rows[1].Value == "100", "First row Value should be '100'")
+	assert(rows[2].Id == "2", "Second row Id should be '2'")
+end)
+
+-- Test fromCSV with comments disabled
+test("fromCSV without comments", function()
+	local csv = "# This should not be filtered\nfoo,bar\n1,2"
+	local rows, err = fromCSV(csv, { hascomments = false })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 3, "Should have 3 rows (including comment)")
+	assert(rows[1][1] == "# This should not be filtered", "Comment line should be preserved")
+end)
+
+-- Test fromCSV with comment at start
+test("fromCSV comment at start", function()
+	local csv = "# Header comment\nId,Name\n1,Test"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows (comment filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+end)
+
+-- Test fromCSV with comment with leading whitespace
+test("fromCSV comment with whitespace", function()
+	local csv = "  # Comment with spaces\nId,Name\n1,Test"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows (comment with spaces filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+end)
+
+-- Test fromCSV with comment with tabs
+test("fromCSV comment with tabs", function()
+	local csv = "\t# Comment with tab\nId,Name\n1,Test"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows (comment with tab filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+end)
+
+-- Test fromCSV with multiple consecutive comments
+test("fromCSV multiple consecutive comments", function()
+	local csv = "# First comment\n# Second comment\n# Third comment\nId,Name\n1,Test"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows (all comments filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+end)
+
+-- Test fromCSV with comment in middle of data
+test("fromCSV comment in middle", function()
+	local csv = "Id,Name\n1,Test\n# Middle comment\n2,Test2"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 3, "Should have 3 rows (comment filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+	assert(rows[2][1] == "1", "Second row should be first data")
+	assert(rows[3][1] == "2", "Third row should be second data")
+end)
+
+-- Test fromCSV with comment at end
+test("fromCSV comment at end", function()
+	local csv = "Id,Name\n1,Test\n# End comment"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows (end comment filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+	assert(rows[2][1] == "1", "Second row should be data")
+end)
+
+-- Test fromCSV with empty comment line
+test("fromCSV empty comment", function()
+	local csv = "#\nId,Name\n1,Test"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows (empty comment filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+end)
+
+-- Test fromCSV with comment and headers
+test("fromCSV comment with headers enabled", function()
+	local csv = "#mercenary_profiles\nId,Name,Value\n1,Test,100\n2,Test2,200"
+	local rows, err = fromCSV(csv, { hasheader = true, hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 data rows")
+	assert(rows[1].Id == "1", "First row Id should be '1'")
+	assert(rows[1].Name == "Test", "First row Name should be 'Test'")
+	assert(rows[2].Id == "2", "Second row Id should be '2'")
+end)
+
+-- Test fromCSV with comment and TSV delimiter
+test("fromCSV comment with tab delimiter", function()
+	local csv = "# Comment\nId\tName\n1\tTest"
+	local rows, err = fromCSV(csv, { delimiter = "\t", hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 rows")
+	assert(rows[1][1] == "Id", "First row should be header")
+	assert(rows[2][1] == "1", "Second row first field should be '1'")
+end)
+
+-- Test fromCSV with comment and headers and TSV
+test("fromCSV comment with headers and TSV", function()
+	local csv = "#mercenary_profiles\nId\tName\tValue\n1\tTest\t100"
+	local rows, err = fromCSV(csv, { delimiter = "\t", hasheader = true, hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 1, "Should have 1 data row")
+	assert(rows[1].Id == "1", "Row Id should be '1'")
+	assert(rows[1].Name == "Test", "Row Name should be 'Test'")
+	assert(rows[1].Value == "100", "Row Value should be '100'")
+end)
+
+-- Test fromCSV with data field starting with # (not a comment)
+test("fromCSV data field starting with hash", function()
+	local csv = "Id,Name\n1,#NotAComment\n2,Test"
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 3, "Should have 3 rows (data with # not filtered)")
+	assert(rows[1][1] == "Id", "First row should be header")
+	assert(rows[2][2] == "#NotAComment", "Second row should have #NotAComment as data")
+end)
+
+-- Test fromCSV with quoted field starting with #
+test("fromCSV quoted field with hash", function()
+	local csv = 'Id,Name\n1,"#NotAComment"\n2,Test'
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 3, "Should have 3 rows (quoted # not filtered)")
+	assert(rows[2][2] == "#NotAComment", "Quoted field with # should be preserved")
+end)
+
+-- Test fromCSV with comment after quoted field
+test("fromCSV comment after quoted field", function()
+	local csv = 'Id,Name\n1,"Test"\n# This is a comment\n2,Test2'
+	local rows, err = fromCSV(csv, { hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 3, "Should have 3 rows (comment filtered)")
+	assert(rows[2][2] == "Test", "Quoted field should be preserved")
+	assert(rows[3][1] == "2", "Third row should be second data row")
+end)
+
 -- Math function tests
 test("min function", function()
 	assert(min(5, 3) == 3, "min(5, 3) should be 3")
@@ -203,11 +391,133 @@ test("isArray function", function()
 	assert(isArray({}) == true, "isArray should return true for empty array")
 	assert(isArray({ a = 1, b = 2 }) == false, "isArray should return false for map")
 	assert(isArray({ 1, 2, [4] = 4 }) == false, "isArray should return false for sparse array")
-	assert(isArray({ [1] = 1, [2] = 2, [3] = 3 }) == true, "isArray should return true for 1-indexed array")
-	assert(isArray({ [0] = 1, [1] = 2 }) == false, "isArray should return false for 0-indexed array")
-	assert(isArray({ [1] = 1, [2] = 2, [4] = 4 }) == false, "isArray should return false for non-sequential array")
+	assert(
+		isArray({ [1] = 1, [2] = 2, [3] = 3 }) == true,
+		"isArray should return true for 1-indexed array"
+	)
+	assert(
+		isArray({ [0] = 1, [1] = 2 }) == false,
+		"isArray should return false for 0-indexed array"
+	)
+	assert(
+		isArray({ [1] = 1, [2] = 2, [4] = 4 }) == false,
+		"isArray should return false for non-sequential array"
+	)
 	assert(isArray("not a table") == false, "isArray should return false for non-table")
 	assert(isArray(123) == false, "isArray should return false for number")
 end)
 
+test("fromCSV assigns header keys correctly", function()
+	local teststr = [[
+#mercenary_profiles
+Id	ModifyStartCost	ModifyStep	ModifyLevelLimit	Health	ResistSheet	WoundSlots	MeleeDamage	MeleeAccuracy	RangeAccuracy	ReceiveAmputationChance	ReceiveWoundChanceMult	AttackWoundChanceMult	Dodge	Los	StarvationLimit	PainThresholdLimit	PainThresholdRegen	TalentPerkId	ActorId	SkinIndex	HairType	HairColorHex	VoiceBank	Immunity	CreatureClass
+john_hawkwood_boss	20	0.1		140	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 8 16 crit 1.60 critchance 0.05	0.5	0.5	0.03	0.5	1.2	0.3	8	2200	16	2	talent_the_man_who_sold_the_world	human_male	0	hair1	#633D08	player		Human
+francis_reid_daly	20	0.1		130	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 7 14 crit 1.70 critchance 0.05	0.5	0.4	0.04	0.9	1	0.3	8	2000	10	1	talent_weapon_durability	human_male	0			player		Human
+]]
+	local rows, err = fromCSV(teststr, { delimiter = "\t", hasheader = true, hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 data rows")
+
+	-- Test first row
+	assert(rows[1].Id == "john_hawkwood_boss", "First row Id should be 'john_hawkwood_boss'")
+	assert(rows[1].ModifyStartCost == "20", "First row ModifyStartCost should be '20'")
+	assert(rows[1].ModifyStep == "0.1", "First row ModifyStep should be '0.1'")
+	assert(rows[1].Health == "140", "First row Health should be '140'")
+	assert(rows[1].ActorId == "human_male", "First row ActorId should be 'human_male'")
+	assert(rows[1].HairColorHex == "#633D08", "First row HairColorHex should be '#633D08'")
+
+	-- Test second row
+	assert(rows[2].Id == "francis_reid_daly", "Second row Id should be 'francis_reid_daly'")
+	assert(rows[2].ModifyStartCost == "20", "Second row ModifyStartCost should be '20'")
+	assert(rows[2].ModifyStep == "0.1", "Second row ModifyStep should be '0.1'")
+	assert(rows[2].Health == "130", "Second row Health should be '130'")
+	assert(rows[2].ActorId == "human_male", "Second row ActorId should be 'human_male'")
+
+	-- Test that numeric indices still work
+	assert(rows[1][1] == "john_hawkwood_boss", "First row first field by index should work")
+	assert(rows[1][2] == "20", "First row second field by index should work")
+end)
+
+test("fromCSV debug header assignment", function()
+	local csv = "Id	Name	Value\n1	Test	100\n2	Test2	200"
+	local rows, err = fromCSV(csv, { delimiter = "\t", hasheader = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(rows[1].Id == "1", "Id should be '1'")
+	assert(rows[1].Name == "Test", "Name should be 'Test'")
+	assert(rows[1].Value == "100", "Value should be '100'")
+end)
+
+test("fromCSV real world mercenary file format", function()
+	local csv = [[#mercenary_profiles
+Id	ModifyStartCost	ModifyStep	ModifyLevelLimit	Health	ResistSheet	WoundSlots	MeleeDamage	MeleeAccuracy	RangeAccuracy	ReceiveAmputationChance	ReceiveWoundChanceMult	AttackWoundChanceMult	Dodge	Los	StarvationLimit	PainThresholdLimit	PainThresholdRegen	TalentPerkId	ActorId	SkinIndex	HairType	HairColorHex	VoiceBank	Immunity	CreatureClass
+john_hawkwood_boss	20	0.1		140	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 8 16 crit 1.60 critchance 0.05	0.5	0.5	0.03	0.5	1.2	0.3	8	2200	16	2	talent_the_man_who_sold_the_world	human_male	0	hair1	#633D08	player		Human
+francis_reid_daly	20	0.1		130	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 7 14 crit 1.70 critchance 0.05	0.5	0.4	0.04	0.9	1	0.3	8	2000	10	1	talent_weapon_durability	human_male	0			player		Human
+]]
+	local rows, err = fromCSV(csv, { delimiter = "\t", hasheader = true, hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows == 2, "Should have 2 data rows")
+
+	assert(rows[1].Id == "john_hawkwood_boss", "First row Id should be 'john_hawkwood_boss'")
+	assert(rows[1].ModifyStartCost == "20", "First row ModifyStartCost should be '20'")
+	assert(rows[2].Id == "francis_reid_daly", "Second row Id should be 'francis_reid_daly'")
+end)
+
+test("full CSV parser complex", function()
+	local original = [[
+#mercenary_profiles																									
+Id	ModifyStartCost	ModifyStep	ModifyLevelLimit	Health	ResistSheet	WoundSlots	MeleeDamage	MeleeAccuracy	RangeAccuracy	ReceiveAmputationChance	ReceiveWoundChanceMult	AttackWoundChanceMult	Dodge	Los	StarvationLimit	PainThresholdLimit	PainThresholdRegen	TalentPerkId	ActorId	SkinIndex	HairType	HairColorHex	VoiceBank	Immunity	CreatureClass
+john_hawkwood_boss	20	0.1		140	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 8 16 crit 1.60 critchance 0.05	0.5	0.5	0.03	0.5	1.2	0.3	8	2200	16	2	talent_the_man_who_sold_the_world	human_male	0	hair1	#633D08	player		Human
+francis_reid_daly	20	0.1		130	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 7 14 crit 1.70 critchance 0.05	0.5	0.4	0.04	0.9	1	0.3	8	2000	10	1	talent_weapon_durability	human_male	0			player		Human
+victoria_boudicca	20	0.1		90	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 5 10 crit 1.70 critchance 0.1	0.4	0.45	0.05	1	1.2	0.3	8	1800	8	1	talent_weapon_distance	human_female	0	hair1	#633D08	player		Human
+persival_fawcett	20	0.1		150	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 6 12 crit 1.70 critchance 0.05	0.5	0.35	0.05	0.6	1	0.25	8	2100	16	1	talent_all_resists	human_male	1	hair1	#633D08	player		Human
+Isabella_capet	20	0.1		100	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 7 14 crit 1.70 critchance 0.15	0.55	0.3	0.03	0.8	1.4	0.35	7	1700	14	2	talent_ignore_infection	human_female	1	hair3	#FF3100	player		Human
+maximilian_rohr	20	0.1		120	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 8 16 crit 1.75 critchance 0.05	0.45	0.45	0.06	0.9	1	0.2	8	2000	14	1	talent_ignore_pain	human_male	0	hair2	#FFC400	player		Human
+priya_marlon	20	0.1		110	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 5 10 crit 1.70 critchance 0.15	0.45	0.35	0.05	1	1.1	0.3	7	2200	12	1	talent_all_consumables_stack	human_female	0	hair2	#FFC400	player		Human
+jacques_kennet	20	0.1		120	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 5 10 crit 1.70 critchance 0.05	0.45	0.35	0.04	0.9	1.2	0.3	8	2300	10	1	talent_reload_time	human_male	0	hair1	#908E87	player		Human
+mirza_aishatu	20	0.1		110	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 7 14 crit 1.70 critchance 0.05	0.55	0.45	0.03	1	1.1	0.25	9	2000	10	1	talent_starving_slower	human_female	1	hair2	#633D08	player		Human
+kenzie_yukio	20	0.1		100	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 5 10 crit 1.70 critchance 0.1	0.6	0.4	0.04	1	1	0.4	7	1600	12	1	talent_weight_dodge_affect	human_male	0	hair2	#633D08	player		Human
+marika_wulfnod	20	0.1		100	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 6 12 crit 1.60 critchance 0.05	0.5	0.5	0.04	1	1	0.3	9	1900	12	1	talent_belt_slots	human_female	0	hair1	#FFC400	player		Human
+auberon_lukas	20	0.1		120	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 4 8 crit 1.60 critchance 0.15	0.45	0.45	0.05	0.8	1	0.2	9	1900	8	2	talent_weapon_slot	human_male	0	hair2	#633D08	player		Human
+niko_medich	20	0.1		120	blunt 0 pierce 0 lacer 0 fire 0 cold 0 poison 0 shock 0 beam 0	HumanHead HumanShoulder HumanArm HumanThigh HumanFeet HumanChest HumanBody HumanStomach HumanKnee	blunt 5 10 crit 1.70 critchance 0.05	0.4	0.45	0.04	1	1.3	0.25	8	2000	10	1	talent_pistol_acc	human_male	0	hair1	#908E87	player		Human
+#end																									
+																									
+#mercenary_classes																									
+Id	ModifyStartCost	ModifyStep	PerkIds																						
+scouts_of_hades	30	0.1	cqc_specialist_basic military_training_basic gear_maintenance_basic blind_fury_basic fire_transfer_basic assault_reflex_basic																						
+ecclipse_blades	30	0.1	berserkgang_basic athletics_basic reaction_training_basic cold_weapon_wielding_basic cannibalism_basic carnage_basic																						
+tifton_elite	30	0.1	heavy_weaponary_basic grenadier_basic selfhealing_basic stationary_defense_basic spray_and_pray_basic shock_awe_basic																						
+tunnel_rats	30	0.1	cautious_basic handmade_shotgun_ammo_basic marauder_basic dirty_shot_basic vicious_symbiosis_basic covermaster_basic																						
+phoenix_brigade	30	0.1	shielding_basic battle_physicist_basic reinforced_battery_basic revealing_flame_basic cauterize_basic scholar_basic																						
+]]
+
+	-- Parse with headers and comments
+	local rows, err = fromCSV(original, { delimiter = "\t", hasheader = true, hascomments = true })
+	if err then error("fromCSV error: " .. err) end
+	assert(#rows > 0, "Should have parsed rows")
+
+	-- Convert back to CSV with headers
+	local csv, err = toCSV(rows, { delimiter = "\t", hasheader = true })
+	if err then error("toCSV error: " .. err) end
+
+	-- Parse again
+	local rows2, err = fromCSV(csv, { delimiter = "\t", hasheader = true, hascomments = false })
+	if err then error("fromCSV error: " .. err) end
+
+	-- Verify identical - same number of rows
+	assert(#rows2 == #rows, "Round trip should have same number of rows")
+
+	-- Verify first row data is identical
+	assert(rows2[1].Id == rows[1].Id, "Round trip first row Id should match")
+	assert(
+		rows2[1].ModifyStartCost == rows[1].ModifyStartCost,
+		"Round trip first row ModifyStartCost should match"
+	)
+	assert(rows2[1].Health == rows[1].Health, "Round trip first row Health should match")
+
+	-- Verify headers are preserved
+	assert(rows2[1].Headers ~= nil, "Round trip rows should have Headers field")
+	assert(#rows2[1].Headers == #rows[1].Headers, "Headers should have same number of elements")
+	assert(rows2[1].Headers[1] == rows[1].Headers[1], "First header should match")
+end)
+
 print("\nAll tests completed!")

processor/luahelper.lua

@@ -48,11 +48,44 @@ function dump(table, depth)
 	end
 end
 
+--- @class ParserOptions
+--- @field delimiter string? The field delimiter (default: ",").
+--- @field hasheader boolean? If true, first non-comment row is treated as headers (default: false).
+--- @field hascomments boolean? If true, lines starting with '#' are skipped (default: false).
+
+--- @type ParserOptions
+parserDefaultOptions = { delimiter = ",", hasheader = false, hascomments = false }
+
+--- Validates options against a set of valid option keys.
+--- @param options ParserOptions? The options table to validate
+--- @return boolean #True if options are valid
+--- @return string? #Error message if invalid, nil if valid
+function areOptionsValid(options)
+	if options == nil then return true, nil end
+
+	if type(options) ~= "table" then return false, "options must be a table" end
+
+	-- Build valid options list from validOptions table
+	local validOptionsStr = ""
+	for k, _ in pairs(parserDefaultOptions) do
+		validOptionsStr = validOptionsStr .. k .. ", "
+	end
+
+	for k, _ in pairs(options) do
+		if parserDefaultOptions[k] == nil then
+			return false,
+				"unknown option: " .. tostring(k) .. " (valid options: " .. validOptionsStr .. ")"
+		end
+	end
+
+	return true, nil
+end
+
 --- Parses CSV text into rows and fields using a minimal RFC 4180 state machine.
 ---
 --- Requirements/assumptions:
 --- - Input is a single string containing the entire CSV content.
---- - Field separators are specified by delimiter parameter (default: comma).
+--- - Field separators are specified by delimiter option (default: comma).
 --- - Newlines between rows may be "\n" or "\r\n". "\r\n" is treated as one line break.
 --- - Fields may be quoted with double quotes (").
 --- - Inside quoted fields, doubled quotes ("") represent a literal quote character.
@@ -61,14 +94,23 @@ end
 --- - Leading/trailing spaces are preserved; no trimming is performed.
 --- - Empty fields and empty rows are preserved.
 --- - The final row is emitted even if the text does not end with a newline.
+--- - Lines starting with '#' (after optional leading whitespace) are treated as comments and skipped if hascomments is true.
 ---
 --- @param csv string The CSV text to parse.
---- @param delimiter string? The field delimiter (default: ",").
---- @param hasHeaders boolean? If true, first row is treated as headers and rows can be accessed by header name (default: false).
---- @return table A table (array) of rows; each row is a table with numeric indices and optionally header-named keys.
-function fromCSV(csv, delimiter, hasHeaders)
-	if delimiter == nil then delimiter = "," end
-	if hasHeaders == nil then hasHeaders = false end
+--- @param options ParserOptions? Options for the parser
+--- @return table #A table (array) of rows; each row is a table with numeric indices and optionally header-named keys.
+--- @return string? #Error message if parsing fails.
+function fromCSV(csv, options)
+	if options == nil then options = {} end
+
+	-- Validate options
+	local isValid, err = areOptionsValid(options)
+	if not isValid then return {}, err end
+
+	local delimiter = options.delimiter or parserDefaultOptions.delimiter
+	local hasheader = options.hasheader or parserDefaultOptions.hasheader
+	local hascomments = options.hascomments or parserDefaultOptions.hascomments
+
 	local allRows = {}
 	local fields = {}
 	local field = {}
@@ -95,7 +137,13 @@ function fromCSV(csv, delimiter, hasHeaders)
 			elseif c == "\r" or c == "\n" then
 				table.insert(fields, table.concat(field))
 				field = {}
-				table.insert(allRows, fields)
+				local shouldAdd = true
+				if hascomments and #fields > 0 then
+					local firstField = fields[1]
+					local trimmed = trim(firstField)
+					if string.sub(trimmed, 1, 1) == "#" then shouldAdd = false end
+				end
+				if shouldAdd then table.insert(allRows, fields) end
 				fields = {}
 				if c == "\r" and i < len and csv:sub(i + 1, i + 1) == "\n" then
 					i = i + 2
@@ -127,7 +175,13 @@ function fromCSV(csv, delimiter, hasHeaders)
 			elseif c == "\r" or c == "\n" then
 				table.insert(fields, table.concat(field))
 				field = {}
-				table.insert(allRows, fields)
+				local shouldAdd = true
+				if hascomments and #fields > 0 then
+					local firstField = fields[1]
+					local trimmed = string.gsub(firstField, "^%s*(.-)%s*$", "%1")
+					if string.sub(trimmed, 1, 1) == "#" then shouldAdd = false end
+				end
+				if shouldAdd then table.insert(allRows, fields) end
 				fields = {}
 				state = STATE_DEFAULT
 				if c == "\r" and i < len and csv:sub(i + 1, i + 1) == "\n" then
@@ -144,25 +198,35 @@ function fromCSV(csv, delimiter, hasHeaders)
 
 	if #field > 0 or #fields > 0 then
 		table.insert(fields, table.concat(field))
-		table.insert(allRows, fields)
+		local shouldAdd = true
+		if hascomments and #fields > 0 then
+			local firstField = fields[1]
+			local trimmed = string.gsub(firstField, "^%s*(.-)%s*$", "%1")
+			if string.sub(trimmed, 1, 1) == "#" then shouldAdd = false end
+		end
+		if shouldAdd then table.insert(allRows, fields) end
 	end
 
-	if hasHeaders and #allRows > 0 then
+	if hasheader and #allRows > 0 then
 		local headers = allRows[1]
 		local rows = {}
-		for i = 2, #allRows do
+		for ii = 2, #allRows do
 			local row = {}
-			local dataRow = allRows[i]
+			local dataRow = allRows[ii]
 			for j = 1, #dataRow do
 				row[j] = dataRow[j]
-				if headers[j] ~= nil and headers[j] ~= "" then row[headers[j]] = dataRow[j] end
+				if headers[j] ~= nil and headers[j] ~= "" then
+					local headerName = trim(headers[j])
+					row[headerName] = dataRow[j]
+				end
 			end
+			row.Headers = headers
 			table.insert(rows, row)
 		end
-		return rows
+		return rows, nil
 	end
 
-	return allRows
+	return allRows, nil
 end
 
 --- Converts a table of rows back to CSV text format (RFC 4180 compliant).
@@ -173,14 +237,46 @@ end
 --- - Fields are quoted if they contain the delimiter, newlines, or double quotes.
 --- - Double quotes inside quoted fields are doubled ("").
 --- - Fields are joined with the specified delimiter; rows are joined with newlines.
+--- - If includeHeaders is true and rows have a Headers field, headers are included as the first row.
 ---
 --- @param rows table Array of rows, where each row is an array of field values.
---- @param delimiter string? The field delimiter (default: ",").
---- @return string CSV-formatted text.
-function toCSV(rows, delimiter)
-	if delimiter == nil then delimiter = "," end
+--- @param options ParserOptions? Options for the parser
+--- @return string? #CSV-formatted text, error string?
+--- @return string? #Error message if conversion fails.
+function toCSV(rows, options)
+	if options == nil then options = {} end
+
+	-- Validate options
+	local isValid, err = areOptionsValid(options)
+	if not isValid then return nil, err end
+
+	local delimiter = options.delimiter or parserDefaultOptions.delimiter
+	local includeHeaders = options.hasheader or parserDefaultOptions.hasheader
 	local rowStrings = {}
 
+	-- Include headers row if requested and available
+	if includeHeaders and #rows > 0 and rows[1].Headers ~= nil then
+		local headerStrings = {}
+		for _, header in ipairs(rows[1].Headers) do
+			local headerStr = tostring(header)
+			local needsQuoting = false
+			if
+				headerStr:find(delimiter)
+				or headerStr:find("\n")
+				or headerStr:find("\r")
+				or headerStr:find('"')
+			then
+				needsQuoting = true
+			end
+			if needsQuoting then
+				headerStr = headerStr:gsub('"', '""')
+				headerStr = '"' .. headerStr .. '"'
+			end
+			table.insert(headerStrings, headerStr)
+		end
+		table.insert(rowStrings, table.concat(headerStrings, delimiter))
+	end
+
 	for _, row in ipairs(rows) do
 		local fieldStrings = {}
 
@@ -208,7 +304,7 @@ function toCSV(rows, delimiter)
 		table.insert(rowStrings, table.concat(fieldStrings, delimiter))
 	end
 
-	return table.concat(rowStrings, "\n")
+	return table.concat(rowStrings, "\n"), nil
 end
 
 -- String to number conversion helper