package main import ( "errors" "flag" "fmt" "os" "sort" "sync" "sync/atomic" "time" "cook/processor" "cook/utils" "gopkg.in/yaml.v3" logger "git.site.quack-lab.dev/dave/cylogger" ) // mainLogger is a scoped logger for the main package. var mainLogger = logger.Default.WithPrefix("main") type GlobalStats struct { TotalMatches int64 TotalModifications int64 ProcessedFiles int64 FailedFiles int64 ModificationsPerCommand sync.Map } var ( stats GlobalStats = GlobalStats{ ModificationsPerCommand: sync.Map{}, } ) func main() { flag.Usage = func() { CreateExampleConfig() fmt.Fprintf(os.Stderr, "Usage: %s [options] <...files_or_globs>\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nOptions:\n") fmt.Fprintf(os.Stderr, " -reset\n") fmt.Fprintf(os.Stderr, " Reset files to their original state\n") fmt.Fprintf(os.Stderr, " -loglevel string\n") fmt.Fprintf(os.Stderr, " Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n") fmt.Fprintf(os.Stderr, " -json\n") fmt.Fprintf(os.Stderr, " Enable JSON mode for processing JSON files\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, " Regex mode (default):\n") fmt.Fprintf(os.Stderr, " %s \"(\\\\d+)\" \"*1.5\" data.xml\n", os.Args[0]) fmt.Fprintf(os.Stderr, " JSON mode:\n") fmt.Fprintf(os.Stderr, " %s -json data.json\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n") fmt.Fprintf(os.Stderr, " s1, s2, etc. are used to refer to capture groups as strings.\n") fmt.Fprintf(os.Stderr, " Helper functions: num(str) converts string to number, str(num) converts number to string\n") fmt.Fprintf(os.Stderr, " is_number(str) checks if a string is numeric\n") fmt.Fprintf(os.Stderr, " If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended\n") fmt.Fprintf(os.Stderr, " You can use any valid Lua code, including if statements, loops, etc.\n") fmt.Fprintf(os.Stderr, " Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)\n") } // TODO: Fix bed shitting when doing *.yml in barotrauma directory flag.Parse() args := flag.Args() logger.InitFlag() mainLogger.Info("Initializing with log level: %s", logger.GetLevel().String()) mainLogger.Trace("Full argv: %v", os.Args) if flag.NArg() == 0 { flag.Usage() return } mainLogger.Debug("Getting database connection") db, err := utils.GetDB() if err != nil { mainLogger.Error("Failed to get database: %v", err) return } mainLogger.Debug("Database connection established") workdone, err := HandleSpecialArgs(args, err, db) if err != nil { mainLogger.Error("Failed to handle special args: %v", err) return } if workdone { mainLogger.Info("Special arguments handled, exiting.") return } // The plan is: // Load all commands mainLogger.Debug("Loading commands from arguments") mainLogger.Trace("Arguments: %v", args) commands, err := utils.LoadCommands(args) if err != nil || len(commands) == 0 { mainLogger.Error("Failed to load commands: %v", err) flag.Usage() return } // Collect global modifiers from special entries and filter them out vars := map[string]interface{}{} filtered := make([]utils.ModifyCommand, 0, len(commands)) for _, c := range commands { if len(c.Modifiers) > 0 && c.Name == "" && c.Regex == "" && len(c.Regexes) == 0 && c.Lua == "" && len(c.Files) == 0 { for k, v := range c.Modifiers { vars[k] = v } continue } filtered = append(filtered, c) } if len(vars) > 0 { mainLogger.Info("Loaded %d global modifiers", len(vars)) processor.SetVariables(vars) } commands = filtered mainLogger.Info("Loaded %d commands", len(commands)) if *utils.Filter != "" { mainLogger.Info("Filtering commands by name: %s", *utils.Filter) commands = utils.FilterCommands(commands, *utils.Filter) mainLogger.Info("Filtered %d commands", len(commands)) } // Then aggregate all the globs and deduplicate them mainLogger.Debug("Aggregating globs and deduplicating") globs := utils.AggregateGlobs(commands) mainLogger.Debug("Aggregated %d globs before deduplication", utils.CountGlobsBeforeDedup(commands)) for _, command := range commands { mainLogger.Trace("Command: %s", command.Name) if len(command.Regexes) > 0 { mainLogger.Trace("Regexes: %v", command.Regexes) } else { mainLogger.Trace("Regex: %s", command.Regex) } mainLogger.Trace("Files: %v", command.Files) mainLogger.Trace("Lua: %s", command.Lua) mainLogger.Trace("Reset: %t", command.Reset) mainLogger.Trace("Isolate: %t", command.Isolate) mainLogger.Trace("LogLevel: %s", command.LogLevel) } // Resolve all the files for all the globs mainLogger.Info("Found %d unique file patterns", len(globs)) mainLogger.Debug("Expanding glob patterns to files") files, err := utils.ExpandGLobs(globs) if err != nil { mainLogger.Error("Failed to expand file patterns: %v", err) return } mainLogger.Info("Found %d files to process", len(files)) mainLogger.Trace("Files to process: %v", files) // Somehow connect files to commands via globs.. // For each file check every glob of every command // Maybe memoize this part // That way we know what commands affect what files mainLogger.Debug("Associating files with commands") associations, err := utils.AssociateFilesWithCommands(files, commands) if err != nil { mainLogger.Error("Failed to associate files with commands: %v", err) return } mainLogger.Debug("Files associated with commands") mainLogger.Trace("File-command associations: %v", associations) // Per-file association summary for better visibility when debugging for file, assoc := range associations { cmdNames := make([]string, 0, len(assoc.Commands)) for _, c := range assoc.Commands { cmdNames = append(cmdNames, c.Name) } isoNames := make([]string, 0, len(assoc.IsolateCommands)) for _, c := range assoc.IsolateCommands { isoNames = append(isoNames, c.Name) } mainLogger.Debug("File %q has %d regular and %d isolate commands", file, len(assoc.Commands), len(assoc.IsolateCommands)) mainLogger.Trace("\tRegular: %v", cmdNames) mainLogger.Trace("\tIsolate: %v", isoNames) } mainLogger.Debug("Resetting files where necessary") err = utils.ResetWhereNecessary(associations, db) if err != nil { mainLogger.Error("Failed to reset files where necessary: %v", err) return } mainLogger.Debug("Files reset where necessary") // Then for each file run all commands associated with the file workers := make(chan struct{}, *utils.ParallelFiles) wg := sync.WaitGroup{} mainLogger.Debug("Starting file processing with %d parallel workers", *utils.ParallelFiles) // Add performance tracking startTime := time.Now() // Create a map to store loggers for each command commandLoggers := make(map[string]*logger.Logger) for _, command := range commands { // Create a named logger for each command cmdName := command.Name if cmdName == "" { // If no name is provided, use a short version of the regex pattern if len(command.Regex) > 20 { cmdName = command.Regex[:17] + "..." } else { cmdName = command.Regex } } // Parse the log level for this specific command cmdLogLevel := logger.ParseLevel(command.LogLevel) // Create a logger with the command name as a field commandLoggers[command.Name] = logger.Default.WithField("command", cmdName) commandLoggers[command.Name].SetLevel(cmdLogLevel) mainLogger.Debug("Created logger for command %q with log level %s", cmdName, cmdLogLevel.String()) } for file, association := range associations { workers <- struct{}{} wg.Add(1) logger.SafeGoWithArgs(func(args ...interface{}) { defer func() { <-workers }() defer wg.Done() // Track per-file processing time fileStartTime := time.Now() mainLogger.Debug("Reading file %q", file) fileData, err := os.ReadFile(file) if err != nil { mainLogger.Error("Failed to read file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } fileDataStr := string(fileData) mainLogger.Trace("File %q content: %s", file, utils.LimitString(fileDataStr, 500)) isChanged := false mainLogger.Debug("Running isolate commands for file %q", file) fileDataStr, err = RunIsolateCommands(association, file, fileDataStr) if err != nil && err != NothingToDo { mainLogger.Error("Failed to run isolate commands for file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } if err != NothingToDo { isChanged = true } mainLogger.Debug("Running other commands for file %q", file) fileDataStr, err = RunOtherCommands(file, fileDataStr, association, commandLoggers) if err != nil && err != NothingToDo { mainLogger.Error("Failed to run other commands for file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } if err != NothingToDo { isChanged = true } if isChanged { mainLogger.Debug("Saving file %q to database", file) err = db.SaveFile(file, fileData) if err != nil { mainLogger.Error("Failed to save file %q to database: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } mainLogger.Debug("File %q saved to database", file) } mainLogger.Debug("Writing file %q", file) err = os.WriteFile(file, []byte(fileDataStr), 0644) if err != nil { mainLogger.Error("Failed to write file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } mainLogger.Debug("File %q written", file) // Only increment ProcessedFiles once per file, after all processing is complete atomic.AddInt64(&stats.ProcessedFiles, 1) mainLogger.Debug("File %q processed in %v", file, time.Since(fileStartTime)) }, file, commands) } wg.Wait() processingTime := time.Since(startTime) mainLogger.Info("Processing completed in %v", processingTime) processedFiles := atomic.LoadInt64(&stats.ProcessedFiles) if processedFiles > 0 { mainLogger.Info("Average time per file: %v", processingTime/time.Duration(processedFiles)) } // TODO: Also give each command its own logger, maybe prefix it with something... Maybe give commands a name? // Do that with logger.WithField("loglevel", level.String()) // Since each command also has its own log level // TODO: Maybe even figure out how to run individual commands...? // TODO: What to do with git? Figure it out .... // if *gitFlag { // mainLogger.Info("Git integration enabled, setting up git repository") // err := setupGit() // if err != nil { // mainLogger.Error("Failed to setup git: %v", err) // fmt.Fprintf(os.Stderr, "Error setting up git: %v\n", err) // return // } // } // mainLogger.Debug("Expanding file patterns") // files, err := expandFilePatterns(filePatterns) // if err != nil { // mainLogger.Error("Failed to expand file patterns: %v", err) // fmt.Fprintf(os.Stderr, "Error expanding file patterns: %v\n", err) // return // } // if *gitFlag { // mainLogger.Info("Cleaning up git files before processing") // err := cleanupGitFiles(files) // if err != nil { // mainLogger.Error("Failed to cleanup git files: %v", err) // fmt.Fprintf(os.Stderr, "Error cleaning up git files: %v\n", err) // return // } // } // if *resetFlag { // mainLogger.Info("Files reset to their original state, nothing more to do") // log.Printf("Files reset to their original state, nothing more to do") // return // } // Print summary totalModifications := atomic.LoadInt64(&stats.TotalModifications) if totalModifications == 0 { mainLogger.Warning("No modifications were made in any files") } else { failedFiles := atomic.LoadInt64(&stats.FailedFiles) mainLogger.Info("Operation complete! Modified %d values in %d/%d files", totalModifications, processedFiles, processedFiles+failedFiles) sortedCommands := []string{} stats.ModificationsPerCommand.Range(func(key, value interface{}) bool { sortedCommands = append(sortedCommands, key.(string)) return true }) sort.Strings(sortedCommands) for _, command := range sortedCommands { count, _ := stats.ModificationsPerCommand.Load(command) if count.(int) > 0 { mainLogger.Info("\tCommand %q made %d modifications", command, count) } else { mainLogger.Warning("\tCommand %q made no modifications", command) } } } } func HandleSpecialArgs(args []string, err error, db utils.DB) (bool, error) { handleSpecialArgsLogger := logger.Default.WithPrefix("HandleSpecialArgs") handleSpecialArgsLogger.Debug("Handling special arguments: %v", args) switch args[0] { case "reset": handleSpecialArgsLogger.Info("Resetting all files") err = utils.ResetAllFiles(db) if err != nil { handleSpecialArgsLogger.Error("Failed to reset all files: %v", err) return true, err } handleSpecialArgsLogger.Info("All files reset") return true, nil case "dump": handleSpecialArgsLogger.Info("Dumping all files from database") err = db.RemoveAllFiles() if err != nil { handleSpecialArgsLogger.Error("Failed to remove all files from database: %v", err) return true, err } handleSpecialArgsLogger.Info("All files removed from database") return true, nil } handleSpecialArgsLogger.Debug("No special arguments handled, returning false") return false, nil } func CreateExampleConfig() { createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig") createExampleConfigLogger.Debug("Creating example configuration file") commands := []utils.ModifyCommand{ // Global modifiers only entry (no name/regex/lua/files) { Modifiers: map[string]interface{}{ "foobar": 4, "multiply": 1.5, "prefix": "NEW_", "enabled": true, }, }, // Multi-regex example using $variable in Lua { Name: "RFToolsMultiply", Regexes: []string{"generatePerTick = !num", "ticksPer\\w+ = !num", "generatorRFPerTick = !num"}, Lua: "* $foobar", Files: []string{"polymc/instances/**/rftools*.toml", `polymc\\instances\\**\\rftools*.toml`}, Reset: true, // LogLevel defaults to INFO }, // Named capture groups with arithmetic and string ops { Name: "UpdateAmountsAndItems", Regex: `(?P!num)\s+units\s+of\s+(?P[A-Za-z_\-]+)`, Lua: `amount = amount * $multiply; item = upper(item); return true`, Files: []string{"data/**/*.txt"}, // INFO log level }, // Full replacement via Lua 'replacement' variable { Name: "BumpMinorVersion", Regex: `version\s*=\s*"(?P!num)\.(?P!num)\.(?P!num)"`, Lua: `replacement = format("version=\"%s.%s.%s\"", major, num(minor)+1, 0); return true`, Files: []string{"config/*.ini", "config/*.cfg"}, }, // Multiline regex example (DOTALL is auto-enabled). Captures numeric in nested XML. { Name: "XMLNestedValueMultiply", Regex: `\s*\s*!any<\/name>\s*\s*(!num)<\/value>\s*\s*<\/item>`, Lua: `* $multiply`, Files: []string{"data/**/*.xml"}, // Demonstrates multiline regex in YAML }, // Multiline regexES array, with different patterns handled by same Lua { Name: "MultiLinePatterns", Regexes: []string{ `\s*\n\s*(?P!num)\s*\n\s*(?P!num)\s*\n\s*`, `\[block\]\nkey=(?P[A-Za-z_]+)\nvalue=(?P!num)`, }, Lua: `if is_number(score) then score = score * 2 end; if is_number(val) then val = val * 3 end; return true`, Files: []string{"examples/**/*.*"}, LogLevel: "DEBUG", }, // Use equals operator shorthand and boolean variable { Name: "EnableFlags", Regex: `enabled\s*=\s*(true|false)`, Lua: `= $enabled`, Files: []string{"**/*.toml"}, }, // Demonstrate NoDedup to allow overlapping replacements { Name: "OverlappingGroups", Regex: `(?P!num)(?P!num)`, Lua: `a = num(a) + 1; b = num(b) + 1; return true`, Files: []string{"overlap/**/*.txt"}, NoDedup: true, }, // Isolate command example operating on entire matched block { Name: "IsolateUppercaseBlock", Regex: `BEGIN\n(?P!any)\nEND`, Lua: `block = upper(block); return true`, Files: []string{"logs/**/*.log"}, Isolate: true, LogLevel: "TRACE", }, // Using !rep placeholder and arrays of files { Name: "RepeatPlaceholderExample", Regex: `name: (.*) !rep(, .* , 2)`, Lua: `-- no-op, just demonstrate placeholder; return false`, Files: []string{"lists/**/*.yml", "lists/**/*.yaml"}, }, // Using string variable in Lua expression { Name: "PrefixKeys", Regex: `(?P[A-Za-z0-9_]+)\s*=`, Lua: `key = $prefix .. key; return true`, Files: []string{"**/*.properties"}, }, // JSON mode examples { Name: "JSONArrayMultiply", JSON: true, Lua: `for i, item in ipairs(data.items) do data.items[i].value = item.value * 2 end; return true`, Files: []string{"data/**/*.json"}, }, { Name: "JSONObjectUpdate", JSON: true, Lua: `data.version = "2.0.0"; data.enabled = true; return true`, Files: []string{"config/**/*.json"}, }, { Name: "JSONNestedModify", JSON: true, Lua: `if data.settings and data.settings.performance then data.settings.performance.multiplier = data.settings.performance.multiplier * 1.5 end; return true`, Files: []string{"settings/**/*.json"}, }, } data, err := yaml.Marshal(commands) if err != nil { createExampleConfigLogger.Error("Failed to marshal example config: %v", err) return } createExampleConfigLogger.Debug("Writing example_cook.yml") err = os.WriteFile("example_cook.yml", data, 0644) if err != nil { createExampleConfigLogger.Error("Failed to write example_cook.yml: %v", err) return } createExampleConfigLogger.Info("Wrote example_cook.yml") } var NothingToDo = errors.New("nothing to do") func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, commandLoggers map[string]*logger.Logger) (string, error) { runOtherCommandsLogger := mainLogger.WithPrefix("RunOtherCommands").WithField("file", file) runOtherCommandsLogger.Debug("Running other commands for file") runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200)) // Separate JSON and regex commands for different processing approaches jsonCommands := []utils.ModifyCommand{} regexCommands := []utils.ModifyCommand{} for _, command := range association.Commands { if command.JSON || *utils.JSON { jsonCommands = append(jsonCommands, command) } else { regexCommands = append(regexCommands, command) } } // Process JSON commands sequentially (each operates on the entire file) for _, command := range jsonCommands { cmdLogger := logger.Default if cmdLog, ok := commandLoggers[command.Name]; ok { cmdLogger = cmdLog } cmdLogger.Debug("Processing file with JSON mode for command %q", command.Name) newModifications, err := processor.ProcessJSON(fileDataStr, command, file) if err != nil { runOtherCommandsLogger.Error("Failed to process file with JSON command %q: %v", command.Name, err) continue } // Apply JSON modifications immediately if len(newModifications) > 0 { var count int fileDataStr, count = utils.ExecuteModifications(newModifications, fileDataStr) atomic.AddInt64(&stats.TotalModifications, int64(count)) cmdLogger.Debug("Applied %d JSON modifications for command %q", count, command.Name) } count, ok := stats.ModificationsPerCommand.Load(command.Name) if !ok { count = 0 } stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications)) } // Aggregate regex modifications and execute them modifications := []utils.ReplaceCommand{} numCommandsConsidered := 0 for _, command := range regexCommands { cmdLogger := logger.Default if cmdLog, ok := commandLoggers[command.Name]; ok { cmdLogger = cmdLog } patterns := command.Regexes if len(patterns) == 0 { patterns = []string{command.Regex} } for idx, pattern := range patterns { tmpCmd := command tmpCmd.Regex = pattern cmdLogger.Debug("Begin processing file with command %q (pattern %d/%d)", command.Name, idx+1, len(patterns)) numCommandsConsidered++ newModifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file) if err != nil { runOtherCommandsLogger.Error("Failed to process file with command %q: %v", command.Name, err) continue } modifications = append(modifications, newModifications...) count, ok := stats.ModificationsPerCommand.Load(command.Name) if !ok { count = 0 } stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications)) cmdLogger.Debug("Command %q generated %d modifications (pattern %d/%d)", command.Name, len(newModifications), idx+1, len(patterns)) cmdLogger.Trace("Modifications generated by command %q: %v", command.Name, newModifications) if len(newModifications) == 0 { cmdLogger.Debug("No modifications yielded by command %q (pattern %d/%d)", command.Name, idx+1, len(patterns)) } } } runOtherCommandsLogger.Debug("Aggregated %d modifications from %d command-pattern runs", len(modifications), numCommandsConsidered) runOtherCommandsLogger.Trace("All aggregated modifications: %v", modifications) if len(modifications) == 0 { runOtherCommandsLogger.Warning("No modifications found for file") return fileDataStr, NothingToDo } runOtherCommandsLogger.Debug("Executing %d modifications for file", len(modifications)) // Sort commands in reverse order for safe replacements var count int fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr) runOtherCommandsLogger.Trace("File data after modifications: %s", utils.LimitString(fileDataStr, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) runOtherCommandsLogger.Info("Executed %d modifications for file", count) return fileDataStr, nil } func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string) (string, error) { runIsolateCommandsLogger := mainLogger.WithPrefix("RunIsolateCommands").WithField("file", file) runIsolateCommandsLogger.Debug("Running isolate commands for file") runIsolateCommandsLogger.Trace("File data before isolate modifications: %s", utils.LimitString(fileDataStr, 200)) anythingDone := false for _, isolateCommand := range association.IsolateCommands { // Check if this isolate command should use JSON mode if isolateCommand.JSON || *utils.JSON { runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name) modifications, err := processor.ProcessJSON(fileDataStr, isolateCommand, file) if err != nil { runIsolateCommandsLogger.Error("Failed to process file with JSON isolate command %q: %v", isolateCommand.Name, err) continue } if len(modifications) == 0 { runIsolateCommandsLogger.Debug("JSON isolate command %q produced no modifications", isolateCommand.Name) continue } anythingDone = true runIsolateCommandsLogger.Debug("Executing %d JSON isolate modifications for file", len(modifications)) runIsolateCommandsLogger.Trace("JSON isolate modifications: %v", modifications) var count int fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr) runIsolateCommandsLogger.Trace("File data after JSON isolate modifications: %s", utils.LimitString(fileDataStr, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) runIsolateCommandsLogger.Info("Executed %d JSON isolate modifications for file", count) } else { // Regular regex processing for isolate commands runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex) patterns := isolateCommand.Regexes if len(patterns) == 0 { patterns = []string{isolateCommand.Regex} } for idx, pattern := range patterns { tmpCmd := isolateCommand tmpCmd.Regex = pattern modifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file) if err != nil { runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err) continue } if len(modifications) == 0 { runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns)) continue } anythingDone = true runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications)) runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications) var count int fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr) runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(fileDataStr, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count) } } } if !anythingDone { runIsolateCommandsLogger.Debug("No isolate modifications were made for file") return fileDataStr, NothingToDo } return fileDataStr, nil }