package main import ( _ "embed" "errors" "os" "sort" "sync" "sync/atomic" "time" "cook/processor" "cook/utils" "github.com/spf13/cobra" logger "git.site.quack-lab.dev/dave/cylogger" ) //go:embed example_cook.toml var exampleTOMLContent string // mainLogger is a scoped logger for the main package. var mainLogger = logger.Default.WithPrefix("main") type GlobalStats struct { TotalMatches int64 TotalModifications int64 ProcessedFiles int64 FailedFiles int64 ModificationsPerCommand sync.Map } var ( stats GlobalStats = GlobalStats{ ModificationsPerCommand: sync.Map{}, } ) // rootCmd represents the base command when called without any subcommands var rootCmd *cobra.Command func init() { rootCmd = &cobra.Command{ Use: "modifier [options] <...files_or_globs>", Short: "A powerful file modification tool with Lua scripting", Long: `Modifier is a powerful file processing tool that supports regex patterns, JSON manipulation, and YAML to TOML conversion with Lua scripting capabilities. Features: - Regex-based pattern matching and replacement - JSON file processing with query support - YAML to TOML conversion - Lua scripting for complex transformations - Parallel file processing - Command filtering and organization`, PersistentPreRun: func(cmd *cobra.Command, args []string) { CreateExampleConfig() logger.InitFlag() mainLogger.Info("Initializing with log level: %s", logger.GetLevel().String()) mainLogger.Trace("Full argv: %v", os.Args) }, Run: func(cmd *cobra.Command, args []string) { if len(args) == 0 { cmd.Usage() return } runModifier(args, cmd) }, } // Global flags rootCmd.PersistentFlags().StringP("loglevel", "l", "INFO", "Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE") // Local flags rootCmd.Flags().IntP("parallel", "P", 100, "Number of files to process in parallel") rootCmd.Flags().StringP("filter", "f", "", "Filter commands before running them") rootCmd.Flags().Bool("json", false, "Enable JSON mode for processing JSON files") rootCmd.Flags().BoolP("conv", "c", false, "Convert YAML files to TOML format") // Set up examples in the help text rootCmd.SetUsageTemplate(`Usage:{{if .Runnable}} {{.UseLine}}{{end}}{{if .HasAvailableSubCommands}} {{.CommandPath}} [command]{{end}} {{if gt (len .Aliases) 0}} Aliases: {{.NameAndAliases}}{{end}}{{if .HasExample}} Examples: {{.Example}}{{end}}{{if .HasAvailableSubCommands}} Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}} {{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}} Flags: {{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}} Global Flags: {{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}} Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}} {{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}} Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}} `) // Add examples rootCmd.Example = ` Regex mode (default): modifier "(\\d+)" "*1.5" data.xml JSON mode: modifier -json data.json YAML to TOML conversion: modifier -conv *.yml modifier -conv **/*.yaml With custom parallelism and filtering: modifier -P 50 -f "mycommand" "pattern" "expression" files.txt Note: v1, v2, etc. are used to refer to capture groups as numbers. s1, s2, etc. are used to refer to capture groups as strings. Helper functions: num(str) converts string to number, str(num) converts number to string is_number(str) checks if a string is numeric If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended You can use any valid Lua code, including if statements, loops, etc. Glob patterns are supported for file selection (*.xml, data/**.xml, etc.) ` + processor.GetLuaFunctionsHelp() } func main() { if err := rootCmd.Execute(); err != nil { mainLogger.Error("Command execution failed: %v", err) os.Exit(1) } } func runModifier(args []string, cmd *cobra.Command) { // Get flag values from Cobra convertFlag, _ := cmd.Flags().GetBool("conv") parallelFlag, _ := cmd.Flags().GetInt("parallel") filterFlag, _ := cmd.Flags().GetString("filter") jsonFlag, _ := cmd.Flags().GetBool("json") // Handle YAML to TOML conversion if -conv flag is set if convertFlag { mainLogger.Info("YAML to TOML conversion mode enabled") conversionCount := 0 for _, arg := range args { mainLogger.Debug("Converting YAML files matching pattern: %s", arg) err := utils.ConvertYAMLToTOML(arg) if err != nil { mainLogger.Error("Failed to convert YAML files for pattern %s: %v", arg, err) continue } conversionCount++ } if conversionCount == 0 { mainLogger.Warning("No files were converted. Please check your patterns.") } else { mainLogger.Info("Conversion completed for %d pattern(s)", conversionCount) } return } mainLogger.Debug("Getting database connection") db, err := utils.GetDB() if err != nil { mainLogger.Error("Failed to get database: %v", err) return } mainLogger.Debug("Database connection established") workdone, err := HandleSpecialArgs(args, db) if err != nil { mainLogger.Error("Failed to handle special args: %v", err) return } if workdone { mainLogger.Info("Special arguments handled, exiting.") return } // The plan is: // Load all commands mainLogger.Debug("Loading commands from arguments") mainLogger.Trace("Arguments: %v", args) commands, err := utils.LoadCommands(args) if err != nil || len(commands) == 0 { mainLogger.Error("Failed to load commands: %v", err) cmd.Usage() return } // Collect global modifiers from special entries and filter them out vars := map[string]interface{}{} filtered := make([]utils.ModifyCommand, 0, len(commands)) for _, c := range commands { if len(c.Modifiers) > 0 && c.Name == "" && c.Regex == "" && len(c.Regexes) == 0 && c.Lua == "" && len(c.Files) == 0 { for k, v := range c.Modifiers { vars[k] = v } continue } filtered = append(filtered, c) } if len(vars) > 0 { mainLogger.Info("Loaded %d global modifiers", len(vars)) processor.SetVariables(vars) } commands = filtered mainLogger.Info("Loaded %d commands", len(commands)) if filterFlag != "" { mainLogger.Info("Filtering commands by name: %s", filterFlag) commands = utils.FilterCommands(commands, filterFlag) mainLogger.Info("Filtered %d commands", len(commands)) } // Then aggregate all the globs and deduplicate them mainLogger.Debug("Aggregating globs and deduplicating") globs := utils.AggregateGlobs(commands) mainLogger.Debug("Aggregated %d globs before deduplication", utils.CountGlobsBeforeDedup(commands)) for _, command := range commands { mainLogger.Trace("Command: %s", command.Name) if len(command.Regexes) > 0 { mainLogger.Trace("Regexes: %v", command.Regexes) } else { mainLogger.Trace("Regex: %s", command.Regex) } mainLogger.Trace("Files: %v", command.Files) mainLogger.Trace("Lua: %s", command.Lua) mainLogger.Trace("Reset: %t", command.Reset) mainLogger.Trace("Isolate: %t", command.Isolate) mainLogger.Trace("LogLevel: %s", command.LogLevel) } // Resolve all the files for all the globs mainLogger.Info("Found %d unique file patterns", len(globs)) mainLogger.Debug("Expanding glob patterns to files") files, err := utils.ExpandGLobs(globs) if err != nil { mainLogger.Error("Failed to expand file patterns: %v", err) return } mainLogger.Info("Found %d files to process", len(files)) mainLogger.Trace("Files to process: %v", files) // Somehow connect files to commands via globs.. // For each file check every glob of every command // Maybe memoize this part // That way we know what commands affect what files mainLogger.Debug("Associating files with commands") associations, err := utils.AssociateFilesWithCommands(files, commands) if err != nil { mainLogger.Error("Failed to associate files with commands: %v", err) return } mainLogger.Debug("Files associated with commands") mainLogger.Trace("File-command associations: %v", associations) // Per-file association summary for better visibility when debugging for file, assoc := range associations { cmdNames := make([]string, 0, len(assoc.Commands)) for _, c := range assoc.Commands { cmdNames = append(cmdNames, c.Name) } isoNames := make([]string, 0, len(assoc.IsolateCommands)) for _, c := range assoc.IsolateCommands { isoNames = append(isoNames, c.Name) } mainLogger.Debug("File %q has %d regular and %d isolate commands", file, len(assoc.Commands), len(assoc.IsolateCommands)) mainLogger.Trace("\tRegular: %v", cmdNames) mainLogger.Trace("\tIsolate: %v", isoNames) } mainLogger.Debug("Resetting files where necessary") err = utils.ResetWhereNecessary(associations, db) if err != nil { mainLogger.Error("Failed to reset files where necessary: %v", err) return } mainLogger.Debug("Files reset where necessary") // Then for each file run all commands associated with the file workers := make(chan struct{}, parallelFlag) wg := sync.WaitGroup{} mainLogger.Debug("Starting file processing with %d parallel workers", parallelFlag) // Add performance tracking startTime := time.Now() // Create a map to store loggers for each command commandLoggers := make(map[string]*logger.Logger) for _, command := range commands { // Create a named logger for each command cmdName := command.Name if cmdName == "" { // If no name is provided, use a short version of the regex pattern if len(command.Regex) > 20 { cmdName = command.Regex[:17] + "..." } else { cmdName = command.Regex } } // Parse the log level for this specific command cmdLogLevel := logger.ParseLevel(command.LogLevel) // Create a logger with the command name as a field commandLoggers[command.Name] = logger.Default.WithField("command", cmdName) commandLoggers[command.Name].SetLevel(cmdLogLevel) mainLogger.Debug("Created logger for command %q with log level %s", cmdName, cmdLogLevel.String()) } for file, association := range associations { workers <- struct{}{} wg.Add(1) logger.SafeGoWithArgs(func(args ...interface{}) { defer func() { <-workers }() defer wg.Done() // Track per-file processing time fileStartTime := time.Now() mainLogger.Debug("Reading file %q", file) fileData, err := os.ReadFile(file) if err != nil { mainLogger.Error("Failed to read file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } fileDataStr := string(fileData) mainLogger.Trace("File %q content: %s", file, utils.LimitString(fileDataStr, 500)) isChanged := false mainLogger.Debug("Running isolate commands for file %q", file) fileDataStr, err = RunIsolateCommands(association, file, fileDataStr, jsonFlag) if err != nil && err != NothingToDo { mainLogger.Error("Failed to run isolate commands for file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } if err != NothingToDo { isChanged = true } mainLogger.Debug("Running other commands for file %q", file) fileDataStr, err = RunOtherCommands(file, fileDataStr, association, commandLoggers, jsonFlag) if err != nil && err != NothingToDo { mainLogger.Error("Failed to run other commands for file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } if err != NothingToDo { isChanged = true } if isChanged { mainLogger.Debug("Saving file %q to database", file) err = db.SaveFile(file, fileData) if err != nil { mainLogger.Error("Failed to save file %q to database: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } mainLogger.Debug("File %q saved to database", file) } mainLogger.Debug("Writing file %q", file) err = os.WriteFile(file, []byte(fileDataStr), 0644) if err != nil { mainLogger.Error("Failed to write file %q: %v", file, err) atomic.AddInt64(&stats.FailedFiles, 1) return } mainLogger.Debug("File %q written", file) // Only increment ProcessedFiles once per file, after all processing is complete atomic.AddInt64(&stats.ProcessedFiles, 1) mainLogger.Debug("File %q processed in %v", file, time.Since(fileStartTime)) }, file, commands) } wg.Wait() processingTime := time.Since(startTime) mainLogger.Info("Processing completed in %v", processingTime) processedFiles := atomic.LoadInt64(&stats.ProcessedFiles) if processedFiles > 0 { mainLogger.Info("Average time per file: %v", processingTime/time.Duration(processedFiles)) } // TODO: Also give each command its own logger, maybe prefix it with something... Maybe give commands a name? // Do that with logger.WithField("loglevel", level.String()) // Since each command also has its own log level // TODO: Maybe even figure out how to run individual commands...? // Print summary totalModifications := atomic.LoadInt64(&stats.TotalModifications) if totalModifications == 0 { mainLogger.Warning("No modifications were made in any files") } else { failedFiles := atomic.LoadInt64(&stats.FailedFiles) mainLogger.Info("Operation complete! Modified %d values in %d/%d files", totalModifications, processedFiles, processedFiles+failedFiles) sortedCommands := []string{} stats.ModificationsPerCommand.Range(func(key, value interface{}) bool { sortedCommands = append(sortedCommands, key.(string)) return true }) sort.Strings(sortedCommands) for _, command := range sortedCommands { count, _ := stats.ModificationsPerCommand.Load(command) if count.(int) > 0 { mainLogger.Info("\tCommand %q made %d modifications", command, count) } else { mainLogger.Warning("\tCommand %q made no modifications", command) } } } } func HandleSpecialArgs(args []string, db utils.DB) (bool, error) { handleSpecialArgsLogger := logger.Default.WithPrefix("HandleSpecialArgs") handleSpecialArgsLogger.Debug("Handling special arguments: %v", args) if len(args) == 0 { handleSpecialArgsLogger.Warning("No arguments provided to HandleSpecialArgs") return false, nil } switch args[0] { case "reset": handleSpecialArgsLogger.Info("Resetting all files to their original state from database") err := utils.ResetAllFiles(db) if err != nil { handleSpecialArgsLogger.Error("Failed to reset all files: %v", err) return true, err } handleSpecialArgsLogger.Info("Successfully reset all files to original state") return true, nil case "dump": handleSpecialArgsLogger.Info("Dumping all files from database (clearing snapshots)") err := db.RemoveAllFiles() if err != nil { handleSpecialArgsLogger.Error("Failed to remove all files from database: %v", err) return true, err } handleSpecialArgsLogger.Info("Successfully cleared all file snapshots from database") return true, nil default: handleSpecialArgsLogger.Debug("Unknown special argument: %q", args[0]) } handleSpecialArgsLogger.Debug("No special arguments handled, returning false") return false, nil } func CreateExampleConfig() { createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig") createExampleConfigLogger.Debug("Creating example configuration file") // Save the embedded TOML content to disk createExampleConfigLogger.Debug("Writing example_cook.toml") err := os.WriteFile("example_cook.toml", []byte(exampleTOMLContent), 0644) if err != nil { createExampleConfigLogger.Error("Failed to write example_cook.toml: %v", err) return } createExampleConfigLogger.Info("Wrote example_cook.toml") } var NothingToDo = errors.New("nothing to do") func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, commandLoggers map[string]*logger.Logger, jsonFlag bool) (string, error) { runOtherCommandsLogger := mainLogger.WithPrefix("RunOtherCommands").WithField("file", file) runOtherCommandsLogger.Debug("Running other commands for file") runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200)) // Separate JSON and regex commands for different processing approaches jsonCommands := []utils.ModifyCommand{} regexCommands := []utils.ModifyCommand{} for _, command := range association.Commands { if command.JSON || jsonFlag { jsonCommands = append(jsonCommands, command) } else { regexCommands = append(regexCommands, command) } } // Process JSON commands sequentially (each operates on the entire file) for _, command := range jsonCommands { cmdLogger := logger.Default if cmdLog, ok := commandLoggers[command.Name]; ok { cmdLogger = cmdLog } cmdLogger.Debug("Processing file with JSON mode for command %q", command.Name) newModifications, err := processor.ProcessJSON(fileDataStr, command, file) if err != nil { runOtherCommandsLogger.Error("Failed to process file with JSON command %q: %v", command.Name, err) continue } // Apply JSON modifications immediately if len(newModifications) > 0 { var count int fileDataStr, count = utils.ExecuteModifications(newModifications, fileDataStr) atomic.AddInt64(&stats.TotalModifications, int64(count)) cmdLogger.Debug("Applied %d JSON modifications for command %q", count, command.Name) } count, ok := stats.ModificationsPerCommand.Load(command.Name) if !ok { count = 0 } stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications)) } // Aggregate regex modifications and execute them modifications := []utils.ReplaceCommand{} numCommandsConsidered := 0 for _, command := range regexCommands { cmdLogger := logger.Default if cmdLog, ok := commandLoggers[command.Name]; ok { cmdLogger = cmdLog } patterns := command.Regexes if len(patterns) == 0 { patterns = []string{command.Regex} } for idx, pattern := range patterns { tmpCmd := command tmpCmd.Regex = pattern cmdLogger.Debug("Begin processing file with command %q (pattern %d/%d)", command.Name, idx+1, len(patterns)) numCommandsConsidered++ newModifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file) if err != nil { runOtherCommandsLogger.Error("Failed to process file with command %q: %v", command.Name, err) continue } modifications = append(modifications, newModifications...) count, ok := stats.ModificationsPerCommand.Load(command.Name) if !ok { count = 0 } stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications)) cmdLogger.Debug("Command %q generated %d modifications (pattern %d/%d)", command.Name, len(newModifications), idx+1, len(patterns)) cmdLogger.Trace("Modifications generated by command %q: %v", command.Name, newModifications) if len(newModifications) == 0 { cmdLogger.Debug("No modifications yielded by command %q (pattern %d/%d)", command.Name, idx+1, len(patterns)) } } } runOtherCommandsLogger.Debug("Aggregated %d modifications from %d command-pattern runs", len(modifications), numCommandsConsidered) runOtherCommandsLogger.Trace("All aggregated modifications: %v", modifications) if len(modifications) == 0 { runOtherCommandsLogger.Warning("No modifications found for file") return fileDataStr, NothingToDo } runOtherCommandsLogger.Debug("Executing %d modifications for file", len(modifications)) // Sort commands in reverse order for safe replacements var count int fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr) runOtherCommandsLogger.Trace("File data after modifications: %s", utils.LimitString(fileDataStr, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) runOtherCommandsLogger.Info("Executed %d modifications for file", count) return fileDataStr, nil } func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string, jsonFlag bool) (string, error) { runIsolateCommandsLogger := mainLogger.WithPrefix("RunIsolateCommands").WithField("file", file) runIsolateCommandsLogger.Debug("Running isolate commands for file") runIsolateCommandsLogger.Trace("File data before isolate modifications: %s", utils.LimitString(fileDataStr, 200)) anythingDone := false currentFileData := fileDataStr for _, isolateCommand := range association.IsolateCommands { // Check if this isolate command should use JSON mode if isolateCommand.JSON || jsonFlag { runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name) modifications, err := processor.ProcessJSON(currentFileData, isolateCommand, file) if err != nil { runIsolateCommandsLogger.Error("Failed to process file with JSON isolate command %q: %v", isolateCommand.Name, err) continue } if len(modifications) == 0 { runIsolateCommandsLogger.Debug("JSON isolate command %q produced no modifications", isolateCommand.Name) continue } anythingDone = true runIsolateCommandsLogger.Debug("Executing %d JSON isolate modifications for file", len(modifications)) runIsolateCommandsLogger.Trace("JSON isolate modifications: %v", modifications) var count int currentFileData, count = utils.ExecuteModifications(modifications, currentFileData) runIsolateCommandsLogger.Trace("File data after JSON isolate modifications: %s", utils.LimitString(currentFileData, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) cmdCount, ok := stats.ModificationsPerCommand.Load(isolateCommand.Name) if !ok { stats.ModificationsPerCommand.Store(isolateCommand.Name, 0) cmdCount = 0 } stats.ModificationsPerCommand.Store(isolateCommand.Name, cmdCount.(int)+len(modifications)) runIsolateCommandsLogger.Info("Executed %d JSON isolate modifications for file", count) } else { // Regular regex processing for isolate commands patterns := isolateCommand.Regexes if len(patterns) == 0 { patterns = []string{isolateCommand.Regex} } for idx, pattern := range patterns { tmpCmd := isolateCommand tmpCmd.Regex = pattern runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns)) modifications, err := processor.ProcessRegex(currentFileData, tmpCmd, file) if err != nil { runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err) continue } if len(modifications) == 0 { runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns)) continue } anythingDone = true runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications)) runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications) var count int currentFileData, count = utils.ExecuteModifications(modifications, currentFileData) runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(currentFileData, 200)) atomic.AddInt64(&stats.TotalModifications, int64(count)) cmdCount, ok := stats.ModificationsPerCommand.Load(isolateCommand.Name) if !ok { stats.ModificationsPerCommand.Store(isolateCommand.Name, 0) cmdCount = 0 } stats.ModificationsPerCommand.Store(isolateCommand.Name, cmdCount.(int)+len(modifications)) runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count) } } } if !anythingDone { runIsolateCommandsLogger.Debug("No isolate modifications were made for file") return fileDataStr, NothingToDo } return currentFileData, nil }