Files
BigChef/main.go

665 lines
23 KiB
Go

package main
import (
_ "embed"
"errors"
"os"
"sort"
"sync"
"sync/atomic"
"time"
"cook/processor"
"cook/utils"
"github.com/spf13/cobra"
logger "git.site.quack-lab.dev/dave/cylogger"
)
//go:embed example_cook.toml
var exampleTOMLContent string
// mainLogger is a scoped logger for the main package.
var mainLogger = logger.Default.WithPrefix("main")
type GlobalStats struct {
TotalMatches int64
TotalModifications int64
ProcessedFiles int64
FailedFiles int64
ModificationsPerCommand sync.Map
}
var (
stats GlobalStats = GlobalStats{
ModificationsPerCommand: sync.Map{},
}
)
// rootCmd represents the base command when called without any subcommands
var rootCmd *cobra.Command
func init() {
rootCmd = &cobra.Command{
Use: "modifier [options] <pattern> <lua_expression> <...files_or_globs>",
Short: "A powerful file modification tool with Lua scripting",
Long: `Modifier is a powerful file processing tool that supports regex patterns,
JSON manipulation, and YAML to TOML conversion with Lua scripting capabilities.
Features:
- Regex-based pattern matching and replacement
- JSON file processing with query support
- YAML to TOML conversion
- Lua scripting for complex transformations
- Parallel file processing
- Command filtering and organization`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
CreateExampleConfig()
logger.InitFlag()
mainLogger.Info("Initializing with log level: %s", logger.GetLevel().String())
mainLogger.Trace("Full argv: %v", os.Args)
},
Run: func(cmd *cobra.Command, args []string) {
if len(args) == 0 {
cmd.Usage()
return
}
runModifier(args, cmd)
},
}
// Global flags
rootCmd.PersistentFlags().StringP("loglevel", "l", "INFO", "Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE")
// Local flags
rootCmd.Flags().IntP("parallel", "P", 100, "Number of files to process in parallel")
rootCmd.Flags().StringP("filter", "f", "", "Filter commands before running them")
rootCmd.Flags().Bool("json", false, "Enable JSON mode for processing JSON files")
rootCmd.Flags().BoolP("conv", "c", false, "Convert YAML files to TOML format")
// Set up examples in the help text
rootCmd.SetUsageTemplate(`Usage:{{if .Runnable}}
{{.UseLine}}{{end}}{{if .HasAvailableSubCommands}}
{{.CommandPath}} [command]{{end}} {{if gt (len .Aliases) 0}}
Aliases:
{{.NameAndAliases}}{{end}}{{if .HasExample}}
Examples:
{{.Example}}{{end}}{{if .HasAvailableSubCommands}}
Available Commands:{{range .Commands}}{{if (or .IsAvailableCommand (eq .Name "help"))}}
{{rpad .Name .NamePadding }} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableLocalFlags}}
Flags:
{{.LocalFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasAvailableInheritedFlags}}
Global Flags:
{{.InheritedFlags.FlagUsages | trimTrailingWhitespaces}}{{end}}{{if .HasHelpSubCommands}}
Additional help topics:{{range .Commands}}{{if .IsAdditionalHelpTopicCommand}}
{{rpad .CommandPath .CommandPathPadding}} {{.Short}}{{end}}{{end}}{{end}}{{if .HasAvailableSubCommands}}
Use "{{.CommandPath}} [command] --help" for more information about a command.{{end}}
`)
// Add examples
rootCmd.Example = ` Regex mode (default):
modifier "<value>(\\d+)</value>" "*1.5" data.xml
JSON mode:
modifier -json data.json
YAML to TOML conversion:
modifier -conv *.yml
modifier -conv **/*.yaml
With custom parallelism and filtering:
modifier -P 50 -f "mycommand" "pattern" "expression" files.txt
Note: v1, v2, etc. are used to refer to capture groups as numbers.
s1, s2, etc. are used to refer to capture groups as strings.
Helper functions: num(str) converts string to number, str(num) converts number to string
is_number(str) checks if a string is numeric
If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended
You can use any valid Lua code, including if statements, loops, etc.
Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)
` + processor.GetLuaFunctionsHelp()
}
func main() {
if err := rootCmd.Execute(); err != nil {
mainLogger.Error("Command execution failed: %v", err)
os.Exit(1)
}
}
func runModifier(args []string, cmd *cobra.Command) {
// Get flag values from Cobra
convertFlag, _ := cmd.Flags().GetBool("conv")
parallelFlag, _ := cmd.Flags().GetInt("parallel")
filterFlag, _ := cmd.Flags().GetString("filter")
jsonFlag, _ := cmd.Flags().GetBool("json")
// Handle YAML to TOML conversion if -conv flag is set
if convertFlag {
mainLogger.Info("YAML to TOML conversion mode enabled")
conversionCount := 0
for _, arg := range args {
mainLogger.Debug("Converting YAML files matching pattern: %s", arg)
err := utils.ConvertYAMLToTOML(arg)
if err != nil {
mainLogger.Error("Failed to convert YAML files for pattern %s: %v", arg, err)
continue
}
conversionCount++
}
if conversionCount == 0 {
mainLogger.Warning("No files were converted. Please check your patterns.")
} else {
mainLogger.Info("Conversion completed for %d pattern(s)", conversionCount)
}
return
}
mainLogger.Debug("Getting database connection")
db, err := utils.GetDB()
if err != nil {
mainLogger.Error("Failed to get database: %v", err)
return
}
mainLogger.Debug("Database connection established")
workdone, err := HandleSpecialArgs(args, db)
if err != nil {
mainLogger.Error("Failed to handle special args: %v", err)
return
}
if workdone {
mainLogger.Info("Special arguments handled, exiting.")
return
}
// The plan is:
// Load all commands
mainLogger.Debug("Loading commands from arguments")
mainLogger.Trace("Arguments: %v", args)
commands, err := utils.LoadCommands(args)
if err != nil || len(commands) == 0 {
mainLogger.Error("Failed to load commands: %v", err)
cmd.Usage()
return
}
// Collect global modifiers from special entries and filter them out
vars := map[string]interface{}{}
filtered := make([]utils.ModifyCommand, 0, len(commands))
for _, c := range commands {
if len(c.Modifiers) > 0 && c.Name == "" && c.Regex == "" && len(c.Regexes) == 0 && c.Lua == "" && len(c.Files) == 0 {
for k, v := range c.Modifiers {
vars[k] = v
}
continue
}
filtered = append(filtered, c)
}
if len(vars) > 0 {
mainLogger.Info("Loaded %d global modifiers", len(vars))
processor.SetVariables(vars)
}
commands = filtered
mainLogger.Info("Loaded %d commands", len(commands))
if filterFlag != "" {
mainLogger.Info("Filtering commands by name: %s", filterFlag)
commands = utils.FilterCommands(commands, filterFlag)
mainLogger.Info("Filtered %d commands", len(commands))
}
// Then aggregate all the globs and deduplicate them
mainLogger.Debug("Aggregating globs and deduplicating")
globs := utils.AggregateGlobs(commands)
mainLogger.Debug("Aggregated %d globs before deduplication", utils.CountGlobsBeforeDedup(commands))
for _, command := range commands {
mainLogger.Trace("Command: %s", command.Name)
if len(command.Regexes) > 0 {
mainLogger.Trace("Regexes: %v", command.Regexes)
} else {
mainLogger.Trace("Regex: %s", command.Regex)
}
mainLogger.Trace("Files: %v", command.Files)
mainLogger.Trace("Lua: %s", command.Lua)
mainLogger.Trace("Reset: %t", command.Reset)
mainLogger.Trace("Isolate: %t", command.Isolate)
mainLogger.Trace("LogLevel: %s", command.LogLevel)
}
// Resolve all the files for all the globs
mainLogger.Info("Found %d unique file patterns", len(globs))
mainLogger.Debug("Expanding glob patterns to files")
files, err := utils.ExpandGLobs(globs)
if err != nil {
mainLogger.Error("Failed to expand file patterns: %v", err)
return
}
mainLogger.Info("Found %d files to process", len(files))
mainLogger.Trace("Files to process: %v", files)
// Somehow connect files to commands via globs..
// For each file check every glob of every command
// Maybe memoize this part
// That way we know what commands affect what files
mainLogger.Debug("Associating files with commands")
associations, err := utils.AssociateFilesWithCommands(files, commands)
if err != nil {
mainLogger.Error("Failed to associate files with commands: %v", err)
return
}
mainLogger.Debug("Files associated with commands")
mainLogger.Trace("File-command associations: %v", associations)
// Per-file association summary for better visibility when debugging
for file, assoc := range associations {
cmdNames := make([]string, 0, len(assoc.Commands))
for _, c := range assoc.Commands {
cmdNames = append(cmdNames, c.Name)
}
isoNames := make([]string, 0, len(assoc.IsolateCommands))
for _, c := range assoc.IsolateCommands {
isoNames = append(isoNames, c.Name)
}
mainLogger.Debug("File %q has %d regular and %d isolate commands", file, len(assoc.Commands), len(assoc.IsolateCommands))
mainLogger.Trace("\tRegular: %v", cmdNames)
mainLogger.Trace("\tIsolate: %v", isoNames)
}
mainLogger.Debug("Resetting files where necessary")
err = utils.ResetWhereNecessary(associations, db)
if err != nil {
mainLogger.Error("Failed to reset files where necessary: %v", err)
return
}
mainLogger.Debug("Files reset where necessary")
// Then for each file run all commands associated with the file
workers := make(chan struct{}, parallelFlag)
wg := sync.WaitGroup{}
mainLogger.Debug("Starting file processing with %d parallel workers", parallelFlag)
// Add performance tracking
startTime := time.Now()
// Create a map to store loggers for each command
commandLoggers := make(map[string]*logger.Logger)
for _, command := range commands {
// Create a named logger for each command
cmdName := command.Name
if cmdName == "" {
// If no name is provided, use a short version of the regex pattern
if len(command.Regex) > 20 {
cmdName = command.Regex[:17] + "..."
} else {
cmdName = command.Regex
}
}
// Parse the log level for this specific command
cmdLogLevel := logger.ParseLevel(command.LogLevel)
// Create a logger with the command name as a field
commandLoggers[command.Name] = logger.Default.WithField("command", cmdName)
commandLoggers[command.Name].SetLevel(cmdLogLevel)
mainLogger.Debug("Created logger for command %q with log level %s", cmdName, cmdLogLevel.String())
}
for file, association := range associations {
workers <- struct{}{}
wg.Add(1)
logger.SafeGoWithArgs(func(args ...interface{}) {
defer func() { <-workers }()
defer wg.Done()
// Track per-file processing time
fileStartTime := time.Now()
mainLogger.Debug("Reading file %q", file)
fileData, err := os.ReadFile(file)
if err != nil {
mainLogger.Error("Failed to read file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
fileDataStr := string(fileData)
mainLogger.Trace("File %q content: %s", file, utils.LimitString(fileDataStr, 500))
isChanged := false
mainLogger.Debug("Running isolate commands for file %q", file)
fileDataStr, err = RunIsolateCommands(association, file, fileDataStr, jsonFlag)
if err != nil && err != NothingToDo {
mainLogger.Error("Failed to run isolate commands for file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
if err != NothingToDo {
isChanged = true
}
mainLogger.Debug("Running other commands for file %q", file)
fileDataStr, err = RunOtherCommands(file, fileDataStr, association, commandLoggers, jsonFlag)
if err != nil && err != NothingToDo {
mainLogger.Error("Failed to run other commands for file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
if err != NothingToDo {
isChanged = true
}
if isChanged {
mainLogger.Debug("Saving file %q to database", file)
err = db.SaveFile(file, fileData)
if err != nil {
mainLogger.Error("Failed to save file %q to database: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
mainLogger.Debug("File %q saved to database", file)
}
mainLogger.Debug("Writing file %q", file)
err = os.WriteFile(file, []byte(fileDataStr), 0644)
if err != nil {
mainLogger.Error("Failed to write file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
mainLogger.Debug("File %q written", file)
// Only increment ProcessedFiles once per file, after all processing is complete
atomic.AddInt64(&stats.ProcessedFiles, 1)
mainLogger.Debug("File %q processed in %v", file, time.Since(fileStartTime))
}, file, commands)
}
wg.Wait()
processingTime := time.Since(startTime)
mainLogger.Info("Processing completed in %v", processingTime)
processedFiles := atomic.LoadInt64(&stats.ProcessedFiles)
if processedFiles > 0 {
mainLogger.Info("Average time per file: %v", processingTime/time.Duration(processedFiles))
}
// TODO: Also give each command its own logger, maybe prefix it with something... Maybe give commands a name?
// Do that with logger.WithField("loglevel", level.String())
// Since each command also has its own log level
// TODO: Maybe even figure out how to run individual commands...?
// Print summary
totalModifications := atomic.LoadInt64(&stats.TotalModifications)
if totalModifications == 0 {
mainLogger.Warning("No modifications were made in any files")
} else {
failedFiles := atomic.LoadInt64(&stats.FailedFiles)
mainLogger.Info("Operation complete! Modified %d values in %d/%d files",
totalModifications, processedFiles, processedFiles+failedFiles)
sortedCommands := []string{}
stats.ModificationsPerCommand.Range(func(key, value interface{}) bool {
sortedCommands = append(sortedCommands, key.(string))
return true
})
sort.Strings(sortedCommands)
for _, command := range sortedCommands {
count, _ := stats.ModificationsPerCommand.Load(command)
if count.(int) > 0 {
mainLogger.Info("\tCommand %q made %d modifications", command, count)
} else {
mainLogger.Warning("\tCommand %q made no modifications", command)
}
}
}
}
func HandleSpecialArgs(args []string, db utils.DB) (bool, error) {
handleSpecialArgsLogger := logger.Default.WithPrefix("HandleSpecialArgs")
handleSpecialArgsLogger.Debug("Handling special arguments: %v", args)
if len(args) == 0 {
handleSpecialArgsLogger.Warning("No arguments provided to HandleSpecialArgs")
return false, nil
}
switch args[0] {
case "reset":
handleSpecialArgsLogger.Info("Resetting all files to their original state from database")
err := utils.ResetAllFiles(db)
if err != nil {
handleSpecialArgsLogger.Error("Failed to reset all files: %v", err)
return true, err
}
handleSpecialArgsLogger.Info("Successfully reset all files to original state")
return true, nil
case "dump":
handleSpecialArgsLogger.Info("Dumping all files from database (clearing snapshots)")
err := db.RemoveAllFiles()
if err != nil {
handleSpecialArgsLogger.Error("Failed to remove all files from database: %v", err)
return true, err
}
handleSpecialArgsLogger.Info("Successfully cleared all file snapshots from database")
return true, nil
default:
handleSpecialArgsLogger.Debug("Unknown special argument: %q", args[0])
}
handleSpecialArgsLogger.Debug("No special arguments handled, returning false")
return false, nil
}
func CreateExampleConfig() {
createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig")
createExampleConfigLogger.Debug("Creating example configuration file")
// Save the embedded TOML content to disk
createExampleConfigLogger.Debug("Writing example_cook.toml")
err := os.WriteFile("example_cook.toml", []byte(exampleTOMLContent), 0644)
if err != nil {
createExampleConfigLogger.Error("Failed to write example_cook.toml: %v", err)
return
}
createExampleConfigLogger.Info("Wrote example_cook.toml")
}
var NothingToDo = errors.New("nothing to do")
func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, commandLoggers map[string]*logger.Logger, jsonFlag bool) (string, error) {
runOtherCommandsLogger := mainLogger.WithPrefix("RunOtherCommands").WithField("file", file)
runOtherCommandsLogger.Debug("Running other commands for file")
runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200))
// Separate JSON and regex commands for different processing approaches
jsonCommands := []utils.ModifyCommand{}
regexCommands := []utils.ModifyCommand{}
for _, command := range association.Commands {
if command.JSON || jsonFlag {
jsonCommands = append(jsonCommands, command)
} else {
regexCommands = append(regexCommands, command)
}
}
// Process JSON commands sequentially (each operates on the entire file)
for _, command := range jsonCommands {
cmdLogger := logger.Default
if cmdLog, ok := commandLoggers[command.Name]; ok {
cmdLogger = cmdLog
}
cmdLogger.Debug("Processing file with JSON mode for command %q", command.Name)
newModifications, err := processor.ProcessJSON(fileDataStr, command, file)
if err != nil {
runOtherCommandsLogger.Error("Failed to process file with JSON command %q: %v", command.Name, err)
continue
}
// Apply JSON modifications immediately
if len(newModifications) > 0 {
var count int
fileDataStr, count = utils.ExecuteModifications(newModifications, fileDataStr)
atomic.AddInt64(&stats.TotalModifications, int64(count))
cmdLogger.Debug("Applied %d JSON modifications for command %q", count, command.Name)
}
count, ok := stats.ModificationsPerCommand.Load(command.Name)
if !ok {
count = 0
}
stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
}
// Aggregate regex modifications and execute them
modifications := []utils.ReplaceCommand{}
numCommandsConsidered := 0
for _, command := range regexCommands {
cmdLogger := logger.Default
if cmdLog, ok := commandLoggers[command.Name]; ok {
cmdLogger = cmdLog
}
patterns := command.Regexes
if len(patterns) == 0 {
patterns = []string{command.Regex}
}
for idx, pattern := range patterns {
tmpCmd := command
tmpCmd.Regex = pattern
cmdLogger.Debug("Begin processing file with command %q (pattern %d/%d)", command.Name, idx+1, len(patterns))
numCommandsConsidered++
newModifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file)
if err != nil {
runOtherCommandsLogger.Error("Failed to process file with command %q: %v", command.Name, err)
continue
}
modifications = append(modifications, newModifications...)
count, ok := stats.ModificationsPerCommand.Load(command.Name)
if !ok {
count = 0
}
stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
cmdLogger.Debug("Command %q generated %d modifications (pattern %d/%d)", command.Name, len(newModifications), idx+1, len(patterns))
cmdLogger.Trace("Modifications generated by command %q: %v", command.Name, newModifications)
if len(newModifications) == 0 {
cmdLogger.Debug("No modifications yielded by command %q (pattern %d/%d)", command.Name, idx+1, len(patterns))
}
}
}
runOtherCommandsLogger.Debug("Aggregated %d modifications from %d command-pattern runs", len(modifications), numCommandsConsidered)
runOtherCommandsLogger.Trace("All aggregated modifications: %v", modifications)
if len(modifications) == 0 {
runOtherCommandsLogger.Warning("No modifications found for file")
return fileDataStr, NothingToDo
}
runOtherCommandsLogger.Debug("Executing %d modifications for file", len(modifications))
// Sort commands in reverse order for safe replacements
var count int
fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
runOtherCommandsLogger.Trace("File data after modifications: %s", utils.LimitString(fileDataStr, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
runOtherCommandsLogger.Info("Executed %d modifications for file", count)
return fileDataStr, nil
}
func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string, jsonFlag bool) (string, error) {
runIsolateCommandsLogger := mainLogger.WithPrefix("RunIsolateCommands").WithField("file", file)
runIsolateCommandsLogger.Debug("Running isolate commands for file")
runIsolateCommandsLogger.Trace("File data before isolate modifications: %s", utils.LimitString(fileDataStr, 200))
anythingDone := false
currentFileData := fileDataStr
for _, isolateCommand := range association.IsolateCommands {
// Check if this isolate command should use JSON mode
if isolateCommand.JSON || jsonFlag {
runIsolateCommandsLogger.Debug("Begin processing file with JSON isolate command %q", isolateCommand.Name)
modifications, err := processor.ProcessJSON(currentFileData, isolateCommand, file)
if err != nil {
runIsolateCommandsLogger.Error("Failed to process file with JSON isolate command %q: %v", isolateCommand.Name, err)
continue
}
if len(modifications) == 0 {
runIsolateCommandsLogger.Debug("JSON isolate command %q produced no modifications", isolateCommand.Name)
continue
}
anythingDone = true
runIsolateCommandsLogger.Debug("Executing %d JSON isolate modifications for file", len(modifications))
runIsolateCommandsLogger.Trace("JSON isolate modifications: %v", modifications)
var count int
currentFileData, count = utils.ExecuteModifications(modifications, currentFileData)
runIsolateCommandsLogger.Trace("File data after JSON isolate modifications: %s", utils.LimitString(currentFileData, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
cmdCount, ok := stats.ModificationsPerCommand.Load(isolateCommand.Name)
if !ok {
stats.ModificationsPerCommand.Store(isolateCommand.Name, 0)
cmdCount = 0
}
stats.ModificationsPerCommand.Store(isolateCommand.Name, cmdCount.(int)+len(modifications))
runIsolateCommandsLogger.Info("Executed %d JSON isolate modifications for file", count)
} else {
// Regular regex processing for isolate commands
patterns := isolateCommand.Regexes
if len(patterns) == 0 {
patterns = []string{isolateCommand.Regex}
}
for idx, pattern := range patterns {
tmpCmd := isolateCommand
tmpCmd.Regex = pattern
runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns))
modifications, err := processor.ProcessRegex(currentFileData, tmpCmd, file)
if err != nil {
runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err)
continue
}
if len(modifications) == 0 {
runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns))
continue
}
anythingDone = true
runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications))
runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications)
var count int
currentFileData, count = utils.ExecuteModifications(modifications, currentFileData)
runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(currentFileData, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
cmdCount, ok := stats.ModificationsPerCommand.Load(isolateCommand.Name)
if !ok {
stats.ModificationsPerCommand.Store(isolateCommand.Name, 0)
cmdCount = 0
}
stats.ModificationsPerCommand.Store(isolateCommand.Name, cmdCount.(int)+len(modifications))
runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count)
}
}
}
if !anythingDone {
runIsolateCommandsLogger.Debug("No isolate modifications were made for file")
return fileDataStr, NothingToDo
}
return currentFileData, nil
}