Files
BigChef/main.go

545 lines
19 KiB
Go

package main
import (
"errors"
"flag"
"fmt"
"os"
"sort"
"sync"
"sync/atomic"
"time"
"cook/processor"
"cook/utils"
"gopkg.in/yaml.v3"
logger "git.site.quack-lab.dev/dave/cylogger"
)
// mainLogger is a scoped logger for the main package.
var mainLogger = logger.Default.WithPrefix("main")
type GlobalStats struct {
TotalMatches int64
TotalModifications int64
ProcessedFiles int64
FailedFiles int64
ModificationsPerCommand sync.Map
}
var (
stats GlobalStats = GlobalStats{
ModificationsPerCommand: sync.Map{},
}
)
func main() {
flag.Usage = func() {
CreateExampleConfig()
fmt.Fprintf(os.Stderr, "Usage: %s [options] <pattern> <lua_expression> <...files_or_globs>\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\nOptions:\n")
fmt.Fprintf(os.Stderr, " -reset\n")
fmt.Fprintf(os.Stderr, " Reset files to their original state\n")
fmt.Fprintf(os.Stderr, " -loglevel string\n")
fmt.Fprintf(os.Stderr, " Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n")
fmt.Fprintf(os.Stderr, "\nExamples:\n")
fmt.Fprintf(os.Stderr, " Regex mode (default):\n")
fmt.Fprintf(os.Stderr, " %s \"<value>(\\\\d+)</value>\" \"*1.5\" data.xml\n", os.Args[0])
fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n")
fmt.Fprintf(os.Stderr, " s1, s2, etc. are used to refer to capture groups as strings.\n")
fmt.Fprintf(os.Stderr, " Helper functions: num(str) converts string to number, str(num) converts number to string\n")
fmt.Fprintf(os.Stderr, " is_number(str) checks if a string is numeric\n")
fmt.Fprintf(os.Stderr, " If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended\n")
fmt.Fprintf(os.Stderr, " You can use any valid Lua code, including if statements, loops, etc.\n")
fmt.Fprintf(os.Stderr, " Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)\n")
}
// TODO: Fix bed shitting when doing *.yml in barotrauma directory
flag.Parse()
args := flag.Args()
logger.InitFlag()
mainLogger.Info("Initializing with log level: %s", logger.GetLevel().String())
mainLogger.Trace("Full argv: %v", os.Args)
if flag.NArg() == 0 {
flag.Usage()
return
}
mainLogger.Debug("Getting database connection")
db, err := utils.GetDB()
if err != nil {
mainLogger.Error("Failed to get database: %v", err)
return
}
mainLogger.Debug("Database connection established")
workdone, err := HandleSpecialArgs(args, err, db)
if err != nil {
mainLogger.Error("Failed to handle special args: %v", err)
return
}
if workdone {
mainLogger.Info("Special arguments handled, exiting.")
return
}
// The plan is:
// Load all commands
mainLogger.Debug("Loading commands from arguments")
mainLogger.Trace("Arguments: %v", args)
commands, err := utils.LoadCommands(args)
if err != nil || len(commands) == 0 {
mainLogger.Error("Failed to load commands: %v", err)
flag.Usage()
return
}
// Collect global modifiers from special entries and filter them out
vars := map[string]interface{}{}
filtered := make([]utils.ModifyCommand, 0, len(commands))
for _, c := range commands {
if len(c.Modifiers) > 0 && c.Name == "" && c.Regex == "" && len(c.Regexes) == 0 && c.Lua == "" && len(c.Files) == 0 {
for k, v := range c.Modifiers {
vars[k] = v
}
continue
}
filtered = append(filtered, c)
}
if len(vars) > 0 {
mainLogger.Info("Loaded %d global modifiers", len(vars))
processor.SetVariables(vars)
}
commands = filtered
mainLogger.Info("Loaded %d commands", len(commands))
if *utils.Filter != "" {
mainLogger.Info("Filtering commands by name: %s", *utils.Filter)
commands = utils.FilterCommands(commands, *utils.Filter)
mainLogger.Info("Filtered %d commands", len(commands))
}
// Then aggregate all the globs and deduplicate them
mainLogger.Debug("Aggregating globs and deduplicating")
globs := utils.AggregateGlobs(commands)
mainLogger.Debug("Aggregated %d globs before deduplication", utils.CountGlobsBeforeDedup(commands))
for _, command := range commands {
mainLogger.Trace("Command: %s", command.Name)
if len(command.Regexes) > 0 {
mainLogger.Trace("Regexes: %v", command.Regexes)
} else {
mainLogger.Trace("Regex: %s", command.Regex)
}
mainLogger.Trace("Files: %v", command.Files)
mainLogger.Trace("Lua: %s", command.Lua)
mainLogger.Trace("Reset: %t", command.Reset)
mainLogger.Trace("Isolate: %t", command.Isolate)
mainLogger.Trace("LogLevel: %s", command.LogLevel)
}
// Resolve all the files for all the globs
mainLogger.Info("Found %d unique file patterns", len(globs))
mainLogger.Debug("Expanding glob patterns to files")
files, err := utils.ExpandGLobs(globs)
if err != nil {
mainLogger.Error("Failed to expand file patterns: %v", err)
return
}
mainLogger.Info("Found %d files to process", len(files))
mainLogger.Trace("Files to process: %v", files)
// Somehow connect files to commands via globs..
// For each file check every glob of every command
// Maybe memoize this part
// That way we know what commands affect what files
mainLogger.Debug("Associating files with commands")
associations, err := utils.AssociateFilesWithCommands(files, commands)
if err != nil {
mainLogger.Error("Failed to associate files with commands: %v", err)
return
}
mainLogger.Debug("Files associated with commands")
mainLogger.Trace("File-command associations: %v", associations)
// Per-file association summary for better visibility when debugging
for file, assoc := range associations {
cmdNames := make([]string, 0, len(assoc.Commands))
for _, c := range assoc.Commands {
cmdNames = append(cmdNames, c.Name)
}
isoNames := make([]string, 0, len(assoc.IsolateCommands))
for _, c := range assoc.IsolateCommands {
isoNames = append(isoNames, c.Name)
}
mainLogger.Debug("File %q has %d regular and %d isolate commands", file, len(assoc.Commands), len(assoc.IsolateCommands))
mainLogger.Trace("\tRegular: %v", cmdNames)
mainLogger.Trace("\tIsolate: %v", isoNames)
}
mainLogger.Debug("Resetting files where necessary")
err = utils.ResetWhereNecessary(associations, db)
if err != nil {
mainLogger.Error("Failed to reset files where necessary: %v", err)
return
}
mainLogger.Debug("Files reset where necessary")
// Then for each file run all commands associated with the file
workers := make(chan struct{}, *utils.ParallelFiles)
wg := sync.WaitGroup{}
mainLogger.Debug("Starting file processing with %d parallel workers", *utils.ParallelFiles)
// Add performance tracking
startTime := time.Now()
// Create a map to store loggers for each command
commandLoggers := make(map[string]*logger.Logger)
for _, command := range commands {
// Create a named logger for each command
cmdName := command.Name
if cmdName == "" {
// If no name is provided, use a short version of the regex pattern
if len(command.Regex) > 20 {
cmdName = command.Regex[:17] + "..."
} else {
cmdName = command.Regex
}
}
// Parse the log level for this specific command
cmdLogLevel := logger.ParseLevel(command.LogLevel)
// Create a logger with the command name as a field
commandLoggers[command.Name] = logger.Default.WithField("command", cmdName)
commandLoggers[command.Name].SetLevel(cmdLogLevel)
mainLogger.Debug("Created logger for command %q with log level %s", cmdName, cmdLogLevel.String())
}
for file, association := range associations {
workers <- struct{}{}
wg.Add(1)
logger.SafeGoWithArgs(func(args ...interface{}) {
defer func() { <-workers }()
defer wg.Done()
// Track per-file processing time
fileStartTime := time.Now()
mainLogger.Debug("Reading file %q", file)
fileData, err := os.ReadFile(file)
if err != nil {
mainLogger.Error("Failed to read file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
fileDataStr := string(fileData)
mainLogger.Trace("File %q content: %s", file, utils.LimitString(fileDataStr, 500))
isChanged := false
mainLogger.Debug("Running isolate commands for file %q", file)
fileDataStr, err = RunIsolateCommands(association, file, fileDataStr)
if err != nil && err != NothingToDo {
mainLogger.Error("Failed to run isolate commands for file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
if err != NothingToDo {
isChanged = true
}
mainLogger.Debug("Running other commands for file %q", file)
fileDataStr, err = RunOtherCommands(file, fileDataStr, association, commandLoggers)
if err != nil && err != NothingToDo {
mainLogger.Error("Failed to run other commands for file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
if err != NothingToDo {
isChanged = true
}
if isChanged {
mainLogger.Debug("Saving file %q to database", file)
err = db.SaveFile(file, fileData)
if err != nil {
mainLogger.Error("Failed to save file %q to database: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
mainLogger.Debug("File %q saved to database", file)
}
mainLogger.Debug("Writing file %q", file)
err = os.WriteFile(file, []byte(fileDataStr), 0644)
if err != nil {
mainLogger.Error("Failed to write file %q: %v", file, err)
atomic.AddInt64(&stats.FailedFiles, 1)
return
}
mainLogger.Debug("File %q written", file)
// Only increment ProcessedFiles once per file, after all processing is complete
atomic.AddInt64(&stats.ProcessedFiles, 1)
mainLogger.Debug("File %q processed in %v", file, time.Since(fileStartTime))
}, file, commands)
}
wg.Wait()
processingTime := time.Since(startTime)
mainLogger.Info("Processing completed in %v", processingTime)
processedFiles := atomic.LoadInt64(&stats.ProcessedFiles)
if processedFiles > 0 {
mainLogger.Info("Average time per file: %v", processingTime/time.Duration(processedFiles))
}
// TODO: Also give each command its own logger, maybe prefix it with something... Maybe give commands a name?
// Do that with logger.WithField("loglevel", level.String())
// Since each command also has its own log level
// TODO: Maybe even figure out how to run individual commands...?
// TODO: What to do with git? Figure it out ....
// if *gitFlag {
// mainLogger.Info("Git integration enabled, setting up git repository")
// err := setupGit()
// if err != nil {
// mainLogger.Error("Failed to setup git: %v", err)
// fmt.Fprintf(os.Stderr, "Error setting up git: %v\n", err)
// return
// }
// }
// mainLogger.Debug("Expanding file patterns")
// files, err := expandFilePatterns(filePatterns)
// if err != nil {
// mainLogger.Error("Failed to expand file patterns: %v", err)
// fmt.Fprintf(os.Stderr, "Error expanding file patterns: %v\n", err)
// return
// }
// if *gitFlag {
// mainLogger.Info("Cleaning up git files before processing")
// err := cleanupGitFiles(files)
// if err != nil {
// mainLogger.Error("Failed to cleanup git files: %v", err)
// fmt.Fprintf(os.Stderr, "Error cleaning up git files: %v\n", err)
// return
// }
// }
// if *resetFlag {
// mainLogger.Info("Files reset to their original state, nothing more to do")
// log.Printf("Files reset to their original state, nothing more to do")
// return
// }
// Print summary
totalModifications := atomic.LoadInt64(&stats.TotalModifications)
if totalModifications == 0 {
mainLogger.Warning("No modifications were made in any files")
} else {
failedFiles := atomic.LoadInt64(&stats.FailedFiles)
mainLogger.Info("Operation complete! Modified %d values in %d/%d files",
totalModifications, processedFiles, processedFiles+failedFiles)
sortedCommands := []string{}
stats.ModificationsPerCommand.Range(func(key, value interface{}) bool {
sortedCommands = append(sortedCommands, key.(string))
return true
})
sort.Strings(sortedCommands)
for _, command := range sortedCommands {
count, _ := stats.ModificationsPerCommand.Load(command)
if count.(int) > 0 {
mainLogger.Info("\tCommand %q made %d modifications", command, count)
} else {
mainLogger.Warning("\tCommand %q made no modifications", command)
}
}
}
}
func HandleSpecialArgs(args []string, err error, db utils.DB) (bool, error) {
handleSpecialArgsLogger := logger.Default.WithPrefix("HandleSpecialArgs")
handleSpecialArgsLogger.Debug("Handling special arguments: %v", args)
switch args[0] {
case "reset":
handleSpecialArgsLogger.Info("Resetting all files")
err = utils.ResetAllFiles(db)
if err != nil {
handleSpecialArgsLogger.Error("Failed to reset all files: %v", err)
return true, err
}
handleSpecialArgsLogger.Info("All files reset")
return true, nil
case "dump":
handleSpecialArgsLogger.Info("Dumping all files from database")
err = db.RemoveAllFiles()
if err != nil {
handleSpecialArgsLogger.Error("Failed to remove all files from database: %v", err)
return true, err
}
handleSpecialArgsLogger.Info("All files removed from database")
return true, nil
}
handleSpecialArgsLogger.Debug("No special arguments handled, returning false")
return false, nil
}
func CreateExampleConfig() {
createExampleConfigLogger := logger.Default.WithPrefix("CreateExampleConfig")
createExampleConfigLogger.Debug("Creating example configuration file")
commands := []utils.ModifyCommand{
{
Name: "DoubleNumericValues",
Regex: "<value>(\\d+)<\\/value>",
Lua: "v1 * 2",
Files: []string{"data/*.xml"},
LogLevel: "INFO",
},
{
Name: "UpdatePrices",
Regex: "price=\"(\\d+)\"",
Lua: "if num(v1) < 100 then return v1 * 1.5 else return v1 end",
Files: []string{"items/*.xml", "shop/*.xml"},
LogLevel: "DEBUG",
},
{
Name: "IsolatedTagUpdate",
Regex: "<tag>(.*?)<\\/tag>",
Lua: "string.upper(s1)",
Files: []string{"config.xml"},
Isolate: true,
NoDedup: true,
LogLevel: "TRACE",
},
}
data, err := yaml.Marshal(commands)
if err != nil {
createExampleConfigLogger.Error("Failed to marshal example config: %v", err)
return
}
createExampleConfigLogger.Debug("Writing example_cook.yml")
err = os.WriteFile("example_cook.yml", data, 0644)
if err != nil {
createExampleConfigLogger.Error("Failed to write example_cook.yml: %v", err)
return
}
createExampleConfigLogger.Info("Wrote example_cook.yml")
}
var NothingToDo = errors.New("nothing to do")
func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, commandLoggers map[string]*logger.Logger) (string, error) {
runOtherCommandsLogger := mainLogger.WithPrefix("RunOtherCommands").WithField("file", file)
runOtherCommandsLogger.Debug("Running other commands for file")
runOtherCommandsLogger.Trace("File data before modifications: %s", utils.LimitString(fileDataStr, 200))
// Aggregate all the modifications and execute them
modifications := []utils.ReplaceCommand{}
numCommandsConsidered := 0
for _, command := range association.Commands {
// Use command-specific logger if available, otherwise fall back to default logger
cmdLogger := logger.Default
if cmdLog, ok := commandLoggers[command.Name]; ok {
cmdLogger = cmdLog
}
patterns := command.Regexes
if len(patterns) == 0 {
patterns = []string{command.Regex}
}
for idx, pattern := range patterns {
tmpCmd := command
tmpCmd.Regex = pattern
cmdLogger.Debug("Begin processing file with command %q (pattern %d/%d)", command.Name, idx+1, len(patterns))
numCommandsConsidered++
newModifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file)
if err != nil {
runOtherCommandsLogger.Error("Failed to process file with command %q: %v", command.Name, err)
continue
}
modifications = append(modifications, newModifications...)
count, ok := stats.ModificationsPerCommand.Load(command.Name)
if !ok {
count = 0
}
stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
cmdLogger.Debug("Command %q generated %d modifications (pattern %d/%d)", command.Name, len(newModifications), idx+1, len(patterns))
cmdLogger.Trace("Modifications generated by command %q: %v", command.Name, newModifications)
if len(newModifications) == 0 {
cmdLogger.Debug("No modifications yielded by command %q (pattern %d/%d)", command.Name, idx+1, len(patterns))
}
}
}
runOtherCommandsLogger.Debug("Aggregated %d modifications from %d command-pattern runs", len(modifications), numCommandsConsidered)
runOtherCommandsLogger.Trace("All aggregated modifications: %v", modifications)
if len(modifications) == 0 {
runOtherCommandsLogger.Warning("No modifications found for file")
return fileDataStr, NothingToDo
}
runOtherCommandsLogger.Debug("Executing %d modifications for file", len(modifications))
// Sort commands in reverse order for safe replacements
var count int
fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
runOtherCommandsLogger.Trace("File data after modifications: %s", utils.LimitString(fileDataStr, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
runOtherCommandsLogger.Info("Executed %d modifications for file", count)
return fileDataStr, nil
}
func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string) (string, error) {
runIsolateCommandsLogger := mainLogger.WithPrefix("RunIsolateCommands").WithField("file", file)
runIsolateCommandsLogger.Debug("Running isolate commands for file")
runIsolateCommandsLogger.Trace("File data before isolate modifications: %s", utils.LimitString(fileDataStr, 200))
anythingDone := false
for _, isolateCommand := range association.IsolateCommands {
runIsolateCommandsLogger.Debug("Begin processing file with isolate command %q", isolateCommand.Regex)
patterns := isolateCommand.Regexes
if len(patterns) == 0 {
patterns = []string{isolateCommand.Regex}
}
for idx, pattern := range patterns {
tmpCmd := isolateCommand
tmpCmd.Regex = pattern
modifications, err := processor.ProcessRegex(fileDataStr, tmpCmd, file)
if err != nil {
runIsolateCommandsLogger.Error("Failed to process file with isolate command %q (pattern %d/%d): %v", isolateCommand.Name, idx+1, len(patterns), err)
continue
}
if len(modifications) == 0 {
runIsolateCommandsLogger.Debug("Isolate command %q produced no modifications (pattern %d/%d)", isolateCommand.Name, idx+1, len(patterns))
continue
}
anythingDone = true
runIsolateCommandsLogger.Debug("Executing %d isolate modifications for file", len(modifications))
runIsolateCommandsLogger.Trace("Isolate modifications: %v", modifications)
var count int
fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
runIsolateCommandsLogger.Trace("File data after isolate modifications: %s", utils.LimitString(fileDataStr, 200))
atomic.AddInt64(&stats.TotalModifications, int64(count))
runIsolateCommandsLogger.Info("Executed %d isolate modifications for file", count)
}
}
if !anythingDone {
runIsolateCommandsLogger.Debug("No isolate modifications were made for file")
return fileDataStr, NothingToDo
}
return fileDataStr, nil
}