package main import ( "flag" "fmt" "log" "os" "path/filepath" "sort" "sync" "time" "github.com/bmatcuk/doublestar/v4" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/object" "modify/logger" "modify/processor" ) type GlobalStats struct { TotalMatches int TotalModifications int ProcessedFiles int FailedFiles int } var stats GlobalStats = GlobalStats{} var ( gitFlag = flag.Bool("git", false, "Use git to manage files") resetFlag = flag.Bool("reset", false, "Reset files to their original state") logLevel = flag.String("loglevel", "INFO", "Set log level: ERROR, WARNING, INFO, DEBUG, TRACE") cookfile = flag.String("cook", "**/cook.yml", "Path to cook config files, can be globbed") parallelfiles = flag.Int("P", 100, "Number of files to process in parallel") repo *git.Repository worktree *git.Worktree ) type ModifyCommand struct { Pattern string `yaml:"pattern"` LuaExpr string `yaml:"lua"` Files []string `yaml:"files"` Git bool `yaml:"git"` Reset bool `yaml:"reset"` LogLevel string `yaml:"loglevel"` } type ReplaceCommand struct { From int To int With string } type CookFile []ModifyCommand func (c *ModifyCommand) Validate() error { if c.Pattern == "" { return fmt.Errorf("pattern is required") } if c.LuaExpr == "" { return fmt.Errorf("lua expression is required") } if len(c.Files) == 0 { return fmt.Errorf("at least one file is required") } if c.LogLevel == "" { c.LogLevel = "INFO" } return nil } func main() { flag.Usage = func() { fmt.Fprintf(os.Stderr, "Usage: %s [options] <...files_or_globs>\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nOptions:\n") fmt.Fprintf(os.Stderr, " -git\n") fmt.Fprintf(os.Stderr, " Use git to manage files\n") fmt.Fprintf(os.Stderr, " -reset\n") fmt.Fprintf(os.Stderr, " Reset files to their original state\n") fmt.Fprintf(os.Stderr, " -loglevel string\n") fmt.Fprintf(os.Stderr, " Set logging level: ERROR, WARNING, INFO, DEBUG, TRACE (default \"INFO\")\n") fmt.Fprintf(os.Stderr, "\nExamples:\n") fmt.Fprintf(os.Stderr, " Regex mode (default):\n") fmt.Fprintf(os.Stderr, " %s \"(\\d+)\" \"*1.5\" data.xml\n", os.Args[0]) fmt.Fprintf(os.Stderr, "\nNote: v1, v2, etc. are used to refer to capture groups as numbers.\n") fmt.Fprintf(os.Stderr, " s1, s2, etc. are used to refer to capture groups as strings.\n") fmt.Fprintf(os.Stderr, " Helper functions: num(str) converts string to number, str(num) converts number to string\n") fmt.Fprintf(os.Stderr, " is_number(str) checks if a string is numeric\n") fmt.Fprintf(os.Stderr, " If expression starts with an operator like *, /, +, -, =, etc., v1 is automatically prepended\n") fmt.Fprintf(os.Stderr, " You can use any valid Lua code, including if statements, loops, etc.\n") fmt.Fprintf(os.Stderr, " Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)\n") } flag.Parse() args := flag.Args() level := logger.ParseLevel(*logLevel) logger.Init(level) logger.Info("Initializing with log level: %s", level.String()) // The plan is: // Load all commands commands, err := LoadCommands(args) if err != nil { logger.Error("Failed to load commands: %v", err) flag.Usage() return } // Then aggregate all the globs and deduplicate them globs := AggregateGlobs(commands) // Resolve all the files for all the globs logger.Info("Found %d unique file patterns", len(globs)) files, err := ExpandGLobs(globs) if err != nil { logger.Error("Failed to expand file patterns: %v", err) return } logger.Info("Found %d files to process", len(files)) // Somehow connect files to commands via globs.. // For each file check every glob of every command // Maybe memoize this part // That way we know what commands affect what files associations, err := AssociateFilesWithCommands(files, commands) if err != nil { logger.Error("Failed to associate files with commands: %v", err) return } // TODO: Utilize parallel workers for this // Then for each file run all commands associated with the file for file, commands := range associations { fileData, err := os.ReadFile(file) if err != nil { logger.Error("Failed to read file %q: %v", file, err) return } logger.Trace("Loaded %d bytes of data for file %q", len(fileData), file) fileDataStr := string(fileData) // Aggregate all the modifications and execute them modifications := []ReplaceCommand{} for _, command := range commands { logger.Info("Processing file %q with command %q", file, command.Pattern) // TODO: Run processor and return modifications } // Sort commands in reverse order for safe replacements sort.Slice(modifications, func(i, j int) bool { return modifications[i].From > modifications[j].From }) logger.Trace("Applying %d replacement commands in reverse order", len(modifications)) fileDataStr = ExecuteModifications(modifications, fileDataStr) err = os.WriteFile(file, []byte(fileDataStr), 0644) if err != nil { logger.Error("Failed to write file %q: %v", file, err) return } } // This will also relieve processor of some of the file loading // But we will also have to rework the tests....... // Also give each command its own logger, maybe prefix it with something... Maybe give commands a name? // Do that with logger.WithField("loglevel", level.String()) // Since each command also has its own log level // TODO: Maybe even figure out how to run individual commands...? // TODO: What to do with git? Figure it out .... // if *gitFlag { // logger.Info("Git integration enabled, setting up git repository") // err := setupGit() // if err != nil { // logger.Error("Failed to setup git: %v", err) // fmt.Fprintf(os.Stderr, "Error setting up git: %v\n", err) // return // } // } // logger.Debug("Expanding file patterns") // files, err := expandFilePatterns(filePatterns) // if err != nil { // logger.Error("Failed to expand file patterns: %v", err) // fmt.Fprintf(os.Stderr, "Error expanding file patterns: %v\n", err) // return // } // if len(files) == 0 { // logger.Warning("No files found matching the specified patterns") // fmt.Fprintf(os.Stderr, "No files found matching the specified patterns\n") // return // } // if *gitFlag { // logger.Info("Cleaning up git files before processing") // err := cleanupGitFiles(files) // if err != nil { // logger.Error("Failed to cleanup git files: %v", err) // fmt.Fprintf(os.Stderr, "Error cleaning up git files: %v\n", err) // return // } // } // if *resetFlag { // logger.Info("Files reset to their original state, nothing more to do") // log.Printf("Files reset to their original state, nothing more to do") // return // } // Create the processor based on mode var proc processor.Processor = &processor.RegexProcessor{} var wg sync.WaitGroup log.Printf("%#v", proc) log.Printf("%#v", wg) // Process each file // for _, file := range files { // wg.Add(1) // logger.SafeGoWithArgs(func(args ...interface{}) { // defer wg.Done() // fileToProcess := args[0].(string) // logger.Debug("Processing file: %s", fileToProcess) // // It's a bit fucked, maybe I could do better to call it from proc... But it'll do for now // modCount, matchCount, err := processor.Process(proc, fileToProcess, pattern, luaExpr) // if err != nil { // logger.Error("Failed to process file %s: %v", fileToProcess, err) // fmt.Fprintf(os.Stderr, "Failed to process file %s: %v\n", fileToProcess, err) // stats.FailedFiles++ // } else { // if modCount > 0 { // logger.Info("Successfully processed file %s: %d modifications from %d matches", // fileToProcess, modCount, matchCount) // } else if matchCount > 0 { // logger.Info("Found %d matches in file %s but made no modifications", // matchCount, fileToProcess) // } else { // logger.Debug("No matches found in file: %s", fileToProcess) // } // stats.ProcessedFiles++ // stats.TotalMatches += matchCount // stats.TotalModifications += modCount // } // }, file) // } // wg.Wait() // Print summary if stats.TotalModifications == 0 { logger.Warning("No modifications were made in any files") fmt.Fprintf(os.Stderr, "No modifications were made in any files\n") } else { logger.Info("Operation complete! Modified %d values in %d/%d files", stats.TotalModifications, stats.ProcessedFiles, stats.ProcessedFiles+stats.FailedFiles) fmt.Printf("Operation complete! Modified %d values in %d/%d files\n", stats.TotalModifications, stats.ProcessedFiles, stats.ProcessedFiles+stats.FailedFiles) } } func ExecuteModifications(modifications []ReplaceCommand, fileData string) string { var err error for _, modification := range modifications { fileData, err = modification.Execute(fileData) if err != nil { logger.Error("Failed to execute modification: %v", err) continue } } return fileData } func (m *ReplaceCommand) Execute(fileDataStr string) (string, error) { err := m.Validate(len(fileDataStr)) if err != nil { return "", fmt.Errorf("failed to validate modification: %v", err) } logger.Trace("Replace pos %d-%d with %q", m.From, m.To, m.With) return fileDataStr[:m.From] + m.With + fileDataStr[m.To:], nil } func (m *ReplaceCommand) Validate(maxsize int) error { if m.To < m.From { return fmt.Errorf("command to is less than from: %v", m) } if m.From > maxsize || m.To > maxsize { return fmt.Errorf("command from or to is greater than replacement length: %v", m) } return nil } func AssociateFilesWithCommands(files []string, commands []ModifyCommand) (map[string][]ModifyCommand, error) { associationCount := 0 fileCommands := make(map[string][]ModifyCommand) for _, file := range files { for _, command := range commands { for _, glob := range command.Files { // TODO: Maybe memoize this function call matches, err := doublestar.Match(glob, file) if err != nil { logger.Trace("Failed to match glob %s with file %s: %v", glob, file, err) continue } if matches { logger.Debug("Found match for file %q and command %q", file, command.Pattern) fileCommands[file] = append(fileCommands[file], command) associationCount++ } } } logger.Debug("Found %d commands for file %q", len(fileCommands[file]), file) if len(fileCommands[file]) == 0 { logger.Info("No commands found for file %q", file) } } logger.Info("Found %d associations between %d files and %d commands", associationCount, len(files), len(commands)) return fileCommands, nil } func AggregateGlobs(commands []ModifyCommand) map[string]struct{} { logger.Info("Aggregating globs for %d commands", len(commands)) globs := make(map[string]struct{}) for _, command := range commands { for _, glob := range command.Files { globs[glob] = struct{}{} } } logger.Info("Found %d unique globs", len(globs)) return globs } func LoadCommands(args []string) ([]ModifyCommand, error) { commands := []ModifyCommand{} logger.Info("Loading commands from cook files: %s", *cookfile) newcommands, err := LoadCommandsFromCookFiles(*cookfile) if err != nil { return nil, fmt.Errorf("failed to load commands from cook files: %w", err) } logger.Info("Successfully loaded %d commands from cook files", len(newcommands)) commands = append(commands, newcommands...) logger.Info("Now total commands: %d", len(commands)) logger.Info("Loading commands from arguments: %v", args) newcommands, err = LoadCommandFromArgs(args) if err != nil { return nil, fmt.Errorf("failed to load commands from args: %w", err) } logger.Info("Successfully loaded %d commands from args", len(newcommands)) commands = append(commands, newcommands...) logger.Info("Now total commands: %d", len(commands)) return commands, nil } func LoadCommandFromArgs(args []string) ([]ModifyCommand, error) { // Cannot reset without git, right? if *resetFlag { *gitFlag = true } if len(args) < 3 { return nil, fmt.Errorf("at least %d arguments are required", 3) } command := ModifyCommand{ Pattern: args[0], LuaExpr: args[1], Files: args[2:], Git: *gitFlag, Reset: *resetFlag, LogLevel: *logLevel, } if err := command.Validate(); err != nil { return nil, fmt.Errorf("invalid command: %w", err) } return []ModifyCommand{command}, nil } func LoadCommandsFromCookFiles(s string) ([]ModifyCommand, error) { return nil, nil } func setupGit() error { cwd, err := os.Getwd() if err != nil { return fmt.Errorf("failed to get current working directory: %w", err) } logger.Debug("Current working directory obtained: %s", cwd) logger.Debug("Attempting to open git repository at %s", cwd) repo, err = git.PlainOpen(cwd) if err != nil { logger.Debug("No existing git repository found at %s, attempting to initialize a new git repository.", cwd) repo, err = git.PlainInit(cwd, false) if err != nil { return fmt.Errorf("failed to initialize a new git repository at %s: %w", cwd, err) } logger.Info("Successfully initialized a new git repository at %s", cwd) } else { logger.Info("Successfully opened existing git repository at %s", cwd) } logger.Debug("Attempting to obtain worktree for repository at %s", cwd) worktree, err = repo.Worktree() if err != nil { return fmt.Errorf("failed to obtain worktree for repository at %s: %w", cwd, err) } logger.Debug("Successfully obtained worktree for repository at %s", cwd) return nil } func ExpandGLobs(patterns map[string]struct{}) ([]string, error) { var files []string filesMap := make(map[string]bool) cwd, err := os.Getwd() if err != nil { return nil, fmt.Errorf("failed to get current working directory: %w", err) } logger.Debug("Expanding patterns from directory: %s", cwd) for pattern, _ := range patterns { logger.Trace("Processing pattern: %s", pattern) matches, _ := doublestar.Glob(os.DirFS(cwd), pattern) logger.Debug("Found %d matches for pattern %s", len(matches), pattern) for _, m := range matches { info, err := os.Stat(m) if err != nil { logger.Warning("Error getting file info for %s: %v", m, err) continue } if !info.IsDir() && !filesMap[m] { logger.Trace("Adding file to process list: %s", m) filesMap[m], files = true, append(files, m) } } } if len(files) > 0 { logger.Debug("Found %d files to process: %v", len(files), files) } return files, nil } func cleanupGitFiles(files []string) error { for _, file := range files { logger.Debug("Checking git status for file: %s", file) status, err := worktree.Status() if err != nil { logger.Error("Error getting worktree status: %v", err) fmt.Fprintf(os.Stderr, "Error getting worktree status: %v\n", err) return fmt.Errorf("error getting worktree status: %w", err) } if status.IsUntracked(file) { logger.Info("Detected untracked file: %s. Adding to git index.", file) _, err = worktree.Add(file) if err != nil { logger.Error("Error adding file to git: %v", err) fmt.Fprintf(os.Stderr, "Error adding file to git: %v\n", err) return fmt.Errorf("error adding file to git: %w", err) } filename := filepath.Base(file) logger.Info("File %s added successfully. Committing with message: 'Track %s'", filename, filename) _, err = worktree.Commit("Track "+filename, &git.CommitOptions{ Author: &object.Signature{ Name: "Big Chef", Email: "bigchef@bigchef.com", When: time.Now(), }, }) if err != nil { logger.Error("Error committing file: %v", err) fmt.Fprintf(os.Stderr, "Error committing file: %v\n", err) return fmt.Errorf("error committing file: %w", err) } logger.Info("Successfully committed file: %s", filename) } else { logger.Info("File %s is already tracked. Restoring it to the working tree.", file) err := worktree.Restore(&git.RestoreOptions{ Files: []string{file}, Staged: true, Worktree: true, }) if err != nil { logger.Error("Error restoring file: %v", err) fmt.Fprintf(os.Stderr, "Error restoring file: %v\n", err) return fmt.Errorf("error restoring file: %w", err) } logger.Info("File %s restored successfully", file) } } return nil }