Reduce some of the reads and writes

It's really not necessary

.vscode/launch.json

@@ -12,11 +12,10 @@
 			"program": "${workspaceFolder}",
 			"cwd": "C:/Users/Administrator/Seafile/Games-Barotrauma",
 			"args": [
-				"loglevel",
+				"-loglevel",
 				"trace",
-				"(?-s)LightComponent!anyrange=\"(!num)\"",
-				"*4",
-				"**/Outpost*.xml"
+				"-cook",
+				"*.yml",
 			]
 		},
 		{

main.go

@@ -4,6 +4,7 @@ import (
 	"flag"
 	"fmt"
 	"os"
+	"sort"
 	"sync"
 	"time"
 
@@ -20,14 +21,14 @@ type GlobalStats struct {
 	TotalModifications      int
 	ProcessedFiles          int
 	FailedFiles             int
-	ModificationsPerCommand map[string]int
+	ModificationsPerCommand sync.Map
 }
 
 var (
 	repo     *git.Repository
 	worktree *git.Worktree
 	stats    GlobalStats = GlobalStats{
-		ModificationsPerCommand: make(map[string]int),
+		ModificationsPerCommand: sync.Map{},
 	}
 )
 
@@ -52,6 +53,7 @@ func main() {
 		fmt.Fprintf(os.Stderr, "      You can use any valid Lua code, including if statements, loops, etc.\n")
 		fmt.Fprintf(os.Stderr, "      Glob patterns are supported for file selection (*.xml, data/**.xml, etc.)\n")
 	}
+	// TODO: Fix bed shitting when doing *.yml in barotrauma directory
 	flag.Parse()
 	args := flag.Args()
 
@@ -68,10 +70,27 @@ func main() {
 		return
 	}
 
+	if *utils.Filter != "" {
+		logger.Info("Filtering commands by name: %s", *utils.Filter)
+		commands = utils.FilterCommands(commands, *utils.Filter)
+		logger.Info("Filtered %d commands", len(commands))
+	}
+
 	// Then aggregate all the globs and deduplicate them
 	globs := utils.AggregateGlobs(commands)
 	logger.Debug("Aggregated %d globs before deduplication", utils.CountGlobsBeforeDedup(commands))
 
+	for _, command := range commands {
+		logger.Trace("Command: %s", command.Name)
+		logger.Trace("Regex: %s", command.Regex)
+		logger.Trace("Files: %v", command.Files)
+		logger.Trace("Lua: %s", command.Lua)
+		logger.Trace("Git: %t", command.Git)
+		logger.Trace("Reset: %t", command.Reset)
+		logger.Trace("Isolate: %t", command.Isolate)
+		logger.Trace("LogLevel: %s", command.LogLevel)
+	}
+
 	// Resolve all the files for all the globs
 	logger.Info("Found %d unique file patterns", len(globs))
 	files, err := utils.ExpandGLobs(globs)
@@ -91,7 +110,6 @@ func main() {
 		return
 	}
 
-	// TODO: Utilize parallel workers for this
 	// Then for each file run all commands associated with the file
 	workers := make(chan struct{}, *utils.ParallelFiles)
 	wg := sync.WaitGroup{}
@@ -100,7 +118,10 @@ func main() {
 	startTime := time.Now()
 	var fileMutex sync.Mutex
 
-	for file, commands := range associations {
+	// This aggregation is great but what if one modification replaces the whole entire file?
+	// Shit......
+	// TODO: Add "Isolate" field to modifications which makes them run alone
+	for file, association := range associations {
 		workers <- struct{}{}
 		wg.Add(1)
 		logger.SafeGoWithArgs(func(args ...interface{}) {
@@ -115,39 +136,19 @@ func main() {
 				logger.Error("Failed to read file %q: %v", file, err)
 				return
 			}
-			logger.Trace("Loaded %d bytes of data for file %q", len(fileData), file)
 			fileDataStr := string(fileData)
 
-			// Aggregate all the modifications and execute them
-			modifications := []utils.ReplaceCommand{}
-			for _, command := range commands {
-				logger.Info("Processing file %q with command %q", file, command.Regex)
-				commands, err := processor.ProcessRegex(fileDataStr, command)
-				if err != nil {
-					logger.Error("Failed to process file %q with command %q: %v", file, command.Regex, err)
-					return
-				}
-				modifications = append(modifications, commands...)
-				// It is not guranteed that all the commands will be executed...
-				// TODO: Make this better
-				// We'd have to pass the map to executemodifications or something...
-				stats.ModificationsPerCommand[command.Name] += len(commands)
-			}
-
-			if len(modifications) == 0 {
-				logger.Info("No modifications found for file %q", file)
+			fileDataStr, err = RunIsolateCommands(association, file, fileDataStr, &fileMutex)
+			if err != nil {
+				logger.Error("Failed to run isolate commands for file %q: %v", file, err)
 				return
 			}
 
-			// Sort commands in reverse order for safe replacements
-			fileDataStr, count := utils.ExecuteModifications(modifications, fileDataStr)
-
-			fileMutex.Lock()
-			stats.ProcessedFiles++
-			stats.TotalModifications += count
-			fileMutex.Unlock()
-
-			logger.Info("Executed %d modifications for file %q", count, file)
+			fileDataStr, err = RunOtherCommands(file, fileDataStr, association, &fileMutex)
+			if err != nil {
+				logger.Error("Failed to run other commands for file %q: %v", file, err)
+				return
+			}
 
 			err = os.WriteFile(file, []byte(fileDataStr), 0644)
 			if err != nil {
@@ -208,11 +209,87 @@ func main() {
 	// Print summary
 	if stats.TotalModifications == 0 {
 		logger.Warning("No modifications were made in any files")
-		fmt.Fprintf(os.Stderr, "No modifications were made in any files\n")
 	} else {
 		logger.Info("Operation complete! Modified %d values in %d/%d files",
 			stats.TotalModifications, stats.ProcessedFiles, stats.ProcessedFiles+stats.FailedFiles)
-		fmt.Printf("Operation complete! Modified %d values in %d/%d files\n",
-			stats.TotalModifications, stats.ProcessedFiles, stats.ProcessedFiles+stats.FailedFiles)
+		sortedCommands := []string{}
+		stats.ModificationsPerCommand.Range(func(key, value interface{}) bool {
+			sortedCommands = append(sortedCommands, key.(string))
+			return true
+		})
+		sort.Strings(sortedCommands)
+
+		for _, command := range sortedCommands {
+			count, _ := stats.ModificationsPerCommand.Load(command)
+			if count.(int) > 0 {
+				logger.Info("\tCommand %q made %d modifications", command, count)
+			} else {
+				logger.Warning("\tCommand %q made no modifications", command)
+			}
+		}
 	}
 }
+
+func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, fileMutex *sync.Mutex) (string, error) {
+	// Aggregate all the modifications and execute them
+	modifications := []utils.ReplaceCommand{}
+	for _, command := range association.Commands {
+		logger.Info("Processing file %q with command %q", file, command.Regex)
+		newModifications, err := processor.ProcessRegex(fileDataStr, command, file)
+		if err != nil {
+			return fileDataStr, fmt.Errorf("failed to process file %q with command %q: %w", file, command.Regex, err)
+		}
+		modifications = append(modifications, newModifications...)
+		// It is not guranteed that all the commands will be executed...
+		// TODO: Make this better
+		// We'd have to pass the map to executemodifications or something...
+		count, ok := stats.ModificationsPerCommand.Load(command.Name)
+		if !ok {
+			count = 0
+		}
+		stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
+	}
+
+	if len(modifications) == 0 {
+		logger.Info("No modifications found for file %q", file)
+		return fileDataStr, nil
+	}
+
+	// Sort commands in reverse order for safe replacements
+	var count int
+	fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
+
+	fileMutex.Lock()
+	stats.ProcessedFiles++
+	stats.TotalModifications += count
+	fileMutex.Unlock()
+
+	logger.Info("Executed %d modifications for file %q", count, file)
+	return fileDataStr, nil
+}
+
+func RunIsolateCommands(association utils.FileCommandAssociation, file string, fileDataStr string, fileMutex *sync.Mutex) (string, error) {
+	for _, isolateCommand := range association.IsolateCommands {
+		logger.Info("Processing file %q with isolate command %q", file, isolateCommand.Regex)
+		modifications, err := processor.ProcessRegex(fileDataStr, isolateCommand, file)
+		if err != nil {
+			return fileDataStr, fmt.Errorf("failed to process file %q with isolate command %q: %w", file, isolateCommand.Regex, err)
+		}
+
+		if len(modifications) == 0 {
+			logger.Warning("No modifications found for file %q", file)
+			return fileDataStr, nil
+		}
+
+		var count int
+		fileDataStr, count = utils.ExecuteModifications(modifications, fileDataStr)
+
+		fileMutex.Lock()
+		stats.ProcessedFiles++
+		stats.TotalModifications += count
+		fileMutex.Unlock()
+
+		logger.Info("Executed %d isolate modifications for file %q", count, file)
+	}
+	return fileDataStr, nil
+}

processor/regex.go

@@ -21,7 +21,9 @@ type CaptureGroup struct {
 }
 
 // ProcessContent applies regex replacement with Lua processing
-func ProcessRegex(content string, command utils.ModifyCommand) ([]utils.ReplaceCommand, error) {
+// The filename here exists ONLY so we can pass it to the lua environment
+// It's not used for anything else
+func ProcessRegex(content string, command utils.ModifyCommand, filename string) ([]utils.ReplaceCommand, error) {
 	var commands []utils.ReplaceCommand
 	logger.Trace("Processing regex: %q", command.Regex)
 
@@ -79,6 +81,7 @@ func ProcessRegex(content string, command utils.ModifyCommand) ([]utils.ReplaceC
 			logger.Error("Error creating Lua state: %v", err)
 			return commands, fmt.Errorf("error creating Lua state: %v", err)
 		}
+		L.SetGlobal("file", lua.LString(filename))
 		// Hmm... Maybe we don't want to defer this..
 		// Maybe we want to close them every iteration
 		// We'll leave it as is for now
@@ -205,7 +208,7 @@ func ProcessRegex(content string, command utils.ModifyCommand) ([]utils.ReplaceC
 
 			for _, capture := range captureGroups {
 				if capture.Value == capture.Updated {
-					logger.Info("Capture group unchanged: %s", capture.Value)
+					logger.Info("Capture group unchanged: %s", LimitString(capture.Value, 50))
 					continue
 				}
 
@@ -250,7 +253,7 @@ func deduplicateGroups(captureGroups []*CaptureGroup) []*CaptureGroup {
 		}
 		if overlaps {
 			// We CAN just continue despite this fuckup
-			logger.Error("Overlapping capture group: %s", group.Name)
+			logger.Warning("Overlapping capture group: %s", group.Name)
 			continue
 		}
 		logger.Debug("No overlap detected for capture group: %s. Adding to deduplicated groups.", group.Name)
@@ -268,8 +271,6 @@ func resolveRegexPlaceholders(pattern string) string {
 	// Handle special pattern modifications
 	if !strings.HasPrefix(pattern, "(?s)") {
 		pattern = "(?s)" + pattern
-		// Use fmt.Printf for test compatibility
-		fmt.Printf("Pattern modified to include (?s): %s\n", pattern)
 	}
 
 	namedGroupNum := regexp.MustCompile(`(?:(\?<[^>]+>)(!num))`)

processor/regex_test.go

@@ -2,7 +2,6 @@ package processor
 
 import (
 	"bytes"
-	"fmt"
 	"io"
 	"modify/utils"
 	"os"
@@ -38,7 +37,7 @@ func ApiAdaptor(content string, regex string, lua string) (string, int, int, err
 		LogLevel: "TRACE",
 	}
 
-	commands, err := ProcessRegex(content, command)
+	commands, err := ProcessRegex(content, command, "test")
 	if err != nil {
 		return "", 0, 0, err
 	}
@@ -979,29 +978,12 @@ func TestPatternWithoutPrefixGetsModified(t *testing.T) {
 	// Setup
 	pattern := "some.*pattern"
 
-	// Redirect stdout to capture fmt.Printf output
-	oldStdout := os.Stdout
-	r, w, _ := os.Pipe()
-	os.Stdout = w
-
 	// Execute function
 	result := resolveRegexPlaceholders(pattern)
 
-	// Restore stdout
-	w.Close()
-	os.Stdout = oldStdout
-
-	// Read captured output
-	var buf bytes.Buffer
-	io.Copy(&buf, r)
-	output := buf.String()
-
 	// Verify results
 	expectedPattern := "(?s)some.*pattern"
 	assert.Equal(t, expectedPattern, result, "Expected pattern to be %q, got %q", expectedPattern, result)
-
-	expectedOutput := fmt.Sprintf("Pattern modified to include (?s): %s\n", expectedPattern)
-	assert.Equal(t, expectedOutput, output, "Expected output message %q, got %q", expectedOutput, output)
 }
 
 // Empty string input returns "(?s)"
@@ -1009,29 +991,12 @@ func TestEmptyStringReturnsWithPrefix(t *testing.T) {
 	// Setup
 	pattern := ""
 
-	// Redirect stdout to capture fmt.Printf output
-	oldStdout := os.Stdout
-	r, w, _ := os.Pipe()
-	os.Stdout = w
-
 	// Execute function
 	result := resolveRegexPlaceholders(pattern)
 
-	// Restore stdout
-	w.Close()
-	os.Stdout = oldStdout
-
-	// Read captured output
-	var buf bytes.Buffer
-	io.Copy(&buf, r)
-	output := buf.String()
-
 	// Verify results
 	expectedPattern := "(?s)"
 	assert.Equal(t, expectedPattern, result, "Expected pattern to be %q, got %q", expectedPattern, result)
-
-	expectedOutput := fmt.Sprintf("Pattern modified to include (?s): %s\n", expectedPattern)
-	assert.Equal(t, expectedOutput, output, "Expected output message %q, got %q", expectedOutput, output)
 }
 
 // Named group with "!num" pattern gets replaced with proper regex for numbers

regression/regression_test.go

@@ -15,7 +15,7 @@ func ApiAdaptor(content string, regex string, lua string) (string, int, int, err
 		LogLevel: "TRACE",
 	}
 
-	commands, err := processor.ProcessRegex(content, command)
+	commands, err := processor.ProcessRegex(content, command, "test")
 	if err != nil {
 		return "", 0, 0, err
 	}

utils/flags.go

@@ -12,4 +12,5 @@ var (
 	LogLevel      = flag.String("loglevel", "INFO", "Set log level: ERROR, WARNING, INFO, DEBUG, TRACE")
 	Cookfile      = flag.String("cook", "**/cook.yml", "Path to cook config files, can be globbed")
 	ParallelFiles = flag.Int("P", 100, "Number of files to process in parallel")
+	Filter        = flag.String("filter", "", "Filter commands before running them")
 )

utils/modifycommand.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"modify/logger"
 	"os"
+	"strings"
 
 	"github.com/bmatcuk/doublestar/v4"
 	"gopkg.in/yaml.v3"
@@ -17,6 +18,7 @@ type ModifyCommand struct {
 	Git      bool     `yaml:"git"`
 	Reset    bool     `yaml:"reset"`
 	LogLevel string   `yaml:"loglevel"`
+	Isolate  bool     `yaml:"isolate"`
 }
 type CookFile []ModifyCommand
 
@@ -36,29 +38,67 @@ func (c *ModifyCommand) Validate() error {
 	return nil
 }
 
-func AssociateFilesWithCommands(files []string, commands []ModifyCommand) (map[string][]ModifyCommand, error) {
+// Ehh.. Not much better... Guess this wasn't the big deal
+var matchesMemoTable map[string]bool = make(map[string]bool)
+
+func Matches(path string, glob string) (bool, error) {
+	key := fmt.Sprintf("%s:%s", path, glob)
+	if matches, ok := matchesMemoTable[key]; ok {
+		logger.Debug("Found match for file %q and glob %q in memo table", path, glob)
+		return matches, nil
+	}
+	matches, err := doublestar.Match(glob, path)
+	if err != nil {
+		return false, fmt.Errorf("failed to match glob %s with file %s: %w", glob, path, err)
+	}
+	matchesMemoTable[key] = matches
+	return matches, nil
+}
+
+type FileCommandAssociation struct {
+	File            string
+	IsolateCommands []ModifyCommand
+	Commands        []ModifyCommand
+}
+
+func AssociateFilesWithCommands(files []string, commands []ModifyCommand) (map[string]FileCommandAssociation, error) {
 	associationCount := 0
-	fileCommands := make(map[string][]ModifyCommand)
+	fileCommands := make(map[string]FileCommandAssociation)
+
 	for _, file := range files {
+		fileCommands[file] = FileCommandAssociation{
+			File:            file,
+			IsolateCommands: []ModifyCommand{},
+			Commands:        []ModifyCommand{},
+		}
 		for _, command := range commands {
 			for _, glob := range command.Files {
-				// TODO: Maybe memoize this function call
-				matches, err := doublestar.Match(glob, file)
+				matches, err := Matches(file, glob)
 				if err != nil {
 					logger.Trace("Failed to match glob %s with file %s: %v", glob, file, err)
 					continue
 				}
 				if matches {
 					logger.Debug("Found match for file %q and command %q", file, command.Regex)
-					fileCommands[file] = append(fileCommands[file], command)
+					association := fileCommands[file]
+
+					if command.Isolate {
+						association.IsolateCommands = append(association.IsolateCommands, command)
+					} else {
+						association.Commands = append(association.Commands, command)
+					}
+					fileCommands[file] = association
 					associationCount++
 				}
 			}
 		}
-		logger.Debug("Found %d commands for file %q", len(fileCommands[file]), file)
-		if len(fileCommands[file]) == 0 {
+		logger.Debug("Found %d commands for file %q", len(fileCommands[file].Commands), file)
+		if len(fileCommands[file].Commands) == 0 {
 			logger.Info("No commands found for file %q", file)
 		}
+		if len(fileCommands[file].IsolateCommands) > 0 {
+			logger.Info("Found %d isolate commands for file %q", len(fileCommands[file].IsolateCommands), file)
+		}
 	}
 	logger.Info("Found %d associations between %d files and %d commands", associationCount, len(files), len(commands))
 	return fileCommands, nil
@@ -206,3 +246,16 @@ func CountGlobsBeforeDedup(commands []ModifyCommand) int {
 	}
 	return count
 }
+
+func FilterCommands(commands []ModifyCommand, filter string) []ModifyCommand {
+	filteredCommands := []ModifyCommand{}
+	filters := strings.Split(filter, ",")
+	for _, cmd := range commands {
+		for _, filter := range filters {
+			if strings.Contains(cmd.Name, filter) {
+				filteredCommands = append(filteredCommands, cmd)
+			}
+		}
+	}
+	return filteredCommands
+}