13 Commits

13 changed files with 351 additions and 100 deletions

5
.vscode/launch.json vendored
View File

@@ -41,9 +41,8 @@
"args": [
"-loglevel",
"trace",
"(?-s)LightComponent!anyrange=\"(!num)\"",
"*4",
"**/Outpost*.xml"
"-cook",
"cookscoop.yml",
]
}
]

2
go.mod
View File

@@ -1,4 +1,4 @@
module modify
module cook
go 1.24.1

44
main.go
View File

@@ -8,12 +8,12 @@ import (
"sync"
"time"
"modify/processor"
"modify/utils"
"cook/processor"
"cook/utils"
"github.com/go-git/go-git/v5"
"modify/logger"
"cook/logger"
)
type GlobalStats struct {
@@ -118,6 +118,30 @@ func main() {
startTime := time.Now()
var fileMutex sync.Mutex
// Create a map to store loggers for each command
commandLoggers := make(map[string]*logger.Logger)
for _, command := range commands {
// Create a named logger for each command
cmdName := command.Name
if cmdName == "" {
// If no name is provided, use a short version of the regex pattern
if len(command.Regex) > 20 {
cmdName = command.Regex[:17] + "..."
} else {
cmdName = command.Regex
}
}
// Parse the log level for this specific command
cmdLogLevel := logger.ParseLevel(command.LogLevel)
// Create a logger with the command name as a field
commandLoggers[command.Name] = logger.WithField("command", cmdName)
commandLoggers[command.Name].SetLevel(cmdLogLevel)
logger.Debug("Created logger for command %q with log level %s", cmdName, cmdLogLevel.String())
}
// This aggregation is great but what if one modification replaces the whole entire file?
// Shit......
// TODO: Add "Isolate" field to modifications which makes them run alone
@@ -144,7 +168,7 @@ func main() {
return
}
fileDataStr, err = RunOtherCommands(file, fileDataStr, association, &fileMutex)
fileDataStr, err = RunOtherCommands(file, fileDataStr, association, &fileMutex, commandLoggers)
if err != nil {
logger.Error("Failed to run other commands for file %q: %v", file, err)
return
@@ -230,11 +254,17 @@ func main() {
}
}
func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, fileMutex *sync.Mutex) (string, error) {
func RunOtherCommands(file string, fileDataStr string, association utils.FileCommandAssociation, fileMutex *sync.Mutex, commandLoggers map[string]*logger.Logger) (string, error) {
// Aggregate all the modifications and execute them
modifications := []utils.ReplaceCommand{}
for _, command := range association.Commands {
logger.Info("Processing file %q with command %q", file, command.Regex)
// Use command-specific logger if available, otherwise fall back to default logger
cmdLogger := logger.DefaultLogger
if cmdLog, ok := commandLoggers[command.Name]; ok {
cmdLogger = cmdLog
}
cmdLogger.Info("Processing file %q with command %q", file, command.Regex)
newModifications, err := processor.ProcessRegex(fileDataStr, command, file)
if err != nil {
return fileDataStr, fmt.Errorf("failed to process file %q with command %q: %w", file, command.Regex, err)
@@ -248,6 +278,8 @@ func RunOtherCommands(file string, fileDataStr string, association utils.FileCom
count = 0
}
stats.ModificationsPerCommand.Store(command.Name, count.(int)+len(newModifications))
cmdLogger.Debug("Command %q generated %d modifications", command.Name, len(newModifications))
}
if len(modifications) == 0 {

View File

@@ -6,7 +6,7 @@ import (
lua "github.com/yuin/gopher-lua"
"modify/logger"
"cook/logger"
)
// Maybe we make this an interface again for the shits and giggles

View File

@@ -9,8 +9,8 @@ import (
lua "github.com/yuin/gopher-lua"
"modify/logger"
"modify/utils"
"cook/logger"
"cook/utils"
)
type CaptureGroup struct {
@@ -33,6 +33,11 @@ func ProcessRegex(content string, command utils.ModifyCommand, filename string)
// We don't HAVE to do this multiple times for a pattern
// But it's quick enough for us to not care
pattern := resolveRegexPlaceholders(command.Regex)
// I'm not too happy about having to trim regex, we could have meaningful whitespace or newlines
// But it's a compromise that allows us to use | in yaml
// Otherwise we would have to escape every god damn pair of quotation marks
// And a bunch of other shit
pattern = strings.TrimSpace(pattern)
logger.Debug("Compiling regex pattern: %s", pattern)
patternCompileStart := time.Now()
@@ -156,7 +161,11 @@ func ProcessRegex(content string, command utils.ModifyCommand, filename string)
}
}
captureGroups = deduplicateGroups(captureGroups)
// Use the DeduplicateGroups flag to control whether to deduplicate capture groups
if !command.NoDedup {
logger.Debug("Deduplicating capture groups as specified in command settings")
captureGroups = deduplicateGroups(captureGroups)
}
if err := toLua(L, captureGroups); err != nil {
logger.Error("Failed to set Lua variables: %v", err)

View File

@@ -2,8 +2,8 @@ package processor
import (
"bytes"
"cook/utils"
"io"
"modify/utils"
"os"
"regexp"
"strings"

View File

@@ -2,7 +2,7 @@ package processor
import (
"io"
"modify/logger"
"cook/logger"
"os"
)

View File

@@ -1,8 +1,8 @@
package regression
import (
"modify/processor"
"modify/utils"
"cook/processor"
"cook/utils"
"os"
"path/filepath"
"testing"

View File

@@ -10,7 +10,6 @@ var (
// Deprecated
ResetFlag = flag.Bool("reset", false, "Reset files to their original state")
LogLevel = flag.String("loglevel", "INFO", "Set log level: ERROR, WARNING, INFO, DEBUG, TRACE")
Cookfile = flag.String("cook", "**/cook.yml", "Path to cook config files, can be globbed")
ParallelFiles = flag.Int("P", 100, "Number of files to process in parallel")
Filter = flag.String("filter", "", "Filter commands before running them")
)

View File

@@ -1,14 +1,14 @@
package utils
import (
"cook/logger"
"fmt"
"modify/logger"
"os"
"path/filepath"
"time"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/object"
)
var (
@@ -94,4 +94,4 @@ func CleanupGitFiles(files []string) error {
}
}
return nil
}
}

View File

@@ -1,9 +1,10 @@
package utils
import (
"cook/logger"
"fmt"
"modify/logger"
"os"
"path/filepath"
"strings"
"github.com/bmatcuk/doublestar/v4"
@@ -19,6 +20,7 @@ type ModifyCommand struct {
Reset bool `yaml:"reset"`
LogLevel string `yaml:"loglevel"`
Isolate bool `yaml:"isolate"`
NoDedup bool `yaml:"nodedup"`
}
type CookFile []ModifyCommand
@@ -73,7 +75,15 @@ func AssociateFilesWithCommands(files []string, commands []ModifyCommand) (map[s
}
for _, command := range commands {
for _, glob := range command.Files {
matches, err := Matches(file, glob)
_, pattern, err := FigureOutGlobRoot(glob)
if err != nil {
logger.Trace("Failed to figure out glob root for %s: %v", glob, err)
continue
}
file = filepath.Clean(file)
file = strings.ReplaceAll(file, "\\", "/")
matches, err := Matches(file, pattern)
if err != nil {
logger.Trace("Failed to match glob %s with file %s: %v", glob, file, err)
continue
@@ -109,6 +119,8 @@ func AggregateGlobs(commands []ModifyCommand) map[string]struct{} {
globs := make(map[string]struct{})
for _, command := range commands {
for _, glob := range command.Files {
glob = strings.ReplaceAll(glob, "~", os.Getenv("USERPROFILE"))
glob = strings.ReplaceAll(glob, "\\", "/")
globs[glob] = struct{}{}
}
}
@@ -116,21 +128,87 @@ func AggregateGlobs(commands []ModifyCommand) map[string]struct{} {
return globs
}
func FigureOutGlobRoot(inputPattern string) (root, pattern string, err error) {
logger.Debug("Starting to figure out glob root for input pattern: %s", inputPattern)
cwd, err := os.Getwd()
if err != nil {
logger.Error("Failed to get current working directory: %v", err)
return "", inputPattern, fmt.Errorf("failed to get current working directory: %w", err)
}
logger.Trace("Current working directory: %s", cwd)
root = inputPattern
if !filepath.IsAbs(inputPattern) {
root = filepath.Join(cwd, inputPattern)
logger.Info("Input pattern is not absolute. Using combined path: %s", root)
}
root = filepath.Clean(root)
logger.Debug("Cleaned root path: %s", root)
// In either case (whatever our root may be), we have to figure out
// Where to start, what our FS will be
// The best place would be the last sure entry
// That is to say the final directory that is not a wildcard
finalroot := ""
// TODO: This will probably explode on linux because oooooooooo we have to be clever oooooooooo / on linux \\ on windows ooooooooooo
parts := strings.Split(root, "\\")
lastIndex := len(parts) - 1
logger.Debug("Split root into parts: %v", parts)
// In the case our pattern ends with a file (and many of them do)
// Look for only the folders, we cannot mount a file as a FS
// In any case we have to match files so they have to be the last part
for i := 0; i < len(parts)-1; i++ {
part := parts[i]
logger.Trace("Processing part: %s", part)
if part == "*" || part == "**" || part == "?" || part == "[" {
lastIndex = i
logger.Debug("Found wildcard part: %s, updating lastIndex to: %d", part, lastIndex)
break
}
// We can't use join here because it joins C: and Users as C:Users
// Instead of C:/Users/
// God damn it
if finalroot != "" {
finalroot = finalroot + "/" + part
} else {
finalroot = finalroot + part
}
}
finalroot = filepath.Clean(finalroot)
logger.Debug("Final root after processing: %s", finalroot)
// After all this juggling our pattern is whatever is left after the finalroot
// Which is, in "worst" case, only a file
pattern = strings.Join(parts[lastIndex:], "/")
logger.Info("Determined pattern: %s", pattern)
return finalroot, pattern, nil
}
func ExpandGLobs(patterns map[string]struct{}) ([]string, error) {
var files []string
filesMap := make(map[string]bool)
cwd, err := os.Getwd()
if err != nil {
return nil, fmt.Errorf("failed to get current working directory: %w", err)
}
logger.Debug("Expanding patterns from directory: %s", cwd)
for pattern := range patterns {
root, pattern, err := FigureOutGlobRoot(pattern)
if err != nil {
return nil, fmt.Errorf("failed to figure out glob root: %w", err)
}
logger.Trace("Processing pattern: %s", pattern)
matches, _ := doublestar.Glob(os.DirFS(cwd), pattern)
matches, err := doublestar.Glob(os.DirFS(root), pattern)
if err != nil {
return nil, fmt.Errorf("failed to glob pattern %s: %w", pattern, err)
}
logger.Debug("Found %d matches for pattern %s", len(matches), pattern)
for _, m := range matches {
m = filepath.Join(root, m)
m = filepath.Clean(m)
m = strings.ReplaceAll(m, "\\", "/")
info, err := os.Stat(m)
if err != nil {
logger.Warning("Error getting file info for %s: %v", m, err)
@@ -152,69 +230,38 @@ func ExpandGLobs(patterns map[string]struct{}) ([]string, error) {
func LoadCommands(args []string) ([]ModifyCommand, error) {
commands := []ModifyCommand{}
logger.Info("Loading commands from cook files: %s", *Cookfile)
newcommands, err := LoadCommandsFromCookFiles(*Cookfile)
if err != nil {
return nil, fmt.Errorf("failed to load commands from cook files: %w", err)
}
logger.Info("Successfully loaded %d commands from cook files", len(newcommands))
commands = append(commands, newcommands...)
logger.Info("Now total commands: %d", len(commands))
logger.Info("Loading commands from arguments: %v", args)
newcommands, err = LoadCommandFromArgs(args)
if err != nil {
if len(commands) == 0 {
return nil, fmt.Errorf("failed to load commands from args: %w", err)
logger.Info("Loading commands from cook files: %s", args)
for _, arg := range args {
newcommands, err := LoadCommandsFromCookFiles(arg)
if err != nil {
return nil, fmt.Errorf("failed to load commands from cook files: %w", err)
}
logger.Warning("Failed to load commands from args: %v", err)
logger.Info("Successfully loaded %d commands from cook iles", len(newcommands))
commands = append(commands, newcommands...)
logger.Info("Now total commands: %d", len(commands))
}
logger.Info("Successfully loaded %d commands from args", len(newcommands))
commands = append(commands, newcommands...)
logger.Info("Now total commands: %d", len(commands))
logger.Info("Loaded %d commands from all cook f", len(commands))
return commands, nil
}
func LoadCommandFromArgs(args []string) ([]ModifyCommand, error) {
// Cannot reset without git, right?
if *ResetFlag {
*GitFlag = true
}
if len(args) < 3 {
return nil, fmt.Errorf("at least %d arguments are required", 3)
}
command := ModifyCommand{
Regex: args[0],
Lua: args[1],
Files: args[2:],
Git: *GitFlag,
Reset: *ResetFlag,
LogLevel: *LogLevel,
}
if err := command.Validate(); err != nil {
return nil, fmt.Errorf("invalid command: %w", err)
}
return []ModifyCommand{command}, nil
}
func LoadCommandsFromCookFiles(s string) ([]ModifyCommand, error) {
cwd, err := os.Getwd()
func LoadCommandsFromCookFiles(pattern string) ([]ModifyCommand, error) {
root, pattern, err := FigureOutGlobRoot(pattern)
if err != nil {
return nil, fmt.Errorf("failed to get current working directory: %w", err)
return nil, fmt.Errorf("failed to figure out glob root: %w", err)
}
commands := []ModifyCommand{}
cookFiles, err := doublestar.Glob(os.DirFS(cwd), *Cookfile)
cookFiles, err := doublestar.Glob(os.DirFS(root), pattern)
if err != nil {
return nil, fmt.Errorf("failed to glob cook files: %w", err)
}
for _, cookFile := range cookFiles {
cookFile = filepath.Clean(cookFile)
cookFile = strings.ReplaceAll(cookFile, "\\", "/")
logger.Info("Loading commands from cook file: %s", cookFile)
cookFileData, err := os.ReadFile(cookFile)
if err != nil {
return nil, fmt.Errorf("failed to read cook file: %w", err)

View File

@@ -3,6 +3,7 @@ package utils
import (
"os"
"path/filepath"
"strings"
"testing"
"github.com/stretchr/testify/assert"
@@ -158,21 +159,24 @@ func TestAssociateFilesWithCommands(t *testing.T) {
// The associations expected depends on the implementation
// Let's check the actual associations and verify they make sense
for file, cmds := range associations {
t.Logf("File %s is associated with %d commands", file, len(cmds))
for i, cmd := range cmds {
for file, assoc := range associations {
t.Logf("File %s is associated with %d commands and %d isolate commands", file, len(assoc.Commands), len(assoc.IsolateCommands))
for i, cmd := range assoc.Commands {
t.Logf(" Command %d: Pattern=%s, Files=%v", i, cmd.Regex, cmd.Files)
}
for i, cmd := range assoc.IsolateCommands {
t.Logf(" Isolate Command %d: Pattern=%s, Files=%v", i, cmd.Regex, cmd.Files)
}
// Specific validation based on our file types
switch file {
case "file1.xml":
if len(cmds) < 1 {
t.Errorf("Expected at least 1 command for file1.xml, got %d", len(cmds))
if len(assoc.Commands) < 1 {
t.Errorf("Expected at least 1 command for file1.xml, got %d", len(assoc.Commands))
}
// Verify at least one command with *.xml pattern
hasXmlGlob := false
for _, cmd := range cmds {
for _, cmd := range assoc.Commands {
for _, glob := range cmd.Files {
if glob == "*.xml" {
hasXmlGlob = true
@@ -187,12 +191,12 @@ func TestAssociateFilesWithCommands(t *testing.T) {
t.Errorf("Expected command with *.xml glob for file1.xml")
}
case "file2.txt":
if len(cmds) < 1 {
t.Errorf("Expected at least 1 command for file2.txt, got %d", len(cmds))
if len(assoc.Commands) < 1 {
t.Errorf("Expected at least 1 command for file2.txt, got %d", len(assoc.Commands))
}
// Verify at least one command with *.txt pattern
hasTxtGlob := false
for _, cmd := range cmds {
for _, cmd := range assoc.Commands {
for _, glob := range cmd.Files {
if glob == "*.txt" {
hasTxtGlob = true
@@ -207,12 +211,12 @@ func TestAssociateFilesWithCommands(t *testing.T) {
t.Errorf("Expected command with *.txt glob for file2.txt")
}
case "subdir/file3.xml":
if len(cmds) < 1 {
t.Errorf("Expected at least 1 command for subdir/file3.xml, got %d", len(cmds))
if len(assoc.Commands) < 1 {
t.Errorf("Expected at least 1 command for subdir/file3.xml, got %d", len(assoc.Commands))
}
// Should match both *.xml and subdir/* patterns
matches := 0
for _, cmd := range cmds {
for _, cmd := range assoc.Commands {
for _, glob := range cmd.Files {
if glob == "*.xml" || glob == "subdir/*" {
matches++
@@ -393,10 +397,10 @@ func TestLoadCommandsFromCookFileSuccess(t *testing.T) {
// Arrange
yamlData := []byte(`
- name: command1
pattern: "*.txt"
regex: "*.txt"
lua: replace
- name: command2
pattern: "*.go"
regex: "*.go"
lua: delete
`)
@@ -420,11 +424,11 @@ func TestLoadCommandsFromCookFileWithComments(t *testing.T) {
yamlData := []byte(`
# This is a comment
- name: command1
pattern: "*.txt"
regex: "*.txt"
lua: replace
# Another comment
- name: command2
pattern: "*.go"
regex: "*.go"
lua: delete
`)
@@ -445,7 +449,7 @@ func TestLoadCommandsFromCookFileWithComments(t *testing.T) {
// Handle different YAML formatting styles (flow vs block)
func TestLoadCommandsFromCookFileWithFlowStyle(t *testing.T) {
// Arrange
yamlData := []byte(`[ { name: command1, pattern: "*.txt", lua: replace }, { name: command2, pattern: "*.go", lua: delete } ]`)
yamlData := []byte(`[ { name: command1, regex: "*.txt", lua: replace }, { name: command2, regex: "*.go", lua: delete } ]`)
// Act
commands, err := LoadCommandsFromCookFile(yamlData)
@@ -496,13 +500,13 @@ func TestLoadCommandsFromCookFileWithMultipleEntries(t *testing.T) {
// Arrange
yamlData := []byte(`
- name: command1
pattern: "*.txt"
regex: "*.txt"
lua: replace
- name: command2
pattern: "*.go"
regex: "*.go"
lua: delete
- name: command3
pattern: "*.md"
regex: "*.md"
lua: append
`)
@@ -1266,3 +1270,164 @@ func TestLoadCommandsFromCookFilesNoYamlFiles(t *testing.T) {
// t.Errorf("Expected 0 commands, got: %d", len(commands))
// }
// }
// Absolute path without wildcards returns correct root and pattern
func TestFigureOutGlobRootWithAbsolutePathNoWildcards(t *testing.T) {
// Setup
absPath := filepath.Join("C:\\Users", "test", "documents", "file.txt")
// Execute
root, pattern, err := FigureOutGlobRoot(absPath)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join("C:\\Users", "test", "documents"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "file.txt", pattern)
}
// Empty input pattern handling - we expect our "Patter", such as it is, to be a folder
func TestFigureOutGlobRootWithEmptyPattern(t *testing.T) {
// Setup
emptyPattern := ""
// Execute
root, pattern, err := FigureOutGlobRoot(emptyPattern)
// Assert
assert.NoError(t, err)
cwd, _ := os.Getwd()
cwdParts := strings.Split(filepath.Clean(cwd), "\\")
expectedRoot := strings.Join(cwdParts[:len(cwdParts)-1], "\\")
expectedPattern := cwdParts[len(cwdParts)-1]
assert.Equal(t, expectedRoot, root)
assert.Equal(t, expectedPattern, pattern)
}
// Relative path is correctly joined with current working directory
func TestFigureOutGlobRootWithRelativePath(t *testing.T) {
// Setup
cwd, err := os.Getwd()
assert.NoError(t, err)
relPath := "test/documents/file.txt"
// Execute
root, pattern, err := FigureOutGlobRoot(relPath)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join(cwd, "test/documents"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "file.txt", pattern)
}
// Path with wildcards correctly identifies the last non-wildcard directory as root
func TestFigureOutGlobRootWithWildcards(t *testing.T) {
// Setup
inputPattern := filepath.Join("C:\\Users", "test", "documents", "*", "file.txt")
// Execute
root, pattern, err := FigureOutGlobRoot(inputPattern)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join("C:\\Users", "test", "documents"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "*/file.txt", pattern)
}
// Windows-style paths are properly handled and converted
func TestFigureOutGlobRootWithRelativePathAndWildcards(t *testing.T) {
// Setup
inputPattern := "documents\\*\\file?.txt"
cwd, _ := os.Getwd()
// Execute
root, pattern, err := FigureOutGlobRoot(inputPattern)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join(cwd, "documents"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "*/file?.txt", pattern)
}
// Path with only wildcards (e.g., "*" or "**")
func TestFigureOutGlobRootWithOnlyWildcards(t *testing.T) {
// Setup
wildcardPattern := "*"
// Execute
root, pattern, err := FigureOutGlobRoot(wildcardPattern)
// Assert
assert.NoError(t, err)
expectedRoot, _ := os.Getwd()
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "*", pattern)
}
// Multiple path segments are correctly processed and joined
func TestFigureOutGlobRootWithRelativePathAndWildcards2(t *testing.T) {
// Setup
cwd, err := os.Getwd()
assert.NoError(t, err)
relPath := filepath.Join("test", "data", "*", "file?.txt")
// Execute
root, pattern, err := FigureOutGlobRoot(relPath)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join(cwd, "test", "data"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "*/file?.txt", pattern)
}
// Path with mixed forward and backward slashes
func TestFigureOutGlobRootWithMixedSlashes(t *testing.T) {
// Setup
mixedPath := "C:\\Users/test\\documents\\file.txt"
// Execute
root, pattern, err := FigureOutGlobRoot(mixedPath)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join("C:\\Users", "test", "documents"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "file.txt", pattern)
}
// Path with wildcards in the first segment
func TestFigureOutGlobRootWithWildcardInFirstSegment(t *testing.T) {
// Setup
inputPattern := "*\\Users\\test\\documents\\file.txt"
cwd, _ := os.Getwd()
// Execute
root, pattern, err := FigureOutGlobRoot(inputPattern)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(cwd)
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "*/Users/test/documents/file.txt", pattern)
}
// Handling of relative paths with ".." or "." components
func TestFigureOutGlobRootWithRelativePathAndDotComponents(t *testing.T) {
// Setup
cwd, err := os.Getwd()
assert.NoError(t, err)
relPath := filepath.Join("..", ".", "test", "documents", "file.txt")
// Execute
root, pattern, err := FigureOutGlobRoot(relPath)
// Assert
assert.NoError(t, err)
expectedRoot := filepath.Clean(filepath.Join(cwd, "..", "test", "documents"))
assert.Equal(t, expectedRoot, root)
assert.Equal(t, "file.txt", pattern)
}

View File

@@ -1,8 +1,8 @@
package utils
import (
"cook/logger"
"fmt"
"modify/logger"
"sort"
)