Implement saving snapshots to a database

.gitignore

@@ -1,2 +1,3 @@
 *.exe
 .qodo
+*.sqlite

go.mod

@@ -38,6 +38,7 @@ require (
 )
 
 require (
+	git.site.quack-lab.dev/dave/cyutils v1.0.0
 	github.com/go-git/go-git/v5 v5.14.0
 	github.com/golang/groupcache v0.0.0-20241129210726-2c02b8208cf8 // indirect
 	golang.org/x/net v0.35.0 // indirect

go.sum

@@ -2,6 +2,8 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
 dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 git.site.quack-lab.dev/dave/cylogger v1.2.2 h1:4xUXASEBlG9NiGxh7f57xHh9imW4unHzakIEpQoKC5E=
 git.site.quack-lab.dev/dave/cylogger v1.2.2/go.mod h1:VS9MI4Y/cwjCBZgel7dSfCQlwtAgHmfvixOoBgBhtKg=
+git.site.quack-lab.dev/dave/cyutils v1.0.0 h1:yp/jkM2M7UZ+UIQuy+vPI7yDvTUdpbEdFL8h0lzUTvA=
+git.site.quack-lab.dev/dave/cyutils v1.0.0/go.mod h1:luGNFimplFhkpRLebhkVTNjG2wYfPAs+pu+UIMhBYbE=
 github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY=
 github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
 github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=

main.go

@@ -25,7 +25,7 @@ type GlobalStats struct {
 }
 
 var (
-	stats    GlobalStats = GlobalStats{
+	stats GlobalStats = GlobalStats{
 		ModificationsPerCommand: sync.Map{},
 	}
 )
@@ -34,8 +34,6 @@ func main() {
 	flag.Usage = func() {
 		fmt.Fprintf(os.Stderr, "Usage: %s [options] <pattern> <lua_expression> <...files_or_globs>\n", os.Args[0])
 		fmt.Fprintf(os.Stderr, "\nOptions:\n")
-		fmt.Fprintf(os.Stderr, "  -git\n")
-		fmt.Fprintf(os.Stderr, "        Use git to manage files\n")
 		fmt.Fprintf(os.Stderr, "  -reset\n")
 		fmt.Fprintf(os.Stderr, "        Reset files to their original state\n")
 		fmt.Fprintf(os.Stderr, "  -loglevel string\n")
@@ -83,7 +81,6 @@ func main() {
 		logger.Trace("Regex: %s", command.Regex)
 		logger.Trace("Files: %v", command.Files)
 		logger.Trace("Lua: %s", command.Lua)
-		logger.Trace("Git: %t", command.Git)
 		logger.Trace("Reset: %t", command.Reset)
 		logger.Trace("Isolate: %t", command.Isolate)
 		logger.Trace("LogLevel: %s", command.LogLevel)
@@ -140,16 +137,22 @@ func main() {
 		logger.Debug("Created logger for command %q with log level %s", cmdName, cmdLogLevel.String())
 	}
 
+	db, err := utils.GetDB()
+	if err != nil {
+		logger.Error("Failed to get database: %v", err)
+		return
+	}
+
 	for file, association := range associations {
 		workers <- struct{}{}
 		wg.Add(1)
 		logger.SafeGoWithArgs(func(args ...interface{}) {
 			defer func() { <-workers }()
 			defer wg.Done()
-
 			// Track per-file processing time
 			fileStartTime := time.Now()
 
+			logger.Debug("Reading file %q", file)
 			fileData, err := os.ReadFile(file)
 			if err != nil {
 				logger.Error("Failed to read file %q: %v", file, err)
@@ -157,18 +160,28 @@ func main() {
 			}
 			fileDataStr := string(fileData)
 
+			logger.Debug("Saving file %q to database", file)
+			err = db.SaveFile(file, fileData)
+			if err != nil {
+				logger.Error("Failed to save file %q to database: %v", file, err)
+				return
+			}
+
+			logger.Debug("Running isolate commands for file %q", file)
 			fileDataStr, err = RunIsolateCommands(association, file, fileDataStr, &fileMutex)
 			if err != nil {
 				logger.Error("Failed to run isolate commands for file %q: %v", file, err)
 				return
 			}
 
+			logger.Debug("Running other commands for file %q", file)
 			fileDataStr, err = RunOtherCommands(file, fileDataStr, association, &fileMutex, commandLoggers)
 			if err != nil {
 				logger.Error("Failed to run other commands for file %q: %v", file, err)
 				return
 			}
 
+			logger.Debug("Writing file %q", file)
 			err = os.WriteFile(file, []byte(fileDataStr), 0644)
 			if err != nil {
 				logger.Error("Failed to write file %q: %v", file, err)
@@ -263,7 +276,6 @@ func CreateExampleConfig() {
 			Regex:    "price=\"(\\d+)\"",
 			Lua:      "if num(v1) < 100 then return v1 * 1.5 else return v1 end",
 			Files:    []string{"items/*.xml", "shop/*.xml"},
-			Git:      true,
 			LogLevel: "DEBUG",
 		},
 		{

utils/db.go

@@ -1,8 +1,11 @@
 package utils
 
 import (
+	"fmt"
 	"path/filepath"
+	"time"
 
+	"git.site.quack-lab.dev/dave/cylogger"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
 )
@@ -10,6 +13,13 @@ import (
 type DB interface {
 	DB() *gorm.DB
 	Raw(sql string, args ...any) *gorm.DB
+	SaveFile(filePath string, fileData []byte) error
+}
+
+type FileSnapshot struct {
+	Date     time.Time `gorm:"primaryKey"`
+	FilePath string    `gorm:"primaryKey"`
+	FileData []byte    `gorm:"type:blob"`
 }
 
 type DBWrapper struct {
@@ -30,11 +40,11 @@ func GetDB() (DB, error) {
 	if err != nil {
 		return nil, err
 	}
+	db.AutoMigrate(&FileSnapshot{})
 
 	return &DBWrapper{db: db}, nil
 }
 
-
 // Just a wrapper
 func (db *DBWrapper) Raw(sql string, args ...any) *gorm.DB {
 	return db.db.Raw(sql, args...)
@@ -43,3 +53,30 @@ func (db *DBWrapper) Raw(sql string, args ...any) *gorm.DB {
 func (db *DBWrapper) DB() *gorm.DB {
 	return db.db
 }
+
+func (db *DBWrapper) FileExists(filePath string) (bool, error) {
+	var count int64
+	err := db.db.Model(&FileSnapshot{}).Where("file_path = ?", filePath).Count(&count).Error
+	return count > 0, err
+}
+
+func (db *DBWrapper) SaveFile(filePath string, fileData []byte) error {
+	log := cylogger.Default.WithPrefix(fmt.Sprintf("SaveFile: %q", filePath))
+	exists, err := db.FileExists(filePath)
+	if err != nil {
+		log.Error("Error checking if file exists: %v", err)
+		return err
+	}
+	log.Debug("File exists: %t", exists)
+	// Nothing to do, file already exists
+	if exists {
+		log.Debug("File already exists, skipping save")
+		return nil
+	}
+	log.Debug("Saving file to database")
+	return db.db.Create(&FileSnapshot{
+		Date:     time.Now(),
+		FilePath: filePath,
+		FileData: fileData,
+	}).Error
+}

utils/modifycommand.go

@@ -16,7 +16,6 @@ type ModifyCommand struct {
 	Regex    string   `yaml:"regex"`
 	Lua      string   `yaml:"lua"`
 	Files    []string `yaml:"files"`
-	Git      bool     `yaml:"git"`
 	Reset    bool     `yaml:"reset"`
 	LogLevel string   `yaml:"loglevel"`
 	Isolate  bool     `yaml:"isolate"`