Add progress logging for processed and compared files

main.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path/filepath"
 	"sync"
+	"sync/atomic"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
 	"github.com/bmatcuk/doublestar/v4"
@@ -17,6 +18,7 @@ import (
 func main() {
 	thresh := flag.Int("thresh", 10, "Threshold for distance")
 	workers := flag.Int("workers", 100, "Number of workers")
+	notifyInterval := flag.Int("notify", 1000, "Notify interval")
 	flag.Parse()
 	logger.InitFlag()
 	hashes := &sync.Map{}
@@ -44,6 +46,8 @@ func main() {
 
 	workerChan := make(chan struct{}, *workers)
 	wg := sync.WaitGroup{}
+
+	var processedFiles uint64
 	for _, file := range files {
 		workerChan <- struct{}{}
 		wg.Add(1)
@@ -54,12 +58,20 @@ func main() {
 			ext := filepath.Ext(file)
 			if ext != ".jpg" && ext != ".jpeg" && ext != ".png" {
 				log.Debug("Skipping non-image file: %s", file)
+				atomic.AddUint64(&processedFiles, 1)
+				if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
+					logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
+				}
 				return
 			}
 
 			imgfile, err := os.Open(file)
 			if err != nil {
 				log.Error("Failed to open file: %v", err)
+				atomic.AddUint64(&processedFiles, 1)
+				if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
+					logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
+				}
 				return
 			}
 			defer imgfile.Close()
@@ -73,16 +85,28 @@ func main() {
 			}
 			if err != nil {
 				log.Error("Failed to decode image: %v", err)
+				atomic.AddUint64(&processedFiles, 1)
+				if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
+					logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
+				}
 				return
 			}
 
 			hash, err := goimagehash.ExtPerceptionHash(img, 8, 8)
 			if err != nil {
 				log.Error("Failed to calculate hash: %v", err)
+				atomic.AddUint64(&processedFiles, 1)
+				if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
+					logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
+				}
 				return
 			}
 			log.Debug("Hashed: %v", hash)
 			hashes.Store(file, hash)
+			atomic.AddUint64(&processedFiles, 1)
+			if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
+				logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
+			}
 		}(file)
 	}
 
@@ -90,6 +114,11 @@ func main() {
 	wg.Wait()
 
 	processed := &sync.Map{}
+
+	// Add progress counter for comparison stage
+	var comparedFiles uint64
+	totalFiles := len(files)
+
 	hashes.Range(func(key, value interface{}) bool {
 		workerChan <- struct{}{}
 		wg.Add(1)
@@ -100,6 +129,10 @@ func main() {
 			hasha := value.(*goimagehash.ExtImageHash)
 
 			if _, ok := processed.Load(filea); ok {
+				atomic.AddUint64(&comparedFiles, 1)
+				if atomic.LoadUint64(&comparedFiles)%uint64(*notifyInterval) == 0 {
+					logger.Info("Compared %d/%d files", atomic.LoadUint64(&comparedFiles), totalFiles)
+				}
 				return
 			}
 
@@ -127,6 +160,10 @@ func main() {
 				groupedImages[filea] = group
 				processed.Store(filea, true)
 			}
+			atomic.AddUint64(&comparedFiles, 1)
+			if atomic.LoadUint64(&comparedFiles)%uint64(*notifyInterval) == 0 {
+				logger.Info("Compared %d/%d files", atomic.LoadUint64(&comparedFiles), totalFiles)
+			}
 		}(key, value)
 		return true
 	})