dave/ImageDeduplicator
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add progress logging for processed and compared files

Browse Source
This commit is contained in:
PhatPhuckDave
2025-05-23 13:26:52 +02:00
parent 6445a2eded
commit 552d0d4057
1 changed files with 37 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
main.go
View File

@@ -8,6 +8,7 @@ import (
"os" "os"
"path/filepath" "path/filepath"
"sync" "sync"
"sync/atomic"
logger "git.site.quack-lab.dev/dave/cylogger" logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/bmatcuk/doublestar/v4" "github.com/bmatcuk/doublestar/v4"
@@ -17,6 +18,7 @@ import (
func main() { func main() {
thresh := flag.Int("thresh", 10, "Threshold for distance") thresh := flag.Int("thresh", 10, "Threshold for distance")
workers := flag.Int("workers", 100, "Number of workers") workers := flag.Int("workers", 100, "Number of workers")
notifyInterval := flag.Int("notify", 1000, "Notify interval")
flag.Parse() flag.Parse()
logger.InitFlag() logger.InitFlag()
hashes := &sync.Map{} hashes := &sync.Map{}
@@ -44,6 +46,8 @@ func main() {
workerChan := make(chan struct{}, *workers) workerChan := make(chan struct{}, *workers)
wg := sync.WaitGroup{} wg := sync.WaitGroup{}
var processedFiles uint64
for _, file := range files { for _, file := range files {
workerChan <- struct{}{} workerChan <- struct{}{}
wg.Add(1) wg.Add(1)
@@ -54,12 +58,20 @@ func main() {
ext := filepath.Ext(file) ext := filepath.Ext(file)
if ext != ".jpg" && ext != ".jpeg" && ext != ".png" { if ext != ".jpg" && ext != ".jpeg" && ext != ".png" {
log.Debug("Skipping non-image file: %s", file) log.Debug("Skipping non-image file: %s", file)
atomic.AddUint64(&processedFiles, 1)
if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
}
return return
} }
imgfile, err := os.Open(file) imgfile, err := os.Open(file)
if err != nil { if err != nil {
log.Error("Failed to open file: %v", err) log.Error("Failed to open file: %v", err)
atomic.AddUint64(&processedFiles, 1)
if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
}
return return
} }
defer imgfile.Close() defer imgfile.Close()
@@ -73,16 +85,28 @@ func main() {
} }
if err != nil { if err != nil {
log.Error("Failed to decode image: %v", err) log.Error("Failed to decode image: %v", err)
atomic.AddUint64(&processedFiles, 1)
if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
}
return return
} }
hash, err := goimagehash.ExtPerceptionHash(img, 8, 8) hash, err := goimagehash.ExtPerceptionHash(img, 8, 8)
if err != nil { if err != nil {
log.Error("Failed to calculate hash: %v", err) log.Error("Failed to calculate hash: %v", err)
atomic.AddUint64(&processedFiles, 1)
if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
}
return return
} }
log.Debug("Hashed: %v", hash) log.Debug("Hashed: %v", hash)
hashes.Store(file, hash) hashes.Store(file, hash)
atomic.AddUint64(&processedFiles, 1)
if atomic.LoadUint64(&processedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Processed %d/%d files", atomic.LoadUint64(&processedFiles), len(files))
}
}(file) }(file)
} }
@@ -90,6 +114,11 @@ func main() {
wg.Wait() wg.Wait()
processed := &sync.Map{} processed := &sync.Map{}
// Add progress counter for comparison stage
var comparedFiles uint64
totalFiles := len(files)
hashes.Range(func(key, value interface{}) bool { hashes.Range(func(key, value interface{}) bool {
workerChan <- struct{}{} workerChan <- struct{}{}
wg.Add(1) wg.Add(1)
@@ -100,6 +129,10 @@ func main() {
hasha := value.(*goimagehash.ExtImageHash) hasha := value.(*goimagehash.ExtImageHash)
if _, ok := processed.Load(filea); ok { if _, ok := processed.Load(filea); ok {
atomic.AddUint64(&comparedFiles, 1)
if atomic.LoadUint64(&comparedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Compared %d/%d files", atomic.LoadUint64(&comparedFiles), totalFiles)
}
return return
} }
@@ -127,6 +160,10 @@ func main() {
groupedImages[filea] = group groupedImages[filea] = group
processed.Store(filea, true) processed.Store(filea, true)
} }
atomic.AddUint64(&comparedFiles, 1)
if atomic.LoadUint64(&comparedFiles)%uint64(*notifyInterval) == 0 {
logger.Info("Compared %d/%d files", atomic.LoadUint64(&comparedFiles), totalFiles)
}
}(key, value) }(key, value)
return true return true
}) })