dave/directory-cleaner
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: replace std log with cylogger, add flag-based init, enhance logs,

Browse Source 
update Go version and deps, and streamline Docker build/deploy scripts
This commit is contained in:
PhatPhuckDave
2025-08-07 10:31:28 +02:00
parent 862f95f730
commit 37b08d27f5
5 changed files with 104 additions and 50 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
build.sh
View File

@@ -1 +0,0 @@
docker build -t directory-cleaner .

3
deploy.sh
View File

@@ -1 +1,2 @@
docker build -t directory-cleaner . docker build -t docker.site.quack-lab.dev/directory-cleaner .
docker push docker.site.quack-lab.dev/directory-cleaner

18
go.mod
View File

@@ -1,7 +1,19 @@
module main module main
go 1.21.6 go 1.23.6
require github.com/djherbis/times v1.6.0 require (
git.site.quack-lab.dev/dave/cylogger v1.3.0
git.site.quack-lab.dev/dave/cyutils v1.1.3
github.com/djherbis/times v1.6.0
)
require golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c // indirect require (
github.com/google/go-cmp v0.5.9 // indirect
github.com/hexops/valast v1.5.0 // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/time v0.12.0 // indirect
golang.org/x/tools v0.4.0 // indirect
mvdan.cc/gofumpt v0.4.0 // indirect
)

33
go.sum
View File

@@ -1,4 +1,35 @@
git.site.quack-lab.dev/dave/cylogger v1.3.0 h1:eTWPUD+ThVi8kGIsRcE0XDeoH3yFb5miFEODyKUdWJw=
git.site.quack-lab.dev/dave/cylogger v1.3.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk=
git.site.quack-lab.dev/dave/cyutils v1.1.3 h1:9Y1GhrPrVLut36hceZwuFm0IMlAFerl6ATRPa9tGHFM=
git.site.quack-lab.dev/dave/cyutils v1.1.3/go.mod h1:fBjALu2Cp2u2bDr+E4zbGVMBeIgFzROg+4TCcTNAiQU=
github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c= github.com/djherbis/times v1.6.0 h1:w2ctJ92J8fBvWPxugmXIv7Nz7Q3iDMKNx9v5ocVH20c=
github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0= github.com/djherbis/times v1.6.0/go.mod h1:gOHeRAz2h+VJNZ5Gmc/o7iD9k4wW7NMVqieYCY99oc0=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c h1:aFV+BgZ4svzjfabn8ERpuB4JI4N6/rdy1iusx77G3oU= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/hexops/autogold v0.8.1 h1:wvyd/bAJ+Dy+DcE09BoLk6r4Fa5R5W+O+GUzmR985WM=
github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hexops/valast v1.5.0 h1:FBTuvVi0wjTngtXJRZXMbkN/Dn6DgsUsBwch2DUJU8Y=
github.com/hexops/valast v1.5.0/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM=
mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ=

95
main.go
View File

@@ -1,7 +1,7 @@
package main package main
import ( import (
"log" "flag"
"os" "os"
"path/filepath" "path/filepath"
"regexp" "regexp"
@@ -9,6 +9,8 @@ import (
"strings" "strings"
"time" "time"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/djherbis/times" "github.com/djherbis/times"
) )
@@ -35,25 +37,25 @@ func parseDuration(date string) int64 {
var parts = strings.Split(date, "_") var parts = strings.Split(date, "_")
for _, part := range parts { for _, part := range parts {
part = strings.TrimSpace(part) part = strings.TrimSpace(part)
log.Printf("Parsing date part: %s\n", part) logger.Info("Parsing date part: %s", part)
var value = valueRegex.FindString(part) var value = valueRegex.FindString(part)
var unit = unitRegex.FindString(part) var unit = unitRegex.FindString(part)
if value == "" || unit == "" { if value == "" || unit == "" {
log.Println("Invalid date part: " + part) logger.Error("Invalid date part: " + part)
continue continue
} }
if _, ok := timeUnits[unit]; !ok { if _, ok := timeUnits[unit]; !ok {
log.Println("Invalid date unit: " + unit) logger.Error("Invalid date unit: " + unit)
continue continue
} }
log.Printf("Part %s parsed as: Value: %s, Unit: %s\n", part, value, unit) logger.Info("Part %s parsed as: Value: %s, Unit: %s", part, value, unit)
var valueMs, _ = strconv.ParseInt(value, 10, 16) var valueMs, _ = strconv.ParseInt(value, 10, 16)
valueMs = valueMs * timeUnits[unit] valueMs = valueMs * timeUnits[unit]
milliseconds += valueMs milliseconds += valueMs
log.Printf("Adding %dms to duration, now: %d\n", valueMs, milliseconds) logger.Info("Adding %dms to duration, now: %d", valueMs, milliseconds)
} }
return milliseconds return milliseconds
@@ -68,31 +70,32 @@ func getEnv(key, def string) string {
} }
func scanRoot() { func scanRoot() {
log.Println("Scanning root directory...") log := logger.Default.WithPrefix("scanRoot")
log.Info("Scanning root directory...")
filepath.Walk(constants.ROOT, func(path string, info os.FileInfo, err error) error { filepath.Walk(constants.ROOT, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
log.Printf("Error scanning %s: %s\n", path, err) log.Error("Error scanning %s: %s", path, err)
return nil return nil
} }
path = filepath.ToSlash(path) path = filepath.ToSlash(path)
if path == constants.ROOT { if path == constants.ROOT {
log.Printf("Skipping root directory %s...\n", path) log.Info("Skipping root directory %s...", path)
return nil return nil
} }
// I forgot why this code was here... It doesn't make sense to me now // I forgot why this code was here... It doesn't make sense to me now
// if info.IsDir() { // if info.IsDir() {
// log.Printf("Skipping directory %s...\n", path) // log.Info("Skipping directory %s...", path)
// return filepath.SkipDir // return filepath.SkipDir
// } // }
// We hope that IGNORED_DIRECTORIES is a small list, so we can afford to iterate over it // We hope that IGNORED_DIRECTORIES is a small list, so we can afford to iterate over it
// In fact iteration should be faster for small lists rather than hashing // In fact iteration should be faster for small lists rather than hashing
for _, ignoredDir := range constants.IGNORED_DIRECTORIES { for _, ignoredDir := range constants.IGNORED_DIRECTORIES {
log.Println(constants.IGNORED_DIRECTORIES, len(constants.IGNORED_DIRECTORIES)) log.Info("Ignored directories: %s", constants.IGNORED_DIRECTORIES)
if strings.HasPrefix(path, ignoredDir) { if strings.HasPrefix(path, ignoredDir) {
log.Printf("Ignoring directory %s\n", path) log.Info("Ignoring directory %s", path)
return filepath.SkipDir return filepath.SkipDir
} }
} }
@@ -103,16 +106,17 @@ func scanRoot() {
} }
func scanArchive() { func scanArchive() {
log.Println("Scanning archive...") log := logger.Default.WithPrefix("scanArchive")
log.Info("Scanning archive...")
filepath.Walk(constants.ROOT_ARCHIVE, func(path string, info os.FileInfo, err error) error { filepath.Walk(constants.ROOT_ARCHIVE, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
log.Printf("Error scanning %s: %s\n", path, err) log.Error("Error scanning %s: %s", path, err)
return nil return nil
} }
path = filepath.ToSlash(path) path = filepath.ToSlash(path)
if path == constants.ROOT_ARCHIVE { if path == constants.ROOT_ARCHIVE {
log.Printf("Skipping root directory %s...\n", path) log.Info("Skipping root directory %s...", path)
return nil return nil
} }
@@ -123,6 +127,7 @@ func scanArchive() {
func processFile(path string, info os.FileInfo) { func processFile(path string, info os.FileInfo) {
var now = time.Now().UnixMilli() var now = time.Now().UnixMilli()
log := logger.Default.WithPrefix("processFile").WithPrefix(path)
var timeType = "accessed" var timeType = "accessed"
if constants.USE_MODTIME { if constants.USE_MODTIME {
@@ -137,15 +142,16 @@ func processFile(path string, info os.FileInfo) {
} }
var timeDelta = now - fileTime var timeDelta = now - fileTime
log.Printf("File %s last %s at %d, %dms ago\n", path, timeType, fileTime, timeDelta) log.Info("File %s last %s at %d, %dms ago", path, timeType, fileTime, timeDelta)
if timeDelta > constants.ARCHIVE_THRESHOLD { if timeDelta > constants.ARCHIVE_THRESHOLD {
log.Printf("File %s was %s more than %dms ago, archiving...\n", path, timeType, constants.ARCHIVE_THRESHOLD) log.Info("File %s was %s more than %dms ago, archiving...", path, timeType, constants.ARCHIVE_THRESHOLD)
archiveFile(path) archiveFile(path)
} }
} }
func processArchiveFile(path string, info os.FileInfo) { func processArchiveFile(path string, info os.FileInfo) {
var now = time.Now().UnixMilli() var now = time.Now().UnixMilli()
log := logger.Default.WithPrefix("processArchiveFile").WithPrefix(path)
var timeType = "accessed" var timeType = "accessed"
if constants.USE_MODTIME { if constants.USE_MODTIME {
@@ -160,10 +166,10 @@ func processArchiveFile(path string, info os.FileInfo) {
} }
var timeDelta = now - int64(fileTime) var timeDelta = now - int64(fileTime)
log.Printf("File %s last %s at %d, %dms ago\n", path, timeType, fileTime, timeDelta) log.Info("File %s last %s at %d, %dms ago", path, timeType, fileTime, timeDelta)
if timeDelta > constants.DELETE_THRESHOLD { if timeDelta > constants.DELETE_THRESHOLD {
log.Printf("File %s was %s more than %dms ago, deleting...\n", path, timeType, constants.DELETE_THRESHOLD) log.Info("File %s was %s more than %dms ago, deleting...", path, timeType, constants.DELETE_THRESHOLD)
deleteFile(path) deleteFile(path)
} }
} }
@@ -171,32 +177,35 @@ func processArchiveFile(path string, info os.FileInfo) {
func archiveFile(path string) { func archiveFile(path string) {
// defer os.Exit(1) // defer os.Exit(1)
var newPath = constants.ROOT_ARCHIVE + strings.Replace(path, constants.ROOT, "", 1) var newPath = constants.ROOT_ARCHIVE + strings.Replace(path, constants.ROOT, "", 1)
log.Printf("Archiving file %s to %s...\n", path, newPath) log := logger.Default.WithPrefix("archiveFile").WithPrefix(path)
log.Info("Archiving file %s to %s...", path, newPath)
os.MkdirAll(filepath.Dir(newPath), os.ModePerm) os.MkdirAll(filepath.Dir(newPath), os.ModePerm)
var err = os.Rename(path, newPath) var err = os.Rename(path, newPath)
if err != nil { if err != nil {
log.Printf("Error archiving file %s: %s\n", path, err) log.Error("Error archiving file %s: %s", path, err)
return return
} }
numFilesArchived++ numFilesArchived++
} }
func deleteFile(path string) { func deleteFile(path string) {
log := logger.Default.WithPrefix("deleteFile").WithPrefix(path)
// defer os.Exit(1) // defer os.Exit(1)
log.Printf("Deleting file %s...\n", path) log.Info("Deleting file %s...", path)
var err = os.Remove(path) var err = os.Remove(path)
if err != nil { if err != nil {
log.Printf("Error deleting file %s: %s\n", path, err) log.Error("Error deleting file %s: %s", path, err)
return return
} }
numFilesDeleted++ numFilesDeleted++
} }
func cleanRoot() { func cleanRoot() {
log := logger.Default.WithPrefix("cleanRoot")
var files, err = os.ReadDir(constants.ROOT) var files, err = os.ReadDir(constants.ROOT)
if err != nil { if err != nil {
log.Printf("Error reading root directory %s: %s\n", constants.ROOT, err) log.Error("Error reading root directory %s: %s", constants.ROOT, err)
return return
} }
for _, file := range files { for _, file := range files {
@@ -205,15 +214,15 @@ func cleanRoot() {
} }
var empty, err = isDirEmpty(constants.ROOT + "/" + file.Name()) var empty, err = isDirEmpty(constants.ROOT + "/" + file.Name())
if err != nil { if err != nil {
log.Printf("Error checking if directory %s is empty: %s\n", file.Name(), err) log.Error("Error checking if directory %s is empty: %s", file.Name(), err)
continue continue
} }
log.Printf("Directory %s isempty: %t\n", file.Name(), empty) log.Info("Directory %s isempty: %t", file.Name(), empty)
if empty { if empty {
log.Printf("Deleting empty directory %s\n", file.Name()) log.Info("Deleting empty directory %s", file.Name())
var err = os.RemoveAll(constants.ROOT + "/" + file.Name()) var err = os.RemoveAll(constants.ROOT + "/" + file.Name())
if err != nil { if err != nil {
log.Printf("Error deleting empty directory %s: %s\n", file.Name(), err) log.Error("Error deleting empty directory %s: %s", file.Name(), err)
} }
} }
} }
@@ -221,16 +230,17 @@ func cleanRoot() {
func isDirEmpty(dirPath string) (bool, error) { func isDirEmpty(dirPath string) (bool, error) {
var empty = true var empty = true
var ferr error = nil var ferr error = nil
log := logger.Default.WithPrefix("isDirEmpty").WithPrefix(dirPath)
filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error { filepath.Walk(dirPath, func(path string, info os.FileInfo, err error) error {
if err != nil { if err != nil {
log.Printf("Error scanning %s: %s\n", path, err) log.Error("Error scanning %s: %s", path, err)
ferr = err ferr = err
return nil return nil
} }
if !info.IsDir() { if !info.IsDir() {
empty = false empty = false
log.Printf("Directory %s is not empty, found %s\n", dirPath, path) log.Info("Directory %s is not empty, found %s", dirPath, path)
return filepath.SkipAll return filepath.SkipAll
} }
return nil return nil
@@ -252,7 +262,7 @@ func doRun() {
scanRoot() scanRoot()
scanArchive() scanArchive()
cleanRoot() cleanRoot()
log.Printf("Archived %d files, deleted %d files\n", numFilesArchived, numFilesDeleted) logger.Info("Archived %d files, deleted %d files", numFilesArchived, numFilesDeleted)
numFilesArchived = 0 numFilesArchived = 0
numFilesDeleted = 0 numFilesDeleted = 0
} }
@@ -260,12 +270,14 @@ func doRun() {
var constants = Constants{} var constants = Constants{}
func main() { func main() {
log.SetFlags(0b111) flag.Parse()
logger.InitFlag()
// Important: Access times don’t accumulate. // Important: Access times don’t accumulate.
// This implies that archiving the file won't alter its access time. // This implies that archiving the file won't alter its access time.
// Therefore, assign X as the ARCHIVE_TIME and X + Y as the DELETE_TIME, // Therefore, assign X as the ARCHIVE_TIME and X + Y as the DELETE_TIME,
// where X represents the duration it can exist in the folder, // where X represents the duration it can exist in the folder,
// and Y represents the duration it can exist in the archive. // and Y represents the duration it can exist in the archive.
logger.Info("Starting directory cleaner")
var ROOT = filepath.ToSlash(strings.TrimSpace(getEnv("ROOT", "/tmp"))) var ROOT = filepath.ToSlash(strings.TrimSpace(getEnv("ROOT", "/tmp")))
var ROOT_ARCHIVE = filepath.ToSlash(strings.TrimSpace(getEnv("ROOT_ARCHIVE", ROOT+"/archive"))) var ROOT_ARCHIVE = filepath.ToSlash(strings.TrimSpace(getEnv("ROOT_ARCHIVE", ROOT+"/archive")))
@@ -284,6 +296,14 @@ func main() {
var DELETE_THRESHOLD = parseDuration(getEnv("DELETE_THRESHOLD", "12h")) var DELETE_THRESHOLD = parseDuration(getEnv("DELETE_THRESHOLD", "12h"))
var SCAN_INTERVAL = time.Duration(parseDuration(getEnv("SCAN_INTERVAL", "1m")) * 1e6) var SCAN_INTERVAL = time.Duration(parseDuration(getEnv("SCAN_INTERVAL", "1m")) * 1e6)
var USE_MODTIME = strings.TrimSpace(getEnv("USE_MODTIME", "false")) == "true" var USE_MODTIME = strings.TrimSpace(getEnv("USE_MODTIME", "false")) == "true"
logger.Info("Input args parsed as:")
logger.Info("ROOT: %s", ROOT)
logger.Info("ROOT_ARCHIVE: %s", ROOT_ARCHIVE)
logger.Info("IGNORED_DIRECTORIES: %s", IGNORED_DIRECTORIES)
logger.Info("ARCHIVE_THRESHOLD: %d", ARCHIVE_THRESHOLD)
logger.Info("DELETE_THRESHOLD: %d", DELETE_THRESHOLD)
logger.Info("SCAN_INTERVAL: %d", SCAN_INTERVAL.Milliseconds())
logger.Info("USE_MODTIME: %s", strconv.FormatBool(USE_MODTIME))
constants.ROOT = ROOT constants.ROOT = ROOT
constants.ROOT_ARCHIVE = ROOT_ARCHIVE constants.ROOT_ARCHIVE = ROOT_ARCHIVE
@@ -293,18 +313,9 @@ func main() {
constants.SCAN_INTERVAL = SCAN_INTERVAL constants.SCAN_INTERVAL = SCAN_INTERVAL
constants.USE_MODTIME = USE_MODTIME constants.USE_MODTIME = USE_MODTIME
log.Println("Input args parsed as:")
log.Printf("ROOT: %s\n", ROOT)
log.Printf("ROOT_ARCHIVE: %s\n", ROOT_ARCHIVE)
log.Printf("IGNORED_DIRECTORIES: %s\n", IGNORED_DIRECTORIES)
log.Printf("ARCHIVE_THRESHOLD: %d\n", ARCHIVE_THRESHOLD)
log.Printf("DELETE_THRESHOLD: %d\n", DELETE_THRESHOLD)
log.Printf("SCAN_INTERVAL: %d\n", SCAN_INTERVAL.Milliseconds())
log.Printf("USE_MODTIME: %s\n", strconv.FormatBool(USE_MODTIME))
doRun() doRun()
for { for {
log.Printf("Running at %d", time.Now().UnixMilli()) logger.Info("Running at %d", time.Now().UnixMilli())
time.Sleep(SCAN_INTERVAL) time.Sleep(SCAN_INTERVAL)
doRun() doRun()
} }