Implement walkdir
This commit is contained in:
141
main.go
141
main.go
@@ -2,33 +2,148 @@ package main
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
|
||||||
"log"
|
"log"
|
||||||
"os"
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
)
|
)
|
||||||
|
|
||||||
var Error *log.Logger
|
var Error *log.Logger
|
||||||
var Warning *log.Logger
|
var Warning *log.Logger
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
|
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
|
||||||
logFile, err := os.Create("main.log")
|
log.SetOutput(os.Stdout)
|
||||||
if err != nil {
|
|
||||||
log.Printf("Error creating log file: %v", err)
|
|
||||||
os.Exit(1)
|
|
||||||
}
|
|
||||||
logger := io.MultiWriter(os.Stdout, logFile)
|
|
||||||
log.SetOutput(logger)
|
|
||||||
|
|
||||||
Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
|
Error = log.New(os.Stderr,
|
||||||
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
|
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
|
||||||
log.Lmicroseconds|log.Lshortfile)
|
log.Lmicroseconds|log.Lshortfile)
|
||||||
Warning = log.New(io.MultiWriter(logFile, os.Stdout),
|
Warning = log.New(os.Stdout,
|
||||||
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
|
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
|
||||||
log.Lmicroseconds|log.Lshortfile)
|
log.Lmicroseconds|log.Lshortfile)
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
log.Println("Hello, World!")
|
files := make(chan string, 10000)
|
||||||
Warning.Println("Hello, World!")
|
status := make(chan error)
|
||||||
Error.Println("Hello, World!")
|
|
||||||
|
go GetSyncFilesRecursively(".", files, status)
|
||||||
|
|
||||||
|
for file := range files {
|
||||||
|
log.Println(file)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func GetSyncFilesRecursively(input string, output chan string, status chan error) {
|
||||||
|
defer close(output)
|
||||||
|
defer close(status)
|
||||||
|
|
||||||
|
var filesProcessed int32
|
||||||
|
var foldersProcessed int32
|
||||||
|
var activeWorkers int32
|
||||||
|
|
||||||
|
progressTicker := time.NewTicker(200 * time.Millisecond)
|
||||||
|
defer progressTicker.Stop()
|
||||||
|
|
||||||
|
done := make(chan struct{})
|
||||||
|
defer close(done)
|
||||||
|
|
||||||
|
directories := make(chan string, 100000)
|
||||||
|
workerPool := make(chan struct{}, 4000)
|
||||||
|
directories <- input
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-progressTicker.C:
|
||||||
|
dirCount := len(directories)
|
||||||
|
workers := atomic.LoadInt32(&activeWorkers)
|
||||||
|
fmt.Printf("\rFiles processed: %8d; Folders processed: %8d; Active workers: %8d; Directory queue: %8d",
|
||||||
|
atomic.LoadInt32(&filesProcessed),
|
||||||
|
atomic.LoadInt32(&foldersProcessed),
|
||||||
|
workers,
|
||||||
|
dirCount)
|
||||||
|
case <-done:
|
||||||
|
// Final progress update
|
||||||
|
fmt.Printf("\nFiles processed: %8d; Folders processed: %8d; Completed successfully\n",
|
||||||
|
atomic.LoadInt32(&filesProcessed),
|
||||||
|
atomic.LoadInt32(&foldersProcessed))
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
allDone := make(chan struct{})
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
var wg sync.WaitGroup
|
||||||
|
|
||||||
|
go func() {
|
||||||
|
for {
|
||||||
|
if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
|
||||||
|
time.Sleep(10 * time.Millisecond)
|
||||||
|
if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
|
||||||
|
close(allDone)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
time.Sleep(50 * time.Millisecond)
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case directory, ok := <-directories:
|
||||||
|
if !ok {
|
||||||
|
wg.Wait()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
atomic.AddInt32(&activeWorkers, 1)
|
||||||
|
|
||||||
|
go func(dir string) {
|
||||||
|
workerPool <- struct{}{}
|
||||||
|
|
||||||
|
atomic.AddInt32(&foldersProcessed, 1)
|
||||||
|
processDirectory(dir, directories, output, &filesProcessed)
|
||||||
|
|
||||||
|
<-workerPool
|
||||||
|
atomic.AddInt32(&activeWorkers, -1)
|
||||||
|
}(directory)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
|
||||||
|
<-allDone
|
||||||
|
|
||||||
|
log.Printf("Files processed: %d; Folders processed: %d",
|
||||||
|
atomic.LoadInt32(&filesProcessed),
|
||||||
|
atomic.LoadInt32(&foldersProcessed))
|
||||||
|
}
|
||||||
|
|
||||||
|
func processDirectory(directory string, directories chan<- string, output chan<- string, filesProcessed *int32) {
|
||||||
|
files, err := os.ReadDir(directory)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("Error reading directory %s: %+v", directory, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, file := range files {
|
||||||
|
if file.IsDir() {
|
||||||
|
directories <- filepath.Join(directory, file.Name())
|
||||||
|
} else {
|
||||||
|
output <- filepath.Join(directory, file.Name())
|
||||||
|
atomic.AddInt32(filesProcessed, 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func NormalizePath(input, workdir string) string {
|
||||||
|
input = filepath.Clean(input)
|
||||||
|
input = filepath.ToSlash(input)
|
||||||
|
input = strings.ReplaceAll(input, "\"", "")
|
||||||
|
return input
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user