Compare commits
3 Commits
3f7fd36f84
...
c94a7ae8ab
Author | SHA1 | Date | |
---|---|---|---|
c94a7ae8ab | |||
ca57ee728e | |||
b53628e698 |
1
.gitignore
vendored
1
.gitignore
vendored
@@ -3,3 +3,4 @@
|
||||
cln
|
||||
cln.log
|
||||
.qodo
|
||||
*.log
|
||||
|
2
build.sh
Normal file
2
build.sh
Normal file
@@ -0,0 +1,2 @@
|
||||
GOOS=windows GOARCH=amd64 go build -o cln.exe .
|
||||
GOOS=linux GOARCH=amd64 go build -o cln .
|
59
main.go
59
main.go
@@ -7,7 +7,6 @@ import (
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
)
|
||||
@@ -20,7 +19,6 @@ const ImportantColor = BRed
|
||||
const DefaultColor = Reset
|
||||
const PathColor = Green
|
||||
|
||||
var FileRegex, _ = regexp.Compile(`sync\.ya?ml$`)
|
||||
var programName = os.Args[0]
|
||||
var undo = false
|
||||
|
||||
@@ -145,14 +143,19 @@ func ReadFromFilesRecursively(input string, output chan *LinkInstruction, status
|
||||
|
||||
files := make(chan string, 128)
|
||||
fileStatus := make(chan error)
|
||||
var wg sync.WaitGroup
|
||||
go GetSyncFilesRecursively(input, files, fileStatus)
|
||||
go func() {
|
||||
wg.Wait()
|
||||
close(files)
|
||||
}()
|
||||
|
||||
go func() {
|
||||
// Collect all files first
|
||||
var syncFiles []string
|
||||
for {
|
||||
file, ok := <-files
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
syncFiles = append(syncFiles, file)
|
||||
}
|
||||
|
||||
// Check for errors from file search
|
||||
for {
|
||||
err, ok := <-fileStatus
|
||||
if !ok {
|
||||
@@ -163,40 +166,26 @@ func ReadFromFilesRecursively(input string, output chan *LinkInstruction, status
|
||||
status <- err
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
file, ok := <-files
|
||||
if !ok {
|
||||
LogInfo("No more files to process")
|
||||
break
|
||||
}
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
LogInfo(file)
|
||||
// Process each file
|
||||
for _, file := range syncFiles {
|
||||
file = NormalizePath(file, workdir)
|
||||
LogInfo("Processing file: %s", FormatPathValue(file))
|
||||
|
||||
// This "has" to be done because instructions are resolved in relation to cwd
|
||||
fileDir := FileRegex.FindStringSubmatch(file)
|
||||
if fileDir == nil {
|
||||
LogError("Failed to extract directory from %s", FormatSourcePath(file))
|
||||
return
|
||||
}
|
||||
LogInfo("Changing directory to %s (for %s)",
|
||||
FormatPathValue(fileDir[1]),
|
||||
FormatPathValue(file))
|
||||
err := os.Chdir(fileDir[1])
|
||||
// Change to the directory containing the sync file
|
||||
fileDir := filepath.Dir(file)
|
||||
originalDir, _ := os.Getwd()
|
||||
err := os.Chdir(fileDir)
|
||||
if err != nil {
|
||||
LogError("Failed to change directory to %s: %v",
|
||||
FormatSourcePath(fileDir[1]), err)
|
||||
return
|
||||
LogError("Failed to change directory to %s: %v", FormatSourcePath(fileDir), err)
|
||||
continue
|
||||
}
|
||||
|
||||
// Read and process the file
|
||||
ReadFromFile(file, output, status, false)
|
||||
// Don't return directory, stay where we are
|
||||
os.Chdir(workdir)
|
||||
}()
|
||||
|
||||
// Return to original directory
|
||||
os.Chdir(originalDir)
|
||||
}
|
||||
}
|
||||
|
||||
|
@@ -16,8 +16,8 @@ fi
|
||||
echo "Tag: $TAG"
|
||||
|
||||
echo "Building the thing..."
|
||||
go build .
|
||||
go install .
|
||||
sh build.sh
|
||||
sh install.sh
|
||||
|
||||
echo "Creating a release..."
|
||||
TOKEN="$GITEA_API_KEY"
|
||||
|
26
synclib.log
26
synclib.log
@@ -1,26 +0,0 @@
|
||||
18:24:37.967093 Using default sync.yaml file
|
||||
18:24:37.967093 Input 'sync.yaml' is not absolute, prepending work dir '.'
|
||||
18:24:37.967593 Reading input from file: [0;32mC:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m
|
||||
18:24:37.967593 Parsing as YAML file
|
||||
18:24:37.967593 [4;31mFailed to parse YAML file [0;35mC:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m: [4;31merror unmarshaling YAML: yaml: unmarshal errors:
|
||||
line 1: cannot unmarshal !!seq into main.YAMLConfig[0m[0m
|
||||
18:24:37.967593 No more instructions to process
|
||||
18:24:37.968092 No instructions were processed
|
||||
18:27:59.691333 Using default sync.yaml file
|
||||
18:27:59.691333 Input 'sync.yaml' is not absolute, prepending work dir '.'
|
||||
18:27:59.691834 Reading input from file: [0;32mC:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m
|
||||
18:27:59.691834 Parsing as YAML file
|
||||
18:27:59.692335 [0;35mExpanding wildcard source \* in YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m
|
||||
18:27:59.692335 Expanded wildcard source [0;35m\*[0m to 0 links
|
||||
18:27:59.692836 Expanded wildcard source [0;35m\*[0m in YAML file [0;35mC:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m to 0 links
|
||||
18:27:59.692836 No more instructions to process
|
||||
18:27:59.692836 No instructions were processed
|
||||
18:28:04.075821 Using default sync.yaml file
|
||||
18:28:04.076320 Input 'sync.yaml' is not absolute, prepending work dir '.'
|
||||
18:28:04.076320 Reading input from file: [0;32mC:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m
|
||||
18:28:04.076320 Parsing as YAML file
|
||||
18:28:04.076320 [0;35mExpanding wildcard source \* in YAML file C:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m
|
||||
18:28:04.076821 Expanded wildcard source [0;35m\*[0m to 0 links
|
||||
18:28:04.076821 Expanded wildcard source [0;35m\*[0m in YAML file [0;35mC:/Users/Administrator/Seafile/Projects-Go/GoProjects/synclib/sync.yaml[0m to 0 links
|
||||
18:28:04.076821 No more instructions to process
|
||||
18:28:04.076821 No instructions were processed
|
117
util.go
117
util.go
@@ -5,9 +5,8 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"github.com/bmatcuk/doublestar/v4"
|
||||
)
|
||||
|
||||
func IsSymlink(path string) (bool, error) {
|
||||
@@ -76,112 +75,24 @@ func GetSyncFilesRecursively(input string, output chan string, status chan error
|
||||
defer close(output)
|
||||
defer close(status)
|
||||
|
||||
var filesProcessed int32
|
||||
var foldersProcessed int32
|
||||
var activeWorkers int32
|
||||
workdir, _ := os.Getwd()
|
||||
input = NormalizePath(input, workdir)
|
||||
LogInfo("Searching for sync files recursively starting in %s", FormatPathValue(input))
|
||||
|
||||
progressTicker := time.NewTicker(200 * time.Millisecond)
|
||||
defer progressTicker.Stop()
|
||||
|
||||
done := make(chan struct{})
|
||||
defer close(done)
|
||||
|
||||
directories := make(chan string, 100000)
|
||||
workerPool := make(chan struct{}, 4000)
|
||||
directories <- input
|
||||
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-progressTicker.C:
|
||||
dirCount := len(directories)
|
||||
workers := atomic.LoadInt32(&activeWorkers)
|
||||
fmt.Printf("\rFiles processed: %d; Folders processed: %d; Active workers: %d; Directory queue: %d",
|
||||
atomic.LoadInt32(&filesProcessed),
|
||||
atomic.LoadInt32(&foldersProcessed),
|
||||
workers,
|
||||
dirCount)
|
||||
case <-done:
|
||||
// Final progress update
|
||||
fmt.Printf("\nFiles processed: %d; Folders processed: %d; Completed successfully\n",
|
||||
atomic.LoadInt32(&filesProcessed),
|
||||
atomic.LoadInt32(&foldersProcessed))
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
allDone := make(chan struct{})
|
||||
|
||||
go func() {
|
||||
// WTF is this waitgroup?
|
||||
// Nowhere is it added...
|
||||
var wg sync.WaitGroup
|
||||
|
||||
go func() {
|
||||
for {
|
||||
if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
|
||||
time.Sleep(10 * time.Millisecond)
|
||||
if atomic.LoadInt32(&activeWorkers) == 0 && len(directories) == 0 {
|
||||
close(allDone)
|
||||
return
|
||||
}
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
}
|
||||
}()
|
||||
|
||||
for {
|
||||
select {
|
||||
case directory, ok := <-directories:
|
||||
if !ok {
|
||||
wg.Wait()
|
||||
return
|
||||
}
|
||||
|
||||
atomic.AddInt32(&activeWorkers, 1)
|
||||
|
||||
go func(dir string) {
|
||||
workerPool <- struct{}{}
|
||||
|
||||
atomic.AddInt32(&foldersProcessed, 1)
|
||||
processDirectory(dir, directories, output, &filesProcessed)
|
||||
|
||||
<-workerPool
|
||||
atomic.AddInt32(&activeWorkers, -1)
|
||||
}(directory)
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
<-allDone
|
||||
|
||||
if atomic.LoadInt32(&filesProcessed) > 0 {
|
||||
LogInfo("Files processed: %d; Folders processed: %d",
|
||||
atomic.LoadInt32(&filesProcessed),
|
||||
atomic.LoadInt32(&foldersProcessed))
|
||||
}
|
||||
}
|
||||
|
||||
func processDirectory(directory string, directories chan<- string, output chan<- string, filesProcessed *int32) {
|
||||
files, err := os.ReadDir(directory)
|
||||
// Use doublestar to find all sync.yml and sync.yaml files recursively
|
||||
pattern := "**/sync.y*ml"
|
||||
files, err := doublestar.Glob(os.DirFS(input), pattern)
|
||||
if err != nil {
|
||||
LogError("Error reading directory %s: %v", directory, err)
|
||||
LogError("Failed to search for pattern %s: %v", pattern, err)
|
||||
status <- err
|
||||
return
|
||||
}
|
||||
|
||||
for _, file := range files {
|
||||
if file.IsDir() {
|
||||
directories <- filepath.Join(directory, file.Name())
|
||||
} else {
|
||||
if IsYAMLSyncFile(file.Name()) {
|
||||
output <- filepath.Join(directory, file.Name())
|
||||
fullPath := filepath.Join(input, file)
|
||||
LogInfo("Found sync file: %s", FormatPathValue(fullPath))
|
||||
output <- fullPath
|
||||
}
|
||||
atomic.AddInt32(filesProcessed, 1)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func IsYAMLSyncFile(filename string) bool {
|
||||
return filename == "sync.yaml" || filename == "sync.yml"
|
||||
LogInfo("Completed recursive search for sync files")
|
||||
}
|
||||
|
Reference in New Issue
Block a user