Use new logger

avorion-ship-design-copy/main.go

@@ -2,35 +2,14 @@ package main
 
 import (
 	"fmt"
-	"io"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"sync"
+
+	logger "git.site.quack-lab.dev/dave/cylogger"
 )
 
-var Error *log.Logger
-var Warning *log.Logger
-
-func init() {
-	log.SetFlags(log.Ldate | log.Lshortfile)
-	logFile, err := os.Create("main.log")
-	if err != nil {
-		log.Printf("Error creating log file: %v", err)
-		os.Exit(1)
-	}
-	logger := io.MultiWriter(os.Stdout, logFile)
-	log.SetOutput(logger)
-
-	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
-		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
-		log.Ldate|log.Lshortfile)
-	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
-		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
-		log.Ldate|log.Lshortfile)
-}
-
 var steamWorkshopContentDir = `C:/Users/Administrator/Seafile/Projects-Go/GoProjects/steamcmd-api/steamcmd/steamapps/workshop/content/445220`
 var avorionRoaming = `C:/Users/Administrator/AppData/Roaming/Avorion`
 
@@ -38,7 +17,7 @@ func main() {
 	steamWorkshopContentDir, _ = filepath.Abs(steamWorkshopContentDir)
 	files, err := os.ReadDir(steamWorkshopContentDir)
 	if err != nil {
-		Error.Printf("Error reading steam workshop content dir: %v", err)
+		logger.Error("Error reading steam workshop content dir: %v", err)
 		return
 	}
 
@@ -57,14 +36,14 @@ func main() {
 			fileHandle, err := os.Open(modFile)
 			if err != nil {
 				if !os.IsNotExist(err) {
-					Error.Printf("Error opening mod file '%s': '%v'", modFile, err)
+					logger.Error("Error opening mod file '%s': '%v'", modFile, err)
 					return
 				} else {
-					log.Printf("Mod '%s' is a ship!", src)
+					logger.Info("Mod '%s' is a ship!", src)
 					dst := filepath.Join(avorionRoaming, "ships", "workshop", file.Name())
 					err2 := Copy(src, dst)
 					if err2 != nil {
-						Error.Printf("Error copying '%s' to '%s': '%v'", src, dst, err2)
+						logger.Error("Error copying '%s' to '%s': '%v'", src, dst, err2)
 						return
 					}
 
@@ -77,35 +56,35 @@ func main() {
 					turretFileDst := filepath.Join(avorionRoaming, "ships", "turrets", file.Name()+".turret.xml")
 
 					if FileExists(shipFileSrc) && !FileExists(shipFileDst) {
-						log.Printf("Copying '%s' to '%s'", shipFileSrc, shipFileDst)
+						logger.Info("Copying '%s' to '%s'", shipFileSrc, shipFileDst)
 						err := Copy(shipFileSrc, shipFileDst)
 						if err != nil {
-							Error.Printf("Error copying '%s' to '%s': '%v'", shipFileSrc, shipFileDst, err)
+							logger.Error("Error copying '%s' to '%s': '%v'", shipFileSrc, shipFileDst, err)
 							return
 						}
 					}
 
 					if FileExists(planFileSrc) && !FileExists(planFileDst) {
-						log.Printf("Copying '%s' to '%s'", planFileSrc, planFileDst)
+						logger.Info("Copying '%s' to '%s'", planFileSrc, planFileDst)
 						err := Copy(planFileSrc, planFileDst)
 						if err != nil {
-							Error.Printf("Error copying '%s' to '%s': '%v'", planFileSrc, planFileDst, err)
+							logger.Error("Error copying '%s' to '%s': '%v'", planFileSrc, planFileDst, err)
 							return
 						}
 					}
 
 					if FileExists(turretFileSrc) && !FileExists(turretFileDst) {
-						log.Printf("Copying '%s' to '%s'", turretFileSrc, turretFileDst)
+						logger.Info("Copying '%s' to '%s'", turretFileSrc, turretFileDst)
 						err := Copy(turretFileSrc, turretFileDst)
 						if err != nil {
-							Error.Printf("Error copying '%s' to '%s': '%v'", turretFileSrc, turretFileDst, err)
+							logger.Error("Error copying '%s' to '%s': '%v'", turretFileSrc, turretFileDst, err)
 							return
 						}
 					}
 				}
 			}
 			if err == nil {
-				log.Printf("'%s' is a mod, not a ship, skipping", src)
+				logger.Info("'%s' is a mod, not a ship, skipping", src)
 				fileHandle.Close()
 				return
 			}

main.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
+	utils "git.site.quack-lab.dev/dave/cyutils"
 
 	"github.com/joho/godotenv"
 )
@@ -28,6 +29,8 @@ func main() {
 	envfile := flag.String("envfile", ".env", "")
 	inputfile := flag.String("if", "", "")
 	pocketbase := flag.Bool("pb", false, "")
+	batchsize := flag.Int("bs", 10, "")
+	workers := flag.Int("w", 30, "")
 	flag.Parse()
 	logger.InitFlag()
 
@@ -63,6 +66,9 @@ func main() {
 		logger.Error("No args specified, please pass space delimited list of workshop ids for downloading or use -pb to get ids from pocketbase")
 		return
 	}
+	toDownload := make([]string, 0, len(args)+len(pocketbaseids))
+	toDownload = append(toDownload, args...)
+	toDownload = append(toDownload, pocketbaseids...)
 
 	env, err := loadEnv(*envfile)
 	if err != nil {
@@ -84,12 +90,10 @@ func main() {
 	}
 	tempfile.Close()
 
-	steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
-	if !ok {
-		steamcmdScriptPath = filepath.Join(steamcmdPath, "script")
-		logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
-		logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
-	}
+	batches := make([][]string, 0, len(toDownload)/(*batchsize))
+	utils.Batched(toDownload, *batchsize, func(batch []string) {
+		batches = append(batches, batch)
+	})
 
 	app, ok := env[appEnvKey]
 	if !ok {
@@ -109,48 +113,55 @@ func main() {
 		logger.Info("Password not found in env, using empty")
 	}
 
-	logger.Info("Using steamcmd at '%s'", steamcmdPath)
-	logger.Info("As user '%s'", username)
-	logger.Info("Downloading %d items for '%s'", len(args), app)
+	utils.WithWorkers(*workers, batches, func(worker int, batch []string) {
+		steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
+		if !ok {
+			steamcmdScriptPath = filepath.Join(steamcmdPath, fmt.Sprintf("script-%d.txt", worker))
+			logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+			logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
+		}
 
-	scriptContents := make([]string, 0, len(args)+4)
-	scriptContents = append(scriptContents, "@ShutdownOnFailedCommand 0")
-	scriptContents = append(scriptContents, "@NoPromptForPassword 1")
-	scriptContents = append(scriptContents, fmt.Sprintf("login %s %s", username, password))
-	for _, arg := range args {
-		scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, arg))
-	}
-	for _, id := range pocketbaseids {
-		scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, id))
-	}
-	scriptContents = append(scriptContents, "quit")
+		logger.Info("Using steamcmd at '%s'", steamcmdPath)
+		logger.Info("As user '%s'", username)
+		logger.Info("Downloading %d items for '%s'", len(batch), app)
 
-	scriptFileHandle, err := os.OpenFile(steamcmdScriptPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755)
-	if err != nil {
-		logger.Error("Could not open steamcmd script file")
-		return
-	}
-	defer scriptFileHandle.Close()
-	_, err = scriptFileHandle.Write([]byte(strings.Join(scriptContents, "\n")))
-	if err != nil {
-		logger.Error("Could not write to steamcmd script file")
-		return
-	}
-	logger.Info("Wrote %d lines to script file", len(scriptContents))
+		scriptContents := make([]string, 0, len(args)+4)
+		scriptContents = append(scriptContents, "@ShutdownOnFailedCommand 0")
+		scriptContents = append(scriptContents, "@NoPromptForPassword 1")
+		scriptContents = append(scriptContents, fmt.Sprintf("login %s %s", username, password))
+		for _, id := range batch {
+			scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, id))
+		}
+		scriptContents = append(scriptContents, "quit")
+
+		scriptFileHandle, err := os.OpenFile(steamcmdScriptPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755)
+		if err != nil {
+			logger.Error("Could not open steamcmd script file")
+			return
+		}
+		defer scriptFileHandle.Close()
+		_, err = scriptFileHandle.Write([]byte(strings.Join(scriptContents, "\n")))
+		if err != nil {
+			logger.Error("Could not write to steamcmd script file")
+			return
+		}
+		logger.Info("Wrote %d lines to script file at '%s'", len(scriptContents), steamcmdScriptPath)
+
+		steamcmdExe, _ = filepath.Abs(steamcmdExe)
+		steamcmdScriptPath, _ = filepath.Abs(steamcmdScriptPath)
+		logger.Info("Running steamcmd at %s", steamcmdExe)
+		cmd := exec.Command(steamcmdExe, "+runscript", steamcmdScriptPath)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Dir = steamcmdPath
+		err = cmd.Run()
+		if err != nil {
+			logger.Error("SteamCMD failed with code %v", err)
+			return
+		}
+		logger.Info("Batch %d finished", worker)
+	})
 
-	steamcmdExe, _ = filepath.Abs(steamcmdExe)
-	steamcmdScriptPath, _ = filepath.Abs(steamcmdScriptPath)
-	logger.Info("Running steamcmd at %s", steamcmdExe)
-	cmd := exec.Command(steamcmdExe, "+runscript", steamcmdScriptPath)
-	cmd.Dir = steamcmdPath
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Stdin = os.Stdin
-	err = cmd.Run()
-	if err != nil {
-		logger.Error("SteamCMD failed with code %v", err)
-		return
-	}
 	logger.Info("All done")
 }