Make parallel

main.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
+	utils "git.site.quack-lab.dev/dave/cyutils"
 
 	"github.com/joho/godotenv"
 )
@@ -28,6 +29,8 @@ func main() {
 	envfile := flag.String("envfile", ".env", "")
 	inputfile := flag.String("if", "", "")
 	pocketbase := flag.Bool("pb", false, "")
+	batchsize := flag.Int("bs", 10, "")
+	workers := flag.Int("w", 30, "")
 	flag.Parse()
 	logger.InitFlag()
 
@@ -63,6 +66,9 @@ func main() {
 		logger.Error("No args specified, please pass space delimited list of workshop ids for downloading or use -pb to get ids from pocketbase")
 		return
 	}
+	toDownload := make([]string, 0, len(args)+len(pocketbaseids))
+	toDownload = append(toDownload, args...)
+	toDownload = append(toDownload, pocketbaseids...)
 
 	env, err := loadEnv(*envfile)
 	if err != nil {
@@ -84,12 +90,10 @@ func main() {
 	}
 	tempfile.Close()
 
-	steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
+	batches := make([][]string, 0, len(toDownload)/(*batchsize))
-	if !ok {
+	utils.Batched(toDownload, *batchsize, func(batch []string) {
-		steamcmdScriptPath = filepath.Join(steamcmdPath, "script")
+		batches = append(batches, batch)
-		logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+	})
-		logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
-	}
 
 	app, ok := env[appEnvKey]
 	if !ok {
@@ -109,48 +113,55 @@ func main() {
 		logger.Info("Password not found in env, using empty")
 	}
 
-	logger.Info("Using steamcmd at '%s'", steamcmdPath)
+	utils.WithWorkers(*workers, batches, func(worker int, batch []string) {
-	logger.Info("As user '%s'", username)
+		steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
-	logger.Info("Downloading %d items for '%s'", len(args), app)
+		if !ok {
+			steamcmdScriptPath = filepath.Join(steamcmdPath, fmt.Sprintf("script-%d.txt", worker))
+			logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+			logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
+		}
 
-	scriptContents := make([]string, 0, len(args)+4)
+		logger.Info("Using steamcmd at '%s'", steamcmdPath)
-	scriptContents = append(scriptContents, "@ShutdownOnFailedCommand 0")
+		logger.Info("As user '%s'", username)
-	scriptContents = append(scriptContents, "@NoPromptForPassword 1")
+		logger.Info("Downloading %d items for '%s'", len(batch), app)
-	scriptContents = append(scriptContents, fmt.Sprintf("login %s %s", username, password))
-	for _, arg := range args {
-		scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, arg))
-	}
-	for _, id := range pocketbaseids {
-		scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, id))
-	}
-	scriptContents = append(scriptContents, "quit")
 
-	scriptFileHandle, err := os.OpenFile(steamcmdScriptPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755)
+		scriptContents := make([]string, 0, len(args)+4)
-	if err != nil {
+		scriptContents = append(scriptContents, "@ShutdownOnFailedCommand 0")
-		logger.Error("Could not open steamcmd script file")
+		scriptContents = append(scriptContents, "@NoPromptForPassword 1")
-		return
+		scriptContents = append(scriptContents, fmt.Sprintf("login %s %s", username, password))
-	}
+		for _, id := range batch {
-	defer scriptFileHandle.Close()
+			scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, id))
-	_, err = scriptFileHandle.Write([]byte(strings.Join(scriptContents, "\n")))
+		}
-	if err != nil {
+		scriptContents = append(scriptContents, "quit")
-		logger.Error("Could not write to steamcmd script file")
+
-		return
+		scriptFileHandle, err := os.OpenFile(steamcmdScriptPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755)
-	}
+		if err != nil {
-	logger.Info("Wrote %d lines to script file", len(scriptContents))
+			logger.Error("Could not open steamcmd script file")
+			return
+		}
+		defer scriptFileHandle.Close()
+		_, err = scriptFileHandle.Write([]byte(strings.Join(scriptContents, "\n")))
+		if err != nil {
+			logger.Error("Could not write to steamcmd script file")
+			return
+		}
+		logger.Info("Wrote %d lines to script file at '%s'", len(scriptContents), steamcmdScriptPath)
+
+		steamcmdExe, _ = filepath.Abs(steamcmdExe)
+		steamcmdScriptPath, _ = filepath.Abs(steamcmdScriptPath)
+		logger.Info("Running steamcmd at %s", steamcmdExe)
+		cmd := exec.Command(steamcmdExe, "+runscript", steamcmdScriptPath)
+		cmd.Stdout = os.Stdout
+		cmd.Stderr = os.Stderr
+		cmd.Dir = steamcmdPath
+		err = cmd.Run()
+		if err != nil {
+			logger.Error("SteamCMD failed with code %v", err)
+			return
+		}
+		logger.Info("Batch %d finished", worker)
+	})
 
-	steamcmdExe, _ = filepath.Abs(steamcmdExe)
-	steamcmdScriptPath, _ = filepath.Abs(steamcmdScriptPath)
-	logger.Info("Running steamcmd at %s", steamcmdExe)
-	cmd := exec.Command(steamcmdExe, "+runscript", steamcmdScriptPath)
-	cmd.Dir = steamcmdPath
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	cmd.Stdin = os.Stdin
-	err = cmd.Run()
-	if err != nil {
-		logger.Error("SteamCMD failed with code %v", err)
-		return
-	}
 	logger.Info("All done")
 }