Make parallel

main.go

@@ -12,6 +12,7 @@ import (
 	"strings"
 
 	logger "git.site.quack-lab.dev/dave/cylogger"
+	utils "git.site.quack-lab.dev/dave/cyutils"
 
 	"github.com/joho/godotenv"
 )
@@ -28,6 +29,8 @@ func main() {
 	envfile := flag.String("envfile", ".env", "")
 	inputfile := flag.String("if", "", "")
 	pocketbase := flag.Bool("pb", false, "")
+	batchsize := flag.Int("bs", 10, "")
+	workers := flag.Int("w", 30, "")
 	flag.Parse()
 	logger.InitFlag()
 
@@ -63,6 +66,9 @@ func main() {
 		logger.Error("No args specified, please pass space delimited list of workshop ids for downloading or use -pb to get ids from pocketbase")
 		return
 	}
+	toDownload := make([]string, 0, len(args)+len(pocketbaseids))
+	toDownload = append(toDownload, args...)
+	toDownload = append(toDownload, pocketbaseids...)
 
 	env, err := loadEnv(*envfile)
 	if err != nil {
@@ -84,12 +90,10 @@ func main() {
 	}
 	tempfile.Close()
 
-	steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
+	batches := make([][]string, 0, len(toDownload)/(*batchsize))
-	if !ok {
+	utils.Batched(toDownload, *batchsize, func(batch []string) {
-		steamcmdScriptPath = filepath.Join(steamcmdPath, "script")
+		batches = append(batches, batch)
-		logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+	})
-		logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
-	}
 
 	app, ok := env[appEnvKey]
 	if !ok {
@@ -109,18 +113,23 @@ func main() {
 		logger.Info("Password not found in env, using empty")
 	}
 
+	utils.WithWorkers(*workers, batches, func(worker int, batch []string) {
+		steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
+		if !ok {
+			steamcmdScriptPath = filepath.Join(steamcmdPath, fmt.Sprintf("script-%d.txt", worker))
+			logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+			logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
+		}
+
 		logger.Info("Using steamcmd at '%s'", steamcmdPath)
 		logger.Info("As user '%s'", username)
-	logger.Info("Downloading %d items for '%s'", len(args), app)
+		logger.Info("Downloading %d items for '%s'", len(batch), app)
 
 		scriptContents := make([]string, 0, len(args)+4)
 		scriptContents = append(scriptContents, "@ShutdownOnFailedCommand 0")
 		scriptContents = append(scriptContents, "@NoPromptForPassword 1")
 		scriptContents = append(scriptContents, fmt.Sprintf("login %s %s", username, password))
-	for _, arg := range args {
+		for _, id := range batch {
-		scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, arg))
-	}
-	for _, id := range pocketbaseids {
 			scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, id))
 		}
 		scriptContents = append(scriptContents, "quit")
@@ -136,21 +145,23 @@ func main() {
 			logger.Error("Could not write to steamcmd script file")
 			return
 		}
-	logger.Info("Wrote %d lines to script file", len(scriptContents))
+		logger.Info("Wrote %d lines to script file at '%s'", len(scriptContents), steamcmdScriptPath)
 
 		steamcmdExe, _ = filepath.Abs(steamcmdExe)
 		steamcmdScriptPath, _ = filepath.Abs(steamcmdScriptPath)
 		logger.Info("Running steamcmd at %s", steamcmdExe)
 		cmd := exec.Command(steamcmdExe, "+runscript", steamcmdScriptPath)
-	cmd.Dir = steamcmdPath
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-	cmd.Stdin = os.Stdin
+		cmd.Dir = steamcmdPath
 		err = cmd.Run()
 		if err != nil {
 			logger.Error("SteamCMD failed with code %v", err)
 			return
 		}
+		logger.Info("Batch %d finished", worker)
+	})
+
 	logger.Info("All done")
 }