Use new logger

avorion-ship-design-copy/main.go

@@ -2,35 +2,14 @@ package main
 
 import (
 	"fmt"
-	"io"
-	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"sync"
+
+	logger "git.site.quack-lab.dev/dave/cylogger"
 )
 
-var Error *log.Logger
-var Warning *log.Logger
-
-func init() {
-	log.SetFlags(log.Ldate | log.Lshortfile)
-	logFile, err := os.Create("main.log")
-	if err != nil {
-		log.Printf("Error creating log file: %v", err)
-		os.Exit(1)
-	}
-	logger := io.MultiWriter(os.Stdout, logFile)
-	log.SetOutput(logger)
-
-	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
-		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
-		log.Ldate|log.Lshortfile)
-	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
-		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
-		log.Ldate|log.Lshortfile)
-}
-
 var steamWorkshopContentDir = `C:/Users/Administrator/Seafile/Projects-Go/GoProjects/steamcmd-api/steamcmd/steamapps/workshop/content/445220`
 var avorionRoaming = `C:/Users/Administrator/AppData/Roaming/Avorion`
 
@@ -38,7 +17,7 @@ func main() {
 	steamWorkshopContentDir, _ = filepath.Abs(steamWorkshopContentDir)
 	files, err := os.ReadDir(steamWorkshopContentDir)
 	if err != nil {
-		Error.Printf("Error reading steam workshop content dir: %v", err)
+		logger.Error("Error reading steam workshop content dir: %v", err)
 		return
 	}
 
@@ -57,14 +36,14 @@ func main() {
 			fileHandle, err := os.Open(modFile)
 			if err != nil {
 				if !os.IsNotExist(err) {
-					Error.Printf("Error opening mod file '%s': '%v'", modFile, err)
+					logger.Error("Error opening mod file '%s': '%v'", modFile, err)
 					return
 				} else {
-					log.Printf("Mod '%s' is a ship!", src)
+					logger.Info("Mod '%s' is a ship!", src)
 					dst := filepath.Join(avorionRoaming, "ships", "workshop", file.Name())
 					err2 := Copy(src, dst)
 					if err2 != nil {
-						Error.Printf("Error copying '%s' to '%s': '%v'", src, dst, err2)
+						logger.Error("Error copying '%s' to '%s': '%v'", src, dst, err2)
 						return
 					}
 
@@ -77,35 +56,35 @@ func main() {
 					turretFileDst := filepath.Join(avorionRoaming, "ships", "turrets", file.Name()+".turret.xml")
 
 					if FileExists(shipFileSrc) && !FileExists(shipFileDst) {
-						log.Printf("Copying '%s' to '%s'", shipFileSrc, shipFileDst)
+						logger.Info("Copying '%s' to '%s'", shipFileSrc, shipFileDst)
 						err := Copy(shipFileSrc, shipFileDst)
 						if err != nil {
-							Error.Printf("Error copying '%s' to '%s': '%v'", shipFileSrc, shipFileDst, err)
+							logger.Error("Error copying '%s' to '%s': '%v'", shipFileSrc, shipFileDst, err)
 							return
 						}
 					}
 
 					if FileExists(planFileSrc) && !FileExists(planFileDst) {
-						log.Printf("Copying '%s' to '%s'", planFileSrc, planFileDst)
+						logger.Info("Copying '%s' to '%s'", planFileSrc, planFileDst)
 						err := Copy(planFileSrc, planFileDst)
 						if err != nil {
-							Error.Printf("Error copying '%s' to '%s': '%v'", planFileSrc, planFileDst, err)
+							logger.Error("Error copying '%s' to '%s': '%v'", planFileSrc, planFileDst, err)
 							return
 						}
 					}
 
 					if FileExists(turretFileSrc) && !FileExists(turretFileDst) {
-						log.Printf("Copying '%s' to '%s'", turretFileSrc, turretFileDst)
+						logger.Info("Copying '%s' to '%s'", turretFileSrc, turretFileDst)
 						err := Copy(turretFileSrc, turretFileDst)
 						if err != nil {
-							Error.Printf("Error copying '%s' to '%s': '%v'", turretFileSrc, turretFileDst, err)
+							logger.Error("Error copying '%s' to '%s': '%v'", turretFileSrc, turretFileDst, err)
 							return
 						}
 					}
 				}
 			}
 			if err == nil {
-				log.Printf("'%s' is a mod, not a ship, skipping", src)
+				logger.Info("'%s' is a mod, not a ship, skipping", src)
 				fileHandle.Close()
 				return
 			}

go.mod

@@ -1,5 +1,18 @@
 module steamcmd-api
 
-go 1.23.0
+go 1.23.6
 
-require github.com/joho/godotenv v1.5.1
+require (
+	git.site.quack-lab.dev/dave/cylogger v1.3.0
+	git.site.quack-lab.dev/dave/cyutils v1.0.0
+	github.com/joho/godotenv v1.5.1
+)
+
+require (
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/hexops/valast v1.5.0 // indirect
+	golang.org/x/mod v0.7.0 // indirect
+	golang.org/x/sys v0.3.0 // indirect
+	golang.org/x/tools v0.4.0 // indirect
+	mvdan.cc/gofumpt v0.4.0 // indirect
+)

go.sum

@@ -1,2 +1,32 @@
+git.site.quack-lab.dev/dave/cylogger v1.3.0 h1:eTWPUD+ThVi8kGIsRcE0XDeoH3yFb5miFEODyKUdWJw=
+git.site.quack-lab.dev/dave/cylogger v1.3.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk=
+git.site.quack-lab.dev/dave/cyutils v1.0.0 h1:yp/jkM2M7UZ+UIQuy+vPI7yDvTUdpbEdFL8h0lzUTvA=
+git.site.quack-lab.dev/dave/cyutils v1.0.0/go.mod h1:luGNFimplFhkpRLebhkVTNjG2wYfPAs+pu+UIMhBYbE=
+github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
+github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
+github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/hexops/autogold v0.8.1 h1:wvyd/bAJ+Dy+DcE09BoLk6r4Fa5R5W+O+GUzmR985WM=
+github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
+github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
+github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
+github.com/hexops/valast v1.5.0 h1:FBTuvVi0wjTngtXJRZXMbkN/Dn6DgsUsBwch2DUJU8Y=
+github.com/hexops/valast v1.5.0/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8=
+github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
+golang.org/x/mod v0.7.0 h1:LapD9S96VoQRhi/GrNTqeBJFrUjs5UHCAtTlgwA5oZA=
+golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
+golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ=
+golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/tools v0.4.0 h1:7mTAgkunk3fr4GAloyyCasadO6h9zSsQZbwvcaIciV4=
+golang.org/x/tools v0.4.0/go.mod h1:UE5sM2OK9E/d67R0ANs2xJizIymRP5gJU295PvKXxjQ=
+mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM=
+mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ=

main.go

@@ -1,34 +1,22 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"io"
-	"log"
+	"net/http"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 
+	logger "git.site.quack-lab.dev/dave/cylogger"
+	utils "git.site.quack-lab.dev/dave/cyutils"
+
 	"github.com/joho/godotenv"
 )
 
-var Error *log.Logger
-var Warning *log.Logger
-
-func init() {
-	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
-	logger := io.MultiWriter(os.Stdout)
-	log.SetOutput(logger)
-
-	Error = log.New(io.MultiWriter(os.Stderr, os.Stdout),
-		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
-		log.Lmicroseconds|log.Lshortfile)
-	Warning = log.New(io.MultiWriter(os.Stdout),
-		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
-		log.Lmicroseconds|log.Lshortfile)
-}
-
 const (
 	steamcmdEnvKey           = "STEAMCMD"
 	steamcmdScriptPathEnvKey = "STEAMCMD_SCRIPT"
@@ -40,122 +28,148 @@ const (
 func main() {
 	envfile := flag.String("envfile", ".env", "")
 	inputfile := flag.String("if", "", "")
+	pocketbase := flag.Bool("pb", false, "")
+	batchsize := flag.Int("bs", 10, "")
+	workers := flag.Int("w", 30, "")
 	flag.Parse()
+	logger.InitFlag()
+
+	pocketbaseids := make([]string, 0)
+	if *pocketbase {
+		ids, err := FromPocketbase(pocketbaseUrl, "445220")
+		if err != nil {
+			logger.Error("Error getting workshop ids: %v", err)
+			return
+		}
+		logger.Info("Got %d workshop ids", len(ids))
+		pocketbaseids = append(pocketbaseids, ids...)
+	}
 
 	args := flag.Args()
 	if *inputfile != "" {
 		filehandle, err := os.Open(*inputfile)
 		if err != nil {
-			Error.Printf("Error opening input file: %v", err)
+			logger.Error("Error opening input file: %v", err)
 			return
 		}
 		defer filehandle.Close()
 		fargs, err := io.ReadAll(filehandle)
 		if err != nil {
-			Error.Printf("Error reading input file: %v", err)
+			logger.Error("Error reading input file: %v", err)
 			return
 		}
 		sargs := strings.Split(string(fargs), "\r\n")
 		args = append(args, sargs...)
 	}
 
-	if len(args) == 0 {
+	if len(args) == 0 && len(pocketbaseids) == 0 {
-		Error.Println("No args specified, please pass space delimited list of workshop ids for downloading")
+		logger.Error("No args specified, please pass space delimited list of workshop ids for downloading or use -pb to get ids from pocketbase")
 		return
 	}
+	toDownload := make([]string, 0, len(args)+len(pocketbaseids))
+	toDownload = append(toDownload, args...)
+	toDownload = append(toDownload, pocketbaseids...)
 
 	env, err := loadEnv(*envfile)
 	if err != nil {
-		Error.Printf("Error leading env file: %v", err)
+		logger.Error("Error leading env file: %v", err)
 		return
 	}
 
 	steamcmdPath, ok := env[steamcmdEnvKey]
 	if !ok {
-		Error.Printf("SteamCMD not found in env, please specify '%s'", steamcmdEnvKey)
+		logger.Error("SteamCMD not found in env, please specify '%s'", steamcmdEnvKey)
 		return
 	}
 	steamcmdPath = filepath.Clean(steamcmdPath)
 	steamcmdExe := filepath.Join(steamcmdPath, "steamcmd.exe")
 	tempfile, err := os.Open(steamcmdExe)
 	if err != nil {
-		Error.Printf("Error opening SteamCMD, does it exist?: %v", err)
+		logger.Error("Error opening SteamCMD, does it exist?: %v", err)
 		return
 	}
 	tempfile.Close()
 
-	steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
+	batches := make([][]string, 0, len(toDownload)/(*batchsize))
-	if !ok {
+	utils.Batched(toDownload, *batchsize, func(batch []string) {
-		steamcmdScriptPath = filepath.Join(steamcmdPath, "script")
+		batches = append(batches, batch)
-		log.Printf("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+	})
-		log.Printf("Specify script path with '%s'", steamcmdScriptPathEnvKey)
-	}
 
 	app, ok := env[appEnvKey]
 	if !ok {
-		Error.Printf("App not found in env, please specify '%s'", appEnvKey)
+		logger.Error("App not found in env, please specify '%s'", appEnvKey)
 		return
 	}
 
 	username, ok := env[usernameEnvKey]
 	if !ok {
 		username = "anonymous"
-		log.Printf("Username not found in env, using '%s'", username)
+		logger.Info("Username not found in env, using '%s'", username)
-		log.Printf("If you want to log in specify '%s' and '%s'", usernameEnvKey, passwordEnvKey)
+		logger.Info("If you want to log in specify '%s' and '%s'", usernameEnvKey, passwordEnvKey)
 	}
 	password, ok := env[passwordEnvKey]
 	if !ok {
 		password = ""
-		log.Printf("Password not found in env, using empty")
+		logger.Info("Password not found in env, using empty")
 	}
 
-	log.Printf("Using steamcmd at '%s'", steamcmdPath)
+	utils.WithWorkers(*workers, batches, func(worker int, batch []string) {
-	log.Printf("As user '%s'", username)
+		steamcmdScriptPath, ok := env[steamcmdScriptPathEnvKey]
-	log.Printf("Downloading %d items for '%s'", len(args), app)
+		if !ok {
+			steamcmdScriptPath = filepath.Join(steamcmdPath, fmt.Sprintf("script-%d.txt", worker))
+			logger.Info("SteamCMD script not found in env, using '%s'", steamcmdScriptPath)
+			logger.Info("Specify script path with '%s'", steamcmdScriptPathEnvKey)
+		}
+
+		logger.Info("Using steamcmd at '%s'", steamcmdPath)
+		logger.Info("As user '%s'", username)
+		logger.Info("Downloading %d items for '%s'", len(batch), app)
 
 		scriptContents := make([]string, 0, len(args)+4)
 		scriptContents = append(scriptContents, "@ShutdownOnFailedCommand 0")
 		scriptContents = append(scriptContents, "@NoPromptForPassword 1")
 		scriptContents = append(scriptContents, fmt.Sprintf("login %s %s", username, password))
-	for _, arg := range args {
+		for _, id := range batch {
-		scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, arg))
+			scriptContents = append(scriptContents, fmt.Sprintf("workshop_download_item %s %s", app, id))
 		}
 		scriptContents = append(scriptContents, "quit")
 
 		scriptFileHandle, err := os.OpenFile(steamcmdScriptPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0755)
 		if err != nil {
-		Error.Println("Could not open steamcmd script file")
+			logger.Error("Could not open steamcmd script file")
 			return
 		}
 		defer scriptFileHandle.Close()
 		_, err = scriptFileHandle.Write([]byte(strings.Join(scriptContents, "\n")))
 		if err != nil {
-		Error.Println("Could not write to steamcmd script file")
+			logger.Error("Could not write to steamcmd script file")
 			return
 		}
-	log.Printf("Wrote %d lines to script file", len(scriptContents))
+		logger.Info("Wrote %d lines to script file at '%s'", len(scriptContents), steamcmdScriptPath)
 
 		steamcmdExe, _ = filepath.Abs(steamcmdExe)
 		steamcmdScriptPath, _ = filepath.Abs(steamcmdScriptPath)
-	log.Printf("Running steamcmd at %s", steamcmdExe)
+		logger.Info("Running steamcmd at %s", steamcmdExe)
 		cmd := exec.Command(steamcmdExe, "+runscript", steamcmdScriptPath)
-	cmd.Dir = steamcmdPath
 		cmd.Stdout = os.Stdout
 		cmd.Stderr = os.Stderr
-	cmd.Stdin = os.Stdin
+		cmd.Dir = steamcmdPath
 		err = cmd.Run()
 		if err != nil {
-		Error.Printf("SteamCMD failed with code %v", err)
+			logger.Error("SteamCMD failed with code %v", err)
 			return
 		}
-	log.Printf("All done")
+		logger.Info("Batch %d finished", worker)
+	})
+
+	logger.Info("All done")
 }
 
 func loadEnv(fpath string) (map[string]string, error) {
 	res := make(map[string]string)
 
 	fpath = filepath.Clean(fpath)
-	log.Printf("Trying to open env file at '%s'", fpath)
+	logger.Info("Trying to open env file at '%s'", fpath)
 
 	file, err := os.Open(fpath)
 	if err != nil {
@@ -170,3 +184,60 @@ func loadEnv(fpath string) (map[string]string, error) {
 
 	return res, nil
 }
+
+type PocketbaseResponse struct {
+	Page       int64          `json:"page"`
+	PerPage    int64          `json:"perPage"`
+	TotalPages int64          `json:"totalPages"`
+	TotalItems int64          `json:"totalItems"`
+	Items      []WorkshopItem `json:"items"`
+}
+
+type WorkshopItem struct {
+	CollectionID   string `json:"collectionId"`
+	CollectionName string `json:"collectionName"`
+	ID             string `json:"id"`
+	Gameid         string `json:"gameid"`
+	Workshopid     string `json:"workshopid"`
+	Title          string `json:"title"`
+	Link           string `json:"link"`
+	LastUpdated    string `json:"lastUpdated"`
+	Subscribed     string `json:"subscribed"`
+	Created        string `json:"created"`
+	Updated        string `json:"updated"`
+}
+
+var pocketbaseUrl = "https://pocketbase-scratch.site.quack-lab.dev/api/collections/steam_workshop/records"
+
+func FromPocketbase(url string, appId string) ([]string, error) {
+	url = fmt.Sprintf("%s?filter=(gameid='%s')&fields=workshopid&perPage=1000", pocketbaseUrl, appId)
+
+	resp, err := http.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+
+	var res PocketbaseResponse
+	err = json.Unmarshal(body, &res)
+	if err != nil {
+		return nil, err
+	}
+
+	// Deduplicate
+	ids := make(map[string]struct{})
+	for _, entry := range res.Items {
+		ids[entry.Workshopid] = struct{}{}
+	}
+	idsarr := make([]string, 0, len(ids))
+	for id := range ids {
+		idsarr = append(idsarr, id)
+	}
+
+	return idsarr, nil
+}