More better rate limiting

downloader/download.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
 	"sync"
 
@@ -12,7 +13,7 @@ import (
 const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
 
 type DownloadWorker struct {
-	id int
+	id    int
 	input chan *DownloadTask
 }
 
@@ -23,7 +24,7 @@ var dl = ytdlp.New().
 	// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
 	FormatSort("res,ext:mp4:m4a").
 	Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
-	LimitRate("10M").
+	LimitRate(fmt.Sprintf("%dM", 100/DOWNLOAD_WORKERS)).
 	// HTTPChunkSize("20M").
 	MarkWatched().
 	SponsorblockMark("all").
@@ -56,13 +57,13 @@ func (w *DownloadWorker) Run() {
 		if err != nil {
 			log.Printf("Failed alerting with %+v", err)
 		}
-	
+
 		_, err = dl.Run(context.TODO(), task.Url)
 		if err != nil {
 			log.Printf("DownloadWorker %d: Failed downloading %s with %+v", w.id, task.Url, err)
 			continue
 		}
-	
+
 		log.Printf("DownloadWorker %d: Downloaded %s", w.id, task.Url)
 		ongoingDownloadsMutex.Lock()
 		delete(ongoingDownloads, task.Url)

downloader/main.go

@@ -2,12 +2,15 @@ package main
 
 import (
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
 	"net/http"
 	"os"
 )
 
+var Error *log.Logger
+var Warning *log.Logger
 func init() {
 	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
 	logFile, err := os.Create("main.log")
@@ -17,10 +20,16 @@ func init() {
 	}
 	logger := io.MultiWriter(os.Stdout, logFile)
 	log.SetOutput(logger)
+
+	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
+		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
+		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
 }
 
-const DOWNLOAD_WORKERS = 10
+const DOWNLOAD_WORKERS = 4
-
 var downloadQueue = make(chan *DownloadTask, 100)
 
 func enableCORS(next http.Handler) http.Handler {