Refactor sanitization

That works a lot faster, pog

.gitignore

@@ -3,3 +3,7 @@ main.exe
 logs.log
 ws-server/deploy.tar
 downloader/main.log
+downloader/vendor/golang.org
+downloader/vendor/github.com/*
+!downloader/vendor/github.com/kkdai
+downloader/vendor/modules.txt

dl/dl.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 )
 
-const URL = `http://localhost:5000/download`
+const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
 
 type Item struct {
 	Link string `json:"link"`

downloader/download.go

@@ -2,70 +2,55 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"log"
-	"sync"
 
 	"github.com/gen2brain/beeep"
 	"github.com/lrstanley/go-ytdlp"
 )
 
-const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
-
-type DownloadWorker struct {
-	id int
-	input chan *DownloadTask
-}
-
-var ongoingDownloads = make(map[string]struct{})
-var ongoingDownloadsMutex = &sync.Mutex{}
 
 var dl = ytdlp.New().
 	// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
 	FormatSort("res,ext:mp4:m4a").
 	Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
-	LimitRate("10M").
+	LimitRate(fmt.Sprintf("%dM", 150/DOWNLOAD_WORKERS)).
 	// HTTPChunkSize("20M").
 	MarkWatched().
 	SponsorblockMark("all").
 	RecodeVideo("mp4").
 	ConcurrentFragments(6)
 
-func (w *DownloadWorker) Run() {
+func Download(url string) error {
-	for {
+	_, ongoing := ongoingDownloads[url]
-		task, ok := <-w.input
-		if !ok {
-			log.Printf("DownloadWorker %d: input channel closed, exiting", w.id)
-			return
-		}
-		_, ongoing := ongoingDownloads[task.Url]
 	if ongoing {
-			log.Printf("DownloadWorker %d: Download %s is already ongoing", w.id, task.Url)
+		// return fmt.Errorf("Download %s is already ongoing", url)
-			continue
+		Warning.Printf("Download %s is already ongoing", url)
+		return nil
 	}
 	ongoingDownloadsMutex.Lock()
-		ongoingDownloads[task.Url] = struct{}{}
+	ongoingDownloads[url] = struct{}{}
 	ongoingDownloadsMutex.Unlock()
 
-		log.Printf("DownloadWorker %d: Downloading %s", w.id, task.Url)
+	log.Printf("Downloading %s", url)
 
 	err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
 	if err != nil {
-			log.Printf("Failed beeping with %+v", err)
+		Warning.Printf("Failed beeping with %+v", err)
 	}
-		err = beeep.Alert("Download Started", task.Url, "assets/information.png")
+	err = beeep.Alert("Download Started", url, "assets/information.png")
 	if err != nil {
-			log.Printf("Failed alerting with %+v", err)
+		Warning.Printf("Failed alerting with %+v", err)
 	}
 
-		_, err = dl.Run(context.TODO(), task.Url)
+	_, err = dl.Run(context.TODO(), url)
 	if err != nil {
-			log.Printf("DownloadWorker %d: Failed downloading %s with %+v", w.id, task.Url, err)
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
-			continue
 	}
 
-		log.Printf("DownloadWorker %d: Downloaded %s", w.id, task.Url)
+	log.Printf("Downloaded %s", url)
 	ongoingDownloadsMutex.Lock()
-		delete(ongoingDownloads, task.Url)
+	delete(ongoingDownloads, url)
 	ongoingDownloadsMutex.Unlock()
-	}
+	return nil
 }

downloader/download_downloadr.go

@@ -0,0 +1,88 @@
+package main
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/gen2brain/beeep"
+	"github.com/kkdai/youtube/v2"
+	ytd "github.com/kkdai/youtube/v2/downloader"
+)
+
+func DownloadR(url string) error {
+	_, ongoing := ongoingDownloads[url]
+	if ongoing {
+		// return fmt.Errorf("Download %s is already ongoing", url)
+		Warning.Printf("Download %s is already ongoing", url)
+		return nil
+	}
+	ongoingDownloadsMutex.Lock()
+	ongoingDownloads[url] = struct{}{}
+	ongoingDownloadsMutex.Unlock()
+	defer func() {
+		ongoingDownloadsMutex.Lock()
+		delete(ongoingDownloads, url)
+		ongoingDownloadsMutex.Unlock()
+	}()
+
+	log.Printf("Downloading %s", url)
+
+	err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
+	if err != nil {
+		Warning.Printf("Failed beeping with %+v", err)
+	}
+	err = beeep.Alert("Download Started", url, "assets/information.png")
+	if err != nil {
+		Warning.Printf("Failed alerting with %+v", err)
+	}
+
+	client := youtube.Client{}
+	video, err := client.GetVideo(url)
+	if err != nil {
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
+	}
+	// $ go run . download -m mp4 -q hd https://www.youtube.com/watch?v=D9trBXaXCgA
+	// This works fine, now I have to figure out how to plug -m and -q into this shit below
+	downloader := ytd.Downloader{
+		OutputDir: OUTPUT_DIR,
+		Client:    client,
+	}
+
+	videoTitle := Sanitize(video.Title)
+	videoAuthor := Sanitize(video.Author)
+
+	videoFileRoot := filepath.Join(OUTPUT_DIR, videoAuthor)
+	err = os.MkdirAll(videoFileRoot, 0755)
+	if err != nil {
+		return fmt.Errorf("failed creating directory %s with %+v", videoFileRoot, err)
+	}
+
+	fullVideoPath := filepath.Join(OUTPUT_DIR, videoAuthor, videoTitle+".mp4")
+	_, err = os.Open(fullVideoPath)
+	if err == nil {
+		log.Printf("File %s already exists, skipping download", fullVideoPath)
+		return nil
+	}
+
+	videoFile := filepath.Join(videoAuthor, videoTitle+".mp4")
+	err = downloader.DownloadComposite(context.Background(), videoFile, video, "hd", "mp4", "")
+	if err != nil {
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
+	}
+
+	log.Printf("Downloaded %s", url)
+	return nil
+}
+
+func Sanitize(s string) string {
+	s = strings.ReplaceAll(s, "&", "and")
+	s = strings.ReplaceAll(s, "?", "")
+	s = strings.ReplaceAll(s, ":", "")
+	s = strings.ReplaceAll(s, ";", "")
+	s = strings.ReplaceAll(s, ",", "")
+	return s
+}

downloader/go.mod

@@ -4,17 +4,31 @@ go 1.22.4
 
 require (
 	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
+	github.com/bitly/go-simplejson v0.5.1 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 // indirect
+	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
 	github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
 	github.com/godbus/dbus/v5 v5.1.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
+	github.com/mattn/go-runewidth v0.0.15 // indirect
 	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
 	github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
+	github.com/vbauerster/mpb/v5 v5.4.0 // indirect
 	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 )
 
 require (
 	github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4
 	github.com/gorilla/websocket v1.5.3
+	github.com/kkdai/youtube/v2 v2.10.1
 	github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99
+	github.com/nsqio/go-nsq v1.1.0
 )

downloader/go.sum

@@ -1,23 +1,76 @@
 github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
 github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
+github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
 github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
+github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
+github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
 github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
 github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
 github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
+github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 h1:O7I1iuzEA7SG+dK8ocOBSlYAA9jBUmCYl/Qa7ey7JAM=
+github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
+github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
 github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI=
 github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
+github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
 github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 h1:qZNfIGkIANxGv/OqtnntR4DfOY2+BgwR60cAcu/i3SE=
 github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4/go.mod h1:kW3HQ4UdaAyrUCSSDR4xUzBKW6O2iA4uHhk7AtyYp10=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
+github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
+github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/kkdai/youtube/v2 v2.10.1 h1:jdPho4R7VxWoRi9Wx4ULMq4+hlzSVOXxh4Zh83f2F9M=
+github.com/kkdai/youtube/v2 v2.10.1/go.mod h1:qL8JZv7Q1IoDs4nnaL51o/hmITXEIvyCIXopB0oqgVM=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 h1:ZAo7qJht9PqefOD7C0ZKQ8dEkpJeM955sYw0FtQnzvo=
 github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
+github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
+github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
 github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
+github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
+github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
 github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk=
 github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o=
+github.com/vbauerster/mpb/v5 v5.4.0 h1:n8JPunifvQvh6P1D1HAl2Ur9YcmKT1tpoUuiea5mlmg=
+github.com/vbauerster/mpb/v5 v5.4.0/go.mod h1:fi4wVo7BVQ22QcvFObm+VwliQXlV1eBT8JDaKXR4JGI=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -33,12 +86,16 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug
 golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
+golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
+golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -54,12 +111,24 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
 golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
 golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
 golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

downloader/main.go

@@ -1,13 +1,22 @@
 package main
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"log"
-	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/nsqio/go-nsq"
 )
 
+var Error *log.Logger
+var Warning *log.Logger
+
 func init() {
 	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
 	logFile, err := os.Create("main.log")
@@ -17,54 +26,83 @@ func init() {
 	}
 	logger := io.MultiWriter(os.Stdout, logFile)
 	log.SetOutput(logger)
+
+	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
+		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
+		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
 }
 
-const DOWNLOAD_WORKERS = 10
+const DOWNLOAD_WORKERS = 2
 
-var downloadQueue = make(chan *DownloadTask, 100)
+type DLHandler struct{}
 
-func enableCORS(next http.Handler) http.Handler {
+func (*DLHandler) HandleMessage(message *nsq.Message) error {
-	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+	log.Printf("Received message '%s' with %d attempts", message.Body, message.Attempts)
-		w.Header().Set("Access-Control-Allow-Origin", "*")
+	data := DownloadRequest{}
-		w.Header().Set("Access-Control-Allow-Methods", "GET, POST, OPTIONS")
+	err := json.Unmarshal(message.Body, &data)
-		w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
+	if err != nil {
+		Error.Printf("Error unmarshalling message: %v", err)
+		return err
+	}
 
-		if r.Method == http.MethodOptions {
+	ctx, cancel := context.WithCancel(context.Background())
-			w.WriteHeader(http.StatusOK)
+	defer cancel()
+
+	go func() {
+		ticker := time.NewTicker(1 * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				message.Touch()
+			case <-ctx.Done():
 				return
 			}
-		next.ServeHTTP(w, r)
-	})
 		}
+	}()
 
-func handleDownload(w http.ResponseWriter, r *http.Request) {
+	err = DownloadR(data.Link)
-	if r.Method != http.MethodPost {
+	if err != nil {
-		http.Error(w, "Invalid request method", http.StatusMethodNotAllowed)
+		Error.Printf("Error downloading %s: %v", data.Link, err)
-		return
+		return err
 	}
-
+	message.Finish()
-	var req DownloadRequest
+	return nil
-	if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
-		log.Printf("Error parsing JSON: %v", err)
-		http.Error(w, "Error parsing JSON", http.StatusBadRequest)
-		return
-	}
-
-	downloadQueue <- &DownloadTask{Url: req.Link}
-	w.WriteHeader(http.StatusOK)
 }
 
 func main() {
+	config := nsq.NewConfig()
+	config.MaxAttempts = 5
+	config.MaxInFlight = DOWNLOAD_WORKERS
+	config.MsgTimeout = 10 * time.Second
+
+	consumer, err := nsq.NewConsumer("ytdqueue", "dl", config)
+	if err != nil {
+		Error.Printf("Error creating consumer: %v", err)
+		return
+	}
 	for i := 0; i < DOWNLOAD_WORKERS; i++ {
-		worker := &DownloadWorker{id: i, input: downloadQueue}
+		consumer.AddHandler(&DLHandler{})
-		go worker.Run()
 	}
 
-	mux := http.NewServeMux()
+	err = consumer.ConnectToNSQD("nsq.site.quack-lab.dev:41505")
-	mux.Handle("/download", enableCORS(http.HandlerFunc(handleDownload)))
-	log.Println("Server starting on :5000")
-	err := http.ListenAndServe(":5000", mux)
 	if err != nil {
-		log.Println("Error starting server:", err)
+		Error.Printf("Error connecting to nsqlookupd: %v", err)
+		return
 	}
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	<-sigChan
+	log.Println("Received signal to terminate. Initiating graceful shutdown...")
+
+	consumer.Stop()
+	<-consumer.StopChan
+
+	log.Println("Graceful shutdown completed.")
 }

downloader/types.go

@@ -1,8 +1,5 @@
 package main
 
-type DownloadTask struct {
-	Url string
-}
 type DownloadRequest struct {
 	Link string `json:"link"`
 }

downloader/utils.go

@@ -0,0 +1,8 @@
+package main
+
+import "sync"
+
+const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
+
+var ongoingDownloads = make(map[string]struct{})
+var ongoingDownloadsMutex = &sync.Mutex{}

downloader/vendor/github.com/kkdai/youtube/v2/.gitignore

@@ -0,0 +1,8 @@
+/.idea
+/.vscode
+download_test
+/bin
+/dist
+/output
+*.out
+.DS_Store

downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml

@@ -0,0 +1,8 @@
+issues:
+  exclude:
+  - Subprocess launched with function call
+  # Excluding configuration per-path, per-linter, per-text and per-source
+  exclude-rules:
+    # Exclude some linters from running on tests files.
+    - path: cmd/
+      text: InsecureSkipVerify

downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml

@@ -0,0 +1,20 @@
+project_name: youtubedr
+
+env:
+  # Build without CGO to don't depend on specific glibc versions
+  - CGO_ENABLED=0
+
+builds:
+  - main: ./cmd/youtubedr
+    binary: youtubedr
+    goos:
+      - windows
+      - darwin
+      - linux
+      - freebsd
+    goarch:
+      - amd64
+      - arm
+      - arm64
+    flags:
+      - -trimpath

downloader/vendor/github.com/kkdai/youtube/v2/LICENSE

@@ -0,0 +1,22 @@
+The MIT License (MIT)
+
+Copyright (c) 2015 Evan Lin
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+

downloader/vendor/github.com/kkdai/youtube/v2/Makefile

@@ -0,0 +1,57 @@
+FILES_TO_FMT      ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print)
+LOGLEVEL	?= debug
+
+## help: Show makefile commands
+.PHONY: help
+help: Makefile
+	@echo "---- Project: kkdai/youtube ----"
+	@echo " Usage: make COMMAND"
+	@echo
+	@echo " Management Commands:"
+	@sed -n 's/^##//p' $< | column -t -s ':' |  sed -e 's/^/ /'
+	@echo
+
+## build: Build project
+.PHONY: build
+build:
+	goreleaser --rm-dist
+
+## deps: Ensures fresh go.mod and go.sum
+.PHONY: deps
+deps:
+	go mod tidy
+	go mod verify
+
+## lint: Run golangci-lint check
+.PHONY: lint
+lint:
+	command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
+	echo "golangci-lint checking..."
+	golangci-lint run --deadline=30m --enable=misspell --enable=gosec --enable=gofmt --enable=goimports --enable=revive ./cmd/... ./...
+	go vet ./...
+
+## format: Formats Go code
+.PHONY: format
+format:
+	@echo ">> formatting code"
+	@gofmt -s -w $(FILES_TO_FMT)
+
+## test-unit: Run all Youtube Go unit tests
+.PHONY: test-unit
+test-unit:
+	LOGLEVEL=${LOGLEVEL} go test -v -cover ./...
+
+## test-integration: Run all Youtube Go integration tests
+.PHONY: test-integration
+test-integration:
+	mkdir -p output
+	rm -f output/*
+	LOGLEVEL=${LOGLEVEL} ARTIFACTS=output go test -v -race -covermode=atomic -coverprofile=coverage.out -tags=integration ./...
+
+.PHONY: coverage.out
+coverage.out:
+
+## clean: Clean files and downloaded videos from builds during development
+.PHONY: clean
+clean:
+	rm -rf dist *.mp4 *.mkv

downloader/vendor/github.com/kkdai/youtube/v2/README.md

@@ -0,0 +1,161 @@
+Download Youtube Video in Golang
+==================
+
+[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/kkdai/youtube/master/LICENSE)
+[![Go Reference](https://pkg.go.dev/badge/github.com/kkdai/youtube.svg)](https://pkg.go.dev/github.com/kkdai/youtube/v2)
+[![Build Status](https://github.com/kkdai/youtube/workflows/go/badge.svg?branch=master)](https://github.com/kkdai/youtube/actions)
+[![Coverage](https://codecov.io/gh/kkdai/youtube/branch/master/graph/badge.svg)](https://codecov.io/gh/kkdai/youtube)
+[![](https://goreportcard.com/badge/github.com/kkdai/youtube)](https://goreportcard.com/badge/github.com/kkdai/youtube)
+
+
+This package is a Youtube video download package, for more detail refer [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) for more download options.
+
+This tool is meant to be used to download CC0 licenced content, we do not support nor recommend using it for illegal activities.
+
+## Overview
+  * [Install](#installation)
+  * [Usage](#usage)
+  * [Example: Download video from \[dotGo 2015 - Rob Pike - Simplicity is Complicated\]](#download-dotGo-2015-rob-pike-video)
+
+## Installation
+
+### Install via go get
+
+Please ensure you have installed Go 1.21 or later.
+
+```shell
+go get github.com/kkdai/youtube/v2
+```
+
+### From source code
+
+```shell
+git clone https://github.com/kkdai/youtube.git
+cd youtube
+go run ./cmd/youtubedr
+```
+
+### Mac
+
+```shell
+brew install youtubedr
+```
+
+### in Termux
+```shell
+pkg install youtubedr
+```
+###  You can also find this package in
+- [archlinux](https://aur.archlinux.org/packages/youtubedr/)  (thanks to [cjsthompson](https://github.com/cjsthompson))
+- [Termux package](https://github.com/termux/termux-packages/tree/master/packages/youtubedr) (thanks to [kcubeterm](https://github.com/kcubeterm))
+- [Homebrew](https://formulae.brew.sh/formula/youtubedr) (thanks to [kkc](https://github.com/kkc))
+
+## Usage
+
+### Use the binary directly
+It's really simple to use, just get the video id from youtube url - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
+
+```shell
+$ youtubedr download QAGDGja7kbs
+$ youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
+```
+
+
+### Use this package in your golang program
+
+Please check out the [example_test.go](example_test.go) for example code.
+
+
+## Example:
+ * ### Get information of dotGo-2015-rob-pike video for downloading
+
+    `go get github.com/kkdai/youtube/v2/youtubedr`
+
+    Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
+
+    ```
+    youtubedr info https://www.youtube.com/watch?v=rFejpH_tAHM
+
+   Title: dotGo 2015 - Rob Pike - Simplicity is Complicated
+   Author: dotconferences
+   -----available streams-----
+   itag:  18 , quality: medium , type: video/mp4; codecs="avc1.42001E, mp4a.40.2"
+   itag:  22 , quality:  hd720 , type: video/mp4; codecs="avc1.64001F, mp4a.40.2"
+   itag: 137 , quality: hd1080 , type: video/mp4; codecs="avc1.640028"
+   itag: 248 , quality: hd1080 , type: video/webm; codecs="vp9"
+   ........
+    ```
+ * ### Download dotGo-2015-rob-pike-video
+
+    `go get github.com/kkdai/youtube/v2/youtubedr`
+
+    Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
+
+    ```
+    youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
+    ```
+
+ * ### Download video to specific folder and name
+
+	`go get github.com/kkdai/youtube/v2/youtubedr`
+
+	Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) to current directory and name the file to simplicity-is-complicated.mp4
+
+	```
+	youtubedr download -d ./ -o simplicity-is-complicated.mp4 https://www.youtube.com/watch?v=rFejpH_tAHM
+	```
+
+ * ### Download video with specific quality
+
+	`go get github.com/kkdai/youtube/v2/youtubedr`
+
+	Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) with specific quality
+
+	```
+	youtubedr download -q medium https://www.youtube.com/watch?v=rFejpH_tAHM
+	```
+
+   #### Special case by quality hd1080:
+   Installation of ffmpeg is necessary for hd1080
+   ```
+   ffmpeg   //check ffmpeg is installed, if not please download ffmpeg and set to your PATH.
+   youtubedr download -q hd1080 https://www.youtube.com/watch?v=rFejpH_tAHM
+   ```
+
+
+ * ### Download video with specific itag
+
+    `go get github.com/kkdai/youtube/v2/youtubedr`
+
+    Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
+
+    ```
+    youtubedr download -q 18 https://www.youtube.com/watch?v=rFejpH_tAHM
+    ```
+
+## How it works
+
+- Parse the video ID you input in URL
+	- ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
+- Get video information via video id.
+	- Use URL: `http://youtube.com/get_video_info?video_id=`
+- Parse and decode video information.
+	- Download URL in "url="
+	- title in "title="
+- Download video from URL
+	- Need the string combination of "url"
+
+## Inspired
+- [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl)
+- [https://github.com/lepidosteus/youtube-dl](https://github.com/lepidosteus/youtube-dl)
+- [拆解 Youtube 影片下載位置](http://hkgoldenmra.blogspot.tw/2013/05/youtube.html)
+- [iawia002/annie](https://github.com/iawia002/annie)
+- [How to get url from obfuscate video info: youtube video downloader with php](https://stackoverflow.com/questions/60607291/youtube-video-downloader-with-php)
+
+
+## Project52
+It is one of my [project 52](https://github.com/kkdai/project52).
+
+
+## License
+This package is licensed under MIT license. See LICENSE for details.

downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md

@@ -0,0 +1,11 @@
+Tracking Youtube decipher change
+
+### 2022/01/21:
+
+#### action objects:
+
+var $z={fA:function(a){a.reverse()},S2:function(a,b){a.splice(0,b)},l6:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
+#### actions function:
+
+Npa=function(a){a=a.split("");$z.S2(a,3);$z.fA(a,45);$z.l6(a,31);$z.S2(a,1);$z.fA(a,63);$z.S2(a,2);$z.fA(a,68);return a.join("")};
+

downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go

@@ -0,0 +1,29 @@
+package youtube
+
+import (
+	"log/slog"
+	"os"
+	"path/filepath"
+)
+
+// destination for artifacts, used by integration tests
+var artifactsFolder = os.Getenv("ARTIFACTS")
+
+func writeArtifact(name string, content []byte) {
+	// Ensure folder exists
+	err := os.MkdirAll(artifactsFolder, os.ModePerm)
+	if err != nil {
+		slog.Error("unable to create artifacts folder", "path", artifactsFolder, "error", err)
+		return
+	}
+
+	path := filepath.Join(artifactsFolder, name)
+	err = os.WriteFile(path, content, 0600)
+
+	log := slog.With("path", path)
+	if err != nil {
+		log.Error("unable to write artifact", "error", err)
+	} else {
+		log.Debug("artifact created")
+	}
+}

downloader/vendor/github.com/kkdai/youtube/v2/client.go

@@ -0,0 +1,618 @@
+package youtube
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"math/rand"
+	"net/http"
+	"net/url"
+	"strconv"
+	"sync/atomic"
+
+	"log/slog"
+)
+
+const (
+	Size1Kb  = 1024
+	Size1Mb  = Size1Kb * 1024
+	Size10Mb = Size1Mb * 10
+
+	playerParams = "CgIQBg=="
+)
+
+var (
+	ErrNoFormat = errors.New("no video format provided")
+)
+
+// DefaultClient type to use. No reason to change but you could if you wanted to.
+var DefaultClient = AndroidClient
+
+// Client offers methods to download video metadata and video streams.
+type Client struct {
+	// HTTPClient can be used to set a custom HTTP client.
+	// If not set, http.DefaultClient will be used
+	HTTPClient *http.Client
+
+	// MaxRoutines to use when downloading a video.
+	MaxRoutines int
+
+	// ChunkSize to use when downloading videos in chunks. Default is Size10Mb.
+	ChunkSize int64
+
+	// playerCache caches the JavaScript code of a player response
+	playerCache playerCache
+
+	client *clientInfo
+
+	consentID string
+}
+
+func (c *Client) assureClient() {
+	if c.client == nil {
+		c.client = &DefaultClient
+	}
+}
+
+// GetVideo fetches video metadata
+func (c *Client) GetVideo(url string) (*Video, error) {
+	return c.GetVideoContext(context.Background(), url)
+}
+
+// GetVideoContext fetches video metadata with a context
+func (c *Client) GetVideoContext(ctx context.Context, url string) (*Video, error) {
+	id, err := ExtractVideoID(url)
+	if err != nil {
+		return nil, fmt.Errorf("extractVideoID failed: %w", err)
+	}
+
+	return c.videoFromID(ctx, id)
+}
+
+func (c *Client) videoFromID(ctx context.Context, id string) (*Video, error) {
+	c.assureClient()
+
+	body, err := c.videoDataByInnertube(ctx, id)
+	if err != nil {
+		return nil, err
+	}
+
+	v := Video{
+		ID: id,
+	}
+
+	// return early if all good
+	if err = v.parseVideoInfo(body); err == nil {
+		return &v, nil
+	}
+
+	// If the uploader has disabled embedding the video on other sites, parse video page
+	if errors.Is(err, ErrNotPlayableInEmbed) {
+		// additional parameters are required to access clips with sensitiv content
+		html, err := c.httpGetBodyBytes(ctx, "https://www.youtube.com/watch?v="+id+"&bpctr=9999999999&has_verified=1")
+		if err != nil {
+			return nil, err
+		}
+
+		return &v, v.parseVideoPage(html)
+	}
+
+	// If the uploader marked the video as inappropriate for some ages, use embed player
+	if errors.Is(err, ErrLoginRequired) {
+		c.client = &EmbeddedClient
+
+		bodyEmbed, errEmbed := c.videoDataByInnertube(ctx, id)
+		if errEmbed == nil {
+			errEmbed = v.parseVideoInfo(bodyEmbed)
+		}
+
+		if errEmbed == nil {
+			return &v, nil
+		}
+
+		// private video clearly not age-restricted and thus should be explicit
+		if errEmbed == ErrVideoPrivate {
+			return &v, errEmbed
+		}
+
+		// wrapping error so its clear whats happened
+		return &v, fmt.Errorf("can't bypass age restriction: %w", errEmbed)
+	}
+
+	// undefined error
+	return &v, err
+}
+
+type innertubeRequest struct {
+	VideoID         string            `json:"videoId,omitempty"`
+	BrowseID        string            `json:"browseId,omitempty"`
+	Continuation    string            `json:"continuation,omitempty"`
+	Context         inntertubeContext `json:"context"`
+	PlaybackContext *playbackContext  `json:"playbackContext,omitempty"`
+	ContentCheckOK  bool              `json:"contentCheckOk,omitempty"`
+	RacyCheckOk     bool              `json:"racyCheckOk,omitempty"`
+	Params          string            `json:"params"`
+}
+
+type playbackContext struct {
+	ContentPlaybackContext contentPlaybackContext `json:"contentPlaybackContext"`
+}
+
+type contentPlaybackContext struct {
+	// SignatureTimestamp string `json:"signatureTimestamp"`
+	HTML5Preference string `json:"html5Preference"`
+}
+
+type inntertubeContext struct {
+	Client innertubeClient `json:"client"`
+}
+
+type innertubeClient struct {
+	HL                string `json:"hl"`
+	GL                string `json:"gl"`
+	ClientName        string `json:"clientName"`
+	ClientVersion     string `json:"clientVersion"`
+	AndroidSDKVersion int    `json:"androidSDKVersion,omitempty"`
+	UserAgent         string `json:"userAgent,omitempty"`
+	TimeZone          string `json:"timeZone"`
+	UTCOffset         int    `json:"utcOffsetMinutes"`
+}
+
+// client info for the innertube API
+type clientInfo struct {
+	name           string
+	key            string
+	version        string
+	userAgent      string
+	androidVersion int
+}
+
+var (
+	// WebClient, better to use Android client but go ahead.
+	WebClient = clientInfo{
+		name:      "WEB",
+		version:   "2.20220801.00.00",
+		key:       "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
+		userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+	}
+
+	// AndroidClient, download go brrrrrr.
+	AndroidClient = clientInfo{
+		name:           "ANDROID",
+		version:        "18.11.34",
+		key:            "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
+		userAgent:      "com.google.android.youtube/18.11.34 (Linux; U; Android 11) gzip",
+		androidVersion: 30,
+	}
+
+	// EmbeddedClient, not really tested.
+	EmbeddedClient = clientInfo{
+		name:      "WEB_EMBEDDED_PLAYER",
+		version:   "1.19700101",
+		key:       "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", // seems like same key works for both clients
+		userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
+	}
+)
+
+func (c *Client) videoDataByInnertube(ctx context.Context, id string) ([]byte, error) {
+	data := innertubeRequest{
+		VideoID:        id,
+		Context:        prepareInnertubeContext(*c.client),
+		ContentCheckOK: true,
+		RacyCheckOk:    true,
+		Params:         playerParams,
+		PlaybackContext: &playbackContext{
+			ContentPlaybackContext: contentPlaybackContext{
+				// SignatureTimestamp: sts,
+				HTML5Preference: "HTML5_PREF_WANTS",
+			},
+		},
+	}
+
+	return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/player?key="+c.client.key, data)
+}
+
+func (c *Client) transcriptDataByInnertube(ctx context.Context, id string, lang string) ([]byte, error) {
+	data := innertubeRequest{
+		Context: prepareInnertubeContext(*c.client),
+		Params:  transcriptVideoID(id, lang),
+	}
+
+	return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/get_transcript?key="+c.client.key, data)
+}
+
+func prepareInnertubeContext(clientInfo clientInfo) inntertubeContext {
+	return inntertubeContext{
+		Client: innertubeClient{
+			HL:                "en",
+			GL:                "US",
+			TimeZone:          "UTC",
+			ClientName:        clientInfo.name,
+			ClientVersion:     clientInfo.version,
+			AndroidSDKVersion: clientInfo.androidVersion,
+			UserAgent:         clientInfo.userAgent,
+		},
+	}
+}
+
+func prepareInnertubePlaylistData(ID string, continuation bool, clientInfo clientInfo) innertubeRequest {
+	context := prepareInnertubeContext(clientInfo)
+
+	if continuation {
+		return innertubeRequest{
+			Context:        context,
+			Continuation:   ID,
+			ContentCheckOK: true,
+			RacyCheckOk:    true,
+			Params:         playerParams,
+		}
+	}
+
+	return innertubeRequest{
+		Context:        context,
+		BrowseID:       "VL" + ID,
+		ContentCheckOK: true,
+		RacyCheckOk:    true,
+		Params:         playerParams,
+	}
+}
+
+// transcriptVideoID encodes the video ID to the param used to fetch transcripts.
+func transcriptVideoID(videoID string, lang string) string {
+	langCode := encTranscriptLang(lang)
+
+	// This can be optionally appened to the Sprintf str, not sure what it means
+	// *3engagement-panel-searchable-transcript-search-panel\x30\x00\x38\x01\x40\x01
+	return base64Enc(fmt.Sprintf("\n\x0b%s\x12\x12%s\x18\x01", videoID, langCode))
+}
+
+func encTranscriptLang(languageCode string) string {
+	s := fmt.Sprintf("\n\x03asr\x12\x02%s\x1a\x00", languageCode)
+	s = base64PadEnc(s)
+
+	return url.QueryEscape(s)
+}
+
+// GetPlaylist fetches playlist metadata
+func (c *Client) GetPlaylist(url string) (*Playlist, error) {
+	return c.GetPlaylistContext(context.Background(), url)
+}
+
+// GetPlaylistContext fetches playlist metadata, with a context, along with a list of Videos, and some basic information
+// for these videos. Playlist entries cannot be downloaded, as they lack all the required metadata, but
+// can be used to enumerate all IDs, Authors, Titles, etc.
+func (c *Client) GetPlaylistContext(ctx context.Context, url string) (*Playlist, error) {
+	c.assureClient()
+
+	id, err := extractPlaylistID(url)
+	if err != nil {
+		return nil, fmt.Errorf("extractPlaylistID failed: %w", err)
+	}
+
+	data := prepareInnertubePlaylistData(id, false, *c.client)
+	body, err := c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+c.client.key, data)
+	if err != nil {
+		return nil, err
+	}
+
+	p := &Playlist{ID: id}
+	return p, p.parsePlaylistInfo(ctx, c, body)
+}
+
+func (c *Client) VideoFromPlaylistEntry(entry *PlaylistEntry) (*Video, error) {
+	return c.videoFromID(context.Background(), entry.ID)
+}
+
+func (c *Client) VideoFromPlaylistEntryContext(ctx context.Context, entry *PlaylistEntry) (*Video, error) {
+	return c.videoFromID(ctx, entry.ID)
+}
+
+// GetStream returns the stream and the total size for a specific format
+func (c *Client) GetStream(video *Video, format *Format) (io.ReadCloser, int64, error) {
+	return c.GetStreamContext(context.Background(), video, format)
+}
+
+// GetStreamContext returns the stream and the total size for a specific format with a context.
+func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *Format) (io.ReadCloser, int64, error) {
+	url, err := c.GetStreamURL(video, format)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, 0, err
+	}
+
+	r, w := io.Pipe()
+	contentLength := format.ContentLength
+
+	if contentLength == 0 {
+		// some videos don't have length information
+		contentLength = c.downloadOnce(req, w, format)
+	} else {
+		// we have length information, let's download by chunks!
+		c.downloadChunked(ctx, req, w, format)
+	}
+
+	return r, contentLength, nil
+}
+
+func (c *Client) downloadOnce(req *http.Request, w *io.PipeWriter, _ *Format) int64 {
+	resp, err := c.httpDo(req)
+	if err != nil {
+		w.CloseWithError(err) //nolint:errcheck
+		return 0
+	}
+
+	go func() {
+		defer resp.Body.Close()
+		_, err := io.Copy(w, resp.Body)
+		if err == nil {
+			w.Close()
+		} else {
+			w.CloseWithError(err) //nolint:errcheck
+		}
+	}()
+
+	contentLength := resp.Header.Get("Content-Length")
+	length, _ := strconv.ParseInt(contentLength, 10, 64)
+
+	return length
+}
+
+func (c *Client) getChunkSize() int64 {
+	if c.ChunkSize > 0 {
+		return c.ChunkSize
+	}
+
+	return Size10Mb
+}
+
+func (c *Client) getMaxRoutines(limit int) int {
+	routines := 10
+
+	if c.MaxRoutines > 0 {
+		routines = c.MaxRoutines
+	}
+
+	if limit > 0 && routines > limit {
+		routines = limit
+	}
+
+	return routines
+}
+
+func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.PipeWriter, format *Format) {
+	chunks := getChunks(format.ContentLength, c.getChunkSize())
+	maxRoutines := c.getMaxRoutines(len(chunks))
+
+	cancelCtx, cancel := context.WithCancel(ctx)
+	abort := func(err error) {
+		w.CloseWithError(err)
+		cancel()
+	}
+
+	currentChunk := atomic.Uint32{}
+	for i := 0; i < maxRoutines; i++ {
+		go func() {
+			for {
+				chunkIndex := int(currentChunk.Add(1)) - 1
+				if chunkIndex >= len(chunks) {
+					// no more chunks
+					return
+				}
+
+				chunk := &chunks[chunkIndex]
+				err := c.downloadChunk(req.Clone(cancelCtx), chunk)
+				close(chunk.data)
+
+				if err != nil {
+					abort(err)
+					return
+				}
+			}
+		}()
+	}
+
+	go func() {
+		// copy chunks into the PipeWriter
+		for i := 0; i < len(chunks); i++ {
+			select {
+			case <-cancelCtx.Done():
+				abort(context.Canceled)
+				return
+			case data := <-chunks[i].data:
+				_, err := io.Copy(w, bytes.NewBuffer(data))
+
+				if err != nil {
+					abort(err)
+				}
+			}
+		}
+
+		// everything succeeded
+		w.Close()
+	}()
+}
+
+// GetStreamURL returns the url for a specific format
+func (c *Client) GetStreamURL(video *Video, format *Format) (string, error) {
+	return c.GetStreamURLContext(context.Background(), video, format)
+}
+
+// GetStreamURLContext returns the url for a specific format with a context
+func (c *Client) GetStreamURLContext(ctx context.Context, video *Video, format *Format) (string, error) {
+	if format == nil {
+		return "", ErrNoFormat
+	}
+
+	c.assureClient()
+
+	if format.URL != "" {
+		if c.client.androidVersion > 0 {
+			return format.URL, nil
+		}
+
+		return c.unThrottle(ctx, video.ID, format.URL)
+	}
+
+	// TODO: check rest of this function, is it redundant?
+
+	cipher := format.Cipher
+	if cipher == "" {
+		return "", ErrCipherNotFound
+	}
+
+	uri, err := c.decipherURL(ctx, video.ID, cipher)
+	if err != nil {
+		return "", err
+	}
+
+	return uri, err
+}
+
+// httpDo sends an HTTP request and returns an HTTP response.
+func (c *Client) httpDo(req *http.Request) (*http.Response, error) {
+	client := c.HTTPClient
+	if client == nil {
+		client = http.DefaultClient
+	}
+
+	req.Header.Set("User-Agent", c.client.userAgent)
+	req.Header.Set("Origin", "https://youtube.com")
+	req.Header.Set("Sec-Fetch-Mode", "navigate")
+
+	if len(c.consentID) == 0 {
+		c.consentID = strconv.Itoa(rand.Intn(899) + 100) //nolint:gosec
+	}
+
+	req.AddCookie(&http.Cookie{
+		Name:   "CONSENT",
+		Value:  "YES+cb.20210328-17-p0.en+FX+" + c.consentID,
+		Path:   "/",
+		Domain: ".youtube.com",
+	})
+
+	res, err := client.Do(req)
+
+	log := slog.With("method", req.Method, "url", req.URL)
+
+	if err != nil {
+		log.Debug("HTTP request failed", "error", err)
+	} else {
+		log.Debug("HTTP request succeeded", "status", res.Status)
+	}
+
+	return res, err
+}
+
+// httpGet does a HTTP GET request, checks the response to be a 200 OK and returns it
+func (c *Client) httpGet(ctx context.Context, url string) (*http.Response, error) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	resp, err := c.httpDo(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, ErrUnexpectedStatusCode(resp.StatusCode)
+	}
+
+	return resp, nil
+}
+
+// httpGetBodyBytes reads the whole HTTP body and returns it
+func (c *Client) httpGetBodyBytes(ctx context.Context, url string) ([]byte, error) {
+	resp, err := c.httpGet(ctx, url)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return io.ReadAll(resp.Body)
+}
+
+// httpPost does a HTTP POST request with a body, checks the response to be a 200 OK and returns it
+func (c *Client) httpPost(ctx context.Context, url string, body interface{}) (*http.Response, error) {
+	data, err := json.Marshal(body)
+	if err != nil {
+		return nil, err
+	}
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
+	if err != nil {
+		return nil, err
+	}
+
+	req.Header.Set("X-Youtube-Client-Name", "3")
+	req.Header.Set("X-Youtube-Client-Version", c.client.version)
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
+
+	resp, err := c.httpDo(req)
+	if err != nil {
+		return nil, err
+	}
+
+	if resp.StatusCode != http.StatusOK {
+		resp.Body.Close()
+		return nil, ErrUnexpectedStatusCode(resp.StatusCode)
+	}
+
+	return resp, nil
+}
+
+// httpPostBodyBytes reads the whole HTTP body and returns it
+func (c *Client) httpPostBodyBytes(ctx context.Context, url string, body interface{}) ([]byte, error) {
+	resp, err := c.httpPost(ctx, url, body)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	return io.ReadAll(resp.Body)
+}
+
+// downloadChunk writes the response data into the data channel of the chunk.
+// Downloading in multiple chunks is much faster:
+// https://github.com/kkdai/youtube/pull/190
+func (c *Client) downloadChunk(req *http.Request, chunk *chunk) error {
+	q := req.URL.Query()
+	q.Set("range", fmt.Sprintf("%d-%d", chunk.start, chunk.end))
+	req.URL.RawQuery = q.Encode()
+
+	resp, err := c.httpDo(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < http.StatusOK && resp.StatusCode >= 300 {
+		return ErrUnexpectedStatusCode(resp.StatusCode)
+	}
+
+	expected := int(chunk.end-chunk.start) + 1
+	data, err := io.ReadAll(resp.Body)
+	n := len(data)
+
+	if err != nil {
+		return err
+	}
+
+	if n != expected {
+		return fmt.Errorf("chunk at offset %d has invalid size: expected=%d actual=%d", chunk.start, expected, n)
+	}
+
+	chunk.data <- data
+
+	return nil
+}

downloader/vendor/github.com/kkdai/youtube/v2/decipher.go

@@ -0,0 +1,285 @@
+package youtube
+
+import (
+	"bytes"
+	"context"
+	"errors"
+	"fmt"
+	"net/url"
+	"regexp"
+	"strconv"
+
+	"github.com/dop251/goja"
+)
+
+func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) {
+	params, err := url.ParseQuery(cipher)
+	if err != nil {
+		return "", err
+	}
+
+	uri, err := url.Parse(params.Get("url"))
+	if err != nil {
+		return "", err
+	}
+	query := uri.Query()
+
+	config, err := c.getPlayerConfig(ctx, videoID)
+	if err != nil {
+		return "", err
+	}
+
+	// decrypt s-parameter
+	bs, err := config.decrypt([]byte(params.Get("s")))
+	if err != nil {
+		return "", err
+	}
+	query.Add(params.Get("sp"), string(bs))
+
+	query, err = c.decryptNParam(config, query)
+	if err != nil {
+		return "", err
+	}
+
+	uri.RawQuery = query.Encode()
+
+	return uri.String(), nil
+}
+
+// see https://github.com/kkdai/youtube/pull/244
+func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) {
+	config, err := c.getPlayerConfig(ctx, videoID)
+	if err != nil {
+		return "", err
+	}
+
+	uri, err := url.Parse(urlString)
+	if err != nil {
+		return "", err
+	}
+
+	// for debugging
+	if artifactsFolder != "" {
+		writeArtifact("video-"+videoID+".url", []byte(uri.String()))
+	}
+
+	query, err := c.decryptNParam(config, uri.Query())
+	if err != nil {
+		return "", err
+	}
+
+	uri.RawQuery = query.Encode()
+	return uri.String(), nil
+}
+
+func (c *Client) decryptNParam(config playerConfig, query url.Values) (url.Values, error) {
+	// decrypt n-parameter
+	nSig := query.Get("v")
+	log := Logger.With("n", nSig)
+
+	if nSig != "" {
+		nDecoded, err := config.decodeNsig(nSig)
+		if err != nil {
+			return nil, fmt.Errorf("unable to decode nSig: %w", err)
+		}
+		query.Set("v", nDecoded)
+		log = log.With("decoded", nDecoded)
+	}
+
+	log.Debug("nParam")
+
+	return query, nil
+}
+
+const (
+	jsvarStr   = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
+	reverseStr = ":function\\(a\\)\\{" +
+		"(?:return )?a\\.reverse\\(\\)" +
+		"\\}"
+	spliceStr = ":function\\(a,b\\)\\{" +
+		"a\\.splice\\(0,b\\)" +
+		"\\}"
+	swapStr = ":function\\(a,b\\)\\{" +
+		"var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
+		"\\}"
+)
+
+var (
+	nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})")
+	actionsObjRegexp    = regexp.MustCompile(fmt.Sprintf(
+		"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr))
+
+	actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
+		"function(?: %s)?\\(a\\)\\{"+
+			"a=a\\.split\\(\"\"\\);\\s*"+
+			"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
+			"return a\\.join\\(\"\"\\)"+
+			"\\}", jsvarStr, jsvarStr, jsvarStr))
+
+	reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
+	spliceRegexp  = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
+	swapRegexp    = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
+)
+
+func (config playerConfig) decodeNsig(encoded string) (string, error) {
+	fBody, err := config.getNFunction()
+	if err != nil {
+		return "", err
+	}
+
+	return evalJavascript(fBody, encoded)
+}
+
+func evalJavascript(jsFunction, arg string) (string, error) {
+	const myName = "myFunction"
+
+	vm := goja.New()
+	_, err := vm.RunString(myName + "=" + jsFunction)
+	if err != nil {
+		return "", err
+	}
+
+	var output func(string) string
+	err = vm.ExportTo(vm.Get(myName), &output)
+	if err != nil {
+		return "", err
+	}
+
+	return output(arg), nil
+}
+
+func (config playerConfig) getNFunction() (string, error) {
+	nameResult := nFunctionNameRegexp.FindSubmatch(config)
+	if len(nameResult) == 0 {
+		return "", errors.New("unable to extract n-function name")
+	}
+
+	var name string
+	if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 {
+		name = string(nameResult[4])
+	} else {
+		name = string(nameResult[1])
+	}
+
+	return config.extraFunction(name)
+
+}
+
+func (config playerConfig) extraFunction(name string) (string, error) {
+	// find the beginning of the function
+	def := []byte(name + "=function(")
+	start := bytes.Index(config, def)
+	if start < 1 {
+		return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def)
+	}
+
+	// start after the first curly bracket
+	pos := start + bytes.IndexByte(config[start:], '{') + 1
+
+	var strChar byte
+
+	// find the bracket closing the function
+	for brackets := 1; brackets > 0; pos++ {
+		b := config[pos]
+		switch b {
+		case '{':
+			if strChar == 0 {
+				brackets++
+			}
+		case '}':
+			if strChar == 0 {
+				brackets--
+			}
+		case '`', '"', '\'':
+			if config[pos-1] == '\\' && config[pos-2] != '\\' {
+				continue
+			}
+			if strChar == 0 {
+				strChar = b
+			} else if strChar == b {
+				strChar = 0
+			}
+		}
+	}
+
+	return string(config[start:pos]), nil
+}
+
+func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) {
+	operations, err := config.parseDecipherOps()
+	if err != nil {
+		return nil, err
+	}
+
+	// apply operations
+	bs := []byte(cyphertext)
+	for _, op := range operations {
+		bs = op(bs)
+	}
+
+	return bs, nil
+}
+
+/*
+parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js
+
+var Mt={
+splice:function(a,b){a.splice(0,b)},
+reverse:function(a){a.reverse()},
+EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
+
+a=a.split("");
+Mt.splice(a,3);
+Mt.EQ(a,39);
+Mt.splice(a,2);
+Mt.EQ(a,1);
+Mt.splice(a,1);
+Mt.EQ(a,35);
+Mt.EQ(a,51);
+Mt.splice(a,2);
+Mt.reverse(a,52);
+return a.join("")
+*/
+func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) {
+	objResult := actionsObjRegexp.FindSubmatch(config)
+	funcResult := actionsFuncRegexp.FindSubmatch(config)
+	if len(objResult) < 3 || len(funcResult) < 2 {
+		return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult))
+	}
+
+	obj := objResult[1]
+	objBody := objResult[2]
+	funcBody := funcResult[1]
+
+	var reverseKey, spliceKey, swapKey string
+
+	if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 {
+		reverseKey = string(result[1])
+	}
+	if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 {
+		spliceKey = string(result[1])
+	}
+	if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 {
+		swapKey = string(result[1])
+	}
+
+	regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey)))
+	if err != nil {
+		return nil, err
+	}
+
+	var ops []DecipherOperation
+	for _, s := range regex.FindAllSubmatch(funcBody, -1) {
+		switch string(s[1]) {
+		case reverseKey:
+			ops = append(ops, reverseFunc)
+		case swapKey:
+			arg, _ := strconv.Atoi(string(s[2]))
+			ops = append(ops, newSwapFunc(arg))
+		case spliceKey:
+			arg, _ := strconv.Atoi(string(s[2]))
+			ops = append(ops, newSpliceFunc(arg))
+		}
+	}
+	return ops, nil
+}

downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go

@@ -0,0 +1,27 @@
+package youtube
+
+type DecipherOperation func([]byte) []byte
+
+func newSpliceFunc(pos int) DecipherOperation {
+	return func(bs []byte) []byte {
+		return bs[pos:]
+	}
+}
+
+func newSwapFunc(arg int) DecipherOperation {
+	return func(bs []byte) []byte {
+		pos := arg % len(bs)
+		bs[0], bs[pos] = bs[pos], bs[0]
+		return bs
+	}
+}
+
+func reverseFunc(bs []byte) []byte {
+	l, r := 0, len(bs)-1
+	for l < r {
+		bs[l], bs[r] = bs[r], bs[l]
+		l++
+		r--
+	}
+	return bs
+}

downloader/vendor/github.com/kkdai/youtube/v2/doc.go

@@ -0,0 +1,4 @@
+/*
+Package youtube implement youtube download package in go.
+*/
+package youtube

downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go

@@ -0,0 +1,203 @@
+package downloader
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"os"
+	"os/exec"
+	"path/filepath"
+
+	"github.com/kkdai/youtube/v2"
+	"github.com/vbauerster/mpb/v5"
+	"github.com/vbauerster/mpb/v5/decor"
+)
+
+// Downloader offers high level functions to download videos into files
+type Downloader struct {
+	youtube.Client
+	OutputDir string // optional directory to store the files
+}
+
+func (dl *Downloader) getOutputFile(v *youtube.Video, format *youtube.Format, outputFile string) (string, error) {
+	if outputFile == "" {
+		outputFile = SanitizeFilename(v.Title)
+		outputFile += pickIdealFileExtension(format.MimeType)
+	}
+
+	if dl.OutputDir != "" {
+		if err := os.MkdirAll(dl.OutputDir, 0o755); err != nil {
+			return "", err
+		}
+		outputFile = filepath.Join(dl.OutputDir, outputFile)
+	}
+
+	return outputFile, nil
+}
+
+// Download : Starting download video by arguments.
+func (dl *Downloader) Download(ctx context.Context, v *youtube.Video, format *youtube.Format, outputFile string) error {
+	youtube.Logger.Info(
+		"Downloading video",
+		"id", v.ID,
+		"quality", format.Quality,
+		"mimeType", format.MimeType,
+	)
+	destFile, err := dl.getOutputFile(v, format, outputFile)
+	if err != nil {
+		return err
+	}
+
+	// Create output file
+	out, err := os.Create(destFile)
+	if err != nil {
+		return err
+	}
+	defer out.Close()
+
+	return dl.videoDLWorker(ctx, out, v, format)
+}
+
+// DownloadComposite : Downloads audio and video streams separately and merges them via ffmpeg.
+func (dl *Downloader) DownloadComposite(ctx context.Context, outputFile string, v *youtube.Video, quality string, mimetype, language string) error {
+	videoFormat, audioFormat, err1 := getVideoAudioFormats(v, quality, mimetype, language)
+	if err1 != nil {
+		return err1
+	}
+
+	log := youtube.Logger.With("id", v.ID)
+
+	log.Info(
+		"Downloading composite video",
+		"videoQuality", videoFormat.QualityLabel,
+		"videoMimeType", videoFormat.MimeType,
+		"audioMimeType", audioFormat.MimeType,
+	)
+
+	destFile, err := dl.getOutputFile(v, videoFormat, outputFile)
+	if err != nil {
+		return err
+	}
+	outputDir := filepath.Dir(destFile)
+
+	// Create temporary video file
+	videoFile, err := os.CreateTemp(outputDir, "youtube_*.m4v")
+	if err != nil {
+		return err
+	}
+	defer Delete(videoFile.Name(), log)
+
+	// Create temporary audio file
+	audioFile, err := os.CreateTemp(outputDir, "youtube_*.m4a")
+	if err != nil {
+		return err
+	}
+	defer Delete(audioFile.Name(), log)
+
+	log.Debug("Downloading video file...")
+	err = dl.videoDLWorker(ctx, videoFile, v, videoFormat)
+	if err != nil {
+		return err
+	}
+	videoFile.Close()
+
+	log.Debug("Downloading audio file...")
+	err = dl.videoDLWorker(ctx, audioFile, v, audioFormat)
+	if err != nil {
+		return err
+	}
+	audioFile.Close()
+
+	//nolint:gosec
+	ffmpegVersionCmd := exec.Command("ffmpeg", "-y",
+		"-i", videoFile.Name(),
+		"-i", audioFile.Name(),
+		"-c", "copy", // Just copy without re-encoding
+		"-shortest", // Finish encoding when the shortest input stream ends
+		destFile,
+		"-loglevel", "warning",
+	)
+	ffmpegVersionCmd.Stderr = os.Stderr
+	ffmpegVersionCmd.Stdout = os.Stdout
+	log.Info("merging video and audio", "output", destFile)
+
+	return ffmpegVersionCmd.Run()
+}
+
+func Delete(path string, log *slog.Logger) {
+	err := os.Remove(path)
+	if err != nil {
+		log.Error("Failed deleting file", "path", path, "error", err)
+	}
+}
+
+func getVideoAudioFormats(v *youtube.Video, quality string, mimetype, language string) (*youtube.Format, *youtube.Format, error) {
+	var videoFormats, audioFormats youtube.FormatList
+
+	formats := v.Formats
+	if mimetype != "" {
+		formats = formats.Type(mimetype)
+	}
+
+	videoFormats = formats.Type("video").AudioChannels(0)
+	audioFormats = formats.Type("audio")
+
+	if quality != "" {
+		videoFormats = videoFormats.Quality(quality)
+	}
+
+	if language != "" {
+		audioFormats = audioFormats.Language(language)
+	}
+
+	if len(videoFormats) == 0 {
+		return nil, nil, errors.New("no video format found after filtering")
+	}
+
+	if len(audioFormats) == 0 {
+		return nil, nil, errors.New("no audio format found after filtering")
+	}
+
+	videoFormats.Sort()
+	audioFormats.Sort()
+
+	return &videoFormats[0], &audioFormats[0], nil
+}
+
+func (dl *Downloader) videoDLWorker(ctx context.Context, out *os.File, video *youtube.Video, format *youtube.Format) error {
+	stream, size, err := dl.GetStreamContext(ctx, video, format)
+	if err != nil {
+		return err
+	}
+
+	prog := &progress{
+		contentLength: float64(size),
+	}
+
+	// create progress bar
+	progress := mpb.New(mpb.WithWidth(64))
+	bar := progress.AddBar(
+		int64(prog.contentLength),
+
+		mpb.PrependDecorators(
+			decor.CountersKibiByte("% .2f / % .2f"),
+			decor.Percentage(decor.WCSyncSpace),
+		),
+		mpb.AppendDecorators(
+			decor.EwmaETA(decor.ET_STYLE_GO, 90),
+			decor.Name(" ] "),
+			decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60),
+		),
+	)
+
+	reader := bar.ProxyReader(stream)
+	mw := io.MultiWriter(out, prog)
+	_, err = io.Copy(mw, reader)
+	if err != nil {
+		return err
+	}
+
+	progress.Wait()
+	return nil
+}

downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go

@@ -0,0 +1,65 @@
+package downloader
+
+import (
+	"mime"
+	"regexp"
+)
+
+const defaultExtension = ".mov"
+
+// Rely on hardcoded canonical mime types, as the ones provided by Go aren't exhaustive [1].
+// This seems to be a recurring problem for youtube downloaders, see [2].
+// The implementation is based on mozilla's list [3], IANA [4] and Youtube's support [5].
+// [1] https://github.com/golang/go/blob/ed7888aea6021e25b0ea58bcad3f26da2b139432/src/mime/type.go#L60
+// [2] https://github.com/ZiTAL/youtube-dl/blob/master/mime.types
+// [3] https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
+// [4] https://www.iana.org/assignments/media-types/media-types.xhtml#video
+// [5] https://support.google.com/youtube/troubleshooter/2888402?hl=en
+var canonicals = map[string]string{
+	"video/quicktime":  ".mov",
+	"video/x-msvideo":  ".avi",
+	"video/x-matroska": ".mkv",
+	"video/mpeg":       ".mpeg",
+	"video/webm":       ".webm",
+	"video/3gpp2":      ".3g2",
+	"video/x-flv":      ".flv",
+	"video/3gpp":       ".3gp",
+	"video/mp4":        ".mp4",
+	"video/ogg":        ".ogv",
+	"video/mp2t":       ".ts",
+}
+
+func pickIdealFileExtension(mediaType string) string {
+	mediaType, _, err := mime.ParseMediaType(mediaType)
+	if err != nil {
+		return defaultExtension
+	}
+
+	if extension, ok := canonicals[mediaType]; ok {
+		return extension
+	}
+
+	// Our last resort is to ask the operating system, but these give multiple results and are rarely canonical.
+	extensions, err := mime.ExtensionsByType(mediaType)
+	if err != nil || extensions == nil {
+		return defaultExtension
+	}
+
+	return extensions[0]
+}
+
+func SanitizeFilename(fileName string) string {
+	// Characters not allowed on mac
+	//	:/
+	// Characters not allowed on linux
+	//	/
+	// Characters not allowed on windows
+	//	<>:"/\|?*
+
+	// Ref https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
+
+	fileName = regexp.MustCompile(`[:/<>\:"\\|?*]`).ReplaceAllString(fileName, "")
+	fileName = regexp.MustCompile(`\s+`).ReplaceAllString(fileName, " ")
+
+	return fileName
+}

downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go

@@ -0,0 +1,17 @@
+package downloader
+
+type progress struct {
+	contentLength     float64
+	totalWrittenBytes float64
+	downloadLevel     float64
+}
+
+func (dl *progress) Write(p []byte) (n int, err error) {
+	n = len(p)
+	dl.totalWrittenBytes = dl.totalWrittenBytes + float64(n)
+	currentPercent := (dl.totalWrittenBytes / dl.contentLength) * 100
+	if (dl.downloadLevel <= currentPercent) && (dl.downloadLevel < 100) {
+		dl.downloadLevel++
+	}
+	return
+}

downloader/vendor/github.com/kkdai/youtube/v2/errors.go

@@ -0,0 +1,47 @@
+package youtube
+
+import (
+	"fmt"
+)
+
+const (
+	ErrCipherNotFound             = constError("cipher not found")
+	ErrSignatureTimestampNotFound = constError("signature timestamp not found")
+	ErrInvalidCharactersInVideoID = constError("invalid characters in video id")
+	ErrVideoIDMinLength           = constError("the video id must be at least 10 characters long")
+	ErrReadOnClosedResBody        = constError("http: read on closed response body")
+	ErrNotPlayableInEmbed         = constError("embedding of this video has been disabled")
+	ErrLoginRequired              = constError("login required to confirm your age")
+	ErrVideoPrivate               = constError("user restricted access to this video")
+	ErrInvalidPlaylist            = constError("no playlist detected or invalid playlist ID")
+)
+
+type constError string
+
+func (e constError) Error() string {
+	return string(e)
+}
+
+type ErrPlayabiltyStatus struct {
+	Status string
+	Reason string
+}
+
+func (err ErrPlayabiltyStatus) Error() string {
+	return fmt.Sprintf("cannot playback and download, status: %s, reason: %s", err.Status, err.Reason)
+}
+
+// ErrUnexpectedStatusCode is returned on unexpected HTTP status codes
+type ErrUnexpectedStatusCode int
+
+func (err ErrUnexpectedStatusCode) Error() string {
+	return fmt.Sprintf("unexpected status code: %d", err)
+}
+
+type ErrPlaylistStatus struct {
+	Reason string
+}
+
+func (err ErrPlaylistStatus) Error() string {
+	return fmt.Sprintf("could not load playlist: %s", err.Reason)
+}

downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go

@@ -0,0 +1,35 @@
+//go:build fetch
+// +build fetch
+
+package youtube
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+)
+
+func init() {
+	FetchTestData()
+}
+
+// ran via go generate to fetch and update the playlist response data
+func FetchTestData() {
+	f, err := os.Create(testPlaylistResponseDataFile)
+	exitOnError(err)
+	requestURL := fmt.Sprintf(playlistFetchURL, testPlaylistID)
+	resp, err := http.Get(requestURL)
+	exitOnError(err)
+	defer resp.Body.Close()
+	n, err := io.Copy(f, resp.Body)
+	exitOnError(err)
+	fmt.Printf("Successfully fetched playlist %s (%d bytes)\n", testPlaylistID, n)
+}
+
+func exitOnError(err error) {
+	if err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}

downloader/vendor/github.com/kkdai/youtube/v2/format_list.go

@@ -0,0 +1,148 @@
+package youtube
+
+import (
+	"sort"
+	"strconv"
+	"strings"
+)
+
+type FormatList []Format
+
+// Type returns a new FormatList filtered by itag
+func (list FormatList) Select(f func(Format) bool) (result FormatList) {
+	for i := range list {
+		if f(list[i]) {
+			result = append(result, list[i])
+		}
+	}
+	return result
+}
+
+// Type returns a new FormatList filtered by itag
+func (list FormatList) Itag(itagNo int) FormatList {
+	return list.Select(func(f Format) bool {
+		return f.ItagNo == itagNo
+	})
+}
+
+// Type returns a new FormatList filtered by mime type
+func (list FormatList) Type(value string) FormatList {
+	return list.Select(func(f Format) bool {
+		return strings.Contains(f.MimeType, value)
+	})
+}
+
+// Type returns a new FormatList filtered by display name
+func (list FormatList) Language(displayName string) FormatList {
+	return list.Select(func(f Format) bool {
+		return f.LanguageDisplayName() == displayName
+	})
+}
+
+// Quality returns a new FormatList filtered by quality, quality label or itag,
+// but not audio quality
+func (list FormatList) Quality(quality string) FormatList {
+	itag, _ := strconv.Atoi(quality)
+
+	return list.Select(func(f Format) bool {
+		return itag == f.ItagNo || strings.Contains(f.Quality, quality) || strings.Contains(f.QualityLabel, quality)
+	})
+}
+
+// AudioChannels returns a new FormatList filtered by the matching AudioChannels
+func (list FormatList) AudioChannels(n int) FormatList {
+	return list.Select(func(f Format) bool {
+		return f.AudioChannels == n
+	})
+}
+
+// AudioChannels returns a new FormatList filtered by the matching AudioChannels
+func (list FormatList) WithAudioChannels() FormatList {
+	return list.Select(func(f Format) bool {
+		return f.AudioChannels > 0
+	})
+}
+
+// FilterQuality reduces the format list to formats matching the quality
+func (v *Video) FilterQuality(quality string) {
+	v.Formats = v.Formats.Quality(quality)
+	v.Formats.Sort()
+}
+
+// Sort sorts all formats fields
+func (list FormatList) Sort() {
+	sort.SliceStable(list, func(i, j int) bool {
+		return sortFormat(i, j, list)
+	})
+}
+
+// sortFormat sorts video by resolution, FPS, codec (av01, vp9, avc1), bitrate
+// sorts audio by default, codec (mp4, opus), channels, bitrate, sample rate
+func sortFormat(i int, j int, formats FormatList) bool {
+
+	// Sort by Width
+	if formats[i].Width == formats[j].Width {
+		// Format 137 downloads slowly, give it less priority
+		// see https://github.com/kkdai/youtube/pull/171
+		switch 137 {
+		case formats[i].ItagNo:
+			return false
+		case formats[j].ItagNo:
+			return true
+		}
+
+		// Sort by FPS
+		if formats[i].FPS == formats[j].FPS {
+			if formats[i].FPS == 0 && formats[i].AudioChannels > 0 && formats[j].AudioChannels > 0 {
+				// Audio
+				// Sort by default
+				if (formats[i].AudioTrack == nil && formats[j].AudioTrack == nil) || (formats[i].AudioTrack != nil && formats[j].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault == formats[j].AudioTrack.AudioIsDefault) {
+					// Sort by codec
+					codec := map[int]int{}
+					for _, index := range []int{i, j} {
+						if strings.Contains(formats[index].MimeType, "mp4") {
+							codec[index] = 1
+						} else if strings.Contains(formats[index].MimeType, "opus") {
+							codec[index] = 2
+						}
+					}
+					if codec[i] == codec[j] {
+						// Sort by Audio Channel
+						if formats[i].AudioChannels == formats[j].AudioChannels {
+							// Sort by Audio Bitrate
+							if formats[i].Bitrate == formats[j].Bitrate {
+								// Sort by Audio Sample Rate
+								return formats[i].AudioSampleRate > formats[j].AudioSampleRate
+							}
+							return formats[i].Bitrate > formats[j].Bitrate
+						}
+						return formats[i].AudioChannels > formats[j].AudioChannels
+					}
+					return codec[i] < codec[j]
+				} else if formats[i].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault {
+					return true
+				}
+				return false
+			}
+			// Video
+			// Sort by codec
+			codec := map[int]int{}
+			for _, index := range []int{i, j} {
+				if strings.Contains(formats[index].MimeType, "av01") {
+					codec[index] = 1
+				} else if strings.Contains(formats[index].MimeType, "vp9") {
+					codec[index] = 2
+				} else if strings.Contains(formats[index].MimeType, "avc1") {
+					codec[index] = 3
+				}
+			}
+			if codec[i] == codec[j] {
+				// Sort by Audio Bitrate
+				return formats[i].Bitrate > formats[j].Bitrate
+			}
+			return codec[i] < codec[j]
+		}
+		return formats[i].FPS > formats[j].FPS
+	}
+	return formats[i].Width > formats[j].Width
+}

downloader/vendor/github.com/kkdai/youtube/v2/logger.go

@@ -0,0 +1,28 @@
+package youtube
+
+import (
+	"fmt"
+	"log/slog"
+	"os"
+)
+
+// The global logger for all Client instances
+var Logger = getLogger(os.Getenv("LOGLEVEL"))
+
+func SetLogLevel(value string) {
+	Logger = getLogger(value)
+}
+
+func getLogger(logLevel string) *slog.Logger {
+	levelVar := slog.LevelVar{}
+
+	if logLevel != "" {
+		if err := levelVar.UnmarshalText([]byte(logLevel)); err != nil {
+			panic(fmt.Sprintf("Invalid log level %s: %v", logLevel, err))
+		}
+	}
+
+	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
+		Level: levelVar.Level(),
+	}))
+}

downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go

@@ -0,0 +1,37 @@
+package youtube
+
+import (
+	"time"
+)
+
+const defaultCacheExpiration = time.Minute * time.Duration(5)
+
+type playerCache struct {
+	key       string
+	expiredAt time.Time
+	config    playerConfig
+}
+
+// Get : get cache  when it has same video id and not expired
+func (s playerCache) Get(key string) playerConfig {
+	return s.GetCacheBefore(key, time.Now())
+}
+
+// GetCacheBefore : can pass time for testing
+func (s playerCache) GetCacheBefore(key string, time time.Time) playerConfig {
+	if key == s.key && s.expiredAt.After(time) {
+		return s.config
+	}
+	return nil
+}
+
+// Set : set cache with default expiration
+func (s *playerCache) Set(key string, operations playerConfig) {
+	s.setWithExpiredTime(key, operations, time.Now().Add(defaultCacheExpiration))
+}
+
+func (s *playerCache) setWithExpiredTime(key string, config playerConfig, time time.Time) {
+	s.key = key
+	s.config = config
+	s.expiredAt = time
+}

downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go

@@ -0,0 +1,59 @@
+package youtube
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+type playerConfig []byte
+
+var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`)
+
+func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) {
+	embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID)
+	embedBody, err := c.httpGetBodyBytes(ctx, embedURL)
+	if err != nil {
+		return nil, err
+	}
+
+	// example: /s/player/f676c671/player_ias.vflset/en_US/base.js
+	playerPath := string(basejsPattern.Find(embedBody))
+	if playerPath == "" {
+		return nil, errors.New("unable to find basejs URL in playerConfig")
+	}
+
+	// for debugging
+	var artifactName string
+	if artifactsFolder != "" {
+		parts := strings.SplitN(playerPath, "/", 5)
+		artifactName = "player-" + parts[3] + ".js"
+		linkName := filepath.Join(artifactsFolder, "video-"+videoID+".js")
+		if err := os.Symlink(artifactName, linkName); err != nil {
+			log.Printf("unable to create symlink %s: %v", linkName, err)
+		}
+	}
+
+	config := c.playerCache.Get(playerPath)
+	if config != nil {
+		return config, nil
+	}
+
+	config, err = c.httpGetBodyBytes(ctx, "https://youtube.com"+playerPath)
+	if err != nil {
+		return nil, err
+	}
+
+	// for debugging
+	if artifactName != "" {
+		writeArtifact(artifactName, config)
+	}
+
+	c.playerCache.Set(playerPath, config)
+	return config, nil
+}

downloader/vendor/github.com/kkdai/youtube/v2/playlist.go

@@ -0,0 +1,256 @@
+package youtube
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"regexp"
+	"runtime/debug"
+	"strconv"
+	"time"
+
+	sjson "github.com/bitly/go-simplejson"
+)
+
+var (
+	playlistIDRegex    = regexp.MustCompile("^[A-Za-z0-9_-]{13,42}$")
+	playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{13,42})(&.*)?$")
+)
+
+type Playlist struct {
+	ID          string
+	Title       string
+	Description string
+	Author      string
+	Videos      []*PlaylistEntry
+}
+
+type PlaylistEntry struct {
+	ID         string
+	Title      string
+	Author     string
+	Duration   time.Duration
+	Thumbnails Thumbnails
+}
+
+func extractPlaylistID(url string) (string, error) {
+	if playlistIDRegex.Match([]byte(url)) {
+		return url, nil
+	}
+
+	matches := playlistInURLRegex.FindStringSubmatch(url)
+
+	if matches != nil {
+		return matches[1], nil
+	}
+
+	return "", ErrInvalidPlaylist
+}
+
+// structs for playlist extraction
+
+// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text
+// Description: metadata.playlistMetadataRenderer.description
+// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text
+
+// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents
+// ID: .videoId
+// Title: title.runs[0].text
+// Author: .shortBylineText.runs[0].text
+// Duration: .lengthSeconds
+// Thumbnails .thumbnails
+
+// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails
+func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body []byte) (err error) {
+	var j *sjson.Json
+	j, err = sjson.NewJson(body)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		stack := debug.Stack()
+		if r := recover(); r != nil {
+			err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack)
+		}
+	}()
+
+	renderer := j.GetPath("alerts").GetIndex(0).GetPath("alertRenderer")
+	if renderer != nil && renderer.GetPath("type").MustString() == "ERROR" {
+		message := renderer.GetPath("text", "runs").GetIndex(0).GetPath("text").MustString()
+
+		return ErrPlaylistStatus{Reason: message}
+	}
+
+	// Metadata can be located in multiple places depending on client type
+	var metadata *sjson.Json
+	if node, ok := j.CheckGet("metadata"); ok {
+		metadata = node
+	} else if node, ok := j.CheckGet("header"); ok {
+		metadata = node
+	} else {
+		return fmt.Errorf("no playlist header / metadata found")
+	}
+
+	metadata = metadata.Get("playlistHeaderRenderer")
+
+	p.Title = sjsonGetText(metadata, "title")
+	p.Description = sjsonGetText(metadata, "description", "descriptionText")
+	p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1).
+		GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs").
+		GetIndex(0).Get("text").MustString()
+
+	if len(p.Author) == 0 {
+		p.Author = sjsonGetText(metadata, "owner", "ownerText")
+	}
+
+	contents, ok := j.CheckGet("contents")
+	if !ok {
+		return fmt.Errorf("contents not found in json body")
+	}
+
+	// contents can have different keys with same child structure
+	firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0).
+		GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0)
+
+	// This extra nested item is only set with the web client
+	if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) {
+		firstPart = n
+	}
+
+	vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON()
+	if err != nil {
+		return err
+	}
+
+	if len(vJSON) <= 4 {
+		return fmt.Errorf("no video data found in JSON")
+	}
+
+	entries, continuation, err := extractPlaylistEntries(vJSON)
+	if err != nil {
+		return err
+	}
+
+	if len(continuation) == 0 {
+		continuation = getContinuation(firstPart.Get("playlistVideoListRenderer"))
+	}
+
+	if len(entries) == 0 {
+		return fmt.Errorf("no videos found in playlist")
+	}
+
+	p.Videos = entries
+
+	for continuation != "" {
+		data := prepareInnertubePlaylistData(continuation, true, *client.client)
+
+		body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data)
+		if err != nil {
+			return err
+		}
+
+		j, err := sjson.NewJson(body)
+		if err != nil {
+			return err
+		}
+
+		next := j.GetPath("onResponseReceivedActions").GetIndex(0).
+			GetPath("appendContinuationItemsAction", "continuationItems")
+
+		if !isValidJSON(next) {
+			next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents")
+		}
+
+		vJSON, err := next.MarshalJSON()
+		if err != nil {
+			return err
+		}
+
+		entries, token, err := extractPlaylistEntries(vJSON)
+		if err != nil {
+			return err
+		}
+
+		if len(token) > 0 {
+			continuation = token
+		} else {
+			continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation"))
+		}
+
+		p.Videos = append(p.Videos, entries...)
+	}
+
+	return err
+}
+
+func extractPlaylistEntries(data []byte) ([]*PlaylistEntry, string, error) {
+	var vids []*videosJSONExtractor
+
+	if err := json.Unmarshal(data, &vids); err != nil {
+		return nil, "", err
+	}
+
+	entries := make([]*PlaylistEntry, 0, len(vids))
+
+	var continuation string
+	for _, v := range vids {
+		if v.Renderer == nil {
+			if v.Continuation.Endpoint.Command.Token != "" {
+				continuation = v.Continuation.Endpoint.Command.Token
+			}
+
+			continue
+		}
+
+		entries = append(entries, v.PlaylistEntry())
+	}
+
+	return entries, continuation, nil
+}
+
+type videosJSONExtractor struct {
+	Renderer *struct {
+		ID        string   `json:"videoId"`
+		Title     withRuns `json:"title"`
+		Author    withRuns `json:"shortBylineText"`
+		Duration  string   `json:"lengthSeconds"`
+		Thumbnail struct {
+			Thumbnails []Thumbnail `json:"thumbnails"`
+		} `json:"thumbnail"`
+	} `json:"playlistVideoRenderer"`
+	Continuation struct {
+		Endpoint struct {
+			Command struct {
+				Token string `json:"token"`
+			} `json:"continuationCommand"`
+		} `json:"continuationEndpoint"`
+	} `json:"continuationItemRenderer"`
+}
+
+func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry {
+	ds, err := strconv.Atoi(vje.Renderer.Duration)
+	if err != nil {
+		panic("invalid video duration: " + vje.Renderer.Duration)
+	}
+	return &PlaylistEntry{
+		ID:         vje.Renderer.ID,
+		Title:      vje.Renderer.Title.String(),
+		Author:     vje.Renderer.Author.String(),
+		Duration:   time.Second * time.Duration(ds),
+		Thumbnails: vje.Renderer.Thumbnail.Thumbnails,
+	}
+}
+
+type withRuns struct {
+	Runs []struct {
+		Text string `json:"text"`
+	} `json:"runs"`
+}
+
+func (wr withRuns) String() string {
+	if len(wr.Runs) > 0 {
+		return wr.Runs[0].Text
+	}
+	return ""
+}

downloader/vendor/github.com/kkdai/youtube/v2/response_data.go

@@ -0,0 +1,153 @@
+package youtube
+
+type playerResponseData struct {
+	Captions struct {
+		PlayerCaptionsTracklistRenderer struct {
+			CaptionTracks []CaptionTrack `json:"captionTracks"`
+			AudioTracks   []struct {
+				CaptionTrackIndices []int `json:"captionTrackIndices"`
+			} `json:"audioTracks"`
+			TranslationLanguages []struct {
+				LanguageCode string `json:"languageCode"`
+				LanguageName struct {
+					SimpleText string `json:"simpleText"`
+				} `json:"languageName"`
+			} `json:"translationLanguages"`
+			DefaultAudioTrackIndex int `json:"defaultAudioTrackIndex"`
+		} `json:"playerCaptionsTracklistRenderer"`
+	} `json:"captions"`
+
+	PlayabilityStatus struct {
+		Status          string `json:"status"`
+		Reason          string `json:"reason"`
+		PlayableInEmbed bool   `json:"playableInEmbed"`
+		Miniplayer      struct {
+			MiniplayerRenderer struct {
+				PlaybackMode string `json:"playbackMode"`
+			} `json:"miniplayerRenderer"`
+		} `json:"miniplayer"`
+		ContextParams string `json:"contextParams"`
+	} `json:"playabilityStatus"`
+	StreamingData struct {
+		ExpiresInSeconds string   `json:"expiresInSeconds"`
+		Formats          []Format `json:"formats"`
+		AdaptiveFormats  []Format `json:"adaptiveFormats"`
+		DashManifestURL  string   `json:"dashManifestUrl"`
+		HlsManifestURL   string   `json:"hlsManifestUrl"`
+	} `json:"streamingData"`
+	VideoDetails struct {
+		VideoID          string   `json:"videoId"`
+		Title            string   `json:"title"`
+		LengthSeconds    string   `json:"lengthSeconds"`
+		Keywords         []string `json:"keywords"`
+		ChannelID        string   `json:"channelId"`
+		IsOwnerViewing   bool     `json:"isOwnerViewing"`
+		ShortDescription string   `json:"shortDescription"`
+		IsCrawlable      bool     `json:"isCrawlable"`
+		Thumbnail        struct {
+			Thumbnails []Thumbnail `json:"thumbnails"`
+		} `json:"thumbnail"`
+		AverageRating     float64 `json:"averageRating"`
+		AllowRatings      bool    `json:"allowRatings"`
+		ViewCount         string  `json:"viewCount"`
+		Author            string  `json:"author"`
+		IsPrivate         bool    `json:"isPrivate"`
+		IsUnpluggedCorpus bool    `json:"isUnpluggedCorpus"`
+		IsLiveContent     bool    `json:"isLiveContent"`
+	} `json:"videoDetails"`
+	Microformat struct {
+		PlayerMicroformatRenderer struct {
+			Thumbnail struct {
+				Thumbnails []struct {
+					URL    string `json:"url"`
+					Width  int    `json:"width"`
+					Height int    `json:"height"`
+				} `json:"thumbnails"`
+			} `json:"thumbnail"`
+			Title struct {
+				SimpleText string `json:"simpleText"`
+			} `json:"title"`
+			Description struct {
+				SimpleText string `json:"simpleText"`
+			} `json:"description"`
+			LengthSeconds      string   `json:"lengthSeconds"`
+			OwnerProfileURL    string   `json:"ownerProfileUrl"`
+			ExternalChannelID  string   `json:"externalChannelId"`
+			IsFamilySafe       bool     `json:"isFamilySafe"`
+			AvailableCountries []string `json:"availableCountries"`
+			IsUnlisted         bool     `json:"isUnlisted"`
+			HasYpcMetadata     bool     `json:"hasYpcMetadata"`
+			ViewCount          string   `json:"viewCount"`
+			Category           string   `json:"category"`
+			PublishDate        string   `json:"publishDate"`
+			OwnerChannelName   string   `json:"ownerChannelName"`
+			UploadDate         string   `json:"uploadDate"`
+		} `json:"playerMicroformatRenderer"`
+	} `json:"microformat"`
+}
+
+type Format struct {
+	ItagNo           int    `json:"itag"`
+	URL              string `json:"url"`
+	MimeType         string `json:"mimeType"`
+	Quality          string `json:"quality"`
+	Cipher           string `json:"signatureCipher"`
+	Bitrate          int    `json:"bitrate"`
+	FPS              int    `json:"fps"`
+	Width            int    `json:"width"`
+	Height           int    `json:"height"`
+	LastModified     string `json:"lastModified"`
+	ContentLength    int64  `json:"contentLength,string"`
+	QualityLabel     string `json:"qualityLabel"`
+	ProjectionType   string `json:"projectionType"`
+	AverageBitrate   int    `json:"averageBitrate"`
+	AudioQuality     string `json:"audioQuality"`
+	ApproxDurationMs string `json:"approxDurationMs"`
+	AudioSampleRate  string `json:"audioSampleRate"`
+	AudioChannels    int    `json:"audioChannels"`
+
+	// InitRange is only available for adaptive formats
+	InitRange *struct {
+		Start string `json:"start"`
+		End   string `json:"end"`
+	} `json:"initRange"`
+
+	// IndexRange is only available for adaptive formats
+	IndexRange *struct {
+		Start string `json:"start"`
+		End   string `json:"end"`
+	} `json:"indexRange"`
+
+	// AudioTrack is only available for videos with multiple audio track languages
+	AudioTrack *struct {
+		DisplayName    string `json:"displayName"`
+		ID             string `json:"id"`
+		AudioIsDefault bool   `json:"audioIsDefault"`
+	}
+}
+
+func (f *Format) LanguageDisplayName() string {
+	if f.AudioTrack == nil {
+		return ""
+	}
+	return f.AudioTrack.DisplayName
+}
+
+type Thumbnails []Thumbnail
+
+type Thumbnail struct {
+	URL    string
+	Width  uint
+	Height uint
+}
+
+type CaptionTrack struct {
+	BaseURL string `json:"baseUrl"`
+	Name    struct {
+		SimpleText string `json:"simpleText"`
+	} `json:"name"`
+	VssID          string `json:"vssId"`
+	LanguageCode   string `json:"languageCode"`
+	Kind           string `json:"kind,omitempty"`
+	IsTranslatable bool   `json:"isTranslatable"`
+}

downloader/vendor/github.com/kkdai/youtube/v2/transcript.go

@@ -0,0 +1,214 @@
+package youtube
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+var (
+	ErrTranscriptDisabled = errors.New("transcript is disabled on this video")
+)
+
+// TranscriptSegment is a single transcipt segment spanning a few milliseconds.
+type TranscriptSegment struct {
+	// Text is the transcipt text.
+	Text string `json:"text"`
+
+	// StartMs is the start timestamp in ms.
+	StartMs int `json:"offset"`
+
+	// OffsetText e.g. '4:00'.
+	OffsetText string `json:"offsetText"`
+
+	// Duration the transcript segment spans in ms.
+	Duration int `json:"duration"`
+}
+
+func (tr TranscriptSegment) String() string {
+	return tr.OffsetText + " - " + strings.TrimSpace(tr.Text)
+}
+
+type VideoTranscript []TranscriptSegment
+
+func (vt VideoTranscript) String() string {
+	var str string
+	for _, tr := range vt {
+		str += tr.String() + "\n"
+	}
+
+	return str
+}
+
+// GetTranscript fetches the video transcript if available.
+//
+// Not all videos have transcripts, only relatively new videos.
+// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
+func (c *Client) GetTranscript(video *Video, lang string) (VideoTranscript, error) {
+	return c.GetTranscriptCtx(context.Background(), video, lang)
+}
+
+// GetTranscriptCtx fetches the video transcript if available.
+//
+// Not all videos have transcripts, only relatively new videos.
+// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
+func (c *Client) GetTranscriptCtx(ctx context.Context, video *Video, lang string) (VideoTranscript, error) {
+	c.assureClient()
+
+	if video == nil || video.ID == "" {
+		return nil, fmt.Errorf("no video provided")
+	}
+
+	body, err := c.transcriptDataByInnertube(ctx, video.ID, lang)
+	if err != nil {
+		return nil, err
+	}
+
+	transcript, err := parseTranscript(body)
+	if err != nil {
+		return nil, err
+	}
+
+	return transcript, nil
+}
+
+func parseTranscript(body []byte) (VideoTranscript, error) {
+	var resp transcriptResp
+	if err := json.Unmarshal(body, &resp); err != nil {
+		return nil, err
+	}
+
+	if len(resp.Actions) > 0 {
+		// Android client response
+		if app := resp.Actions[0].AppSegment; app != nil {
+			return getSegments(app)
+		}
+
+		// Web client response
+		if web := resp.Actions[0].WebSegment; web != nil {
+			return nil, fmt.Errorf("not implemented")
+		}
+	}
+
+	return nil, ErrTranscriptDisabled
+}
+
+type segmenter interface {
+	ParseSegments() []TranscriptSegment
+}
+
+func getSegments(f segmenter) (VideoTranscript, error) {
+	if segments := f.ParseSegments(); len(segments) > 0 {
+		return segments, nil
+	}
+
+	return nil, ErrTranscriptDisabled
+}
+
+// transcriptResp is the JSON structure as returned by the transcript API.
+type transcriptResp struct {
+	Actions []struct {
+		AppSegment *appData `json:"elementsCommand"`
+		WebSegment *webData `json:"updateEngagementPanelAction"`
+	} `json:"actions"`
+}
+
+type appData struct {
+	TEC struct {
+		Args struct {
+			ListArgs struct {
+				Ow struct {
+					InitialSeg []struct {
+						TranscriptSegment struct {
+							StartMs string `json:"startMs"`
+							EndMs   string `json:"endMs"`
+							Text    struct {
+								String struct {
+									// Content is the actual transctipt text
+									Content string `json:"content"`
+								} `json:"elementsAttributedString"`
+							} `json:"snippet"`
+							StartTimeText struct {
+								String struct {
+									// Content is the fomratted timestamp, e.g. '4:00'
+									Content string `json:"content"`
+								} `json:"elementsAttributedString"`
+							} `json:"startTimeText"`
+						} `json:"transcriptSegmentRenderer"`
+					} `json:"initialSegments"`
+				} `json:"overwrite"`
+			} `json:"transformTranscriptSegmentListArguments"`
+		} `json:"arguments"`
+	} `json:"transformEntityCommand"`
+}
+
+func (s *appData) ParseSegments() []TranscriptSegment {
+	rawSegments := s.TEC.Args.ListArgs.Ow.InitialSeg
+	segments := make([]TranscriptSegment, 0, len(rawSegments))
+
+	for _, segment := range rawSegments {
+		startMs, _ := strconv.Atoi(segment.TranscriptSegment.StartMs)
+		endMs, _ := strconv.Atoi(segment.TranscriptSegment.EndMs)
+
+		segments = append(segments, TranscriptSegment{
+			Text:       segment.TranscriptSegment.Text.String.Content,
+			StartMs:    startMs,
+			OffsetText: segment.TranscriptSegment.StartTimeText.String.Content,
+			Duration:   endMs - startMs,
+		})
+	}
+
+	return segments
+}
+
+type webData struct {
+	Content struct {
+		TR struct {
+			Body struct {
+				TBR struct {
+					Cues []struct {
+						Transcript struct {
+							FormattedStartOffset struct {
+								SimpleText string `json:"simpleText"`
+							} `json:"formattedStartOffset"`
+							Cues []struct {
+								TranscriptCueRenderer struct {
+									Cue struct {
+										SimpleText string `json:"simpleText"`
+									} `json:"cue"`
+									StartOffsetMs string `json:"startOffsetMs"`
+									DurationMs    string `json:"durationMs"`
+								} `json:"transcriptCueRenderer"`
+							} `json:"cues"`
+						} `json:"transcriptCueGroupRenderer"`
+					} `json:"cueGroups"`
+				} `json:"transcriptSearchPanelRenderer"`
+			} `json:"content"`
+		} `json:"transcriptRenderer"`
+	} `json:"content"`
+}
+
+func (s *webData) ParseSegments() []TranscriptSegment {
+	// TODO: doesn't actually work now, check json.
+	cues := s.Content.TR.Body.TBR.Cues
+	segments := make([]TranscriptSegment, 0, len(cues))
+
+	for _, s := range cues {
+		formatted := s.Transcript.FormattedStartOffset.SimpleText
+		segment := s.Transcript.Cues[0].TranscriptCueRenderer
+		start, _ := strconv.Atoi(segment.StartOffsetMs)
+		duration, _ := strconv.Atoi(segment.DurationMs)
+
+		segments = append(segments, TranscriptSegment{
+			Text:       segment.Cue.SimpleText,
+			StartMs:    start,
+			OffsetText: formatted,
+			Duration:   duration,
+		})
+	}
+
+	return segments
+}

downloader/vendor/github.com/kkdai/youtube/v2/utils.go

@@ -0,0 +1,97 @@
+package youtube
+
+import (
+	"encoding/base64"
+
+	sjson "github.com/bitly/go-simplejson"
+)
+
+type chunk struct {
+	start int64
+	end   int64
+	data  chan []byte
+}
+
+func getChunks(totalSize, chunkSize int64) []chunk {
+	var chunks []chunk
+
+	for start := int64(0); start < totalSize; start += chunkSize {
+		end := chunkSize + start - 1
+		if end > totalSize-1 {
+			end = totalSize - 1
+		}
+
+		chunks = append(chunks, chunk{start, end, make(chan []byte, 1)})
+	}
+
+	return chunks
+}
+
+func getFirstKeyJSON(j *sjson.Json) *sjson.Json {
+	m, err := j.Map()
+	if err != nil {
+		return j
+	}
+
+	for key := range m {
+		return j.Get(key)
+	}
+
+	return j
+}
+
+func isValidJSON(j *sjson.Json) bool {
+	b, err := j.MarshalJSON()
+	if err != nil {
+		return false
+	}
+
+	if len(b) <= 4 {
+		return false
+	}
+
+	return true
+}
+
+func sjsonGetText(j *sjson.Json, paths ...string) string {
+	for _, path := range paths {
+		if isValidJSON(j.Get(path)) {
+			j = j.Get(path)
+		}
+	}
+
+	if text, err := j.String(); err == nil {
+		return text
+	}
+
+	if isValidJSON(j.Get("text")) {
+		return j.Get("text").MustString()
+	}
+
+	if p := j.Get("runs"); isValidJSON(p) {
+		var text string
+
+		for i := 0; i < len(p.MustArray()); i++ {
+			if textNode := p.GetIndex(i).Get("text"); isValidJSON(textNode) {
+				text += textNode.MustString()
+			}
+		}
+
+		return text
+	}
+
+	return ""
+}
+
+func getContinuation(j *sjson.Json) string {
+	return j.GetPath("continuations").
+		GetIndex(0).GetPath("nextContinuationData", "continuation").MustString()
+}
+
+func base64PadEnc(str string) string {
+	return base64.StdEncoding.EncodeToString([]byte(str))
+}
+
+func base64Enc(str string) string {
+	return base64.RawStdEncoding.EncodeToString([]byte(str))
+}

downloader/vendor/github.com/kkdai/youtube/v2/video.go

@@ -0,0 +1,147 @@
+package youtube
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/url"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+type Video struct {
+	ID              string
+	Title           string
+	Description     string
+	Author          string
+	ChannelID       string
+	ChannelHandle   string
+	Views           int
+	Duration        time.Duration
+	PublishDate     time.Time
+	Formats         FormatList
+	Thumbnails      Thumbnails
+	DASHManifestURL string // URI of the DASH manifest file
+	HLSManifestURL  string // URI of the HLS manifest file
+	CaptionTracks   []CaptionTrack
+}
+
+const dateFormat = "2006-01-02"
+
+func (v *Video) parseVideoInfo(body []byte) error {
+	var prData playerResponseData
+	if err := json.Unmarshal(body, &prData); err != nil {
+		return fmt.Errorf("unable to parse player response JSON: %w", err)
+	}
+
+	if err := v.isVideoFromInfoDownloadable(prData); err != nil {
+		return err
+	}
+
+	return v.extractDataFromPlayerResponse(prData)
+}
+
+func (v *Video) isVideoFromInfoDownloadable(prData playerResponseData) error {
+	return v.isVideoDownloadable(prData, false)
+}
+
+var playerResponsePattern = regexp.MustCompile(`var ytInitialPlayerResponse\s*=\s*(\{.+?\});`)
+
+func (v *Video) parseVideoPage(body []byte) error {
+	initialPlayerResponse := playerResponsePattern.FindSubmatch(body)
+	if initialPlayerResponse == nil || len(initialPlayerResponse) < 2 {
+		return errors.New("no ytInitialPlayerResponse found in the server's answer")
+	}
+
+	var prData playerResponseData
+	if err := json.Unmarshal(initialPlayerResponse[1], &prData); err != nil {
+		return fmt.Errorf("unable to parse player response JSON: %w", err)
+	}
+
+	if err := v.isVideoFromPageDownloadable(prData); err != nil {
+		return err
+	}
+
+	return v.extractDataFromPlayerResponse(prData)
+}
+
+func (v *Video) isVideoFromPageDownloadable(prData playerResponseData) error {
+	return v.isVideoDownloadable(prData, true)
+}
+
+func (v *Video) isVideoDownloadable(prData playerResponseData, isVideoPage bool) error {
+	// Check if video is downloadable
+	switch prData.PlayabilityStatus.Status {
+	case "OK":
+		return nil
+	case "LOGIN_REQUIRED":
+		// for some reason they use same status message for age-restricted and private videos
+		if strings.HasPrefix(prData.PlayabilityStatus.Reason, "This video is private") {
+			return ErrVideoPrivate
+		}
+		return ErrLoginRequired
+	}
+
+	if !isVideoPage && !prData.PlayabilityStatus.PlayableInEmbed {
+		return ErrNotPlayableInEmbed
+	}
+
+	return &ErrPlayabiltyStatus{
+		Status: prData.PlayabilityStatus.Status,
+		Reason: prData.PlayabilityStatus.Reason,
+	}
+}
+
+func (v *Video) extractDataFromPlayerResponse(prData playerResponseData) error {
+	v.Title = prData.VideoDetails.Title
+	v.Description = prData.VideoDetails.ShortDescription
+	v.Author = prData.VideoDetails.Author
+	v.Thumbnails = prData.VideoDetails.Thumbnail.Thumbnails
+	v.ChannelID = prData.VideoDetails.ChannelID
+	v.CaptionTracks = prData.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks
+
+	if views, _ := strconv.Atoi(prData.VideoDetails.ViewCount); views > 0 {
+		v.Views = views
+	}
+
+	if seconds, _ := strconv.Atoi(prData.VideoDetails.LengthSeconds); seconds > 0 {
+		v.Duration = time.Duration(seconds) * time.Second
+	}
+
+	if seconds, _ := strconv.Atoi(prData.Microformat.PlayerMicroformatRenderer.LengthSeconds); seconds > 0 {
+		v.Duration = time.Duration(seconds) * time.Second
+	}
+
+	if str := prData.Microformat.PlayerMicroformatRenderer.PublishDate; str != "" {
+		v.PublishDate, _ = time.Parse(dateFormat, str)
+	}
+
+	if profileURL, err := url.Parse(prData.Microformat.PlayerMicroformatRenderer.OwnerProfileURL); err == nil && len(profileURL.Path) > 1 {
+		v.ChannelHandle = profileURL.Path[1:]
+	}
+
+	// Assign Streams
+	v.Formats = append(prData.StreamingData.Formats, prData.StreamingData.AdaptiveFormats...)
+	if len(v.Formats) == 0 {
+		return errors.New("no formats found in the server's answer")
+	}
+
+	// Sort formats by bitrate
+	sort.SliceStable(v.Formats, v.SortBitrateDesc)
+
+	v.HLSManifestURL = prData.StreamingData.HlsManifestURL
+	v.DASHManifestURL = prData.StreamingData.DashManifestURL
+
+	return nil
+}
+
+func (v *Video) SortBitrateDesc(i int, j int) bool {
+	return v.Formats[i].Bitrate > v.Formats[j].Bitrate
+}
+
+func (v *Video) SortBitrateAsc(i int, j int) bool {
+	return v.Formats[i].Bitrate < v.Formats[j].Bitrate
+}

downloader/vendor/github.com/kkdai/youtube/v2/video_id.go

@@ -0,0 +1,34 @@
+package youtube
+
+import (
+	"regexp"
+	"strings"
+)
+
+var videoRegexpList = []*regexp.Regexp{
+	regexp.MustCompile(`(?:v|embed|shorts|watch\?v)(?:=|/)([^"&?/=%]{11})`),
+	regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`),
+	regexp.MustCompile(`([^"&?/=%]{11})`),
+}
+
+// ExtractVideoID extracts the videoID from the given string
+func ExtractVideoID(videoID string) (string, error) {
+	if strings.Contains(videoID, "youtu") || strings.ContainsAny(videoID, "\"?&/<%=") {
+		for _, re := range videoRegexpList {
+			if isMatch := re.MatchString(videoID); isMatch {
+				subs := re.FindStringSubmatch(videoID)
+				videoID = subs[1]
+			}
+		}
+	}
+
+	if strings.ContainsAny(videoID, "?&/<%=") {
+		return "", ErrInvalidCharactersInVideoID
+	}
+
+	if len(videoID) < 10 {
+		return "", ErrVideoIDMinLength
+	}
+
+	return videoID, nil
+}

hotkey.js

@@ -22,7 +22,7 @@ if (!(titleNode && titleNode.matches(selector))) {
 		link = window.location.origin + link;
 	}
 	console.log(link);
-	fetch("http://localhost:5000/download", {
+	fetch("https://nsq.site.quack-lab.dev/pub?topic=ytdqueue", {
 		method: "POST",
 		headers: {
 			"Content-Type": "application/json",