Implement writing urls to nsq for downloading

.gitignore

@@ -9,3 +9,5 @@ downloader/vendor/github.com/*
 downloader/vendor/modules.txt
 downloader/ytdl.exe
 downloader/ytdl.log
+*.log
+*.xml

downloader/discordNotifier.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+)
+
+func NotifyDiscordErrorless(message string) {
+	err := NotifyDiscord(message)
+	if err != nil {
+		log.Printf("Error notifying discord: %v", err)
+	}
+}
+
+func NotifyDiscord(message string) error {
+	webhookURL := os.Getenv("YTDL_DISCORD_WEBHOOK_URL")
+	if webhookURL == "" {
+		return fmt.Errorf("error notifying discord: webhook URL is not set in environment variables")
+	}
+
+	jsonData := map[string]string{"content": message}
+	jsonBytes, err := json.Marshal(jsonData)
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error marshalling JSON: %v", err)
+	}
+	req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(jsonBytes))
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error creating request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error sending request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error reading response body: %v", err)
+	}
+
+	log.Printf("Response from Discord: %s", string(body))
+	return nil
+}

downloader/downloaders/utils.go

@@ -8,4 +8,4 @@ const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
 var ongoingDownloads = make(map[string]struct{})
 var ongoingDownloadsMutex = &sync.Mutex{}
 
-const DOWNLOAD_WORKERS = 2
+const DOWNLOAD_WORKERS = 4

downloader/main.go

@@ -6,11 +6,11 @@ import (
 	"fmt"
 	"io"
 	"log"
-	"ytdl/downloaders"
 	"os"
 	"os/signal"
 	"syscall"
 	"time"
+	"ytdl/downloaders"
 
 	"github.com/nsqio/go-nsq"
 )
@@ -36,18 +36,21 @@ func init() {
 		log.Lmicroseconds|log.Lshortfile)
 }
 
-//var downloader downloaders.Downloader = &downloaders.YTDLPRawDownloader{}
+// var downloader downloaders.Downloader = &downloaders.YTDLPRawDownloader{}
-var downloader downloaders.Downloader = &downloaders.KidaiDownloader{}
+// var downloader downloaders.Downloader = &downloaders.KidaiDownloader{}
-//var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
+var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
 
 type DLHandler struct{}
 
 func (*DLHandler) HandleMessage(message *nsq.Message) error {
 	log.Printf("Received message '%s' with %d attempts", message.Body, message.Attempts)
+	NotifyDiscordErrorless(fmt.Sprintf("Received message '%s' with %d attempts", message.Body, message.Attempts))
+
 	data := DownloadRequest{}
 	err := json.Unmarshal(message.Body, &data)
 	if err != nil {
 		Error.Printf("Error unmarshalling message: %v", err)
+		NotifyDiscordErrorless(fmt.Sprintf("Error unmarshalling message: %v", err))
 		return err
 	}
 
@@ -71,18 +74,20 @@ func (*DLHandler) HandleMessage(message *nsq.Message) error {
 	err = downloader.Download(data.Link)
 	if err != nil {
 		Error.Printf("Error downloading %s: %v", data.Link, err)
+		NotifyDiscordErrorless(fmt.Sprintf("Error downloading %s: %v", data.Link, err))
 		return err
 	}
 	message.Finish()
+	NotifyDiscordErrorless(fmt.Sprintf("Downloaded %s", data.Link))
 	return nil
 }
 
 func main() {
-//	err := downloader.Download("https://www.youtube.com/watch?v=SiKjprtiPaw")
+	//	err := downloader.Download("https://www.youtube.com/watch?v=SiKjprtiPaw")
-//	if err != nil {
+	//	if err != nil {
-//		Error.Printf("Error downloading: %v", err)
+	//		Error.Printf("Error downloading: %v", err)
-//	}
+	//	}
-//	return
+	//	return
 
 	config := nsq.NewConfig()
 	config.MaxAttempts = 5
@@ -98,11 +103,14 @@ func main() {
 		consumer.AddHandler(&DLHandler{})
 	}
 
-	err = consumer.ConnectToNSQD("nsq.site.quack-lab.dev:41505")
+	url := "192.168.1.123:41505"
+	log.Printf("Connecting to nsq at %s", url)
+	err = consumer.ConnectToNSQD(url)
 	if err != nil {
-		Error.Printf("Error connecting to nsqlookupd: %v", err)
+		Error.Printf("Error connecting to nsq: %v", err)
 		return
 	}
+	log.Printf("Connected to nsq at %s", url)
 
 	sigChan := make(chan os.Signal, 1)
 	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

downloader/vendor/github.com/kkdai/youtube/v2/.gitignore

@@ -1,8 +0,0 @@
-/.idea
-/.vscode
-download_test
-/bin
-/dist
-/output
-*.out
-.DS_Store

downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml

@@ -1,8 +0,0 @@
-issues:
-  exclude:
-  - Subprocess launched with function call
-  # Excluding configuration per-path, per-linter, per-text and per-source
-  exclude-rules:
-    # Exclude some linters from running on tests files.
-    - path: cmd/
-      text: InsecureSkipVerify

downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml

@@ -1,20 +0,0 @@
-project_name: youtubedr
-
-env:
-  # Build without CGO to don't depend on specific glibc versions
-  - CGO_ENABLED=0
-
-builds:
-  - main: ./cmd/youtubedr
-    binary: youtubedr
-    goos:
-      - windows
-      - darwin
-      - linux
-      - freebsd
-    goarch:
-      - amd64
-      - arm
-      - arm64
-    flags:
-      - -trimpath

downloader/vendor/github.com/kkdai/youtube/v2/LICENSE

@@ -1,22 +0,0 @@
-The MIT License (MIT)
-
-Copyright (c) 2015 Evan Lin
-
-Permission is hereby granted, free of charge, to any person obtaining a copy
-of this software and associated documentation files (the "Software"), to deal
-in the Software without restriction, including without limitation the rights
-to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
-copies of the Software, and to permit persons to whom the Software is
-furnished to do so, subject to the following conditions:
-
-The above copyright notice and this permission notice shall be included in all
-copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
-AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
-LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
-OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
-SOFTWARE.
-

downloader/vendor/github.com/kkdai/youtube/v2/Makefile

@@ -1,57 +0,0 @@
-FILES_TO_FMT      ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print)
-LOGLEVEL	?= debug
-
-## help: Show makefile commands
-.PHONY: help
-help: Makefile
-	@echo "---- Project: kkdai/youtube ----"
-	@echo " Usage: make COMMAND"
-	@echo
-	@echo " Management Commands:"
-	@sed -n 's/^##//p' $< | column -t -s ':' |  sed -e 's/^/ /'
-	@echo
-
-## build: Build project
-.PHONY: build
-build:
-	goreleaser --rm-dist
-
-## deps: Ensures fresh go.mod and go.sum
-.PHONY: deps
-deps:
-	go mod tidy
-	go mod verify
-
-## lint: Run golangci-lint check
-.PHONY: lint
-lint:
-	command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
-	echo "golangci-lint checking..."
-	golangci-lint run --timeout=30m --enable=misspell --enable=gosec --enable=gofmt --enable=goimports --enable=revive ./cmd/... ./...
-	go vet ./...
-
-## format: Formats Go code
-.PHONY: format
-format:
-	@echo ">> formatting code"
-	@gofmt -s -w $(FILES_TO_FMT)
-
-## test-unit: Run all Youtube Go unit tests
-.PHONY: test-unit
-test-unit:
-	LOGLEVEL=${LOGLEVEL} go test -v -cover ./...
-
-## test-integration: Run all Youtube Go integration tests
-.PHONY: test-integration
-test-integration:
-	mkdir -p output
-	rm -f output/*
-	LOGLEVEL=${LOGLEVEL} ARTIFACTS=output go test -v -race -covermode=atomic -coverprofile=coverage.out -tags=integration ./...
-
-.PHONY: coverage.out
-coverage.out:
-
-## clean: Clean files and downloaded videos from builds during development
-.PHONY: clean
-clean:
-	rm -rf dist *.mp4 *.mkv

downloader/vendor/github.com/kkdai/youtube/v2/README.md

@@ -1,161 +0,0 @@
-Download Youtube Video in Golang
-==================
-
-[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/kkdai/youtube/master/LICENSE)
-[![Go Reference](https://pkg.go.dev/badge/github.com/kkdai/youtube.svg)](https://pkg.go.dev/github.com/kkdai/youtube/v2)
-[![Build Status](https://github.com/kkdai/youtube/workflows/go/badge.svg?branch=master)](https://github.com/kkdai/youtube/actions)
-[![Coverage](https://codecov.io/gh/kkdai/youtube/branch/master/graph/badge.svg)](https://codecov.io/gh/kkdai/youtube)
-[![](https://goreportcard.com/badge/github.com/kkdai/youtube)](https://goreportcard.com/badge/github.com/kkdai/youtube)
-
-
-This package is a Youtube video download package, for more detail refer [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) for more download options.
-
-This tool is meant to be used to download CC0 licenced content, we do not support nor recommend using it for illegal activities.
-
-## Overview
-  * [Install](#installation)
-  * [Usage](#usage)
-  * [Example: Download video from \[dotGo 2015 - Rob Pike - Simplicity is Complicated\]](#download-dotGo-2015-rob-pike-video)
-
-## Installation
-
-### Install via go get
-
-Please ensure you have installed Go 1.22 or later.
-
-```shell
-go get github.com/kkdai/youtube/v2
-```
-
-### From source code
-
-```shell
-git clone https://github.com/kkdai/youtube.git
-cd youtube
-go run ./cmd/youtubedr
-```
-
-### Mac
-
-```shell
-brew install youtubedr
-```
-
-### in Termux
-```shell
-pkg install youtubedr
-```
-###  You can also find this package in
-- [archlinux](https://aur.archlinux.org/packages/youtubedr/)  (thanks to [cjsthompson](https://github.com/cjsthompson))
-- [Termux package](https://github.com/termux/termux-packages/tree/master/packages/youtubedr) (thanks to [kcubeterm](https://github.com/kcubeterm))
-- [Homebrew](https://formulae.brew.sh/formula/youtubedr) (thanks to [kkc](https://github.com/kkc))
-
-## Usage
-
-### Use the binary directly
-It's really simple to use, just get the video id from youtube url - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
-
-```shell
-$ youtubedr download QAGDGja7kbs
-$ youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
-```
-
-
-### Use this package in your golang program
-
-Please check out the [example_test.go](example_test.go) for example code.
-
-
-## Example:
- * ### Get information of dotGo-2015-rob-pike video for downloading
-
-    `go get github.com/kkdai/youtube/v2/youtubedr`
-
-    Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
-
-    ```
-    youtubedr info https://www.youtube.com/watch?v=rFejpH_tAHM
-
-   Title: dotGo 2015 - Rob Pike - Simplicity is Complicated
-   Author: dotconferences
-   -----available streams-----
-   itag:  18 , quality: medium , type: video/mp4; codecs="avc1.42001E, mp4a.40.2"
-   itag:  22 , quality:  hd720 , type: video/mp4; codecs="avc1.64001F, mp4a.40.2"
-   itag: 137 , quality: hd1080 , type: video/mp4; codecs="avc1.640028"
-   itag: 248 , quality: hd1080 , type: video/webm; codecs="vp9"
-   ........
-    ```
- * ### Download dotGo-2015-rob-pike-video
-
-    `go get github.com/kkdai/youtube/v2/youtubedr`
-
-    Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
-
-    ```
-    youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
-    ```
-
- * ### Download video to specific folder and name
-
-	`go get github.com/kkdai/youtube/v2/youtubedr`
-
-	Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) to current directory and name the file to simplicity-is-complicated.mp4
-
-	```
-	youtubedr download -d ./ -o simplicity-is-complicated.mp4 https://www.youtube.com/watch?v=rFejpH_tAHM
-	```
-
- * ### Download video with specific quality
-
-	`go get github.com/kkdai/youtube/v2/youtubedr`
-
-	Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) with specific quality
-
-	```
-	youtubedr download -q medium https://www.youtube.com/watch?v=rFejpH_tAHM
-	```
-
-   #### Special case by quality hd1080:
-   Installation of ffmpeg is necessary for hd1080
-   ```
-   ffmpeg   //check ffmpeg is installed, if not please download ffmpeg and set to your PATH.
-   youtubedr download -q hd1080 https://www.youtube.com/watch?v=rFejpH_tAHM
-   ```
-
-
- * ### Download video with specific itag
-
-    `go get github.com/kkdai/youtube/v2/youtubedr`
-
-    Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
-
-    ```
-    youtubedr download -q 18 https://www.youtube.com/watch?v=rFejpH_tAHM
-    ```
-
-## How it works
-
-- Parse the video ID you input in URL
-	- ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
-- Get video information via video id.
-	- Use URL: `http://youtube.com/get_video_info?video_id=`
-- Parse and decode video information.
-	- Download URL in "url="
-	- title in "title="
-- Download video from URL
-	- Need the string combination of "url"
-
-## Inspired
-- [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl)
-- [https://github.com/lepidosteus/youtube-dl](https://github.com/lepidosteus/youtube-dl)
-- [拆解 Youtube 影片下載位置](http://hkgoldenmra.blogspot.tw/2013/05/youtube.html)
-- [iawia002/annie](https://github.com/iawia002/annie)
-- [How to get url from obfuscate video info: youtube video downloader with php](https://stackoverflow.com/questions/60607291/youtube-video-downloader-with-php)
-
-
-## Project52
-It is one of my [project 52](https://github.com/kkdai/project52).
-
-
-## License
-This package is licensed under MIT license. See LICENSE for details.

downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md

@@ -1,11 +0,0 @@
-Tracking Youtube decipher change
-
-### 2022/01/21:
-
-#### action objects:
-
-var $z={fA:function(a){a.reverse()},S2:function(a,b){a.splice(0,b)},l6:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
-#### actions function:
-
-Npa=function(a){a=a.split("");$z.S2(a,3);$z.fA(a,45);$z.l6(a,31);$z.S2(a,1);$z.fA(a,63);$z.S2(a,2);$z.fA(a,68);return a.join("")};
-

downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go

@@ -1,29 +0,0 @@
-package youtube
-
-import (
-	"log/slog"
-	"os"
-	"path/filepath"
-)
-
-// destination for artifacts, used by integration tests
-var artifactsFolder = os.Getenv("ARTIFACTS")
-
-func writeArtifact(name string, content []byte) {
-	// Ensure folder exists
-	err := os.MkdirAll(artifactsFolder, os.ModePerm)
-	if err != nil {
-		slog.Error("unable to create artifacts folder", "path", artifactsFolder, "error", err)
-		return
-	}
-
-	path := filepath.Join(artifactsFolder, name)
-	err = os.WriteFile(path, content, 0600)
-
-	log := slog.With("path", path)
-	if err != nil {
-		log.Error("unable to write artifact", "error", err)
-	} else {
-		log.Debug("artifact created")
-	}
-}

downloader/vendor/github.com/kkdai/youtube/v2/client.go

@@ -1,632 +0,0 @@
-package youtube
-
-import (
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"log/slog"
-	"math/rand"
-	"net/http"
-	"net/url"
-	"strconv"
-	"sync/atomic"
-)
-
-const (
-	Size1Kb  = 1024
-	Size1Mb  = Size1Kb * 1024
-	Size10Mb = Size1Mb * 10
-
-	playerParams = "CgIQBg=="
-)
-
-var ErrNoFormat = errors.New("no video format provided")
-
-// DefaultClient type to use. No reason to change but you could if you wanted to.
-var DefaultClient = IOSClient
-
-// Client offers methods to download video metadata and video streams.
-type Client struct {
-	// HTTPClient can be used to set a custom HTTP client.
-	// If not set, http.DefaultClient will be used
-	HTTPClient *http.Client
-
-	// MaxRoutines to use when downloading a video.
-	MaxRoutines int
-
-	// ChunkSize to use when downloading videos in chunks. Default is Size10Mb.
-	ChunkSize int64
-
-	// playerCache caches the JavaScript code of a player response
-	playerCache playerCache
-
-	client *clientInfo
-
-	consentID string
-}
-
-func (c *Client) assureClient() {
-	if c.client == nil {
-		c.client = &DefaultClient
-	}
-}
-
-// GetVideo fetches video metadata
-func (c *Client) GetVideo(url string) (*Video, error) {
-	return c.GetVideoContext(context.Background(), url)
-}
-
-// GetVideoContext fetches video metadata with a context
-func (c *Client) GetVideoContext(ctx context.Context, url string) (*Video, error) {
-	id, err := ExtractVideoID(url)
-	if err != nil {
-		return nil, fmt.Errorf("extractVideoID failed: %w", err)
-	}
-
-	return c.videoFromID(ctx, id)
-}
-
-func (c *Client) videoFromID(ctx context.Context, id string) (*Video, error) {
-	c.assureClient()
-
-	body, err := c.videoDataByInnertube(ctx, id)
-	if err != nil {
-		return nil, err
-	}
-
-	v := Video{
-		ID: id,
-	}
-
-	// return early if all good
-	if err = v.parseVideoInfo(body); err == nil {
-		return &v, nil
-	}
-
-	// If the uploader has disabled embedding the video on other sites, parse video page
-	if errors.Is(err, ErrNotPlayableInEmbed) {
-		// additional parameters are required to access clips with sensitiv content
-		html, err := c.httpGetBodyBytes(ctx, "https://www.youtube.com/watch?v="+id+"&bpctr=9999999999&has_verified=1")
-		if err != nil {
-			return nil, err
-		}
-
-		return &v, v.parseVideoPage(html)
-	}
-
-	// If the uploader marked the video as inappropriate for some ages, use embed player
-	if errors.Is(err, ErrLoginRequired) {
-		c.client = &EmbeddedClient
-
-		bodyEmbed, errEmbed := c.videoDataByInnertube(ctx, id)
-		if errEmbed == nil {
-			errEmbed = v.parseVideoInfo(bodyEmbed)
-		}
-
-		if errEmbed == nil {
-			return &v, nil
-		}
-
-		// private video clearly not age-restricted and thus should be explicit
-		if errEmbed == ErrVideoPrivate {
-			return &v, errEmbed
-		}
-
-		// wrapping error so its clear whats happened
-		return &v, fmt.Errorf("can't bypass age restriction: %w", errEmbed)
-	}
-
-	// undefined error
-	return &v, err
-}
-
-type innertubeRequest struct {
-	VideoID         string            `json:"videoId,omitempty"`
-	BrowseID        string            `json:"browseId,omitempty"`
-	Continuation    string            `json:"continuation,omitempty"`
-	Context         inntertubeContext `json:"context"`
-	PlaybackContext *playbackContext  `json:"playbackContext,omitempty"`
-	ContentCheckOK  bool              `json:"contentCheckOk,omitempty"`
-	RacyCheckOk     bool              `json:"racyCheckOk,omitempty"`
-	Params          string            `json:"params"`
-}
-
-type playbackContext struct {
-	ContentPlaybackContext contentPlaybackContext `json:"contentPlaybackContext"`
-}
-
-type contentPlaybackContext struct {
-	// SignatureTimestamp string `json:"signatureTimestamp"`
-	HTML5Preference string `json:"html5Preference"`
-}
-
-type inntertubeContext struct {
-	Client innertubeClient `json:"client"`
-}
-
-type innertubeClient struct {
-	HL                string `json:"hl"`
-	GL                string `json:"gl"`
-	ClientName        string `json:"clientName"`
-	ClientVersion     string `json:"clientVersion"`
-	AndroidSDKVersion int    `json:"androidSDKVersion,omitempty"`
-	UserAgent         string `json:"userAgent,omitempty"`
-	TimeZone          string `json:"timeZone"`
-	UTCOffset         int    `json:"utcOffsetMinutes"`
-	DeviceModel       string `json:"deviceModel,omitempty"`
-}
-
-// client info for the innertube API
-type clientInfo struct {
-	name           string
-	key            string
-	version        string
-	userAgent      string
-	androidVersion int
-	deviceModel    string
-}
-
-var (
-	// WebClient, better to use Android client but go ahead.
-	WebClient = clientInfo{
-		name:      "WEB",
-		version:   "2.20220801.00.00",
-		key:       "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
-		userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-	}
-
-	// AndroidClient, download go brrrrrr.
-	AndroidClient = clientInfo{
-		name:           "ANDROID",
-		version:        "18.11.34",
-		key:            "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
-		userAgent:      "com.google.android.youtube/18.11.34 (Linux; U; Android 11) gzip",
-		androidVersion: 30,
-	}
-
-	// IOSClient Client based brrrr.
-	IOSClient = clientInfo{
-		name:        "IOS",
-		version:     "19.45.4",
-		key:         "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
-		userAgent:   "com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)",
-		deviceModel: "iPhone16,2",
-	}
-
-	// EmbeddedClient, not really tested.
-	EmbeddedClient = clientInfo{
-		name:      "WEB_EMBEDDED_PLAYER",
-		version:   "1.19700101",
-		key:       "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", // seems like same key works for both clients
-		userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
-	}
-)
-
-func (c *Client) videoDataByInnertube(ctx context.Context, id string) ([]byte, error) {
-	data := innertubeRequest{
-		VideoID:        id,
-		Context:        prepareInnertubeContext(*c.client),
-		ContentCheckOK: true,
-		RacyCheckOk:    true,
-		// Params:                   playerParams,
-		PlaybackContext: &playbackContext{
-			ContentPlaybackContext: contentPlaybackContext{
-				// SignatureTimestamp: sts,
-				HTML5Preference: "HTML5_PREF_WANTS",
-			},
-		},
-	}
-
-	return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/player?key="+c.client.key, data)
-}
-
-func (c *Client) transcriptDataByInnertube(ctx context.Context, id string, lang string) ([]byte, error) {
-	data := innertubeRequest{
-		Context: prepareInnertubeContext(*c.client),
-		Params:  transcriptVideoID(id, lang),
-	}
-
-	return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/get_transcript?key="+c.client.key, data)
-}
-
-func prepareInnertubeContext(clientInfo clientInfo) inntertubeContext {
-	return inntertubeContext{
-		Client: innertubeClient{
-			HL:                "en",
-			GL:                "US",
-			TimeZone:          "UTC",
-			DeviceModel:       clientInfo.deviceModel,
-			ClientName:        clientInfo.name,
-			ClientVersion:     clientInfo.version,
-			AndroidSDKVersion: clientInfo.androidVersion,
-			UserAgent:         clientInfo.userAgent,
-		},
-	}
-}
-
-func prepareInnertubePlaylistData(ID string, continuation bool, clientInfo clientInfo) innertubeRequest {
-	context := prepareInnertubeContext(clientInfo)
-
-	if continuation {
-		return innertubeRequest{
-			Context:        context,
-			Continuation:   ID,
-			ContentCheckOK: true,
-			RacyCheckOk:    true,
-			Params:         playerParams,
-		}
-	}
-
-	return innertubeRequest{
-		Context:        context,
-		BrowseID:       "VL" + ID,
-		ContentCheckOK: true,
-		RacyCheckOk:    true,
-		Params:         playerParams,
-	}
-}
-
-// transcriptVideoID encodes the video ID to the param used to fetch transcripts.
-func transcriptVideoID(videoID string, lang string) string {
-	langCode := encTranscriptLang(lang)
-
-	// This can be optionally appened to the Sprintf str, not sure what it means
-	// *3engagement-panel-searchable-transcript-search-panel\x30\x00\x38\x01\x40\x01
-	return base64Enc(fmt.Sprintf("\n\x0b%s\x12\x12%s\x18\x01", videoID, langCode))
-}
-
-func encTranscriptLang(languageCode string) string {
-	s := fmt.Sprintf("\n\x03asr\x12\x02%s\x1a\x00", languageCode)
-	s = base64PadEnc(s)
-
-	return url.QueryEscape(s)
-}
-
-// GetPlaylist fetches playlist metadata
-func (c *Client) GetPlaylist(url string) (*Playlist, error) {
-	return c.GetPlaylistContext(context.Background(), url)
-}
-
-// GetPlaylistContext fetches playlist metadata, with a context, along with a list of Videos, and some basic information
-// for these videos. Playlist entries cannot be downloaded, as they lack all the required metadata, but
-// can be used to enumerate all IDs, Authors, Titles, etc.
-func (c *Client) GetPlaylistContext(ctx context.Context, url string) (*Playlist, error) {
-	c.assureClient()
-
-	id, err := extractPlaylistID(url)
-	if err != nil {
-		return nil, fmt.Errorf("extractPlaylistID failed: %w", err)
-	}
-
-	data := prepareInnertubePlaylistData(id, false, *c.client)
-	body, err := c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+c.client.key, data)
-	if err != nil {
-		return nil, err
-	}
-
-	p := &Playlist{ID: id}
-	return p, p.parsePlaylistInfo(ctx, c, body)
-}
-
-func (c *Client) VideoFromPlaylistEntry(entry *PlaylistEntry) (*Video, error) {
-	return c.videoFromID(context.Background(), entry.ID)
-}
-
-func (c *Client) VideoFromPlaylistEntryContext(ctx context.Context, entry *PlaylistEntry) (*Video, error) {
-	return c.videoFromID(ctx, entry.ID)
-}
-
-// GetStream returns the stream and the total size for a specific format
-func (c *Client) GetStream(video *Video, format *Format) (io.ReadCloser, int64, error) {
-	return c.GetStreamContext(context.Background(), video, format)
-}
-
-// GetStreamContext returns the stream and the total size for a specific format with a context.
-func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *Format) (io.ReadCloser, int64, error) {
-	url, err := c.GetStreamURL(video, format)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, 0, err
-	}
-
-	r, w := io.Pipe()
-	contentLength := format.ContentLength
-
-	if contentLength == 0 {
-		// some videos don't have length information
-		contentLength = c.downloadOnce(req, w, format)
-	} else {
-		// we have length information, let's download by chunks!
-		c.downloadChunked(ctx, req, w, format)
-	}
-
-	return r, contentLength, nil
-}
-
-func (c *Client) downloadOnce(req *http.Request, w *io.PipeWriter, _ *Format) int64 {
-	resp, err := c.httpDo(req)
-	if err != nil {
-		w.CloseWithError(err) //nolint:errcheck
-		return 0
-	}
-
-	go func() {
-		defer resp.Body.Close()
-		_, err := io.Copy(w, resp.Body)
-		if err == nil {
-			w.Close()
-		} else {
-			w.CloseWithError(err) //nolint:errcheck
-		}
-	}()
-
-	contentLength := resp.Header.Get("Content-Length")
-	length, _ := strconv.ParseInt(contentLength, 10, 64)
-
-	return length
-}
-
-func (c *Client) getChunkSize() int64 {
-	if c.ChunkSize > 0 {
-		return c.ChunkSize
-	}
-
-	return Size10Mb
-}
-
-func (c *Client) getMaxRoutines(limit int) int {
-	routines := 10
-
-	if c.MaxRoutines > 0 {
-		routines = c.MaxRoutines
-	}
-
-	if limit > 0 && routines > limit {
-		routines = limit
-	}
-
-	return routines
-}
-
-func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.PipeWriter, format *Format) {
-	chunks := getChunks(format.ContentLength, c.getChunkSize())
-	maxRoutines := c.getMaxRoutines(len(chunks))
-
-	cancelCtx, cancel := context.WithCancel(ctx)
-	abort := func(err error) {
-		w.CloseWithError(err)
-		cancel()
-	}
-
-	currentChunk := atomic.Uint32{}
-	for i := 0; i < maxRoutines; i++ {
-		go func() {
-			for {
-				chunkIndex := int(currentChunk.Add(1)) - 1
-				if chunkIndex >= len(chunks) {
-					// no more chunks
-					return
-				}
-
-				chunk := &chunks[chunkIndex]
-				err := c.downloadChunk(req.Clone(cancelCtx), chunk)
-				close(chunk.data)
-
-				if err != nil {
-					abort(err)
-					return
-				}
-			}
-		}()
-	}
-
-	go func() {
-		// copy chunks into the PipeWriter
-		for i := 0; i < len(chunks); i++ {
-			select {
-			case <-cancelCtx.Done():
-				abort(context.Canceled)
-				return
-			case data := <-chunks[i].data:
-				_, err := io.Copy(w, bytes.NewBuffer(data))
-				if err != nil {
-					abort(err)
-				}
-			}
-		}
-
-		// everything succeeded
-		w.Close()
-	}()
-}
-
-// GetStreamURL returns the url for a specific format
-func (c *Client) GetStreamURL(video *Video, format *Format) (string, error) {
-	return c.GetStreamURLContext(context.Background(), video, format)
-}
-
-// GetStreamURLContext returns the url for a specific format with a context
-func (c *Client) GetStreamURLContext(ctx context.Context, video *Video, format *Format) (string, error) {
-	if format == nil {
-		return "", ErrNoFormat
-	}
-
-	c.assureClient()
-
-	if format.URL != "" {
-		if c.client.androidVersion > 0 {
-			return format.URL, nil
-		}
-
-		return c.unThrottle(ctx, video.ID, format.URL)
-	}
-
-	// TODO: check rest of this function, is it redundant?
-
-	cipher := format.Cipher
-	if cipher == "" {
-		return "", ErrCipherNotFound
-	}
-
-	uri, err := c.decipherURL(ctx, video.ID, cipher)
-	if err != nil {
-		return "", err
-	}
-
-	return uri, err
-}
-
-// httpDo sends an HTTP request and returns an HTTP response.
-func (c *Client) httpDo(req *http.Request) (*http.Response, error) {
-	client := c.HTTPClient
-	if client == nil {
-		client = http.DefaultClient
-	}
-
-	req.Header.Set("User-Agent", c.client.userAgent)
-	req.Header.Set("Origin", "https://youtube.com")
-	req.Header.Set("Sec-Fetch-Mode", "navigate")
-
-	if len(c.consentID) == 0 {
-		c.consentID = strconv.Itoa(rand.Intn(899) + 100) //nolint:gosec
-	}
-
-	req.AddCookie(&http.Cookie{
-		Name:   "CONSENT",
-		Value:  "YES+cb.20210328-17-p0.en+FX+" + c.consentID,
-		Path:   "/",
-		Domain: ".youtube.com",
-	})
-
-	res, err := client.Do(req)
-
-	log := slog.With("method", req.Method, "url", req.URL)
-
-	if err == nil && res.StatusCode != http.StatusOK {
-		err = ErrUnexpectedStatusCode(res.StatusCode)
-		res.Body.Close()
-		res = nil
-	}
-
-	if err != nil {
-		log.Debug("HTTP request failed", "error", err)
-	} else {
-		log.Debug("HTTP request succeeded", "status", res.Status)
-	}
-
-	return res, err
-}
-
-// httpGet does a HTTP GET request, checks the response to be a 200 OK and returns it
-func (c *Client) httpGet(ctx context.Context, url string) (*http.Response, error) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	resp, err := c.httpDo(req)
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		resp.Body.Close()
-		return nil, ErrUnexpectedStatusCode(resp.StatusCode)
-	}
-
-	return resp, nil
-}
-
-// httpGetBodyBytes reads the whole HTTP body and returns it
-func (c *Client) httpGetBodyBytes(ctx context.Context, url string) ([]byte, error) {
-	resp, err := c.httpGet(ctx, url)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	return io.ReadAll(resp.Body)
-}
-
-// httpPost does a HTTP POST request with a body, checks the response to be a 200 OK and returns it
-func (c *Client) httpPost(ctx context.Context, url string, body interface{}) (*http.Response, error) {
-	data, err := json.Marshal(body)
-	if err != nil {
-		return nil, err
-	}
-
-	req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
-	if err != nil {
-		return nil, err
-	}
-
-	req.Header.Set("X-Youtube-Client-Name", "3")
-	req.Header.Set("X-Youtube-Client-Version", c.client.version)
-	req.Header.Set("Content-Type", "application/json")
-	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-
-	resp, err := c.httpDo(req)
-	if err != nil {
-		return nil, err
-	}
-
-	if resp.StatusCode != http.StatusOK {
-		resp.Body.Close()
-		return nil, ErrUnexpectedStatusCode(resp.StatusCode)
-	}
-
-	return resp, nil
-}
-
-// httpPostBodyBytes reads the whole HTTP body and returns it
-func (c *Client) httpPostBodyBytes(ctx context.Context, url string, body interface{}) ([]byte, error) {
-	resp, err := c.httpPost(ctx, url, body)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-
-	return io.ReadAll(resp.Body)
-}
-
-// downloadChunk writes the response data into the data channel of the chunk.
-// Downloading in multiple chunks is much faster:
-// https://github.com/kkdai/youtube/pull/190
-func (c *Client) downloadChunk(req *http.Request, chunk *chunk) error {
-	q := req.URL.Query()
-	q.Set("range", fmt.Sprintf("%d-%d", chunk.start, chunk.end))
-	req.URL.RawQuery = q.Encode()
-
-	resp, err := c.httpDo(req)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return ErrUnexpectedStatusCode(resp.StatusCode)
-	}
-
-	expected := int(chunk.end-chunk.start) + 1
-	data, err := io.ReadAll(resp.Body)
-	n := len(data)
-
-	if err != nil {
-		return err
-	}
-
-	if n != expected {
-		return fmt.Errorf("chunk at offset %d has invalid size: expected=%d actual=%d", chunk.start, expected, n)
-	}
-
-	chunk.data <- data
-
-	return nil
-}

downloader/vendor/github.com/kkdai/youtube/v2/decipher.go

@@ -1,285 +0,0 @@
-package youtube
-
-import (
-	"bytes"
-	"context"
-	"errors"
-	"fmt"
-	"net/url"
-	"regexp"
-	"strconv"
-
-	"github.com/dop251/goja"
-)
-
-func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) {
-	params, err := url.ParseQuery(cipher)
-	if err != nil {
-		return "", err
-	}
-
-	uri, err := url.Parse(params.Get("url"))
-	if err != nil {
-		return "", err
-	}
-	query := uri.Query()
-
-	config, err := c.getPlayerConfig(ctx, videoID)
-	if err != nil {
-		return "", err
-	}
-
-	// decrypt s-parameter
-	bs, err := config.decrypt([]byte(params.Get("s")))
-	if err != nil {
-		return "", err
-	}
-	query.Add(params.Get("sp"), string(bs))
-
-	query, err = c.decryptNParam(config, query)
-	if err != nil {
-		return "", err
-	}
-
-	uri.RawQuery = query.Encode()
-
-	return uri.String(), nil
-}
-
-// see https://github.com/kkdai/youtube/pull/244
-func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) {
-	config, err := c.getPlayerConfig(ctx, videoID)
-	if err != nil {
-		return "", err
-	}
-
-	uri, err := url.Parse(urlString)
-	if err != nil {
-		return "", err
-	}
-
-	// for debugging
-	if artifactsFolder != "" {
-		writeArtifact("video-"+videoID+".url", []byte(uri.String()))
-	}
-
-	query, err := c.decryptNParam(config, uri.Query())
-	if err != nil {
-		return "", err
-	}
-
-	uri.RawQuery = query.Encode()
-	return uri.String(), nil
-}
-
-func (c *Client) decryptNParam(config playerConfig, query url.Values) (url.Values, error) {
-	// decrypt n-parameter
-	nSig := query.Get("v")
-	log := Logger.With("n", nSig)
-
-	if nSig != "" {
-		nDecoded, err := config.decodeNsig(nSig)
-		if err != nil {
-			return nil, fmt.Errorf("unable to decode nSig: %w", err)
-		}
-		query.Set("v", nDecoded)
-		log = log.With("decoded", nDecoded)
-	}
-
-	log.Debug("nParam")
-
-	return query, nil
-}
-
-const (
-	jsvarStr   = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
-	reverseStr = ":function\\(a\\)\\{" +
-		"(?:return )?a\\.reverse\\(\\)" +
-		"\\}"
-	spliceStr = ":function\\(a,b\\)\\{" +
-		"a\\.splice\\(0,b\\)" +
-		"\\}"
-	swapStr = ":function\\(a,b\\)\\{" +
-		"var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
-		"\\}"
-)
-
-var (
-	nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})")
-	actionsObjRegexp    = regexp.MustCompile(fmt.Sprintf(
-		"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr))
-
-	actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
-		"function(?: %s)?\\(a\\)\\{"+
-			"a=a\\.split\\(\"\"\\);\\s*"+
-			"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
-			"return a\\.join\\(\"\"\\)"+
-			"\\}", jsvarStr, jsvarStr, jsvarStr))
-
-	reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
-	spliceRegexp  = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
-	swapRegexp    = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
-)
-
-func (config playerConfig) decodeNsig(encoded string) (string, error) {
-	fBody, err := config.getNFunction()
-	if err != nil {
-		return "", err
-	}
-
-	return evalJavascript(fBody, encoded)
-}
-
-func evalJavascript(jsFunction, arg string) (string, error) {
-	const myName = "myFunction"
-
-	vm := goja.New()
-	_, err := vm.RunString(myName + "=" + jsFunction)
-	if err != nil {
-		return "", err
-	}
-
-	var output func(string) string
-	err = vm.ExportTo(vm.Get(myName), &output)
-	if err != nil {
-		return "", err
-	}
-
-	return output(arg), nil
-}
-
-func (config playerConfig) getNFunction() (string, error) {
-	nameResult := nFunctionNameRegexp.FindSubmatch(config)
-	if len(nameResult) == 0 {
-		return "", errors.New("unable to extract n-function name")
-	}
-
-	var name string
-	if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 {
-		name = string(nameResult[4])
-	} else {
-		name = string(nameResult[1])
-	}
-
-	return config.extraFunction(name)
-
-}
-
-func (config playerConfig) extraFunction(name string) (string, error) {
-	// find the beginning of the function
-	def := []byte(name + "=function(")
-	start := bytes.Index(config, def)
-	if start < 1 {
-		return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def)
-	}
-
-	// start after the first curly bracket
-	pos := start + bytes.IndexByte(config[start:], '{') + 1
-
-	var strChar byte
-
-	// find the bracket closing the function
-	for brackets := 1; brackets > 0; pos++ {
-		b := config[pos]
-		switch b {
-		case '{':
-			if strChar == 0 {
-				brackets++
-			}
-		case '}':
-			if strChar == 0 {
-				brackets--
-			}
-		case '`', '"', '\'':
-			if config[pos-1] == '\\' && config[pos-2] != '\\' {
-				continue
-			}
-			if strChar == 0 {
-				strChar = b
-			} else if strChar == b {
-				strChar = 0
-			}
-		}
-	}
-
-	return string(config[start:pos]), nil
-}
-
-func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) {
-	operations, err := config.parseDecipherOps()
-	if err != nil {
-		return nil, err
-	}
-
-	// apply operations
-	bs := []byte(cyphertext)
-	for _, op := range operations {
-		bs = op(bs)
-	}
-
-	return bs, nil
-}
-
-/*
-parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js
-
-var Mt={
-splice:function(a,b){a.splice(0,b)},
-reverse:function(a){a.reverse()},
-EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
-
-a=a.split("");
-Mt.splice(a,3);
-Mt.EQ(a,39);
-Mt.splice(a,2);
-Mt.EQ(a,1);
-Mt.splice(a,1);
-Mt.EQ(a,35);
-Mt.EQ(a,51);
-Mt.splice(a,2);
-Mt.reverse(a,52);
-return a.join("")
-*/
-func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) {
-	objResult := actionsObjRegexp.FindSubmatch(config)
-	funcResult := actionsFuncRegexp.FindSubmatch(config)
-	if len(objResult) < 3 || len(funcResult) < 2 {
-		return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult))
-	}
-
-	obj := objResult[1]
-	objBody := objResult[2]
-	funcBody := funcResult[1]
-
-	var reverseKey, spliceKey, swapKey string
-
-	if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 {
-		reverseKey = string(result[1])
-	}
-	if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 {
-		spliceKey = string(result[1])
-	}
-	if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 {
-		swapKey = string(result[1])
-	}
-
-	regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey)))
-	if err != nil {
-		return nil, err
-	}
-
-	var ops []DecipherOperation
-	for _, s := range regex.FindAllSubmatch(funcBody, -1) {
-		switch string(s[1]) {
-		case reverseKey:
-			ops = append(ops, reverseFunc)
-		case swapKey:
-			arg, _ := strconv.Atoi(string(s[2]))
-			ops = append(ops, newSwapFunc(arg))
-		case spliceKey:
-			arg, _ := strconv.Atoi(string(s[2]))
-			ops = append(ops, newSpliceFunc(arg))
-		}
-	}
-	return ops, nil
-}

downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go

@@ -1,27 +0,0 @@
-package youtube
-
-type DecipherOperation func([]byte) []byte
-
-func newSpliceFunc(pos int) DecipherOperation {
-	return func(bs []byte) []byte {
-		return bs[pos:]
-	}
-}
-
-func newSwapFunc(arg int) DecipherOperation {
-	return func(bs []byte) []byte {
-		pos := arg % len(bs)
-		bs[0], bs[pos] = bs[pos], bs[0]
-		return bs
-	}
-}
-
-func reverseFunc(bs []byte) []byte {
-	l, r := 0, len(bs)-1
-	for l < r {
-		bs[l], bs[r] = bs[r], bs[l]
-		l++
-		r--
-	}
-	return bs
-}

downloader/vendor/github.com/kkdai/youtube/v2/doc.go

@@ -1,4 +0,0 @@
-/*
-Package youtube implement youtube download package in go.
-*/
-package youtube

downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go

@@ -1,203 +0,0 @@
-package downloader
-
-import (
-	"context"
-	"errors"
-	"io"
-	"log/slog"
-	"os"
-	"os/exec"
-	"path/filepath"
-
-	"github.com/kkdai/youtube/v2"
-	"github.com/vbauerster/mpb/v5"
-	"github.com/vbauerster/mpb/v5/decor"
-)
-
-// Downloader offers high level functions to download videos into files
-type Downloader struct {
-	youtube.Client
-	OutputDir string // optional directory to store the files
-}
-
-func (dl *Downloader) getOutputFile(v *youtube.Video, format *youtube.Format, outputFile string) (string, error) {
-	if outputFile == "" {
-		outputFile = SanitizeFilename(v.Title)
-		outputFile += pickIdealFileExtension(format.MimeType)
-	}
-
-	if dl.OutputDir != "" {
-		if err := os.MkdirAll(dl.OutputDir, 0o755); err != nil {
-			return "", err
-		}
-		outputFile = filepath.Join(dl.OutputDir, outputFile)
-	}
-
-	return outputFile, nil
-}
-
-// Download : Starting download video by arguments.
-func (dl *Downloader) Download(ctx context.Context, v *youtube.Video, format *youtube.Format, outputFile string) error {
-	youtube.Logger.Info(
-		"Downloading video",
-		"id", v.ID,
-		"quality", format.Quality,
-		"mimeType", format.MimeType,
-	)
-	destFile, err := dl.getOutputFile(v, format, outputFile)
-	if err != nil {
-		return err
-	}
-
-	// Create output file
-	out, err := os.Create(destFile)
-	if err != nil {
-		return err
-	}
-	defer out.Close()
-
-	return dl.videoDLWorker(ctx, out, v, format)
-}
-
-// DownloadComposite : Downloads audio and video streams separately and merges them via ffmpeg.
-func (dl *Downloader) DownloadComposite(ctx context.Context, outputFile string, v *youtube.Video, quality string, mimetype, language string) error {
-	videoFormat, audioFormat, err1 := getVideoAudioFormats(v, quality, mimetype, language)
-	if err1 != nil {
-		return err1
-	}
-
-	log := youtube.Logger.With("id", v.ID)
-
-	log.Info(
-		"Downloading composite video",
-		"videoQuality", videoFormat.QualityLabel,
-		"videoMimeType", videoFormat.MimeType,
-		"audioMimeType", audioFormat.MimeType,
-	)
-
-	destFile, err := dl.getOutputFile(v, videoFormat, outputFile)
-	if err != nil {
-		return err
-	}
-	outputDir := filepath.Dir(destFile)
-
-	// Create temporary video file
-	videoFile, err := os.CreateTemp(outputDir, "youtube_*.m4v")
-	if err != nil {
-		return err
-	}
-	defer Delete(videoFile.Name(), log)
-
-	// Create temporary audio file
-	audioFile, err := os.CreateTemp(outputDir, "youtube_*.m4a")
-	if err != nil {
-		return err
-	}
-	defer Delete(audioFile.Name(), log)
-
-	log.Debug("Downloading video file...")
-	err = dl.videoDLWorker(ctx, videoFile, v, videoFormat)
-	if err != nil {
-		return err
-	}
-	videoFile.Close()
-
-	log.Debug("Downloading audio file...")
-	err = dl.videoDLWorker(ctx, audioFile, v, audioFormat)
-	if err != nil {
-		return err
-	}
-	audioFile.Close()
-
-	//nolint:gosec
-	ffmpegVersionCmd := exec.Command("ffmpeg", "-y",
-		"-i", videoFile.Name(),
-		"-i", audioFile.Name(),
-		"-c", "copy", // Just copy without re-encoding
-		"-shortest", // Finish encoding when the shortest input stream ends
-		destFile,
-		"-loglevel", "warning",
-	)
-	ffmpegVersionCmd.Stderr = os.Stderr
-	ffmpegVersionCmd.Stdout = os.Stdout
-	log.Info("merging video and audio", "output", destFile)
-
-	return ffmpegVersionCmd.Run()
-}
-
-func Delete(path string, log *slog.Logger) {
-	err := os.Remove(path)
-	if err != nil {
-		log.Error("Failed deleting file", "path", path, "error", err)
-	}
-}
-
-func getVideoAudioFormats(v *youtube.Video, quality string, mimetype, language string) (*youtube.Format, *youtube.Format, error) {
-	var videoFormats, audioFormats youtube.FormatList
-
-	formats := v.Formats
-	if mimetype != "" {
-		formats = formats.Type(mimetype)
-	}
-
-	videoFormats = formats.Type("video").AudioChannels(0)
-	audioFormats = formats.Type("audio")
-
-	if quality != "" {
-		videoFormats = videoFormats.Quality(quality)
-	}
-
-	if language != "" {
-		audioFormats = audioFormats.Language(language)
-	}
-
-	if len(videoFormats) == 0 {
-		return nil, nil, errors.New("no video format found after filtering")
-	}
-
-	if len(audioFormats) == 0 {
-		return nil, nil, errors.New("no audio format found after filtering")
-	}
-
-	videoFormats.Sort()
-	audioFormats.Sort()
-
-	return &videoFormats[0], &audioFormats[0], nil
-}
-
-func (dl *Downloader) videoDLWorker(ctx context.Context, out *os.File, video *youtube.Video, format *youtube.Format) error {
-	stream, size, err := dl.GetStreamContext(ctx, video, format)
-	if err != nil {
-		return err
-	}
-
-	prog := &progress{
-		contentLength: float64(size),
-	}
-
-	// create progress bar
-	progress := mpb.New(mpb.WithWidth(64))
-	bar := progress.AddBar(
-		int64(prog.contentLength),
-
-		mpb.PrependDecorators(
-			decor.CountersKibiByte("% .2f / % .2f"),
-			decor.Percentage(decor.WCSyncSpace),
-		),
-		mpb.AppendDecorators(
-			decor.EwmaETA(decor.ET_STYLE_GO, 90),
-			decor.Name(" ] "),
-			decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60),
-		),
-	)
-
-	reader := bar.ProxyReader(stream)
-	mw := io.MultiWriter(out, prog)
-	_, err = io.Copy(mw, reader)
-	if err != nil {
-		return err
-	}
-
-	progress.Wait()
-	return nil
-}

downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go

@@ -1,65 +0,0 @@
-package downloader
-
-import (
-	"mime"
-	"regexp"
-)
-
-const defaultExtension = ".mov"
-
-// Rely on hardcoded canonical mime types, as the ones provided by Go aren't exhaustive [1].
-// This seems to be a recurring problem for youtube downloaders, see [2].
-// The implementation is based on mozilla's list [3], IANA [4] and Youtube's support [5].
-// [1] https://github.com/golang/go/blob/ed7888aea6021e25b0ea58bcad3f26da2b139432/src/mime/type.go#L60
-// [2] https://github.com/ZiTAL/youtube-dl/blob/master/mime.types
-// [3] https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
-// [4] https://www.iana.org/assignments/media-types/media-types.xhtml#video
-// [5] https://support.google.com/youtube/troubleshooter/2888402?hl=en
-var canonicals = map[string]string{
-	"video/quicktime":  ".mov",
-	"video/x-msvideo":  ".avi",
-	"video/x-matroska": ".mkv",
-	"video/mpeg":       ".mpeg",
-	"video/webm":       ".webm",
-	"video/3gpp2":      ".3g2",
-	"video/x-flv":      ".flv",
-	"video/3gpp":       ".3gp",
-	"video/mp4":        ".mp4",
-	"video/ogg":        ".ogv",
-	"video/mp2t":       ".ts",
-}
-
-func pickIdealFileExtension(mediaType string) string {
-	mediaType, _, err := mime.ParseMediaType(mediaType)
-	if err != nil {
-		return defaultExtension
-	}
-
-	if extension, ok := canonicals[mediaType]; ok {
-		return extension
-	}
-
-	// Our last resort is to ask the operating system, but these give multiple results and are rarely canonical.
-	extensions, err := mime.ExtensionsByType(mediaType)
-	if err != nil || extensions == nil {
-		return defaultExtension
-	}
-
-	return extensions[0]
-}
-
-func SanitizeFilename(fileName string) string {
-	// Characters not allowed on mac
-	//	:/
-	// Characters not allowed on linux
-	//	/
-	// Characters not allowed on windows
-	//	<>:"/\|?*
-
-	// Ref https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
-
-	fileName = regexp.MustCompile(`[:/<>\:"\\|?*]`).ReplaceAllString(fileName, "")
-	fileName = regexp.MustCompile(`\s+`).ReplaceAllString(fileName, " ")
-
-	return fileName
-}

downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go

@@ -1,17 +0,0 @@
-package downloader
-
-type progress struct {
-	contentLength     float64
-	totalWrittenBytes float64
-	downloadLevel     float64
-}
-
-func (dl *progress) Write(p []byte) (n int, err error) {
-	n = len(p)
-	dl.totalWrittenBytes = dl.totalWrittenBytes + float64(n)
-	currentPercent := (dl.totalWrittenBytes / dl.contentLength) * 100
-	if (dl.downloadLevel <= currentPercent) && (dl.downloadLevel < 100) {
-		dl.downloadLevel++
-	}
-	return
-}

downloader/vendor/github.com/kkdai/youtube/v2/errors.go

@@ -1,47 +0,0 @@
-package youtube
-
-import (
-	"fmt"
-)
-
-const (
-	ErrCipherNotFound             = constError("cipher not found")
-	ErrSignatureTimestampNotFound = constError("signature timestamp not found")
-	ErrInvalidCharactersInVideoID = constError("invalid characters in video id")
-	ErrVideoIDMinLength           = constError("the video id must be at least 10 characters long")
-	ErrReadOnClosedResBody        = constError("http: read on closed response body")
-	ErrNotPlayableInEmbed         = constError("embedding of this video has been disabled")
-	ErrLoginRequired              = constError("login required to confirm your age")
-	ErrVideoPrivate               = constError("user restricted access to this video")
-	ErrInvalidPlaylist            = constError("no playlist detected or invalid playlist ID")
-)
-
-type constError string
-
-func (e constError) Error() string {
-	return string(e)
-}
-
-type ErrPlayabiltyStatus struct {
-	Status string
-	Reason string
-}
-
-func (err ErrPlayabiltyStatus) Error() string {
-	return fmt.Sprintf("cannot playback and download, status: %s, reason: %s", err.Status, err.Reason)
-}
-
-// ErrUnexpectedStatusCode is returned on unexpected HTTP status codes
-type ErrUnexpectedStatusCode int
-
-func (err ErrUnexpectedStatusCode) Error() string {
-	return fmt.Sprintf("unexpected status code: %d", err)
-}
-
-type ErrPlaylistStatus struct {
-	Reason string
-}
-
-func (err ErrPlaylistStatus) Error() string {
-	return fmt.Sprintf("could not load playlist: %s", err.Reason)
-}

downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go

@@ -1,35 +0,0 @@
-//go:build fetch
-// +build fetch
-
-package youtube
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-)
-
-func init() {
-	FetchTestData()
-}
-
-// ran via go generate to fetch and update the playlist response data
-func FetchTestData() {
-	f, err := os.Create(testPlaylistResponseDataFile)
-	exitOnError(err)
-	requestURL := fmt.Sprintf(playlistFetchURL, testPlaylistID)
-	resp, err := http.Get(requestURL)
-	exitOnError(err)
-	defer resp.Body.Close()
-	n, err := io.Copy(f, resp.Body)
-	exitOnError(err)
-	fmt.Printf("Successfully fetched playlist %s (%d bytes)\n", testPlaylistID, n)
-}
-
-func exitOnError(err error) {
-	if err != nil {
-		fmt.Println(err)
-		os.Exit(1)
-	}
-}

downloader/vendor/github.com/kkdai/youtube/v2/format_list.go

@@ -1,148 +0,0 @@
-package youtube
-
-import (
-	"sort"
-	"strconv"
-	"strings"
-)
-
-type FormatList []Format
-
-// Type returns a new FormatList filtered by itag
-func (list FormatList) Select(f func(Format) bool) (result FormatList) {
-	for i := range list {
-		if f(list[i]) {
-			result = append(result, list[i])
-		}
-	}
-	return result
-}
-
-// Type returns a new FormatList filtered by itag
-func (list FormatList) Itag(itagNo int) FormatList {
-	return list.Select(func(f Format) bool {
-		return f.ItagNo == itagNo
-	})
-}
-
-// Type returns a new FormatList filtered by mime type
-func (list FormatList) Type(value string) FormatList {
-	return list.Select(func(f Format) bool {
-		return strings.Contains(f.MimeType, value)
-	})
-}
-
-// Type returns a new FormatList filtered by display name
-func (list FormatList) Language(displayName string) FormatList {
-	return list.Select(func(f Format) bool {
-		return f.LanguageDisplayName() == displayName
-	})
-}
-
-// Quality returns a new FormatList filtered by quality, quality label or itag,
-// but not audio quality
-func (list FormatList) Quality(quality string) FormatList {
-	itag, _ := strconv.Atoi(quality)
-
-	return list.Select(func(f Format) bool {
-		return itag == f.ItagNo || strings.Contains(f.Quality, quality) || strings.Contains(f.QualityLabel, quality)
-	})
-}
-
-// AudioChannels returns a new FormatList filtered by the matching AudioChannels
-func (list FormatList) AudioChannels(n int) FormatList {
-	return list.Select(func(f Format) bool {
-		return f.AudioChannels == n
-	})
-}
-
-// AudioChannels returns a new FormatList filtered by the matching AudioChannels
-func (list FormatList) WithAudioChannels() FormatList {
-	return list.Select(func(f Format) bool {
-		return f.AudioChannels > 0
-	})
-}
-
-// FilterQuality reduces the format list to formats matching the quality
-func (v *Video) FilterQuality(quality string) {
-	v.Formats = v.Formats.Quality(quality)
-	v.Formats.Sort()
-}
-
-// Sort sorts all formats fields
-func (list FormatList) Sort() {
-	sort.SliceStable(list, func(i, j int) bool {
-		return sortFormat(i, j, list)
-	})
-}
-
-// sortFormat sorts video by resolution, FPS, codec (av01, vp9, avc1), bitrate
-// sorts audio by default, codec (mp4, opus), channels, bitrate, sample rate
-func sortFormat(i int, j int, formats FormatList) bool {
-
-	// Sort by Width
-	if formats[i].Width == formats[j].Width {
-		// Format 137 downloads slowly, give it less priority
-		// see https://github.com/kkdai/youtube/pull/171
-		switch 137 {
-		case formats[i].ItagNo:
-			return false
-		case formats[j].ItagNo:
-			return true
-		}
-
-		// Sort by FPS
-		if formats[i].FPS == formats[j].FPS {
-			if formats[i].FPS == 0 && formats[i].AudioChannels > 0 && formats[j].AudioChannels > 0 {
-				// Audio
-				// Sort by default
-				if (formats[i].AudioTrack == nil && formats[j].AudioTrack == nil) || (formats[i].AudioTrack != nil && formats[j].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault == formats[j].AudioTrack.AudioIsDefault) {
-					// Sort by codec
-					codec := map[int]int{}
-					for _, index := range []int{i, j} {
-						if strings.Contains(formats[index].MimeType, "mp4") {
-							codec[index] = 1
-						} else if strings.Contains(formats[index].MimeType, "opus") {
-							codec[index] = 2
-						}
-					}
-					if codec[i] == codec[j] {
-						// Sort by Audio Channel
-						if formats[i].AudioChannels == formats[j].AudioChannels {
-							// Sort by Audio Bitrate
-							if formats[i].Bitrate == formats[j].Bitrate {
-								// Sort by Audio Sample Rate
-								return formats[i].AudioSampleRate > formats[j].AudioSampleRate
-							}
-							return formats[i].Bitrate > formats[j].Bitrate
-						}
-						return formats[i].AudioChannels > formats[j].AudioChannels
-					}
-					return codec[i] < codec[j]
-				} else if formats[i].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault {
-					return true
-				}
-				return false
-			}
-			// Video
-			// Sort by codec
-			codec := map[int]int{}
-			for _, index := range []int{i, j} {
-				if strings.Contains(formats[index].MimeType, "av01") {
-					codec[index] = 1
-				} else if strings.Contains(formats[index].MimeType, "vp9") {
-					codec[index] = 2
-				} else if strings.Contains(formats[index].MimeType, "avc1") {
-					codec[index] = 3
-				}
-			}
-			if codec[i] == codec[j] {
-				// Sort by Audio Bitrate
-				return formats[i].Bitrate > formats[j].Bitrate
-			}
-			return codec[i] < codec[j]
-		}
-		return formats[i].FPS > formats[j].FPS
-	}
-	return formats[i].Width > formats[j].Width
-}

downloader/vendor/github.com/kkdai/youtube/v2/logger.go

@@ -1,28 +0,0 @@
-package youtube
-
-import (
-	"fmt"
-	"log/slog"
-	"os"
-)
-
-// The global logger for all Client instances
-var Logger = getLogger(os.Getenv("LOGLEVEL"))
-
-func SetLogLevel(value string) {
-	Logger = getLogger(value)
-}
-
-func getLogger(logLevel string) *slog.Logger {
-	levelVar := slog.LevelVar{}
-
-	if logLevel != "" {
-		if err := levelVar.UnmarshalText([]byte(logLevel)); err != nil {
-			panic(fmt.Sprintf("Invalid log level %s: %v", logLevel, err))
-		}
-	}
-
-	return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
-		Level: levelVar.Level(),
-	}))
-}

downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go

@@ -1,37 +0,0 @@
-package youtube
-
-import (
-	"time"
-)
-
-const defaultCacheExpiration = time.Minute * time.Duration(5)
-
-type playerCache struct {
-	key       string
-	expiredAt time.Time
-	config    playerConfig
-}
-
-// Get : get cache  when it has same video id and not expired
-func (s playerCache) Get(key string) playerConfig {
-	return s.GetCacheBefore(key, time.Now())
-}
-
-// GetCacheBefore : can pass time for testing
-func (s playerCache) GetCacheBefore(key string, time time.Time) playerConfig {
-	if key == s.key && s.expiredAt.After(time) {
-		return s.config
-	}
-	return nil
-}
-
-// Set : set cache with default expiration
-func (s *playerCache) Set(key string, operations playerConfig) {
-	s.setWithExpiredTime(key, operations, time.Now().Add(defaultCacheExpiration))
-}
-
-func (s *playerCache) setWithExpiredTime(key string, config playerConfig, time time.Time) {
-	s.key = key
-	s.config = config
-	s.expiredAt = time
-}

downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go

@@ -1,59 +0,0 @@
-package youtube
-
-import (
-	"context"
-	"errors"
-	"fmt"
-	"log"
-	"os"
-	"path/filepath"
-	"regexp"
-	"strings"
-)
-
-type playerConfig []byte
-
-var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`)
-
-func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) {
-	embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID)
-	embedBody, err := c.httpGetBodyBytes(ctx, embedURL)
-	if err != nil {
-		return nil, err
-	}
-
-	// example: /s/player/f676c671/player_ias.vflset/en_US/base.js
-	playerPath := string(basejsPattern.Find(embedBody))
-	if playerPath == "" {
-		return nil, errors.New("unable to find basejs URL in playerConfig")
-	}
-
-	// for debugging
-	var artifactName string
-	if artifactsFolder != "" {
-		parts := strings.SplitN(playerPath, "/", 5)
-		artifactName = "player-" + parts[3] + ".js"
-		linkName := filepath.Join(artifactsFolder, "video-"+videoID+".js")
-		if err := os.Symlink(artifactName, linkName); err != nil {
-			log.Printf("unable to create symlink %s: %v", linkName, err)
-		}
-	}
-
-	config := c.playerCache.Get(playerPath)
-	if config != nil {
-		return config, nil
-	}
-
-	config, err = c.httpGetBodyBytes(ctx, "https://youtube.com"+playerPath)
-	if err != nil {
-		return nil, err
-	}
-
-	// for debugging
-	if artifactName != "" {
-		writeArtifact(artifactName, config)
-	}
-
-	c.playerCache.Set(playerPath, config)
-	return config, nil
-}

downloader/vendor/github.com/kkdai/youtube/v2/playlist.go

@@ -1,256 +0,0 @@
-package youtube
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"regexp"
-	"runtime/debug"
-	"strconv"
-	"time"
-
-	sjson "github.com/bitly/go-simplejson"
-)
-
-var (
-	playlistIDRegex    = regexp.MustCompile("^[A-Za-z0-9_-]{13,42}$")
-	playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{13,42})(&.*)?$")
-)
-
-type Playlist struct {
-	ID          string
-	Title       string
-	Description string
-	Author      string
-	Videos      []*PlaylistEntry
-}
-
-type PlaylistEntry struct {
-	ID         string
-	Title      string
-	Author     string
-	Duration   time.Duration
-	Thumbnails Thumbnails
-}
-
-func extractPlaylistID(url string) (string, error) {
-	if playlistIDRegex.Match([]byte(url)) {
-		return url, nil
-	}
-
-	matches := playlistInURLRegex.FindStringSubmatch(url)
-
-	if matches != nil {
-		return matches[1], nil
-	}
-
-	return "", ErrInvalidPlaylist
-}
-
-// structs for playlist extraction
-
-// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text
-// Description: metadata.playlistMetadataRenderer.description
-// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text
-
-// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents
-// ID: .videoId
-// Title: title.runs[0].text
-// Author: .shortBylineText.runs[0].text
-// Duration: .lengthSeconds
-// Thumbnails .thumbnails
-
-// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails
-func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body []byte) (err error) {
-	var j *sjson.Json
-	j, err = sjson.NewJson(body)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		stack := debug.Stack()
-		if r := recover(); r != nil {
-			err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack)
-		}
-	}()
-
-	renderer := j.GetPath("alerts").GetIndex(0).GetPath("alertRenderer")
-	if renderer != nil && renderer.GetPath("type").MustString() == "ERROR" {
-		message := renderer.GetPath("text", "runs").GetIndex(0).GetPath("text").MustString()
-
-		return ErrPlaylistStatus{Reason: message}
-	}
-
-	// Metadata can be located in multiple places depending on client type
-	var metadata *sjson.Json
-	if node, ok := j.CheckGet("metadata"); ok {
-		metadata = node
-	} else if node, ok := j.CheckGet("header"); ok {
-		metadata = node
-	} else {
-		return fmt.Errorf("no playlist header / metadata found")
-	}
-
-	metadata = metadata.Get("playlistHeaderRenderer")
-
-	p.Title = sjsonGetText(metadata, "title")
-	p.Description = sjsonGetText(metadata, "description", "descriptionText")
-	p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1).
-		GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs").
-		GetIndex(0).Get("text").MustString()
-
-	if len(p.Author) == 0 {
-		p.Author = sjsonGetText(metadata, "owner", "ownerText")
-	}
-
-	contents, ok := j.CheckGet("contents")
-	if !ok {
-		return fmt.Errorf("contents not found in json body")
-	}
-
-	// contents can have different keys with same child structure
-	firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0).
-		GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0)
-
-	// This extra nested item is only set with the web client
-	if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) {
-		firstPart = n
-	}
-
-	vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON()
-	if err != nil {
-		return err
-	}
-
-	if len(vJSON) <= 4 {
-		return fmt.Errorf("no video data found in JSON")
-	}
-
-	entries, continuation, err := extractPlaylistEntries(vJSON)
-	if err != nil {
-		return err
-	}
-
-	if len(continuation) == 0 {
-		continuation = getContinuation(firstPart.Get("playlistVideoListRenderer"))
-	}
-
-	if len(entries) == 0 {
-		return fmt.Errorf("no videos found in playlist")
-	}
-
-	p.Videos = entries
-
-	for continuation != "" {
-		data := prepareInnertubePlaylistData(continuation, true, *client.client)
-
-		body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data)
-		if err != nil {
-			return err
-		}
-
-		j, err := sjson.NewJson(body)
-		if err != nil {
-			return err
-		}
-
-		next := j.GetPath("onResponseReceivedActions").GetIndex(0).
-			GetPath("appendContinuationItemsAction", "continuationItems")
-
-		if !isValidJSON(next) {
-			next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents")
-		}
-
-		vJSON, err := next.MarshalJSON()
-		if err != nil {
-			return err
-		}
-
-		entries, token, err := extractPlaylistEntries(vJSON)
-		if err != nil {
-			return err
-		}
-
-		if len(token) > 0 {
-			continuation = token
-		} else {
-			continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation"))
-		}
-
-		p.Videos = append(p.Videos, entries...)
-	}
-
-	return err
-}
-
-func extractPlaylistEntries(data []byte) ([]*PlaylistEntry, string, error) {
-	var vids []*videosJSONExtractor
-
-	if err := json.Unmarshal(data, &vids); err != nil {
-		return nil, "", err
-	}
-
-	entries := make([]*PlaylistEntry, 0, len(vids))
-
-	var continuation string
-	for _, v := range vids {
-		if v.Renderer == nil {
-			if v.Continuation.Endpoint.Command.Token != "" {
-				continuation = v.Continuation.Endpoint.Command.Token
-			}
-
-			continue
-		}
-
-		entries = append(entries, v.PlaylistEntry())
-	}
-
-	return entries, continuation, nil
-}
-
-type videosJSONExtractor struct {
-	Renderer *struct {
-		ID        string   `json:"videoId"`
-		Title     withRuns `json:"title"`
-		Author    withRuns `json:"shortBylineText"`
-		Duration  string   `json:"lengthSeconds"`
-		Thumbnail struct {
-			Thumbnails []Thumbnail `json:"thumbnails"`
-		} `json:"thumbnail"`
-	} `json:"playlistVideoRenderer"`
-	Continuation struct {
-		Endpoint struct {
-			Command struct {
-				Token string `json:"token"`
-			} `json:"continuationCommand"`
-		} `json:"continuationEndpoint"`
-	} `json:"continuationItemRenderer"`
-}
-
-func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry {
-	ds, err := strconv.Atoi(vje.Renderer.Duration)
-	if err != nil {
-		panic("invalid video duration: " + vje.Renderer.Duration)
-	}
-	return &PlaylistEntry{
-		ID:         vje.Renderer.ID,
-		Title:      vje.Renderer.Title.String(),
-		Author:     vje.Renderer.Author.String(),
-		Duration:   time.Second * time.Duration(ds),
-		Thumbnails: vje.Renderer.Thumbnail.Thumbnails,
-	}
-}
-
-type withRuns struct {
-	Runs []struct {
-		Text string `json:"text"`
-	} `json:"runs"`
-}
-
-func (wr withRuns) String() string {
-	if len(wr.Runs) > 0 {
-		return wr.Runs[0].Text
-	}
-	return ""
-}

downloader/vendor/github.com/kkdai/youtube/v2/response_data.go

@@ -1,153 +0,0 @@
-package youtube
-
-type playerResponseData struct {
-	Captions struct {
-		PlayerCaptionsTracklistRenderer struct {
-			CaptionTracks []CaptionTrack `json:"captionTracks"`
-			AudioTracks   []struct {
-				CaptionTrackIndices []int `json:"captionTrackIndices"`
-			} `json:"audioTracks"`
-			TranslationLanguages []struct {
-				LanguageCode string `json:"languageCode"`
-				LanguageName struct {
-					SimpleText string `json:"simpleText"`
-				} `json:"languageName"`
-			} `json:"translationLanguages"`
-			DefaultAudioTrackIndex int `json:"defaultAudioTrackIndex"`
-		} `json:"playerCaptionsTracklistRenderer"`
-	} `json:"captions"`
-
-	PlayabilityStatus struct {
-		Status          string `json:"status"`
-		Reason          string `json:"reason"`
-		PlayableInEmbed bool   `json:"playableInEmbed"`
-		Miniplayer      struct {
-			MiniplayerRenderer struct {
-				PlaybackMode string `json:"playbackMode"`
-			} `json:"miniplayerRenderer"`
-		} `json:"miniplayer"`
-		ContextParams string `json:"contextParams"`
-	} `json:"playabilityStatus"`
-	StreamingData struct {
-		ExpiresInSeconds string   `json:"expiresInSeconds"`
-		Formats          []Format `json:"formats"`
-		AdaptiveFormats  []Format `json:"adaptiveFormats"`
-		DashManifestURL  string   `json:"dashManifestUrl"`
-		HlsManifestURL   string   `json:"hlsManifestUrl"`
-	} `json:"streamingData"`
-	VideoDetails struct {
-		VideoID          string   `json:"videoId"`
-		Title            string   `json:"title"`
-		LengthSeconds    string   `json:"lengthSeconds"`
-		Keywords         []string `json:"keywords"`
-		ChannelID        string   `json:"channelId"`
-		IsOwnerViewing   bool     `json:"isOwnerViewing"`
-		ShortDescription string   `json:"shortDescription"`
-		IsCrawlable      bool     `json:"isCrawlable"`
-		Thumbnail        struct {
-			Thumbnails []Thumbnail `json:"thumbnails"`
-		} `json:"thumbnail"`
-		AverageRating     float64 `json:"averageRating"`
-		AllowRatings      bool    `json:"allowRatings"`
-		ViewCount         string  `json:"viewCount"`
-		Author            string  `json:"author"`
-		IsPrivate         bool    `json:"isPrivate"`
-		IsUnpluggedCorpus bool    `json:"isUnpluggedCorpus"`
-		IsLiveContent     bool    `json:"isLiveContent"`
-	} `json:"videoDetails"`
-	Microformat struct {
-		PlayerMicroformatRenderer struct {
-			Thumbnail struct {
-				Thumbnails []struct {
-					URL    string `json:"url"`
-					Width  int    `json:"width"`
-					Height int    `json:"height"`
-				} `json:"thumbnails"`
-			} `json:"thumbnail"`
-			Title struct {
-				SimpleText string `json:"simpleText"`
-			} `json:"title"`
-			Description struct {
-				SimpleText string `json:"simpleText"`
-			} `json:"description"`
-			LengthSeconds      string   `json:"lengthSeconds"`
-			OwnerProfileURL    string   `json:"ownerProfileUrl"`
-			ExternalChannelID  string   `json:"externalChannelId"`
-			IsFamilySafe       bool     `json:"isFamilySafe"`
-			AvailableCountries []string `json:"availableCountries"`
-			IsUnlisted         bool     `json:"isUnlisted"`
-			HasYpcMetadata     bool     `json:"hasYpcMetadata"`
-			ViewCount          string   `json:"viewCount"`
-			Category           string   `json:"category"`
-			PublishDate        string   `json:"publishDate"`
-			OwnerChannelName   string   `json:"ownerChannelName"`
-			UploadDate         string   `json:"uploadDate"`
-		} `json:"playerMicroformatRenderer"`
-	} `json:"microformat"`
-}
-
-type Format struct {
-	ItagNo           int    `json:"itag"`
-	URL              string `json:"url"`
-	MimeType         string `json:"mimeType"`
-	Quality          string `json:"quality"`
-	Cipher           string `json:"signatureCipher"`
-	Bitrate          int    `json:"bitrate"`
-	FPS              int    `json:"fps"`
-	Width            int    `json:"width"`
-	Height           int    `json:"height"`
-	LastModified     string `json:"lastModified"`
-	ContentLength    int64  `json:"contentLength,string"`
-	QualityLabel     string `json:"qualityLabel"`
-	ProjectionType   string `json:"projectionType"`
-	AverageBitrate   int    `json:"averageBitrate"`
-	AudioQuality     string `json:"audioQuality"`
-	ApproxDurationMs string `json:"approxDurationMs"`
-	AudioSampleRate  string `json:"audioSampleRate"`
-	AudioChannels    int    `json:"audioChannels"`
-
-	// InitRange is only available for adaptive formats
-	InitRange *struct {
-		Start string `json:"start"`
-		End   string `json:"end"`
-	} `json:"initRange"`
-
-	// IndexRange is only available for adaptive formats
-	IndexRange *struct {
-		Start string `json:"start"`
-		End   string `json:"end"`
-	} `json:"indexRange"`
-
-	// AudioTrack is only available for videos with multiple audio track languages
-	AudioTrack *struct {
-		DisplayName    string `json:"displayName"`
-		ID             string `json:"id"`
-		AudioIsDefault bool   `json:"audioIsDefault"`
-	}
-}
-
-func (f *Format) LanguageDisplayName() string {
-	if f.AudioTrack == nil {
-		return ""
-	}
-	return f.AudioTrack.DisplayName
-}
-
-type Thumbnails []Thumbnail
-
-type Thumbnail struct {
-	URL    string
-	Width  uint
-	Height uint
-}
-
-type CaptionTrack struct {
-	BaseURL string `json:"baseUrl"`
-	Name    struct {
-		SimpleText string `json:"simpleText"`
-	} `json:"name"`
-	VssID          string `json:"vssId"`
-	LanguageCode   string `json:"languageCode"`
-	Kind           string `json:"kind,omitempty"`
-	IsTranslatable bool   `json:"isTranslatable"`
-}

downloader/vendor/github.com/kkdai/youtube/v2/transcript.go

@@ -1,214 +0,0 @@
-package youtube
-
-import (
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"strconv"
-	"strings"
-)
-
-var (
-	ErrTranscriptDisabled = errors.New("transcript is disabled on this video")
-)
-
-// TranscriptSegment is a single transcipt segment spanning a few milliseconds.
-type TranscriptSegment struct {
-	// Text is the transcipt text.
-	Text string `json:"text"`
-
-	// StartMs is the start timestamp in ms.
-	StartMs int `json:"offset"`
-
-	// OffsetText e.g. '4:00'.
-	OffsetText string `json:"offsetText"`
-
-	// Duration the transcript segment spans in ms.
-	Duration int `json:"duration"`
-}
-
-func (tr TranscriptSegment) String() string {
-	return tr.OffsetText + " - " + strings.TrimSpace(tr.Text)
-}
-
-type VideoTranscript []TranscriptSegment
-
-func (vt VideoTranscript) String() string {
-	var str string
-	for _, tr := range vt {
-		str += tr.String() + "\n"
-	}
-
-	return str
-}
-
-// GetTranscript fetches the video transcript if available.
-//
-// Not all videos have transcripts, only relatively new videos.
-// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
-func (c *Client) GetTranscript(video *Video, lang string) (VideoTranscript, error) {
-	return c.GetTranscriptCtx(context.Background(), video, lang)
-}
-
-// GetTranscriptCtx fetches the video transcript if available.
-//
-// Not all videos have transcripts, only relatively new videos.
-// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
-func (c *Client) GetTranscriptCtx(ctx context.Context, video *Video, lang string) (VideoTranscript, error) {
-	c.assureClient()
-
-	if video == nil || video.ID == "" {
-		return nil, fmt.Errorf("no video provided")
-	}
-
-	body, err := c.transcriptDataByInnertube(ctx, video.ID, lang)
-	if err != nil {
-		return nil, err
-	}
-
-	transcript, err := parseTranscript(body)
-	if err != nil {
-		return nil, err
-	}
-
-	return transcript, nil
-}
-
-func parseTranscript(body []byte) (VideoTranscript, error) {
-	var resp transcriptResp
-	if err := json.Unmarshal(body, &resp); err != nil {
-		return nil, err
-	}
-
-	if len(resp.Actions) > 0 {
-		// Android client response
-		if app := resp.Actions[0].AppSegment; app != nil {
-			return getSegments(app)
-		}
-
-		// Web client response
-		if web := resp.Actions[0].WebSegment; web != nil {
-			return nil, fmt.Errorf("not implemented")
-		}
-	}
-
-	return nil, ErrTranscriptDisabled
-}
-
-type segmenter interface {
-	ParseSegments() []TranscriptSegment
-}
-
-func getSegments(f segmenter) (VideoTranscript, error) {
-	if segments := f.ParseSegments(); len(segments) > 0 {
-		return segments, nil
-	}
-
-	return nil, ErrTranscriptDisabled
-}
-
-// transcriptResp is the JSON structure as returned by the transcript API.
-type transcriptResp struct {
-	Actions []struct {
-		AppSegment *appData `json:"elementsCommand"`
-		WebSegment *webData `json:"updateEngagementPanelAction"`
-	} `json:"actions"`
-}
-
-type appData struct {
-	TEC struct {
-		Args struct {
-			ListArgs struct {
-				Ow struct {
-					InitialSeg []struct {
-						TranscriptSegment struct {
-							StartMs string `json:"startMs"`
-							EndMs   string `json:"endMs"`
-							Text    struct {
-								String struct {
-									// Content is the actual transctipt text
-									Content string `json:"content"`
-								} `json:"elementsAttributedString"`
-							} `json:"snippet"`
-							StartTimeText struct {
-								String struct {
-									// Content is the fomratted timestamp, e.g. '4:00'
-									Content string `json:"content"`
-								} `json:"elementsAttributedString"`
-							} `json:"startTimeText"`
-						} `json:"transcriptSegmentRenderer"`
-					} `json:"initialSegments"`
-				} `json:"overwrite"`
-			} `json:"transformTranscriptSegmentListArguments"`
-		} `json:"arguments"`
-	} `json:"transformEntityCommand"`
-}
-
-func (s *appData) ParseSegments() []TranscriptSegment {
-	rawSegments := s.TEC.Args.ListArgs.Ow.InitialSeg
-	segments := make([]TranscriptSegment, 0, len(rawSegments))
-
-	for _, segment := range rawSegments {
-		startMs, _ := strconv.Atoi(segment.TranscriptSegment.StartMs)
-		endMs, _ := strconv.Atoi(segment.TranscriptSegment.EndMs)
-
-		segments = append(segments, TranscriptSegment{
-			Text:       segment.TranscriptSegment.Text.String.Content,
-			StartMs:    startMs,
-			OffsetText: segment.TranscriptSegment.StartTimeText.String.Content,
-			Duration:   endMs - startMs,
-		})
-	}
-
-	return segments
-}
-
-type webData struct {
-	Content struct {
-		TR struct {
-			Body struct {
-				TBR struct {
-					Cues []struct {
-						Transcript struct {
-							FormattedStartOffset struct {
-								SimpleText string `json:"simpleText"`
-							} `json:"formattedStartOffset"`
-							Cues []struct {
-								TranscriptCueRenderer struct {
-									Cue struct {
-										SimpleText string `json:"simpleText"`
-									} `json:"cue"`
-									StartOffsetMs string `json:"startOffsetMs"`
-									DurationMs    string `json:"durationMs"`
-								} `json:"transcriptCueRenderer"`
-							} `json:"cues"`
-						} `json:"transcriptCueGroupRenderer"`
-					} `json:"cueGroups"`
-				} `json:"transcriptSearchPanelRenderer"`
-			} `json:"content"`
-		} `json:"transcriptRenderer"`
-	} `json:"content"`
-}
-
-func (s *webData) ParseSegments() []TranscriptSegment {
-	// TODO: doesn't actually work now, check json.
-	cues := s.Content.TR.Body.TBR.Cues
-	segments := make([]TranscriptSegment, 0, len(cues))
-
-	for _, s := range cues {
-		formatted := s.Transcript.FormattedStartOffset.SimpleText
-		segment := s.Transcript.Cues[0].TranscriptCueRenderer
-		start, _ := strconv.Atoi(segment.StartOffsetMs)
-		duration, _ := strconv.Atoi(segment.DurationMs)
-
-		segments = append(segments, TranscriptSegment{
-			Text:       segment.Cue.SimpleText,
-			StartMs:    start,
-			OffsetText: formatted,
-			Duration:   duration,
-		})
-	}
-
-	return segments
-}

downloader/vendor/github.com/kkdai/youtube/v2/utils.go

@@ -1,97 +0,0 @@
-package youtube
-
-import (
-	"encoding/base64"
-
-	sjson "github.com/bitly/go-simplejson"
-)
-
-type chunk struct {
-	start int64
-	end   int64
-	data  chan []byte
-}
-
-func getChunks(totalSize, chunkSize int64) []chunk {
-	var chunks []chunk
-
-	for start := int64(0); start < totalSize; start += chunkSize {
-		end := chunkSize + start - 1
-		if end > totalSize-1 {
-			end = totalSize - 1
-		}
-
-		chunks = append(chunks, chunk{start, end, make(chan []byte, 1)})
-	}
-
-	return chunks
-}
-
-func getFirstKeyJSON(j *sjson.Json) *sjson.Json {
-	m, err := j.Map()
-	if err != nil {
-		return j
-	}
-
-	for key := range m {
-		return j.Get(key)
-	}
-
-	return j
-}
-
-func isValidJSON(j *sjson.Json) bool {
-	b, err := j.MarshalJSON()
-	if err != nil {
-		return false
-	}
-
-	if len(b) <= 4 {
-		return false
-	}
-
-	return true
-}
-
-func sjsonGetText(j *sjson.Json, paths ...string) string {
-	for _, path := range paths {
-		if isValidJSON(j.Get(path)) {
-			j = j.Get(path)
-		}
-	}
-
-	if text, err := j.String(); err == nil {
-		return text
-	}
-
-	if isValidJSON(j.Get("text")) {
-		return j.Get("text").MustString()
-	}
-
-	if p := j.Get("runs"); isValidJSON(p) {
-		var text string
-
-		for i := 0; i < len(p.MustArray()); i++ {
-			if textNode := p.GetIndex(i).Get("text"); isValidJSON(textNode) {
-				text += textNode.MustString()
-			}
-		}
-
-		return text
-	}
-
-	return ""
-}
-
-func getContinuation(j *sjson.Json) string {
-	return j.GetPath("continuations").
-		GetIndex(0).GetPath("nextContinuationData", "continuation").MustString()
-}
-
-func base64PadEnc(str string) string {
-	return base64.StdEncoding.EncodeToString([]byte(str))
-}
-
-func base64Enc(str string) string {
-	return base64.RawStdEncoding.EncodeToString([]byte(str))
-}

downloader/vendor/github.com/kkdai/youtube/v2/video.go

@@ -1,147 +0,0 @@
-package youtube
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"net/url"
-	"regexp"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-)
-
-type Video struct {
-	ID              string
-	Title           string
-	Description     string
-	Author          string
-	ChannelID       string
-	ChannelHandle   string
-	Views           int
-	Duration        time.Duration
-	PublishDate     time.Time
-	Formats         FormatList
-	Thumbnails      Thumbnails
-	DASHManifestURL string // URI of the DASH manifest file
-	HLSManifestURL  string // URI of the HLS manifest file
-	CaptionTracks   []CaptionTrack
-}
-
-const dateFormat = "2006-01-02"
-
-func (v *Video) parseVideoInfo(body []byte) error {
-	var prData playerResponseData
-	if err := json.Unmarshal(body, &prData); err != nil {
-		return fmt.Errorf("unable to parse player response JSON: %w", err)
-	}
-
-	if err := v.isVideoFromInfoDownloadable(prData); err != nil {
-		return err
-	}
-
-	return v.extractDataFromPlayerResponse(prData)
-}
-
-func (v *Video) isVideoFromInfoDownloadable(prData playerResponseData) error {
-	return v.isVideoDownloadable(prData, false)
-}
-
-var playerResponsePattern = regexp.MustCompile(`var ytInitialPlayerResponse\s*=\s*(\{.+?\});`)
-
-func (v *Video) parseVideoPage(body []byte) error {
-	initialPlayerResponse := playerResponsePattern.FindSubmatch(body)
-	if initialPlayerResponse == nil || len(initialPlayerResponse) < 2 {
-		return errors.New("no ytInitialPlayerResponse found in the server's answer")
-	}
-
-	var prData playerResponseData
-	if err := json.Unmarshal(initialPlayerResponse[1], &prData); err != nil {
-		return fmt.Errorf("unable to parse player response JSON: %w", err)
-	}
-
-	if err := v.isVideoFromPageDownloadable(prData); err != nil {
-		return err
-	}
-
-	return v.extractDataFromPlayerResponse(prData)
-}
-
-func (v *Video) isVideoFromPageDownloadable(prData playerResponseData) error {
-	return v.isVideoDownloadable(prData, true)
-}
-
-func (v *Video) isVideoDownloadable(prData playerResponseData, isVideoPage bool) error {
-	// Check if video is downloadable
-	switch prData.PlayabilityStatus.Status {
-	case "OK":
-		return nil
-	case "LOGIN_REQUIRED":
-		// for some reason they use same status message for age-restricted and private videos
-		if strings.HasPrefix(prData.PlayabilityStatus.Reason, "This video is private") {
-			return ErrVideoPrivate
-		}
-		return ErrLoginRequired
-	}
-
-	if !isVideoPage && !prData.PlayabilityStatus.PlayableInEmbed {
-		return ErrNotPlayableInEmbed
-	}
-
-	return &ErrPlayabiltyStatus{
-		Status: prData.PlayabilityStatus.Status,
-		Reason: prData.PlayabilityStatus.Reason,
-	}
-}
-
-func (v *Video) extractDataFromPlayerResponse(prData playerResponseData) error {
-	v.Title = prData.VideoDetails.Title
-	v.Description = prData.VideoDetails.ShortDescription
-	v.Author = prData.VideoDetails.Author
-	v.Thumbnails = prData.VideoDetails.Thumbnail.Thumbnails
-	v.ChannelID = prData.VideoDetails.ChannelID
-	v.CaptionTracks = prData.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks
-
-	if views, _ := strconv.Atoi(prData.VideoDetails.ViewCount); views > 0 {
-		v.Views = views
-	}
-
-	if seconds, _ := strconv.Atoi(prData.VideoDetails.LengthSeconds); seconds > 0 {
-		v.Duration = time.Duration(seconds) * time.Second
-	}
-
-	if seconds, _ := strconv.Atoi(prData.Microformat.PlayerMicroformatRenderer.LengthSeconds); seconds > 0 {
-		v.Duration = time.Duration(seconds) * time.Second
-	}
-
-	if str := prData.Microformat.PlayerMicroformatRenderer.PublishDate; str != "" {
-		v.PublishDate, _ = time.Parse(dateFormat, str)
-	}
-
-	if profileURL, err := url.Parse(prData.Microformat.PlayerMicroformatRenderer.OwnerProfileURL); err == nil && len(profileURL.Path) > 1 {
-		v.ChannelHandle = profileURL.Path[1:]
-	}
-
-	// Assign Streams
-	v.Formats = append(prData.StreamingData.Formats, prData.StreamingData.AdaptiveFormats...)
-	if len(v.Formats) == 0 {
-		return errors.New("no formats found in the server's answer")
-	}
-
-	// Sort formats by bitrate
-	sort.SliceStable(v.Formats, v.SortBitrateDesc)
-
-	v.HLSManifestURL = prData.StreamingData.HlsManifestURL
-	v.DASHManifestURL = prData.StreamingData.DashManifestURL
-
-	return nil
-}
-
-func (v *Video) SortBitrateDesc(i int, j int) bool {
-	return v.Formats[i].Bitrate > v.Formats[j].Bitrate
-}
-
-func (v *Video) SortBitrateAsc(i int, j int) bool {
-	return v.Formats[i].Bitrate < v.Formats[j].Bitrate
-}

downloader/vendor/github.com/kkdai/youtube/v2/video_id.go

@@ -1,34 +0,0 @@
-package youtube
-
-import (
-	"regexp"
-	"strings"
-)
-
-var videoRegexpList = []*regexp.Regexp{
-	regexp.MustCompile(`(?:v|embed|shorts|watch\?v)(?:=|/)([^"&?/=%]{11})`),
-	regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`),
-	regexp.MustCompile(`([^"&?/=%]{11})`),
-}
-
-// ExtractVideoID extracts the videoID from the given string
-func ExtractVideoID(videoID string) (string, error) {
-	if strings.Contains(videoID, "youtu") || strings.ContainsAny(videoID, "\"?&/<%=") {
-		for _, re := range videoRegexpList {
-			if isMatch := re.MatchString(videoID); isMatch {
-				subs := re.FindStringSubmatch(videoID)
-				videoID = subs[1]
-			}
-		}
-	}
-
-	if strings.ContainsAny(videoID, "?&/<%=") {
-		return "", ErrInvalidCharactersInVideoID
-	}
-
-	if len(videoID) < 10 {
-		return "", ErrVideoIDMinLength
-	}
-
-	return videoID, nil
-}

youtubeWatcher/go.mod

@@ -0,0 +1,3 @@
+module ywatcher
+
+go 1.24.2

youtubeWatcher/main.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"sync"
+	"time"
+)
+
+type DownloadRequest struct {
+	Link string `json:"link"`
+}
+type RssWatcher struct {
+	Feed *RssFeed
+}
+
+var videoRegex = regexp.MustCompile(`yt:video:(?<videoid>[^ ]+) (?:[^ ]+ ){2}(?<videotitle>.+?)https(?:[^ ]+ ){2}(?<date>[^ ]+)`)
+var feeds []*RssFeed = []*RssFeed{
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCMwJJL5FJFuTRT55ksbQ4GQ", Id: "@AsmongoldClips"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
+}
+
+func (w *RssWatcher) Watch(videoUrls chan string) error {
+	ticker := time.NewTicker(1 * time.Minute)
+	defer ticker.Stop()
+
+	log.Printf("[%s]: Watcher started, checking every minute.", w.Feed.Id)
+
+	w.CheckFeed(videoUrls)
+	for {
+		select {
+		case <-ticker.C:
+			log.Printf("[%s]: Checking feed", w.Feed.Id)
+			err := w.CheckFeed(videoUrls)
+			if err != nil {
+				return fmt.Errorf("watcher %s failed to check feed: %w", w.Feed.Id, err)
+			}
+			log.Printf("[%s]: Successfully checked feed", w.Feed.Id)
+		}
+	}
+}
+
+func (w *RssWatcher) CheckFeed(videoUrls chan string) error {
+	log.Printf("Checking feed URL: %s", w.Feed.Url)
+	resp, err := http.Get(w.Feed.Url)
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
+	}
+	defer resp.Body.Close()
+
+	log.Printf("Received response with status code: %d", resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
+	}
+
+	// os.WriteFile("cache.xml", body, 0644)
+	// body, err := os.ReadFile("cache.xml")
+	// if err != nil {
+	// 	return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
+	// }
+
+	var feed Feed
+	err = xml.Unmarshal(body, &feed)
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to unmarshal feed: %w", w.Feed.Id, err)
+	}
+
+	for _, entry := range feed.Entry {
+		uploaded, err := time.Parse(time.RFC3339, entry.Published)
+		if err != nil {
+			return fmt.Errorf("[%s]: failed to parse published date: %w", w.Feed.Id, err)
+		}
+		if uploaded.Before(w.Feed.LastSeen) {
+			log.Printf("[%s]: Skipping video titled %q because it was uploaded before %s", w.Feed.Id, entry.Title, w.Feed.LastSeen.Format(time.RFC3339))
+			continue
+		}
+		log.Printf("[%s]: Found new video titled %q with url %q", w.Feed.Id, entry.Title, entry.Link.Href)
+		videoUrls <- entry.Link.Href
+	}
+	err = w.Feed.WriteLastSeen(time.Now())
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to write last seen: %w", w.Feed.Id, err)
+	}
+	w.Feed.LastSeen = time.Now()
+
+	return nil
+}
+
+var Error *log.Logger
+var Warning *log.Logger
+
+func init() {
+	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
+	logFile, err := os.Create("ywatcher.log")
+	if err != nil {
+		log.Printf("Error creating log file: %v", err)
+		os.Exit(1)
+	}
+	logger := io.MultiWriter(os.Stdout, logFile)
+	log.SetOutput(logger)
+
+	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
+		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
+		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+}
+
+func main() {
+	videoUrls := make(chan string, 12)
+
+	wg := &sync.WaitGroup{}
+	for _, feed := range feeds {
+		wg.Add(1)
+		go func(feed *RssFeed) {
+			err := feed.UpdateLastSeen()
+			if err != nil {
+				Error.Printf("failed to update lastseen for feed %s: %v", feed.Id, err)
+				panic(err)
+			}
+
+			defer wg.Done()
+			watcher := RssWatcher{
+				Feed: feed,
+			}
+			err = watcher.Watch(videoUrls)
+			if err != nil {
+				Error.Printf("watcher %s failed to watch feed: %v", feed.Id, err)
+				panic(err)
+			}
+		}(feed)
+	}
+
+	go func() {
+		for videoUrl := range videoUrls {
+			log.Printf("Got new video with url %q", videoUrl)
+			err := Download(videoUrl)
+			if err != nil {
+				Error.Printf("failed to download video: %v", err)
+				panic(err)
+			}
+		}
+	}()
+
+	wg.Wait()
+}

youtubeWatcher/nsqWriter.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+)
+
+const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
+
+type Item struct {
+	Link string `json:"link"`
+}
+
+func Download(url string) error {
+	log.Printf("Starting download for URL: %s", url)
+
+	req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil)
+	if err != nil {
+		return fmt.Errorf("error creating POST request: %++v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	item := new(Item)
+	item.Link = url
+
+	body, err := json.Marshal(item)
+	if err != nil {
+		return fmt.Errorf("error marshalling subscription body: %++v", err)
+	}
+	req.Body = io.NopCloser(bytes.NewReader(body))
+
+	client := http.Client{}
+	log.Printf("Sending POST request to %s", URL)
+	res, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error sending POST request: %++v", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("non-OK HTTP status: %d", res.StatusCode)
+	}
+	log.Printf("Successfully enqueued %s", url)
+	return nil
+}

youtubeWatcher/rssfeed.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sync"
+	"time"
+)
+
+type RssFeed struct {
+	Url       string
+	Id        string
+	LastSeen  time.Time
+	fileMutex sync.Mutex
+}
+
+func (f *RssFeed) UpdateLastSeen() error {
+	log.Printf("[%s]: Updating last seen time", f.Id)
+	lastSeen, err := f.ReadLastSeen()
+	if err != nil {
+		return fmt.Errorf("failed to read lastseen file for feed %s: %w", f.Id, err)
+	}
+	f.LastSeen = lastSeen
+	log.Printf("[%s]: Last seen time updated to: %s", f.Id, f.LastSeen)
+	return nil
+}
+func (f *RssFeed) ReadLastSeen() (time.Time, error) {
+	f.fileMutex.Lock()
+	defer f.fileMutex.Unlock()
+
+	lastSeen := time.Now()
+	log.Printf("[%s]: Attempting to open lastseen file...", f.Id)
+	lastSeenFile, err := os.Open("lastseen" + f.Id)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Printf("[%s]: lastseen file does not exist, creating a new one...", f.Id)
+			lastSeenFile, err = os.Create("lastseen" + f.Id)
+			if err != nil {
+				return lastSeen, fmt.Errorf("failed to create lastseen file: %w", err)
+			}
+			log.Printf("[%s]: Writing current time to lastseen file...", f.Id)
+			lastSeenFile.Write([]byte(lastSeen.Format(time.RFC3339)))
+			lastSeenFile.Sync()
+			log.Printf("[%s]: Successfully created lastseen file with current time.", f.Id)
+			return lastSeen, nil
+		} else {
+			log.Printf("[%s]: Error opening lastseen file: %v", f.Id, err)
+			return lastSeen, fmt.Errorf("failed to read lastseen file: %w", err)
+		}
+	}
+	log.Printf("[%s]: Reading contents of lastseen file...", f.Id)
+	lastSeenBytes, err := io.ReadAll(lastSeenFile)
+	if err != nil {
+		log.Printf("[%s]: Error reading lastseen file: %v", f.Id, err)
+		return lastSeen, fmt.Errorf("failed to read lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Parsing lastseen time...", f.Id)
+	lastSeen, err = time.Parse(time.RFC3339, string(lastSeenBytes))
+	if err != nil {
+		log.Printf("[%s]: Error parsing lastseen file: %v", f.Id, err)
+		return lastSeen, fmt.Errorf("failed to parse lastseen file: %w", err)
+	}
+	lastSeenFile.Close()
+	log.Printf("[%s]: Last seen time read: %s", f.Id, lastSeen)
+	return lastSeen, nil
+}
+
+func (f *RssFeed) WriteLastSeen(when time.Time) error {
+	f.fileMutex.Lock()
+	defer f.fileMutex.Unlock()
+
+	log.Printf("[%s]: Attempting to create lastseen file...", f.Id)
+	lastSeenFile, err := os.Create("lastseen" + f.Id)
+	if err != nil {
+		log.Printf("[%s]: Error creating lastseen file: %v", f.Id, err)
+		return fmt.Errorf("failed to create lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Successfully created lastseen file: lastseen%s", f.Id, f.Id)
+
+	log.Printf("[%s]: Writing last seen time to file: %s", f.Id, when.Format(time.RFC3339))
+	_, err = lastSeenFile.Write([]byte(when.Format(time.RFC3339)))
+	if err != nil {
+		log.Printf("[%s]: Error writing to lastseen file: %v", f.Id, err)
+		return fmt.Errorf("failed to write to lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Successfully wrote last seen time to file.", f.Id)
+
+	err = lastSeenFile.Close()
+	if err != nil {
+		log.Printf("[%s]: Error closing lastseen file: %v", f.Id, err)
+		return fmt.Errorf("failed to close lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Lastseen file closed successfully.", f.Id)
+	return nil
+}

youtubeWatcher/rssfeedResType.go

@@ -0,0 +1,86 @@
+package main
+
+import "encoding/xml"
+
+type Feed struct {
+	XMLName   xml.Name `xml:"feed"`
+	Text      string   `xml:",chardata"`
+	Yt        string   `xml:"yt,attr"`
+	Media     string   `xml:"media,attr"`
+	Xmlns     string   `xml:"xmlns,attr"`
+	Link      []Link   `xml:"link"`
+	ID        string   `xml:"id"`
+	ChannelId string   `xml:"channelId"`
+	Title     string   `xml:"title"`
+	Author    Author   `xml:"author"`
+	Published string   `xml:"published"`
+	Entry     []Entry  `xml:"entry"`
+}
+
+type Link struct {
+	Text string `xml:",chardata"`
+	Rel  string `xml:"rel,attr"`
+	Href string `xml:"href,attr"`
+}
+
+type Author struct {
+	Text string `xml:",chardata"`
+	Name string `xml:"name"`
+	URI  string `xml:"uri"`
+}
+
+type Entry struct {
+	Text      string `xml:",chardata"`
+	ID        string `xml:"id"`
+	VideoId   string `xml:"videoId"`
+	ChannelId string `xml:"channelId"`
+	Title     string `xml:"title"`
+	Link      Link   `xml:"link"`
+	Author    Author `xml:"author"`
+	Published string `xml:"published"`
+	Updated   string `xml:"updated"`
+	Group     Group  `xml:"group"`
+}
+
+type Group struct {
+	Text        string    `xml:",chardata"`
+	Title       string    `xml:"title"`
+	Content     Content   `xml:"content"`
+	Thumbnail   Thumbnail `xml:"thumbnail"`
+	Description string    `xml:"description"`
+	Community   Community `xml:"community"`
+}
+
+type Content struct {
+	Text   string `xml:",chardata"`
+	URL    string `xml:"url,attr"`
+	Type   string `xml:"type,attr"`
+	Width  string `xml:"width,attr"`
+	Height string `xml:"height,attr"`
+}
+
+type Thumbnail struct {
+	Text   string `xml:",chardata"`
+	URL    string `xml:"url,attr"`
+	Width  string `xml:"width,attr"`
+	Height string `xml:"height,attr"`
+}
+
+type Community struct {
+	Text       string     `xml:",chardata"`
+	StarRating StarRating `xml:"starRating"`
+	Statistics Statistics `xml:"statistics"`
+}
+
+type StarRating struct {
+	Text    string `xml:",chardata"`
+	Count   string `xml:"count,attr"`
+	Average string `xml:"average,attr"`
+	Min     string `xml:"min,attr"`
+	Max     string `xml:"max,attr"`
+}
+
+type Statistics struct {
+	Text  string `xml:",chardata"`
+	Views string `xml:"views,attr"`
+}