diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/.gitignore b/downloader/vendor/github.com/kkdai/youtube/v2/.gitignore deleted file mode 100644 index 818294a..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/.gitignore +++ /dev/null @@ -1,8 +0,0 @@ -/.idea -/.vscode -download_test -/bin -/dist -/output -*.out -.DS_Store diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml b/downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml deleted file mode 100644 index c0319a2..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml +++ /dev/null @@ -1,8 +0,0 @@ -issues: - exclude: - - Subprocess launched with function call - # Excluding configuration per-path, per-linter, per-text and per-source - exclude-rules: - # Exclude some linters from running on tests files. - - path: cmd/ - text: InsecureSkipVerify diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml b/downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml deleted file mode 100644 index f7d7c44..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml +++ /dev/null @@ -1,20 +0,0 @@ -project_name: youtubedr - -env: - # Build without CGO to don't depend on specific glibc versions - - CGO_ENABLED=0 - -builds: - - main: ./cmd/youtubedr - binary: youtubedr - goos: - - windows - - darwin - - linux - - freebsd - goarch: - - amd64 - - arm - - arm64 - flags: - - -trimpath diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/LICENSE b/downloader/vendor/github.com/kkdai/youtube/v2/LICENSE deleted file mode 100644 index d252f53..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/LICENSE +++ /dev/null @@ -1,22 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2015 Evan Lin - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. - diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/Makefile b/downloader/vendor/github.com/kkdai/youtube/v2/Makefile deleted file mode 100644 index 3548843..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/Makefile +++ /dev/null @@ -1,57 +0,0 @@ -FILES_TO_FMT ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print) -LOGLEVEL ?= debug - -## help: Show makefile commands -.PHONY: help -help: Makefile - @echo "---- Project: kkdai/youtube ----" - @echo " Usage: make COMMAND" - @echo - @echo " Management Commands:" - @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' - @echo - -## build: Build project -.PHONY: build -build: - goreleaser --rm-dist - -## deps: Ensures fresh go.mod and go.sum -.PHONY: deps -deps: - go mod tidy - go mod verify - -## lint: Run golangci-lint check -.PHONY: lint -lint: - command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION) - echo "golangci-lint checking..." - golangci-lint run --timeout=30m --enable=misspell --enable=gosec --enable=gofmt --enable=goimports --enable=revive ./cmd/... ./... - go vet ./... - -## format: Formats Go code -.PHONY: format -format: - @echo ">> formatting code" - @gofmt -s -w $(FILES_TO_FMT) - -## test-unit: Run all Youtube Go unit tests -.PHONY: test-unit -test-unit: - LOGLEVEL=${LOGLEVEL} go test -v -cover ./... - -## test-integration: Run all Youtube Go integration tests -.PHONY: test-integration -test-integration: - mkdir -p output - rm -f output/* - LOGLEVEL=${LOGLEVEL} ARTIFACTS=output go test -v -race -covermode=atomic -coverprofile=coverage.out -tags=integration ./... - -.PHONY: coverage.out -coverage.out: - -## clean: Clean files and downloaded videos from builds during development -.PHONY: clean -clean: - rm -rf dist *.mp4 *.mkv diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/README.md b/downloader/vendor/github.com/kkdai/youtube/v2/README.md deleted file mode 100644 index cc34eb0..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/README.md +++ /dev/null @@ -1,161 +0,0 @@ -Download Youtube Video in Golang -================== - -[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/kkdai/youtube/master/LICENSE) -[![Go Reference](https://pkg.go.dev/badge/github.com/kkdai/youtube.svg)](https://pkg.go.dev/github.com/kkdai/youtube/v2) -[![Build Status](https://github.com/kkdai/youtube/workflows/go/badge.svg?branch=master)](https://github.com/kkdai/youtube/actions) -[![Coverage](https://codecov.io/gh/kkdai/youtube/branch/master/graph/badge.svg)](https://codecov.io/gh/kkdai/youtube) -[![](https://goreportcard.com/badge/github.com/kkdai/youtube)](https://goreportcard.com/badge/github.com/kkdai/youtube) - - -This package is a Youtube video download package, for more detail refer [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) for more download options. - -This tool is meant to be used to download CC0 licenced content, we do not support nor recommend using it for illegal activities. - -## Overview - * [Install](#installation) - * [Usage](#usage) - * [Example: Download video from \[dotGo 2015 - Rob Pike - Simplicity is Complicated\]](#download-dotGo-2015-rob-pike-video) - -## Installation - -### Install via go get - -Please ensure you have installed Go 1.22 or later. - -```shell -go get github.com/kkdai/youtube/v2 -``` - -### From source code - -```shell -git clone https://github.com/kkdai/youtube.git -cd youtube -go run ./cmd/youtubedr -``` - -### Mac - -```shell -brew install youtubedr -``` - -### in Termux -```shell -pkg install youtubedr -``` -### You can also find this package in -- [archlinux](https://aur.archlinux.org/packages/youtubedr/) (thanks to [cjsthompson](https://github.com/cjsthompson)) -- [Termux package](https://github.com/termux/termux-packages/tree/master/packages/youtubedr) (thanks to [kcubeterm](https://github.com/kcubeterm)) -- [Homebrew](https://formulae.brew.sh/formula/youtubedr) (thanks to [kkc](https://github.com/kkc)) - -## Usage - -### Use the binary directly -It's really simple to use, just get the video id from youtube url - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM` - -```shell -$ youtubedr download QAGDGja7kbs -$ youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM -``` - - -### Use this package in your golang program - -Please check out the [example_test.go](example_test.go) for example code. - - -## Example: - * ### Get information of dotGo-2015-rob-pike video for downloading - - `go get github.com/kkdai/youtube/v2/youtubedr` - - Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) - - ``` - youtubedr info https://www.youtube.com/watch?v=rFejpH_tAHM - - Title: dotGo 2015 - Rob Pike - Simplicity is Complicated - Author: dotconferences - -----available streams----- - itag: 18 , quality: medium , type: video/mp4; codecs="avc1.42001E, mp4a.40.2" - itag: 22 , quality: hd720 , type: video/mp4; codecs="avc1.64001F, mp4a.40.2" - itag: 137 , quality: hd1080 , type: video/mp4; codecs="avc1.640028" - itag: 248 , quality: hd1080 , type: video/webm; codecs="vp9" - ........ - ``` - * ### Download dotGo-2015-rob-pike-video - - `go get github.com/kkdai/youtube/v2/youtubedr` - - Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) - - ``` - youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM - ``` - - * ### Download video to specific folder and name - - `go get github.com/kkdai/youtube/v2/youtubedr` - - Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) to current directory and name the file to simplicity-is-complicated.mp4 - - ``` - youtubedr download -d ./ -o simplicity-is-complicated.mp4 https://www.youtube.com/watch?v=rFejpH_tAHM - ``` - - * ### Download video with specific quality - - `go get github.com/kkdai/youtube/v2/youtubedr` - - Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) with specific quality - - ``` - youtubedr download -q medium https://www.youtube.com/watch?v=rFejpH_tAHM - ``` - - #### Special case by quality hd1080: - Installation of ffmpeg is necessary for hd1080 - ``` - ffmpeg //check ffmpeg is installed, if not please download ffmpeg and set to your PATH. - youtubedr download -q hd1080 https://www.youtube.com/watch?v=rFejpH_tAHM - ``` - - - * ### Download video with specific itag - - `go get github.com/kkdai/youtube/v2/youtubedr` - - Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) - - ``` - youtubedr download -q 18 https://www.youtube.com/watch?v=rFejpH_tAHM - ``` - -## How it works - -- Parse the video ID you input in URL - - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM` -- Get video information via video id. - - Use URL: `http://youtube.com/get_video_info?video_id=` -- Parse and decode video information. - - Download URL in "url=" - - title in "title=" -- Download video from URL - - Need the string combination of "url" - -## Inspired -- [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) -- [https://github.com/lepidosteus/youtube-dl](https://github.com/lepidosteus/youtube-dl) -- [拆解 Youtube 影片下載位置](http://hkgoldenmra.blogspot.tw/2013/05/youtube.html) -- [iawia002/annie](https://github.com/iawia002/annie) -- [How to get url from obfuscate video info: youtube video downloader with php](https://stackoverflow.com/questions/60607291/youtube-video-downloader-with-php) - - -## Project52 -It is one of my [project 52](https://github.com/kkdai/project52). - - -## License -This package is licensed under MIT license. See LICENSE for details. diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md b/downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md deleted file mode 100644 index 179112a..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md +++ /dev/null @@ -1,11 +0,0 @@ -Tracking Youtube decipher change - -### 2022/01/21: - -#### action objects: - -var $z={fA:function(a){a.reverse()},S2:function(a,b){a.splice(0,b)},l6:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}}; -#### actions function: - -Npa=function(a){a=a.split("");$z.S2(a,3);$z.fA(a,45);$z.l6(a,31);$z.S2(a,1);$z.fA(a,63);$z.S2(a,2);$z.fA(a,68);return a.join("")}; - diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go b/downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go deleted file mode 100644 index 2f8b399..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go +++ /dev/null @@ -1,29 +0,0 @@ -package youtube - -import ( - "log/slog" - "os" - "path/filepath" -) - -// destination for artifacts, used by integration tests -var artifactsFolder = os.Getenv("ARTIFACTS") - -func writeArtifact(name string, content []byte) { - // Ensure folder exists - err := os.MkdirAll(artifactsFolder, os.ModePerm) - if err != nil { - slog.Error("unable to create artifacts folder", "path", artifactsFolder, "error", err) - return - } - - path := filepath.Join(artifactsFolder, name) - err = os.WriteFile(path, content, 0600) - - log := slog.With("path", path) - if err != nil { - log.Error("unable to write artifact", "error", err) - } else { - log.Debug("artifact created") - } -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/client.go b/downloader/vendor/github.com/kkdai/youtube/v2/client.go deleted file mode 100644 index 587bbb1..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/client.go +++ /dev/null @@ -1,632 +0,0 @@ -package youtube - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "log/slog" - "math/rand" - "net/http" - "net/url" - "strconv" - "sync/atomic" -) - -const ( - Size1Kb = 1024 - Size1Mb = Size1Kb * 1024 - Size10Mb = Size1Mb * 10 - - playerParams = "CgIQBg==" -) - -var ErrNoFormat = errors.New("no video format provided") - -// DefaultClient type to use. No reason to change but you could if you wanted to. -var DefaultClient = IOSClient - -// Client offers methods to download video metadata and video streams. -type Client struct { - // HTTPClient can be used to set a custom HTTP client. - // If not set, http.DefaultClient will be used - HTTPClient *http.Client - - // MaxRoutines to use when downloading a video. - MaxRoutines int - - // ChunkSize to use when downloading videos in chunks. Default is Size10Mb. - ChunkSize int64 - - // playerCache caches the JavaScript code of a player response - playerCache playerCache - - client *clientInfo - - consentID string -} - -func (c *Client) assureClient() { - if c.client == nil { - c.client = &DefaultClient - } -} - -// GetVideo fetches video metadata -func (c *Client) GetVideo(url string) (*Video, error) { - return c.GetVideoContext(context.Background(), url) -} - -// GetVideoContext fetches video metadata with a context -func (c *Client) GetVideoContext(ctx context.Context, url string) (*Video, error) { - id, err := ExtractVideoID(url) - if err != nil { - return nil, fmt.Errorf("extractVideoID failed: %w", err) - } - - return c.videoFromID(ctx, id) -} - -func (c *Client) videoFromID(ctx context.Context, id string) (*Video, error) { - c.assureClient() - - body, err := c.videoDataByInnertube(ctx, id) - if err != nil { - return nil, err - } - - v := Video{ - ID: id, - } - - // return early if all good - if err = v.parseVideoInfo(body); err == nil { - return &v, nil - } - - // If the uploader has disabled embedding the video on other sites, parse video page - if errors.Is(err, ErrNotPlayableInEmbed) { - // additional parameters are required to access clips with sensitiv content - html, err := c.httpGetBodyBytes(ctx, "https://www.youtube.com/watch?v="+id+"&bpctr=9999999999&has_verified=1") - if err != nil { - return nil, err - } - - return &v, v.parseVideoPage(html) - } - - // If the uploader marked the video as inappropriate for some ages, use embed player - if errors.Is(err, ErrLoginRequired) { - c.client = &EmbeddedClient - - bodyEmbed, errEmbed := c.videoDataByInnertube(ctx, id) - if errEmbed == nil { - errEmbed = v.parseVideoInfo(bodyEmbed) - } - - if errEmbed == nil { - return &v, nil - } - - // private video clearly not age-restricted and thus should be explicit - if errEmbed == ErrVideoPrivate { - return &v, errEmbed - } - - // wrapping error so its clear whats happened - return &v, fmt.Errorf("can't bypass age restriction: %w", errEmbed) - } - - // undefined error - return &v, err -} - -type innertubeRequest struct { - VideoID string `json:"videoId,omitempty"` - BrowseID string `json:"browseId,omitempty"` - Continuation string `json:"continuation,omitempty"` - Context inntertubeContext `json:"context"` - PlaybackContext *playbackContext `json:"playbackContext,omitempty"` - ContentCheckOK bool `json:"contentCheckOk,omitempty"` - RacyCheckOk bool `json:"racyCheckOk,omitempty"` - Params string `json:"params"` -} - -type playbackContext struct { - ContentPlaybackContext contentPlaybackContext `json:"contentPlaybackContext"` -} - -type contentPlaybackContext struct { - // SignatureTimestamp string `json:"signatureTimestamp"` - HTML5Preference string `json:"html5Preference"` -} - -type inntertubeContext struct { - Client innertubeClient `json:"client"` -} - -type innertubeClient struct { - HL string `json:"hl"` - GL string `json:"gl"` - ClientName string `json:"clientName"` - ClientVersion string `json:"clientVersion"` - AndroidSDKVersion int `json:"androidSDKVersion,omitempty"` - UserAgent string `json:"userAgent,omitempty"` - TimeZone string `json:"timeZone"` - UTCOffset int `json:"utcOffsetMinutes"` - DeviceModel string `json:"deviceModel,omitempty"` -} - -// client info for the innertube API -type clientInfo struct { - name string - key string - version string - userAgent string - androidVersion int - deviceModel string -} - -var ( - // WebClient, better to use Android client but go ahead. - WebClient = clientInfo{ - name: "WEB", - version: "2.20220801.00.00", - key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", - userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", - } - - // AndroidClient, download go brrrrrr. - AndroidClient = clientInfo{ - name: "ANDROID", - version: "18.11.34", - key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", - userAgent: "com.google.android.youtube/18.11.34 (Linux; U; Android 11) gzip", - androidVersion: 30, - } - - // IOSClient Client based brrrr. - IOSClient = clientInfo{ - name: "IOS", - version: "19.45.4", - key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", - userAgent: "com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)", - deviceModel: "iPhone16,2", - } - - // EmbeddedClient, not really tested. - EmbeddedClient = clientInfo{ - name: "WEB_EMBEDDED_PLAYER", - version: "1.19700101", - key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", // seems like same key works for both clients - userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", - } -) - -func (c *Client) videoDataByInnertube(ctx context.Context, id string) ([]byte, error) { - data := innertubeRequest{ - VideoID: id, - Context: prepareInnertubeContext(*c.client), - ContentCheckOK: true, - RacyCheckOk: true, - // Params: playerParams, - PlaybackContext: &playbackContext{ - ContentPlaybackContext: contentPlaybackContext{ - // SignatureTimestamp: sts, - HTML5Preference: "HTML5_PREF_WANTS", - }, - }, - } - - return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/player?key="+c.client.key, data) -} - -func (c *Client) transcriptDataByInnertube(ctx context.Context, id string, lang string) ([]byte, error) { - data := innertubeRequest{ - Context: prepareInnertubeContext(*c.client), - Params: transcriptVideoID(id, lang), - } - - return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/get_transcript?key="+c.client.key, data) -} - -func prepareInnertubeContext(clientInfo clientInfo) inntertubeContext { - return inntertubeContext{ - Client: innertubeClient{ - HL: "en", - GL: "US", - TimeZone: "UTC", - DeviceModel: clientInfo.deviceModel, - ClientName: clientInfo.name, - ClientVersion: clientInfo.version, - AndroidSDKVersion: clientInfo.androidVersion, - UserAgent: clientInfo.userAgent, - }, - } -} - -func prepareInnertubePlaylistData(ID string, continuation bool, clientInfo clientInfo) innertubeRequest { - context := prepareInnertubeContext(clientInfo) - - if continuation { - return innertubeRequest{ - Context: context, - Continuation: ID, - ContentCheckOK: true, - RacyCheckOk: true, - Params: playerParams, - } - } - - return innertubeRequest{ - Context: context, - BrowseID: "VL" + ID, - ContentCheckOK: true, - RacyCheckOk: true, - Params: playerParams, - } -} - -// transcriptVideoID encodes the video ID to the param used to fetch transcripts. -func transcriptVideoID(videoID string, lang string) string { - langCode := encTranscriptLang(lang) - - // This can be optionally appened to the Sprintf str, not sure what it means - // *3engagement-panel-searchable-transcript-search-panel\x30\x00\x38\x01\x40\x01 - return base64Enc(fmt.Sprintf("\n\x0b%s\x12\x12%s\x18\x01", videoID, langCode)) -} - -func encTranscriptLang(languageCode string) string { - s := fmt.Sprintf("\n\x03asr\x12\x02%s\x1a\x00", languageCode) - s = base64PadEnc(s) - - return url.QueryEscape(s) -} - -// GetPlaylist fetches playlist metadata -func (c *Client) GetPlaylist(url string) (*Playlist, error) { - return c.GetPlaylistContext(context.Background(), url) -} - -// GetPlaylistContext fetches playlist metadata, with a context, along with a list of Videos, and some basic information -// for these videos. Playlist entries cannot be downloaded, as they lack all the required metadata, but -// can be used to enumerate all IDs, Authors, Titles, etc. -func (c *Client) GetPlaylistContext(ctx context.Context, url string) (*Playlist, error) { - c.assureClient() - - id, err := extractPlaylistID(url) - if err != nil { - return nil, fmt.Errorf("extractPlaylistID failed: %w", err) - } - - data := prepareInnertubePlaylistData(id, false, *c.client) - body, err := c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+c.client.key, data) - if err != nil { - return nil, err - } - - p := &Playlist{ID: id} - return p, p.parsePlaylistInfo(ctx, c, body) -} - -func (c *Client) VideoFromPlaylistEntry(entry *PlaylistEntry) (*Video, error) { - return c.videoFromID(context.Background(), entry.ID) -} - -func (c *Client) VideoFromPlaylistEntryContext(ctx context.Context, entry *PlaylistEntry) (*Video, error) { - return c.videoFromID(ctx, entry.ID) -} - -// GetStream returns the stream and the total size for a specific format -func (c *Client) GetStream(video *Video, format *Format) (io.ReadCloser, int64, error) { - return c.GetStreamContext(context.Background(), video, format) -} - -// GetStreamContext returns the stream and the total size for a specific format with a context. -func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *Format) (io.ReadCloser, int64, error) { - url, err := c.GetStreamURL(video, format) - if err != nil { - return nil, 0, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, 0, err - } - - r, w := io.Pipe() - contentLength := format.ContentLength - - if contentLength == 0 { - // some videos don't have length information - contentLength = c.downloadOnce(req, w, format) - } else { - // we have length information, let's download by chunks! - c.downloadChunked(ctx, req, w, format) - } - - return r, contentLength, nil -} - -func (c *Client) downloadOnce(req *http.Request, w *io.PipeWriter, _ *Format) int64 { - resp, err := c.httpDo(req) - if err != nil { - w.CloseWithError(err) //nolint:errcheck - return 0 - } - - go func() { - defer resp.Body.Close() - _, err := io.Copy(w, resp.Body) - if err == nil { - w.Close() - } else { - w.CloseWithError(err) //nolint:errcheck - } - }() - - contentLength := resp.Header.Get("Content-Length") - length, _ := strconv.ParseInt(contentLength, 10, 64) - - return length -} - -func (c *Client) getChunkSize() int64 { - if c.ChunkSize > 0 { - return c.ChunkSize - } - - return Size10Mb -} - -func (c *Client) getMaxRoutines(limit int) int { - routines := 10 - - if c.MaxRoutines > 0 { - routines = c.MaxRoutines - } - - if limit > 0 && routines > limit { - routines = limit - } - - return routines -} - -func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.PipeWriter, format *Format) { - chunks := getChunks(format.ContentLength, c.getChunkSize()) - maxRoutines := c.getMaxRoutines(len(chunks)) - - cancelCtx, cancel := context.WithCancel(ctx) - abort := func(err error) { - w.CloseWithError(err) - cancel() - } - - currentChunk := atomic.Uint32{} - for i := 0; i < maxRoutines; i++ { - go func() { - for { - chunkIndex := int(currentChunk.Add(1)) - 1 - if chunkIndex >= len(chunks) { - // no more chunks - return - } - - chunk := &chunks[chunkIndex] - err := c.downloadChunk(req.Clone(cancelCtx), chunk) - close(chunk.data) - - if err != nil { - abort(err) - return - } - } - }() - } - - go func() { - // copy chunks into the PipeWriter - for i := 0; i < len(chunks); i++ { - select { - case <-cancelCtx.Done(): - abort(context.Canceled) - return - case data := <-chunks[i].data: - _, err := io.Copy(w, bytes.NewBuffer(data)) - if err != nil { - abort(err) - } - } - } - - // everything succeeded - w.Close() - }() -} - -// GetStreamURL returns the url for a specific format -func (c *Client) GetStreamURL(video *Video, format *Format) (string, error) { - return c.GetStreamURLContext(context.Background(), video, format) -} - -// GetStreamURLContext returns the url for a specific format with a context -func (c *Client) GetStreamURLContext(ctx context.Context, video *Video, format *Format) (string, error) { - if format == nil { - return "", ErrNoFormat - } - - c.assureClient() - - if format.URL != "" { - if c.client.androidVersion > 0 { - return format.URL, nil - } - - return c.unThrottle(ctx, video.ID, format.URL) - } - - // TODO: check rest of this function, is it redundant? - - cipher := format.Cipher - if cipher == "" { - return "", ErrCipherNotFound - } - - uri, err := c.decipherURL(ctx, video.ID, cipher) - if err != nil { - return "", err - } - - return uri, err -} - -// httpDo sends an HTTP request and returns an HTTP response. -func (c *Client) httpDo(req *http.Request) (*http.Response, error) { - client := c.HTTPClient - if client == nil { - client = http.DefaultClient - } - - req.Header.Set("User-Agent", c.client.userAgent) - req.Header.Set("Origin", "https://youtube.com") - req.Header.Set("Sec-Fetch-Mode", "navigate") - - if len(c.consentID) == 0 { - c.consentID = strconv.Itoa(rand.Intn(899) + 100) //nolint:gosec - } - - req.AddCookie(&http.Cookie{ - Name: "CONSENT", - Value: "YES+cb.20210328-17-p0.en+FX+" + c.consentID, - Path: "/", - Domain: ".youtube.com", - }) - - res, err := client.Do(req) - - log := slog.With("method", req.Method, "url", req.URL) - - if err == nil && res.StatusCode != http.StatusOK { - err = ErrUnexpectedStatusCode(res.StatusCode) - res.Body.Close() - res = nil - } - - if err != nil { - log.Debug("HTTP request failed", "error", err) - } else { - log.Debug("HTTP request succeeded", "status", res.Status) - } - - return res, err -} - -// httpGet does a HTTP GET request, checks the response to be a 200 OK and returns it -func (c *Client) httpGet(ctx context.Context, url string) (*http.Response, error) { - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) - if err != nil { - return nil, err - } - - resp, err := c.httpDo(req) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, ErrUnexpectedStatusCode(resp.StatusCode) - } - - return resp, nil -} - -// httpGetBodyBytes reads the whole HTTP body and returns it -func (c *Client) httpGetBodyBytes(ctx context.Context, url string) ([]byte, error) { - resp, err := c.httpGet(ctx, url) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - return io.ReadAll(resp.Body) -} - -// httpPost does a HTTP POST request with a body, checks the response to be a 200 OK and returns it -func (c *Client) httpPost(ctx context.Context, url string, body interface{}) (*http.Response, error) { - data, err := json.Marshal(body) - if err != nil { - return nil, err - } - - req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) - if err != nil { - return nil, err - } - - req.Header.Set("X-Youtube-Client-Name", "3") - req.Header.Set("X-Youtube-Client-Version", c.client.version) - req.Header.Set("Content-Type", "application/json") - req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") - - resp, err := c.httpDo(req) - if err != nil { - return nil, err - } - - if resp.StatusCode != http.StatusOK { - resp.Body.Close() - return nil, ErrUnexpectedStatusCode(resp.StatusCode) - } - - return resp, nil -} - -// httpPostBodyBytes reads the whole HTTP body and returns it -func (c *Client) httpPostBodyBytes(ctx context.Context, url string, body interface{}) ([]byte, error) { - resp, err := c.httpPost(ctx, url, body) - if err != nil { - return nil, err - } - defer resp.Body.Close() - - return io.ReadAll(resp.Body) -} - -// downloadChunk writes the response data into the data channel of the chunk. -// Downloading in multiple chunks is much faster: -// https://github.com/kkdai/youtube/pull/190 -func (c *Client) downloadChunk(req *http.Request, chunk *chunk) error { - q := req.URL.Query() - q.Set("range", fmt.Sprintf("%d-%d", chunk.start, chunk.end)) - req.URL.RawQuery = q.Encode() - - resp, err := c.httpDo(req) - if err != nil { - return err - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusOK { - return ErrUnexpectedStatusCode(resp.StatusCode) - } - - expected := int(chunk.end-chunk.start) + 1 - data, err := io.ReadAll(resp.Body) - n := len(data) - - if err != nil { - return err - } - - if n != expected { - return fmt.Errorf("chunk at offset %d has invalid size: expected=%d actual=%d", chunk.start, expected, n) - } - - chunk.data <- data - - return nil -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/decipher.go b/downloader/vendor/github.com/kkdai/youtube/v2/decipher.go deleted file mode 100644 index 3dd4c1c..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/decipher.go +++ /dev/null @@ -1,285 +0,0 @@ -package youtube - -import ( - "bytes" - "context" - "errors" - "fmt" - "net/url" - "regexp" - "strconv" - - "github.com/dop251/goja" -) - -func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) { - params, err := url.ParseQuery(cipher) - if err != nil { - return "", err - } - - uri, err := url.Parse(params.Get("url")) - if err != nil { - return "", err - } - query := uri.Query() - - config, err := c.getPlayerConfig(ctx, videoID) - if err != nil { - return "", err - } - - // decrypt s-parameter - bs, err := config.decrypt([]byte(params.Get("s"))) - if err != nil { - return "", err - } - query.Add(params.Get("sp"), string(bs)) - - query, err = c.decryptNParam(config, query) - if err != nil { - return "", err - } - - uri.RawQuery = query.Encode() - - return uri.String(), nil -} - -// see https://github.com/kkdai/youtube/pull/244 -func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) { - config, err := c.getPlayerConfig(ctx, videoID) - if err != nil { - return "", err - } - - uri, err := url.Parse(urlString) - if err != nil { - return "", err - } - - // for debugging - if artifactsFolder != "" { - writeArtifact("video-"+videoID+".url", []byte(uri.String())) - } - - query, err := c.decryptNParam(config, uri.Query()) - if err != nil { - return "", err - } - - uri.RawQuery = query.Encode() - return uri.String(), nil -} - -func (c *Client) decryptNParam(config playerConfig, query url.Values) (url.Values, error) { - // decrypt n-parameter - nSig := query.Get("v") - log := Logger.With("n", nSig) - - if nSig != "" { - nDecoded, err := config.decodeNsig(nSig) - if err != nil { - return nil, fmt.Errorf("unable to decode nSig: %w", err) - } - query.Set("v", nDecoded) - log = log.With("decoded", nDecoded) - } - - log.Debug("nParam") - - return query, nil -} - -const ( - jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*" - reverseStr = ":function\\(a\\)\\{" + - "(?:return )?a\\.reverse\\(\\)" + - "\\}" - spliceStr = ":function\\(a,b\\)\\{" + - "a\\.splice\\(0,b\\)" + - "\\}" - swapStr = ":function\\(a,b\\)\\{" + - "var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" + - "\\}" -) - -var ( - nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})") - actionsObjRegexp = regexp.MustCompile(fmt.Sprintf( - "var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr)) - - actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf( - "function(?: %s)?\\(a\\)\\{"+ - "a=a\\.split\\(\"\"\\);\\s*"+ - "((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+ - "return a\\.join\\(\"\"\\)"+ - "\\}", jsvarStr, jsvarStr, jsvarStr)) - - reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr)) - spliceRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr)) - swapRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr)) -) - -func (config playerConfig) decodeNsig(encoded string) (string, error) { - fBody, err := config.getNFunction() - if err != nil { - return "", err - } - - return evalJavascript(fBody, encoded) -} - -func evalJavascript(jsFunction, arg string) (string, error) { - const myName = "myFunction" - - vm := goja.New() - _, err := vm.RunString(myName + "=" + jsFunction) - if err != nil { - return "", err - } - - var output func(string) string - err = vm.ExportTo(vm.Get(myName), &output) - if err != nil { - return "", err - } - - return output(arg), nil -} - -func (config playerConfig) getNFunction() (string, error) { - nameResult := nFunctionNameRegexp.FindSubmatch(config) - if len(nameResult) == 0 { - return "", errors.New("unable to extract n-function name") - } - - var name string - if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 { - name = string(nameResult[4]) - } else { - name = string(nameResult[1]) - } - - return config.extraFunction(name) - -} - -func (config playerConfig) extraFunction(name string) (string, error) { - // find the beginning of the function - def := []byte(name + "=function(") - start := bytes.Index(config, def) - if start < 1 { - return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def) - } - - // start after the first curly bracket - pos := start + bytes.IndexByte(config[start:], '{') + 1 - - var strChar byte - - // find the bracket closing the function - for brackets := 1; brackets > 0; pos++ { - b := config[pos] - switch b { - case '{': - if strChar == 0 { - brackets++ - } - case '}': - if strChar == 0 { - brackets-- - } - case '`', '"', '\'': - if config[pos-1] == '\\' && config[pos-2] != '\\' { - continue - } - if strChar == 0 { - strChar = b - } else if strChar == b { - strChar = 0 - } - } - } - - return string(config[start:pos]), nil -} - -func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) { - operations, err := config.parseDecipherOps() - if err != nil { - return nil, err - } - - // apply operations - bs := []byte(cyphertext) - for _, op := range operations { - bs = op(bs) - } - - return bs, nil -} - -/* -parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js - -var Mt={ -splice:function(a,b){a.splice(0,b)}, -reverse:function(a){a.reverse()}, -EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}}; - -a=a.split(""); -Mt.splice(a,3); -Mt.EQ(a,39); -Mt.splice(a,2); -Mt.EQ(a,1); -Mt.splice(a,1); -Mt.EQ(a,35); -Mt.EQ(a,51); -Mt.splice(a,2); -Mt.reverse(a,52); -return a.join("") -*/ -func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) { - objResult := actionsObjRegexp.FindSubmatch(config) - funcResult := actionsFuncRegexp.FindSubmatch(config) - if len(objResult) < 3 || len(funcResult) < 2 { - return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult)) - } - - obj := objResult[1] - objBody := objResult[2] - funcBody := funcResult[1] - - var reverseKey, spliceKey, swapKey string - - if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 { - reverseKey = string(result[1]) - } - if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 { - spliceKey = string(result[1]) - } - if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 { - swapKey = string(result[1]) - } - - regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey))) - if err != nil { - return nil, err - } - - var ops []DecipherOperation - for _, s := range regex.FindAllSubmatch(funcBody, -1) { - switch string(s[1]) { - case reverseKey: - ops = append(ops, reverseFunc) - case swapKey: - arg, _ := strconv.Atoi(string(s[2])) - ops = append(ops, newSwapFunc(arg)) - case spliceKey: - arg, _ := strconv.Atoi(string(s[2])) - ops = append(ops, newSpliceFunc(arg)) - } - } - return ops, nil -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go b/downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go deleted file mode 100644 index 7abb5d1..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go +++ /dev/null @@ -1,27 +0,0 @@ -package youtube - -type DecipherOperation func([]byte) []byte - -func newSpliceFunc(pos int) DecipherOperation { - return func(bs []byte) []byte { - return bs[pos:] - } -} - -func newSwapFunc(arg int) DecipherOperation { - return func(bs []byte) []byte { - pos := arg % len(bs) - bs[0], bs[pos] = bs[pos], bs[0] - return bs - } -} - -func reverseFunc(bs []byte) []byte { - l, r := 0, len(bs)-1 - for l < r { - bs[l], bs[r] = bs[r], bs[l] - l++ - r-- - } - return bs -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/doc.go b/downloader/vendor/github.com/kkdai/youtube/v2/doc.go deleted file mode 100644 index 821dc69..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -/* -Package youtube implement youtube download package in go. -*/ -package youtube diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go deleted file mode 100644 index 04dc834..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go +++ /dev/null @@ -1,203 +0,0 @@ -package downloader - -import ( - "context" - "errors" - "io" - "log/slog" - "os" - "os/exec" - "path/filepath" - - "github.com/kkdai/youtube/v2" - "github.com/vbauerster/mpb/v5" - "github.com/vbauerster/mpb/v5/decor" -) - -// Downloader offers high level functions to download videos into files -type Downloader struct { - youtube.Client - OutputDir string // optional directory to store the files -} - -func (dl *Downloader) getOutputFile(v *youtube.Video, format *youtube.Format, outputFile string) (string, error) { - if outputFile == "" { - outputFile = SanitizeFilename(v.Title) - outputFile += pickIdealFileExtension(format.MimeType) - } - - if dl.OutputDir != "" { - if err := os.MkdirAll(dl.OutputDir, 0o755); err != nil { - return "", err - } - outputFile = filepath.Join(dl.OutputDir, outputFile) - } - - return outputFile, nil -} - -// Download : Starting download video by arguments. -func (dl *Downloader) Download(ctx context.Context, v *youtube.Video, format *youtube.Format, outputFile string) error { - youtube.Logger.Info( - "Downloading video", - "id", v.ID, - "quality", format.Quality, - "mimeType", format.MimeType, - ) - destFile, err := dl.getOutputFile(v, format, outputFile) - if err != nil { - return err - } - - // Create output file - out, err := os.Create(destFile) - if err != nil { - return err - } - defer out.Close() - - return dl.videoDLWorker(ctx, out, v, format) -} - -// DownloadComposite : Downloads audio and video streams separately and merges them via ffmpeg. -func (dl *Downloader) DownloadComposite(ctx context.Context, outputFile string, v *youtube.Video, quality string, mimetype, language string) error { - videoFormat, audioFormat, err1 := getVideoAudioFormats(v, quality, mimetype, language) - if err1 != nil { - return err1 - } - - log := youtube.Logger.With("id", v.ID) - - log.Info( - "Downloading composite video", - "videoQuality", videoFormat.QualityLabel, - "videoMimeType", videoFormat.MimeType, - "audioMimeType", audioFormat.MimeType, - ) - - destFile, err := dl.getOutputFile(v, videoFormat, outputFile) - if err != nil { - return err - } - outputDir := filepath.Dir(destFile) - - // Create temporary video file - videoFile, err := os.CreateTemp(outputDir, "youtube_*.m4v") - if err != nil { - return err - } - defer Delete(videoFile.Name(), log) - - // Create temporary audio file - audioFile, err := os.CreateTemp(outputDir, "youtube_*.m4a") - if err != nil { - return err - } - defer Delete(audioFile.Name(), log) - - log.Debug("Downloading video file...") - err = dl.videoDLWorker(ctx, videoFile, v, videoFormat) - if err != nil { - return err - } - videoFile.Close() - - log.Debug("Downloading audio file...") - err = dl.videoDLWorker(ctx, audioFile, v, audioFormat) - if err != nil { - return err - } - audioFile.Close() - - //nolint:gosec - ffmpegVersionCmd := exec.Command("ffmpeg", "-y", - "-i", videoFile.Name(), - "-i", audioFile.Name(), - "-c", "copy", // Just copy without re-encoding - "-shortest", // Finish encoding when the shortest input stream ends - destFile, - "-loglevel", "warning", - ) - ffmpegVersionCmd.Stderr = os.Stderr - ffmpegVersionCmd.Stdout = os.Stdout - log.Info("merging video and audio", "output", destFile) - - return ffmpegVersionCmd.Run() -} - -func Delete(path string, log *slog.Logger) { - err := os.Remove(path) - if err != nil { - log.Error("Failed deleting file", "path", path, "error", err) - } -} - -func getVideoAudioFormats(v *youtube.Video, quality string, mimetype, language string) (*youtube.Format, *youtube.Format, error) { - var videoFormats, audioFormats youtube.FormatList - - formats := v.Formats - if mimetype != "" { - formats = formats.Type(mimetype) - } - - videoFormats = formats.Type("video").AudioChannels(0) - audioFormats = formats.Type("audio") - - if quality != "" { - videoFormats = videoFormats.Quality(quality) - } - - if language != "" { - audioFormats = audioFormats.Language(language) - } - - if len(videoFormats) == 0 { - return nil, nil, errors.New("no video format found after filtering") - } - - if len(audioFormats) == 0 { - return nil, nil, errors.New("no audio format found after filtering") - } - - videoFormats.Sort() - audioFormats.Sort() - - return &videoFormats[0], &audioFormats[0], nil -} - -func (dl *Downloader) videoDLWorker(ctx context.Context, out *os.File, video *youtube.Video, format *youtube.Format) error { - stream, size, err := dl.GetStreamContext(ctx, video, format) - if err != nil { - return err - } - - prog := &progress{ - contentLength: float64(size), - } - - // create progress bar - progress := mpb.New(mpb.WithWidth(64)) - bar := progress.AddBar( - int64(prog.contentLength), - - mpb.PrependDecorators( - decor.CountersKibiByte("% .2f / % .2f"), - decor.Percentage(decor.WCSyncSpace), - ), - mpb.AppendDecorators( - decor.EwmaETA(decor.ET_STYLE_GO, 90), - decor.Name(" ] "), - decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60), - ), - ) - - reader := bar.ProxyReader(stream) - mw := io.MultiWriter(out, prog) - _, err = io.Copy(mw, reader) - if err != nil { - return err - } - - progress.Wait() - return nil -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go deleted file mode 100644 index e65ee27..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go +++ /dev/null @@ -1,65 +0,0 @@ -package downloader - -import ( - "mime" - "regexp" -) - -const defaultExtension = ".mov" - -// Rely on hardcoded canonical mime types, as the ones provided by Go aren't exhaustive [1]. -// This seems to be a recurring problem for youtube downloaders, see [2]. -// The implementation is based on mozilla's list [3], IANA [4] and Youtube's support [5]. -// [1] https://github.com/golang/go/blob/ed7888aea6021e25b0ea58bcad3f26da2b139432/src/mime/type.go#L60 -// [2] https://github.com/ZiTAL/youtube-dl/blob/master/mime.types -// [3] https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types -// [4] https://www.iana.org/assignments/media-types/media-types.xhtml#video -// [5] https://support.google.com/youtube/troubleshooter/2888402?hl=en -var canonicals = map[string]string{ - "video/quicktime": ".mov", - "video/x-msvideo": ".avi", - "video/x-matroska": ".mkv", - "video/mpeg": ".mpeg", - "video/webm": ".webm", - "video/3gpp2": ".3g2", - "video/x-flv": ".flv", - "video/3gpp": ".3gp", - "video/mp4": ".mp4", - "video/ogg": ".ogv", - "video/mp2t": ".ts", -} - -func pickIdealFileExtension(mediaType string) string { - mediaType, _, err := mime.ParseMediaType(mediaType) - if err != nil { - return defaultExtension - } - - if extension, ok := canonicals[mediaType]; ok { - return extension - } - - // Our last resort is to ask the operating system, but these give multiple results and are rarely canonical. - extensions, err := mime.ExtensionsByType(mediaType) - if err != nil || extensions == nil { - return defaultExtension - } - - return extensions[0] -} - -func SanitizeFilename(fileName string) string { - // Characters not allowed on mac - // :/ - // Characters not allowed on linux - // / - // Characters not allowed on windows - // <>:"/\|?* - - // Ref https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions - - fileName = regexp.MustCompile(`[:/<>\:"\\|?*]`).ReplaceAllString(fileName, "") - fileName = regexp.MustCompile(`\s+`).ReplaceAllString(fileName, " ") - - return fileName -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go deleted file mode 100644 index 6d88773..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go +++ /dev/null @@ -1,17 +0,0 @@ -package downloader - -type progress struct { - contentLength float64 - totalWrittenBytes float64 - downloadLevel float64 -} - -func (dl *progress) Write(p []byte) (n int, err error) { - n = len(p) - dl.totalWrittenBytes = dl.totalWrittenBytes + float64(n) - currentPercent := (dl.totalWrittenBytes / dl.contentLength) * 100 - if (dl.downloadLevel <= currentPercent) && (dl.downloadLevel < 100) { - dl.downloadLevel++ - } - return -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/errors.go b/downloader/vendor/github.com/kkdai/youtube/v2/errors.go deleted file mode 100644 index 526d64d..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/errors.go +++ /dev/null @@ -1,47 +0,0 @@ -package youtube - -import ( - "fmt" -) - -const ( - ErrCipherNotFound = constError("cipher not found") - ErrSignatureTimestampNotFound = constError("signature timestamp not found") - ErrInvalidCharactersInVideoID = constError("invalid characters in video id") - ErrVideoIDMinLength = constError("the video id must be at least 10 characters long") - ErrReadOnClosedResBody = constError("http: read on closed response body") - ErrNotPlayableInEmbed = constError("embedding of this video has been disabled") - ErrLoginRequired = constError("login required to confirm your age") - ErrVideoPrivate = constError("user restricted access to this video") - ErrInvalidPlaylist = constError("no playlist detected or invalid playlist ID") -) - -type constError string - -func (e constError) Error() string { - return string(e) -} - -type ErrPlayabiltyStatus struct { - Status string - Reason string -} - -func (err ErrPlayabiltyStatus) Error() string { - return fmt.Sprintf("cannot playback and download, status: %s, reason: %s", err.Status, err.Reason) -} - -// ErrUnexpectedStatusCode is returned on unexpected HTTP status codes -type ErrUnexpectedStatusCode int - -func (err ErrUnexpectedStatusCode) Error() string { - return fmt.Sprintf("unexpected status code: %d", err) -} - -type ErrPlaylistStatus struct { - Reason string -} - -func (err ErrPlaylistStatus) Error() string { - return fmt.Sprintf("could not load playlist: %s", err.Reason) -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go b/downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go deleted file mode 100644 index 989970c..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go +++ /dev/null @@ -1,35 +0,0 @@ -//go:build fetch -// +build fetch - -package youtube - -import ( - "fmt" - "io" - "net/http" - "os" -) - -func init() { - FetchTestData() -} - -// ran via go generate to fetch and update the playlist response data -func FetchTestData() { - f, err := os.Create(testPlaylistResponseDataFile) - exitOnError(err) - requestURL := fmt.Sprintf(playlistFetchURL, testPlaylistID) - resp, err := http.Get(requestURL) - exitOnError(err) - defer resp.Body.Close() - n, err := io.Copy(f, resp.Body) - exitOnError(err) - fmt.Printf("Successfully fetched playlist %s (%d bytes)\n", testPlaylistID, n) -} - -func exitOnError(err error) { - if err != nil { - fmt.Println(err) - os.Exit(1) - } -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/format_list.go b/downloader/vendor/github.com/kkdai/youtube/v2/format_list.go deleted file mode 100644 index 57c98dc..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/format_list.go +++ /dev/null @@ -1,148 +0,0 @@ -package youtube - -import ( - "sort" - "strconv" - "strings" -) - -type FormatList []Format - -// Type returns a new FormatList filtered by itag -func (list FormatList) Select(f func(Format) bool) (result FormatList) { - for i := range list { - if f(list[i]) { - result = append(result, list[i]) - } - } - return result -} - -// Type returns a new FormatList filtered by itag -func (list FormatList) Itag(itagNo int) FormatList { - return list.Select(func(f Format) bool { - return f.ItagNo == itagNo - }) -} - -// Type returns a new FormatList filtered by mime type -func (list FormatList) Type(value string) FormatList { - return list.Select(func(f Format) bool { - return strings.Contains(f.MimeType, value) - }) -} - -// Type returns a new FormatList filtered by display name -func (list FormatList) Language(displayName string) FormatList { - return list.Select(func(f Format) bool { - return f.LanguageDisplayName() == displayName - }) -} - -// Quality returns a new FormatList filtered by quality, quality label or itag, -// but not audio quality -func (list FormatList) Quality(quality string) FormatList { - itag, _ := strconv.Atoi(quality) - - return list.Select(func(f Format) bool { - return itag == f.ItagNo || strings.Contains(f.Quality, quality) || strings.Contains(f.QualityLabel, quality) - }) -} - -// AudioChannels returns a new FormatList filtered by the matching AudioChannels -func (list FormatList) AudioChannels(n int) FormatList { - return list.Select(func(f Format) bool { - return f.AudioChannels == n - }) -} - -// AudioChannels returns a new FormatList filtered by the matching AudioChannels -func (list FormatList) WithAudioChannels() FormatList { - return list.Select(func(f Format) bool { - return f.AudioChannels > 0 - }) -} - -// FilterQuality reduces the format list to formats matching the quality -func (v *Video) FilterQuality(quality string) { - v.Formats = v.Formats.Quality(quality) - v.Formats.Sort() -} - -// Sort sorts all formats fields -func (list FormatList) Sort() { - sort.SliceStable(list, func(i, j int) bool { - return sortFormat(i, j, list) - }) -} - -// sortFormat sorts video by resolution, FPS, codec (av01, vp9, avc1), bitrate -// sorts audio by default, codec (mp4, opus), channels, bitrate, sample rate -func sortFormat(i int, j int, formats FormatList) bool { - - // Sort by Width - if formats[i].Width == formats[j].Width { - // Format 137 downloads slowly, give it less priority - // see https://github.com/kkdai/youtube/pull/171 - switch 137 { - case formats[i].ItagNo: - return false - case formats[j].ItagNo: - return true - } - - // Sort by FPS - if formats[i].FPS == formats[j].FPS { - if formats[i].FPS == 0 && formats[i].AudioChannels > 0 && formats[j].AudioChannels > 0 { - // Audio - // Sort by default - if (formats[i].AudioTrack == nil && formats[j].AudioTrack == nil) || (formats[i].AudioTrack != nil && formats[j].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault == formats[j].AudioTrack.AudioIsDefault) { - // Sort by codec - codec := map[int]int{} - for _, index := range []int{i, j} { - if strings.Contains(formats[index].MimeType, "mp4") { - codec[index] = 1 - } else if strings.Contains(formats[index].MimeType, "opus") { - codec[index] = 2 - } - } - if codec[i] == codec[j] { - // Sort by Audio Channel - if formats[i].AudioChannels == formats[j].AudioChannels { - // Sort by Audio Bitrate - if formats[i].Bitrate == formats[j].Bitrate { - // Sort by Audio Sample Rate - return formats[i].AudioSampleRate > formats[j].AudioSampleRate - } - return formats[i].Bitrate > formats[j].Bitrate - } - return formats[i].AudioChannels > formats[j].AudioChannels - } - return codec[i] < codec[j] - } else if formats[i].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault { - return true - } - return false - } - // Video - // Sort by codec - codec := map[int]int{} - for _, index := range []int{i, j} { - if strings.Contains(formats[index].MimeType, "av01") { - codec[index] = 1 - } else if strings.Contains(formats[index].MimeType, "vp9") { - codec[index] = 2 - } else if strings.Contains(formats[index].MimeType, "avc1") { - codec[index] = 3 - } - } - if codec[i] == codec[j] { - // Sort by Audio Bitrate - return formats[i].Bitrate > formats[j].Bitrate - } - return codec[i] < codec[j] - } - return formats[i].FPS > formats[j].FPS - } - return formats[i].Width > formats[j].Width -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/logger.go b/downloader/vendor/github.com/kkdai/youtube/v2/logger.go deleted file mode 100644 index 11a1541..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/logger.go +++ /dev/null @@ -1,28 +0,0 @@ -package youtube - -import ( - "fmt" - "log/slog" - "os" -) - -// The global logger for all Client instances -var Logger = getLogger(os.Getenv("LOGLEVEL")) - -func SetLogLevel(value string) { - Logger = getLogger(value) -} - -func getLogger(logLevel string) *slog.Logger { - levelVar := slog.LevelVar{} - - if logLevel != "" { - if err := levelVar.UnmarshalText([]byte(logLevel)); err != nil { - panic(fmt.Sprintf("Invalid log level %s: %v", logLevel, err)) - } - } - - return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ - Level: levelVar.Level(), - })) -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go b/downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go deleted file mode 100644 index b2b147f..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go +++ /dev/null @@ -1,37 +0,0 @@ -package youtube - -import ( - "time" -) - -const defaultCacheExpiration = time.Minute * time.Duration(5) - -type playerCache struct { - key string - expiredAt time.Time - config playerConfig -} - -// Get : get cache when it has same video id and not expired -func (s playerCache) Get(key string) playerConfig { - return s.GetCacheBefore(key, time.Now()) -} - -// GetCacheBefore : can pass time for testing -func (s playerCache) GetCacheBefore(key string, time time.Time) playerConfig { - if key == s.key && s.expiredAt.After(time) { - return s.config - } - return nil -} - -// Set : set cache with default expiration -func (s *playerCache) Set(key string, operations playerConfig) { - s.setWithExpiredTime(key, operations, time.Now().Add(defaultCacheExpiration)) -} - -func (s *playerCache) setWithExpiredTime(key string, config playerConfig, time time.Time) { - s.key = key - s.config = config - s.expiredAt = time -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go b/downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go deleted file mode 100644 index 7790eeb..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go +++ /dev/null @@ -1,59 +0,0 @@ -package youtube - -import ( - "context" - "errors" - "fmt" - "log" - "os" - "path/filepath" - "regexp" - "strings" -) - -type playerConfig []byte - -var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`) - -func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) { - embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID) - embedBody, err := c.httpGetBodyBytes(ctx, embedURL) - if err != nil { - return nil, err - } - - // example: /s/player/f676c671/player_ias.vflset/en_US/base.js - playerPath := string(basejsPattern.Find(embedBody)) - if playerPath == "" { - return nil, errors.New("unable to find basejs URL in playerConfig") - } - - // for debugging - var artifactName string - if artifactsFolder != "" { - parts := strings.SplitN(playerPath, "/", 5) - artifactName = "player-" + parts[3] + ".js" - linkName := filepath.Join(artifactsFolder, "video-"+videoID+".js") - if err := os.Symlink(artifactName, linkName); err != nil { - log.Printf("unable to create symlink %s: %v", linkName, err) - } - } - - config := c.playerCache.Get(playerPath) - if config != nil { - return config, nil - } - - config, err = c.httpGetBodyBytes(ctx, "https://youtube.com"+playerPath) - if err != nil { - return nil, err - } - - // for debugging - if artifactName != "" { - writeArtifact(artifactName, config) - } - - c.playerCache.Set(playerPath, config) - return config, nil -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/playlist.go b/downloader/vendor/github.com/kkdai/youtube/v2/playlist.go deleted file mode 100644 index 60462f5..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/playlist.go +++ /dev/null @@ -1,256 +0,0 @@ -package youtube - -import ( - "context" - "encoding/json" - "fmt" - "regexp" - "runtime/debug" - "strconv" - "time" - - sjson "github.com/bitly/go-simplejson" -) - -var ( - playlistIDRegex = regexp.MustCompile("^[A-Za-z0-9_-]{13,42}$") - playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{13,42})(&.*)?$") -) - -type Playlist struct { - ID string - Title string - Description string - Author string - Videos []*PlaylistEntry -} - -type PlaylistEntry struct { - ID string - Title string - Author string - Duration time.Duration - Thumbnails Thumbnails -} - -func extractPlaylistID(url string) (string, error) { - if playlistIDRegex.Match([]byte(url)) { - return url, nil - } - - matches := playlistInURLRegex.FindStringSubmatch(url) - - if matches != nil { - return matches[1], nil - } - - return "", ErrInvalidPlaylist -} - -// structs for playlist extraction - -// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text -// Description: metadata.playlistMetadataRenderer.description -// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text - -// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents -// ID: .videoId -// Title: title.runs[0].text -// Author: .shortBylineText.runs[0].text -// Duration: .lengthSeconds -// Thumbnails .thumbnails - -// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails -func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body []byte) (err error) { - var j *sjson.Json - j, err = sjson.NewJson(body) - if err != nil { - return err - } - - defer func() { - stack := debug.Stack() - if r := recover(); r != nil { - err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack) - } - }() - - renderer := j.GetPath("alerts").GetIndex(0).GetPath("alertRenderer") - if renderer != nil && renderer.GetPath("type").MustString() == "ERROR" { - message := renderer.GetPath("text", "runs").GetIndex(0).GetPath("text").MustString() - - return ErrPlaylistStatus{Reason: message} - } - - // Metadata can be located in multiple places depending on client type - var metadata *sjson.Json - if node, ok := j.CheckGet("metadata"); ok { - metadata = node - } else if node, ok := j.CheckGet("header"); ok { - metadata = node - } else { - return fmt.Errorf("no playlist header / metadata found") - } - - metadata = metadata.Get("playlistHeaderRenderer") - - p.Title = sjsonGetText(metadata, "title") - p.Description = sjsonGetText(metadata, "description", "descriptionText") - p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1). - GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs"). - GetIndex(0).Get("text").MustString() - - if len(p.Author) == 0 { - p.Author = sjsonGetText(metadata, "owner", "ownerText") - } - - contents, ok := j.CheckGet("contents") - if !ok { - return fmt.Errorf("contents not found in json body") - } - - // contents can have different keys with same child structure - firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0). - GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0) - - // This extra nested item is only set with the web client - if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) { - firstPart = n - } - - vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON() - if err != nil { - return err - } - - if len(vJSON) <= 4 { - return fmt.Errorf("no video data found in JSON") - } - - entries, continuation, err := extractPlaylistEntries(vJSON) - if err != nil { - return err - } - - if len(continuation) == 0 { - continuation = getContinuation(firstPart.Get("playlistVideoListRenderer")) - } - - if len(entries) == 0 { - return fmt.Errorf("no videos found in playlist") - } - - p.Videos = entries - - for continuation != "" { - data := prepareInnertubePlaylistData(continuation, true, *client.client) - - body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data) - if err != nil { - return err - } - - j, err := sjson.NewJson(body) - if err != nil { - return err - } - - next := j.GetPath("onResponseReceivedActions").GetIndex(0). - GetPath("appendContinuationItemsAction", "continuationItems") - - if !isValidJSON(next) { - next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents") - } - - vJSON, err := next.MarshalJSON() - if err != nil { - return err - } - - entries, token, err := extractPlaylistEntries(vJSON) - if err != nil { - return err - } - - if len(token) > 0 { - continuation = token - } else { - continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation")) - } - - p.Videos = append(p.Videos, entries...) - } - - return err -} - -func extractPlaylistEntries(data []byte) ([]*PlaylistEntry, string, error) { - var vids []*videosJSONExtractor - - if err := json.Unmarshal(data, &vids); err != nil { - return nil, "", err - } - - entries := make([]*PlaylistEntry, 0, len(vids)) - - var continuation string - for _, v := range vids { - if v.Renderer == nil { - if v.Continuation.Endpoint.Command.Token != "" { - continuation = v.Continuation.Endpoint.Command.Token - } - - continue - } - - entries = append(entries, v.PlaylistEntry()) - } - - return entries, continuation, nil -} - -type videosJSONExtractor struct { - Renderer *struct { - ID string `json:"videoId"` - Title withRuns `json:"title"` - Author withRuns `json:"shortBylineText"` - Duration string `json:"lengthSeconds"` - Thumbnail struct { - Thumbnails []Thumbnail `json:"thumbnails"` - } `json:"thumbnail"` - } `json:"playlistVideoRenderer"` - Continuation struct { - Endpoint struct { - Command struct { - Token string `json:"token"` - } `json:"continuationCommand"` - } `json:"continuationEndpoint"` - } `json:"continuationItemRenderer"` -} - -func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry { - ds, err := strconv.Atoi(vje.Renderer.Duration) - if err != nil { - panic("invalid video duration: " + vje.Renderer.Duration) - } - return &PlaylistEntry{ - ID: vje.Renderer.ID, - Title: vje.Renderer.Title.String(), - Author: vje.Renderer.Author.String(), - Duration: time.Second * time.Duration(ds), - Thumbnails: vje.Renderer.Thumbnail.Thumbnails, - } -} - -type withRuns struct { - Runs []struct { - Text string `json:"text"` - } `json:"runs"` -} - -func (wr withRuns) String() string { - if len(wr.Runs) > 0 { - return wr.Runs[0].Text - } - return "" -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/response_data.go b/downloader/vendor/github.com/kkdai/youtube/v2/response_data.go deleted file mode 100644 index 154a572..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/response_data.go +++ /dev/null @@ -1,153 +0,0 @@ -package youtube - -type playerResponseData struct { - Captions struct { - PlayerCaptionsTracklistRenderer struct { - CaptionTracks []CaptionTrack `json:"captionTracks"` - AudioTracks []struct { - CaptionTrackIndices []int `json:"captionTrackIndices"` - } `json:"audioTracks"` - TranslationLanguages []struct { - LanguageCode string `json:"languageCode"` - LanguageName struct { - SimpleText string `json:"simpleText"` - } `json:"languageName"` - } `json:"translationLanguages"` - DefaultAudioTrackIndex int `json:"defaultAudioTrackIndex"` - } `json:"playerCaptionsTracklistRenderer"` - } `json:"captions"` - - PlayabilityStatus struct { - Status string `json:"status"` - Reason string `json:"reason"` - PlayableInEmbed bool `json:"playableInEmbed"` - Miniplayer struct { - MiniplayerRenderer struct { - PlaybackMode string `json:"playbackMode"` - } `json:"miniplayerRenderer"` - } `json:"miniplayer"` - ContextParams string `json:"contextParams"` - } `json:"playabilityStatus"` - StreamingData struct { - ExpiresInSeconds string `json:"expiresInSeconds"` - Formats []Format `json:"formats"` - AdaptiveFormats []Format `json:"adaptiveFormats"` - DashManifestURL string `json:"dashManifestUrl"` - HlsManifestURL string `json:"hlsManifestUrl"` - } `json:"streamingData"` - VideoDetails struct { - VideoID string `json:"videoId"` - Title string `json:"title"` - LengthSeconds string `json:"lengthSeconds"` - Keywords []string `json:"keywords"` - ChannelID string `json:"channelId"` - IsOwnerViewing bool `json:"isOwnerViewing"` - ShortDescription string `json:"shortDescription"` - IsCrawlable bool `json:"isCrawlable"` - Thumbnail struct { - Thumbnails []Thumbnail `json:"thumbnails"` - } `json:"thumbnail"` - AverageRating float64 `json:"averageRating"` - AllowRatings bool `json:"allowRatings"` - ViewCount string `json:"viewCount"` - Author string `json:"author"` - IsPrivate bool `json:"isPrivate"` - IsUnpluggedCorpus bool `json:"isUnpluggedCorpus"` - IsLiveContent bool `json:"isLiveContent"` - } `json:"videoDetails"` - Microformat struct { - PlayerMicroformatRenderer struct { - Thumbnail struct { - Thumbnails []struct { - URL string `json:"url"` - Width int `json:"width"` - Height int `json:"height"` - } `json:"thumbnails"` - } `json:"thumbnail"` - Title struct { - SimpleText string `json:"simpleText"` - } `json:"title"` - Description struct { - SimpleText string `json:"simpleText"` - } `json:"description"` - LengthSeconds string `json:"lengthSeconds"` - OwnerProfileURL string `json:"ownerProfileUrl"` - ExternalChannelID string `json:"externalChannelId"` - IsFamilySafe bool `json:"isFamilySafe"` - AvailableCountries []string `json:"availableCountries"` - IsUnlisted bool `json:"isUnlisted"` - HasYpcMetadata bool `json:"hasYpcMetadata"` - ViewCount string `json:"viewCount"` - Category string `json:"category"` - PublishDate string `json:"publishDate"` - OwnerChannelName string `json:"ownerChannelName"` - UploadDate string `json:"uploadDate"` - } `json:"playerMicroformatRenderer"` - } `json:"microformat"` -} - -type Format struct { - ItagNo int `json:"itag"` - URL string `json:"url"` - MimeType string `json:"mimeType"` - Quality string `json:"quality"` - Cipher string `json:"signatureCipher"` - Bitrate int `json:"bitrate"` - FPS int `json:"fps"` - Width int `json:"width"` - Height int `json:"height"` - LastModified string `json:"lastModified"` - ContentLength int64 `json:"contentLength,string"` - QualityLabel string `json:"qualityLabel"` - ProjectionType string `json:"projectionType"` - AverageBitrate int `json:"averageBitrate"` - AudioQuality string `json:"audioQuality"` - ApproxDurationMs string `json:"approxDurationMs"` - AudioSampleRate string `json:"audioSampleRate"` - AudioChannels int `json:"audioChannels"` - - // InitRange is only available for adaptive formats - InitRange *struct { - Start string `json:"start"` - End string `json:"end"` - } `json:"initRange"` - - // IndexRange is only available for adaptive formats - IndexRange *struct { - Start string `json:"start"` - End string `json:"end"` - } `json:"indexRange"` - - // AudioTrack is only available for videos with multiple audio track languages - AudioTrack *struct { - DisplayName string `json:"displayName"` - ID string `json:"id"` - AudioIsDefault bool `json:"audioIsDefault"` - } -} - -func (f *Format) LanguageDisplayName() string { - if f.AudioTrack == nil { - return "" - } - return f.AudioTrack.DisplayName -} - -type Thumbnails []Thumbnail - -type Thumbnail struct { - URL string - Width uint - Height uint -} - -type CaptionTrack struct { - BaseURL string `json:"baseUrl"` - Name struct { - SimpleText string `json:"simpleText"` - } `json:"name"` - VssID string `json:"vssId"` - LanguageCode string `json:"languageCode"` - Kind string `json:"kind,omitempty"` - IsTranslatable bool `json:"isTranslatable"` -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/transcript.go b/downloader/vendor/github.com/kkdai/youtube/v2/transcript.go deleted file mode 100644 index 18724a5..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/transcript.go +++ /dev/null @@ -1,214 +0,0 @@ -package youtube - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "strconv" - "strings" -) - -var ( - ErrTranscriptDisabled = errors.New("transcript is disabled on this video") -) - -// TranscriptSegment is a single transcipt segment spanning a few milliseconds. -type TranscriptSegment struct { - // Text is the transcipt text. - Text string `json:"text"` - - // StartMs is the start timestamp in ms. - StartMs int `json:"offset"` - - // OffsetText e.g. '4:00'. - OffsetText string `json:"offsetText"` - - // Duration the transcript segment spans in ms. - Duration int `json:"duration"` -} - -func (tr TranscriptSegment) String() string { - return tr.OffsetText + " - " + strings.TrimSpace(tr.Text) -} - -type VideoTranscript []TranscriptSegment - -func (vt VideoTranscript) String() string { - var str string - for _, tr := range vt { - str += tr.String() + "\n" - } - - return str -} - -// GetTranscript fetches the video transcript if available. -// -// Not all videos have transcripts, only relatively new videos. -// If transcripts are disabled or not available, ErrTranscriptDisabled is returned. -func (c *Client) GetTranscript(video *Video, lang string) (VideoTranscript, error) { - return c.GetTranscriptCtx(context.Background(), video, lang) -} - -// GetTranscriptCtx fetches the video transcript if available. -// -// Not all videos have transcripts, only relatively new videos. -// If transcripts are disabled or not available, ErrTranscriptDisabled is returned. -func (c *Client) GetTranscriptCtx(ctx context.Context, video *Video, lang string) (VideoTranscript, error) { - c.assureClient() - - if video == nil || video.ID == "" { - return nil, fmt.Errorf("no video provided") - } - - body, err := c.transcriptDataByInnertube(ctx, video.ID, lang) - if err != nil { - return nil, err - } - - transcript, err := parseTranscript(body) - if err != nil { - return nil, err - } - - return transcript, nil -} - -func parseTranscript(body []byte) (VideoTranscript, error) { - var resp transcriptResp - if err := json.Unmarshal(body, &resp); err != nil { - return nil, err - } - - if len(resp.Actions) > 0 { - // Android client response - if app := resp.Actions[0].AppSegment; app != nil { - return getSegments(app) - } - - // Web client response - if web := resp.Actions[0].WebSegment; web != nil { - return nil, fmt.Errorf("not implemented") - } - } - - return nil, ErrTranscriptDisabled -} - -type segmenter interface { - ParseSegments() []TranscriptSegment -} - -func getSegments(f segmenter) (VideoTranscript, error) { - if segments := f.ParseSegments(); len(segments) > 0 { - return segments, nil - } - - return nil, ErrTranscriptDisabled -} - -// transcriptResp is the JSON structure as returned by the transcript API. -type transcriptResp struct { - Actions []struct { - AppSegment *appData `json:"elementsCommand"` - WebSegment *webData `json:"updateEngagementPanelAction"` - } `json:"actions"` -} - -type appData struct { - TEC struct { - Args struct { - ListArgs struct { - Ow struct { - InitialSeg []struct { - TranscriptSegment struct { - StartMs string `json:"startMs"` - EndMs string `json:"endMs"` - Text struct { - String struct { - // Content is the actual transctipt text - Content string `json:"content"` - } `json:"elementsAttributedString"` - } `json:"snippet"` - StartTimeText struct { - String struct { - // Content is the fomratted timestamp, e.g. '4:00' - Content string `json:"content"` - } `json:"elementsAttributedString"` - } `json:"startTimeText"` - } `json:"transcriptSegmentRenderer"` - } `json:"initialSegments"` - } `json:"overwrite"` - } `json:"transformTranscriptSegmentListArguments"` - } `json:"arguments"` - } `json:"transformEntityCommand"` -} - -func (s *appData) ParseSegments() []TranscriptSegment { - rawSegments := s.TEC.Args.ListArgs.Ow.InitialSeg - segments := make([]TranscriptSegment, 0, len(rawSegments)) - - for _, segment := range rawSegments { - startMs, _ := strconv.Atoi(segment.TranscriptSegment.StartMs) - endMs, _ := strconv.Atoi(segment.TranscriptSegment.EndMs) - - segments = append(segments, TranscriptSegment{ - Text: segment.TranscriptSegment.Text.String.Content, - StartMs: startMs, - OffsetText: segment.TranscriptSegment.StartTimeText.String.Content, - Duration: endMs - startMs, - }) - } - - return segments -} - -type webData struct { - Content struct { - TR struct { - Body struct { - TBR struct { - Cues []struct { - Transcript struct { - FormattedStartOffset struct { - SimpleText string `json:"simpleText"` - } `json:"formattedStartOffset"` - Cues []struct { - TranscriptCueRenderer struct { - Cue struct { - SimpleText string `json:"simpleText"` - } `json:"cue"` - StartOffsetMs string `json:"startOffsetMs"` - DurationMs string `json:"durationMs"` - } `json:"transcriptCueRenderer"` - } `json:"cues"` - } `json:"transcriptCueGroupRenderer"` - } `json:"cueGroups"` - } `json:"transcriptSearchPanelRenderer"` - } `json:"content"` - } `json:"transcriptRenderer"` - } `json:"content"` -} - -func (s *webData) ParseSegments() []TranscriptSegment { - // TODO: doesn't actually work now, check json. - cues := s.Content.TR.Body.TBR.Cues - segments := make([]TranscriptSegment, 0, len(cues)) - - for _, s := range cues { - formatted := s.Transcript.FormattedStartOffset.SimpleText - segment := s.Transcript.Cues[0].TranscriptCueRenderer - start, _ := strconv.Atoi(segment.StartOffsetMs) - duration, _ := strconv.Atoi(segment.DurationMs) - - segments = append(segments, TranscriptSegment{ - Text: segment.Cue.SimpleText, - StartMs: start, - OffsetText: formatted, - Duration: duration, - }) - } - - return segments -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/utils.go b/downloader/vendor/github.com/kkdai/youtube/v2/utils.go deleted file mode 100644 index 80c84b4..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/utils.go +++ /dev/null @@ -1,97 +0,0 @@ -package youtube - -import ( - "encoding/base64" - - sjson "github.com/bitly/go-simplejson" -) - -type chunk struct { - start int64 - end int64 - data chan []byte -} - -func getChunks(totalSize, chunkSize int64) []chunk { - var chunks []chunk - - for start := int64(0); start < totalSize; start += chunkSize { - end := chunkSize + start - 1 - if end > totalSize-1 { - end = totalSize - 1 - } - - chunks = append(chunks, chunk{start, end, make(chan []byte, 1)}) - } - - return chunks -} - -func getFirstKeyJSON(j *sjson.Json) *sjson.Json { - m, err := j.Map() - if err != nil { - return j - } - - for key := range m { - return j.Get(key) - } - - return j -} - -func isValidJSON(j *sjson.Json) bool { - b, err := j.MarshalJSON() - if err != nil { - return false - } - - if len(b) <= 4 { - return false - } - - return true -} - -func sjsonGetText(j *sjson.Json, paths ...string) string { - for _, path := range paths { - if isValidJSON(j.Get(path)) { - j = j.Get(path) - } - } - - if text, err := j.String(); err == nil { - return text - } - - if isValidJSON(j.Get("text")) { - return j.Get("text").MustString() - } - - if p := j.Get("runs"); isValidJSON(p) { - var text string - - for i := 0; i < len(p.MustArray()); i++ { - if textNode := p.GetIndex(i).Get("text"); isValidJSON(textNode) { - text += textNode.MustString() - } - } - - return text - } - - return "" -} - -func getContinuation(j *sjson.Json) string { - return j.GetPath("continuations"). - GetIndex(0).GetPath("nextContinuationData", "continuation").MustString() -} - -func base64PadEnc(str string) string { - return base64.StdEncoding.EncodeToString([]byte(str)) -} - -func base64Enc(str string) string { - return base64.RawStdEncoding.EncodeToString([]byte(str)) -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/video.go b/downloader/vendor/github.com/kkdai/youtube/v2/video.go deleted file mode 100644 index cbb1520..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/video.go +++ /dev/null @@ -1,147 +0,0 @@ -package youtube - -import ( - "encoding/json" - "errors" - "fmt" - "net/url" - "regexp" - "sort" - "strconv" - "strings" - "time" -) - -type Video struct { - ID string - Title string - Description string - Author string - ChannelID string - ChannelHandle string - Views int - Duration time.Duration - PublishDate time.Time - Formats FormatList - Thumbnails Thumbnails - DASHManifestURL string // URI of the DASH manifest file - HLSManifestURL string // URI of the HLS manifest file - CaptionTracks []CaptionTrack -} - -const dateFormat = "2006-01-02" - -func (v *Video) parseVideoInfo(body []byte) error { - var prData playerResponseData - if err := json.Unmarshal(body, &prData); err != nil { - return fmt.Errorf("unable to parse player response JSON: %w", err) - } - - if err := v.isVideoFromInfoDownloadable(prData); err != nil { - return err - } - - return v.extractDataFromPlayerResponse(prData) -} - -func (v *Video) isVideoFromInfoDownloadable(prData playerResponseData) error { - return v.isVideoDownloadable(prData, false) -} - -var playerResponsePattern = regexp.MustCompile(`var ytInitialPlayerResponse\s*=\s*(\{.+?\});`) - -func (v *Video) parseVideoPage(body []byte) error { - initialPlayerResponse := playerResponsePattern.FindSubmatch(body) - if initialPlayerResponse == nil || len(initialPlayerResponse) < 2 { - return errors.New("no ytInitialPlayerResponse found in the server's answer") - } - - var prData playerResponseData - if err := json.Unmarshal(initialPlayerResponse[1], &prData); err != nil { - return fmt.Errorf("unable to parse player response JSON: %w", err) - } - - if err := v.isVideoFromPageDownloadable(prData); err != nil { - return err - } - - return v.extractDataFromPlayerResponse(prData) -} - -func (v *Video) isVideoFromPageDownloadable(prData playerResponseData) error { - return v.isVideoDownloadable(prData, true) -} - -func (v *Video) isVideoDownloadable(prData playerResponseData, isVideoPage bool) error { - // Check if video is downloadable - switch prData.PlayabilityStatus.Status { - case "OK": - return nil - case "LOGIN_REQUIRED": - // for some reason they use same status message for age-restricted and private videos - if strings.HasPrefix(prData.PlayabilityStatus.Reason, "This video is private") { - return ErrVideoPrivate - } - return ErrLoginRequired - } - - if !isVideoPage && !prData.PlayabilityStatus.PlayableInEmbed { - return ErrNotPlayableInEmbed - } - - return &ErrPlayabiltyStatus{ - Status: prData.PlayabilityStatus.Status, - Reason: prData.PlayabilityStatus.Reason, - } -} - -func (v *Video) extractDataFromPlayerResponse(prData playerResponseData) error { - v.Title = prData.VideoDetails.Title - v.Description = prData.VideoDetails.ShortDescription - v.Author = prData.VideoDetails.Author - v.Thumbnails = prData.VideoDetails.Thumbnail.Thumbnails - v.ChannelID = prData.VideoDetails.ChannelID - v.CaptionTracks = prData.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks - - if views, _ := strconv.Atoi(prData.VideoDetails.ViewCount); views > 0 { - v.Views = views - } - - if seconds, _ := strconv.Atoi(prData.VideoDetails.LengthSeconds); seconds > 0 { - v.Duration = time.Duration(seconds) * time.Second - } - - if seconds, _ := strconv.Atoi(prData.Microformat.PlayerMicroformatRenderer.LengthSeconds); seconds > 0 { - v.Duration = time.Duration(seconds) * time.Second - } - - if str := prData.Microformat.PlayerMicroformatRenderer.PublishDate; str != "" { - v.PublishDate, _ = time.Parse(dateFormat, str) - } - - if profileURL, err := url.Parse(prData.Microformat.PlayerMicroformatRenderer.OwnerProfileURL); err == nil && len(profileURL.Path) > 1 { - v.ChannelHandle = profileURL.Path[1:] - } - - // Assign Streams - v.Formats = append(prData.StreamingData.Formats, prData.StreamingData.AdaptiveFormats...) - if len(v.Formats) == 0 { - return errors.New("no formats found in the server's answer") - } - - // Sort formats by bitrate - sort.SliceStable(v.Formats, v.SortBitrateDesc) - - v.HLSManifestURL = prData.StreamingData.HlsManifestURL - v.DASHManifestURL = prData.StreamingData.DashManifestURL - - return nil -} - -func (v *Video) SortBitrateDesc(i int, j int) bool { - return v.Formats[i].Bitrate > v.Formats[j].Bitrate -} - -func (v *Video) SortBitrateAsc(i int, j int) bool { - return v.Formats[i].Bitrate < v.Formats[j].Bitrate -} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/video_id.go b/downloader/vendor/github.com/kkdai/youtube/v2/video_id.go deleted file mode 100644 index 7de09cb..0000000 --- a/downloader/vendor/github.com/kkdai/youtube/v2/video_id.go +++ /dev/null @@ -1,34 +0,0 @@ -package youtube - -import ( - "regexp" - "strings" -) - -var videoRegexpList = []*regexp.Regexp{ - regexp.MustCompile(`(?:v|embed|shorts|watch\?v)(?:=|/)([^"&?/=%]{11})`), - regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`), - regexp.MustCompile(`([^"&?/=%]{11})`), -} - -// ExtractVideoID extracts the videoID from the given string -func ExtractVideoID(videoID string) (string, error) { - if strings.Contains(videoID, "youtu") || strings.ContainsAny(videoID, "\"?&/<%=") { - for _, re := range videoRegexpList { - if isMatch := re.MatchString(videoID); isMatch { - subs := re.FindStringSubmatch(videoID) - videoID = subs[1] - } - } - } - - if strings.ContainsAny(videoID, "?&/<%=") { - return "", ErrInvalidCharactersInVideoID - } - - if len(videoID) < 10 { - return "", ErrVideoIDMinLength - } - - return videoID, nil -}