diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/.gitignore b/downloader/vendor/github.com/kkdai/youtube/v2/.gitignore new file mode 100644 index 0000000..818294a --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/.gitignore @@ -0,0 +1,8 @@ +/.idea +/.vscode +download_test +/bin +/dist +/output +*.out +.DS_Store diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml b/downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml new file mode 100644 index 0000000..c0319a2 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/.golangci.yml @@ -0,0 +1,8 @@ +issues: + exclude: + - Subprocess launched with function call + # Excluding configuration per-path, per-linter, per-text and per-source + exclude-rules: + # Exclude some linters from running on tests files. + - path: cmd/ + text: InsecureSkipVerify diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml b/downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml new file mode 100644 index 0000000..f7d7c44 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/.goreleaser.yml @@ -0,0 +1,20 @@ +project_name: youtubedr + +env: + # Build without CGO to don't depend on specific glibc versions + - CGO_ENABLED=0 + +builds: + - main: ./cmd/youtubedr + binary: youtubedr + goos: + - windows + - darwin + - linux + - freebsd + goarch: + - amd64 + - arm + - arm64 + flags: + - -trimpath diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/LICENSE b/downloader/vendor/github.com/kkdai/youtube/v2/LICENSE new file mode 100644 index 0000000..d252f53 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Evan Lin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/Makefile b/downloader/vendor/github.com/kkdai/youtube/v2/Makefile new file mode 100644 index 0000000..02b6aed --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/Makefile @@ -0,0 +1,57 @@ +FILES_TO_FMT ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print) +LOGLEVEL ?= debug + +## help: Show makefile commands +.PHONY: help +help: Makefile + @echo "---- Project: kkdai/youtube ----" + @echo " Usage: make COMMAND" + @echo + @echo " Management Commands:" + @sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /' + @echo + +## build: Build project +.PHONY: build +build: + goreleaser --rm-dist + +## deps: Ensures fresh go.mod and go.sum +.PHONY: deps +deps: + go mod tidy + go mod verify + +## lint: Run golangci-lint check +.PHONY: lint +lint: + command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION) + echo "golangci-lint checking..." + golangci-lint run --deadline=30m --enable=misspell --enable=gosec --enable=gofmt --enable=goimports --enable=revive ./cmd/... ./... + go vet ./... + +## format: Formats Go code +.PHONY: format +format: + @echo ">> formatting code" + @gofmt -s -w $(FILES_TO_FMT) + +## test-unit: Run all Youtube Go unit tests +.PHONY: test-unit +test-unit: + LOGLEVEL=${LOGLEVEL} go test -v -cover ./... + +## test-integration: Run all Youtube Go integration tests +.PHONY: test-integration +test-integration: + mkdir -p output + rm -f output/* + LOGLEVEL=${LOGLEVEL} ARTIFACTS=output go test -v -race -covermode=atomic -coverprofile=coverage.out -tags=integration ./... + +.PHONY: coverage.out +coverage.out: + +## clean: Clean files and downloaded videos from builds during development +.PHONY: clean +clean: + rm -rf dist *.mp4 *.mkv diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/README.md b/downloader/vendor/github.com/kkdai/youtube/v2/README.md new file mode 100644 index 0000000..74de98b --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/README.md @@ -0,0 +1,161 @@ +Download Youtube Video in Golang +================== + +[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/kkdai/youtube/master/LICENSE) +[![Go Reference](https://pkg.go.dev/badge/github.com/kkdai/youtube.svg)](https://pkg.go.dev/github.com/kkdai/youtube/v2) +[![Build Status](https://github.com/kkdai/youtube/workflows/go/badge.svg?branch=master)](https://github.com/kkdai/youtube/actions) +[![Coverage](https://codecov.io/gh/kkdai/youtube/branch/master/graph/badge.svg)](https://codecov.io/gh/kkdai/youtube) +[![](https://goreportcard.com/badge/github.com/kkdai/youtube)](https://goreportcard.com/badge/github.com/kkdai/youtube) + + +This package is a Youtube video download package, for more detail refer [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) for more download options. + +This tool is meant to be used to download CC0 licenced content, we do not support nor recommend using it for illegal activities. + +## Overview + * [Install](#installation) + * [Usage](#usage) + * [Example: Download video from \[dotGo 2015 - Rob Pike - Simplicity is Complicated\]](#download-dotGo-2015-rob-pike-video) + +## Installation + +### Install via go get + +Please ensure you have installed Go 1.21 or later. + +```shell +go get github.com/kkdai/youtube/v2 +``` + +### From source code + +```shell +git clone https://github.com/kkdai/youtube.git +cd youtube +go run ./cmd/youtubedr +``` + +### Mac + +```shell +brew install youtubedr +``` + +### in Termux +```shell +pkg install youtubedr +``` +### You can also find this package in +- [archlinux](https://aur.archlinux.org/packages/youtubedr/) (thanks to [cjsthompson](https://github.com/cjsthompson)) +- [Termux package](https://github.com/termux/termux-packages/tree/master/packages/youtubedr) (thanks to [kcubeterm](https://github.com/kcubeterm)) +- [Homebrew](https://formulae.brew.sh/formula/youtubedr) (thanks to [kkc](https://github.com/kkc)) + +## Usage + +### Use the binary directly +It's really simple to use, just get the video id from youtube url - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM` + +```shell +$ youtubedr download QAGDGja7kbs +$ youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM +``` + + +### Use this package in your golang program + +Please check out the [example_test.go](example_test.go) for example code. + + +## Example: + * ### Get information of dotGo-2015-rob-pike video for downloading + + `go get github.com/kkdai/youtube/v2/youtubedr` + + Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) + + ``` + youtubedr info https://www.youtube.com/watch?v=rFejpH_tAHM + + Title: dotGo 2015 - Rob Pike - Simplicity is Complicated + Author: dotconferences + -----available streams----- + itag: 18 , quality: medium , type: video/mp4; codecs="avc1.42001E, mp4a.40.2" + itag: 22 , quality: hd720 , type: video/mp4; codecs="avc1.64001F, mp4a.40.2" + itag: 137 , quality: hd1080 , type: video/mp4; codecs="avc1.640028" + itag: 248 , quality: hd1080 , type: video/webm; codecs="vp9" + ........ + ``` + * ### Download dotGo-2015-rob-pike-video + + `go get github.com/kkdai/youtube/v2/youtubedr` + + Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) + + ``` + youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM + ``` + + * ### Download video to specific folder and name + + `go get github.com/kkdai/youtube/v2/youtubedr` + + Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) to current directory and name the file to simplicity-is-complicated.mp4 + + ``` + youtubedr download -d ./ -o simplicity-is-complicated.mp4 https://www.youtube.com/watch?v=rFejpH_tAHM + ``` + + * ### Download video with specific quality + + `go get github.com/kkdai/youtube/v2/youtubedr` + + Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) with specific quality + + ``` + youtubedr download -q medium https://www.youtube.com/watch?v=rFejpH_tAHM + ``` + + #### Special case by quality hd1080: + Installation of ffmpeg is necessary for hd1080 + ``` + ffmpeg //check ffmpeg is installed, if not please download ffmpeg and set to your PATH. + youtubedr download -q hd1080 https://www.youtube.com/watch?v=rFejpH_tAHM + ``` + + + * ### Download video with specific itag + + `go get github.com/kkdai/youtube/v2/youtubedr` + + Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) + + ``` + youtubedr download -q 18 https://www.youtube.com/watch?v=rFejpH_tAHM + ``` + +## How it works + +- Parse the video ID you input in URL + - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM` +- Get video information via video id. + - Use URL: `http://youtube.com/get_video_info?video_id=` +- Parse and decode video information. + - Download URL in "url=" + - title in "title=" +- Download video from URL + - Need the string combination of "url" + +## Inspired +- [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) +- [https://github.com/lepidosteus/youtube-dl](https://github.com/lepidosteus/youtube-dl) +- [拆解 Youtube 影片下載位置](http://hkgoldenmra.blogspot.tw/2013/05/youtube.html) +- [iawia002/annie](https://github.com/iawia002/annie) +- [How to get url from obfuscate video info: youtube video downloader with php](https://stackoverflow.com/questions/60607291/youtube-video-downloader-with-php) + + +## Project52 +It is one of my [project 52](https://github.com/kkdai/project52). + + +## License +This package is licensed under MIT license. See LICENSE for details. diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md b/downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md new file mode 100644 index 0000000..179112a --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/YoutubeChange.md @@ -0,0 +1,11 @@ +Tracking Youtube decipher change + +### 2022/01/21: + +#### action objects: + +var $z={fA:function(a){a.reverse()},S2:function(a,b){a.splice(0,b)},l6:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}}; +#### actions function: + +Npa=function(a){a=a.split("");$z.S2(a,3);$z.fA(a,45);$z.l6(a,31);$z.S2(a,1);$z.fA(a,63);$z.S2(a,2);$z.fA(a,68);return a.join("")}; + diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go b/downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go new file mode 100644 index 0000000..2f8b399 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/artifacts.go @@ -0,0 +1,29 @@ +package youtube + +import ( + "log/slog" + "os" + "path/filepath" +) + +// destination for artifacts, used by integration tests +var artifactsFolder = os.Getenv("ARTIFACTS") + +func writeArtifact(name string, content []byte) { + // Ensure folder exists + err := os.MkdirAll(artifactsFolder, os.ModePerm) + if err != nil { + slog.Error("unable to create artifacts folder", "path", artifactsFolder, "error", err) + return + } + + path := filepath.Join(artifactsFolder, name) + err = os.WriteFile(path, content, 0600) + + log := slog.With("path", path) + if err != nil { + log.Error("unable to write artifact", "error", err) + } else { + log.Debug("artifact created") + } +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/client.go b/downloader/vendor/github.com/kkdai/youtube/v2/client.go new file mode 100644 index 0000000..289401b --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/client.go @@ -0,0 +1,618 @@ +package youtube + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "strconv" + "sync/atomic" + + "log/slog" +) + +const ( + Size1Kb = 1024 + Size1Mb = Size1Kb * 1024 + Size10Mb = Size1Mb * 10 + + playerParams = "CgIQBg==" +) + +var ( + ErrNoFormat = errors.New("no video format provided") +) + +// DefaultClient type to use. No reason to change but you could if you wanted to. +var DefaultClient = AndroidClient + +// Client offers methods to download video metadata and video streams. +type Client struct { + // HTTPClient can be used to set a custom HTTP client. + // If not set, http.DefaultClient will be used + HTTPClient *http.Client + + // MaxRoutines to use when downloading a video. + MaxRoutines int + + // ChunkSize to use when downloading videos in chunks. Default is Size10Mb. + ChunkSize int64 + + // playerCache caches the JavaScript code of a player response + playerCache playerCache + + client *clientInfo + + consentID string +} + +func (c *Client) assureClient() { + if c.client == nil { + c.client = &DefaultClient + } +} + +// GetVideo fetches video metadata +func (c *Client) GetVideo(url string) (*Video, error) { + return c.GetVideoContext(context.Background(), url) +} + +// GetVideoContext fetches video metadata with a context +func (c *Client) GetVideoContext(ctx context.Context, url string) (*Video, error) { + id, err := ExtractVideoID(url) + if err != nil { + return nil, fmt.Errorf("extractVideoID failed: %w", err) + } + + return c.videoFromID(ctx, id) +} + +func (c *Client) videoFromID(ctx context.Context, id string) (*Video, error) { + c.assureClient() + + body, err := c.videoDataByInnertube(ctx, id) + if err != nil { + return nil, err + } + + v := Video{ + ID: id, + } + + // return early if all good + if err = v.parseVideoInfo(body); err == nil { + return &v, nil + } + + // If the uploader has disabled embedding the video on other sites, parse video page + if errors.Is(err, ErrNotPlayableInEmbed) { + // additional parameters are required to access clips with sensitiv content + html, err := c.httpGetBodyBytes(ctx, "https://www.youtube.com/watch?v="+id+"&bpctr=9999999999&has_verified=1") + if err != nil { + return nil, err + } + + return &v, v.parseVideoPage(html) + } + + // If the uploader marked the video as inappropriate for some ages, use embed player + if errors.Is(err, ErrLoginRequired) { + c.client = &EmbeddedClient + + bodyEmbed, errEmbed := c.videoDataByInnertube(ctx, id) + if errEmbed == nil { + errEmbed = v.parseVideoInfo(bodyEmbed) + } + + if errEmbed == nil { + return &v, nil + } + + // private video clearly not age-restricted and thus should be explicit + if errEmbed == ErrVideoPrivate { + return &v, errEmbed + } + + // wrapping error so its clear whats happened + return &v, fmt.Errorf("can't bypass age restriction: %w", errEmbed) + } + + // undefined error + return &v, err +} + +type innertubeRequest struct { + VideoID string `json:"videoId,omitempty"` + BrowseID string `json:"browseId,omitempty"` + Continuation string `json:"continuation,omitempty"` + Context inntertubeContext `json:"context"` + PlaybackContext *playbackContext `json:"playbackContext,omitempty"` + ContentCheckOK bool `json:"contentCheckOk,omitempty"` + RacyCheckOk bool `json:"racyCheckOk,omitempty"` + Params string `json:"params"` +} + +type playbackContext struct { + ContentPlaybackContext contentPlaybackContext `json:"contentPlaybackContext"` +} + +type contentPlaybackContext struct { + // SignatureTimestamp string `json:"signatureTimestamp"` + HTML5Preference string `json:"html5Preference"` +} + +type inntertubeContext struct { + Client innertubeClient `json:"client"` +} + +type innertubeClient struct { + HL string `json:"hl"` + GL string `json:"gl"` + ClientName string `json:"clientName"` + ClientVersion string `json:"clientVersion"` + AndroidSDKVersion int `json:"androidSDKVersion,omitempty"` + UserAgent string `json:"userAgent,omitempty"` + TimeZone string `json:"timeZone"` + UTCOffset int `json:"utcOffsetMinutes"` +} + +// client info for the innertube API +type clientInfo struct { + name string + key string + version string + userAgent string + androidVersion int +} + +var ( + // WebClient, better to use Android client but go ahead. + WebClient = clientInfo{ + name: "WEB", + version: "2.20220801.00.00", + key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + } + + // AndroidClient, download go brrrrrr. + AndroidClient = clientInfo{ + name: "ANDROID", + version: "18.11.34", + key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w", + userAgent: "com.google.android.youtube/18.11.34 (Linux; U; Android 11) gzip", + androidVersion: 30, + } + + // EmbeddedClient, not really tested. + EmbeddedClient = clientInfo{ + name: "WEB_EMBEDDED_PLAYER", + version: "1.19700101", + key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", // seems like same key works for both clients + userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36", + } +) + +func (c *Client) videoDataByInnertube(ctx context.Context, id string) ([]byte, error) { + data := innertubeRequest{ + VideoID: id, + Context: prepareInnertubeContext(*c.client), + ContentCheckOK: true, + RacyCheckOk: true, + Params: playerParams, + PlaybackContext: &playbackContext{ + ContentPlaybackContext: contentPlaybackContext{ + // SignatureTimestamp: sts, + HTML5Preference: "HTML5_PREF_WANTS", + }, + }, + } + + return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/player?key="+c.client.key, data) +} + +func (c *Client) transcriptDataByInnertube(ctx context.Context, id string, lang string) ([]byte, error) { + data := innertubeRequest{ + Context: prepareInnertubeContext(*c.client), + Params: transcriptVideoID(id, lang), + } + + return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/get_transcript?key="+c.client.key, data) +} + +func prepareInnertubeContext(clientInfo clientInfo) inntertubeContext { + return inntertubeContext{ + Client: innertubeClient{ + HL: "en", + GL: "US", + TimeZone: "UTC", + ClientName: clientInfo.name, + ClientVersion: clientInfo.version, + AndroidSDKVersion: clientInfo.androidVersion, + UserAgent: clientInfo.userAgent, + }, + } +} + +func prepareInnertubePlaylistData(ID string, continuation bool, clientInfo clientInfo) innertubeRequest { + context := prepareInnertubeContext(clientInfo) + + if continuation { + return innertubeRequest{ + Context: context, + Continuation: ID, + ContentCheckOK: true, + RacyCheckOk: true, + Params: playerParams, + } + } + + return innertubeRequest{ + Context: context, + BrowseID: "VL" + ID, + ContentCheckOK: true, + RacyCheckOk: true, + Params: playerParams, + } +} + +// transcriptVideoID encodes the video ID to the param used to fetch transcripts. +func transcriptVideoID(videoID string, lang string) string { + langCode := encTranscriptLang(lang) + + // This can be optionally appened to the Sprintf str, not sure what it means + // *3engagement-panel-searchable-transcript-search-panel\x30\x00\x38\x01\x40\x01 + return base64Enc(fmt.Sprintf("\n\x0b%s\x12\x12%s\x18\x01", videoID, langCode)) +} + +func encTranscriptLang(languageCode string) string { + s := fmt.Sprintf("\n\x03asr\x12\x02%s\x1a\x00", languageCode) + s = base64PadEnc(s) + + return url.QueryEscape(s) +} + +// GetPlaylist fetches playlist metadata +func (c *Client) GetPlaylist(url string) (*Playlist, error) { + return c.GetPlaylistContext(context.Background(), url) +} + +// GetPlaylistContext fetches playlist metadata, with a context, along with a list of Videos, and some basic information +// for these videos. Playlist entries cannot be downloaded, as they lack all the required metadata, but +// can be used to enumerate all IDs, Authors, Titles, etc. +func (c *Client) GetPlaylistContext(ctx context.Context, url string) (*Playlist, error) { + c.assureClient() + + id, err := extractPlaylistID(url) + if err != nil { + return nil, fmt.Errorf("extractPlaylistID failed: %w", err) + } + + data := prepareInnertubePlaylistData(id, false, *c.client) + body, err := c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+c.client.key, data) + if err != nil { + return nil, err + } + + p := &Playlist{ID: id} + return p, p.parsePlaylistInfo(ctx, c, body) +} + +func (c *Client) VideoFromPlaylistEntry(entry *PlaylistEntry) (*Video, error) { + return c.videoFromID(context.Background(), entry.ID) +} + +func (c *Client) VideoFromPlaylistEntryContext(ctx context.Context, entry *PlaylistEntry) (*Video, error) { + return c.videoFromID(ctx, entry.ID) +} + +// GetStream returns the stream and the total size for a specific format +func (c *Client) GetStream(video *Video, format *Format) (io.ReadCloser, int64, error) { + return c.GetStreamContext(context.Background(), video, format) +} + +// GetStreamContext returns the stream and the total size for a specific format with a context. +func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *Format) (io.ReadCloser, int64, error) { + url, err := c.GetStreamURL(video, format) + if err != nil { + return nil, 0, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, 0, err + } + + r, w := io.Pipe() + contentLength := format.ContentLength + + if contentLength == 0 { + // some videos don't have length information + contentLength = c.downloadOnce(req, w, format) + } else { + // we have length information, let's download by chunks! + c.downloadChunked(ctx, req, w, format) + } + + return r, contentLength, nil +} + +func (c *Client) downloadOnce(req *http.Request, w *io.PipeWriter, _ *Format) int64 { + resp, err := c.httpDo(req) + if err != nil { + w.CloseWithError(err) //nolint:errcheck + return 0 + } + + go func() { + defer resp.Body.Close() + _, err := io.Copy(w, resp.Body) + if err == nil { + w.Close() + } else { + w.CloseWithError(err) //nolint:errcheck + } + }() + + contentLength := resp.Header.Get("Content-Length") + length, _ := strconv.ParseInt(contentLength, 10, 64) + + return length +} + +func (c *Client) getChunkSize() int64 { + if c.ChunkSize > 0 { + return c.ChunkSize + } + + return Size10Mb +} + +func (c *Client) getMaxRoutines(limit int) int { + routines := 10 + + if c.MaxRoutines > 0 { + routines = c.MaxRoutines + } + + if limit > 0 && routines > limit { + routines = limit + } + + return routines +} + +func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.PipeWriter, format *Format) { + chunks := getChunks(format.ContentLength, c.getChunkSize()) + maxRoutines := c.getMaxRoutines(len(chunks)) + + cancelCtx, cancel := context.WithCancel(ctx) + abort := func(err error) { + w.CloseWithError(err) + cancel() + } + + currentChunk := atomic.Uint32{} + for i := 0; i < maxRoutines; i++ { + go func() { + for { + chunkIndex := int(currentChunk.Add(1)) - 1 + if chunkIndex >= len(chunks) { + // no more chunks + return + } + + chunk := &chunks[chunkIndex] + err := c.downloadChunk(req.Clone(cancelCtx), chunk) + close(chunk.data) + + if err != nil { + abort(err) + return + } + } + }() + } + + go func() { + // copy chunks into the PipeWriter + for i := 0; i < len(chunks); i++ { + select { + case <-cancelCtx.Done(): + abort(context.Canceled) + return + case data := <-chunks[i].data: + _, err := io.Copy(w, bytes.NewBuffer(data)) + + if err != nil { + abort(err) + } + } + } + + // everything succeeded + w.Close() + }() +} + +// GetStreamURL returns the url for a specific format +func (c *Client) GetStreamURL(video *Video, format *Format) (string, error) { + return c.GetStreamURLContext(context.Background(), video, format) +} + +// GetStreamURLContext returns the url for a specific format with a context +func (c *Client) GetStreamURLContext(ctx context.Context, video *Video, format *Format) (string, error) { + if format == nil { + return "", ErrNoFormat + } + + c.assureClient() + + if format.URL != "" { + if c.client.androidVersion > 0 { + return format.URL, nil + } + + return c.unThrottle(ctx, video.ID, format.URL) + } + + // TODO: check rest of this function, is it redundant? + + cipher := format.Cipher + if cipher == "" { + return "", ErrCipherNotFound + } + + uri, err := c.decipherURL(ctx, video.ID, cipher) + if err != nil { + return "", err + } + + return uri, err +} + +// httpDo sends an HTTP request and returns an HTTP response. +func (c *Client) httpDo(req *http.Request) (*http.Response, error) { + client := c.HTTPClient + if client == nil { + client = http.DefaultClient + } + + req.Header.Set("User-Agent", c.client.userAgent) + req.Header.Set("Origin", "https://youtube.com") + req.Header.Set("Sec-Fetch-Mode", "navigate") + + if len(c.consentID) == 0 { + c.consentID = strconv.Itoa(rand.Intn(899) + 100) //nolint:gosec + } + + req.AddCookie(&http.Cookie{ + Name: "CONSENT", + Value: "YES+cb.20210328-17-p0.en+FX+" + c.consentID, + Path: "/", + Domain: ".youtube.com", + }) + + res, err := client.Do(req) + + log := slog.With("method", req.Method, "url", req.URL) + + if err != nil { + log.Debug("HTTP request failed", "error", err) + } else { + log.Debug("HTTP request succeeded", "status", res.Status) + } + + return res, err +} + +// httpGet does a HTTP GET request, checks the response to be a 200 OK and returns it +func (c *Client) httpGet(ctx context.Context, url string) (*http.Response, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return nil, err + } + + resp, err := c.httpDo(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, ErrUnexpectedStatusCode(resp.StatusCode) + } + + return resp, nil +} + +// httpGetBodyBytes reads the whole HTTP body and returns it +func (c *Client) httpGetBodyBytes(ctx context.Context, url string) ([]byte, error) { + resp, err := c.httpGet(ctx, url) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return io.ReadAll(resp.Body) +} + +// httpPost does a HTTP POST request with a body, checks the response to be a 200 OK and returns it +func (c *Client) httpPost(ctx context.Context, url string, body interface{}) (*http.Response, error) { + data, err := json.Marshal(body) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data)) + if err != nil { + return nil, err + } + + req.Header.Set("X-Youtube-Client-Name", "3") + req.Header.Set("X-Youtube-Client-Version", c.client.version) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8") + + resp, err := c.httpDo(req) + if err != nil { + return nil, err + } + + if resp.StatusCode != http.StatusOK { + resp.Body.Close() + return nil, ErrUnexpectedStatusCode(resp.StatusCode) + } + + return resp, nil +} + +// httpPostBodyBytes reads the whole HTTP body and returns it +func (c *Client) httpPostBodyBytes(ctx context.Context, url string, body interface{}) ([]byte, error) { + resp, err := c.httpPost(ctx, url, body) + if err != nil { + return nil, err + } + defer resp.Body.Close() + + return io.ReadAll(resp.Body) +} + +// downloadChunk writes the response data into the data channel of the chunk. +// Downloading in multiple chunks is much faster: +// https://github.com/kkdai/youtube/pull/190 +func (c *Client) downloadChunk(req *http.Request, chunk *chunk) error { + q := req.URL.Query() + q.Set("range", fmt.Sprintf("%d-%d", chunk.start, chunk.end)) + req.URL.RawQuery = q.Encode() + + resp, err := c.httpDo(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode < http.StatusOK && resp.StatusCode >= 300 { + return ErrUnexpectedStatusCode(resp.StatusCode) + } + + expected := int(chunk.end-chunk.start) + 1 + data, err := io.ReadAll(resp.Body) + n := len(data) + + if err != nil { + return err + } + + if n != expected { + return fmt.Errorf("chunk at offset %d has invalid size: expected=%d actual=%d", chunk.start, expected, n) + } + + chunk.data <- data + + return nil +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/decipher.go b/downloader/vendor/github.com/kkdai/youtube/v2/decipher.go new file mode 100644 index 0000000..3dd4c1c --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/decipher.go @@ -0,0 +1,285 @@ +package youtube + +import ( + "bytes" + "context" + "errors" + "fmt" + "net/url" + "regexp" + "strconv" + + "github.com/dop251/goja" +) + +func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) { + params, err := url.ParseQuery(cipher) + if err != nil { + return "", err + } + + uri, err := url.Parse(params.Get("url")) + if err != nil { + return "", err + } + query := uri.Query() + + config, err := c.getPlayerConfig(ctx, videoID) + if err != nil { + return "", err + } + + // decrypt s-parameter + bs, err := config.decrypt([]byte(params.Get("s"))) + if err != nil { + return "", err + } + query.Add(params.Get("sp"), string(bs)) + + query, err = c.decryptNParam(config, query) + if err != nil { + return "", err + } + + uri.RawQuery = query.Encode() + + return uri.String(), nil +} + +// see https://github.com/kkdai/youtube/pull/244 +func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) { + config, err := c.getPlayerConfig(ctx, videoID) + if err != nil { + return "", err + } + + uri, err := url.Parse(urlString) + if err != nil { + return "", err + } + + // for debugging + if artifactsFolder != "" { + writeArtifact("video-"+videoID+".url", []byte(uri.String())) + } + + query, err := c.decryptNParam(config, uri.Query()) + if err != nil { + return "", err + } + + uri.RawQuery = query.Encode() + return uri.String(), nil +} + +func (c *Client) decryptNParam(config playerConfig, query url.Values) (url.Values, error) { + // decrypt n-parameter + nSig := query.Get("v") + log := Logger.With("n", nSig) + + if nSig != "" { + nDecoded, err := config.decodeNsig(nSig) + if err != nil { + return nil, fmt.Errorf("unable to decode nSig: %w", err) + } + query.Set("v", nDecoded) + log = log.With("decoded", nDecoded) + } + + log.Debug("nParam") + + return query, nil +} + +const ( + jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*" + reverseStr = ":function\\(a\\)\\{" + + "(?:return )?a\\.reverse\\(\\)" + + "\\}" + spliceStr = ":function\\(a,b\\)\\{" + + "a\\.splice\\(0,b\\)" + + "\\}" + swapStr = ":function\\(a,b\\)\\{" + + "var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" + + "\\}" +) + +var ( + nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})") + actionsObjRegexp = regexp.MustCompile(fmt.Sprintf( + "var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr)) + + actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf( + "function(?: %s)?\\(a\\)\\{"+ + "a=a\\.split\\(\"\"\\);\\s*"+ + "((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+ + "return a\\.join\\(\"\"\\)"+ + "\\}", jsvarStr, jsvarStr, jsvarStr)) + + reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr)) + spliceRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr)) + swapRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr)) +) + +func (config playerConfig) decodeNsig(encoded string) (string, error) { + fBody, err := config.getNFunction() + if err != nil { + return "", err + } + + return evalJavascript(fBody, encoded) +} + +func evalJavascript(jsFunction, arg string) (string, error) { + const myName = "myFunction" + + vm := goja.New() + _, err := vm.RunString(myName + "=" + jsFunction) + if err != nil { + return "", err + } + + var output func(string) string + err = vm.ExportTo(vm.Get(myName), &output) + if err != nil { + return "", err + } + + return output(arg), nil +} + +func (config playerConfig) getNFunction() (string, error) { + nameResult := nFunctionNameRegexp.FindSubmatch(config) + if len(nameResult) == 0 { + return "", errors.New("unable to extract n-function name") + } + + var name string + if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 { + name = string(nameResult[4]) + } else { + name = string(nameResult[1]) + } + + return config.extraFunction(name) + +} + +func (config playerConfig) extraFunction(name string) (string, error) { + // find the beginning of the function + def := []byte(name + "=function(") + start := bytes.Index(config, def) + if start < 1 { + return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def) + } + + // start after the first curly bracket + pos := start + bytes.IndexByte(config[start:], '{') + 1 + + var strChar byte + + // find the bracket closing the function + for brackets := 1; brackets > 0; pos++ { + b := config[pos] + switch b { + case '{': + if strChar == 0 { + brackets++ + } + case '}': + if strChar == 0 { + brackets-- + } + case '`', '"', '\'': + if config[pos-1] == '\\' && config[pos-2] != '\\' { + continue + } + if strChar == 0 { + strChar = b + } else if strChar == b { + strChar = 0 + } + } + } + + return string(config[start:pos]), nil +} + +func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) { + operations, err := config.parseDecipherOps() + if err != nil { + return nil, err + } + + // apply operations + bs := []byte(cyphertext) + for _, op := range operations { + bs = op(bs) + } + + return bs, nil +} + +/* +parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js + +var Mt={ +splice:function(a,b){a.splice(0,b)}, +reverse:function(a){a.reverse()}, +EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}}; + +a=a.split(""); +Mt.splice(a,3); +Mt.EQ(a,39); +Mt.splice(a,2); +Mt.EQ(a,1); +Mt.splice(a,1); +Mt.EQ(a,35); +Mt.EQ(a,51); +Mt.splice(a,2); +Mt.reverse(a,52); +return a.join("") +*/ +func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) { + objResult := actionsObjRegexp.FindSubmatch(config) + funcResult := actionsFuncRegexp.FindSubmatch(config) + if len(objResult) < 3 || len(funcResult) < 2 { + return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult)) + } + + obj := objResult[1] + objBody := objResult[2] + funcBody := funcResult[1] + + var reverseKey, spliceKey, swapKey string + + if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 { + reverseKey = string(result[1]) + } + if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 { + spliceKey = string(result[1]) + } + if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 { + swapKey = string(result[1]) + } + + regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey))) + if err != nil { + return nil, err + } + + var ops []DecipherOperation + for _, s := range regex.FindAllSubmatch(funcBody, -1) { + switch string(s[1]) { + case reverseKey: + ops = append(ops, reverseFunc) + case swapKey: + arg, _ := strconv.Atoi(string(s[2])) + ops = append(ops, newSwapFunc(arg)) + case spliceKey: + arg, _ := strconv.Atoi(string(s[2])) + ops = append(ops, newSpliceFunc(arg)) + } + } + return ops, nil +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go b/downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go new file mode 100644 index 0000000..7abb5d1 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/decipher_operations.go @@ -0,0 +1,27 @@ +package youtube + +type DecipherOperation func([]byte) []byte + +func newSpliceFunc(pos int) DecipherOperation { + return func(bs []byte) []byte { + return bs[pos:] + } +} + +func newSwapFunc(arg int) DecipherOperation { + return func(bs []byte) []byte { + pos := arg % len(bs) + bs[0], bs[pos] = bs[pos], bs[0] + return bs + } +} + +func reverseFunc(bs []byte) []byte { + l, r := 0, len(bs)-1 + for l < r { + bs[l], bs[r] = bs[r], bs[l] + l++ + r-- + } + return bs +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/doc.go b/downloader/vendor/github.com/kkdai/youtube/v2/doc.go new file mode 100644 index 0000000..821dc69 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/doc.go @@ -0,0 +1,4 @@ +/* +Package youtube implement youtube download package in go. +*/ +package youtube diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go new file mode 100644 index 0000000..b2a61dc --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/downloader.go @@ -0,0 +1,193 @@ +package downloader + +import ( + "context" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + + "github.com/kkdai/youtube/v2" + "github.com/vbauerster/mpb/v5" + "github.com/vbauerster/mpb/v5/decor" +) + +// Downloader offers high level functions to download videos into files +type Downloader struct { + youtube.Client + OutputDir string // optional directory to store the files +} + +func (dl *Downloader) getOutputFile(v *youtube.Video, format *youtube.Format, outputFile string) (string, error) { + if outputFile == "" { + outputFile = SanitizeFilename(v.Title) + outputFile += pickIdealFileExtension(format.MimeType) + } + + if dl.OutputDir != "" { + if err := os.MkdirAll(dl.OutputDir, 0o755); err != nil { + return "", err + } + outputFile = filepath.Join(dl.OutputDir, outputFile) + } + + return outputFile, nil +} + +// Download : Starting download video by arguments. +func (dl *Downloader) Download(ctx context.Context, v *youtube.Video, format *youtube.Format, outputFile string) error { + youtube.Logger.Info( + "Downloading video", + "id", v.ID, + "quality", format.Quality, + "mimeType", format.MimeType, + ) + destFile, err := dl.getOutputFile(v, format, outputFile) + if err != nil { + return err + } + + // Create output file + out, err := os.Create(destFile) + if err != nil { + return err + } + defer out.Close() + + return dl.videoDLWorker(ctx, out, v, format) +} + +// DownloadComposite : Downloads audio and video streams separately and merges them via ffmpeg. +func (dl *Downloader) DownloadComposite(ctx context.Context, outputFile string, v *youtube.Video, quality string, mimetype, language string) error { + videoFormat, audioFormat, err1 := getVideoAudioFormats(v, quality, mimetype, language) + if err1 != nil { + return err1 + } + + log := youtube.Logger.With("id", v.ID) + + log.Info( + "Downloading composite video", + "videoQuality", videoFormat.QualityLabel, + "videoMimeType", videoFormat.MimeType, + "audioMimeType", audioFormat.MimeType, + ) + + destFile, err := dl.getOutputFile(v, videoFormat, outputFile) + if err != nil { + return err + } + outputDir := filepath.Dir(destFile) + + // Create temporary video file + videoFile, err := os.CreateTemp(outputDir, "youtube_*.m4v") + if err != nil { + return err + } + defer os.Remove(videoFile.Name()) + + // Create temporary audio file + audioFile, err := os.CreateTemp(outputDir, "youtube_*.m4a") + if err != nil { + return err + } + defer os.Remove(audioFile.Name()) + + log.Debug("Downloading video file...") + err = dl.videoDLWorker(ctx, videoFile, v, videoFormat) + if err != nil { + return err + } + + log.Debug("Downloading audio file...") + err = dl.videoDLWorker(ctx, audioFile, v, audioFormat) + if err != nil { + return err + } + + //nolint:gosec + ffmpegVersionCmd := exec.Command("ffmpeg", "-y", + "-i", videoFile.Name(), + "-i", audioFile.Name(), + "-c", "copy", // Just copy without re-encoding + "-shortest", // Finish encoding when the shortest input stream ends + destFile, + "-loglevel", "warning", + ) + ffmpegVersionCmd.Stderr = os.Stderr + ffmpegVersionCmd.Stdout = os.Stdout + log.Info("merging video and audio", "output", destFile) + + return ffmpegVersionCmd.Run() +} + +func getVideoAudioFormats(v *youtube.Video, quality string, mimetype, language string) (*youtube.Format, *youtube.Format, error) { + var videoFormats, audioFormats youtube.FormatList + + formats := v.Formats + if mimetype != "" { + formats = formats.Type(mimetype) + } + + videoFormats = formats.Type("video").AudioChannels(0) + audioFormats = formats.Type("audio") + + if quality != "" { + videoFormats = videoFormats.Quality(quality) + } + + if language != "" { + audioFormats = audioFormats.Language(language) + } + + if len(videoFormats) == 0 { + return nil, nil, errors.New("no video format found after filtering") + } + + if len(audioFormats) == 0 { + return nil, nil, errors.New("no audio format found after filtering") + } + + videoFormats.Sort() + audioFormats.Sort() + + return &videoFormats[0], &audioFormats[0], nil +} + +func (dl *Downloader) videoDLWorker(ctx context.Context, out *os.File, video *youtube.Video, format *youtube.Format) error { + stream, size, err := dl.GetStreamContext(ctx, video, format) + if err != nil { + return err + } + + prog := &progress{ + contentLength: float64(size), + } + + // create progress bar + progress := mpb.New(mpb.WithWidth(64)) + bar := progress.AddBar( + int64(prog.contentLength), + + mpb.PrependDecorators( + decor.CountersKibiByte("% .2f / % .2f"), + decor.Percentage(decor.WCSyncSpace), + ), + mpb.AppendDecorators( + decor.EwmaETA(decor.ET_STYLE_GO, 90), + decor.Name(" ] "), + decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60), + ), + ) + + reader := bar.ProxyReader(stream) + mw := io.MultiWriter(out, prog) + _, err = io.Copy(mw, reader) + if err != nil { + return err + } + + progress.Wait() + return nil +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go new file mode 100644 index 0000000..e65ee27 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/file_utils.go @@ -0,0 +1,65 @@ +package downloader + +import ( + "mime" + "regexp" +) + +const defaultExtension = ".mov" + +// Rely on hardcoded canonical mime types, as the ones provided by Go aren't exhaustive [1]. +// This seems to be a recurring problem for youtube downloaders, see [2]. +// The implementation is based on mozilla's list [3], IANA [4] and Youtube's support [5]. +// [1] https://github.com/golang/go/blob/ed7888aea6021e25b0ea58bcad3f26da2b139432/src/mime/type.go#L60 +// [2] https://github.com/ZiTAL/youtube-dl/blob/master/mime.types +// [3] https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types +// [4] https://www.iana.org/assignments/media-types/media-types.xhtml#video +// [5] https://support.google.com/youtube/troubleshooter/2888402?hl=en +var canonicals = map[string]string{ + "video/quicktime": ".mov", + "video/x-msvideo": ".avi", + "video/x-matroska": ".mkv", + "video/mpeg": ".mpeg", + "video/webm": ".webm", + "video/3gpp2": ".3g2", + "video/x-flv": ".flv", + "video/3gpp": ".3gp", + "video/mp4": ".mp4", + "video/ogg": ".ogv", + "video/mp2t": ".ts", +} + +func pickIdealFileExtension(mediaType string) string { + mediaType, _, err := mime.ParseMediaType(mediaType) + if err != nil { + return defaultExtension + } + + if extension, ok := canonicals[mediaType]; ok { + return extension + } + + // Our last resort is to ask the operating system, but these give multiple results and are rarely canonical. + extensions, err := mime.ExtensionsByType(mediaType) + if err != nil || extensions == nil { + return defaultExtension + } + + return extensions[0] +} + +func SanitizeFilename(fileName string) string { + // Characters not allowed on mac + // :/ + // Characters not allowed on linux + // / + // Characters not allowed on windows + // <>:"/\|?* + + // Ref https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions + + fileName = regexp.MustCompile(`[:/<>\:"\\|?*]`).ReplaceAllString(fileName, "") + fileName = regexp.MustCompile(`\s+`).ReplaceAllString(fileName, " ") + + return fileName +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go new file mode 100644 index 0000000..6d88773 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/downloader/progress.go @@ -0,0 +1,17 @@ +package downloader + +type progress struct { + contentLength float64 + totalWrittenBytes float64 + downloadLevel float64 +} + +func (dl *progress) Write(p []byte) (n int, err error) { + n = len(p) + dl.totalWrittenBytes = dl.totalWrittenBytes + float64(n) + currentPercent := (dl.totalWrittenBytes / dl.contentLength) * 100 + if (dl.downloadLevel <= currentPercent) && (dl.downloadLevel < 100) { + dl.downloadLevel++ + } + return +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/errors.go b/downloader/vendor/github.com/kkdai/youtube/v2/errors.go new file mode 100644 index 0000000..526d64d --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/errors.go @@ -0,0 +1,47 @@ +package youtube + +import ( + "fmt" +) + +const ( + ErrCipherNotFound = constError("cipher not found") + ErrSignatureTimestampNotFound = constError("signature timestamp not found") + ErrInvalidCharactersInVideoID = constError("invalid characters in video id") + ErrVideoIDMinLength = constError("the video id must be at least 10 characters long") + ErrReadOnClosedResBody = constError("http: read on closed response body") + ErrNotPlayableInEmbed = constError("embedding of this video has been disabled") + ErrLoginRequired = constError("login required to confirm your age") + ErrVideoPrivate = constError("user restricted access to this video") + ErrInvalidPlaylist = constError("no playlist detected or invalid playlist ID") +) + +type constError string + +func (e constError) Error() string { + return string(e) +} + +type ErrPlayabiltyStatus struct { + Status string + Reason string +} + +func (err ErrPlayabiltyStatus) Error() string { + return fmt.Sprintf("cannot playback and download, status: %s, reason: %s", err.Status, err.Reason) +} + +// ErrUnexpectedStatusCode is returned on unexpected HTTP status codes +type ErrUnexpectedStatusCode int + +func (err ErrUnexpectedStatusCode) Error() string { + return fmt.Sprintf("unexpected status code: %d", err) +} + +type ErrPlaylistStatus struct { + Reason string +} + +func (err ErrPlaylistStatus) Error() string { + return fmt.Sprintf("could not load playlist: %s", err.Reason) +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go b/downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go new file mode 100644 index 0000000..989970c --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/fetch_testdata_helper.go @@ -0,0 +1,35 @@ +//go:build fetch +// +build fetch + +package youtube + +import ( + "fmt" + "io" + "net/http" + "os" +) + +func init() { + FetchTestData() +} + +// ran via go generate to fetch and update the playlist response data +func FetchTestData() { + f, err := os.Create(testPlaylistResponseDataFile) + exitOnError(err) + requestURL := fmt.Sprintf(playlistFetchURL, testPlaylistID) + resp, err := http.Get(requestURL) + exitOnError(err) + defer resp.Body.Close() + n, err := io.Copy(f, resp.Body) + exitOnError(err) + fmt.Printf("Successfully fetched playlist %s (%d bytes)\n", testPlaylistID, n) +} + +func exitOnError(err error) { + if err != nil { + fmt.Println(err) + os.Exit(1) + } +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/format_list.go b/downloader/vendor/github.com/kkdai/youtube/v2/format_list.go new file mode 100644 index 0000000..57c98dc --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/format_list.go @@ -0,0 +1,148 @@ +package youtube + +import ( + "sort" + "strconv" + "strings" +) + +type FormatList []Format + +// Type returns a new FormatList filtered by itag +func (list FormatList) Select(f func(Format) bool) (result FormatList) { + for i := range list { + if f(list[i]) { + result = append(result, list[i]) + } + } + return result +} + +// Type returns a new FormatList filtered by itag +func (list FormatList) Itag(itagNo int) FormatList { + return list.Select(func(f Format) bool { + return f.ItagNo == itagNo + }) +} + +// Type returns a new FormatList filtered by mime type +func (list FormatList) Type(value string) FormatList { + return list.Select(func(f Format) bool { + return strings.Contains(f.MimeType, value) + }) +} + +// Type returns a new FormatList filtered by display name +func (list FormatList) Language(displayName string) FormatList { + return list.Select(func(f Format) bool { + return f.LanguageDisplayName() == displayName + }) +} + +// Quality returns a new FormatList filtered by quality, quality label or itag, +// but not audio quality +func (list FormatList) Quality(quality string) FormatList { + itag, _ := strconv.Atoi(quality) + + return list.Select(func(f Format) bool { + return itag == f.ItagNo || strings.Contains(f.Quality, quality) || strings.Contains(f.QualityLabel, quality) + }) +} + +// AudioChannels returns a new FormatList filtered by the matching AudioChannels +func (list FormatList) AudioChannels(n int) FormatList { + return list.Select(func(f Format) bool { + return f.AudioChannels == n + }) +} + +// AudioChannels returns a new FormatList filtered by the matching AudioChannels +func (list FormatList) WithAudioChannels() FormatList { + return list.Select(func(f Format) bool { + return f.AudioChannels > 0 + }) +} + +// FilterQuality reduces the format list to formats matching the quality +func (v *Video) FilterQuality(quality string) { + v.Formats = v.Formats.Quality(quality) + v.Formats.Sort() +} + +// Sort sorts all formats fields +func (list FormatList) Sort() { + sort.SliceStable(list, func(i, j int) bool { + return sortFormat(i, j, list) + }) +} + +// sortFormat sorts video by resolution, FPS, codec (av01, vp9, avc1), bitrate +// sorts audio by default, codec (mp4, opus), channels, bitrate, sample rate +func sortFormat(i int, j int, formats FormatList) bool { + + // Sort by Width + if formats[i].Width == formats[j].Width { + // Format 137 downloads slowly, give it less priority + // see https://github.com/kkdai/youtube/pull/171 + switch 137 { + case formats[i].ItagNo: + return false + case formats[j].ItagNo: + return true + } + + // Sort by FPS + if formats[i].FPS == formats[j].FPS { + if formats[i].FPS == 0 && formats[i].AudioChannels > 0 && formats[j].AudioChannels > 0 { + // Audio + // Sort by default + if (formats[i].AudioTrack == nil && formats[j].AudioTrack == nil) || (formats[i].AudioTrack != nil && formats[j].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault == formats[j].AudioTrack.AudioIsDefault) { + // Sort by codec + codec := map[int]int{} + for _, index := range []int{i, j} { + if strings.Contains(formats[index].MimeType, "mp4") { + codec[index] = 1 + } else if strings.Contains(formats[index].MimeType, "opus") { + codec[index] = 2 + } + } + if codec[i] == codec[j] { + // Sort by Audio Channel + if formats[i].AudioChannels == formats[j].AudioChannels { + // Sort by Audio Bitrate + if formats[i].Bitrate == formats[j].Bitrate { + // Sort by Audio Sample Rate + return formats[i].AudioSampleRate > formats[j].AudioSampleRate + } + return formats[i].Bitrate > formats[j].Bitrate + } + return formats[i].AudioChannels > formats[j].AudioChannels + } + return codec[i] < codec[j] + } else if formats[i].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault { + return true + } + return false + } + // Video + // Sort by codec + codec := map[int]int{} + for _, index := range []int{i, j} { + if strings.Contains(formats[index].MimeType, "av01") { + codec[index] = 1 + } else if strings.Contains(formats[index].MimeType, "vp9") { + codec[index] = 2 + } else if strings.Contains(formats[index].MimeType, "avc1") { + codec[index] = 3 + } + } + if codec[i] == codec[j] { + // Sort by Audio Bitrate + return formats[i].Bitrate > formats[j].Bitrate + } + return codec[i] < codec[j] + } + return formats[i].FPS > formats[j].FPS + } + return formats[i].Width > formats[j].Width +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/logger.go b/downloader/vendor/github.com/kkdai/youtube/v2/logger.go new file mode 100644 index 0000000..11a1541 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/logger.go @@ -0,0 +1,28 @@ +package youtube + +import ( + "fmt" + "log/slog" + "os" +) + +// The global logger for all Client instances +var Logger = getLogger(os.Getenv("LOGLEVEL")) + +func SetLogLevel(value string) { + Logger = getLogger(value) +} + +func getLogger(logLevel string) *slog.Logger { + levelVar := slog.LevelVar{} + + if logLevel != "" { + if err := levelVar.UnmarshalText([]byte(logLevel)); err != nil { + panic(fmt.Sprintf("Invalid log level %s: %v", logLevel, err)) + } + } + + return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{ + Level: levelVar.Level(), + })) +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go b/downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go new file mode 100644 index 0000000..b2b147f --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/player_cache.go @@ -0,0 +1,37 @@ +package youtube + +import ( + "time" +) + +const defaultCacheExpiration = time.Minute * time.Duration(5) + +type playerCache struct { + key string + expiredAt time.Time + config playerConfig +} + +// Get : get cache when it has same video id and not expired +func (s playerCache) Get(key string) playerConfig { + return s.GetCacheBefore(key, time.Now()) +} + +// GetCacheBefore : can pass time for testing +func (s playerCache) GetCacheBefore(key string, time time.Time) playerConfig { + if key == s.key && s.expiredAt.After(time) { + return s.config + } + return nil +} + +// Set : set cache with default expiration +func (s *playerCache) Set(key string, operations playerConfig) { + s.setWithExpiredTime(key, operations, time.Now().Add(defaultCacheExpiration)) +} + +func (s *playerCache) setWithExpiredTime(key string, config playerConfig, time time.Time) { + s.key = key + s.config = config + s.expiredAt = time +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go b/downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go new file mode 100644 index 0000000..7790eeb --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/player_parse.go @@ -0,0 +1,59 @@ +package youtube + +import ( + "context" + "errors" + "fmt" + "log" + "os" + "path/filepath" + "regexp" + "strings" +) + +type playerConfig []byte + +var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`) + +func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) { + embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID) + embedBody, err := c.httpGetBodyBytes(ctx, embedURL) + if err != nil { + return nil, err + } + + // example: /s/player/f676c671/player_ias.vflset/en_US/base.js + playerPath := string(basejsPattern.Find(embedBody)) + if playerPath == "" { + return nil, errors.New("unable to find basejs URL in playerConfig") + } + + // for debugging + var artifactName string + if artifactsFolder != "" { + parts := strings.SplitN(playerPath, "/", 5) + artifactName = "player-" + parts[3] + ".js" + linkName := filepath.Join(artifactsFolder, "video-"+videoID+".js") + if err := os.Symlink(artifactName, linkName); err != nil { + log.Printf("unable to create symlink %s: %v", linkName, err) + } + } + + config := c.playerCache.Get(playerPath) + if config != nil { + return config, nil + } + + config, err = c.httpGetBodyBytes(ctx, "https://youtube.com"+playerPath) + if err != nil { + return nil, err + } + + // for debugging + if artifactName != "" { + writeArtifact(artifactName, config) + } + + c.playerCache.Set(playerPath, config) + return config, nil +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/playlist.go b/downloader/vendor/github.com/kkdai/youtube/v2/playlist.go new file mode 100644 index 0000000..60462f5 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/playlist.go @@ -0,0 +1,256 @@ +package youtube + +import ( + "context" + "encoding/json" + "fmt" + "regexp" + "runtime/debug" + "strconv" + "time" + + sjson "github.com/bitly/go-simplejson" +) + +var ( + playlistIDRegex = regexp.MustCompile("^[A-Za-z0-9_-]{13,42}$") + playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{13,42})(&.*)?$") +) + +type Playlist struct { + ID string + Title string + Description string + Author string + Videos []*PlaylistEntry +} + +type PlaylistEntry struct { + ID string + Title string + Author string + Duration time.Duration + Thumbnails Thumbnails +} + +func extractPlaylistID(url string) (string, error) { + if playlistIDRegex.Match([]byte(url)) { + return url, nil + } + + matches := playlistInURLRegex.FindStringSubmatch(url) + + if matches != nil { + return matches[1], nil + } + + return "", ErrInvalidPlaylist +} + +// structs for playlist extraction + +// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text +// Description: metadata.playlistMetadataRenderer.description +// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text + +// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents +// ID: .videoId +// Title: title.runs[0].text +// Author: .shortBylineText.runs[0].text +// Duration: .lengthSeconds +// Thumbnails .thumbnails + +// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails +func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body []byte) (err error) { + var j *sjson.Json + j, err = sjson.NewJson(body) + if err != nil { + return err + } + + defer func() { + stack := debug.Stack() + if r := recover(); r != nil { + err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack) + } + }() + + renderer := j.GetPath("alerts").GetIndex(0).GetPath("alertRenderer") + if renderer != nil && renderer.GetPath("type").MustString() == "ERROR" { + message := renderer.GetPath("text", "runs").GetIndex(0).GetPath("text").MustString() + + return ErrPlaylistStatus{Reason: message} + } + + // Metadata can be located in multiple places depending on client type + var metadata *sjson.Json + if node, ok := j.CheckGet("metadata"); ok { + metadata = node + } else if node, ok := j.CheckGet("header"); ok { + metadata = node + } else { + return fmt.Errorf("no playlist header / metadata found") + } + + metadata = metadata.Get("playlistHeaderRenderer") + + p.Title = sjsonGetText(metadata, "title") + p.Description = sjsonGetText(metadata, "description", "descriptionText") + p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1). + GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs"). + GetIndex(0).Get("text").MustString() + + if len(p.Author) == 0 { + p.Author = sjsonGetText(metadata, "owner", "ownerText") + } + + contents, ok := j.CheckGet("contents") + if !ok { + return fmt.Errorf("contents not found in json body") + } + + // contents can have different keys with same child structure + firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0). + GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0) + + // This extra nested item is only set with the web client + if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) { + firstPart = n + } + + vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON() + if err != nil { + return err + } + + if len(vJSON) <= 4 { + return fmt.Errorf("no video data found in JSON") + } + + entries, continuation, err := extractPlaylistEntries(vJSON) + if err != nil { + return err + } + + if len(continuation) == 0 { + continuation = getContinuation(firstPart.Get("playlistVideoListRenderer")) + } + + if len(entries) == 0 { + return fmt.Errorf("no videos found in playlist") + } + + p.Videos = entries + + for continuation != "" { + data := prepareInnertubePlaylistData(continuation, true, *client.client) + + body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data) + if err != nil { + return err + } + + j, err := sjson.NewJson(body) + if err != nil { + return err + } + + next := j.GetPath("onResponseReceivedActions").GetIndex(0). + GetPath("appendContinuationItemsAction", "continuationItems") + + if !isValidJSON(next) { + next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents") + } + + vJSON, err := next.MarshalJSON() + if err != nil { + return err + } + + entries, token, err := extractPlaylistEntries(vJSON) + if err != nil { + return err + } + + if len(token) > 0 { + continuation = token + } else { + continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation")) + } + + p.Videos = append(p.Videos, entries...) + } + + return err +} + +func extractPlaylistEntries(data []byte) ([]*PlaylistEntry, string, error) { + var vids []*videosJSONExtractor + + if err := json.Unmarshal(data, &vids); err != nil { + return nil, "", err + } + + entries := make([]*PlaylistEntry, 0, len(vids)) + + var continuation string + for _, v := range vids { + if v.Renderer == nil { + if v.Continuation.Endpoint.Command.Token != "" { + continuation = v.Continuation.Endpoint.Command.Token + } + + continue + } + + entries = append(entries, v.PlaylistEntry()) + } + + return entries, continuation, nil +} + +type videosJSONExtractor struct { + Renderer *struct { + ID string `json:"videoId"` + Title withRuns `json:"title"` + Author withRuns `json:"shortBylineText"` + Duration string `json:"lengthSeconds"` + Thumbnail struct { + Thumbnails []Thumbnail `json:"thumbnails"` + } `json:"thumbnail"` + } `json:"playlistVideoRenderer"` + Continuation struct { + Endpoint struct { + Command struct { + Token string `json:"token"` + } `json:"continuationCommand"` + } `json:"continuationEndpoint"` + } `json:"continuationItemRenderer"` +} + +func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry { + ds, err := strconv.Atoi(vje.Renderer.Duration) + if err != nil { + panic("invalid video duration: " + vje.Renderer.Duration) + } + return &PlaylistEntry{ + ID: vje.Renderer.ID, + Title: vje.Renderer.Title.String(), + Author: vje.Renderer.Author.String(), + Duration: time.Second * time.Duration(ds), + Thumbnails: vje.Renderer.Thumbnail.Thumbnails, + } +} + +type withRuns struct { + Runs []struct { + Text string `json:"text"` + } `json:"runs"` +} + +func (wr withRuns) String() string { + if len(wr.Runs) > 0 { + return wr.Runs[0].Text + } + return "" +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/response_data.go b/downloader/vendor/github.com/kkdai/youtube/v2/response_data.go new file mode 100644 index 0000000..154a572 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/response_data.go @@ -0,0 +1,153 @@ +package youtube + +type playerResponseData struct { + Captions struct { + PlayerCaptionsTracklistRenderer struct { + CaptionTracks []CaptionTrack `json:"captionTracks"` + AudioTracks []struct { + CaptionTrackIndices []int `json:"captionTrackIndices"` + } `json:"audioTracks"` + TranslationLanguages []struct { + LanguageCode string `json:"languageCode"` + LanguageName struct { + SimpleText string `json:"simpleText"` + } `json:"languageName"` + } `json:"translationLanguages"` + DefaultAudioTrackIndex int `json:"defaultAudioTrackIndex"` + } `json:"playerCaptionsTracklistRenderer"` + } `json:"captions"` + + PlayabilityStatus struct { + Status string `json:"status"` + Reason string `json:"reason"` + PlayableInEmbed bool `json:"playableInEmbed"` + Miniplayer struct { + MiniplayerRenderer struct { + PlaybackMode string `json:"playbackMode"` + } `json:"miniplayerRenderer"` + } `json:"miniplayer"` + ContextParams string `json:"contextParams"` + } `json:"playabilityStatus"` + StreamingData struct { + ExpiresInSeconds string `json:"expiresInSeconds"` + Formats []Format `json:"formats"` + AdaptiveFormats []Format `json:"adaptiveFormats"` + DashManifestURL string `json:"dashManifestUrl"` + HlsManifestURL string `json:"hlsManifestUrl"` + } `json:"streamingData"` + VideoDetails struct { + VideoID string `json:"videoId"` + Title string `json:"title"` + LengthSeconds string `json:"lengthSeconds"` + Keywords []string `json:"keywords"` + ChannelID string `json:"channelId"` + IsOwnerViewing bool `json:"isOwnerViewing"` + ShortDescription string `json:"shortDescription"` + IsCrawlable bool `json:"isCrawlable"` + Thumbnail struct { + Thumbnails []Thumbnail `json:"thumbnails"` + } `json:"thumbnail"` + AverageRating float64 `json:"averageRating"` + AllowRatings bool `json:"allowRatings"` + ViewCount string `json:"viewCount"` + Author string `json:"author"` + IsPrivate bool `json:"isPrivate"` + IsUnpluggedCorpus bool `json:"isUnpluggedCorpus"` + IsLiveContent bool `json:"isLiveContent"` + } `json:"videoDetails"` + Microformat struct { + PlayerMicroformatRenderer struct { + Thumbnail struct { + Thumbnails []struct { + URL string `json:"url"` + Width int `json:"width"` + Height int `json:"height"` + } `json:"thumbnails"` + } `json:"thumbnail"` + Title struct { + SimpleText string `json:"simpleText"` + } `json:"title"` + Description struct { + SimpleText string `json:"simpleText"` + } `json:"description"` + LengthSeconds string `json:"lengthSeconds"` + OwnerProfileURL string `json:"ownerProfileUrl"` + ExternalChannelID string `json:"externalChannelId"` + IsFamilySafe bool `json:"isFamilySafe"` + AvailableCountries []string `json:"availableCountries"` + IsUnlisted bool `json:"isUnlisted"` + HasYpcMetadata bool `json:"hasYpcMetadata"` + ViewCount string `json:"viewCount"` + Category string `json:"category"` + PublishDate string `json:"publishDate"` + OwnerChannelName string `json:"ownerChannelName"` + UploadDate string `json:"uploadDate"` + } `json:"playerMicroformatRenderer"` + } `json:"microformat"` +} + +type Format struct { + ItagNo int `json:"itag"` + URL string `json:"url"` + MimeType string `json:"mimeType"` + Quality string `json:"quality"` + Cipher string `json:"signatureCipher"` + Bitrate int `json:"bitrate"` + FPS int `json:"fps"` + Width int `json:"width"` + Height int `json:"height"` + LastModified string `json:"lastModified"` + ContentLength int64 `json:"contentLength,string"` + QualityLabel string `json:"qualityLabel"` + ProjectionType string `json:"projectionType"` + AverageBitrate int `json:"averageBitrate"` + AudioQuality string `json:"audioQuality"` + ApproxDurationMs string `json:"approxDurationMs"` + AudioSampleRate string `json:"audioSampleRate"` + AudioChannels int `json:"audioChannels"` + + // InitRange is only available for adaptive formats + InitRange *struct { + Start string `json:"start"` + End string `json:"end"` + } `json:"initRange"` + + // IndexRange is only available for adaptive formats + IndexRange *struct { + Start string `json:"start"` + End string `json:"end"` + } `json:"indexRange"` + + // AudioTrack is only available for videos with multiple audio track languages + AudioTrack *struct { + DisplayName string `json:"displayName"` + ID string `json:"id"` + AudioIsDefault bool `json:"audioIsDefault"` + } +} + +func (f *Format) LanguageDisplayName() string { + if f.AudioTrack == nil { + return "" + } + return f.AudioTrack.DisplayName +} + +type Thumbnails []Thumbnail + +type Thumbnail struct { + URL string + Width uint + Height uint +} + +type CaptionTrack struct { + BaseURL string `json:"baseUrl"` + Name struct { + SimpleText string `json:"simpleText"` + } `json:"name"` + VssID string `json:"vssId"` + LanguageCode string `json:"languageCode"` + Kind string `json:"kind,omitempty"` + IsTranslatable bool `json:"isTranslatable"` +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/transcript.go b/downloader/vendor/github.com/kkdai/youtube/v2/transcript.go new file mode 100644 index 0000000..18724a5 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/transcript.go @@ -0,0 +1,214 @@ +package youtube + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "strconv" + "strings" +) + +var ( + ErrTranscriptDisabled = errors.New("transcript is disabled on this video") +) + +// TranscriptSegment is a single transcipt segment spanning a few milliseconds. +type TranscriptSegment struct { + // Text is the transcipt text. + Text string `json:"text"` + + // StartMs is the start timestamp in ms. + StartMs int `json:"offset"` + + // OffsetText e.g. '4:00'. + OffsetText string `json:"offsetText"` + + // Duration the transcript segment spans in ms. + Duration int `json:"duration"` +} + +func (tr TranscriptSegment) String() string { + return tr.OffsetText + " - " + strings.TrimSpace(tr.Text) +} + +type VideoTranscript []TranscriptSegment + +func (vt VideoTranscript) String() string { + var str string + for _, tr := range vt { + str += tr.String() + "\n" + } + + return str +} + +// GetTranscript fetches the video transcript if available. +// +// Not all videos have transcripts, only relatively new videos. +// If transcripts are disabled or not available, ErrTranscriptDisabled is returned. +func (c *Client) GetTranscript(video *Video, lang string) (VideoTranscript, error) { + return c.GetTranscriptCtx(context.Background(), video, lang) +} + +// GetTranscriptCtx fetches the video transcript if available. +// +// Not all videos have transcripts, only relatively new videos. +// If transcripts are disabled or not available, ErrTranscriptDisabled is returned. +func (c *Client) GetTranscriptCtx(ctx context.Context, video *Video, lang string) (VideoTranscript, error) { + c.assureClient() + + if video == nil || video.ID == "" { + return nil, fmt.Errorf("no video provided") + } + + body, err := c.transcriptDataByInnertube(ctx, video.ID, lang) + if err != nil { + return nil, err + } + + transcript, err := parseTranscript(body) + if err != nil { + return nil, err + } + + return transcript, nil +} + +func parseTranscript(body []byte) (VideoTranscript, error) { + var resp transcriptResp + if err := json.Unmarshal(body, &resp); err != nil { + return nil, err + } + + if len(resp.Actions) > 0 { + // Android client response + if app := resp.Actions[0].AppSegment; app != nil { + return getSegments(app) + } + + // Web client response + if web := resp.Actions[0].WebSegment; web != nil { + return nil, fmt.Errorf("not implemented") + } + } + + return nil, ErrTranscriptDisabled +} + +type segmenter interface { + ParseSegments() []TranscriptSegment +} + +func getSegments(f segmenter) (VideoTranscript, error) { + if segments := f.ParseSegments(); len(segments) > 0 { + return segments, nil + } + + return nil, ErrTranscriptDisabled +} + +// transcriptResp is the JSON structure as returned by the transcript API. +type transcriptResp struct { + Actions []struct { + AppSegment *appData `json:"elementsCommand"` + WebSegment *webData `json:"updateEngagementPanelAction"` + } `json:"actions"` +} + +type appData struct { + TEC struct { + Args struct { + ListArgs struct { + Ow struct { + InitialSeg []struct { + TranscriptSegment struct { + StartMs string `json:"startMs"` + EndMs string `json:"endMs"` + Text struct { + String struct { + // Content is the actual transctipt text + Content string `json:"content"` + } `json:"elementsAttributedString"` + } `json:"snippet"` + StartTimeText struct { + String struct { + // Content is the fomratted timestamp, e.g. '4:00' + Content string `json:"content"` + } `json:"elementsAttributedString"` + } `json:"startTimeText"` + } `json:"transcriptSegmentRenderer"` + } `json:"initialSegments"` + } `json:"overwrite"` + } `json:"transformTranscriptSegmentListArguments"` + } `json:"arguments"` + } `json:"transformEntityCommand"` +} + +func (s *appData) ParseSegments() []TranscriptSegment { + rawSegments := s.TEC.Args.ListArgs.Ow.InitialSeg + segments := make([]TranscriptSegment, 0, len(rawSegments)) + + for _, segment := range rawSegments { + startMs, _ := strconv.Atoi(segment.TranscriptSegment.StartMs) + endMs, _ := strconv.Atoi(segment.TranscriptSegment.EndMs) + + segments = append(segments, TranscriptSegment{ + Text: segment.TranscriptSegment.Text.String.Content, + StartMs: startMs, + OffsetText: segment.TranscriptSegment.StartTimeText.String.Content, + Duration: endMs - startMs, + }) + } + + return segments +} + +type webData struct { + Content struct { + TR struct { + Body struct { + TBR struct { + Cues []struct { + Transcript struct { + FormattedStartOffset struct { + SimpleText string `json:"simpleText"` + } `json:"formattedStartOffset"` + Cues []struct { + TranscriptCueRenderer struct { + Cue struct { + SimpleText string `json:"simpleText"` + } `json:"cue"` + StartOffsetMs string `json:"startOffsetMs"` + DurationMs string `json:"durationMs"` + } `json:"transcriptCueRenderer"` + } `json:"cues"` + } `json:"transcriptCueGroupRenderer"` + } `json:"cueGroups"` + } `json:"transcriptSearchPanelRenderer"` + } `json:"content"` + } `json:"transcriptRenderer"` + } `json:"content"` +} + +func (s *webData) ParseSegments() []TranscriptSegment { + // TODO: doesn't actually work now, check json. + cues := s.Content.TR.Body.TBR.Cues + segments := make([]TranscriptSegment, 0, len(cues)) + + for _, s := range cues { + formatted := s.Transcript.FormattedStartOffset.SimpleText + segment := s.Transcript.Cues[0].TranscriptCueRenderer + start, _ := strconv.Atoi(segment.StartOffsetMs) + duration, _ := strconv.Atoi(segment.DurationMs) + + segments = append(segments, TranscriptSegment{ + Text: segment.Cue.SimpleText, + StartMs: start, + OffsetText: formatted, + Duration: duration, + }) + } + + return segments +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/utils.go b/downloader/vendor/github.com/kkdai/youtube/v2/utils.go new file mode 100644 index 0000000..80c84b4 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/utils.go @@ -0,0 +1,97 @@ +package youtube + +import ( + "encoding/base64" + + sjson "github.com/bitly/go-simplejson" +) + +type chunk struct { + start int64 + end int64 + data chan []byte +} + +func getChunks(totalSize, chunkSize int64) []chunk { + var chunks []chunk + + for start := int64(0); start < totalSize; start += chunkSize { + end := chunkSize + start - 1 + if end > totalSize-1 { + end = totalSize - 1 + } + + chunks = append(chunks, chunk{start, end, make(chan []byte, 1)}) + } + + return chunks +} + +func getFirstKeyJSON(j *sjson.Json) *sjson.Json { + m, err := j.Map() + if err != nil { + return j + } + + for key := range m { + return j.Get(key) + } + + return j +} + +func isValidJSON(j *sjson.Json) bool { + b, err := j.MarshalJSON() + if err != nil { + return false + } + + if len(b) <= 4 { + return false + } + + return true +} + +func sjsonGetText(j *sjson.Json, paths ...string) string { + for _, path := range paths { + if isValidJSON(j.Get(path)) { + j = j.Get(path) + } + } + + if text, err := j.String(); err == nil { + return text + } + + if isValidJSON(j.Get("text")) { + return j.Get("text").MustString() + } + + if p := j.Get("runs"); isValidJSON(p) { + var text string + + for i := 0; i < len(p.MustArray()); i++ { + if textNode := p.GetIndex(i).Get("text"); isValidJSON(textNode) { + text += textNode.MustString() + } + } + + return text + } + + return "" +} + +func getContinuation(j *sjson.Json) string { + return j.GetPath("continuations"). + GetIndex(0).GetPath("nextContinuationData", "continuation").MustString() +} + +func base64PadEnc(str string) string { + return base64.StdEncoding.EncodeToString([]byte(str)) +} + +func base64Enc(str string) string { + return base64.RawStdEncoding.EncodeToString([]byte(str)) +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/video.go b/downloader/vendor/github.com/kkdai/youtube/v2/video.go new file mode 100644 index 0000000..cbb1520 --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/video.go @@ -0,0 +1,147 @@ +package youtube + +import ( + "encoding/json" + "errors" + "fmt" + "net/url" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +type Video struct { + ID string + Title string + Description string + Author string + ChannelID string + ChannelHandle string + Views int + Duration time.Duration + PublishDate time.Time + Formats FormatList + Thumbnails Thumbnails + DASHManifestURL string // URI of the DASH manifest file + HLSManifestURL string // URI of the HLS manifest file + CaptionTracks []CaptionTrack +} + +const dateFormat = "2006-01-02" + +func (v *Video) parseVideoInfo(body []byte) error { + var prData playerResponseData + if err := json.Unmarshal(body, &prData); err != nil { + return fmt.Errorf("unable to parse player response JSON: %w", err) + } + + if err := v.isVideoFromInfoDownloadable(prData); err != nil { + return err + } + + return v.extractDataFromPlayerResponse(prData) +} + +func (v *Video) isVideoFromInfoDownloadable(prData playerResponseData) error { + return v.isVideoDownloadable(prData, false) +} + +var playerResponsePattern = regexp.MustCompile(`var ytInitialPlayerResponse\s*=\s*(\{.+?\});`) + +func (v *Video) parseVideoPage(body []byte) error { + initialPlayerResponse := playerResponsePattern.FindSubmatch(body) + if initialPlayerResponse == nil || len(initialPlayerResponse) < 2 { + return errors.New("no ytInitialPlayerResponse found in the server's answer") + } + + var prData playerResponseData + if err := json.Unmarshal(initialPlayerResponse[1], &prData); err != nil { + return fmt.Errorf("unable to parse player response JSON: %w", err) + } + + if err := v.isVideoFromPageDownloadable(prData); err != nil { + return err + } + + return v.extractDataFromPlayerResponse(prData) +} + +func (v *Video) isVideoFromPageDownloadable(prData playerResponseData) error { + return v.isVideoDownloadable(prData, true) +} + +func (v *Video) isVideoDownloadable(prData playerResponseData, isVideoPage bool) error { + // Check if video is downloadable + switch prData.PlayabilityStatus.Status { + case "OK": + return nil + case "LOGIN_REQUIRED": + // for some reason they use same status message for age-restricted and private videos + if strings.HasPrefix(prData.PlayabilityStatus.Reason, "This video is private") { + return ErrVideoPrivate + } + return ErrLoginRequired + } + + if !isVideoPage && !prData.PlayabilityStatus.PlayableInEmbed { + return ErrNotPlayableInEmbed + } + + return &ErrPlayabiltyStatus{ + Status: prData.PlayabilityStatus.Status, + Reason: prData.PlayabilityStatus.Reason, + } +} + +func (v *Video) extractDataFromPlayerResponse(prData playerResponseData) error { + v.Title = prData.VideoDetails.Title + v.Description = prData.VideoDetails.ShortDescription + v.Author = prData.VideoDetails.Author + v.Thumbnails = prData.VideoDetails.Thumbnail.Thumbnails + v.ChannelID = prData.VideoDetails.ChannelID + v.CaptionTracks = prData.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks + + if views, _ := strconv.Atoi(prData.VideoDetails.ViewCount); views > 0 { + v.Views = views + } + + if seconds, _ := strconv.Atoi(prData.VideoDetails.LengthSeconds); seconds > 0 { + v.Duration = time.Duration(seconds) * time.Second + } + + if seconds, _ := strconv.Atoi(prData.Microformat.PlayerMicroformatRenderer.LengthSeconds); seconds > 0 { + v.Duration = time.Duration(seconds) * time.Second + } + + if str := prData.Microformat.PlayerMicroformatRenderer.PublishDate; str != "" { + v.PublishDate, _ = time.Parse(dateFormat, str) + } + + if profileURL, err := url.Parse(prData.Microformat.PlayerMicroformatRenderer.OwnerProfileURL); err == nil && len(profileURL.Path) > 1 { + v.ChannelHandle = profileURL.Path[1:] + } + + // Assign Streams + v.Formats = append(prData.StreamingData.Formats, prData.StreamingData.AdaptiveFormats...) + if len(v.Formats) == 0 { + return errors.New("no formats found in the server's answer") + } + + // Sort formats by bitrate + sort.SliceStable(v.Formats, v.SortBitrateDesc) + + v.HLSManifestURL = prData.StreamingData.HlsManifestURL + v.DASHManifestURL = prData.StreamingData.DashManifestURL + + return nil +} + +func (v *Video) SortBitrateDesc(i int, j int) bool { + return v.Formats[i].Bitrate > v.Formats[j].Bitrate +} + +func (v *Video) SortBitrateAsc(i int, j int) bool { + return v.Formats[i].Bitrate < v.Formats[j].Bitrate +} diff --git a/downloader/vendor/github.com/kkdai/youtube/v2/video_id.go b/downloader/vendor/github.com/kkdai/youtube/v2/video_id.go new file mode 100644 index 0000000..7de09cb --- /dev/null +++ b/downloader/vendor/github.com/kkdai/youtube/v2/video_id.go @@ -0,0 +1,34 @@ +package youtube + +import ( + "regexp" + "strings" +) + +var videoRegexpList = []*regexp.Regexp{ + regexp.MustCompile(`(?:v|embed|shorts|watch\?v)(?:=|/)([^"&?/=%]{11})`), + regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`), + regexp.MustCompile(`([^"&?/=%]{11})`), +} + +// ExtractVideoID extracts the videoID from the given string +func ExtractVideoID(videoID string) (string, error) { + if strings.Contains(videoID, "youtu") || strings.ContainsAny(videoID, "\"?&/<%=") { + for _, re := range videoRegexpList { + if isMatch := re.MatchString(videoID); isMatch { + subs := re.FindStringSubmatch(videoID) + videoID = subs[1] + } + } + } + + if strings.ContainsAny(videoID, "?&/<%=") { + return "", ErrInvalidCharactersInVideoID + } + + if len(videoID) < 10 { + return "", ErrVideoIDMinLength + } + + return videoID, nil +}