Compare commits

...

17 Commits

Author SHA1 Message Date
f867606a6f Use tv player for more HD shit 2025-10-07 10:29:49 +02:00
0002b08256 Hallucinate a fix to ytdlp 2025-10-06 13:57:27 +02:00
ccefc21f53 Update dependencies 2025-10-06 13:47:58 +02:00
8f45686dfc Remove some unused trash 2025-10-06 13:47:58 +02:00
92ea0b43d0 Add nssm scripts 2025-10-06 13:47:58 +02:00
eb4f715ce3 Have downloader use new logging utilities 2025-10-06 13:47:58 +02:00
4601a0fc60 Add deploy script 2025-04-13 17:33:34 +02:00
d420f78ab8 Implement writing urls to nsq for downloading 2025-04-13 17:28:58 +02:00
014e99751e Implement skipping videos that we've already seen 2025-04-13 17:22:44 +02:00
91106bd391 Implement reading xml rss feed and extracting videos 2025-04-13 17:21:06 +02:00
3a74d63963 Implement basic rss reader structure 2025-04-13 17:17:00 +02:00
586c49558e Refine the discord api a little 2025-04-13 16:30:28 +02:00
00ed4e9fc1 Implement discord webhook notifications 2025-04-13 16:27:38 +02:00
c859499676 Change nsq ip 2025-04-13 16:19:32 +02:00
7adc1e2f8c Undo vendor 2025-04-13 16:19:24 +02:00
0c1a643d93 Update hotkey script to work with multiple selectors
Because, of course, videos are drawn differently on home page, search
results, video recommendations and subscriptions
THANKS GOOGL
2024-12-21 11:34:29 +01:00
ad4aac1061 Rename package 2024-12-21 11:25:31 +01:00
46 changed files with 597 additions and 3232 deletions

5
.gitignore vendored
View File

@@ -7,3 +7,8 @@ downloader/vendor/golang.org
downloader/vendor/github.com/*
!downloader/vendor/github.com/kkdai
downloader/vendor/modules.txt
downloader/ytdl.exe
downloader/ytdl.log
*.log
*.xml
*.exe

View File

@@ -1,75 +0,0 @@
package main
// import (
// "bytes"
// "context"
// "encoding/json"
// "fmt"
// "io"
// "log"
// "net/http"
// )
// type APIError struct {
// Code int `json:"code"`
// Message string `json:"message"`
// Data APIErrorData `json:"data"`
// }
// type APIErrorData struct {
// Link APIErrorLink `json:"link"`
// }
// type APIErrorLink struct {
// Code string `json:"code"`
// Message string `json:"message"`
// }
// func SetDownloaded(item PBEvent) (err error) {
// req, err := http.NewRequestWithContext(context.Background(), "PATCH", FULL_URL+"/"+item.Record.Id, nil)
// if err != nil {
// log.Printf("Error creating PATCH request: %++v", err)
// return err
// }
// req.Header.Set("Content-Type", "application/json")
// partialItem := new(PBEvent)
// partialItem.Record = item.Record
// partialItem.Record.Downloaded = true
// body, err := json.Marshal(partialItem.Record)
// if err != nil {
// log.Printf("Error marshalling subscription body: %++v", err)
// return err
// }
// req.Body = io.NopCloser(bytes.NewReader(body))
// client := http.Client{}
// res, err := client.Do(req)
// if err != nil {
// log.Printf("Error sending PATCH request: %++v", err)
// return err
// }
// defer res.Body.Close()
// if res.StatusCode != http.StatusOK {
// log.Printf("Non-OK HTTP status: %d", res.StatusCode)
// body, err = io.ReadAll(res.Body)
// if err != nil {
// log.Printf("Error reading response body: %++v", err)
// return err
// }
// var data APIError
// err = json.Unmarshal(body, &data)
// if err != nil {
// log.Printf("Error unmarshaling JSON: %++v", err)
// return err
// }
// log.Printf("API error: %++v", data)
// return fmt.Errorf("Non-OK HTTP status, err: %++v", data)
// }
// return nil
// }

View File

@@ -1,3 +1,3 @@
nssm stop YoutubeDownloader && \
go build main && \
go build . && \
nssm start YoutubeDownloader

View File

@@ -8,4 +8,4 @@ const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
var ongoingDownloads = make(map[string]struct{})
var ongoingDownloadsMutex = &sync.Mutex{}
const DOWNLOAD_WORKERS = 2
const DOWNLOAD_WORKERS = 4

View File

@@ -3,18 +3,23 @@ package downloaders
import (
"context"
"fmt"
"log"
"github.com/gen2brain/beeep"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/lrstanley/go-ytdlp"
)
func init() {
// Ensure yt-dlp is installed/up-to-date for this environment.
ytdlp.MustInstall(context.TODO(), nil)
}
var dl = ytdlp.New().
// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
FormatSort("res,ext:mp4:m4a").
FormatSort("best,ext:mp4:m4a").
Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
LimitRate(fmt.Sprintf("%dM", 150/DOWNLOAD_WORKERS)).
// HTTPChunkSize("20M").
ExtractorArgs("youtube:player_client=tv").
MarkWatched().
SponsorblockMark("all").
RecodeVideo("mp4").
@@ -23,35 +28,36 @@ var dl = ytdlp.New().
type YTDLPLibDownloader struct{}
func (d *YTDLPLibDownloader) Download(url string) error {
downloadlogger := logger.Default.WithPrefix(fmt.Sprintf("url=%q", url))
_, ongoing := ongoingDownloads[url]
if ongoing {
// return fmt.Errorf("Download %s is already ongoing", url)
log.Printf("Download %s is already ongoing", url)
downloadlogger.Info("Download is already ongoing")
return nil
}
ongoingDownloadsMutex.Lock()
ongoingDownloads[url] = struct{}{}
ongoingDownloadsMutex.Unlock()
log.Printf("YTDLPLib downloading %s", url)
downloadlogger.Info("YTDLPLib downloading")
go func() {
err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
if err != nil {
log.Printf("Failed beeping with %+v", err)
}
err = beeep.Alert("Download Started", url, "assets/information.png")
if err != nil {
log.Printf("Failed alerting with %+v", err)
}
}()
// go func() {
// err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
// if err != nil {
// downloadlogger.Error("Failed beeping with %+v", err)
// }
// err = beeep.Alert("Download Started", url, "assets/information.png")
// if err != nil {
// downloadlogger.Error("Failed alerting with %+v", err)
// }
// }()
_, err := dl.Run(context.TODO(), url)
if err != nil {
return fmt.Errorf("failed downloading %s with %+v", url, err)
}
log.Printf("Downloaded %s", url)
downloadlogger.Info("Downloaded")
ongoingDownloadsMutex.Lock()
delete(ongoingDownloads, url)
ongoingDownloadsMutex.Unlock()

View File

@@ -1,34 +1,46 @@
module main
module ytdl
go 1.22.4
go 1.24.0
require (
github.com/ProtonMail/go-crypto v1.1.3 // indirect
git.sr.ht/~jackmordaunt/go-toast v1.1.2 // indirect
github.com/ProtonMail/go-crypto v1.3.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/bitly/go-simplejson v0.5.1 // indirect
github.com/cloudflare/circl v1.5.0 // indirect
github.com/dlclark/regexp2 v1.11.4 // indirect
github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd // indirect
github.com/clipperhouse/uax29/v2 v2.2.0 // indirect
github.com/cloudflare/circl v1.6.1 // indirect
github.com/dlclark/regexp2 v1.11.5 // indirect
github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994 // indirect
github.com/esiqveland/notify v0.13.3 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/golang/snappy v1.0.0 // indirect
github.com/google/go-cmp v0.7.0 // indirect
github.com/google/pprof v0.0.0-20251002213607-436353cc1ee6 // indirect
github.com/hexops/valast v1.5.0 // indirect
github.com/jackmordaunt/icns/v3 v3.0.1 // indirect
github.com/mattn/go-runewidth v0.0.19 // indirect
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 // indirect
github.com/sergeymakinen/go-bmp v1.0.0 // indirect
github.com/sergeymakinen/go-ico v1.0.0-beta.0 // indirect
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
github.com/ulikunitz/xz v0.5.15 // indirect
github.com/vbauerster/mpb/v5 v5.4.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/crypto v0.42.0 // indirect
golang.org/x/mod v0.28.0 // indirect
golang.org/x/sync v0.17.0 // indirect
golang.org/x/sys v0.36.0 // indirect
golang.org/x/text v0.29.0 // indirect
golang.org/x/tools v0.37.0 // indirect
mvdan.cc/gofumpt v0.9.1 // indirect
)
require (
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4
github.com/gorilla/websocket v1.5.3
github.com/kkdai/youtube/v2 v2.10.2
github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd
git.site.quack-lab.dev/dave/cylogger v1.4.0
github.com/gen2brain/beeep v0.11.1
github.com/kkdai/youtube/v2 v2.10.4
github.com/lrstanley/go-ytdlp v1.2.6
github.com/nsqio/go-nsq v1.1.0
)

View File

@@ -1,7 +1,11 @@
git.site.quack-lab.dev/dave/cylogger v1.4.0 h1:3Ca7V5JWvruARJd5S8xDFwW9LnZ9QInqkYLRdrEFvuY=
git.site.quack-lab.dev/dave/cylogger v1.4.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk=
git.sr.ht/~jackmordaunt/go-toast v1.1.2 h1:/yrfI55LRt1M7H1vkaw+NaH1+L1CDxrqDltwm5euVuE=
git.sr.ht/~jackmordaunt/go-toast v1.1.2/go.mod h1:jA4OqHKTQ4AFBdwrSnwnskUIIS3HYzlJSgdzCKqfavo=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk=
github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
github.com/ProtonMail/go-crypto v1.3.0 h1:ILq8+Sf5If5DCpHQp4PbZdS1J7HDFRXz/+xKBiRGFrw=
github.com/ProtonMail/go-crypto v1.3.0/go.mod h1:9whxjD8Rbs29b4XWbB8irEcE8KHMqaR2e7GWU1R+/PE=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
@@ -9,62 +13,104 @@ github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpH
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/clipperhouse/uax29/v2 v2.2.0 h1:ChwIKnQN3kcZteTXMgb1wztSgaU+ZemkgWdohwgs8tY=
github.com/clipperhouse/uax29/v2 v2.2.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM=
github.com/cloudflare/circl v1.6.1 h1:zqIqSPIndyBh1bjLVVDHMPpVKqp8Su/V+6MeDzzQBQ0=
github.com/cloudflare/circl v1.6.1/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd h1:QMSNEh9uQkDjyPwu/J541GgSH+4hw+0skJDIj9HJ3mE=
github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI=
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4=
github.com/dlclark/regexp2 v1.11.5 h1:Q/sSnsKerHeCkc/jSTNq1oCm7KiVgUMZRDUoRu0JQZQ=
github.com/dlclark/regexp2 v1.11.5/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994 h1:aQYWswi+hRL2zJqGacdCZx32XjKYV8ApXFGntw79XAM=
github.com/dop251/goja v0.0.0-20250630131328-58d95d85e994/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
github.com/esiqveland/notify v0.13.3 h1:QCMw6o1n+6rl+oLUfg8P1IIDSFsDEb2WlXvVvIJbI/o=
github.com/esiqveland/notify v0.13.3/go.mod h1:hesw/IRYTO0x99u1JPweAl4+5mwXJibQVUcP0Iu5ORE=
github.com/gen2brain/beeep v0.11.1 h1:EbSIhrQZFDj1K2fzlMpAYlFOzV8YuNe721A58XcCTYI=
github.com/gen2brain/beeep v0.11.1/go.mod h1:jQVvuwnLuwOcdctHn/uyh8horSBNJ8uGb9Cn2W4tvoc=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78=
github.com/go-quicktest/qt v1.101.0 h1:O1K29Txy5P2OK0dGo59b7b0LR6wKfIhttaAhHUyn7eI=
github.com/go-quicktest/qt v1.101.0/go.mod h1:14Bz/f7NwaXPtdYEgzsx46kqSxVwTbzVZsDC26tQJow=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 h1:qZNfIGkIANxGv/OqtnntR4DfOY2+BgwR60cAcu/i3SE=
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4/go.mod h1:kW3HQ4UdaAyrUCSSDR4xUzBKW6O2iA4uHhk7AtyYp10=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/kkdai/youtube/v2 v2.10.2 h1:e3JslUDiKEfjMzxFyrOh3O59C/aLfKNZyrcav00MZV0=
github.com/kkdai/youtube/v2 v2.10.2/go.mod h1:4y1MIg7f1o5/kQfkr7nwXFtv8PGSoe4kChOB9/iMA88=
github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd h1:lLajTMgNTs/W4H05uQYnJDRIbIvHk6XXy7DQNFRbvzU=
github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4=
github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs=
github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8=
github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU=
github.com/google/pprof v0.0.0-20251002213607-436353cc1ee6 h1:/WHh/1k4thM/w+PAZEIiZK9NwCMFahw5tUzKUCnUtds=
github.com/google/pprof v0.0.0-20251002213607-436353cc1ee6/go.mod h1:I6V7YzU0XDpsHqbsyrghnFZLO1gwK6NPTNvmetQIk9U=
github.com/hexops/autogold v0.8.1 h1:wvyd/bAJ+Dy+DcE09BoLk6r4Fa5R5W+O+GUzmR985WM=
github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hexops/valast v1.5.0 h1:FBTuvVi0wjTngtXJRZXMbkN/Dn6DgsUsBwch2DUJU8Y=
github.com/hexops/valast v1.5.0/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4=
github.com/jackmordaunt/icns/v3 v3.0.1 h1:xxot6aNuGrU+lNgxz5I5H0qSeCjNKp8uTXB1j8D4S3o=
github.com/jackmordaunt/icns/v3 v3.0.1/go.mod h1:5sHL59nqTd2ynTnowxB/MDQFhKNqkK8X687uKNygaSQ=
github.com/kkdai/youtube/v2 v2.10.4 h1:T3VAQ65EB4eHptwcQIigpFvUJlV9EcKRGJJdSVUy3aU=
github.com/kkdai/youtube/v2 v2.10.4/go.mod h1:pm4RuJ2tRIIaOvz4YMIpCY8Ls4Fm7IVtnZQyule61MU=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lrstanley/go-ytdlp v1.2.6 h1:LJ1I+uaP2KviRAfe3tUN0Sd4yI9XlCJBG37RCH+sfq8=
github.com/lrstanley/go-ytdlp v1.2.6/go.mod h1:38IL64XM6gULrWtKTiR0+TTNCVbxesNSbTyaFG2CGTI=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/mattn/go-runewidth v0.0.19 h1:v++JhqYnZuu5jSKrk9RbgF5v4CGUjqRfBm05byFGLdw=
github.com/mattn/go-runewidth v0.0.19/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/sergeymakinen/go-bmp v1.0.0 h1:SdGTzp9WvCV0A1V0mBeaS7kQAwNLdVJbmHlqNWq0R+M=
github.com/sergeymakinen/go-bmp v1.0.0/go.mod h1:/mxlAQZRLxSvJFNIEGGLBE/m40f3ZnUifpgVDlcUIEY=
github.com/sergeymakinen/go-ico v1.0.0-beta.0 h1:m5qKH7uPKLdrygMWxbamVn+tl2HfiA3K6MFJw4GfZvQ=
github.com/sergeymakinen/go-ico v1.0.0-beta.0/go.mod h1:wQ47mTczswBO5F0NoDt7O0IXgnV4Xy3ojrroMQzyhUk=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk=
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o=
github.com/ulikunitz/xz v0.5.15 h1:9DNdB5s+SgV3bQ2ApL10xRc35ck0DuIX/isZvIk+ubY=
github.com/ulikunitz/xz v0.5.15/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
github.com/vbauerster/mpb/v5 v5.4.0 h1:n8JPunifvQvh6P1D1HAl2Ur9YcmKT1tpoUuiea5mlmg=
github.com/vbauerster/mpb/v5 v5.4.0/go.mod h1:fi4wVo7BVQ22QcvFObm+VwliQXlV1eBT8JDaKXR4JGI=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
golang.org/x/mod v0.28.0 h1:gQBtGhjxykdjY9YhZpSlZIsbnaE2+PgjfLWUQTnoZ1U=
golang.org/x/mod v0.28.0/go.mod h1:yfB/L0NOf/kmEbXjzCPOx1iK1fRutOydrCMsqRhEBxI=
golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I=
golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY=
golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
golang.org/x/tools v0.37.0 h1:DVSRzp7FwePZW356yEAChSdNcQo6Nsp+fex1SUW09lE=
golang.org/x/tools v0.37.0/go.mod h1:MBN5QPQtLMHVdvsbtarmTNukZDdgwdwlO5qGacAzF0w=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
mvdan.cc/gofumpt v0.9.1 h1:p5YT2NfFWsYyTieYgwcQ8aKV3xRvFH4uuN/zB2gBbMQ=
mvdan.cc/gofumpt v0.9.1/go.mod h1:3xYtNemnKiXaTh6R4VtlqDATFwBbdXI8lJvH/4qk7mw=

View File

@@ -3,51 +3,32 @@ package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"main/downloaders"
"os"
"os/signal"
"syscall"
"time"
"ytdl/downloaders"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/nsqio/go-nsq"
)
var Error *log.Logger
var Warning *log.Logger
func init() {
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
logFile, err := os.Create("main.log")
if err != nil {
log.Printf("Error creating log file: %v", err)
os.Exit(1)
}
logger := io.MultiWriter(os.Stdout, logFile)
log.SetOutput(logger)
Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
Warning = log.New(io.MultiWriter(logFile, os.Stdout),
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
}
//var downloader downloaders.Downloader = &downloaders.YTDLPRawDownloader{}
var downloader downloaders.Downloader = &downloaders.KidaiDownloader{}
//var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
// var downloader downloaders.Downloader = &downloaders.YTDLPRawDownloader{}
// var downloader downloaders.Downloader = &downloaders.KidaiDownloader{}
var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
type DLHandler struct{}
func (*DLHandler) HandleMessage(message *nsq.Message) error {
log.Printf("Received message '%s' with %d attempts", message.Body, message.Attempts)
messagelog := logger.Default.WithPrefix(fmt.Sprintf("message=%q", message.Body)).WithPrefix(fmt.Sprintf("attempts=%d", message.Attempts))
messagelog.Info("Received message")
data := DownloadRequest{}
err := json.Unmarshal(message.Body, &data)
if err != nil {
Error.Printf("Error unmarshalling message: %v", err)
messagelog.Error("Error unmarshalling message: %v", err)
return err
}
@@ -55,12 +36,14 @@ func (*DLHandler) HandleMessage(message *nsq.Message) error {
defer cancel()
go func() {
messagelog.Debug("Starting touch ticker")
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ticker.C:
messagelog.Debug("Touching message")
message.Touch()
case <-ctx.Done():
return
@@ -68,50 +51,63 @@ func (*DLHandler) HandleMessage(message *nsq.Message) error {
}
}()
messagelog.Debug("Downloading %q", data.Link)
err = downloader.Download(data.Link)
if err != nil {
Error.Printf("Error downloading %s: %v", data.Link, err)
messagelog.Error("Error downloading %s: %v", data.Link, err)
return err
}
messagelog.Info("Downloaded %q", data.Link)
message.Finish()
return nil
}
func main() {
// err := downloader.Download("https://www.youtube.com/watch?v=SiKjprtiPaw")
// if err != nil {
// Error.Printf("Error downloading: %v", err)
// }
// return
flag.Parse()
logger.InitFlag()
// err := downloader.Download("https://www.youtube.com/watch?v=SiKjprtiPaw")
// if err != nil {
// Error.Printf("Error downloading: %v", err)
// }
// return
logger.Info("Starting downloader")
config := nsq.NewConfig()
config.MaxAttempts = 5
config.MaxInFlight = downloaders.DOWNLOAD_WORKERS
config.MsgTimeout = 10 * time.Second
logger.Info("Creating consumer")
consumer, err := nsq.NewConsumer("ytdqueue", "dl", config)
if err != nil {
Error.Printf("Error creating consumer: %v", err)
logger.Error("Error creating consumer: %v", err)
return
}
logger.Info("Creating handlers")
for i := 0; i < downloaders.DOWNLOAD_WORKERS; i++ {
consumer.AddHandler(&DLHandler{})
}
err = consumer.ConnectToNSQD("nsq.site.quack-lab.dev:41505")
url := "192.168.1.123:41505"
logger.Info("Connecting to nsq at %s", url)
err = consumer.ConnectToNSQD(url)
if err != nil {
Error.Printf("Error connecting to nsqlookupd: %v", err)
logger.Error("Error connecting to nsq: %v", err)
return
}
logger.Info("Connected to nsq at %s", url)
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
logger.Info("Waiting for signal to terminate")
<-sigChan
log.Println("Received signal to terminate. Initiating graceful shutdown...")
logger.Info("Received signal to terminate. Initiating graceful shutdown...")
consumer.Stop()
<-consumer.StopChan
log.Println("Graceful shutdown completed.")
logger.Info("Graceful shutdown completed.")
}

9
downloader/nssm.sh Normal file
View File

@@ -0,0 +1,9 @@
nssm install YoutubeDownloader 'C:\Users\Administrator\Seafile\Projects-Go\GoProjects\youtube-downloader\downloader\ytdl.exe'
nssm set YoutubeDownloader AppDirectory 'C:\Users\Administrator\Seafile\Projects-Go\GoProjects\youtube-downloader\downloader'
nssm set YoutubeDownloader AppExit Default Restart
nssm set YoutubeDownloader AppEnvironmentExtra :PATH='C:\Users\Administrator\scoop\shims'
nssm set YoutubeDownloader AppEnvironmentExtra +YTDL_DISCORD_WEBHOOK_URL=https://discord.com/api/webhooks/1360983696366112980/NEYWb9mc_E5-x3uCKoRkNBX-G3pyXH3YcXIf7YOZibdsGP10C-u0N32LQKuf5Hfu-hi7
nssm set YoutubeDownloader DisplayName YoutubeDownloader
nssm set YoutubeDownloader ObjectName LocalSystem
nssm set YoutubeDownloader Start SERVICE_AUTO_START
nssm set YoutubeDownloader Type SERVICE_WIN32_OWN_PROCESS

View File

@@ -1,89 +0,0 @@
package main
// import (
// "bytes"
// "encoding/json"
// "log"
// "net/http"
// "github.com/r3labs/sse"
// )
// type RealtimeListener struct {
// Url string
// Collections []string
// Create chan PBEvent
// Update chan PBEvent
// Delete chan PBEvent
// client *sse.Client
// }
// type Subscription struct {
// ClientId string `json:"clientId"`
// Subscriptions []string `json:"subscriptions"`
// }
// func (listener RealtimeListener) handlePbEvent(msg *sse.Event) {
// pbEvent := new(PBEvent)
// err := json.Unmarshal(msg.Data, &pbEvent)
// if err != nil {
// log.Printf("Error unmarshalling event: %v\n", err)
// return
// }
// log.Printf("Received event: %++v", pbEvent)
// if pbEvent.ClientId != "" {
// listener.doSubscribe(pbEvent.ClientId)
// }
// if pbEvent.Action != "" {
// go listener.shipEvent(*pbEvent)
// }
// }
// func (listener RealtimeListener) shipEvent(event PBEvent) {
// switch event.Action {
// case "create":
// listener.Create <- event
// case "update":
// listener.Update <- event
// case "delete":
// listener.Delete <- event
// default:
// log.Printf("Unknown action: %v\n", event.Action)
// }
// }
// func (listener RealtimeListener) doSubscribe(clientId string) {
// subscription := Subscription{
// ClientId: clientId,
// Subscriptions: listener.Collections,
// }
// log.Printf("Subscribing client: %v to %++v", clientId, subscription)
// body, err := json.Marshal(subscription)
// if err != nil {
// log.Printf("Error marshalling subscription body: %v\n", err)
// return
// }
// resp, err := http.Post(POCKETBASE_REALTIME, "application/json", bytes.NewBuffer(body))
// if err != nil {
// log.Printf("Error posting subscription: %v\n", err)
// return
// }
// defer resp.Body.Close()
// if resp.StatusCode != http.StatusNoContent {
// log.Printf("Subscription request failed with status: %v\n", resp.Status)
// }
// }
// func (listener *RealtimeListener) initialize() {
// listener.Update = make(chan PBEvent, 32)
// listener.Create = make(chan PBEvent, 32)
// listener.Delete = make(chan PBEvent, 32)
// log.Print("Initialized")
// listener.client = sse.NewClient(listener.Url)
// go listener.client.Subscribe("", listener.handlePbEvent)
// }

1
downloader/stop.sh Normal file
View File

@@ -0,0 +1 @@
nssm stop YoutubeDownloader

View File

@@ -1,8 +0,0 @@
/.idea
/.vscode
download_test
/bin
/dist
/output
*.out
.DS_Store

View File

@@ -1,8 +0,0 @@
issues:
exclude:
- Subprocess launched with function call
# Excluding configuration per-path, per-linter, per-text and per-source
exclude-rules:
# Exclude some linters from running on tests files.
- path: cmd/
text: InsecureSkipVerify

View File

@@ -1,20 +0,0 @@
project_name: youtubedr
env:
# Build without CGO to don't depend on specific glibc versions
- CGO_ENABLED=0
builds:
- main: ./cmd/youtubedr
binary: youtubedr
goos:
- windows
- darwin
- linux
- freebsd
goarch:
- amd64
- arm
- arm64
flags:
- -trimpath

View File

@@ -1,22 +0,0 @@
The MIT License (MIT)
Copyright (c) 2015 Evan Lin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

View File

@@ -1,57 +0,0 @@
FILES_TO_FMT ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print)
LOGLEVEL ?= debug
## help: Show makefile commands
.PHONY: help
help: Makefile
@echo "---- Project: kkdai/youtube ----"
@echo " Usage: make COMMAND"
@echo
@echo " Management Commands:"
@sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /'
@echo
## build: Build project
.PHONY: build
build:
goreleaser --rm-dist
## deps: Ensures fresh go.mod and go.sum
.PHONY: deps
deps:
go mod tidy
go mod verify
## lint: Run golangci-lint check
.PHONY: lint
lint:
command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
echo "golangci-lint checking..."
golangci-lint run --timeout=30m --enable=misspell --enable=gosec --enable=gofmt --enable=goimports --enable=revive ./cmd/... ./...
go vet ./...
## format: Formats Go code
.PHONY: format
format:
@echo ">> formatting code"
@gofmt -s -w $(FILES_TO_FMT)
## test-unit: Run all Youtube Go unit tests
.PHONY: test-unit
test-unit:
LOGLEVEL=${LOGLEVEL} go test -v -cover ./...
## test-integration: Run all Youtube Go integration tests
.PHONY: test-integration
test-integration:
mkdir -p output
rm -f output/*
LOGLEVEL=${LOGLEVEL} ARTIFACTS=output go test -v -race -covermode=atomic -coverprofile=coverage.out -tags=integration ./...
.PHONY: coverage.out
coverage.out:
## clean: Clean files and downloaded videos from builds during development
.PHONY: clean
clean:
rm -rf dist *.mp4 *.mkv

View File

@@ -1,161 +0,0 @@
Download Youtube Video in Golang
==================
[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/kkdai/youtube/master/LICENSE)
[![Go Reference](https://pkg.go.dev/badge/github.com/kkdai/youtube.svg)](https://pkg.go.dev/github.com/kkdai/youtube/v2)
[![Build Status](https://github.com/kkdai/youtube/workflows/go/badge.svg?branch=master)](https://github.com/kkdai/youtube/actions)
[![Coverage](https://codecov.io/gh/kkdai/youtube/branch/master/graph/badge.svg)](https://codecov.io/gh/kkdai/youtube)
[![](https://goreportcard.com/badge/github.com/kkdai/youtube)](https://goreportcard.com/badge/github.com/kkdai/youtube)
This package is a Youtube video download package, for more detail refer [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) for more download options.
This tool is meant to be used to download CC0 licenced content, we do not support nor recommend using it for illegal activities.
## Overview
* [Install](#installation)
* [Usage](#usage)
* [Example: Download video from \[dotGo 2015 - Rob Pike - Simplicity is Complicated\]](#download-dotGo-2015-rob-pike-video)
## Installation
### Install via go get
Please ensure you have installed Go 1.22 or later.
```shell
go get github.com/kkdai/youtube/v2
```
### From source code
```shell
git clone https://github.com/kkdai/youtube.git
cd youtube
go run ./cmd/youtubedr
```
### Mac
```shell
brew install youtubedr
```
### in Termux
```shell
pkg install youtubedr
```
### You can also find this package in
- [archlinux](https://aur.archlinux.org/packages/youtubedr/) (thanks to [cjsthompson](https://github.com/cjsthompson))
- [Termux package](https://github.com/termux/termux-packages/tree/master/packages/youtubedr) (thanks to [kcubeterm](https://github.com/kcubeterm))
- [Homebrew](https://formulae.brew.sh/formula/youtubedr) (thanks to [kkc](https://github.com/kkc))
## Usage
### Use the binary directly
It's really simple to use, just get the video id from youtube url - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
```shell
$ youtubedr download QAGDGja7kbs
$ youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
```
### Use this package in your golang program
Please check out the [example_test.go](example_test.go) for example code.
## Example:
* ### Get information of dotGo-2015-rob-pike video for downloading
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
```
youtubedr info https://www.youtube.com/watch?v=rFejpH_tAHM
Title: dotGo 2015 - Rob Pike - Simplicity is Complicated
Author: dotconferences
-----available streams-----
itag: 18 , quality: medium , type: video/mp4; codecs="avc1.42001E, mp4a.40.2"
itag: 22 , quality: hd720 , type: video/mp4; codecs="avc1.64001F, mp4a.40.2"
itag: 137 , quality: hd1080 , type: video/mp4; codecs="avc1.640028"
itag: 248 , quality: hd1080 , type: video/webm; codecs="vp9"
........
```
* ### Download dotGo-2015-rob-pike-video
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
```
youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
```
* ### Download video to specific folder and name
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) to current directory and name the file to simplicity-is-complicated.mp4
```
youtubedr download -d ./ -o simplicity-is-complicated.mp4 https://www.youtube.com/watch?v=rFejpH_tAHM
```
* ### Download video with specific quality
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) with specific quality
```
youtubedr download -q medium https://www.youtube.com/watch?v=rFejpH_tAHM
```
#### Special case by quality hd1080:
Installation of ffmpeg is necessary for hd1080
```
ffmpeg //check ffmpeg is installed, if not please download ffmpeg and set to your PATH.
youtubedr download -q hd1080 https://www.youtube.com/watch?v=rFejpH_tAHM
```
* ### Download video with specific itag
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
```
youtubedr download -q 18 https://www.youtube.com/watch?v=rFejpH_tAHM
```
## How it works
- Parse the video ID you input in URL
- ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
- Get video information via video id.
- Use URL: `http://youtube.com/get_video_info?video_id=`
- Parse and decode video information.
- Download URL in "url="
- title in "title="
- Download video from URL
- Need the string combination of "url"
## Inspired
- [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl)
- [https://github.com/lepidosteus/youtube-dl](https://github.com/lepidosteus/youtube-dl)
- [拆解 Youtube 影片下載位置](http://hkgoldenmra.blogspot.tw/2013/05/youtube.html)
- [iawia002/annie](https://github.com/iawia002/annie)
- [How to get url from obfuscate video info: youtube video downloader with php](https://stackoverflow.com/questions/60607291/youtube-video-downloader-with-php)
## Project52
It is one of my [project 52](https://github.com/kkdai/project52).
## License
This package is licensed under MIT license. See LICENSE for details.

View File

@@ -1,11 +0,0 @@
Tracking Youtube decipher change
### 2022/01/21:
#### action objects:
var $z={fA:function(a){a.reverse()},S2:function(a,b){a.splice(0,b)},l6:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
#### actions function:
Npa=function(a){a=a.split("");$z.S2(a,3);$z.fA(a,45);$z.l6(a,31);$z.S2(a,1);$z.fA(a,63);$z.S2(a,2);$z.fA(a,68);return a.join("")};

View File

@@ -1,29 +0,0 @@
package youtube
import (
"log/slog"
"os"
"path/filepath"
)
// destination for artifacts, used by integration tests
var artifactsFolder = os.Getenv("ARTIFACTS")
func writeArtifact(name string, content []byte) {
// Ensure folder exists
err := os.MkdirAll(artifactsFolder, os.ModePerm)
if err != nil {
slog.Error("unable to create artifacts folder", "path", artifactsFolder, "error", err)
return
}
path := filepath.Join(artifactsFolder, name)
err = os.WriteFile(path, content, 0600)
log := slog.With("path", path)
if err != nil {
log.Error("unable to write artifact", "error", err)
} else {
log.Debug("artifact created")
}
}

View File

@@ -1,632 +0,0 @@
package youtube
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"log/slog"
"math/rand"
"net/http"
"net/url"
"strconv"
"sync/atomic"
)
const (
Size1Kb = 1024
Size1Mb = Size1Kb * 1024
Size10Mb = Size1Mb * 10
playerParams = "CgIQBg=="
)
var ErrNoFormat = errors.New("no video format provided")
// DefaultClient type to use. No reason to change but you could if you wanted to.
var DefaultClient = IOSClient
// Client offers methods to download video metadata and video streams.
type Client struct {
// HTTPClient can be used to set a custom HTTP client.
// If not set, http.DefaultClient will be used
HTTPClient *http.Client
// MaxRoutines to use when downloading a video.
MaxRoutines int
// ChunkSize to use when downloading videos in chunks. Default is Size10Mb.
ChunkSize int64
// playerCache caches the JavaScript code of a player response
playerCache playerCache
client *clientInfo
consentID string
}
func (c *Client) assureClient() {
if c.client == nil {
c.client = &DefaultClient
}
}
// GetVideo fetches video metadata
func (c *Client) GetVideo(url string) (*Video, error) {
return c.GetVideoContext(context.Background(), url)
}
// GetVideoContext fetches video metadata with a context
func (c *Client) GetVideoContext(ctx context.Context, url string) (*Video, error) {
id, err := ExtractVideoID(url)
if err != nil {
return nil, fmt.Errorf("extractVideoID failed: %w", err)
}
return c.videoFromID(ctx, id)
}
func (c *Client) videoFromID(ctx context.Context, id string) (*Video, error) {
c.assureClient()
body, err := c.videoDataByInnertube(ctx, id)
if err != nil {
return nil, err
}
v := Video{
ID: id,
}
// return early if all good
if err = v.parseVideoInfo(body); err == nil {
return &v, nil
}
// If the uploader has disabled embedding the video on other sites, parse video page
if errors.Is(err, ErrNotPlayableInEmbed) {
// additional parameters are required to access clips with sensitiv content
html, err := c.httpGetBodyBytes(ctx, "https://www.youtube.com/watch?v="+id+"&bpctr=9999999999&has_verified=1")
if err != nil {
return nil, err
}
return &v, v.parseVideoPage(html)
}
// If the uploader marked the video as inappropriate for some ages, use embed player
if errors.Is(err, ErrLoginRequired) {
c.client = &EmbeddedClient
bodyEmbed, errEmbed := c.videoDataByInnertube(ctx, id)
if errEmbed == nil {
errEmbed = v.parseVideoInfo(bodyEmbed)
}
if errEmbed == nil {
return &v, nil
}
// private video clearly not age-restricted and thus should be explicit
if errEmbed == ErrVideoPrivate {
return &v, errEmbed
}
// wrapping error so its clear whats happened
return &v, fmt.Errorf("can't bypass age restriction: %w", errEmbed)
}
// undefined error
return &v, err
}
type innertubeRequest struct {
VideoID string `json:"videoId,omitempty"`
BrowseID string `json:"browseId,omitempty"`
Continuation string `json:"continuation,omitempty"`
Context inntertubeContext `json:"context"`
PlaybackContext *playbackContext `json:"playbackContext,omitempty"`
ContentCheckOK bool `json:"contentCheckOk,omitempty"`
RacyCheckOk bool `json:"racyCheckOk,omitempty"`
Params string `json:"params"`
}
type playbackContext struct {
ContentPlaybackContext contentPlaybackContext `json:"contentPlaybackContext"`
}
type contentPlaybackContext struct {
// SignatureTimestamp string `json:"signatureTimestamp"`
HTML5Preference string `json:"html5Preference"`
}
type inntertubeContext struct {
Client innertubeClient `json:"client"`
}
type innertubeClient struct {
HL string `json:"hl"`
GL string `json:"gl"`
ClientName string `json:"clientName"`
ClientVersion string `json:"clientVersion"`
AndroidSDKVersion int `json:"androidSDKVersion,omitempty"`
UserAgent string `json:"userAgent,omitempty"`
TimeZone string `json:"timeZone"`
UTCOffset int `json:"utcOffsetMinutes"`
DeviceModel string `json:"deviceModel,omitempty"`
}
// client info for the innertube API
type clientInfo struct {
name string
key string
version string
userAgent string
androidVersion int
deviceModel string
}
var (
// WebClient, better to use Android client but go ahead.
WebClient = clientInfo{
name: "WEB",
version: "2.20220801.00.00",
key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
}
// AndroidClient, download go brrrrrr.
AndroidClient = clientInfo{
name: "ANDROID",
version: "18.11.34",
key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
userAgent: "com.google.android.youtube/18.11.34 (Linux; U; Android 11) gzip",
androidVersion: 30,
}
// IOSClient Client based brrrr.
IOSClient = clientInfo{
name: "IOS",
version: "19.45.4",
key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
userAgent: "com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)",
deviceModel: "iPhone16,2",
}
// EmbeddedClient, not really tested.
EmbeddedClient = clientInfo{
name: "WEB_EMBEDDED_PLAYER",
version: "1.19700101",
key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", // seems like same key works for both clients
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
}
)
func (c *Client) videoDataByInnertube(ctx context.Context, id string) ([]byte, error) {
data := innertubeRequest{
VideoID: id,
Context: prepareInnertubeContext(*c.client),
ContentCheckOK: true,
RacyCheckOk: true,
// Params: playerParams,
PlaybackContext: &playbackContext{
ContentPlaybackContext: contentPlaybackContext{
// SignatureTimestamp: sts,
HTML5Preference: "HTML5_PREF_WANTS",
},
},
}
return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/player?key="+c.client.key, data)
}
func (c *Client) transcriptDataByInnertube(ctx context.Context, id string, lang string) ([]byte, error) {
data := innertubeRequest{
Context: prepareInnertubeContext(*c.client),
Params: transcriptVideoID(id, lang),
}
return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/get_transcript?key="+c.client.key, data)
}
func prepareInnertubeContext(clientInfo clientInfo) inntertubeContext {
return inntertubeContext{
Client: innertubeClient{
HL: "en",
GL: "US",
TimeZone: "UTC",
DeviceModel: clientInfo.deviceModel,
ClientName: clientInfo.name,
ClientVersion: clientInfo.version,
AndroidSDKVersion: clientInfo.androidVersion,
UserAgent: clientInfo.userAgent,
},
}
}
func prepareInnertubePlaylistData(ID string, continuation bool, clientInfo clientInfo) innertubeRequest {
context := prepareInnertubeContext(clientInfo)
if continuation {
return innertubeRequest{
Context: context,
Continuation: ID,
ContentCheckOK: true,
RacyCheckOk: true,
Params: playerParams,
}
}
return innertubeRequest{
Context: context,
BrowseID: "VL" + ID,
ContentCheckOK: true,
RacyCheckOk: true,
Params: playerParams,
}
}
// transcriptVideoID encodes the video ID to the param used to fetch transcripts.
func transcriptVideoID(videoID string, lang string) string {
langCode := encTranscriptLang(lang)
// This can be optionally appened to the Sprintf str, not sure what it means
// *3engagement-panel-searchable-transcript-search-panel\x30\x00\x38\x01\x40\x01
return base64Enc(fmt.Sprintf("\n\x0b%s\x12\x12%s\x18\x01", videoID, langCode))
}
func encTranscriptLang(languageCode string) string {
s := fmt.Sprintf("\n\x03asr\x12\x02%s\x1a\x00", languageCode)
s = base64PadEnc(s)
return url.QueryEscape(s)
}
// GetPlaylist fetches playlist metadata
func (c *Client) GetPlaylist(url string) (*Playlist, error) {
return c.GetPlaylistContext(context.Background(), url)
}
// GetPlaylistContext fetches playlist metadata, with a context, along with a list of Videos, and some basic information
// for these videos. Playlist entries cannot be downloaded, as they lack all the required metadata, but
// can be used to enumerate all IDs, Authors, Titles, etc.
func (c *Client) GetPlaylistContext(ctx context.Context, url string) (*Playlist, error) {
c.assureClient()
id, err := extractPlaylistID(url)
if err != nil {
return nil, fmt.Errorf("extractPlaylistID failed: %w", err)
}
data := prepareInnertubePlaylistData(id, false, *c.client)
body, err := c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+c.client.key, data)
if err != nil {
return nil, err
}
p := &Playlist{ID: id}
return p, p.parsePlaylistInfo(ctx, c, body)
}
func (c *Client) VideoFromPlaylistEntry(entry *PlaylistEntry) (*Video, error) {
return c.videoFromID(context.Background(), entry.ID)
}
func (c *Client) VideoFromPlaylistEntryContext(ctx context.Context, entry *PlaylistEntry) (*Video, error) {
return c.videoFromID(ctx, entry.ID)
}
// GetStream returns the stream and the total size for a specific format
func (c *Client) GetStream(video *Video, format *Format) (io.ReadCloser, int64, error) {
return c.GetStreamContext(context.Background(), video, format)
}
// GetStreamContext returns the stream and the total size for a specific format with a context.
func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *Format) (io.ReadCloser, int64, error) {
url, err := c.GetStreamURL(video, format)
if err != nil {
return nil, 0, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, 0, err
}
r, w := io.Pipe()
contentLength := format.ContentLength
if contentLength == 0 {
// some videos don't have length information
contentLength = c.downloadOnce(req, w, format)
} else {
// we have length information, let's download by chunks!
c.downloadChunked(ctx, req, w, format)
}
return r, contentLength, nil
}
func (c *Client) downloadOnce(req *http.Request, w *io.PipeWriter, _ *Format) int64 {
resp, err := c.httpDo(req)
if err != nil {
w.CloseWithError(err) //nolint:errcheck
return 0
}
go func() {
defer resp.Body.Close()
_, err := io.Copy(w, resp.Body)
if err == nil {
w.Close()
} else {
w.CloseWithError(err) //nolint:errcheck
}
}()
contentLength := resp.Header.Get("Content-Length")
length, _ := strconv.ParseInt(contentLength, 10, 64)
return length
}
func (c *Client) getChunkSize() int64 {
if c.ChunkSize > 0 {
return c.ChunkSize
}
return Size10Mb
}
func (c *Client) getMaxRoutines(limit int) int {
routines := 10
if c.MaxRoutines > 0 {
routines = c.MaxRoutines
}
if limit > 0 && routines > limit {
routines = limit
}
return routines
}
func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.PipeWriter, format *Format) {
chunks := getChunks(format.ContentLength, c.getChunkSize())
maxRoutines := c.getMaxRoutines(len(chunks))
cancelCtx, cancel := context.WithCancel(ctx)
abort := func(err error) {
w.CloseWithError(err)
cancel()
}
currentChunk := atomic.Uint32{}
for i := 0; i < maxRoutines; i++ {
go func() {
for {
chunkIndex := int(currentChunk.Add(1)) - 1
if chunkIndex >= len(chunks) {
// no more chunks
return
}
chunk := &chunks[chunkIndex]
err := c.downloadChunk(req.Clone(cancelCtx), chunk)
close(chunk.data)
if err != nil {
abort(err)
return
}
}
}()
}
go func() {
// copy chunks into the PipeWriter
for i := 0; i < len(chunks); i++ {
select {
case <-cancelCtx.Done():
abort(context.Canceled)
return
case data := <-chunks[i].data:
_, err := io.Copy(w, bytes.NewBuffer(data))
if err != nil {
abort(err)
}
}
}
// everything succeeded
w.Close()
}()
}
// GetStreamURL returns the url for a specific format
func (c *Client) GetStreamURL(video *Video, format *Format) (string, error) {
return c.GetStreamURLContext(context.Background(), video, format)
}
// GetStreamURLContext returns the url for a specific format with a context
func (c *Client) GetStreamURLContext(ctx context.Context, video *Video, format *Format) (string, error) {
if format == nil {
return "", ErrNoFormat
}
c.assureClient()
if format.URL != "" {
if c.client.androidVersion > 0 {
return format.URL, nil
}
return c.unThrottle(ctx, video.ID, format.URL)
}
// TODO: check rest of this function, is it redundant?
cipher := format.Cipher
if cipher == "" {
return "", ErrCipherNotFound
}
uri, err := c.decipherURL(ctx, video.ID, cipher)
if err != nil {
return "", err
}
return uri, err
}
// httpDo sends an HTTP request and returns an HTTP response.
func (c *Client) httpDo(req *http.Request) (*http.Response, error) {
client := c.HTTPClient
if client == nil {
client = http.DefaultClient
}
req.Header.Set("User-Agent", c.client.userAgent)
req.Header.Set("Origin", "https://youtube.com")
req.Header.Set("Sec-Fetch-Mode", "navigate")
if len(c.consentID) == 0 {
c.consentID = strconv.Itoa(rand.Intn(899) + 100) //nolint:gosec
}
req.AddCookie(&http.Cookie{
Name: "CONSENT",
Value: "YES+cb.20210328-17-p0.en+FX+" + c.consentID,
Path: "/",
Domain: ".youtube.com",
})
res, err := client.Do(req)
log := slog.With("method", req.Method, "url", req.URL)
if err == nil && res.StatusCode != http.StatusOK {
err = ErrUnexpectedStatusCode(res.StatusCode)
res.Body.Close()
res = nil
}
if err != nil {
log.Debug("HTTP request failed", "error", err)
} else {
log.Debug("HTTP request succeeded", "status", res.Status)
}
return res, err
}
// httpGet does a HTTP GET request, checks the response to be a 200 OK and returns it
func (c *Client) httpGet(ctx context.Context, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := c.httpDo(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, ErrUnexpectedStatusCode(resp.StatusCode)
}
return resp, nil
}
// httpGetBodyBytes reads the whole HTTP body and returns it
func (c *Client) httpGetBodyBytes(ctx context.Context, url string) ([]byte, error) {
resp, err := c.httpGet(ctx, url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// httpPost does a HTTP POST request with a body, checks the response to be a 200 OK and returns it
func (c *Client) httpPost(ctx context.Context, url string, body interface{}) (*http.Response, error) {
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return nil, err
}
req.Header.Set("X-Youtube-Client-Name", "3")
req.Header.Set("X-Youtube-Client-Version", c.client.version)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := c.httpDo(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, ErrUnexpectedStatusCode(resp.StatusCode)
}
return resp, nil
}
// httpPostBodyBytes reads the whole HTTP body and returns it
func (c *Client) httpPostBodyBytes(ctx context.Context, url string, body interface{}) ([]byte, error) {
resp, err := c.httpPost(ctx, url, body)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// downloadChunk writes the response data into the data channel of the chunk.
// Downloading in multiple chunks is much faster:
// https://github.com/kkdai/youtube/pull/190
func (c *Client) downloadChunk(req *http.Request, chunk *chunk) error {
q := req.URL.Query()
q.Set("range", fmt.Sprintf("%d-%d", chunk.start, chunk.end))
req.URL.RawQuery = q.Encode()
resp, err := c.httpDo(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return ErrUnexpectedStatusCode(resp.StatusCode)
}
expected := int(chunk.end-chunk.start) + 1
data, err := io.ReadAll(resp.Body)
n := len(data)
if err != nil {
return err
}
if n != expected {
return fmt.Errorf("chunk at offset %d has invalid size: expected=%d actual=%d", chunk.start, expected, n)
}
chunk.data <- data
return nil
}

View File

@@ -1,285 +0,0 @@
package youtube
import (
"bytes"
"context"
"errors"
"fmt"
"net/url"
"regexp"
"strconv"
"github.com/dop251/goja"
)
func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) {
params, err := url.ParseQuery(cipher)
if err != nil {
return "", err
}
uri, err := url.Parse(params.Get("url"))
if err != nil {
return "", err
}
query := uri.Query()
config, err := c.getPlayerConfig(ctx, videoID)
if err != nil {
return "", err
}
// decrypt s-parameter
bs, err := config.decrypt([]byte(params.Get("s")))
if err != nil {
return "", err
}
query.Add(params.Get("sp"), string(bs))
query, err = c.decryptNParam(config, query)
if err != nil {
return "", err
}
uri.RawQuery = query.Encode()
return uri.String(), nil
}
// see https://github.com/kkdai/youtube/pull/244
func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) {
config, err := c.getPlayerConfig(ctx, videoID)
if err != nil {
return "", err
}
uri, err := url.Parse(urlString)
if err != nil {
return "", err
}
// for debugging
if artifactsFolder != "" {
writeArtifact("video-"+videoID+".url", []byte(uri.String()))
}
query, err := c.decryptNParam(config, uri.Query())
if err != nil {
return "", err
}
uri.RawQuery = query.Encode()
return uri.String(), nil
}
func (c *Client) decryptNParam(config playerConfig, query url.Values) (url.Values, error) {
// decrypt n-parameter
nSig := query.Get("v")
log := Logger.With("n", nSig)
if nSig != "" {
nDecoded, err := config.decodeNsig(nSig)
if err != nil {
return nil, fmt.Errorf("unable to decode nSig: %w", err)
}
query.Set("v", nDecoded)
log = log.With("decoded", nDecoded)
}
log.Debug("nParam")
return query, nil
}
const (
jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
reverseStr = ":function\\(a\\)\\{" +
"(?:return )?a\\.reverse\\(\\)" +
"\\}"
spliceStr = ":function\\(a,b\\)\\{" +
"a\\.splice\\(0,b\\)" +
"\\}"
swapStr = ":function\\(a,b\\)\\{" +
"var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
"\\}"
)
var (
nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})")
actionsObjRegexp = regexp.MustCompile(fmt.Sprintf(
"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr))
actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
"function(?: %s)?\\(a\\)\\{"+
"a=a\\.split\\(\"\"\\);\\s*"+
"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
"return a\\.join\\(\"\"\\)"+
"\\}", jsvarStr, jsvarStr, jsvarStr))
reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
spliceRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
swapRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
)
func (config playerConfig) decodeNsig(encoded string) (string, error) {
fBody, err := config.getNFunction()
if err != nil {
return "", err
}
return evalJavascript(fBody, encoded)
}
func evalJavascript(jsFunction, arg string) (string, error) {
const myName = "myFunction"
vm := goja.New()
_, err := vm.RunString(myName + "=" + jsFunction)
if err != nil {
return "", err
}
var output func(string) string
err = vm.ExportTo(vm.Get(myName), &output)
if err != nil {
return "", err
}
return output(arg), nil
}
func (config playerConfig) getNFunction() (string, error) {
nameResult := nFunctionNameRegexp.FindSubmatch(config)
if len(nameResult) == 0 {
return "", errors.New("unable to extract n-function name")
}
var name string
if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 {
name = string(nameResult[4])
} else {
name = string(nameResult[1])
}
return config.extraFunction(name)
}
func (config playerConfig) extraFunction(name string) (string, error) {
// find the beginning of the function
def := []byte(name + "=function(")
start := bytes.Index(config, def)
if start < 1 {
return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def)
}
// start after the first curly bracket
pos := start + bytes.IndexByte(config[start:], '{') + 1
var strChar byte
// find the bracket closing the function
for brackets := 1; brackets > 0; pos++ {
b := config[pos]
switch b {
case '{':
if strChar == 0 {
brackets++
}
case '}':
if strChar == 0 {
brackets--
}
case '`', '"', '\'':
if config[pos-1] == '\\' && config[pos-2] != '\\' {
continue
}
if strChar == 0 {
strChar = b
} else if strChar == b {
strChar = 0
}
}
}
return string(config[start:pos]), nil
}
func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) {
operations, err := config.parseDecipherOps()
if err != nil {
return nil, err
}
// apply operations
bs := []byte(cyphertext)
for _, op := range operations {
bs = op(bs)
}
return bs, nil
}
/*
parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js
var Mt={
splice:function(a,b){a.splice(0,b)},
reverse:function(a){a.reverse()},
EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
a=a.split("");
Mt.splice(a,3);
Mt.EQ(a,39);
Mt.splice(a,2);
Mt.EQ(a,1);
Mt.splice(a,1);
Mt.EQ(a,35);
Mt.EQ(a,51);
Mt.splice(a,2);
Mt.reverse(a,52);
return a.join("")
*/
func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) {
objResult := actionsObjRegexp.FindSubmatch(config)
funcResult := actionsFuncRegexp.FindSubmatch(config)
if len(objResult) < 3 || len(funcResult) < 2 {
return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult))
}
obj := objResult[1]
objBody := objResult[2]
funcBody := funcResult[1]
var reverseKey, spliceKey, swapKey string
if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 {
reverseKey = string(result[1])
}
if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 {
spliceKey = string(result[1])
}
if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 {
swapKey = string(result[1])
}
regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey)))
if err != nil {
return nil, err
}
var ops []DecipherOperation
for _, s := range regex.FindAllSubmatch(funcBody, -1) {
switch string(s[1]) {
case reverseKey:
ops = append(ops, reverseFunc)
case swapKey:
arg, _ := strconv.Atoi(string(s[2]))
ops = append(ops, newSwapFunc(arg))
case spliceKey:
arg, _ := strconv.Atoi(string(s[2]))
ops = append(ops, newSpliceFunc(arg))
}
}
return ops, nil
}

View File

@@ -1,27 +0,0 @@
package youtube
type DecipherOperation func([]byte) []byte
func newSpliceFunc(pos int) DecipherOperation {
return func(bs []byte) []byte {
return bs[pos:]
}
}
func newSwapFunc(arg int) DecipherOperation {
return func(bs []byte) []byte {
pos := arg % len(bs)
bs[0], bs[pos] = bs[pos], bs[0]
return bs
}
}
func reverseFunc(bs []byte) []byte {
l, r := 0, len(bs)-1
for l < r {
bs[l], bs[r] = bs[r], bs[l]
l++
r--
}
return bs
}

View File

@@ -1,4 +0,0 @@
/*
Package youtube implement youtube download package in go.
*/
package youtube

View File

@@ -1,203 +0,0 @@
package downloader
import (
"context"
"errors"
"io"
"log/slog"
"os"
"os/exec"
"path/filepath"
"github.com/kkdai/youtube/v2"
"github.com/vbauerster/mpb/v5"
"github.com/vbauerster/mpb/v5/decor"
)
// Downloader offers high level functions to download videos into files
type Downloader struct {
youtube.Client
OutputDir string // optional directory to store the files
}
func (dl *Downloader) getOutputFile(v *youtube.Video, format *youtube.Format, outputFile string) (string, error) {
if outputFile == "" {
outputFile = SanitizeFilename(v.Title)
outputFile += pickIdealFileExtension(format.MimeType)
}
if dl.OutputDir != "" {
if err := os.MkdirAll(dl.OutputDir, 0o755); err != nil {
return "", err
}
outputFile = filepath.Join(dl.OutputDir, outputFile)
}
return outputFile, nil
}
// Download : Starting download video by arguments.
func (dl *Downloader) Download(ctx context.Context, v *youtube.Video, format *youtube.Format, outputFile string) error {
youtube.Logger.Info(
"Downloading video",
"id", v.ID,
"quality", format.Quality,
"mimeType", format.MimeType,
)
destFile, err := dl.getOutputFile(v, format, outputFile)
if err != nil {
return err
}
// Create output file
out, err := os.Create(destFile)
if err != nil {
return err
}
defer out.Close()
return dl.videoDLWorker(ctx, out, v, format)
}
// DownloadComposite : Downloads audio and video streams separately and merges them via ffmpeg.
func (dl *Downloader) DownloadComposite(ctx context.Context, outputFile string, v *youtube.Video, quality string, mimetype, language string) error {
videoFormat, audioFormat, err1 := getVideoAudioFormats(v, quality, mimetype, language)
if err1 != nil {
return err1
}
log := youtube.Logger.With("id", v.ID)
log.Info(
"Downloading composite video",
"videoQuality", videoFormat.QualityLabel,
"videoMimeType", videoFormat.MimeType,
"audioMimeType", audioFormat.MimeType,
)
destFile, err := dl.getOutputFile(v, videoFormat, outputFile)
if err != nil {
return err
}
outputDir := filepath.Dir(destFile)
// Create temporary video file
videoFile, err := os.CreateTemp(outputDir, "youtube_*.m4v")
if err != nil {
return err
}
defer Delete(videoFile.Name(), log)
// Create temporary audio file
audioFile, err := os.CreateTemp(outputDir, "youtube_*.m4a")
if err != nil {
return err
}
defer Delete(audioFile.Name(), log)
log.Debug("Downloading video file...")
err = dl.videoDLWorker(ctx, videoFile, v, videoFormat)
if err != nil {
return err
}
videoFile.Close()
log.Debug("Downloading audio file...")
err = dl.videoDLWorker(ctx, audioFile, v, audioFormat)
if err != nil {
return err
}
audioFile.Close()
//nolint:gosec
ffmpegVersionCmd := exec.Command("ffmpeg", "-y",
"-i", videoFile.Name(),
"-i", audioFile.Name(),
"-c", "copy", // Just copy without re-encoding
"-shortest", // Finish encoding when the shortest input stream ends
destFile,
"-loglevel", "warning",
)
ffmpegVersionCmd.Stderr = os.Stderr
ffmpegVersionCmd.Stdout = os.Stdout
log.Info("merging video and audio", "output", destFile)
return ffmpegVersionCmd.Run()
}
func Delete(path string, log *slog.Logger) {
err := os.Remove(path)
if err != nil {
log.Error("Failed deleting file", "path", path, "error", err)
}
}
func getVideoAudioFormats(v *youtube.Video, quality string, mimetype, language string) (*youtube.Format, *youtube.Format, error) {
var videoFormats, audioFormats youtube.FormatList
formats := v.Formats
if mimetype != "" {
formats = formats.Type(mimetype)
}
videoFormats = formats.Type("video").AudioChannels(0)
audioFormats = formats.Type("audio")
if quality != "" {
videoFormats = videoFormats.Quality(quality)
}
if language != "" {
audioFormats = audioFormats.Language(language)
}
if len(videoFormats) == 0 {
return nil, nil, errors.New("no video format found after filtering")
}
if len(audioFormats) == 0 {
return nil, nil, errors.New("no audio format found after filtering")
}
videoFormats.Sort()
audioFormats.Sort()
return &videoFormats[0], &audioFormats[0], nil
}
func (dl *Downloader) videoDLWorker(ctx context.Context, out *os.File, video *youtube.Video, format *youtube.Format) error {
stream, size, err := dl.GetStreamContext(ctx, video, format)
if err != nil {
return err
}
prog := &progress{
contentLength: float64(size),
}
// create progress bar
progress := mpb.New(mpb.WithWidth(64))
bar := progress.AddBar(
int64(prog.contentLength),
mpb.PrependDecorators(
decor.CountersKibiByte("% .2f / % .2f"),
decor.Percentage(decor.WCSyncSpace),
),
mpb.AppendDecorators(
decor.EwmaETA(decor.ET_STYLE_GO, 90),
decor.Name(" ] "),
decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60),
),
)
reader := bar.ProxyReader(stream)
mw := io.MultiWriter(out, prog)
_, err = io.Copy(mw, reader)
if err != nil {
return err
}
progress.Wait()
return nil
}

View File

@@ -1,65 +0,0 @@
package downloader
import (
"mime"
"regexp"
)
const defaultExtension = ".mov"
// Rely on hardcoded canonical mime types, as the ones provided by Go aren't exhaustive [1].
// This seems to be a recurring problem for youtube downloaders, see [2].
// The implementation is based on mozilla's list [3], IANA [4] and Youtube's support [5].
// [1] https://github.com/golang/go/blob/ed7888aea6021e25b0ea58bcad3f26da2b139432/src/mime/type.go#L60
// [2] https://github.com/ZiTAL/youtube-dl/blob/master/mime.types
// [3] https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
// [4] https://www.iana.org/assignments/media-types/media-types.xhtml#video
// [5] https://support.google.com/youtube/troubleshooter/2888402?hl=en
var canonicals = map[string]string{
"video/quicktime": ".mov",
"video/x-msvideo": ".avi",
"video/x-matroska": ".mkv",
"video/mpeg": ".mpeg",
"video/webm": ".webm",
"video/3gpp2": ".3g2",
"video/x-flv": ".flv",
"video/3gpp": ".3gp",
"video/mp4": ".mp4",
"video/ogg": ".ogv",
"video/mp2t": ".ts",
}
func pickIdealFileExtension(mediaType string) string {
mediaType, _, err := mime.ParseMediaType(mediaType)
if err != nil {
return defaultExtension
}
if extension, ok := canonicals[mediaType]; ok {
return extension
}
// Our last resort is to ask the operating system, but these give multiple results and are rarely canonical.
extensions, err := mime.ExtensionsByType(mediaType)
if err != nil || extensions == nil {
return defaultExtension
}
return extensions[0]
}
func SanitizeFilename(fileName string) string {
// Characters not allowed on mac
// :/
// Characters not allowed on linux
// /
// Characters not allowed on windows
// <>:"/\|?*
// Ref https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
fileName = regexp.MustCompile(`[:/<>\:"\\|?*]`).ReplaceAllString(fileName, "")
fileName = regexp.MustCompile(`\s+`).ReplaceAllString(fileName, " ")
return fileName
}

View File

@@ -1,17 +0,0 @@
package downloader
type progress struct {
contentLength float64
totalWrittenBytes float64
downloadLevel float64
}
func (dl *progress) Write(p []byte) (n int, err error) {
n = len(p)
dl.totalWrittenBytes = dl.totalWrittenBytes + float64(n)
currentPercent := (dl.totalWrittenBytes / dl.contentLength) * 100
if (dl.downloadLevel <= currentPercent) && (dl.downloadLevel < 100) {
dl.downloadLevel++
}
return
}

View File

@@ -1,47 +0,0 @@
package youtube
import (
"fmt"
)
const (
ErrCipherNotFound = constError("cipher not found")
ErrSignatureTimestampNotFound = constError("signature timestamp not found")
ErrInvalidCharactersInVideoID = constError("invalid characters in video id")
ErrVideoIDMinLength = constError("the video id must be at least 10 characters long")
ErrReadOnClosedResBody = constError("http: read on closed response body")
ErrNotPlayableInEmbed = constError("embedding of this video has been disabled")
ErrLoginRequired = constError("login required to confirm your age")
ErrVideoPrivate = constError("user restricted access to this video")
ErrInvalidPlaylist = constError("no playlist detected or invalid playlist ID")
)
type constError string
func (e constError) Error() string {
return string(e)
}
type ErrPlayabiltyStatus struct {
Status string
Reason string
}
func (err ErrPlayabiltyStatus) Error() string {
return fmt.Sprintf("cannot playback and download, status: %s, reason: %s", err.Status, err.Reason)
}
// ErrUnexpectedStatusCode is returned on unexpected HTTP status codes
type ErrUnexpectedStatusCode int
func (err ErrUnexpectedStatusCode) Error() string {
return fmt.Sprintf("unexpected status code: %d", err)
}
type ErrPlaylistStatus struct {
Reason string
}
func (err ErrPlaylistStatus) Error() string {
return fmt.Sprintf("could not load playlist: %s", err.Reason)
}

View File

@@ -1,35 +0,0 @@
//go:build fetch
// +build fetch
package youtube
import (
"fmt"
"io"
"net/http"
"os"
)
func init() {
FetchTestData()
}
// ran via go generate to fetch and update the playlist response data
func FetchTestData() {
f, err := os.Create(testPlaylistResponseDataFile)
exitOnError(err)
requestURL := fmt.Sprintf(playlistFetchURL, testPlaylistID)
resp, err := http.Get(requestURL)
exitOnError(err)
defer resp.Body.Close()
n, err := io.Copy(f, resp.Body)
exitOnError(err)
fmt.Printf("Successfully fetched playlist %s (%d bytes)\n", testPlaylistID, n)
}
func exitOnError(err error) {
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}

View File

@@ -1,148 +0,0 @@
package youtube
import (
"sort"
"strconv"
"strings"
)
type FormatList []Format
// Type returns a new FormatList filtered by itag
func (list FormatList) Select(f func(Format) bool) (result FormatList) {
for i := range list {
if f(list[i]) {
result = append(result, list[i])
}
}
return result
}
// Type returns a new FormatList filtered by itag
func (list FormatList) Itag(itagNo int) FormatList {
return list.Select(func(f Format) bool {
return f.ItagNo == itagNo
})
}
// Type returns a new FormatList filtered by mime type
func (list FormatList) Type(value string) FormatList {
return list.Select(func(f Format) bool {
return strings.Contains(f.MimeType, value)
})
}
// Type returns a new FormatList filtered by display name
func (list FormatList) Language(displayName string) FormatList {
return list.Select(func(f Format) bool {
return f.LanguageDisplayName() == displayName
})
}
// Quality returns a new FormatList filtered by quality, quality label or itag,
// but not audio quality
func (list FormatList) Quality(quality string) FormatList {
itag, _ := strconv.Atoi(quality)
return list.Select(func(f Format) bool {
return itag == f.ItagNo || strings.Contains(f.Quality, quality) || strings.Contains(f.QualityLabel, quality)
})
}
// AudioChannels returns a new FormatList filtered by the matching AudioChannels
func (list FormatList) AudioChannels(n int) FormatList {
return list.Select(func(f Format) bool {
return f.AudioChannels == n
})
}
// AudioChannels returns a new FormatList filtered by the matching AudioChannels
func (list FormatList) WithAudioChannels() FormatList {
return list.Select(func(f Format) bool {
return f.AudioChannels > 0
})
}
// FilterQuality reduces the format list to formats matching the quality
func (v *Video) FilterQuality(quality string) {
v.Formats = v.Formats.Quality(quality)
v.Formats.Sort()
}
// Sort sorts all formats fields
func (list FormatList) Sort() {
sort.SliceStable(list, func(i, j int) bool {
return sortFormat(i, j, list)
})
}
// sortFormat sorts video by resolution, FPS, codec (av01, vp9, avc1), bitrate
// sorts audio by default, codec (mp4, opus), channels, bitrate, sample rate
func sortFormat(i int, j int, formats FormatList) bool {
// Sort by Width
if formats[i].Width == formats[j].Width {
// Format 137 downloads slowly, give it less priority
// see https://github.com/kkdai/youtube/pull/171
switch 137 {
case formats[i].ItagNo:
return false
case formats[j].ItagNo:
return true
}
// Sort by FPS
if formats[i].FPS == formats[j].FPS {
if formats[i].FPS == 0 && formats[i].AudioChannels > 0 && formats[j].AudioChannels > 0 {
// Audio
// Sort by default
if (formats[i].AudioTrack == nil && formats[j].AudioTrack == nil) || (formats[i].AudioTrack != nil && formats[j].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault == formats[j].AudioTrack.AudioIsDefault) {
// Sort by codec
codec := map[int]int{}
for _, index := range []int{i, j} {
if strings.Contains(formats[index].MimeType, "mp4") {
codec[index] = 1
} else if strings.Contains(formats[index].MimeType, "opus") {
codec[index] = 2
}
}
if codec[i] == codec[j] {
// Sort by Audio Channel
if formats[i].AudioChannels == formats[j].AudioChannels {
// Sort by Audio Bitrate
if formats[i].Bitrate == formats[j].Bitrate {
// Sort by Audio Sample Rate
return formats[i].AudioSampleRate > formats[j].AudioSampleRate
}
return formats[i].Bitrate > formats[j].Bitrate
}
return formats[i].AudioChannels > formats[j].AudioChannels
}
return codec[i] < codec[j]
} else if formats[i].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault {
return true
}
return false
}
// Video
// Sort by codec
codec := map[int]int{}
for _, index := range []int{i, j} {
if strings.Contains(formats[index].MimeType, "av01") {
codec[index] = 1
} else if strings.Contains(formats[index].MimeType, "vp9") {
codec[index] = 2
} else if strings.Contains(formats[index].MimeType, "avc1") {
codec[index] = 3
}
}
if codec[i] == codec[j] {
// Sort by Audio Bitrate
return formats[i].Bitrate > formats[j].Bitrate
}
return codec[i] < codec[j]
}
return formats[i].FPS > formats[j].FPS
}
return formats[i].Width > formats[j].Width
}

View File

@@ -1,28 +0,0 @@
package youtube
import (
"fmt"
"log/slog"
"os"
)
// The global logger for all Client instances
var Logger = getLogger(os.Getenv("LOGLEVEL"))
func SetLogLevel(value string) {
Logger = getLogger(value)
}
func getLogger(logLevel string) *slog.Logger {
levelVar := slog.LevelVar{}
if logLevel != "" {
if err := levelVar.UnmarshalText([]byte(logLevel)); err != nil {
panic(fmt.Sprintf("Invalid log level %s: %v", logLevel, err))
}
}
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
Level: levelVar.Level(),
}))
}

View File

@@ -1,37 +0,0 @@
package youtube
import (
"time"
)
const defaultCacheExpiration = time.Minute * time.Duration(5)
type playerCache struct {
key string
expiredAt time.Time
config playerConfig
}
// Get : get cache when it has same video id and not expired
func (s playerCache) Get(key string) playerConfig {
return s.GetCacheBefore(key, time.Now())
}
// GetCacheBefore : can pass time for testing
func (s playerCache) GetCacheBefore(key string, time time.Time) playerConfig {
if key == s.key && s.expiredAt.After(time) {
return s.config
}
return nil
}
// Set : set cache with default expiration
func (s *playerCache) Set(key string, operations playerConfig) {
s.setWithExpiredTime(key, operations, time.Now().Add(defaultCacheExpiration))
}
func (s *playerCache) setWithExpiredTime(key string, config playerConfig, time time.Time) {
s.key = key
s.config = config
s.expiredAt = time
}

View File

@@ -1,59 +0,0 @@
package youtube
import (
"context"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"strings"
)
type playerConfig []byte
var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`)
func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) {
embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID)
embedBody, err := c.httpGetBodyBytes(ctx, embedURL)
if err != nil {
return nil, err
}
// example: /s/player/f676c671/player_ias.vflset/en_US/base.js
playerPath := string(basejsPattern.Find(embedBody))
if playerPath == "" {
return nil, errors.New("unable to find basejs URL in playerConfig")
}
// for debugging
var artifactName string
if artifactsFolder != "" {
parts := strings.SplitN(playerPath, "/", 5)
artifactName = "player-" + parts[3] + ".js"
linkName := filepath.Join(artifactsFolder, "video-"+videoID+".js")
if err := os.Symlink(artifactName, linkName); err != nil {
log.Printf("unable to create symlink %s: %v", linkName, err)
}
}
config := c.playerCache.Get(playerPath)
if config != nil {
return config, nil
}
config, err = c.httpGetBodyBytes(ctx, "https://youtube.com"+playerPath)
if err != nil {
return nil, err
}
// for debugging
if artifactName != "" {
writeArtifact(artifactName, config)
}
c.playerCache.Set(playerPath, config)
return config, nil
}

View File

@@ -1,256 +0,0 @@
package youtube
import (
"context"
"encoding/json"
"fmt"
"regexp"
"runtime/debug"
"strconv"
"time"
sjson "github.com/bitly/go-simplejson"
)
var (
playlistIDRegex = regexp.MustCompile("^[A-Za-z0-9_-]{13,42}$")
playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{13,42})(&.*)?$")
)
type Playlist struct {
ID string
Title string
Description string
Author string
Videos []*PlaylistEntry
}
type PlaylistEntry struct {
ID string
Title string
Author string
Duration time.Duration
Thumbnails Thumbnails
}
func extractPlaylistID(url string) (string, error) {
if playlistIDRegex.Match([]byte(url)) {
return url, nil
}
matches := playlistInURLRegex.FindStringSubmatch(url)
if matches != nil {
return matches[1], nil
}
return "", ErrInvalidPlaylist
}
// structs for playlist extraction
// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text
// Description: metadata.playlistMetadataRenderer.description
// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text
// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents
// ID: .videoId
// Title: title.runs[0].text
// Author: .shortBylineText.runs[0].text
// Duration: .lengthSeconds
// Thumbnails .thumbnails
// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails
func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body []byte) (err error) {
var j *sjson.Json
j, err = sjson.NewJson(body)
if err != nil {
return err
}
defer func() {
stack := debug.Stack()
if r := recover(); r != nil {
err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack)
}
}()
renderer := j.GetPath("alerts").GetIndex(0).GetPath("alertRenderer")
if renderer != nil && renderer.GetPath("type").MustString() == "ERROR" {
message := renderer.GetPath("text", "runs").GetIndex(0).GetPath("text").MustString()
return ErrPlaylistStatus{Reason: message}
}
// Metadata can be located in multiple places depending on client type
var metadata *sjson.Json
if node, ok := j.CheckGet("metadata"); ok {
metadata = node
} else if node, ok := j.CheckGet("header"); ok {
metadata = node
} else {
return fmt.Errorf("no playlist header / metadata found")
}
metadata = metadata.Get("playlistHeaderRenderer")
p.Title = sjsonGetText(metadata, "title")
p.Description = sjsonGetText(metadata, "description", "descriptionText")
p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1).
GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs").
GetIndex(0).Get("text").MustString()
if len(p.Author) == 0 {
p.Author = sjsonGetText(metadata, "owner", "ownerText")
}
contents, ok := j.CheckGet("contents")
if !ok {
return fmt.Errorf("contents not found in json body")
}
// contents can have different keys with same child structure
firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0).
GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0)
// This extra nested item is only set with the web client
if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) {
firstPart = n
}
vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON()
if err != nil {
return err
}
if len(vJSON) <= 4 {
return fmt.Errorf("no video data found in JSON")
}
entries, continuation, err := extractPlaylistEntries(vJSON)
if err != nil {
return err
}
if len(continuation) == 0 {
continuation = getContinuation(firstPart.Get("playlistVideoListRenderer"))
}
if len(entries) == 0 {
return fmt.Errorf("no videos found in playlist")
}
p.Videos = entries
for continuation != "" {
data := prepareInnertubePlaylistData(continuation, true, *client.client)
body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data)
if err != nil {
return err
}
j, err := sjson.NewJson(body)
if err != nil {
return err
}
next := j.GetPath("onResponseReceivedActions").GetIndex(0).
GetPath("appendContinuationItemsAction", "continuationItems")
if !isValidJSON(next) {
next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents")
}
vJSON, err := next.MarshalJSON()
if err != nil {
return err
}
entries, token, err := extractPlaylistEntries(vJSON)
if err != nil {
return err
}
if len(token) > 0 {
continuation = token
} else {
continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation"))
}
p.Videos = append(p.Videos, entries...)
}
return err
}
func extractPlaylistEntries(data []byte) ([]*PlaylistEntry, string, error) {
var vids []*videosJSONExtractor
if err := json.Unmarshal(data, &vids); err != nil {
return nil, "", err
}
entries := make([]*PlaylistEntry, 0, len(vids))
var continuation string
for _, v := range vids {
if v.Renderer == nil {
if v.Continuation.Endpoint.Command.Token != "" {
continuation = v.Continuation.Endpoint.Command.Token
}
continue
}
entries = append(entries, v.PlaylistEntry())
}
return entries, continuation, nil
}
type videosJSONExtractor struct {
Renderer *struct {
ID string `json:"videoId"`
Title withRuns `json:"title"`
Author withRuns `json:"shortBylineText"`
Duration string `json:"lengthSeconds"`
Thumbnail struct {
Thumbnails []Thumbnail `json:"thumbnails"`
} `json:"thumbnail"`
} `json:"playlistVideoRenderer"`
Continuation struct {
Endpoint struct {
Command struct {
Token string `json:"token"`
} `json:"continuationCommand"`
} `json:"continuationEndpoint"`
} `json:"continuationItemRenderer"`
}
func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry {
ds, err := strconv.Atoi(vje.Renderer.Duration)
if err != nil {
panic("invalid video duration: " + vje.Renderer.Duration)
}
return &PlaylistEntry{
ID: vje.Renderer.ID,
Title: vje.Renderer.Title.String(),
Author: vje.Renderer.Author.String(),
Duration: time.Second * time.Duration(ds),
Thumbnails: vje.Renderer.Thumbnail.Thumbnails,
}
}
type withRuns struct {
Runs []struct {
Text string `json:"text"`
} `json:"runs"`
}
func (wr withRuns) String() string {
if len(wr.Runs) > 0 {
return wr.Runs[0].Text
}
return ""
}

View File

@@ -1,153 +0,0 @@
package youtube
type playerResponseData struct {
Captions struct {
PlayerCaptionsTracklistRenderer struct {
CaptionTracks []CaptionTrack `json:"captionTracks"`
AudioTracks []struct {
CaptionTrackIndices []int `json:"captionTrackIndices"`
} `json:"audioTracks"`
TranslationLanguages []struct {
LanguageCode string `json:"languageCode"`
LanguageName struct {
SimpleText string `json:"simpleText"`
} `json:"languageName"`
} `json:"translationLanguages"`
DefaultAudioTrackIndex int `json:"defaultAudioTrackIndex"`
} `json:"playerCaptionsTracklistRenderer"`
} `json:"captions"`
PlayabilityStatus struct {
Status string `json:"status"`
Reason string `json:"reason"`
PlayableInEmbed bool `json:"playableInEmbed"`
Miniplayer struct {
MiniplayerRenderer struct {
PlaybackMode string `json:"playbackMode"`
} `json:"miniplayerRenderer"`
} `json:"miniplayer"`
ContextParams string `json:"contextParams"`
} `json:"playabilityStatus"`
StreamingData struct {
ExpiresInSeconds string `json:"expiresInSeconds"`
Formats []Format `json:"formats"`
AdaptiveFormats []Format `json:"adaptiveFormats"`
DashManifestURL string `json:"dashManifestUrl"`
HlsManifestURL string `json:"hlsManifestUrl"`
} `json:"streamingData"`
VideoDetails struct {
VideoID string `json:"videoId"`
Title string `json:"title"`
LengthSeconds string `json:"lengthSeconds"`
Keywords []string `json:"keywords"`
ChannelID string `json:"channelId"`
IsOwnerViewing bool `json:"isOwnerViewing"`
ShortDescription string `json:"shortDescription"`
IsCrawlable bool `json:"isCrawlable"`
Thumbnail struct {
Thumbnails []Thumbnail `json:"thumbnails"`
} `json:"thumbnail"`
AverageRating float64 `json:"averageRating"`
AllowRatings bool `json:"allowRatings"`
ViewCount string `json:"viewCount"`
Author string `json:"author"`
IsPrivate bool `json:"isPrivate"`
IsUnpluggedCorpus bool `json:"isUnpluggedCorpus"`
IsLiveContent bool `json:"isLiveContent"`
} `json:"videoDetails"`
Microformat struct {
PlayerMicroformatRenderer struct {
Thumbnail struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"thumbnail"`
Title struct {
SimpleText string `json:"simpleText"`
} `json:"title"`
Description struct {
SimpleText string `json:"simpleText"`
} `json:"description"`
LengthSeconds string `json:"lengthSeconds"`
OwnerProfileURL string `json:"ownerProfileUrl"`
ExternalChannelID string `json:"externalChannelId"`
IsFamilySafe bool `json:"isFamilySafe"`
AvailableCountries []string `json:"availableCountries"`
IsUnlisted bool `json:"isUnlisted"`
HasYpcMetadata bool `json:"hasYpcMetadata"`
ViewCount string `json:"viewCount"`
Category string `json:"category"`
PublishDate string `json:"publishDate"`
OwnerChannelName string `json:"ownerChannelName"`
UploadDate string `json:"uploadDate"`
} `json:"playerMicroformatRenderer"`
} `json:"microformat"`
}
type Format struct {
ItagNo int `json:"itag"`
URL string `json:"url"`
MimeType string `json:"mimeType"`
Quality string `json:"quality"`
Cipher string `json:"signatureCipher"`
Bitrate int `json:"bitrate"`
FPS int `json:"fps"`
Width int `json:"width"`
Height int `json:"height"`
LastModified string `json:"lastModified"`
ContentLength int64 `json:"contentLength,string"`
QualityLabel string `json:"qualityLabel"`
ProjectionType string `json:"projectionType"`
AverageBitrate int `json:"averageBitrate"`
AudioQuality string `json:"audioQuality"`
ApproxDurationMs string `json:"approxDurationMs"`
AudioSampleRate string `json:"audioSampleRate"`
AudioChannels int `json:"audioChannels"`
// InitRange is only available for adaptive formats
InitRange *struct {
Start string `json:"start"`
End string `json:"end"`
} `json:"initRange"`
// IndexRange is only available for adaptive formats
IndexRange *struct {
Start string `json:"start"`
End string `json:"end"`
} `json:"indexRange"`
// AudioTrack is only available for videos with multiple audio track languages
AudioTrack *struct {
DisplayName string `json:"displayName"`
ID string `json:"id"`
AudioIsDefault bool `json:"audioIsDefault"`
}
}
func (f *Format) LanguageDisplayName() string {
if f.AudioTrack == nil {
return ""
}
return f.AudioTrack.DisplayName
}
type Thumbnails []Thumbnail
type Thumbnail struct {
URL string
Width uint
Height uint
}
type CaptionTrack struct {
BaseURL string `json:"baseUrl"`
Name struct {
SimpleText string `json:"simpleText"`
} `json:"name"`
VssID string `json:"vssId"`
LanguageCode string `json:"languageCode"`
Kind string `json:"kind,omitempty"`
IsTranslatable bool `json:"isTranslatable"`
}

View File

@@ -1,214 +0,0 @@
package youtube
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)
var (
ErrTranscriptDisabled = errors.New("transcript is disabled on this video")
)
// TranscriptSegment is a single transcipt segment spanning a few milliseconds.
type TranscriptSegment struct {
// Text is the transcipt text.
Text string `json:"text"`
// StartMs is the start timestamp in ms.
StartMs int `json:"offset"`
// OffsetText e.g. '4:00'.
OffsetText string `json:"offsetText"`
// Duration the transcript segment spans in ms.
Duration int `json:"duration"`
}
func (tr TranscriptSegment) String() string {
return tr.OffsetText + " - " + strings.TrimSpace(tr.Text)
}
type VideoTranscript []TranscriptSegment
func (vt VideoTranscript) String() string {
var str string
for _, tr := range vt {
str += tr.String() + "\n"
}
return str
}
// GetTranscript fetches the video transcript if available.
//
// Not all videos have transcripts, only relatively new videos.
// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
func (c *Client) GetTranscript(video *Video, lang string) (VideoTranscript, error) {
return c.GetTranscriptCtx(context.Background(), video, lang)
}
// GetTranscriptCtx fetches the video transcript if available.
//
// Not all videos have transcripts, only relatively new videos.
// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
func (c *Client) GetTranscriptCtx(ctx context.Context, video *Video, lang string) (VideoTranscript, error) {
c.assureClient()
if video == nil || video.ID == "" {
return nil, fmt.Errorf("no video provided")
}
body, err := c.transcriptDataByInnertube(ctx, video.ID, lang)
if err != nil {
return nil, err
}
transcript, err := parseTranscript(body)
if err != nil {
return nil, err
}
return transcript, nil
}
func parseTranscript(body []byte) (VideoTranscript, error) {
var resp transcriptResp
if err := json.Unmarshal(body, &resp); err != nil {
return nil, err
}
if len(resp.Actions) > 0 {
// Android client response
if app := resp.Actions[0].AppSegment; app != nil {
return getSegments(app)
}
// Web client response
if web := resp.Actions[0].WebSegment; web != nil {
return nil, fmt.Errorf("not implemented")
}
}
return nil, ErrTranscriptDisabled
}
type segmenter interface {
ParseSegments() []TranscriptSegment
}
func getSegments(f segmenter) (VideoTranscript, error) {
if segments := f.ParseSegments(); len(segments) > 0 {
return segments, nil
}
return nil, ErrTranscriptDisabled
}
// transcriptResp is the JSON structure as returned by the transcript API.
type transcriptResp struct {
Actions []struct {
AppSegment *appData `json:"elementsCommand"`
WebSegment *webData `json:"updateEngagementPanelAction"`
} `json:"actions"`
}
type appData struct {
TEC struct {
Args struct {
ListArgs struct {
Ow struct {
InitialSeg []struct {
TranscriptSegment struct {
StartMs string `json:"startMs"`
EndMs string `json:"endMs"`
Text struct {
String struct {
// Content is the actual transctipt text
Content string `json:"content"`
} `json:"elementsAttributedString"`
} `json:"snippet"`
StartTimeText struct {
String struct {
// Content is the fomratted timestamp, e.g. '4:00'
Content string `json:"content"`
} `json:"elementsAttributedString"`
} `json:"startTimeText"`
} `json:"transcriptSegmentRenderer"`
} `json:"initialSegments"`
} `json:"overwrite"`
} `json:"transformTranscriptSegmentListArguments"`
} `json:"arguments"`
} `json:"transformEntityCommand"`
}
func (s *appData) ParseSegments() []TranscriptSegment {
rawSegments := s.TEC.Args.ListArgs.Ow.InitialSeg
segments := make([]TranscriptSegment, 0, len(rawSegments))
for _, segment := range rawSegments {
startMs, _ := strconv.Atoi(segment.TranscriptSegment.StartMs)
endMs, _ := strconv.Atoi(segment.TranscriptSegment.EndMs)
segments = append(segments, TranscriptSegment{
Text: segment.TranscriptSegment.Text.String.Content,
StartMs: startMs,
OffsetText: segment.TranscriptSegment.StartTimeText.String.Content,
Duration: endMs - startMs,
})
}
return segments
}
type webData struct {
Content struct {
TR struct {
Body struct {
TBR struct {
Cues []struct {
Transcript struct {
FormattedStartOffset struct {
SimpleText string `json:"simpleText"`
} `json:"formattedStartOffset"`
Cues []struct {
TranscriptCueRenderer struct {
Cue struct {
SimpleText string `json:"simpleText"`
} `json:"cue"`
StartOffsetMs string `json:"startOffsetMs"`
DurationMs string `json:"durationMs"`
} `json:"transcriptCueRenderer"`
} `json:"cues"`
} `json:"transcriptCueGroupRenderer"`
} `json:"cueGroups"`
} `json:"transcriptSearchPanelRenderer"`
} `json:"content"`
} `json:"transcriptRenderer"`
} `json:"content"`
}
func (s *webData) ParseSegments() []TranscriptSegment {
// TODO: doesn't actually work now, check json.
cues := s.Content.TR.Body.TBR.Cues
segments := make([]TranscriptSegment, 0, len(cues))
for _, s := range cues {
formatted := s.Transcript.FormattedStartOffset.SimpleText
segment := s.Transcript.Cues[0].TranscriptCueRenderer
start, _ := strconv.Atoi(segment.StartOffsetMs)
duration, _ := strconv.Atoi(segment.DurationMs)
segments = append(segments, TranscriptSegment{
Text: segment.Cue.SimpleText,
StartMs: start,
OffsetText: formatted,
Duration: duration,
})
}
return segments
}

View File

@@ -1,97 +0,0 @@
package youtube
import (
"encoding/base64"
sjson "github.com/bitly/go-simplejson"
)
type chunk struct {
start int64
end int64
data chan []byte
}
func getChunks(totalSize, chunkSize int64) []chunk {
var chunks []chunk
for start := int64(0); start < totalSize; start += chunkSize {
end := chunkSize + start - 1
if end > totalSize-1 {
end = totalSize - 1
}
chunks = append(chunks, chunk{start, end, make(chan []byte, 1)})
}
return chunks
}
func getFirstKeyJSON(j *sjson.Json) *sjson.Json {
m, err := j.Map()
if err != nil {
return j
}
for key := range m {
return j.Get(key)
}
return j
}
func isValidJSON(j *sjson.Json) bool {
b, err := j.MarshalJSON()
if err != nil {
return false
}
if len(b) <= 4 {
return false
}
return true
}
func sjsonGetText(j *sjson.Json, paths ...string) string {
for _, path := range paths {
if isValidJSON(j.Get(path)) {
j = j.Get(path)
}
}
if text, err := j.String(); err == nil {
return text
}
if isValidJSON(j.Get("text")) {
return j.Get("text").MustString()
}
if p := j.Get("runs"); isValidJSON(p) {
var text string
for i := 0; i < len(p.MustArray()); i++ {
if textNode := p.GetIndex(i).Get("text"); isValidJSON(textNode) {
text += textNode.MustString()
}
}
return text
}
return ""
}
func getContinuation(j *sjson.Json) string {
return j.GetPath("continuations").
GetIndex(0).GetPath("nextContinuationData", "continuation").MustString()
}
func base64PadEnc(str string) string {
return base64.StdEncoding.EncodeToString([]byte(str))
}
func base64Enc(str string) string {
return base64.RawStdEncoding.EncodeToString([]byte(str))
}

View File

@@ -1,147 +0,0 @@
package youtube
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
type Video struct {
ID string
Title string
Description string
Author string
ChannelID string
ChannelHandle string
Views int
Duration time.Duration
PublishDate time.Time
Formats FormatList
Thumbnails Thumbnails
DASHManifestURL string // URI of the DASH manifest file
HLSManifestURL string // URI of the HLS manifest file
CaptionTracks []CaptionTrack
}
const dateFormat = "2006-01-02"
func (v *Video) parseVideoInfo(body []byte) error {
var prData playerResponseData
if err := json.Unmarshal(body, &prData); err != nil {
return fmt.Errorf("unable to parse player response JSON: %w", err)
}
if err := v.isVideoFromInfoDownloadable(prData); err != nil {
return err
}
return v.extractDataFromPlayerResponse(prData)
}
func (v *Video) isVideoFromInfoDownloadable(prData playerResponseData) error {
return v.isVideoDownloadable(prData, false)
}
var playerResponsePattern = regexp.MustCompile(`var ytInitialPlayerResponse\s*=\s*(\{.+?\});`)
func (v *Video) parseVideoPage(body []byte) error {
initialPlayerResponse := playerResponsePattern.FindSubmatch(body)
if initialPlayerResponse == nil || len(initialPlayerResponse) < 2 {
return errors.New("no ytInitialPlayerResponse found in the server's answer")
}
var prData playerResponseData
if err := json.Unmarshal(initialPlayerResponse[1], &prData); err != nil {
return fmt.Errorf("unable to parse player response JSON: %w", err)
}
if err := v.isVideoFromPageDownloadable(prData); err != nil {
return err
}
return v.extractDataFromPlayerResponse(prData)
}
func (v *Video) isVideoFromPageDownloadable(prData playerResponseData) error {
return v.isVideoDownloadable(prData, true)
}
func (v *Video) isVideoDownloadable(prData playerResponseData, isVideoPage bool) error {
// Check if video is downloadable
switch prData.PlayabilityStatus.Status {
case "OK":
return nil
case "LOGIN_REQUIRED":
// for some reason they use same status message for age-restricted and private videos
if strings.HasPrefix(prData.PlayabilityStatus.Reason, "This video is private") {
return ErrVideoPrivate
}
return ErrLoginRequired
}
if !isVideoPage && !prData.PlayabilityStatus.PlayableInEmbed {
return ErrNotPlayableInEmbed
}
return &ErrPlayabiltyStatus{
Status: prData.PlayabilityStatus.Status,
Reason: prData.PlayabilityStatus.Reason,
}
}
func (v *Video) extractDataFromPlayerResponse(prData playerResponseData) error {
v.Title = prData.VideoDetails.Title
v.Description = prData.VideoDetails.ShortDescription
v.Author = prData.VideoDetails.Author
v.Thumbnails = prData.VideoDetails.Thumbnail.Thumbnails
v.ChannelID = prData.VideoDetails.ChannelID
v.CaptionTracks = prData.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks
if views, _ := strconv.Atoi(prData.VideoDetails.ViewCount); views > 0 {
v.Views = views
}
if seconds, _ := strconv.Atoi(prData.VideoDetails.LengthSeconds); seconds > 0 {
v.Duration = time.Duration(seconds) * time.Second
}
if seconds, _ := strconv.Atoi(prData.Microformat.PlayerMicroformatRenderer.LengthSeconds); seconds > 0 {
v.Duration = time.Duration(seconds) * time.Second
}
if str := prData.Microformat.PlayerMicroformatRenderer.PublishDate; str != "" {
v.PublishDate, _ = time.Parse(dateFormat, str)
}
if profileURL, err := url.Parse(prData.Microformat.PlayerMicroformatRenderer.OwnerProfileURL); err == nil && len(profileURL.Path) > 1 {
v.ChannelHandle = profileURL.Path[1:]
}
// Assign Streams
v.Formats = append(prData.StreamingData.Formats, prData.StreamingData.AdaptiveFormats...)
if len(v.Formats) == 0 {
return errors.New("no formats found in the server's answer")
}
// Sort formats by bitrate
sort.SliceStable(v.Formats, v.SortBitrateDesc)
v.HLSManifestURL = prData.StreamingData.HlsManifestURL
v.DASHManifestURL = prData.StreamingData.DashManifestURL
return nil
}
func (v *Video) SortBitrateDesc(i int, j int) bool {
return v.Formats[i].Bitrate > v.Formats[j].Bitrate
}
func (v *Video) SortBitrateAsc(i int, j int) bool {
return v.Formats[i].Bitrate < v.Formats[j].Bitrate
}

View File

@@ -1,34 +0,0 @@
package youtube
import (
"regexp"
"strings"
)
var videoRegexpList = []*regexp.Regexp{
regexp.MustCompile(`(?:v|embed|shorts|watch\?v)(?:=|/)([^"&?/=%]{11})`),
regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`),
regexp.MustCompile(`([^"&?/=%]{11})`),
}
// ExtractVideoID extracts the videoID from the given string
func ExtractVideoID(videoID string) (string, error) {
if strings.Contains(videoID, "youtu") || strings.ContainsAny(videoID, "\"?&/<%=") {
for _, re := range videoRegexpList {
if isMatch := re.MatchString(videoID); isMatch {
subs := re.FindStringSubmatch(videoID)
videoID = subs[1]
}
}
}
if strings.ContainsAny(videoID, "?&/<%=") {
return "", ErrInvalidCharactersInVideoID
}
if len(videoID) < 10 {
return "", ErrVideoIDMinLength
}
return videoID, nil
}

View File

@@ -1,143 +0,0 @@
package main
import (
"log"
"sync"
"time"
"github.com/gorilla/websocket"
)
const TIMEOUT = 6
const IDLE_TIMEOUT = TIMEOUT * time.Second
const PING_INTERVAL = (TIMEOUT / 2) * time.Second
type WSConnection struct {
alive bool
url string
conn *websocket.Conn
errChan chan error
writeLock sync.Mutex
ReadChan chan string
WriteChan chan string
Dead chan error
}
func (ws *WSConnection) messageReader() {
log.Printf("Starting reader")
for {
if !ws.alive {
break
}
_, message, err := ws.conn.ReadMessage()
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
if err != nil {
ws.errChan <- err
break
}
log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan))
ws.ReadChan <- string(message)
}
log.Printf("Reader done")
}
func (ws *WSConnection) messageSender() {
log.Printf("Starting sender")
for {
msg, ok := <-ws.WriteChan
if !ok {
break
}
ws.doSend(msg)
}
log.Printf("Sender done")
}
func (ws *WSConnection) doSend(msg string) {
ws.writeLock.Lock()
defer ws.writeLock.Unlock()
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
if err != nil {
log.Printf("Error during message writing: %v", err)
ws.errChan <- err
return
}
}
func (ws *WSConnection) pinger() {
log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL)
for {
if !ws.alive {
break
}
ws.doPing()
time.Sleep(PING_INTERVAL)
}
log.Printf("Pinger done")
}
func (ws *WSConnection) doPing() {
ws.writeLock.Lock()
defer ws.writeLock.Unlock()
// log.Printf("Ping")
err := ws.conn.WriteMessage(websocket.PingMessage, nil)
if err != nil {
log.Println("Error during ping:", err)
ws.errChan <- err
return
}
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
// log.Printf("Ping OK")
}
func (ws *WSConnection) handleError() {
for {
err := <-ws.errChan
log.Printf("Client error: %+v", err)
ws.alive = false
ws.conn.Close()
close(ws.ReadChan)
close(ws.WriteChan)
close(ws.errChan)
ws.Dead <- err
return
}
}
func (ws *WSConnection) Open() {
log.Printf("Connecting to %s", ws.url)
ws.Dead = make(chan error, 1)
conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil)
if err != nil {
log.Println("Error during connection:", err)
ws.Dead <- err
return
}
log.Printf("Connected")
ws.conn = conn
ws.alive = true
ws.errChan = make(chan error, 1)
ws.ReadChan = make(chan string, 128)
ws.WriteChan = make(chan string, 128)
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetPongHandler(func(string) error {
// log.Println("Pong")
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
return nil
})
go ws.handleError()
go ws.messageReader()
go ws.messageSender()
go ws.pinger()
}

View File

@@ -1,20 +1,26 @@
nodes = document.querySelectorAll(":hover");
i = 1;
console.log(nodes);
titleNode = nodes[nodes.length - i];
selector = "a#video-title-link";
selectors = ["a#video-title-link", "a#video-title"];
invidious = false;
if (window.location.href.includes("invidious.site")) {
selector = "a";
invidious = true;
}
while (titleNode && !titleNode.matches(selector)) {
titleNode = nodes[nodes.length - i];
outer:
while (titleNode) {
for (let selector of selectors) {
if (titleNode.matches(selector)) {
break outer;
}
}
titleNode = titleNode.parentElement;
console.log(titleNode);
}
if (!(titleNode && titleNode.matches(selector))) {
if (!titleNode) {
console.error("No video element found.");
} else {
link = titleNode.href;

3
youtubeWatcher/deploy.sh Normal file
View File

@@ -0,0 +1,3 @@
nssm stop YoutubeWatcher && \
go build . && \
nssm start YoutubeWatcher

3
youtubeWatcher/go.mod Normal file
View File

@@ -0,0 +1,3 @@
module ywatcher
go 1.24.2

156
youtubeWatcher/main.go Normal file
View File

@@ -0,0 +1,156 @@
package main
import (
"encoding/xml"
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"sync"
"time"
)
type DownloadRequest struct {
Link string `json:"link"`
}
type RssWatcher struct {
Feed *RssFeed
}
var videoRegex = regexp.MustCompile(`yt:video:(?<videoid>[^ ]+) (?:[^ ]+ ){2}(?<videotitle>.+?)https(?:[^ ]+ ){2}(?<date>[^ ]+)`)
var feeds []*RssFeed = []*RssFeed{
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCMwJJL5FJFuTRT55ksbQ4GQ", Id: "@AsmongoldClips"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
}
func (w *RssWatcher) Watch(videoUrls chan string) error {
ticker := time.NewTicker(1 * time.Minute)
defer ticker.Stop()
log.Printf("[%s]: Watcher started, checking every minute.", w.Feed.Id)
w.CheckFeed(videoUrls)
for {
select {
case <-ticker.C:
log.Printf("[%s]: Checking feed", w.Feed.Id)
err := w.CheckFeed(videoUrls)
if err != nil {
return fmt.Errorf("watcher %s failed to check feed: %w", w.Feed.Id, err)
}
log.Printf("[%s]: Successfully checked feed", w.Feed.Id)
}
}
}
func (w *RssWatcher) CheckFeed(videoUrls chan string) error {
log.Printf("Checking feed URL: %s", w.Feed.Url)
resp, err := http.Get(w.Feed.Url)
if err != nil {
return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
}
defer resp.Body.Close()
log.Printf("Received response with status code: %d", resp.StatusCode)
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
}
// os.WriteFile("cache.xml", body, 0644)
// body, err := os.ReadFile("cache.xml")
// if err != nil {
// return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
// }
var feed Feed
err = xml.Unmarshal(body, &feed)
if err != nil {
return fmt.Errorf("[%s]: failed to unmarshal feed: %w", w.Feed.Id, err)
}
for _, entry := range feed.Entry {
uploaded, err := time.Parse(time.RFC3339, entry.Published)
if err != nil {
return fmt.Errorf("[%s]: failed to parse published date: %w", w.Feed.Id, err)
}
if uploaded.Before(w.Feed.LastSeen) {
log.Printf("[%s]: Skipping video titled %q because it was uploaded before %s", w.Feed.Id, entry.Title, w.Feed.LastSeen.Format(time.RFC3339))
continue
}
log.Printf("[%s]: Found new video titled %q with url %q", w.Feed.Id, entry.Title, entry.Link.Href)
videoUrls <- entry.Link.Href
}
err = w.Feed.WriteLastSeen(time.Now())
if err != nil {
return fmt.Errorf("[%s]: failed to write last seen: %w", w.Feed.Id, err)
}
w.Feed.LastSeen = time.Now()
return nil
}
var Error *log.Logger
var Warning *log.Logger
func init() {
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
logFile, err := os.Create("ywatcher.log")
if err != nil {
log.Printf("Error creating log file: %v", err)
os.Exit(1)
}
logger := io.MultiWriter(os.Stdout, logFile)
log.SetOutput(logger)
Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
Warning = log.New(io.MultiWriter(logFile, os.Stdout),
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
}
func main() {
videoUrls := make(chan string, 12)
wg := &sync.WaitGroup{}
for _, feed := range feeds {
wg.Add(1)
go func(feed *RssFeed) {
err := feed.UpdateLastSeen()
if err != nil {
Error.Printf("failed to update lastseen for feed %s: %v", feed.Id, err)
panic(err)
}
defer wg.Done()
watcher := RssWatcher{
Feed: feed,
}
err = watcher.Watch(videoUrls)
if err != nil {
Error.Printf("watcher %s failed to watch feed: %v", feed.Id, err)
panic(err)
}
}(feed)
}
go func() {
for videoUrl := range videoUrls {
log.Printf("Got new video with url %q", videoUrl)
err := Download(videoUrl)
if err != nil {
Error.Printf("failed to download video: %v", err)
panic(err)
}
}
}()
wg.Wait()
}

View File

@@ -0,0 +1,50 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
)
const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
type Item struct {
Link string `json:"link"`
}
func Download(url string) error {
log.Printf("Starting download for URL: %s", url)
req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil)
if err != nil {
return fmt.Errorf("error creating POST request: %++v", err)
}
req.Header.Set("Content-Type", "application/json")
item := new(Item)
item.Link = url
body, err := json.Marshal(item)
if err != nil {
return fmt.Errorf("error marshalling subscription body: %++v", err)
}
req.Body = io.NopCloser(bytes.NewReader(body))
client := http.Client{}
log.Printf("Sending POST request to %s", URL)
res, err := client.Do(req)
if err != nil {
return fmt.Errorf("error sending POST request: %++v", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return fmt.Errorf("non-OK HTTP status: %d", res.StatusCode)
}
log.Printf("Successfully enqueued %s", url)
return nil
}

97
youtubeWatcher/rssfeed.go Normal file
View File

@@ -0,0 +1,97 @@
package main
import (
"fmt"
"io"
"log"
"os"
"sync"
"time"
)
type RssFeed struct {
Url string
Id string
LastSeen time.Time
fileMutex sync.Mutex
}
func (f *RssFeed) UpdateLastSeen() error {
log.Printf("[%s]: Updating last seen time", f.Id)
lastSeen, err := f.ReadLastSeen()
if err != nil {
return fmt.Errorf("failed to read lastseen file for feed %s: %w", f.Id, err)
}
f.LastSeen = lastSeen
log.Printf("[%s]: Last seen time updated to: %s", f.Id, f.LastSeen)
return nil
}
func (f *RssFeed) ReadLastSeen() (time.Time, error) {
f.fileMutex.Lock()
defer f.fileMutex.Unlock()
lastSeen := time.Now()
log.Printf("[%s]: Attempting to open lastseen file...", f.Id)
lastSeenFile, err := os.Open("lastseen" + f.Id)
if err != nil {
if os.IsNotExist(err) {
log.Printf("[%s]: lastseen file does not exist, creating a new one...", f.Id)
lastSeenFile, err = os.Create("lastseen" + f.Id)
if err != nil {
return lastSeen, fmt.Errorf("failed to create lastseen file: %w", err)
}
log.Printf("[%s]: Writing current time to lastseen file...", f.Id)
lastSeenFile.Write([]byte(lastSeen.Format(time.RFC3339)))
lastSeenFile.Sync()
log.Printf("[%s]: Successfully created lastseen file with current time.", f.Id)
return lastSeen, nil
} else {
log.Printf("[%s]: Error opening lastseen file: %v", f.Id, err)
return lastSeen, fmt.Errorf("failed to read lastseen file: %w", err)
}
}
log.Printf("[%s]: Reading contents of lastseen file...", f.Id)
lastSeenBytes, err := io.ReadAll(lastSeenFile)
if err != nil {
log.Printf("[%s]: Error reading lastseen file: %v", f.Id, err)
return lastSeen, fmt.Errorf("failed to read lastseen file: %w", err)
}
log.Printf("[%s]: Parsing lastseen time...", f.Id)
lastSeen, err = time.Parse(time.RFC3339, string(lastSeenBytes))
if err != nil {
log.Printf("[%s]: Error parsing lastseen file: %v", f.Id, err)
return lastSeen, fmt.Errorf("failed to parse lastseen file: %w", err)
}
lastSeenFile.Close()
log.Printf("[%s]: Last seen time read: %s", f.Id, lastSeen)
return lastSeen, nil
}
func (f *RssFeed) WriteLastSeen(when time.Time) error {
f.fileMutex.Lock()
defer f.fileMutex.Unlock()
log.Printf("[%s]: Attempting to create lastseen file...", f.Id)
lastSeenFile, err := os.Create("lastseen" + f.Id)
if err != nil {
log.Printf("[%s]: Error creating lastseen file: %v", f.Id, err)
return fmt.Errorf("failed to create lastseen file: %w", err)
}
log.Printf("[%s]: Successfully created lastseen file: lastseen%s", f.Id, f.Id)
log.Printf("[%s]: Writing last seen time to file: %s", f.Id, when.Format(time.RFC3339))
_, err = lastSeenFile.Write([]byte(when.Format(time.RFC3339)))
if err != nil {
log.Printf("[%s]: Error writing to lastseen file: %v", f.Id, err)
return fmt.Errorf("failed to write to lastseen file: %w", err)
}
log.Printf("[%s]: Successfully wrote last seen time to file.", f.Id)
err = lastSeenFile.Close()
if err != nil {
log.Printf("[%s]: Error closing lastseen file: %v", f.Id, err)
return fmt.Errorf("failed to close lastseen file: %w", err)
}
log.Printf("[%s]: Lastseen file closed successfully.", f.Id)
return nil
}

View File

@@ -0,0 +1,86 @@
package main
import "encoding/xml"
type Feed struct {
XMLName xml.Name `xml:"feed"`
Text string `xml:",chardata"`
Yt string `xml:"yt,attr"`
Media string `xml:"media,attr"`
Xmlns string `xml:"xmlns,attr"`
Link []Link `xml:"link"`
ID string `xml:"id"`
ChannelId string `xml:"channelId"`
Title string `xml:"title"`
Author Author `xml:"author"`
Published string `xml:"published"`
Entry []Entry `xml:"entry"`
}
type Link struct {
Text string `xml:",chardata"`
Rel string `xml:"rel,attr"`
Href string `xml:"href,attr"`
}
type Author struct {
Text string `xml:",chardata"`
Name string `xml:"name"`
URI string `xml:"uri"`
}
type Entry struct {
Text string `xml:",chardata"`
ID string `xml:"id"`
VideoId string `xml:"videoId"`
ChannelId string `xml:"channelId"`
Title string `xml:"title"`
Link Link `xml:"link"`
Author Author `xml:"author"`
Published string `xml:"published"`
Updated string `xml:"updated"`
Group Group `xml:"group"`
}
type Group struct {
Text string `xml:",chardata"`
Title string `xml:"title"`
Content Content `xml:"content"`
Thumbnail Thumbnail `xml:"thumbnail"`
Description string `xml:"description"`
Community Community `xml:"community"`
}
type Content struct {
Text string `xml:",chardata"`
URL string `xml:"url,attr"`
Type string `xml:"type,attr"`
Width string `xml:"width,attr"`
Height string `xml:"height,attr"`
}
type Thumbnail struct {
Text string `xml:",chardata"`
URL string `xml:"url,attr"`
Width string `xml:"width,attr"`
Height string `xml:"height,attr"`
}
type Community struct {
Text string `xml:",chardata"`
StarRating StarRating `xml:"starRating"`
Statistics Statistics `xml:"statistics"`
}
type StarRating struct {
Text string `xml:",chardata"`
Count string `xml:"count,attr"`
Average string `xml:"average,attr"`
Min string `xml:"min,attr"`
Max string `xml:"max,attr"`
}
type Statistics struct {
Text string `xml:",chardata"`
Views string `xml:"views,attr"`
}