Compare commits

...

41 Commits

Author SHA1 Message Date
f639545e9b Add more shit to strip 2024-11-12 11:29:11 +01:00
18d7177099 Less workers more better? 2024-11-12 10:51:03 +01:00
07c20f4582 Close video file that exists when checking 2024-11-05 16:03:38 +01:00
9b38ea7068 Betterify sanitization 2024-11-05 16:03:32 +01:00
8ab4b4c280 Refactor sanitization 2024-11-05 15:59:29 +01:00
0ad8722bf9 Remove from ongoing downloads even if failed 2024-11-05 15:59:29 +01:00
d8c4c343f4 Skip downloading existing videos 2024-11-05 15:59:29 +01:00
52fdcc64ef Shorter timeout 2024-11-05 15:58:44 +01:00
fa0f1cb48c Sanitize video titles and author names 2024-11-05 15:43:13 +01:00
63899ac4bd Download videos to authorname folder 2024-11-05 15:38:44 +01:00
f5c072360b Add alternate downloader
That works a lot faster, pog
2024-11-05 15:36:08 +01:00
7c02cbf0e4 Update ignore 2024-11-05 15:35:56 +01:00
9b41f9114c Improve deletification
It was meant to run once and not as a service...
So it's bursting at the seams
But it will work
I will make it work
2024-11-05 15:33:30 +01:00
60dc43fd9b add new youtubedl dependency 2024-11-05 15:29:49 +01:00
584084c1bc Touch messages so they don't time out while processing 2024-10-18 17:52:30 +02:00
ed8eaf67a2 Update dl url 2024-10-18 17:52:21 +02:00
268351f31e Cleanup 2024-10-13 23:30:48 +02:00
43efe20d8c Update hotkey script 2024-10-13 23:30:34 +02:00
8d55bcb5ee Refactor everything to use nsq instead of hosting a http server
The point of this is (hopefully) some sort of resiliency
I do not want to lose any messages ever
And I want to be able to kill this process whenever it is misbehaving
Hopefully this achieves that goal
2024-10-13 23:20:06 +02:00
48179c3a67 More better rate limiting 2024-10-13 22:53:43 +02:00
ca3c3423b1 Discover go install 2024-10-03 10:53:35 +02:00
0ce14bdc53 Update hotkey 2024-08-17 15:12:01 +02:00
PhatPhuckDave
676748f614 Make text green when download started
Finally, feedback
2024-07-22 15:45:53 +02:00
PhatPhuckDave
b92a325b99 Update hotkey to work with any youtube page (hopefully) 2024-07-21 23:29:24 +02:00
PhatPhuckDave
7e12d9e939 Enable CORS 2024-07-21 18:59:57 +02:00
PhatPhuckDave
38449a7676 Add hotkey for download 2024-07-21 18:56:11 +02:00
PhatPhuckDave
7cf0cf5033 Add no workey extension 2024-07-21 18:56:08 +02:00
c95613e240 Merge branch 'rework' 2024-07-16 17:53:20 +02:00
8a51da97c2 Remove websockets 2024-07-14 14:59:34 +02:00
468de9e401 Simplify everything 2024-07-14 14:59:12 +02:00
4fd836c0b6 Update instrumentation 2024-07-14 14:40:36 +02:00
cdcdb18c57 Fix issue with reader disconnecting due to closd channel 2024-06-27 12:21:55 +02:00
f368436325 Fix build script 2024-06-27 12:07:29 +02:00
2cb95c397d Mod tidy 2024-06-27 11:56:59 +02:00
7966b06f1b Fix issue with client reconnect and improve client resiliency 2024-06-27 11:54:14 +02:00
51a94b6636 Fix memory leak in server
Remove server instrumentation
2024-06-27 11:22:16 +02:00
4ca3053243 Improve client disconnect detection 2024-06-27 11:05:30 +02:00
4acb89cdb1 Rework WS server 2024-06-27 10:42:17 +02:00
c930aeb737 Enable middle click download on subscriptions as well 2024-06-25 13:34:58 +02:00
aad190d94c Add alerts to downloader 2024-06-25 13:28:58 +02:00
a7babdbba4 Fix issues with slow 2024-06-17 16:14:46 +02:00
56 changed files with 3290 additions and 445 deletions

5
.gitignore vendored
View File

@@ -2,3 +2,8 @@
main.exe
logs.log
ws-server/deploy.tar
downloader/main.log
downloader/vendor/golang.org
downloader/vendor/github.com/*
!downloader/vendor/github.com/kkdai
downloader/vendor/modules.txt

View File

@@ -11,7 +11,7 @@ import (
"sync"
)
const URL = `https://youtube-download-ws-server.site.quack-lab.dev/download`
const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
type Item struct {
Link string `json:"link"`

View File

@@ -1,3 +1,3 @@
module main
module dl
go 1.22.4

View File

@@ -1 +0,0 @@
main.exe,"C:\Program Files\Git\usr\bin\dl.exe",t

33
download.go Normal file
View File

@@ -0,0 +1,33 @@
package main
import (
"context"
"log"
"github.com/lrstanley/go-ytdlp"
)
const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
var dl = ytdlp.New().
// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
FormatSort("res,ext:mp4:m4a").
Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
LimitRate("50M").
// HTTPChunkSize("20M").
MarkWatched().
SponsorblockMark("all").
PrintJSON().
RecodeVideo("mp4").
ConcurrentFragments(4)
func Download(event PBEvent, status chan error) {
_, err := dl.Run(context.TODO(), event.Record.Link)
if err != nil {
status <- err
return
}
log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link)
SetDownloaded(event)
}

Binary file not shown.

After

Width:  |  Height:  |  Size: 4.4 KiB

View File

@@ -2,43 +2,55 @@ package main
import (
"context"
"fmt"
"log"
"github.com/gen2brain/beeep"
"github.com/lrstanley/go-ytdlp"
)
const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
var dl = ytdlp.New().
// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
FormatSort("res,ext:mp4:m4a").
Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
LimitRate("5M").
LimitRate(fmt.Sprintf("%dM", 150/DOWNLOAD_WORKERS)).
// HTTPChunkSize("20M").
MarkWatched().
SponsorblockMark("all").
RecodeVideo("mp4").
ConcurrentFragments(6)
// func Download(event PBEvent, status chan error) {
// _, err := dl.Run(context.TODO(), event.Record.Link)
// if err != nil {
// status <- err
// return
// }
func Download(url string) error {
_, ongoing := ongoingDownloads[url]
if ongoing {
// return fmt.Errorf("Download %s is already ongoing", url)
Warning.Printf("Download %s is already ongoing", url)
return nil
}
ongoingDownloadsMutex.Lock()
ongoingDownloads[url] = struct{}{}
ongoingDownloadsMutex.Unlock()
// log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link)
// SetDownloaded(event)
// }
func DownloadURL(url string, status chan error) {
log.Printf("Downloading %s", url)
_, err := dl.Run(context.TODO(), url)
err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
if err != nil {
status <- err
return
Warning.Printf("Failed beeping with %+v", err)
}
err = beeep.Alert("Download Started", url, "assets/information.png")
if err != nil {
Warning.Printf("Failed alerting with %+v", err)
}
log.Printf("Downloaded %s", url)
close(status)
_, err = dl.Run(context.TODO(), url)
if err != nil {
return fmt.Errorf("failed downloading %s with %+v", url, err)
}
log.Printf("Downloaded %s", url)
ongoingDownloadsMutex.Lock()
delete(ongoingDownloads, url)
ongoingDownloadsMutex.Unlock()
return nil
}

View File

@@ -0,0 +1,89 @@
package main
import (
"context"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"strings"
"github.com/gen2brain/beeep"
"github.com/kkdai/youtube/v2"
ytd "github.com/kkdai/youtube/v2/downloader"
)
func DownloadR(url string) error {
_, ongoing := ongoingDownloads[url]
if ongoing {
// return fmt.Errorf("Download %s is already ongoing", url)
Warning.Printf("Download %s is already ongoing", url)
return nil
}
ongoingDownloadsMutex.Lock()
ongoingDownloads[url] = struct{}{}
ongoingDownloadsMutex.Unlock()
defer func() {
ongoingDownloadsMutex.Lock()
delete(ongoingDownloads, url)
ongoingDownloadsMutex.Unlock()
}()
log.Printf("Downloading %s", url)
err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
if err != nil {
Warning.Printf("Failed beeping with %+v", err)
}
err = beeep.Alert("Download Started", url, "assets/information.png")
if err != nil {
Warning.Printf("Failed alerting with %+v", err)
}
client := youtube.Client{}
video, err := client.GetVideo(url)
if err != nil {
return fmt.Errorf("failed downloading %s with %+v", url, err)
}
// $ go run . download -m mp4 -q hd https://www.youtube.com/watch?v=D9trBXaXCgA
// This works fine, now I have to figure out how to plug -m and -q into this shit below
downloader := ytd.Downloader{
OutputDir: OUTPUT_DIR,
Client: client,
}
videoTitle := Sanitize(video.Title)
videoAuthor := Sanitize(video.Author)
videoFileRoot := filepath.Join(OUTPUT_DIR, videoAuthor)
err = os.MkdirAll(videoFileRoot, 0755)
if err != nil {
return fmt.Errorf("failed creating directory %s with %+v", videoFileRoot, err)
}
fullVideoPath := filepath.Join(OUTPUT_DIR, videoAuthor, videoTitle+".mp4")
videoFileHandle, err := os.Open(fullVideoPath)
if err == nil {
videoFileHandle.Close()
log.Printf("File %s already exists, skipping download", fullVideoPath)
return nil
}
videoFile := filepath.Join(videoAuthor, videoTitle+".mp4")
err = downloader.DownloadComposite(context.Background(), videoFile, video, "hd", "mp4", "")
if err != nil {
return fmt.Errorf("failed downloading %s with %+v", url, err)
}
log.Printf("Downloaded %s", url)
return nil
}
var re = regexp.MustCompile(`[#!$%&/()";:?;,|*]`)
func Sanitize(s string) string {
s = strings.ReplaceAll(s, "&", "and")
s = re.ReplaceAllString(s, "")
return s
}

View File

@@ -2,25 +2,33 @@ module main
go 1.22.4
require github.com/r3labs/sse/v2 v2.10.0
require (
github.com/ProtonMail/go-crypto v1.0.0 // indirect
github.com/VividCortex/ewma v1.2.0 // indirect
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
github.com/bitly/go-simplejson v0.5.1 // indirect
github.com/cloudflare/circl v1.3.7 // indirect
github.com/dlclark/regexp2 v1.11.0 // indirect
github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 // indirect
github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/golang/snappy v0.0.1 // indirect
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect
github.com/mattn/go-runewidth v0.0.15 // indirect
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
github.com/vbauerster/mpb/v5 v5.4.0 // indirect
golang.org/x/crypto v0.21.0 // indirect
golang.org/x/net v0.21.0 // indirect
golang.org/x/sys v0.18.0 // indirect
golang.org/x/text v0.14.0 // indirect
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
)
require (
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4
github.com/gorilla/websocket v1.5.3
github.com/kkdai/youtube/v2 v2.10.1
github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99
github.com/nsqio/go-nsq v1.1.0
)

View File

@@ -1,5 +1,11 @@
github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
@@ -9,20 +15,35 @@ github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUK
github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dlclark/regexp2 v1.11.0 h1:G/nrcoOa7ZXlpoa/91N3X7mM3r8eIlMBBJZvsz/mxKI=
github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 h1:O7I1iuzEA7SG+dK8ocOBSlYAA9jBUmCYl/Qa7ey7JAM=
github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI=
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible h1:W1iEw64niKVGogNgBN3ePyLFfuisuzeidWPMPWmECqU=
github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 h1:qZNfIGkIANxGv/OqtnntR4DfOY2+BgwR60cAcu/i3SE=
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4/go.mod h1:kW3HQ4UdaAyrUCSSDR4xUzBKW6O2iA4uHhk7AtyYp10=
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 h1:y3N7Bm7Y9/CtpiVkw/ZWj6lSlDF3F74SfKwfTCer72Q=
github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
github.com/kkdai/youtube/v2 v2.10.1 h1:jdPho4R7VxWoRi9Wx4ULMq4+hlzSVOXxh4Zh83f2F9M=
github.com/kkdai/youtube/v2 v2.10.1/go.mod h1:qL8JZv7Q1IoDs4nnaL51o/hmITXEIvyCIXopB0oqgVM=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
@@ -31,14 +52,25 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 h1:ZAo7qJht9PqefOD7C0ZKQ8dEkpJeM955sYw0FtQnzvo=
github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o=
github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8=
github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-runewidth v0.0.15 h1:UNAjwbU9l54TA3KzvqLGxwWjHmMgBUVhBiTjelZgg3U=
github.com/mattn/go-runewidth v0.0.15/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk=
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o=
github.com/vbauerster/mpb/v5 v5.4.0 h1:n8JPunifvQvh6P1D1HAl2Ur9YcmKT1tpoUuiea5mlmg=
github.com/vbauerster/mpb/v5 v5.4.0/go.mod h1:fi4wVo7BVQ22QcvFObm+VwliQXlV1eBT8JDaKXR4JGI=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -49,19 +81,19 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOM
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc=
golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -85,18 +117,18 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@@ -1,72 +1,108 @@
package main
import (
"context"
"encoding/json"
"fmt"
"io"
"log"
"os"
"os/signal"
"syscall"
"time"
"github.com/nsqio/go-nsq"
)
const WEBSOCKET_SERVER = "ws://youtube-download-ws-server.site.quack-lab.dev/ws"
const WEBSOCKET_SERVER_ALT = "ws://localhost:8080/ws"
var Error *log.Logger
var Warning *log.Logger
func init() {
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
logFile, err := os.Create("main.log")
if err != nil {
log.Printf("Error creating log file: %v", err)
os.Exit(1)
}
logger := io.MultiWriter(os.Stdout, logFile)
log.SetOutput(logger)
Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
Warning = log.New(io.MultiWriter(logFile, os.Stdout),
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
}
const DOWNLOAD_WORKERS = 1
type DLHandler struct{}
func (*DLHandler) HandleMessage(message *nsq.Message) error {
log.Printf("Received message '%s' with %d attempts", message.Body, message.Attempts)
data := DownloadRequest{}
err := json.Unmarshal(message.Body, &data)
if err != nil {
Error.Printf("Error unmarshalling message: %v", err)
return err
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
go func() {
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ticker.C:
message.Touch()
case <-ctx.Done():
return
}
}
}()
err = DownloadR(data.Link)
if err != nil {
Error.Printf("Error downloading %s: %v", data.Link, err)
return err
}
message.Finish()
return nil
}
func main() {
log.SetFlags(log.Lmicroseconds)
config := nsq.NewConfig()
config.MaxAttempts = 5
config.MaxInFlight = DOWNLOAD_WORKERS
config.MsgTimeout = 10 * time.Second
// res, err := http.Get(FULL_URL)
// if err != nil {
// log.Fatal(err)
// }
// defer res.Body.Close()
// body, err := io.ReadAll(res.Body)
// if err != nil {
// log.Printf("Error reading response body: %+v\n", err)
// return
// }
// if res.StatusCode != http.StatusOK {
// log.Printf("Non-OK HTTP status: %d\nResponse body: %s\n", res.StatusCode, body)
// return
// }
// var data APIResponse
// err = json.Unmarshal(body, &data)
// if err != nil {
// log.Printf("Error unmarshaling JSON: %+v\n", err)
// return
// }
// log.Printf("Data: %+v\n", data)
// listener := new(RealtimeListener)
// listener.Url = POCKETBASE_REALTIME
// listener.Collections = []string{COLLECTION_NAME}
// listener.initialize()
ws := new(WSConnection)
ws.url = WEBSOCKET_SERVER
ws.Open()
sem := make(chan struct{}, 4)
for {
select {
case event := <-ws.ReadChan:
eventCopy := event
status := make(chan error)
sem <- struct{}{}
log.Printf("New event: %+v; semaphore at: %d", eventCopy, len(sem))
go func() {
defer func() {
<-sem
log.Printf("Semaphore at: %d", len(sem))
}()
// Download(eventCopy, status)
DownloadURL(eventCopy, status)
// go DownloadNative(event, status)
for status := range status {
log.Printf("Status: %s\n", status)
}
}()
case <-time.After(1 * time.Minute):
// Perform some action or simply continue to avoid deadlock
log.Println("Consumer is alive, but has no new events.")
}
consumer, err := nsq.NewConsumer("ytdqueue", "dl", config)
if err != nil {
Error.Printf("Error creating consumer: %v", err)
return
}
for i := 0; i < DOWNLOAD_WORKERS; i++ {
consumer.AddHandler(&DLHandler{})
}
err = consumer.ConnectToNSQD("nsq.site.quack-lab.dev:41505")
if err != nil {
Error.Printf("Error connecting to nsqlookupd: %v", err)
return
}
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
<-sigChan
log.Println("Received signal to terminate. Initiating graceful shutdown...")
consumer.Stop()
<-consumer.StopChan
log.Println("Graceful shutdown completed.")
}

View File

@@ -1,25 +1,5 @@
package main
type APIResponse struct {
Page int `json:"page"`
PerPage int `json:"perPage"`
TotalItems int `json:"totalItems"`
TotalPages int `json:"totalPages"`
Items []APIItem `json:"items"`
}
type APIItem struct {
CollectionId string `json:"collectionId"`
CollectionName string `json:"collectionName"`
Created string `json:"created"`
Downloaded bool `json:"downloaded"`
Id string `json:"id"`
Link string `json:"link"`
Updated string `json:"updated"`
}
type PBEvent struct {
ClientId string `json:"clientId"`
Action string `json:"action"`
Record APIItem `json:"record"`
}
type DownloadRequest struct {
Link string `json:"link"`
}

8
downloader/utils.go Normal file
View File

@@ -0,0 +1,8 @@
package main
import "sync"
const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
var ongoingDownloads = make(map[string]struct{})
var ongoingDownloadsMutex = &sync.Mutex{}

View File

@@ -0,0 +1,8 @@
/.idea
/.vscode
download_test
/bin
/dist
/output
*.out
.DS_Store

View File

@@ -0,0 +1,8 @@
issues:
exclude:
- Subprocess launched with function call
# Excluding configuration per-path, per-linter, per-text and per-source
exclude-rules:
# Exclude some linters from running on tests files.
- path: cmd/
text: InsecureSkipVerify

View File

@@ -0,0 +1,20 @@
project_name: youtubedr
env:
# Build without CGO to don't depend on specific glibc versions
- CGO_ENABLED=0
builds:
- main: ./cmd/youtubedr
binary: youtubedr
goos:
- windows
- darwin
- linux
- freebsd
goarch:
- amd64
- arm
- arm64
flags:
- -trimpath

22
downloader/vendor/github.com/kkdai/youtube/v2/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,22 @@
The MIT License (MIT)
Copyright (c) 2015 Evan Lin
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

57
downloader/vendor/github.com/kkdai/youtube/v2/Makefile generated vendored Normal file
View File

@@ -0,0 +1,57 @@
FILES_TO_FMT ?= $(shell find . -path ./vendor -prune -o -name '*.go' -print)
LOGLEVEL ?= debug
## help: Show makefile commands
.PHONY: help
help: Makefile
@echo "---- Project: kkdai/youtube ----"
@echo " Usage: make COMMAND"
@echo
@echo " Management Commands:"
@sed -n 's/^##//p' $< | column -t -s ':' | sed -e 's/^/ /'
@echo
## build: Build project
.PHONY: build
build:
goreleaser --rm-dist
## deps: Ensures fresh go.mod and go.sum
.PHONY: deps
deps:
go mod tidy
go mod verify
## lint: Run golangci-lint check
.PHONY: lint
lint:
command -v golangci-lint || curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $$(go env GOPATH)/bin $(GOLANGCI_LINT_VERSION)
echo "golangci-lint checking..."
golangci-lint run --deadline=30m --enable=misspell --enable=gosec --enable=gofmt --enable=goimports --enable=revive ./cmd/... ./...
go vet ./...
## format: Formats Go code
.PHONY: format
format:
@echo ">> formatting code"
@gofmt -s -w $(FILES_TO_FMT)
## test-unit: Run all Youtube Go unit tests
.PHONY: test-unit
test-unit:
LOGLEVEL=${LOGLEVEL} go test -v -cover ./...
## test-integration: Run all Youtube Go integration tests
.PHONY: test-integration
test-integration:
mkdir -p output
rm -f output/*
LOGLEVEL=${LOGLEVEL} ARTIFACTS=output go test -v -race -covermode=atomic -coverprofile=coverage.out -tags=integration ./...
.PHONY: coverage.out
coverage.out:
## clean: Clean files and downloaded videos from builds during development
.PHONY: clean
clean:
rm -rf dist *.mp4 *.mkv

161
downloader/vendor/github.com/kkdai/youtube/v2/README.md generated vendored Normal file
View File

@@ -0,0 +1,161 @@
Download Youtube Video in Golang
==================
[![GitHub license](https://img.shields.io/badge/license-MIT-blue.svg)](https://raw.githubusercontent.com/kkdai/youtube/master/LICENSE)
[![Go Reference](https://pkg.go.dev/badge/github.com/kkdai/youtube.svg)](https://pkg.go.dev/github.com/kkdai/youtube/v2)
[![Build Status](https://github.com/kkdai/youtube/workflows/go/badge.svg?branch=master)](https://github.com/kkdai/youtube/actions)
[![Coverage](https://codecov.io/gh/kkdai/youtube/branch/master/graph/badge.svg)](https://codecov.io/gh/kkdai/youtube)
[![](https://goreportcard.com/badge/github.com/kkdai/youtube)](https://goreportcard.com/badge/github.com/kkdai/youtube)
This package is a Youtube video download package, for more detail refer [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl) for more download options.
This tool is meant to be used to download CC0 licenced content, we do not support nor recommend using it for illegal activities.
## Overview
* [Install](#installation)
* [Usage](#usage)
* [Example: Download video from \[dotGo 2015 - Rob Pike - Simplicity is Complicated\]](#download-dotGo-2015-rob-pike-video)
## Installation
### Install via go get
Please ensure you have installed Go 1.21 or later.
```shell
go get github.com/kkdai/youtube/v2
```
### From source code
```shell
git clone https://github.com/kkdai/youtube.git
cd youtube
go run ./cmd/youtubedr
```
### Mac
```shell
brew install youtubedr
```
### in Termux
```shell
pkg install youtubedr
```
### You can also find this package in
- [archlinux](https://aur.archlinux.org/packages/youtubedr/) (thanks to [cjsthompson](https://github.com/cjsthompson))
- [Termux package](https://github.com/termux/termux-packages/tree/master/packages/youtubedr) (thanks to [kcubeterm](https://github.com/kcubeterm))
- [Homebrew](https://formulae.brew.sh/formula/youtubedr) (thanks to [kkc](https://github.com/kkc))
## Usage
### Use the binary directly
It's really simple to use, just get the video id from youtube url - ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
```shell
$ youtubedr download QAGDGja7kbs
$ youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
```
### Use this package in your golang program
Please check out the [example_test.go](example_test.go) for example code.
## Example:
* ### Get information of dotGo-2015-rob-pike video for downloading
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
```
youtubedr info https://www.youtube.com/watch?v=rFejpH_tAHM
Title: dotGo 2015 - Rob Pike - Simplicity is Complicated
Author: dotconferences
-----available streams-----
itag: 18 , quality: medium , type: video/mp4; codecs="avc1.42001E, mp4a.40.2"
itag: 22 , quality: hd720 , type: video/mp4; codecs="avc1.64001F, mp4a.40.2"
itag: 137 , quality: hd1080 , type: video/mp4; codecs="avc1.640028"
itag: 248 , quality: hd1080 , type: video/webm; codecs="vp9"
........
```
* ### Download dotGo-2015-rob-pike-video
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
```
youtubedr download https://www.youtube.com/watch?v=rFejpH_tAHM
```
* ### Download video to specific folder and name
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) to current directory and name the file to simplicity-is-complicated.mp4
```
youtubedr download -d ./ -o simplicity-is-complicated.mp4 https://www.youtube.com/watch?v=rFejpH_tAHM
```
* ### Download video with specific quality
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM) with specific quality
```
youtubedr download -q medium https://www.youtube.com/watch?v=rFejpH_tAHM
```
#### Special case by quality hd1080:
Installation of ffmpeg is necessary for hd1080
```
ffmpeg //check ffmpeg is installed, if not please download ffmpeg and set to your PATH.
youtubedr download -q hd1080 https://www.youtube.com/watch?v=rFejpH_tAHM
```
* ### Download video with specific itag
`go get github.com/kkdai/youtube/v2/youtubedr`
Download video from [dotGo 2015 - Rob Pike - Simplicity is Complicated](https://www.youtube.com/watch?v=rFejpH_tAHM)
```
youtubedr download -q 18 https://www.youtube.com/watch?v=rFejpH_tAHM
```
## How it works
- Parse the video ID you input in URL
- ex: `https://www.youtube.com/watch?v=rFejpH_tAHM`, the video id is `rFejpH_tAHM`
- Get video information via video id.
- Use URL: `http://youtube.com/get_video_info?video_id=`
- Parse and decode video information.
- Download URL in "url="
- title in "title="
- Download video from URL
- Need the string combination of "url"
## Inspired
- [https://github.com/ytdl-org/youtube-dl](https://github.com/ytdl-org/youtube-dl)
- [https://github.com/lepidosteus/youtube-dl](https://github.com/lepidosteus/youtube-dl)
- [拆解 Youtube 影片下載位置](http://hkgoldenmra.blogspot.tw/2013/05/youtube.html)
- [iawia002/annie](https://github.com/iawia002/annie)
- [How to get url from obfuscate video info: youtube video downloader with php](https://stackoverflow.com/questions/60607291/youtube-video-downloader-with-php)
## Project52
It is one of my [project 52](https://github.com/kkdai/project52).
## License
This package is licensed under MIT license. See LICENSE for details.

View File

@@ -0,0 +1,11 @@
Tracking Youtube decipher change
### 2022/01/21:
#### action objects:
var $z={fA:function(a){a.reverse()},S2:function(a,b){a.splice(0,b)},l6:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
#### actions function:
Npa=function(a){a=a.split("");$z.S2(a,3);$z.fA(a,45);$z.l6(a,31);$z.S2(a,1);$z.fA(a,63);$z.S2(a,2);$z.fA(a,68);return a.join("")};

View File

@@ -0,0 +1,29 @@
package youtube
import (
"log/slog"
"os"
"path/filepath"
)
// destination for artifacts, used by integration tests
var artifactsFolder = os.Getenv("ARTIFACTS")
func writeArtifact(name string, content []byte) {
// Ensure folder exists
err := os.MkdirAll(artifactsFolder, os.ModePerm)
if err != nil {
slog.Error("unable to create artifacts folder", "path", artifactsFolder, "error", err)
return
}
path := filepath.Join(artifactsFolder, name)
err = os.WriteFile(path, content, 0600)
log := slog.With("path", path)
if err != nil {
log.Error("unable to write artifact", "error", err)
} else {
log.Debug("artifact created")
}
}

618
downloader/vendor/github.com/kkdai/youtube/v2/client.go generated vendored Normal file
View File

@@ -0,0 +1,618 @@
package youtube
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"math/rand"
"net/http"
"net/url"
"strconv"
"sync/atomic"
"log/slog"
)
const (
Size1Kb = 1024
Size1Mb = Size1Kb * 1024
Size10Mb = Size1Mb * 10
playerParams = "CgIQBg=="
)
var (
ErrNoFormat = errors.New("no video format provided")
)
// DefaultClient type to use. No reason to change but you could if you wanted to.
var DefaultClient = AndroidClient
// Client offers methods to download video metadata and video streams.
type Client struct {
// HTTPClient can be used to set a custom HTTP client.
// If not set, http.DefaultClient will be used
HTTPClient *http.Client
// MaxRoutines to use when downloading a video.
MaxRoutines int
// ChunkSize to use when downloading videos in chunks. Default is Size10Mb.
ChunkSize int64
// playerCache caches the JavaScript code of a player response
playerCache playerCache
client *clientInfo
consentID string
}
func (c *Client) assureClient() {
if c.client == nil {
c.client = &DefaultClient
}
}
// GetVideo fetches video metadata
func (c *Client) GetVideo(url string) (*Video, error) {
return c.GetVideoContext(context.Background(), url)
}
// GetVideoContext fetches video metadata with a context
func (c *Client) GetVideoContext(ctx context.Context, url string) (*Video, error) {
id, err := ExtractVideoID(url)
if err != nil {
return nil, fmt.Errorf("extractVideoID failed: %w", err)
}
return c.videoFromID(ctx, id)
}
func (c *Client) videoFromID(ctx context.Context, id string) (*Video, error) {
c.assureClient()
body, err := c.videoDataByInnertube(ctx, id)
if err != nil {
return nil, err
}
v := Video{
ID: id,
}
// return early if all good
if err = v.parseVideoInfo(body); err == nil {
return &v, nil
}
// If the uploader has disabled embedding the video on other sites, parse video page
if errors.Is(err, ErrNotPlayableInEmbed) {
// additional parameters are required to access clips with sensitiv content
html, err := c.httpGetBodyBytes(ctx, "https://www.youtube.com/watch?v="+id+"&bpctr=9999999999&has_verified=1")
if err != nil {
return nil, err
}
return &v, v.parseVideoPage(html)
}
// If the uploader marked the video as inappropriate for some ages, use embed player
if errors.Is(err, ErrLoginRequired) {
c.client = &EmbeddedClient
bodyEmbed, errEmbed := c.videoDataByInnertube(ctx, id)
if errEmbed == nil {
errEmbed = v.parseVideoInfo(bodyEmbed)
}
if errEmbed == nil {
return &v, nil
}
// private video clearly not age-restricted and thus should be explicit
if errEmbed == ErrVideoPrivate {
return &v, errEmbed
}
// wrapping error so its clear whats happened
return &v, fmt.Errorf("can't bypass age restriction: %w", errEmbed)
}
// undefined error
return &v, err
}
type innertubeRequest struct {
VideoID string `json:"videoId,omitempty"`
BrowseID string `json:"browseId,omitempty"`
Continuation string `json:"continuation,omitempty"`
Context inntertubeContext `json:"context"`
PlaybackContext *playbackContext `json:"playbackContext,omitempty"`
ContentCheckOK bool `json:"contentCheckOk,omitempty"`
RacyCheckOk bool `json:"racyCheckOk,omitempty"`
Params string `json:"params"`
}
type playbackContext struct {
ContentPlaybackContext contentPlaybackContext `json:"contentPlaybackContext"`
}
type contentPlaybackContext struct {
// SignatureTimestamp string `json:"signatureTimestamp"`
HTML5Preference string `json:"html5Preference"`
}
type inntertubeContext struct {
Client innertubeClient `json:"client"`
}
type innertubeClient struct {
HL string `json:"hl"`
GL string `json:"gl"`
ClientName string `json:"clientName"`
ClientVersion string `json:"clientVersion"`
AndroidSDKVersion int `json:"androidSDKVersion,omitempty"`
UserAgent string `json:"userAgent,omitempty"`
TimeZone string `json:"timeZone"`
UTCOffset int `json:"utcOffsetMinutes"`
}
// client info for the innertube API
type clientInfo struct {
name string
key string
version string
userAgent string
androidVersion int
}
var (
// WebClient, better to use Android client but go ahead.
WebClient = clientInfo{
name: "WEB",
version: "2.20220801.00.00",
key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8",
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
}
// AndroidClient, download go brrrrrr.
AndroidClient = clientInfo{
name: "ANDROID",
version: "18.11.34",
key: "AIzaSyA8eiZmM1FaDVjRy-df2KTyQ_vz_yYM39w",
userAgent: "com.google.android.youtube/18.11.34 (Linux; U; Android 11) gzip",
androidVersion: 30,
}
// EmbeddedClient, not really tested.
EmbeddedClient = clientInfo{
name: "WEB_EMBEDDED_PLAYER",
version: "1.19700101",
key: "AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8", // seems like same key works for both clients
userAgent: "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
}
)
func (c *Client) videoDataByInnertube(ctx context.Context, id string) ([]byte, error) {
data := innertubeRequest{
VideoID: id,
Context: prepareInnertubeContext(*c.client),
ContentCheckOK: true,
RacyCheckOk: true,
Params: playerParams,
PlaybackContext: &playbackContext{
ContentPlaybackContext: contentPlaybackContext{
// SignatureTimestamp: sts,
HTML5Preference: "HTML5_PREF_WANTS",
},
},
}
return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/player?key="+c.client.key, data)
}
func (c *Client) transcriptDataByInnertube(ctx context.Context, id string, lang string) ([]byte, error) {
data := innertubeRequest{
Context: prepareInnertubeContext(*c.client),
Params: transcriptVideoID(id, lang),
}
return c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/get_transcript?key="+c.client.key, data)
}
func prepareInnertubeContext(clientInfo clientInfo) inntertubeContext {
return inntertubeContext{
Client: innertubeClient{
HL: "en",
GL: "US",
TimeZone: "UTC",
ClientName: clientInfo.name,
ClientVersion: clientInfo.version,
AndroidSDKVersion: clientInfo.androidVersion,
UserAgent: clientInfo.userAgent,
},
}
}
func prepareInnertubePlaylistData(ID string, continuation bool, clientInfo clientInfo) innertubeRequest {
context := prepareInnertubeContext(clientInfo)
if continuation {
return innertubeRequest{
Context: context,
Continuation: ID,
ContentCheckOK: true,
RacyCheckOk: true,
Params: playerParams,
}
}
return innertubeRequest{
Context: context,
BrowseID: "VL" + ID,
ContentCheckOK: true,
RacyCheckOk: true,
Params: playerParams,
}
}
// transcriptVideoID encodes the video ID to the param used to fetch transcripts.
func transcriptVideoID(videoID string, lang string) string {
langCode := encTranscriptLang(lang)
// This can be optionally appened to the Sprintf str, not sure what it means
// *3engagement-panel-searchable-transcript-search-panel\x30\x00\x38\x01\x40\x01
return base64Enc(fmt.Sprintf("\n\x0b%s\x12\x12%s\x18\x01", videoID, langCode))
}
func encTranscriptLang(languageCode string) string {
s := fmt.Sprintf("\n\x03asr\x12\x02%s\x1a\x00", languageCode)
s = base64PadEnc(s)
return url.QueryEscape(s)
}
// GetPlaylist fetches playlist metadata
func (c *Client) GetPlaylist(url string) (*Playlist, error) {
return c.GetPlaylistContext(context.Background(), url)
}
// GetPlaylistContext fetches playlist metadata, with a context, along with a list of Videos, and some basic information
// for these videos. Playlist entries cannot be downloaded, as they lack all the required metadata, but
// can be used to enumerate all IDs, Authors, Titles, etc.
func (c *Client) GetPlaylistContext(ctx context.Context, url string) (*Playlist, error) {
c.assureClient()
id, err := extractPlaylistID(url)
if err != nil {
return nil, fmt.Errorf("extractPlaylistID failed: %w", err)
}
data := prepareInnertubePlaylistData(id, false, *c.client)
body, err := c.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+c.client.key, data)
if err != nil {
return nil, err
}
p := &Playlist{ID: id}
return p, p.parsePlaylistInfo(ctx, c, body)
}
func (c *Client) VideoFromPlaylistEntry(entry *PlaylistEntry) (*Video, error) {
return c.videoFromID(context.Background(), entry.ID)
}
func (c *Client) VideoFromPlaylistEntryContext(ctx context.Context, entry *PlaylistEntry) (*Video, error) {
return c.videoFromID(ctx, entry.ID)
}
// GetStream returns the stream and the total size for a specific format
func (c *Client) GetStream(video *Video, format *Format) (io.ReadCloser, int64, error) {
return c.GetStreamContext(context.Background(), video, format)
}
// GetStreamContext returns the stream and the total size for a specific format with a context.
func (c *Client) GetStreamContext(ctx context.Context, video *Video, format *Format) (io.ReadCloser, int64, error) {
url, err := c.GetStreamURL(video, format)
if err != nil {
return nil, 0, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, 0, err
}
r, w := io.Pipe()
contentLength := format.ContentLength
if contentLength == 0 {
// some videos don't have length information
contentLength = c.downloadOnce(req, w, format)
} else {
// we have length information, let's download by chunks!
c.downloadChunked(ctx, req, w, format)
}
return r, contentLength, nil
}
func (c *Client) downloadOnce(req *http.Request, w *io.PipeWriter, _ *Format) int64 {
resp, err := c.httpDo(req)
if err != nil {
w.CloseWithError(err) //nolint:errcheck
return 0
}
go func() {
defer resp.Body.Close()
_, err := io.Copy(w, resp.Body)
if err == nil {
w.Close()
} else {
w.CloseWithError(err) //nolint:errcheck
}
}()
contentLength := resp.Header.Get("Content-Length")
length, _ := strconv.ParseInt(contentLength, 10, 64)
return length
}
func (c *Client) getChunkSize() int64 {
if c.ChunkSize > 0 {
return c.ChunkSize
}
return Size10Mb
}
func (c *Client) getMaxRoutines(limit int) int {
routines := 10
if c.MaxRoutines > 0 {
routines = c.MaxRoutines
}
if limit > 0 && routines > limit {
routines = limit
}
return routines
}
func (c *Client) downloadChunked(ctx context.Context, req *http.Request, w *io.PipeWriter, format *Format) {
chunks := getChunks(format.ContentLength, c.getChunkSize())
maxRoutines := c.getMaxRoutines(len(chunks))
cancelCtx, cancel := context.WithCancel(ctx)
abort := func(err error) {
w.CloseWithError(err)
cancel()
}
currentChunk := atomic.Uint32{}
for i := 0; i < maxRoutines; i++ {
go func() {
for {
chunkIndex := int(currentChunk.Add(1)) - 1
if chunkIndex >= len(chunks) {
// no more chunks
return
}
chunk := &chunks[chunkIndex]
err := c.downloadChunk(req.Clone(cancelCtx), chunk)
close(chunk.data)
if err != nil {
abort(err)
return
}
}
}()
}
go func() {
// copy chunks into the PipeWriter
for i := 0; i < len(chunks); i++ {
select {
case <-cancelCtx.Done():
abort(context.Canceled)
return
case data := <-chunks[i].data:
_, err := io.Copy(w, bytes.NewBuffer(data))
if err != nil {
abort(err)
}
}
}
// everything succeeded
w.Close()
}()
}
// GetStreamURL returns the url for a specific format
func (c *Client) GetStreamURL(video *Video, format *Format) (string, error) {
return c.GetStreamURLContext(context.Background(), video, format)
}
// GetStreamURLContext returns the url for a specific format with a context
func (c *Client) GetStreamURLContext(ctx context.Context, video *Video, format *Format) (string, error) {
if format == nil {
return "", ErrNoFormat
}
c.assureClient()
if format.URL != "" {
if c.client.androidVersion > 0 {
return format.URL, nil
}
return c.unThrottle(ctx, video.ID, format.URL)
}
// TODO: check rest of this function, is it redundant?
cipher := format.Cipher
if cipher == "" {
return "", ErrCipherNotFound
}
uri, err := c.decipherURL(ctx, video.ID, cipher)
if err != nil {
return "", err
}
return uri, err
}
// httpDo sends an HTTP request and returns an HTTP response.
func (c *Client) httpDo(req *http.Request) (*http.Response, error) {
client := c.HTTPClient
if client == nil {
client = http.DefaultClient
}
req.Header.Set("User-Agent", c.client.userAgent)
req.Header.Set("Origin", "https://youtube.com")
req.Header.Set("Sec-Fetch-Mode", "navigate")
if len(c.consentID) == 0 {
c.consentID = strconv.Itoa(rand.Intn(899) + 100) //nolint:gosec
}
req.AddCookie(&http.Cookie{
Name: "CONSENT",
Value: "YES+cb.20210328-17-p0.en+FX+" + c.consentID,
Path: "/",
Domain: ".youtube.com",
})
res, err := client.Do(req)
log := slog.With("method", req.Method, "url", req.URL)
if err != nil {
log.Debug("HTTP request failed", "error", err)
} else {
log.Debug("HTTP request succeeded", "status", res.Status)
}
return res, err
}
// httpGet does a HTTP GET request, checks the response to be a 200 OK and returns it
func (c *Client) httpGet(ctx context.Context, url string) (*http.Response, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
if err != nil {
return nil, err
}
resp, err := c.httpDo(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, ErrUnexpectedStatusCode(resp.StatusCode)
}
return resp, nil
}
// httpGetBodyBytes reads the whole HTTP body and returns it
func (c *Client) httpGetBodyBytes(ctx context.Context, url string) ([]byte, error) {
resp, err := c.httpGet(ctx, url)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// httpPost does a HTTP POST request with a body, checks the response to be a 200 OK and returns it
func (c *Client) httpPost(ctx context.Context, url string, body interface{}) (*http.Response, error) {
data, err := json.Marshal(body)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(ctx, http.MethodPost, url, bytes.NewReader(data))
if err != nil {
return nil, err
}
req.Header.Set("X-Youtube-Client-Name", "3")
req.Header.Set("X-Youtube-Client-Version", c.client.version)
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
resp, err := c.httpDo(req)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
resp.Body.Close()
return nil, ErrUnexpectedStatusCode(resp.StatusCode)
}
return resp, nil
}
// httpPostBodyBytes reads the whole HTTP body and returns it
func (c *Client) httpPostBodyBytes(ctx context.Context, url string, body interface{}) ([]byte, error) {
resp, err := c.httpPost(ctx, url, body)
if err != nil {
return nil, err
}
defer resp.Body.Close()
return io.ReadAll(resp.Body)
}
// downloadChunk writes the response data into the data channel of the chunk.
// Downloading in multiple chunks is much faster:
// https://github.com/kkdai/youtube/pull/190
func (c *Client) downloadChunk(req *http.Request, chunk *chunk) error {
q := req.URL.Query()
q.Set("range", fmt.Sprintf("%d-%d", chunk.start, chunk.end))
req.URL.RawQuery = q.Encode()
resp, err := c.httpDo(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode < http.StatusOK && resp.StatusCode >= 300 {
return ErrUnexpectedStatusCode(resp.StatusCode)
}
expected := int(chunk.end-chunk.start) + 1
data, err := io.ReadAll(resp.Body)
n := len(data)
if err != nil {
return err
}
if n != expected {
return fmt.Errorf("chunk at offset %d has invalid size: expected=%d actual=%d", chunk.start, expected, n)
}
chunk.data <- data
return nil
}

View File

@@ -0,0 +1,285 @@
package youtube
import (
"bytes"
"context"
"errors"
"fmt"
"net/url"
"regexp"
"strconv"
"github.com/dop251/goja"
)
func (c *Client) decipherURL(ctx context.Context, videoID string, cipher string) (string, error) {
params, err := url.ParseQuery(cipher)
if err != nil {
return "", err
}
uri, err := url.Parse(params.Get("url"))
if err != nil {
return "", err
}
query := uri.Query()
config, err := c.getPlayerConfig(ctx, videoID)
if err != nil {
return "", err
}
// decrypt s-parameter
bs, err := config.decrypt([]byte(params.Get("s")))
if err != nil {
return "", err
}
query.Add(params.Get("sp"), string(bs))
query, err = c.decryptNParam(config, query)
if err != nil {
return "", err
}
uri.RawQuery = query.Encode()
return uri.String(), nil
}
// see https://github.com/kkdai/youtube/pull/244
func (c *Client) unThrottle(ctx context.Context, videoID string, urlString string) (string, error) {
config, err := c.getPlayerConfig(ctx, videoID)
if err != nil {
return "", err
}
uri, err := url.Parse(urlString)
if err != nil {
return "", err
}
// for debugging
if artifactsFolder != "" {
writeArtifact("video-"+videoID+".url", []byte(uri.String()))
}
query, err := c.decryptNParam(config, uri.Query())
if err != nil {
return "", err
}
uri.RawQuery = query.Encode()
return uri.String(), nil
}
func (c *Client) decryptNParam(config playerConfig, query url.Values) (url.Values, error) {
// decrypt n-parameter
nSig := query.Get("v")
log := Logger.With("n", nSig)
if nSig != "" {
nDecoded, err := config.decodeNsig(nSig)
if err != nil {
return nil, fmt.Errorf("unable to decode nSig: %w", err)
}
query.Set("v", nDecoded)
log = log.With("decoded", nDecoded)
}
log.Debug("nParam")
return query, nil
}
const (
jsvarStr = "[a-zA-Z_\\$][a-zA-Z_0-9]*"
reverseStr = ":function\\(a\\)\\{" +
"(?:return )?a\\.reverse\\(\\)" +
"\\}"
spliceStr = ":function\\(a,b\\)\\{" +
"a\\.splice\\(0,b\\)" +
"\\}"
swapStr = ":function\\(a,b\\)\\{" +
"var c=a\\[0\\];a\\[0\\]=a\\[b(?:%a\\.length)?\\];a\\[b(?:%a\\.length)?\\]=c(?:;return a)?" +
"\\}"
)
var (
nFunctionNameRegexp = regexp.MustCompile("\\.get\\(\"n\"\\)\\)&&\\(b=([a-zA-Z0-9$]{0,3})\\[(\\d+)\\](.+)\\|\\|([a-zA-Z0-9]{0,3})")
actionsObjRegexp = regexp.MustCompile(fmt.Sprintf(
"var (%s)=\\{((?:(?:%s%s|%s%s|%s%s),?\\n?)+)\\};", jsvarStr, jsvarStr, swapStr, jsvarStr, spliceStr, jsvarStr, reverseStr))
actionsFuncRegexp = regexp.MustCompile(fmt.Sprintf(
"function(?: %s)?\\(a\\)\\{"+
"a=a\\.split\\(\"\"\\);\\s*"+
"((?:(?:a=)?%s\\.%s\\(a,\\d+\\);)+)"+
"return a\\.join\\(\"\"\\)"+
"\\}", jsvarStr, jsvarStr, jsvarStr))
reverseRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, reverseStr))
spliceRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, spliceStr))
swapRegexp = regexp.MustCompile(fmt.Sprintf("(?m)(?:^|,)(%s)%s", jsvarStr, swapStr))
)
func (config playerConfig) decodeNsig(encoded string) (string, error) {
fBody, err := config.getNFunction()
if err != nil {
return "", err
}
return evalJavascript(fBody, encoded)
}
func evalJavascript(jsFunction, arg string) (string, error) {
const myName = "myFunction"
vm := goja.New()
_, err := vm.RunString(myName + "=" + jsFunction)
if err != nil {
return "", err
}
var output func(string) string
err = vm.ExportTo(vm.Get(myName), &output)
if err != nil {
return "", err
}
return output(arg), nil
}
func (config playerConfig) getNFunction() (string, error) {
nameResult := nFunctionNameRegexp.FindSubmatch(config)
if len(nameResult) == 0 {
return "", errors.New("unable to extract n-function name")
}
var name string
if idx, _ := strconv.Atoi(string(nameResult[2])); idx == 0 {
name = string(nameResult[4])
} else {
name = string(nameResult[1])
}
return config.extraFunction(name)
}
func (config playerConfig) extraFunction(name string) (string, error) {
// find the beginning of the function
def := []byte(name + "=function(")
start := bytes.Index(config, def)
if start < 1 {
return "", fmt.Errorf("unable to extract n-function body: looking for '%s'", def)
}
// start after the first curly bracket
pos := start + bytes.IndexByte(config[start:], '{') + 1
var strChar byte
// find the bracket closing the function
for brackets := 1; brackets > 0; pos++ {
b := config[pos]
switch b {
case '{':
if strChar == 0 {
brackets++
}
case '}':
if strChar == 0 {
brackets--
}
case '`', '"', '\'':
if config[pos-1] == '\\' && config[pos-2] != '\\' {
continue
}
if strChar == 0 {
strChar = b
} else if strChar == b {
strChar = 0
}
}
}
return string(config[start:pos]), nil
}
func (config playerConfig) decrypt(cyphertext []byte) ([]byte, error) {
operations, err := config.parseDecipherOps()
if err != nil {
return nil, err
}
// apply operations
bs := []byte(cyphertext)
for _, op := range operations {
bs = op(bs)
}
return bs, nil
}
/*
parses decipher operations from https://youtube.com/s/player/4fbb4d5b/player_ias.vflset/en_US/base.js
var Mt={
splice:function(a,b){a.splice(0,b)},
reverse:function(a){a.reverse()},
EQ:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c}};
a=a.split("");
Mt.splice(a,3);
Mt.EQ(a,39);
Mt.splice(a,2);
Mt.EQ(a,1);
Mt.splice(a,1);
Mt.EQ(a,35);
Mt.EQ(a,51);
Mt.splice(a,2);
Mt.reverse(a,52);
return a.join("")
*/
func (config playerConfig) parseDecipherOps() (operations []DecipherOperation, err error) {
objResult := actionsObjRegexp.FindSubmatch(config)
funcResult := actionsFuncRegexp.FindSubmatch(config)
if len(objResult) < 3 || len(funcResult) < 2 {
return nil, fmt.Errorf("error parsing signature tokens (#obj=%d, #func=%d)", len(objResult), len(funcResult))
}
obj := objResult[1]
objBody := objResult[2]
funcBody := funcResult[1]
var reverseKey, spliceKey, swapKey string
if result := reverseRegexp.FindSubmatch(objBody); len(result) > 1 {
reverseKey = string(result[1])
}
if result := spliceRegexp.FindSubmatch(objBody); len(result) > 1 {
spliceKey = string(result[1])
}
if result := swapRegexp.FindSubmatch(objBody); len(result) > 1 {
swapKey = string(result[1])
}
regex, err := regexp.Compile(fmt.Sprintf("(?:a=)?%s\\.(%s|%s|%s)\\(a,(\\d+)\\)", regexp.QuoteMeta(string(obj)), regexp.QuoteMeta(reverseKey), regexp.QuoteMeta(spliceKey), regexp.QuoteMeta(swapKey)))
if err != nil {
return nil, err
}
var ops []DecipherOperation
for _, s := range regex.FindAllSubmatch(funcBody, -1) {
switch string(s[1]) {
case reverseKey:
ops = append(ops, reverseFunc)
case swapKey:
arg, _ := strconv.Atoi(string(s[2]))
ops = append(ops, newSwapFunc(arg))
case spliceKey:
arg, _ := strconv.Atoi(string(s[2]))
ops = append(ops, newSpliceFunc(arg))
}
}
return ops, nil
}

View File

@@ -0,0 +1,27 @@
package youtube
type DecipherOperation func([]byte) []byte
func newSpliceFunc(pos int) DecipherOperation {
return func(bs []byte) []byte {
return bs[pos:]
}
}
func newSwapFunc(arg int) DecipherOperation {
return func(bs []byte) []byte {
pos := arg % len(bs)
bs[0], bs[pos] = bs[pos], bs[0]
return bs
}
}
func reverseFunc(bs []byte) []byte {
l, r := 0, len(bs)-1
for l < r {
bs[l], bs[r] = bs[r], bs[l]
l++
r--
}
return bs
}

4
downloader/vendor/github.com/kkdai/youtube/v2/doc.go generated vendored Normal file
View File

@@ -0,0 +1,4 @@
/*
Package youtube implement youtube download package in go.
*/
package youtube

View File

@@ -0,0 +1,203 @@
package downloader
import (
"context"
"errors"
"io"
"log/slog"
"os"
"os/exec"
"path/filepath"
"github.com/kkdai/youtube/v2"
"github.com/vbauerster/mpb/v5"
"github.com/vbauerster/mpb/v5/decor"
)
// Downloader offers high level functions to download videos into files
type Downloader struct {
youtube.Client
OutputDir string // optional directory to store the files
}
func (dl *Downloader) getOutputFile(v *youtube.Video, format *youtube.Format, outputFile string) (string, error) {
if outputFile == "" {
outputFile = SanitizeFilename(v.Title)
outputFile += pickIdealFileExtension(format.MimeType)
}
if dl.OutputDir != "" {
if err := os.MkdirAll(dl.OutputDir, 0o755); err != nil {
return "", err
}
outputFile = filepath.Join(dl.OutputDir, outputFile)
}
return outputFile, nil
}
// Download : Starting download video by arguments.
func (dl *Downloader) Download(ctx context.Context, v *youtube.Video, format *youtube.Format, outputFile string) error {
youtube.Logger.Info(
"Downloading video",
"id", v.ID,
"quality", format.Quality,
"mimeType", format.MimeType,
)
destFile, err := dl.getOutputFile(v, format, outputFile)
if err != nil {
return err
}
// Create output file
out, err := os.Create(destFile)
if err != nil {
return err
}
defer out.Close()
return dl.videoDLWorker(ctx, out, v, format)
}
// DownloadComposite : Downloads audio and video streams separately and merges them via ffmpeg.
func (dl *Downloader) DownloadComposite(ctx context.Context, outputFile string, v *youtube.Video, quality string, mimetype, language string) error {
videoFormat, audioFormat, err1 := getVideoAudioFormats(v, quality, mimetype, language)
if err1 != nil {
return err1
}
log := youtube.Logger.With("id", v.ID)
log.Info(
"Downloading composite video",
"videoQuality", videoFormat.QualityLabel,
"videoMimeType", videoFormat.MimeType,
"audioMimeType", audioFormat.MimeType,
)
destFile, err := dl.getOutputFile(v, videoFormat, outputFile)
if err != nil {
return err
}
outputDir := filepath.Dir(destFile)
// Create temporary video file
videoFile, err := os.CreateTemp(outputDir, "youtube_*.m4v")
if err != nil {
return err
}
defer Delete(videoFile.Name(), log)
// Create temporary audio file
audioFile, err := os.CreateTemp(outputDir, "youtube_*.m4a")
if err != nil {
return err
}
defer Delete(audioFile.Name(), log)
log.Debug("Downloading video file...")
err = dl.videoDLWorker(ctx, videoFile, v, videoFormat)
if err != nil {
return err
}
videoFile.Close()
log.Debug("Downloading audio file...")
err = dl.videoDLWorker(ctx, audioFile, v, audioFormat)
if err != nil {
return err
}
audioFile.Close()
//nolint:gosec
ffmpegVersionCmd := exec.Command("ffmpeg", "-y",
"-i", videoFile.Name(),
"-i", audioFile.Name(),
"-c", "copy", // Just copy without re-encoding
"-shortest", // Finish encoding when the shortest input stream ends
destFile,
"-loglevel", "warning",
)
ffmpegVersionCmd.Stderr = os.Stderr
ffmpegVersionCmd.Stdout = os.Stdout
log.Info("merging video and audio", "output", destFile)
return ffmpegVersionCmd.Run()
}
func Delete(path string, log *slog.Logger) {
err := os.Remove(path)
if err != nil {
log.Error("Failed deleting file", "path", path, "error", err)
}
}
func getVideoAudioFormats(v *youtube.Video, quality string, mimetype, language string) (*youtube.Format, *youtube.Format, error) {
var videoFormats, audioFormats youtube.FormatList
formats := v.Formats
if mimetype != "" {
formats = formats.Type(mimetype)
}
videoFormats = formats.Type("video").AudioChannels(0)
audioFormats = formats.Type("audio")
if quality != "" {
videoFormats = videoFormats.Quality(quality)
}
if language != "" {
audioFormats = audioFormats.Language(language)
}
if len(videoFormats) == 0 {
return nil, nil, errors.New("no video format found after filtering")
}
if len(audioFormats) == 0 {
return nil, nil, errors.New("no audio format found after filtering")
}
videoFormats.Sort()
audioFormats.Sort()
return &videoFormats[0], &audioFormats[0], nil
}
func (dl *Downloader) videoDLWorker(ctx context.Context, out *os.File, video *youtube.Video, format *youtube.Format) error {
stream, size, err := dl.GetStreamContext(ctx, video, format)
if err != nil {
return err
}
prog := &progress{
contentLength: float64(size),
}
// create progress bar
progress := mpb.New(mpb.WithWidth(64))
bar := progress.AddBar(
int64(prog.contentLength),
mpb.PrependDecorators(
decor.CountersKibiByte("% .2f / % .2f"),
decor.Percentage(decor.WCSyncSpace),
),
mpb.AppendDecorators(
decor.EwmaETA(decor.ET_STYLE_GO, 90),
decor.Name(" ] "),
decor.EwmaSpeed(decor.UnitKiB, "% .2f", 60),
),
)
reader := bar.ProxyReader(stream)
mw := io.MultiWriter(out, prog)
_, err = io.Copy(mw, reader)
if err != nil {
return err
}
progress.Wait()
return nil
}

View File

@@ -0,0 +1,65 @@
package downloader
import (
"mime"
"regexp"
)
const defaultExtension = ".mov"
// Rely on hardcoded canonical mime types, as the ones provided by Go aren't exhaustive [1].
// This seems to be a recurring problem for youtube downloaders, see [2].
// The implementation is based on mozilla's list [3], IANA [4] and Youtube's support [5].
// [1] https://github.com/golang/go/blob/ed7888aea6021e25b0ea58bcad3f26da2b139432/src/mime/type.go#L60
// [2] https://github.com/ZiTAL/youtube-dl/blob/master/mime.types
// [3] https://developer.mozilla.org/en-US/docs/Web/HTTP/Basics_of_HTTP/MIME_types/Common_types
// [4] https://www.iana.org/assignments/media-types/media-types.xhtml#video
// [5] https://support.google.com/youtube/troubleshooter/2888402?hl=en
var canonicals = map[string]string{
"video/quicktime": ".mov",
"video/x-msvideo": ".avi",
"video/x-matroska": ".mkv",
"video/mpeg": ".mpeg",
"video/webm": ".webm",
"video/3gpp2": ".3g2",
"video/x-flv": ".flv",
"video/3gpp": ".3gp",
"video/mp4": ".mp4",
"video/ogg": ".ogv",
"video/mp2t": ".ts",
}
func pickIdealFileExtension(mediaType string) string {
mediaType, _, err := mime.ParseMediaType(mediaType)
if err != nil {
return defaultExtension
}
if extension, ok := canonicals[mediaType]; ok {
return extension
}
// Our last resort is to ask the operating system, but these give multiple results and are rarely canonical.
extensions, err := mime.ExtensionsByType(mediaType)
if err != nil || extensions == nil {
return defaultExtension
}
return extensions[0]
}
func SanitizeFilename(fileName string) string {
// Characters not allowed on mac
// :/
// Characters not allowed on linux
// /
// Characters not allowed on windows
// <>:"/\|?*
// Ref https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file#naming-conventions
fileName = regexp.MustCompile(`[:/<>\:"\\|?*]`).ReplaceAllString(fileName, "")
fileName = regexp.MustCompile(`\s+`).ReplaceAllString(fileName, " ")
return fileName
}

View File

@@ -0,0 +1,17 @@
package downloader
type progress struct {
contentLength float64
totalWrittenBytes float64
downloadLevel float64
}
func (dl *progress) Write(p []byte) (n int, err error) {
n = len(p)
dl.totalWrittenBytes = dl.totalWrittenBytes + float64(n)
currentPercent := (dl.totalWrittenBytes / dl.contentLength) * 100
if (dl.downloadLevel <= currentPercent) && (dl.downloadLevel < 100) {
dl.downloadLevel++
}
return
}

View File

@@ -0,0 +1,47 @@
package youtube
import (
"fmt"
)
const (
ErrCipherNotFound = constError("cipher not found")
ErrSignatureTimestampNotFound = constError("signature timestamp not found")
ErrInvalidCharactersInVideoID = constError("invalid characters in video id")
ErrVideoIDMinLength = constError("the video id must be at least 10 characters long")
ErrReadOnClosedResBody = constError("http: read on closed response body")
ErrNotPlayableInEmbed = constError("embedding of this video has been disabled")
ErrLoginRequired = constError("login required to confirm your age")
ErrVideoPrivate = constError("user restricted access to this video")
ErrInvalidPlaylist = constError("no playlist detected or invalid playlist ID")
)
type constError string
func (e constError) Error() string {
return string(e)
}
type ErrPlayabiltyStatus struct {
Status string
Reason string
}
func (err ErrPlayabiltyStatus) Error() string {
return fmt.Sprintf("cannot playback and download, status: %s, reason: %s", err.Status, err.Reason)
}
// ErrUnexpectedStatusCode is returned on unexpected HTTP status codes
type ErrUnexpectedStatusCode int
func (err ErrUnexpectedStatusCode) Error() string {
return fmt.Sprintf("unexpected status code: %d", err)
}
type ErrPlaylistStatus struct {
Reason string
}
func (err ErrPlaylistStatus) Error() string {
return fmt.Sprintf("could not load playlist: %s", err.Reason)
}

View File

@@ -0,0 +1,35 @@
//go:build fetch
// +build fetch
package youtube
import (
"fmt"
"io"
"net/http"
"os"
)
func init() {
FetchTestData()
}
// ran via go generate to fetch and update the playlist response data
func FetchTestData() {
f, err := os.Create(testPlaylistResponseDataFile)
exitOnError(err)
requestURL := fmt.Sprintf(playlistFetchURL, testPlaylistID)
resp, err := http.Get(requestURL)
exitOnError(err)
defer resp.Body.Close()
n, err := io.Copy(f, resp.Body)
exitOnError(err)
fmt.Printf("Successfully fetched playlist %s (%d bytes)\n", testPlaylistID, n)
}
func exitOnError(err error) {
if err != nil {
fmt.Println(err)
os.Exit(1)
}
}

View File

@@ -0,0 +1,148 @@
package youtube
import (
"sort"
"strconv"
"strings"
)
type FormatList []Format
// Type returns a new FormatList filtered by itag
func (list FormatList) Select(f func(Format) bool) (result FormatList) {
for i := range list {
if f(list[i]) {
result = append(result, list[i])
}
}
return result
}
// Type returns a new FormatList filtered by itag
func (list FormatList) Itag(itagNo int) FormatList {
return list.Select(func(f Format) bool {
return f.ItagNo == itagNo
})
}
// Type returns a new FormatList filtered by mime type
func (list FormatList) Type(value string) FormatList {
return list.Select(func(f Format) bool {
return strings.Contains(f.MimeType, value)
})
}
// Type returns a new FormatList filtered by display name
func (list FormatList) Language(displayName string) FormatList {
return list.Select(func(f Format) bool {
return f.LanguageDisplayName() == displayName
})
}
// Quality returns a new FormatList filtered by quality, quality label or itag,
// but not audio quality
func (list FormatList) Quality(quality string) FormatList {
itag, _ := strconv.Atoi(quality)
return list.Select(func(f Format) bool {
return itag == f.ItagNo || strings.Contains(f.Quality, quality) || strings.Contains(f.QualityLabel, quality)
})
}
// AudioChannels returns a new FormatList filtered by the matching AudioChannels
func (list FormatList) AudioChannels(n int) FormatList {
return list.Select(func(f Format) bool {
return f.AudioChannels == n
})
}
// AudioChannels returns a new FormatList filtered by the matching AudioChannels
func (list FormatList) WithAudioChannels() FormatList {
return list.Select(func(f Format) bool {
return f.AudioChannels > 0
})
}
// FilterQuality reduces the format list to formats matching the quality
func (v *Video) FilterQuality(quality string) {
v.Formats = v.Formats.Quality(quality)
v.Formats.Sort()
}
// Sort sorts all formats fields
func (list FormatList) Sort() {
sort.SliceStable(list, func(i, j int) bool {
return sortFormat(i, j, list)
})
}
// sortFormat sorts video by resolution, FPS, codec (av01, vp9, avc1), bitrate
// sorts audio by default, codec (mp4, opus), channels, bitrate, sample rate
func sortFormat(i int, j int, formats FormatList) bool {
// Sort by Width
if formats[i].Width == formats[j].Width {
// Format 137 downloads slowly, give it less priority
// see https://github.com/kkdai/youtube/pull/171
switch 137 {
case formats[i].ItagNo:
return false
case formats[j].ItagNo:
return true
}
// Sort by FPS
if formats[i].FPS == formats[j].FPS {
if formats[i].FPS == 0 && formats[i].AudioChannels > 0 && formats[j].AudioChannels > 0 {
// Audio
// Sort by default
if (formats[i].AudioTrack == nil && formats[j].AudioTrack == nil) || (formats[i].AudioTrack != nil && formats[j].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault == formats[j].AudioTrack.AudioIsDefault) {
// Sort by codec
codec := map[int]int{}
for _, index := range []int{i, j} {
if strings.Contains(formats[index].MimeType, "mp4") {
codec[index] = 1
} else if strings.Contains(formats[index].MimeType, "opus") {
codec[index] = 2
}
}
if codec[i] == codec[j] {
// Sort by Audio Channel
if formats[i].AudioChannels == formats[j].AudioChannels {
// Sort by Audio Bitrate
if formats[i].Bitrate == formats[j].Bitrate {
// Sort by Audio Sample Rate
return formats[i].AudioSampleRate > formats[j].AudioSampleRate
}
return formats[i].Bitrate > formats[j].Bitrate
}
return formats[i].AudioChannels > formats[j].AudioChannels
}
return codec[i] < codec[j]
} else if formats[i].AudioTrack != nil && formats[i].AudioTrack.AudioIsDefault {
return true
}
return false
}
// Video
// Sort by codec
codec := map[int]int{}
for _, index := range []int{i, j} {
if strings.Contains(formats[index].MimeType, "av01") {
codec[index] = 1
} else if strings.Contains(formats[index].MimeType, "vp9") {
codec[index] = 2
} else if strings.Contains(formats[index].MimeType, "avc1") {
codec[index] = 3
}
}
if codec[i] == codec[j] {
// Sort by Audio Bitrate
return formats[i].Bitrate > formats[j].Bitrate
}
return codec[i] < codec[j]
}
return formats[i].FPS > formats[j].FPS
}
return formats[i].Width > formats[j].Width
}

View File

@@ -0,0 +1,28 @@
package youtube
import (
"fmt"
"log/slog"
"os"
)
// The global logger for all Client instances
var Logger = getLogger(os.Getenv("LOGLEVEL"))
func SetLogLevel(value string) {
Logger = getLogger(value)
}
func getLogger(logLevel string) *slog.Logger {
levelVar := slog.LevelVar{}
if logLevel != "" {
if err := levelVar.UnmarshalText([]byte(logLevel)); err != nil {
panic(fmt.Sprintf("Invalid log level %s: %v", logLevel, err))
}
}
return slog.New(slog.NewTextHandler(os.Stderr, &slog.HandlerOptions{
Level: levelVar.Level(),
}))
}

View File

@@ -0,0 +1,37 @@
package youtube
import (
"time"
)
const defaultCacheExpiration = time.Minute * time.Duration(5)
type playerCache struct {
key string
expiredAt time.Time
config playerConfig
}
// Get : get cache when it has same video id and not expired
func (s playerCache) Get(key string) playerConfig {
return s.GetCacheBefore(key, time.Now())
}
// GetCacheBefore : can pass time for testing
func (s playerCache) GetCacheBefore(key string, time time.Time) playerConfig {
if key == s.key && s.expiredAt.After(time) {
return s.config
}
return nil
}
// Set : set cache with default expiration
func (s *playerCache) Set(key string, operations playerConfig) {
s.setWithExpiredTime(key, operations, time.Now().Add(defaultCacheExpiration))
}
func (s *playerCache) setWithExpiredTime(key string, config playerConfig, time time.Time) {
s.key = key
s.config = config
s.expiredAt = time
}

View File

@@ -0,0 +1,59 @@
package youtube
import (
"context"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"regexp"
"strings"
)
type playerConfig []byte
var basejsPattern = regexp.MustCompile(`(/s/player/\w+/player_ias.vflset/\w+/base.js)`)
func (c *Client) getPlayerConfig(ctx context.Context, videoID string) (playerConfig, error) {
embedURL := fmt.Sprintf("https://youtube.com/embed/%s?hl=en", videoID)
embedBody, err := c.httpGetBodyBytes(ctx, embedURL)
if err != nil {
return nil, err
}
// example: /s/player/f676c671/player_ias.vflset/en_US/base.js
playerPath := string(basejsPattern.Find(embedBody))
if playerPath == "" {
return nil, errors.New("unable to find basejs URL in playerConfig")
}
// for debugging
var artifactName string
if artifactsFolder != "" {
parts := strings.SplitN(playerPath, "/", 5)
artifactName = "player-" + parts[3] + ".js"
linkName := filepath.Join(artifactsFolder, "video-"+videoID+".js")
if err := os.Symlink(artifactName, linkName); err != nil {
log.Printf("unable to create symlink %s: %v", linkName, err)
}
}
config := c.playerCache.Get(playerPath)
if config != nil {
return config, nil
}
config, err = c.httpGetBodyBytes(ctx, "https://youtube.com"+playerPath)
if err != nil {
return nil, err
}
// for debugging
if artifactName != "" {
writeArtifact(artifactName, config)
}
c.playerCache.Set(playerPath, config)
return config, nil
}

View File

@@ -0,0 +1,256 @@
package youtube
import (
"context"
"encoding/json"
"fmt"
"regexp"
"runtime/debug"
"strconv"
"time"
sjson "github.com/bitly/go-simplejson"
)
var (
playlistIDRegex = regexp.MustCompile("^[A-Za-z0-9_-]{13,42}$")
playlistInURLRegex = regexp.MustCompile("[&?]list=([A-Za-z0-9_-]{13,42})(&.*)?$")
)
type Playlist struct {
ID string
Title string
Description string
Author string
Videos []*PlaylistEntry
}
type PlaylistEntry struct {
ID string
Title string
Author string
Duration time.Duration
Thumbnails Thumbnails
}
func extractPlaylistID(url string) (string, error) {
if playlistIDRegex.Match([]byte(url)) {
return url, nil
}
matches := playlistInURLRegex.FindStringSubmatch(url)
if matches != nil {
return matches[1], nil
}
return "", ErrInvalidPlaylist
}
// structs for playlist extraction
// Title: metadata.playlistMetadataRenderer.title | sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.title.runs[0].text
// Description: metadata.playlistMetadataRenderer.description
// Author: sidebar.playlistSidebarRenderer.items[1].playlistSidebarSecondaryInfoRenderer.videoOwner.videoOwnerRenderer.title.runs[0].text
// Videos: contents.twoColumnBrowseResultsRenderer.tabs[0].tabRenderer.content.sectionListRenderer.contents[0].itemSectionRenderer.contents[0].playlistVideoListRenderer.contents
// ID: .videoId
// Title: title.runs[0].text
// Author: .shortBylineText.runs[0].text
// Duration: .lengthSeconds
// Thumbnails .thumbnails
// TODO?: Author thumbnails: sidebar.playlistSidebarRenderer.items[0].playlistSidebarPrimaryInfoRenderer.thumbnailRenderer.playlistVideoThumbnailRenderer.thumbnail.thumbnails
func (p *Playlist) parsePlaylistInfo(ctx context.Context, client *Client, body []byte) (err error) {
var j *sjson.Json
j, err = sjson.NewJson(body)
if err != nil {
return err
}
defer func() {
stack := debug.Stack()
if r := recover(); r != nil {
err = fmt.Errorf("JSON parsing error: %v\n%s", r, stack)
}
}()
renderer := j.GetPath("alerts").GetIndex(0).GetPath("alertRenderer")
if renderer != nil && renderer.GetPath("type").MustString() == "ERROR" {
message := renderer.GetPath("text", "runs").GetIndex(0).GetPath("text").MustString()
return ErrPlaylistStatus{Reason: message}
}
// Metadata can be located in multiple places depending on client type
var metadata *sjson.Json
if node, ok := j.CheckGet("metadata"); ok {
metadata = node
} else if node, ok := j.CheckGet("header"); ok {
metadata = node
} else {
return fmt.Errorf("no playlist header / metadata found")
}
metadata = metadata.Get("playlistHeaderRenderer")
p.Title = sjsonGetText(metadata, "title")
p.Description = sjsonGetText(metadata, "description", "descriptionText")
p.Author = j.GetPath("sidebar", "playlistSidebarRenderer", "items").GetIndex(1).
GetPath("playlistSidebarSecondaryInfoRenderer", "videoOwner", "videoOwnerRenderer", "title", "runs").
GetIndex(0).Get("text").MustString()
if len(p.Author) == 0 {
p.Author = sjsonGetText(metadata, "owner", "ownerText")
}
contents, ok := j.CheckGet("contents")
if !ok {
return fmt.Errorf("contents not found in json body")
}
// contents can have different keys with same child structure
firstPart := getFirstKeyJSON(contents).GetPath("tabs").GetIndex(0).
GetPath("tabRenderer", "content", "sectionListRenderer", "contents").GetIndex(0)
// This extra nested item is only set with the web client
if n := firstPart.GetPath("itemSectionRenderer", "contents").GetIndex(0); isValidJSON(n) {
firstPart = n
}
vJSON, err := firstPart.GetPath("playlistVideoListRenderer", "contents").MarshalJSON()
if err != nil {
return err
}
if len(vJSON) <= 4 {
return fmt.Errorf("no video data found in JSON")
}
entries, continuation, err := extractPlaylistEntries(vJSON)
if err != nil {
return err
}
if len(continuation) == 0 {
continuation = getContinuation(firstPart.Get("playlistVideoListRenderer"))
}
if len(entries) == 0 {
return fmt.Errorf("no videos found in playlist")
}
p.Videos = entries
for continuation != "" {
data := prepareInnertubePlaylistData(continuation, true, *client.client)
body, err := client.httpPostBodyBytes(ctx, "https://www.youtube.com/youtubei/v1/browse?key="+client.client.key, data)
if err != nil {
return err
}
j, err := sjson.NewJson(body)
if err != nil {
return err
}
next := j.GetPath("onResponseReceivedActions").GetIndex(0).
GetPath("appendContinuationItemsAction", "continuationItems")
if !isValidJSON(next) {
next = j.GetPath("continuationContents", "playlistVideoListContinuation", "contents")
}
vJSON, err := next.MarshalJSON()
if err != nil {
return err
}
entries, token, err := extractPlaylistEntries(vJSON)
if err != nil {
return err
}
if len(token) > 0 {
continuation = token
} else {
continuation = getContinuation(j.GetPath("continuationContents", "playlistVideoListContinuation"))
}
p.Videos = append(p.Videos, entries...)
}
return err
}
func extractPlaylistEntries(data []byte) ([]*PlaylistEntry, string, error) {
var vids []*videosJSONExtractor
if err := json.Unmarshal(data, &vids); err != nil {
return nil, "", err
}
entries := make([]*PlaylistEntry, 0, len(vids))
var continuation string
for _, v := range vids {
if v.Renderer == nil {
if v.Continuation.Endpoint.Command.Token != "" {
continuation = v.Continuation.Endpoint.Command.Token
}
continue
}
entries = append(entries, v.PlaylistEntry())
}
return entries, continuation, nil
}
type videosJSONExtractor struct {
Renderer *struct {
ID string `json:"videoId"`
Title withRuns `json:"title"`
Author withRuns `json:"shortBylineText"`
Duration string `json:"lengthSeconds"`
Thumbnail struct {
Thumbnails []Thumbnail `json:"thumbnails"`
} `json:"thumbnail"`
} `json:"playlistVideoRenderer"`
Continuation struct {
Endpoint struct {
Command struct {
Token string `json:"token"`
} `json:"continuationCommand"`
} `json:"continuationEndpoint"`
} `json:"continuationItemRenderer"`
}
func (vje videosJSONExtractor) PlaylistEntry() *PlaylistEntry {
ds, err := strconv.Atoi(vje.Renderer.Duration)
if err != nil {
panic("invalid video duration: " + vje.Renderer.Duration)
}
return &PlaylistEntry{
ID: vje.Renderer.ID,
Title: vje.Renderer.Title.String(),
Author: vje.Renderer.Author.String(),
Duration: time.Second * time.Duration(ds),
Thumbnails: vje.Renderer.Thumbnail.Thumbnails,
}
}
type withRuns struct {
Runs []struct {
Text string `json:"text"`
} `json:"runs"`
}
func (wr withRuns) String() string {
if len(wr.Runs) > 0 {
return wr.Runs[0].Text
}
return ""
}

View File

@@ -0,0 +1,153 @@
package youtube
type playerResponseData struct {
Captions struct {
PlayerCaptionsTracklistRenderer struct {
CaptionTracks []CaptionTrack `json:"captionTracks"`
AudioTracks []struct {
CaptionTrackIndices []int `json:"captionTrackIndices"`
} `json:"audioTracks"`
TranslationLanguages []struct {
LanguageCode string `json:"languageCode"`
LanguageName struct {
SimpleText string `json:"simpleText"`
} `json:"languageName"`
} `json:"translationLanguages"`
DefaultAudioTrackIndex int `json:"defaultAudioTrackIndex"`
} `json:"playerCaptionsTracklistRenderer"`
} `json:"captions"`
PlayabilityStatus struct {
Status string `json:"status"`
Reason string `json:"reason"`
PlayableInEmbed bool `json:"playableInEmbed"`
Miniplayer struct {
MiniplayerRenderer struct {
PlaybackMode string `json:"playbackMode"`
} `json:"miniplayerRenderer"`
} `json:"miniplayer"`
ContextParams string `json:"contextParams"`
} `json:"playabilityStatus"`
StreamingData struct {
ExpiresInSeconds string `json:"expiresInSeconds"`
Formats []Format `json:"formats"`
AdaptiveFormats []Format `json:"adaptiveFormats"`
DashManifestURL string `json:"dashManifestUrl"`
HlsManifestURL string `json:"hlsManifestUrl"`
} `json:"streamingData"`
VideoDetails struct {
VideoID string `json:"videoId"`
Title string `json:"title"`
LengthSeconds string `json:"lengthSeconds"`
Keywords []string `json:"keywords"`
ChannelID string `json:"channelId"`
IsOwnerViewing bool `json:"isOwnerViewing"`
ShortDescription string `json:"shortDescription"`
IsCrawlable bool `json:"isCrawlable"`
Thumbnail struct {
Thumbnails []Thumbnail `json:"thumbnails"`
} `json:"thumbnail"`
AverageRating float64 `json:"averageRating"`
AllowRatings bool `json:"allowRatings"`
ViewCount string `json:"viewCount"`
Author string `json:"author"`
IsPrivate bool `json:"isPrivate"`
IsUnpluggedCorpus bool `json:"isUnpluggedCorpus"`
IsLiveContent bool `json:"isLiveContent"`
} `json:"videoDetails"`
Microformat struct {
PlayerMicroformatRenderer struct {
Thumbnail struct {
Thumbnails []struct {
URL string `json:"url"`
Width int `json:"width"`
Height int `json:"height"`
} `json:"thumbnails"`
} `json:"thumbnail"`
Title struct {
SimpleText string `json:"simpleText"`
} `json:"title"`
Description struct {
SimpleText string `json:"simpleText"`
} `json:"description"`
LengthSeconds string `json:"lengthSeconds"`
OwnerProfileURL string `json:"ownerProfileUrl"`
ExternalChannelID string `json:"externalChannelId"`
IsFamilySafe bool `json:"isFamilySafe"`
AvailableCountries []string `json:"availableCountries"`
IsUnlisted bool `json:"isUnlisted"`
HasYpcMetadata bool `json:"hasYpcMetadata"`
ViewCount string `json:"viewCount"`
Category string `json:"category"`
PublishDate string `json:"publishDate"`
OwnerChannelName string `json:"ownerChannelName"`
UploadDate string `json:"uploadDate"`
} `json:"playerMicroformatRenderer"`
} `json:"microformat"`
}
type Format struct {
ItagNo int `json:"itag"`
URL string `json:"url"`
MimeType string `json:"mimeType"`
Quality string `json:"quality"`
Cipher string `json:"signatureCipher"`
Bitrate int `json:"bitrate"`
FPS int `json:"fps"`
Width int `json:"width"`
Height int `json:"height"`
LastModified string `json:"lastModified"`
ContentLength int64 `json:"contentLength,string"`
QualityLabel string `json:"qualityLabel"`
ProjectionType string `json:"projectionType"`
AverageBitrate int `json:"averageBitrate"`
AudioQuality string `json:"audioQuality"`
ApproxDurationMs string `json:"approxDurationMs"`
AudioSampleRate string `json:"audioSampleRate"`
AudioChannels int `json:"audioChannels"`
// InitRange is only available for adaptive formats
InitRange *struct {
Start string `json:"start"`
End string `json:"end"`
} `json:"initRange"`
// IndexRange is only available for adaptive formats
IndexRange *struct {
Start string `json:"start"`
End string `json:"end"`
} `json:"indexRange"`
// AudioTrack is only available for videos with multiple audio track languages
AudioTrack *struct {
DisplayName string `json:"displayName"`
ID string `json:"id"`
AudioIsDefault bool `json:"audioIsDefault"`
}
}
func (f *Format) LanguageDisplayName() string {
if f.AudioTrack == nil {
return ""
}
return f.AudioTrack.DisplayName
}
type Thumbnails []Thumbnail
type Thumbnail struct {
URL string
Width uint
Height uint
}
type CaptionTrack struct {
BaseURL string `json:"baseUrl"`
Name struct {
SimpleText string `json:"simpleText"`
} `json:"name"`
VssID string `json:"vssId"`
LanguageCode string `json:"languageCode"`
Kind string `json:"kind,omitempty"`
IsTranslatable bool `json:"isTranslatable"`
}

View File

@@ -0,0 +1,214 @@
package youtube
import (
"context"
"encoding/json"
"errors"
"fmt"
"strconv"
"strings"
)
var (
ErrTranscriptDisabled = errors.New("transcript is disabled on this video")
)
// TranscriptSegment is a single transcipt segment spanning a few milliseconds.
type TranscriptSegment struct {
// Text is the transcipt text.
Text string `json:"text"`
// StartMs is the start timestamp in ms.
StartMs int `json:"offset"`
// OffsetText e.g. '4:00'.
OffsetText string `json:"offsetText"`
// Duration the transcript segment spans in ms.
Duration int `json:"duration"`
}
func (tr TranscriptSegment) String() string {
return tr.OffsetText + " - " + strings.TrimSpace(tr.Text)
}
type VideoTranscript []TranscriptSegment
func (vt VideoTranscript) String() string {
var str string
for _, tr := range vt {
str += tr.String() + "\n"
}
return str
}
// GetTranscript fetches the video transcript if available.
//
// Not all videos have transcripts, only relatively new videos.
// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
func (c *Client) GetTranscript(video *Video, lang string) (VideoTranscript, error) {
return c.GetTranscriptCtx(context.Background(), video, lang)
}
// GetTranscriptCtx fetches the video transcript if available.
//
// Not all videos have transcripts, only relatively new videos.
// If transcripts are disabled or not available, ErrTranscriptDisabled is returned.
func (c *Client) GetTranscriptCtx(ctx context.Context, video *Video, lang string) (VideoTranscript, error) {
c.assureClient()
if video == nil || video.ID == "" {
return nil, fmt.Errorf("no video provided")
}
body, err := c.transcriptDataByInnertube(ctx, video.ID, lang)
if err != nil {
return nil, err
}
transcript, err := parseTranscript(body)
if err != nil {
return nil, err
}
return transcript, nil
}
func parseTranscript(body []byte) (VideoTranscript, error) {
var resp transcriptResp
if err := json.Unmarshal(body, &resp); err != nil {
return nil, err
}
if len(resp.Actions) > 0 {
// Android client response
if app := resp.Actions[0].AppSegment; app != nil {
return getSegments(app)
}
// Web client response
if web := resp.Actions[0].WebSegment; web != nil {
return nil, fmt.Errorf("not implemented")
}
}
return nil, ErrTranscriptDisabled
}
type segmenter interface {
ParseSegments() []TranscriptSegment
}
func getSegments(f segmenter) (VideoTranscript, error) {
if segments := f.ParseSegments(); len(segments) > 0 {
return segments, nil
}
return nil, ErrTranscriptDisabled
}
// transcriptResp is the JSON structure as returned by the transcript API.
type transcriptResp struct {
Actions []struct {
AppSegment *appData `json:"elementsCommand"`
WebSegment *webData `json:"updateEngagementPanelAction"`
} `json:"actions"`
}
type appData struct {
TEC struct {
Args struct {
ListArgs struct {
Ow struct {
InitialSeg []struct {
TranscriptSegment struct {
StartMs string `json:"startMs"`
EndMs string `json:"endMs"`
Text struct {
String struct {
// Content is the actual transctipt text
Content string `json:"content"`
} `json:"elementsAttributedString"`
} `json:"snippet"`
StartTimeText struct {
String struct {
// Content is the fomratted timestamp, e.g. '4:00'
Content string `json:"content"`
} `json:"elementsAttributedString"`
} `json:"startTimeText"`
} `json:"transcriptSegmentRenderer"`
} `json:"initialSegments"`
} `json:"overwrite"`
} `json:"transformTranscriptSegmentListArguments"`
} `json:"arguments"`
} `json:"transformEntityCommand"`
}
func (s *appData) ParseSegments() []TranscriptSegment {
rawSegments := s.TEC.Args.ListArgs.Ow.InitialSeg
segments := make([]TranscriptSegment, 0, len(rawSegments))
for _, segment := range rawSegments {
startMs, _ := strconv.Atoi(segment.TranscriptSegment.StartMs)
endMs, _ := strconv.Atoi(segment.TranscriptSegment.EndMs)
segments = append(segments, TranscriptSegment{
Text: segment.TranscriptSegment.Text.String.Content,
StartMs: startMs,
OffsetText: segment.TranscriptSegment.StartTimeText.String.Content,
Duration: endMs - startMs,
})
}
return segments
}
type webData struct {
Content struct {
TR struct {
Body struct {
TBR struct {
Cues []struct {
Transcript struct {
FormattedStartOffset struct {
SimpleText string `json:"simpleText"`
} `json:"formattedStartOffset"`
Cues []struct {
TranscriptCueRenderer struct {
Cue struct {
SimpleText string `json:"simpleText"`
} `json:"cue"`
StartOffsetMs string `json:"startOffsetMs"`
DurationMs string `json:"durationMs"`
} `json:"transcriptCueRenderer"`
} `json:"cues"`
} `json:"transcriptCueGroupRenderer"`
} `json:"cueGroups"`
} `json:"transcriptSearchPanelRenderer"`
} `json:"content"`
} `json:"transcriptRenderer"`
} `json:"content"`
}
func (s *webData) ParseSegments() []TranscriptSegment {
// TODO: doesn't actually work now, check json.
cues := s.Content.TR.Body.TBR.Cues
segments := make([]TranscriptSegment, 0, len(cues))
for _, s := range cues {
formatted := s.Transcript.FormattedStartOffset.SimpleText
segment := s.Transcript.Cues[0].TranscriptCueRenderer
start, _ := strconv.Atoi(segment.StartOffsetMs)
duration, _ := strconv.Atoi(segment.DurationMs)
segments = append(segments, TranscriptSegment{
Text: segment.Cue.SimpleText,
StartMs: start,
OffsetText: formatted,
Duration: duration,
})
}
return segments
}

97
downloader/vendor/github.com/kkdai/youtube/v2/utils.go generated vendored Normal file
View File

@@ -0,0 +1,97 @@
package youtube
import (
"encoding/base64"
sjson "github.com/bitly/go-simplejson"
)
type chunk struct {
start int64
end int64
data chan []byte
}
func getChunks(totalSize, chunkSize int64) []chunk {
var chunks []chunk
for start := int64(0); start < totalSize; start += chunkSize {
end := chunkSize + start - 1
if end > totalSize-1 {
end = totalSize - 1
}
chunks = append(chunks, chunk{start, end, make(chan []byte, 1)})
}
return chunks
}
func getFirstKeyJSON(j *sjson.Json) *sjson.Json {
m, err := j.Map()
if err != nil {
return j
}
for key := range m {
return j.Get(key)
}
return j
}
func isValidJSON(j *sjson.Json) bool {
b, err := j.MarshalJSON()
if err != nil {
return false
}
if len(b) <= 4 {
return false
}
return true
}
func sjsonGetText(j *sjson.Json, paths ...string) string {
for _, path := range paths {
if isValidJSON(j.Get(path)) {
j = j.Get(path)
}
}
if text, err := j.String(); err == nil {
return text
}
if isValidJSON(j.Get("text")) {
return j.Get("text").MustString()
}
if p := j.Get("runs"); isValidJSON(p) {
var text string
for i := 0; i < len(p.MustArray()); i++ {
if textNode := p.GetIndex(i).Get("text"); isValidJSON(textNode) {
text += textNode.MustString()
}
}
return text
}
return ""
}
func getContinuation(j *sjson.Json) string {
return j.GetPath("continuations").
GetIndex(0).GetPath("nextContinuationData", "continuation").MustString()
}
func base64PadEnc(str string) string {
return base64.StdEncoding.EncodeToString([]byte(str))
}
func base64Enc(str string) string {
return base64.RawStdEncoding.EncodeToString([]byte(str))
}

147
downloader/vendor/github.com/kkdai/youtube/v2/video.go generated vendored Normal file
View File

@@ -0,0 +1,147 @@
package youtube
import (
"encoding/json"
"errors"
"fmt"
"net/url"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
type Video struct {
ID string
Title string
Description string
Author string
ChannelID string
ChannelHandle string
Views int
Duration time.Duration
PublishDate time.Time
Formats FormatList
Thumbnails Thumbnails
DASHManifestURL string // URI of the DASH manifest file
HLSManifestURL string // URI of the HLS manifest file
CaptionTracks []CaptionTrack
}
const dateFormat = "2006-01-02"
func (v *Video) parseVideoInfo(body []byte) error {
var prData playerResponseData
if err := json.Unmarshal(body, &prData); err != nil {
return fmt.Errorf("unable to parse player response JSON: %w", err)
}
if err := v.isVideoFromInfoDownloadable(prData); err != nil {
return err
}
return v.extractDataFromPlayerResponse(prData)
}
func (v *Video) isVideoFromInfoDownloadable(prData playerResponseData) error {
return v.isVideoDownloadable(prData, false)
}
var playerResponsePattern = regexp.MustCompile(`var ytInitialPlayerResponse\s*=\s*(\{.+?\});`)
func (v *Video) parseVideoPage(body []byte) error {
initialPlayerResponse := playerResponsePattern.FindSubmatch(body)
if initialPlayerResponse == nil || len(initialPlayerResponse) < 2 {
return errors.New("no ytInitialPlayerResponse found in the server's answer")
}
var prData playerResponseData
if err := json.Unmarshal(initialPlayerResponse[1], &prData); err != nil {
return fmt.Errorf("unable to parse player response JSON: %w", err)
}
if err := v.isVideoFromPageDownloadable(prData); err != nil {
return err
}
return v.extractDataFromPlayerResponse(prData)
}
func (v *Video) isVideoFromPageDownloadable(prData playerResponseData) error {
return v.isVideoDownloadable(prData, true)
}
func (v *Video) isVideoDownloadable(prData playerResponseData, isVideoPage bool) error {
// Check if video is downloadable
switch prData.PlayabilityStatus.Status {
case "OK":
return nil
case "LOGIN_REQUIRED":
// for some reason they use same status message for age-restricted and private videos
if strings.HasPrefix(prData.PlayabilityStatus.Reason, "This video is private") {
return ErrVideoPrivate
}
return ErrLoginRequired
}
if !isVideoPage && !prData.PlayabilityStatus.PlayableInEmbed {
return ErrNotPlayableInEmbed
}
return &ErrPlayabiltyStatus{
Status: prData.PlayabilityStatus.Status,
Reason: prData.PlayabilityStatus.Reason,
}
}
func (v *Video) extractDataFromPlayerResponse(prData playerResponseData) error {
v.Title = prData.VideoDetails.Title
v.Description = prData.VideoDetails.ShortDescription
v.Author = prData.VideoDetails.Author
v.Thumbnails = prData.VideoDetails.Thumbnail.Thumbnails
v.ChannelID = prData.VideoDetails.ChannelID
v.CaptionTracks = prData.Captions.PlayerCaptionsTracklistRenderer.CaptionTracks
if views, _ := strconv.Atoi(prData.VideoDetails.ViewCount); views > 0 {
v.Views = views
}
if seconds, _ := strconv.Atoi(prData.VideoDetails.LengthSeconds); seconds > 0 {
v.Duration = time.Duration(seconds) * time.Second
}
if seconds, _ := strconv.Atoi(prData.Microformat.PlayerMicroformatRenderer.LengthSeconds); seconds > 0 {
v.Duration = time.Duration(seconds) * time.Second
}
if str := prData.Microformat.PlayerMicroformatRenderer.PublishDate; str != "" {
v.PublishDate, _ = time.Parse(dateFormat, str)
}
if profileURL, err := url.Parse(prData.Microformat.PlayerMicroformatRenderer.OwnerProfileURL); err == nil && len(profileURL.Path) > 1 {
v.ChannelHandle = profileURL.Path[1:]
}
// Assign Streams
v.Formats = append(prData.StreamingData.Formats, prData.StreamingData.AdaptiveFormats...)
if len(v.Formats) == 0 {
return errors.New("no formats found in the server's answer")
}
// Sort formats by bitrate
sort.SliceStable(v.Formats, v.SortBitrateDesc)
v.HLSManifestURL = prData.StreamingData.HlsManifestURL
v.DASHManifestURL = prData.StreamingData.DashManifestURL
return nil
}
func (v *Video) SortBitrateDesc(i int, j int) bool {
return v.Formats[i].Bitrate > v.Formats[j].Bitrate
}
func (v *Video) SortBitrateAsc(i int, j int) bool {
return v.Formats[i].Bitrate < v.Formats[j].Bitrate
}

View File

@@ -0,0 +1,34 @@
package youtube
import (
"regexp"
"strings"
)
var videoRegexpList = []*regexp.Regexp{
regexp.MustCompile(`(?:v|embed|shorts|watch\?v)(?:=|/)([^"&?/=%]{11})`),
regexp.MustCompile(`(?:=|/)([^"&?/=%]{11})`),
regexp.MustCompile(`([^"&?/=%]{11})`),
}
// ExtractVideoID extracts the videoID from the given string
func ExtractVideoID(videoID string) (string, error) {
if strings.Contains(videoID, "youtu") || strings.ContainsAny(videoID, "\"?&/<%=") {
for _, re := range videoRegexpList {
if isMatch := re.MatchString(videoID); isMatch {
subs := re.FindStringSubmatch(videoID)
videoID = subs[1]
}
}
}
if strings.ContainsAny(videoID, "?&/<%=") {
return "", ErrInvalidCharactersInVideoID
}
if len(videoID) < 10 {
return "", ErrVideoIDMinLength
}
return videoID, nil
}

View File

@@ -13,99 +13,131 @@ const IDLE_TIMEOUT = TIMEOUT * time.Second
const PING_INTERVAL = (TIMEOUT / 2) * time.Second
type WSConnection struct {
alive bool
url string
conn *websocket.Conn
errChan chan error
writeLock sync.Mutex
ReadChan chan string
WriteChan chan string
Dead chan error
}
func (ws *WSConnection) messageReader() {
log.Printf("Reading messages")
log.Printf("Starting reader")
for {
if !ws.alive {
break
}
_, message, err := ws.conn.ReadMessage()
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
if err != nil {
ws.errChan <- err
return
break
}
log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan))
ws.ReadChan <- string(message)
}
log.Printf("Reader done")
}
func (ws *WSConnection) messageSender() {
log.Printf("Sending messages")
log.Printf("Starting sender")
for {
msg := <-ws.WriteChan
ws.writeLock.Lock()
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
if err != nil {
log.Printf("Error during message writing: %v", err)
ws.errChan <- err
return
msg, ok := <-ws.WriteChan
if !ok {
break
}
ws.writeLock.Unlock()
ws.doSend(msg)
}
log.Printf("Sender done")
}
func (ws *WSConnection) doSend(msg string) {
ws.writeLock.Lock()
defer ws.writeLock.Unlock()
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
if err != nil {
log.Printf("Error during message writing: %v", err)
ws.errChan <- err
return
}
}
func (ws *WSConnection) pinger() {
log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL)
for {
time.Sleep(PING_INTERVAL)
log.Printf("Ping")
ws.writeLock.Lock()
err := ws.conn.WriteMessage(websocket.PingMessage, nil)
if err != nil {
log.Println("Error during ping:", err)
ws.errChan <- err
return
if !ws.alive {
break
}
ws.writeLock.Unlock()
ws.doPing()
time.Sleep(PING_INTERVAL)
}
log.Printf("Pinger done")
}
func (ws *WSConnection) doPing() {
ws.writeLock.Lock()
defer ws.writeLock.Unlock()
// log.Printf("Ping")
err := ws.conn.WriteMessage(websocket.PingMessage, nil)
if err != nil {
log.Println("Error during ping:", err)
ws.errChan <- err
return
}
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
// log.Printf("Ping OK")
}
func (ws *WSConnection) handleError() {
for {
err := <-ws.errChan
log.Println("Error during message reading:", err)
time.Sleep(5 * time.Second)
ws.Open()
log.Printf("Client error: %+v", err)
ws.alive = false
ws.conn.Close()
close(ws.ReadChan)
close(ws.WriteChan)
close(ws.errChan)
ws.Dead <- err
return
}
}
func (ws *WSConnection) Open() {
log.Printf("Connecting to %s", ws.url)
ws.Dead = make(chan error, 1)
conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil)
if err != nil {
log.Println("Error during connection:", err)
ws.errChan <- err
ws.Dead <- err
return
}
log.Printf("Connected")
ws.conn = conn
ws.errChan = make(chan error)
ws.ReadChan = make(chan string, 1024)
ws.WriteChan = make(chan string, 1024)
ws.alive = true
ws.errChan = make(chan error, 1)
ws.ReadChan = make(chan string, 128)
ws.WriteChan = make(chan string, 128)
ws.conn.SetReadLimit(1024)
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetPongHandler(func(string) error {
log.Println("Pong")
// log.Println("Pong")
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
return nil
})
go ws.handleError()
go ws.messageReader()
go ws.messageSender()
go ws.handleError()
go ws.pinger()
}

10
extension/background.js Normal file
View File

@@ -0,0 +1,10 @@
browser.menus.create({
id: "youtube-download",
title: "Download",
contexts: ["all"],
documentUrlPatterns: ["*://*.youtube.com/*"],
});
browser.menus.onShown.addListener((info, tab) => {
browser.tabs.sendMessage(tab.id, { action: "check-element", info: info });
});

34
extension/content.js Normal file
View File

@@ -0,0 +1,34 @@
const URL = `http://localhost:5000/download`;
let lastRightClickCoords = { x: 0, y: 0 };
document.addEventListener("contextmenu", (event) => {
lastRightClickCoords = { x: event.clientX, y: event.clientY };
console.log("Right-click coordinates:", lastRightClickCoords);
});
browser.runtime.onMessage.addListener((message, sender, sendResponse) => {
const { x, y } = lastRightClickCoords;
let element = document.elementFromPoint(x, y);
while (element && element.tagName != "YTD-RICH-ITEM-RENDERER") {
element = element.parentElement;
}
if (!element.tagName == "YTD-RICH-ITEM-RENDERER") {
console.error("No video element found.");
return;
}
const link = element.querySelector("a#video-title-link").href;
fetch(URL, {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
link: link,
}),
}).then((res) => {
console.log(res);
res.json().then((data) => console.log(data));
});
});

20
extension/manifest.json Normal file
View File

@@ -0,0 +1,20 @@
{
"manifest_version": 2,
"name": "Youtube Downloader",
"version": "1.0",
"permissions": [
"menus",
"activeTab",
"tabs",
"http://localhost:5000/*"
],
"background": {
"scripts": ["background.js"]
},
"content_scripts": [
{
"matches": ["*://*.youtube.com/*"],
"js": ["content.js"]
}
]
}

41
hotkey.js Normal file
View File

@@ -0,0 +1,41 @@
nodes = document.querySelectorAll(":hover");
i = 1;
console.log(nodes);
titleNode = nodes[nodes.length - i];
selector = "a#video-title-link";
invidious = false;
if (window.location.href.includes("invidious.site")) {
selector = "a";
invidious = true;
}
while (titleNode && !titleNode.matches(selector)) {
titleNode = titleNode.parentElement;
console.log(titleNode);
}
if (!(titleNode && titleNode.matches(selector))) {
console.error("No video element found.");
} else {
link = titleNode.href;
if (link.startsWith("/")) {
link = window.location.origin + link;
}
console.log(link);
fetch("https://nsq.site.quack-lab.dev/pub?topic=ytdqueue", {
method: "POST",
headers: {
"Content-Type": "application/json",
},
body: JSON.stringify({
link: link,
}),
}).then((res) => {
textNode = titleNode.querySelector("yt-formatted-string");
if (invidious) {
textNode = titleNode.querySelector("p");
}
textNode.style.setProperty("color", "green", "important");
res.json().then((data) => console.log(data));
});
}

View File

@@ -63,7 +63,7 @@ function hookVideo(videoElement) {
}
async function main() {
const videosContainer = await waitForElement(document, "div#contents.style-scope.ytd-rich-grid-renderer");
const videosContainer = await waitForElement(document, "ytd-rich-grid-renderer > div#contents");
for (const video of videosContainer.querySelectorAll("ytd-rich-item-renderer")) {
parseVideo(video);

View File

@@ -1,5 +0,0 @@
module main
go 1.22.4
require github.com/gorilla/websocket v1.5.3

View File

@@ -1,2 +0,0 @@
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

View File

@@ -1,73 +0,0 @@
package main
import (
"log"
"time"
"github.com/gorilla/websocket"
)
type WSConnection struct {
url string
conn *websocket.Conn
errChan chan error
}
func (ws *WSConnection) readMessage() {
log.Printf("Reading messages")
for {
_, message, err := ws.conn.ReadMessage()
if err != nil {
ws.errChan <- err
return
}
log.Printf("Received: %s", message)
}
}
func (ws *WSConnection) writeMessage(message string) {
err := ws.conn.WriteMessage(websocket.TextMessage, []byte(message))
if err != nil {
log.Printf("Error during message writing: %v", err)
ws.errChan <- err
return
}
}
func (ws *WSConnection) handleError() {
for {
err := <-ws.errChan
log.Println("Error during message reading:", err)
time.Sleep(5 * time.Second)
ws.open()
}
}
func (ws *WSConnection) open() {
log.Printf("Connecting to %s", ws.url)
conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil)
if err != nil {
log.Println("Error during connection:", err)
ws.errChan <- err
return
}
log.Printf("Connected")
ws.conn = conn
ws.errChan = make(chan error)
go ws.readMessage()
go ws.handleError()
}
func main() {
log.SetFlags(log.Lmicroseconds)
wsConn := WSConnection{
url: "ws://localhost:8080/ws",
}
wsConn.open()
for {
log.Printf("zzz...")
time.Sleep(30 * time.Second)
}
}

View File

@@ -1,3 +0,0 @@
# docker build -t youtube-download-ws-server .
tar -cf deploy.tar captain-definition dockerfile main.go go.mod go.sum

View File

@@ -1,4 +0,0 @@
{
"schemaVersion": 2,
"dockerfilePath": "./dockerfile"
}

View File

@@ -1,21 +0,0 @@
FROM golang:1.22.4 as base
WORKDIR $GOPATH/src/app/
COPY . .
RUN go mod download
RUN go mod verify
RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /main .
FROM scratch
COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
COPY --from=base /etc/passwd /etc/passwd
COPY --from=base /etc/group /etc/group
COPY --from=base /main .
CMD ["/main"]

View File

@@ -1,5 +0,0 @@
module main
go 1.22.4
require github.com/gorilla/websocket v1.5.3

View File

@@ -1,2 +0,0 @@
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

View File

@@ -1,169 +0,0 @@
package main
import (
"encoding/json"
"fmt"
"io"
"log"
"net/http"
"sync"
"time"
"github.com/gorilla/websocket"
)
var upgrader = websocket.Upgrader{}
var wsBroadcast = make(chan string, 128)
var connections = make(map[*WSConnection]bool)
const TIMEOUT = 6
const IDLE_TIMEOUT = TIMEOUT * time.Second
const PING_INTERVAL = (TIMEOUT / 2) * time.Second
type WSConnection struct {
conn *websocket.Conn
writeLock sync.Mutex
ReadChan chan string
WriteChan chan string
ErrorChan chan error
}
func (ws *WSConnection) messageReader() {
log.Printf("Reading messages")
for {
_, message, err := ws.conn.ReadMessage()
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
if err != nil {
ws.ErrorChan <- err
return
}
log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan))
ws.ReadChan <- string(message)
}
}
func (ws *WSConnection) messageSender() {
log.Printf("Sending messages")
for {
msg := <-ws.WriteChan
ws.writeLock.Lock()
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
if err != nil {
log.Printf("Error during message writing: %v", err)
ws.ErrorChan <- err
return
}
ws.writeLock.Unlock()
}
}
func (ws *WSConnection) pinger() {
log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL)
for {
time.Sleep(PING_INTERVAL)
log.Printf("Ping")
ws.writeLock.Lock()
err := ws.conn.WriteMessage(websocket.PingMessage, nil)
if err != nil {
log.Println("Error during ping:", err)
ws.ErrorChan <- err
return
}
ws.writeLock.Unlock()
}
}
func (ws *WSConnection) Open() {
log.Printf("Client connected")
ws.ReadChan = make(chan string, 1024)
ws.WriteChan = make(chan string, 1024)
ws.ErrorChan = make(chan error, 16)
ws.conn.SetReadLimit(1024)
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetPongHandler(func(string) error {
log.Println("Pong")
ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
return nil
})
connections[ws] = true
go ws.messageReader()
go ws.messageSender()
go ws.pinger()
go func() {
for {
select {
case err := <-ws.ErrorChan:
log.Printf("Error: %v", err)
ws.conn.Close()
log.Printf("Client disconnected")
connections[ws] = false
return
// case msg := <-wsBroadcast:
// ws.WriteChan <- msg
}
}
}()
}
func wsHandler(responseWriter http.ResponseWriter, request *http.Request) {
conn, err := upgrader.Upgrade(responseWriter, request, nil)
if err != nil {
fmt.Println("Error during connection upgrade:", err)
return
}
ws := new(WSConnection)
ws.conn = conn
ws.Open()
}
type DownloadReq struct {
Link string `json:"link"`
}
func handleDownload(responseWriter http.ResponseWriter, request *http.Request) {
body, err := io.ReadAll(request.Body)
if err != nil {
log.Printf("Error reading request body: %v", err)
http.Error(responseWriter, "Error reading request body", http.StatusBadRequest)
return
}
defer request.Body.Close()
req := DownloadReq{}
err = json.Unmarshal(body, &req)
if err != nil {
log.Printf("Error parsing JSON: %v", err)
http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest)
return
}
log.Printf("Received download request: %s, %d in channel", req.Link, len(wsBroadcast))
go func() {
for ws := range connections {
ws.WriteChan <- req.Link
}
}()
// wsBroadcast <- req.Link
}
func main() {
log.SetFlags(log.Lmicroseconds)
http.HandleFunc("/ws", wsHandler)
http.HandleFunc("/download", handleDownload)
log.Println("Server starting on :8080")
err := http.ListenAndServe(":8080", nil)
if err != nil {
log.Println("Error starting server:", err)
}
}