diff --git a/.gitignore b/.gitignore index 854f053..0c31076 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ .idea main.exe logs.log +ws-server/deploy.tar +downloader/main.log diff --git a/api.go b/api.go deleted file mode 100644 index 68db7d5..0000000 --- a/api.go +++ /dev/null @@ -1,75 +0,0 @@ -package main - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" -) - -type APIError struct { - Code int `json:"code"` - Message string `json:"message"` - Data APIErrorData `json:"data"` -} - -type APIErrorData struct { - Link APIErrorLink `json:"link"` -} - -type APIErrorLink struct { - Code string `json:"code"` - Message string `json:"message"` -} - -func SetDownloaded(item PBEvent) (err error) { - req, err := http.NewRequestWithContext(context.Background(), "PATCH", FULL_URL+"/"+item.Record.Id, nil) - if err != nil { - log.Printf("Error creating PATCH request: %++v", err) - return err - } - req.Header.Set("Content-Type", "application/json") - - partialItem := new(PBEvent) - partialItem.Record = item.Record - partialItem.Record.Downloaded = true - - body, err := json.Marshal(partialItem.Record) - if err != nil { - log.Printf("Error marshalling subscription body: %++v", err) - return err - } - req.Body = io.NopCloser(bytes.NewReader(body)) - - client := http.Client{} - res, err := client.Do(req) - if err != nil { - log.Printf("Error sending PATCH request: %++v", err) - return err - } - defer res.Body.Close() - - if res.StatusCode != http.StatusOK { - log.Printf("Non-OK HTTP status: %d", res.StatusCode) - - body, err = io.ReadAll(res.Body) - if err != nil { - log.Printf("Error reading response body: %++v", err) - return err - } - var data APIError - err = json.Unmarshal(body, &data) - if err != nil { - log.Printf("Error unmarshaling JSON: %++v", err) - return err - } - - log.Printf("API error: %++v", data) - return fmt.Errorf("Non-OK HTTP status, err: %++v", data) - } - - return nil -} diff --git a/build.sh b/build.sh deleted file mode 100644 index 166a66b..0000000 --- a/build.sh +++ /dev/null @@ -1 +0,0 @@ -docker build -t youtube-downloader . \ No newline at end of file diff --git a/dl/dl.go b/dl/dl.go new file mode 100644 index 0000000..57ab548 --- /dev/null +++ b/dl/dl.go @@ -0,0 +1,73 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "io" + "log" + "net/http" + "os" + "sync" +) + +const URL = `http://localhost:5000/download` + +type Item struct { + Link string `json:"link"` +} + +var wg sync.WaitGroup + +func main() { + log.SetFlags(log.Lmicroseconds) + + for _, url := range os.Args[1:] { + log.Printf("Downloading %s", url) + wg.Add(1) + go Download(url) + } + wg.Wait() +} + +func Download(url string) { + defer wg.Done() + + req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil) + if err != nil { + log.Printf("Error creating POST request: %++v", err) + return + } + req.Header.Set("Content-Type", "application/json") + + item := new(Item) + item.Link = url + + body, err := json.Marshal(item) + if err != nil { + log.Printf("Error marshalling subscription body: %++v", err) + return + } + req.Body = io.NopCloser(bytes.NewReader(body)) + + client := http.Client{} + res, err := client.Do(req) + if err != nil { + log.Printf("Error sending POST request: %++v", err) + return + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + log.Printf("Non-OK HTTP status: %d", res.StatusCode) + + body, err = io.ReadAll(res.Body) + if err != nil { + log.Printf("Error reading response body: %++v", err) + return + } + return + } else { + log.Printf("Enqueued %s", url) + } +} diff --git a/dl/go.mod b/dl/go.mod new file mode 100644 index 0000000..9a9693e --- /dev/null +++ b/dl/go.mod @@ -0,0 +1,3 @@ +module main + +go 1.22.4 diff --git a/dl/sync b/dl/sync new file mode 100644 index 0000000..e517c85 --- /dev/null +++ b/dl/sync @@ -0,0 +1 @@ +main.exe,"C:\Program Files\Git\usr\bin\dl.exe",t \ No newline at end of file diff --git a/downloader/api.go b/downloader/api.go new file mode 100644 index 0000000..2529490 --- /dev/null +++ b/downloader/api.go @@ -0,0 +1,75 @@ +package main + +// import ( +// "bytes" +// "context" +// "encoding/json" +// "fmt" +// "io" +// "log" +// "net/http" +// ) + +// type APIError struct { +// Code int `json:"code"` +// Message string `json:"message"` +// Data APIErrorData `json:"data"` +// } + +// type APIErrorData struct { +// Link APIErrorLink `json:"link"` +// } + +// type APIErrorLink struct { +// Code string `json:"code"` +// Message string `json:"message"` +// } + +// func SetDownloaded(item PBEvent) (err error) { +// req, err := http.NewRequestWithContext(context.Background(), "PATCH", FULL_URL+"/"+item.Record.Id, nil) +// if err != nil { +// log.Printf("Error creating PATCH request: %++v", err) +// return err +// } +// req.Header.Set("Content-Type", "application/json") + +// partialItem := new(PBEvent) +// partialItem.Record = item.Record +// partialItem.Record.Downloaded = true + +// body, err := json.Marshal(partialItem.Record) +// if err != nil { +// log.Printf("Error marshalling subscription body: %++v", err) +// return err +// } +// req.Body = io.NopCloser(bytes.NewReader(body)) + +// client := http.Client{} +// res, err := client.Do(req) +// if err != nil { +// log.Printf("Error sending PATCH request: %++v", err) +// return err +// } +// defer res.Body.Close() + +// if res.StatusCode != http.StatusOK { +// log.Printf("Non-OK HTTP status: %d", res.StatusCode) + +// body, err = io.ReadAll(res.Body) +// if err != nil { +// log.Printf("Error reading response body: %++v", err) +// return err +// } +// var data APIError +// err = json.Unmarshal(body, &data) +// if err != nil { +// log.Printf("Error unmarshaling JSON: %++v", err) +// return err +// } + +// log.Printf("API error: %++v", data) +// return fmt.Errorf("Non-OK HTTP status, err: %++v", data) +// } + +// return nil +// } diff --git a/downloader/assets/information.png b/downloader/assets/information.png new file mode 100644 index 0000000..d47243d Binary files /dev/null and b/downloader/assets/information.png differ diff --git a/downloader/build.sh b/downloader/build.sh new file mode 100644 index 0000000..d16e268 --- /dev/null +++ b/downloader/build.sh @@ -0,0 +1,3 @@ +nssm stop YoutubeDownloader && \ +go build main && \ +nssm start YoutubeDownloader \ No newline at end of file diff --git a/dockerfile b/downloader/dockerfile similarity index 100% rename from dockerfile rename to downloader/dockerfile diff --git a/downloader/download.go b/downloader/download.go new file mode 100644 index 0000000..105f37d --- /dev/null +++ b/downloader/download.go @@ -0,0 +1,71 @@ +package main + +import ( + "context" + "log" + "sync" + + "github.com/gen2brain/beeep" + "github.com/lrstanley/go-ytdlp" +) + +const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos" + +type DownloadWorker struct { + id int + input chan *DownloadTask +} + +var ongoingDownloads = make(map[string]struct{}) +var ongoingDownloadsMutex = &sync.Mutex{} + +var dl = ytdlp.New(). + // FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]"). + FormatSort("res,ext:mp4:m4a"). + Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s"). + LimitRate("10M"). + // HTTPChunkSize("20M"). + MarkWatched(). + SponsorblockMark("all"). + RecodeVideo("mp4"). + ConcurrentFragments(6) + +func (w *DownloadWorker) Run() { + for { + task, ok := <-w.input + if !ok { + log.Printf("DownloadWorker %d: input channel closed, exiting", w.id) + return + } + _, ongoing := ongoingDownloads[task.Url] + if ongoing { + log.Printf("DownloadWorker %d: Download %s is already ongoing", w.id, task.Url) + continue + } + ongoingDownloadsMutex.Lock() + ongoingDownloads[task.Url] = struct{}{} + ongoingDownloadsMutex.Unlock() + + log.Printf("DownloadWorker %d: Downloading %s", w.id, task.Url) + + err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration) + if err != nil { + log.Printf("Failed beeping with %+v", err) + } + err = beeep.Alert("Download Started", task.Url, "assets/information.png") + if err != nil { + log.Printf("Failed alerting with %+v", err) + } + + _, err = dl.Run(context.TODO(), task.Url) + if err != nil { + log.Printf("DownloadWorker %d: Failed downloading %s with %+v", w.id, task.Url, err) + continue + } + + log.Printf("DownloadWorker %d: Downloaded %s", w.id, task.Url) + ongoingDownloadsMutex.Lock() + delete(ongoingDownloads, task.Url) + ongoingDownloadsMutex.Unlock() + } +} diff --git a/downloader/go.mod b/downloader/go.mod new file mode 100644 index 0000000..143c4c1 --- /dev/null +++ b/downloader/go.mod @@ -0,0 +1,20 @@ +module main + +go 1.22.4 + +require ( + github.com/ProtonMail/go-crypto v1.0.0 // indirect + github.com/cloudflare/circl v1.3.7 // indirect + github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect + github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect + github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/sys v0.18.0 // indirect +) + +require ( + github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 + github.com/gorilla/websocket v1.5.3 + github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 +) diff --git a/go.sum b/downloader/go.sum similarity index 52% rename from go.sum rename to downloader/go.sum index 445796d..76dc1dc 100644 --- a/go.sum +++ b/downloader/go.sum @@ -1,42 +1,23 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= -github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= -github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= -github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= -github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= -github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= -github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI= +github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4= +github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 h1:qZNfIGkIANxGv/OqtnntR4DfOY2+BgwR60cAcu/i3SE= +github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4/go.mod h1:kW3HQ4UdaAyrUCSSDR4xUzBKW6O2iA4uHhk7AtyYp10= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 h1:ZAo7qJht9PqefOD7C0ZKQ8dEkpJeM955sYw0FtQnzvo= github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o= -github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8= -github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= +github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= +github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk= +github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= @@ -47,21 +28,17 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOM golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -77,24 +54,12 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= -gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/downloader/main.go b/downloader/main.go new file mode 100644 index 0000000..cd08f73 --- /dev/null +++ b/downloader/main.go @@ -0,0 +1,49 @@ +package main + +import ( + "encoding/json" + "io" + "log" + "net/http" + "os" +) + +func init() { + log.SetFlags(log.Lmicroseconds | log.Lshortfile) + logFile, err := os.Create("main.log") + if err != nil { + log.Printf("Error creating log file: %v", err) + os.Exit(1) + } + logger := io.MultiWriter(os.Stdout, logFile) + log.SetOutput(logger) +} + +const DOWNLOAD_WORKERS = 10 + +func main() { + downloadQueue := make(chan *DownloadTask, 100) + for i := 0; i < DOWNLOAD_WORKERS; i++ { + worker := &DownloadWorker{id: i, input: downloadQueue} + go worker.Run() + } + + http.HandleFunc("/download", func(responseWriter http.ResponseWriter, request *http.Request) { + defer request.Body.Close() + + req := DownloadRequest{} + err := json.NewDecoder(request.Body).Decode(&req) + if err != nil { + log.Printf("Error parsing JSON: %v", err) + http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest) + return + } + + downloadQueue <- &DownloadTask{Url: req.Link} + }) + log.Println("Server starting on :5000") + err := http.ListenAndServe(":5000", nil) + if err != nil { + log.Println("Error starting server:", err) + } +} diff --git a/downloader/realtime.go b/downloader/realtime.go new file mode 100644 index 0000000..462ea8c --- /dev/null +++ b/downloader/realtime.go @@ -0,0 +1,89 @@ +package main + +// import ( +// "bytes" +// "encoding/json" +// "log" +// "net/http" + +// "github.com/r3labs/sse" +// ) + +// type RealtimeListener struct { +// Url string +// Collections []string +// Create chan PBEvent +// Update chan PBEvent +// Delete chan PBEvent +// client *sse.Client +// } + +// type Subscription struct { +// ClientId string `json:"clientId"` +// Subscriptions []string `json:"subscriptions"` +// } + +// func (listener RealtimeListener) handlePbEvent(msg *sse.Event) { +// pbEvent := new(PBEvent) +// err := json.Unmarshal(msg.Data, &pbEvent) +// if err != nil { +// log.Printf("Error unmarshalling event: %v\n", err) +// return +// } +// log.Printf("Received event: %++v", pbEvent) + +// if pbEvent.ClientId != "" { +// listener.doSubscribe(pbEvent.ClientId) +// } + +// if pbEvent.Action != "" { +// go listener.shipEvent(*pbEvent) +// } +// } + +// func (listener RealtimeListener) shipEvent(event PBEvent) { +// switch event.Action { +// case "create": +// listener.Create <- event +// case "update": +// listener.Update <- event +// case "delete": +// listener.Delete <- event +// default: +// log.Printf("Unknown action: %v\n", event.Action) +// } +// } + +// func (listener RealtimeListener) doSubscribe(clientId string) { +// subscription := Subscription{ +// ClientId: clientId, +// Subscriptions: listener.Collections, +// } +// log.Printf("Subscribing client: %v to %++v", clientId, subscription) + +// body, err := json.Marshal(subscription) +// if err != nil { +// log.Printf("Error marshalling subscription body: %v\n", err) +// return +// } + +// resp, err := http.Post(POCKETBASE_REALTIME, "application/json", bytes.NewBuffer(body)) +// if err != nil { +// log.Printf("Error posting subscription: %v\n", err) +// return +// } +// defer resp.Body.Close() + +// if resp.StatusCode != http.StatusNoContent { +// log.Printf("Subscription request failed with status: %v\n", resp.Status) +// } +// } + +// func (listener *RealtimeListener) initialize() { +// listener.Update = make(chan PBEvent, 32) +// listener.Create = make(chan PBEvent, 32) +// listener.Delete = make(chan PBEvent, 32) +// log.Print("Initialized") +// listener.client = sse.NewClient(listener.Url) +// go listener.client.Subscribe("", listener.handlePbEvent) +// } diff --git a/downloader/types.go b/downloader/types.go new file mode 100644 index 0000000..a51b886 --- /dev/null +++ b/downloader/types.go @@ -0,0 +1,8 @@ +package main + +type DownloadTask struct { + Url string +} +type DownloadRequest struct { + Link string `json:"link"` +} \ No newline at end of file diff --git a/downloader/ws-client.go b/downloader/ws-client.go new file mode 100644 index 0000000..54fc790 --- /dev/null +++ b/downloader/ws-client.go @@ -0,0 +1,143 @@ +package main + +import ( + "log" + "sync" + "time" + + "github.com/gorilla/websocket" +) + +const TIMEOUT = 6 +const IDLE_TIMEOUT = TIMEOUT * time.Second +const PING_INTERVAL = (TIMEOUT / 2) * time.Second + +type WSConnection struct { + alive bool + url string + conn *websocket.Conn + errChan chan error + writeLock sync.Mutex + ReadChan chan string + WriteChan chan string + Dead chan error +} + +func (ws *WSConnection) messageReader() { + log.Printf("Starting reader") + for { + if !ws.alive { + break + } + + _, message, err := ws.conn.ReadMessage() + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + if err != nil { + ws.errChan <- err + break + } + + log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) + ws.ReadChan <- string(message) + } + log.Printf("Reader done") +} + +func (ws *WSConnection) messageSender() { + log.Printf("Starting sender") + for { + msg, ok := <-ws.WriteChan + if !ok { + break + } + ws.doSend(msg) + } + log.Printf("Sender done") +} + +func (ws *WSConnection) doSend(msg string) { + ws.writeLock.Lock() + defer ws.writeLock.Unlock() + + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Error during message writing: %v", err) + ws.errChan <- err + return + } +} + +func (ws *WSConnection) pinger() { + log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL) + for { + if !ws.alive { + break + } + ws.doPing() + time.Sleep(PING_INTERVAL) + } + log.Printf("Pinger done") +} + +func (ws *WSConnection) doPing() { + ws.writeLock.Lock() + defer ws.writeLock.Unlock() + + // log.Printf("Ping") + err := ws.conn.WriteMessage(websocket.PingMessage, nil) + if err != nil { + log.Println("Error during ping:", err) + ws.errChan <- err + return + } + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + // log.Printf("Ping OK") +} + +func (ws *WSConnection) handleError() { + for { + err := <-ws.errChan + log.Printf("Client error: %+v", err) + ws.alive = false + ws.conn.Close() + close(ws.ReadChan) + close(ws.WriteChan) + close(ws.errChan) + ws.Dead <- err + return + } +} + +func (ws *WSConnection) Open() { + log.Printf("Connecting to %s", ws.url) + ws.Dead = make(chan error, 1) + + conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil) + if err != nil { + log.Println("Error during connection:", err) + ws.Dead <- err + return + } + log.Printf("Connected") + ws.conn = conn + ws.alive = true + + ws.errChan = make(chan error, 1) + ws.ReadChan = make(chan string, 128) + ws.WriteChan = make(chan string, 128) + + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetPongHandler(func(string) error { + // log.Println("Pong") + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + return nil + }) + + go ws.handleError() + go ws.messageReader() + go ws.messageSender() + go ws.pinger() +} diff --git a/go.mod b/go.mod deleted file mode 100644 index fa0d15d..0000000 --- a/go.mod +++ /dev/null @@ -1,23 +0,0 @@ -module main - -go 1.22.4 - -require github.com/r3labs/sse/v2 v2.10.0 - -require ( - github.com/ProtonMail/go-crypto v1.0.0 // indirect - github.com/bitly/go-simplejson v0.5.1 // indirect - github.com/cloudflare/circl v1.3.7 // indirect - github.com/dlclark/regexp2 v1.11.0 // indirect - github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 // indirect - github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect - github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect - github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect - golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.21.0 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect -) - -require github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 diff --git a/main.go b/main.go deleted file mode 100644 index 7bdfb79..0000000 --- a/main.go +++ /dev/null @@ -1,62 +0,0 @@ -package main - -import ( - "encoding/json" - "io" - "log" - "net/http" - "time" -) - -const POCKETBASE_URL = `https://pocketbase-scratch.site.quack-lab.dev/api/collections` -const POCKETBASE_REALTIME = `https://pocketbase-scratch.site.quack-lab.dev/api/realtime` -const COLLECTION_NAME = "youtubedownload" -const FULL_URL = POCKETBASE_URL + "/" + COLLECTION_NAME + "/records" - -func main() { - log.SetFlags(log.Lmicroseconds) - log.Println(FULL_URL) - - res, err := http.Get(FULL_URL) - if err != nil { - log.Fatal(err) - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - log.Printf("Error reading response body: %+v\n", err) - return - } - if res.StatusCode != http.StatusOK { - log.Printf("Non-OK HTTP status: %d\nResponse body: %s\n", res.StatusCode, body) - return - } - - var data APIResponse - err = json.Unmarshal(body, &data) - if err != nil { - log.Printf("Error unmarshaling JSON: %+v\n", err) - return - } - // log.Printf("Data: %+v\n", data) - - listener := new(RealtimeListener) - listener.Url = POCKETBASE_REALTIME - listener.Collections = []string{COLLECTION_NAME} - listener.initialize() - - status := make(chan error) - for event := range listener.Create { - log.Printf("Create event: %+v\n", event) - eventCopy := event - go func() { - Download(eventCopy, status) - // go DownloadNative(event, status) - for status := range status { - log.Printf("Status: %s\n", status) - } - }() - } - - time.Sleep(1 * time.Hour) -} diff --git a/realtime.go b/realtime.go deleted file mode 100644 index 32f2571..0000000 --- a/realtime.go +++ /dev/null @@ -1,87 +0,0 @@ -package main - -import ( - "bytes" - "encoding/json" - "log" - "net/http" - - "github.com/r3labs/sse" -) - -type RealtimeListener struct { - Url string - Collections []string - Create chan PBEvent - Update chan PBEvent - Delete chan PBEvent - client *sse.Client -} - -type Subscription struct { - ClientId string `json:"clientId"` - Subscriptions []string `json:"subscriptions"` -} - -func (listener RealtimeListener) handlePbEvent(msg *sse.Event) { - var pbEvent = new(PBEvent) - err := json.Unmarshal(msg.Data, &pbEvent) - if err != nil { - log.Printf("Error unmarshalling event: %v\n", err) - return - } - - if pbEvent.ClientId != "" { - listener.doSubscribe(pbEvent.ClientId) - } - - if pbEvent.Action != "" { - go listener.shipEvent(*pbEvent) - } -} - -func (listener RealtimeListener) shipEvent(event PBEvent) { - switch event.Action { - case "create": - listener.Create <- event - case "update": - listener.Update <- event - case "delete": - listener.Delete <- event - default: - log.Printf("Unknown action: %v\n", event.Action) - } -} - -func (listener RealtimeListener) doSubscribe(clientId string) { - subscription := Subscription{ - ClientId: clientId, - Subscriptions: listener.Collections, - } - - body, err := json.Marshal(subscription) - if err != nil { - log.Printf("Error marshalling subscription body: %v\n", err) - return - } - - resp, err := http.Post(POCKETBASE_REALTIME, "application/json", bytes.NewBuffer(body)) - if err != nil { - log.Printf("Error posting subscription: %v\n", err) - return - } - defer resp.Body.Close() - - if resp.StatusCode != http.StatusNoContent { - log.Printf("Subscription request failed with status: %v\n", resp.Status) - } -} - -func (listener *RealtimeListener) initialize() { - listener.Update = make(chan PBEvent, 32) - listener.Create = make(chan PBEvent, 32) - listener.Delete = make(chan PBEvent, 32) - log.Print("initialized") - listener.client = sse.NewClient(listener.Url) - go listener.client.Subscribe("", listener.handlePbEvent) -} diff --git a/tampermonkey.js b/tampermonkey.js new file mode 100644 index 0000000..7ba6c71 --- /dev/null +++ b/tampermonkey.js @@ -0,0 +1,86 @@ +// ==UserScript== +// @name Youtube Downloader +// @author Cyka +// @match https://www.youtube.com/* +// @version 1.19 +// @run-at document-idle +// @noframes +// ==/UserScript== + +const URL = `https://youtube-download-ws-server.site.quack-lab.dev/download`; + +function waitForElement(element, selector) { + return new Promise((resolve) => { + if (element.querySelector(selector)) { + return resolve(element.querySelector(selector)); + } + + const observer = new MutationObserver((mutations) => { + if (element.querySelector(selector)) { + resolve(element.querySelector(selector)); + observer.disconnect(); + } + }); + + observer.observe(element, { + childList: true, + subtree: true, + }); + }); +} + +function parseVideo(videoElement) { + hookVideo(videoElement); +} + +function hookVideo(videoElement) { + videoElement.addEventListener( + "mousedown", + function (e) { + if (e.button === 1) { + e.preventDefault(); + e.stopPropagation(); + e.stopImmediatePropagation(); + + const link = videoElement.querySelector("a#video-title-link").href; + console.log(link); + fetch(URL, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + link: link, + }), + }).then((res) => { + console.log(res); + res.json().then((data) => console.log(data)); + }); + } + }, + false + ); +} + +async function main() { + const videosContainer = await waitForElement(document, "ytd-rich-grid-renderer > div#contents"); + + for (const video of videosContainer.querySelectorAll("ytd-rich-item-renderer")) { + parseVideo(video); + } + + new MutationObserver((mutations) => { + mutations = mutations.filter((mutation) => mutation.addedNodes.length > 0); + + for (const mutation of mutations) { + if (mutation.target.tagName == "YTD-RICH-ITEM-RENDERER") { + parseVideo(mutation.target); + } + } + }).observe(videosContainer, { + childList: true, + subtree: true, + }); +} + +main(); diff --git a/types.go b/types.go deleted file mode 100644 index 884eab2..0000000 --- a/types.go +++ /dev/null @@ -1,25 +0,0 @@ -package main - -type APIResponse struct { - Page int `json:"page"` - PerPage int `json:"perPage"` - TotalItems int `json:"totalItems"` - TotalPages int `json:"totalPages"` - Items []APIItem `json:"items"` -} - -type APIItem struct { - CollectionId string `json:"collectionId"` - CollectionName string `json:"collectionName"` - Created string `json:"created"` - Downloaded bool `json:"downloaded"` - Id string `json:"id"` - Link string `json:"link"` - Updated string `json:"updated"` -} - -type PBEvent struct { - ClientId string `json:"clientId"` - Action string `json:"action"` - Record APIItem `json:"record"` -}