From 3663c6c214f0deb49e4c03e216ac2b65b7d69cb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Mon, 17 Jun 2024 16:14:46 +0200 Subject: [PATCH 01/35] Fix issues with slow --- download.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/download.go b/download.go index 6585c37..a0147cf 100644 --- a/download.go +++ b/download.go @@ -14,8 +14,11 @@ var dl = ytdlp.New(). FormatSort("res,ext:mp4:m4a"). Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s"). LimitRate("50M"). - HTTPChunkSize("20M"). - MarkWatched() + // HTTPChunkSize("20M"). + MarkWatched(). + SponsorblockMark("all"). + RecodeVideo("mp4"). + ConcurrentFragments(4) func Download(event PBEvent, status chan error) { _, err := dl.Run(context.TODO(), event.Record.Link) From 6f2b472c839a7118d48ccaba7b333d0b9588379d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Mon, 17 Jun 2024 16:35:19 +0200 Subject: [PATCH 02/35] Add tampermonkey script --- tampermonkey.js | 89 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) create mode 100644 tampermonkey.js diff --git a/tampermonkey.js b/tampermonkey.js new file mode 100644 index 0000000..d17234e --- /dev/null +++ b/tampermonkey.js @@ -0,0 +1,89 @@ +// ==UserScript== +// @name Youtube Downloader +// @author Cyka +// @match https://www.youtube.com/* +// @version 1.19 +// @run-at document-idle +// @noframes +// ==/UserScript== + +const API = "https://pocketbase-scratch.site.quack-lab.dev/api/collections/youtubedownload/records"; + +function waitForElement(element, selector) { + return new Promise((resolve) => { + if (element.querySelector(selector)) { + return resolve(element.querySelector(selector)); + } + + const observer = new MutationObserver((mutations) => { + if (element.querySelector(selector)) { + resolve(element.querySelector(selector)); + observer.disconnect(); + } + }); + + observer.observe(element, { + childList: true, + subtree: true, + }); + }); +} + +function parseVideo(videoElement) { + console.log(videoElement); + hookVideo(videoElement); +} + +function hookVideo(videoElement) { + console.log("Hooking", videoElement); + + videoElement.addEventListener( + "mousedown", + function (e) { + if (e.button === 1) { + e.preventDefault(); + e.stopPropagation(); + e.stopImmediatePropagation(); + + const link = videoElement.querySelector("a#video-title-link").href; + console.log(link); + fetch(API, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + link: link, + }), + }).then((res) => { + console.log(res); + res.json().then((data) => console.log(data)); + }); + } + }, + false + ); +} + +async function main() { + const videosContainer = await waitForElement(document, "div#contents.style-scope.ytd-rich-grid-renderer"); + + for (const video of videosContainer.querySelectorAll("ytd-rich-item-renderer")) { + parseVideo(video); + } + + new MutationObserver((mutations) => { + mutations = mutations.filter((mutation) => mutation.addedNodes.length > 0); + + for (const mutation of mutations) { + if (mutation.target.tagName == "YTD-RICH-ITEM-RENDERER") { + parseVideo(mutation.target); + } + } + }).observe(videosContainer, { + childList: true, + subtree: true, + }); +} + +main(); From eb4397685767a102dc9864ca3d524080d0f99789 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Mon, 17 Jun 2024 21:11:19 +0200 Subject: [PATCH 03/35] Fix issue with fake deadlock --- main.go | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/main.go b/main.go index 7bdfb79..877e364 100644 --- a/main.go +++ b/main.go @@ -46,17 +46,21 @@ func main() { listener.initialize() status := make(chan error) - for event := range listener.Create { - log.Printf("Create event: %+v\n", event) - eventCopy := event - go func() { - Download(eventCopy, status) - // go DownloadNative(event, status) - for status := range status { - log.Printf("Status: %s\n", status) - } - }() + for { + select { + case event := <-listener.Create: + log.Printf("Create event: %+v\n", event) + eventCopy := event + go func() { + Download(eventCopy, status) + // go DownloadNative(event, status) + for status := range status { + log.Printf("Status: %s\n", status) + } + }() + case <-time.After(1 * time.Minute): + // Perform some action or simply continue to avoid deadlock + log.Println("Consumer is alive, but has no new events.") + } } - - time.Sleep(1 * time.Hour) } From 6e5dc31856090c401481b5784b20f110d9bb273b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Mon, 17 Jun 2024 21:11:23 +0200 Subject: [PATCH 04/35] Add dl module --- dl/dl.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ dl/go.mod | 3 +++ 2 files changed, 79 insertions(+) create mode 100644 dl/dl.go create mode 100644 dl/go.mod diff --git a/dl/dl.go b/dl/dl.go new file mode 100644 index 0000000..11a55c6 --- /dev/null +++ b/dl/dl.go @@ -0,0 +1,76 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "os" +) + +const POCKETBASE_URL = `https://pocketbase-scratch.site.quack-lab.dev/api/collections` +const POCKETBASE_REALTIME = `https://pocketbase-scratch.site.quack-lab.dev/api/realtime` +const COLLECTION_NAME = "youtubedownload" +const FULL_URL = POCKETBASE_URL + "/" + COLLECTION_NAME + "/records" + +type Item struct { + Link string `json:"link"` +} + +func main() { + log.SetFlags(log.Lmicroseconds) + + for _, url := range os.Args[1:] { + log.Printf("Downloading %s", url) + go Download(url) + } +} + +func Download(url string) { + req, err := http.NewRequestWithContext(context.Background(), "POST", FULL_URL, nil) + if err != nil { + log.Printf("Error creating PATCH request: %++v", err) + return + } + req.Header.Set("Content-Type", "application/json") + + item := new(Item) + item.Link = url + + body, err := json.Marshal(item) + if err != nil { + log.Printf("Error marshalling subscription body: %++v", err) + return + } + req.Body = io.NopCloser(bytes.NewReader(body)) + + client := http.Client{} + res, err := client.Do(req) + if err != nil { + log.Printf("Error sending PATCH request: %++v", err) + return + } + defer res.Body.Close() + + if res.StatusCode != http.StatusOK { + log.Printf("Non-OK HTTP status: %d", res.StatusCode) + + body, err = io.ReadAll(res.Body) + if err != nil { + log.Printf("Error reading response body: %++v", err) + return + } + var data APIError + err = json.Unmarshal(body, &data) + if err != nil { + log.Printf("Error unmarshaling JSON: %++v", err) + return + } + + log.Printf("API error: %++v", data) + return + } +} diff --git a/dl/go.mod b/dl/go.mod new file mode 100644 index 0000000..9a9693e --- /dev/null +++ b/dl/go.mod @@ -0,0 +1,3 @@ +module main + +go 1.22.4 From bb7af20f8388a78dbe1a7e06b4a2b0158fd03464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Mon, 17 Jun 2024 21:19:34 +0200 Subject: [PATCH 05/35] Add dl to enqueue downloads --- dl/dl.go | 27 ++++++++++++++++++++++++--- dl/sync | 1 + 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 dl/sync diff --git a/dl/dl.go b/dl/dl.go index 11a55c6..9018e9b 100644 --- a/dl/dl.go +++ b/dl/dl.go @@ -4,11 +4,11 @@ import ( "bytes" "context" "encoding/json" - "fmt" "io" "log" "net/http" "os" + "sync" ) const POCKETBASE_URL = `https://pocketbase-scratch.site.quack-lab.dev/api/collections` @@ -20,19 +20,38 @@ type Item struct { Link string `json:"link"` } +type APIError struct { + Code int `json:"code"` + Message string `json:"message"` + Data APIErrorData `json:"data"` +} +type APIErrorData struct { + Link APIErrorLink `json:"link"` +} +type APIErrorLink struct { + Code string `json:"code"` + Message string `json:"message"` +} + +var wg sync.WaitGroup + func main() { log.SetFlags(log.Lmicroseconds) for _, url := range os.Args[1:] { log.Printf("Downloading %s", url) + wg.Add(1) go Download(url) } + wg.Wait() } func Download(url string) { + defer wg.Done() + req, err := http.NewRequestWithContext(context.Background(), "POST", FULL_URL, nil) if err != nil { - log.Printf("Error creating PATCH request: %++v", err) + log.Printf("Error creating POST request: %++v", err) return } req.Header.Set("Content-Type", "application/json") @@ -50,7 +69,7 @@ func Download(url string) { client := http.Client{} res, err := client.Do(req) if err != nil { - log.Printf("Error sending PATCH request: %++v", err) + log.Printf("Error sending POST request: %++v", err) return } defer res.Body.Close() @@ -72,5 +91,7 @@ func Download(url string) { log.Printf("API error: %++v", data) return + } else { + log.Printf("Enqueued %s", url) } } diff --git a/dl/sync b/dl/sync new file mode 100644 index 0000000..42b139b --- /dev/null +++ b/dl/sync @@ -0,0 +1 @@ +main.exe,"C:\Program Files\Git\usr\bin\dl.exe" \ No newline at end of file From 2ec00ea02d612e34a4bbd776e8295528ac0340ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 00:07:56 +0200 Subject: [PATCH 06/35] Cleanup --- dl/sync | 2 +- main.go | 43 ++++++++++++++++++++----------------------- realtime.go | 6 ++++-- 3 files changed, 25 insertions(+), 26 deletions(-) diff --git a/dl/sync b/dl/sync index 42b139b..e517c85 100644 --- a/dl/sync +++ b/dl/sync @@ -1 +1 @@ -main.exe,"C:\Program Files\Git\usr\bin\dl.exe" \ No newline at end of file +main.exe,"C:\Program Files\Git\usr\bin\dl.exe",t \ No newline at end of file diff --git a/main.go b/main.go index 877e364..145fbd0 100644 --- a/main.go +++ b/main.go @@ -1,10 +1,7 @@ package main import ( - "encoding/json" - "io" "log" - "net/http" "time" ) @@ -17,27 +14,27 @@ func main() { log.SetFlags(log.Lmicroseconds) log.Println(FULL_URL) - res, err := http.Get(FULL_URL) - if err != nil { - log.Fatal(err) - } - defer res.Body.Close() - body, err := io.ReadAll(res.Body) - if err != nil { - log.Printf("Error reading response body: %+v\n", err) - return - } - if res.StatusCode != http.StatusOK { - log.Printf("Non-OK HTTP status: %d\nResponse body: %s\n", res.StatusCode, body) - return - } + // res, err := http.Get(FULL_URL) + // if err != nil { + // log.Fatal(err) + // } + // defer res.Body.Close() + // body, err := io.ReadAll(res.Body) + // if err != nil { + // log.Printf("Error reading response body: %+v\n", err) + // return + // } + // if res.StatusCode != http.StatusOK { + // log.Printf("Non-OK HTTP status: %d\nResponse body: %s\n", res.StatusCode, body) + // return + // } - var data APIResponse - err = json.Unmarshal(body, &data) - if err != nil { - log.Printf("Error unmarshaling JSON: %+v\n", err) - return - } + // var data APIResponse + // err = json.Unmarshal(body, &data) + // if err != nil { + // log.Printf("Error unmarshaling JSON: %+v\n", err) + // return + // } // log.Printf("Data: %+v\n", data) listener := new(RealtimeListener) diff --git a/realtime.go b/realtime.go index 32f2571..681db0f 100644 --- a/realtime.go +++ b/realtime.go @@ -24,12 +24,13 @@ type Subscription struct { } func (listener RealtimeListener) handlePbEvent(msg *sse.Event) { - var pbEvent = new(PBEvent) + pbEvent := new(PBEvent) err := json.Unmarshal(msg.Data, &pbEvent) if err != nil { log.Printf("Error unmarshalling event: %v\n", err) return } + log.Printf("Received event: %++v", pbEvent) if pbEvent.ClientId != "" { listener.doSubscribe(pbEvent.ClientId) @@ -58,6 +59,7 @@ func (listener RealtimeListener) doSubscribe(clientId string) { ClientId: clientId, Subscriptions: listener.Collections, } + log.Printf("Subscribing client: %v to %++v", clientId, subscription) body, err := json.Marshal(subscription) if err != nil { @@ -81,7 +83,7 @@ func (listener *RealtimeListener) initialize() { listener.Update = make(chan PBEvent, 32) listener.Create = make(chan PBEvent, 32) listener.Delete = make(chan PBEvent, 32) - log.Print("initialized") + log.Print("Initialized") listener.client = sse.NewClient(listener.Url) go listener.client.Subscribe("", listener.handlePbEvent) } From f01e13a82da700579f5f2aa252bc203400803915 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 00:09:58 +0200 Subject: [PATCH 07/35] Refactor --- api.go => downloader/api.go | 0 build.sh => downloader/build.sh | 0 dockerfile => downloader/dockerfile | 0 download.go => downloader/download.go | 0 go.mod => downloader/go.mod | 0 go.sum => downloader/go.sum | 0 main.go => downloader/main.go | 0 realtime.go => downloader/realtime.go | 0 types.go => downloader/types.go | 0 9 files changed, 0 insertions(+), 0 deletions(-) rename api.go => downloader/api.go (100%) rename build.sh => downloader/build.sh (100%) rename dockerfile => downloader/dockerfile (100%) rename download.go => downloader/download.go (100%) rename go.mod => downloader/go.mod (100%) rename go.sum => downloader/go.sum (100%) rename main.go => downloader/main.go (100%) rename realtime.go => downloader/realtime.go (100%) rename types.go => downloader/types.go (100%) diff --git a/api.go b/downloader/api.go similarity index 100% rename from api.go rename to downloader/api.go diff --git a/build.sh b/downloader/build.sh similarity index 100% rename from build.sh rename to downloader/build.sh diff --git a/dockerfile b/downloader/dockerfile similarity index 100% rename from dockerfile rename to downloader/dockerfile diff --git a/download.go b/downloader/download.go similarity index 100% rename from download.go rename to downloader/download.go diff --git a/go.mod b/downloader/go.mod similarity index 100% rename from go.mod rename to downloader/go.mod diff --git a/go.sum b/downloader/go.sum similarity index 100% rename from go.sum rename to downloader/go.sum diff --git a/main.go b/downloader/main.go similarity index 100% rename from main.go rename to downloader/main.go diff --git a/realtime.go b/downloader/realtime.go similarity index 100% rename from realtime.go rename to downloader/realtime.go diff --git a/types.go b/downloader/types.go similarity index 100% rename from types.go rename to downloader/types.go From fa6be7f2cf30bafc0e3140a2ae378bd98357dd62 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 00:10:00 +0200 Subject: [PATCH 08/35] Add ws server --- ws-server/go.mod | 5 +++++ ws-server/go.sum | 2 ++ ws-server/main.go | 11 +++++++++++ 3 files changed, 18 insertions(+) create mode 100644 ws-server/go.mod create mode 100644 ws-server/go.sum create mode 100644 ws-server/main.go diff --git a/ws-server/go.mod b/ws-server/go.mod new file mode 100644 index 0000000..3de883c --- /dev/null +++ b/ws-server/go.mod @@ -0,0 +1,5 @@ +module main + +go 1.22.4 + +require github.com/gorilla/websocket v1.5.3 diff --git a/ws-server/go.sum b/ws-server/go.sum new file mode 100644 index 0000000..25a9fc4 --- /dev/null +++ b/ws-server/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/ws-server/main.go b/ws-server/main.go new file mode 100644 index 0000000..29361da --- /dev/null +++ b/ws-server/main.go @@ -0,0 +1,11 @@ +package main + +import ( + "fmt" + + "github.com/gorilla/websocket" +) + +func main() { + fmt.Println("Hello, World!") +} \ No newline at end of file From 762e62614d3f1aac210e5cb55966bad1f9dae050 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 00:47:10 +0200 Subject: [PATCH 09/35] Implement basic ws server and client --- ws-client/go.mod | 5 ++++ ws-client/go.sum | 2 ++ ws-client/main.go | 76 +++++++++++++++++++++++++++++++++++++++++++++++ ws-server/main.go | 42 ++++++++++++++++++++++++-- 4 files changed, 123 insertions(+), 2 deletions(-) create mode 100644 ws-client/go.mod create mode 100644 ws-client/go.sum create mode 100644 ws-client/main.go diff --git a/ws-client/go.mod b/ws-client/go.mod new file mode 100644 index 0000000..3de883c --- /dev/null +++ b/ws-client/go.mod @@ -0,0 +1,5 @@ +module main + +go 1.22.4 + +require github.com/gorilla/websocket v1.5.3 diff --git a/ws-client/go.sum b/ws-client/go.sum new file mode 100644 index 0000000..25a9fc4 --- /dev/null +++ b/ws-client/go.sum @@ -0,0 +1,2 @@ +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/ws-client/main.go b/ws-client/main.go new file mode 100644 index 0000000..b87a3a3 --- /dev/null +++ b/ws-client/main.go @@ -0,0 +1,76 @@ +package main + +import ( + "log" + "time" + + "github.com/gorilla/websocket" +) + +type WSConnection struct { + url string + conn *websocket.Conn + errChan chan error +} + +func (ws *WSConnection) readMessage() { + log.Printf("Reading messages") + for { + _, message, err := ws.conn.ReadMessage() + if err != nil { + ws.errChan <- err + return + } + log.Printf("Received: %s", message) + } +} + +func (ws *WSConnection) writeMessage(message string) { + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(message)) + if err != nil { + log.Printf("Error during message writing: %v", err) + ws.errChan <- err + return + } +} + +func (ws *WSConnection) handleError() { + for { + err := <-ws.errChan + log.Println("Error during message reading:", err) + + time.Sleep(5 * time.Second) + ws.open() + } +} + +func (ws *WSConnection) open() { + log.Printf("Connecting to %s", ws.url) + conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil) + if err != nil { + log.Println("Error during connection:", err) + ws.errChan <- err + return + } + log.Printf("Connected") + ws.conn = conn + ws.errChan = make(chan error) + go ws.readMessage() + go ws.handleError() +} + +func main() { + log.SetFlags(log.Lmicroseconds) + + wsConn := WSConnection{ + url: "ws://localhost:8080/ws", + } + wsConn.open() + + ticker := time.NewTicker(time.Second) + defer ticker.Stop() + + for timestamp := range ticker.C { + wsConn.writeMessage(timestamp.String()) + } +} diff --git a/ws-server/main.go b/ws-server/main.go index 29361da..86fbd5f 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -2,10 +2,48 @@ package main import ( "fmt" + "log" + "net/http" "github.com/gorilla/websocket" ) +var upgrader = websocket.Upgrader{} + +// upgrader.CheckOrigin = func(r *http.Request) bool { return true } + +func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { + conn, err := upgrader.Upgrade(responseWriter, request, nil) + if err != nil { + fmt.Println("Error during connection upgrade:", err) + return + } + defer conn.Close() + + for { + messageType, packet, err := conn.ReadMessage() + + if err != nil { + fmt.Println("Error during message reading:", err) + break + } + log.Printf("Received: %v %s", messageType, packet) + + // err = conn.WriteMessage(messageType, packet) + // if err != nil { + // fmt.Println("Error during message writing:", err) + // break + // } + } +} + func main() { - fmt.Println("Hello, World!") -} \ No newline at end of file + log.SetFlags(log.Lmicroseconds) + + http.HandleFunc("/ws", wsHandler) + log.Println("Server starting on :8080") + err := http.ListenAndServe(":8080", nil) + if err != nil { + log.Println("Error starting server:", err) + } +} From eccd113e908c1dfb409dfb5b5c654d3089763371 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 01:03:11 +0200 Subject: [PATCH 10/35] Assemble useful client-server communication --- ws-client/main.go | 9 +++---- ws-server/main.go | 60 ++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 52 insertions(+), 17 deletions(-) diff --git a/ws-client/main.go b/ws-client/main.go index b87a3a3..50ab8d8 100644 --- a/ws-client/main.go +++ b/ws-client/main.go @@ -66,11 +66,8 @@ func main() { url: "ws://localhost:8080/ws", } wsConn.open() - - ticker := time.NewTicker(time.Second) - defer ticker.Stop() - - for timestamp := range ticker.C { - wsConn.writeMessage(timestamp.String()) + for { + log.Printf("zzz...") + time.Sleep(30 * time.Second) } } diff --git a/ws-server/main.go b/ws-server/main.go index 86fbd5f..ef4bb4a 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -1,7 +1,9 @@ package main import ( + "encoding/json" "fmt" + "io" "log" "net/http" @@ -9,8 +11,7 @@ import ( ) var upgrader = websocket.Upgrader{} - -// upgrader.CheckOrigin = func(r *http.Request) bool { return true } +var wsBroadcast = make(chan []byte, 100) func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { conn, err := upgrader.Upgrade(responseWriter, request, nil) @@ -18,29 +19,66 @@ func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { fmt.Println("Error during connection upgrade:", err) return } - defer conn.Close() + go wsHandleRead(conn) + wsHandleWrite(conn) +} + +func wsHandleRead(conn *websocket.Conn) { + log.Printf("Starting read handler") for { messageType, packet, err := conn.ReadMessage() - if err != nil { fmt.Println("Error during message reading:", err) - break + return } log.Printf("Received: %v %s", messageType, packet) - - // err = conn.WriteMessage(messageType, packet) - // if err != nil { - // fmt.Println("Error during message writing:", err) - // break - // } } } +func wsHandleWrite(conn *websocket.Conn) { + log.Println("Starting write handler") + for { + packet := <-wsBroadcast + log.Printf("Broadcasting: %s", packet) + err := conn.WriteMessage(websocket.TextMessage, packet) + if err != nil { + fmt.Println("Error during message writing:", err) + return + } + } +} + +type DownloadReq struct { + Link string `json:"link"` +} + +func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { + body, err := io.ReadAll(request.Body) + if err != nil { + log.Printf("Error reading request body: %v", err) + http.Error(responseWriter, "Error reading request body", http.StatusBadRequest) + return + } + defer request.Body.Close() + + req := DownloadReq{} + err = json.Unmarshal(body, &req) + if err != nil { + log.Printf("Error parsing JSON: %v", err) + http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest) + return + } + + log.Printf("Received download request: %s", req.Link) + wsBroadcast <- []byte(req.Link) +} + func main() { log.SetFlags(log.Lmicroseconds) http.HandleFunc("/ws", wsHandler) + http.HandleFunc("/download", handleDownload) log.Println("Server starting on :8080") err := http.ListenAndServe(":8080", nil) if err != nil { From 0e9ddcd7a4260cabc190c5f49a4bee8355d67fc8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 09:38:48 +0200 Subject: [PATCH 11/35] Add server build files --- ws-server/build.sh | 3 +++ ws-server/captain-definition | 4 ++++ ws-server/deploy.tar | Bin 0 -> 10240 bytes ws-server/dockerfile | 21 +++++++++++++++++++++ 4 files changed, 28 insertions(+) create mode 100644 ws-server/build.sh create mode 100644 ws-server/captain-definition create mode 100644 ws-server/deploy.tar create mode 100644 ws-server/dockerfile diff --git a/ws-server/build.sh b/ws-server/build.sh new file mode 100644 index 0000000..5e455f9 --- /dev/null +++ b/ws-server/build.sh @@ -0,0 +1,3 @@ +# docker build -t youtube-download-ws-server . + +tar -cf deploy.tar captain-definition dockerfile main.go go.mod go.sum \ No newline at end of file diff --git a/ws-server/captain-definition b/ws-server/captain-definition new file mode 100644 index 0000000..774ff1a --- /dev/null +++ b/ws-server/captain-definition @@ -0,0 +1,4 @@ +{ + "schemaVersion": 2, + "dockerfilePath": "./dockerfile" +} diff --git a/ws-server/deploy.tar b/ws-server/deploy.tar new file mode 100644 index 0000000000000000000000000000000000000000..eaebdb91f8cd56a13d4462908b5dbefd7d28f6b0 GIT binary patch literal 10240 zcmeHLSyS6c5a!jc;y=u)Hd{^tI(!n>`oWkh8^FXs4pOPjSQ^O+I?T)nLQ?zRw|ivC z1{@$EPSs}R2WvFy^Xp?;vgue-KU#H|>qoxy<7nxJ|4jHb>-8nGX_~F3x>}}bE#W@? zELdChX02YWSv91yU}Y{5^M_k9M_?i(<%BGCoe%&Jl2aM;Ids41)|cfnA)2s9ETpd( z7mxs2n^afSwiDZ@jJtlo4yYW#zNQ<~J#D%C&vN+}#pVCv)-;@p^B8OVw)$_j8hQO! z>(CdhYSU~M^}o1?OeJp(`g_ES0~&d4ORrY-I-!CLsbI_HMB{8m&d`-xh3UT42 z)am+blpc?haZ0F`^Ie*{-`m?D?=*#n_I?`1?_Qx~-HX4Q|2~X&)PLQq*3d+`^ zSpR0VIREbh?SpB?)IOyiBbW!6dww{MAEK3UJ2_rMmjDx&*PO zt`U>QNXjwXAeHCKQ8Lu+I5fPN`$0gB3pNxuOd$JXq&kP8D;ay7I*gMJ$)D@Tlg%$M zTOiSeSm)4AY$_zsh4hqF>=BK~yZ6IO$yNwVl1iYPL^crb(8viGuW%;DaU|Gl?n}^s z*6D*Qdn<&qvxI@zCzGe@?lFOpsAR`cv_cr?q}|Ew%xt?RJQuV=B0m6QO1`UDf9v3e zl2-!c^aJikGKebLCg(9HP6GEHK~>Dg$vacj)|6_LRN_n~93oQ!IN%?eV&>z3QKy2; zNSAFYce7o^as`*1hdh}g-a{G6N{DA_q+8JrVJwzt9|t`I5Jy-w)a8Qhh*2qILO|{u zU5+6Rm`F_Kn4w_+ICb_?qKfin4luVMGm)<>083_Jnc{-LluEOf517sTbLOokl+A3waha2fvq_YeDCR!eo^=Woj%hVh zpsIH$>T8y-kV&1-69NJhBP5Va#5#2VIP66#VzV`0f){h>U$fThy&x703kuwT2bfwJ zdLRZHX9@GALlANOI0`9;c_^r)(EI@`_Xc1LtO&qghyDF;c6^rk-VTHDvvJ6ZIiD*t zC?+)m&ghtB=+R7dL9I8iF|Bqy?kdx!cgW;cKs`}G)w`i@b3ho1%@HZeDpu*y zvIXZ%MKdn2uVh`9qxPFL*(i*@tNxiq3CVLORUcs-Mr5dzNza9|puJ|UnX2DTYG29U zg)bB=>PF5Xa#XaG>b8UDik>v9b2UP#o$0G-O+R?3CtTyc*SqjmFc>V}m-~kFMqJ?Tj`%MXUK4IZ@~+5GW8R5GW8R5GW8R5GW8R5GWA% HH4*q1vQz47 literal 0 HcmV?d00001 diff --git a/ws-server/dockerfile b/ws-server/dockerfile new file mode 100644 index 0000000..5c514db --- /dev/null +++ b/ws-server/dockerfile @@ -0,0 +1,21 @@ +FROM golang:1.22.4 as base + +WORKDIR $GOPATH/src/app/ + +COPY . . + +RUN go mod download +RUN go mod verify + +RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /main . + +FROM scratch + +COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=base /etc/passwd /etc/passwd +COPY --from=base /etc/group /etc/group + +COPY --from=base /main . + +CMD ["/main"] \ No newline at end of file From b4d161e8e97e68b9fd05e8d0834024c147799097 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 09:38:55 +0200 Subject: [PATCH 12/35] Integrate ws client into downloader --- downloader/download.go | 11 +++++++ downloader/go.mod | 5 +++- downloader/go.sum | 2 ++ downloader/main.go | 23 +++++++++------ downloader/ws-client.go | 63 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 95 insertions(+), 9 deletions(-) create mode 100644 downloader/ws-client.go diff --git a/downloader/download.go b/downloader/download.go index a0147cf..c4c725c 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -30,3 +30,14 @@ func Download(event PBEvent, status chan error) { log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link) SetDownloaded(event) } + +func DownloadURL(url string, status chan error) { + log.Printf("Downloading %s", url) + _, err := dl.Run(context.TODO(), url) + if err != nil { + status <- err + return + } + + log.Printf("Downloaded %s", url) +} diff --git a/downloader/go.mod b/downloader/go.mod index fa0d15d..5157924 100644 --- a/downloader/go.mod +++ b/downloader/go.mod @@ -20,4 +20,7 @@ require ( gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect ) -require github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 +require ( + github.com/gorilla/websocket v1.5.3 + github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 +) diff --git a/downloader/go.sum b/downloader/go.sum index 445796d..1b4075d 100644 --- a/downloader/go.sum +++ b/downloader/go.sum @@ -20,6 +20,8 @@ github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8 github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= diff --git a/downloader/main.go b/downloader/main.go index 145fbd0..d51217e 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -9,6 +9,8 @@ const POCKETBASE_URL = `https://pocketbase-scratch.site.quack-lab.dev/api/collec const POCKETBASE_REALTIME = `https://pocketbase-scratch.site.quack-lab.dev/api/realtime` const COLLECTION_NAME = "youtubedownload" const FULL_URL = POCKETBASE_URL + "/" + COLLECTION_NAME + "/records" +const WEBSOCKET_SERVER = "ws://youtube-download-ws-server.site.quack-lab.dev/ws" +const WEBSOCKET_SERVER_ALT = "ws://localhost:8080/ws" func main() { log.SetFlags(log.Lmicroseconds) @@ -37,19 +39,24 @@ func main() { // } // log.Printf("Data: %+v\n", data) - listener := new(RealtimeListener) - listener.Url = POCKETBASE_REALTIME - listener.Collections = []string{COLLECTION_NAME} - listener.initialize() + // listener := new(RealtimeListener) + // listener.Url = POCKETBASE_REALTIME + // listener.Collections = []string{COLLECTION_NAME} + // listener.initialize() - status := make(chan error) + ws := new(WSConnection) + ws.url = WEBSOCKET_SERVER + ws.Open() + + status := make(chan error, 16) for { select { - case event := <-listener.Create: - log.Printf("Create event: %+v\n", event) + case event := <-ws.ReadChan: eventCopy := event + log.Printf("New event: %+v", eventCopy) go func() { - Download(eventCopy, status) + // Download(eventCopy, status) + DownloadURL(eventCopy, status) // go DownloadNative(event, status) for status := range status { log.Printf("Status: %s\n", status) diff --git a/downloader/ws-client.go b/downloader/ws-client.go new file mode 100644 index 0000000..ca9357f --- /dev/null +++ b/downloader/ws-client.go @@ -0,0 +1,63 @@ +package main + +import ( + "log" + "time" + + "github.com/gorilla/websocket" +) + +type WSConnection struct { + url string + conn *websocket.Conn + errChan chan error + ReadChan chan string +} + +func (ws *WSConnection) readMessage() { + log.Printf("Reading messages") + for { + _, message, err := ws.conn.ReadMessage() + if err != nil { + ws.errChan <- err + return + } + log.Printf("Received: %s", message) + ws.ReadChan <- string(message) + } +} + +func (ws *WSConnection) writeMessage(message string) { + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(message)) + if err != nil { + log.Printf("Error during message writing: %v", err) + ws.errChan <- err + return + } +} + +func (ws *WSConnection) handleError() { + for { + err := <-ws.errChan + log.Println("Error during message reading:", err) + + time.Sleep(5 * time.Second) + ws.Open() + } +} + +func (ws *WSConnection) Open() { + log.Printf("Connecting to %s", ws.url) + conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil) + if err != nil { + log.Println("Error during connection:", err) + ws.errChan <- err + return + } + log.Printf("Connected") + ws.conn = conn + ws.errChan = make(chan error) + ws.ReadChan = make(chan string, 1024) + go ws.readMessage() + go ws.handleError() +} From d5a70a27006907714a91514867c01824040f1a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 18:08:25 +0200 Subject: [PATCH 13/35] Rework websockets and add timeouts --- downloader/ws-client.go | 72 +++++++++++++++++++++----- ws-server/main.go | 109 ++++++++++++++++++++++++++++++---------- 2 files changed, 142 insertions(+), 39 deletions(-) diff --git a/downloader/ws-client.go b/downloader/ws-client.go index ca9357f..ba982a9 100644 --- a/downloader/ws-client.go +++ b/downloader/ws-client.go @@ -2,22 +2,30 @@ package main import ( "log" + "sync" "time" "github.com/gorilla/websocket" ) +const TIMEOUT = 6 +const IDLE_TIMEOUT = TIMEOUT * time.Second +const PING_INTERVAL = (TIMEOUT - 1) * time.Second + type WSConnection struct { - url string - conn *websocket.Conn - errChan chan error - ReadChan chan string + url string + conn *websocket.Conn + errChan chan error + writeLock sync.Mutex + ReadChan chan string + WriteChan chan string } -func (ws *WSConnection) readMessage() { +func (ws *WSConnection) messageReader() { log.Printf("Reading messages") for { _, message, err := ws.conn.ReadMessage() + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) if err != nil { ws.errChan <- err return @@ -27,12 +35,38 @@ func (ws *WSConnection) readMessage() { } } -func (ws *WSConnection) writeMessage(message string) { - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(message)) - if err != nil { - log.Printf("Error during message writing: %v", err) - ws.errChan <- err - return +func (ws *WSConnection) messageSender() { + log.Printf("Sending messages") + for { + msg := <-ws.WriteChan + ws.writeLock.Lock() + + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + log.Printf("Sending: %s", msg) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Error during message writing: %v", err) + ws.errChan <- err + return + } + ws.writeLock.Unlock() + } +} + +func (ws *WSConnection) pinger() { + log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL) + for { + time.Sleep(PING_INTERVAL) + + // log.Printf("Ping") + ws.writeLock.Lock() + err := ws.conn.WriteMessage(websocket.PingMessage, nil) + if err != nil { + log.Println("Error during ping:", err) + ws.errChan <- err + return + } + ws.writeLock.Unlock() } } @@ -58,6 +92,20 @@ func (ws *WSConnection) Open() { ws.conn = conn ws.errChan = make(chan error) ws.ReadChan = make(chan string, 1024) - go ws.readMessage() + ws.WriteChan = make(chan string, 1024) + + ws.conn.SetReadLimit(1024) + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetPongHandler(func(string) error { + // log.Println("Pong") + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + return nil + }) + + go ws.messageReader() + go ws.messageSender() go ws.handleError() + go ws.pinger() } diff --git a/ws-server/main.go b/ws-server/main.go index ef4bb4a..b39dd8c 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -6,6 +6,8 @@ import ( "io" "log" "net/http" + "sync" + "time" "github.com/gorilla/websocket" ) @@ -13,6 +15,83 @@ import ( var upgrader = websocket.Upgrader{} var wsBroadcast = make(chan []byte, 100) +const TIMEOUT = 6 +const IDLE_TIMEOUT = TIMEOUT * time.Second +const PING_INTERVAL = (TIMEOUT - 1) * time.Second + +type WSConnection struct { + conn *websocket.Conn + writeLock sync.Mutex + ReadChan chan string + WriteChan chan string +} + +func (ws *WSConnection) messageReader() { + log.Printf("Reading messages") + for { + _, message, err := ws.conn.ReadMessage() + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + if err != nil { + return + } + log.Printf("Received: %s", message) + ws.ReadChan <- string(message) + } +} + +func (ws *WSConnection) messageSender() { + log.Printf("Sending messages") + for { + msg := <-ws.WriteChan + ws.writeLock.Lock() + + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + log.Printf("Sending: %s", msg) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Error during message writing: %v", err) + return + } + ws.writeLock.Unlock() + } +} + +func (ws *WSConnection) pinger() { + log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL) + for { + time.Sleep(PING_INTERVAL) + + // log.Printf("Ping") + ws.writeLock.Lock() + err := ws.conn.WriteMessage(websocket.PingMessage, nil) + if err != nil { + log.Println("Error during ping:", err) + return + } + ws.writeLock.Unlock() + } +} + +func (ws *WSConnection) Open() { + log.Printf("Client connected") + ws.ReadChan = make(chan string, 1024) + ws.WriteChan = make(chan string, 1024) + + ws.conn.SetReadLimit(1024) + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetPongHandler(func(string) error { + // log.Println("Pong") + ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + return nil + }) + + go ws.messageReader() + go ws.messageSender() + go ws.pinger() +} + func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { conn, err := upgrader.Upgrade(responseWriter, request, nil) if err != nil { @@ -20,33 +99,9 @@ func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { return } - go wsHandleRead(conn) - wsHandleWrite(conn) -} - -func wsHandleRead(conn *websocket.Conn) { - log.Printf("Starting read handler") - for { - messageType, packet, err := conn.ReadMessage() - if err != nil { - fmt.Println("Error during message reading:", err) - return - } - log.Printf("Received: %v %s", messageType, packet) - } -} - -func wsHandleWrite(conn *websocket.Conn) { - log.Println("Starting write handler") - for { - packet := <-wsBroadcast - log.Printf("Broadcasting: %s", packet) - err := conn.WriteMessage(websocket.TextMessage, packet) - if err != nil { - fmt.Println("Error during message writing:", err) - return - } - } + ws := new(WSConnection) + ws.conn = conn + ws.Open() } type DownloadReq struct { From eb6764538b44d875cddd38b6230ef4a89fba13aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 18:09:28 +0200 Subject: [PATCH 14/35] Rework DL to use new endpoint --- dl/dl.go | 28 ++-------------------------- 1 file changed, 2 insertions(+), 26 deletions(-) diff --git a/dl/dl.go b/dl/dl.go index 9018e9b..14e61eb 100644 --- a/dl/dl.go +++ b/dl/dl.go @@ -11,28 +11,12 @@ import ( "sync" ) -const POCKETBASE_URL = `https://pocketbase-scratch.site.quack-lab.dev/api/collections` -const POCKETBASE_REALTIME = `https://pocketbase-scratch.site.quack-lab.dev/api/realtime` -const COLLECTION_NAME = "youtubedownload" -const FULL_URL = POCKETBASE_URL + "/" + COLLECTION_NAME + "/records" +const URL = `https://youtube-download-ws-server.site.quack-lab.dev/download` type Item struct { Link string `json:"link"` } -type APIError struct { - Code int `json:"code"` - Message string `json:"message"` - Data APIErrorData `json:"data"` -} -type APIErrorData struct { - Link APIErrorLink `json:"link"` -} -type APIErrorLink struct { - Code string `json:"code"` - Message string `json:"message"` -} - var wg sync.WaitGroup func main() { @@ -49,7 +33,7 @@ func main() { func Download(url string) { defer wg.Done() - req, err := http.NewRequestWithContext(context.Background(), "POST", FULL_URL, nil) + req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil) if err != nil { log.Printf("Error creating POST request: %++v", err) return @@ -82,14 +66,6 @@ func Download(url string) { log.Printf("Error reading response body: %++v", err) return } - var data APIError - err = json.Unmarshal(body, &data) - if err != nil { - log.Printf("Error unmarshaling JSON: %++v", err) - return - } - - log.Printf("API error: %++v", data) return } else { log.Printf("Enqueued %s", url) From e0635c3bc99e0f785640e613a8276ee10700b668 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 18:16:16 +0200 Subject: [PATCH 15/35] Remove deploy --- .gitignore | 1 + ws-server/deploy.tar | Bin 10240 -> 0 bytes 2 files changed, 1 insertion(+) delete mode 100644 ws-server/deploy.tar diff --git a/.gitignore b/.gitignore index 854f053..faf7207 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ .idea main.exe logs.log +ws-server/deploy.tar diff --git a/ws-server/deploy.tar b/ws-server/deploy.tar deleted file mode 100644 index eaebdb91f8cd56a13d4462908b5dbefd7d28f6b0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 10240 zcmeHLSyS6c5a!jc;y=u)Hd{^tI(!n>`oWkh8^FXs4pOPjSQ^O+I?T)nLQ?zRw|ivC z1{@$EPSs}R2WvFy^Xp?;vgue-KU#H|>qoxy<7nxJ|4jHb>-8nGX_~F3x>}}bE#W@? zELdChX02YWSv91yU}Y{5^M_k9M_?i(<%BGCoe%&Jl2aM;Ids41)|cfnA)2s9ETpd( z7mxs2n^afSwiDZ@jJtlo4yYW#zNQ<~J#D%C&vN+}#pVCv)-;@p^B8OVw)$_j8hQO! z>(CdhYSU~M^}o1?OeJp(`g_ES0~&d4ORrY-I-!CLsbI_HMB{8m&d`-xh3UT42 z)am+blpc?haZ0F`^Ie*{-`m?D?=*#n_I?`1?_Qx~-HX4Q|2~X&)PLQq*3d+`^ zSpR0VIREbh?SpB?)IOyiBbW!6dww{MAEK3UJ2_rMmjDx&*PO zt`U>QNXjwXAeHCKQ8Lu+I5fPN`$0gB3pNxuOd$JXq&kP8D;ay7I*gMJ$)D@Tlg%$M zTOiSeSm)4AY$_zsh4hqF>=BK~yZ6IO$yNwVl1iYPL^crb(8viGuW%;DaU|Gl?n}^s z*6D*Qdn<&qvxI@zCzGe@?lFOpsAR`cv_cr?q}|Ew%xt?RJQuV=B0m6QO1`UDf9v3e zl2-!c^aJikGKebLCg(9HP6GEHK~>Dg$vacj)|6_LRN_n~93oQ!IN%?eV&>z3QKy2; zNSAFYce7o^as`*1hdh}g-a{G6N{DA_q+8JrVJwzt9|t`I5Jy-w)a8Qhh*2qILO|{u zU5+6Rm`F_Kn4w_+ICb_?qKfin4luVMGm)<>083_Jnc{-LluEOf517sTbLOokl+A3waha2fvq_YeDCR!eo^=Woj%hVh zpsIH$>T8y-kV&1-69NJhBP5Va#5#2VIP66#VzV`0f){h>U$fThy&x703kuwT2bfwJ zdLRZHX9@GALlANOI0`9;c_^r)(EI@`_Xc1LtO&qghyDF;c6^rk-VTHDvvJ6ZIiD*t zC?+)m&ghtB=+R7dL9I8iF|Bqy?kdx!cgW;cKs`}G)w`i@b3ho1%@HZeDpu*y zvIXZ%MKdn2uVh`9qxPFL*(i*@tNxiq3CVLORUcs-Mr5dzNza9|puJ|UnX2DTYG29U zg)bB=>PF5Xa#XaG>b8UDik>v9b2UP#o$0G-O+R?3CtTyc*SqjmFc>V}m-~kFMqJ?Tj`%MXUK4IZ@~+5GW8R5GW8R5GW8R5GW8R5GWA% HH4*q1vQz47 From d94b581d41a29e8f91aa66dab7bd4916bc00023a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 18:20:16 +0200 Subject: [PATCH 16/35] Remove unused api and realtime --- downloader/api.go | 124 +++++++++++++++++----------------- downloader/download.go | 18 ++--- downloader/main.go | 5 -- downloader/realtime.go | 150 ++++++++++++++++++++--------------------- 4 files changed, 146 insertions(+), 151 deletions(-) diff --git a/downloader/api.go b/downloader/api.go index 68db7d5..2529490 100644 --- a/downloader/api.go +++ b/downloader/api.go @@ -1,75 +1,75 @@ package main -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "log" - "net/http" -) +// import ( +// "bytes" +// "context" +// "encoding/json" +// "fmt" +// "io" +// "log" +// "net/http" +// ) -type APIError struct { - Code int `json:"code"` - Message string `json:"message"` - Data APIErrorData `json:"data"` -} +// type APIError struct { +// Code int `json:"code"` +// Message string `json:"message"` +// Data APIErrorData `json:"data"` +// } -type APIErrorData struct { - Link APIErrorLink `json:"link"` -} +// type APIErrorData struct { +// Link APIErrorLink `json:"link"` +// } -type APIErrorLink struct { - Code string `json:"code"` - Message string `json:"message"` -} +// type APIErrorLink struct { +// Code string `json:"code"` +// Message string `json:"message"` +// } -func SetDownloaded(item PBEvent) (err error) { - req, err := http.NewRequestWithContext(context.Background(), "PATCH", FULL_URL+"/"+item.Record.Id, nil) - if err != nil { - log.Printf("Error creating PATCH request: %++v", err) - return err - } - req.Header.Set("Content-Type", "application/json") +// func SetDownloaded(item PBEvent) (err error) { +// req, err := http.NewRequestWithContext(context.Background(), "PATCH", FULL_URL+"/"+item.Record.Id, nil) +// if err != nil { +// log.Printf("Error creating PATCH request: %++v", err) +// return err +// } +// req.Header.Set("Content-Type", "application/json") - partialItem := new(PBEvent) - partialItem.Record = item.Record - partialItem.Record.Downloaded = true +// partialItem := new(PBEvent) +// partialItem.Record = item.Record +// partialItem.Record.Downloaded = true - body, err := json.Marshal(partialItem.Record) - if err != nil { - log.Printf("Error marshalling subscription body: %++v", err) - return err - } - req.Body = io.NopCloser(bytes.NewReader(body)) +// body, err := json.Marshal(partialItem.Record) +// if err != nil { +// log.Printf("Error marshalling subscription body: %++v", err) +// return err +// } +// req.Body = io.NopCloser(bytes.NewReader(body)) - client := http.Client{} - res, err := client.Do(req) - if err != nil { - log.Printf("Error sending PATCH request: %++v", err) - return err - } - defer res.Body.Close() +// client := http.Client{} +// res, err := client.Do(req) +// if err != nil { +// log.Printf("Error sending PATCH request: %++v", err) +// return err +// } +// defer res.Body.Close() - if res.StatusCode != http.StatusOK { - log.Printf("Non-OK HTTP status: %d", res.StatusCode) +// if res.StatusCode != http.StatusOK { +// log.Printf("Non-OK HTTP status: %d", res.StatusCode) - body, err = io.ReadAll(res.Body) - if err != nil { - log.Printf("Error reading response body: %++v", err) - return err - } - var data APIError - err = json.Unmarshal(body, &data) - if err != nil { - log.Printf("Error unmarshaling JSON: %++v", err) - return err - } +// body, err = io.ReadAll(res.Body) +// if err != nil { +// log.Printf("Error reading response body: %++v", err) +// return err +// } +// var data APIError +// err = json.Unmarshal(body, &data) +// if err != nil { +// log.Printf("Error unmarshaling JSON: %++v", err) +// return err +// } - log.Printf("API error: %++v", data) - return fmt.Errorf("Non-OK HTTP status, err: %++v", data) - } +// log.Printf("API error: %++v", data) +// return fmt.Errorf("Non-OK HTTP status, err: %++v", data) +// } - return nil -} +// return nil +// } diff --git a/downloader/download.go b/downloader/download.go index c4c725c..8d34778 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -20,16 +20,16 @@ var dl = ytdlp.New(). RecodeVideo("mp4"). ConcurrentFragments(4) -func Download(event PBEvent, status chan error) { - _, err := dl.Run(context.TODO(), event.Record.Link) - if err != nil { - status <- err - return - } +// func Download(event PBEvent, status chan error) { +// _, err := dl.Run(context.TODO(), event.Record.Link) +// if err != nil { +// status <- err +// return +// } - log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link) - SetDownloaded(event) -} +// log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link) +// SetDownloaded(event) +// } func DownloadURL(url string, status chan error) { log.Printf("Downloading %s", url) diff --git a/downloader/main.go b/downloader/main.go index d51217e..f54b382 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -5,16 +5,11 @@ import ( "time" ) -const POCKETBASE_URL = `https://pocketbase-scratch.site.quack-lab.dev/api/collections` -const POCKETBASE_REALTIME = `https://pocketbase-scratch.site.quack-lab.dev/api/realtime` -const COLLECTION_NAME = "youtubedownload" -const FULL_URL = POCKETBASE_URL + "/" + COLLECTION_NAME + "/records" const WEBSOCKET_SERVER = "ws://youtube-download-ws-server.site.quack-lab.dev/ws" const WEBSOCKET_SERVER_ALT = "ws://localhost:8080/ws" func main() { log.SetFlags(log.Lmicroseconds) - log.Println(FULL_URL) // res, err := http.Get(FULL_URL) // if err != nil { diff --git a/downloader/realtime.go b/downloader/realtime.go index 681db0f..462ea8c 100644 --- a/downloader/realtime.go +++ b/downloader/realtime.go @@ -1,89 +1,89 @@ package main -import ( - "bytes" - "encoding/json" - "log" - "net/http" +// import ( +// "bytes" +// "encoding/json" +// "log" +// "net/http" - "github.com/r3labs/sse" -) +// "github.com/r3labs/sse" +// ) -type RealtimeListener struct { - Url string - Collections []string - Create chan PBEvent - Update chan PBEvent - Delete chan PBEvent - client *sse.Client -} +// type RealtimeListener struct { +// Url string +// Collections []string +// Create chan PBEvent +// Update chan PBEvent +// Delete chan PBEvent +// client *sse.Client +// } -type Subscription struct { - ClientId string `json:"clientId"` - Subscriptions []string `json:"subscriptions"` -} +// type Subscription struct { +// ClientId string `json:"clientId"` +// Subscriptions []string `json:"subscriptions"` +// } -func (listener RealtimeListener) handlePbEvent(msg *sse.Event) { - pbEvent := new(PBEvent) - err := json.Unmarshal(msg.Data, &pbEvent) - if err != nil { - log.Printf("Error unmarshalling event: %v\n", err) - return - } - log.Printf("Received event: %++v", pbEvent) +// func (listener RealtimeListener) handlePbEvent(msg *sse.Event) { +// pbEvent := new(PBEvent) +// err := json.Unmarshal(msg.Data, &pbEvent) +// if err != nil { +// log.Printf("Error unmarshalling event: %v\n", err) +// return +// } +// log.Printf("Received event: %++v", pbEvent) - if pbEvent.ClientId != "" { - listener.doSubscribe(pbEvent.ClientId) - } +// if pbEvent.ClientId != "" { +// listener.doSubscribe(pbEvent.ClientId) +// } - if pbEvent.Action != "" { - go listener.shipEvent(*pbEvent) - } -} +// if pbEvent.Action != "" { +// go listener.shipEvent(*pbEvent) +// } +// } -func (listener RealtimeListener) shipEvent(event PBEvent) { - switch event.Action { - case "create": - listener.Create <- event - case "update": - listener.Update <- event - case "delete": - listener.Delete <- event - default: - log.Printf("Unknown action: %v\n", event.Action) - } -} +// func (listener RealtimeListener) shipEvent(event PBEvent) { +// switch event.Action { +// case "create": +// listener.Create <- event +// case "update": +// listener.Update <- event +// case "delete": +// listener.Delete <- event +// default: +// log.Printf("Unknown action: %v\n", event.Action) +// } +// } -func (listener RealtimeListener) doSubscribe(clientId string) { - subscription := Subscription{ - ClientId: clientId, - Subscriptions: listener.Collections, - } - log.Printf("Subscribing client: %v to %++v", clientId, subscription) +// func (listener RealtimeListener) doSubscribe(clientId string) { +// subscription := Subscription{ +// ClientId: clientId, +// Subscriptions: listener.Collections, +// } +// log.Printf("Subscribing client: %v to %++v", clientId, subscription) - body, err := json.Marshal(subscription) - if err != nil { - log.Printf("Error marshalling subscription body: %v\n", err) - return - } +// body, err := json.Marshal(subscription) +// if err != nil { +// log.Printf("Error marshalling subscription body: %v\n", err) +// return +// } - resp, err := http.Post(POCKETBASE_REALTIME, "application/json", bytes.NewBuffer(body)) - if err != nil { - log.Printf("Error posting subscription: %v\n", err) - return - } - defer resp.Body.Close() +// resp, err := http.Post(POCKETBASE_REALTIME, "application/json", bytes.NewBuffer(body)) +// if err != nil { +// log.Printf("Error posting subscription: %v\n", err) +// return +// } +// defer resp.Body.Close() - if resp.StatusCode != http.StatusNoContent { - log.Printf("Subscription request failed with status: %v\n", resp.Status) - } -} +// if resp.StatusCode != http.StatusNoContent { +// log.Printf("Subscription request failed with status: %v\n", resp.Status) +// } +// } -func (listener *RealtimeListener) initialize() { - listener.Update = make(chan PBEvent, 32) - listener.Create = make(chan PBEvent, 32) - listener.Delete = make(chan PBEvent, 32) - log.Print("Initialized") - listener.client = sse.NewClient(listener.Url) - go listener.client.Subscribe("", listener.handlePbEvent) -} +// func (listener *RealtimeListener) initialize() { +// listener.Update = make(chan PBEvent, 32) +// listener.Create = make(chan PBEvent, 32) +// listener.Delete = make(chan PBEvent, 32) +// log.Print("Initialized") +// listener.client = sse.NewClient(listener.Url) +// go listener.client.Subscribe("", listener.handlePbEvent) +// } From be1f62d27d57a76af08b6ffa2a52f5e281d7cdfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 18:23:58 +0200 Subject: [PATCH 17/35] Fix issue with server not broadcasting --- ws-server/main.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/ws-server/main.go b/ws-server/main.go index b39dd8c..6aeddf1 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -13,7 +13,7 @@ import ( ) var upgrader = websocket.Upgrader{} -var wsBroadcast = make(chan []byte, 100) +var wsBroadcast = make(chan string, 128) const TIMEOUT = 6 const IDLE_TIMEOUT = TIMEOUT * time.Second @@ -90,6 +90,12 @@ func (ws *WSConnection) Open() { go ws.messageReader() go ws.messageSender() go ws.pinger() + go func() { + for { + msg := <-wsBroadcast + ws.WriteChan <- msg + } + }() } func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { @@ -126,7 +132,7 @@ func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { } log.Printf("Received download request: %s", req.Link) - wsBroadcast <- []byte(req.Link) + wsBroadcast <- req.Link } func main() { From 85a0dcd8f5e117f0f87860f3ab48adca5a8ad536 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 18:45:18 +0200 Subject: [PATCH 18/35] Update tampermonkey script --- tampermonkey.js | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/tampermonkey.js b/tampermonkey.js index d17234e..d58e91c 100644 --- a/tampermonkey.js +++ b/tampermonkey.js @@ -7,7 +7,7 @@ // @noframes // ==/UserScript== -const API = "https://pocketbase-scratch.site.quack-lab.dev/api/collections/youtubedownload/records"; +const URL = `https://youtube-download-ws-server.site.quack-lab.dev/download`; function waitForElement(element, selector) { return new Promise((resolve) => { @@ -30,13 +30,10 @@ function waitForElement(element, selector) { } function parseVideo(videoElement) { - console.log(videoElement); hookVideo(videoElement); } function hookVideo(videoElement) { - console.log("Hooking", videoElement); - videoElement.addEventListener( "mousedown", function (e) { @@ -47,7 +44,7 @@ function hookVideo(videoElement) { const link = videoElement.querySelector("a#video-title-link").href; console.log(link); - fetch(API, { + fetch(URL, { method: "POST", headers: { "Content-Type": "application/json", From 7cac378d049cceebcbe55ce8fee990918036064f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 18 Jun 2024 20:14:26 +0200 Subject: [PATCH 19/35] add downloader build --- downloader/build.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/downloader/build.sh b/downloader/build.sh index 166a66b..d16e268 100644 --- a/downloader/build.sh +++ b/downloader/build.sh @@ -1 +1,3 @@ -docker build -t youtube-downloader . \ No newline at end of file +nssm stop YoutubeDownloader && \ +go build main && \ +nssm start YoutubeDownloader \ No newline at end of file From 0174f3d9b926acaade4a926b4672c8ebb3fd0c8c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Wed, 19 Jun 2024 21:40:31 +0200 Subject: [PATCH 20/35] Fix issue with download order not being propagated I assume it's because dead sockets are consuming broadcast messages --- downloader/ws-client.go | 10 +++++----- ws-server/main.go | 26 ++++++++++++++++++-------- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/downloader/ws-client.go b/downloader/ws-client.go index ba982a9..907839f 100644 --- a/downloader/ws-client.go +++ b/downloader/ws-client.go @@ -10,7 +10,7 @@ import ( const TIMEOUT = 6 const IDLE_TIMEOUT = TIMEOUT * time.Second -const PING_INTERVAL = (TIMEOUT - 1) * time.Second +const PING_INTERVAL = (TIMEOUT / 2) * time.Second type WSConnection struct { url string @@ -30,7 +30,7 @@ func (ws *WSConnection) messageReader() { ws.errChan <- err return } - log.Printf("Received: %s", message) + log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) ws.ReadChan <- string(message) } } @@ -42,7 +42,7 @@ func (ws *WSConnection) messageSender() { ws.writeLock.Lock() ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) - log.Printf("Sending: %s", msg) + log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) if err != nil { log.Printf("Error during message writing: %v", err) @@ -58,7 +58,7 @@ func (ws *WSConnection) pinger() { for { time.Sleep(PING_INTERVAL) - // log.Printf("Ping") + log.Printf("Ping") ws.writeLock.Lock() err := ws.conn.WriteMessage(websocket.PingMessage, nil) if err != nil { @@ -98,7 +98,7 @@ func (ws *WSConnection) Open() { ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetPongHandler(func(string) error { - // log.Println("Pong") + log.Println("Pong") ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) return nil diff --git a/ws-server/main.go b/ws-server/main.go index 6aeddf1..d9a3c9b 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -17,13 +17,14 @@ var wsBroadcast = make(chan string, 128) const TIMEOUT = 6 const IDLE_TIMEOUT = TIMEOUT * time.Second -const PING_INTERVAL = (TIMEOUT - 1) * time.Second +const PING_INTERVAL = (TIMEOUT / 2) * time.Second type WSConnection struct { conn *websocket.Conn writeLock sync.Mutex ReadChan chan string WriteChan chan string + ErrorChan chan error } func (ws *WSConnection) messageReader() { @@ -32,9 +33,10 @@ func (ws *WSConnection) messageReader() { _, message, err := ws.conn.ReadMessage() ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) if err != nil { + ws.ErrorChan <- err return } - log.Printf("Received: %s", message) + log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) ws.ReadChan <- string(message) } } @@ -46,10 +48,11 @@ func (ws *WSConnection) messageSender() { ws.writeLock.Lock() ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) - log.Printf("Sending: %s", msg) + log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) if err != nil { log.Printf("Error during message writing: %v", err) + ws.ErrorChan <- err return } ws.writeLock.Unlock() @@ -61,11 +64,12 @@ func (ws *WSConnection) pinger() { for { time.Sleep(PING_INTERVAL) - // log.Printf("Ping") + log.Printf("Ping") ws.writeLock.Lock() err := ws.conn.WriteMessage(websocket.PingMessage, nil) if err != nil { log.Println("Error during ping:", err) + ws.ErrorChan <- err return } ws.writeLock.Unlock() @@ -81,7 +85,7 @@ func (ws *WSConnection) Open() { ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetPongHandler(func(string) error { - // log.Println("Pong") + log.Println("Pong") ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) return nil @@ -92,8 +96,14 @@ func (ws *WSConnection) Open() { go ws.pinger() go func() { for { - msg := <-wsBroadcast - ws.WriteChan <- msg + select { + case err := <-ws.ErrorChan: + log.Printf("Error: %v", err) + ws.conn.Close() + return + case msg := <-wsBroadcast: + ws.WriteChan <- msg + } } }() } @@ -131,7 +141,7 @@ func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { return } - log.Printf("Received download request: %s", req.Link) + log.Printf("Received download request: %s, %d in channel", req.Link, len(wsBroadcast)) wsBroadcast <- req.Link } From b6e70406628ac4c2d8c55f09e14a6b05c204aba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 20 Jun 2024 17:46:43 +0200 Subject: [PATCH 21/35] Fix broadcast --- ws-server/main.go | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/ws-server/main.go b/ws-server/main.go index d9a3c9b..b1e0d86 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -14,6 +14,7 @@ import ( var upgrader = websocket.Upgrader{} var wsBroadcast = make(chan string, 128) +var connections = make(map[*WSConnection]bool) const TIMEOUT = 6 const IDLE_TIMEOUT = TIMEOUT * time.Second @@ -91,6 +92,8 @@ func (ws *WSConnection) Open() { return nil }) + connections[ws] = true + go ws.messageReader() go ws.messageSender() go ws.pinger() @@ -100,9 +103,10 @@ func (ws *WSConnection) Open() { case err := <-ws.ErrorChan: log.Printf("Error: %v", err) ws.conn.Close() + connections[ws] = false return - case msg := <-wsBroadcast: - ws.WriteChan <- msg + // case msg := <-wsBroadcast: + // ws.WriteChan <- msg } } }() @@ -142,7 +146,12 @@ func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { } log.Printf("Received download request: %s, %d in channel", req.Link, len(wsBroadcast)) - wsBroadcast <- req.Link + go func() { + for ws := range connections { + ws.WriteChan <- req.Link + } + }() + // wsBroadcast <- req.Link } func main() { From be38b3d602d2a8582a113e0ac59c668f2c554dfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 20 Jun 2024 17:51:48 +0200 Subject: [PATCH 22/35] Fix error handling --- ws-server/main.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ws-server/main.go b/ws-server/main.go index b1e0d86..06f8be9 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -81,6 +81,7 @@ func (ws *WSConnection) Open() { log.Printf("Client connected") ws.ReadChan = make(chan string, 1024) ws.WriteChan = make(chan string, 1024) + ws.ErrorChan = make(chan error, 16) ws.conn.SetReadLimit(1024) ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) @@ -103,6 +104,7 @@ func (ws *WSConnection) Open() { case err := <-ws.ErrorChan: log.Printf("Error: %v", err) ws.conn.Close() + log.Printf("Client disconnected") connections[ws] = false return // case msg := <-wsBroadcast: From 089e7f637f844e1e522dc946be7a59ffbdc4b2c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Fri, 21 Jun 2024 22:23:02 +0200 Subject: [PATCH 23/35] Limit download speed to not kill internet --- downloader/download.go | 5 +++-- downloader/main.go | 11 +++++++++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/downloader/download.go b/downloader/download.go index 8d34778..762c29c 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -13,12 +13,12 @@ var dl = ytdlp.New(). // FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]"). FormatSort("res,ext:mp4:m4a"). Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s"). - LimitRate("50M"). + LimitRate("5M"). // HTTPChunkSize("20M"). MarkWatched(). SponsorblockMark("all"). RecodeVideo("mp4"). - ConcurrentFragments(4) + ConcurrentFragments(6) // func Download(event PBEvent, status chan error) { // _, err := dl.Run(context.TODO(), event.Record.Link) @@ -40,4 +40,5 @@ func DownloadURL(url string, status chan error) { } log.Printf("Downloaded %s", url) + close(status) } diff --git a/downloader/main.go b/downloader/main.go index f54b382..d13f204 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -43,13 +43,20 @@ func main() { ws.url = WEBSOCKET_SERVER ws.Open() - status := make(chan error, 16) + sem := make(chan struct{}, 4) for { select { case event := <-ws.ReadChan: eventCopy := event - log.Printf("New event: %+v", eventCopy) + status := make(chan error) + sem <- struct{}{} + + log.Printf("New event: %+v; semaphore at: %d", eventCopy, len(sem)) go func() { + defer func() { + <-sem + log.Printf("Semaphore at: %d", len(sem)) + }() // Download(eventCopy, status) DownloadURL(eventCopy, status) // go DownloadNative(event, status) From aad190d94c5aed51cad20bd69cad402bc09c6fab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Tue, 25 Jun 2024 13:28:58 +0200 Subject: [PATCH 24/35] Add alerts to downloader --- downloader/assets/information.png | Bin 0 -> 4486 bytes downloader/download.go | 13 ++++++++++++- downloader/go.mod | 5 +++++ downloader/go.sum | 10 ++++++++++ 4 files changed, 27 insertions(+), 1 deletion(-) create mode 100644 downloader/assets/information.png diff --git a/downloader/assets/information.png b/downloader/assets/information.png new file mode 100644 index 0000000000000000000000000000000000000000..d47243dd9ab1adf0ac5520f80c698cb0ee6d2a46 GIT binary patch literal 4486 zcmV;15qa*3P)1TvDqdkIP2 zeeX`^oc-$``%HIuLtYFn?C-7f?!D*iv(Ns0f8+k`eUyms|C#hJdjP*|a%Z}rlu}rL z)GIff(?S9mnL*W6=Xs@+0~7%taCr4=i2)*@1vEq?%3o{)Kq=(|l}~-)ZI@pAp382k zlq=_Xo>w{fXPz#lIeC8-1i{o)ZEDATcmDLoFMaw4TY)JNX(iWF0F+WLFnIlqZ+gQ= zK7QTpEl&7T;W&{rbKSKw)|<`KZ86;-%y!061uE`hd}g zIBm=(qmV?-i6%%#TfvBdEVQdU0J#AhB*&1X%@We5X9vmOz&3uFqH<9Zt9KxU$uDG3s5 zqsio+(*!dI01D7zaweci)jRpEM5RV+O+sNJAq=d9!5J%VSrL$fDr)BW&Igf10QE{`UrbodXlaC|C>p*n_}6;^)+6>0->Em>tbyX zE3~|ISx{p0Fa|o(`yLuVo1;q<4W(l+l8B&DD1w$DXc=1dh+5Tf;pn@$V*PJ1GO!F| z3{eyjh9Smuqo2@BKBvDt!t2)j1{bZphJ7zR%_F<-;K1ZhDEB+~B?r&9T9K%HHa-m! zC9c~79`n{ood772$RcLFN)CkB!6as9gjNs{)D5+%kYUzy{ndX(r94CwMYLKi;v@?I ztrs?>NVH}3;`4aTlE2{a(dW5y>*uL94pOeTct+!S+VV^yJZUqFR154hpv|q$=mTOr zAnylagA)VIx}jAIsUA1H{oGG;>AJTeA_PHT?^J5e3~?bD;l6FR@U!vn z&_Co5m0b!Y1v)l}g!x$V2>^5X01ar1_>4nF47BQoW-X%evd<^4y_KQL7*P}~FhC4A8NTfX)1 zr!ivz#%TPq1{F(QB;`S*D+;ItjXVI#NYv)3la3H*)+6fEArgQ=5S$k7byJiPsTtURZmmFM)c>bwEQR`+q= zmjCLK^xPo4mg~;Hf$GsFK_en)MQvkGj~-#L%V16H14OWKRUnFlX5BD78FKmZ53_u9 z&AbLS+fnq(tXQ_1Ldl^}b|{t|`UYKAFJ0Hwc0OQSdEvWQ?rmUtGNf5IL~*v6@E~G& z>!dCKMA|h#L}=9`nzay_8swd?|JW%ruarV-&3muuJ&00vS9(D0CS1POk6`2TQ1f}BGFGAQ7YSoYrT>e=+x6sweLTUZlD>-ZB z_u0PhC-j#`SiAhSIC{o9EzmL3>yYSBWigkmdN13KeiO%caeO<+keH9iq&{HAOi<;+ zpdgq4!j>?3q{Ue)FYcMOoeCBFGMB7-OZt1ZjB!46)%thy;CFAK)bHR1pgntDvM@Mn zBBb((kO%}Vp)nnC*_rR5Tn%#xASEa|&|#`G?nf`jKMluDw2fmr}hfHJws zkcsS+wlN)W$?M*F()oNRHG>9^@4TNKdmiJ!__MTvIxEJ`;y2%TJ*$?Sy#RK5Kl`FH zui=3`+bIkvoJgUaI1ex>06k5>0w5+e0TWpe>QfP(Q#d8{-1M`%x$XO(ry11kM5dq- zfrC4q<O+Tk-F`Bxy~f$iVq!1NZpq9$w!6Z>o2z3Jch)Cc~4!TslHcwi|(!-hg5f|eu` zY+nw00P^jxdhI*Y@2)QJ<@+`>Hdv-uRs;>7iDL(UVSZv{ za2Y`m5d{Vlfw=LMxW{~WMiUSOV;~F-VJNhMrmf)gU~WDXhGF{s*yIuF)d*uiyWsko zfx#sw)#q$&MXc`%LxV9otv}9Y0txB734l#DvvPfP)HGq@*b68fm-CabE+i(~4LN)D#Ta9{%4Q2WCQcl}aTVGTJ2On@0%;yFTS+WR3kVh#!7n--I~^Z%(_OFKyXzA6~&Bsb!~Oy+^&ThN$DXx2l{UA2KIic%Bkb7o{Nz47C#hXDmJc7v^6! zJ#nX<2NV^`g=Cc|mj}$K0h-hak{;SsIG$!;#OJ5Ge#qh27tF-qY?5oVY8;(9falpN zd(e!KrpMUO$^|o=f7a0xFLM9ZJ17r(IG#pn)y7NC3F=M&awV~3(AMsh0>@W)Wkq?w zVdH~0FGw{-gvqI+IIf@^8=k_@;OY`#7|vURW&`*??)e%ZM<|$hI*F7 zl$gu}5-_wACo5w9=O5v@gHJ99 z;LT5dkDUh}VQ8t3UvhA~Y&k4#*P|XVYe_6Qlva4YqCBV>Sy|+c`@h5^KmWl=*}gFb zM|-?$!zXA=1+*Fwj;AOMxZLx|zjM>AALSpvb32KG8ZT!<6H&9#@Q5i1a`U>qTjFA*f zNLGmEULDrPL>)U81Tdg9PC?-q&ES|v)HM9dcR$1Lzx_roJ@=YZ7y)0s`$jI?^#C7w>t`qx=H5{kMG;d|Q)sQ@B{xZJeZCZgEjB)MGyl8w+YF3( z3@M@*%w!B%}# zTv}1ef}oUxb`*Z8#LI_|^R@5&C13o+_ZP(2T(ooFPx;Z-Z}ao#AEq?uvSe+E@_?-@ zS*ItqmI1MgAahTKl+_f|<_83WO4i_WiIn4G+bM^k$Y=eccb$|f&qd?M4zhFa zV?6Z4UA!={n_|Ubc!f`8*rV9z;Q9)u5SP9?)ZRM08~~E;cAI^9TyhgZ`#V|Vm-OBw zK}seY#t8Kj5pTZcw`T-k`<_QRd}I%Ul~D%!Mi{J&Fwi%GR+?)4I8(J_RBOjMRvqWL zgHQ78!6!IY9Y;HgQoqCEH6@A_heAo?cs3$sCGWhgx33E(=a@k*ohHzBV@3whu_o9* ziIEDLLZcRO`P%oecxV;1TCMB;2e)nH_WS<w>b4_p4|VaHAcignm}&KlMN!p`W`2{TiT4NDtr$@RVITyx2(P}F-+kX7GhMB< zIk$Rqnmadrh0Rah#n4ip@{kL0nJo&0D6r3QJi80baqR{u$5FTiI|tF)f|4+Ahm%#V zGcXhNH?ht4vj4W}aL=NsO0uRl?;oqY01_CY5KI8gnxS8v!JmHkKM9+rErEIT>3iAu z(9KK-#~5BwU|`I}E9$sEhqkB`^QGbltsI5p*oc#Cq01!Mq0M&?{+RbqOdb_kcVt?B`lJeOLfVi!lKqLjl{Ai+< zp*dwZaacHK?L|Dc{Q-WoWh1-yKgIFs38w3lcm=ylWAqH4ay&2he8KT0m0%)=&J87r zvzmV($uh}K=2HN2U}#iV551#62jZgQ-C_cwKy=7MPk+Zoz*u6sPqAcK0k5cWJzGX| zeQUdRKVPTrE}XiUOuo&)y#3w#@iX`UX>))U1e<1hMHpJ#GK zl8&n<0I41|tI~AJlJ0`&0zoU3&XVd*GKrm>7U~UX8YUN%ylA=0T5x|plFTn zZ5~~5L1B4hvZbs}5YqEA@_G6s-}`ah&+BB~(JqJgHjiT89;;z_P5}skX`s6Kwuy&- z=WkZM+b#HRejyc`iwc>Qg~$e0Sscrf-Zt{_=s~e&ENz zZeUzQB>&o+4=e}H2F?S{1qQIMm%N&(1xx}vfgQju;E=@c4tBmcSq4V2Z&r;0!`K&$ zUd>boUIvZ;hk+v^QcEuOdUe(X27xl*@vHIbY=Cuq=?jwybpeoGuztbiR|9}hW?=OH Y0K- Date: Tue, 25 Jun 2024 13:34:58 +0200 Subject: [PATCH 25/35] Enable middle click download on subscriptions as well --- tampermonkey.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tampermonkey.js b/tampermonkey.js index d58e91c..7ba6c71 100644 --- a/tampermonkey.js +++ b/tampermonkey.js @@ -63,7 +63,7 @@ function hookVideo(videoElement) { } async function main() { - const videosContainer = await waitForElement(document, "div#contents.style-scope.ytd-rich-grid-renderer"); + const videosContainer = await waitForElement(document, "ytd-rich-grid-renderer > div#contents"); for (const video of videosContainer.querySelectorAll("ytd-rich-item-renderer")) { parseVideo(video); From 4acb89cdb107108f4c57aedc1c08f95bc8651211 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 10:42:17 +0200 Subject: [PATCH 26/35] Rework WS server --- ws-server/connection.go | 98 ++++++++++++++++++++++++++++++++ ws-server/main.go | 121 ++-------------------------------------- ws-server/server.go | 50 +++++++++++++++++ 3 files changed, 152 insertions(+), 117 deletions(-) create mode 100644 ws-server/connection.go create mode 100644 ws-server/server.go diff --git a/ws-server/connection.go b/ws-server/connection.go new file mode 100644 index 0000000..84488dd --- /dev/null +++ b/ws-server/connection.go @@ -0,0 +1,98 @@ +package main + +import ( + "log" + "sync" + "time" + + "github.com/gorilla/websocket" +) + +type WSConnection struct { + conn *websocket.Conn + writeLock sync.Mutex + alive bool + ReadChan chan string + WriteChan chan string + ErrorChan chan error + IdleTimeout time.Duration + PingInterval time.Duration +} + +func NewConn(conn *websocket.Conn, server *WSServer) *WSConnection { + wsconn := &WSConnection{ + conn: conn, + alive: true, + IdleTimeout: server.IdleTimeout, + PingInterval: server.PingInterval, + ReadChan: make(chan string, 1024), + WriteChan: make(chan string, 1024), + ErrorChan: make(chan error, 1), + } + + return wsconn +} + +func (ws *WSConnection) Open() { + go ws.messageReader() + go ws.messageSender() + // go ws.pinger() +} + +func (ws *WSConnection) messageReader() { + log.Printf("Reading messages") + for { + _, message, err := ws.conn.ReadMessage() + if !ws.alive { + return + } + ws.conn.SetReadDeadline(time.Now().Add(ws.IdleTimeout)) + if err != nil { + ws.ErrorChan <- err + return + } + log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) + ws.ReadChan <- string(message) + } +} + +func (ws *WSConnection) messageSender() { + log.Printf("Sending messages") + for { + msg := <-ws.WriteChan + if !ws.alive { + return + } + ws.writeLock.Lock() + + ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) + log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Error during message writing: %v", err) + ws.ErrorChan <- err + return + } + ws.writeLock.Unlock() + } +} + +// func (ws *WSConnection) pinger() { +// log.Printf("Starting pinger, sleeping for %v", ws.PingInterval) +// for { +// time.Sleep(ws.PingInterval) +// if !ws.alive { +// return +// } + +// log.Printf("Ping") +// ws.writeLock.Lock() +// err := ws.conn.WriteMessage(websocket.PingMessage, nil) +// if err != nil { +// log.Println("Error during ping:", err) +// ws.ErrorChan <- err +// return +// } +// ws.writeLock.Unlock() +// } +// } diff --git a/ws-server/main.go b/ws-server/main.go index 06f8be9..0eb84b4 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -6,124 +6,18 @@ import ( "io" "log" "net/http" - "sync" "time" - - "github.com/gorilla/websocket" ) -var upgrader = websocket.Upgrader{} -var wsBroadcast = make(chan string, 128) -var connections = make(map[*WSConnection]bool) - -const TIMEOUT = 6 -const IDLE_TIMEOUT = TIMEOUT * time.Second -const PING_INTERVAL = (TIMEOUT / 2) * time.Second - -type WSConnection struct { - conn *websocket.Conn - writeLock sync.Mutex - ReadChan chan string - WriteChan chan string - ErrorChan chan error -} - -func (ws *WSConnection) messageReader() { - log.Printf("Reading messages") - for { - _, message, err := ws.conn.ReadMessage() - ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) - if err != nil { - ws.ErrorChan <- err - return - } - log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) - ws.ReadChan <- string(message) - } -} - -func (ws *WSConnection) messageSender() { - log.Printf("Sending messages") - for { - msg := <-ws.WriteChan - ws.writeLock.Lock() - - ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) - log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) - if err != nil { - log.Printf("Error during message writing: %v", err) - ws.ErrorChan <- err - return - } - ws.writeLock.Unlock() - } -} - -func (ws *WSConnection) pinger() { - log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL) - for { - time.Sleep(PING_INTERVAL) - - log.Printf("Ping") - ws.writeLock.Lock() - err := ws.conn.WriteMessage(websocket.PingMessage, nil) - if err != nil { - log.Println("Error during ping:", err) - ws.ErrorChan <- err - return - } - ws.writeLock.Unlock() - } -} - -func (ws *WSConnection) Open() { - log.Printf("Client connected") - ws.ReadChan = make(chan string, 1024) - ws.WriteChan = make(chan string, 1024) - ws.ErrorChan = make(chan error, 16) - - ws.conn.SetReadLimit(1024) - ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) - ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) - ws.conn.SetPongHandler(func(string) error { - log.Println("Pong") - ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) - ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) - return nil - }) - - connections[ws] = true - - go ws.messageReader() - go ws.messageSender() - go ws.pinger() - go func() { - for { - select { - case err := <-ws.ErrorChan: - log.Printf("Error: %v", err) - ws.conn.Close() - log.Printf("Client disconnected") - connections[ws] = false - return - // case msg := <-wsBroadcast: - // ws.WriteChan <- msg - } - } - }() -} +var server = New(10 * time.Second) func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { - conn, err := upgrader.Upgrade(responseWriter, request, nil) + conn, err := server.Upgrader.Upgrade(responseWriter, request, nil) if err != nil { fmt.Println("Error during connection upgrade:", err) return } - - ws := new(WSConnection) - ws.conn = conn - ws.Open() + server.HandleNew(conn) } type DownloadReq struct { @@ -146,14 +40,7 @@ func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest) return } - - log.Printf("Received download request: %s, %d in channel", req.Link, len(wsBroadcast)) - go func() { - for ws := range connections { - ws.WriteChan <- req.Link - } - }() - // wsBroadcast <- req.Link + server.Broadcast <- req.Link } func main() { diff --git a/ws-server/server.go b/ws-server/server.go new file mode 100644 index 0000000..b990b3f --- /dev/null +++ b/ws-server/server.go @@ -0,0 +1,50 @@ +package main + +import ( + "log" + "time" + + "github.com/gorilla/websocket" +) + +type WSServer struct { + connections map[*WSConnection]bool + Upgrader websocket.Upgrader + Broadcast chan string + IdleTimeout time.Duration + PingInterval time.Duration +} + +func New(timeout time.Duration) *WSServer { + server := &WSServer{ + connections: make(map[*WSConnection]bool), + Upgrader: websocket.Upgrader{}, + Broadcast: make(chan string, 128), + IdleTimeout: timeout, + PingInterval: timeout / 2, + } + go func() { + for { + msg := <-server.Broadcast + for conn := range server.connections { + conn.WriteChan <- msg + } + } + }() + return server +} + +func (server *WSServer) HandleNew(conn *websocket.Conn) { + log.Printf("Client connected, now %d clients", len(server.connections)+1) + + wsconn := NewConn(conn, server) + go wsconn.Open() + server.connections[wsconn] = true + + go func() { + <-wsconn.ErrorChan + wsconn.alive = false + delete(server.connections, wsconn) + log.Printf("Client disconnected, now %d clients", len(server.connections)) + }() +} From 4ca305324310ff5c9ee5fb2e6575b83fb7aea4bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 11:05:30 +0200 Subject: [PATCH 27/35] Improve client disconnect detection --- ws-server/connection.go | 87 +++++++++++++++++++++++------------------ ws-server/main.go | 6 ++- ws-server/server.go | 8 +++- 3 files changed, 60 insertions(+), 41 deletions(-) diff --git a/ws-server/connection.go b/ws-server/connection.go index 84488dd..b366815 100644 --- a/ws-server/connection.go +++ b/ws-server/connection.go @@ -3,12 +3,14 @@ package main import ( "log" "sync" + "sync/atomic" "time" "github.com/gorilla/websocket" ) type WSConnection struct { + id int32 conn *websocket.Conn writeLock sync.Mutex alive bool @@ -21,6 +23,7 @@ type WSConnection struct { func NewConn(conn *websocket.Conn, server *WSServer) *WSConnection { wsconn := &WSConnection{ + id: server.clientId, conn: conn, alive: true, IdleTimeout: server.IdleTimeout, @@ -29,6 +32,7 @@ func NewConn(conn *websocket.Conn, server *WSServer) *WSConnection { WriteChan: make(chan string, 1024), ErrorChan: make(chan error, 1), } + atomic.AddInt32(&server.clientId, 1) return wsconn } @@ -36,11 +40,16 @@ func NewConn(conn *websocket.Conn, server *WSServer) *WSConnection { func (ws *WSConnection) Open() { go ws.messageReader() go ws.messageSender() - // go ws.pinger() + go ws.pinger() + ws.conn.SetPongHandler(func(string) error { + // log.Printf("Client %d: Pong OK", ws.id) + ws.conn.SetReadDeadline(time.Now().Add(ws.IdleTimeout)) + return nil + }) } func (ws *WSConnection) messageReader() { - log.Printf("Reading messages") + log.Printf("Client %d: Reading messages", ws.id) for { _, message, err := ws.conn.ReadMessage() if !ws.alive { @@ -51,48 +60,52 @@ func (ws *WSConnection) messageReader() { ws.ErrorChan <- err return } - log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) + log.Printf("Client %d: Received: %s, %d in output channel", ws.id, message, len(ws.ReadChan)) ws.ReadChan <- string(message) } } func (ws *WSConnection) messageSender() { - log.Printf("Sending messages") + log.Printf("Client %d: Sending messages", ws.id) for { - msg := <-ws.WriteChan - if !ws.alive { - return - } - ws.writeLock.Lock() - - ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) - log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) - if err != nil { - log.Printf("Error during message writing: %v", err) - ws.ErrorChan <- err - return - } - ws.writeLock.Unlock() + func() { + msg, ok := <-ws.WriteChan + if !ok || !ws.alive { + return + } + ws.writeLock.Lock() + defer ws.writeLock.Unlock() + + ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) + log.Printf("Client %d: Sending: %s, %d in input channel", ws.id, msg, len(ws.WriteChan)) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Client %d: Error during message writing: %v", ws.id, err) + ws.ErrorChan <- err + return + } + }() } } -// func (ws *WSConnection) pinger() { -// log.Printf("Starting pinger, sleeping for %v", ws.PingInterval) -// for { -// time.Sleep(ws.PingInterval) -// if !ws.alive { -// return -// } +func (ws *WSConnection) pinger() { + log.Printf("Client %d: Starting pinger, sleeping for %v", ws.id, ws.PingInterval) + for { + time.Sleep(ws.PingInterval) + if !ws.alive { + return + } -// log.Printf("Ping") -// ws.writeLock.Lock() -// err := ws.conn.WriteMessage(websocket.PingMessage, nil) -// if err != nil { -// log.Println("Error during ping:", err) -// ws.ErrorChan <- err -// return -// } -// ws.writeLock.Unlock() -// } -// } + // log.Printf("Client %d: Ping", ws.id) + ws.writeLock.Lock() + err := ws.conn.WriteMessage(websocket.PingMessage, nil) + if err != nil { + log.Printf("Client %d: Error during ping: %+v", ws.id, err) + ws.ErrorChan <- err + return + } + ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) + ws.writeLock.Unlock() + // log.Printf("Client %d: Ping OK", ws.id) + } +} diff --git a/ws-server/main.go b/ws-server/main.go index 0eb84b4..af888ef 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -43,9 +43,11 @@ func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { server.Broadcast <- req.Link } -func main() { - log.SetFlags(log.Lmicroseconds) +func init() { + log.SetFlags(log.Lmicroseconds | log.Lshortfile) +} +func main() { http.HandleFunc("/ws", wsHandler) http.HandleFunc("/download", handleDownload) log.Println("Server starting on :8080") diff --git a/ws-server/server.go b/ws-server/server.go index b990b3f..f6deaea 100644 --- a/ws-server/server.go +++ b/ws-server/server.go @@ -9,6 +9,7 @@ import ( type WSServer struct { connections map[*WSConnection]bool + clientId int32 Upgrader websocket.Upgrader Broadcast chan string IdleTimeout time.Duration @@ -42,9 +43,12 @@ func (server *WSServer) HandleNew(conn *websocket.Conn) { server.connections[wsconn] = true go func() { - <-wsconn.ErrorChan + err := <-wsconn.ErrorChan wsconn.alive = false + close(wsconn.ReadChan) + close(wsconn.WriteChan) + close(wsconn.ErrorChan) + log.Printf("Client %d: disconnected due to %+v, now %d clients", wsconn.id, err, len(server.connections)) delete(server.connections, wsconn) - log.Printf("Client disconnected, now %d clients", len(server.connections)) }() } From 51a94b6636166d5ee5510bf36822207d11a7be8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 11:16:25 +0200 Subject: [PATCH 28/35] Fix memory leak in server Remove server instrumentation --- ws-server/connection.go | 44 +++++++++++++++++++++-------------------- ws-server/main.go | 24 +++++++++++++++++++++- ws-server/server.go | 1 + 3 files changed, 47 insertions(+), 22 deletions(-) diff --git a/ws-server/connection.go b/ws-server/connection.go index b366815..542cdf7 100644 --- a/ws-server/connection.go +++ b/ws-server/connection.go @@ -53,39 +53,40 @@ func (ws *WSConnection) messageReader() { for { _, message, err := ws.conn.ReadMessage() if !ws.alive { - return + break } ws.conn.SetReadDeadline(time.Now().Add(ws.IdleTimeout)) if err != nil { ws.ErrorChan <- err - return + break } log.Printf("Client %d: Received: %s, %d in output channel", ws.id, message, len(ws.ReadChan)) ws.ReadChan <- string(message) } + log.Printf("Client %d: Stopped reading messages", ws.id) } func (ws *WSConnection) messageSender() { log.Printf("Client %d: Sending messages", ws.id) for { - func() { - msg, ok := <-ws.WriteChan - if !ok || !ws.alive { - return - } - ws.writeLock.Lock() - defer ws.writeLock.Unlock() - - ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) - log.Printf("Client %d: Sending: %s, %d in input channel", ws.id, msg, len(ws.WriteChan)) - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) - if err != nil { - log.Printf("Client %d: Error during message writing: %v", ws.id, err) - ws.ErrorChan <- err - return - } - }() + msg, ok := <-ws.WriteChan + if !ok || !ws.alive { + break + } + ws.writeLock.Lock() + + ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) + log.Printf("Client %d: Sending: %s, %d in input channel", ws.id, msg, len(ws.WriteChan)) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Client %d: Error during message writing: %v", ws.id, err) + ws.ErrorChan <- err + ws.writeLock.Unlock() + break + } + ws.writeLock.Unlock() } + log.Printf("Client %d: Stopped sending messages", ws.id) } func (ws *WSConnection) pinger() { @@ -93,7 +94,7 @@ func (ws *WSConnection) pinger() { for { time.Sleep(ws.PingInterval) if !ws.alive { - return + break } // log.Printf("Client %d: Ping", ws.id) @@ -102,10 +103,11 @@ func (ws *WSConnection) pinger() { if err != nil { log.Printf("Client %d: Error during ping: %+v", ws.id, err) ws.ErrorChan <- err - return + break } ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) ws.writeLock.Unlock() // log.Printf("Client %d: Ping OK", ws.id) } + log.Printf("Client %d: Stopped pinger", ws.id) } diff --git a/ws-server/main.go b/ws-server/main.go index af888ef..2b8992a 100644 --- a/ws-server/main.go +++ b/ws-server/main.go @@ -44,10 +44,32 @@ func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { } func init() { - log.SetFlags(log.Lmicroseconds | log.Lshortfile) + // log.SetFlags(log.Lmicroseconds | log.Lshortfile) + log.SetFlags(log.Lmicroseconds) } +// Mainly used to detect memory leaks +// func instrument() { +// numGoroutines := runtime.NumGoroutine() + +// var m runtime.MemStats +// runtime.ReadMemStats(&m) + +// malloc := float64(m.Alloc) +// ramUsedMB := malloc / 1024 / 1024 +// kbPerGoroutine := malloc / 1024 / float64(numGoroutines) + +// log.Printf("Number of active goroutines: %d; RAM used: %.2f MB; KB per goroutine: %.2f", numGoroutines, ramUsedMB, kbPerGoroutine) +// } + func main() { + // go func() { + // for { + // time.Sleep(1 * time.Second) + // instrument() + // } + // }() + http.HandleFunc("/ws", wsHandler) http.HandleFunc("/download", handleDownload) log.Println("Server starting on :8080") diff --git a/ws-server/server.go b/ws-server/server.go index f6deaea..508e5b5 100644 --- a/ws-server/server.go +++ b/ws-server/server.go @@ -49,6 +49,7 @@ func (server *WSServer) HandleNew(conn *websocket.Conn) { close(wsconn.WriteChan) close(wsconn.ErrorChan) log.Printf("Client %d: disconnected due to %+v, now %d clients", wsconn.id, err, len(server.connections)) + wsconn.conn.Close() delete(server.connections, wsconn) }() } From 7966b06f1b9092cab4ddd2891a17be7a5d6d4721 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 11:54:14 +0200 Subject: [PATCH 29/35] Fix issue with client reconnect and improve client resiliency --- downloader/main.go | 42 ++++++++++++++-- downloader/ws-client.go | 104 ++++++++++++++++++++++++++-------------- 2 files changed, 105 insertions(+), 41 deletions(-) diff --git a/downloader/main.go b/downloader/main.go index d13f204..8f768b8 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -8,8 +8,31 @@ import ( const WEBSOCKET_SERVER = "ws://youtube-download-ws-server.site.quack-lab.dev/ws" const WEBSOCKET_SERVER_ALT = "ws://localhost:8080/ws" -func main() { +func init() { + // log.SetFlags(log.Lmicroseconds | log.Lshortfile) log.SetFlags(log.Lmicroseconds) +} + +// func instrument() { +// numGoroutines := runtime.NumGoroutine() + +// var m runtime.MemStats +// runtime.ReadMemStats(&m) + +// malloc := float64(m.Alloc) +// ramUsedMB := malloc / 1024 / 1024 +// kbPerGoroutine := malloc / 1024 / float64(numGoroutines) + +// log.Printf("Number of active goroutines: %d; RAM used: %.2f MB; KB per goroutine: %.2f", numGoroutines, ramUsedMB, kbPerGoroutine) +// } + +func main() { + // go func() { + // for { + // instrument() + // time.Sleep(1 * time.Second) + // } + // }() // res, err := http.Get(FULL_URL) // if err != nil { @@ -39,9 +62,18 @@ func main() { // listener.Collections = []string{COLLECTION_NAME} // listener.initialize() - ws := new(WSConnection) - ws.url = WEBSOCKET_SERVER - ws.Open() + var ws WSConnection + go func() { + for { + ws := WSConnection{ + url: WEBSOCKET_SERVER_ALT, + } + ws.Open() + <-ws.Dead + log.Printf("Reconnecting in 5 seconds...") + time.Sleep(5 * time.Second) + } + }() sem := make(chan struct{}, 4) for { @@ -50,7 +82,7 @@ func main() { eventCopy := event status := make(chan error) sem <- struct{}{} - + log.Printf("New event: %+v; semaphore at: %d", eventCopy, len(sem)) go func() { defer func() { diff --git a/downloader/ws-client.go b/downloader/ws-client.go index 907839f..54fc790 100644 --- a/downloader/ws-client.go +++ b/downloader/ws-client.go @@ -13,99 +13,131 @@ const IDLE_TIMEOUT = TIMEOUT * time.Second const PING_INTERVAL = (TIMEOUT / 2) * time.Second type WSConnection struct { + alive bool url string conn *websocket.Conn errChan chan error writeLock sync.Mutex ReadChan chan string WriteChan chan string + Dead chan error } func (ws *WSConnection) messageReader() { - log.Printf("Reading messages") + log.Printf("Starting reader") for { + if !ws.alive { + break + } + _, message, err := ws.conn.ReadMessage() ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) if err != nil { ws.errChan <- err - return + break } + log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan)) ws.ReadChan <- string(message) } + log.Printf("Reader done") } func (ws *WSConnection) messageSender() { - log.Printf("Sending messages") + log.Printf("Starting sender") for { - msg := <-ws.WriteChan - ws.writeLock.Lock() - - ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) - log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) - if err != nil { - log.Printf("Error during message writing: %v", err) - ws.errChan <- err - return + msg, ok := <-ws.WriteChan + if !ok { + break } - ws.writeLock.Unlock() + ws.doSend(msg) + } + log.Printf("Sender done") +} + +func (ws *WSConnection) doSend(msg string) { + ws.writeLock.Lock() + defer ws.writeLock.Unlock() + + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan)) + err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) + if err != nil { + log.Printf("Error during message writing: %v", err) + ws.errChan <- err + return } } func (ws *WSConnection) pinger() { log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL) for { - time.Sleep(PING_INTERVAL) - - log.Printf("Ping") - ws.writeLock.Lock() - err := ws.conn.WriteMessage(websocket.PingMessage, nil) - if err != nil { - log.Println("Error during ping:", err) - ws.errChan <- err - return + if !ws.alive { + break } - ws.writeLock.Unlock() + ws.doPing() + time.Sleep(PING_INTERVAL) } + log.Printf("Pinger done") +} + +func (ws *WSConnection) doPing() { + ws.writeLock.Lock() + defer ws.writeLock.Unlock() + + // log.Printf("Ping") + err := ws.conn.WriteMessage(websocket.PingMessage, nil) + if err != nil { + log.Println("Error during ping:", err) + ws.errChan <- err + return + } + ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) + // log.Printf("Ping OK") } func (ws *WSConnection) handleError() { for { err := <-ws.errChan - log.Println("Error during message reading:", err) - - time.Sleep(5 * time.Second) - ws.Open() + log.Printf("Client error: %+v", err) + ws.alive = false + ws.conn.Close() + close(ws.ReadChan) + close(ws.WriteChan) + close(ws.errChan) + ws.Dead <- err + return } } func (ws *WSConnection) Open() { log.Printf("Connecting to %s", ws.url) + ws.Dead = make(chan error, 1) + conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil) if err != nil { log.Println("Error during connection:", err) - ws.errChan <- err + ws.Dead <- err return } log.Printf("Connected") ws.conn = conn - ws.errChan = make(chan error) - ws.ReadChan = make(chan string, 1024) - ws.WriteChan = make(chan string, 1024) + ws.alive = true + + ws.errChan = make(chan error, 1) + ws.ReadChan = make(chan string, 128) + ws.WriteChan = make(chan string, 128) - ws.conn.SetReadLimit(1024) ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) ws.conn.SetPongHandler(func(string) error { - log.Println("Pong") + // log.Println("Pong") ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT)) - ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT)) return nil }) + go ws.handleError() go ws.messageReader() go ws.messageSender() - go ws.handleError() go ws.pinger() } From 2cb95c397dbad6aba04bfc33fed14aeeebc3aadf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 11:56:59 +0200 Subject: [PATCH 30/35] Mod tidy --- downloader/go.mod | 11 ----------- downloader/go.sum | 47 ---------------------------------------------- downloader/main.go | 2 +- 3 files changed, 1 insertion(+), 59 deletions(-) diff --git a/downloader/go.mod b/downloader/go.mod index bd1e950..143c4c1 100644 --- a/downloader/go.mod +++ b/downloader/go.mod @@ -2,26 +2,15 @@ module main go 1.22.4 -require github.com/r3labs/sse/v2 v2.10.0 - require ( github.com/ProtonMail/go-crypto v1.0.0 // indirect - github.com/bitly/go-simplejson v0.5.1 // indirect github.com/cloudflare/circl v1.3.7 // indirect - github.com/dlclark/regexp2 v1.11.0 // indirect - github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 // indirect - github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect - github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect golang.org/x/crypto v0.21.0 // indirect - golang.org/x/net v0.21.0 // indirect golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect - gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect ) require ( diff --git a/downloader/go.sum b/downloader/go.sum index baf12a8..76dc1dc 100644 --- a/downloader/go.sum +++ b/downloader/go.sum @@ -1,52 +1,21 @@ github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78= github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= -github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= -github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= -github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU= github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= -github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8= -github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk= -github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4= -github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= -github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM= github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI= github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4= -github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 h1:qZNfIGkIANxGv/OqtnntR4DfOY2+BgwR60cAcu/i3SE= github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4/go.mod h1:kW3HQ4UdaAyrUCSSDR4xUzBKW6O2iA4uHhk7AtyYp10= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg= -github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 h1:ZAo7qJht9PqefOD7C0ZKQ8dEkpJeM955sYw0FtQnzvo= github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ= github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o= -github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8= -github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk= github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= @@ -59,21 +28,17 @@ golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOM golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -89,24 +54,12 @@ golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y= -gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/downloader/main.go b/downloader/main.go index 8f768b8..f9211b2 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -66,7 +66,7 @@ func main() { go func() { for { ws := WSConnection{ - url: WEBSOCKET_SERVER_ALT, + url: WEBSOCKET_SERVER, } ws.Open() <-ws.Dead From f36843632592f934a27bfcf9d5b7ccbcbdf39c30 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 12:07:29 +0200 Subject: [PATCH 31/35] Fix build script --- ws-server/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ws-server/build.sh b/ws-server/build.sh index 5e455f9..36de773 100644 --- a/ws-server/build.sh +++ b/ws-server/build.sh @@ -1,3 +1,3 @@ # docker build -t youtube-download-ws-server . -tar -cf deploy.tar captain-definition dockerfile main.go go.mod go.sum \ No newline at end of file +tar -cf deploy.tar captain-definition dockerfile *.go go.mod go.sum \ No newline at end of file From cdcdb18c5771c4eec9b8039cb486c1fc15c7ca48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Thu, 27 Jun 2024 12:21:55 +0200 Subject: [PATCH 32/35] Fix issue with reader disconnecting due to closd channel --- downloader/main.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/downloader/main.go b/downloader/main.go index f9211b2..3e86bd0 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -63,12 +63,20 @@ func main() { // listener.initialize() var ws WSConnection + read := make(chan string) go func() { for { - ws := WSConnection{ + ws = WSConnection{ url: WEBSOCKET_SERVER, } ws.Open() + for { + msg, ok := <-ws.ReadChan + if !ok { + break + } + read <- msg + } <-ws.Dead log.Printf("Reconnecting in 5 seconds...") time.Sleep(5 * time.Second) @@ -78,7 +86,7 @@ func main() { sem := make(chan struct{}, 4) for { select { - case event := <-ws.ReadChan: + case event := <-read: eventCopy := event status := make(chan error) sem <- struct{}{} From 4fd836c0b656b847b8cf4a1cb08748a71a9e3d96 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Fri, 28 Jun 2024 17:23:13 +0200 Subject: [PATCH 33/35] Update instrumentation --- .gitignore | 1 + downloader/main.go | 21 ++++++++++++++++----- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index faf7207..0c31076 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ main.exe logs.log ws-server/deploy.tar +downloader/main.log diff --git a/downloader/main.go b/downloader/main.go index 3e86bd0..a971674 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -18,12 +18,23 @@ func init() { // var m runtime.MemStats // runtime.ReadMemStats(&m) +// // log.Printf("%+v", m) -// malloc := float64(m.Alloc) -// ramUsedMB := malloc / 1024 / 1024 -// kbPerGoroutine := malloc / 1024 / float64(numGoroutines) +// sys := float64(m.Sys) +// ramUsedMB := sys / 1024 / 1024 +// kbPerGoroutine := sys / 1024 / float64(numGoroutines) -// log.Printf("Number of active goroutines: %d; RAM used: %.2f MB; KB per goroutine: %.2f", numGoroutines, ramUsedMB, kbPerGoroutine) +// var numGoroutinesPretty string +// switch { +// case numGoroutines >= 1_000_000: +// numGoroutinesPretty = fmt.Sprintf("%.2fM", float64(numGoroutines)/1_000_000) +// case numGoroutines >= 1_000: +// numGoroutinesPretty = fmt.Sprintf("%.2fk", float64(numGoroutines)/1_000) +// default: +// numGoroutinesPretty = fmt.Sprintf("%d", numGoroutines) +// } + +// log.Printf("Number of active goroutines: %d (%s); RAM used: %.2f MB; KB per goroutine: %.2f", numGoroutines, numGoroutinesPretty, ramUsedMB, kbPerGoroutine) // } func main() { @@ -67,7 +78,7 @@ func main() { go func() { for { ws = WSConnection{ - url: WEBSOCKET_SERVER, + url: WEBSOCKET_SERVER_ALT, } ws.Open() for { From 468de9e40179fbbf3bf3cc0246f77084e5973b08 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Sun, 14 Jul 2024 14:59:12 +0200 Subject: [PATCH 34/35] Simplify everything --- dl/dl.go | 2 +- downloader/download.go | 76 ++++++++++++++--------- downloader/main.go | 138 ++++++++++------------------------------- downloader/types.go | 27 ++------ 4 files changed, 84 insertions(+), 159 deletions(-) diff --git a/dl/dl.go b/dl/dl.go index 14e61eb..57ab548 100644 --- a/dl/dl.go +++ b/dl/dl.go @@ -11,7 +11,7 @@ import ( "sync" ) -const URL = `https://youtube-download-ws-server.site.quack-lab.dev/download` +const URL = `http://localhost:5000/download` type Item struct { Link string `json:"link"` diff --git a/downloader/download.go b/downloader/download.go index 0825132..105f37d 100644 --- a/downloader/download.go +++ b/downloader/download.go @@ -3,53 +3,69 @@ package main import ( "context" "log" + "sync" - "github.com/lrstanley/go-ytdlp" "github.com/gen2brain/beeep" + "github.com/lrstanley/go-ytdlp" ) const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos" +type DownloadWorker struct { + id int + input chan *DownloadTask +} + +var ongoingDownloads = make(map[string]struct{}) +var ongoingDownloadsMutex = &sync.Mutex{} + var dl = ytdlp.New(). // FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]"). FormatSort("res,ext:mp4:m4a"). Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s"). - LimitRate("5M"). + LimitRate("10M"). // HTTPChunkSize("20M"). MarkWatched(). SponsorblockMark("all"). RecodeVideo("mp4"). ConcurrentFragments(6) -// func Download(event PBEvent, status chan error) { -// _, err := dl.Run(context.TODO(), event.Record.Link) -// if err != nil { -// status <- err -// return -// } +func (w *DownloadWorker) Run() { + for { + task, ok := <-w.input + if !ok { + log.Printf("DownloadWorker %d: input channel closed, exiting", w.id) + return + } + _, ongoing := ongoingDownloads[task.Url] + if ongoing { + log.Printf("DownloadWorker %d: Download %s is already ongoing", w.id, task.Url) + continue + } + ongoingDownloadsMutex.Lock() + ongoingDownloads[task.Url] = struct{}{} + ongoingDownloadsMutex.Unlock() -// log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link) -// SetDownloaded(event) -// } + log.Printf("DownloadWorker %d: Downloading %s", w.id, task.Url) -func DownloadURL(url string, status chan error) { - log.Printf("Downloading %s", url) - - err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration) - if err != nil { - log.Printf("Failed beeping with %+v", err) + err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration) + if err != nil { + log.Printf("Failed beeping with %+v", err) + } + err = beeep.Alert("Download Started", task.Url, "assets/information.png") + if err != nil { + log.Printf("Failed alerting with %+v", err) + } + + _, err = dl.Run(context.TODO(), task.Url) + if err != nil { + log.Printf("DownloadWorker %d: Failed downloading %s with %+v", w.id, task.Url, err) + continue + } + + log.Printf("DownloadWorker %d: Downloaded %s", w.id, task.Url) + ongoingDownloadsMutex.Lock() + delete(ongoingDownloads, task.Url) + ongoingDownloadsMutex.Unlock() } - err = beeep.Alert("Download Started", url, "assets/information.png") - if err != nil { - log.Printf("Failed alerting with %+v", err) - } - - _, err = dl.Run(context.TODO(), url) - if err != nil { - status <- err - return - } - - log.Printf("Downloaded %s", url) - close(status) } diff --git a/downloader/main.go b/downloader/main.go index a971674..cd08f73 100644 --- a/downloader/main.go +++ b/downloader/main.go @@ -1,123 +1,49 @@ package main import ( + "encoding/json" + "io" "log" - "time" + "net/http" + "os" ) -const WEBSOCKET_SERVER = "ws://youtube-download-ws-server.site.quack-lab.dev/ws" -const WEBSOCKET_SERVER_ALT = "ws://localhost:8080/ws" - func init() { - // log.SetFlags(log.Lmicroseconds | log.Lshortfile) - log.SetFlags(log.Lmicroseconds) + log.SetFlags(log.Lmicroseconds | log.Lshortfile) + logFile, err := os.Create("main.log") + if err != nil { + log.Printf("Error creating log file: %v", err) + os.Exit(1) + } + logger := io.MultiWriter(os.Stdout, logFile) + log.SetOutput(logger) } -// func instrument() { -// numGoroutines := runtime.NumGoroutine() - -// var m runtime.MemStats -// runtime.ReadMemStats(&m) -// // log.Printf("%+v", m) - -// sys := float64(m.Sys) -// ramUsedMB := sys / 1024 / 1024 -// kbPerGoroutine := sys / 1024 / float64(numGoroutines) - -// var numGoroutinesPretty string -// switch { -// case numGoroutines >= 1_000_000: -// numGoroutinesPretty = fmt.Sprintf("%.2fM", float64(numGoroutines)/1_000_000) -// case numGoroutines >= 1_000: -// numGoroutinesPretty = fmt.Sprintf("%.2fk", float64(numGoroutines)/1_000) -// default: -// numGoroutinesPretty = fmt.Sprintf("%d", numGoroutines) -// } - -// log.Printf("Number of active goroutines: %d (%s); RAM used: %.2f MB; KB per goroutine: %.2f", numGoroutines, numGoroutinesPretty, ramUsedMB, kbPerGoroutine) -// } +const DOWNLOAD_WORKERS = 10 func main() { - // go func() { - // for { - // instrument() - // time.Sleep(1 * time.Second) - // } - // }() + downloadQueue := make(chan *DownloadTask, 100) + for i := 0; i < DOWNLOAD_WORKERS; i++ { + worker := &DownloadWorker{id: i, input: downloadQueue} + go worker.Run() + } - // res, err := http.Get(FULL_URL) - // if err != nil { - // log.Fatal(err) - // } - // defer res.Body.Close() - // body, err := io.ReadAll(res.Body) - // if err != nil { - // log.Printf("Error reading response body: %+v\n", err) - // return - // } - // if res.StatusCode != http.StatusOK { - // log.Printf("Non-OK HTTP status: %d\nResponse body: %s\n", res.StatusCode, body) - // return - // } + http.HandleFunc("/download", func(responseWriter http.ResponseWriter, request *http.Request) { + defer request.Body.Close() - // var data APIResponse - // err = json.Unmarshal(body, &data) - // if err != nil { - // log.Printf("Error unmarshaling JSON: %+v\n", err) - // return - // } - // log.Printf("Data: %+v\n", data) - - // listener := new(RealtimeListener) - // listener.Url = POCKETBASE_REALTIME - // listener.Collections = []string{COLLECTION_NAME} - // listener.initialize() - - var ws WSConnection - read := make(chan string) - go func() { - for { - ws = WSConnection{ - url: WEBSOCKET_SERVER_ALT, - } - ws.Open() - for { - msg, ok := <-ws.ReadChan - if !ok { - break - } - read <- msg - } - <-ws.Dead - log.Printf("Reconnecting in 5 seconds...") - time.Sleep(5 * time.Second) + req := DownloadRequest{} + err := json.NewDecoder(request.Body).Decode(&req) + if err != nil { + log.Printf("Error parsing JSON: %v", err) + http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest) + return } - }() - sem := make(chan struct{}, 4) - for { - select { - case event := <-read: - eventCopy := event - status := make(chan error) - sem <- struct{}{} - - log.Printf("New event: %+v; semaphore at: %d", eventCopy, len(sem)) - go func() { - defer func() { - <-sem - log.Printf("Semaphore at: %d", len(sem)) - }() - // Download(eventCopy, status) - DownloadURL(eventCopy, status) - // go DownloadNative(event, status) - for status := range status { - log.Printf("Status: %s\n", status) - } - }() - case <-time.After(1 * time.Minute): - // Perform some action or simply continue to avoid deadlock - log.Println("Consumer is alive, but has no new events.") - } + downloadQueue <- &DownloadTask{Url: req.Link} + }) + log.Println("Server starting on :5000") + err := http.ListenAndServe(":5000", nil) + if err != nil { + log.Println("Error starting server:", err) } } diff --git a/downloader/types.go b/downloader/types.go index 884eab2..a51b886 100644 --- a/downloader/types.go +++ b/downloader/types.go @@ -1,25 +1,8 @@ package main -type APIResponse struct { - Page int `json:"page"` - PerPage int `json:"perPage"` - TotalItems int `json:"totalItems"` - TotalPages int `json:"totalPages"` - Items []APIItem `json:"items"` -} - -type APIItem struct { - CollectionId string `json:"collectionId"` - CollectionName string `json:"collectionName"` - Created string `json:"created"` - Downloaded bool `json:"downloaded"` - Id string `json:"id"` - Link string `json:"link"` - Updated string `json:"updated"` -} - -type PBEvent struct { - ClientId string `json:"clientId"` - Action string `json:"action"` - Record APIItem `json:"record"` +type DownloadTask struct { + Url string } +type DownloadRequest struct { + Link string `json:"link"` +} \ No newline at end of file From 8a51da97c2c000829caed8a0b58f9770c2ecce91 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?David=20Majdand=C5=BEi=C4=87?= Date: Sun, 14 Jul 2024 14:59:34 +0200 Subject: [PATCH 35/35] Remove websockets --- ws-client/go.mod | 5 -- ws-client/go.sum | 2 - ws-client/main.go | 73 ---------------------- ws-server/build.sh | 3 - ws-server/captain-definition | 4 -- ws-server/connection.go | 113 ----------------------------------- ws-server/dockerfile | 21 ------- ws-server/go.mod | 5 -- ws-server/go.sum | 2 - ws-server/main.go | 80 ------------------------- ws-server/server.go | 55 ----------------- 11 files changed, 363 deletions(-) delete mode 100644 ws-client/go.mod delete mode 100644 ws-client/go.sum delete mode 100644 ws-client/main.go delete mode 100644 ws-server/build.sh delete mode 100644 ws-server/captain-definition delete mode 100644 ws-server/connection.go delete mode 100644 ws-server/dockerfile delete mode 100644 ws-server/go.mod delete mode 100644 ws-server/go.sum delete mode 100644 ws-server/main.go delete mode 100644 ws-server/server.go diff --git a/ws-client/go.mod b/ws-client/go.mod deleted file mode 100644 index 3de883c..0000000 --- a/ws-client/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module main - -go 1.22.4 - -require github.com/gorilla/websocket v1.5.3 diff --git a/ws-client/go.sum b/ws-client/go.sum deleted file mode 100644 index 25a9fc4..0000000 --- a/ws-client/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/ws-client/main.go b/ws-client/main.go deleted file mode 100644 index 50ab8d8..0000000 --- a/ws-client/main.go +++ /dev/null @@ -1,73 +0,0 @@ -package main - -import ( - "log" - "time" - - "github.com/gorilla/websocket" -) - -type WSConnection struct { - url string - conn *websocket.Conn - errChan chan error -} - -func (ws *WSConnection) readMessage() { - log.Printf("Reading messages") - for { - _, message, err := ws.conn.ReadMessage() - if err != nil { - ws.errChan <- err - return - } - log.Printf("Received: %s", message) - } -} - -func (ws *WSConnection) writeMessage(message string) { - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(message)) - if err != nil { - log.Printf("Error during message writing: %v", err) - ws.errChan <- err - return - } -} - -func (ws *WSConnection) handleError() { - for { - err := <-ws.errChan - log.Println("Error during message reading:", err) - - time.Sleep(5 * time.Second) - ws.open() - } -} - -func (ws *WSConnection) open() { - log.Printf("Connecting to %s", ws.url) - conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil) - if err != nil { - log.Println("Error during connection:", err) - ws.errChan <- err - return - } - log.Printf("Connected") - ws.conn = conn - ws.errChan = make(chan error) - go ws.readMessage() - go ws.handleError() -} - -func main() { - log.SetFlags(log.Lmicroseconds) - - wsConn := WSConnection{ - url: "ws://localhost:8080/ws", - } - wsConn.open() - for { - log.Printf("zzz...") - time.Sleep(30 * time.Second) - } -} diff --git a/ws-server/build.sh b/ws-server/build.sh deleted file mode 100644 index 36de773..0000000 --- a/ws-server/build.sh +++ /dev/null @@ -1,3 +0,0 @@ -# docker build -t youtube-download-ws-server . - -tar -cf deploy.tar captain-definition dockerfile *.go go.mod go.sum \ No newline at end of file diff --git a/ws-server/captain-definition b/ws-server/captain-definition deleted file mode 100644 index 774ff1a..0000000 --- a/ws-server/captain-definition +++ /dev/null @@ -1,4 +0,0 @@ -{ - "schemaVersion": 2, - "dockerfilePath": "./dockerfile" -} diff --git a/ws-server/connection.go b/ws-server/connection.go deleted file mode 100644 index 542cdf7..0000000 --- a/ws-server/connection.go +++ /dev/null @@ -1,113 +0,0 @@ -package main - -import ( - "log" - "sync" - "sync/atomic" - "time" - - "github.com/gorilla/websocket" -) - -type WSConnection struct { - id int32 - conn *websocket.Conn - writeLock sync.Mutex - alive bool - ReadChan chan string - WriteChan chan string - ErrorChan chan error - IdleTimeout time.Duration - PingInterval time.Duration -} - -func NewConn(conn *websocket.Conn, server *WSServer) *WSConnection { - wsconn := &WSConnection{ - id: server.clientId, - conn: conn, - alive: true, - IdleTimeout: server.IdleTimeout, - PingInterval: server.PingInterval, - ReadChan: make(chan string, 1024), - WriteChan: make(chan string, 1024), - ErrorChan: make(chan error, 1), - } - atomic.AddInt32(&server.clientId, 1) - - return wsconn -} - -func (ws *WSConnection) Open() { - go ws.messageReader() - go ws.messageSender() - go ws.pinger() - ws.conn.SetPongHandler(func(string) error { - // log.Printf("Client %d: Pong OK", ws.id) - ws.conn.SetReadDeadline(time.Now().Add(ws.IdleTimeout)) - return nil - }) -} - -func (ws *WSConnection) messageReader() { - log.Printf("Client %d: Reading messages", ws.id) - for { - _, message, err := ws.conn.ReadMessage() - if !ws.alive { - break - } - ws.conn.SetReadDeadline(time.Now().Add(ws.IdleTimeout)) - if err != nil { - ws.ErrorChan <- err - break - } - log.Printf("Client %d: Received: %s, %d in output channel", ws.id, message, len(ws.ReadChan)) - ws.ReadChan <- string(message) - } - log.Printf("Client %d: Stopped reading messages", ws.id) -} - -func (ws *WSConnection) messageSender() { - log.Printf("Client %d: Sending messages", ws.id) - for { - msg, ok := <-ws.WriteChan - if !ok || !ws.alive { - break - } - ws.writeLock.Lock() - - ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) - log.Printf("Client %d: Sending: %s, %d in input channel", ws.id, msg, len(ws.WriteChan)) - err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg)) - if err != nil { - log.Printf("Client %d: Error during message writing: %v", ws.id, err) - ws.ErrorChan <- err - ws.writeLock.Unlock() - break - } - ws.writeLock.Unlock() - } - log.Printf("Client %d: Stopped sending messages", ws.id) -} - -func (ws *WSConnection) pinger() { - log.Printf("Client %d: Starting pinger, sleeping for %v", ws.id, ws.PingInterval) - for { - time.Sleep(ws.PingInterval) - if !ws.alive { - break - } - - // log.Printf("Client %d: Ping", ws.id) - ws.writeLock.Lock() - err := ws.conn.WriteMessage(websocket.PingMessage, nil) - if err != nil { - log.Printf("Client %d: Error during ping: %+v", ws.id, err) - ws.ErrorChan <- err - break - } - ws.conn.SetWriteDeadline(time.Now().Add(ws.IdleTimeout)) - ws.writeLock.Unlock() - // log.Printf("Client %d: Ping OK", ws.id) - } - log.Printf("Client %d: Stopped pinger", ws.id) -} diff --git a/ws-server/dockerfile b/ws-server/dockerfile deleted file mode 100644 index 5c514db..0000000 --- a/ws-server/dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM golang:1.22.4 as base - -WORKDIR $GOPATH/src/app/ - -COPY . . - -RUN go mod download -RUN go mod verify - -RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /main . - -FROM scratch - -COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo -COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=base /etc/passwd /etc/passwd -COPY --from=base /etc/group /etc/group - -COPY --from=base /main . - -CMD ["/main"] \ No newline at end of file diff --git a/ws-server/go.mod b/ws-server/go.mod deleted file mode 100644 index 3de883c..0000000 --- a/ws-server/go.mod +++ /dev/null @@ -1,5 +0,0 @@ -module main - -go 1.22.4 - -require github.com/gorilla/websocket v1.5.3 diff --git a/ws-server/go.sum b/ws-server/go.sum deleted file mode 100644 index 25a9fc4..0000000 --- a/ws-server/go.sum +++ /dev/null @@ -1,2 +0,0 @@ -github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= -github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= diff --git a/ws-server/main.go b/ws-server/main.go deleted file mode 100644 index 2b8992a..0000000 --- a/ws-server/main.go +++ /dev/null @@ -1,80 +0,0 @@ -package main - -import ( - "encoding/json" - "fmt" - "io" - "log" - "net/http" - "time" -) - -var server = New(10 * time.Second) - -func wsHandler(responseWriter http.ResponseWriter, request *http.Request) { - conn, err := server.Upgrader.Upgrade(responseWriter, request, nil) - if err != nil { - fmt.Println("Error during connection upgrade:", err) - return - } - server.HandleNew(conn) -} - -type DownloadReq struct { - Link string `json:"link"` -} - -func handleDownload(responseWriter http.ResponseWriter, request *http.Request) { - body, err := io.ReadAll(request.Body) - if err != nil { - log.Printf("Error reading request body: %v", err) - http.Error(responseWriter, "Error reading request body", http.StatusBadRequest) - return - } - defer request.Body.Close() - - req := DownloadReq{} - err = json.Unmarshal(body, &req) - if err != nil { - log.Printf("Error parsing JSON: %v", err) - http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest) - return - } - server.Broadcast <- req.Link -} - -func init() { - // log.SetFlags(log.Lmicroseconds | log.Lshortfile) - log.SetFlags(log.Lmicroseconds) -} - -// Mainly used to detect memory leaks -// func instrument() { -// numGoroutines := runtime.NumGoroutine() - -// var m runtime.MemStats -// runtime.ReadMemStats(&m) - -// malloc := float64(m.Alloc) -// ramUsedMB := malloc / 1024 / 1024 -// kbPerGoroutine := malloc / 1024 / float64(numGoroutines) - -// log.Printf("Number of active goroutines: %d; RAM used: %.2f MB; KB per goroutine: %.2f", numGoroutines, ramUsedMB, kbPerGoroutine) -// } - -func main() { - // go func() { - // for { - // time.Sleep(1 * time.Second) - // instrument() - // } - // }() - - http.HandleFunc("/ws", wsHandler) - http.HandleFunc("/download", handleDownload) - log.Println("Server starting on :8080") - err := http.ListenAndServe(":8080", nil) - if err != nil { - log.Println("Error starting server:", err) - } -} diff --git a/ws-server/server.go b/ws-server/server.go deleted file mode 100644 index 508e5b5..0000000 --- a/ws-server/server.go +++ /dev/null @@ -1,55 +0,0 @@ -package main - -import ( - "log" - "time" - - "github.com/gorilla/websocket" -) - -type WSServer struct { - connections map[*WSConnection]bool - clientId int32 - Upgrader websocket.Upgrader - Broadcast chan string - IdleTimeout time.Duration - PingInterval time.Duration -} - -func New(timeout time.Duration) *WSServer { - server := &WSServer{ - connections: make(map[*WSConnection]bool), - Upgrader: websocket.Upgrader{}, - Broadcast: make(chan string, 128), - IdleTimeout: timeout, - PingInterval: timeout / 2, - } - go func() { - for { - msg := <-server.Broadcast - for conn := range server.connections { - conn.WriteChan <- msg - } - } - }() - return server -} - -func (server *WSServer) HandleNew(conn *websocket.Conn) { - log.Printf("Client connected, now %d clients", len(server.connections)+1) - - wsconn := NewConn(conn, server) - go wsconn.Open() - server.connections[wsconn] = true - - go func() { - err := <-wsconn.ErrorChan - wsconn.alive = false - close(wsconn.ReadChan) - close(wsconn.WriteChan) - close(wsconn.ErrorChan) - log.Printf("Client %d: disconnected due to %+v, now %d clients", wsconn.id, err, len(server.connections)) - wsconn.conn.Close() - delete(server.connections, wsconn) - }() -}