Add deploy script

Because, of course, videos are drawn differently on home page, search
results, video recommendations and subscriptions
THANKS GOOGL

.gitignore

@@ -2,3 +2,13 @@
 main.exe
 logs.log
 ws-server/deploy.tar
+downloader/main.log
+downloader/vendor/golang.org
+downloader/vendor/github.com/*
+!downloader/vendor/github.com/kkdai
+downloader/vendor/modules.txt
+downloader/ytdl.exe
+downloader/ytdl.log
+*.log
+*.xml
+*.exe

dl/dl.go

@@ -11,7 +11,7 @@ import (
 	"sync"
 )
 
-const URL = `https://youtube-download-ws-server.site.quack-lab.dev/download`
+const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
 
 type Item struct {
 	Link string `json:"link"`

dl/go.mod

@@ -1,3 +1,3 @@
-module main
+module dl
 
 go 1.22.4

dl/sync

@@ -1 +0,0 @@
-main.exe,"C:\Program Files\Git\usr\bin\dl.exe",t

download.go

@@ -0,0 +1,33 @@
+package main
+
+import (
+	"context"
+	"log"
+
+	"github.com/lrstanley/go-ytdlp"
+)
+
+const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
+
+var dl = ytdlp.New().
+	// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
+	FormatSort("res,ext:mp4:m4a").
+	Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
+	LimitRate("50M").
+	// HTTPChunkSize("20M").
+	MarkWatched().
+	SponsorblockMark("all").
+	PrintJSON().
+	RecodeVideo("mp4").
+	ConcurrentFragments(4)
+
+func Download(event PBEvent, status chan error) {
+	_, err := dl.Run(context.TODO(), event.Record.Link)
+	if err != nil {
+		status <- err
+		return
+	}
+
+	log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link)
+	SetDownloaded(event)
+}

downloader/deploy.sh

@@ -1,3 +1,3 @@
 nssm stop YoutubeDownloader && \
-go build main && \
+go build . && \
 nssm start YoutubeDownloader

downloader/discordNotifier.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+)
+
+func NotifyDiscordErrorless(message string) {
+	err := NotifyDiscord(message)
+	if err != nil {
+		log.Printf("Error notifying discord: %v", err)
+	}
+}
+
+func NotifyDiscord(message string) error {
+	webhookURL := os.Getenv("YTDL_DISCORD_WEBHOOK_URL")
+	if webhookURL == "" {
+		return fmt.Errorf("error notifying discord: webhook URL is not set in environment variables")
+	}
+
+	jsonData := map[string]string{"content": message}
+	jsonBytes, err := json.Marshal(jsonData)
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error marshalling JSON: %v", err)
+	}
+	req, err := http.NewRequest("POST", webhookURL, bytes.NewBuffer(jsonBytes))
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error creating request: %v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error sending request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("error notifying discord: error reading response body: %v", err)
+	}
+
+	log.Printf("Response from Discord: %s", string(body))
+	return nil
+}

downloader/download.go

@@ -1,44 +0,0 @@
-package main
-
-import (
-	"context"
-	"log"
-
-	"github.com/lrstanley/go-ytdlp"
-)
-
-const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
-
-var dl = ytdlp.New().
-	// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
-	FormatSort("res,ext:mp4:m4a").
-	Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
-	LimitRate("5M").
-	// HTTPChunkSize("20M").
-	MarkWatched().
-	SponsorblockMark("all").
-	RecodeVideo("mp4").
-	ConcurrentFragments(6)
-
-// func Download(event PBEvent, status chan error) {
-// 	_, err := dl.Run(context.TODO(), event.Record.Link)
-// 	if err != nil {
-// 		status <- err
-// 		return
-// 	}
-
-// 	log.Printf("Downloaded %s (%s)", event.Record.Id, event.Record.Link)
-// 	SetDownloaded(event)
-// }
-
-func DownloadURL(url string, status chan error) {
-	log.Printf("Downloading %s", url)
-	_, err := dl.Run(context.TODO(), url)
-	if err != nil {
-		status <- err
-		return
-	}
-
-	log.Printf("Downloaded %s", url)	
-	close(status)
-}

downloader/downloaders/downloader.go

@@ -0,0 +1,5 @@
+package downloaders
+
+type Downloader interface {
+	Download(url string) error
+}

downloader/downloaders/kidai.go

@@ -0,0 +1,93 @@
+package downloaders
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/gen2brain/beeep"
+	"github.com/kkdai/youtube/v2"
+	ytd "github.com/kkdai/youtube/v2/downloader"
+)
+
+type KidaiDownloader struct{}
+
+func (d *KidaiDownloader) Download(url string) error {
+	_, ongoing := ongoingDownloads[url]
+	if ongoing {
+		// return fmt.Errorf("Download %s is already ongoing", url)
+		log.Printf("Download %s is already ongoing", url)
+		return nil
+	}
+	ongoingDownloadsMutex.Lock()
+	ongoingDownloads[url] = struct{}{}
+	ongoingDownloadsMutex.Unlock()
+	defer func() {
+		ongoingDownloadsMutex.Lock()
+		delete(ongoingDownloads, url)
+		ongoingDownloadsMutex.Unlock()
+	}()
+
+	log.Printf("Kidai downloading %s", url)
+
+	go func() {
+		err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
+		if err != nil {
+			log.Printf("Failed beeping with %+v", err)
+		}
+		err = beeep.Alert("Download Started", url, "assets/information.png")
+		if err != nil {
+			log.Printf("Failed alerting with %+v", err)
+		}
+	}()
+
+	client := youtube.Client{}
+	video, err := client.GetVideo(url)
+	if err != nil {
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
+	}
+	// $ go run . download -m mp4 -q hd https://www.youtube.com/watch?v=D9trBXaXCgA
+	// This works fine, now I have to figure out how to plug -m and -q into this shit below
+	downloader := ytd.Downloader{
+		OutputDir: OUTPUT_DIR,
+		Client:    client,
+	}
+
+	videoTitle := Sanitize(video.Title)
+	videoAuthor := Sanitize(video.Author)
+
+	videoFileRoot := filepath.Join(OUTPUT_DIR, videoAuthor)
+	err = os.MkdirAll(videoFileRoot, 0755)
+	if err != nil {
+		return fmt.Errorf("failed creating directory %s with %+v", videoFileRoot, err)
+	}
+
+	fullVideoPath := filepath.Join(OUTPUT_DIR, videoAuthor, videoTitle+".mp4")
+	videoFileHandle, err := os.Open(fullVideoPath)
+	if err == nil {
+		videoFileHandle.Close()
+		log.Printf("File %s already exists, skipping download", fullVideoPath)
+		return nil
+	}
+
+	videoFile := filepath.Join(videoAuthor, videoTitle+".mp4")
+	err = downloader.DownloadComposite(context.Background(), videoFile, video, "hd", "mp4", "")
+	if err != nil {
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
+	}
+
+	log.Printf("Downloaded %s", url)
+	return nil
+}
+
+var re = regexp.MustCompile(`[#!$%&/()";:?;,|*]`)
+
+func Sanitize(s string) string {
+	s = strings.ReplaceAll(s, "&", "and")
+	s = re.ReplaceAllString(s, "")
+	return s
+}

downloader/downloaders/utils.go

@@ -0,0 +1,11 @@
+package downloaders
+
+import "sync"
+
+const OUTPUT_DIR = "C:/Users/Administrator/ytdlpVideos"
+//const OUTPUT_DIR = "."
+
+var ongoingDownloads = make(map[string]struct{})
+var ongoingDownloadsMutex = &sync.Mutex{}
+
+const DOWNLOAD_WORKERS = 4

downloader/downloaders/ytdlp-lib.go

@@ -0,0 +1,59 @@
+package downloaders
+
+import (
+	"context"
+	"fmt"
+	"log"
+
+	"github.com/gen2brain/beeep"
+	"github.com/lrstanley/go-ytdlp"
+)
+
+var dl = ytdlp.New().
+	// FormatSort("bestvideo[ext=mp4]+bestaudio[ext=m4a]").
+	FormatSort("res,ext:mp4:m4a").
+	Output("C:/Users/Administrator/ytdlpVideos/%(uploader)s/%(title)s.%(ext)s").
+	LimitRate(fmt.Sprintf("%dM", 150/DOWNLOAD_WORKERS)).
+	// HTTPChunkSize("20M").
+	MarkWatched().
+	SponsorblockMark("all").
+	RecodeVideo("mp4").
+	ConcurrentFragments(6)
+
+type YTDLPLibDownloader struct{}
+
+func (d *YTDLPLibDownloader) Download(url string) error {
+	_, ongoing := ongoingDownloads[url]
+	if ongoing {
+		// return fmt.Errorf("Download %s is already ongoing", url)
+		log.Printf("Download %s is already ongoing", url)
+		return nil
+	}
+	ongoingDownloadsMutex.Lock()
+	ongoingDownloads[url] = struct{}{}
+	ongoingDownloadsMutex.Unlock()
+
+	log.Printf("YTDLPLib downloading %s", url)
+
+	go func() {
+		err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
+		if err != nil {
+			log.Printf("Failed beeping with %+v", err)
+		}
+		err = beeep.Alert("Download Started", url, "assets/information.png")
+		if err != nil {
+			log.Printf("Failed alerting with %+v", err)
+		}
+	}()
+
+	_, err := dl.Run(context.TODO(), url)
+	if err != nil {
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
+	}
+
+	log.Printf("Downloaded %s", url)
+	ongoingDownloadsMutex.Lock()
+	delete(ongoingDownloads, url)
+	ongoingDownloadsMutex.Unlock()
+	return nil
+}

downloader/downloaders/ytdlp-raw.go

@@ -0,0 +1,51 @@
+package downloaders
+
+import (
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+
+	"github.com/gen2brain/beeep"
+)
+
+type YTDLPRawDownloader struct{}
+
+func (d *YTDLPRawDownloader) Download(url string) error {
+	_, ongoing := ongoingDownloads[url]
+	if ongoing {
+		// return fmt.Errorf("Download %s is already ongoing", url)
+		log.Printf("Download %s is already ongoing", url)
+		return nil
+	}
+	ongoingDownloadsMutex.Lock()
+	ongoingDownloads[url] = struct{}{}
+	ongoingDownloadsMutex.Unlock()
+
+	log.Printf("YTDLPRaw downloading %s", url)
+
+	go func() {
+		err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
+		if err != nil {
+			log.Printf("Failed beeping with %+v", err)
+		}
+		err = beeep.Alert("Download Started", url, "assets/information.png")
+		if err != nil {
+			log.Printf("Failed alerting with %+v", err)
+		}
+	}()
+
+	cmd := exec.Command("yt-dlp", "-o", "C:/Users/Administrator/ytdlpVideos/%(title)s.%(ext)s", "-f", "bestvideo[ext=mp4]+bestaudio[ext=m4a]", "-N", "12", "--sponsorblock-mark", "all", url)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("failed downloading %s with %+v", url, err)
+	}
+
+	log.Printf("Downloaded %s", url)
+	ongoingDownloadsMutex.Lock()
+	delete(ongoingDownloads, url)
+	ongoingDownloadsMutex.Unlock()
+	return nil
+}

downloader/go.mod

@@ -1,26 +1,34 @@
-module main
+module ytdl
 
 go 1.22.4
 
-require github.com/r3labs/sse/v2 v2.10.0
-
 require (
-	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/ProtonMail/go-crypto v1.1.3 // indirect
+	github.com/VividCortex/ewma v1.2.0 // indirect
+	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect
 	github.com/bitly/go-simplejson v0.5.1 // indirect
-	github.com/cloudflare/circl v1.3.7 // indirect
+	github.com/cloudflare/circl v1.5.0 // indirect
-	github.com/dlclark/regexp2 v1.11.0 // indirect
+	github.com/dlclark/regexp2 v1.11.4 // indirect
-	github.com/dop251/goja v0.0.0-20240220182346-e401ed450204 // indirect
+	github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd // indirect
-	github.com/go-sourcemap/sourcemap v2.1.3+incompatible // indirect
+	github.com/go-sourcemap/sourcemap v2.1.4+incompatible // indirect
-	github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7 // indirect
+	github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
-	github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
-	golang.org/x/crypto v0.21.0 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
-	golang.org/x/net v0.21.0 // indirect
+	github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
-	golang.org/x/sys v0.18.0 // indirect
+	github.com/mattn/go-runewidth v0.0.16 // indirect
-	golang.org/x/text v0.14.0 // indirect
+	github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
-	gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
+	github.com/rivo/uniseg v0.4.7 // indirect
+	github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
+	github.com/vbauerster/mpb/v5 v5.4.0 // indirect
+	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/text v0.21.0 // indirect
 )
 
 require (
+	github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4
 	github.com/gorilla/websocket v1.5.3
-	github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99
+	github.com/kkdai/youtube/v2 v2.10.2
+	github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd
+	github.com/nsqio/go-nsq v1.1.0
 )

downloader/go.sum

@@ -1,102 +1,70 @@
-github.com/ProtonMail/go-crypto v1.0.0 h1:LRuvITjQWX+WIfr930YHG2HNfjR1uOfyf5vE0kC2U78=
+github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
-github.com/ProtonMail/go-crypto v1.0.0/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0=
+github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
+github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk=
+github.com/ProtonMail/go-crypto v1.1.3/go.mod h1:rA3QumHc/FZ8pAHreoekgiAbzpNsfQAosU5td4SnOrE=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
+github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/bitly/go-simplejson v0.5.1 h1:xgwPbetQScXt1gh9BmoJ6j9JMr3TElvuIyjR8pgdoow=
 github.com/bitly/go-simplejson v0.5.1/go.mod h1:YOPVLzCfwK14b4Sff3oP1AmGhI9T9Vsg84etUnlyp+Q=
-github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0=
+github.com/cloudflare/circl v1.5.0 h1:hxIWksrX6XN5a1L2TI/h53AGPhNHoUBo+TD1ms9+pys=
-github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY=
+github.com/cloudflare/circl v1.5.0/go.mod h1:uddAzsPgqdMAYatqJ0lsjX1oECcQLIlRpzZh3pJrofs=
-github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
-github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA=
+github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yAo=
-github.com/cloudflare/circl v1.3.7 h1:qlCDlTPz2n9fu58M0Nh1J/JzcFpfgkFHHX3O35r5vcU=
+github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
-github.com/cloudflare/circl v1.3.7/go.mod h1:sRTcRWXGLrKw6yIGJ+l7amYJFfAXbZG0kBSc8r4zxgA=
+github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd h1:QMSNEh9uQkDjyPwu/J541GgSH+4hw+0skJDIj9HJ3mE=
-github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
-github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI=
-github.com/dlclark/regexp2 v1.4.1-0.20201116162257-a2a8dda75c91/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc=
+github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4=
-github.com/dlclark/regexp2 v1.7.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
-github.com/dlclark/regexp2 v1.11.0/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
+github.com/go-sourcemap/sourcemap v2.1.4+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
-github.com/dop251/goja v0.0.0-20211022113120-dc8c55024d06/go.mod h1:R9ET47fwRVRPZnOGvHxxhuZcbrMCuiqOz3Rlrh4KSnk=
+github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 h1:qZNfIGkIANxGv/OqtnntR4DfOY2+BgwR60cAcu/i3SE=
-github.com/dop251/goja v0.0.0-20240220182346-e401ed450204/go.mod h1:QMWlm50DNe14hD7t24KEqZuUdC9sOTy8W6XbCU1mlw4=
+github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4/go.mod h1:kW3HQ4UdaAyrUCSSDR4xUzBKW6O2iA4uHhk7AtyYp10=
-github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
-github.com/dop251/goja_nodejs v0.0.0-20211022123610-8dd9abb0616d/go.mod h1:DngW8aVqWbuLRMHItjPUyqdj+HWPvnQe8V8y1nDpIbM=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg=
+github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
-github.com/google/pprof v0.0.0-20230207041349-798e818bf904/go.mod h1:uglQLonpP8qtYCYyzA+8c/9qtqgA3qsXGYqCPKARAFg=
+github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
-github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik=
+github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
+github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
 github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
 github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
-github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w=
+github.com/kkdai/youtube/v2 v2.10.2 h1:e3JslUDiKEfjMzxFyrOh3O59C/aLfKNZyrcav00MZV0=
-github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kkdai/youtube/v2 v2.10.2/go.mod h1:4y1MIg7f1o5/kQfkr7nwXFtv8PGSoe4kChOB9/iMA88=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd h1:lLajTMgNTs/W4H05uQYnJDRIbIvHk6XXy7DQNFRbvzU=
-github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
+github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4=
-github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc=
-github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99 h1:ZAo7qJht9PqefOD7C0ZKQ8dEkpJeM955sYw0FtQnzvo=
+github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE=
-github.com/lrstanley/go-ytdlp v0.0.0-20240616011628-f35a10876c99/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4=
+github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY=
-github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d h1:VhgPp6v9qf9Agr/56bj7Y/xa04UccTW04VP0Qed4vnQ=
-github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc h1:zAsgcP8MhzAbhMnB1QQ2O7ZhWYVGYSR2iVcjzQuPV+o=
+github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d/go.mod h1:YUTz3bUH2ZwIWBy3CJBeOBEugqcmXREj14T+iG/4k4U=
-github.com/r3labs/sse v0.0.0-20210224172625-26fe804710bc/go.mod h1:S8xSOnV3CgpNrWd0GQ/OoQfMtlg2uPRSuTzcSGrzwK8=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
-github.com/r3labs/sse/v2 v2.10.0/go.mod h1:Igau6Whc+F17QUgML1fYe1VPZzTV6EMCnYktEmkNJ7I=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
+github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
-github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk=
-golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af/go.mod h1:4F09kP5F+am0jAwlQLddpoMDM+iewkxxt6nxUQ5nq5o=
-golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4=
+github.com/vbauerster/mpb/v5 v5.4.0 h1:n8JPunifvQvh6P1D1HAl2Ur9YcmKT1tpoUuiea5mlmg=
-golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU=
+github.com/vbauerster/mpb/v5 v5.4.0/go.mod h1:fi4wVo7BVQ22QcvFObm+VwliQXlV1eBT8JDaKXR4JGI=
-golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
-golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
-golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/net v0.0.0-20191116160921-f9c825593386/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
-golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY=
-golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
-golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc=
-golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
-golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
-golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
-golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U=
-golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
-golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
-golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
-golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
-golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
-golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
-golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
-golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
-golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
-golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
-gopkg.in/cenkalti/backoff.v1 v1.1.0/go.mod h1:J6Vskwqd+OMVJl8C33mmtxTBs2gyzfv7UDAkHu8BrjI=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
-gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
-gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

downloader/main.go

@@ -1,72 +1,125 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
 	"log"
+	"os"
+	"os/signal"
+	"syscall"
 	"time"
+	"ytdl/downloaders"
+
+	"github.com/nsqio/go-nsq"
 )
 
-const WEBSOCKET_SERVER = "ws://youtube-download-ws-server.site.quack-lab.dev/ws"
+var Error *log.Logger
-const WEBSOCKET_SERVER_ALT = "ws://localhost:8080/ws"
+var Warning *log.Logger
+
+func init() {
+	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
+	logFile, err := os.Create("ytdl.log")
+	if err != nil {
+		log.Printf("Error creating log file: %v", err)
+		os.Exit(1)
+	}
+	logger := io.MultiWriter(os.Stdout, logFile)
+	log.SetOutput(logger)
+
+	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
+		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
+		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+}
+
+// var downloader downloaders.Downloader = &downloaders.YTDLPRawDownloader{}
+// var downloader downloaders.Downloader = &downloaders.KidaiDownloader{}
+var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
+
+type DLHandler struct{}
+
+func (*DLHandler) HandleMessage(message *nsq.Message) error {
+	log.Printf("Received message '%s' with %d attempts", message.Body, message.Attempts)
+	NotifyDiscordErrorless(fmt.Sprintf("Received message '%s' with %d attempts", message.Body, message.Attempts))
+
+	data := DownloadRequest{}
+	err := json.Unmarshal(message.Body, &data)
+	if err != nil {
+		Error.Printf("Error unmarshalling message: %v", err)
+		NotifyDiscordErrorless(fmt.Sprintf("Error unmarshalling message: %v", err))
+		return err
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
+	go func() {
+		ticker := time.NewTicker(1 * time.Second)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				message.Touch()
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+
+	err = downloader.Download(data.Link)
+	if err != nil {
+		Error.Printf("Error downloading %s: %v", data.Link, err)
+		NotifyDiscordErrorless(fmt.Sprintf("Error downloading %s: %v", data.Link, err))
+		return err
+	}
+	message.Finish()
+	NotifyDiscordErrorless(fmt.Sprintf("Downloaded %s", data.Link))
+	return nil
+}
 
 func main() {
-	log.SetFlags(log.Lmicroseconds)
+	//	err := downloader.Download("https://www.youtube.com/watch?v=SiKjprtiPaw")
+	//	if err != nil {
+	//		Error.Printf("Error downloading: %v", err)
+	//	}
+	//	return
 
-	// res, err := http.Get(FULL_URL)
+	config := nsq.NewConfig()
-	// if err != nil {
+	config.MaxAttempts = 5
-	// 	log.Fatal(err)
+	config.MaxInFlight = downloaders.DOWNLOAD_WORKERS
-	// }
+	config.MsgTimeout = 10 * time.Second
-	// defer res.Body.Close()
-	// body, err := io.ReadAll(res.Body)
-	// if err != nil {
-	// 	log.Printf("Error reading response body: %+v\n", err)
-	// 	return
-	// }
-	// if res.StatusCode != http.StatusOK {
-	// 	log.Printf("Non-OK HTTP status: %d\nResponse body: %s\n", res.StatusCode, body)
-	// 	return
-	// }
 
-	// var data APIResponse
+	consumer, err := nsq.NewConsumer("ytdqueue", "dl", config)
-	// err = json.Unmarshal(body, &data)
+	if err != nil {
-	// if err != nil {
+		Error.Printf("Error creating consumer: %v", err)
-	// 	log.Printf("Error unmarshaling JSON: %+v\n", err)
+		return
-	// 	return
-	// }
-	// log.Printf("Data: %+v\n", data)
-
-	// listener := new(RealtimeListener)
-	// listener.Url = POCKETBASE_REALTIME
-	// listener.Collections = []string{COLLECTION_NAME}
-	// listener.initialize()
-
-	ws := new(WSConnection)
-	ws.url = WEBSOCKET_SERVER
-	ws.Open()
-
-	sem := make(chan struct{}, 4)
-	for {
-		select {
-		case event := <-ws.ReadChan:
-			eventCopy := event
-			status := make(chan error)
-			sem <- struct{}{}
-			
-			log.Printf("New event: %+v; semaphore at: %d", eventCopy, len(sem))
-			go func() {
-				defer func() {
-					<-sem
-					log.Printf("Semaphore at: %d", len(sem))
-				}()
-				// Download(eventCopy, status)
-				DownloadURL(eventCopy, status)
-				// go DownloadNative(event, status)
-				for status := range status {
-					log.Printf("Status: %s\n", status)
-				}
-			}()
-		case <-time.After(1 * time.Minute):
-			// Perform some action or simply continue to avoid deadlock
-			log.Println("Consumer is alive, but has no new events.")
-		}
 	}
+	for i := 0; i < downloaders.DOWNLOAD_WORKERS; i++ {
+		consumer.AddHandler(&DLHandler{})
+	}
+
+	url := "192.168.1.123:41505"
+	log.Printf("Connecting to nsq at %s", url)
+	err = consumer.ConnectToNSQD(url)
+	if err != nil {
+		Error.Printf("Error connecting to nsq: %v", err)
+		return
+	}
+	log.Printf("Connected to nsq at %s", url)
+
+	sigChan := make(chan os.Signal, 1)
+	signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
+
+	<-sigChan
+	log.Println("Received signal to terminate. Initiating graceful shutdown...")
+
+	consumer.Stop()
+	<-consumer.StopChan
+
+	log.Println("Graceful shutdown completed.")
 }

downloader/types.go

@@ -1,25 +1,5 @@
 package main
 
-type APIResponse struct {
+type DownloadRequest struct {
-	Page       int       `json:"page"`
+	Link string `json:"link"`
-	PerPage    int       `json:"perPage"`
+}
-	TotalItems int       `json:"totalItems"`
-	TotalPages int       `json:"totalPages"`
-	Items      []APIItem `json:"items"`
-}
-
-type APIItem struct {
-	CollectionId   string `json:"collectionId"`
-	CollectionName string `json:"collectionName"`
-	Created        string `json:"created"`
-	Downloaded     bool   `json:"downloaded"`
-	Id             string `json:"id"`
-	Link           string `json:"link"`
-	Updated        string `json:"updated"`
-}
-
-type PBEvent struct {
-	ClientId string  `json:"clientId"`
-	Action   string  `json:"action"`
-	Record   APIItem `json:"record"`
-}

downloader/ws-client.go

@@ -13,99 +13,131 @@ const IDLE_TIMEOUT = TIMEOUT * time.Second
 const PING_INTERVAL = (TIMEOUT / 2) * time.Second
 
 type WSConnection struct {
+	alive     bool
 	url       string
 	conn      *websocket.Conn
 	errChan   chan error
 	writeLock sync.Mutex
 	ReadChan  chan string
 	WriteChan chan string
+	Dead      chan error
 }
 
 func (ws *WSConnection) messageReader() {
-	log.Printf("Reading messages")
+	log.Printf("Starting reader")
 	for {
+		if !ws.alive {
+			break
+		}
+
 		_, message, err := ws.conn.ReadMessage()
 		ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
 		if err != nil {
 			ws.errChan <- err
-			return
+			break
 		}
+
 		log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan))
 		ws.ReadChan <- string(message)
 	}
+	log.Printf("Reader done")
 }
 
 func (ws *WSConnection) messageSender() {
-	log.Printf("Sending messages")
+	log.Printf("Starting sender")
 	for {
-		msg := <-ws.WriteChan
+		msg, ok := <-ws.WriteChan
-		ws.writeLock.Lock()
+		if !ok {
-		
+			break
-		ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
-		log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
-		err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
-		if err != nil {
-			log.Printf("Error during message writing: %v", err)
-			ws.errChan <- err
-			return
 		}
-		ws.writeLock.Unlock()
+		ws.doSend(msg)
+	}
+	log.Printf("Sender done")
+}
+
+func (ws *WSConnection) doSend(msg string) {
+	ws.writeLock.Lock()
+	defer ws.writeLock.Unlock()
+
+	ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
+	log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
+	err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
+	if err != nil {
+		log.Printf("Error during message writing: %v", err)
+		ws.errChan <- err
+		return
 	}
 }
 
 func (ws *WSConnection) pinger() {
 	log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL)
 	for {
-		time.Sleep(PING_INTERVAL)
+		if !ws.alive {
-
+			break
-		log.Printf("Ping")
-		ws.writeLock.Lock()
-		err := ws.conn.WriteMessage(websocket.PingMessage, nil)
-		if err != nil {
-			log.Println("Error during ping:", err)
-			ws.errChan <- err
-			return
 		}
-		ws.writeLock.Unlock()
+		ws.doPing()
+		time.Sleep(PING_INTERVAL)
 	}
+	log.Printf("Pinger done")
+}
+
+func (ws *WSConnection) doPing() {
+	ws.writeLock.Lock()
+	defer ws.writeLock.Unlock()
+
+	// log.Printf("Ping")
+	err := ws.conn.WriteMessage(websocket.PingMessage, nil)
+	if err != nil {
+		log.Println("Error during ping:", err)
+		ws.errChan <- err
+		return
+	}
+	ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
+	// log.Printf("Ping OK")
 }
 
 func (ws *WSConnection) handleError() {
 	for {
 		err := <-ws.errChan
-		log.Println("Error during message reading:", err)
+		log.Printf("Client error: %+v", err)
-
+		ws.alive = false
-		time.Sleep(5 * time.Second)
+		ws.conn.Close()
-		ws.Open()
+		close(ws.ReadChan)
+		close(ws.WriteChan)
+		close(ws.errChan)
+		ws.Dead <- err
+		return
 	}
 }
 
 func (ws *WSConnection) Open() {
 	log.Printf("Connecting to %s", ws.url)
+	ws.Dead = make(chan error, 1)
+
 	conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil)
 	if err != nil {
 		log.Println("Error during connection:", err)
-		ws.errChan <- err
+		ws.Dead <- err
 		return
 	}
 	log.Printf("Connected")
 	ws.conn = conn
-	ws.errChan = make(chan error)
+	ws.alive = true
-	ws.ReadChan = make(chan string, 1024)
+
-	ws.WriteChan = make(chan string, 1024)
+	ws.errChan = make(chan error, 1)
+	ws.ReadChan = make(chan string, 128)
+	ws.WriteChan = make(chan string, 128)
 
-	ws.conn.SetReadLimit(1024)
 	ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
 	ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
 	ws.conn.SetPongHandler(func(string) error {
-		log.Println("Pong")
+		// log.Println("Pong")
 		ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
-		ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
 		return nil
 	})
 
+	go ws.handleError()
 	go ws.messageReader()
 	go ws.messageSender()
-	go ws.handleError()
 	go ws.pinger()
 }

extension/background.js

@@ -0,0 +1,10 @@
+browser.menus.create({
+	id: "youtube-download",
+	title: "Download",
+	contexts: ["all"],
+	documentUrlPatterns: ["*://*.youtube.com/*"],
+});
+
+browser.menus.onShown.addListener((info, tab) => {
+	browser.tabs.sendMessage(tab.id, { action: "check-element", info: info });
+});

extension/content.js

@@ -0,0 +1,34 @@
+const URL = `http://localhost:5000/download`;
+let lastRightClickCoords = { x: 0, y: 0 };
+
+document.addEventListener("contextmenu", (event) => {
+	lastRightClickCoords = { x: event.clientX, y: event.clientY };
+	console.log("Right-click coordinates:", lastRightClickCoords);
+});
+
+browser.runtime.onMessage.addListener((message, sender, sendResponse) => {
+	const { x, y } = lastRightClickCoords;
+	let element = document.elementFromPoint(x, y);
+
+	while (element && element.tagName != "YTD-RICH-ITEM-RENDERER") {
+		element = element.parentElement;
+	}
+	if (!element.tagName == "YTD-RICH-ITEM-RENDERER") {
+		console.error("No video element found.");
+		return;
+	}
+
+	const link = element.querySelector("a#video-title-link").href;
+	fetch(URL, {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+		},
+		body: JSON.stringify({
+			link: link,
+		}),
+	}).then((res) => {
+		console.log(res);
+		res.json().then((data) => console.log(data));
+	});
+});

extension/manifest.json

@@ -0,0 +1,20 @@
+{
+	"manifest_version": 2,
+	"name": "Youtube Downloader",
+	"version": "1.0",
+	"permissions": [
+		"menus",
+		"activeTab",
+		"tabs",
+		"http://localhost:5000/*"
+	],
+	"background": {
+		"scripts": ["background.js"]
+	},
+	"content_scripts": [
+		{
+			"matches": ["*://*.youtube.com/*"],
+			"js": ["content.js"]
+		}
+	]
+}

hotkey.js

@@ -0,0 +1,47 @@
+nodes = document.querySelectorAll(":hover");
+i = 1;
+console.log(nodes);
+
+selectors = ["a#video-title-link", "a#video-title"];
+invidious = false;
+if (window.location.href.includes("invidious.site")) {
+	selector = "a";
+	invidious = true;
+}
+
+titleNode = nodes[nodes.length - i];
+outer:
+while (titleNode) {
+	for (let selector of selectors) {
+		if (titleNode.matches(selector)) {
+			break outer;
+		}
+	}
+	titleNode = titleNode.parentElement;
+	console.log(titleNode);
+}
+if (!titleNode) {
+	console.error("No video element found.");
+} else {
+	link = titleNode.href;
+	if (link.startsWith("/")) {
+		link = window.location.origin + link;
+	}
+	console.log(link);
+	fetch("https://nsq.site.quack-lab.dev/pub?topic=ytdqueue", {
+		method: "POST",
+		headers: {
+			"Content-Type": "application/json",
+		},
+		body: JSON.stringify({
+			link: link,
+		}),
+	}).then((res) => {
+		textNode = titleNode.querySelector("yt-formatted-string");
+		if (invidious) {
+			textNode = titleNode.querySelector("p");
+		}
+		textNode.style.setProperty("color", "green", "important");
+		res.json().then((data) => console.log(data));
+	});
+}

tampermonkey.js

@@ -63,7 +63,7 @@ function hookVideo(videoElement) {
 }
 
 async function main() {
-	const videosContainer = await waitForElement(document, "div#contents.style-scope.ytd-rich-grid-renderer");
+	const videosContainer = await waitForElement(document, "ytd-rich-grid-renderer > div#contents");
 
 	for (const video of videosContainer.querySelectorAll("ytd-rich-item-renderer")) {
 		parseVideo(video);

ws-client/go.mod

@@ -1,5 +0,0 @@
-module main
-
-go 1.22.4
-
-require github.com/gorilla/websocket v1.5.3

ws-client/go.sum

@@ -1,2 +0,0 @@
-github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
-github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

ws-client/main.go

@@ -1,73 +0,0 @@
-package main
-
-import (
-	"log"
-	"time"
-
-	"github.com/gorilla/websocket"
-)
-
-type WSConnection struct {
-	url     string
-	conn    *websocket.Conn
-	errChan chan error
-}
-
-func (ws *WSConnection) readMessage() {
-	log.Printf("Reading messages")
-	for {
-		_, message, err := ws.conn.ReadMessage()
-		if err != nil {
-			ws.errChan <- err
-			return
-		}
-		log.Printf("Received: %s", message)
-	}
-}
-
-func (ws *WSConnection) writeMessage(message string) {
-	err := ws.conn.WriteMessage(websocket.TextMessage, []byte(message))
-	if err != nil {
-		log.Printf("Error during message writing: %v", err)
-		ws.errChan <- err
-		return
-	}
-}
-
-func (ws *WSConnection) handleError() {
-	for {
-		err := <-ws.errChan
-		log.Println("Error during message reading:", err)
-
-		time.Sleep(5 * time.Second)
-		ws.open()
-	}
-}
-
-func (ws *WSConnection) open() {
-	log.Printf("Connecting to %s", ws.url)
-	conn, _, err := websocket.DefaultDialer.Dial(ws.url, nil)
-	if err != nil {
-		log.Println("Error during connection:", err)
-		ws.errChan <- err
-		return
-	}
-	log.Printf("Connected")
-	ws.conn = conn
-	ws.errChan = make(chan error)
-	go ws.readMessage()
-	go ws.handleError()
-}
-
-func main() {
-	log.SetFlags(log.Lmicroseconds)
-
-	wsConn := WSConnection{
-		url: "ws://localhost:8080/ws",
-	}
-	wsConn.open()
-	for {
-		log.Printf("zzz...")
-		time.Sleep(30 * time.Second)
-	}
-}

ws-server/build.sh

@@ -1,3 +0,0 @@
-# docker build -t youtube-download-ws-server .
-
-tar -cf deploy.tar captain-definition dockerfile main.go go.mod go.sum

ws-server/captain-definition

@@ -1,4 +0,0 @@
-{
-  "schemaVersion": 2,
-  "dockerfilePath": "./dockerfile"
-}

ws-server/dockerfile

@@ -1,21 +0,0 @@
-FROM golang:1.22.4 as base
-
-WORKDIR $GOPATH/src/app/
-
-COPY . .
-
-RUN go mod download
-RUN go mod verify
-
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -o /main .
-
-FROM scratch
-
-COPY --from=base /usr/share/zoneinfo /usr/share/zoneinfo
-COPY --from=base /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
-COPY --from=base /etc/passwd /etc/passwd
-COPY --from=base /etc/group /etc/group
-
-COPY --from=base /main .
-
-CMD ["/main"]

ws-server/go.mod

@@ -1,5 +0,0 @@
-module main
-
-go 1.22.4
-
-require github.com/gorilla/websocket v1.5.3

ws-server/go.sum

@@ -1,2 +0,0 @@
-github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
-github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=

ws-server/main.go

@@ -1,169 +0,0 @@
-package main
-
-import (
-	"encoding/json"
-	"fmt"
-	"io"
-	"log"
-	"net/http"
-	"sync"
-	"time"
-
-	"github.com/gorilla/websocket"
-)
-
-var upgrader = websocket.Upgrader{}
-var wsBroadcast = make(chan string, 128)
-var connections = make(map[*WSConnection]bool)
-
-const TIMEOUT = 6
-const IDLE_TIMEOUT = TIMEOUT * time.Second
-const PING_INTERVAL = (TIMEOUT / 2) * time.Second
-
-type WSConnection struct {
-	conn      *websocket.Conn
-	writeLock sync.Mutex
-	ReadChan  chan string
-	WriteChan chan string
-	ErrorChan chan error
-}
-
-func (ws *WSConnection) messageReader() {
-	log.Printf("Reading messages")
-	for {
-		_, message, err := ws.conn.ReadMessage()
-		ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
-		if err != nil {
-			ws.ErrorChan <- err
-			return
-		}
-		log.Printf("Received: %s, %d in output channel", message, len(ws.ReadChan))
-		ws.ReadChan <- string(message)
-	}
-}
-
-func (ws *WSConnection) messageSender() {
-	log.Printf("Sending messages")
-	for {
-		msg := <-ws.WriteChan
-		ws.writeLock.Lock()
-
-		ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
-		log.Printf("Sending: %s, %d in input channel", msg, len(ws.WriteChan))
-		err := ws.conn.WriteMessage(websocket.TextMessage, []byte(msg))
-		if err != nil {
-			log.Printf("Error during message writing: %v", err)
-			ws.ErrorChan <- err
-			return
-		}
-		ws.writeLock.Unlock()
-	}
-}
-
-func (ws *WSConnection) pinger() {
-	log.Printf("Starting pinger, sleeping for %v", PING_INTERVAL)
-	for {
-		time.Sleep(PING_INTERVAL)
-
-		log.Printf("Ping")
-		ws.writeLock.Lock()
-		err := ws.conn.WriteMessage(websocket.PingMessage, nil)
-		if err != nil {
-			log.Println("Error during ping:", err)
-			ws.ErrorChan <- err
-			return
-		}
-		ws.writeLock.Unlock()
-	}
-}
-
-func (ws *WSConnection) Open() {
-	log.Printf("Client connected")
-	ws.ReadChan = make(chan string, 1024)
-	ws.WriteChan = make(chan string, 1024)
-	ws.ErrorChan = make(chan error, 16)
-
-	ws.conn.SetReadLimit(1024)
-	ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
-	ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
-	ws.conn.SetPongHandler(func(string) error {
-		log.Println("Pong")
-		ws.conn.SetReadDeadline(time.Now().Add(IDLE_TIMEOUT))
-		ws.conn.SetWriteDeadline(time.Now().Add(IDLE_TIMEOUT))
-		return nil
-	})
-
-	connections[ws] = true
-
-	go ws.messageReader()
-	go ws.messageSender()
-	go ws.pinger()
-	go func() {
-		for {
-			select {
-			case err := <-ws.ErrorChan:
-				log.Printf("Error: %v", err)
-				ws.conn.Close()
-				log.Printf("Client disconnected")
-				connections[ws] = false
-				return
-			// case msg := <-wsBroadcast:
-			// 	ws.WriteChan <- msg
-			}
-		}
-	}()
-}
-
-func wsHandler(responseWriter http.ResponseWriter, request *http.Request) {
-	conn, err := upgrader.Upgrade(responseWriter, request, nil)
-	if err != nil {
-		fmt.Println("Error during connection upgrade:", err)
-		return
-	}
-
-	ws := new(WSConnection)
-	ws.conn = conn
-	ws.Open()
-}
-
-type DownloadReq struct {
-	Link string `json:"link"`
-}
-
-func handleDownload(responseWriter http.ResponseWriter, request *http.Request) {
-	body, err := io.ReadAll(request.Body)
-	if err != nil {
-		log.Printf("Error reading request body: %v", err)
-		http.Error(responseWriter, "Error reading request body", http.StatusBadRequest)
-		return
-	}
-	defer request.Body.Close()
-
-	req := DownloadReq{}
-	err = json.Unmarshal(body, &req)
-	if err != nil {
-		log.Printf("Error parsing JSON: %v", err)
-		http.Error(responseWriter, "Error parsing JSON", http.StatusBadRequest)
-		return
-	}
-
-	log.Printf("Received download request: %s, %d in channel", req.Link, len(wsBroadcast))
-	go func() {
-		for ws := range connections {
-			ws.WriteChan <- req.Link
-		}
-	}()
-	// wsBroadcast <- req.Link
-}
-
-func main() {
-	log.SetFlags(log.Lmicroseconds)
-
-	http.HandleFunc("/ws", wsHandler)
-	http.HandleFunc("/download", handleDownload)
-	log.Println("Server starting on :8080")
-	err := http.ListenAndServe(":8080", nil)
-	if err != nil {
-		log.Println("Error starting server:", err)
-	}
-}

youtubeWatcher/deploy.sh

@@ -0,0 +1,3 @@
+nssm stop YoutubeWatcher && \
+go build . && \
+nssm start YoutubeWatcher

youtubeWatcher/go.mod

@@ -0,0 +1,3 @@
+module ywatcher
+
+go 1.24.2

youtubeWatcher/main.go

@@ -0,0 +1,156 @@
+package main
+
+import (
+	"encoding/xml"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+	"os"
+	"regexp"
+	"sync"
+	"time"
+)
+
+type DownloadRequest struct {
+	Link string `json:"link"`
+}
+type RssWatcher struct {
+	Feed *RssFeed
+}
+
+var videoRegex = regexp.MustCompile(`yt:video:(?<videoid>[^ ]+) (?:[^ ]+ ){2}(?<videotitle>.+?)https(?:[^ ]+ ){2}(?<date>[^ ]+)`)
+var feeds []*RssFeed = []*RssFeed{
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCMwJJL5FJFuTRT55ksbQ4GQ", Id: "@AsmongoldClips"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
+}
+
+func (w *RssWatcher) Watch(videoUrls chan string) error {
+	ticker := time.NewTicker(1 * time.Minute)
+	defer ticker.Stop()
+
+	log.Printf("[%s]: Watcher started, checking every minute.", w.Feed.Id)
+
+	w.CheckFeed(videoUrls)
+	for {
+		select {
+		case <-ticker.C:
+			log.Printf("[%s]: Checking feed", w.Feed.Id)
+			err := w.CheckFeed(videoUrls)
+			if err != nil {
+				return fmt.Errorf("watcher %s failed to check feed: %w", w.Feed.Id, err)
+			}
+			log.Printf("[%s]: Successfully checked feed", w.Feed.Id)
+		}
+	}
+}
+
+func (w *RssWatcher) CheckFeed(videoUrls chan string) error {
+	log.Printf("Checking feed URL: %s", w.Feed.Url)
+	resp, err := http.Get(w.Feed.Url)
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
+	}
+	defer resp.Body.Close()
+
+	log.Printf("Received response with status code: %d", resp.StatusCode)
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
+	}
+
+	// os.WriteFile("cache.xml", body, 0644)
+	// body, err := os.ReadFile("cache.xml")
+	// if err != nil {
+	// 	return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
+	// }
+
+	var feed Feed
+	err = xml.Unmarshal(body, &feed)
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to unmarshal feed: %w", w.Feed.Id, err)
+	}
+
+	for _, entry := range feed.Entry {
+		uploaded, err := time.Parse(time.RFC3339, entry.Published)
+		if err != nil {
+			return fmt.Errorf("[%s]: failed to parse published date: %w", w.Feed.Id, err)
+		}
+		if uploaded.Before(w.Feed.LastSeen) {
+			log.Printf("[%s]: Skipping video titled %q because it was uploaded before %s", w.Feed.Id, entry.Title, w.Feed.LastSeen.Format(time.RFC3339))
+			continue
+		}
+		log.Printf("[%s]: Found new video titled %q with url %q", w.Feed.Id, entry.Title, entry.Link.Href)
+		videoUrls <- entry.Link.Href
+	}
+	err = w.Feed.WriteLastSeen(time.Now())
+	if err != nil {
+		return fmt.Errorf("[%s]: failed to write last seen: %w", w.Feed.Id, err)
+	}
+	w.Feed.LastSeen = time.Now()
+
+	return nil
+}
+
+var Error *log.Logger
+var Warning *log.Logger
+
+func init() {
+	log.SetFlags(log.Lmicroseconds | log.Lshortfile)
+	logFile, err := os.Create("ywatcher.log")
+	if err != nil {
+		log.Printf("Error creating log file: %v", err)
+		os.Exit(1)
+	}
+	logger := io.MultiWriter(os.Stdout, logFile)
+	log.SetOutput(logger)
+
+	Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
+		fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+	Warning = log.New(io.MultiWriter(logFile, os.Stdout),
+		fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
+		log.Lmicroseconds|log.Lshortfile)
+}
+
+func main() {
+	videoUrls := make(chan string, 12)
+
+	wg := &sync.WaitGroup{}
+	for _, feed := range feeds {
+		wg.Add(1)
+		go func(feed *RssFeed) {
+			err := feed.UpdateLastSeen()
+			if err != nil {
+				Error.Printf("failed to update lastseen for feed %s: %v", feed.Id, err)
+				panic(err)
+			}
+
+			defer wg.Done()
+			watcher := RssWatcher{
+				Feed: feed,
+			}
+			err = watcher.Watch(videoUrls)
+			if err != nil {
+				Error.Printf("watcher %s failed to watch feed: %v", feed.Id, err)
+				panic(err)
+			}
+		}(feed)
+	}
+
+	go func() {
+		for videoUrl := range videoUrls {
+			log.Printf("Got new video with url %q", videoUrl)
+			err := Download(videoUrl)
+			if err != nil {
+				Error.Printf("failed to download video: %v", err)
+				panic(err)
+			}
+		}
+	}()
+
+	wg.Wait()
+}

youtubeWatcher/nsqWriter.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+)
+
+const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
+
+type Item struct {
+	Link string `json:"link"`
+}
+
+func Download(url string) error {
+	log.Printf("Starting download for URL: %s", url)
+
+	req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil)
+	if err != nil {
+		return fmt.Errorf("error creating POST request: %++v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	item := new(Item)
+	item.Link = url
+
+	body, err := json.Marshal(item)
+	if err != nil {
+		return fmt.Errorf("error marshalling subscription body: %++v", err)
+	}
+	req.Body = io.NopCloser(bytes.NewReader(body))
+
+	client := http.Client{}
+	log.Printf("Sending POST request to %s", URL)
+	res, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error sending POST request: %++v", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("non-OK HTTP status: %d", res.StatusCode)
+	}
+	log.Printf("Successfully enqueued %s", url)
+	return nil
+}

youtubeWatcher/rssfeed.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"log"
+	"os"
+	"sync"
+	"time"
+)
+
+type RssFeed struct {
+	Url       string
+	Id        string
+	LastSeen  time.Time
+	fileMutex sync.Mutex
+}
+
+func (f *RssFeed) UpdateLastSeen() error {
+	log.Printf("[%s]: Updating last seen time", f.Id)
+	lastSeen, err := f.ReadLastSeen()
+	if err != nil {
+		return fmt.Errorf("failed to read lastseen file for feed %s: %w", f.Id, err)
+	}
+	f.LastSeen = lastSeen
+	log.Printf("[%s]: Last seen time updated to: %s", f.Id, f.LastSeen)
+	return nil
+}
+func (f *RssFeed) ReadLastSeen() (time.Time, error) {
+	f.fileMutex.Lock()
+	defer f.fileMutex.Unlock()
+
+	lastSeen := time.Now()
+	log.Printf("[%s]: Attempting to open lastseen file...", f.Id)
+	lastSeenFile, err := os.Open("lastseen" + f.Id)
+	if err != nil {
+		if os.IsNotExist(err) {
+			log.Printf("[%s]: lastseen file does not exist, creating a new one...", f.Id)
+			lastSeenFile, err = os.Create("lastseen" + f.Id)
+			if err != nil {
+				return lastSeen, fmt.Errorf("failed to create lastseen file: %w", err)
+			}
+			log.Printf("[%s]: Writing current time to lastseen file...", f.Id)
+			lastSeenFile.Write([]byte(lastSeen.Format(time.RFC3339)))
+			lastSeenFile.Sync()
+			log.Printf("[%s]: Successfully created lastseen file with current time.", f.Id)
+			return lastSeen, nil
+		} else {
+			log.Printf("[%s]: Error opening lastseen file: %v", f.Id, err)
+			return lastSeen, fmt.Errorf("failed to read lastseen file: %w", err)
+		}
+	}
+	log.Printf("[%s]: Reading contents of lastseen file...", f.Id)
+	lastSeenBytes, err := io.ReadAll(lastSeenFile)
+	if err != nil {
+		log.Printf("[%s]: Error reading lastseen file: %v", f.Id, err)
+		return lastSeen, fmt.Errorf("failed to read lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Parsing lastseen time...", f.Id)
+	lastSeen, err = time.Parse(time.RFC3339, string(lastSeenBytes))
+	if err != nil {
+		log.Printf("[%s]: Error parsing lastseen file: %v", f.Id, err)
+		return lastSeen, fmt.Errorf("failed to parse lastseen file: %w", err)
+	}
+	lastSeenFile.Close()
+	log.Printf("[%s]: Last seen time read: %s", f.Id, lastSeen)
+	return lastSeen, nil
+}
+
+func (f *RssFeed) WriteLastSeen(when time.Time) error {
+	f.fileMutex.Lock()
+	defer f.fileMutex.Unlock()
+
+	log.Printf("[%s]: Attempting to create lastseen file...", f.Id)
+	lastSeenFile, err := os.Create("lastseen" + f.Id)
+	if err != nil {
+		log.Printf("[%s]: Error creating lastseen file: %v", f.Id, err)
+		return fmt.Errorf("failed to create lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Successfully created lastseen file: lastseen%s", f.Id, f.Id)
+
+	log.Printf("[%s]: Writing last seen time to file: %s", f.Id, when.Format(time.RFC3339))
+	_, err = lastSeenFile.Write([]byte(when.Format(time.RFC3339)))
+	if err != nil {
+		log.Printf("[%s]: Error writing to lastseen file: %v", f.Id, err)
+		return fmt.Errorf("failed to write to lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Successfully wrote last seen time to file.", f.Id)
+
+	err = lastSeenFile.Close()
+	if err != nil {
+		log.Printf("[%s]: Error closing lastseen file: %v", f.Id, err)
+		return fmt.Errorf("failed to close lastseen file: %w", err)
+	}
+	log.Printf("[%s]: Lastseen file closed successfully.", f.Id)
+	return nil
+}

youtubeWatcher/rssfeedResType.go

@@ -0,0 +1,86 @@
+package main
+
+import "encoding/xml"
+
+type Feed struct {
+	XMLName   xml.Name `xml:"feed"`
+	Text      string   `xml:",chardata"`
+	Yt        string   `xml:"yt,attr"`
+	Media     string   `xml:"media,attr"`
+	Xmlns     string   `xml:"xmlns,attr"`
+	Link      []Link   `xml:"link"`
+	ID        string   `xml:"id"`
+	ChannelId string   `xml:"channelId"`
+	Title     string   `xml:"title"`
+	Author    Author   `xml:"author"`
+	Published string   `xml:"published"`
+	Entry     []Entry  `xml:"entry"`
+}
+
+type Link struct {
+	Text string `xml:",chardata"`
+	Rel  string `xml:"rel,attr"`
+	Href string `xml:"href,attr"`
+}
+
+type Author struct {
+	Text string `xml:",chardata"`
+	Name string `xml:"name"`
+	URI  string `xml:"uri"`
+}
+
+type Entry struct {
+	Text      string `xml:",chardata"`
+	ID        string `xml:"id"`
+	VideoId   string `xml:"videoId"`
+	ChannelId string `xml:"channelId"`
+	Title     string `xml:"title"`
+	Link      Link   `xml:"link"`
+	Author    Author `xml:"author"`
+	Published string `xml:"published"`
+	Updated   string `xml:"updated"`
+	Group     Group  `xml:"group"`
+}
+
+type Group struct {
+	Text        string    `xml:",chardata"`
+	Title       string    `xml:"title"`
+	Content     Content   `xml:"content"`
+	Thumbnail   Thumbnail `xml:"thumbnail"`
+	Description string    `xml:"description"`
+	Community   Community `xml:"community"`
+}
+
+type Content struct {
+	Text   string `xml:",chardata"`
+	URL    string `xml:"url,attr"`
+	Type   string `xml:"type,attr"`
+	Width  string `xml:"width,attr"`
+	Height string `xml:"height,attr"`
+}
+
+type Thumbnail struct {
+	Text   string `xml:",chardata"`
+	URL    string `xml:"url,attr"`
+	Width  string `xml:"width,attr"`
+	Height string `xml:"height,attr"`
+}
+
+type Community struct {
+	Text       string     `xml:",chardata"`
+	StarRating StarRating `xml:"starRating"`
+	Statistics Statistics `xml:"statistics"`
+}
+
+type StarRating struct {
+	Text    string `xml:",chardata"`
+	Count   string `xml:"count,attr"`
+	Average string `xml:"average,attr"`
+	Min     string `xml:"min,attr"`
+	Max     string `xml:"max,attr"`
+}
+
+type Statistics struct {
+	Text  string `xml:",chardata"`
+	Views string `xml:"views,attr"`
+}