Implement writing urls to nsq for downloading

youtubeWatcher/main.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"log"
+	"net/http"
 	"os"
 	"regexp"
 	"sync"
@@ -21,10 +22,10 @@ type RssWatcher struct {
 var videoRegex = regexp.MustCompile(`yt:video:(?<videoid>[^ ]+) (?:[^ ]+ ){2}(?<videotitle>.+?)https(?:[^ ]+ ){2}(?<date>[^ ]+)`)
 var feeds []*RssFeed = []*RssFeed{
 	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCMwJJL5FJFuTRT55ksbQ4GQ", Id: "@AsmongoldClips"},
-	// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
-	// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
-	// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
-	// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
+	{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
 }
 
 func (w *RssWatcher) Watch(videoUrls chan string) error {
@@ -48,24 +49,24 @@ func (w *RssWatcher) Watch(videoUrls chan string) error {
 }
 
 func (w *RssWatcher) CheckFeed(videoUrls chan string) error {
-	// log.Printf("Checking feed URL: %s", w.Feed.Url)
+	log.Printf("Checking feed URL: %s", w.Feed.Url)
-	// resp, err := http.Get(w.Feed.Url)
+	resp, err := http.Get(w.Feed.Url)
-	// if err != nil {
+	if err != nil {
-	// 	return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
+		return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
-	// }
+	}
-	// defer resp.Body.Close()
+	defer resp.Body.Close()
 
-	// log.Printf("Received response with status code: %d", resp.StatusCode)
+	log.Printf("Received response with status code: %d", resp.StatusCode)
-	// body, err := io.ReadAll(resp.Body)
+	body, err := io.ReadAll(resp.Body)
-	// if err != nil {
+	if err != nil {
-	// 	return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
+		return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
-	// }
+	}
 
 	// os.WriteFile("cache.xml", body, 0644)
-	body, err := os.ReadFile("cache.xml")
+	// body, err := os.ReadFile("cache.xml")
-	if err != nil {
+	// if err != nil {
-		return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
+	// 	return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
-	}
+	// }
 
 	var feed Feed
 	err = xml.Unmarshal(body, &feed)
@@ -143,6 +144,11 @@ func main() {
 	go func() {
 		for videoUrl := range videoUrls {
 			log.Printf("Got new video with url %q", videoUrl)
+			err := Download(videoUrl)
+			if err != nil {
+				Error.Printf("failed to download video: %v", err)
+				panic(err)
+			}
 		}
 	}()
 

youtubeWatcher/nsqWriter.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"net/http"
+)
+
+const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
+
+type Item struct {
+	Link string `json:"link"`
+}
+
+func Download(url string) error {
+	log.Printf("Starting download for URL: %s", url)
+
+	req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil)
+	if err != nil {
+		return fmt.Errorf("error creating POST request: %++v", err)
+	}
+	req.Header.Set("Content-Type", "application/json")
+
+	item := new(Item)
+	item.Link = url
+
+	body, err := json.Marshal(item)
+	if err != nil {
+		return fmt.Errorf("error marshalling subscription body: %++v", err)
+	}
+	req.Body = io.NopCloser(bytes.NewReader(body))
+
+	client := http.Client{}
+	log.Printf("Sending POST request to %s", URL)
+	res, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error sending POST request: %++v", err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return fmt.Errorf("non-OK HTTP status: %d", res.StatusCode)
+	}
+	log.Printf("Successfully enqueued %s", url)
+	return nil
+}

youtubeWatcher/rssfeed.go

@@ -32,7 +32,7 @@ func (f *RssFeed) ReadLastSeen() (time.Time, error) {
 
 	lastSeen := time.Now()
 	log.Printf("[%s]: Attempting to open lastseen file...", f.Id)
-	lastSeenFile, err := os.Open("lastseen")
+	lastSeenFile, err := os.Open("lastseen" + f.Id)
 	if err != nil {
 		if os.IsNotExist(err) {
 			log.Printf("[%s]: lastseen file does not exist, creating a new one...", f.Id)