dave/youtube-downloader
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Implement writing urls to nsq for downloading

Browse Source
This commit is contained in:
PhatPhuckDave
2025-04-13 17:28:58 +02:00
parent 014e99751e
commit d420f78ab8
3 changed files with 76 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files

44
youtubeWatcher/main.go
View File

@@ -5,6 +5,7 @@ import (
"fmt"
"io"
"log"
"net/http"
"os"
"regexp"
"sync"
@@ -21,10 +22,10 @@ type RssWatcher struct {
var videoRegex = regexp.MustCompile(`yt:video:(?<videoid>[^ ]+) (?:[^ ]+ ){2}(?<videotitle>.+?)https(?:[^ ]+ ){2}(?<date>[^ ]+)`)
var feeds []*RssFeed = []*RssFeed{
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCMwJJL5FJFuTRT55ksbQ4GQ", Id: "@AsmongoldClips"},
// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
// {Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC8nZUXCwCTffxthKLtOp6ng", Id: "@Splattercatgaming"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UC2THf0jmDDeBujMzG1sD2-Q", Id: "@thesingleplayersquad"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCmtyQOKKmrMVaKuRXz02jbQ", Id: "@SebastianLague"},
{Url: "https://www.youtube.com/feeds/videos.xml?channel_id=UCywBfpGBYhsczNuyyh6Cf6w", Id: "@WorthABuyreviews"},
}
func (w *RssWatcher) Watch(videoUrls chan string) error {
@@ -48,24 +49,24 @@ func (w *RssWatcher) Watch(videoUrls chan string) error {
}
func (w *RssWatcher) CheckFeed(videoUrls chan string) error {
// log.Printf("Checking feed URL: %s", w.Feed.Url)
// resp, err := http.Get(w.Feed.Url)
// if err != nil {
// return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
// }
// defer resp.Body.Close()
log.Printf("Checking feed URL: %s", w.Feed.Url)
resp, err := http.Get(w.Feed.Url)
if err != nil {
return fmt.Errorf("[%s]: failed to create request: %w", w.Feed.Id, err)
}
defer resp.Body.Close()
// log.Printf("Received response with status code: %d", resp.StatusCode)
// body, err := io.ReadAll(resp.Body)
// if err != nil {
// return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
// }
log.Printf("Received response with status code: %d", resp.StatusCode)
body, err := io.ReadAll(resp.Body)
if err != nil {
return fmt.Errorf("[%s]: failed to read response body: %w", w.Feed.Id, err)
}
// os.WriteFile("cache.xml", body, 0644)
body, err := os.ReadFile("cache.xml")
if err != nil {
return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
}
// body, err := os.ReadFile("cache.xml")
// if err != nil {
// return fmt.Errorf("[%s]: failed to read cache file: %w", w.Feed.Id, err)
// }
var feed Feed
err = xml.Unmarshal(body, &feed)
@@ -143,6 +144,11 @@ func main() {
go func() {
for videoUrl := range videoUrls {
log.Printf("Got new video with url %q", videoUrl)
err := Download(videoUrl)
if err != nil {
Error.Printf("failed to download video: %v", err)
panic(err)
}
}
}()

50
youtubeWatcher/nsqWriter.go Normal file
View File

@@ -0,0 +1,50 @@
package main
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"log"
"net/http"
)
const URL = `https://nsq.site.quack-lab.dev/pub?topic=ytdqueue`
type Item struct {
Link string `json:"link"`
}
func Download(url string) error {
log.Printf("Starting download for URL: %s", url)
req, err := http.NewRequestWithContext(context.Background(), "POST", URL, nil)
if err != nil {
return fmt.Errorf("error creating POST request: %++v", err)
}
req.Header.Set("Content-Type", "application/json")
item := new(Item)
item.Link = url
body, err := json.Marshal(item)
if err != nil {
return fmt.Errorf("error marshalling subscription body: %++v", err)
}
req.Body = io.NopCloser(bytes.NewReader(body))
client := http.Client{}
log.Printf("Sending POST request to %s", URL)
res, err := client.Do(req)
if err != nil {
return fmt.Errorf("error sending POST request: %++v", err)
}
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return fmt.Errorf("non-OK HTTP status: %d", res.StatusCode)
}
log.Printf("Successfully enqueued %s", url)
return nil
}

2
youtubeWatcher/rssfeed.go
View File

@@ -32,7 +32,7 @@ func (f *RssFeed) ReadLastSeen() (time.Time, error) {
lastSeen := time.Now()
log.Printf("[%s]: Attempting to open lastseen file...", f.Id)
lastSeenFile, err := os.Open("lastseen")
lastSeenFile, err := os.Open("lastseen" + f.Id)
if err != nil {
if os.IsNotExist(err) {
log.Printf("[%s]: lastseen file does not exist, creating a new one...", f.Id)