Have downloader use new logging utilities

This commit is contained in:
2025-10-06 13:43:13 +02:00
parent 4601a0fc60
commit eb4f715ce3
4 changed files with 79 additions and 52 deletions

View File

@@ -3,9 +3,8 @@ package downloaders
import (
"context"
"fmt"
"log"
"github.com/gen2brain/beeep"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/lrstanley/go-ytdlp"
)
@@ -23,35 +22,36 @@ var dl = ytdlp.New().
type YTDLPLibDownloader struct{}
func (d *YTDLPLibDownloader) Download(url string) error {
downloadlogger := logger.Default.WithPrefix(fmt.Sprintf("url=%q", url))
_, ongoing := ongoingDownloads[url]
if ongoing {
// return fmt.Errorf("Download %s is already ongoing", url)
log.Printf("Download %s is already ongoing", url)
downloadlogger.Info("Download is already ongoing")
return nil
}
ongoingDownloadsMutex.Lock()
ongoingDownloads[url] = struct{}{}
ongoingDownloadsMutex.Unlock()
log.Printf("YTDLPLib downloading %s", url)
downloadlogger.Info("YTDLPLib downloading")
go func() {
err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
if err != nil {
log.Printf("Failed beeping with %+v", err)
}
err = beeep.Alert("Download Started", url, "assets/information.png")
if err != nil {
log.Printf("Failed alerting with %+v", err)
}
}()
// go func() {
// err := beeep.Beep(beeep.DefaultFreq, beeep.DefaultDuration)
// if err != nil {
// downloadlogger.Error("Failed beeping with %+v", err)
// }
// err = beeep.Alert("Download Started", url, "assets/information.png")
// if err != nil {
// downloadlogger.Error("Failed alerting with %+v", err)
// }
// }()
_, err := dl.Run(context.TODO(), url)
if err != nil {
return fmt.Errorf("failed downloading %s with %+v", url, err)
}
log.Printf("Downloaded %s", url)
downloadlogger.Info("Downloaded")
ongoingDownloadsMutex.Lock()
delete(ongoingDownloads, url)
ongoingDownloadsMutex.Unlock()

View File

@@ -1,6 +1,6 @@
module ytdl
go 1.22.4
go 1.23.6
require (
github.com/ProtonMail/go-crypto v1.1.3 // indirect
@@ -14,18 +14,27 @@ require (
github.com/go-toast/toast v0.0.0-20190211030409-01e6764cf0a4 // indirect
github.com/godbus/dbus/v5 v5.1.0 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad // indirect
github.com/hexops/valast v1.5.0 // indirect
github.com/mattn/go-runewidth v0.0.16 // indirect
github.com/nu7hatch/gouuid v0.0.0-20131221200532-179d4d0c4d8d // indirect
github.com/rivo/uniseg v0.4.7 // indirect
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af // indirect
github.com/vbauerster/mpb/v5 v5.4.0 // indirect
golang.org/x/crypto v0.31.0 // indirect
golang.org/x/mod v0.17.0 // indirect
golang.org/x/sync v0.10.0 // indirect
golang.org/x/sys v0.28.0 // indirect
golang.org/x/text v0.21.0 // indirect
golang.org/x/time v0.12.0 // indirect
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect
mvdan.cc/gofumpt v0.4.0 // indirect
)
require (
git.site.quack-lab.dev/dave/cylogger v1.4.0
git.site.quack-lab.dev/dave/cyutils v1.4.0
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4
github.com/gorilla/websocket v1.5.3
github.com/kkdai/youtube/v2 v2.10.2

View File

@@ -1,3 +1,7 @@
git.site.quack-lab.dev/dave/cylogger v1.4.0 h1:3Ca7V5JWvruARJd5S8xDFwW9LnZ9QInqkYLRdrEFvuY=
git.site.quack-lab.dev/dave/cylogger v1.4.0/go.mod h1:wctgZplMvroA4X6p8f4B/LaCKtiBcT1Pp+L14kcS8jk=
git.site.quack-lab.dev/dave/cyutils v1.4.0 h1:/Xo3QfLIFNab+axHneWmUK4MyfuObl+qq8whF9vTQpk=
git.site.quack-lab.dev/dave/cyutils v1.4.0/go.mod h1:fBjALu2Cp2u2bDr+E4zbGVMBeIgFzROg+4TCcTNAiQU=
github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0rYXWg0=
github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
github.com/ProtonMail/go-crypto v1.1.3 h1:nRBOetoydLeUb4nHajyO2bKqMLfWQ/ZPwkXqXxPxCFk=
@@ -17,6 +21,8 @@ github.com/dlclark/regexp2 v1.11.4 h1:rPYF9/LECdNymJufQKmri9gV604RvvABwgOA8un7yA
github.com/dlclark/regexp2 v1.11.4/go.mod h1:DHkYz0B9wPfa6wondMfaivmHpzrQ3v9q8cnmRbL6yW8=
github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd h1:QMSNEh9uQkDjyPwu/J541GgSH+4hw+0skJDIj9HJ3mE=
github.com/dop251/goja v0.0.0-20241024094426-79f3a7efcdbd/go.mod h1:MxLav0peU43GgvwVgNbLAj1s/bSGboKkhuULvq/7hx4=
github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE=
github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps=
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4 h1:ygs9POGDQpQGLJPlq4+0LBUmMBNox1N4JSpw+OETcvI=
github.com/gen2brain/beeep v0.0.0-20240516210008-9c006672e7f4/go.mod h1:0W7dI87PvXJ1Sjs0QPvWXKcQmNERY77e8l7GFhZB/s4=
github.com/go-sourcemap/sourcemap v2.1.4+incompatible h1:a+iTbH5auLKxaNwQFg0B+TCYl6lbukKPc7b5x0n1s6Q=
@@ -28,12 +34,24 @@ github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad h1:a6HEuzUHeKH6hwfN/ZoQgRgVIWFJljSWa/zetS2WTvg=
github.com/google/pprof v0.0.0-20241210010833-40e02aabc2ad/go.mod h1:vavhavw2zAxS5dIdcRluK6cSGGPlZynqzFM8NdvU144=
github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg=
github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
github.com/hexops/autogold v0.8.1 h1:wvyd/bAJ+Dy+DcE09BoLk6r4Fa5R5W+O+GUzmR985WM=
github.com/hexops/autogold v0.8.1/go.mod h1:97HLDXyG23akzAoRYJh/2OBs3kd80eHyKPvZw0S5ZBY=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hexops/valast v1.5.0 h1:FBTuvVi0wjTngtXJRZXMbkN/Dn6DgsUsBwch2DUJU8Y=
github.com/hexops/valast v1.5.0/go.mod h1:Jcy1pNH7LNraVaAZDLyv21hHg2WBv9Nf9FL6fGxU7o4=
github.com/kkdai/youtube/v2 v2.10.2 h1:e3JslUDiKEfjMzxFyrOh3O59C/aLfKNZyrcav00MZV0=
github.com/kkdai/youtube/v2 v2.10.2/go.mod h1:4y1MIg7f1o5/kQfkr7nwXFtv8PGSoe4kChOB9/iMA88=
github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd h1:lLajTMgNTs/W4H05uQYnJDRIbIvHk6XXy7DQNFRbvzU=
github.com/lrstanley/go-ytdlp v0.0.0-20241221063727-6717edbb36dd/go.mod h1:75ujbafjqiJugIGw4K6o52/p8C0m/kt+DrYwgClXYT4=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
@@ -48,6 +66,8 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tadvi/systray v0.0.0-20190226123456-11a2b8fa57af h1:6yITBqGTE2lEeTPG04SN9W+iWHCRyHqlVYILiSXziwk=
@@ -56,15 +76,25 @@ github.com/vbauerster/mpb/v5 v5.4.0 h1:n8JPunifvQvh6P1D1HAl2Ur9YcmKT1tpoUuiea5ml
github.com/vbauerster/mpb/v5 v5.4.0/go.mod h1:fi4wVo7BVQ22QcvFObm+VwliQXlV1eBT8JDaKXR4JGI=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/mod v0.17.0 h1:zY54UmvipHiNd+pm+m0x9KhZ9hl1/7QNMyxXbc6ICqA=
golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c=
golang.org/x/net v0.31.0 h1:68CPQngjLL0r2AlUKiSxtQFKvzRVbnzLwMUn5SzcLHo=
golang.org/x/net v0.31.0/go.mod h1:P4fl1q7dY2hnZFxEk4pPSkDHF+QqjitcnDjUQyMM+pM=
golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg=
golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk=
gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
mvdan.cc/gofumpt v0.4.0 h1:JVf4NN1mIpHogBj7ABpgOyZc65/UUOkKQFkoURsz4MM=
mvdan.cc/gofumpt v0.4.0/go.mod h1:PljLOHDeZqgS8opHRKLzp2It2VBuSdteAgqUfzMTxlQ=

View File

@@ -3,39 +3,18 @@ package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"io"
"log"
"os"
"os/signal"
"syscall"
"time"
"ytdl/downloaders"
logger "git.site.quack-lab.dev/dave/cylogger"
"github.com/nsqio/go-nsq"
)
var Error *log.Logger
var Warning *log.Logger
func init() {
log.SetFlags(log.Lmicroseconds | log.Lshortfile)
logFile, err := os.Create("ytdl.log")
if err != nil {
log.Printf("Error creating log file: %v", err)
os.Exit(1)
}
logger := io.MultiWriter(os.Stdout, logFile)
log.SetOutput(logger)
Error = log.New(io.MultiWriter(logFile, os.Stderr, os.Stdout),
fmt.Sprintf("%sERROR:%s ", "\033[0;101m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
Warning = log.New(io.MultiWriter(logFile, os.Stdout),
fmt.Sprintf("%sWarning:%s ", "\033[0;93m", "\033[0m"),
log.Lmicroseconds|log.Lshortfile)
}
// var downloader downloaders.Downloader = &downloaders.YTDLPRawDownloader{}
// var downloader downloaders.Downloader = &downloaders.KidaiDownloader{}
var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
@@ -43,14 +22,13 @@ var downloader downloaders.Downloader = &downloaders.YTDLPLibDownloader{}
type DLHandler struct{}
func (*DLHandler) HandleMessage(message *nsq.Message) error {
log.Printf("Received message '%s' with %d attempts", message.Body, message.Attempts)
NotifyDiscordErrorless(fmt.Sprintf("Received message '%s' with %d attempts", message.Body, message.Attempts))
messagelog := logger.Default.WithPrefix(fmt.Sprintf("message=%q", message.Body)).WithPrefix(fmt.Sprintf("attempts=%d", message.Attempts))
messagelog.Info("Received message")
data := DownloadRequest{}
err := json.Unmarshal(message.Body, &data)
if err != nil {
Error.Printf("Error unmarshalling message: %v", err)
NotifyDiscordErrorless(fmt.Sprintf("Error unmarshalling message: %v", err))
messagelog.Error("Error unmarshalling message: %v", err)
return err
}
@@ -58,12 +36,14 @@ func (*DLHandler) HandleMessage(message *nsq.Message) error {
defer cancel()
go func() {
messagelog.Debug("Starting touch ticker")
ticker := time.NewTicker(1 * time.Second)
defer ticker.Stop()
for {
select {
case <-ticker.C:
messagelog.Debug("Touching message")
message.Touch()
case <-ctx.Done():
return
@@ -71,55 +51,63 @@ func (*DLHandler) HandleMessage(message *nsq.Message) error {
}
}()
messagelog.Debug("Downloading %q", data.Link)
err = downloader.Download(data.Link)
if err != nil {
Error.Printf("Error downloading %s: %v", data.Link, err)
NotifyDiscordErrorless(fmt.Sprintf("Error downloading %s: %v", data.Link, err))
messagelog.Error("Error downloading %s: %v", data.Link, err)
return err
}
messagelog.Info("Downloaded %q", data.Link)
message.Finish()
NotifyDiscordErrorless(fmt.Sprintf("Downloaded %s", data.Link))
return nil
}
func main() {
flag.Parse()
logger.InitFlag()
// err := downloader.Download("https://www.youtube.com/watch?v=SiKjprtiPaw")
// if err != nil {
// Error.Printf("Error downloading: %v", err)
// }
// return
logger.Info("Starting downloader")
config := nsq.NewConfig()
config.MaxAttempts = 5
config.MaxInFlight = downloaders.DOWNLOAD_WORKERS
config.MsgTimeout = 10 * time.Second
logger.Info("Creating consumer")
consumer, err := nsq.NewConsumer("ytdqueue", "dl", config)
if err != nil {
Error.Printf("Error creating consumer: %v", err)
logger.Error("Error creating consumer: %v", err)
return
}
logger.Info("Creating handlers")
for i := 0; i < downloaders.DOWNLOAD_WORKERS; i++ {
consumer.AddHandler(&DLHandler{})
}
url := "192.168.1.123:41505"
log.Printf("Connecting to nsq at %s", url)
logger.Info("Connecting to nsq at %s", url)
err = consumer.ConnectToNSQD(url)
if err != nil {
Error.Printf("Error connecting to nsq: %v", err)
logger.Error("Error connecting to nsq: %v", err)
return
}
log.Printf("Connected to nsq at %s", url)
logger.Info("Connected to nsq at %s", url)
sigChan := make(chan os.Signal, 1)
signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)
logger.Info("Waiting for signal to terminate")
<-sigChan
log.Println("Received signal to terminate. Initiating graceful shutdown...")
logger.Info("Received signal to terminate. Initiating graceful shutdown...")
consumer.Stop()
<-consumer.StopChan
log.Println("Graceful shutdown completed.")
logger.Info("Graceful shutdown completed.")
}