Refactor sanitization

downloader/download_downloadr.go

@@ -23,6 +23,11 @@ func DownloadR(url string) error {
 	ongoingDownloadsMutex.Lock()
 	ongoingDownloads[url] = struct{}{}
 	ongoingDownloadsMutex.Unlock()
+	defer func() {
+		ongoingDownloadsMutex.Lock()
+		delete(ongoingDownloads, url)
+		ongoingDownloadsMutex.Unlock()
+	}()
 
 	log.Printf("Downloading %s", url)
 
@@ -47,20 +52,22 @@ func DownloadR(url string) error {
 		Client:    client,
 	}
 
-	videoTitle := video.Title
-	videoAuthor := video.Author
-	videoTitle = strings.ReplaceAll(videoTitle, " ", "-")
-	videoTitle = strings.ReplaceAll(videoTitle, "&", "and")
-	videoTitle = strings.ReplaceAll(videoTitle, "?", "")
-	videoAuthor = strings.ReplaceAll(videoAuthor, " ", "-")
-	videoAuthor = strings.ReplaceAll(videoAuthor, "&", "and")
-	videoAuthor = strings.ReplaceAll(videoAuthor, "?", "")
+	videoTitle := Sanitize(video.Title)
+	videoAuthor := Sanitize(video.Author)
 
 	videoFileRoot := filepath.Join(OUTPUT_DIR, videoAuthor)
 	err = os.MkdirAll(videoFileRoot, 0755)
 	if err != nil {
 		return fmt.Errorf("failed creating directory %s with %+v", videoFileRoot, err)
 	}
+
+	fullVideoPath := filepath.Join(OUTPUT_DIR, videoAuthor, videoTitle+".mp4")
+	_, err = os.Open(fullVideoPath)
+	if err == nil {
+		log.Printf("File %s already exists, skipping download", fullVideoPath)
+		return nil
+	}
+
 	videoFile := filepath.Join(videoAuthor, videoTitle+".mp4")
 	err = downloader.DownloadComposite(context.Background(), videoFile, video, "hd", "mp4", "")
 	if err != nil {
@@ -68,8 +75,14 @@ func DownloadR(url string) error {
 	}
 
 	log.Printf("Downloaded %s", url)
-	ongoingDownloadsMutex.Lock()
-	delete(ongoingDownloads, url)
-	ongoingDownloadsMutex.Unlock()
 	return nil
 }
+
+func Sanitize(s string) string {
+	s = strings.ReplaceAll(s, "&", "and")
+	s = strings.ReplaceAll(s, "?", "")
+	s = strings.ReplaceAll(s, ":", "")
+	s = strings.ReplaceAll(s, ";", "")
+	s = strings.ReplaceAll(s, ",", "")
+	return s
+}

downloader/main.go

@@ -35,7 +35,7 @@ func init() {
 		log.Lmicroseconds|log.Lshortfile)
 }
 
-const DOWNLOAD_WORKERS = 1
+const DOWNLOAD_WORKERS = 2
 
 type DLHandler struct{}
 
@@ -52,7 +52,7 @@ func (*DLHandler) HandleMessage(message *nsq.Message) error {
 	defer cancel()
 
 	go func() {
-		ticker := time.NewTicker(5 * time.Second)
+		ticker := time.NewTicker(1 * time.Second)
 		defer ticker.Stop()
 
 		for {