More faster

html-getter.go

@@ -9,7 +9,7 @@ import (
 	"golang.org/x/time/rate"
 )
 
-var LIMITER = rate.NewLimiter(rate.Limit(8), 16)
+var LIMITER = rate.NewLimiter(rate.Limit(32), 32)
 
 func Fetch(url string) (string, error) {
 	err := LIMITER.Wait(context.Background())

main.go

@@ -48,7 +48,7 @@ func main() {
 		Error.Printf("Error fetching %s: %v", apiUrl, err)
 		return
 	}
-
+	
 	//os.WriteFile("test.html", []byte(res), 0644)
 	pages, err := ParseHTML(res)
 	if err != nil {
@@ -57,27 +57,33 @@ func main() {
 	}
 	for _, page := range pages {
 		log.Printf("Processing page %s", page)
-		pname := pageNameExtractor.FindStringSubmatch(page)
+		err := ParseDocFull(page)
-		if len(pname) != 2 {
-			Error.Printf("Failed to extract page name from %s", page)
-			continue
-		}
-
-		res, err := Fetch(rootUrl + page)
 		if err != nil {
-			Error.Printf("Error fetching %s: %v", rootUrl+page, err)
+			Error.Printf("Error parsing doc: %v", err)
-		}
+			return
-		
-		function, err := ParseDoc(res)
-		if err != nil {
-			Error.Printf("Error parsing HTML: %v", err)
-			continue
-		}
-		function.Name = pname[1]
-		err = function.WriteFile(outDir)
-		if err != nil {
-			Error.Printf("Error writing file: %v", err)
-			continue
 		}
 	}
 }
+
+func ParseDocFull(url string) error {
+	pname := pageNameExtractor.FindStringSubmatch(url)
+	if len(pname) != 2 {
+		return fmt.Errorf("Failed to extract page name from %s", url)
+	}
+
+	res, err := Fetch(rootUrl + url)
+	if err != nil {
+		return fmt.Errorf("Error fetching %s: %v", rootUrl+url, err)
+	}
+
+	function, err := ParseDoc(res)
+	if err != nil {
+		return fmt.Errorf("Error parsing HTML for %s: %v", url, err)
+	}
+	function.Name = pname[1]
+	err = function.WriteFile(outDir)
+	if err != nil {
+		return fmt.Errorf("Error writing file for %s: %v", url, err)
+	}
+	return nil
+}