Initial commit

html-getter.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+
+	"golang.org/x/time/rate"
+)
+
+var LIMITER = rate.NewLimiter(rate.Limit(1), 2)
+
+func FetchFull(url string) (string, error) {
+	res, err := http.Get(url)
+	if err != nil {
+		return "", fmt.Errorf("Error fetching %s: %v", url, err)
+	}
+	defer res.Body.Close()
+
+	if res.StatusCode != http.StatusOK {
+		return "", fmt.Errorf("Error fetching %s, returned code %d: %v", url, res.StatusCode, err)
+	}
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return "", fmt.Errorf("Error reading body of %s: %v", url, err)
+	}
+
+	return string(body), nil
+}