41 lines
832 B
Go
41 lines
832 B
Go
package main
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
var LIMITER = rate.NewLimiter(rate.Limit(32), 32)
|
|
|
|
func Fetch(url string) (string, error) {
|
|
err := LIMITER.Wait(context.Background())
|
|
if err != nil {
|
|
return "", fmt.Errorf("Error waiting for rate limiter: %v", err)
|
|
}
|
|
|
|
res, err := http.Get(url)
|
|
if err != nil {
|
|
return "", fmt.Errorf("Error fetching %s: %v", url, err)
|
|
}
|
|
defer res.Body.Close()
|
|
|
|
if res.StatusCode != http.StatusOK {
|
|
return "", fmt.Errorf("Error fetching %s, returned code %d: %v", url, res.StatusCode, err)
|
|
}
|
|
|
|
body, err := io.ReadAll(res.Body)
|
|
if err != nil {
|
|
return "", fmt.Errorf("Error reading body of %s: %v", url, err)
|
|
}
|
|
|
|
return string(body), nil
|
|
}
|
|
|
|
func FetchDocs(url string) (string, error) {
|
|
return "", nil
|
|
}
|