151 lines
3.2 KiB
Go
151 lines
3.2 KiB
Go
package cyutils
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io"
|
|
"net/http"
|
|
"os"
|
|
"path/filepath"
|
|
"sync"
|
|
|
|
"golang.org/x/time/rate"
|
|
)
|
|
|
|
func WithWorkers[K comparable, V any, C []V | map[K]V](
|
|
workers int,
|
|
items C,
|
|
fn func(worker int, key K, value V),
|
|
) {
|
|
type KV struct {
|
|
// A bit of type weaving...
|
|
Key any
|
|
Value V
|
|
}
|
|
|
|
n := len(items)
|
|
ingestChan := make(chan KV, n)
|
|
|
|
go func() {
|
|
switch v := any(items).(type) {
|
|
case []V:
|
|
for i, it := range v {
|
|
ingestChan <- KV{Key: i, Value: it}
|
|
}
|
|
case map[K]V:
|
|
for key, it := range v {
|
|
ingestChan <- KV{Key: key, Value: it}
|
|
}
|
|
}
|
|
close(ingestChan)
|
|
}()
|
|
|
|
var wg sync.WaitGroup
|
|
wg.Add(workers)
|
|
for i := 0; i < workers; i++ {
|
|
go func(worker int) {
|
|
defer wg.Done()
|
|
for v := range ingestChan {
|
|
fn(worker, v.Key.(K), v.Value)
|
|
}
|
|
}(i)
|
|
}
|
|
wg.Wait()
|
|
}
|
|
|
|
func Batched[T any](arr []T, batchSize int, fn func(batch []T)) {
|
|
for i := 0; i < len(arr); i += batchSize {
|
|
start := i
|
|
end := min(i+batchSize, len(arr))
|
|
batch := arr[start:end]
|
|
fn(batch)
|
|
}
|
|
}
|
|
|
|
type RateLimitedTransport struct {
|
|
base http.RoundTripper // Underlying transport
|
|
limiter *rate.Limiter // Rate limiter
|
|
}
|
|
|
|
// RoundTrip enforces rate limiting before executing the request
|
|
func (t *RateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
|
ctx := req.Context()
|
|
if err := t.limiter.Wait(ctx); err != nil {
|
|
return nil, err // Handle context cancellation/timeout
|
|
}
|
|
return t.base.RoundTrip(req)
|
|
}
|
|
func LimitedHttp(rps float64, burst int) *http.Client {
|
|
baseTransport := &http.Transport{
|
|
MaxIdleConns: 100,
|
|
MaxIdleConnsPerHost: 10,
|
|
}
|
|
|
|
// Initialize rate limiter
|
|
limiter := rate.NewLimiter(rate.Limit(rps), burst)
|
|
|
|
// Wrap transport with rate limiting
|
|
transport := &RateLimitedTransport{
|
|
base: baseTransport,
|
|
limiter: limiter,
|
|
}
|
|
|
|
// Return configured client
|
|
return &http.Client{
|
|
Transport: transport,
|
|
}
|
|
}
|
|
|
|
var client = http.DefaultClient
|
|
|
|
func WithClient(nclient *http.Client) {
|
|
client = nclient
|
|
}
|
|
|
|
func RequestCached[T any](req *http.Request, filename string) (T, error) {
|
|
var zero T
|
|
|
|
data, err := os.ReadFile(filename)
|
|
if err != nil {
|
|
if !os.IsNotExist(err) {
|
|
return zero, fmt.Errorf("failed to read cache: %w", err)
|
|
}
|
|
|
|
resp, httpErr := client.Do(req)
|
|
if httpErr != nil {
|
|
return zero, fmt.Errorf("HTTP request failed: %w", httpErr)
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, readErr := io.ReadAll(resp.Body)
|
|
if readErr != nil {
|
|
return zero, fmt.Errorf("failed to read response body: %w", readErr)
|
|
}
|
|
|
|
dir := filepath.Dir(filename)
|
|
if dir != "." && dir != "" {
|
|
if mkErr := os.MkdirAll(dir, 0o755); mkErr != nil {
|
|
return zero, fmt.Errorf("failed to create cache dir %s: %w", dir, mkErr)
|
|
}
|
|
}
|
|
if writeErr := os.WriteFile(filename, body, 0o644); writeErr != nil {
|
|
return zero, fmt.Errorf("failed to write cache %s: %w", filename, writeErr)
|
|
}
|
|
data = body
|
|
}
|
|
|
|
if _, ok := any(zero).([]byte); ok {
|
|
return any(data).(T), nil
|
|
}
|
|
|
|
var out T
|
|
if err := json.Unmarshal(data, &out); err != nil {
|
|
return zero, fmt.Errorf("failed to unmarshal cached response: %w", err)
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
func RequestCachedBytes(req *http.Request, filename string) ([]byte, error) {
|
|
return RequestCached[[]byte](req, filename)
|
|
}
|