Compare commits
4 Commits
Author | SHA1 | Date | |
---|---|---|---|
b2c5d8f41c | |||
4d2c1622d3 | |||
82d1c5a8c8 | |||
c07fb20a8a |
2
go.mod
2
go.mod
@@ -1,3 +1,5 @@
|
|||||||
module git.site.quack-lab.dev/dave/cyutils
|
module git.site.quack-lab.dev/dave/cyutils
|
||||||
|
|
||||||
go 1.23.6
|
go 1.23.6
|
||||||
|
|
||||||
|
require golang.org/x/time v0.12.0
|
||||||
|
2
go.sum
Normal file
2
go.sum
Normal file
@@ -0,0 +1,2 @@
|
|||||||
|
golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
|
||||||
|
golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
|
37
main.go
37
main.go
@@ -1,7 +1,10 @@
|
|||||||
package cyutils
|
package cyutils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net/http"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
|
"golang.org/x/time/rate"
|
||||||
)
|
)
|
||||||
|
|
||||||
func WithWorkers[T any](workers int, arr []T, fn func(worker int, item T)) {
|
func WithWorkers[T any](workers int, arr []T, fn func(worker int, item T)) {
|
||||||
@@ -40,3 +43,37 @@ func Batched[T any](arr []T, batchSize int, fn func(batch []T)) {
|
|||||||
fn(batch)
|
fn(batch)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type RateLimitedTransport struct {
|
||||||
|
base http.RoundTripper // Underlying transport
|
||||||
|
limiter *rate.Limiter // Rate limiter
|
||||||
|
}
|
||||||
|
|
||||||
|
// RoundTrip enforces rate limiting before executing the request
|
||||||
|
func (t *RateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||||
|
ctx := req.Context()
|
||||||
|
if err := t.limiter.Wait(ctx); err != nil {
|
||||||
|
return nil, err // Handle context cancellation/timeout
|
||||||
|
}
|
||||||
|
return t.base.RoundTrip(req)
|
||||||
|
}
|
||||||
|
func LimitedHttp(rps float64, burst int) *http.Client {
|
||||||
|
baseTransport := &http.Transport{
|
||||||
|
MaxIdleConns: 100,
|
||||||
|
MaxIdleConnsPerHost: 10,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Initialize rate limiter
|
||||||
|
limiter := rate.NewLimiter(rate.Limit(rps), burst)
|
||||||
|
|
||||||
|
// Wrap transport with rate limiting
|
||||||
|
transport := &RateLimitedTransport{
|
||||||
|
base: baseTransport,
|
||||||
|
limiter: limiter,
|
||||||
|
}
|
||||||
|
|
||||||
|
// Return configured client
|
||||||
|
return &http.Client{
|
||||||
|
Transport: transport,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Reference in New Issue
Block a user