diff --git a/main.go b/main.go index 6b70e26..52b30d7 100644 --- a/main.go +++ b/main.go @@ -1,3 +1,4 @@ +// Package cyutils provides utility functions for concurrent processing, HTTP requests with caching, and rate limiting. package cyutils import ( @@ -12,6 +13,20 @@ import ( "golang.org/x/time/rate" ) +// WithWorkers processes items concurrently using a specified number of worker goroutines. +// It accepts either a slice or map of items and applies the provided function to each item. +// The function receives the worker ID, key, and value for each item. +// +// Parameters: +// - workers: Number of concurrent workers to spawn +// - items: Collection of items to process (slice or map) +// - fn: Function to apply to each item, receives (workerID, key, value) +// +// Example: +// +// WithWorkers(3, []string{"a", "b", "c"}, func(worker, key, value string) { +// fmt.Printf("Worker %d processing %s\n", worker, value) +// }) func WithWorkers[K comparable, V any, C []V | map[K]V]( workers int, items C, @@ -53,6 +68,19 @@ func WithWorkers[K comparable, V any, C []V | map[K]V]( wg.Wait() } +// Batched processes an array in batches of the specified size. +// The provided function is called for each batch with the batch slice. +// +// Parameters: +// - arr: Array to process in batches +// - batchSize: Maximum number of items per batch +// - fn: Function to call for each batch +// +// Example: +// +// Batched([]int{1, 2, 3, 4, 5}, 2, func(batch []int) { +// fmt.Printf("Processing batch: %v\n", batch) +// }) func Batched[T any](arr []T, batchSize int, fn func(batch []T)) { for i := 0; i < len(arr); i += batchSize { start := i @@ -62,12 +90,16 @@ func Batched[T any](arr []T, batchSize int, fn func(batch []T)) { } } +// RateLimitedTransport wraps an HTTP transport with rate limiting capabilities. +// It enforces request rate limits before executing HTTP requests. type RateLimitedTransport struct { base http.RoundTripper // Underlying transport limiter *rate.Limiter // Rate limiter } -// RoundTrip enforces rate limiting before executing the request +// RoundTrip enforces rate limiting before executing the HTTP request. +// It waits for rate limiter permission before delegating to the underlying transport. +// Returns an error if the context is cancelled or times out while waiting. func (t *RateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, error) { ctx := req.Context() if err := t.limiter.Wait(ctx); err != nil { @@ -75,7 +107,16 @@ func (t *RateLimitedTransport) RoundTrip(req *http.Request) (*http.Response, err } return t.base.RoundTrip(req) } -func LimitedHttp(rps float64, burst int) *http.Client { + +// LimitedHTTP creates an HTTP client with rate limiting capabilities. +// The client will respect the specified requests per second (rps) and burst limits. +// +// Parameters: +// - rps: Maximum requests per second +// - burst: Maximum burst capacity for rate limiting +// +// Returns an HTTP client configured with rate limiting and connection pooling. +func LimitedHTTP(rps float64, burst int) *http.Client { baseTransport := &http.Transport{ MaxIdleConns: 100, MaxIdleConnsPerHost: 10, @@ -98,10 +139,30 @@ func LimitedHttp(rps float64, burst int) *http.Client { var client = http.DefaultClient +// WithClient sets the default HTTP client used by caching functions. +// This allows customization of the HTTP client for all subsequent cached requests. +// +// Parameters: +// - nclient: The HTTP client to use for future requests func WithClient(nclient *http.Client) { client = nclient } +// RequestCached performs an HTTP request with file-based caching. +// If the cache file exists, it returns the cached data. Otherwise, it makes the HTTP request, +// caches the response, and returns the result. For non-[]byte types, it attempts JSON unmarshaling. +// +// Parameters: +// - req: HTTP request to execute +// - filename: Path to cache file +// +// Returns the cached or fetched data of type T, or an error if the request fails. +// +// Example: +// +// var result MyStruct +// req, _ := http.NewRequest("GET", "https://api.example.com/data", nil) +// err := RequestCached(req, "cache/data.json", &result) func RequestCached[T any](req *http.Request, filename string) (T, error) { var zero T @@ -145,6 +206,14 @@ func RequestCached[T any](req *http.Request, filename string) (T, error) { return out, nil } +// RequestCachedBytes is a convenience function for RequestCached that returns raw bytes. +// It's equivalent to calling RequestCached[[]byte] but provides a cleaner API for byte data. +// +// Parameters: +// - req: HTTP request to execute +// - filename: Path to cache file +// +// Returns the cached or fetched data as []byte, or an error if the request fails. func RequestCachedBytes(req *http.Request, filename string) ([]byte, error) { return RequestCached[[]byte](req, filename) }