httpx-hedged 0.2.0__py3-none-any.whl

httpx_hedged/_stats.py

@@ -0,0 +1,160 @@
+"""Thread-safe, per-key statistics for hedge operations."""
+
+from __future__ import annotations
+
+import threading
+from dataclasses import dataclass
+
+from httpx_hedged._bounded import BoundedRegistry
+
+
+class Stats:
+    """Thread-safe counters for hedge operations on a single key.
+
+    All fields use a lock for atomic updates and are safe to read concurrently.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self.total_requests: int = 0
+        self.hedged_requests: int = 0
+        self.hedge_wins: int = 0
+        self.primary_wins: int = 0
+        self.budget_exhausted: int = 0
+        self.warmup_requests: int = 0
+        self.circuit_blocked: int = 0
+        self.errors: int = 0
+
+    def increment_total(self) -> None:
+        with self._lock:
+            self.total_requests += 1
+
+    def increment_hedged(self) -> None:
+        with self._lock:
+            self.hedged_requests += 1
+
+    def increment_hedge_wins(self) -> None:
+        with self._lock:
+            self.hedge_wins += 1
+
+    def increment_primary_wins(self) -> None:
+        with self._lock:
+            self.primary_wins += 1
+
+    def increment_budget_exhausted(self) -> None:
+        with self._lock:
+            self.budget_exhausted += 1
+
+    def increment_warmup(self) -> None:
+        with self._lock:
+            self.warmup_requests += 1
+
+    def increment_circuit_blocked(self) -> None:
+        with self._lock:
+            self.circuit_blocked += 1
+
+    def increment_errors(self) -> None:
+        with self._lock:
+            self.errors += 1
+
+    def snapshot(self, key: str = "") -> StatsSnapshot:
+        """Take a consistent point-in-time copy of all counters."""
+        with self._lock:
+            return StatsSnapshot(
+                key=key,
+                total_requests=self.total_requests,
+                hedged_requests=self.hedged_requests,
+                hedge_wins=self.hedge_wins,
+                primary_wins=self.primary_wins,
+                budget_exhausted=self.budget_exhausted,
+                warmup_requests=self.warmup_requests,
+                circuit_blocked=self.circuit_blocked,
+                errors=self.errors,
+            )
+
+    def hedge_rate(self) -> float:
+        """Return hedged_requests / total_requests, or 0.0 if no requests."""
+        with self._lock:
+            if self.total_requests == 0:
+                return 0.0
+            return self.hedged_requests / self.total_requests
+
+
+@dataclass(frozen=True, slots=True)
+class StatsSnapshot:
+    """Immutable point-in-time snapshot of Stats for a single key."""
+
+    key: str
+    total_requests: int
+    hedged_requests: int
+    hedge_wins: int
+    primary_wins: int
+    budget_exhausted: int
+    warmup_requests: int
+    circuit_blocked: int
+    errors: int
+
+    def __add__(self, other: StatsSnapshot) -> StatsSnapshot:
+        return StatsSnapshot(
+            key="*",
+            total_requests=self.total_requests + other.total_requests,
+            hedged_requests=self.hedged_requests + other.hedged_requests,
+            hedge_wins=self.hedge_wins + other.hedge_wins,
+            primary_wins=self.primary_wins + other.primary_wins,
+            budget_exhausted=self.budget_exhausted + other.budget_exhausted,
+            warmup_requests=self.warmup_requests + other.warmup_requests,
+            circuit_blocked=self.circuit_blocked + other.circuit_blocked,
+            errors=self.errors + other.errors,
+        )
+
+
+_EMPTY_SNAPSHOT = StatsSnapshot(
+    key="*",
+    total_requests=0,
+    hedged_requests=0,
+    hedge_wins=0,
+    primary_wins=0,
+    budget_exhausted=0,
+    warmup_requests=0,
+    circuit_blocked=0,
+    errors=0,
+)
+
+
+class StatsRegistry:
+    """Owns one ``Stats`` object per tracked key plus a global aggregate view.
+
+    hedge-python only ever exposes a single global ``Stats`` object; since
+    this library tracks hedge behavior per endpoint, per-key breakdown is
+    the point of observability here.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._stats: BoundedRegistry[Stats] = BoundedRegistry()
+
+    def for_key(self, key: str) -> Stats:
+        """Get or create the ``Stats`` object for a key."""
+        with self._lock:
+            return self._stats.get_or_create(key, Stats)
+
+    def snapshot(self, key: str) -> StatsSnapshot | None:
+        """Return a snapshot for a key, or None if the key has no recorded stats."""
+        with self._lock:
+            stats = self._stats.get(key)
+        if stats is None:
+            return None
+        return stats.snapshot(key)
+
+    def all_snapshots(self) -> dict[str, StatsSnapshot]:
+        """Return a snapshot for every tracked key."""
+        with self._lock:
+            items = list(self._stats.items())
+        return {key: stats.snapshot(key) for key, stats in items}
+
+    def global_snapshot(self) -> StatsSnapshot:
+        """Return the sum of every tracked key's stats."""
+        total = _EMPTY_SNAPSHOT
+        for snapshot in self.all_snapshots().values():
+            total = total + snapshot
+        return total

httpx_hedged/budget/__init__.py

@@ -0,0 +1,7 @@
+"""Hedge-rate budget primitives."""
+
+from __future__ import annotations
+
+from httpx_hedged.budget._token_bucket import TokenBucket
+
+__all__ = ["TokenBucket"]

httpx_hedged/budget/_token_bucket.py

@@ -0,0 +1,54 @@
+"""Token bucket algorithm for controlling hedge request rate."""
+
+from __future__ import annotations
+
+import threading
+import time
+
+
+class TokenBucket:
+    """Controls hedge request rate using a token bucket algorithm.
+
+    The bucket refills at ``estimated_rps * budget_percent / 100`` tokens per
+    second. During genuine outages the bucket drains and hedging stops,
+    preventing the load-doubling spiral that would deepen the incident.
+
+    Args:
+        budget_percent: Max hedge rate as percent of total traffic.
+        estimated_rps: Expected requests per second.
+    """
+
+    def __init__(
+        self, budget_percent: float = 10.0, estimated_rps: float = 100.0
+    ) -> None:
+        self._lock = threading.Lock()
+        self._budget_percent = budget_percent
+        self._rate = estimated_rps * (budget_percent / 100.0)
+        self._max_burst = max(self._rate * 2, 1.0)
+        self._tokens = self._max_burst
+        self._last_refill = time.monotonic()
+
+    def try_acquire(self) -> bool:
+        """Return True if a hedge token is available, False if over budget."""
+        with self._lock:
+            now = time.monotonic()
+            elapsed = now - self._last_refill
+            self._last_refill = now
+
+            self._tokens += elapsed * self._rate
+            if self._tokens > self._max_burst:
+                self._tokens = self._max_burst
+
+            if self._tokens < 1.0:
+                return False
+            self._tokens -= 1.0
+            return True
+
+    def set_rps(self, rps: float) -> None:
+        """Update the hedge rate as traffic changes, preserving the budget %."""
+        with self._lock:
+            self._rate = rps * (self._budget_percent / 100.0)
+            max_burst = max(self._rate * 2, 1.0)
+            self._max_burst = max_burst
+            if self._tokens > self._max_burst:
+                self._tokens = self._max_burst

httpx_hedged/sketch/__init__.py

@@ -0,0 +1,8 @@
+"""Quantile-sketch primitives used to estimate per-key latency percentiles."""
+
+from __future__ import annotations
+
+from httpx_hedged.sketch._ddsketch import DDSketch
+from httpx_hedged.sketch._windowed import WindowedSketch
+
+__all__ = ["DDSketch", "WindowedSketch"]

httpx_hedged/sketch/_ddsketch.py

@@ -0,0 +1,188 @@
+"""DDSketch streaming quantile estimator.
+
+Based on Masson et al., "DDSketch: A fast and fully-mergeable quantile sketch
+with relative-error guarantees", VLDB 2019. Adapted from hedge-python
+(https://github.com/sunhailin-Leo/hedge-python).
+"""
+
+from __future__ import annotations
+
+import math
+from collections import defaultdict
+
+
+class _LogMapping:
+    """Maps values to bucket indices using logarithmic scaling.
+
+    For a given relative accuracy alpha, gamma = (1+alpha)/(1-alpha).
+    A positive value x maps to bucket index ceil(ln(x) / ln(gamma)).
+
+    The guarantee: any value in bucket i is within a factor of gamma^0.5 of
+    the bucket's representative value, giving relative error <= alpha.
+    """
+
+    __slots__ = ("gamma", "multiplier")
+
+    def __init__(self, relative_accuracy: float) -> None:
+        self.gamma = (1 + relative_accuracy) / (1 - relative_accuracy)
+        self.multiplier = 1.0 / math.log(self.gamma)
+
+    def index(self, value: float) -> int:
+        """Return the bucket index for a strictly positive value."""
+        return math.ceil(math.log(value) * self.multiplier)
+
+    def value(self, index: int) -> float:
+        """Return the representative value (geometric midpoint) for a bucket."""
+        return math.exp((index - 0.5) / self.multiplier)
+
+
+class _Store:
+    """Sparse map of bucket indices to cumulative counts with lazy sorted key cache."""
+
+    __slots__ = ("_sorted_keys_cache", "bins", "count")
+
+    def __init__(self) -> None:
+        self.bins: dict[int, float] = defaultdict(float)
+        self.count: float = 0.0
+        self._sorted_keys_cache: list[int] | None = None
+
+    @property
+    def sorted_keys(self) -> list[int]:
+        """Return sorted bin keys, building from cache when possible."""
+        if self._sorted_keys_cache is None:
+            self._sorted_keys_cache = sorted(self.bins.keys())
+        return self._sorted_keys_cache
+
+    def add(self, index: int) -> None:
+        if index not in self.bins:
+            self._sorted_keys_cache = None
+        self.bins[index] += 1.0
+        self.count += 1.0
+
+    def merge(self, other: _Store) -> None:
+        for idx, cnt in other.bins.items():
+            self.bins[idx] += cnt
+        self.count += other.count
+        self._sorted_keys_cache = None
+
+    def reset(self) -> None:
+        self.bins = defaultdict(float)
+        self.count = 0.0
+        self._sorted_keys_cache = None
+
+
+class DDSketch:
+    """Streaming quantile sketch with relative-error guarantees.
+
+    Positive and negative values are stored in separate sparse bucket maps.
+    Zero values are counted separately. Min and max are tracked exactly.
+
+    Property: for any quantile q, the returned estimate satisfies
+        |estimate - true_value| / |true_value| <= relative_accuracy
+
+    Args:
+        relative_accuracy: Target relative accuracy. 0.01 means estimates
+            are within +/-1% of the true value. Must be in (0, 1).
+    """
+
+    def __init__(self, relative_accuracy: float = 0.01) -> None:
+        if not (0 < relative_accuracy < 1):
+            raise ValueError("relative_accuracy must be in (0, 1)")
+        self._mapping = _LogMapping(relative_accuracy)
+        self._positive = _Store()
+        self._negative = _Store()
+        self._zero_count: float = 0.0
+        self._count: int = 0
+        self._min: float = math.inf
+        self._max: float = -math.inf
+
+    @property
+    def count(self) -> int:
+        """Total number of values added."""
+        return self._count
+
+    def add(self, value: float) -> None:
+        """Record a single value. O(1) per insert.
+
+        NaN and infinite values are silently ignored.
+        """
+        if math.isnan(value) or math.isinf(value):
+            return
+
+        if value > 0:
+            self._positive.add(self._mapping.index(value))
+        elif value < 0:
+            self._negative.add(self._mapping.index(-value))
+        else:
+            self._zero_count += 1.0
+
+        self._count += 1
+        if value < self._min:
+            self._min = value
+        if value > self._max:
+            self._max = value
+
+    def quantile(self, q: float) -> float:
+        """Return the estimated value at quantile q in [0, 1].
+
+        Returns math.nan if the sketch is empty.
+
+        The estimate satisfies the relative-error guarantee:
+            |estimate - true_value| / |true_value| <= relative_accuracy
+        """
+        if self._count == 0:
+            return math.nan
+        if q <= 0:
+            return self._min
+        if q >= 1:
+            return self._max
+
+        rank: float = float(math.ceil(q * self._count))
+
+        # Negative values: iterate descending (most negative -> least negative)
+        if self._negative.count > 0:
+            cumulative = 0.0
+            neg_bins = self._negative.bins
+            for idx in reversed(self._negative.sorted_keys):
+                cumulative += neg_bins[idx]
+                if cumulative >= rank:
+                    return -self._mapping.value(idx)
+            rank -= self._negative.count
+
+        # Zero values
+        if self._zero_count > 0:
+            rank -= self._zero_count
+            if rank <= 0:
+                return 0.0
+
+        # Positive values: iterate ascending (least positive -> most positive)
+        if self._positive.count > 0:
+            cumulative = 0.0
+            pos_bins = self._positive.bins
+            for idx in self._positive.sorted_keys:
+                cumulative += pos_bins[idx]
+                if cumulative >= rank:
+                    return self._mapping.value(idx)
+
+        return self._max
+
+    def merge(self, other: DDSketch) -> None:
+        """Combine other into self. The merge is exact: no error accumulates."""
+        self._positive.merge(other._positive)
+        self._negative.merge(other._negative)
+        self._zero_count += other._zero_count
+        self._count += other._count
+        if other._count > 0:
+            if other._min < self._min:
+                self._min = other._min
+            if other._max > self._max:
+                self._max = other._max
+
+    def reset(self) -> None:
+        """Clear all state. Used for windowed / tumbling-window decay."""
+        self._positive.reset()
+        self._negative.reset()
+        self._zero_count = 0.0
+        self._count = 0
+        self._min = math.inf
+        self._max = -math.inf

httpx_hedged/sketch/_windowed.py

@@ -0,0 +1,92 @@
+"""WindowedSketch: sliding-window quantile estimation over DDSketch pairs."""
+
+from __future__ import annotations
+
+import math
+import threading
+import time
+
+from httpx_hedged._rotation import RotateAction, next_action
+from httpx_hedged.sketch._ddsketch import DDSketch
+
+_DEFAULT_WINDOW_DURATION = 30.0  # seconds
+
+
+class WindowedSketch:
+    """Maintains a sliding window over two DDSketches that rotate lazily.
+
+    Quantile queries merge both sketches, giving a window that spans 1x to
+    2x the configured duration. ``add`` always writes to the current
+    sketch. Rotation is decided lazily on each call rather than by a
+    background thread/task -- see ``httpx_hedged._rotation`` -- since a
+    service can have many independently-tracked endpoints, and spinning one
+    rotation task per endpoint does not scale.
+
+    The rotation scheme::
+
+        t=0:  current=A, previous=empty
+        t=30: current=B, previous=A       (A covers [0,30))
+        t=60: current=C, previous=B       (A is dropped)
+        t=90+ (idle):                      hard reset, both sketches emptied
+
+    This class is thread-safe.
+
+    Args:
+        relative_accuracy: DDSketch relative accuracy (default: 0.01).
+        window_duration: Rotation interval in seconds (default: 30.0).
+    """
+
+    def __init__(
+        self,
+        relative_accuracy: float = 0.01,
+        window_duration: float = _DEFAULT_WINDOW_DURATION,
+    ) -> None:
+        if window_duration <= 0:
+            window_duration = _DEFAULT_WINDOW_DURATION
+        self._relative_accuracy = relative_accuracy
+        self._window_duration = window_duration
+        self._lock = threading.Lock()
+        self._current = DDSketch(relative_accuracy)
+        self._previous = DDSketch(relative_accuracy)
+        self._window_start = time.monotonic()
+
+    def _maybe_rotate_locked(self) -> None:
+        """Rotate or reset if enough time has passed. Caller must hold the lock."""
+        now = time.monotonic()
+        action = next_action(self._window_start, self._window_duration, now)
+        if action is RotateAction.NONE:
+            return
+        if action is RotateAction.ROTATE:
+            self._previous = self._current
+            self._current = DDSketch(self._relative_accuracy)
+        else:  # RESET
+            self._previous = DDSketch(self._relative_accuracy)
+            self._current = DDSketch(self._relative_accuracy)
+        self._window_start = now
+
+    def add(self, value: float) -> None:
+        """Record a latency sample (in seconds) to the current sketch."""
+        with self._lock:
+            self._maybe_rotate_locked()
+            self._current.add(value)
+
+    def quantile(self, q: float) -> float:
+        """Return the estimated quantile q in [0, 1] over the sliding window.
+
+        Returns math.nan if no data has been recorded.
+        """
+        with self._lock:
+            self._maybe_rotate_locked()
+            if self._current.count == 0 and self._previous.count == 0:
+                return math.nan
+            merged = DDSketch(self._relative_accuracy)
+            merged.merge(self._previous)
+            merged.merge(self._current)
+            return merged.quantile(q)
+
+    def rotate(self) -> None:
+        """Force an immediate rotation. Mostly useful for testing."""
+        with self._lock:
+            self._previous = self._current
+            self._current = DDSketch(self._relative_accuracy)
+            self._window_start = time.monotonic()

httpx_hedged/transport.py

@@ -0,0 +1,176 @@
+"""Hedged transport for httpx.AsyncClient.
+
+Usage::
+
+    import httpx
+    from httpx_hedged import EndpointConfig, HedgeConfig, HedgedTransport
+
+    transport = HedgedTransport()
+    transport.register("GET", "/api/v1/fast-lookup", EndpointConfig(percentile=0.90))
+    transport.register("GET", "/api/v1/bulk-export", EndpointConfig(percentile=0.90))
+
+    async with httpx.AsyncClient(transport=transport) as client:
+        resp = await client.get("https://api.example.com/api/v1/fast-lookup")
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+
+import httpx
+
+from httpx_hedged._config import EndpointConfig, HedgeConfig, resolve
+from httpx_hedged._health import HealthRegistry
+from httpx_hedged._matcher import EndpointMatcher, Route
+from httpx_hedged._scheduler import HedgeScheduler, extract_host
+from httpx_hedged._stats import StatsRegistry
+
+_IDEMPOTENT_METHODS = ("GET", "HEAD", "OPTIONS")
+
+
+def _has_body(request: httpx.Request) -> bool:
+    """Whether a request carries a body that a hedge can't safely re-send.
+
+    The primary and hedge both send the same ``httpx.Request`` object; a
+    body backed by a one-shot async stream (e.g. ``content=some_generator``)
+    would be consumed by whichever of the two reads it first, corrupting or
+    failing the other. Idempotent methods essentially never carry a body in
+    normal use, so this only ever changes behavior for that edge case.
+    """
+    content_length = request.headers.get("content-length")
+    if content_length not in (None, "0"):
+        return True
+    return request.headers.get("transfer-encoding", "").lower() == "chunked"
+
+
+class HedgedTransport(httpx.AsyncBaseTransport):
+    """An httpx async transport that adds adaptive, per-endpoint hedged requests.
+
+    Wraps a single inner transport (default: ``httpx.AsyncHTTPTransport``,
+    one connection pool) and races a backup request when the primary
+    exceeds its estimated latency percentile -- or a hardcoded delay, for
+    endpoints registered with ``EndpointConfig(hedge_delay=...)``.
+
+    Endpoints are identified by registering method + path patterns via
+    ``register()``; a request can also be tagged directly with
+    ``extensions={"hedge_endpoint": "name"}`` to bypass pattern matching.
+    Unmatched requests fall back to a default config scoped per host.
+
+    A circuit breaker independently tracks host-level and endpoint-level
+    health; when either trips, hedging is suppressed for that scope (the
+    primary request is always still sent and its result/exception is always
+    returned normally).
+
+    Args:
+        inner: The underlying transport to wrap. Defaults to a new
+            ``httpx.AsyncHTTPTransport()``.
+        default_config: Hedge configuration used for any request that
+            doesn't match a registered endpoint. Defaults to ``HedgeConfig()``.
+        routes: Endpoints to register up front (equivalent to calling
+            ``register()`` for each one after construction).
+        on_hedge_fired: Called with the key each time a hedge request is
+            actually launched. Intended for metrics -- see the README's
+            observability section for an example.
+        on_circuit_open: Called as ``on_circuit_open(scope, key)`` each
+            time a host- or endpoint-scoped circuit breaker trips open
+            (``scope`` is ``"host"`` or ``"endpoint"``). Intended for
+            alerting -- see the README's observability section for an
+            example.
+    """
+
+    def __init__(
+        self,
+        inner: httpx.AsyncBaseTransport | None = None,
+        default_config: HedgeConfig | None = None,
+        routes: list[Route] | None = None,
+        on_hedge_fired: Callable[[str], None] | None = None,
+        on_circuit_open: Callable[[str, str], None] | None = None,
+    ) -> None:
+        self._inner = inner or httpx.AsyncHTTPTransport()
+        self._default_config = default_config or HedgeConfig()
+        self._matcher = EndpointMatcher()
+        self._stats = StatsRegistry()
+        self._health = HealthRegistry(on_circuit_open=on_circuit_open)
+        self._scheduler = HedgeScheduler(
+            self._health,
+            self._stats,
+            self._default_config.circuit_breaker,
+            on_hedge_fired=on_hedge_fired,
+        )
+
+        for route in routes or []:
+            self.register(
+                route.method, route.path_pattern, route.config, name=route.name
+            )
+
+    def register(
+        self,
+        method: str,
+        path_pattern: str,
+        config: EndpointConfig,
+        *,
+        name: str | None = None,
+    ) -> str:
+        """Register a per-endpoint hedge config for a method + path pattern.
+
+        ``path_pattern`` segments may contain ``{name}`` placeholders or a
+        bare ``*`` to match any single path segment (e.g.
+        ``/api/v1/users/{id}``). Routes are matched in registration order,
+        first match wins -- register more specific patterns first.
+
+        Returns the resolved endpoint name (used as the key in ``stats``
+        and as the value for ``extensions={"hedge_endpoint": name}``).
+        """
+        return self._matcher.register(method, path_pattern, config, name=name)
+
+    @property
+    def stats(self) -> StatsRegistry:
+        """Per-endpoint and aggregate hedge statistics."""
+        return self._stats
+
+    @property
+    def health(self) -> HealthRegistry:
+        """Host- and endpoint-level circuit breaker state."""
+        return self._health
+
+    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
+        """Handle an outgoing request with adaptive, per-endpoint hedging."""
+        host = extract_host(str(request.url))
+        route = self._matcher.match(request)
+
+        if route is not None:
+            key = f"endpoint:{route.name}"
+            resolved = resolve(route.config, self._default_config)
+        else:
+            key = f"host:{host}"
+            resolved = resolve(None, self._default_config)
+
+        can_hedge = request.method.upper() in _IDEMPOTENT_METHODS and not _has_body(
+            request
+        )
+        treat_5xx_as_failure = resolved.circuit_breaker.treat_5xx_as_failure
+
+        async def do_request() -> httpx.Response:
+            return await self._inner.handle_async_request(request)
+
+        def classify(response: httpx.Response) -> bool:
+            return not (treat_5xx_as_failure and response.status_code >= 500)
+
+        async def discard(response: httpx.Response) -> None:
+            await response.aclose()
+
+        return await self._scheduler.execute_with_hedge(
+            key=key,
+            host=host,
+            config=resolved,
+            primary_func=do_request,
+            hedge_func=do_request,
+            classify=classify,
+            can_hedge=can_hedge,
+            discard=discard,
+        )
+
+    async def aclose(self) -> None:
+        """Close the transport and the wrapped inner transport."""
+        self._scheduler.close()
+        await self._inner.aclose()