httpx-hedged 0.2.0__py3-none-any.whl

httpx_hedged/__init__.py

@@ -0,0 +1,38 @@
+"""Adaptive, per-endpoint request hedging transport for httpx.
+
+Learns per-endpoint latency distributions using DDSketch, fires a backup
+request when the primary exceeds its estimated percentile (or a hardcoded
+delay), caps hedge rate with a token bucket, and stops hedging entirely
+(without blocking the primary request) when a host or endpoint circuit
+breaker trips.
+
+Inspired by hedge-python (https://github.com/sunhailin-Leo/hedge-python),
+adapted to key hedge state per endpoint rather than per host so a single
+service with many routes of very different latency/RPS profiles doesn't
+have those profiles mixed into one shared estimate.
+"""
+
+from __future__ import annotations
+
+from httpx_hedged._config import CircuitBreakerConfig, EndpointConfig, HedgeConfig
+from httpx_hedged._health import CircuitBreaker, CircuitState, HealthRegistry
+from httpx_hedged._matcher import Route, UnknownHedgeEndpointError
+from httpx_hedged._stats import Stats, StatsRegistry, StatsSnapshot
+from httpx_hedged.transport import HedgedTransport
+
+__all__ = [
+    "CircuitBreaker",
+    "CircuitBreakerConfig",
+    "CircuitState",
+    "EndpointConfig",
+    "HealthRegistry",
+    "HedgeConfig",
+    "HedgedTransport",
+    "Route",
+    "Stats",
+    "StatsRegistry",
+    "StatsSnapshot",
+    "UnknownHedgeEndpointError",
+]
+
+__version__ = "0.2.0"

httpx_hedged/_bounded.py

@@ -0,0 +1,45 @@
+"""A get-or-create map bounded to a max size, evicting least-recently-used keys.
+
+Per-key state (sketches, breakers, stats) is created lazily, keyed by
+endpoint name or a per-host fallback. Nothing ever evicted those keys, so a
+caller that talks to high-cardinality hosts (per-tenant subdomains, or -- in
+the pathological case -- one key per unique URL when a request has no
+parseable host) would grow these maps forever. Bounding them with LRU
+eviction keeps memory bounded without requiring callers to know about it.
+"""
+
+from __future__ import annotations
+
+from collections import OrderedDict
+from collections.abc import Callable
+
+DEFAULT_MAX_SIZE = 10_000
+
+
+class BoundedRegistry[V]:
+    """Not thread-safe by itself -- callers hold their own lock, as with the
+    plain dicts this replaces."""
+
+    def __init__(self, max_size: int = DEFAULT_MAX_SIZE) -> None:
+        self._max_size = max_size
+        self._items: OrderedDict[str, V] = OrderedDict()
+
+    def get_or_create(self, key: str, factory: Callable[[], V]) -> V:
+        value = self._items.get(key)
+        if value is not None:
+            self._items.move_to_end(key)
+            return value
+        value = factory()
+        self._items[key] = value
+        if len(self._items) > self._max_size:
+            self._items.popitem(last=False)
+        return value
+
+    def get(self, key: str) -> V | None:
+        value = self._items.get(key)
+        if value is not None:
+            self._items.move_to_end(key)
+        return value
+
+    def items(self) -> list[tuple[str, V]]:
+        return list(self._items.items())

httpx_hedged/_config.py

@@ -0,0 +1,221 @@
+"""Configuration for hedge behavior, and default/per-endpoint override resolution."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+
+def _validate_common(config: HedgeConfig | EndpointConfig) -> None:
+    """Validate the fields shared by ``HedgeConfig`` and ``EndpointConfig``.
+
+    Maybe we should just use pydantic to get this stuff build in...
+
+    Every check is skipped for a ``None`` value, since ``EndpointConfig``
+    fields default to None (meaning "inherit the transport default") and
+    are validated for real once ``resolve()`` merges them onto a
+    ``HedgeConfig``, which has no ``None`` fields.
+    """
+    if config.percentile is not None and not 0 < config.percentile < 1:
+        raise ValueError(f"percentile must be between 0 and 1, got {config.percentile}")
+    if config.budget_percent is not None and config.budget_percent < 0:
+        raise ValueError(f"budget_percent must be >= 0, got {config.budget_percent}")
+    if config.estimated_rps is not None and config.estimated_rps <= 0:
+        raise ValueError(f"estimated_rps must be > 0, got {config.estimated_rps}")
+    if config.rps_window_duration is not None and config.rps_window_duration <= 0:
+        raise ValueError(
+            f"rps_window_duration must be > 0, got {config.rps_window_duration}"
+        )
+    if config.min_delay is not None and config.min_delay < 0:
+        raise ValueError(f"min_delay must be >= 0, got {config.min_delay}")
+    if config.warmup_requests is not None and config.warmup_requests < 0:
+        raise ValueError(f"warmup_requests must be >= 0, got {config.warmup_requests}")
+    if config.warmup_delay is not None and config.warmup_delay < 0:
+        raise ValueError(f"warmup_delay must be >= 0, got {config.warmup_delay}")
+    if config.window_duration is not None and config.window_duration <= 0:
+        raise ValueError(f"window_duration must be > 0, got {config.window_duration}")
+
+
+@dataclass(slots=True)
+class CircuitBreakerConfig:
+    """Configuration for the health circuit breaker that gates hedging.
+
+    Tripping the breaker only ever suppresses the *hedge* request -- the
+    primary request is always sent and its result or exception is always
+    returned to the caller normally.
+    """
+
+    #: Fraction of failed requests (0-1) that trips the breaker open.
+    error_rate_threshold: float = 0.5
+
+    #: Minimum number of samples in the current window before the breaker
+    #: is allowed to trip, avoiding trips on tiny samples.
+    min_samples: int = 20
+
+    #: Rolling window size in seconds for the error-rate estimate.
+    window_duration: float = 30.0
+
+    #: Seconds the breaker stays open before allowing a half-open trial.
+    cooldown: float = 30.0
+
+    #: Number of trial requests allowed through while half-open before
+    #: deciding to close or re-open.
+    half_open_max_trial: int = 5
+
+    #: Whether an HTTP 5xx response counts as a failure for
+    #: circuit-breaker purposes.
+    treat_5xx_as_failure: bool = True
+
+    def __post_init__(self) -> None:
+        if not 0 <= self.error_rate_threshold <= 1:
+            raise ValueError(
+                "error_rate_threshold must be between 0 and 1, got "
+                f"{self.error_rate_threshold}"
+            )
+        if self.min_samples < 1:
+            raise ValueError(f"min_samples must be >= 1, got {self.min_samples}")
+        if self.window_duration <= 0:
+            raise ValueError(f"window_duration must be > 0, got {self.window_duration}")
+        if self.cooldown < 0:
+            raise ValueError(f"cooldown must be >= 0, got {self.cooldown}")
+        if self.half_open_max_trial < 1:
+            raise ValueError(
+                f"half_open_max_trial must be >= 1, got {self.half_open_max_trial}"
+            )
+
+
+@dataclass(slots=True)
+class HedgeConfig:
+    """Transport-wide default hedge configuration."""
+
+    #: Sketch quantile used as hedge trigger.
+    percentile: float = 0.90
+
+    #: Max hedge rate as percent of total traffic.
+    budget_percent: float = 10.0
+
+    #: Expected requests per second. If None, the rate is estimated
+    #: automatically from observed traffic instead of requiring a manual
+    #: guess.
+    estimated_rps: float | None = None
+
+    #: Rolling window in seconds for RPS auto-estimation.
+    rps_window_duration: float = 10.0
+
+    #: Floor on the hedge delay in seconds.
+    min_delay: float = 0.001
+
+    #: Number of initial requests using a fixed delay before the sketch
+    #: is trusted.
+    warmup_requests: int = 20
+
+    #: Fixed hedge delay during warmup, in seconds.
+    warmup_delay: float = 0.01
+
+    #: Latency sketch window rotation interval in seconds.
+    window_duration: float = 30.0
+
+    #: Health circuit-breaker configuration.
+    circuit_breaker: CircuitBreakerConfig = field(default_factory=CircuitBreakerConfig)
+
+    def __post_init__(self) -> None:
+        _validate_common(self)
+
+
+@dataclass(slots=True)
+class EndpointConfig:
+    """Per-endpoint override of the transport's default ``HedgeConfig``.
+
+    Every field defaults to None, meaning "inherit the transport default."
+    """
+
+    #: Hardcode a fixed hedge delay for this endpoint instead of learning
+    #: one adaptively from a DDSketch. The endpoint still records latency
+    #: into its sketch for observability; it just isn't consulted for the
+    #: hedge-delay decision.
+    hedge_delay: float | None = None
+
+    #: Sketch quantile used as hedge trigger.
+    percentile: float | None = None
+
+    #: Max hedge rate as percent of total traffic.
+    budget_percent: float | None = None
+
+    #: Expected requests per second. If None, the rate is estimated
+    #: automatically from observed traffic instead of requiring a manual
+    #: guess.
+    estimated_rps: float | None = None
+
+    #: Rolling window in seconds for RPS auto-estimation.
+    rps_window_duration: float | None = None
+
+    #: Floor on the hedge delay in seconds.
+    min_delay: float | None = None
+
+    #: Number of initial requests using a fixed delay before the sketch
+    #: is trusted.
+    warmup_requests: int | None = None
+
+    #: Fixed hedge delay during warmup, in seconds.
+    warmup_delay: float | None = None
+
+    #: Latency sketch window rotation interval in seconds.
+    window_duration: float | None = None
+
+    #: Health circuit-breaker configuration. When set, replaces the
+    #: default breaker config as a whole object rather than being merged
+    #: field-by-field -- for a partial override, use
+    #: ``dataclasses.replace(default.circuit_breaker, ...)``.
+    circuit_breaker: CircuitBreakerConfig | None = None
+
+    def __post_init__(self) -> None:
+        if self.hedge_delay is not None and self.hedge_delay < 0:
+            raise ValueError(f"hedge_delay must be >= 0, got {self.hedge_delay}")
+        _validate_common(self)
+
+
+@dataclass(frozen=True, slots=True)
+class EffectiveConfig:
+    """Fully-resolved hedge configuration for a single key (no more None fields)."""
+
+    percentile: float
+    budget_percent: float
+    estimated_rps: float | None
+    rps_window_duration: float
+    min_delay: float
+    warmup_requests: int
+    warmup_delay: float
+    window_duration: float
+    circuit_breaker: CircuitBreakerConfig
+    hedge_delay: float | None = None
+
+    @property
+    def is_hardcoded(self) -> bool:
+        return self.hedge_delay is not None
+
+
+def _pick[T](override: T | None, fallback: T) -> T:
+    return override if override is not None else fallback
+
+
+def resolve(endpoint: EndpointConfig | None, default: HedgeConfig) -> EffectiveConfig:
+    """Merge a per-endpoint override onto the transport default.
+
+    For each field, the endpoint's value is used if it is not None,
+    otherwise the default's value is used. Cheap enough to call on every
+    request rather than caching the result.
+    """
+    override = endpoint if endpoint is not None else EndpointConfig()
+    return EffectiveConfig(
+        percentile=_pick(override.percentile, default.percentile),
+        budget_percent=_pick(override.budget_percent, default.budget_percent),
+        estimated_rps=_pick(override.estimated_rps, default.estimated_rps),
+        rps_window_duration=_pick(
+            override.rps_window_duration, default.rps_window_duration
+        ),
+        min_delay=_pick(override.min_delay, default.min_delay),
+        warmup_requests=_pick(override.warmup_requests, default.warmup_requests),
+        warmup_delay=_pick(override.warmup_delay, default.warmup_delay),
+        window_duration=_pick(override.window_duration, default.window_duration),
+        circuit_breaker=_pick(override.circuit_breaker, default.circuit_breaker),
+        hedge_delay=override.hedge_delay,
+    )

httpx_hedged/_health.py

@@ -0,0 +1,284 @@
+"""Health-based circuit breaker that suppresses hedging during outages.
+
+hedge-python has no concept of request success/failure at all -- its token
+bucket caps hedge *volume* but has no idea whether the backend is actually
+healthy. This module adds a circuit breaker, tracked independently at both
+the host level and the per-endpoint level, so that either "one endpoint is
+struggling" or "the whole host is struggling" stops hedging without
+requiring the two to be conflated.
+
+Tripping the breaker only ever suppresses the *hedge* request. The primary
+request always goes through and its result or exception is always returned
+to the caller -- this is deliberately not a request-blocking circuit
+breaker, only a hedge-suppressing one, so hedging can't pile extra load
+onto an already-failing backend.
+
+Known limitation: health is recorded from the winning task's outcome only.
+A cancelled loser's real outcome is unknowable, and losers are cancelled
+deliberately -- not doing so would defeat the breaker's purpose of reducing
+load on a struggling backend.
+"""
+
+from __future__ import annotations
+
+import threading
+import time
+from collections.abc import Callable
+from enum import Enum, auto
+
+from httpx_hedged._bounded import BoundedRegistry
+from httpx_hedged._config import CircuitBreakerConfig
+from httpx_hedged._rotation import RotateAction, next_action
+
+
+class CircuitState(Enum):
+    CLOSED = auto()
+    OPEN = auto()
+    HALF_OPEN = auto()
+
+
+class _ErrorWindow:
+    """Lazy dual-window success/failure counter, same rotation scheme as sketches."""
+
+    def __init__(self, window_duration: float) -> None:
+        self._window_duration = window_duration
+        self._current_total = 0
+        self._current_failures = 0
+        self._previous_total = 0
+        self._previous_failures = 0
+        self._window_start = time.monotonic()
+
+    def _maybe_rotate(self) -> None:
+        now = time.monotonic()
+        action = next_action(self._window_start, self._window_duration, now)
+        if action is RotateAction.NONE:
+            return
+        if action is RotateAction.ROTATE:
+            self._previous_total = self._current_total
+            self._previous_failures = self._current_failures
+        else:  # RESET
+            self._previous_total = 0
+            self._previous_failures = 0
+        self._current_total = 0
+        self._current_failures = 0
+        self._window_start = now
+
+    def record(self, ok: bool) -> None:
+        self._maybe_rotate()
+        self._current_total += 1
+        if not ok:
+            self._current_failures += 1
+
+    def sample_count(self) -> int:
+        self._maybe_rotate()
+        return self._current_total + self._previous_total
+
+    def error_rate(self) -> float:
+        self._maybe_rotate()
+        total = self._current_total + self._previous_total
+        if total == 0:
+            return 0.0
+        failures = self._current_failures + self._previous_failures
+        return failures / total
+
+    def reset(self) -> None:
+        self._current_total = 0
+        self._current_failures = 0
+        self._previous_total = 0
+        self._previous_failures = 0
+        self._window_start = time.monotonic()
+
+
+class CircuitBreaker:
+    """A closed/open/half-open circuit breaker gating whether hedging is allowed.
+
+    Not itself thread-safe across concurrent mutation from multiple
+    threads; callers (``HealthRegistry``) hold their own lock.
+
+    Args:
+        config: Breaker thresholds and timing.
+        on_open: Called (with no arguments) each time the breaker
+            transitions into the OPEN state, whether from CLOSED or from a
+            failed HALF_OPEN trial. Useful for alerting -- see the README's
+            observability section for a logging example.
+    """
+
+    def __init__(
+        self,
+        config: CircuitBreakerConfig,
+        on_open: Callable[[], None] | None = None,
+    ) -> None:
+        self._config = config
+        self._on_open = on_open
+        self._window = _ErrorWindow(config.window_duration)
+        self._state = CircuitState.CLOSED
+        self._opened_at: float = 0.0
+        self._half_open_trials = 0
+        self._half_open_failures = 0
+
+    @property
+    def state(self) -> CircuitState:
+        return self._state
+
+    def record_result(self, ok: bool) -> None:
+        if self._state is CircuitState.HALF_OPEN:
+            self._half_open_trials += 1
+            if not ok:
+                self._half_open_failures += 1
+            if self._half_open_trials >= self._config.half_open_max_trial:
+                if (
+                    self._half_open_failures / self._half_open_trials
+                    >= self._config.error_rate_threshold
+                ):
+                    self._reopen()
+                else:
+                    self._close()
+            return
+
+        # CLOSED (or OPEN, where a result can still arrive from a primary
+        # request even though hedging is suppressed -- keep tracking it).
+        self._window.record(ok)
+        if (
+            self._state is CircuitState.CLOSED
+            and self._window.sample_count() >= self._config.min_samples
+            and self._window.error_rate() >= self._config.error_rate_threshold
+        ):
+            self._open()
+
+    def allow_hedge(self) -> bool:
+        if self._state is CircuitState.CLOSED:
+            return True
+        if self._state is CircuitState.OPEN:
+            if time.monotonic() - self._opened_at >= self._config.cooldown:
+                self._enter_half_open()
+                return True
+            return False
+        # HALF_OPEN
+        return self._half_open_trials < self._config.half_open_max_trial
+
+    def _open(self) -> None:
+        self._state = CircuitState.OPEN
+        self._opened_at = time.monotonic()
+        if self._on_open is not None:
+            self._on_open()
+
+    def _reopen(self) -> None:
+        self._state = CircuitState.OPEN
+        self._opened_at = time.monotonic()
+        self._half_open_trials = 0
+        self._half_open_failures = 0
+        if self._on_open is not None:
+            self._on_open()
+
+    def _enter_half_open(self) -> None:
+        self._state = CircuitState.HALF_OPEN
+        self._half_open_trials = 0
+        self._half_open_failures = 0
+
+    def _close(self) -> None:
+        self._state = CircuitState.CLOSED
+        self._half_open_trials = 0
+        self._half_open_failures = 0
+        self._window.reset()
+
+
+class HealthRegistry:
+    """Owns one ``CircuitBreaker`` per host and one per endpoint key.
+
+    Both tiers are consulted independently: a host-level trip disables
+    hedging for every endpoint on that host, while an endpoint-level trip
+    disables hedging only for that endpoint, leaving sibling endpoints on
+    the same host unaffected.
+
+    Args:
+        on_circuit_open: Called each time a breaker (host- or
+            endpoint-scoped) transitions into the OPEN state, as
+            ``on_circuit_open(scope, key)`` where ``scope`` is ``"host"``
+            or ``"endpoint"`` and ``key`` is the host name or endpoint key
+            that tripped. Intended for alerting -- see the README's
+            observability section for a logging example.
+    """
+
+    def __init__(
+        self, on_circuit_open: Callable[[str, str], None] | None = None
+    ) -> None:
+        self._lock = threading.Lock()
+        self._on_circuit_open = on_circuit_open
+        self._host_breakers: BoundedRegistry[CircuitBreaker] = BoundedRegistry()
+        self._endpoint_breakers: BoundedRegistry[CircuitBreaker] = BoundedRegistry()
+
+    def breaker_for_host(
+        self, host: str, config: CircuitBreakerConfig
+    ) -> CircuitBreaker:
+        with self._lock:
+            return self._get_host_locked(host, config)
+
+    def breaker_for_endpoint(
+        self, key: str, config: CircuitBreakerConfig
+    ) -> CircuitBreaker:
+        with self._lock:
+            return self._get_endpoint_locked(key, config)
+
+    def _get_host_locked(
+        self, host: str, config: CircuitBreakerConfig
+    ) -> CircuitBreaker:
+        """Caller must hold ``self._lock``."""
+        return self._host_breakers.get_or_create(
+            host, lambda: CircuitBreaker(config, self._on_open_callback("host", host))
+        )
+
+    def _get_endpoint_locked(
+        self, key: str, config: CircuitBreakerConfig
+    ) -> CircuitBreaker:
+        """Caller must hold ``self._lock``."""
+        return self._endpoint_breakers.get_or_create(
+            key,
+            lambda: CircuitBreaker(config, self._on_open_callback("endpoint", key)),
+        )
+
+    def _on_open_callback(self, scope: str, key: str) -> Callable[[], None] | None:
+        if self._on_circuit_open is None:
+            return None
+        callback = self._on_circuit_open
+        return lambda: callback(scope, key)
+
+    def record_result(
+        self,
+        host: str,
+        key: str,
+        host_config: CircuitBreakerConfig,
+        key_config: CircuitBreakerConfig,
+        ok: bool,
+    ) -> None:
+        # CircuitBreaker is documented as not being thread-safe on its own
+        # ("callers hold their own lock") -- the lookup AND the mutation
+        # below must happen under one lock acquisition, not just the
+        # lookup, or concurrent callers can race on the same breaker's
+        # internal counters/state transitions.
+        with self._lock:
+            host_breaker = self._get_host_locked(host, host_config)
+            endpoint_breaker = self._get_endpoint_locked(key, key_config)
+            host_breaker.record_result(ok)
+            endpoint_breaker.record_result(ok)
+
+    def hedging_allowed(
+        self,
+        host: str,
+        key: str,
+        host_config: CircuitBreakerConfig,
+        key_config: CircuitBreakerConfig,
+    ) -> bool:
+        with self._lock:
+            host_breaker = self._get_host_locked(host, host_config)
+            endpoint_breaker = self._get_endpoint_locked(key, key_config)
+            return host_breaker.allow_hedge() and endpoint_breaker.allow_hedge()
+
+    def host_state(self, host: str) -> CircuitState | None:
+        with self._lock:
+            breaker = self._host_breakers.get(host)
+        return breaker.state if breaker else None
+
+    def endpoint_state(self, key: str) -> CircuitState | None:
+        with self._lock:
+            breaker = self._endpoint_breakers.get(key)
+        return breaker.state if breaker else None