clientwright 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- clientwright/__init__.py +179 -0
- clientwright/__version__.py +1 -0
- clientwright/adapters/__init__.py +3 -0
- clientwright/adapters/_httpx_shared.py +831 -0
- clientwright/adapters/_lazy.py +30 -0
- clientwright/adapters/aiohttp/__init__.py +45 -0
- clientwright/adapters/aiohttp/_imports.py +30 -0
- clientwright/adapters/aiohttp/adapter.py +236 -0
- clientwright/adapters/aiohttp/capabilities.py +81 -0
- clientwright/adapters/aiohttp/classify.py +59 -0
- clientwright/adapters/aiohttp/errors.py +57 -0
- clientwright/adapters/aiohttp/middleware.py +103 -0
- clientwright/adapters/aiohttp/normalize.py +64 -0
- clientwright/adapters/aiohttp/options.py +16 -0
- clientwright/adapters/aiohttp/trace.py +109 -0
- clientwright/adapters/aiohttp/views.py +108 -0
- clientwright/adapters/httpx/__init__.py +45 -0
- clientwright/adapters/httpx/_imports.py +17 -0
- clientwright/adapters/httpx/adapter.py +39 -0
- clientwright/adapters/httpx/capabilities.py +9 -0
- clientwright/adapters/httpx/classify.py +14 -0
- clientwright/adapters/httpx/errors.py +35 -0
- clientwright/adapters/httpx/normalize.py +27 -0
- clientwright/adapters/httpx/normalize_sync.py +27 -0
- clientwright/adapters/httpx/transport.py +40 -0
- clientwright/adapters/httpx/views.py +46 -0
- clientwright/adapters/httpx2/__init__.py +46 -0
- clientwright/adapters/httpx2/_imports.py +18 -0
- clientwright/adapters/httpx2/adapter.py +37 -0
- clientwright/adapters/httpx2/capabilities.py +9 -0
- clientwright/adapters/httpx2/classify.py +14 -0
- clientwright/adapters/httpx2/errors.py +36 -0
- clientwright/adapters/httpx2/normalize.py +27 -0
- clientwright/adapters/httpx2/normalize_sync.py +27 -0
- clientwright/adapters/httpx2/transport.py +35 -0
- clientwright/adapters/httpx2/views.py +45 -0
- clientwright/adapters/observability/__init__.py +26 -0
- clientwright/adapters/observability/_metrics/__init__.py +1 -0
- clientwright/adapters/observability/_metrics/prometheus.py +200 -0
- clientwright/adapters/observability/_tracing/__init__.py +3 -0
- clientwright/adapters/observability/_tracing/otel.py +61 -0
- clientwright/adapters/requests/__init__.py +45 -0
- clientwright/adapters/requests/_imports.py +21 -0
- clientwright/adapters/requests/adapter.py +206 -0
- clientwright/adapters/requests/capabilities.py +80 -0
- clientwright/adapters/requests/classify.py +62 -0
- clientwright/adapters/requests/errors.py +40 -0
- clientwright/adapters/requests/normalize.py +63 -0
- clientwright/adapters/requests/views.py +121 -0
- clientwright/adapters/urllib3/__init__.py +48 -0
- clientwright/adapters/urllib3/_imports.py +18 -0
- clientwright/adapters/urllib3/adapter.py +260 -0
- clientwright/adapters/urllib3/capabilities.py +86 -0
- clientwright/adapters/urllib3/classify.py +46 -0
- clientwright/adapters/urllib3/errors.py +55 -0
- clientwright/adapters/urllib3/normalize.py +51 -0
- clientwright/adapters/urllib3/views.py +119 -0
- clientwright/contrib/__init__.py +3 -0
- clientwright/contrib/deadline.py +107 -0
- clientwright/contrib/dishka.py +80 -0
- clientwright/core/__init__.py +6 -0
- clientwright/core/balancer/__init__.py +1 -0
- clientwright/core/balancer/policy.py +23 -0
- clientwright/core/capabilities.py +156 -0
- clientwright/core/config.py +367 -0
- clientwright/core/contracts/__init__.py +33 -0
- clientwright/core/contracts/adapter.py +62 -0
- clientwright/core/contracts/context.py +31 -0
- clientwright/core/contracts/message.py +118 -0
- clientwright/core/contracts/observability.py +92 -0
- clientwright/core/contracts/settings.py +140 -0
- clientwright/core/engine/__init__.py +1 -0
- clientwright/core/engine/aio.py +246 -0
- clientwright/core/engine/base.py +65 -0
- clientwright/core/engine/redirects.py +64 -0
- clientwright/core/engine/suppress.py +30 -0
- clientwright/core/engine/sync.py +241 -0
- clientwright/core/errors.py +113 -0
- clientwright/core/model.py +138 -0
- clientwright/core/native.py +84 -0
- clientwright/core/options.py +46 -0
- clientwright/core/plan.py +190 -0
- clientwright/core/policy/__init__.py +1 -0
- clientwright/core/policy/budget.py +76 -0
- clientwright/core/policy/circuit.py +169 -0
- clientwright/core/policy/concurrency.py +98 -0
- clientwright/core/policy/retry.py +80 -0
- clientwright/core/policy/timeout.py +64 -0
- clientwright/core/registry.py +75 -0
- clientwright/core/telemetry/__init__.py +6 -0
- clientwright/core/telemetry/emitter.py +163 -0
- clientwright/core/telemetry/names.py +61 -0
- clientwright/core/telemetry/null.py +89 -0
- clientwright/core/telemetry/redaction.py +24 -0
- clientwright/core/testing/__init__.py +7 -0
- clientwright/core/testing/doubles.py +107 -0
- clientwright/core/testing/origin.py +201 -0
- clientwright/py.typed +0 -0
- clientwright-0.1.0.dist-info/METADATA +210 -0
- clientwright-0.1.0.dist-info/RECORD +102 -0
- clientwright-0.1.0.dist-info/WHEEL +4 -0
- clientwright-0.1.0.dist-info/licenses/LICENSE +201 -0
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Compiled call plans, the APP-scope runtime and client handles."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import time
|
|
6
|
+
import weakref
|
|
7
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
8
|
+
from dataclasses import dataclass
|
|
9
|
+
from random import Random
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
from .capabilities import (
|
|
13
|
+
AdapterCapabilities,
|
|
14
|
+
Capability,
|
|
15
|
+
ConfigApplicationReport,
|
|
16
|
+
Support,
|
|
17
|
+
dead_retryable_kinds,
|
|
18
|
+
)
|
|
19
|
+
from .config import ClientConfig, RetryMode, resolve
|
|
20
|
+
from .model import FailureKind, ResolvedTimeouts
|
|
21
|
+
from .policy.budget import RetryBudgetRegistry
|
|
22
|
+
from .policy.circuit import CircuitRegistry, StateListener
|
|
23
|
+
from .policy.concurrency import AsyncOriginLimiter, SyncOriginLimiter
|
|
24
|
+
from .policy.retry import DefaultRetryPolicy
|
|
25
|
+
from .policy.timeout import TimeoutPlanner, base_timeouts
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class ClientRuntime:
|
|
29
|
+
"""State that must outlive REQUEST-scoped clients: circuits, budgets, limiters.
|
|
30
|
+
|
|
31
|
+
Give this APP scope in DI. A fresh runtime per client makes the circuit
|
|
32
|
+
breaker and retry budget decorative.
|
|
33
|
+
"""
|
|
34
|
+
|
|
35
|
+
def __init__(
|
|
36
|
+
self,
|
|
37
|
+
*,
|
|
38
|
+
clock: Callable[[], float] | None = None,
|
|
39
|
+
rng: Random | None = None,
|
|
40
|
+
circuits: CircuitRegistry | None = None,
|
|
41
|
+
retry_budgets: RetryBudgetRegistry | None = None,
|
|
42
|
+
async_limiter: AsyncOriginLimiter | None = None,
|
|
43
|
+
sync_limiter: SyncOriginLimiter | None = None,
|
|
44
|
+
) -> None:
|
|
45
|
+
self.clock: Callable[[], float] = clock or time.monotonic
|
|
46
|
+
self.rng = rng or Random() # noqa: S311 - jitter, not cryptography
|
|
47
|
+
self.circuits = circuits
|
|
48
|
+
self.retry_budgets = retry_budgets
|
|
49
|
+
self.async_limiter = async_limiter
|
|
50
|
+
self.sync_limiter = sync_limiter
|
|
51
|
+
|
|
52
|
+
@classmethod
|
|
53
|
+
def for_config(
|
|
54
|
+
cls,
|
|
55
|
+
config: ClientConfig,
|
|
56
|
+
*,
|
|
57
|
+
clock: Callable[[], float] | None = None,
|
|
58
|
+
rng: Random | None = None,
|
|
59
|
+
circuit_listener: StateListener | None = None,
|
|
60
|
+
) -> ClientRuntime:
|
|
61
|
+
resolved_clock = clock or time.monotonic
|
|
62
|
+
circuits = (
|
|
63
|
+
CircuitRegistry(config.circuit_breaker, resolved_clock, circuit_listener)
|
|
64
|
+
if config.circuit_breaker is not None
|
|
65
|
+
else None
|
|
66
|
+
)
|
|
67
|
+
retry_budgets = (
|
|
68
|
+
RetryBudgetRegistry(config.retry.budget_ratio)
|
|
69
|
+
if config.retry is not None and config.retry.budget_ratio is not None
|
|
70
|
+
else None
|
|
71
|
+
)
|
|
72
|
+
per_host = resolve(config.pool.max_connections_per_host, None)
|
|
73
|
+
async_limiter = AsyncOriginLimiter(per_host) if per_host is not None else None
|
|
74
|
+
sync_limiter = SyncOriginLimiter(per_host) if per_host is not None else None
|
|
75
|
+
return cls(
|
|
76
|
+
clock=resolved_clock,
|
|
77
|
+
rng=rng,
|
|
78
|
+
circuits=circuits,
|
|
79
|
+
retry_budgets=retry_budgets,
|
|
80
|
+
async_limiter=async_limiter,
|
|
81
|
+
sync_limiter=sync_limiter,
|
|
82
|
+
)
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
@dataclass(frozen=True, slots=True)
|
|
86
|
+
class CallPlan:
|
|
87
|
+
"""Everything the engine needs, compiled once at build time."""
|
|
88
|
+
|
|
89
|
+
config: ClientConfig
|
|
90
|
+
capabilities: AdapterCapabilities
|
|
91
|
+
planner: TimeoutPlanner
|
|
92
|
+
retry_policy: DefaultRetryPolicy | None
|
|
93
|
+
report: ConfigApplicationReport
|
|
94
|
+
use_origin_limiter: bool
|
|
95
|
+
# False under RetryMode.DELEGATED: attempts live below the seam, inside the
|
|
96
|
+
# native retry machinery, and a single fake attempt record would be a lie.
|
|
97
|
+
emit_attempt_metrics: bool = True
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def compile_plan(
|
|
101
|
+
config: ClientConfig,
|
|
102
|
+
capabilities: AdapterCapabilities,
|
|
103
|
+
*,
|
|
104
|
+
native_timeout_defaults: ResolvedTimeouts,
|
|
105
|
+
applied_natively: frozenset[Capability] = frozenset(),
|
|
106
|
+
emulated: frozenset[Capability] = frozenset(),
|
|
107
|
+
dropped: Mapping[Capability, str] | None = None,
|
|
108
|
+
native_overrides: Mapping[str, tuple[str, ...]] | None = None,
|
|
109
|
+
) -> CallPlan:
|
|
110
|
+
"""Compile config against capabilities; the caller enforces the report."""
|
|
111
|
+
dead: frozenset[FailureKind] = frozenset()
|
|
112
|
+
retry_policy: DefaultRetryPolicy | None = None
|
|
113
|
+
if config.retry is not None and config.retry.mode is RetryMode.OWNED:
|
|
114
|
+
retry_policy = DefaultRetryPolicy(config.retry)
|
|
115
|
+
dead = dead_retryable_kinds(config.retry.retryable_kinds, capabilities)
|
|
116
|
+
report = ConfigApplicationReport(
|
|
117
|
+
adapter=capabilities.adapter,
|
|
118
|
+
applied_natively=applied_natively,
|
|
119
|
+
emulated=emulated,
|
|
120
|
+
dropped=dict(dropped or {}),
|
|
121
|
+
dead_retryable_kinds=dead,
|
|
122
|
+
collapsed_kinds=dict(capabilities.collapses),
|
|
123
|
+
native_overrides=dict(native_overrides or {}),
|
|
124
|
+
)
|
|
125
|
+
per_host = resolve(config.pool.max_connections_per_host, None)
|
|
126
|
+
use_origin_limiter = (
|
|
127
|
+
per_host is not None and capabilities.support_of(Capability.POOL_LIMIT_PER_HOST) is Support.EMULATED
|
|
128
|
+
)
|
|
129
|
+
delegated = config.retry is not None and config.retry.mode is RetryMode.DELEGATED
|
|
130
|
+
return CallPlan(
|
|
131
|
+
config=config,
|
|
132
|
+
capabilities=capabilities,
|
|
133
|
+
planner=TimeoutPlanner(base_timeouts(config.timeout, native_timeout_defaults), config.caller_override),
|
|
134
|
+
retry_policy=retry_policy,
|
|
135
|
+
report=report,
|
|
136
|
+
use_origin_limiter=use_origin_limiter,
|
|
137
|
+
emit_attempt_metrics=not delegated,
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@dataclass(slots=True)
|
|
142
|
+
class ClientHandle[ClientT]:
|
|
143
|
+
"""The real native client plus everything clientwright knows about it."""
|
|
144
|
+
|
|
145
|
+
client: ClientT
|
|
146
|
+
adapter: str
|
|
147
|
+
capabilities: AdapterCapabilities
|
|
148
|
+
report: ConfigApplicationReport
|
|
149
|
+
runtime: ClientRuntime
|
|
150
|
+
plan: CallPlan
|
|
151
|
+
aclose: Callable[[], Awaitable[None]] | None = None
|
|
152
|
+
close: Callable[[], None] | None = None
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
# The handle lives ON the client instance: the client<->handle reference cycle
|
|
156
|
+
# is collectable by the garbage collector once the caller drops both, unlike a
|
|
157
|
+
# module-global registry whose strong value would pin the client forever.
|
|
158
|
+
_HANDLE_ATTRIBUTE = "_clientwright_handle"
|
|
159
|
+
|
|
160
|
+
# Fallback for exotic clients without __dict__; entries here live as long as
|
|
161
|
+
# the handle does, which is as long as the client (documented limitation).
|
|
162
|
+
_HANDLE_FALLBACK: weakref.WeakKeyDictionary[Any, ClientHandle[Any]] = weakref.WeakKeyDictionary()
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def register_handle(client: object, handle: ClientHandle[Any]) -> None:
|
|
166
|
+
try:
|
|
167
|
+
object.__setattr__(client, _HANDLE_ATTRIBUTE, handle)
|
|
168
|
+
except (AttributeError, TypeError):
|
|
169
|
+
_HANDLE_FALLBACK[client] = handle
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def inspect_client(client: object) -> ClientHandle[Any] | None:
|
|
173
|
+
"""The handle of a built client, or None for foreign objects."""
|
|
174
|
+
handle = getattr(client, _HANDLE_ATTRIBUTE, None)
|
|
175
|
+
if handle is not None:
|
|
176
|
+
return handle # type: ignore[no-any-return]
|
|
177
|
+
try:
|
|
178
|
+
return _HANDLE_FALLBACK.get(client)
|
|
179
|
+
except TypeError: # not weak-referenceable: certainly not a client we built
|
|
180
|
+
return None
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
__all__ = [
|
|
184
|
+
"CallPlan",
|
|
185
|
+
"ClientHandle",
|
|
186
|
+
"ClientRuntime",
|
|
187
|
+
"compile_plan",
|
|
188
|
+
"inspect_client",
|
|
189
|
+
"register_handle",
|
|
190
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Pure policies: no I/O, no awaits, one implementation for both runtimes."""
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Monotonic deadlines and per-origin retry budgets."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import threading
|
|
6
|
+
from collections import OrderedDict
|
|
7
|
+
from collections.abc import Callable
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class Deadline:
|
|
11
|
+
"""Wall-of-time budget of one logical call, measured on a monotonic clock."""
|
|
12
|
+
|
|
13
|
+
__slots__ = ("_clock", "_expires_at", "total")
|
|
14
|
+
|
|
15
|
+
def __init__(self, total: float | None, clock: Callable[[], float]) -> None:
|
|
16
|
+
self.total = total
|
|
17
|
+
self._clock = clock
|
|
18
|
+
self._expires_at = None if total is None else clock() + total
|
|
19
|
+
|
|
20
|
+
@classmethod
|
|
21
|
+
def intersect(cls, clock: Callable[[], float], *totals: float | None) -> Deadline:
|
|
22
|
+
"""The tightest of several budgets; ``None`` entries do not constrain."""
|
|
23
|
+
bounded = [total for total in totals if total is not None]
|
|
24
|
+
return cls(min(bounded) if bounded else None, clock)
|
|
25
|
+
|
|
26
|
+
def remaining(self) -> float | None:
|
|
27
|
+
if self._expires_at is None:
|
|
28
|
+
return None
|
|
29
|
+
return self._expires_at - self._clock()
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def expired(self) -> bool:
|
|
33
|
+
remaining = self.remaining()
|
|
34
|
+
return remaining is not None and remaining <= 0
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class RetryBudgetRegistry:
|
|
38
|
+
"""Token-bucket retry budget per origin.
|
|
39
|
+
|
|
40
|
+
Every logical call earns ``ratio`` tokens; every retry spends one. With the
|
|
41
|
+
default ratio of 0.1 at most ~10% of traffic can be retries, which prevents
|
|
42
|
+
retry storms against a struggling upstream.
|
|
43
|
+
"""
|
|
44
|
+
|
|
45
|
+
_MAX_TOKENS = 10.0
|
|
46
|
+
|
|
47
|
+
def __init__(self, ratio: float | None, max_origins: int = 512) -> None:
|
|
48
|
+
self._ratio = ratio
|
|
49
|
+
self._max_origins = max_origins
|
|
50
|
+
self._tokens: OrderedDict[str, float] = OrderedDict()
|
|
51
|
+
self._lock = threading.Lock()
|
|
52
|
+
|
|
53
|
+
def earn(self, origin: str) -> None:
|
|
54
|
+
if self._ratio is None:
|
|
55
|
+
return
|
|
56
|
+
with self._lock:
|
|
57
|
+
tokens = self._tokens.pop(origin, None)
|
|
58
|
+
# A fresh origin starts with a FULL bucket: an initial burst of
|
|
59
|
+
# retries is allowed, the sustained rate is capped by the ratio.
|
|
60
|
+
self._tokens[origin] = self._MAX_TOKENS if tokens is None else min(tokens + self._ratio, self._MAX_TOKENS)
|
|
61
|
+
while len(self._tokens) > self._max_origins:
|
|
62
|
+
self._tokens.popitem(last=False)
|
|
63
|
+
|
|
64
|
+
def try_spend(self, origin: str) -> bool:
|
|
65
|
+
if self._ratio is None:
|
|
66
|
+
return True
|
|
67
|
+
with self._lock:
|
|
68
|
+
tokens = self._tokens.get(origin, 0.0)
|
|
69
|
+
if tokens < 1.0:
|
|
70
|
+
return False
|
|
71
|
+
self._tokens[origin] = tokens - 1.0
|
|
72
|
+
self._tokens.move_to_end(origin)
|
|
73
|
+
return True
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
__all__ = ["Deadline", "RetryBudgetRegistry"]
|
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""Circuit breaker registry: monotonic clock, LRU-capped keys, thread-safe.
|
|
2
|
+
|
|
3
|
+
One signal per logical call: the engine calls ``check`` once before the attempt
|
|
4
|
+
loop and ``record`` once with the final outcome, so retries cannot pump the
|
|
5
|
+
failure counter of a single call. A call that dies without a classified outcome
|
|
6
|
+
(cancellation, adapter fault) is ``record_aborted`` - it releases its half-open
|
|
7
|
+
slot but is neither a success nor a failure signal.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import logging
|
|
13
|
+
import threading
|
|
14
|
+
from collections import OrderedDict
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from dataclasses import dataclass
|
|
17
|
+
from enum import StrEnum
|
|
18
|
+
|
|
19
|
+
from ..config import CircuitBreakerConfig
|
|
20
|
+
from ..errors import CircuitOpenError
|
|
21
|
+
from ..model import FailureKind, Outcome
|
|
22
|
+
|
|
23
|
+
logger = logging.getLogger("clientwright.circuit")
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class CircuitState(StrEnum):
|
|
27
|
+
CLOSED = "closed"
|
|
28
|
+
OPEN = "open"
|
|
29
|
+
HALF_OPEN = "half_open"
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
@dataclass(slots=True)
|
|
33
|
+
class _KeyState:
|
|
34
|
+
state: CircuitState = CircuitState.CLOSED
|
|
35
|
+
failures: int = 0
|
|
36
|
+
opened_at: float = 0.0
|
|
37
|
+
half_open_in_flight: int = 0
|
|
38
|
+
|
|
39
|
+
@property
|
|
40
|
+
def evictable(self) -> bool:
|
|
41
|
+
"""Only cold, fully closed circuits may be LRU-evicted."""
|
|
42
|
+
return self.state is CircuitState.CLOSED and self.failures == 0 and self.half_open_in_flight == 0
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
StateListener = Callable[[str, CircuitState], None]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
@dataclass(frozen=True, slots=True)
|
|
49
|
+
class CircuitSnapshot:
|
|
50
|
+
state: CircuitState
|
|
51
|
+
failures: int
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class CircuitRegistry:
|
|
55
|
+
"""Per-key circuit state machines behind one lock (operations are O(1)).
|
|
56
|
+
|
|
57
|
+
The state listener is invoked AFTER the lock is released, so a slow or
|
|
58
|
+
re-entrant listener (one that calls ``snapshot()`` or triggers another
|
|
59
|
+
call on a shared runtime) can neither stall other callers nor deadlock.
|
|
60
|
+
"""
|
|
61
|
+
|
|
62
|
+
def __init__(
|
|
63
|
+
self,
|
|
64
|
+
config: CircuitBreakerConfig,
|
|
65
|
+
clock: Callable[[], float],
|
|
66
|
+
on_state_change: StateListener | None = None,
|
|
67
|
+
) -> None:
|
|
68
|
+
self._config = config
|
|
69
|
+
self._clock = clock
|
|
70
|
+
self._on_state_change = on_state_change
|
|
71
|
+
self._states: OrderedDict[str, _KeyState] = OrderedDict()
|
|
72
|
+
self._lock = threading.Lock()
|
|
73
|
+
|
|
74
|
+
def _get(self, key: str) -> _KeyState:
|
|
75
|
+
state = self._states.pop(key, None)
|
|
76
|
+
if state is None:
|
|
77
|
+
state = _KeyState()
|
|
78
|
+
self._states[key] = state
|
|
79
|
+
if len(self._states) > self._config.max_keys:
|
|
80
|
+
# Evict the least recently used COLD entry; an OPEN or HALF_OPEN
|
|
81
|
+
# circuit must never be silently disarmed, so the cap is soft when
|
|
82
|
+
# every entry is hot.
|
|
83
|
+
for candidate_key, candidate in self._states.items():
|
|
84
|
+
if candidate_key != key and candidate.evictable:
|
|
85
|
+
del self._states[candidate_key]
|
|
86
|
+
break
|
|
87
|
+
return state
|
|
88
|
+
|
|
89
|
+
def _notify(self, transitions: list[tuple[str, CircuitState]]) -> None:
|
|
90
|
+
if self._on_state_change is None:
|
|
91
|
+
return
|
|
92
|
+
for key, state in transitions:
|
|
93
|
+
try:
|
|
94
|
+
self._on_state_change(key, state)
|
|
95
|
+
except Exception:
|
|
96
|
+
logger.debug("Circuit state listener failed for %s", key, exc_info=True)
|
|
97
|
+
|
|
98
|
+
def check(self, key: str) -> None:
|
|
99
|
+
"""Admit or reject the logical call; may transition OPEN -> HALF_OPEN.
|
|
100
|
+
|
|
101
|
+
The call that performs the transition consumes one half-open slot, so at
|
|
102
|
+
most ``half_open_max_calls`` trial calls are in flight (the legacy kit
|
|
103
|
+
let the transitioning call fly for free).
|
|
104
|
+
"""
|
|
105
|
+
config = self._config
|
|
106
|
+
transitions: list[tuple[str, CircuitState]] = []
|
|
107
|
+
try:
|
|
108
|
+
with self._lock:
|
|
109
|
+
state = self._get(key)
|
|
110
|
+
if state.state is CircuitState.OPEN:
|
|
111
|
+
elapsed = self._clock() - state.opened_at
|
|
112
|
+
if elapsed < config.recovery_timeout:
|
|
113
|
+
raise CircuitOpenError(key, retry_after=config.recovery_timeout - elapsed)
|
|
114
|
+
state.state = CircuitState.HALF_OPEN
|
|
115
|
+
state.half_open_in_flight = 0
|
|
116
|
+
transitions.append((key, CircuitState.HALF_OPEN))
|
|
117
|
+
if state.state is CircuitState.HALF_OPEN:
|
|
118
|
+
if state.half_open_in_flight >= config.half_open_max_calls:
|
|
119
|
+
raise CircuitOpenError(key)
|
|
120
|
+
state.half_open_in_flight += 1
|
|
121
|
+
finally:
|
|
122
|
+
self._notify(transitions)
|
|
123
|
+
|
|
124
|
+
def _trips(self, outcome: Outcome) -> bool:
|
|
125
|
+
kind = outcome.kind
|
|
126
|
+
if kind is None or kind not in self._config.trip_kinds:
|
|
127
|
+
return False
|
|
128
|
+
if kind is FailureKind.STATUS:
|
|
129
|
+
return outcome.status_code is not None and outcome.status_code >= 500
|
|
130
|
+
return True
|
|
131
|
+
|
|
132
|
+
def record(self, key: str, outcome: Outcome) -> None:
|
|
133
|
+
"""Record the FINAL outcome of one admitted logical call."""
|
|
134
|
+
config = self._config
|
|
135
|
+
transitions: list[tuple[str, CircuitState]] = []
|
|
136
|
+
with self._lock:
|
|
137
|
+
state = self._get(key)
|
|
138
|
+
if state.state is CircuitState.HALF_OPEN and state.half_open_in_flight > 0:
|
|
139
|
+
state.half_open_in_flight -= 1
|
|
140
|
+
if self._trips(outcome):
|
|
141
|
+
state.failures += 1
|
|
142
|
+
if state.state is CircuitState.HALF_OPEN or (
|
|
143
|
+
state.state is CircuitState.CLOSED and state.failures >= config.fail_threshold
|
|
144
|
+
):
|
|
145
|
+
state.state = CircuitState.OPEN
|
|
146
|
+
state.opened_at = self._clock()
|
|
147
|
+
state.half_open_in_flight = 0
|
|
148
|
+
transitions.append((key, CircuitState.OPEN))
|
|
149
|
+
else:
|
|
150
|
+
if state.state is CircuitState.HALF_OPEN:
|
|
151
|
+
state.state = CircuitState.CLOSED
|
|
152
|
+
transitions.append((key, CircuitState.CLOSED))
|
|
153
|
+
state.failures = 0
|
|
154
|
+
self._notify(transitions)
|
|
155
|
+
|
|
156
|
+
def record_aborted(self, key: str) -> None:
|
|
157
|
+
"""An admitted call died without a classified outcome (cancellation,
|
|
158
|
+
adapter fault): release its half-open slot, count nothing."""
|
|
159
|
+
with self._lock:
|
|
160
|
+
state = self._get(key)
|
|
161
|
+
if state.state is CircuitState.HALF_OPEN and state.half_open_in_flight > 0:
|
|
162
|
+
state.half_open_in_flight -= 1
|
|
163
|
+
|
|
164
|
+
def snapshot(self) -> dict[str, CircuitSnapshot]:
|
|
165
|
+
with self._lock:
|
|
166
|
+
return {key: CircuitSnapshot(state=s.state, failures=s.failures) for key, s in self._states.items()}
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
__all__ = ["CircuitRegistry", "CircuitSnapshot", "CircuitState"]
|
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Per-origin in-flight limiters emulating a per-host connection limit.
|
|
2
|
+
|
|
3
|
+
This limits in-flight REQUESTS, not connections - close but not identical,
|
|
4
|
+
which is why adapters relying on it declare ``POOL_LIMIT_PER_HOST: EMULATED``.
|
|
5
|
+
|
|
6
|
+
The async limiter keys its semaphores by the RUNNING EVENT LOOP: an APP-scoped
|
|
7
|
+
runtime may serve several loops over its lifetime (worker threads, sequential
|
|
8
|
+
``asyncio.run`` calls, test suites), and an ``asyncio.Semaphore`` binds to the
|
|
9
|
+
loop of its first contended acquire. Per-loop state also dies with its loop,
|
|
10
|
+
so slots consumed in a dead loop cannot shrink the limit forever.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import asyncio
|
|
16
|
+
import threading
|
|
17
|
+
import weakref
|
|
18
|
+
from types import TracebackType
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class AsyncOriginLimiter:
|
|
22
|
+
"""Per-(event loop, origin) semaphores; safe to share across loops and threads."""
|
|
23
|
+
|
|
24
|
+
def __init__(self, limit: int) -> None:
|
|
25
|
+
self._limit = limit
|
|
26
|
+
self._per_loop: weakref.WeakKeyDictionary[asyncio.AbstractEventLoop, dict[str, asyncio.Semaphore]] = (
|
|
27
|
+
weakref.WeakKeyDictionary()
|
|
28
|
+
)
|
|
29
|
+
self._lock = threading.Lock()
|
|
30
|
+
|
|
31
|
+
def _semaphore(self, origin: str) -> asyncio.Semaphore:
|
|
32
|
+
loop = asyncio.get_running_loop()
|
|
33
|
+
with self._lock:
|
|
34
|
+
per_origin = self._per_loop.get(loop)
|
|
35
|
+
if per_origin is None:
|
|
36
|
+
per_origin = {}
|
|
37
|
+
self._per_loop[loop] = per_origin
|
|
38
|
+
semaphore = per_origin.get(origin)
|
|
39
|
+
if semaphore is None:
|
|
40
|
+
semaphore = asyncio.Semaphore(self._limit)
|
|
41
|
+
per_origin[origin] = semaphore
|
|
42
|
+
return semaphore
|
|
43
|
+
|
|
44
|
+
def acquire(self, origin: str) -> _AsyncSlot:
|
|
45
|
+
return _AsyncSlot(self._semaphore(origin))
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class _AsyncSlot:
|
|
49
|
+
def __init__(self, semaphore: asyncio.Semaphore) -> None:
|
|
50
|
+
self._semaphore = semaphore
|
|
51
|
+
|
|
52
|
+
async def __aenter__(self) -> None:
|
|
53
|
+
await self._semaphore.acquire()
|
|
54
|
+
|
|
55
|
+
async def __aexit__(
|
|
56
|
+
self,
|
|
57
|
+
exc_type: type[BaseException] | None,
|
|
58
|
+
exc: BaseException | None,
|
|
59
|
+
tb: TracebackType | None,
|
|
60
|
+
) -> None:
|
|
61
|
+
self._semaphore.release()
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
class SyncOriginLimiter:
|
|
65
|
+
def __init__(self, limit: int) -> None:
|
|
66
|
+
self._limit = limit
|
|
67
|
+
self._semaphores: dict[str, threading.BoundedSemaphore] = {}
|
|
68
|
+
self._lock = threading.Lock()
|
|
69
|
+
|
|
70
|
+
def _semaphore(self, origin: str) -> threading.BoundedSemaphore:
|
|
71
|
+
with self._lock:
|
|
72
|
+
semaphore = self._semaphores.get(origin)
|
|
73
|
+
if semaphore is None:
|
|
74
|
+
semaphore = threading.BoundedSemaphore(self._limit)
|
|
75
|
+
self._semaphores[origin] = semaphore
|
|
76
|
+
return semaphore
|
|
77
|
+
|
|
78
|
+
def acquire(self, origin: str) -> _SyncSlot:
|
|
79
|
+
return _SyncSlot(self._semaphore(origin))
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _SyncSlot:
|
|
83
|
+
def __init__(self, semaphore: threading.BoundedSemaphore) -> None:
|
|
84
|
+
self._semaphore = semaphore
|
|
85
|
+
|
|
86
|
+
def __enter__(self) -> None:
|
|
87
|
+
self._semaphore.acquire()
|
|
88
|
+
|
|
89
|
+
def __exit__(
|
|
90
|
+
self,
|
|
91
|
+
exc_type: type[BaseException] | None,
|
|
92
|
+
exc: BaseException | None,
|
|
93
|
+
tb: TracebackType | None,
|
|
94
|
+
) -> None:
|
|
95
|
+
self._semaphore.release()
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
__all__ = ["AsyncOriginLimiter", "SyncOriginLimiter"]
|
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
"""Retry decision policy: a pure function shared by the sync and async engines.
|
|
2
|
+
|
|
3
|
+
``decide`` never sleeps, never logs and knows nothing about coroutines; the
|
|
4
|
+
engine owns the loop, the sleeping and the budget accounting.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Sequence
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from random import Random
|
|
12
|
+
|
|
13
|
+
from ..config import RetryConfig
|
|
14
|
+
from ..model import Attempt, FailureKind, RequestInfo
|
|
15
|
+
|
|
16
|
+
# A retry whose backoff would land this close to the deadline is pointless.
|
|
17
|
+
_DEADLINE_SLACK = 0.001
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
@dataclass(frozen=True, slots=True)
|
|
21
|
+
class RetryDecision:
|
|
22
|
+
retry: bool
|
|
23
|
+
delay: float = 0.0
|
|
24
|
+
reason: str = ""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class DefaultRetryPolicy:
|
|
28
|
+
"""Config-driven policy implementing the org-wide retry semantics."""
|
|
29
|
+
|
|
30
|
+
def __init__(self, config: RetryConfig) -> None:
|
|
31
|
+
self._config = config
|
|
32
|
+
|
|
33
|
+
def _wants_retry(self, attempt: Attempt) -> str | None:
|
|
34
|
+
"""Reason string when the last outcome is retry-worthy, else None."""
|
|
35
|
+
outcome = attempt.outcome
|
|
36
|
+
if outcome.status_code is not None and outcome.status_code in self._config.retryable_status:
|
|
37
|
+
return f"status_{outcome.status_code}"
|
|
38
|
+
kind = outcome.kind
|
|
39
|
+
if kind is not None and kind is not FailureKind.STATUS and kind in self._config.retryable_kinds:
|
|
40
|
+
return f"kind_{kind.value}"
|
|
41
|
+
return None
|
|
42
|
+
|
|
43
|
+
def _backoff(self, attempt_index: int, retry_after: float | None, rng: Random) -> float:
|
|
44
|
+
config = self._config
|
|
45
|
+
if config.respect_retry_after and retry_after is not None:
|
|
46
|
+
return min(retry_after, config.retry_after_max)
|
|
47
|
+
delay = min(config.initial_backoff * config.multiplier ** (attempt_index - 1), config.max_backoff)
|
|
48
|
+
if config.jitter:
|
|
49
|
+
delay *= 1.0 + rng.uniform(-config.jitter, config.jitter)
|
|
50
|
+
return max(delay, 0.0)
|
|
51
|
+
|
|
52
|
+
def decide(
|
|
53
|
+
self,
|
|
54
|
+
*,
|
|
55
|
+
info: RequestInfo,
|
|
56
|
+
history: Sequence[Attempt],
|
|
57
|
+
remaining: float | None,
|
|
58
|
+
replayable: bool,
|
|
59
|
+
rng: Random,
|
|
60
|
+
) -> RetryDecision:
|
|
61
|
+
if not history:
|
|
62
|
+
return RetryDecision(retry=False, reason="no_attempts")
|
|
63
|
+
config = self._config
|
|
64
|
+
last = history[-1]
|
|
65
|
+
reason = self._wants_retry(last)
|
|
66
|
+
if reason is None:
|
|
67
|
+
return RetryDecision(retry=False, reason="final")
|
|
68
|
+
if len(history) >= config.max_attempts:
|
|
69
|
+
return RetryDecision(retry=False, reason="attempts")
|
|
70
|
+
if info.method not in config.methods and not info.idempotent:
|
|
71
|
+
return RetryDecision(retry=False, reason="method")
|
|
72
|
+
if config.require_replayable_body and not replayable:
|
|
73
|
+
return RetryDecision(retry=False, reason="non_replayable")
|
|
74
|
+
delay = self._backoff(len(history), last.outcome.retry_after, rng)
|
|
75
|
+
if remaining is not None and delay + _DEADLINE_SLACK >= remaining:
|
|
76
|
+
return RetryDecision(retry=False, reason="deadline")
|
|
77
|
+
return RetryDecision(retry=True, delay=delay, reason=reason)
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
__all__ = ["DefaultRetryPolicy", "RetryDecision"]
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""Per-attempt timeout planning.
|
|
2
|
+
|
|
3
|
+
The planner starts from base phases (config knobs already resolved against the
|
|
4
|
+
adapter's native defaults at build time), merges the caller's per-call override
|
|
5
|
+
according to ``CallerOverride``, then clamps every phase and the attempt
|
|
6
|
+
ceiling by the remaining total budget.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
from ..config import CallerOverride, TimeoutConfig, resolve
|
|
12
|
+
from ..errors import CallError
|
|
13
|
+
from ..model import ResolvedTimeouts
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
class CallerOverrideForbiddenError(CallError):
|
|
17
|
+
def __init__(self) -> None:
|
|
18
|
+
super().__init__("Per-call timeout overrides are forbidden by config (caller_override=raise)")
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def base_timeouts(config: TimeoutConfig, native_defaults: ResolvedTimeouts) -> ResolvedTimeouts:
|
|
22
|
+
"""Resolve UNSET phases against the adapter's native defaults, once, at build."""
|
|
23
|
+
return ResolvedTimeouts(
|
|
24
|
+
connect=resolve(config.connect, native_defaults.connect),
|
|
25
|
+
read=resolve(config.read, native_defaults.read),
|
|
26
|
+
write=resolve(config.write, native_defaults.write),
|
|
27
|
+
pool_acquire=resolve(config.pool_acquire, native_defaults.pool_acquire),
|
|
28
|
+
attempt=resolve(config.attempt, None),
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class TimeoutPlanner:
|
|
33
|
+
__slots__ = ("_base", "_caller_override")
|
|
34
|
+
|
|
35
|
+
def __init__(self, base: ResolvedTimeouts, caller_override: CallerOverride) -> None:
|
|
36
|
+
self._base = base
|
|
37
|
+
self._caller_override = caller_override
|
|
38
|
+
|
|
39
|
+
@staticmethod
|
|
40
|
+
def _clamp(phase: float | None, remaining: float | None) -> float | None:
|
|
41
|
+
if remaining is None:
|
|
42
|
+
return phase
|
|
43
|
+
if phase is None:
|
|
44
|
+
return remaining
|
|
45
|
+
return min(phase, remaining)
|
|
46
|
+
|
|
47
|
+
def plan(self, *, remaining: float | None, caller: ResolvedTimeouts | None) -> ResolvedTimeouts:
|
|
48
|
+
base = self._base
|
|
49
|
+
if caller is not None:
|
|
50
|
+
if self._caller_override is CallerOverride.RAISE:
|
|
51
|
+
raise CallerOverrideForbiddenError
|
|
52
|
+
if self._caller_override is CallerOverride.CALLER_WINS:
|
|
53
|
+
base = caller
|
|
54
|
+
attempt = self._clamp(base.attempt, remaining)
|
|
55
|
+
return ResolvedTimeouts(
|
|
56
|
+
connect=self._clamp(base.connect, remaining),
|
|
57
|
+
read=self._clamp(base.read, remaining),
|
|
58
|
+
write=self._clamp(base.write, remaining),
|
|
59
|
+
pool_acquire=self._clamp(base.pool_acquire, remaining),
|
|
60
|
+
attempt=attempt,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
__all__ = ["CallerOverrideForbiddenError", "TimeoutPlanner", "base_timeouts"]
|