interlock-cb 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- interlock/__init__.py +53 -0
- interlock/_classify.py +24 -0
- interlock/_clock.py +18 -0
- interlock/_detect.py +34 -0
- interlock/_engine.py +232 -0
- interlock/_state_machine.py +197 -0
- interlock/_typing.py +35 -0
- interlock/_windows.py +118 -0
- interlock/breaker.py +169 -0
- interlock/config.py +69 -0
- interlock/errors.py +72 -0
- interlock/httpx2.py +158 -0
- interlock/listeners.py +38 -0
- interlock/otel.py +64 -0
- interlock/outcome.py +30 -0
- interlock/protocols.py +89 -0
- interlock/py.typed +0 -0
- interlock/registry.py +69 -0
- interlock/state.py +28 -0
- interlock/timeout.py +33 -0
- interlock/version.py +15 -0
- interlock/window.py +47 -0
- interlock_cb-1.0.0.dist-info/METADATA +153 -0
- interlock_cb-1.0.0.dist-info/RECORD +26 -0
- interlock_cb-1.0.0.dist-info/WHEEL +4 -0
- interlock_cb-1.0.0.dist-info/licenses/LICENSE +21 -0
interlock/_windows.py
ADDED
|
@@ -0,0 +1,118 @@
|
|
|
1
|
+
"""Concrete sliding-window implementations injected into the core.
|
|
2
|
+
|
|
3
|
+
Two strategies satisfy the ``SlidingWindow`` protocol: ``CountBasedSlidingWindow``
|
|
4
|
+
keeps the last N calls in a ring buffer; ``TimeBasedSlidingWindow`` keeps calls
|
|
5
|
+
from the last N seconds in per-second buckets. Both expose running aggregates so
|
|
6
|
+
``snapshot`` stays cheap, and both are constructed by the core from a validated
|
|
7
|
+
``Config`` — they trust their ``size`` and never validate it themselves.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
|
|
12
|
+
from interlock.config import Config
|
|
13
|
+
from interlock.outcome import Outcome
|
|
14
|
+
from interlock.protocols import Clock, SlidingWindow
|
|
15
|
+
from interlock.window import WindowSnapshot, WindowType
|
|
16
|
+
|
|
17
|
+
__all__ = ('CountBasedSlidingWindow', 'TimeBasedSlidingWindow', 'build_window')
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def build_window(*, config: Config, clock: Clock) -> SlidingWindow:
|
|
21
|
+
"""Construct the window implementation selected by ``config``.
|
|
22
|
+
|
|
23
|
+
Time-based windows need the clock; count-based ones ignore it. The core
|
|
24
|
+
calls this whenever it needs fresh metrics — on construction and whenever a
|
|
25
|
+
breaker returns to ``CLOSED``.
|
|
26
|
+
"""
|
|
27
|
+
if config.window_type is WindowType.COUNT_BASED:
|
|
28
|
+
return CountBasedSlidingWindow(size=config.window_size)
|
|
29
|
+
|
|
30
|
+
return TimeBasedSlidingWindow(size=config.window_size, clock=clock)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
class CountBasedSlidingWindow:
|
|
34
|
+
"""Aggregates the most recent ``size`` outcomes via a ring buffer.
|
|
35
|
+
|
|
36
|
+
Running counters are adjusted on each record — incremented for the new
|
|
37
|
+
outcome, decremented for the one it evicts — so ``snapshot`` is O(1).
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
def __init__(self, *, size: int) -> None:
|
|
41
|
+
self._buffer: list[Outcome | None] = [None] * size
|
|
42
|
+
self._head = 0
|
|
43
|
+
self._total = 0
|
|
44
|
+
self._failed = 0
|
|
45
|
+
self._slow = 0
|
|
46
|
+
|
|
47
|
+
def record(self, outcome: Outcome) -> None:
|
|
48
|
+
evicted = self._buffer[self._head]
|
|
49
|
+
if evicted is not None:
|
|
50
|
+
self._total -= 1
|
|
51
|
+
self._failed -= evicted.is_failure
|
|
52
|
+
self._slow -= evicted.is_slow
|
|
53
|
+
|
|
54
|
+
self._buffer[self._head] = outcome
|
|
55
|
+
self._total += 1
|
|
56
|
+
self._failed += outcome.is_failure
|
|
57
|
+
self._slow += outcome.is_slow
|
|
58
|
+
|
|
59
|
+
self._head = (self._head + 1) % len(self._buffer)
|
|
60
|
+
|
|
61
|
+
def snapshot(self) -> WindowSnapshot:
|
|
62
|
+
return WindowSnapshot(
|
|
63
|
+
total_calls=self._total,
|
|
64
|
+
failed_calls=self._failed,
|
|
65
|
+
slow_calls=self._slow,
|
|
66
|
+
)
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(slots=True)
|
|
70
|
+
class _Bucket:
|
|
71
|
+
"""Per-second aggregate. ``epoch_second`` tags which second it holds."""
|
|
72
|
+
|
|
73
|
+
epoch_second: int
|
|
74
|
+
total: int = 0
|
|
75
|
+
failed: int = 0
|
|
76
|
+
slow: int = 0
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
class TimeBasedSlidingWindow:
|
|
80
|
+
"""Aggregates outcomes from the last ``size`` seconds in per-second buckets.
|
|
81
|
+
|
|
82
|
+
Bucket ``i`` holds the second ``epoch % size == i``; touching a bucket whose
|
|
83
|
+
tag no longer matches the current second resets it, so a wrapped index never
|
|
84
|
+
carries a stale count. ``snapshot`` sums only buckets whose tag still falls
|
|
85
|
+
inside the window, which also expires calls as time passes without records.
|
|
86
|
+
"""
|
|
87
|
+
|
|
88
|
+
def __init__(self, *, size: int, clock: Clock) -> None:
|
|
89
|
+
self._size = size
|
|
90
|
+
self._clock = clock
|
|
91
|
+
self._buckets = [_Bucket(epoch_second=-1) for _ in range(size)]
|
|
92
|
+
|
|
93
|
+
def _now_second(self) -> int:
|
|
94
|
+
return int(self._clock.monotonic())
|
|
95
|
+
|
|
96
|
+
def record(self, outcome: Outcome) -> None:
|
|
97
|
+
now = self._now_second()
|
|
98
|
+
bucket = self._buckets[now % self._size]
|
|
99
|
+
if bucket.epoch_second != now:
|
|
100
|
+
bucket.epoch_second = now
|
|
101
|
+
bucket.total = bucket.failed = bucket.slow = 0
|
|
102
|
+
|
|
103
|
+
bucket.total += 1
|
|
104
|
+
bucket.failed += outcome.is_failure
|
|
105
|
+
bucket.slow += outcome.is_slow
|
|
106
|
+
|
|
107
|
+
def snapshot(self) -> WindowSnapshot:
|
|
108
|
+
now = self._now_second()
|
|
109
|
+
oldest = max(0, now - self._size + 1)
|
|
110
|
+
|
|
111
|
+
total = failed = slow = 0
|
|
112
|
+
for bucket in self._buckets:
|
|
113
|
+
if oldest <= bucket.epoch_second <= now:
|
|
114
|
+
total += bucket.total
|
|
115
|
+
failed += bucket.failed
|
|
116
|
+
slow += bucket.slow
|
|
117
|
+
|
|
118
|
+
return WindowSnapshot(total_calls=total, failed_calls=failed, slow_calls=slow)
|
interlock/breaker.py
ADDED
|
@@ -0,0 +1,169 @@
|
|
|
1
|
+
"""The public circuit breaker.
|
|
2
|
+
|
|
3
|
+
``CircuitBreaker`` is a single class serving both sync and async code. It offers
|
|
4
|
+
three ways to protect work over the one ``call()`` primitive:
|
|
5
|
+
|
|
6
|
+
- **decorator** — ``@breaker`` wraps a function, preserving its signature *and*
|
|
7
|
+
its sync/async nature via ``@overload`` + ``ParamSpec``;
|
|
8
|
+
- **context manager** — the same instance is both a sync (``with``) and async
|
|
9
|
+
(``async with``) context manager guarding a block;
|
|
10
|
+
- **``breaker.call(fn, ...)``** — the breaker executes the callable.
|
|
11
|
+
|
|
12
|
+
A documented contract difference: the decorator and ``call`` see a callable, so
|
|
13
|
+
result-based classification and slow-call detection both apply. The context
|
|
14
|
+
manager sees only the block — its exception and duration — so result-based
|
|
15
|
+
classification is unavailable there.
|
|
16
|
+
"""
|
|
17
|
+
|
|
18
|
+
import functools
|
|
19
|
+
from collections.abc import Awaitable
|
|
20
|
+
from types import TracebackType
|
|
21
|
+
from typing import Literal, Self, cast, overload
|
|
22
|
+
|
|
23
|
+
from interlock._clock import SystemClock
|
|
24
|
+
from interlock._detect import is_async_callable
|
|
25
|
+
from interlock._engine import Engine
|
|
26
|
+
from interlock._typing import AsyncCallable, P, R, SyncCallable
|
|
27
|
+
from interlock.config import Config
|
|
28
|
+
from interlock.protocols import Clock, EventListener, FailureClassifier
|
|
29
|
+
from interlock.state import State
|
|
30
|
+
from interlock.window import WindowSnapshot
|
|
31
|
+
|
|
32
|
+
__all__ = ('CircuitBreaker',)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class CircuitBreaker:
|
|
36
|
+
"""A named circuit breaker for sync and async callables.
|
|
37
|
+
|
|
38
|
+
Args:
|
|
39
|
+
name: Identifies the breaker; surfaced on ``CircuitOpenError``.
|
|
40
|
+
config: Thresholds, window and timing. Defaults to ``Config()``.
|
|
41
|
+
clock: Time source. Defaults to ``SystemClock`` (real monotonic time);
|
|
42
|
+
inject a fake for deterministic tests.
|
|
43
|
+
classifier: Decides which outcomes count as failures. Defaults to any
|
|
44
|
+
raised exception being a failure.
|
|
45
|
+
listener: Observability hooks (state changes, calls, rejections,
|
|
46
|
+
resets). Defaults to no observation.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
def __init__(
|
|
50
|
+
self,
|
|
51
|
+
*,
|
|
52
|
+
name: str,
|
|
53
|
+
config: Config | None = None,
|
|
54
|
+
clock: Clock | None = None,
|
|
55
|
+
classifier: FailureClassifier | None = None,
|
|
56
|
+
listener: EventListener | None = None,
|
|
57
|
+
) -> None:
|
|
58
|
+
self._name = name
|
|
59
|
+
self._engine = Engine(
|
|
60
|
+
name=name,
|
|
61
|
+
config=config if config is not None else Config(),
|
|
62
|
+
clock=clock if clock is not None else SystemClock(),
|
|
63
|
+
classifier=classifier,
|
|
64
|
+
listener=listener,
|
|
65
|
+
)
|
|
66
|
+
self._block_starts: list[float] = []
|
|
67
|
+
|
|
68
|
+
@property
|
|
69
|
+
def name(self) -> str:
|
|
70
|
+
"""The breaker's name."""
|
|
71
|
+
return self._name
|
|
72
|
+
|
|
73
|
+
@property
|
|
74
|
+
def state(self) -> State:
|
|
75
|
+
"""The breaker's current lifecycle state."""
|
|
76
|
+
return self._engine.state
|
|
77
|
+
|
|
78
|
+
def snapshot(self) -> WindowSnapshot:
|
|
79
|
+
"""An immutable view of the current window aggregates."""
|
|
80
|
+
return self._engine.snapshot()
|
|
81
|
+
|
|
82
|
+
def reset(self) -> None:
|
|
83
|
+
"""Return the breaker to ``CLOSED`` with a fresh window."""
|
|
84
|
+
self._engine.reset()
|
|
85
|
+
|
|
86
|
+
def force_open(self) -> None:
|
|
87
|
+
"""Force the breaker ``FORCED_OPEN``: reject all traffic until reset."""
|
|
88
|
+
self._engine.force_open()
|
|
89
|
+
|
|
90
|
+
def disable(self) -> None:
|
|
91
|
+
"""Disable the breaker: admit all traffic and record nothing."""
|
|
92
|
+
self._engine.disable()
|
|
93
|
+
|
|
94
|
+
def metrics_only(self) -> None:
|
|
95
|
+
"""Put the breaker in shadow mode: admit all traffic, record, never trip."""
|
|
96
|
+
self._engine.metrics_only()
|
|
97
|
+
|
|
98
|
+
def call(
|
|
99
|
+
self,
|
|
100
|
+
fn: AsyncCallable[P, R] | SyncCallable[P, R],
|
|
101
|
+
/,
|
|
102
|
+
*args: P.args,
|
|
103
|
+
**kwargs: P.kwargs,
|
|
104
|
+
) -> Awaitable[R] | R:
|
|
105
|
+
"""Execute ``fn`` under protection, dispatching on its sync/async nature.
|
|
106
|
+
|
|
107
|
+
Returns an awaitable for a coroutine function and the plain result for a
|
|
108
|
+
synchronous one.
|
|
109
|
+
|
|
110
|
+
Raises:
|
|
111
|
+
CircuitOpenError: If the breaker rejects the call.
|
|
112
|
+
"""
|
|
113
|
+
return self._engine.call(fn, *args, **kwargs)
|
|
114
|
+
|
|
115
|
+
@overload
|
|
116
|
+
def __call__(self, fn: AsyncCallable[P, R]) -> AsyncCallable[P, R]: ...
|
|
117
|
+
|
|
118
|
+
@overload
|
|
119
|
+
def __call__(self, fn: SyncCallable[P, R]) -> SyncCallable[P, R]: ...
|
|
120
|
+
|
|
121
|
+
# mypy cannot reconcile this union implementation with the ParamSpec
|
|
122
|
+
# overloads above (a known limitation of overloaded decorators); pyright
|
|
123
|
+
# accepts it, and the overloads are what callers see.
|
|
124
|
+
def __call__( # type: ignore[misc]
|
|
125
|
+
self, fn: AsyncCallable[P, R] | SyncCallable[P, R]
|
|
126
|
+
) -> AsyncCallable[P, R] | SyncCallable[P, R]:
|
|
127
|
+
"""Decorate ``fn``, preserving its signature and sync/async nature."""
|
|
128
|
+
if is_async_callable(fn):
|
|
129
|
+
async_fn = cast('AsyncCallable[P, R]', fn)
|
|
130
|
+
|
|
131
|
+
@functools.wraps(async_fn)
|
|
132
|
+
async def async_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
133
|
+
return await self._engine.call_async(async_fn, *args, **kwargs)
|
|
134
|
+
|
|
135
|
+
return async_wrapper
|
|
136
|
+
|
|
137
|
+
sync_fn = cast('SyncCallable[P, R]', fn)
|
|
138
|
+
|
|
139
|
+
@functools.wraps(sync_fn)
|
|
140
|
+
def sync_wrapper(*args: P.args, **kwargs: P.kwargs) -> R:
|
|
141
|
+
return self._engine.call_sync(sync_fn, *args, **kwargs)
|
|
142
|
+
|
|
143
|
+
return sync_wrapper
|
|
144
|
+
|
|
145
|
+
def __enter__(self) -> Self:
|
|
146
|
+
self._block_starts.append(self._engine.enter_block())
|
|
147
|
+
return self
|
|
148
|
+
|
|
149
|
+
def __exit__(
|
|
150
|
+
self,
|
|
151
|
+
_exc_type: type[BaseException] | None,
|
|
152
|
+
exc: BaseException | None,
|
|
153
|
+
_traceback: TracebackType | None,
|
|
154
|
+
) -> Literal[False]:
|
|
155
|
+
self._engine.exit_block(start=self._block_starts.pop(), exception=exc)
|
|
156
|
+
return False
|
|
157
|
+
|
|
158
|
+
async def __aenter__(self) -> Self:
|
|
159
|
+
self._block_starts.append(self._engine.enter_block())
|
|
160
|
+
return self
|
|
161
|
+
|
|
162
|
+
async def __aexit__(
|
|
163
|
+
self,
|
|
164
|
+
_exc_type: type[BaseException] | None,
|
|
165
|
+
exc: BaseException | None,
|
|
166
|
+
_traceback: TracebackType | None,
|
|
167
|
+
) -> Literal[False]:
|
|
168
|
+
self._engine.exit_block(start=self._block_starts.pop(), exception=exc)
|
|
169
|
+
return False
|
interlock/config.py
ADDED
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
"""Immutable circuit breaker configuration with eager validation."""
|
|
2
|
+
|
|
3
|
+
from dataclasses import dataclass
|
|
4
|
+
|
|
5
|
+
from interlock.window import WindowType
|
|
6
|
+
|
|
7
|
+
__all__ = ('Config',)
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
@dataclass(frozen=True, kw_only=True, slots=True)
|
|
11
|
+
class Config:
|
|
12
|
+
"""Thresholds, window and timing for a circuit breaker.
|
|
13
|
+
|
|
14
|
+
Reusable across breakers: the registry shares one config and overrides per
|
|
15
|
+
name. Failure classification (which exceptions/results count) is a separate
|
|
16
|
+
concern, handled by the ``FailureClassifier``, not here.
|
|
17
|
+
|
|
18
|
+
Defaults follow resilience4j: trip at 50% failures over at least 10 calls,
|
|
19
|
+
treat calls slower than 60s as slow (but never trip on slowness alone until
|
|
20
|
+
tuned), stay open 60s before a single probe is allowed.
|
|
21
|
+
|
|
22
|
+
Raises:
|
|
23
|
+
ValueError: If any value is out of range or inconsistent.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
failure_rate_threshold: float = 0.5
|
|
27
|
+
minimum_number_of_calls: int = 10
|
|
28
|
+
slow_call_duration_threshold: float = 60.0
|
|
29
|
+
slow_call_rate_threshold: float = 1.0
|
|
30
|
+
permitted_calls_in_half_open: int = 10
|
|
31
|
+
max_concurrent_probes: int = 1
|
|
32
|
+
wait_duration_in_open: float = 60.0
|
|
33
|
+
window_type: WindowType = WindowType.COUNT_BASED
|
|
34
|
+
window_size: int = 100
|
|
35
|
+
|
|
36
|
+
def __post_init__(self) -> None:
|
|
37
|
+
if not 0.0 < self.failure_rate_threshold <= 1.0:
|
|
38
|
+
raise ValueError(
|
|
39
|
+
f'failure_rate_threshold must be in (0, 1], got {self.failure_rate_threshold!r}'
|
|
40
|
+
)
|
|
41
|
+
if not 0.0 < self.slow_call_rate_threshold <= 1.0:
|
|
42
|
+
raise ValueError(
|
|
43
|
+
f'slow_call_rate_threshold must be in (0, 1], got {self.slow_call_rate_threshold!r}'
|
|
44
|
+
)
|
|
45
|
+
if self.minimum_number_of_calls < 1:
|
|
46
|
+
raise ValueError(
|
|
47
|
+
f'minimum_number_of_calls must be >= 1, got {self.minimum_number_of_calls!r}'
|
|
48
|
+
)
|
|
49
|
+
if self.slow_call_duration_threshold <= 0.0:
|
|
50
|
+
raise ValueError(
|
|
51
|
+
f'slow_call_duration_threshold must be > 0, '
|
|
52
|
+
f'got {self.slow_call_duration_threshold!r}'
|
|
53
|
+
)
|
|
54
|
+
if self.wait_duration_in_open <= 0.0:
|
|
55
|
+
raise ValueError(
|
|
56
|
+
f'wait_duration_in_open must be > 0, got {self.wait_duration_in_open!r}'
|
|
57
|
+
)
|
|
58
|
+
if self.permitted_calls_in_half_open < 1:
|
|
59
|
+
raise ValueError(
|
|
60
|
+
f'permitted_calls_in_half_open must be >= 1, '
|
|
61
|
+
f'got {self.permitted_calls_in_half_open!r}'
|
|
62
|
+
)
|
|
63
|
+
if not 1 <= self.max_concurrent_probes <= self.permitted_calls_in_half_open:
|
|
64
|
+
raise ValueError(
|
|
65
|
+
f'max_concurrent_probes must be in [1, {self.permitted_calls_in_half_open}], '
|
|
66
|
+
f'got {self.max_concurrent_probes!r}'
|
|
67
|
+
)
|
|
68
|
+
if self.window_size < 1:
|
|
69
|
+
raise ValueError(f'window_size must be >= 1, got {self.window_size!r}')
|
interlock/errors.py
ADDED
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""Exception and warning hierarchy for interlock."""
|
|
2
|
+
|
|
3
|
+
__all__ = (
|
|
4
|
+
'CallTimeoutError',
|
|
5
|
+
'CircuitOpenError',
|
|
6
|
+
'InterlockDeprecationWarning',
|
|
7
|
+
'InterlockError',
|
|
8
|
+
)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class InterlockError(Exception):
|
|
12
|
+
"""Base class for all errors raised by interlock."""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class CircuitOpenError(InterlockError):
|
|
16
|
+
"""Raised when a call is rejected because the circuit is not closed.
|
|
17
|
+
|
|
18
|
+
Carries enough context to act on the rejection without inspecting the
|
|
19
|
+
breaker: which breaker rejected the call, roughly how long until the next
|
|
20
|
+
probe is allowed, and the most recent recorded failure (if any).
|
|
21
|
+
|
|
22
|
+
Args:
|
|
23
|
+
breaker_name: Name of the breaker that rejected the call.
|
|
24
|
+
retry_after: Seconds until the next probe is allowed, or ``None`` when
|
|
25
|
+
the breaker cannot estimate it (e.g. ``FORCED_OPEN``).
|
|
26
|
+
last_failure: The most recent recorded failure, if any.
|
|
27
|
+
"""
|
|
28
|
+
|
|
29
|
+
def __init__(
|
|
30
|
+
self,
|
|
31
|
+
breaker_name: str,
|
|
32
|
+
*,
|
|
33
|
+
retry_after: float | None = None,
|
|
34
|
+
last_failure: BaseException | None = None,
|
|
35
|
+
) -> None:
|
|
36
|
+
self.breaker_name = breaker_name
|
|
37
|
+
self.retry_after = retry_after
|
|
38
|
+
self.last_failure = last_failure
|
|
39
|
+
|
|
40
|
+
super().__init__(self._build_message())
|
|
41
|
+
|
|
42
|
+
def _build_message(self) -> str:
|
|
43
|
+
message = f'Circuit {self.breaker_name!r} is open'
|
|
44
|
+
if self.retry_after is not None:
|
|
45
|
+
message = f'{message}; retry in ~{self.retry_after:.3f}s'
|
|
46
|
+
|
|
47
|
+
return message
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class CallTimeoutError(InterlockError):
|
|
51
|
+
"""Raised when a guarded operation exceeds its timeout deadline.
|
|
52
|
+
|
|
53
|
+
A call that hangs forever would never be counted as slow or failed — it just
|
|
54
|
+
holds a resource; the timeout converts it into a failure a surrounding
|
|
55
|
+
breaker can observe.
|
|
56
|
+
|
|
57
|
+
Args:
|
|
58
|
+
timeout: The deadline, in seconds, that was exceeded.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
def __init__(self, timeout: float) -> None:
|
|
62
|
+
self.timeout = timeout
|
|
63
|
+
message = f'Operation exceeded its {timeout:.3f}s timeout'
|
|
64
|
+
super().__init__(message)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class InterlockDeprecationWarning(UserWarning):
|
|
68
|
+
"""Deprecation warning that is visible by default.
|
|
69
|
+
|
|
70
|
+
Subclasses ``UserWarning`` rather than ``DeprecationWarning`` so it is
|
|
71
|
+
shown to end users without enabling the deprecation filter.
|
|
72
|
+
"""
|
interlock/httpx2.py
ADDED
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""httpx2 transport integration — requires the ``httpx2`` extra.
|
|
2
|
+
|
|
3
|
+
This module imports ``httpx2`` and is deliberately *not* re-exported from
|
|
4
|
+
``interlock`` so the core stays zero-dependency. Install with
|
|
5
|
+
``pip install interlock[httpx2]`` and wrap your transport explicitly::
|
|
6
|
+
|
|
7
|
+
import httpx2
|
|
8
|
+
from interlock.httpx2 import CircuitBreakerTransport
|
|
9
|
+
|
|
10
|
+
transport = CircuitBreakerTransport(httpx2.HTTPTransport())
|
|
11
|
+
client = httpx2.Client(transport=transport)
|
|
12
|
+
|
|
13
|
+
The wrapper applies one circuit breaker **per host** transparently: no
|
|
14
|
+
decorators in user code. Each host gets its own breaker (a slow or failing
|
|
15
|
+
``api.a`` must not trip ``api.b``), created lazily and shared across requests.
|
|
16
|
+
|
|
17
|
+
By default a response counts as a failure when its status is one of
|
|
18
|
+
``HttpStatusClassifier``'s — the canonical retryable set ``429, 500, 502, 503,
|
|
19
|
+
504`` — and any transport exception (connect/read errors) is a failure. Supply
|
|
20
|
+
a custom ``classifier`` to change that policy.
|
|
21
|
+
"""
|
|
22
|
+
|
|
23
|
+
import http
|
|
24
|
+
from typing import cast
|
|
25
|
+
|
|
26
|
+
from httpx2 import AsyncBaseTransport, BaseTransport, Request, Response
|
|
27
|
+
from interlock.config import Config
|
|
28
|
+
from interlock.protocols import Clock, EventListener, FailureClassifier
|
|
29
|
+
from interlock.registry import Registry
|
|
30
|
+
|
|
31
|
+
__all__ = (
|
|
32
|
+
'AsyncCircuitBreakerTransport',
|
|
33
|
+
'CircuitBreakerTransport',
|
|
34
|
+
'HttpStatusClassifier',
|
|
35
|
+
)
|
|
36
|
+
|
|
37
|
+
# Mirrors urllib3's recommended ``status_forcelist`` (also used by AWS/Google):
|
|
38
|
+
# transient server-side conditions where the dependency is unhealthy or
|
|
39
|
+
# overloaded. Permanent 5xx (501 Not Implemented, 505) are excluded — retrying
|
|
40
|
+
# or tripping the breaker cannot help a contract/protocol error.
|
|
41
|
+
_FAILURE_STATUSES = frozenset(
|
|
42
|
+
{
|
|
43
|
+
http.HTTPStatus.TOO_MANY_REQUESTS, # 429
|
|
44
|
+
http.HTTPStatus.INTERNAL_SERVER_ERROR, # 500
|
|
45
|
+
http.HTTPStatus.BAD_GATEWAY, # 502
|
|
46
|
+
http.HTTPStatus.SERVICE_UNAVAILABLE, # 503
|
|
47
|
+
http.HTTPStatus.GATEWAY_TIMEOUT, # 504
|
|
48
|
+
}
|
|
49
|
+
)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class HttpStatusClassifier:
|
|
53
|
+
"""Counts transport exceptions and unhealthy-status responses as failures.
|
|
54
|
+
|
|
55
|
+
A returned response is a failure when its status is in the canonical
|
|
56
|
+
retryable set (``429, 500, 502, 503, 504``); any raised exception is a
|
|
57
|
+
failure. Other responses — including ``4xx`` client mistakes like ``404`` —
|
|
58
|
+
are successes, so they never trip the breaker.
|
|
59
|
+
"""
|
|
60
|
+
|
|
61
|
+
def is_failure(self, *, result: object, exception: BaseException | None) -> bool:
|
|
62
|
+
"""Return whether a completed request counts as a failure."""
|
|
63
|
+
if exception is not None:
|
|
64
|
+
return True
|
|
65
|
+
|
|
66
|
+
return cast('Response', result).status_code in _FAILURE_STATUSES
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _build_registry(
|
|
70
|
+
config: Config | None,
|
|
71
|
+
clock: Clock | None,
|
|
72
|
+
classifier: FailureClassifier | None,
|
|
73
|
+
listener: EventListener | None,
|
|
74
|
+
) -> Registry:
|
|
75
|
+
return Registry(
|
|
76
|
+
config=config,
|
|
77
|
+
clock=clock,
|
|
78
|
+
classifier=classifier if classifier is not None else HttpStatusClassifier(),
|
|
79
|
+
listener=listener,
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
class CircuitBreakerTransport(BaseTransport):
|
|
84
|
+
"""A synchronous transport that guards each host with a circuit breaker.
|
|
85
|
+
|
|
86
|
+
Args:
|
|
87
|
+
transport: The wrapped transport that performs the actual request.
|
|
88
|
+
config: Thresholds, window and timing for every host's breaker.
|
|
89
|
+
clock: Time source for the breakers; inject a fake for deterministic
|
|
90
|
+
tests.
|
|
91
|
+
classifier: Failure policy. Defaults to ``HttpStatusClassifier``.
|
|
92
|
+
listener: Observability hooks shared by every host's breaker.
|
|
93
|
+
"""
|
|
94
|
+
|
|
95
|
+
def __init__(
|
|
96
|
+
self,
|
|
97
|
+
transport: BaseTransport,
|
|
98
|
+
*,
|
|
99
|
+
config: Config | None = None,
|
|
100
|
+
clock: Clock | None = None,
|
|
101
|
+
classifier: FailureClassifier | None = None,
|
|
102
|
+
listener: EventListener | None = None,
|
|
103
|
+
) -> None:
|
|
104
|
+
self._transport = transport
|
|
105
|
+
self._registry = _build_registry(config, clock, classifier, listener)
|
|
106
|
+
|
|
107
|
+
def handle_request(self, request: Request) -> Response:
|
|
108
|
+
"""Run the request under its host's breaker.
|
|
109
|
+
|
|
110
|
+
Raises:
|
|
111
|
+
CircuitOpenError: If the host's breaker is open.
|
|
112
|
+
"""
|
|
113
|
+
breaker = self._registry.get(request.url.host)
|
|
114
|
+
guarded = breaker(self._transport.handle_request)
|
|
115
|
+
return guarded(request)
|
|
116
|
+
|
|
117
|
+
def close(self) -> None:
|
|
118
|
+
"""Close the wrapped transport, releasing its connection pool."""
|
|
119
|
+
self._transport.close()
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
class AsyncCircuitBreakerTransport(AsyncBaseTransport):
|
|
123
|
+
"""An asynchronous transport that guards each host with a circuit breaker.
|
|
124
|
+
|
|
125
|
+
Args:
|
|
126
|
+
transport: The wrapped async transport that performs the request.
|
|
127
|
+
config: Thresholds, window and timing for every host's breaker.
|
|
128
|
+
clock: Time source for the breakers; inject a fake for deterministic
|
|
129
|
+
tests.
|
|
130
|
+
classifier: Failure policy. Defaults to ``HttpStatusClassifier``.
|
|
131
|
+
listener: Observability hooks shared by every host's breaker.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
def __init__(
|
|
135
|
+
self,
|
|
136
|
+
transport: AsyncBaseTransport,
|
|
137
|
+
*,
|
|
138
|
+
config: Config | None = None,
|
|
139
|
+
clock: Clock | None = None,
|
|
140
|
+
classifier: FailureClassifier | None = None,
|
|
141
|
+
listener: EventListener | None = None,
|
|
142
|
+
) -> None:
|
|
143
|
+
self._transport = transport
|
|
144
|
+
self._registry = _build_registry(config, clock, classifier, listener)
|
|
145
|
+
|
|
146
|
+
async def handle_async_request(self, request: Request) -> Response:
|
|
147
|
+
"""Run the request under its host's breaker.
|
|
148
|
+
|
|
149
|
+
Raises:
|
|
150
|
+
CircuitOpenError: If the host's breaker is open.
|
|
151
|
+
"""
|
|
152
|
+
breaker = self._registry.get(request.url.host)
|
|
153
|
+
guarded = breaker(self._transport.handle_async_request)
|
|
154
|
+
return await guarded(request)
|
|
155
|
+
|
|
156
|
+
async def aclose(self) -> None:
|
|
157
|
+
"""Close the wrapped transport, releasing its connection pool."""
|
|
158
|
+
await self._transport.aclose()
|
interlock/listeners.py
ADDED
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
"""Built-in ``EventListener`` implementations (zero-dependency)."""
|
|
2
|
+
|
|
3
|
+
import logging
|
|
4
|
+
|
|
5
|
+
from interlock.outcome import Outcome
|
|
6
|
+
from interlock.state import State
|
|
7
|
+
|
|
8
|
+
__all__ = ('LoggingEventListener',)
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
class LoggingEventListener:
|
|
12
|
+
"""An ``EventListener`` that logs every breaker event via stdlib logging.
|
|
13
|
+
|
|
14
|
+
State changes and rejections log at ``WARNING`` (operationally significant),
|
|
15
|
+
resets at ``INFO``, and individual calls at ``DEBUG`` (high volume).
|
|
16
|
+
|
|
17
|
+
Args:
|
|
18
|
+
logger: Logger to write to. Defaults to ``logging.getLogger('interlock')``.
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
def __init__(self, logger: logging.Logger | None = None) -> None:
|
|
22
|
+
self._log = logger if logger is not None else logging.getLogger('interlock')
|
|
23
|
+
|
|
24
|
+
def on_state_change(self, *, name: str, old: State, new: State) -> None:
|
|
25
|
+
"""Log a state transition at WARNING."""
|
|
26
|
+
self._log.warning('circuit %r: state %s -> %s', name, old, new)
|
|
27
|
+
|
|
28
|
+
def on_call(self, *, name: str, outcome: Outcome, duration: float) -> None:
|
|
29
|
+
"""Log a completed call at DEBUG."""
|
|
30
|
+
self._log.debug('circuit %r: call %s in %.3fs', name, outcome, duration)
|
|
31
|
+
|
|
32
|
+
def on_rejected(self, *, name: str) -> None:
|
|
33
|
+
"""Log a rejected call at WARNING."""
|
|
34
|
+
self._log.warning('circuit %r: call rejected (open)', name)
|
|
35
|
+
|
|
36
|
+
def on_reset(self, *, name: str) -> None:
|
|
37
|
+
"""Log a manual reset at INFO."""
|
|
38
|
+
self._log.info('circuit %r: reset', name)
|
interlock/otel.py
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""OpenTelemetry metrics adapter — requires the ``otel`` extra.
|
|
2
|
+
|
|
3
|
+
This module imports ``opentelemetry`` and is deliberately *not* re-exported from
|
|
4
|
+
``interlock`` so the core stays zero-dependency. Install with
|
|
5
|
+
``pip install interlock[otel]`` and import explicitly::
|
|
6
|
+
|
|
7
|
+
from interlock.otel import OTelEventListener
|
|
8
|
+
|
|
9
|
+
It maps breaker events onto three instruments: a duration histogram per call, a
|
|
10
|
+
counter of rejected calls, and a counter of state transitions (plus resets).
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from opentelemetry import metrics
|
|
14
|
+
from opentelemetry.metrics import Meter
|
|
15
|
+
|
|
16
|
+
from interlock.outcome import Outcome
|
|
17
|
+
from interlock.state import State
|
|
18
|
+
|
|
19
|
+
__all__ = ('OTelEventListener',)
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class OTelEventListener:
|
|
23
|
+
"""Records breaker events as OpenTelemetry metrics.
|
|
24
|
+
|
|
25
|
+
Args:
|
|
26
|
+
meter: Meter to create instruments on. Defaults to
|
|
27
|
+
``opentelemetry.metrics.get_meter('interlock')``.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
def __init__(self, meter: Meter | None = None) -> None:
|
|
31
|
+
meter = meter if meter is not None else metrics.get_meter('interlock')
|
|
32
|
+
self._calls = meter.create_histogram(
|
|
33
|
+
'interlock.call.duration',
|
|
34
|
+
unit='s',
|
|
35
|
+
description='Duration of protected calls.',
|
|
36
|
+
)
|
|
37
|
+
self._rejected = meter.create_counter(
|
|
38
|
+
'interlock.call.rejected',
|
|
39
|
+
description='Calls rejected because the circuit was open.',
|
|
40
|
+
)
|
|
41
|
+
self._state_changes = meter.create_counter(
|
|
42
|
+
'interlock.state.changes',
|
|
43
|
+
description='Circuit breaker state transitions.',
|
|
44
|
+
)
|
|
45
|
+
self._resets = meter.create_counter(
|
|
46
|
+
'interlock.reset',
|
|
47
|
+
description='Manual breaker resets.',
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
def on_state_change(self, *, name: str, old: State, new: State) -> None:
|
|
51
|
+
"""Count a state transition, labelled with the breaker and direction."""
|
|
52
|
+
self._state_changes.add(1, {'breaker': name, 'from': str(old), 'to': str(new)})
|
|
53
|
+
|
|
54
|
+
def on_call(self, *, name: str, outcome: Outcome, duration: float) -> None:
|
|
55
|
+
"""Record the call's duration, labelled with the breaker and outcome."""
|
|
56
|
+
self._calls.record(duration, {'breaker': name, 'outcome': str(outcome)})
|
|
57
|
+
|
|
58
|
+
def on_rejected(self, *, name: str) -> None:
|
|
59
|
+
"""Count a rejected call, labelled with the breaker."""
|
|
60
|
+
self._rejected.add(1, {'breaker': name})
|
|
61
|
+
|
|
62
|
+
def on_reset(self, *, name: str) -> None:
|
|
63
|
+
"""Count a manual reset, labelled with the breaker."""
|
|
64
|
+
self._resets.add(1, {'breaker': name})
|