httpx-hedged 0.2.0__tar.gz → 0.3.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/.github/workflows/ci.yml +11 -4
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/.github/workflows/release.yml +2 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/.gitignore +3 -0
- httpx_hedged-0.3.1/LICENSE +21 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/PKG-INFO +40 -18
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/README.md +37 -17
- httpx_hedged-0.3.1/examples/README.md +41 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/examples/app.py +3 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/examples/example_usage.py +8 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/examples/run_example.sh +1 -1
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/pyproject.toml +12 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/__init__.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_bounded.py +3 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_config.py +12 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_health.py +8 -8
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_matcher.py +1 -1
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_rate.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_rotation.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_scheduler.py +18 -9
- httpx_hedged-0.3.1/src/httpx_hedged/_version.py +24 -0
- httpx_hedged-0.3.1/src/httpx_hedged/sketch/__init__.py +7 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/sketch/_windowed.py +5 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/transport.py +21 -4
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/integration/test_httpx_transport.py +18 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_rate_counter.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_scheduler.py +28 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_windowed_sketch.py +6 -5
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/uv.lock +53 -2
- httpx_hedged-0.2.0/LICENSE +0 -28
- httpx_hedged-0.2.0/src/httpx_hedged/sketch/__init__.py +0 -8
- httpx_hedged-0.2.0/src/httpx_hedged/sketch/_ddsketch.py +0 -188
- httpx_hedged-0.2.0/tests/unit/test_ddsketch.py +0 -91
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/.python-version +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/_stats.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/budget/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/src/httpx_hedged/budget/_token_bucket.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/conftest.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/integration/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_config.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_health.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_matcher.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_rotation.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_stats.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.1}/tests/unit/test_token_bucket.py +0 -0
|
@@ -15,6 +15,8 @@ jobs:
|
|
|
15
15
|
steps:
|
|
16
16
|
- name: Checkout repository
|
|
17
17
|
uses: actions/checkout@v4
|
|
18
|
+
with:
|
|
19
|
+
fetch-depth: 0 # full history, so hatch-vcs can see tags
|
|
18
20
|
|
|
19
21
|
- name: Set up uv
|
|
20
22
|
uses: astral-sh/setup-uv@v5
|
|
@@ -25,12 +27,17 @@ jobs:
|
|
|
25
27
|
- name: Set up Python ${{ matrix.python-version }}
|
|
26
28
|
run: uv python install ${{ matrix.python-version }}
|
|
27
29
|
|
|
28
|
-
|
|
30
|
+
- name: Install dependencies
|
|
31
|
+
run: uv sync --locked --group dev
|
|
32
|
+
|
|
29
33
|
- name: Run Ruff Linter
|
|
30
|
-
run: uv run
|
|
34
|
+
run: uv run ruff check .
|
|
31
35
|
|
|
32
36
|
- name: Run Ruff Formatter Check
|
|
33
|
-
run: uv run
|
|
37
|
+
run: uv run ruff format --check .
|
|
38
|
+
|
|
39
|
+
- name: Run Mypy
|
|
40
|
+
run: uv run mypy src
|
|
34
41
|
|
|
35
42
|
- name: Run Tests with Pytest
|
|
36
|
-
run: uv run
|
|
43
|
+
run: uv run pytest -q
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Brendan Fahy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: httpx-hedged
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.1
|
|
4
4
|
Summary: Adaptive, per-endpoint request hedging transport for httpx. Learns latency percentiles per endpoint with DDSketch, caps hedge rate with a token bucket, and stops hedging under a host/endpoint circuit breaker.
|
|
5
|
+
License-Expression: MIT
|
|
5
6
|
License-File: LICENSE
|
|
6
7
|
Requires-Python: >=3.13
|
|
8
|
+
Requires-Dist: ddsketch>=3.0.1
|
|
7
9
|
Requires-Dist: httpx>=0.27.0
|
|
8
10
|
Description-Content-Type: text/markdown
|
|
9
11
|
|
|
@@ -32,6 +34,10 @@ async def main():
|
|
|
32
34
|
asyncio.run(main())
|
|
33
35
|
```
|
|
34
36
|
|
|
37
|
+
For a runnable, end-to-end demo (a small backend with different latency
|
|
38
|
+
profiles per route, driven by a client that prints a hedge/latency/circuit
|
|
39
|
+
breaker report), see [`examples/`](examples/README.md).
|
|
40
|
+
|
|
35
41
|
With no configuration, `HedgedTransport` learns a p90 latency estimate per
|
|
36
42
|
host (via a [DDSketch](https://arxiv.org/abs/2004.08604) quantile sketch)
|
|
37
43
|
and fires a hedge request whenever the primary exceeds it.
|
|
@@ -39,7 +45,7 @@ and fires a hedge request whenever the primary exceeds it.
|
|
|
39
45
|
## Why per-endpoint?
|
|
40
46
|
|
|
41
47
|
A single host can host wildly different endpoints. Learning one latency
|
|
42
|
-
distribution per *host
|
|
48
|
+
distribution per *host*, rather than per endpoint, means a handful of
|
|
43
49
|
calls to a slow endpoint skew the hedge trigger for a fast one sharing the
|
|
44
50
|
same host:
|
|
45
51
|
|
|
@@ -50,7 +56,7 @@ GET /api/v1/bulk-export median 900ms, low RPS
|
|
|
50
56
|
|
|
51
57
|
`HedgedTransport` lets you register per-endpoint config up front. Each
|
|
52
58
|
registered endpoint gets its own latency sketch, rate estimate, and hedge
|
|
53
|
-
budget
|
|
59
|
+
budget, all still funneled through a **single inner transport and
|
|
54
60
|
connection pool**, unlike using `httpx` `mounts={}`, which would mean one
|
|
55
61
|
connection pool per pattern:
|
|
56
62
|
|
|
@@ -68,7 +74,7 @@ endpoints at all).
|
|
|
68
74
|
|
|
69
75
|
Route patterns may contain `{name}` placeholders or a bare `*` for a single
|
|
70
76
|
path segment, e.g. `/api/v1/users/{id}`. Patterns are matched in
|
|
71
|
-
registration order
|
|
77
|
+
registration order, so register more specific patterns first.
|
|
72
78
|
|
|
73
79
|
## Hardcoded vs. adaptive delay
|
|
74
80
|
|
|
@@ -79,8 +85,8 @@ percentile:
|
|
|
79
85
|
transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.95))
|
|
80
86
|
```
|
|
81
87
|
|
|
82
|
-
For an endpoint where you already know the right delay
|
|
83
|
-
deterministic behavior without a warmup period
|
|
88
|
+
For an endpoint where you already know the right delay, or want
|
|
89
|
+
deterministic behavior without a warmup period, hardcode it instead:
|
|
84
90
|
|
|
85
91
|
```python
|
|
86
92
|
transport.register("GET", "/api/v1/health", EndpointConfig(hedge_delay=0.05))
|
|
@@ -92,7 +98,7 @@ observability; it just isn't consulted for the hedge-delay decision.
|
|
|
92
98
|
## Explicit endpoint override
|
|
93
99
|
|
|
94
100
|
Auto-matching not precise enough for a particular call site (or you'd
|
|
95
|
-
rather not register a pattern)? Tag the request directly
|
|
101
|
+
rather not register a pattern)? Tag the request directly; this bypasses
|
|
96
102
|
pattern matching entirely:
|
|
97
103
|
|
|
98
104
|
```python
|
|
@@ -159,7 +165,7 @@ request ──────┤
|
|
|
159
165
|
|
|
160
166
|
Only idempotent methods (`GET`, `HEAD`, `OPTIONS`) are ever hedged, to avoid
|
|
161
167
|
duplicating side effects. A request with a body is also never hedged, even
|
|
162
|
-
if the method is idempotent
|
|
168
|
+
if the method is idempotent: the primary and hedge send the same
|
|
163
169
|
`httpx.Request` object, and a body backed by a one-shot stream can't be
|
|
164
170
|
safely read twice.
|
|
165
171
|
|
|
@@ -168,7 +174,9 @@ safely read twice.
|
|
|
168
174
|
Each tracked key (an endpoint, or the per-host fallback) gets its own
|
|
169
175
|
sliding-window DDSketch pair that rotates every `window_duration` seconds.
|
|
170
176
|
DDSketch gives relative-error quantile guarantees regardless of the
|
|
171
|
-
underlying latency distribution's shape.
|
|
177
|
+
underlying latency distribution's shape. The sketch itself comes from the
|
|
178
|
+
[`ddsketch`](https://github.com/DataDog/sketches-py) package (DataDog's
|
|
179
|
+
reference implementation of the paper below).
|
|
172
180
|
|
|
173
181
|
### Token bucket budget
|
|
174
182
|
|
|
@@ -184,13 +192,13 @@ endpoint.
|
|
|
184
192
|
|
|
185
193
|
A closed / open / half-open circuit breaker tracks request success/failure
|
|
186
194
|
at **two independent tiers**: one breaker per host, one breaker per
|
|
187
|
-
endpoint key. Either tripping open suppresses hedging for its scope
|
|
195
|
+
endpoint key. Either tripping open suppresses hedging for its scope: a
|
|
188
196
|
host-level trip disables hedging for every endpoint on that host, while an
|
|
189
197
|
endpoint-level trip disables hedging only for that one endpoint.
|
|
190
198
|
|
|
191
199
|
Crucially, the breaker **only ever suppresses the hedge request**. The
|
|
192
200
|
primary request is always sent, and its result or exception is always
|
|
193
|
-
returned to the caller normally
|
|
201
|
+
returned to the caller normally. This is not a request-blocking circuit
|
|
194
202
|
breaker, so hedging can't pile extra load onto a backend that's already
|
|
195
203
|
struggling.
|
|
196
204
|
|
|
@@ -201,7 +209,7 @@ HALF_OPEN ──(trial requests mostly succeed)──▶ CLOSED
|
|
|
201
209
|
HALF_OPEN ──(trial requests mostly fail)────▶ OPEN
|
|
202
210
|
```
|
|
203
211
|
|
|
204
|
-
Note: health is recorded from the *winning* task's outcome only
|
|
212
|
+
Note: health is recorded from the *winning* task's outcome only. A
|
|
205
213
|
cancelled loser's real outcome is unknowable, and losers are cancelled
|
|
206
214
|
deliberately (not doing so would defeat the point of reducing load on a
|
|
207
215
|
struggling backend).
|
|
@@ -225,6 +233,20 @@ print(transport.health.host_state("api.example.com"))
|
|
|
225
233
|
`primary_wins`, `budget_exhausted`, `warmup_requests`, `circuit_blocked`,
|
|
226
234
|
and `errors` per key, plus a global aggregate.
|
|
227
235
|
|
|
236
|
+
To see the learned latency estimate itself (e.g. the current p90 driving
|
|
237
|
+
the hedge trigger), query `latency_quantile()` with the same key format:
|
|
238
|
+
|
|
239
|
+
```python
|
|
240
|
+
name = transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.90))
|
|
241
|
+
# ... after running some traffic ...
|
|
242
|
+
p90 = transport.latency_quantile(f"endpoint:{name}", 0.9)
|
|
243
|
+
if p90 is not None:
|
|
244
|
+
print(f"p90: {p90 * 1000:.1f}ms")
|
|
245
|
+
```
|
|
246
|
+
|
|
247
|
+
Returns `None` if the key hasn't recorded any samples yet. You can query
|
|
248
|
+
any quantile, not just the one driving the hedge decision.
|
|
249
|
+
|
|
228
250
|
### Push-based hooks: metrics and alerting
|
|
229
251
|
|
|
230
252
|
Polling snapshots works for dashboards, but alerting on a circuit-breaker
|
|
@@ -267,14 +289,14 @@ async with httpx.AsyncClient(transport=transport) as client:
|
|
|
267
289
|
```
|
|
268
290
|
|
|
269
291
|
`on_hedge_fired` is called with the key each time a hedge request is
|
|
270
|
-
actually launched
|
|
292
|
+
actually launched, after the idempotency, circuit-breaker, and budget
|
|
271
293
|
gates have all passed, so it only fires for hedges that were genuinely
|
|
272
294
|
sent. `on_circuit_open` is called once per OPEN transition (not on every
|
|
273
295
|
suppressed hedge while it stays open), so it's safe to wire straight into
|
|
274
296
|
an alerting/paging pipeline without flooding it.
|
|
275
297
|
|
|
276
298
|
Both callbacks run synchronously on the request path, so keep them fast
|
|
277
|
-
(increment a counter, log a line)
|
|
299
|
+
(increment a counter, log a line); don't do network I/O in them directly.
|
|
278
300
|
|
|
279
301
|
## Relationship to hedge-python
|
|
280
302
|
|
|
@@ -290,11 +312,11 @@ for the motivating scenario.
|
|
|
290
312
|
|
|
291
313
|
## References
|
|
292
314
|
|
|
293
|
-
- [The Tail at Scale](https://research.google/pubs/pub40801/)
|
|
294
|
-
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604)
|
|
295
|
-
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python)
|
|
315
|
+
- [The Tail at Scale](https://research.google/pubs/pub40801/): Google's paper on tail latency
|
|
316
|
+
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604): Masson et al., VLDB 2019
|
|
317
|
+
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python): the project this one is modeled after
|
|
296
318
|
- [httpx documentation](https://www.python-httpx.org/)
|
|
297
319
|
|
|
298
320
|
## License
|
|
299
321
|
|
|
300
|
-
|
|
322
|
+
MIT License. See [LICENSE](LICENSE) file for details.
|
|
@@ -23,6 +23,10 @@ async def main():
|
|
|
23
23
|
asyncio.run(main())
|
|
24
24
|
```
|
|
25
25
|
|
|
26
|
+
For a runnable, end-to-end demo (a small backend with different latency
|
|
27
|
+
profiles per route, driven by a client that prints a hedge/latency/circuit
|
|
28
|
+
breaker report), see [`examples/`](examples/README.md).
|
|
29
|
+
|
|
26
30
|
With no configuration, `HedgedTransport` learns a p90 latency estimate per
|
|
27
31
|
host (via a [DDSketch](https://arxiv.org/abs/2004.08604) quantile sketch)
|
|
28
32
|
and fires a hedge request whenever the primary exceeds it.
|
|
@@ -30,7 +34,7 @@ and fires a hedge request whenever the primary exceeds it.
|
|
|
30
34
|
## Why per-endpoint?
|
|
31
35
|
|
|
32
36
|
A single host can host wildly different endpoints. Learning one latency
|
|
33
|
-
distribution per *host
|
|
37
|
+
distribution per *host*, rather than per endpoint, means a handful of
|
|
34
38
|
calls to a slow endpoint skew the hedge trigger for a fast one sharing the
|
|
35
39
|
same host:
|
|
36
40
|
|
|
@@ -41,7 +45,7 @@ GET /api/v1/bulk-export median 900ms, low RPS
|
|
|
41
45
|
|
|
42
46
|
`HedgedTransport` lets you register per-endpoint config up front. Each
|
|
43
47
|
registered endpoint gets its own latency sketch, rate estimate, and hedge
|
|
44
|
-
budget
|
|
48
|
+
budget, all still funneled through a **single inner transport and
|
|
45
49
|
connection pool**, unlike using `httpx` `mounts={}`, which would mean one
|
|
46
50
|
connection pool per pattern:
|
|
47
51
|
|
|
@@ -59,7 +63,7 @@ endpoints at all).
|
|
|
59
63
|
|
|
60
64
|
Route patterns may contain `{name}` placeholders or a bare `*` for a single
|
|
61
65
|
path segment, e.g. `/api/v1/users/{id}`. Patterns are matched in
|
|
62
|
-
registration order
|
|
66
|
+
registration order, so register more specific patterns first.
|
|
63
67
|
|
|
64
68
|
## Hardcoded vs. adaptive delay
|
|
65
69
|
|
|
@@ -70,8 +74,8 @@ percentile:
|
|
|
70
74
|
transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.95))
|
|
71
75
|
```
|
|
72
76
|
|
|
73
|
-
For an endpoint where you already know the right delay
|
|
74
|
-
deterministic behavior without a warmup period
|
|
77
|
+
For an endpoint where you already know the right delay, or want
|
|
78
|
+
deterministic behavior without a warmup period, hardcode it instead:
|
|
75
79
|
|
|
76
80
|
```python
|
|
77
81
|
transport.register("GET", "/api/v1/health", EndpointConfig(hedge_delay=0.05))
|
|
@@ -83,7 +87,7 @@ observability; it just isn't consulted for the hedge-delay decision.
|
|
|
83
87
|
## Explicit endpoint override
|
|
84
88
|
|
|
85
89
|
Auto-matching not precise enough for a particular call site (or you'd
|
|
86
|
-
rather not register a pattern)? Tag the request directly
|
|
90
|
+
rather not register a pattern)? Tag the request directly; this bypasses
|
|
87
91
|
pattern matching entirely:
|
|
88
92
|
|
|
89
93
|
```python
|
|
@@ -150,7 +154,7 @@ request ──────┤
|
|
|
150
154
|
|
|
151
155
|
Only idempotent methods (`GET`, `HEAD`, `OPTIONS`) are ever hedged, to avoid
|
|
152
156
|
duplicating side effects. A request with a body is also never hedged, even
|
|
153
|
-
if the method is idempotent
|
|
157
|
+
if the method is idempotent: the primary and hedge send the same
|
|
154
158
|
`httpx.Request` object, and a body backed by a one-shot stream can't be
|
|
155
159
|
safely read twice.
|
|
156
160
|
|
|
@@ -159,7 +163,9 @@ safely read twice.
|
|
|
159
163
|
Each tracked key (an endpoint, or the per-host fallback) gets its own
|
|
160
164
|
sliding-window DDSketch pair that rotates every `window_duration` seconds.
|
|
161
165
|
DDSketch gives relative-error quantile guarantees regardless of the
|
|
162
|
-
underlying latency distribution's shape.
|
|
166
|
+
underlying latency distribution's shape. The sketch itself comes from the
|
|
167
|
+
[`ddsketch`](https://github.com/DataDog/sketches-py) package (DataDog's
|
|
168
|
+
reference implementation of the paper below).
|
|
163
169
|
|
|
164
170
|
### Token bucket budget
|
|
165
171
|
|
|
@@ -175,13 +181,13 @@ endpoint.
|
|
|
175
181
|
|
|
176
182
|
A closed / open / half-open circuit breaker tracks request success/failure
|
|
177
183
|
at **two independent tiers**: one breaker per host, one breaker per
|
|
178
|
-
endpoint key. Either tripping open suppresses hedging for its scope
|
|
184
|
+
endpoint key. Either tripping open suppresses hedging for its scope: a
|
|
179
185
|
host-level trip disables hedging for every endpoint on that host, while an
|
|
180
186
|
endpoint-level trip disables hedging only for that one endpoint.
|
|
181
187
|
|
|
182
188
|
Crucially, the breaker **only ever suppresses the hedge request**. The
|
|
183
189
|
primary request is always sent, and its result or exception is always
|
|
184
|
-
returned to the caller normally
|
|
190
|
+
returned to the caller normally. This is not a request-blocking circuit
|
|
185
191
|
breaker, so hedging can't pile extra load onto a backend that's already
|
|
186
192
|
struggling.
|
|
187
193
|
|
|
@@ -192,7 +198,7 @@ HALF_OPEN ──(trial requests mostly succeed)──▶ CLOSED
|
|
|
192
198
|
HALF_OPEN ──(trial requests mostly fail)────▶ OPEN
|
|
193
199
|
```
|
|
194
200
|
|
|
195
|
-
Note: health is recorded from the *winning* task's outcome only
|
|
201
|
+
Note: health is recorded from the *winning* task's outcome only. A
|
|
196
202
|
cancelled loser's real outcome is unknowable, and losers are cancelled
|
|
197
203
|
deliberately (not doing so would defeat the point of reducing load on a
|
|
198
204
|
struggling backend).
|
|
@@ -216,6 +222,20 @@ print(transport.health.host_state("api.example.com"))
|
|
|
216
222
|
`primary_wins`, `budget_exhausted`, `warmup_requests`, `circuit_blocked`,
|
|
217
223
|
and `errors` per key, plus a global aggregate.
|
|
218
224
|
|
|
225
|
+
To see the learned latency estimate itself (e.g. the current p90 driving
|
|
226
|
+
the hedge trigger), query `latency_quantile()` with the same key format:
|
|
227
|
+
|
|
228
|
+
```python
|
|
229
|
+
name = transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.90))
|
|
230
|
+
# ... after running some traffic ...
|
|
231
|
+
p90 = transport.latency_quantile(f"endpoint:{name}", 0.9)
|
|
232
|
+
if p90 is not None:
|
|
233
|
+
print(f"p90: {p90 * 1000:.1f}ms")
|
|
234
|
+
```
|
|
235
|
+
|
|
236
|
+
Returns `None` if the key hasn't recorded any samples yet. You can query
|
|
237
|
+
any quantile, not just the one driving the hedge decision.
|
|
238
|
+
|
|
219
239
|
### Push-based hooks: metrics and alerting
|
|
220
240
|
|
|
221
241
|
Polling snapshots works for dashboards, but alerting on a circuit-breaker
|
|
@@ -258,14 +278,14 @@ async with httpx.AsyncClient(transport=transport) as client:
|
|
|
258
278
|
```
|
|
259
279
|
|
|
260
280
|
`on_hedge_fired` is called with the key each time a hedge request is
|
|
261
|
-
actually launched
|
|
281
|
+
actually launched, after the idempotency, circuit-breaker, and budget
|
|
262
282
|
gates have all passed, so it only fires for hedges that were genuinely
|
|
263
283
|
sent. `on_circuit_open` is called once per OPEN transition (not on every
|
|
264
284
|
suppressed hedge while it stays open), so it's safe to wire straight into
|
|
265
285
|
an alerting/paging pipeline without flooding it.
|
|
266
286
|
|
|
267
287
|
Both callbacks run synchronously on the request path, so keep them fast
|
|
268
|
-
(increment a counter, log a line)
|
|
288
|
+
(increment a counter, log a line); don't do network I/O in them directly.
|
|
269
289
|
|
|
270
290
|
## Relationship to hedge-python
|
|
271
291
|
|
|
@@ -281,11 +301,11 @@ for the motivating scenario.
|
|
|
281
301
|
|
|
282
302
|
## References
|
|
283
303
|
|
|
284
|
-
- [The Tail at Scale](https://research.google/pubs/pub40801/)
|
|
285
|
-
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604)
|
|
286
|
-
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python)
|
|
304
|
+
- [The Tail at Scale](https://research.google/pubs/pub40801/): Google's paper on tail latency
|
|
305
|
+
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604): Masson et al., VLDB 2019
|
|
306
|
+
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python): the project this one is modeled after
|
|
287
307
|
- [httpx documentation](https://www.python-httpx.org/)
|
|
288
308
|
|
|
289
309
|
## License
|
|
290
310
|
|
|
291
|
-
|
|
311
|
+
MIT License. See [LICENSE](LICENSE) file for details.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Examples
|
|
2
|
+
|
|
3
|
+
A runnable demo of `HedgedTransport` against a small local backend with
|
|
4
|
+
three different latency profiles.
|
|
5
|
+
|
|
6
|
+
| File | What it is |
|
|
7
|
+
|---|---|
|
|
8
|
+
| `app.py` | A tiny FastAPI backend exposing `/fast`, `/slow`, and `/flaky` routes, each with a different latency profile. |
|
|
9
|
+
| `example_usage.py` | A client that registers per-endpoint hedge config, drives load through `HedgedTransport` against `app.py`, and prints a stats/latency/circuit-breaker report. |
|
|
10
|
+
| `run_example.sh` | Installs deps, starts `app.py`, runs `example_usage.py` against it, and tears the server down. The easiest way to see the whole thing end to end. |
|
|
11
|
+
|
|
12
|
+
## Running it
|
|
13
|
+
|
|
14
|
+
From the repo root:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
./examples/run_example.sh
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Or run the pieces yourself:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uv sync --group examples
|
|
24
|
+
uv run uvicorn examples.app:app --host 127.0.0.1 --port 8000 &
|
|
25
|
+
uv run python examples/example_usage.py
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## What to look for in the output
|
|
29
|
+
|
|
30
|
+
- `/fast` rarely triggers a hedge, since the primary is almost always
|
|
31
|
+
faster than the learned delay.
|
|
32
|
+
- `/slow` hedges often, but the hedge rarely *wins*, because the backup is
|
|
33
|
+
just as slow as the primary on this uniformly slow endpoint.
|
|
34
|
+
- `/flaky` is the interesting case: after warmup, the learned p90 sits near
|
|
35
|
+
the fast latency, so the rare slow primary gets hedged against a fresh
|
|
36
|
+
(usually fast) request and the hedge usually wins the race.
|
|
37
|
+
|
|
38
|
+
The final report prints per-endpoint stats (`total`, `hedged`,
|
|
39
|
+
`hedge_wins`, `primary_wins`, ...), each endpoint's learned p90 latency
|
|
40
|
+
estimate (via `transport.latency_quantile()`), and each endpoint's circuit
|
|
41
|
+
breaker state.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""Tiny FastAPI backend with three latency profiles, for the hedging demo.
|
|
2
2
|
|
|
3
|
-
GET /fast
|
|
4
|
-
GET /slow
|
|
5
|
-
GET /flaky
|
|
3
|
+
GET /fast always responds in ~5-15ms
|
|
4
|
+
GET /slow always takes 800ms
|
|
5
|
+
GET /flaky 90% of requests are fast (~5-15ms), 10% take ~900ms
|
|
6
6
|
|
|
7
7
|
Run directly with uvicorn, e.g.:
|
|
8
8
|
|
|
@@ -39,7 +39,7 @@ CONCURRENCY = 6
|
|
|
39
39
|
|
|
40
40
|
# The hedge token bucket auto-estimates RPS over a rolling ~10s window by
|
|
41
41
|
# default, which under-counts traffic sent in a shorter burst like this
|
|
42
|
-
# demo's
|
|
42
|
+
# demo's, so budget starves and hedges rarely fire. Pinning estimated_rps
|
|
43
43
|
# (roughly concurrency / expected latency) avoids that ramp-up and keeps
|
|
44
44
|
# the demo's hedge behavior visible run over run. /fast is left on the
|
|
45
45
|
# default auto-estimate since it barely ever needs to hedge anyway.
|
|
@@ -91,6 +91,12 @@ def print_report(transport: HedgedTransport) -> None:
|
|
|
91
91
|
print(f"\ntotal requests sent: {global_snap.total_requests}")
|
|
92
92
|
print(f"total hedges fired: {global_snap.hedged_requests}")
|
|
93
93
|
|
|
94
|
+
print("\n=== learned p90 latency estimates ===")
|
|
95
|
+
for route in ROUTES:
|
|
96
|
+
key = f"endpoint:{route}"
|
|
97
|
+
p90 = transport.latency_quantile(key, 0.9)
|
|
98
|
+
print(f"{key:<16} {f'{p90 * 1000:.1f}ms' if p90 is not None else 'n/a'}")
|
|
99
|
+
|
|
94
100
|
print("\n=== circuit breaker state ===")
|
|
95
101
|
for route in ROUTES:
|
|
96
102
|
key = f"endpoint:{route}"
|
|
@@ -104,7 +110,7 @@ async def wait_for_server(client: httpx.AsyncClient) -> None:
|
|
|
104
110
|
response.raise_for_status()
|
|
105
111
|
except httpx.HTTPError:
|
|
106
112
|
print(
|
|
107
|
-
f"Could not reach {BASE_URL}
|
|
113
|
+
f"Could not reach {BASE_URL}: is examples/app.py running?",
|
|
108
114
|
file=sys.stderr,
|
|
109
115
|
)
|
|
110
116
|
sys.exit(1)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# Installs deps, starts the demo FastAPI backend, runs the hedging demo
|
|
3
|
-
# client against it, and prints the stats report
|
|
3
|
+
# client against it, and prints the stats report, then tears the server
|
|
4
4
|
# down. Run from anywhere:
|
|
5
5
|
#
|
|
6
6
|
# ./examples/run_example.sh
|
|
@@ -1,10 +1,13 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "httpx-hedged"
|
|
3
|
-
|
|
3
|
+
dynamic = ["version"]
|
|
4
4
|
description = "Adaptive, per-endpoint request hedging transport for httpx. Learns latency percentiles per endpoint with DDSketch, caps hedge rate with a token bucket, and stops hedging under a host/endpoint circuit breaker."
|
|
5
5
|
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
6
8
|
requires-python = ">=3.13"
|
|
7
9
|
dependencies = [
|
|
10
|
+
"ddsketch>=3.0.1",
|
|
8
11
|
"httpx>=0.27.0",
|
|
9
12
|
]
|
|
10
13
|
|
|
@@ -15,6 +18,7 @@ dev = [
|
|
|
15
18
|
"pytest>=9.0.1",
|
|
16
19
|
"pytest-asyncio>=1.3.0",
|
|
17
20
|
"python-lsp-server[all]>=1.13.2",
|
|
21
|
+
"ruff>=0.14.0",
|
|
18
22
|
]
|
|
19
23
|
examples = [
|
|
20
24
|
"fastapi>=0.115",
|
|
@@ -22,12 +26,18 @@ examples = [
|
|
|
22
26
|
]
|
|
23
27
|
|
|
24
28
|
[build-system]
|
|
25
|
-
requires = ["hatchling"]
|
|
29
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
26
30
|
build-backend = "hatchling.build"
|
|
27
31
|
|
|
28
32
|
[tool.hatch.build.targets.wheel]
|
|
29
33
|
packages = ["src/httpx_hedged"]
|
|
30
34
|
|
|
35
|
+
[tool.hatch.version]
|
|
36
|
+
source = "vcs"
|
|
37
|
+
|
|
38
|
+
[tool.hatch.build.hooks.vcs]
|
|
39
|
+
version-file = "src/httpx_hedged/_version.py"
|
|
40
|
+
|
|
31
41
|
[tool.pytest.ini_options]
|
|
32
42
|
testpaths = ["tests"]
|
|
33
43
|
asyncio_mode = "auto"
|
|
@@ -18,6 +18,7 @@ from httpx_hedged._config import CircuitBreakerConfig, EndpointConfig, HedgeConf
|
|
|
18
18
|
from httpx_hedged._health import CircuitBreaker, CircuitState, HealthRegistry
|
|
19
19
|
from httpx_hedged._matcher import Route, UnknownHedgeEndpointError
|
|
20
20
|
from httpx_hedged._stats import Stats, StatsRegistry, StatsSnapshot
|
|
21
|
+
from httpx_hedged._version import __version__
|
|
21
22
|
from httpx_hedged.transport import HedgedTransport
|
|
22
23
|
|
|
23
24
|
__all__ = [
|
|
@@ -33,6 +34,5 @@ __all__ = [
|
|
|
33
34
|
"StatsRegistry",
|
|
34
35
|
"StatsSnapshot",
|
|
35
36
|
"UnknownHedgeEndpointError",
|
|
37
|
+
"__version__",
|
|
36
38
|
]
|
|
37
|
-
|
|
38
|
-
__version__ = "0.2.0"
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Per-key state (sketches, breakers, stats) is created lazily, keyed by
|
|
4
4
|
endpoint name or a per-host fallback. Nothing ever evicted those keys, so a
|
|
5
|
-
caller that talks to high-cardinality hosts (per-tenant subdomains, or
|
|
6
|
-
the pathological case
|
|
5
|
+
caller that talks to high-cardinality hosts (per-tenant subdomains, or, in
|
|
6
|
+
the pathological case, one key per unique URL when a request has no
|
|
7
7
|
parseable host) would grow these maps forever. Bounding them with LRU
|
|
8
8
|
eviction keeps memory bounded without requiring callers to know about it.
|
|
9
9
|
"""
|
|
@@ -17,7 +17,7 @@ DEFAULT_MAX_SIZE = 10_000
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class BoundedRegistry[V]:
|
|
20
|
-
"""Not thread-safe by itself
|
|
20
|
+
"""Not thread-safe by itself. Callers hold their own lock, as with the
|
|
21
21
|
plain dicts this replaces."""
|
|
22
22
|
|
|
23
23
|
def __init__(self, max_size: int = DEFAULT_MAX_SIZE) -> None:
|
|
@@ -39,7 +39,7 @@ def _validate_common(config: HedgeConfig | EndpointConfig) -> None:
|
|
|
39
39
|
class CircuitBreakerConfig:
|
|
40
40
|
"""Configuration for the health circuit breaker that gates hedging.
|
|
41
41
|
|
|
42
|
-
Tripping the breaker only ever suppresses the *hedge* request
|
|
42
|
+
Tripping the breaker only ever suppresses the *hedge* request. The
|
|
43
43
|
primary request is always sent and its result or exception is always
|
|
44
44
|
returned to the caller normally.
|
|
45
45
|
"""
|
|
@@ -163,7 +163,7 @@ class EndpointConfig:
|
|
|
163
163
|
|
|
164
164
|
#: Health circuit-breaker configuration. When set, replaces the
|
|
165
165
|
#: default breaker config as a whole object rather than being merged
|
|
166
|
-
#: field-by-field
|
|
166
|
+
#: field-by-field. For a partial override, use
|
|
167
167
|
#: ``dataclasses.replace(default.circuit_breaker, ...)``.
|
|
168
168
|
circuit_breaker: CircuitBreakerConfig | None = None
|
|
169
169
|
|
|
@@ -175,7 +175,16 @@ class EndpointConfig:
|
|
|
175
175
|
|
|
176
176
|
@dataclass(frozen=True, slots=True)
|
|
177
177
|
class EffectiveConfig:
|
|
178
|
-
"""Fully-resolved hedge configuration for a single key
|
|
178
|
+
"""Fully-resolved hedge configuration for a single key.
|
|
179
|
+
|
|
180
|
+
Every "inherit the default" field from ``EndpointConfig`` has been
|
|
181
|
+
resolved away by ``resolve()``. ``estimated_rps`` and ``hedge_delay``
|
|
182
|
+
stay ``Optional`` here regardless, since their ``None`` means something
|
|
183
|
+
different than "unresolved": ``estimated_rps=None`` means "auto-estimate
|
|
184
|
+
from traffic" and ``hedge_delay=None`` means "no hardcoded delay, learn
|
|
185
|
+
one from the sketch" (see ``is_hardcoded``). Both are real, permanent
|
|
186
|
+
states, not resolution artifacts.
|
|
187
|
+
"""
|
|
179
188
|
|
|
180
189
|
percentile: float
|
|
181
190
|
budget_percent: float
|