httpx-hedged 0.2.0__tar.gz → 0.3.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/.github/workflows/ci.yml +11 -4
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/.github/workflows/release.yml +2 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/.gitignore +3 -0
- httpx_hedged-0.3.0/LICENSE +21 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/PKG-INFO +36 -17
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/README.md +34 -16
- httpx_hedged-0.3.0/examples/README.md +41 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/examples/app.py +3 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/examples/example_usage.py +8 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/examples/run_example.sh +1 -1
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/pyproject.toml +11 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/__init__.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_bounded.py +3 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_config.py +12 -3
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_health.py +8 -8
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_matcher.py +1 -1
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_rate.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_rotation.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_scheduler.py +18 -9
- httpx_hedged-0.3.0/src/httpx_hedged/_version.py +24 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/sketch/_windowed.py +1 -1
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/transport.py +21 -4
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/integration/test_httpx_transport.py +18 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_rate_counter.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_scheduler.py +28 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_windowed_sketch.py +2 -2
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/uv.lock +27 -1
- httpx_hedged-0.2.0/LICENSE +0 -28
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/.python-version +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/_stats.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/budget/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/budget/_token_bucket.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/sketch/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/src/httpx_hedged/sketch/_ddsketch.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/conftest.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/integration/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/__init__.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_config.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_ddsketch.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_health.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_matcher.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_rotation.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_stats.py +0 -0
- {httpx_hedged-0.2.0 → httpx_hedged-0.3.0}/tests/unit/test_token_bucket.py +0 -0
|
@@ -15,6 +15,8 @@ jobs:
|
|
|
15
15
|
steps:
|
|
16
16
|
- name: Checkout repository
|
|
17
17
|
uses: actions/checkout@v4
|
|
18
|
+
with:
|
|
19
|
+
fetch-depth: 0 # full history, so hatch-vcs can see tags
|
|
18
20
|
|
|
19
21
|
- name: Set up uv
|
|
20
22
|
uses: astral-sh/setup-uv@v5
|
|
@@ -25,12 +27,17 @@ jobs:
|
|
|
25
27
|
- name: Set up Python ${{ matrix.python-version }}
|
|
26
28
|
run: uv python install ${{ matrix.python-version }}
|
|
27
29
|
|
|
28
|
-
|
|
30
|
+
- name: Install dependencies
|
|
31
|
+
run: uv sync --locked --group dev
|
|
32
|
+
|
|
29
33
|
- name: Run Ruff Linter
|
|
30
|
-
run: uv run
|
|
34
|
+
run: uv run ruff check .
|
|
31
35
|
|
|
32
36
|
- name: Run Ruff Formatter Check
|
|
33
|
-
run: uv run
|
|
37
|
+
run: uv run ruff format --check .
|
|
38
|
+
|
|
39
|
+
- name: Run Mypy
|
|
40
|
+
run: uv run mypy src
|
|
34
41
|
|
|
35
42
|
- name: Run Tests with Pytest
|
|
36
|
-
run: uv run
|
|
43
|
+
run: uv run pytest -q
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2025 Brendan Fahy
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -1,7 +1,8 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: httpx-hedged
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.3.0
|
|
4
4
|
Summary: Adaptive, per-endpoint request hedging transport for httpx. Learns latency percentiles per endpoint with DDSketch, caps hedge rate with a token bucket, and stops hedging under a host/endpoint circuit breaker.
|
|
5
|
+
License-Expression: MIT
|
|
5
6
|
License-File: LICENSE
|
|
6
7
|
Requires-Python: >=3.13
|
|
7
8
|
Requires-Dist: httpx>=0.27.0
|
|
@@ -32,6 +33,10 @@ async def main():
|
|
|
32
33
|
asyncio.run(main())
|
|
33
34
|
```
|
|
34
35
|
|
|
36
|
+
For a runnable, end-to-end demo (a small backend with different latency
|
|
37
|
+
profiles per route, driven by a client that prints a hedge/latency/circuit
|
|
38
|
+
breaker report), see [`examples/`](examples/README.md).
|
|
39
|
+
|
|
35
40
|
With no configuration, `HedgedTransport` learns a p90 latency estimate per
|
|
36
41
|
host (via a [DDSketch](https://arxiv.org/abs/2004.08604) quantile sketch)
|
|
37
42
|
and fires a hedge request whenever the primary exceeds it.
|
|
@@ -39,7 +44,7 @@ and fires a hedge request whenever the primary exceeds it.
|
|
|
39
44
|
## Why per-endpoint?
|
|
40
45
|
|
|
41
46
|
A single host can host wildly different endpoints. Learning one latency
|
|
42
|
-
distribution per *host
|
|
47
|
+
distribution per *host*, rather than per endpoint, means a handful of
|
|
43
48
|
calls to a slow endpoint skew the hedge trigger for a fast one sharing the
|
|
44
49
|
same host:
|
|
45
50
|
|
|
@@ -50,7 +55,7 @@ GET /api/v1/bulk-export median 900ms, low RPS
|
|
|
50
55
|
|
|
51
56
|
`HedgedTransport` lets you register per-endpoint config up front. Each
|
|
52
57
|
registered endpoint gets its own latency sketch, rate estimate, and hedge
|
|
53
|
-
budget
|
|
58
|
+
budget, all still funneled through a **single inner transport and
|
|
54
59
|
connection pool**, unlike using `httpx` `mounts={}`, which would mean one
|
|
55
60
|
connection pool per pattern:
|
|
56
61
|
|
|
@@ -68,7 +73,7 @@ endpoints at all).
|
|
|
68
73
|
|
|
69
74
|
Route patterns may contain `{name}` placeholders or a bare `*` for a single
|
|
70
75
|
path segment, e.g. `/api/v1/users/{id}`. Patterns are matched in
|
|
71
|
-
registration order
|
|
76
|
+
registration order, so register more specific patterns first.
|
|
72
77
|
|
|
73
78
|
## Hardcoded vs. adaptive delay
|
|
74
79
|
|
|
@@ -79,8 +84,8 @@ percentile:
|
|
|
79
84
|
transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.95))
|
|
80
85
|
```
|
|
81
86
|
|
|
82
|
-
For an endpoint where you already know the right delay
|
|
83
|
-
deterministic behavior without a warmup period
|
|
87
|
+
For an endpoint where you already know the right delay, or want
|
|
88
|
+
deterministic behavior without a warmup period, hardcode it instead:
|
|
84
89
|
|
|
85
90
|
```python
|
|
86
91
|
transport.register("GET", "/api/v1/health", EndpointConfig(hedge_delay=0.05))
|
|
@@ -92,7 +97,7 @@ observability; it just isn't consulted for the hedge-delay decision.
|
|
|
92
97
|
## Explicit endpoint override
|
|
93
98
|
|
|
94
99
|
Auto-matching not precise enough for a particular call site (or you'd
|
|
95
|
-
rather not register a pattern)? Tag the request directly
|
|
100
|
+
rather not register a pattern)? Tag the request directly; this bypasses
|
|
96
101
|
pattern matching entirely:
|
|
97
102
|
|
|
98
103
|
```python
|
|
@@ -159,7 +164,7 @@ request ──────┤
|
|
|
159
164
|
|
|
160
165
|
Only idempotent methods (`GET`, `HEAD`, `OPTIONS`) are ever hedged, to avoid
|
|
161
166
|
duplicating side effects. A request with a body is also never hedged, even
|
|
162
|
-
if the method is idempotent
|
|
167
|
+
if the method is idempotent: the primary and hedge send the same
|
|
163
168
|
`httpx.Request` object, and a body backed by a one-shot stream can't be
|
|
164
169
|
safely read twice.
|
|
165
170
|
|
|
@@ -184,13 +189,13 @@ endpoint.
|
|
|
184
189
|
|
|
185
190
|
A closed / open / half-open circuit breaker tracks request success/failure
|
|
186
191
|
at **two independent tiers**: one breaker per host, one breaker per
|
|
187
|
-
endpoint key. Either tripping open suppresses hedging for its scope
|
|
192
|
+
endpoint key. Either tripping open suppresses hedging for its scope: a
|
|
188
193
|
host-level trip disables hedging for every endpoint on that host, while an
|
|
189
194
|
endpoint-level trip disables hedging only for that one endpoint.
|
|
190
195
|
|
|
191
196
|
Crucially, the breaker **only ever suppresses the hedge request**. The
|
|
192
197
|
primary request is always sent, and its result or exception is always
|
|
193
|
-
returned to the caller normally
|
|
198
|
+
returned to the caller normally. This is not a request-blocking circuit
|
|
194
199
|
breaker, so hedging can't pile extra load onto a backend that's already
|
|
195
200
|
struggling.
|
|
196
201
|
|
|
@@ -201,7 +206,7 @@ HALF_OPEN ──(trial requests mostly succeed)──▶ CLOSED
|
|
|
201
206
|
HALF_OPEN ──(trial requests mostly fail)────▶ OPEN
|
|
202
207
|
```
|
|
203
208
|
|
|
204
|
-
Note: health is recorded from the *winning* task's outcome only
|
|
209
|
+
Note: health is recorded from the *winning* task's outcome only. A
|
|
205
210
|
cancelled loser's real outcome is unknowable, and losers are cancelled
|
|
206
211
|
deliberately (not doing so would defeat the point of reducing load on a
|
|
207
212
|
struggling backend).
|
|
@@ -225,6 +230,20 @@ print(transport.health.host_state("api.example.com"))
|
|
|
225
230
|
`primary_wins`, `budget_exhausted`, `warmup_requests`, `circuit_blocked`,
|
|
226
231
|
and `errors` per key, plus a global aggregate.
|
|
227
232
|
|
|
233
|
+
To see the learned latency estimate itself (e.g. the current p90 driving
|
|
234
|
+
the hedge trigger), query `latency_quantile()` with the same key format:
|
|
235
|
+
|
|
236
|
+
```python
|
|
237
|
+
name = transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.90))
|
|
238
|
+
# ... after running some traffic ...
|
|
239
|
+
p90 = transport.latency_quantile(f"endpoint:{name}", 0.9)
|
|
240
|
+
if p90 is not None:
|
|
241
|
+
print(f"p90: {p90 * 1000:.1f}ms")
|
|
242
|
+
```
|
|
243
|
+
|
|
244
|
+
Returns `None` if the key hasn't recorded any samples yet. You can query
|
|
245
|
+
any quantile, not just the one driving the hedge decision.
|
|
246
|
+
|
|
228
247
|
### Push-based hooks: metrics and alerting
|
|
229
248
|
|
|
230
249
|
Polling snapshots works for dashboards, but alerting on a circuit-breaker
|
|
@@ -267,14 +286,14 @@ async with httpx.AsyncClient(transport=transport) as client:
|
|
|
267
286
|
```
|
|
268
287
|
|
|
269
288
|
`on_hedge_fired` is called with the key each time a hedge request is
|
|
270
|
-
actually launched
|
|
289
|
+
actually launched, after the idempotency, circuit-breaker, and budget
|
|
271
290
|
gates have all passed, so it only fires for hedges that were genuinely
|
|
272
291
|
sent. `on_circuit_open` is called once per OPEN transition (not on every
|
|
273
292
|
suppressed hedge while it stays open), so it's safe to wire straight into
|
|
274
293
|
an alerting/paging pipeline without flooding it.
|
|
275
294
|
|
|
276
295
|
Both callbacks run synchronously on the request path, so keep them fast
|
|
277
|
-
(increment a counter, log a line)
|
|
296
|
+
(increment a counter, log a line); don't do network I/O in them directly.
|
|
278
297
|
|
|
279
298
|
## Relationship to hedge-python
|
|
280
299
|
|
|
@@ -290,11 +309,11 @@ for the motivating scenario.
|
|
|
290
309
|
|
|
291
310
|
## References
|
|
292
311
|
|
|
293
|
-
- [The Tail at Scale](https://research.google/pubs/pub40801/)
|
|
294
|
-
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604)
|
|
295
|
-
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python)
|
|
312
|
+
- [The Tail at Scale](https://research.google/pubs/pub40801/): Google's paper on tail latency
|
|
313
|
+
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604): Masson et al., VLDB 2019
|
|
314
|
+
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python): the project this one is modeled after
|
|
296
315
|
- [httpx documentation](https://www.python-httpx.org/)
|
|
297
316
|
|
|
298
317
|
## License
|
|
299
318
|
|
|
300
|
-
|
|
319
|
+
MIT License. See [LICENSE](LICENSE) file for details.
|
|
@@ -23,6 +23,10 @@ async def main():
|
|
|
23
23
|
asyncio.run(main())
|
|
24
24
|
```
|
|
25
25
|
|
|
26
|
+
For a runnable, end-to-end demo (a small backend with different latency
|
|
27
|
+
profiles per route, driven by a client that prints a hedge/latency/circuit
|
|
28
|
+
breaker report), see [`examples/`](examples/README.md).
|
|
29
|
+
|
|
26
30
|
With no configuration, `HedgedTransport` learns a p90 latency estimate per
|
|
27
31
|
host (via a [DDSketch](https://arxiv.org/abs/2004.08604) quantile sketch)
|
|
28
32
|
and fires a hedge request whenever the primary exceeds it.
|
|
@@ -30,7 +34,7 @@ and fires a hedge request whenever the primary exceeds it.
|
|
|
30
34
|
## Why per-endpoint?
|
|
31
35
|
|
|
32
36
|
A single host can host wildly different endpoints. Learning one latency
|
|
33
|
-
distribution per *host
|
|
37
|
+
distribution per *host*, rather than per endpoint, means a handful of
|
|
34
38
|
calls to a slow endpoint skew the hedge trigger for a fast one sharing the
|
|
35
39
|
same host:
|
|
36
40
|
|
|
@@ -41,7 +45,7 @@ GET /api/v1/bulk-export median 900ms, low RPS
|
|
|
41
45
|
|
|
42
46
|
`HedgedTransport` lets you register per-endpoint config up front. Each
|
|
43
47
|
registered endpoint gets its own latency sketch, rate estimate, and hedge
|
|
44
|
-
budget
|
|
48
|
+
budget, all still funneled through a **single inner transport and
|
|
45
49
|
connection pool**, unlike using `httpx` `mounts={}`, which would mean one
|
|
46
50
|
connection pool per pattern:
|
|
47
51
|
|
|
@@ -59,7 +63,7 @@ endpoints at all).
|
|
|
59
63
|
|
|
60
64
|
Route patterns may contain `{name}` placeholders or a bare `*` for a single
|
|
61
65
|
path segment, e.g. `/api/v1/users/{id}`. Patterns are matched in
|
|
62
|
-
registration order
|
|
66
|
+
registration order, so register more specific patterns first.
|
|
63
67
|
|
|
64
68
|
## Hardcoded vs. adaptive delay
|
|
65
69
|
|
|
@@ -70,8 +74,8 @@ percentile:
|
|
|
70
74
|
transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.95))
|
|
71
75
|
```
|
|
72
76
|
|
|
73
|
-
For an endpoint where you already know the right delay
|
|
74
|
-
deterministic behavior without a warmup period
|
|
77
|
+
For an endpoint where you already know the right delay, or want
|
|
78
|
+
deterministic behavior without a warmup period, hardcode it instead:
|
|
75
79
|
|
|
76
80
|
```python
|
|
77
81
|
transport.register("GET", "/api/v1/health", EndpointConfig(hedge_delay=0.05))
|
|
@@ -83,7 +87,7 @@ observability; it just isn't consulted for the hedge-delay decision.
|
|
|
83
87
|
## Explicit endpoint override
|
|
84
88
|
|
|
85
89
|
Auto-matching not precise enough for a particular call site (or you'd
|
|
86
|
-
rather not register a pattern)? Tag the request directly
|
|
90
|
+
rather not register a pattern)? Tag the request directly; this bypasses
|
|
87
91
|
pattern matching entirely:
|
|
88
92
|
|
|
89
93
|
```python
|
|
@@ -150,7 +154,7 @@ request ──────┤
|
|
|
150
154
|
|
|
151
155
|
Only idempotent methods (`GET`, `HEAD`, `OPTIONS`) are ever hedged, to avoid
|
|
152
156
|
duplicating side effects. A request with a body is also never hedged, even
|
|
153
|
-
if the method is idempotent
|
|
157
|
+
if the method is idempotent: the primary and hedge send the same
|
|
154
158
|
`httpx.Request` object, and a body backed by a one-shot stream can't be
|
|
155
159
|
safely read twice.
|
|
156
160
|
|
|
@@ -175,13 +179,13 @@ endpoint.
|
|
|
175
179
|
|
|
176
180
|
A closed / open / half-open circuit breaker tracks request success/failure
|
|
177
181
|
at **two independent tiers**: one breaker per host, one breaker per
|
|
178
|
-
endpoint key. Either tripping open suppresses hedging for its scope
|
|
182
|
+
endpoint key. Either tripping open suppresses hedging for its scope: a
|
|
179
183
|
host-level trip disables hedging for every endpoint on that host, while an
|
|
180
184
|
endpoint-level trip disables hedging only for that one endpoint.
|
|
181
185
|
|
|
182
186
|
Crucially, the breaker **only ever suppresses the hedge request**. The
|
|
183
187
|
primary request is always sent, and its result or exception is always
|
|
184
|
-
returned to the caller normally
|
|
188
|
+
returned to the caller normally. This is not a request-blocking circuit
|
|
185
189
|
breaker, so hedging can't pile extra load onto a backend that's already
|
|
186
190
|
struggling.
|
|
187
191
|
|
|
@@ -192,7 +196,7 @@ HALF_OPEN ──(trial requests mostly succeed)──▶ CLOSED
|
|
|
192
196
|
HALF_OPEN ──(trial requests mostly fail)────▶ OPEN
|
|
193
197
|
```
|
|
194
198
|
|
|
195
|
-
Note: health is recorded from the *winning* task's outcome only
|
|
199
|
+
Note: health is recorded from the *winning* task's outcome only. A
|
|
196
200
|
cancelled loser's real outcome is unknowable, and losers are cancelled
|
|
197
201
|
deliberately (not doing so would defeat the point of reducing load on a
|
|
198
202
|
struggling backend).
|
|
@@ -216,6 +220,20 @@ print(transport.health.host_state("api.example.com"))
|
|
|
216
220
|
`primary_wins`, `budget_exhausted`, `warmup_requests`, `circuit_blocked`,
|
|
217
221
|
and `errors` per key, plus a global aggregate.
|
|
218
222
|
|
|
223
|
+
To see the learned latency estimate itself (e.g. the current p90 driving
|
|
224
|
+
the hedge trigger), query `latency_quantile()` with the same key format:
|
|
225
|
+
|
|
226
|
+
```python
|
|
227
|
+
name = transport.register("GET", "/api/v1/search", EndpointConfig(percentile=0.90))
|
|
228
|
+
# ... after running some traffic ...
|
|
229
|
+
p90 = transport.latency_quantile(f"endpoint:{name}", 0.9)
|
|
230
|
+
if p90 is not None:
|
|
231
|
+
print(f"p90: {p90 * 1000:.1f}ms")
|
|
232
|
+
```
|
|
233
|
+
|
|
234
|
+
Returns `None` if the key hasn't recorded any samples yet. You can query
|
|
235
|
+
any quantile, not just the one driving the hedge decision.
|
|
236
|
+
|
|
219
237
|
### Push-based hooks: metrics and alerting
|
|
220
238
|
|
|
221
239
|
Polling snapshots works for dashboards, but alerting on a circuit-breaker
|
|
@@ -258,14 +276,14 @@ async with httpx.AsyncClient(transport=transport) as client:
|
|
|
258
276
|
```
|
|
259
277
|
|
|
260
278
|
`on_hedge_fired` is called with the key each time a hedge request is
|
|
261
|
-
actually launched
|
|
279
|
+
actually launched, after the idempotency, circuit-breaker, and budget
|
|
262
280
|
gates have all passed, so it only fires for hedges that were genuinely
|
|
263
281
|
sent. `on_circuit_open` is called once per OPEN transition (not on every
|
|
264
282
|
suppressed hedge while it stays open), so it's safe to wire straight into
|
|
265
283
|
an alerting/paging pipeline without flooding it.
|
|
266
284
|
|
|
267
285
|
Both callbacks run synchronously on the request path, so keep them fast
|
|
268
|
-
(increment a counter, log a line)
|
|
286
|
+
(increment a counter, log a line); don't do network I/O in them directly.
|
|
269
287
|
|
|
270
288
|
## Relationship to hedge-python
|
|
271
289
|
|
|
@@ -281,11 +299,11 @@ for the motivating scenario.
|
|
|
281
299
|
|
|
282
300
|
## References
|
|
283
301
|
|
|
284
|
-
- [The Tail at Scale](https://research.google/pubs/pub40801/)
|
|
285
|
-
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604)
|
|
286
|
-
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python)
|
|
302
|
+
- [The Tail at Scale](https://research.google/pubs/pub40801/): Google's paper on tail latency
|
|
303
|
+
- [DDSketch: A fast and fully-mergeable quantile sketch with relative-error guarantees](https://arxiv.org/abs/2004.08604): Masson et al., VLDB 2019
|
|
304
|
+
- [hedge-python](https://github.com/sunhailin-Leo/hedge-python): the project this one is modeled after
|
|
287
305
|
- [httpx documentation](https://www.python-httpx.org/)
|
|
288
306
|
|
|
289
307
|
## License
|
|
290
308
|
|
|
291
|
-
|
|
309
|
+
MIT License. See [LICENSE](LICENSE) file for details.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
# Examples
|
|
2
|
+
|
|
3
|
+
A runnable demo of `HedgedTransport` against a small local backend with
|
|
4
|
+
three different latency profiles.
|
|
5
|
+
|
|
6
|
+
| File | What it is |
|
|
7
|
+
|---|---|
|
|
8
|
+
| `app.py` | A tiny FastAPI backend exposing `/fast`, `/slow`, and `/flaky` routes, each with a different latency profile. |
|
|
9
|
+
| `example_usage.py` | A client that registers per-endpoint hedge config, drives load through `HedgedTransport` against `app.py`, and prints a stats/latency/circuit-breaker report. |
|
|
10
|
+
| `run_example.sh` | Installs deps, starts `app.py`, runs `example_usage.py` against it, and tears the server down. The easiest way to see the whole thing end to end. |
|
|
11
|
+
|
|
12
|
+
## Running it
|
|
13
|
+
|
|
14
|
+
From the repo root:
|
|
15
|
+
|
|
16
|
+
```bash
|
|
17
|
+
./examples/run_example.sh
|
|
18
|
+
```
|
|
19
|
+
|
|
20
|
+
Or run the pieces yourself:
|
|
21
|
+
|
|
22
|
+
```bash
|
|
23
|
+
uv sync --group examples
|
|
24
|
+
uv run uvicorn examples.app:app --host 127.0.0.1 --port 8000 &
|
|
25
|
+
uv run python examples/example_usage.py
|
|
26
|
+
```
|
|
27
|
+
|
|
28
|
+
## What to look for in the output
|
|
29
|
+
|
|
30
|
+
- `/fast` rarely triggers a hedge, since the primary is almost always
|
|
31
|
+
faster than the learned delay.
|
|
32
|
+
- `/slow` hedges often, but the hedge rarely *wins*, because the backup is
|
|
33
|
+
just as slow as the primary on this uniformly slow endpoint.
|
|
34
|
+
- `/flaky` is the interesting case: after warmup, the learned p90 sits near
|
|
35
|
+
the fast latency, so the rare slow primary gets hedged against a fresh
|
|
36
|
+
(usually fast) request and the hedge usually wins the race.
|
|
37
|
+
|
|
38
|
+
The final report prints per-endpoint stats (`total`, `hedged`,
|
|
39
|
+
`hedge_wins`, `primary_wins`, ...), each endpoint's learned p90 latency
|
|
40
|
+
estimate (via `transport.latency_quantile()`), and each endpoint's circuit
|
|
41
|
+
breaker state.
|
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
"""Tiny FastAPI backend with three latency profiles, for the hedging demo.
|
|
2
2
|
|
|
3
|
-
GET /fast
|
|
4
|
-
GET /slow
|
|
5
|
-
GET /flaky
|
|
3
|
+
GET /fast always responds in ~5-15ms
|
|
4
|
+
GET /slow always takes 800ms
|
|
5
|
+
GET /flaky 90% of requests are fast (~5-15ms), 10% take ~900ms
|
|
6
6
|
|
|
7
7
|
Run directly with uvicorn, e.g.:
|
|
8
8
|
|
|
@@ -39,7 +39,7 @@ CONCURRENCY = 6
|
|
|
39
39
|
|
|
40
40
|
# The hedge token bucket auto-estimates RPS over a rolling ~10s window by
|
|
41
41
|
# default, which under-counts traffic sent in a shorter burst like this
|
|
42
|
-
# demo's
|
|
42
|
+
# demo's, so budget starves and hedges rarely fire. Pinning estimated_rps
|
|
43
43
|
# (roughly concurrency / expected latency) avoids that ramp-up and keeps
|
|
44
44
|
# the demo's hedge behavior visible run over run. /fast is left on the
|
|
45
45
|
# default auto-estimate since it barely ever needs to hedge anyway.
|
|
@@ -91,6 +91,12 @@ def print_report(transport: HedgedTransport) -> None:
|
|
|
91
91
|
print(f"\ntotal requests sent: {global_snap.total_requests}")
|
|
92
92
|
print(f"total hedges fired: {global_snap.hedged_requests}")
|
|
93
93
|
|
|
94
|
+
print("\n=== learned p90 latency estimates ===")
|
|
95
|
+
for route in ROUTES:
|
|
96
|
+
key = f"endpoint:{route}"
|
|
97
|
+
p90 = transport.latency_quantile(key, 0.9)
|
|
98
|
+
print(f"{key:<16} {f'{p90 * 1000:.1f}ms' if p90 is not None else 'n/a'}")
|
|
99
|
+
|
|
94
100
|
print("\n=== circuit breaker state ===")
|
|
95
101
|
for route in ROUTES:
|
|
96
102
|
key = f"endpoint:{route}"
|
|
@@ -104,7 +110,7 @@ async def wait_for_server(client: httpx.AsyncClient) -> None:
|
|
|
104
110
|
response.raise_for_status()
|
|
105
111
|
except httpx.HTTPError:
|
|
106
112
|
print(
|
|
107
|
-
f"Could not reach {BASE_URL}
|
|
113
|
+
f"Could not reach {BASE_URL}: is examples/app.py running?",
|
|
108
114
|
file=sys.stderr,
|
|
109
115
|
)
|
|
110
116
|
sys.exit(1)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
#!/usr/bin/env bash
|
|
2
2
|
# Installs deps, starts the demo FastAPI backend, runs the hedging demo
|
|
3
|
-
# client against it, and prints the stats report
|
|
3
|
+
# client against it, and prints the stats report, then tears the server
|
|
4
4
|
# down. Run from anywhere:
|
|
5
5
|
#
|
|
6
6
|
# ./examples/run_example.sh
|
|
@@ -1,8 +1,10 @@
|
|
|
1
1
|
[project]
|
|
2
2
|
name = "httpx-hedged"
|
|
3
|
-
|
|
3
|
+
dynamic = ["version"]
|
|
4
4
|
description = "Adaptive, per-endpoint request hedging transport for httpx. Learns latency percentiles per endpoint with DDSketch, caps hedge rate with a token bucket, and stops hedging under a host/endpoint circuit breaker."
|
|
5
5
|
readme = "README.md"
|
|
6
|
+
license = "MIT"
|
|
7
|
+
license-files = ["LICENSE"]
|
|
6
8
|
requires-python = ">=3.13"
|
|
7
9
|
dependencies = [
|
|
8
10
|
"httpx>=0.27.0",
|
|
@@ -15,6 +17,7 @@ dev = [
|
|
|
15
17
|
"pytest>=9.0.1",
|
|
16
18
|
"pytest-asyncio>=1.3.0",
|
|
17
19
|
"python-lsp-server[all]>=1.13.2",
|
|
20
|
+
"ruff>=0.14.0",
|
|
18
21
|
]
|
|
19
22
|
examples = [
|
|
20
23
|
"fastapi>=0.115",
|
|
@@ -22,12 +25,18 @@ examples = [
|
|
|
22
25
|
]
|
|
23
26
|
|
|
24
27
|
[build-system]
|
|
25
|
-
requires = ["hatchling"]
|
|
28
|
+
requires = ["hatchling", "hatch-vcs"]
|
|
26
29
|
build-backend = "hatchling.build"
|
|
27
30
|
|
|
28
31
|
[tool.hatch.build.targets.wheel]
|
|
29
32
|
packages = ["src/httpx_hedged"]
|
|
30
33
|
|
|
34
|
+
[tool.hatch.version]
|
|
35
|
+
source = "vcs"
|
|
36
|
+
|
|
37
|
+
[tool.hatch.build.hooks.vcs]
|
|
38
|
+
version-file = "src/httpx_hedged/_version.py"
|
|
39
|
+
|
|
31
40
|
[tool.pytest.ini_options]
|
|
32
41
|
testpaths = ["tests"]
|
|
33
42
|
asyncio_mode = "auto"
|
|
@@ -18,6 +18,7 @@ from httpx_hedged._config import CircuitBreakerConfig, EndpointConfig, HedgeConf
|
|
|
18
18
|
from httpx_hedged._health import CircuitBreaker, CircuitState, HealthRegistry
|
|
19
19
|
from httpx_hedged._matcher import Route, UnknownHedgeEndpointError
|
|
20
20
|
from httpx_hedged._stats import Stats, StatsRegistry, StatsSnapshot
|
|
21
|
+
from httpx_hedged._version import __version__
|
|
21
22
|
from httpx_hedged.transport import HedgedTransport
|
|
22
23
|
|
|
23
24
|
__all__ = [
|
|
@@ -33,6 +34,5 @@ __all__ = [
|
|
|
33
34
|
"StatsRegistry",
|
|
34
35
|
"StatsSnapshot",
|
|
35
36
|
"UnknownHedgeEndpointError",
|
|
37
|
+
"__version__",
|
|
36
38
|
]
|
|
37
|
-
|
|
38
|
-
__version__ = "0.2.0"
|
|
@@ -2,8 +2,8 @@
|
|
|
2
2
|
|
|
3
3
|
Per-key state (sketches, breakers, stats) is created lazily, keyed by
|
|
4
4
|
endpoint name or a per-host fallback. Nothing ever evicted those keys, so a
|
|
5
|
-
caller that talks to high-cardinality hosts (per-tenant subdomains, or
|
|
6
|
-
the pathological case
|
|
5
|
+
caller that talks to high-cardinality hosts (per-tenant subdomains, or, in
|
|
6
|
+
the pathological case, one key per unique URL when a request has no
|
|
7
7
|
parseable host) would grow these maps forever. Bounding them with LRU
|
|
8
8
|
eviction keeps memory bounded without requiring callers to know about it.
|
|
9
9
|
"""
|
|
@@ -17,7 +17,7 @@ DEFAULT_MAX_SIZE = 10_000
|
|
|
17
17
|
|
|
18
18
|
|
|
19
19
|
class BoundedRegistry[V]:
|
|
20
|
-
"""Not thread-safe by itself
|
|
20
|
+
"""Not thread-safe by itself. Callers hold their own lock, as with the
|
|
21
21
|
plain dicts this replaces."""
|
|
22
22
|
|
|
23
23
|
def __init__(self, max_size: int = DEFAULT_MAX_SIZE) -> None:
|
|
@@ -39,7 +39,7 @@ def _validate_common(config: HedgeConfig | EndpointConfig) -> None:
|
|
|
39
39
|
class CircuitBreakerConfig:
|
|
40
40
|
"""Configuration for the health circuit breaker that gates hedging.
|
|
41
41
|
|
|
42
|
-
Tripping the breaker only ever suppresses the *hedge* request
|
|
42
|
+
Tripping the breaker only ever suppresses the *hedge* request. The
|
|
43
43
|
primary request is always sent and its result or exception is always
|
|
44
44
|
returned to the caller normally.
|
|
45
45
|
"""
|
|
@@ -163,7 +163,7 @@ class EndpointConfig:
|
|
|
163
163
|
|
|
164
164
|
#: Health circuit-breaker configuration. When set, replaces the
|
|
165
165
|
#: default breaker config as a whole object rather than being merged
|
|
166
|
-
#: field-by-field
|
|
166
|
+
#: field-by-field. For a partial override, use
|
|
167
167
|
#: ``dataclasses.replace(default.circuit_breaker, ...)``.
|
|
168
168
|
circuit_breaker: CircuitBreakerConfig | None = None
|
|
169
169
|
|
|
@@ -175,7 +175,16 @@ class EndpointConfig:
|
|
|
175
175
|
|
|
176
176
|
@dataclass(frozen=True, slots=True)
|
|
177
177
|
class EffectiveConfig:
|
|
178
|
-
"""Fully-resolved hedge configuration for a single key
|
|
178
|
+
"""Fully-resolved hedge configuration for a single key.
|
|
179
|
+
|
|
180
|
+
Every "inherit the default" field from ``EndpointConfig`` has been
|
|
181
|
+
resolved away by ``resolve()``. ``estimated_rps`` and ``hedge_delay``
|
|
182
|
+
stay ``Optional`` here regardless, since their ``None`` means something
|
|
183
|
+
different than "unresolved": ``estimated_rps=None`` means "auto-estimate
|
|
184
|
+
from traffic" and ``hedge_delay=None`` means "no hardcoded delay, learn
|
|
185
|
+
one from the sketch" (see ``is_hardcoded``). Both are real, permanent
|
|
186
|
+
states, not resolution artifacts.
|
|
187
|
+
"""
|
|
179
188
|
|
|
180
189
|
percentile: float
|
|
181
190
|
budget_percent: float
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"""Health-based circuit breaker that suppresses hedging during outages.
|
|
2
2
|
|
|
3
|
-
hedge-python has no concept of request success/failure at all
|
|
3
|
+
hedge-python has no concept of request success/failure at all: its token
|
|
4
4
|
bucket caps hedge *volume* but has no idea whether the backend is actually
|
|
5
5
|
healthy. This module adds a circuit breaker, tracked independently at both
|
|
6
6
|
the host level and the per-endpoint level, so that either "one endpoint is
|
|
@@ -9,14 +9,14 @@ requiring the two to be conflated.
|
|
|
9
9
|
|
|
10
10
|
Tripping the breaker only ever suppresses the *hedge* request. The primary
|
|
11
11
|
request always goes through and its result or exception is always returned
|
|
12
|
-
to the caller
|
|
12
|
+
to the caller. This is deliberately not a request-blocking circuit
|
|
13
13
|
breaker, only a hedge-suppressing one, so hedging can't pile extra load
|
|
14
14
|
onto an already-failing backend.
|
|
15
15
|
|
|
16
16
|
Known limitation: health is recorded from the winning task's outcome only.
|
|
17
17
|
A cancelled loser's real outcome is unknowable, and losers are cancelled
|
|
18
|
-
deliberately
|
|
19
|
-
load on a struggling backend.
|
|
18
|
+
deliberately, since not doing so would defeat the breaker's purpose of
|
|
19
|
+
reducing load on a struggling backend.
|
|
20
20
|
"""
|
|
21
21
|
|
|
22
22
|
from __future__ import annotations
|
|
@@ -99,7 +99,7 @@ class CircuitBreaker:
|
|
|
99
99
|
config: Breaker thresholds and timing.
|
|
100
100
|
on_open: Called (with no arguments) each time the breaker
|
|
101
101
|
transitions into the OPEN state, whether from CLOSED or from a
|
|
102
|
-
failed HALF_OPEN trial. Useful for alerting
|
|
102
|
+
failed HALF_OPEN trial. Useful for alerting; see the README's
|
|
103
103
|
observability section for a logging example.
|
|
104
104
|
"""
|
|
105
105
|
|
|
@@ -136,7 +136,7 @@ class CircuitBreaker:
|
|
|
136
136
|
return
|
|
137
137
|
|
|
138
138
|
# CLOSED (or OPEN, where a result can still arrive from a primary
|
|
139
|
-
# request even though hedging is suppressed
|
|
139
|
+
# request even though hedging is suppressed; keep tracking it).
|
|
140
140
|
self._window.record(ok)
|
|
141
141
|
if (
|
|
142
142
|
self._state is CircuitState.CLOSED
|
|
@@ -195,7 +195,7 @@ class HealthRegistry:
|
|
|
195
195
|
endpoint-scoped) transitions into the OPEN state, as
|
|
196
196
|
``on_circuit_open(scope, key)`` where ``scope`` is ``"host"``
|
|
197
197
|
or ``"endpoint"`` and ``key`` is the host name or endpoint key
|
|
198
|
-
that tripped. Intended for alerting
|
|
198
|
+
that tripped. Intended for alerting; see the README's
|
|
199
199
|
observability section for a logging example.
|
|
200
200
|
"""
|
|
201
201
|
|
|
@@ -251,7 +251,7 @@ class HealthRegistry:
|
|
|
251
251
|
ok: bool,
|
|
252
252
|
) -> None:
|
|
253
253
|
# CircuitBreaker is documented as not being thread-safe on its own
|
|
254
|
-
# ("callers hold their own lock")
|
|
254
|
+
# ("callers hold their own lock"). The lookup AND the mutation
|
|
255
255
|
# below must happen under one lock acquisition, not just the
|
|
256
256
|
# lookup, or concurrent callers can race on the same breaker's
|
|
257
257
|
# internal counters/state transitions.
|
|
@@ -4,7 +4,7 @@ hedge-python keys its latency sketch per *host* only. A service with many
|
|
|
4
4
|
routes of very different latency/RPS profiles (e.g. ``GET /api/v1/foo`` vs.
|
|
5
5
|
``GET /api/v1/bar``) would have all of their latencies mixed into one
|
|
6
6
|
sketch. This module lets callers register per-route ``EndpointConfig``
|
|
7
|
-
objects up front, matched against every request's method and path
|
|
7
|
+
objects up front, matched against every request's method and path, all
|
|
8
8
|
still funneled through a single ``httpx.AsyncBaseTransport`` / connection
|
|
9
9
|
pool, so supporting many endpoints does not multiply connection pools the
|
|
10
10
|
way ``httpx`` ``mounts={}`` would.
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
hedge-python requires callers to hand-tune a single ``estimated_rps`` value
|
|
4
4
|
per host. That is workable for one host, but this library tracks state per
|
|
5
5
|
*endpoint*, and a service with many endpoints of very different traffic
|
|
6
|
-
volumes would require hand-tuning a guess for each one
|
|
6
|
+
volumes would require hand-tuning a guess for each one, exactly the kind
|
|
7
7
|
of per-endpoint configuration burden this rewrite is trying to avoid. By
|
|
8
8
|
default we estimate the rate automatically instead.
|
|
9
9
|
"""
|
|
@@ -61,7 +61,7 @@ class RollingRateCounter:
|
|
|
61
61
|
Weights the previous window's count by the fraction of it that
|
|
62
62
|
still falls within the trailing ``window_duration``-sized lookback
|
|
63
63
|
from now, rather than always dividing by a fixed ``2 *
|
|
64
|
-
window_duration
|
|
64
|
+
window_duration``. That fixed divisor systematically
|
|
65
65
|
underestimates by up to 2x right after a rotation, when the
|
|
66
66
|
current window has accumulated almost nothing yet but the full
|
|
67
67
|
previous window's count is still discounted as if it were only
|
|
@@ -5,8 +5,8 @@ Every windowed structure in this package (``WindowedSketch``,
|
|
|
5
5
|
current/previous pair of accumulators and decides whether to rotate or reset
|
|
6
6
|
them lazily, on the next call that touches the structure, rather than via a
|
|
7
7
|
background thread or asyncio task. This avoids spinning one rotation task per
|
|
8
|
-
tracked key
|
|
9
|
-
per-host, since a service can have dozens of endpoints.
|
|
8
|
+
tracked key, which matters once state is tracked per-endpoint rather than
|
|
9
|
+
just per-host, since a service can have dozens of endpoints.
|
|
10
10
|
"""
|
|
11
11
|
|
|
12
12
|
from __future__ import annotations
|
|
@@ -5,7 +5,7 @@ two changes: state is keyed per *endpoint* rather than only per *host* (see
|
|
|
5
5
|
``_matcher.py``), and the win/lose bookkeeping always records the outcome
|
|
6
6
|
(latency + health) even when the winning task raised an exception. In
|
|
7
7
|
hedge-python, ``winner_task.result()`` is called before recording, so an
|
|
8
|
-
exception there silently skips recording entirely
|
|
8
|
+
exception there silently skips recording entirely. This rewrite records
|
|
9
9
|
first, then re-raises.
|
|
10
10
|
"""
|
|
11
11
|
|
|
@@ -86,15 +86,15 @@ class HedgeScheduler:
|
|
|
86
86
|
stats_registry: Shared per-key statistics registry.
|
|
87
87
|
host_circuit_breaker: Circuit-breaker configuration used for the
|
|
88
88
|
*host* tier, independent of whichever endpoint's config happens
|
|
89
|
-
to be resolved for a given request
|
|
89
|
+
to be resolved for a given request. A host isn't owned by any
|
|
90
90
|
one endpoint, so its breaker thresholds must not depend on
|
|
91
91
|
request arrival order (see ``_should_hedge``/``_finish``, which
|
|
92
92
|
always pass this rather than the per-request resolved config
|
|
93
93
|
for the host side of ``HealthRegistry`` calls).
|
|
94
94
|
on_hedge_fired: Called with the key each time a hedge request is
|
|
95
|
-
actually launched
|
|
96
|
-
|
|
97
|
-
|
|
95
|
+
actually launched, after the idempotency, circuit-breaker, and
|
|
96
|
+
budget gates have all passed. Intended for metrics; see the
|
|
97
|
+
README's observability section for an example.
|
|
98
98
|
"""
|
|
99
99
|
|
|
100
100
|
def __init__(
|
|
@@ -116,11 +116,20 @@ class HedgeScheduler:
|
|
|
116
116
|
key, lambda: _EndpointState(config, self._stats_registry.for_key(key))
|
|
117
117
|
)
|
|
118
118
|
|
|
119
|
+
def latency_quantile(self, key: str, q: float) -> float | None:
|
|
120
|
+
"""Return the current estimated latency (seconds) at quantile ``q``
|
|
121
|
+
for a tracked key, or None if the key isn't tracked yet or has no
|
|
122
|
+
recorded samples."""
|
|
123
|
+
state = self._states.get(key)
|
|
124
|
+
if state is None:
|
|
125
|
+
return None
|
|
126
|
+
estimate = state.sketch.quantile(q)
|
|
127
|
+
return None if math.isnan(estimate) else estimate
|
|
128
|
+
|
|
119
129
|
def compute_hedge_delay(self, state: _EndpointState) -> float:
|
|
120
130
|
"""Compute the hedge delay in seconds for the current request on this key."""
|
|
121
131
|
config = state.config
|
|
122
|
-
if config.
|
|
123
|
-
assert config.hedge_delay is not None
|
|
132
|
+
if config.hedge_delay is not None:
|
|
124
133
|
return max(config.hedge_delay, config.min_delay)
|
|
125
134
|
|
|
126
135
|
if state.counter <= config.warmup_requests:
|
|
@@ -219,7 +228,7 @@ class HedgeScheduler:
|
|
|
219
228
|
return await self._finish(state, host, key, winner_task, start, classify)
|
|
220
229
|
finally:
|
|
221
230
|
# Reached on the happy path too, where every task is already
|
|
222
|
-
# done and this is a no-op
|
|
231
|
+
# done and this is a no-op. But if this coroutine itself is
|
|
223
232
|
# cancelled (e.g. the caller wrapped the request in a timeout)
|
|
224
233
|
# while blocked on one of the awaits above, asyncio.wait does
|
|
225
234
|
# not cancel the tasks it was waiting on, so they'd otherwise
|
|
@@ -274,7 +283,7 @@ class HedgeScheduler:
|
|
|
274
283
|
"""Record latency and health outcome, then return the result or re-raise.
|
|
275
284
|
|
|
276
285
|
Recording always happens before the result is returned or the
|
|
277
|
-
exception is re-raised
|
|
286
|
+
exception is re-raised. In hedge-python, by contrast, an exception
|
|
278
287
|
from ``winner_task.result()`` silently skips recording.
|
|
279
288
|
"""
|
|
280
289
|
elapsed = time.monotonic() - start
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
# file generated by vcs-versioning
|
|
2
|
+
# don't change, don't track in version control
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
__all__ = [
|
|
6
|
+
"__version__",
|
|
7
|
+
"__version_tuple__",
|
|
8
|
+
"version",
|
|
9
|
+
"version_tuple",
|
|
10
|
+
"__commit_id__",
|
|
11
|
+
"commit_id",
|
|
12
|
+
]
|
|
13
|
+
|
|
14
|
+
version: str
|
|
15
|
+
__version__: str
|
|
16
|
+
__version_tuple__: tuple[int | str, ...]
|
|
17
|
+
version_tuple: tuple[int | str, ...]
|
|
18
|
+
commit_id: str | None
|
|
19
|
+
__commit_id__: str | None
|
|
20
|
+
|
|
21
|
+
__version__ = version = '0.3.0'
|
|
22
|
+
__version_tuple__ = version_tuple = (0, 3, 0)
|
|
23
|
+
|
|
24
|
+
__commit_id__ = commit_id = None
|
|
@@ -18,7 +18,7 @@ class WindowedSketch:
|
|
|
18
18
|
Quantile queries merge both sketches, giving a window that spans 1x to
|
|
19
19
|
2x the configured duration. ``add`` always writes to the current
|
|
20
20
|
sketch. Rotation is decided lazily on each call rather than by a
|
|
21
|
-
background thread/task
|
|
21
|
+
background thread/task (see ``httpx_hedged._rotation``), since a
|
|
22
22
|
service can have many independently-tracked endpoints, and spinning one
|
|
23
23
|
rotation task per endpoint does not scale.
|
|
24
24
|
|
|
@@ -48,7 +48,7 @@ class HedgedTransport(httpx.AsyncBaseTransport):
|
|
|
48
48
|
|
|
49
49
|
Wraps a single inner transport (default: ``httpx.AsyncHTTPTransport``,
|
|
50
50
|
one connection pool) and races a backup request when the primary
|
|
51
|
-
exceeds its estimated latency percentile
|
|
51
|
+
exceeds its estimated latency percentile, or a hardcoded delay, for
|
|
52
52
|
endpoints registered with ``EndpointConfig(hedge_delay=...)``.
|
|
53
53
|
|
|
54
54
|
Endpoints are identified by registering method + path patterns via
|
|
@@ -69,12 +69,12 @@ class HedgedTransport(httpx.AsyncBaseTransport):
|
|
|
69
69
|
routes: Endpoints to register up front (equivalent to calling
|
|
70
70
|
``register()`` for each one after construction).
|
|
71
71
|
on_hedge_fired: Called with the key each time a hedge request is
|
|
72
|
-
actually launched. Intended for metrics
|
|
72
|
+
actually launched. Intended for metrics; see the README's
|
|
73
73
|
observability section for an example.
|
|
74
74
|
on_circuit_open: Called as ``on_circuit_open(scope, key)`` each
|
|
75
75
|
time a host- or endpoint-scoped circuit breaker trips open
|
|
76
76
|
(``scope`` is ``"host"`` or ``"endpoint"``). Intended for
|
|
77
|
-
alerting
|
|
77
|
+
alerting; see the README's observability section for an
|
|
78
78
|
example.
|
|
79
79
|
"""
|
|
80
80
|
|
|
@@ -116,7 +116,7 @@ class HedgedTransport(httpx.AsyncBaseTransport):
|
|
|
116
116
|
``path_pattern`` segments may contain ``{name}`` placeholders or a
|
|
117
117
|
bare ``*`` to match any single path segment (e.g.
|
|
118
118
|
``/api/v1/users/{id}``). Routes are matched in registration order,
|
|
119
|
-
first match wins
|
|
119
|
+
first match wins, so register more specific patterns first.
|
|
120
120
|
|
|
121
121
|
Returns the resolved endpoint name (used as the key in ``stats``
|
|
122
122
|
and as the value for ``extensions={"hedge_endpoint": name}``).
|
|
@@ -133,6 +133,23 @@ class HedgedTransport(httpx.AsyncBaseTransport):
|
|
|
133
133
|
"""Host- and endpoint-level circuit breaker state."""
|
|
134
134
|
return self._health
|
|
135
135
|
|
|
136
|
+
def latency_quantile(self, key: str, q: float) -> float | None:
|
|
137
|
+
"""Return the current estimated latency (seconds) at quantile ``q``
|
|
138
|
+
for a tracked key, or None if nothing has been recorded for that
|
|
139
|
+
key yet.
|
|
140
|
+
|
|
141
|
+
``key`` uses the same ``"endpoint:<name>"`` / ``"host:<hostname>"``
|
|
142
|
+
format as ``stats`` and ``health``. For example, after
|
|
143
|
+
``name = transport.register("GET", "/search", ...)``, query it with
|
|
144
|
+
``transport.latency_quantile(f"endpoint:{name}", 0.9)`` for the
|
|
145
|
+
current learned p90::
|
|
146
|
+
|
|
147
|
+
p90 = transport.latency_quantile(f"endpoint:{name}", 0.9)
|
|
148
|
+
if p90 is not None:
|
|
149
|
+
print(f"{name} p90: {p90 * 1000:.1f}ms")
|
|
150
|
+
"""
|
|
151
|
+
return self._scheduler.latency_quantile(key, q)
|
|
152
|
+
|
|
136
153
|
async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
|
|
137
154
|
"""Handle an outgoing request with adaptive, per-endpoint hedging."""
|
|
138
155
|
host = extract_host(str(request.url))
|
|
@@ -38,7 +38,7 @@ async def test_two_endpoints_share_one_transport_with_independent_delays() -> No
|
|
|
38
38
|
transport.register("GET", "/api/v1/fast-lookup", EndpointConfig(percentile=0.9))
|
|
39
39
|
transport.register("GET", "/api/v1/bulk-export", EndpointConfig(percentile=0.9))
|
|
40
40
|
|
|
41
|
-
# Prime each endpoint's own sketch directly
|
|
41
|
+
# Prime each endpoint's own sketch directly; this is what accumulates
|
|
42
42
|
# from real traffic via handle_async_request over time; injecting
|
|
43
43
|
# samples here keeps the test fast and deterministic. The scheduler is
|
|
44
44
|
# reached through the transport's internals deliberately, to prove the
|
|
@@ -119,6 +119,22 @@ async def test_post_never_hedges_even_when_slow() -> None:
|
|
|
119
119
|
await transport.aclose()
|
|
120
120
|
|
|
121
121
|
|
|
122
|
+
async def test_latency_quantile_reflects_traffic_through_the_transport() -> None:
|
|
123
|
+
inner = ScriptedTransport([delayed_response(0.01)])
|
|
124
|
+
transport = HedgedTransport(inner=inner, default_config=HedgeConfig(min_delay=0.0))
|
|
125
|
+
transport.register("GET", "/api/v1/search", EndpointConfig())
|
|
126
|
+
|
|
127
|
+
assert transport.latency_quantile("endpoint:GET /api/v1/search", 0.9) is None
|
|
128
|
+
|
|
129
|
+
async with httpx.AsyncClient(transport=transport) as client:
|
|
130
|
+
await client.get("https://api.example.com/api/v1/search")
|
|
131
|
+
|
|
132
|
+
p90 = transport.latency_quantile("endpoint:GET /api/v1/search", 0.9)
|
|
133
|
+
assert p90 is not None
|
|
134
|
+
assert p90 > 0
|
|
135
|
+
await transport.aclose()
|
|
136
|
+
|
|
137
|
+
|
|
122
138
|
async def test_get_with_streamed_body_never_hedges_even_when_slow() -> None:
|
|
123
139
|
inner = ScriptedTransport([delayed_response(0.05)])
|
|
124
140
|
transport = HedgedTransport(
|
|
@@ -157,7 +173,7 @@ async def test_host_breaker_uses_transport_default_not_first_endpoint_touched()
|
|
|
157
173
|
await client.get("https://api.example.com/aggressive")
|
|
158
174
|
|
|
159
175
|
# A single failure trips the endpoint breaker (min_samples=1) but must
|
|
160
|
-
# not trip the host breaker
|
|
176
|
+
# not trip the host breaker: the host tier always uses the
|
|
161
177
|
# transport-wide default config, regardless of which endpoint's
|
|
162
178
|
# override happens to touch the host first.
|
|
163
179
|
assert (
|
|
@@ -44,7 +44,7 @@ def test_rate_weights_previous_window_by_remaining_overlap(
|
|
|
44
44
|
fake_clock: Callable[[float], None],
|
|
45
45
|
) -> None:
|
|
46
46
|
"""At steady state, the estimate should track the true rate throughout
|
|
47
|
-
the window
|
|
47
|
+
the window, not just right after a rotation, by discounting the
|
|
48
48
|
previous window's count as it falls further outside the trailing
|
|
49
49
|
window_duration-sized lookback from now."""
|
|
50
50
|
counter = RollingRateCounter(window_duration=10.0)
|
|
@@ -63,5 +63,5 @@ def test_long_idle_resets_rate_to_zero(fake_clock: Callable[[float], None]) -> N
|
|
|
63
63
|
counter = RollingRateCounter(window_duration=10.0)
|
|
64
64
|
for _ in range(20):
|
|
65
65
|
counter.increment()
|
|
66
|
-
fake_clock(25.0) # beyond 2x window
|
|
66
|
+
fake_clock(25.0) # beyond 2x window, reset
|
|
67
67
|
assert counter.rate_per_second() == 0.0
|
|
@@ -84,6 +84,32 @@ def test_min_delay_floors_the_result() -> None:
|
|
|
84
84
|
assert scheduler.compute_hedge_delay(state) == 0.05
|
|
85
85
|
|
|
86
86
|
|
|
87
|
+
# --- latency_quantile ---------------------------------------------------------
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_latency_quantile_is_none_for_an_untracked_key() -> None:
|
|
91
|
+
scheduler, _health, _stats = make_scheduler()
|
|
92
|
+
assert scheduler.latency_quantile("never-seen", 0.9) is None
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
def test_latency_quantile_is_none_before_any_samples() -> None:
|
|
96
|
+
scheduler, _health, _stats = make_scheduler()
|
|
97
|
+
config = resolve(None, HedgeConfig())
|
|
98
|
+
scheduler.state_for("k", config) # creates the state, but adds no samples
|
|
99
|
+
assert scheduler.latency_quantile("k", 0.9) is None
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def test_latency_quantile_reflects_recorded_samples() -> None:
|
|
103
|
+
scheduler, _health, _stats = make_scheduler()
|
|
104
|
+
config = resolve(None, HedgeConfig())
|
|
105
|
+
state = scheduler.state_for("k", config)
|
|
106
|
+
for v in range(1, 101):
|
|
107
|
+
state.sketch.add(v / 1000.0) # 0.001..0.1 seconds
|
|
108
|
+
p90 = scheduler.latency_quantile("k", 0.9)
|
|
109
|
+
assert p90 is not None
|
|
110
|
+
assert 0.085 <= p90 <= 0.095
|
|
111
|
+
|
|
112
|
+
|
|
87
113
|
# --- execute_with_hedge: race behavior ---------------------------------------
|
|
88
114
|
|
|
89
115
|
|
|
@@ -304,7 +330,7 @@ async def test_external_cancellation_does_not_leak_the_primary_task() -> None:
|
|
|
304
330
|
|
|
305
331
|
async def test_discard_releases_a_loser_that_completed_successfully() -> None:
|
|
306
332
|
"""When the primary and hedge finish in the same event-loop pass, the
|
|
307
|
-
non-winning task is never cancelled (it's already done)
|
|
333
|
+
non-winning task is never cancelled (it's already done), so its result
|
|
308
334
|
must still be handed to ``discard`` so callers can release it (e.g.
|
|
309
335
|
closing an httpx.Response to free its pooled connection)."""
|
|
310
336
|
scheduler, _health, _stats = make_scheduler()
|
|
@@ -356,7 +382,7 @@ async def test_discard_skips_a_loser_that_raised() -> None:
|
|
|
356
382
|
|
|
357
383
|
async def test_discard_not_called_for_a_genuinely_cancelled_loser_in_a_race() -> None:
|
|
358
384
|
"""The common case: the loser is still mid-flight when cancelled, so it
|
|
359
|
-
never produced a result
|
|
385
|
+
never produced a result, and discard must not be invoked for it."""
|
|
360
386
|
scheduler, _health, _stats = make_scheduler()
|
|
361
387
|
config = hardcoded_config(0.01)
|
|
362
388
|
discarded: list[str] = []
|
|
@@ -38,7 +38,7 @@ def test_quantile_reflects_data_within_window(
|
|
|
38
38
|
sketch = WindowedSketch(window_duration=30.0)
|
|
39
39
|
for v in range(1, 51):
|
|
40
40
|
sketch.add(float(v))
|
|
41
|
-
fake_clock(15.0) # still within the first window
|
|
41
|
+
fake_clock(15.0) # still within the first window, no rotation
|
|
42
42
|
assert sketch.quantile(0.0) == 1.0
|
|
43
43
|
assert sketch.quantile(1.0) == 50.0
|
|
44
44
|
|
|
@@ -62,7 +62,7 @@ def test_idle_beyond_two_windows_hard_resets(
|
|
|
62
62
|
) -> None:
|
|
63
63
|
sketch = WindowedSketch(window_duration=10.0)
|
|
64
64
|
sketch.add(5.0)
|
|
65
|
-
fake_clock(25.0) # more than 2x window_duration has elapsed
|
|
65
|
+
fake_clock(25.0) # more than 2x window_duration has elapsed: reset, not rotate
|
|
66
66
|
assert math.isnan(sketch.quantile(0.5))
|
|
67
67
|
|
|
68
68
|
|
|
@@ -200,7 +200,6 @@ wheels = [
|
|
|
200
200
|
|
|
201
201
|
[[package]]
|
|
202
202
|
name = "httpx-hedged"
|
|
203
|
-
version = "0.2.0"
|
|
204
203
|
source = { editable = "." }
|
|
205
204
|
dependencies = [
|
|
206
205
|
{ name = "httpx" },
|
|
@@ -213,6 +212,7 @@ dev = [
|
|
|
213
212
|
{ name = "pytest" },
|
|
214
213
|
{ name = "pytest-asyncio" },
|
|
215
214
|
{ name = "python-lsp-server", extra = ["all"] },
|
|
215
|
+
{ name = "ruff" },
|
|
216
216
|
]
|
|
217
217
|
examples = [
|
|
218
218
|
{ name = "fastapi" },
|
|
@@ -229,6 +229,7 @@ dev = [
|
|
|
229
229
|
{ name = "pytest", specifier = ">=9.0.1" },
|
|
230
230
|
{ name = "pytest-asyncio", specifier = ">=1.3.0" },
|
|
231
231
|
{ name = "python-lsp-server", extras = ["all"], specifier = ">=1.13.2" },
|
|
232
|
+
{ name = "ruff", specifier = ">=0.14.0" },
|
|
232
233
|
]
|
|
233
234
|
examples = [
|
|
234
235
|
{ name = "fastapi", specifier = ">=0.115" },
|
|
@@ -625,6 +626,31 @@ wheels = [
|
|
|
625
626
|
{ url = "https://files.pythonhosted.org/packages/75/35/130469d1901da2b3a5a377539b4ffcd8a5c983f1c9e3ba5ffdd8d71ae314/rope-1.14.0-py3-none-any.whl", hash = "sha256:00a7ea8c0c376fc0b053b2f2f8ef3bfb8b50fecf1ebf3eb80e4f8bd7f1941918", size = 207143, upload-time = "2025-07-12T17:46:05.928Z" },
|
|
626
627
|
]
|
|
627
628
|
|
|
629
|
+
[[package]]
|
|
630
|
+
name = "ruff"
|
|
631
|
+
version = "0.15.21"
|
|
632
|
+
source = { registry = "https://pypi.org/simple" }
|
|
633
|
+
sdist = { url = "https://files.pythonhosted.org/packages/0f/36/6f65aa9989acdec45d417192d8f4e7921931d8a6cf87ac74bce3eed98a8e/ruff-0.15.21.tar.gz", hash = "sha256:d0cfc841c572283c36548f82664a54ce6565567f1b0d5b4cf2caac693d8b7500", size = 4769401, upload-time = "2026-07-09T20:01:34.005Z" }
|
|
634
|
+
wheels = [
|
|
635
|
+
{ url = "https://files.pythonhosted.org/packages/d0/c6/ede15cac6839f3dbce52565c8f5164a8210e669c7bc4decb03e5bdf47d0d/ruff-0.15.21-py3-none-linux_armv6l.whl", hash = "sha256:63ea0e965e5d73c90e95b2434beeafc70820536717f561b32ab6e777cb9bdf5d", size = 10854342, upload-time = "2026-07-09T20:00:53.998Z" },
|
|
636
|
+
{ url = "https://files.pythonhosted.org/packages/28/9d/d825b07ee7ea9e2d61df92a860033c94e06e7300d50a1c2653aac27d24fe/ruff-0.15.21-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:0f212c5d7d54c01bbfe6dcab02b724a39300f3e34ed7acbe995ccb320a2c58bd", size = 11139539, upload-time = "2026-07-09T20:00:57.809Z" },
|
|
637
|
+
{ url = "https://files.pythonhosted.org/packages/f5/de/3b107712e642f063c7a9e0887c427b22cb44097de5aab36c05f2e280670c/ruff-0.15.21-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e6312e41bc96791299614995ea3a977c5857c3b5662b1ecef6755b02b87cb646", size = 10595437, upload-time = "2026-07-09T20:01:00.006Z" },
|
|
638
|
+
{ url = "https://files.pythonhosted.org/packages/9a/6f/b4523cc90ba239ede441447a19d0c968846a3012e5a0b0c5b62831a3d5e3/ruff-0.15.21-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:01d65b4831c6b2a4ba8ee6faa84049d44d982b7a706e622c4094c509e51673be", size = 10990053, upload-time = "2026-07-09T20:01:02.187Z" },
|
|
639
|
+
{ url = "https://files.pythonhosted.org/packages/92/cc/c6a9872a5375f0628875481cf2f66b13d7d865bf3ca2e57f91c7e762d976/ruff-0.15.21-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:2c5a913a589120ce67933d5d05fd6ddbcc2481c6a054980ee767f7414c72b4fd", size = 10666096, upload-time = "2026-07-09T20:01:04.299Z" },
|
|
640
|
+
{ url = "https://files.pythonhosted.org/packages/ab/97/c621f7a17e097f1790fa3af6374138823b330b2d03fc38337945daca212c/ruff-0.15.21-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5ef04b681d02ad4dc9620f00f83ac5c22f652d0e9a9cfe431d219b16ad5ccc41", size = 11537011, upload-time = "2026-07-09T20:01:06.771Z" },
|
|
641
|
+
{ url = "https://files.pythonhosted.org/packages/ea/51/d928727e476e25ccc57c6f449ffd80241a651a973ad949d39cfb2a771d28/ruff-0.15.21-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:16d090c0740916594157e75b80d666eab8e78083b39b3b0e1d698f4670a17b86", size = 12347101, upload-time = "2026-07-09T20:01:08.859Z" },
|
|
642
|
+
{ url = "https://files.pythonhosted.org/packages/1e/88/8cd62026802b16018ad06931d87997cf795ba2a6239ab659606c87d96bf0/ruff-0.15.21-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3a10e74757dd65004d779b73e2f3c5210156d9980b41224d50d2ebcf1db51e67", size = 11572001, upload-time = "2026-07-09T20:01:11.092Z" },
|
|
643
|
+
{ url = "https://files.pythonhosted.org/packages/b2/97/f63084cf55444fc110e8cb985ebfcc592af47f597d44453d778cb81bc156/ruff-0.15.21-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bab0905d2f29e0d9fbc3c373ed23db0095edaa3f71f1f4f519ec15134d9e85c8", size = 11549239, upload-time = "2026-07-09T20:01:13.27Z" },
|
|
644
|
+
{ url = "https://files.pythonhosted.org/packages/9d/77/f107da4a2874b7715914b03f09ba9c54424de3ff8a1cc5d015d3ee2ce0ac/ruff-0.15.21-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:00eca240af5789fec6fe7df74c088cc1f9644ed83027113468efba7c92b94075", size = 11535340, upload-time = "2026-07-09T20:01:15.206Z" },
|
|
645
|
+
{ url = "https://files.pythonhosted.org/packages/d5/e9/601deb322d3303a7bf212b0100ead6f2ee3f6a044d89c30f2f92bf83c731/ruff-0.15.21-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:262ab31557a75141325e32d3357f3597645a7f084e732b6b054dde428ecd9341", size = 10964048, upload-time = "2026-07-09T20:01:17.723Z" },
|
|
646
|
+
{ url = "https://files.pythonhosted.org/packages/ea/2e/0f2176d1e99c15192caea19c8c3a0a955246b4cb4de795042eeb616345cd/ruff-0.15.21-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:659c4e7a4212f83306045ec7c5e5a356d16d9a6ef4ae0c7a4d872914fc655d9d", size = 10667055, upload-time = "2026-07-09T20:01:19.73Z" },
|
|
647
|
+
{ url = "https://files.pythonhosted.org/packages/48/60/abd74a02e0c4214f12a68becfd30af7165cfdcb0e661ecdc60bbb949c09a/ruff-0.15.21-py3-none-musllinux_1_2_i686.whl", hash = "sha256:9e866eab611a5f959d36df2d10e446973a3610bc42b0c15b31dc27977d59c233", size = 11242043, upload-time = "2026-07-09T20:01:21.947Z" },
|
|
648
|
+
{ url = "https://files.pythonhosted.org/packages/b2/c6/583075d8ccabb4b229345edcaf1545eb3d8d6be90f686a479d7e94088bbf/ruff-0.15.21-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:e89bc93c0d3803ba870b55c29671bad9dc6d94bb1eb181b056b52eb05b52854f", size = 11648064, upload-time = "2026-07-09T20:01:24.023Z" },
|
|
649
|
+
{ url = "https://files.pythonhosted.org/packages/3a/3c/37d0ecb729a7cc2d393ea7dce316fc585680f35d93b8d62139d7d0a3700c/ruff-0.15.21-py3-none-win32.whl", hash = "sha256:01f8d5be84823c172b389e123174f781f9daf86d6c58719d603f941932195cdd", size = 10896555, upload-time = "2026-07-09T20:01:26.941Z" },
|
|
650
|
+
{ url = "https://files.pythonhosted.org/packages/c0/b8/e43466b2a6067ce91e669068f6e28d6c719a920f014b070d5c8731725de3/ruff-0.15.21-py3-none-win_amd64.whl", hash = "sha256:d4b8d9a2f0f12b816b50447f6eccb9f4bb01a6b82c86b50fb3b5354b458dc6d3", size = 12038772, upload-time = "2026-07-09T20:01:29.497Z" },
|
|
651
|
+
{ url = "https://files.pythonhosted.org/packages/dd/75/e90ab9aeece218a9fc5a5bc3ec97d0ee6bb3c4ff95869463c1de58e29a1c/ruff-0.15.21-py3-none-win_arm64.whl", hash = "sha256:6e83115d4b9377c1cbc13abf0e051f069fab0ef815ea0504a8a008cee24dd0a8", size = 11375265, upload-time = "2026-07-09T20:01:31.772Z" },
|
|
652
|
+
]
|
|
653
|
+
|
|
628
654
|
[[package]]
|
|
629
655
|
name = "sniffio"
|
|
630
656
|
version = "1.3.1"
|
httpx_hedged-0.2.0/LICENSE
DELETED
|
@@ -1,28 +0,0 @@
|
|
|
1
|
-
BSD 3-Clause License
|
|
2
|
-
|
|
3
|
-
Copyright (c) 2025, Brendan Fahy
|
|
4
|
-
|
|
5
|
-
Redistribution and use in source and binary forms, with or without
|
|
6
|
-
modification, are permitted provided that the following conditions are met:
|
|
7
|
-
|
|
8
|
-
1. Redistributions of source code must retain the above copyright notice, this
|
|
9
|
-
list of conditions and the following disclaimer.
|
|
10
|
-
|
|
11
|
-
2. Redistributions in binary form must reproduce the above copyright notice,
|
|
12
|
-
this list of conditions and the following disclaimer in the documentation
|
|
13
|
-
and/or other materials provided with the distribution.
|
|
14
|
-
|
|
15
|
-
3. Neither the name of the copyright holder nor the names of its
|
|
16
|
-
contributors may be used to endorse or promote products derived from
|
|
17
|
-
this software without specific prior written permission.
|
|
18
|
-
|
|
19
|
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
|
20
|
-
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
|
21
|
-
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
|
22
|
-
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
|
23
|
-
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
|
24
|
-
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
|
25
|
-
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
|
26
|
-
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
|
27
|
-
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
|
28
|
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|