hedge-python 0.1.0__tar.gz → 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {hedge_python-0.1.0 → hedge_python-0.2.0}/.github/workflows/ci.yml +4 -1
- {hedge_python-0.1.0 → hedge_python-0.2.0}/CHANGELOG.md +34 -1
- {hedge_python-0.1.0 → hedge_python-0.2.0}/Makefile +8 -8
- {hedge_python-0.1.0 → hedge_python-0.2.0}/PKG-INFO +72 -7
- {hedge_python-0.1.0 → hedge_python-0.2.0}/README.ja.md +59 -3
- {hedge_python-0.1.0 → hedge_python-0.2.0}/README.md +62 -5
- {hedge_python-0.1.0 → hedge_python-0.2.0}/README.zh-CN.md +54 -3
- hedge_python-0.2.0/benchmark/plot_optimization.py +221 -0
- hedge_python-0.2.0/eval.png +0 -0
- hedge_python-0.2.0/eval_multi_framework.png +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/examples/README.md +3 -0
- hedge_python-0.2.0/examples/niquests_basic.py +53 -0
- hedge_python-0.2.0/examples/openai_hedged.py +104 -0
- hedge_python-0.2.0/examples/tornado_basic.py +52 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/pyproject.toml +14 -3
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/__init__.py +1 -1
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/_stats.py +12 -10
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/interceptor/_grpc.py +1 -2
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/sketch/_ddsketch.py +20 -6
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/sketch/_windowed.py +1 -1
- hedge_python-0.2.0/src/hedge/transport/__init__.py +36 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/transport/_aiohttp.py +3 -6
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/transport/_base.py +31 -7
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/transport/_httpx.py +3 -4
- hedge_python-0.2.0/src/hedge/transport/_niquests.py +133 -0
- hedge_python-0.2.0/src/hedge/transport/_tornado.py +130 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/benchmark/test_bench_hedge_comparison.py +116 -105
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/benchmark/test_bench_multi_framework.py +67 -49
- hedge_python-0.2.0/tests/conftest.py +26 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/test_grpc_interceptor.py +21 -62
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_ddsketch.py +32 -2
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_grpc_interceptor_branches.py +67 -21
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_hedge_scheduler.py +36 -4
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_lazy_imports.py +10 -0
- hedge_python-0.2.0/tests/unit/test_niquests_transport.py +155 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_token_bucket.py +12 -0
- hedge_python-0.2.0/tests/unit/test_tornado_transport.py +163 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_transport_import_errors.py +26 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_windowed_sketch.py +18 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/uv.lock +382 -3
- hedge_python-0.1.0/eval.png +0 -0
- hedge_python-0.1.0/eval_multi_framework.png +0 -0
- hedge_python-0.1.0/src/hedge/transport/__init__.py +0 -24
- hedge_python-0.1.0/tests/conftest.py +0 -9
- {hedge_python-0.1.0 → hedge_python-0.2.0}/.github/workflows/release.yml +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/.gitignore +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/LICENSE +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/benchmark/plot.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/examples/aiohttp_basic.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/examples/grpc_stream.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/examples/grpc_unary.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/examples/httpx_basic.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/_options.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/budget/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/budget/_token_bucket.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/interceptor/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/py.typed +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/src/hedge/sketch/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/benchmark/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/benchmark/test_bench_ddsketch.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/benchmark/test_bench_token_bucket.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/proto/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/proto/testservice.proto +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/proto/testservice_pb2.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/proto/testservice_pb2_grpc.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/test_aiohttp_session.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/integration/test_httpx_transport.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/__init__.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_options.py +0 -0
- {hedge_python-0.1.0 → hedge_python-0.2.0}/tests/unit/test_stats.py +0 -0
|
@@ -10,6 +10,9 @@ concurrency:
|
|
|
10
10
|
group: ${{ github.workflow }}-${{ github.ref }}
|
|
11
11
|
cancel-in-progress: true
|
|
12
12
|
|
|
13
|
+
env:
|
|
14
|
+
FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
|
|
15
|
+
|
|
13
16
|
jobs:
|
|
14
17
|
lint:
|
|
15
18
|
name: Lint & Type Check
|
|
@@ -64,7 +67,7 @@ jobs:
|
|
|
64
67
|
with:
|
|
65
68
|
python-version: "3.13"
|
|
66
69
|
- run: uv sync --all-extras
|
|
67
|
-
- run: uv run pytest tests/ --cov=src/hedge --cov-report=xml --cov-report=term-missing
|
|
70
|
+
- run: uv run pytest tests/ --cov=src/hedge --cov-report=xml --cov-report=term-missing --ignore=tests/benchmark
|
|
68
71
|
- uses: codecov/codecov-action@v4
|
|
69
72
|
with:
|
|
70
73
|
file: coverage.xml
|
|
@@ -5,6 +5,38 @@ All notable changes to this project will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.2.0] - 2026-06-12
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- **niquests adapter**: `HedgedNiquestsSession` — drop-in wrapper for
|
|
13
|
+
`niquests.AsyncSession` with adaptive hedging on `GET`/`HEAD`/`OPTIONS`.
|
|
14
|
+
- **tornado adapter**: `HedgedTornadoClient` — wraps
|
|
15
|
+
`tornado.httpclient.AsyncHTTPClient` with adaptive hedging via `fetch()`.
|
|
16
|
+
- **OpenAI integration example** (`examples/openai_hedged.py`): demonstrates
|
|
17
|
+
injecting `HedgedHttpxTransport` into `openai.AsyncOpenAI` via `http_client`.
|
|
18
|
+
- New examples: `examples/niquests_basic.py`, `examples/tornado_basic.py`.
|
|
19
|
+
- Lazy-import registry refactored to a data-driven `_LAZY_IMPORTS` dict
|
|
20
|
+
in `hedge.transport.__init__` for easier extensibility.
|
|
21
|
+
- `pyproject.toml` optional extras: `[niquests]`, `[tornado]` (also added
|
|
22
|
+
to `[all]` and `[dev]`).
|
|
23
|
+
- Comprehensive unit tests for niquests and tornado adapters.
|
|
24
|
+
- Import-error tests for `niquests`, `tornado`, and `grpc` modules.
|
|
25
|
+
- Additional coverage tests: `TokenBucket.set_rps` truncation, `WindowedSketch`
|
|
26
|
+
double-start idempotency, `DDSketch` quantile edge cases, gRPC interceptor
|
|
27
|
+
`current_task()` None branch and `_PrependedStream` EOF.
|
|
28
|
+
|
|
29
|
+
### Changed
|
|
30
|
+
|
|
31
|
+
- `Makefile`: `test`, `test-unit`, `test-integration`, and `coverage` targets
|
|
32
|
+
now depend on `install` to ensure all extras are available before running.
|
|
33
|
+
- `examples/README.md` updated with new framework entries.
|
|
34
|
+
|
|
35
|
+
### Other
|
|
36
|
+
|
|
37
|
+
- Test coverage: **97%** (150 tests).
|
|
38
|
+
- Supports Python 3.9 → 3.14.
|
|
39
|
+
|
|
8
40
|
## [0.1.0] - 2026-04-23
|
|
9
41
|
|
|
10
42
|
### Added
|
|
@@ -61,5 +93,6 @@ _Initial release — nothing removed._
|
|
|
61
93
|
`[grpc]`, `[all]`, `[dev]`.
|
|
62
94
|
- Supports Python 3.9 → 3.14.
|
|
63
95
|
|
|
64
|
-
[Unreleased]: https://github.com/sunhailin-Leo/hedge-python/compare/v0.
|
|
96
|
+
[Unreleased]: https://github.com/sunhailin-Leo/hedge-python/compare/v0.2.0...HEAD
|
|
97
|
+
[0.2.0]: https://github.com/sunhailin-Leo/hedge-python/compare/v0.1.0...v0.2.0
|
|
65
98
|
[0.1.0]: https://github.com/sunhailin-Leo/hedge-python/releases/tag/v0.1.0
|
|
@@ -17,16 +17,16 @@ format:
|
|
|
17
17
|
typecheck:
|
|
18
18
|
uv run mypy src/hedge/
|
|
19
19
|
|
|
20
|
-
# Run all tests
|
|
21
|
-
test:
|
|
22
|
-
uv run pytest tests/ -v --tb=short
|
|
20
|
+
# Run all tests (unit + integration, excludes benchmarks)
|
|
21
|
+
test: install
|
|
22
|
+
uv run pytest tests/ -v --tb=short --ignore=tests/benchmark
|
|
23
23
|
|
|
24
24
|
# Unit tests only
|
|
25
|
-
test-unit:
|
|
25
|
+
test-unit: install
|
|
26
26
|
uv run pytest tests/unit/ -v --tb=short
|
|
27
27
|
|
|
28
28
|
# Integration tests only
|
|
29
|
-
test-integration:
|
|
29
|
+
test-integration: install
|
|
30
30
|
uv run pytest tests/integration/ -v --tb=short -m integration
|
|
31
31
|
|
|
32
32
|
# Benchmark tests
|
|
@@ -45,9 +45,9 @@ bench-multi:
|
|
|
45
45
|
bench-plot:
|
|
46
46
|
uv run python benchmark/plot.py
|
|
47
47
|
|
|
48
|
-
# Coverage report
|
|
49
|
-
coverage:
|
|
50
|
-
uv run pytest tests/ --cov=src/hedge --cov-report=term-missing --cov-report=html
|
|
48
|
+
# Coverage report (excludes benchmarks for speed)
|
|
49
|
+
coverage: install
|
|
50
|
+
uv run pytest tests/ --cov=src/hedge --cov-report=term-missing --cov-report=html --ignore=tests/benchmark
|
|
51
51
|
|
|
52
52
|
# Run full CI checks locally
|
|
53
53
|
ci: lint typecheck test coverage
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: hedge-python
|
|
3
|
-
Version: 0.
|
|
3
|
+
Version: 0.2.0
|
|
4
4
|
Summary: Adaptive hedged request library for Python. Learns per-host latency via DDSketch, fires backup requests at estimated p90, caps hedge rate with token bucket.
|
|
5
5
|
Author-email: LeoSun <sunhailin.shl@antgroup.com>
|
|
6
6
|
License: MIT
|
|
7
7
|
License-File: LICENSE
|
|
8
|
-
Keywords: aiohttp,ddsketch,grpc,hedge,httpx,latency,tail-latency
|
|
8
|
+
Keywords: aiohttp,ddsketch,grpc,hedge,httpx,latency,niquests,tail-latency,tornado
|
|
9
9
|
Classifier: Development Status :: 3 - Alpha
|
|
10
10
|
Classifier: Framework :: AsyncIO
|
|
11
11
|
Classifier: Intended Audience :: Developers
|
|
@@ -26,7 +26,9 @@ Provides-Extra: all
|
|
|
26
26
|
Requires-Dist: aiohttp>=3.9.0; extra == 'all'
|
|
27
27
|
Requires-Dist: grpcio>=1.50.0; extra == 'all'
|
|
28
28
|
Requires-Dist: httpx>=0.24.0; extra == 'all'
|
|
29
|
+
Requires-Dist: niquests>=3.0.0; extra == 'all'
|
|
29
30
|
Requires-Dist: protobuf>=4.21.0; extra == 'all'
|
|
31
|
+
Requires-Dist: tornado>=6.0.0; extra == 'all'
|
|
30
32
|
Provides-Extra: dev
|
|
31
33
|
Requires-Dist: aiohttp>=3.9.0; extra == 'dev'
|
|
32
34
|
Requires-Dist: grpcio-tools>=1.50.0; extra == 'dev'
|
|
@@ -34,6 +36,7 @@ Requires-Dist: grpcio>=1.50.0; extra == 'dev'
|
|
|
34
36
|
Requires-Dist: httpx>=0.24.0; extra == 'dev'
|
|
35
37
|
Requires-Dist: matplotlib>=3.7.0; extra == 'dev'
|
|
36
38
|
Requires-Dist: mypy>=1.0; extra == 'dev'
|
|
39
|
+
Requires-Dist: niquests>=3.0.0; extra == 'dev'
|
|
37
40
|
Requires-Dist: protobuf>=4.21.0; extra == 'dev'
|
|
38
41
|
Requires-Dist: pytest-asyncio>=0.21.0; extra == 'dev'
|
|
39
42
|
Requires-Dist: pytest-benchmark>=4.0; extra == 'dev'
|
|
@@ -41,11 +44,16 @@ Requires-Dist: pytest-cov>=4.0; extra == 'dev'
|
|
|
41
44
|
Requires-Dist: pytest-timeout>=2.1.0; extra == 'dev'
|
|
42
45
|
Requires-Dist: pytest>=7.0; extra == 'dev'
|
|
43
46
|
Requires-Dist: ruff>=0.4.0; extra == 'dev'
|
|
47
|
+
Requires-Dist: tornado>=6.0.0; extra == 'dev'
|
|
44
48
|
Provides-Extra: grpc
|
|
45
49
|
Requires-Dist: grpcio>=1.50.0; extra == 'grpc'
|
|
46
50
|
Requires-Dist: protobuf>=4.21.0; extra == 'grpc'
|
|
47
51
|
Provides-Extra: httpx
|
|
48
52
|
Requires-Dist: httpx>=0.24.0; extra == 'httpx'
|
|
53
|
+
Provides-Extra: niquests
|
|
54
|
+
Requires-Dist: niquests>=3.0.0; extra == 'niquests'
|
|
55
|
+
Provides-Extra: tornado
|
|
56
|
+
Requires-Dist: tornado>=6.0.0; extra == 'tornado'
|
|
49
57
|
Description-Content-Type: text/markdown
|
|
50
58
|
|
|
51
59
|
# hedge-python
|
|
@@ -54,7 +62,7 @@ Description-Content-Type: text/markdown
|
|
|
54
62
|
|
|
55
63
|
[](https://github.com/sunhailin-Leo/hedge-python/actions)
|
|
56
64
|
[](#testing)
|
|
57
|
-
[](pyproject.toml)
|
|
58
66
|
[](LICENSE)
|
|
59
67
|
|
|
60
68
|
Python port of [bhope/hedge](https://github.com/bhope/hedge) — **adaptive hedged
|
|
@@ -64,8 +72,8 @@ requests for tail-latency optimisation**.
|
|
|
64
72
|
[DDSketch](https://arxiv.org/abs/2004.08604), races a backup request when the
|
|
65
73
|
primary exceeds its estimated p90, and caps the hedge rate with a token bucket
|
|
66
74
|
to prevent load amplification during outages. Zero configuration required.
|
|
67
|
-
First-class support for **httpx**, **aiohttp**, and **gRPC** (unary +
|
|
68
|
-
server-streaming).
|
|
75
|
+
First-class support for **httpx**, **aiohttp**, **niquests**, **tornado**, and **gRPC** (unary +
|
|
76
|
+
server-streaming). Works out of the box with **OpenAI's Python SDK**.
|
|
69
77
|
|
|
70
78
|
Inspired by Dean & Barroso, [_The Tail at Scale_](https://research.google/pubs/the-tail-at-scale/) (CACM 2013).
|
|
71
79
|
|
|
@@ -101,6 +109,8 @@ Across all three frameworks, p99 latency drops by **60–66%** at the cost of
|
|
|
101
109
|
# Install with your preferred framework
|
|
102
110
|
pip install hedge-python[httpx]
|
|
103
111
|
pip install hedge-python[aiohttp]
|
|
112
|
+
pip install hedge-python[niquests]
|
|
113
|
+
pip install hedge-python[tornado]
|
|
104
114
|
pip install hedge-python[grpc]
|
|
105
115
|
pip install hedge-python[all] # all frameworks
|
|
106
116
|
```
|
|
@@ -166,6 +176,59 @@ async def make_channel():
|
|
|
166
176
|
)
|
|
167
177
|
```
|
|
168
178
|
|
|
179
|
+
### niquests
|
|
180
|
+
|
|
181
|
+
```python
|
|
182
|
+
import asyncio
|
|
183
|
+
from hedge import HedgeConfig
|
|
184
|
+
from hedge.transport import HedgedNiquestsSession
|
|
185
|
+
|
|
186
|
+
async def main():
|
|
187
|
+
async with HedgedNiquestsSession(config=HedgeConfig()) as session:
|
|
188
|
+
resp = await session.get("https://api.example.com/data")
|
|
189
|
+
print(resp.status_code)
|
|
190
|
+
|
|
191
|
+
asyncio.run(main())
|
|
192
|
+
```
|
|
193
|
+
|
|
194
|
+
### tornado
|
|
195
|
+
|
|
196
|
+
```python
|
|
197
|
+
import asyncio
|
|
198
|
+
from hedge import HedgeConfig
|
|
199
|
+
from hedge.transport import HedgedTornadoClient
|
|
200
|
+
|
|
201
|
+
async def main():
|
|
202
|
+
async with HedgedTornadoClient(config=HedgeConfig()) as client:
|
|
203
|
+
resp = await client.fetch("https://api.example.com/data")
|
|
204
|
+
print(resp.code)
|
|
205
|
+
|
|
206
|
+
asyncio.run(main())
|
|
207
|
+
```
|
|
208
|
+
|
|
209
|
+
### OpenAI SDK
|
|
210
|
+
|
|
211
|
+
Since the OpenAI Python SDK uses httpx under the hood, you can inject
|
|
212
|
+
`HedgedHttpxTransport` directly via the `http_client` parameter:
|
|
213
|
+
|
|
214
|
+
```python
|
|
215
|
+
import httpx
|
|
216
|
+
from openai import AsyncOpenAI
|
|
217
|
+
from hedge import HedgeConfig
|
|
218
|
+
from hedge.transport import HedgedHttpxTransport
|
|
219
|
+
|
|
220
|
+
transport = HedgedHttpxTransport(config=HedgeConfig(percentile=0.95))
|
|
221
|
+
client = AsyncOpenAI(
|
|
222
|
+
api_key="sk-...",
|
|
223
|
+
http_client=httpx.AsyncClient(transport=transport),
|
|
224
|
+
)
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
> **Note**: OpenAI's core APIs (Chat Completions, Embeddings, etc.) use POST,
|
|
228
|
+
> so they are **not** hedged by default — avoiding double billing. Only GET
|
|
229
|
+
> endpoints (e.g. model listing) are hedged. See
|
|
230
|
+
> [`examples/openai_hedged.py`](examples/openai_hedged.py) for a full example.
|
|
231
|
+
|
|
169
232
|
For server streaming, the hedge signal is **time-to-first-message (TTFM)**: if
|
|
170
233
|
the primary stream doesn't yield its first chunk within the estimated p90,
|
|
171
234
|
a backup stream is started. Whichever yields first wins and continues
|
|
@@ -305,7 +368,7 @@ make ci # lint + typecheck + test + coverage
|
|
|
305
368
|
* **Benchmarks** (`tests/benchmark/`): DDSketch microbench, token bucket
|
|
306
369
|
microbench, four-config comparison, three-framework comparison.
|
|
307
370
|
|
|
308
|
-
Current coverage: **97%** (
|
|
371
|
+
Current coverage: **97%** (150 tests, ~7 seconds).
|
|
309
372
|
|
|
310
373
|
---
|
|
311
374
|
|
|
@@ -325,7 +388,9 @@ hedge-python/
|
|
|
325
388
|
│ ├── transport/
|
|
326
389
|
│ │ ├── _base.py # Shared HedgeScheduler logic
|
|
327
390
|
│ │ ├── _httpx.py # httpx AsyncBaseTransport adapter
|
|
328
|
-
│ │
|
|
391
|
+
│ │ ├── _aiohttp.py # aiohttp session wrapper
|
|
392
|
+
│ │ ├── _niquests.py # niquests session wrapper
|
|
393
|
+
│ │ └── _tornado.py # tornado AsyncHTTPClient wrapper
|
|
329
394
|
│ └── interceptor/
|
|
330
395
|
│ └── _grpc.py # gRPC unary + server-stream interceptors
|
|
331
396
|
├── tests/
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[](https://github.com/sunhailin-Leo/hedge-python/actions)
|
|
6
6
|
[](#テスト)
|
|
7
|
-
[](pyproject.toml)
|
|
8
8
|
[](LICENSE)
|
|
9
9
|
|
|
10
10
|
> 📘 公式ドキュメントは **[英語版](README.md)** が主です。本書は概要をすばやく把握するための日本語訳です。
|
|
@@ -15,7 +15,8 @@
|
|
|
15
15
|
`hedge-python` は [DDSketch](https://arxiv.org/abs/2004.08604) を用いてホストごとのレイテンシ分布を学習し、
|
|
16
16
|
プライマリリクエストが推定 p90 を超えた時点でバックアップリクエストを発射、
|
|
17
17
|
さらにトークンバケットでヘッジレートを制限することで、障害時の負荷増幅を防ぎます。
|
|
18
|
-
**設定不要** で、**httpx**、**aiohttp**、**gRPC**(unary + server-streaming)を第一級でサポートします。
|
|
18
|
+
**設定不要** で、**httpx**、**aiohttp**、**niquests**、**tornado**、**gRPC**(unary + server-streaming)を第一級でサポートします。
|
|
19
|
+
`http_client` パラメータ経由で **OpenAI Python SDK** とのシームレスな統合もサポートしています。
|
|
19
20
|
|
|
20
21
|
Dean & Barroso の [_The Tail at Scale_](https://research.google/pubs/the-tail-at-scale/)(CACM 2013)に着想を得ています。
|
|
21
22
|
|
|
@@ -52,6 +53,8 @@ Dean & Barroso の [_The Tail at Scale_](https://research.google/pubs/the-tail-a
|
|
|
52
53
|
# 必要なフレームワーク向けにインストール
|
|
53
54
|
pip install hedge-python[httpx]
|
|
54
55
|
pip install hedge-python[aiohttp]
|
|
56
|
+
pip install hedge-python[niquests]
|
|
57
|
+
pip install hedge-python[tornado]
|
|
55
58
|
pip install hedge-python[grpc]
|
|
56
59
|
pip install hedge-python[all] # 全フレームワーク
|
|
57
60
|
```
|
|
@@ -117,6 +120,59 @@ async def make_channel():
|
|
|
117
120
|
)
|
|
118
121
|
```
|
|
119
122
|
|
|
123
|
+
### niquests
|
|
124
|
+
|
|
125
|
+
```python
|
|
126
|
+
import asyncio
|
|
127
|
+
from hedge import HedgeConfig
|
|
128
|
+
from hedge.transport import HedgedNiquestsSession
|
|
129
|
+
|
|
130
|
+
async def main():
|
|
131
|
+
async with HedgedNiquestsSession(config=HedgeConfig()) as session:
|
|
132
|
+
resp = await session.get("https://api.example.com/data")
|
|
133
|
+
print(resp.status_code)
|
|
134
|
+
|
|
135
|
+
asyncio.run(main())
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
### tornado
|
|
139
|
+
|
|
140
|
+
```python
|
|
141
|
+
import asyncio
|
|
142
|
+
from hedge import HedgeConfig
|
|
143
|
+
from hedge.transport import HedgedTornadoClient
|
|
144
|
+
|
|
145
|
+
async def main():
|
|
146
|
+
async with HedgedTornadoClient(config=HedgeConfig()) as client:
|
|
147
|
+
resp = await client.fetch("https://api.example.com/data")
|
|
148
|
+
print(resp.code)
|
|
149
|
+
|
|
150
|
+
asyncio.run(main())
|
|
151
|
+
```
|
|
152
|
+
|
|
153
|
+
### OpenAI SDK
|
|
154
|
+
|
|
155
|
+
OpenAI Python SDK は内部で httpx を使用しているため、`http_client` パラメータ経由で
|
|
156
|
+
`HedgedHttpxTransport` を直接注入できます:
|
|
157
|
+
|
|
158
|
+
```python
|
|
159
|
+
import httpx
|
|
160
|
+
from openai import AsyncOpenAI
|
|
161
|
+
from hedge import HedgeConfig
|
|
162
|
+
from hedge.transport import HedgedHttpxTransport
|
|
163
|
+
|
|
164
|
+
transport = HedgedHttpxTransport(config=HedgeConfig(percentile=0.95))
|
|
165
|
+
client = AsyncOpenAI(
|
|
166
|
+
api_key="sk-...",
|
|
167
|
+
http_client=httpx.AsyncClient(transport=transport),
|
|
168
|
+
)
|
|
169
|
+
```
|
|
170
|
+
|
|
171
|
+
> **注意**: OpenAI のコア API(Chat Completions、Embeddings など)は POST を使用するため、
|
|
172
|
+
> デフォルトではヘッジ **されません** —— 二重課金を回避するためです。GET エンドポイント
|
|
173
|
+
>(モデル一覧など)のみがヘッジされます。完全な例は
|
|
174
|
+
> [`examples/openai_hedged.py`](examples/openai_hedged.py) を参照してください。
|
|
175
|
+
|
|
120
176
|
server-streaming におけるヘッジ信号は **TTFM(Time To First Message)** です。
|
|
121
177
|
プライマリストリームが推定 p90 までに最初の chunk を返さなければ、バックアップストリームを起動します。
|
|
122
178
|
先に最初の chunk を返した側が勝ち、以降のストリーミングを引き継ぎます。
|
|
@@ -240,7 +296,7 @@ make ci # lint + typecheck + test + coverage
|
|
|
240
296
|
* **結合テスト** (`tests/integration/`): 実 httpx transport、実 aiohttp session、**実ローカル gRPC サーバ**(`.proto` + 生成 pb2 込み)。
|
|
241
297
|
* **ベンチマーク** (`tests/benchmark/`): DDSketch マイクロベンチ、トークンバケットマイクロベンチ、4 構成比較、3 フレームワーク比較。
|
|
242
298
|
|
|
243
|
-
現在のカバレッジ: **97%**(
|
|
299
|
+
現在のカバレッジ: **97%**(150 テスト、約 7 秒)。
|
|
244
300
|
|
|
245
301
|
---
|
|
246
302
|
|
|
@@ -4,7 +4,7 @@
|
|
|
4
4
|
|
|
5
5
|
[](https://github.com/sunhailin-Leo/hedge-python/actions)
|
|
6
6
|
[](#testing)
|
|
7
|
-
[](pyproject.toml)
|
|
8
8
|
[](LICENSE)
|
|
9
9
|
|
|
10
10
|
Python port of [bhope/hedge](https://github.com/bhope/hedge) — **adaptive hedged
|
|
@@ -14,8 +14,8 @@ requests for tail-latency optimisation**.
|
|
|
14
14
|
[DDSketch](https://arxiv.org/abs/2004.08604), races a backup request when the
|
|
15
15
|
primary exceeds its estimated p90, and caps the hedge rate with a token bucket
|
|
16
16
|
to prevent load amplification during outages. Zero configuration required.
|
|
17
|
-
First-class support for **httpx**, **aiohttp**, and **gRPC** (unary +
|
|
18
|
-
server-streaming).
|
|
17
|
+
First-class support for **httpx**, **aiohttp**, **niquests**, **tornado**, and **gRPC** (unary +
|
|
18
|
+
server-streaming). Works out of the box with **OpenAI's Python SDK**.
|
|
19
19
|
|
|
20
20
|
Inspired by Dean & Barroso, [_The Tail at Scale_](https://research.google/pubs/the-tail-at-scale/) (CACM 2013).
|
|
21
21
|
|
|
@@ -51,6 +51,8 @@ Across all three frameworks, p99 latency drops by **60–66%** at the cost of
|
|
|
51
51
|
# Install with your preferred framework
|
|
52
52
|
pip install hedge-python[httpx]
|
|
53
53
|
pip install hedge-python[aiohttp]
|
|
54
|
+
pip install hedge-python[niquests]
|
|
55
|
+
pip install hedge-python[tornado]
|
|
54
56
|
pip install hedge-python[grpc]
|
|
55
57
|
pip install hedge-python[all] # all frameworks
|
|
56
58
|
```
|
|
@@ -116,6 +118,59 @@ async def make_channel():
|
|
|
116
118
|
)
|
|
117
119
|
```
|
|
118
120
|
|
|
121
|
+
### niquests
|
|
122
|
+
|
|
123
|
+
```python
|
|
124
|
+
import asyncio
|
|
125
|
+
from hedge import HedgeConfig
|
|
126
|
+
from hedge.transport import HedgedNiquestsSession
|
|
127
|
+
|
|
128
|
+
async def main():
|
|
129
|
+
async with HedgedNiquestsSession(config=HedgeConfig()) as session:
|
|
130
|
+
resp = await session.get("https://api.example.com/data")
|
|
131
|
+
print(resp.status_code)
|
|
132
|
+
|
|
133
|
+
asyncio.run(main())
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
### tornado
|
|
137
|
+
|
|
138
|
+
```python
|
|
139
|
+
import asyncio
|
|
140
|
+
from hedge import HedgeConfig
|
|
141
|
+
from hedge.transport import HedgedTornadoClient
|
|
142
|
+
|
|
143
|
+
async def main():
|
|
144
|
+
async with HedgedTornadoClient(config=HedgeConfig()) as client:
|
|
145
|
+
resp = await client.fetch("https://api.example.com/data")
|
|
146
|
+
print(resp.code)
|
|
147
|
+
|
|
148
|
+
asyncio.run(main())
|
|
149
|
+
```
|
|
150
|
+
|
|
151
|
+
### OpenAI SDK
|
|
152
|
+
|
|
153
|
+
Since the OpenAI Python SDK uses httpx under the hood, you can inject
|
|
154
|
+
`HedgedHttpxTransport` directly via the `http_client` parameter:
|
|
155
|
+
|
|
156
|
+
```python
|
|
157
|
+
import httpx
|
|
158
|
+
from openai import AsyncOpenAI
|
|
159
|
+
from hedge import HedgeConfig
|
|
160
|
+
from hedge.transport import HedgedHttpxTransport
|
|
161
|
+
|
|
162
|
+
transport = HedgedHttpxTransport(config=HedgeConfig(percentile=0.95))
|
|
163
|
+
client = AsyncOpenAI(
|
|
164
|
+
api_key="sk-...",
|
|
165
|
+
http_client=httpx.AsyncClient(transport=transport),
|
|
166
|
+
)
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
> **Note**: OpenAI's core APIs (Chat Completions, Embeddings, etc.) use POST,
|
|
170
|
+
> so they are **not** hedged by default — avoiding double billing. Only GET
|
|
171
|
+
> endpoints (e.g. model listing) are hedged. See
|
|
172
|
+
> [`examples/openai_hedged.py`](examples/openai_hedged.py) for a full example.
|
|
173
|
+
|
|
119
174
|
For server streaming, the hedge signal is **time-to-first-message (TTFM)**: if
|
|
120
175
|
the primary stream doesn't yield its first chunk within the estimated p90,
|
|
121
176
|
a backup stream is started. Whichever yields first wins and continues
|
|
@@ -255,7 +310,7 @@ make ci # lint + typecheck + test + coverage
|
|
|
255
310
|
* **Benchmarks** (`tests/benchmark/`): DDSketch microbench, token bucket
|
|
256
311
|
microbench, four-config comparison, three-framework comparison.
|
|
257
312
|
|
|
258
|
-
Current coverage: **97%** (
|
|
313
|
+
Current coverage: **97%** (150 tests, ~7 seconds).
|
|
259
314
|
|
|
260
315
|
---
|
|
261
316
|
|
|
@@ -275,7 +330,9 @@ hedge-python/
|
|
|
275
330
|
│ ├── transport/
|
|
276
331
|
│ │ ├── _base.py # Shared HedgeScheduler logic
|
|
277
332
|
│ │ ├── _httpx.py # httpx AsyncBaseTransport adapter
|
|
278
|
-
│ │
|
|
333
|
+
│ │ ├── _aiohttp.py # aiohttp session wrapper
|
|
334
|
+
│ │ ├── _niquests.py # niquests session wrapper
|
|
335
|
+
│ │ └── _tornado.py # tornado AsyncHTTPClient wrapper
|
|
279
336
|
│ └── interceptor/
|
|
280
337
|
│ └── _grpc.py # gRPC unary + server-stream interceptors
|
|
281
338
|
├── tests/
|
|
@@ -4,14 +4,14 @@
|
|
|
4
4
|
|
|
5
5
|
[](https://github.com/sunhailin-Leo/hedge-python/actions)
|
|
6
6
|
[](#测试)
|
|
7
|
-
[](pyproject.toml)
|
|
8
8
|
[](LICENSE)
|
|
9
9
|
|
|
10
10
|
> 📘 完整文档以 **[英文版](README.md)** 为主,本文为中文摘要,便于快速了解。
|
|
11
11
|
|
|
12
12
|
[bhope/hedge](https://github.com/bhope/hedge) 的 Python 移植版本 —— **面向尾延迟优化的自适应对冲请求库**。
|
|
13
13
|
|
|
14
|
-
`hedge-python` 使用 [DDSketch](https://arxiv.org/abs/2004.08604) 学习每个目标主机的延迟分布,当主请求超过估算的 p90 时立即发起备份请求,并通过令牌桶限制对冲速率,避免在故障期间放大流量。**零配置开箱即用**,原生支持 **httpx**、**aiohttp** 和 **gRPC**(unary + server-streaming
|
|
14
|
+
`hedge-python` 使用 [DDSketch](https://arxiv.org/abs/2004.08604) 学习每个目标主机的延迟分布,当主请求超过估算的 p90 时立即发起备份请求,并通过令牌桶限制对冲速率,避免在故障期间放大流量。**零配置开箱即用**,原生支持 **httpx**、**aiohttp**、**niquests**、**tornado** 和 **gRPC**(unary + server-streaming)。同时支持通过 `http_client` 参数无缝集成 **OpenAI Python SDK**。
|
|
15
15
|
|
|
16
16
|
灵感来自 Dean & Barroso 的 [_The Tail at Scale_](https://research.google/pubs/the-tail-at-scale/)(CACM 2013)。
|
|
17
17
|
|
|
@@ -44,6 +44,8 @@
|
|
|
44
44
|
# 按需安装框架支持
|
|
45
45
|
pip install hedge-python[httpx]
|
|
46
46
|
pip install hedge-python[aiohttp]
|
|
47
|
+
pip install hedge-python[niquests]
|
|
48
|
+
pip install hedge-python[tornado]
|
|
47
49
|
pip install hedge-python[grpc]
|
|
48
50
|
pip install hedge-python[all] # 全部框架
|
|
49
51
|
```
|
|
@@ -109,6 +111,55 @@ async def make_channel():
|
|
|
109
111
|
)
|
|
110
112
|
```
|
|
111
113
|
|
|
114
|
+
### niquests
|
|
115
|
+
|
|
116
|
+
```python
|
|
117
|
+
import asyncio
|
|
118
|
+
from hedge import HedgeConfig
|
|
119
|
+
from hedge.transport import HedgedNiquestsSession
|
|
120
|
+
|
|
121
|
+
async def main():
|
|
122
|
+
async with HedgedNiquestsSession(config=HedgeConfig()) as session:
|
|
123
|
+
resp = await session.get("https://api.example.com/data")
|
|
124
|
+
print(resp.status_code)
|
|
125
|
+
|
|
126
|
+
asyncio.run(main())
|
|
127
|
+
```
|
|
128
|
+
|
|
129
|
+
### tornado
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
import asyncio
|
|
133
|
+
from hedge import HedgeConfig
|
|
134
|
+
from hedge.transport import HedgedTornadoClient
|
|
135
|
+
|
|
136
|
+
async def main():
|
|
137
|
+
async with HedgedTornadoClient(config=HedgeConfig()) as client:
|
|
138
|
+
resp = await client.fetch("https://api.example.com/data")
|
|
139
|
+
print(resp.code)
|
|
140
|
+
|
|
141
|
+
asyncio.run(main())
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
### OpenAI SDK
|
|
145
|
+
|
|
146
|
+
OpenAI Python SDK 底层使用 httpx,可通过 `http_client` 参数直接注入 `HedgedHttpxTransport`:
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
import httpx
|
|
150
|
+
from openai import AsyncOpenAI
|
|
151
|
+
from hedge import HedgeConfig
|
|
152
|
+
from hedge.transport import HedgedHttpxTransport
|
|
153
|
+
|
|
154
|
+
transport = HedgedHttpxTransport(config=HedgeConfig(percentile=0.95))
|
|
155
|
+
client = AsyncOpenAI(
|
|
156
|
+
api_key="sk-...",
|
|
157
|
+
http_client=httpx.AsyncClient(transport=transport),
|
|
158
|
+
)
|
|
159
|
+
```
|
|
160
|
+
|
|
161
|
+
> **注意**:OpenAI 核心 API(Chat Completions、Embeddings 等)使用 POST,默认**不会**被对冲——避免双倍计费。仅 GET 端点(如模型列表)会被对冲。完整示例见 [`examples/openai_hedged.py`](examples/openai_hedged.py)。
|
|
162
|
+
|
|
112
163
|
对于 server-streaming,对冲信号是 **首消息到达时间(TTFM)**:若主流在估算的 p90 内仍未返回首个 chunk,则发起备份流。先返回首 chunk 的流胜出并继续接管,败者在传输层被取消。
|
|
113
164
|
|
|
114
165
|
> 每个框架的可运行示例位于 [`examples/`](examples/) 目录 —— gRPC 示例**完全自包含**(启动本地 server + 注入 straggler,无需任何外部依赖即可看到对冲触发)。详见 [`examples/README.md`](examples/README.md)。
|
|
@@ -220,7 +271,7 @@ make ci # lint + typecheck + test + coverage
|
|
|
220
271
|
* **集成测试** (`tests/integration/`):真实 httpx transport、真实 aiohttp session、**真实本地 gRPC server**(含 `.proto` 与生成的 pb2)。
|
|
221
272
|
* **基准测试** (`tests/benchmark/`):DDSketch 微基准、令牌桶微基准、四配置对比、三框架对比。
|
|
222
273
|
|
|
223
|
-
当前覆盖率:**97%**(
|
|
274
|
+
当前覆盖率:**97%**(150 个测试,约 7 秒)。
|
|
224
275
|
|
|
225
276
|
---
|
|
226
277
|
|