xtr-http-client 4.0.2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- xtr_http_client/.agents/skills/xtr-http-client/SKILL.md +289 -0
- xtr_http_client/__init__.py +47 -0
- xtr_http_client/bundle/__init__.py +17 -0
- xtr_http_client/bundle/_clients.py +217 -0
- xtr_http_client/bundle/client_options_config.py +79 -0
- xtr_http_client/bundle/http_client_bundle.py +85 -0
- xtr_http_client/bundle/http_client_config.py +75 -0
- xtr_http_client/bundle/retry_failed_config.py +77 -0
- xtr_http_client/bundle/scoped_client_config.py +99 -0
- xtr_http_client/exception/__init__.py +40 -0
- xtr_http_client/exception/_http_error_message.py +129 -0
- xtr_http_client/http_client.py +57 -0
- xtr_http_client/http_client_mixin.py +176 -0
- xtr_http_client/httpx_http_client.py +327 -0
- xtr_http_client/internal/__init__.py +13 -0
- xtr_http_client/internal/debug_type.py +14 -0
- xtr_http_client/internal/httpx_body.py +88 -0
- xtr_http_client/internal/httpx_clients.py +203 -0
- xtr_http_client/internal/httpx_errors.py +63 -0
- xtr_http_client/internal/httpx_redirect.py +63 -0
- xtr_http_client/internal/httpx_request.py +272 -0
- xtr_http_client/internal/mock_head.py +144 -0
- xtr_http_client/internal/mock_info.py +86 -0
- xtr_http_client/internal/option_merge.py +108 -0
- xtr_http_client/internal/response_body.py +270 -0
- xtr_http_client/internal/retry_count_response.py +148 -0
- xtr_http_client/internal/retry_delay.py +45 -0
- xtr_http_client/internal/retry_options.py +98 -0
- xtr_http_client/mock_http_client.py +253 -0
- xtr_http_client/no_private_network_http_client.py +342 -0
- xtr_http_client/py.typed +0 -0
- xtr_http_client/recipe/__init__.py +15 -0
- xtr_http_client/recipe/manifest.toml +23 -0
- xtr_http_client/response/__init__.py +10 -0
- xtr_http_client/response/http_response.py +412 -0
- xtr_http_client/response/httpx_response.py +173 -0
- xtr_http_client/response/json_mock_response.py +119 -0
- xtr_http_client/response/mock_response.py +375 -0
- xtr_http_client/retry/__init__.py +25 -0
- xtr_http_client/retry/generic_retry_strategy.py +215 -0
- xtr_http_client/retry/retry_context.py +73 -0
- xtr_http_client/retry/retry_strategy_interface.py +74 -0
- xtr_http_client/retryable_http_client.py +346 -0
- xtr_http_client/scoping_http_client.py +303 -0
- xtr_http_client/throttling_http_client.py +143 -0
- xtr_http_client/traceable_http_client.py +406 -0
- xtr_http_client-4.0.2.dist-info/METADATA +505 -0
- xtr_http_client-4.0.2.dist-info/RECORD +51 -0
- xtr_http_client-4.0.2.dist-info/WHEEL +4 -0
- xtr_http_client-4.0.2.dist-info/entry_points.txt +6 -0
- xtr_http_client-4.0.2.dist-info/licenses/LICENSE +22 -0
|
@@ -0,0 +1,289 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: xtr-http-client
|
|
3
|
+
description: How to call out over HTTP with xtr-http-client and the xtr-http-client-contracts interfaces it implements — typed request options, a response whose body is read on demand, scoped clients per service, retries, a pace, a trace, a private-network guard, and a mock client for tests. Use when code must fetch or post to another service; when a 4xx or 5xx has to be handled rather than raised; when you need per-service credentials, base URIs, timeouts, redirects without leaking an Authorization header, retry with backoff, rate-paced outbound calls, or an SSRF guard on a URL that came from a user; when writing a test that must not touch the network; and when adding HttpClientBundle to an application or configuring its default and scoped clients.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# xtr-http-client
|
|
7
|
+
|
|
8
|
+
A library that calls out depends on `HttpClientInterface` from
|
|
9
|
+
[xtr-http-client-contracts](https://github.com/xterr/python-xtr/tree/main/packages/xtr-http-client-contracts);
|
|
10
|
+
this package is what an application puts behind it. Every feature is a client wrapping a
|
|
11
|
+
client, so the calling code sees one type however deep the stack is.
|
|
12
|
+
|
|
13
|
+
## Quick reference
|
|
14
|
+
|
|
15
|
+
- `HttpClient.create(max_host_connections=6, **default_options)` → an `HttpClientInterface`.
|
|
16
|
+
- `await client.request(method, url, **options)` returns once the status line and headers are
|
|
17
|
+
in. **It never raises because of the status** — a 404 is an answer.
|
|
18
|
+
- `await client.send_request(message_request, **options)` sends a message request; the request
|
|
19
|
+
wins wherever both have an opinion.
|
|
20
|
+
- `client.with_options(**options)` → another client of the same type with those defaults.
|
|
21
|
+
- Reading: `await response.get_content(throw=True)`, `await response.to_array(throw=True)`,
|
|
22
|
+
`response.get_body()` (the message stream), `response.get_info(key=None)`,
|
|
23
|
+
`await response.cancel()`. `throw` is what raises `ClientError` / `ServerError` /
|
|
24
|
+
`RedirectionError`.
|
|
25
|
+
- `async with await client.request(...) as response:` frees the connection on the way out.
|
|
26
|
+
- Clients: `HttpxHttpClient` (the transport), `ScopingHttpClient`, `RetryableHttpClient`,
|
|
27
|
+
`ThrottlingHttpClient`, `TraceableHttpClient`, `NoPrivateNetworkHttpClient`,
|
|
28
|
+
`MockHttpClient`.
|
|
29
|
+
- Options are the contract's `RequestOptions`. An option it does not know is refused, never
|
|
30
|
+
ignored.
|
|
31
|
+
|
|
32
|
+
## Make a request
|
|
33
|
+
|
|
34
|
+
```python
|
|
35
|
+
from xtr_http_client import HttpClient
|
|
36
|
+
|
|
37
|
+
client = HttpClient.create(base_uri="https://api.example.com/", timeout=5)
|
|
38
|
+
|
|
39
|
+
async with await client.request("GET", "orders", query={"paid": "1"}) as response:
|
|
40
|
+
if response.get_status_code() == 404:
|
|
41
|
+
missing = await response.get_content(throw=False)
|
|
42
|
+
else:
|
|
43
|
+
orders = await response.to_array()
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
- The body is read on demand and **kept** — in a temporary file past 2 MiB — so two reads give
|
|
47
|
+
the same bytes, and `response.get_body()` is the message stream over that kept body.
|
|
48
|
+
- `get_info()` never waits and never raises, so it is readable mid-transfer: `"http_code"`,
|
|
49
|
+
`"url"`, `"redirect_count"`, `"redirect_url"`, `"total_time"`, `"response_headers"` (the raw
|
|
50
|
+
lines of every hop), `"canceled"`, `"error"`, `"user_data"`.
|
|
51
|
+
- `timeout` is how long the transfer may go quiet for, and `None` means 60 seconds, not "no
|
|
52
|
+
limit". `max_duration` and `max_connect_duration` of `0` *are* "no limit".
|
|
53
|
+
|
|
54
|
+
```python
|
|
55
|
+
from xtr_http_message import Request
|
|
56
|
+
|
|
57
|
+
request = Request("POST", "/orders", headers={"Idempotency-Key": "k-1"}, body=b'{"total":42}')
|
|
58
|
+
response = await client.send_request(request, headers={"Accept": "application/json"})
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
The request's header is the one sent, the options fill the names it lacks, a relative URI is
|
|
62
|
+
resolved against `base_uri`, `query` is merged into the URI, and `auth_bearer` applies only
|
|
63
|
+
where the request has no `Authorization`. A `body` or `json` option plus a request that already
|
|
64
|
+
carries a body is refused.
|
|
65
|
+
|
|
66
|
+
## Give each service its own defaults
|
|
67
|
+
|
|
68
|
+
```python
|
|
69
|
+
from xtr_http_client import ScopingHttpClient
|
|
70
|
+
|
|
71
|
+
github = ScopingHttpClient.for_base_uri(
|
|
72
|
+
inner,
|
|
73
|
+
"https://api.github.com/",
|
|
74
|
+
{"auth_bearer": token, "headers": {"Accept": "application/vnd.github+json"}},
|
|
75
|
+
)
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
`ScopingHttpClient(inner, {regexp: options}, default_regexp=None)` is the general form: the
|
|
79
|
+
first pattern matching the start of a URL applies, so order from most specific to least. What
|
|
80
|
+
the request asked for always wins; `headers` is the one option merged name by name.
|
|
81
|
+
`for_base_uris(inner, [...])` covers a service answering at several, the first resolving
|
|
82
|
+
relative URLs.
|
|
83
|
+
|
|
84
|
+
## Retry what is worth retrying
|
|
85
|
+
|
|
86
|
+
```python
|
|
87
|
+
from xtr_http_client import GenericRetryStrategy, RetryableHttpClient
|
|
88
|
+
|
|
89
|
+
client = RetryableHttpClient(inner, GenericRetryStrategy([500, 502, 503]), max_retries=2)
|
|
90
|
+
```
|
|
91
|
+
|
|
92
|
+
- `max_retries` is how many **further** attempts are allowed; a request may change it with the
|
|
93
|
+
`max_retries` option. The response reports `get_info("retry_count")`.
|
|
94
|
+
- A `Retry-After` header wins over the strategy's delay.
|
|
95
|
+
- A body that cannot be produced twice is sent once — a second attempt would send a different
|
|
96
|
+
request.
|
|
97
|
+
- `base_uri` naming several makes each attempt move one along them.
|
|
98
|
+
- `GenericRetryStrategy(status_codes, delay_ms=1000, multiplier=2.0, max_delay_ms=0,
|
|
99
|
+
jitter=0.1)`: the default `DEFAULT_RETRY_STATUS_CODES` retries `423`, `425`, `429`, `502` and
|
|
100
|
+
`503` for every method, and `0` (nothing answered), `500`, `504`, `507`, `510` for
|
|
101
|
+
`IDEMPOTENT_METHODS` only.
|
|
102
|
+
- A strategy of your own implements `RetryStrategyInterface`: `await should_retry(context,
|
|
103
|
+
content, error)` answers `True`, `False`, or `None` for "not without the body" — asked once
|
|
104
|
+
more with it — and `get_delay(...)` returns milliseconds. `RetryContext` carries `method`,
|
|
105
|
+
`url`, `options`, `retry_count`, `response`, `get_status_code()` (`0` when nothing answered).
|
|
106
|
+
|
|
107
|
+
## Pace, trace, guard
|
|
108
|
+
|
|
109
|
+
```python
|
|
110
|
+
from xtr_http_client import NoPrivateNetworkHttpClient, ThrottlingHttpClient, TraceableHttpClient
|
|
111
|
+
|
|
112
|
+
paced = ThrottlingHttpClient(inner, limiter) # xtr-rate-limiter LimiterInterface, [throttling]
|
|
113
|
+
traced = TraceableHttpClient(inner)
|
|
114
|
+
guarded = NoPrivateNetworkHttpClient(inner, allow_list=("10.1.2.3",))
|
|
115
|
+
```
|
|
116
|
+
|
|
117
|
+
- **Throttling** books a token before each request and waits for its turn, so the pace holds
|
|
118
|
+
under concurrency. Requests sharing a limiter share the pace; `with_options()` keeps it.
|
|
119
|
+
- **Tracing** fills `client.get_traced_requests()` as requests go on — a `TracedRequest` has
|
|
120
|
+
`method`, `url`, `options`, `info`, `content`. `extra={"trace_content": False}` drops the
|
|
121
|
+
bodies (`content` is then `False`); `disabled=lambda: ...` records nothing while it says so.
|
|
122
|
+
- **The guard** is the SSRF answer for a URL that came from outside. It checks what the host
|
|
123
|
+
*resolves to*, refuses the whole request if any address is refused, pins the request to the
|
|
124
|
+
checked address through `resolve`, and follows redirects itself so no hop goes unchecked.
|
|
125
|
+
Refusal is `TransportError`: `Host "<host>" is blocked for "<url>".`. Without `subnets` it
|
|
126
|
+
refuses `IpUtils.PRIVATE_SUBNETS`; `allow_list` opens one receiver. Addresses and CIDR
|
|
127
|
+
networks only — never host names.
|
|
128
|
+
|
|
129
|
+
## Use the transport directly
|
|
130
|
+
|
|
131
|
+
```python
|
|
132
|
+
from xtr_http_client import HttpxHttpClient
|
|
133
|
+
|
|
134
|
+
async with HttpxHttpClient({"timeout": 5}, max_host_connections=4) as client:
|
|
135
|
+
response = await client.request("GET", "https://api.example.com/orders")
|
|
136
|
+
```
|
|
137
|
+
|
|
138
|
+
Connections are pooled by how they connect — proxy, trust store, client certificate, protocol
|
|
139
|
+
version, deadlines — so each combination gets a pool, made on first use; `max_host_connections`
|
|
140
|
+
is each pool's ceiling. `http_proxy`, `https_proxy` and `no_proxy` are read from the
|
|
141
|
+
environment and are part of what picks a pool; the `proxy` option overrides them. Redirects are
|
|
142
|
+
followed here, not by the backend, so an `Authorization` is dropped when a chain leaves its host
|
|
143
|
+
and restored only if it returns. `await client.aclose()` closes every pool.
|
|
144
|
+
|
|
145
|
+
## Testing
|
|
146
|
+
|
|
147
|
+
`MockHttpClient` is the same client with the transfer replaced by a script: the same refusals,
|
|
148
|
+
the same defaults, the same merging. Nothing is opened.
|
|
149
|
+
|
|
150
|
+
```python
|
|
151
|
+
import pytest
|
|
152
|
+
from xtr_http_client import JsonMockResponse, MockHttpClient, MockResponse
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
@pytest.mark.anyio
|
|
156
|
+
async def test_it_reads_the_created_order() -> None:
|
|
157
|
+
client = MockHttpClient(JsonMockResponse({"id": 7}, {"http_code": 201}))
|
|
158
|
+
|
|
159
|
+
async with await client.request("POST", "/orders") as response:
|
|
160
|
+
assert response.get_status_code() == 201
|
|
161
|
+
assert await response.to_array() == {"id": 7}
|
|
162
|
+
|
|
163
|
+
assert client.get_requests_count() == 1
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
- Answers come from nothing (every request an empty 200), one `MockResponse`, an iterable of
|
|
167
|
+
them one per request, or a callable `(method, url, options) -> MockResponse` — which may be
|
|
168
|
+
async, and is how a test asserts on the request that was made. An iterable may hold callables.
|
|
169
|
+
- A request past the end of an iterable raises `TransportError`: a test making more requests
|
|
170
|
+
than it wrote answers for is not testing what it says.
|
|
171
|
+
- `MockResponse(body, info)`: `body` is bytes, text, or an iterable of bytes handed over piece
|
|
172
|
+
by piece; `info` takes `http_code` (200 by default), `response_headers`, and `error` for a
|
|
173
|
+
transfer that fails instead of answering. `JsonMockResponse(value, info)` encodes a value and
|
|
174
|
+
labels it `application/json`; `JsonMockResponse.from_file(path)` reads a recorded document.
|
|
175
|
+
- Nothing is followed and nothing is resolved — a 3xx is the answer — and `base_uri` starts at
|
|
176
|
+
`https://example.com`, so `"/orders"` means something.
|
|
177
|
+
- Assert the request on the issued response: `get_request_method()`, `get_request_url()`,
|
|
178
|
+
`get_request_options()`. `set_response_factory(...)` changes the answers and keeps the count;
|
|
179
|
+
`await client.reset()` clears the count and keeps the answers.
|
|
180
|
+
- A client of your own proves it is one by subclassing `HttpClientTestCase` from
|
|
181
|
+
`xtr_http_client_contracts.testing` (the `testing` extra) and returning it from
|
|
182
|
+
`create_client()`; the suite runs the whole contract against its own fixture server.
|
|
183
|
+
|
|
184
|
+
## Use in an application
|
|
185
|
+
|
|
186
|
+
`uv run xtr-recipes recipes:sync` applies the recipe shipped with this package: it lists
|
|
187
|
+
`HttpClientBundle`. There is no config file, environment or ignore line to write; it prints the
|
|
188
|
+
step to name the services the application calls, which a recipe cannot make for you.
|
|
189
|
+
|
|
190
|
+
1. **Install** — `uv add "xtr-http-client[di]"`; add `throttling` to pace a client with an
|
|
191
|
+
xtr-rate-limiter limiter, `http2` to ask for HTTP/2.
|
|
192
|
+
2. **Activate** — `HttpClientBundle: {"all": True}` in `BUNDLES` in `<app>/bundles.py`
|
|
193
|
+
(`from xtr_http_client.bundle import HttpClientBundle`).
|
|
194
|
+
3. **Brings along** — the rate limiter bundle when xtr-rate-limiter is installed.
|
|
195
|
+
4. **Configure** — optional: with no configuration there is one client sending requests as they
|
|
196
|
+
are given. Otherwise `<app>/config/http_client.py`:
|
|
197
|
+
|
|
198
|
+
```python
|
|
199
|
+
from xtr_dependency_injection import configure, env
|
|
200
|
+
from xtr_http_client.bundle import (
|
|
201
|
+
ClientOptionsConfig,
|
|
202
|
+
HttpClientConfig,
|
|
203
|
+
RetryFailedConfig,
|
|
204
|
+
ScopedClientConfig,
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
@configure
|
|
209
|
+
def http_client() -> HttpClientConfig:
|
|
210
|
+
return HttpClientConfig(
|
|
211
|
+
default_options=ClientOptionsConfig(
|
|
212
|
+
timeout=10, retry_failed=RetryFailedConfig(enabled=True)
|
|
213
|
+
),
|
|
214
|
+
scoped_clients={
|
|
215
|
+
"github": ScopedClientConfig(
|
|
216
|
+
base_uri="https://api.github.com/",
|
|
217
|
+
headers={"Accept": "application/vnd.github+json"},
|
|
218
|
+
auth_bearer=env("GITHUB_TOKEN"),
|
|
219
|
+
),
|
|
220
|
+
},
|
|
221
|
+
)
|
|
222
|
+
```
|
|
223
|
+
|
|
224
|
+
`HttpClientConfig` also takes `max_host_connections` (`6`), `mock_response_factory`
|
|
225
|
+
(`False` — answer from `MockHttpClient` instead of the network) and `tracing` (`False`).
|
|
226
|
+
`ScopedClientConfig` needs one of `scope` or `base_uri`, and adds `auth_basic`,
|
|
227
|
+
`auth_bearer`, `query` and its own `mock_response_factory`.
|
|
228
|
+
5. **Environment** — nothing. A credential given as `env(...)` is read when the client needing
|
|
229
|
+
it is built.
|
|
230
|
+
6. **Use** — the default client is the unqualified `HttpClientInterface`; a scoped one is the
|
|
231
|
+
same interface qualified by its name:
|
|
232
|
+
|
|
233
|
+
```python
|
|
234
|
+
from typing import Annotated
|
|
235
|
+
|
|
236
|
+
from xtr_dependency_injection import Target, as_service
|
|
237
|
+
from xtr_http_client_contracts import HttpClientInterface
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
@as_service
|
|
241
|
+
class GithubRepositories:
|
|
242
|
+
def __init__(self, github: Annotated[HttpClientInterface, Target("github")]) -> None:
|
|
243
|
+
self._github = github
|
|
244
|
+
```
|
|
245
|
+
|
|
246
|
+
7. **Check** — `debug:bundles` shows `http_client` as `listed` and `active`.
|
|
247
|
+
8. **Remove** — drop the `BUNDLES` entry, delete `<app>/config/http_client.py`, then
|
|
248
|
+
`uv remove xtr-http-client`.
|
|
249
|
+
|
|
250
|
+
## Errors
|
|
251
|
+
|
|
252
|
+
Everything derives from `HttpClientError`; import from `xtr_http_client.exception` or from
|
|
253
|
+
`xtr_http_client_contracts`.
|
|
254
|
+
|
|
255
|
+
| Error | Raised when |
|
|
256
|
+
| --- | --- |
|
|
257
|
+
| `TransportError` | no answer came — refused connection, broken transfer, cancelled response, guarded address |
|
|
258
|
+
| `TransportTimeoutError` | no answer came in time; a `TransportError` |
|
|
259
|
+
| `HttpError` | there is an answer and it is not a success |
|
|
260
|
+
| `RedirectionError` | a 3xx read with `throw`, once `max_redirects` was reached |
|
|
261
|
+
| `ClientError` | a 4xx read with `throw` |
|
|
262
|
+
| `ServerError` | a 5xx read with `throw` |
|
|
263
|
+
| `DecodingError` | the body is not what it was read as |
|
|
264
|
+
| `JsonError` | the body is not the JSON object it was read as; a `DecodingError` |
|
|
265
|
+
| `InvalidArgumentError` | a request a client refuses to make, before anything is sent |
|
|
266
|
+
|
|
267
|
+
An `HttpError` names itself: `HTTP/1.1 404 Not Found returned for "<url>".`, or the `title` and
|
|
268
|
+
`detail` of the body when that body is a JSON error document.
|
|
269
|
+
|
|
270
|
+
## Do not
|
|
271
|
+
|
|
272
|
+
- Do not expect `request()` to raise for a 404 or a 503 — read the status, or read the body
|
|
273
|
+
with `throw` left on.
|
|
274
|
+
- Do not read a body without owning the transfer: use `async with`, or `await
|
|
275
|
+
response.cancel()`, or the connection is held until the response is collected.
|
|
276
|
+
- Do not pass an option the contract does not name — it is refused. Put what one client
|
|
277
|
+
understands and another ignores in `extra`.
|
|
278
|
+
- Do not set `auth_basic` and `auth_bearer` together, or `json` and `body` together: both are
|
|
279
|
+
refused.
|
|
280
|
+
- Do not set `timeout=None` meaning "wait forever" — it is 60 seconds. Use `max_duration=0` for
|
|
281
|
+
no whole-request limit.
|
|
282
|
+
- Do not fetch a URL that came from a user without `NoPrivateNetworkHttpClient`, and do not
|
|
283
|
+
give it host names in `subnets` or `allow_list`.
|
|
284
|
+
- Do not retry a non-idempotent request by listing its status code for every method; keep
|
|
285
|
+
`IDEMPOTENT_METHODS` for the codes where a repeat could change something twice.
|
|
286
|
+
- Do not read a `MockResponse` no client issued — it raises `InvalidArgumentError`. Let
|
|
287
|
+
`MockHttpClient` issue it.
|
|
288
|
+
- Do not reach for `xtr_http_client.internal`: it is this package's own, and nothing there is
|
|
289
|
+
part of the contract.
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
"""An async HTTP client: typed options in, a response read on demand out.
|
|
2
|
+
|
|
3
|
+
One interface with clients behind it — one that opens real connections and
|
|
4
|
+
streams what comes back, one that answers from a script a test wrote, five
|
|
5
|
+
that wrap another and add what every application writes again (a set of
|
|
6
|
+
defaults per service, a refusal to reach inwards, a request made again while
|
|
7
|
+
that is worth doing, a pace requests are held to, and a record of every
|
|
8
|
+
request made), and :class:`HttpClient`, the one call that hands one over so
|
|
9
|
+
nothing else has to name it.
|
|
10
|
+
|
|
11
|
+
A request returns once the status line and headers have arrived, whatever the
|
|
12
|
+
status; the body is read when the caller asks for it, as a whole or piece by
|
|
13
|
+
piece.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
from .http_client import HttpClient
|
|
19
|
+
from .http_client_mixin import HttpClientMixin
|
|
20
|
+
from .httpx_http_client import HttpxHttpClient
|
|
21
|
+
from .mock_http_client import MockHttpClient
|
|
22
|
+
from .no_private_network_http_client import NoPrivateNetworkHttpClient
|
|
23
|
+
from .response import HttpResponse, JsonMockResponse, MockResponse
|
|
24
|
+
from .retry import GenericRetryStrategy, RetryContext, RetryStrategyInterface
|
|
25
|
+
from .retryable_http_client import RetryableHttpClient
|
|
26
|
+
from .scoping_http_client import ScopingHttpClient
|
|
27
|
+
from .throttling_http_client import ThrottlingHttpClient
|
|
28
|
+
from .traceable_http_client import TraceableHttpClient, TracedRequest
|
|
29
|
+
|
|
30
|
+
__all__ = [
|
|
31
|
+
"GenericRetryStrategy",
|
|
32
|
+
"HttpClient",
|
|
33
|
+
"HttpClientMixin",
|
|
34
|
+
"HttpResponse",
|
|
35
|
+
"HttpxHttpClient",
|
|
36
|
+
"JsonMockResponse",
|
|
37
|
+
"MockHttpClient",
|
|
38
|
+
"MockResponse",
|
|
39
|
+
"NoPrivateNetworkHttpClient",
|
|
40
|
+
"RetryContext",
|
|
41
|
+
"RetryStrategyInterface",
|
|
42
|
+
"RetryableHttpClient",
|
|
43
|
+
"ScopingHttpClient",
|
|
44
|
+
"ThrottlingHttpClient",
|
|
45
|
+
"TraceableHttpClient",
|
|
46
|
+
"TracedRequest",
|
|
47
|
+
]
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""The xtr-dependency-injection bundle for xtr-http-client."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from .client_options_config import ClientOptionsConfig
|
|
6
|
+
from .http_client_bundle import HttpClientBundle
|
|
7
|
+
from .http_client_config import HttpClientConfig
|
|
8
|
+
from .retry_failed_config import RetryFailedConfig
|
|
9
|
+
from .scoped_client_config import ScopedClientConfig
|
|
10
|
+
|
|
11
|
+
__all__ = [
|
|
12
|
+
"ClientOptionsConfig",
|
|
13
|
+
"HttpClientBundle",
|
|
14
|
+
"HttpClientConfig",
|
|
15
|
+
"RetryFailedConfig",
|
|
16
|
+
"ScopedClientConfig",
|
|
17
|
+
]
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
"""What the bundle registers: the transports, and every client built over one.
|
|
2
|
+
|
|
3
|
+
A client is a transport wrapped in the layers its configuration turns on, built
|
|
4
|
+
by a factory of its own. What that factory is handed depends on the
|
|
5
|
+
configuration — the transport it sends through, the limiter it waits its turn
|
|
6
|
+
from, the strategy deciding its retries — so the annotations the container reads
|
|
7
|
+
are written onto it here rather than in its signature, and a dependency it does
|
|
8
|
+
not need is given as ``None`` rather than asked for.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import AsyncIterator, Callable, Collection, Sequence
|
|
14
|
+
from typing import TYPE_CHECKING, Annotated, Protocol
|
|
15
|
+
|
|
16
|
+
from xtr_dependency_injection import ServiceConfigurator, Target, named_factory
|
|
17
|
+
from xtr_http_client_contracts import HttpClientInterface
|
|
18
|
+
from xtr_service_contracts import ResetInterface
|
|
19
|
+
|
|
20
|
+
from xtr_http_client.httpx_http_client import HttpxHttpClient
|
|
21
|
+
from xtr_http_client.mock_http_client import MockHttpClient
|
|
22
|
+
from xtr_http_client.retry.generic_retry_strategy import (
|
|
23
|
+
DEFAULT_RETRY_STATUS_CODES,
|
|
24
|
+
GenericRetryStrategy,
|
|
25
|
+
)
|
|
26
|
+
from xtr_http_client.retry.retry_strategy_interface import RetryStrategyInterface
|
|
27
|
+
from xtr_http_client.retryable_http_client import RetryableHttpClient
|
|
28
|
+
from xtr_http_client.scoping_http_client import ScopingHttpClient
|
|
29
|
+
from xtr_http_client.throttling_http_client import ThrottlingHttpClient
|
|
30
|
+
from xtr_http_client.traceable_http_client import TraceableHttpClient
|
|
31
|
+
|
|
32
|
+
from .http_client_config import HttpClientConfig
|
|
33
|
+
|
|
34
|
+
if TYPE_CHECKING:
|
|
35
|
+
from xtr_rate_limiter import RateLimiterFactoryInterface
|
|
36
|
+
|
|
37
|
+
from .client_options_config import ClientOptionsConfig
|
|
38
|
+
from .retry_failed_config import RetryFailedConfig
|
|
39
|
+
from .scoped_client_config import ScopedClientConfig
|
|
40
|
+
|
|
41
|
+
__all__ = ["register_client", "register_transports"]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
class _ResettableHttpClient(HttpClientInterface, ResetInterface, Protocol):
|
|
45
|
+
"""The client a factory builds: the public interface, plus a reset that drops what it holds."""
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def register_transports(services: ServiceConfigurator, mocked: Collection[bool]) -> None:
|
|
49
|
+
"""Register the transports the clients send through — only the ones some client uses.
|
|
50
|
+
|
|
51
|
+
Args:
|
|
52
|
+
services: Where the bundle registers its services.
|
|
53
|
+
mocked: Whether each client answers from the mock transport.
|
|
54
|
+
"""
|
|
55
|
+
if not all(mocked):
|
|
56
|
+
_ = services.set(_httpx_transport)
|
|
57
|
+
if any(mocked):
|
|
58
|
+
_ = services.set(_mock_transport)
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def register_client(
|
|
62
|
+
services: ServiceConfigurator,
|
|
63
|
+
name: str | None,
|
|
64
|
+
options: ClientOptionsConfig,
|
|
65
|
+
*,
|
|
66
|
+
mocked: bool,
|
|
67
|
+
) -> None:
|
|
68
|
+
"""Register the client ``name`` — the default one for ``None`` — as ``options`` build it.
|
|
69
|
+
|
|
70
|
+
Args:
|
|
71
|
+
services: Where the bundle registers its services.
|
|
72
|
+
name: The scoped client's name, which qualifies it; ``None`` for the
|
|
73
|
+
default client.
|
|
74
|
+
options: Its options, with what it inherits already filled in.
|
|
75
|
+
mocked: Whether it answers from the mock transport.
|
|
76
|
+
"""
|
|
77
|
+
annotations: dict[str, object] = {
|
|
78
|
+
"config": HttpClientConfig,
|
|
79
|
+
"transport": MockHttpClient if mocked else HttpxHttpClient,
|
|
80
|
+
"return": _ResettableHttpClient,
|
|
81
|
+
}
|
|
82
|
+
unused: dict[str, object] = {}
|
|
83
|
+
if options.rate_limiter is None:
|
|
84
|
+
unused["limiters"] = None
|
|
85
|
+
else:
|
|
86
|
+
annotations["limiters"] = _limiters(options.rate_limiter)
|
|
87
|
+
retry = options.retry_failed
|
|
88
|
+
if retry.enabled and retry.retry_strategy is not None:
|
|
89
|
+
annotations["strategy"] = Annotated[RetryStrategyInterface, Target(retry.retry_strategy)]
|
|
90
|
+
else:
|
|
91
|
+
unused["strategy"] = None
|
|
92
|
+
|
|
93
|
+
factory = _client_factory(name)
|
|
94
|
+
factory.__annotations__ = annotations
|
|
95
|
+
definition = services.set(
|
|
96
|
+
named_factory(factory, "http_client" if name is None else f"http_client_{name}"),
|
|
97
|
+
qualifier=name,
|
|
98
|
+
)
|
|
99
|
+
if unused:
|
|
100
|
+
_ = definition.set_arguments(unused)
|
|
101
|
+
_ = definition.add_tag("kernel.reset", method="reset")
|
|
102
|
+
services.alias(
|
|
103
|
+
HttpClientInterface, _ResettableHttpClient, alias_qualifier=name, target_qualifier=name
|
|
104
|
+
)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
async def _httpx_transport(config: HttpClientConfig) -> AsyncIterator[HttpxHttpClient]:
|
|
108
|
+
"""Build the transport opening real connections, and close them with the container.
|
|
109
|
+
|
|
110
|
+
The default options are the transport's own, so every client sending
|
|
111
|
+
through it starts from them.
|
|
112
|
+
"""
|
|
113
|
+
transport = HttpxHttpClient(config.default_options.to_options(), config.max_host_connections)
|
|
114
|
+
try:
|
|
115
|
+
yield transport
|
|
116
|
+
finally:
|
|
117
|
+
await transport.aclose()
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _mock_transport() -> MockHttpClient:
|
|
121
|
+
"""Build the transport answering from a script: an empty 200 until a test sets one."""
|
|
122
|
+
return MockHttpClient()
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
def _limiters(rate_limiter: str) -> object:
|
|
126
|
+
"""Return the annotation asking for the limiter factory ``rate_limiter`` names."""
|
|
127
|
+
# The throttling extra is optional: a client is only paced while the rate
|
|
128
|
+
# limiter bundle is active, which proves it installed.
|
|
129
|
+
from xtr_rate_limiter import RateLimiterFactoryInterface # noqa: PLC0415
|
|
130
|
+
|
|
131
|
+
return Annotated[RateLimiterFactoryInterface, Target(rate_limiter)]
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _client_factory(name: str | None) -> Callable[..., HttpClientInterface]:
|
|
135
|
+
"""Return the factory of the client ``name``, one function per client.
|
|
136
|
+
|
|
137
|
+
Its options are read from the configuration it is handed, so an ``env()``
|
|
138
|
+
placeholder in them is read when the client is built.
|
|
139
|
+
"""
|
|
140
|
+
|
|
141
|
+
def http_client(
|
|
142
|
+
config: HttpClientConfig,
|
|
143
|
+
transport: HttpClientInterface,
|
|
144
|
+
limiters: RateLimiterFactoryInterface | None,
|
|
145
|
+
strategy: RetryStrategyInterface | None,
|
|
146
|
+
) -> HttpClientInterface:
|
|
147
|
+
client = (
|
|
148
|
+
transport if limiters is None else ThrottlingHttpClient(transport, limiters.create())
|
|
149
|
+
)
|
|
150
|
+
if name is None:
|
|
151
|
+
retry = config.default_options.retry_failed
|
|
152
|
+
client = _retried(client, retry, strategy, retry.base_uris)
|
|
153
|
+
else:
|
|
154
|
+
scoped = config.scoped_clients[name].inheriting(config.default_options)
|
|
155
|
+
client = _scoped(client, scoped, strategy)
|
|
156
|
+
|
|
157
|
+
return TraceableHttpClient(client) if config.tracing else client
|
|
158
|
+
|
|
159
|
+
return http_client
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _scoped(
|
|
163
|
+
client: HttpClientInterface,
|
|
164
|
+
scoped: ScopedClientConfig,
|
|
165
|
+
strategy: RetryStrategyInterface | None,
|
|
166
|
+
) -> HttpClientInterface:
|
|
167
|
+
"""Wrap ``client`` in the scope of one service, and in its retries.
|
|
168
|
+
|
|
169
|
+
Retries go inside the scope, unless they move along fallback base URIs:
|
|
170
|
+
the scope resolves a URL and consumes ``base_uri``, so a base URI chosen
|
|
171
|
+
beneath it would never be applied — the retries wrap the scope instead,
|
|
172
|
+
and the scope matches every URI they may move to.
|
|
173
|
+
"""
|
|
174
|
+
retry = scoped.retry_failed
|
|
175
|
+
base_uris = [] if scoped.base_uri is None else [scoped.base_uri, *retry.base_uris]
|
|
176
|
+
walks = len(base_uris) > 1
|
|
177
|
+
inner = client if walks else _retried(client, retry, strategy)
|
|
178
|
+
options = scoped.to_options()
|
|
179
|
+
|
|
180
|
+
if scoped.scope is not None:
|
|
181
|
+
scoping = ScopingHttpClient(inner, {scoped.scope: options}, scoped.scope)
|
|
182
|
+
elif walks:
|
|
183
|
+
scoping = ScopingHttpClient.for_base_uris(inner, base_uris, options)
|
|
184
|
+
else:
|
|
185
|
+
scoping = ScopingHttpClient.for_base_uri(inner, base_uris[0], options)
|
|
186
|
+
|
|
187
|
+
return _retried(scoping, retry, strategy, base_uris) if walks else scoping
|
|
188
|
+
|
|
189
|
+
|
|
190
|
+
def _retried(
|
|
191
|
+
client: HttpClientInterface,
|
|
192
|
+
retry: RetryFailedConfig,
|
|
193
|
+
strategy: RetryStrategyInterface | None,
|
|
194
|
+
base_uris: Sequence[str] = (),
|
|
195
|
+
) -> HttpClientInterface:
|
|
196
|
+
"""Wrap ``client`` in the retries ``retry`` turns on, walking ``base_uris`` if any."""
|
|
197
|
+
if not retry.enabled:
|
|
198
|
+
return client
|
|
199
|
+
|
|
200
|
+
retrying = RetryableHttpClient(
|
|
201
|
+
client,
|
|
202
|
+
strategy if strategy is not None else _generic_strategy(retry),
|
|
203
|
+
retry.max_retries,
|
|
204
|
+
)
|
|
205
|
+
|
|
206
|
+
return retrying.with_options(base_uri=list(base_uris)) if base_uris else retrying
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _generic_strategy(retry: RetryFailedConfig) -> GenericRetryStrategy:
|
|
210
|
+
"""Build the strategy ``retry``'s numbers describe."""
|
|
211
|
+
return GenericRetryStrategy(
|
|
212
|
+
DEFAULT_RETRY_STATUS_CODES if retry.http_codes is None else retry.http_codes,
|
|
213
|
+
retry.delay,
|
|
214
|
+
retry.multiplier,
|
|
215
|
+
retry.max_delay,
|
|
216
|
+
retry.jitter,
|
|
217
|
+
)
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
"""What every request a configured client makes starts from."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Mapping, Sequence
|
|
6
|
+
from dataclasses import dataclass, field, fields
|
|
7
|
+
from typing import TYPE_CHECKING, ClassVar, cast
|
|
8
|
+
|
|
9
|
+
from xtr_http_client.internal.option_merge import as_options
|
|
10
|
+
|
|
11
|
+
from .retry_failed_config import RetryFailedConfig
|
|
12
|
+
|
|
13
|
+
if TYPE_CHECKING:
|
|
14
|
+
from xtr_http_client_contracts import RequestOptions
|
|
15
|
+
|
|
16
|
+
__all__ = ["ClientOptionsConfig"]
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True, slots=True)
|
|
20
|
+
class ClientOptionsConfig:
|
|
21
|
+
"""The request options a client starts from, and the layers wrapped around it.
|
|
22
|
+
|
|
23
|
+
Each option field is the request option of the same name, and ``None``
|
|
24
|
+
leaves it to the client's own default. ``retry_failed`` and
|
|
25
|
+
``rate_limiter`` are not request options: they say whether the client
|
|
26
|
+
makes a failed request again, and which rate it is held to.
|
|
27
|
+
|
|
28
|
+
Attributes:
|
|
29
|
+
headers: Headers to send, a mapping of name to one value or several.
|
|
30
|
+
max_redirects: How many redirects to follow.
|
|
31
|
+
http_version: The HTTP version to ask for.
|
|
32
|
+
timeout: How long the transfer may go quiet for, in seconds.
|
|
33
|
+
max_duration: How long a whole request and response may take, in seconds.
|
|
34
|
+
max_connect_duration: How long connecting may take, in seconds.
|
|
35
|
+
resolve: Host to IP address, used instead of asking a resolver.
|
|
36
|
+
proxy: The proxy to go through, as a URL.
|
|
37
|
+
verify_peer: Whether to check that the peer's certificate is trusted.
|
|
38
|
+
verify_host: Whether to check that the certificate is for the host.
|
|
39
|
+
cafile: A file of certificates to trust instead of the defaults.
|
|
40
|
+
capath: A directory of certificates to trust instead of the defaults.
|
|
41
|
+
local_cert: A client certificate to present.
|
|
42
|
+
local_pk: The private key of ``local_cert``, when it is kept apart.
|
|
43
|
+
passphrase: The passphrase of ``local_pk``; kept out of the ``repr``.
|
|
44
|
+
extra: Additional options a client may ignore.
|
|
45
|
+
retry_failed: Whether and how a failed request is made again.
|
|
46
|
+
rate_limiter: The name of the xtr-rate-limiter limiter every request
|
|
47
|
+
waits its turn from; needs the ``throttling`` extra.
|
|
48
|
+
"""
|
|
49
|
+
|
|
50
|
+
_NOT_OPTIONS: ClassVar[frozenset[str]] = frozenset({"retry_failed", "rate_limiter"})
|
|
51
|
+
|
|
52
|
+
headers: Mapping[str, str | Sequence[str]] | None = None
|
|
53
|
+
max_redirects: int | None = None
|
|
54
|
+
http_version: str | None = None
|
|
55
|
+
timeout: float | None = None
|
|
56
|
+
max_duration: float | None = None
|
|
57
|
+
max_connect_duration: float | None = None
|
|
58
|
+
resolve: Mapping[str, str] | None = None
|
|
59
|
+
proxy: str | None = None
|
|
60
|
+
verify_peer: bool | None = None
|
|
61
|
+
verify_host: bool | None = None
|
|
62
|
+
cafile: str | None = None
|
|
63
|
+
capath: str | None = None
|
|
64
|
+
local_cert: str | None = None
|
|
65
|
+
local_pk: str | None = None
|
|
66
|
+
passphrase: str | None = field(default=None, repr=False)
|
|
67
|
+
extra: Mapping[str, object] | None = None
|
|
68
|
+
retry_failed: RetryFailedConfig = field(default_factory=RetryFailedConfig)
|
|
69
|
+
rate_limiter: str | None = None
|
|
70
|
+
|
|
71
|
+
def to_options(self) -> RequestOptions:
|
|
72
|
+
"""Return the request options that are set, as a client takes them."""
|
|
73
|
+
options: dict[str, object] = {}
|
|
74
|
+
for entry in fields(self):
|
|
75
|
+
value = cast("object", getattr(self, entry.name))
|
|
76
|
+
if entry.name not in self._NOT_OPTIONS and value is not None:
|
|
77
|
+
options[entry.name] = value
|
|
78
|
+
|
|
79
|
+
return as_options(options)
|