syshttp 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- syshttp/AGENTS.md +817 -0
- syshttp/README.md +177 -0
- syshttp/__init__.py +71 -0
- syshttp/_fileio.py +78 -0
- syshttp/_parse.py +30 -0
- syshttp/auth.py +17 -0
- syshttp/autopatch.py +382 -0
- syshttp/client/__init__.py +70 -0
- syshttp/client/_body.py +156 -0
- syshttp/client/_client.py +413 -0
- syshttp/client/_context.py +107 -0
- syshttp/client/_deadline.py +94 -0
- syshttp/client/_default.py +72 -0
- syshttp/client/_fetch.py +98 -0
- syshttp/client/_follow.py +243 -0
- syshttp/client/_headers.py +238 -0
- syshttp/client/_pool.py +170 -0
- syshttp/client/_prepare.py +77 -0
- syshttp/client/_response.py +301 -0
- syshttp/client/_route.py +75 -0
- syshttp/client/_settings.py +12 -0
- syshttp/client/_state.py +322 -0
- syshttp/client/_status.py +59 -0
- syshttp/client/_tlstunnel.py +114 -0
- syshttp/client/_transport.py +550 -0
- syshttp/cookies.py +391 -0
- syshttp/env.py +144 -0
- syshttp/exceptions.py +29 -0
- syshttp/hooks.py +314 -0
- syshttp/jar.py +133 -0
- syshttp/proxy.py +341 -0
- syshttp/py.typed +0 -0
- syshttp/retry.py +34 -0
- syshttp/tls.py +60 -0
- syshttp-0.1.0.dist-info/METADATA +219 -0
- syshttp-0.1.0.dist-info/RECORD +39 -0
- syshttp-0.1.0.dist-info/WHEEL +4 -0
- syshttp-0.1.0.dist-info/entry_points.txt +6 -0
- syshttp-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,413 @@
|
|
|
1
|
+
"""The client: a transport, the headers and timeout every request is sent with,
|
|
2
|
+
the redirect rules above the transport, and what a client keeps between
|
|
3
|
+
requests (cookies, tokens, hooks, retries)."""
|
|
4
|
+
|
|
5
|
+
from __future__ import annotations
|
|
6
|
+
|
|
7
|
+
import os as _os
|
|
8
|
+
import ssl as _ssl
|
|
9
|
+
import threading as _threading
|
|
10
|
+
import time as _time
|
|
11
|
+
import typing as _ty
|
|
12
|
+
|
|
13
|
+
from .. import __version__
|
|
14
|
+
from .. import hooks as _hooks
|
|
15
|
+
from ..env import HTTPEnv
|
|
16
|
+
from ..jar import CookieJar, TokenJar
|
|
17
|
+
from ..retry import TRANSIENT_STATUSES, retry_after
|
|
18
|
+
from . import _follow, _settings
|
|
19
|
+
from ._deadline import Deadline, check_deadline
|
|
20
|
+
from ._default import transport_source
|
|
21
|
+
from ._headers import check_headers, check_url, has_header, merge_headers
|
|
22
|
+
from ._response import HTTPResponse, too_large
|
|
23
|
+
from ._state import StateTransport, cookie_aliases, make_cookies, make_tokens
|
|
24
|
+
|
|
25
|
+
if _ty.TYPE_CHECKING:
|
|
26
|
+
from ..auth import AuthProvider
|
|
27
|
+
|
|
28
|
+
DEFAULT_TIMEOUT = (10, 60)
|
|
29
|
+
"""`(connect, read)` timeout, in seconds, a client sends with every request
|
|
30
|
+
unless it is given `timeout` (`HTTPClient(timeout=...)` or per request);
|
|
31
|
+
`timeout=None` waits without a limit."""
|
|
32
|
+
|
|
33
|
+
_UNSET: _ty.Any = object()
|
|
34
|
+
"""The `timeout` or `deadline` of a call that did not give one: the client's own."""
|
|
35
|
+
|
|
36
|
+
_RETRIED_METHODS = frozenset({"GET", "HEAD", "OPTIONS", "PUT", "DELETE", "TRACE", "PROPFIND"})
|
|
37
|
+
"""Methods a status or a failed connection is retried for: the ones that are
|
|
38
|
+
safe to send twice."""
|
|
39
|
+
|
|
40
|
+
_REQUEST_OPTIONS = frozenset({"data", "headers", "stream", "allow_redirects", "timeout", "deadline", "auth"})
|
|
41
|
+
"""The keyword arguments of `request()`, which a hook may change."""
|
|
42
|
+
|
|
43
|
+
MAX_RETRY_WAIT = 120.0
|
|
44
|
+
"""Most seconds a client waits before a retry, whatever `Retry-After` asks."""
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _retry_wait(attempt: int, backoff: float, headers: "_ty.Mapping[str, str] | None") -> float:
|
|
48
|
+
"""Seconds to wait before retry number `attempt` (1 is the first): none
|
|
49
|
+
before the first, then `backoff` doubled each time; never less than a
|
|
50
|
+
`Retry-After` that `headers` carry; at most `MAX_RETRY_WAIT`."""
|
|
51
|
+
wait = 0.0 if attempt <= 1 else backoff * 2 ** (attempt - 1)
|
|
52
|
+
asked = retry_after(headers.items()) if headers else None
|
|
53
|
+
if asked is not None:
|
|
54
|
+
wait = max(wait, asked)
|
|
55
|
+
return min(wait, MAX_RETRY_WAIT)
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class HTTPClient:
|
|
59
|
+
"""A transport with the settings every request is sent with, the rules for
|
|
60
|
+
what happens after a redirect, and the state that belongs to a client.
|
|
61
|
+
|
|
62
|
+
`transport` is any object with a `send()` method as `HTTPTransport.send()`;
|
|
63
|
+
one it is given stays the caller's. Without one the client builds the
|
|
64
|
+
transport the `SYSHTTP_DEFAULT_TRANSPORT` setting names, else an
|
|
65
|
+
`HTTPTransport`, when it is made, and closes it with `close()` or at the
|
|
66
|
+
end of a `with` block. `proxymap`, `ssl_context`, `proxy_ssl_context` and
|
|
67
|
+
`verify` configure that `HTTPTransport` and are `ValueError` beside a
|
|
68
|
+
transport of your own; `env` (default `HTTPEnv()`) is where the client and
|
|
69
|
+
its transport read their settings. `headers` are sent with every request,
|
|
70
|
+
request-specific headers merged over them by name, over a `User-Agent` of
|
|
71
|
+
`user_agent` (`syshttp/<version>` by default) and `Accept: */*`.
|
|
72
|
+
Every request gets `timeout=DEFAULT_TIMEOUT` (`(10, 60)` seconds) unless
|
|
73
|
+
`timeout` here or in the call says otherwise (`None` waits without limit).
|
|
74
|
+
`timeout` bounds each wait; `deadline` (seconds, `None` for no limit, the
|
|
75
|
+
default) bounds the whole request: redirects, retries and the reading of
|
|
76
|
+
the body included, each wait's timeout is held to the time that remains,
|
|
77
|
+
and `TimeoutError` is raised once it has passed. A read already blocked can
|
|
78
|
+
overrun it by at most the timeout of that wait.
|
|
79
|
+
|
|
80
|
+
The client, not the transport, follows redirects. `GET`, `HEAD` and
|
|
81
|
+
`OPTIONS` follow up to 30; a redirect that changes the scheme, host or
|
|
82
|
+
port drops `Authorization`, `Cookie` and `Proxy-Authorization` (and nothing
|
|
83
|
+
else; the jar's cookies are chosen again for each hop). Any other method is sent
|
|
84
|
+
without following: a 307 or 308 (for `PROPFIND` also a 301, 302 or 303) to
|
|
85
|
+
the same scheme, host and port is sent once more with the same method and
|
|
86
|
+
body, and any other 3xx, a redirect to another origin or a second redirect
|
|
87
|
+
raises `OSError(EIO)`.
|
|
88
|
+
|
|
89
|
+
**State**, chosen by the host of the URL asked for (the origin), never by a
|
|
90
|
+
proxy's or a gateway's. A client keeps the cookies its responses set, in
|
|
91
|
+
memory and, with `cookies`, in a jar: `True` is a file jar under
|
|
92
|
+
`jar.store_root()` by host, a string one key for every host, a jar object
|
|
93
|
+
is used as given. `cookie_hosts` (default `SYSHTTP_COOKIE_HOSTS`) declares
|
|
94
|
+
groups of hosts that share cookies. `tokens` is read the same way and
|
|
95
|
+
becomes a per-request `Authorization` (`Bearer <token>`, or verbatim when
|
|
96
|
+
it names a scheme) that follows no redirect to another origin. In
|
|
97
|
+
`request()` (not in `redirect()`): `auth_provider.ensure(client, url)` runs
|
|
98
|
+
first, its errors swallowed; the `SYSHTTP_HOOKS_<KEY>_PY` hooks run on the
|
|
99
|
+
caller's URL, and a request a hook makes through the client runs none of
|
|
100
|
+
them again (the guard is per thread); a status in `status_forcelist`, or a
|
|
101
|
+
failed connection (`ConnectionError`), is retried up to `retries` times for
|
|
102
|
+
a method that is safe to send twice, after `backoff` seconds doubled each
|
|
103
|
+
time and never less than the `Retry-After` of the reply, and a status that
|
|
104
|
+
outlasts the retries is returned, not raised."""
|
|
105
|
+
|
|
106
|
+
def __init__(
|
|
107
|
+
self,
|
|
108
|
+
transport: "_ty.Any" = None,
|
|
109
|
+
*,
|
|
110
|
+
headers: "_ty.Mapping[str, str] | None" = None,
|
|
111
|
+
timeout: "float | tuple | None" = DEFAULT_TIMEOUT,
|
|
112
|
+
deadline: "float | None" = None,
|
|
113
|
+
user_agent: "str | None" = None,
|
|
114
|
+
env: "HTTPEnv | None" = None,
|
|
115
|
+
proxymap: "_ty.Any" = None,
|
|
116
|
+
ssl_context: "_ssl.SSLContext | None" = None,
|
|
117
|
+
proxy_ssl_context: "_ssl.SSLContext | None" = None,
|
|
118
|
+
verify: "bool | str | _os.PathLike[str]" = True,
|
|
119
|
+
cookies: "bool | str | CookieJar | None" = False,
|
|
120
|
+
tokens: "bool | str | TokenJar | None" = False,
|
|
121
|
+
auth_provider: "AuthProvider | None" = None,
|
|
122
|
+
retries: int = 3,
|
|
123
|
+
backoff: float = 1,
|
|
124
|
+
status_forcelist: "_ty.Iterable[int]" = TRANSIENT_STATUSES,
|
|
125
|
+
cookie_hosts: "_ty.Iterable[_ty.Iterable[str]] | None" = None,
|
|
126
|
+
):
|
|
127
|
+
if transport is not None and not callable(getattr(transport, "send", None)):
|
|
128
|
+
raise TypeError("a transport is an object with a send() method")
|
|
129
|
+
if isinstance(retries, bool) or not isinstance(retries, int) or retries < 0:
|
|
130
|
+
raise ValueError("retries is a count of retries, zero or more")
|
|
131
|
+
if isinstance(backoff, bool) or not isinstance(backoff, (int, float)) or not 0 <= backoff < float("inf"):
|
|
132
|
+
raise ValueError("backoff is a number of seconds, zero or more")
|
|
133
|
+
self._env = HTTPEnv() if env is None else env
|
|
134
|
+
options = {
|
|
135
|
+
"proxymap": proxymap,
|
|
136
|
+
"ssl_context": ssl_context,
|
|
137
|
+
"proxy_ssl_context": proxy_ssl_context,
|
|
138
|
+
"verify": verify,
|
|
139
|
+
}
|
|
140
|
+
self._transport = transport
|
|
141
|
+
self._built = None
|
|
142
|
+
self._transport_lock = _threading.Lock()
|
|
143
|
+
if transport is not None:
|
|
144
|
+
if any(value is not (True if name == "verify" else None) for name, value in options.items()):
|
|
145
|
+
raise ValueError(
|
|
146
|
+
"proxymap, ssl_context, proxy_ssl_context and verify configure the default transport: "
|
|
147
|
+
"a client given a transport leaves them to it"
|
|
148
|
+
)
|
|
149
|
+
else:
|
|
150
|
+
self._make, self._owned = transport_source(self._env, options)
|
|
151
|
+
self.transport # made now, so a setting that cannot be used is refused now
|
|
152
|
+
self._cookies = make_cookies(cookies, self._env, cookie_aliases(cookie_hosts, self._env))
|
|
153
|
+
self._tokens = make_tokens(tokens, self._env)
|
|
154
|
+
self._auth_provider = auth_provider
|
|
155
|
+
self._retries = retries
|
|
156
|
+
self._backoff = float(backoff)
|
|
157
|
+
self._status_forcelist = frozenset(status_forcelist)
|
|
158
|
+
self._busy = _threading.local()
|
|
159
|
+
self.headers = dict(headers or {})
|
|
160
|
+
self.timeout = timeout
|
|
161
|
+
self.deadline = check_deadline(deadline)
|
|
162
|
+
self.user_agent = user_agent or f"syshttp/{__version__}"
|
|
163
|
+
|
|
164
|
+
def __repr__(self) -> str:
|
|
165
|
+
# Headers are left out: they carry credentials.
|
|
166
|
+
return f"{type(self).__name__}()"
|
|
167
|
+
|
|
168
|
+
@property
|
|
169
|
+
def transport(self):
|
|
170
|
+
"""The transport: the one the client was given, or the one it made
|
|
171
|
+
(again, after `close()`). Read once per request."""
|
|
172
|
+
transport = self._transport
|
|
173
|
+
if transport is None:
|
|
174
|
+
with self._transport_lock:
|
|
175
|
+
if self._transport is None:
|
|
176
|
+
self._transport = self._make()
|
|
177
|
+
if self._owned:
|
|
178
|
+
self._built = self._transport
|
|
179
|
+
transport = self._transport
|
|
180
|
+
return transport
|
|
181
|
+
|
|
182
|
+
def close(self) -> None:
|
|
183
|
+
"""Closes the transport this client built for itself; one it was given
|
|
184
|
+
stays the caller's. Harmless when called again."""
|
|
185
|
+
with self._transport_lock:
|
|
186
|
+
built, self._built = self._built, None
|
|
187
|
+
if built is not None:
|
|
188
|
+
self._transport = None
|
|
189
|
+
if built is not None:
|
|
190
|
+
built.close()
|
|
191
|
+
|
|
192
|
+
def __enter__(self):
|
|
193
|
+
return self
|
|
194
|
+
|
|
195
|
+
def __exit__(self, *exc_info) -> None:
|
|
196
|
+
self.close()
|
|
197
|
+
|
|
198
|
+
def _sent_headers(self, headers, auth) -> "dict[str, str]":
|
|
199
|
+
"""The headers every request starts from: the base ones, the client's,
|
|
200
|
+
the call's, and the `Authorization` that `auth` stands for."""
|
|
201
|
+
base = {"User-Agent": self.user_agent, "Accept": "*/*"}
|
|
202
|
+
sent = merge_headers(base, self.headers, headers)
|
|
203
|
+
if auth is not None and not has_header(sent, "Authorization"):
|
|
204
|
+
sent["Authorization"] = auth if isinstance(auth, str) else _follow.basic_authorization(auth)
|
|
205
|
+
return sent
|
|
206
|
+
|
|
207
|
+
def _clock(self, deadline, method, url, first=None) -> "Deadline | None":
|
|
208
|
+
"""The deadline of a request: the call's, else the client's."""
|
|
209
|
+
seconds = self.deadline if deadline is _UNSET else check_deadline(deadline)
|
|
210
|
+
return None if seconds is None else Deadline(seconds, f"{method} {_follow.shown(first or url)}")
|
|
211
|
+
|
|
212
|
+
def _dispatch(
|
|
213
|
+
self,
|
|
214
|
+
method,
|
|
215
|
+
url,
|
|
216
|
+
sent,
|
|
217
|
+
data,
|
|
218
|
+
allow_redirects,
|
|
219
|
+
stream,
|
|
220
|
+
timeout,
|
|
221
|
+
deadline,
|
|
222
|
+
first=None,
|
|
223
|
+
) -> HTTPResponse:
|
|
224
|
+
check_url(url)
|
|
225
|
+
check_headers(sent)
|
|
226
|
+
if timeout is _UNSET:
|
|
227
|
+
timeout = self.timeout
|
|
228
|
+
if isinstance(data, str):
|
|
229
|
+
data = data.encode("utf-8")
|
|
230
|
+
transport = StateTransport(self.transport, self._cookies)
|
|
231
|
+
# Always streamed from the transport, so that no reply is read whole
|
|
232
|
+
# before it is known to be small enough (see `stream` below).
|
|
233
|
+
if method.upper() in _follow.FOLLOWED_METHODS and data is None:
|
|
234
|
+
if allow_redirects is False and first is None:
|
|
235
|
+
resp = transport.send(
|
|
236
|
+
method,
|
|
237
|
+
url,
|
|
238
|
+
headers=sent,
|
|
239
|
+
timeout=timeout if deadline is None else deadline.cap(timeout),
|
|
240
|
+
stream=True,
|
|
241
|
+
)
|
|
242
|
+
else:
|
|
243
|
+
resp = _follow.send_following(
|
|
244
|
+
transport,
|
|
245
|
+
method,
|
|
246
|
+
url,
|
|
247
|
+
headers=sent,
|
|
248
|
+
timeout=timeout,
|
|
249
|
+
stream=True,
|
|
250
|
+
first=first,
|
|
251
|
+
deadline=deadline,
|
|
252
|
+
)
|
|
253
|
+
else:
|
|
254
|
+
resp = _follow.send_once(
|
|
255
|
+
transport,
|
|
256
|
+
method,
|
|
257
|
+
url,
|
|
258
|
+
headers=sent,
|
|
259
|
+
body=data,
|
|
260
|
+
timeout=timeout,
|
|
261
|
+
stream=True,
|
|
262
|
+
deadline=deadline,
|
|
263
|
+
)
|
|
264
|
+
if deadline is not None:
|
|
265
|
+
deadline.bound(resp)
|
|
266
|
+
if not stream:
|
|
267
|
+
limit = _settings.MAX_REPLY_BYTES
|
|
268
|
+
resp._buffered(limit, too_large(_follow.shown(url), limit))
|
|
269
|
+
return resp
|
|
270
|
+
|
|
271
|
+
def _retried(self, method: str, data) -> bool:
|
|
272
|
+
"""Whether this request may be sent again after it failed."""
|
|
273
|
+
return self._retries > 0 and method.upper() in _RETRIED_METHODS and _follow.can_resend_body(data)
|
|
274
|
+
|
|
275
|
+
def _send(self, method, url, data, headers, stream, allow_redirects, timeout, deadline, auth) -> HTTPResponse:
|
|
276
|
+
"""One request with its token, retries and deadline."""
|
|
277
|
+
if isinstance(data, str):
|
|
278
|
+
data = data.encode("utf-8")
|
|
279
|
+
if auth is None and self._tokens is not None:
|
|
280
|
+
auth = self._tokens.authorization(url)
|
|
281
|
+
sent = self._sent_headers(headers, auth)
|
|
282
|
+
clock = self._clock(deadline, method, url)
|
|
283
|
+
retried = self._retried(method, data)
|
|
284
|
+
attempt = 0
|
|
285
|
+
while True:
|
|
286
|
+
try:
|
|
287
|
+
resp = self._dispatch(method, url, sent, data, allow_redirects, stream, timeout, clock)
|
|
288
|
+
except ConnectionError:
|
|
289
|
+
if not retried or attempt >= self._retries:
|
|
290
|
+
raise
|
|
291
|
+
reply_headers = None
|
|
292
|
+
else:
|
|
293
|
+
if not (retried and attempt < self._retries and resp.status in self._status_forcelist):
|
|
294
|
+
return resp
|
|
295
|
+
reply_headers = resp.headers
|
|
296
|
+
_follow.discard(resp)
|
|
297
|
+
attempt += 1
|
|
298
|
+
wait = _retry_wait(attempt, self._backoff, reply_headers)
|
|
299
|
+
if clock is not None:
|
|
300
|
+
wait = min(wait, clock.check())
|
|
301
|
+
if wait > 0:
|
|
302
|
+
_time.sleep(wait)
|
|
303
|
+
|
|
304
|
+
def _guarded(self, name: str) -> bool:
|
|
305
|
+
return bool(getattr(self._busy, name, False))
|
|
306
|
+
|
|
307
|
+
def _ensure(self, url: str) -> None:
|
|
308
|
+
"""Runs the auth provider for `url`, unless this thread is inside it."""
|
|
309
|
+
if self._auth_provider is None or self._guarded("ensuring"):
|
|
310
|
+
return
|
|
311
|
+
self._busy.ensuring = True
|
|
312
|
+
try:
|
|
313
|
+
self._auth_provider.ensure(self, url)
|
|
314
|
+
except Exception:
|
|
315
|
+
pass
|
|
316
|
+
finally:
|
|
317
|
+
self._busy.ensuring = False
|
|
318
|
+
|
|
319
|
+
def request(
|
|
320
|
+
self,
|
|
321
|
+
method: str,
|
|
322
|
+
url: str,
|
|
323
|
+
*,
|
|
324
|
+
data=None,
|
|
325
|
+
headers: "_ty.Mapping[str, str] | None" = None,
|
|
326
|
+
stream: bool = False,
|
|
327
|
+
allow_redirects: "bool | None" = None,
|
|
328
|
+
timeout=_UNSET,
|
|
329
|
+
deadline=_UNSET,
|
|
330
|
+
auth: "tuple[str | bytes, str | bytes] | str | None" = None,
|
|
331
|
+
) -> HTTPResponse:
|
|
332
|
+
"""Send `method` to `url` and return the final response. A `data` that
|
|
333
|
+
is not `bytes` (a file, an iterable) cannot be sent twice, so it is not
|
|
334
|
+
re-sent after a redirect, a dropped connection or a retry.
|
|
335
|
+
`allow_redirects=False` returns the first answer of a `GET`, `HEAD` or
|
|
336
|
+
`OPTIONS` as it is. `auth` is a `(user, password)` pair sent as
|
|
337
|
+
`Authorization: Basic`, or a `str` sent as the `Authorization` value
|
|
338
|
+
(`"Bearer <token>"`), to the origin of `url` unless a header names an
|
|
339
|
+
`Authorization`; it beats a token of the client. `ValueError` for a URL
|
|
340
|
+
(credentials in it included) or a header with a control character,
|
|
341
|
+
before anything is sent.
|
|
342
|
+
|
|
343
|
+
The auth provider, the hooks, the token and the retries of the client
|
|
344
|
+
apply here. A hook is called as `hook(client, method, url, kwargs)` with
|
|
345
|
+
the keyword arguments given (`data`, `headers`, `stream`,
|
|
346
|
+
`allow_redirects`, `timeout`, `deadline`, `auth`), which it may change
|
|
347
|
+
in place, and may return a response to be used instead of sending.
|
|
348
|
+
|
|
349
|
+
With `stream=False` the body is read before this returns, and a decoded
|
|
350
|
+
body over `MAX_REPLY_BYTES` is `OSError(EFBIG)`; with `stream=True` the
|
|
351
|
+
caller reads it, and closes the response."""
|
|
352
|
+
given: "dict[str, _ty.Any]" = {
|
|
353
|
+
name: value
|
|
354
|
+
for name, value, default in (
|
|
355
|
+
("data", data, None),
|
|
356
|
+
("headers", headers, None),
|
|
357
|
+
("stream", stream, False),
|
|
358
|
+
("allow_redirects", allow_redirects, None),
|
|
359
|
+
("timeout", timeout, _UNSET),
|
|
360
|
+
("deadline", deadline, _UNSET),
|
|
361
|
+
("auth", auth, None),
|
|
362
|
+
)
|
|
363
|
+
if value is not default
|
|
364
|
+
}
|
|
365
|
+
self._ensure(url)
|
|
366
|
+
if not self._guarded("in_hook"):
|
|
367
|
+
self._busy.in_hook = True
|
|
368
|
+
try:
|
|
369
|
+
hooked = _hooks.call_py_hooks(self, method, url, given, env=self._env)
|
|
370
|
+
finally:
|
|
371
|
+
self._busy.in_hook = False
|
|
372
|
+
if hooked is not None:
|
|
373
|
+
return hooked
|
|
374
|
+
unknown = sorted(set(given) - _REQUEST_OPTIONS)
|
|
375
|
+
if unknown:
|
|
376
|
+
raise TypeError("a hook set %s, which request() does not take" % ", ".join(unknown))
|
|
377
|
+
return self._send(
|
|
378
|
+
method,
|
|
379
|
+
url,
|
|
380
|
+
given.get("data"),
|
|
381
|
+
given.get("headers"),
|
|
382
|
+
given.get("stream", False),
|
|
383
|
+
given.get("allow_redirects"),
|
|
384
|
+
given.get("timeout", _UNSET),
|
|
385
|
+
given.get("deadline", _UNSET),
|
|
386
|
+
given.get("auth"),
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
def redirect(
|
|
390
|
+
self,
|
|
391
|
+
method: str,
|
|
392
|
+
url: str,
|
|
393
|
+
resp: HTTPResponse,
|
|
394
|
+
*,
|
|
395
|
+
headers: "_ty.Mapping[str, str] | None" = None,
|
|
396
|
+
stream: bool = False,
|
|
397
|
+
timeout=_UNSET,
|
|
398
|
+
deadline=_UNSET,
|
|
399
|
+
auth: "tuple[str | bytes, str | bytes] | str | None" = None,
|
|
400
|
+
) -> HTTPResponse:
|
|
401
|
+
"""Send `method` to where `resp`, the answer to a request for `url`,
|
|
402
|
+
redirects to, and follow what it redirects to in turn. The request
|
|
403
|
+
carries what `url` is sent with, except that `Authorization`, `Cookie`
|
|
404
|
+
and `Proxy-Authorization` are left off whenever it goes to another
|
|
405
|
+
scheme, host or port than `url`'s. A
|
|
406
|
+
redirect with no usable `Location` is `OSError(EIO)`. Cookies apply to
|
|
407
|
+
every hop; the token, hooks and retries belong to `request()`."""
|
|
408
|
+
sent = self._sent_headers(headers, auth)
|
|
409
|
+
location = _follow.redirect_location(url, resp)
|
|
410
|
+
if location is None:
|
|
411
|
+
raise _follow.refused_redirect(method, url, resp, None, "no usable Location")
|
|
412
|
+
clock = self._clock(deadline, method, location, first=url)
|
|
413
|
+
return self._dispatch(method, location, sent, None, None, stream, timeout, clock, first=url)
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
"""Which TLS context a connection is made with: the caller's, a CA bundle, the
|
|
2
|
+
system's, or none that verifies."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import os as _os
|
|
7
|
+
import ssl as _ssl
|
|
8
|
+
import threading as _threading
|
|
9
|
+
|
|
10
|
+
from .. import tls as _tls
|
|
11
|
+
from ..env import HTTPEnv
|
|
12
|
+
from ..exceptions import ConfigurationError
|
|
13
|
+
|
|
14
|
+
CA_BUNDLE = "CA_BUNDLE"
|
|
15
|
+
"""The setting naming a CA bundle (a PEM file) that replaces the system's trust
|
|
16
|
+
store for origins when verification is on and the caller gave no context."""
|
|
17
|
+
|
|
18
|
+
_VERIFIED: "dict[tuple[str, int], _ssl.SSLContext]" = {}
|
|
19
|
+
_UNVERIFIED: "list[_ssl.SSLContext]" = []
|
|
20
|
+
_LOCK = _threading.Lock()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def _unverified_context() -> "_ssl.SSLContext":
|
|
24
|
+
"""One context that accepts any certificate, for `verify=False`."""
|
|
25
|
+
with _LOCK:
|
|
26
|
+
if not _UNVERIFIED:
|
|
27
|
+
context = _ssl.create_default_context()
|
|
28
|
+
context.check_hostname = False
|
|
29
|
+
context.verify_mode = _ssl.CERT_NONE
|
|
30
|
+
_UNVERIFIED.append(context)
|
|
31
|
+
return _UNVERIFIED[0]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _bundle_context(path: str, source: str) -> "_ssl.SSLContext":
|
|
35
|
+
"""A context trusting only the PEM file `path`, shared and renewed when the
|
|
36
|
+
file changes. `source` is what a message names: the variable or argument
|
|
37
|
+
that gave the path."""
|
|
38
|
+
try:
|
|
39
|
+
key = (_os.path.abspath(path), _os.stat(path).st_mtime_ns)
|
|
40
|
+
with _LOCK:
|
|
41
|
+
context = _VERIFIED.get(key)
|
|
42
|
+
if context is None:
|
|
43
|
+
context = _tls.new_os_context(cafile=path, client_certificates=False)
|
|
44
|
+
with _LOCK:
|
|
45
|
+
for stale in [k for k in _VERIFIED if k[0] == key[0]]:
|
|
46
|
+
del _VERIFIED[stale]
|
|
47
|
+
_VERIFIED[key] = context
|
|
48
|
+
return context
|
|
49
|
+
except (OSError, _ssl.SSLError) as error:
|
|
50
|
+
raise ConfigurationError(
|
|
51
|
+
"%s names a CA bundle that cannot be read (%s)" % (source, type(error).__name__)
|
|
52
|
+
) from None
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
class TLSChoice:
|
|
56
|
+
"""The TLS settings of one transport.
|
|
57
|
+
|
|
58
|
+
`ssl_context` is used as given for every https origin, and its own
|
|
59
|
+
`verify_mode` decides, so `verify` must stay `True` beside it. Without one,
|
|
60
|
+
`verify=True` verifies with the CA bundle `CA_BUNDLE` names (read through
|
|
61
|
+
`env` at each connection) and else with the system's context, `verify=False`
|
|
62
|
+
verifies nothing, and a path as `verify` names a bundle. `proxy_ssl_context`
|
|
63
|
+
is what an `https:` proxy or gateway is spoken to with; without it, the
|
|
64
|
+
system's context."""
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
env: HTTPEnv,
|
|
69
|
+
ssl_context: "_ssl.SSLContext | None",
|
|
70
|
+
proxy_ssl_context: "_ssl.SSLContext | None",
|
|
71
|
+
verify: "bool | str | _os.PathLike[str]",
|
|
72
|
+
):
|
|
73
|
+
for name, value in (("ssl_context", ssl_context), ("proxy_ssl_context", proxy_ssl_context)):
|
|
74
|
+
if value is not None and not isinstance(value, _ssl.SSLContext):
|
|
75
|
+
raise TypeError("%s must be an ssl.SSLContext, not %s" % (name, type(value).__name__))
|
|
76
|
+
if not isinstance(verify, (bool, str, _os.PathLike)):
|
|
77
|
+
raise TypeError("verify is True, False or the path of a CA bundle, not %s" % type(verify).__name__)
|
|
78
|
+
if ssl_context is not None and verify is not True:
|
|
79
|
+
raise ValueError("verify=%r and ssl_context contradict: the context's verify_mode decides" % (verify,))
|
|
80
|
+
self._env = env
|
|
81
|
+
self._ssl_context = ssl_context
|
|
82
|
+
self._proxy_ssl_context = proxy_ssl_context
|
|
83
|
+
self._verify: "bool | str" = verify if isinstance(verify, bool) else _os.fspath(verify)
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def unverified(self) -> bool:
|
|
87
|
+
"""Whether this transport turns certificate checking off for origins."""
|
|
88
|
+
return self._ssl_context is None and self._verify is False
|
|
89
|
+
|
|
90
|
+
def origin(self) -> "_ssl.SSLContext":
|
|
91
|
+
"""The context for an https origin."""
|
|
92
|
+
if self._ssl_context is not None:
|
|
93
|
+
return self._ssl_context
|
|
94
|
+
if self._verify is False:
|
|
95
|
+
return _unverified_context()
|
|
96
|
+
if isinstance(self._verify, str):
|
|
97
|
+
return _bundle_context(self._verify, "verify")
|
|
98
|
+
found = self._env.find(CA_BUNDLE)
|
|
99
|
+
if found is not None:
|
|
100
|
+
return _bundle_context(found[1], found[0])
|
|
101
|
+
return _tls.os_ssl_context()
|
|
102
|
+
|
|
103
|
+
def proxy(self) -> "_ssl.SSLContext":
|
|
104
|
+
"""The context for an `https:` proxy or gateway."""
|
|
105
|
+
if self._proxy_ssl_context is not None:
|
|
106
|
+
return self._proxy_ssl_context
|
|
107
|
+
return _tls.os_ssl_context()
|
|
@@ -0,0 +1,94 @@
|
|
|
1
|
+
"""A time limit for a whole request, above the per-wait timeout."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math as _math
|
|
6
|
+
import time as _time
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def check_deadline(deadline) -> "float | None":
|
|
10
|
+
"""`deadline` as seconds, or `None` for no limit. `TypeError` for anything
|
|
11
|
+
but a number or `None`, `ValueError` for a number that is not positive."""
|
|
12
|
+
if deadline is None:
|
|
13
|
+
return None
|
|
14
|
+
if isinstance(deadline, bool) or not isinstance(deadline, (int, float)):
|
|
15
|
+
raise TypeError("a deadline is a number of seconds or None")
|
|
16
|
+
if _math.isnan(deadline) or deadline <= 0:
|
|
17
|
+
raise ValueError("a deadline is a positive number of seconds, or None")
|
|
18
|
+
return float(deadline)
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
class Deadline:
|
|
22
|
+
"""The moment a request must be finished by. It lives for one request,
|
|
23
|
+
redirects included."""
|
|
24
|
+
|
|
25
|
+
__slots__ = ("_seconds", "_at", "_what")
|
|
26
|
+
|
|
27
|
+
def __init__(self, seconds: float, what: str):
|
|
28
|
+
self._seconds = seconds
|
|
29
|
+
self._at = _time.monotonic() + seconds
|
|
30
|
+
self._what = what
|
|
31
|
+
|
|
32
|
+
def check(self) -> float:
|
|
33
|
+
"""The seconds that remain; `TimeoutError` once there are none."""
|
|
34
|
+
left = self._at - _time.monotonic()
|
|
35
|
+
if left <= 0:
|
|
36
|
+
raise TimeoutError(f"the {self._seconds:g} s deadline of {self._what} passed")
|
|
37
|
+
return left
|
|
38
|
+
|
|
39
|
+
def cap(self, timeout):
|
|
40
|
+
"""`timeout` (a number, `None` or a `(connect, read)` pair) with each
|
|
41
|
+
wait held to the time that remains."""
|
|
42
|
+
left = self.check()
|
|
43
|
+
if isinstance(timeout, tuple):
|
|
44
|
+
return tuple(self._capped(part, left) for part in timeout)
|
|
45
|
+
return self._capped(timeout, left)
|
|
46
|
+
|
|
47
|
+
@staticmethod
|
|
48
|
+
def _capped(wait, left):
|
|
49
|
+
return left if wait is None else min(wait, left)
|
|
50
|
+
|
|
51
|
+
def bound(self, response):
|
|
52
|
+
"""Make `response` check the deadline before each read of its body."""
|
|
53
|
+
source = getattr(response, "_source", None)
|
|
54
|
+
if source is not None:
|
|
55
|
+
response._source = _Bounded(source, self)
|
|
56
|
+
return response
|
|
57
|
+
|
|
58
|
+
def bound_async(self, response):
|
|
59
|
+
"""`bound()` for a response read with coroutines."""
|
|
60
|
+
source = getattr(response, "_source", None)
|
|
61
|
+
if source is not None:
|
|
62
|
+
response._source = _AsyncBounded(source, self)
|
|
63
|
+
return response
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class _Bounded:
|
|
67
|
+
"""A response body that raises `TimeoutError` instead of reading once the
|
|
68
|
+
deadline has passed."""
|
|
69
|
+
|
|
70
|
+
def __init__(self, source, deadline: Deadline):
|
|
71
|
+
self._source = source
|
|
72
|
+
self._deadline = deadline
|
|
73
|
+
|
|
74
|
+
def read(self, amount: int) -> bytes:
|
|
75
|
+
self._deadline.check()
|
|
76
|
+
return self._source.read(amount)
|
|
77
|
+
|
|
78
|
+
def close(self) -> None:
|
|
79
|
+
self._source.close()
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class _AsyncBounded:
|
|
83
|
+
"""`_Bounded` for a body read with coroutines."""
|
|
84
|
+
|
|
85
|
+
def __init__(self, source, deadline: Deadline):
|
|
86
|
+
self._source = source
|
|
87
|
+
self._deadline = deadline
|
|
88
|
+
|
|
89
|
+
async def read(self, amount: int) -> bytes:
|
|
90
|
+
self._deadline.check()
|
|
91
|
+
return await self._source.read(amount)
|
|
92
|
+
|
|
93
|
+
async def aclose(self) -> None:
|
|
94
|
+
await self._source.aclose()
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"""The transport a client uses when it is given none: the one the
|
|
2
|
+
`SYSHTTP_DEFAULT_TRANSPORT` setting names, else syshttp's own."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import typing as _ty
|
|
7
|
+
|
|
8
|
+
from ..env import HTTPEnv
|
|
9
|
+
from ..exceptions import ConfigurationError
|
|
10
|
+
from ..hooks import load_object
|
|
11
|
+
from ._transport import HTTPTransport, Transport
|
|
12
|
+
|
|
13
|
+
DEFAULT_TRANSPORT = "SYSHTTP_DEFAULT_TRANSPORT"
|
|
14
|
+
"""The setting naming the transport of a client given none: an object reference
|
|
15
|
+
`module:attr` or `<file>.py:attr` (a dotted `attr` is allowed). The object is
|
|
16
|
+
the transport when it has `send()`; otherwise it is called with no arguments
|
|
17
|
+
to build one (a class, a factory). It names code to import: never let an
|
|
18
|
+
untrusted party set it."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _resolve(value: str) -> _ty.Any:
|
|
22
|
+
"""The object a setting value names. Short names for known transports
|
|
23
|
+
would be resolved here."""
|
|
24
|
+
return load_object(value, "a transport")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def transport_source(env: HTTPEnv, options: "_ty.Mapping[str, _ty.Any]") -> "tuple[_ty.Callable[[], Transport], bool]":
|
|
28
|
+
"""How a client with no `transport=` gets one: `(make, owned)`, where
|
|
29
|
+
`make()` returns the transport and `owned` says whether the client built
|
|
30
|
+
it and so closes it. `options` are the keywords of `HTTPTransport`.
|
|
31
|
+
|
|
32
|
+
Without the setting `make` builds an `HTTPTransport` from `options`. With
|
|
33
|
+
it, `options` must be left at their defaults: they configure syshttp's own
|
|
34
|
+
transport. Anything that cannot be loaded, called or used as a transport is
|
|
35
|
+
a `ConfigurationError` naming the variable."""
|
|
36
|
+
found = env.find(DEFAULT_TRANSPORT)
|
|
37
|
+
if found is None:
|
|
38
|
+
return (lambda: HTTPTransport(env=env, **options)), True
|
|
39
|
+
variable, value = found
|
|
40
|
+
given = sorted(name for name, option in options.items() if option is not (True if name == "verify" else None))
|
|
41
|
+
if given:
|
|
42
|
+
raise ConfigurationError(
|
|
43
|
+
"%s names a transport of its own: %s configure syshttp's, not that one" % (variable, ", ".join(given))
|
|
44
|
+
)
|
|
45
|
+
try:
|
|
46
|
+
named = _resolve(value)
|
|
47
|
+
except ConfigurationError as error:
|
|
48
|
+
raise ConfigurationError("%s: %s" % (variable, error)) from None
|
|
49
|
+
except Exception as error:
|
|
50
|
+
raise ConfigurationError(
|
|
51
|
+
"%s names %r, which cannot be loaded (%s: %s)" % (variable, value, type(error).__name__, error)
|
|
52
|
+
) from error
|
|
53
|
+
if not isinstance(named, type) and callable(getattr(named, "send", None)):
|
|
54
|
+
return (lambda: named), False
|
|
55
|
+
if not callable(named):
|
|
56
|
+
raise ConfigurationError(
|
|
57
|
+
"%s names %r, which is neither a transport nor something that makes one" % (variable, value)
|
|
58
|
+
)
|
|
59
|
+
|
|
60
|
+
def make() -> Transport:
|
|
61
|
+
try:
|
|
62
|
+
made = named()
|
|
63
|
+
except Exception as error:
|
|
64
|
+
raise ConfigurationError(
|
|
65
|
+
"%s names %r, which failed to make a transport (%s: %s)"
|
|
66
|
+
% (variable, value, type(error).__name__, error)
|
|
67
|
+
) from error
|
|
68
|
+
if not callable(getattr(made, "send", None)):
|
|
69
|
+
raise ConfigurationError("%s names %r, which makes no object with send()" % (variable, value))
|
|
70
|
+
return made
|
|
71
|
+
|
|
72
|
+
return make, True
|