sdo-client 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sdo/__init__.py +74 -0
- sdo/_decompress.py +253 -0
- sdo/_encoding.py +739 -0
- sdo/_json.py +50 -0
- sdo/_logging.py +93 -0
- sdo/_secret.py +74 -0
- sdo/_transport.py +639 -0
- sdo/_urls.py +80 -0
- sdo/_version.py +8 -0
- sdo/_wire.py +126 -0
- sdo/client.py +459 -0
- sdo/config.py +452 -0
- sdo/errors.py +247 -0
- sdo/jobs.py +490 -0
- sdo/plugins.py +143 -0
- sdo/py.typed +0 -0
- sdo/response.py +177 -0
- sdo/usage.py +62 -0
- sdo_client-0.1.0.dist-info/METADATA +377 -0
- sdo_client-0.1.0.dist-info/RECORD +23 -0
- sdo_client-0.1.0.dist-info/WHEEL +4 -0
- sdo_client-0.1.0.dist-info/licenses/LICENSE +201 -0
- sdo_client-0.1.0.dist-info/licenses/NOTICE +8 -0
sdo/_transport.py
ADDED
|
@@ -0,0 +1,639 @@
|
|
|
1
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
2
|
+
"""HTTP transport: the one request path for scrape, plugins and async jobs.
|
|
3
|
+
|
|
4
|
+
Every request goes through :meth:`Transport.send`, so every path shares the
|
|
5
|
+
same guarantees:
|
|
6
|
+
|
|
7
|
+
* ``Accept-Encoding`` is explicit and the body is decompressed here, bounded:
|
|
8
|
+
the raw read and every decoding step stop at ``max_response_bytes`` before
|
|
9
|
+
the inflated buffer is materialized (see :mod:`sdo._decompress`);
|
|
10
|
+
* the client timeout is a total wall-clock deadline per attempt over connect,
|
|
11
|
+
headers and the whole body. httpx can only enforce it between reads (each
|
|
12
|
+
read is bounded by the same timeout), so a peer that trickles bytes can
|
|
13
|
+
stretch one attempt to about twice the timeout;
|
|
14
|
+
* redirects are never followed, whatever the injected ``httpx.Client`` says;
|
|
15
|
+
* URLs are absolute and the query string is built by the SDK, byte-exact;
|
|
16
|
+
* the headers sent are exactly the SDK's own plus the caller's, compared
|
|
17
|
+
case-insensitively; httpx's defaults are not sent;
|
|
18
|
+
* every failure is an :class:`~sdo.errors.SdoError` whose message, retained
|
|
19
|
+
body and (absent) cause carry no credential, and no frame that is live when
|
|
20
|
+
it is raised holds the token;
|
|
21
|
+
* retries follow one policy: retryable statuses (only 429 for methods that can
|
|
22
|
+
have side effects on the target), and failures before anything was sent
|
|
23
|
+
(idempotent methods only). A failure after the request was sent is never
|
|
24
|
+
retried: the gateway may still run, and bill, it.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import random
|
|
30
|
+
import re
|
|
31
|
+
import time
|
|
32
|
+
import traceback
|
|
33
|
+
import weakref
|
|
34
|
+
from collections.abc import Mapping
|
|
35
|
+
from dataclasses import dataclass, field
|
|
36
|
+
|
|
37
|
+
import httpx
|
|
38
|
+
|
|
39
|
+
from . import _json, _logging
|
|
40
|
+
from ._decompress import BodyDecoder, DecodeError, TooLarge
|
|
41
|
+
from ._secret import Secret
|
|
42
|
+
from ._wire import pct
|
|
43
|
+
from .errors import (
|
|
44
|
+
SdoAPIError,
|
|
45
|
+
SdoAuthError,
|
|
46
|
+
SdoConcurrencyError,
|
|
47
|
+
SdoConfigError,
|
|
48
|
+
SdoConnectionError,
|
|
49
|
+
SdoError,
|
|
50
|
+
SdoGatewayError,
|
|
51
|
+
SdoResponseTooLargeError,
|
|
52
|
+
SdoTimeoutError,
|
|
53
|
+
SdoTransportError,
|
|
54
|
+
SdoValidationError,
|
|
55
|
+
redact_secrets,
|
|
56
|
+
)
|
|
57
|
+
from .response import parse_request_cost
|
|
58
|
+
|
|
59
|
+
#: Statuses where the same request may succeed when retried. 510 is not one:
|
|
60
|
+
#: the gateway answers 510 only when the HTTP client cancelled the request
|
|
61
|
+
#: before the gateway finished it, so a retry would re-send an abandoned request.
|
|
62
|
+
_RETRYABLE_STATUS = frozenset({429, 502, 503, 504, 520, 521, 522, 523, 524})
|
|
63
|
+
#: The gateway's status for a request its HTTP client cancelled.
|
|
64
|
+
_CLIENT_CANCELED_STATUS = 510
|
|
65
|
+
#: Methods that can have side effects on the target: retried only on 429, which
|
|
66
|
+
#: the gateway returns before contacting the target.
|
|
67
|
+
_NON_IDEMPOTENT = frozenset({"POST", "PUT", "PATCH", "DELETE"})
|
|
68
|
+
_NON_IDEMPOTENT_RETRY_STATUS = frozenset({429})
|
|
69
|
+
#: The documented billing table, used only for an error without a readable
|
|
70
|
+
#: ``Scrape.do-Request-Cost`` (the header is authoritative): of the statuses
|
|
71
|
+
#: that can be in an error position, only 404 and 410, which are results
|
|
72
|
+
#: anyway. The gateway's own 400, 401, 429, 502 and 510 are not billed.
|
|
73
|
+
_BILLED_STATUS = frozenset({404, 410})
|
|
74
|
+
#: Default ceiling for a response body, raw and decoded (32 MiB).
|
|
75
|
+
_MAX_BYTES = 32 * 1024 * 1024
|
|
76
|
+
#: How much of an error body is read: enough for any envelope.
|
|
77
|
+
_ERROR_BODY_BYTES = 64 * 1024
|
|
78
|
+
_MAX_BACKOFF = 30.0
|
|
79
|
+
_RETAINED_BODY_CHARS = 2048
|
|
80
|
+
#: With a gateway ``timeout`` set, one attempt waits at least that plus this.
|
|
81
|
+
_GATEWAY_TIMEOUT_MARGIN = 10
|
|
82
|
+
#: With transparent_response, this header marks the target's own answer.
|
|
83
|
+
_TARGET_ORIGIN_HEADER = "scrape.do-initial-status-code"
|
|
84
|
+
#: The credits the gateway charged for the request.
|
|
85
|
+
_REQUEST_COST_HEADER = "scrape.do-request-cost"
|
|
86
|
+
|
|
87
|
+
SYNC = "sync"
|
|
88
|
+
ASYNC = "async"
|
|
89
|
+
#: Where the token travels: the ``token`` query parameter, or ``X-Token``.
|
|
90
|
+
QUERY = "query"
|
|
91
|
+
HEADER = "header"
|
|
92
|
+
|
|
93
|
+
_DEFAULT_MESSAGES = {
|
|
94
|
+
"validation": "The gateway rejected the request parameters.",
|
|
95
|
+
"auth": "Authentication failed — check your token, plan permissions, and remaining credits.",
|
|
96
|
+
"concurrency": "Concurrent-request limit reached.",
|
|
97
|
+
"gateway": "Gateway-origin upstream failure (HTTP {status}).",
|
|
98
|
+
"canceled": "The request was canceled by the HTTP client (HTTP {status}).",
|
|
99
|
+
"api": "Request failed (HTTP {status}).",
|
|
100
|
+
"redirect": "Unexpected redirect (HTTP {status}); redirects are never followed.",
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
_DIGITS_RE = re.compile(r"[0-9]+", re.ASCII)
|
|
104
|
+
_SIGNED_RE = re.compile(r"-?[0-9]+", re.ASCII)
|
|
105
|
+
#: RFC 9110 token.
|
|
106
|
+
_HEADER_NAME_RE = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
|
|
107
|
+
#: Printable ASCII plus space and tab.
|
|
108
|
+
_HEADER_VALUE_RE = re.compile(r"[\t\x20-\x7e]*")
|
|
109
|
+
#: Managed by the HTTP client; a caller value would corrupt the framing.
|
|
110
|
+
_FORBIDDEN_HEADERS = frozenset({"host", "content-length", "transfer-encoding", "connection"})
|
|
111
|
+
#: Of the headers httpx puts on a built request, only these framing ones stay.
|
|
112
|
+
_FRAMING_HEADERS = frozenset({"host", "content-length", "transfer-encoding"})
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
@dataclass
|
|
116
|
+
class Reply:
|
|
117
|
+
"""A complete response: status, every header as received, decoded body."""
|
|
118
|
+
|
|
119
|
+
status_code: int
|
|
120
|
+
headers: list[tuple[str, str]]
|
|
121
|
+
body: bytes = field(default=b"")
|
|
122
|
+
#: False when an error body could not be read whole (it failed to
|
|
123
|
+
#: decompress, dropped or timed out): its envelope is then not trusted.
|
|
124
|
+
body_intact: bool = True
|
|
125
|
+
|
|
126
|
+
def header(self, name: str) -> str | None:
|
|
127
|
+
name = name.lower()
|
|
128
|
+
for k, v in self.headers:
|
|
129
|
+
if k.lower() == name:
|
|
130
|
+
return v
|
|
131
|
+
return None
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def retry_after_seconds(raw: str | None) -> int | None:
|
|
135
|
+
"""``Retry-After`` as whole seconds (RFC 9110 delay-seconds), else ``None``.
|
|
136
|
+
|
|
137
|
+
Only ASCII digits count: a date, a negative, a fraction, ``NaN`` or
|
|
138
|
+
``'١٢'`` fall back to the exponential backoff.
|
|
139
|
+
"""
|
|
140
|
+
if raw is None:
|
|
141
|
+
return None
|
|
142
|
+
raw = raw.strip(" \t")
|
|
143
|
+
return int(raw) if _DIGITS_RE.fullmatch(raw) else None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def effective_timeout(client_seconds: float, config_timeout_ms: int | None) -> float:
|
|
147
|
+
"""The per-attempt timeout: with a gateway ``timeout`` (milliseconds) set,
|
|
148
|
+
at least that plus a margin, so the client never gives up on a request the
|
|
149
|
+
gateway was told it may take that long for."""
|
|
150
|
+
if not config_timeout_ms:
|
|
151
|
+
return client_seconds
|
|
152
|
+
return max(client_seconds, config_timeout_ms / 1000 + _GATEWAY_TIMEOUT_MARGIN)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def _int_or_none(raw: str | None) -> int | None:
|
|
156
|
+
if raw is None:
|
|
157
|
+
return None
|
|
158
|
+
raw = raw.strip(" \t")
|
|
159
|
+
return int(raw) if _SIGNED_RE.fullmatch(raw) else None
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _ascii_lower(text: str) -> str:
|
|
163
|
+
"""Lowercase A-Z only (``str.lower`` would also fold ``"İ"`` or a Kelvin sign)."""
|
|
164
|
+
return text.translate(_ASCII_LOWER)
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
_ASCII_LOWER = {c: c + 32 for c in range(ord("A"), ord("Z") + 1)}
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
def _envelope_text(value: object) -> str | None:
|
|
171
|
+
if isinstance(value, list):
|
|
172
|
+
return "; ".join(m for m in value if isinstance(m, str) and m) or None
|
|
173
|
+
if isinstance(value, str):
|
|
174
|
+
return value or None
|
|
175
|
+
return None
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _sync_envelope(data: object) -> str | None:
|
|
179
|
+
"""A ``message`` key, else an ``error`` key, each matched in any ASCII
|
|
180
|
+
letter case: the gateway writes ``{"Message": ["...", ...]}``, a plugin
|
|
181
|
+
``{"error": "code", "message": "text"}``. The value is a string or a list
|
|
182
|
+
(string elements joined with ``"; "``). Several keys matching one name are
|
|
183
|
+
tried in byte order (``"Message"`` before ``"message"``); one that yields
|
|
184
|
+
nothing falls through to the next, then to ``error``."""
|
|
185
|
+
if not isinstance(data, dict):
|
|
186
|
+
return None
|
|
187
|
+
keys = sorted(k for k in data if isinstance(k, str))
|
|
188
|
+
for name in ("message", "error"):
|
|
189
|
+
for key in keys:
|
|
190
|
+
if _ascii_lower(key) == name:
|
|
191
|
+
text = _envelope_text(data[key])
|
|
192
|
+
if text is not None:
|
|
193
|
+
return text
|
|
194
|
+
return None
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
def _async_envelope(data: object) -> str | None:
|
|
198
|
+
"""``{"Error": "...", "Code": n}``: only ``Error`` is read."""
|
|
199
|
+
if isinstance(data, dict):
|
|
200
|
+
error = data.get("Error")
|
|
201
|
+
if isinstance(error, str) and error:
|
|
202
|
+
return error
|
|
203
|
+
return None
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
def envelope_message(body: bytes, api: str = SYNC) -> str | None:
|
|
207
|
+
"""The message an error body carries (a leading BOM is ignored)."""
|
|
208
|
+
try:
|
|
209
|
+
data = _json.loads(body)
|
|
210
|
+
except ValueError:
|
|
211
|
+
return None
|
|
212
|
+
return _async_envelope(data) if api == ASYNC else _sync_envelope(data)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _classify(status: int) -> tuple[type[SdoAPIError], str]:
|
|
216
|
+
if status in (401, 403):
|
|
217
|
+
# The gateway returns 401 for a depleted credit balance too, so the
|
|
218
|
+
# default message names both causes.
|
|
219
|
+
return SdoAuthError, "auth"
|
|
220
|
+
if status == 429:
|
|
221
|
+
return SdoConcurrencyError, "concurrency"
|
|
222
|
+
if status == 400:
|
|
223
|
+
return SdoValidationError, "validation"
|
|
224
|
+
if status >= 500:
|
|
225
|
+
return SdoGatewayError, "gateway"
|
|
226
|
+
if 300 <= status < 400:
|
|
227
|
+
return SdoAPIError, "redirect"
|
|
228
|
+
return SdoAPIError, "api"
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def build_api_error(reply: Reply, *, api: str = SYNC, token: str | None = None) -> SdoAPIError:
|
|
232
|
+
"""Map an error response to the taxonomy. The message comes from the error
|
|
233
|
+
envelope when there is one; message and retained body are redacted, then
|
|
234
|
+
the body is truncated."""
|
|
235
|
+
status = reply.status_code
|
|
236
|
+
cls, kind = _classify(status)
|
|
237
|
+
if status == _CLIENT_CANCELED_STATUS:
|
|
238
|
+
kind = "canceled" # still an SdoGatewayError; not an upstream failure
|
|
239
|
+
envelope = envelope_message(reply.body, api) if reply.body_intact else None
|
|
240
|
+
message = redact_secrets(envelope or _DEFAULT_MESSAGES[kind].format(status=status), token)
|
|
241
|
+
# The cost header is authoritative; the documented table is the fallback.
|
|
242
|
+
cost = request_cost(reply) if api == SYNC else None
|
|
243
|
+
billed = api == SYNC and (status in _BILLED_STATUS if cost is None else cost > 0)
|
|
244
|
+
retryable = status in _RETRYABLE_STATUS
|
|
245
|
+
retry_after = retry_after_seconds(reply.header("retry-after"))
|
|
246
|
+
text = reply.body.decode("utf-8", errors="replace")
|
|
247
|
+
body = redact_secrets(text, token)[:_RETAINED_BODY_CHARS]
|
|
248
|
+
if cls is SdoConcurrencyError:
|
|
249
|
+
return SdoConcurrencyError(
|
|
250
|
+
message,
|
|
251
|
+
status_code=status,
|
|
252
|
+
billed=billed,
|
|
253
|
+
is_retryable=retryable,
|
|
254
|
+
retry_after=retry_after,
|
|
255
|
+
body=body,
|
|
256
|
+
remaining_credits=_int_or_none(reply.header("scrape.do-remaining-credits")),
|
|
257
|
+
cost=cost,
|
|
258
|
+
)
|
|
259
|
+
return cls(
|
|
260
|
+
message,
|
|
261
|
+
status_code=status,
|
|
262
|
+
billed=billed,
|
|
263
|
+
is_retryable=retryable,
|
|
264
|
+
retry_after=retry_after,
|
|
265
|
+
body=body,
|
|
266
|
+
cost=cost,
|
|
267
|
+
)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def is_result(reply: Reply, *, api: str, transparent: bool) -> bool:
|
|
271
|
+
"""Whether a response is a delivered result rather than an error."""
|
|
272
|
+
status = reply.status_code
|
|
273
|
+
if api == ASYNC:
|
|
274
|
+
return 200 <= status < 300
|
|
275
|
+
if transparent and reply.header(_TARGET_ORIGIN_HEADER) is not None:
|
|
276
|
+
return True # the target's own answer, whatever its status
|
|
277
|
+
if target_answered(reply):
|
|
278
|
+
return True
|
|
279
|
+
return status < 400 or status in (404, 410)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def request_cost(reply: Reply) -> int | None:
|
|
283
|
+
"""``Scrape.do-Request-Cost`` read as :attr:`ScrapeResult.cost` reads it:
|
|
284
|
+
exactly one value, ASCII digits, at most 2**53 - 1; else ``None``."""
|
|
285
|
+
values = [v for k, v in reply.headers if k.lower() == _REQUEST_COST_HEADER]
|
|
286
|
+
return parse_request_cost(values[0]) if len(values) == 1 else None
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def target_answered(reply: Reply) -> bool:
|
|
290
|
+
"""Whether the gateway billed the response, which makes it the target's
|
|
291
|
+
answer whatever its status.
|
|
292
|
+
|
|
293
|
+
The gateway charges the target statuses it counts as delivered (400, 401,
|
|
294
|
+
404, 405 and more) and states the charge in ``Scrape.do-Request-Cost``; its
|
|
295
|
+
own errors — a parameter it cannot parse (``render=maybe``), a refused token,
|
|
296
|
+
a blocked URL — are written before any cost is known and carry no cost
|
|
297
|
+
header, and a failed fetch carries a cost of 0. A billed response is a
|
|
298
|
+
result, so it is never retried. Without a readable positive cost the status
|
|
299
|
+
keeps its gateway meaning, so a gateway message is never passed off as the
|
|
300
|
+
target's page.
|
|
301
|
+
"""
|
|
302
|
+
cost = request_cost(reply)
|
|
303
|
+
return cost is not None and cost > 0
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def merge_headers(*layers: Mapping[str, str] | None) -> dict[str, str]:
|
|
307
|
+
"""Validate header layers and merge them case-insensitively: a later
|
|
308
|
+
layer's value replaces an earlier one's (never duplicated or joined)."""
|
|
309
|
+
merged: dict[str, tuple[str, str]] = {}
|
|
310
|
+
for layer in layers:
|
|
311
|
+
if layer is None:
|
|
312
|
+
continue
|
|
313
|
+
if not isinstance(layer, Mapping):
|
|
314
|
+
raise SdoConfigError("headers must be a mapping of header names to str values.")
|
|
315
|
+
for name, value in layer.items():
|
|
316
|
+
if not isinstance(name, str) or not _HEADER_NAME_RE.fullmatch(name):
|
|
317
|
+
raise SdoConfigError(f"Invalid header name {name!r}: it must be an HTTP token.")
|
|
318
|
+
if name.lower() in _FORBIDDEN_HEADERS:
|
|
319
|
+
raise SdoConfigError(
|
|
320
|
+
f"The {name!r} header cannot be set: the HTTP client manages it."
|
|
321
|
+
)
|
|
322
|
+
if not isinstance(value, str) or not _HEADER_VALUE_RE.fullmatch(value):
|
|
323
|
+
raise SdoConfigError(
|
|
324
|
+
f"Invalid value for header {name!r}: it must be a str of printable ASCII "
|
|
325
|
+
"(no CR, LF, NUL or other control characters)."
|
|
326
|
+
)
|
|
327
|
+
merged[name.lower()] = (name, value)
|
|
328
|
+
return dict(merged.values())
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def _wire_headers(built: httpx.Headers, ours: Mapping[str, str]) -> httpx.Headers:
|
|
332
|
+
"""Only the framing headers httpx computed, plus ours: its defaults
|
|
333
|
+
(Accept, Connection, its own User-Agent) are not sent."""
|
|
334
|
+
framing = [(k, v) for k, v in built.raw if k.lower().decode() in _FRAMING_HEADERS]
|
|
335
|
+
return httpx.Headers([*framing, *((k.encode(), v.encode()) for k, v in ours.items())])
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _clear_frames_below(exc: BaseException) -> None:
|
|
339
|
+
"""Clear every finished frame below the current one in ``exc``'s
|
|
340
|
+
traceback (and in its chained exceptions'), keeping the type, message and
|
|
341
|
+
the traceback's shape; a frame still executing is skipped."""
|
|
342
|
+
seen: set[int] = set()
|
|
343
|
+
current: BaseException | None = exc
|
|
344
|
+
while current is not None and id(current) not in seen:
|
|
345
|
+
seen.add(id(current))
|
|
346
|
+
tb = current.__traceback__
|
|
347
|
+
traceback.clear_frames(tb.tb_next if tb is not None and current is exc else tb)
|
|
348
|
+
current = current.__cause__ or current.__context__
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
@dataclass
|
|
352
|
+
class _Failure:
|
|
353
|
+
"""An attempt that produced no usable response."""
|
|
354
|
+
|
|
355
|
+
error: SdoError
|
|
356
|
+
#: True when the request provably never left the client (DNS, connect,
|
|
357
|
+
#: TLS, connect timeout): only then is it safe to send it again.
|
|
358
|
+
unsent: bool = False
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
class _Stop(Exception):
|
|
362
|
+
"""Carries a :class:`_Failure` out of the body read."""
|
|
363
|
+
|
|
364
|
+
def __init__(self, failure: _Failure) -> None:
|
|
365
|
+
super().__init__(failure.error.message)
|
|
366
|
+
self.failure = failure
|
|
367
|
+
|
|
368
|
+
|
|
369
|
+
class Transport:
|
|
370
|
+
"""Owns the httpx client and the request / retry / error-mapping loop.
|
|
371
|
+
|
|
372
|
+
The token is held as a :class:`~sdo._secret.Secret` and added to the request
|
|
373
|
+
(query or ``X-Token``) only inside :meth:`_attempt`, which has returned by
|
|
374
|
+
the time :meth:`send` raises.
|
|
375
|
+
"""
|
|
376
|
+
|
|
377
|
+
def __init__(
|
|
378
|
+
self,
|
|
379
|
+
*,
|
|
380
|
+
token: Secret,
|
|
381
|
+
timeout: float,
|
|
382
|
+
accept_encoding: str,
|
|
383
|
+
retries: int,
|
|
384
|
+
backoff_factor: float,
|
|
385
|
+
user_agent: str,
|
|
386
|
+
max_response_bytes: int = _MAX_BYTES,
|
|
387
|
+
http_client: httpx.Client | None = None,
|
|
388
|
+
) -> None:
|
|
389
|
+
self._token = token
|
|
390
|
+
self._timeout = timeout
|
|
391
|
+
self._accept_encoding = accept_encoding
|
|
392
|
+
self._retries = max(0, retries)
|
|
393
|
+
self._backoff = backoff_factor
|
|
394
|
+
self._ua = user_agent
|
|
395
|
+
self._max_bytes = max_response_bytes
|
|
396
|
+
self._owns_client = http_client is None
|
|
397
|
+
# No base_url: every request carries an absolute URL, so an injected
|
|
398
|
+
# client cannot re-route the sync or the async API. The default client
|
|
399
|
+
# ignores the environment (HTTPS_PROXY, ALL_PROXY, SSL_CERT_FILE,
|
|
400
|
+
# .netrc): a variable planted there would otherwise route the token
|
|
401
|
+
# through a proxy or trust a foreign CA. An injected client keeps its
|
|
402
|
+
# own settings, which is how a proxy is configured.
|
|
403
|
+
self._client = http_client or httpx.Client(follow_redirects=False, trust_env=False)
|
|
404
|
+
self._closed = False
|
|
405
|
+
_logging.register_token(token.reveal())
|
|
406
|
+
# Unregister on close(), or when the transport is garbage-collected.
|
|
407
|
+
self._forget_token = weakref.finalize(self, _logging.unregister_token, token.reveal())
|
|
408
|
+
|
|
409
|
+
# -- public ----------------------------------------------------------------
|
|
410
|
+
def send(
|
|
411
|
+
self,
|
|
412
|
+
method: str,
|
|
413
|
+
url: str,
|
|
414
|
+
*,
|
|
415
|
+
query: str = "",
|
|
416
|
+
auth: str = QUERY,
|
|
417
|
+
headers: Mapping[str, str] | None = None,
|
|
418
|
+
content: bytes | None = None,
|
|
419
|
+
api: str = SYNC,
|
|
420
|
+
transparent: bool = False,
|
|
421
|
+
timeout: float | None = None,
|
|
422
|
+
gateway_timeout_ms: int | None = None,
|
|
423
|
+
retry_status: bool = True,
|
|
424
|
+
) -> Reply:
|
|
425
|
+
"""Send a request through the retry loop and return the complete reply,
|
|
426
|
+
or raise the mapped :class:`~sdo.errors.SdoError`.
|
|
427
|
+
|
|
428
|
+
``url`` is the absolute URL without a query; ``query`` is the encoded
|
|
429
|
+
query string without the token, which ``auth`` places. Without
|
|
430
|
+
``timeout``, an attempt waits the client's timeout, raised to
|
|
431
|
+
``gateway_timeout_ms`` (a plugin's ``timeout`` param) plus the margin.
|
|
432
|
+
``retry_status=False`` keeps a retryable status from being retried (a
|
|
433
|
+
failure before anything was sent still is): for ``/info``, whose 429 is
|
|
434
|
+
a per-minute rate limit a quick retry only extends.
|
|
435
|
+
"""
|
|
436
|
+
if self._closed or self._client.is_closed:
|
|
437
|
+
raise SdoConfigError(
|
|
438
|
+
"This client is closed. Create a new Client, or use it inside its `with` block."
|
|
439
|
+
)
|
|
440
|
+
sdk_headers = {"User-Agent": self._ua, "Accept-Encoding": self._accept_encoding}
|
|
441
|
+
req_headers = merge_headers(sdk_headers, headers)
|
|
442
|
+
idempotent = method not in _NON_IDEMPOTENT
|
|
443
|
+
per_attempt = (
|
|
444
|
+
effective_timeout(self._timeout, gateway_timeout_ms) if timeout is None else timeout
|
|
445
|
+
)
|
|
446
|
+
|
|
447
|
+
attempt = 0
|
|
448
|
+
while True:
|
|
449
|
+
outcome = self._attempt(
|
|
450
|
+
method, url, query, auth, req_headers, content, api, transparent, per_attempt
|
|
451
|
+
)
|
|
452
|
+
can_retry = attempt < self._retries
|
|
453
|
+
if isinstance(outcome, _Failure):
|
|
454
|
+
if can_retry and outcome.unsent and idempotent:
|
|
455
|
+
self._sleep(attempt, None)
|
|
456
|
+
attempt += 1
|
|
457
|
+
continue
|
|
458
|
+
raise outcome.error
|
|
459
|
+
if is_result(outcome, api=api, transparent=transparent):
|
|
460
|
+
return outcome
|
|
461
|
+
err = build_api_error(outcome, api=api, token=self._token.reveal())
|
|
462
|
+
allowed = _RETRYABLE_STATUS if idempotent else _NON_IDEMPOTENT_RETRY_STATUS
|
|
463
|
+
if can_retry and retry_status and err.status_code in allowed:
|
|
464
|
+
self._sleep(attempt, err.retry_after)
|
|
465
|
+
attempt += 1
|
|
466
|
+
continue
|
|
467
|
+
raise err
|
|
468
|
+
|
|
469
|
+
def close(self) -> None:
|
|
470
|
+
if not self._closed:
|
|
471
|
+
self._closed = True
|
|
472
|
+
self._forget_token()
|
|
473
|
+
if self._owns_client:
|
|
474
|
+
self._client.close()
|
|
475
|
+
|
|
476
|
+
# -- internals -------------------------------------------------------------
|
|
477
|
+
def _redact(self, text: str) -> str:
|
|
478
|
+
return redact_secrets(text, self._token.reveal())
|
|
479
|
+
|
|
480
|
+
def _authorize(
|
|
481
|
+
self, url: str, query: str, auth: str, headers: dict[str, str]
|
|
482
|
+
) -> tuple[str, dict[str, str]]:
|
|
483
|
+
"""The URL and headers with the token in place."""
|
|
484
|
+
if auth == QUERY:
|
|
485
|
+
token_pair = "token=" + pct(self._token.reveal())
|
|
486
|
+
return f"{url}?{token_pair}&{query}" if query else f"{url}?{token_pair}", headers
|
|
487
|
+
target = f"{url}?{query}" if query else url
|
|
488
|
+
return target, {**headers, "X-Token": self._token.reveal()}
|
|
489
|
+
|
|
490
|
+
def _attempt(
|
|
491
|
+
self,
|
|
492
|
+
method: str,
|
|
493
|
+
url: str,
|
|
494
|
+
query: str,
|
|
495
|
+
auth: str,
|
|
496
|
+
headers: dict[str, str],
|
|
497
|
+
content: bytes | None,
|
|
498
|
+
api: str,
|
|
499
|
+
transparent: bool,
|
|
500
|
+
timeout: float,
|
|
501
|
+
) -> Reply | _Failure:
|
|
502
|
+
"""One request. Failures are returned, not raised, so the SdoError the
|
|
503
|
+
caller finally raises has no ``__context__`` pointing at an httpx
|
|
504
|
+
exception whose text or ``.request`` holds the token, and this frame
|
|
505
|
+
(whose locals do) is gone by then."""
|
|
506
|
+
deadline = time.monotonic() + timeout
|
|
507
|
+
target, wire_headers = self._authorize(url, query, auth, headers)
|
|
508
|
+
request: httpx.Request | None = None
|
|
509
|
+
try:
|
|
510
|
+
request = self._client.build_request(method, target, content=content, timeout=timeout)
|
|
511
|
+
request.headers = _wire_headers(request.headers, wire_headers)
|
|
512
|
+
response = self._client.send(request, stream=True, follow_redirects=False)
|
|
513
|
+
try:
|
|
514
|
+
reply = Reply(response.status_code, response.headers.multi_items())
|
|
515
|
+
if is_result(reply, api=api, transparent=transparent):
|
|
516
|
+
reply.body = self._read(response, deadline, timeout)
|
|
517
|
+
else:
|
|
518
|
+
reply.body, reply.body_intact = self._read_error_body(response, deadline)
|
|
519
|
+
finally:
|
|
520
|
+
response.close()
|
|
521
|
+
except httpx.TimeoutException as exc:
|
|
522
|
+
return _Failure(
|
|
523
|
+
SdoTimeoutError(
|
|
524
|
+
self._redact(
|
|
525
|
+
f"Request timed out (client timeout {timeout:g}s; "
|
|
526
|
+
f"{type(exc).__name__}: {exc})."
|
|
527
|
+
)
|
|
528
|
+
),
|
|
529
|
+
unsent=isinstance(exc, (httpx.ConnectTimeout, httpx.PoolTimeout)),
|
|
530
|
+
)
|
|
531
|
+
except httpx.TransportError as exc:
|
|
532
|
+
return _Failure(
|
|
533
|
+
SdoConnectionError(
|
|
534
|
+
self._redact(f"Connection failed ({type(exc).__name__}: {exc}).")
|
|
535
|
+
),
|
|
536
|
+
unsent=isinstance(exc, httpx.ConnectError),
|
|
537
|
+
)
|
|
538
|
+
except _Stop as stop:
|
|
539
|
+
return stop.failure
|
|
540
|
+
except Exception as exc: # anything else, e.g. from an injected transport
|
|
541
|
+
return _Failure(
|
|
542
|
+
SdoTransportError(self._redact(f"Request failed ({type(exc).__name__}: {exc})."))
|
|
543
|
+
)
|
|
544
|
+
except BaseException as exc:
|
|
545
|
+
# KeyboardInterrupt and the like propagate through this frame: drop
|
|
546
|
+
# the credential-bearing locals so their traceback does not hold them,
|
|
547
|
+
# and clear the finished frames below it (httpx's, an injected
|
|
548
|
+
# transport's), whose locals hold the request and its token URL.
|
|
549
|
+
del target, wire_headers, request
|
|
550
|
+
_clear_frames_below(exc)
|
|
551
|
+
raise
|
|
552
|
+
return reply
|
|
553
|
+
|
|
554
|
+
def _read(self, response: httpx.Response, deadline: float, timeout: float) -> bytes:
|
|
555
|
+
"""Read a result body and decode it, enforcing the size cap on the raw
|
|
556
|
+
bytes and on every decoding step, and the total deadline per chunk."""
|
|
557
|
+
decoder = BodyDecoder(response.headers.get("content-encoding"), self._max_bytes)
|
|
558
|
+
raw = 0
|
|
559
|
+
try:
|
|
560
|
+
if response.is_stream_consumed:
|
|
561
|
+
# A custom transport (e.g. httpx.MockTransport) returned a
|
|
562
|
+
# response it had already read and decoded: only the cap is left.
|
|
563
|
+
if len(response.content) > self._max_bytes:
|
|
564
|
+
raise TooLarge
|
|
565
|
+
return response.content
|
|
566
|
+
self._check_deadline(deadline, timeout) # connect + headers
|
|
567
|
+
for chunk in response.iter_raw():
|
|
568
|
+
raw += len(chunk)
|
|
569
|
+
if raw > self._max_bytes:
|
|
570
|
+
raise TooLarge
|
|
571
|
+
decoder.feed(chunk)
|
|
572
|
+
self._check_deadline(deadline, timeout)
|
|
573
|
+
return decoder.finish()
|
|
574
|
+
except TooLarge:
|
|
575
|
+
err: SdoError = SdoResponseTooLargeError(
|
|
576
|
+
f"Response exceeded the {self._max_bytes}-byte limit; aborted before "
|
|
577
|
+
"decompressing it fully. Raise max_response_bytes if you expect payloads "
|
|
578
|
+
"this large.",
|
|
579
|
+
limit=self._max_bytes,
|
|
580
|
+
)
|
|
581
|
+
except DecodeError as exc:
|
|
582
|
+
err = SdoTransportError(
|
|
583
|
+
self._redact(f"The response body could not be decompressed ({exc}).")
|
|
584
|
+
)
|
|
585
|
+
raise _Stop(_Failure(err)) from None
|
|
586
|
+
|
|
587
|
+
def _read_error_body(self, response: httpx.Response, deadline: float) -> tuple[bytes, bool]:
|
|
588
|
+
"""Read up to 64 KiB of an error body: ``(body, intact)``. The status is
|
|
589
|
+
the answer, so a body that is larger, cannot be decompressed, drops or
|
|
590
|
+
times out mid-read yields what was read, never a different error. A
|
|
591
|
+
larger body is intact (its first 64 KiB are what an envelope is parsed
|
|
592
|
+
from); the other failures are not."""
|
|
593
|
+
limit = min(_ERROR_BODY_BYTES, self._max_bytes)
|
|
594
|
+
if response.is_stream_consumed:
|
|
595
|
+
return response.content[:limit], True
|
|
596
|
+
decoder = BodyDecoder(response.headers.get("content-encoding"), limit)
|
|
597
|
+
raw = 0
|
|
598
|
+
try:
|
|
599
|
+
for chunk in response.iter_raw():
|
|
600
|
+
raw += len(chunk)
|
|
601
|
+
decoder.feed(chunk)
|
|
602
|
+
if raw > self._max_bytes:
|
|
603
|
+
return decoder.partial(), True
|
|
604
|
+
if time.monotonic() > deadline:
|
|
605
|
+
return decoder.partial(), False
|
|
606
|
+
return decoder.finish(), True
|
|
607
|
+
except TooLarge:
|
|
608
|
+
return decoder.partial(), True
|
|
609
|
+
except Exception: # DecodeError, a read error or timeout
|
|
610
|
+
return decoder.partial(), False
|
|
611
|
+
|
|
612
|
+
def _check_deadline(self, deadline: float, timeout: float) -> None:
|
|
613
|
+
# Each read is also bounded by the same timeout (passed to httpx), so a
|
|
614
|
+
# stalled read cannot overshoot by more than one timeout.
|
|
615
|
+
if time.monotonic() > deadline:
|
|
616
|
+
error = SdoTimeoutError(f"Request exceeded the client's total timeout of {timeout:g}s.")
|
|
617
|
+
raise _Stop(_Failure(error))
|
|
618
|
+
|
|
619
|
+
def _sleep(self, attempt: int, retry_after: int | None) -> None:
|
|
620
|
+
if retry_after is not None:
|
|
621
|
+
time.sleep(min(retry_after, _MAX_BACKOFF))
|
|
622
|
+
return
|
|
623
|
+
delay = self._backoff * (2**attempt)
|
|
624
|
+
delay += random.uniform(0, delay * 0.1) # jitter, proportional so 0 stays 0
|
|
625
|
+
time.sleep(min(delay, _MAX_BACKOFF))
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
__all__ = [
|
|
629
|
+
"ASYNC",
|
|
630
|
+
"HEADER",
|
|
631
|
+
"QUERY",
|
|
632
|
+
"SYNC",
|
|
633
|
+
"Reply",
|
|
634
|
+
"Transport",
|
|
635
|
+
"build_api_error",
|
|
636
|
+
"effective_timeout",
|
|
637
|
+
"merge_headers",
|
|
638
|
+
"retry_after_seconds",
|
|
639
|
+
]
|