sdo-client 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
sdo/_transport.py ADDED
@@ -0,0 +1,639 @@
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ """HTTP transport: the one request path for scrape, plugins and async jobs.
3
+
4
+ Every request goes through :meth:`Transport.send`, so every path shares the
5
+ same guarantees:
6
+
7
+ * ``Accept-Encoding`` is explicit and the body is decompressed here, bounded:
8
+ the raw read and every decoding step stop at ``max_response_bytes`` before
9
+ the inflated buffer is materialized (see :mod:`sdo._decompress`);
10
+ * the client timeout is a total wall-clock deadline per attempt over connect,
11
+ headers and the whole body. httpx can only enforce it between reads (each
12
+ read is bounded by the same timeout), so a peer that trickles bytes can
13
+ stretch one attempt to about twice the timeout;
14
+ * redirects are never followed, whatever the injected ``httpx.Client`` says;
15
+ * URLs are absolute and the query string is built by the SDK, byte-exact;
16
+ * the headers sent are exactly the SDK's own plus the caller's, compared
17
+ case-insensitively; httpx's defaults are not sent;
18
+ * every failure is an :class:`~sdo.errors.SdoError` whose message, retained
19
+ body and (absent) cause carry no credential, and no frame that is live when
20
+ it is raised holds the token;
21
+ * retries follow one policy: retryable statuses (only 429 for methods that can
22
+ have side effects on the target), and failures before anything was sent
23
+ (idempotent methods only). A failure after the request was sent is never
24
+ retried: the gateway may still run, and bill, it.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ import random
30
+ import re
31
+ import time
32
+ import traceback
33
+ import weakref
34
+ from collections.abc import Mapping
35
+ from dataclasses import dataclass, field
36
+
37
+ import httpx
38
+
39
+ from . import _json, _logging
40
+ from ._decompress import BodyDecoder, DecodeError, TooLarge
41
+ from ._secret import Secret
42
+ from ._wire import pct
43
+ from .errors import (
44
+ SdoAPIError,
45
+ SdoAuthError,
46
+ SdoConcurrencyError,
47
+ SdoConfigError,
48
+ SdoConnectionError,
49
+ SdoError,
50
+ SdoGatewayError,
51
+ SdoResponseTooLargeError,
52
+ SdoTimeoutError,
53
+ SdoTransportError,
54
+ SdoValidationError,
55
+ redact_secrets,
56
+ )
57
+ from .response import parse_request_cost
58
+
59
+ #: Statuses where the same request may succeed when retried. 510 is not one:
60
+ #: the gateway answers 510 only when the HTTP client cancelled the request
61
+ #: before the gateway finished it, so a retry would re-send an abandoned request.
62
+ _RETRYABLE_STATUS = frozenset({429, 502, 503, 504, 520, 521, 522, 523, 524})
63
+ #: The gateway's status for a request its HTTP client cancelled.
64
+ _CLIENT_CANCELED_STATUS = 510
65
+ #: Methods that can have side effects on the target: retried only on 429, which
66
+ #: the gateway returns before contacting the target.
67
+ _NON_IDEMPOTENT = frozenset({"POST", "PUT", "PATCH", "DELETE"})
68
+ _NON_IDEMPOTENT_RETRY_STATUS = frozenset({429})
69
+ #: The documented billing table, used only for an error without a readable
70
+ #: ``Scrape.do-Request-Cost`` (the header is authoritative): of the statuses
71
+ #: that can be in an error position, only 404 and 410, which are results
72
+ #: anyway. The gateway's own 400, 401, 429, 502 and 510 are not billed.
73
+ _BILLED_STATUS = frozenset({404, 410})
74
+ #: Default ceiling for a response body, raw and decoded (32 MiB).
75
+ _MAX_BYTES = 32 * 1024 * 1024
76
+ #: How much of an error body is read: enough for any envelope.
77
+ _ERROR_BODY_BYTES = 64 * 1024
78
+ _MAX_BACKOFF = 30.0
79
+ _RETAINED_BODY_CHARS = 2048
80
+ #: With a gateway ``timeout`` set, one attempt waits at least that plus this.
81
+ _GATEWAY_TIMEOUT_MARGIN = 10
82
+ #: With transparent_response, this header marks the target's own answer.
83
+ _TARGET_ORIGIN_HEADER = "scrape.do-initial-status-code"
84
+ #: The credits the gateway charged for the request.
85
+ _REQUEST_COST_HEADER = "scrape.do-request-cost"
86
+
87
+ SYNC = "sync"
88
+ ASYNC = "async"
89
+ #: Where the token travels: the ``token`` query parameter, or ``X-Token``.
90
+ QUERY = "query"
91
+ HEADER = "header"
92
+
93
+ _DEFAULT_MESSAGES = {
94
+ "validation": "The gateway rejected the request parameters.",
95
+ "auth": "Authentication failed — check your token, plan permissions, and remaining credits.",
96
+ "concurrency": "Concurrent-request limit reached.",
97
+ "gateway": "Gateway-origin upstream failure (HTTP {status}).",
98
+ "canceled": "The request was canceled by the HTTP client (HTTP {status}).",
99
+ "api": "Request failed (HTTP {status}).",
100
+ "redirect": "Unexpected redirect (HTTP {status}); redirects are never followed.",
101
+ }
102
+
103
+ _DIGITS_RE = re.compile(r"[0-9]+", re.ASCII)
104
+ _SIGNED_RE = re.compile(r"-?[0-9]+", re.ASCII)
105
+ #: RFC 9110 token.
106
+ _HEADER_NAME_RE = re.compile(r"[!#$%&'*+\-.^_`|~0-9A-Za-z]+")
107
+ #: Printable ASCII plus space and tab.
108
+ _HEADER_VALUE_RE = re.compile(r"[\t\x20-\x7e]*")
109
+ #: Managed by the HTTP client; a caller value would corrupt the framing.
110
+ _FORBIDDEN_HEADERS = frozenset({"host", "content-length", "transfer-encoding", "connection"})
111
+ #: Of the headers httpx puts on a built request, only these framing ones stay.
112
+ _FRAMING_HEADERS = frozenset({"host", "content-length", "transfer-encoding"})
113
+
114
+
115
+ @dataclass
116
+ class Reply:
117
+ """A complete response: status, every header as received, decoded body."""
118
+
119
+ status_code: int
120
+ headers: list[tuple[str, str]]
121
+ body: bytes = field(default=b"")
122
+ #: False when an error body could not be read whole (it failed to
123
+ #: decompress, dropped or timed out): its envelope is then not trusted.
124
+ body_intact: bool = True
125
+
126
+ def header(self, name: str) -> str | None:
127
+ name = name.lower()
128
+ for k, v in self.headers:
129
+ if k.lower() == name:
130
+ return v
131
+ return None
132
+
133
+
134
+ def retry_after_seconds(raw: str | None) -> int | None:
135
+ """``Retry-After`` as whole seconds (RFC 9110 delay-seconds), else ``None``.
136
+
137
+ Only ASCII digits count: a date, a negative, a fraction, ``NaN`` or
138
+ ``'١٢'`` fall back to the exponential backoff.
139
+ """
140
+ if raw is None:
141
+ return None
142
+ raw = raw.strip(" \t")
143
+ return int(raw) if _DIGITS_RE.fullmatch(raw) else None
144
+
145
+
146
+ def effective_timeout(client_seconds: float, config_timeout_ms: int | None) -> float:
147
+ """The per-attempt timeout: with a gateway ``timeout`` (milliseconds) set,
148
+ at least that plus a margin, so the client never gives up on a request the
149
+ gateway was told it may take that long for."""
150
+ if not config_timeout_ms:
151
+ return client_seconds
152
+ return max(client_seconds, config_timeout_ms / 1000 + _GATEWAY_TIMEOUT_MARGIN)
153
+
154
+
155
+ def _int_or_none(raw: str | None) -> int | None:
156
+ if raw is None:
157
+ return None
158
+ raw = raw.strip(" \t")
159
+ return int(raw) if _SIGNED_RE.fullmatch(raw) else None
160
+
161
+
162
+ def _ascii_lower(text: str) -> str:
163
+ """Lowercase A-Z only (``str.lower`` would also fold ``"İ"`` or a Kelvin sign)."""
164
+ return text.translate(_ASCII_LOWER)
165
+
166
+
167
+ _ASCII_LOWER = {c: c + 32 for c in range(ord("A"), ord("Z") + 1)}
168
+
169
+
170
+ def _envelope_text(value: object) -> str | None:
171
+ if isinstance(value, list):
172
+ return "; ".join(m for m in value if isinstance(m, str) and m) or None
173
+ if isinstance(value, str):
174
+ return value or None
175
+ return None
176
+
177
+
178
+ def _sync_envelope(data: object) -> str | None:
179
+ """A ``message`` key, else an ``error`` key, each matched in any ASCII
180
+ letter case: the gateway writes ``{"Message": ["...", ...]}``, a plugin
181
+ ``{"error": "code", "message": "text"}``. The value is a string or a list
182
+ (string elements joined with ``"; "``). Several keys matching one name are
183
+ tried in byte order (``"Message"`` before ``"message"``); one that yields
184
+ nothing falls through to the next, then to ``error``."""
185
+ if not isinstance(data, dict):
186
+ return None
187
+ keys = sorted(k for k in data if isinstance(k, str))
188
+ for name in ("message", "error"):
189
+ for key in keys:
190
+ if _ascii_lower(key) == name:
191
+ text = _envelope_text(data[key])
192
+ if text is not None:
193
+ return text
194
+ return None
195
+
196
+
197
+ def _async_envelope(data: object) -> str | None:
198
+ """``{"Error": "...", "Code": n}``: only ``Error`` is read."""
199
+ if isinstance(data, dict):
200
+ error = data.get("Error")
201
+ if isinstance(error, str) and error:
202
+ return error
203
+ return None
204
+
205
+
206
+ def envelope_message(body: bytes, api: str = SYNC) -> str | None:
207
+ """The message an error body carries (a leading BOM is ignored)."""
208
+ try:
209
+ data = _json.loads(body)
210
+ except ValueError:
211
+ return None
212
+ return _async_envelope(data) if api == ASYNC else _sync_envelope(data)
213
+
214
+
215
+ def _classify(status: int) -> tuple[type[SdoAPIError], str]:
216
+ if status in (401, 403):
217
+ # The gateway returns 401 for a depleted credit balance too, so the
218
+ # default message names both causes.
219
+ return SdoAuthError, "auth"
220
+ if status == 429:
221
+ return SdoConcurrencyError, "concurrency"
222
+ if status == 400:
223
+ return SdoValidationError, "validation"
224
+ if status >= 500:
225
+ return SdoGatewayError, "gateway"
226
+ if 300 <= status < 400:
227
+ return SdoAPIError, "redirect"
228
+ return SdoAPIError, "api"
229
+
230
+
231
+ def build_api_error(reply: Reply, *, api: str = SYNC, token: str | None = None) -> SdoAPIError:
232
+ """Map an error response to the taxonomy. The message comes from the error
233
+ envelope when there is one; message and retained body are redacted, then
234
+ the body is truncated."""
235
+ status = reply.status_code
236
+ cls, kind = _classify(status)
237
+ if status == _CLIENT_CANCELED_STATUS:
238
+ kind = "canceled" # still an SdoGatewayError; not an upstream failure
239
+ envelope = envelope_message(reply.body, api) if reply.body_intact else None
240
+ message = redact_secrets(envelope or _DEFAULT_MESSAGES[kind].format(status=status), token)
241
+ # The cost header is authoritative; the documented table is the fallback.
242
+ cost = request_cost(reply) if api == SYNC else None
243
+ billed = api == SYNC and (status in _BILLED_STATUS if cost is None else cost > 0)
244
+ retryable = status in _RETRYABLE_STATUS
245
+ retry_after = retry_after_seconds(reply.header("retry-after"))
246
+ text = reply.body.decode("utf-8", errors="replace")
247
+ body = redact_secrets(text, token)[:_RETAINED_BODY_CHARS]
248
+ if cls is SdoConcurrencyError:
249
+ return SdoConcurrencyError(
250
+ message,
251
+ status_code=status,
252
+ billed=billed,
253
+ is_retryable=retryable,
254
+ retry_after=retry_after,
255
+ body=body,
256
+ remaining_credits=_int_or_none(reply.header("scrape.do-remaining-credits")),
257
+ cost=cost,
258
+ )
259
+ return cls(
260
+ message,
261
+ status_code=status,
262
+ billed=billed,
263
+ is_retryable=retryable,
264
+ retry_after=retry_after,
265
+ body=body,
266
+ cost=cost,
267
+ )
268
+
269
+
270
+ def is_result(reply: Reply, *, api: str, transparent: bool) -> bool:
271
+ """Whether a response is a delivered result rather than an error."""
272
+ status = reply.status_code
273
+ if api == ASYNC:
274
+ return 200 <= status < 300
275
+ if transparent and reply.header(_TARGET_ORIGIN_HEADER) is not None:
276
+ return True # the target's own answer, whatever its status
277
+ if target_answered(reply):
278
+ return True
279
+ return status < 400 or status in (404, 410)
280
+
281
+
282
+ def request_cost(reply: Reply) -> int | None:
283
+ """``Scrape.do-Request-Cost`` read as :attr:`ScrapeResult.cost` reads it:
284
+ exactly one value, ASCII digits, at most 2**53 - 1; else ``None``."""
285
+ values = [v for k, v in reply.headers if k.lower() == _REQUEST_COST_HEADER]
286
+ return parse_request_cost(values[0]) if len(values) == 1 else None
287
+
288
+
289
+ def target_answered(reply: Reply) -> bool:
290
+ """Whether the gateway billed the response, which makes it the target's
291
+ answer whatever its status.
292
+
293
+ The gateway charges the target statuses it counts as delivered (400, 401,
294
+ 404, 405 and more) and states the charge in ``Scrape.do-Request-Cost``; its
295
+ own errors — a parameter it cannot parse (``render=maybe``), a refused token,
296
+ a blocked URL — are written before any cost is known and carry no cost
297
+ header, and a failed fetch carries a cost of 0. A billed response is a
298
+ result, so it is never retried. Without a readable positive cost the status
299
+ keeps its gateway meaning, so a gateway message is never passed off as the
300
+ target's page.
301
+ """
302
+ cost = request_cost(reply)
303
+ return cost is not None and cost > 0
304
+
305
+
306
+ def merge_headers(*layers: Mapping[str, str] | None) -> dict[str, str]:
307
+ """Validate header layers and merge them case-insensitively: a later
308
+ layer's value replaces an earlier one's (never duplicated or joined)."""
309
+ merged: dict[str, tuple[str, str]] = {}
310
+ for layer in layers:
311
+ if layer is None:
312
+ continue
313
+ if not isinstance(layer, Mapping):
314
+ raise SdoConfigError("headers must be a mapping of header names to str values.")
315
+ for name, value in layer.items():
316
+ if not isinstance(name, str) or not _HEADER_NAME_RE.fullmatch(name):
317
+ raise SdoConfigError(f"Invalid header name {name!r}: it must be an HTTP token.")
318
+ if name.lower() in _FORBIDDEN_HEADERS:
319
+ raise SdoConfigError(
320
+ f"The {name!r} header cannot be set: the HTTP client manages it."
321
+ )
322
+ if not isinstance(value, str) or not _HEADER_VALUE_RE.fullmatch(value):
323
+ raise SdoConfigError(
324
+ f"Invalid value for header {name!r}: it must be a str of printable ASCII "
325
+ "(no CR, LF, NUL or other control characters)."
326
+ )
327
+ merged[name.lower()] = (name, value)
328
+ return dict(merged.values())
329
+
330
+
331
+ def _wire_headers(built: httpx.Headers, ours: Mapping[str, str]) -> httpx.Headers:
332
+ """Only the framing headers httpx computed, plus ours: its defaults
333
+ (Accept, Connection, its own User-Agent) are not sent."""
334
+ framing = [(k, v) for k, v in built.raw if k.lower().decode() in _FRAMING_HEADERS]
335
+ return httpx.Headers([*framing, *((k.encode(), v.encode()) for k, v in ours.items())])
336
+
337
+
338
+ def _clear_frames_below(exc: BaseException) -> None:
339
+ """Clear every finished frame below the current one in ``exc``'s
340
+ traceback (and in its chained exceptions'), keeping the type, message and
341
+ the traceback's shape; a frame still executing is skipped."""
342
+ seen: set[int] = set()
343
+ current: BaseException | None = exc
344
+ while current is not None and id(current) not in seen:
345
+ seen.add(id(current))
346
+ tb = current.__traceback__
347
+ traceback.clear_frames(tb.tb_next if tb is not None and current is exc else tb)
348
+ current = current.__cause__ or current.__context__
349
+
350
+
351
+ @dataclass
352
+ class _Failure:
353
+ """An attempt that produced no usable response."""
354
+
355
+ error: SdoError
356
+ #: True when the request provably never left the client (DNS, connect,
357
+ #: TLS, connect timeout): only then is it safe to send it again.
358
+ unsent: bool = False
359
+
360
+
361
+ class _Stop(Exception):
362
+ """Carries a :class:`_Failure` out of the body read."""
363
+
364
+ def __init__(self, failure: _Failure) -> None:
365
+ super().__init__(failure.error.message)
366
+ self.failure = failure
367
+
368
+
369
+ class Transport:
370
+ """Owns the httpx client and the request / retry / error-mapping loop.
371
+
372
+ The token is held as a :class:`~sdo._secret.Secret` and added to the request
373
+ (query or ``X-Token``) only inside :meth:`_attempt`, which has returned by
374
+ the time :meth:`send` raises.
375
+ """
376
+
377
+ def __init__(
378
+ self,
379
+ *,
380
+ token: Secret,
381
+ timeout: float,
382
+ accept_encoding: str,
383
+ retries: int,
384
+ backoff_factor: float,
385
+ user_agent: str,
386
+ max_response_bytes: int = _MAX_BYTES,
387
+ http_client: httpx.Client | None = None,
388
+ ) -> None:
389
+ self._token = token
390
+ self._timeout = timeout
391
+ self._accept_encoding = accept_encoding
392
+ self._retries = max(0, retries)
393
+ self._backoff = backoff_factor
394
+ self._ua = user_agent
395
+ self._max_bytes = max_response_bytes
396
+ self._owns_client = http_client is None
397
+ # No base_url: every request carries an absolute URL, so an injected
398
+ # client cannot re-route the sync or the async API. The default client
399
+ # ignores the environment (HTTPS_PROXY, ALL_PROXY, SSL_CERT_FILE,
400
+ # .netrc): a variable planted there would otherwise route the token
401
+ # through a proxy or trust a foreign CA. An injected client keeps its
402
+ # own settings, which is how a proxy is configured.
403
+ self._client = http_client or httpx.Client(follow_redirects=False, trust_env=False)
404
+ self._closed = False
405
+ _logging.register_token(token.reveal())
406
+ # Unregister on close(), or when the transport is garbage-collected.
407
+ self._forget_token = weakref.finalize(self, _logging.unregister_token, token.reveal())
408
+
409
+ # -- public ----------------------------------------------------------------
410
+ def send(
411
+ self,
412
+ method: str,
413
+ url: str,
414
+ *,
415
+ query: str = "",
416
+ auth: str = QUERY,
417
+ headers: Mapping[str, str] | None = None,
418
+ content: bytes | None = None,
419
+ api: str = SYNC,
420
+ transparent: bool = False,
421
+ timeout: float | None = None,
422
+ gateway_timeout_ms: int | None = None,
423
+ retry_status: bool = True,
424
+ ) -> Reply:
425
+ """Send a request through the retry loop and return the complete reply,
426
+ or raise the mapped :class:`~sdo.errors.SdoError`.
427
+
428
+ ``url`` is the absolute URL without a query; ``query`` is the encoded
429
+ query string without the token, which ``auth`` places. Without
430
+ ``timeout``, an attempt waits the client's timeout, raised to
431
+ ``gateway_timeout_ms`` (a plugin's ``timeout`` param) plus the margin.
432
+ ``retry_status=False`` keeps a retryable status from being retried (a
433
+ failure before anything was sent still is): for ``/info``, whose 429 is
434
+ a per-minute rate limit a quick retry only extends.
435
+ """
436
+ if self._closed or self._client.is_closed:
437
+ raise SdoConfigError(
438
+ "This client is closed. Create a new Client, or use it inside its `with` block."
439
+ )
440
+ sdk_headers = {"User-Agent": self._ua, "Accept-Encoding": self._accept_encoding}
441
+ req_headers = merge_headers(sdk_headers, headers)
442
+ idempotent = method not in _NON_IDEMPOTENT
443
+ per_attempt = (
444
+ effective_timeout(self._timeout, gateway_timeout_ms) if timeout is None else timeout
445
+ )
446
+
447
+ attempt = 0
448
+ while True:
449
+ outcome = self._attempt(
450
+ method, url, query, auth, req_headers, content, api, transparent, per_attempt
451
+ )
452
+ can_retry = attempt < self._retries
453
+ if isinstance(outcome, _Failure):
454
+ if can_retry and outcome.unsent and idempotent:
455
+ self._sleep(attempt, None)
456
+ attempt += 1
457
+ continue
458
+ raise outcome.error
459
+ if is_result(outcome, api=api, transparent=transparent):
460
+ return outcome
461
+ err = build_api_error(outcome, api=api, token=self._token.reveal())
462
+ allowed = _RETRYABLE_STATUS if idempotent else _NON_IDEMPOTENT_RETRY_STATUS
463
+ if can_retry and retry_status and err.status_code in allowed:
464
+ self._sleep(attempt, err.retry_after)
465
+ attempt += 1
466
+ continue
467
+ raise err
468
+
469
+ def close(self) -> None:
470
+ if not self._closed:
471
+ self._closed = True
472
+ self._forget_token()
473
+ if self._owns_client:
474
+ self._client.close()
475
+
476
+ # -- internals -------------------------------------------------------------
477
+ def _redact(self, text: str) -> str:
478
+ return redact_secrets(text, self._token.reveal())
479
+
480
+ def _authorize(
481
+ self, url: str, query: str, auth: str, headers: dict[str, str]
482
+ ) -> tuple[str, dict[str, str]]:
483
+ """The URL and headers with the token in place."""
484
+ if auth == QUERY:
485
+ token_pair = "token=" + pct(self._token.reveal())
486
+ return f"{url}?{token_pair}&{query}" if query else f"{url}?{token_pair}", headers
487
+ target = f"{url}?{query}" if query else url
488
+ return target, {**headers, "X-Token": self._token.reveal()}
489
+
490
+ def _attempt(
491
+ self,
492
+ method: str,
493
+ url: str,
494
+ query: str,
495
+ auth: str,
496
+ headers: dict[str, str],
497
+ content: bytes | None,
498
+ api: str,
499
+ transparent: bool,
500
+ timeout: float,
501
+ ) -> Reply | _Failure:
502
+ """One request. Failures are returned, not raised, so the SdoError the
503
+ caller finally raises has no ``__context__`` pointing at an httpx
504
+ exception whose text or ``.request`` holds the token, and this frame
505
+ (whose locals do) is gone by then."""
506
+ deadline = time.monotonic() + timeout
507
+ target, wire_headers = self._authorize(url, query, auth, headers)
508
+ request: httpx.Request | None = None
509
+ try:
510
+ request = self._client.build_request(method, target, content=content, timeout=timeout)
511
+ request.headers = _wire_headers(request.headers, wire_headers)
512
+ response = self._client.send(request, stream=True, follow_redirects=False)
513
+ try:
514
+ reply = Reply(response.status_code, response.headers.multi_items())
515
+ if is_result(reply, api=api, transparent=transparent):
516
+ reply.body = self._read(response, deadline, timeout)
517
+ else:
518
+ reply.body, reply.body_intact = self._read_error_body(response, deadline)
519
+ finally:
520
+ response.close()
521
+ except httpx.TimeoutException as exc:
522
+ return _Failure(
523
+ SdoTimeoutError(
524
+ self._redact(
525
+ f"Request timed out (client timeout {timeout:g}s; "
526
+ f"{type(exc).__name__}: {exc})."
527
+ )
528
+ ),
529
+ unsent=isinstance(exc, (httpx.ConnectTimeout, httpx.PoolTimeout)),
530
+ )
531
+ except httpx.TransportError as exc:
532
+ return _Failure(
533
+ SdoConnectionError(
534
+ self._redact(f"Connection failed ({type(exc).__name__}: {exc}).")
535
+ ),
536
+ unsent=isinstance(exc, httpx.ConnectError),
537
+ )
538
+ except _Stop as stop:
539
+ return stop.failure
540
+ except Exception as exc: # anything else, e.g. from an injected transport
541
+ return _Failure(
542
+ SdoTransportError(self._redact(f"Request failed ({type(exc).__name__}: {exc})."))
543
+ )
544
+ except BaseException as exc:
545
+ # KeyboardInterrupt and the like propagate through this frame: drop
546
+ # the credential-bearing locals so their traceback does not hold them,
547
+ # and clear the finished frames below it (httpx's, an injected
548
+ # transport's), whose locals hold the request and its token URL.
549
+ del target, wire_headers, request
550
+ _clear_frames_below(exc)
551
+ raise
552
+ return reply
553
+
554
+ def _read(self, response: httpx.Response, deadline: float, timeout: float) -> bytes:
555
+ """Read a result body and decode it, enforcing the size cap on the raw
556
+ bytes and on every decoding step, and the total deadline per chunk."""
557
+ decoder = BodyDecoder(response.headers.get("content-encoding"), self._max_bytes)
558
+ raw = 0
559
+ try:
560
+ if response.is_stream_consumed:
561
+ # A custom transport (e.g. httpx.MockTransport) returned a
562
+ # response it had already read and decoded: only the cap is left.
563
+ if len(response.content) > self._max_bytes:
564
+ raise TooLarge
565
+ return response.content
566
+ self._check_deadline(deadline, timeout) # connect + headers
567
+ for chunk in response.iter_raw():
568
+ raw += len(chunk)
569
+ if raw > self._max_bytes:
570
+ raise TooLarge
571
+ decoder.feed(chunk)
572
+ self._check_deadline(deadline, timeout)
573
+ return decoder.finish()
574
+ except TooLarge:
575
+ err: SdoError = SdoResponseTooLargeError(
576
+ f"Response exceeded the {self._max_bytes}-byte limit; aborted before "
577
+ "decompressing it fully. Raise max_response_bytes if you expect payloads "
578
+ "this large.",
579
+ limit=self._max_bytes,
580
+ )
581
+ except DecodeError as exc:
582
+ err = SdoTransportError(
583
+ self._redact(f"The response body could not be decompressed ({exc}).")
584
+ )
585
+ raise _Stop(_Failure(err)) from None
586
+
587
+ def _read_error_body(self, response: httpx.Response, deadline: float) -> tuple[bytes, bool]:
588
+ """Read up to 64 KiB of an error body: ``(body, intact)``. The status is
589
+ the answer, so a body that is larger, cannot be decompressed, drops or
590
+ times out mid-read yields what was read, never a different error. A
591
+ larger body is intact (its first 64 KiB are what an envelope is parsed
592
+ from); the other failures are not."""
593
+ limit = min(_ERROR_BODY_BYTES, self._max_bytes)
594
+ if response.is_stream_consumed:
595
+ return response.content[:limit], True
596
+ decoder = BodyDecoder(response.headers.get("content-encoding"), limit)
597
+ raw = 0
598
+ try:
599
+ for chunk in response.iter_raw():
600
+ raw += len(chunk)
601
+ decoder.feed(chunk)
602
+ if raw > self._max_bytes:
603
+ return decoder.partial(), True
604
+ if time.monotonic() > deadline:
605
+ return decoder.partial(), False
606
+ return decoder.finish(), True
607
+ except TooLarge:
608
+ return decoder.partial(), True
609
+ except Exception: # DecodeError, a read error or timeout
610
+ return decoder.partial(), False
611
+
612
+ def _check_deadline(self, deadline: float, timeout: float) -> None:
613
+ # Each read is also bounded by the same timeout (passed to httpx), so a
614
+ # stalled read cannot overshoot by more than one timeout.
615
+ if time.monotonic() > deadline:
616
+ error = SdoTimeoutError(f"Request exceeded the client's total timeout of {timeout:g}s.")
617
+ raise _Stop(_Failure(error))
618
+
619
+ def _sleep(self, attempt: int, retry_after: int | None) -> None:
620
+ if retry_after is not None:
621
+ time.sleep(min(retry_after, _MAX_BACKOFF))
622
+ return
623
+ delay = self._backoff * (2**attempt)
624
+ delay += random.uniform(0, delay * 0.1) # jitter, proportional so 0 stays 0
625
+ time.sleep(min(delay, _MAX_BACKOFF))
626
+
627
+
628
+ __all__ = [
629
+ "ASYNC",
630
+ "HEADER",
631
+ "QUERY",
632
+ "SYNC",
633
+ "Reply",
634
+ "Transport",
635
+ "build_api_error",
636
+ "effective_timeout",
637
+ "merge_headers",
638
+ "retry_after_seconds",
639
+ ]