parse-sdk 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- parse_sdk/__init__.py +57 -0
- parse_sdk/_labels.py +51 -0
- parse_sdk/_oauth.py +301 -0
- parse_sdk/_project.py +184 -0
- parse_sdk/_runtime.py +2142 -0
- parse_sdk/_sanitize.py +15 -0
- parse_sdk/_sync.py +887 -0
- parse_sdk/checks.py +601 -0
- parse_sdk/cli.py +1667 -0
- parse_sdk/cli_help.py +110 -0
- parse_sdk/codegen_reconcile.py +157 -0
- parse_sdk/codegen_v2.py +3361 -0
- parse_sdk/config.py +349 -0
- parse_sdk/docgen.py +587 -0
- parse_sdk/doctor.py +348 -0
- parse_sdk/migrate.py +212 -0
- parse_sdk/preview.py +588 -0
- parse_sdk/py.typed +0 -0
- parse_sdk/resource_surface.py +196 -0
- parse_sdk/sample_gate.py +245 -0
- parse_sdk/scaffold.py +273 -0
- parse_sdk-0.1.0.dist-info/METADATA +177 -0
- parse_sdk-0.1.0.dist-info/RECORD +26 -0
- parse_sdk-0.1.0.dist-info/WHEEL +4 -0
- parse_sdk-0.1.0.dist-info/entry_points.txt +2 -0
- parse_sdk-0.1.0.dist-info/licenses/LICENSE +21 -0
parse_sdk/_runtime.py
ADDED
|
@@ -0,0 +1,2142 @@
|
|
|
1
|
+
"""Runtime base classes for generated per-API modules.
|
|
2
|
+
|
|
3
|
+
Generated code (``parse_apis/<slug>/__init__.py``) imports from here.
|
|
4
|
+
Anything user-facing lives in ``parse_sdk.__init__`` / ``parse_sdk.errors``
|
|
5
|
+
so users get clean import paths; this file is the implementation surface.
|
|
6
|
+
|
|
7
|
+
The design is "the SDK module's classes are real Python classes, the
|
|
8
|
+
runtime base lives here." A generated class for a Reddit Post looks like:
|
|
9
|
+
|
|
10
|
+
@resource(name="Post", keyed_by="id")
|
|
11
|
+
class Post(Resource):
|
|
12
|
+
id: str
|
|
13
|
+
title: str
|
|
14
|
+
...
|
|
15
|
+
def refresh(self) -> Post: ...
|
|
16
|
+
|
|
17
|
+
with the methods produced by codegen calling into ``BaseAPI._request``.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
import os
|
|
24
|
+
import random
|
|
25
|
+
import time
|
|
26
|
+
import types
|
|
27
|
+
import urllib.parse
|
|
28
|
+
from collections import deque
|
|
29
|
+
from dataclasses import dataclass
|
|
30
|
+
from datetime import datetime, timezone
|
|
31
|
+
from decimal import Decimal, InvalidOperation
|
|
32
|
+
from enum import Enum
|
|
33
|
+
from typing import (
|
|
34
|
+
Any,
|
|
35
|
+
Callable,
|
|
36
|
+
ClassVar,
|
|
37
|
+
Dict,
|
|
38
|
+
Generic,
|
|
39
|
+
Iterable,
|
|
40
|
+
Iterator,
|
|
41
|
+
List,
|
|
42
|
+
Optional,
|
|
43
|
+
TYPE_CHECKING,
|
|
44
|
+
Type,
|
|
45
|
+
TypeVar,
|
|
46
|
+
Union,
|
|
47
|
+
get_args,
|
|
48
|
+
get_origin,
|
|
49
|
+
get_type_hints,
|
|
50
|
+
)
|
|
51
|
+
|
|
52
|
+
from parse_sdk._labels import split_top, strip_wrapper
|
|
53
|
+
from parse_sdk._sanitize import CONTROL_BYTES as _EXCERPT_UNSAFE
|
|
54
|
+
from parse_sdk.config import CANONICAL_HOST as _CANONICAL_HOST
|
|
55
|
+
from parse_sdk.config import _host_is_trusted
|
|
56
|
+
from parse_sdk.config import resolve as _resolve_config
|
|
57
|
+
|
|
58
|
+
# Identifies SDK-originated calls on the wire so the backend can attribute usage
|
|
59
|
+
# (e.g. PostHog) to the typed SDK + its version. Computed lazily + cached: the
|
|
60
|
+
# version literal lives in ``parse_sdk.__init__``, which imports THIS module, so a
|
|
61
|
+
# top-level import would cycle — by request time the package is fully loaded.
|
|
62
|
+
_CLIENT_HEADER: Optional[str] = None
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _client_header() -> str:
|
|
66
|
+
global _CLIENT_HEADER
|
|
67
|
+
if _CLIENT_HEADER is None:
|
|
68
|
+
try:
|
|
69
|
+
from parse_sdk import __version__ as _v
|
|
70
|
+
except Exception: # pragma: no cover — never block a request on this
|
|
71
|
+
_v = "unknown"
|
|
72
|
+
_CLIENT_HEADER = f"parse-sdk-python/{_v}"
|
|
73
|
+
return _CLIENT_HEADER
|
|
74
|
+
|
|
75
|
+
if TYPE_CHECKING: # avoid runtime import cycle
|
|
76
|
+
# httpx is imported lazily (function-local in BaseAPI.__init__/_request):
|
|
77
|
+
# importing it costs ~32-35 ms, and `import parse_sdk` pulls this module,
|
|
78
|
+
# so every `parse` CLI invocation paid that before any transport ran (see
|
|
79
|
+
# plan 003). Kept here so the httpx annotations below resolve statically.
|
|
80
|
+
import httpx
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
# ---------------------------------------------------------------------------
|
|
84
|
+
# Request metadata (cost / quota / rate-limit control loop)
|
|
85
|
+
#
|
|
86
|
+
# The executor emits X-Credits-* and X-RateLimit-* headers on every response
|
|
87
|
+
# so callers can self-pace. RequestMeta parses them once into typed Optional
|
|
88
|
+
# fields; surfaced on success via ``client.last_meta`` and on errors via
|
|
89
|
+
# ``err.meta``.
|
|
90
|
+
# ---------------------------------------------------------------------------
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
@dataclass
|
|
94
|
+
class RequestMeta:
|
|
95
|
+
credits_charged: Optional[int] = None
|
|
96
|
+
credits_remaining: Optional[int] = None
|
|
97
|
+
credits_limit: Optional[int] = None
|
|
98
|
+
rate_limit_limit: Optional[int] = None
|
|
99
|
+
rate_limit_remaining: Optional[int] = None
|
|
100
|
+
rate_limit_reset: Optional[int] = None # epoch seconds
|
|
101
|
+
burst_limit: Optional[int] = None
|
|
102
|
+
refill_per_min: Optional[int] = None
|
|
103
|
+
daily_limit: Optional[int] = None
|
|
104
|
+
daily_remaining: Optional[int] = None
|
|
105
|
+
daily_reset: Optional[int] = None # duration seconds (NOT epoch — unlike rate_limit_reset)
|
|
106
|
+
retry_after: Optional[int] = None
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def from_headers(cls, headers: Any) -> "RequestMeta":
|
|
110
|
+
def _i(name: str) -> Optional[int]:
|
|
111
|
+
v = headers.get(name)
|
|
112
|
+
if v is None and isinstance(headers, dict):
|
|
113
|
+
lower = name.lower()
|
|
114
|
+
for k, candidate in headers.items():
|
|
115
|
+
if isinstance(k, str) and k.lower() == lower:
|
|
116
|
+
v = candidate
|
|
117
|
+
break
|
|
118
|
+
if v is None:
|
|
119
|
+
return None
|
|
120
|
+
try:
|
|
121
|
+
return int(v)
|
|
122
|
+
except (TypeError, ValueError):
|
|
123
|
+
return None
|
|
124
|
+
|
|
125
|
+
return cls(
|
|
126
|
+
credits_charged=_i("x-credits-charged"),
|
|
127
|
+
credits_remaining=_i("x-credits-remaining"),
|
|
128
|
+
credits_limit=_i("x-credits-limit"),
|
|
129
|
+
rate_limit_limit=_i("x-ratelimit-limit"),
|
|
130
|
+
rate_limit_remaining=_i("x-ratelimit-remaining"),
|
|
131
|
+
rate_limit_reset=_i("x-ratelimit-reset"),
|
|
132
|
+
burst_limit=_i("x-ratelimit-burst"),
|
|
133
|
+
refill_per_min=_i("x-ratelimit-refill-per-min"),
|
|
134
|
+
daily_limit=_i("x-ratelimit-daily-limit"),
|
|
135
|
+
daily_remaining=_i("x-ratelimit-daily-remaining"),
|
|
136
|
+
daily_reset=_i("x-ratelimit-daily-reset"),
|
|
137
|
+
retry_after=_i("retry-after"),
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
# ---------------------------------------------------------------------------
|
|
142
|
+
# Error hierarchy
|
|
143
|
+
#
|
|
144
|
+
# All errors inherit from ``ParseError``. Subclasses map to common HTTP
|
|
145
|
+
# semantics; generated per-API modules subclass ``ParseError`` further for
|
|
146
|
+
# domain-specific errors (e.g. ``SubredditPrivate``).
|
|
147
|
+
# ---------------------------------------------------------------------------
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
class ParseError(RuntimeError):
|
|
151
|
+
"""Base class for all SDK-raised errors.
|
|
152
|
+
|
|
153
|
+
Attributes:
|
|
154
|
+
status_code: the PARSE GATEWAY's HTTP status (the response the SDK
|
|
155
|
+
received), or 0 if the request never reached the server.
|
|
156
|
+
When the gateway relays a target-site failure this is the
|
|
157
|
+
gateway's wrapper status (e.g. 502), NOT the target's —
|
|
158
|
+
see ``upstream_status_code`` for that.
|
|
159
|
+
body: parsed JSON body when the server returned one, raw text otherwise.
|
|
160
|
+
code: the wire error ``kind`` from the server's error envelope
|
|
161
|
+
(e.g. ``input_not_found``) when one was sent, else None.
|
|
162
|
+
Stable across APIs — branch on it for portable handling
|
|
163
|
+
even where no domain error class matched.
|
|
164
|
+
snippet: the target site's response-body excerpt relayed by the
|
|
165
|
+
gateway's upstream-error envelope, else None.
|
|
166
|
+
upstream_status_code: the TARGET SITE's HTTP status relayed by the
|
|
167
|
+
gateway's upstream-error envelope (e.g. 404 while
|
|
168
|
+
``status_code`` is the gateway's 502), else None.
|
|
169
|
+
url: the target-site URL the gateway reported in its error
|
|
170
|
+
envelope (the page the scraper was fetching when it
|
|
171
|
+
failed), else None.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
def __init__(self, status_code: int, message: str, body: Any = None) -> None:
|
|
175
|
+
super().__init__(message)
|
|
176
|
+
self.status_code = status_code
|
|
177
|
+
self.body = body
|
|
178
|
+
# Control-loop metadata, populated by _request when a response exists.
|
|
179
|
+
# Declared here so `except ParseError as e: e.meta` is always safe —
|
|
180
|
+
# errors raised before a response (keyless auth, network) keep None.
|
|
181
|
+
self.meta: Optional[RequestMeta] = None
|
|
182
|
+
# Envelope-derived fields, populated by _build_error when the server's
|
|
183
|
+
# error body carries them. Declared here (same always-safe contract as
|
|
184
|
+
# ``meta``) so typed consumers can read them on ANY ParseError without
|
|
185
|
+
# hasattr probing.
|
|
186
|
+
self.code: Optional[str] = None
|
|
187
|
+
self.snippet: Optional[str] = None
|
|
188
|
+
self.upstream_status_code: Optional[int] = None
|
|
189
|
+
self.url: Optional[str] = None
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
class AuthError(ParseError):
|
|
193
|
+
"""401 — missing / invalid API key, or session expired."""
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
class ForbiddenError(ParseError):
|
|
197
|
+
"""403 — caller is authenticated but not allowed (rate-limited tier,
|
|
198
|
+
plan gating, etc.)."""
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
class NotFoundError(ParseError):
|
|
202
|
+
"""404 — resource doesn't exist or caller can't see it."""
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
class BadRequestError(ParseError):
|
|
206
|
+
"""400 — request shape was wrong; usually a missing required param
|
|
207
|
+
or a value the upstream rejected."""
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
class RateLimitError(ParseError):
|
|
211
|
+
"""429 — caller exceeded their token bucket or daily cap.
|
|
212
|
+
|
|
213
|
+
``retry_after`` is the seconds-to-wait suggestion from the server, if any.
|
|
214
|
+
"""
|
|
215
|
+
|
|
216
|
+
def __init__(self, status_code: int, message: str, body: Any = None,
|
|
217
|
+
retry_after: Optional[int] = None) -> None:
|
|
218
|
+
super().__init__(status_code, message, body=body)
|
|
219
|
+
self.retry_after = retry_after
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
class QuotaExceededError(ParseError):
|
|
223
|
+
"""402 — the account ran out of credits / quota.
|
|
224
|
+
|
|
225
|
+
The executor returns 402 on paid-tier credit exhaustion (an account-wide
|
|
226
|
+
billing state, distinct from a malformed-param 400). The ``X-Credits-*``
|
|
227
|
+
headers are on ``err.meta``. Subclasses ``ParseError`` directly so a generic
|
|
228
|
+
``except ParseError`` still catches it."""
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
class UpstreamError(ParseError):
|
|
232
|
+
"""502 / 503 / 504 — the target site was unreachable or returned a
|
|
233
|
+
non-success response that the scraper couldn't recover from.
|
|
234
|
+
|
|
235
|
+
``status_code`` is the gateway's wrapper status (typically 502);
|
|
236
|
+
``upstream_status_code`` is the target site's own status when the envelope
|
|
237
|
+
relayed one, and ``snippet`` is the target's response-body excerpt. A
|
|
238
|
+
sanitized, truncated snippet is appended to the message so ``str(e)`` and
|
|
239
|
+
uncaught tracebacks show the upstream payload; the full raw body stays on
|
|
240
|
+
``.body`` / ``.snippet``."""
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
class PaginationError(ParseError):
|
|
244
|
+
"""The declared ``items_path`` is present in a successful response but does
|
|
245
|
+
NOT resolve to an array (``{jobs: {...}}`` / ``{jobs: 5}``) — the spec's
|
|
246
|
+
``items_path`` is mis-derived (it points at a scalar/object, not the item
|
|
247
|
+
list), or the site's shape changed. Fail loud rather than yield silent
|
|
248
|
+
garbage.
|
|
249
|
+
|
|
250
|
+
Narrowed: an *absent* ``items_path`` on a successful response
|
|
251
|
+
is no longer a blanket error — a zero-result page that simply omits the key
|
|
252
|
+
(``{}`` / a top-level-list / an envelope-null body) returns ``[]``. The
|
|
253
|
+
raise is reserved for the unambiguous mis-derivation (key present, value not
|
|
254
|
+
a list) plus, in regenerated modules, the sample-proven case (the key was in
|
|
255
|
+
the captured sample but is absent now). See ``_extract_items`` and the
|
|
256
|
+
emitted discriminator."""
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
class PaginationLimitError(ParseError):
|
|
260
|
+
"""A ``Paginator`` could not advance further: stall, fetch budget, or the
|
|
261
|
+
upstream's result window ran out.
|
|
262
|
+
|
|
263
|
+
Raised when a paginated operation re-returns the same page of items (a
|
|
264
|
+
stalled pagination contract — e.g. the server doesn't surface a usable
|
|
265
|
+
next-cursor and the SDK's fabricated cursor is ignored), runs past
|
|
266
|
+
``max_fetches``, or — mid-iteration, after at least one successful page —
|
|
267
|
+
the upstream refuses deeper pages with a 4xx (a result-window cap, e.g.
|
|
268
|
+
GitHub search stops serving past 1000 results). Carries no HTTP status —
|
|
269
|
+
it's a client-side guard, not a server response — so ``status_code`` is 0;
|
|
270
|
+
on the window-exhaustion path ``__cause__`` is the underlying
|
|
271
|
+
:class:`UpstreamError` (with ``upstream_status_code`` / ``snippet``).
|
|
272
|
+
|
|
273
|
+
Items already yielded before the raise are valid. ``pages_fetched`` /
|
|
274
|
+
``items_yielded`` say how far iteration got. ``partial_items`` is
|
|
275
|
+
populated ONLY when the failure surfaces through :meth:`Paginator.list`;
|
|
276
|
+
on the direct ``__next__`` path — manual ``for`` iteration, ``list(
|
|
277
|
+
paginator)``, comprehensions — it stays ``None``. That is deliberate: a
|
|
278
|
+
``for`` consumer already holds each item it was handed, and the streaming
|
|
279
|
+
paginator does not retain yielded items (a documented memory decision), so
|
|
280
|
+
there is nothing to backfill. Prefer ``.list()`` over ``list(paginator)``
|
|
281
|
+
when you need the materialized prefix to survive the raise.
|
|
282
|
+
"""
|
|
283
|
+
|
|
284
|
+
def __init__(
|
|
285
|
+
self,
|
|
286
|
+
message: str,
|
|
287
|
+
*,
|
|
288
|
+
pages_fetched: Optional[int] = None,
|
|
289
|
+
items_yielded: Optional[int] = None,
|
|
290
|
+
partial_items: Optional[List[Any]] = None,
|
|
291
|
+
) -> None:
|
|
292
|
+
super().__init__(0, message, body=None)
|
|
293
|
+
self.pages_fetched = pages_fetched
|
|
294
|
+
self.items_yielded = items_yielded
|
|
295
|
+
self.partial_items = partial_items
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
# Console-hostile bytes stripped from message-embedded excerpts: C0 controls
|
|
299
|
+
# minus \n\t (so \r IS stripped — a bare carriage return rewrites/hides
|
|
300
|
+
# earlier terminal output even piped), DEL, and C1 (incl. the single-byte CSI
|
|
301
|
+
# \x9b). The class is the single source in ``parse_sdk._sanitize`` (imported as
|
|
302
|
+
# ``_EXCERPT_UNSAFE``), shared with ``_sync``/``cli`` so the two threat models
|
|
303
|
+
# cannot drift. The raw bytes remain available on ``.body`` / ``.snippet``.
|
|
304
|
+
_EXCERPT_LIMIT = 300
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _sanitize_excerpt(text: str, limit: int = _EXCERPT_LIMIT) -> str:
|
|
308
|
+
"""Control-strip + hard-truncate an untrusted string for message embedding."""
|
|
309
|
+
cleaned = _EXCERPT_UNSAFE.sub("", text)
|
|
310
|
+
if len(cleaned) > limit:
|
|
311
|
+
return cleaned[:limit] + "…"
|
|
312
|
+
return cleaned
|
|
313
|
+
|
|
314
|
+
|
|
315
|
+
def _error_envelope(body: Any) -> Dict[str, Any]:
|
|
316
|
+
"""The server's nested error envelope (``body['error']``) when present.
|
|
317
|
+
|
|
318
|
+
The executor raises ``HTTPException(status, detail={...})`` and its app-level
|
|
319
|
+
handler renames ``detail`` → ``error``, so envelope fields (``kind``,
|
|
320
|
+
``snippet``, ``upstream_status_code``, ``retry_after``) live one level down.
|
|
321
|
+
Returns ``{}`` for any non-conforming body so callers can ``.get`` freely.
|
|
322
|
+
"""
|
|
323
|
+
if isinstance(body, dict):
|
|
324
|
+
env = body.get("error")
|
|
325
|
+
if isinstance(env, dict):
|
|
326
|
+
return env
|
|
327
|
+
return {}
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def _build_error(
|
|
331
|
+
status_code: int,
|
|
332
|
+
body: Any,
|
|
333
|
+
fallback: str = "",
|
|
334
|
+
*,
|
|
335
|
+
error_classes: Optional[Dict[str, type]] = None,
|
|
336
|
+
headers: Optional["httpx.Headers"] = None,
|
|
337
|
+
params: Optional[Dict[str, Any]] = None,
|
|
338
|
+
) -> ParseError:
|
|
339
|
+
"""Pick the right exception subclass for an HTTP error response.
|
|
340
|
+
|
|
341
|
+
The executor emits a closed wire ``kind`` (``body['error']['kind']``, e.g.
|
|
342
|
+
``input_not_found``). When the generated client passes its ``_ERROR_CLASSES``
|
|
343
|
+
registry (``{kind: GeneratedErrorClass}``) and the body's ``kind`` matches,
|
|
344
|
+
the DOMAIN class is raised BEFORE the coarse HTTP-status dispatch — so
|
|
345
|
+
``except BookNotFound`` fires. The legacy ``body['error_code']`` spelling is
|
|
346
|
+
also honored. Falls through to the status table when there is no registry
|
|
347
|
+
or no match.
|
|
348
|
+
|
|
349
|
+
Every constructed error is finished uniformly: ``code`` (the wire kind),
|
|
350
|
+
``snippet``, ``upstream_status_code`` and ``url`` are lifted from the
|
|
351
|
+
envelope, and —
|
|
352
|
+
for a matched DOMAIN class — declared ``carries`` are hydrated from the
|
|
353
|
+
caller's REQUEST PARAMS via the class's generated ``_CARRY_PARAMS`` map
|
|
354
|
+
(``{attr_ident: wire_param_name}``). Carries hydrate only when the request
|
|
355
|
+
actually sent the named param and the attribute is still None, so a future
|
|
356
|
+
server-provided carry value is never clobbered. The response body is never
|
|
357
|
+
read for carry values.
|
|
358
|
+
|
|
359
|
+
For 429, ``retry_after`` is read from the ``Retry-After`` *header* first
|
|
360
|
+
(the reliable, CORS-exposed signal the executor's rate_limit_service sets),
|
|
361
|
+
then from the body. The executor nests the body value at
|
|
362
|
+
``body['error']['retry_after']`` (its custom handler renames detail→error),
|
|
363
|
+
so the body fallback walks BOTH the flat and nested paths.
|
|
364
|
+
"""
|
|
365
|
+
kind = _wire_error_kind(body)
|
|
366
|
+
msg = _structured_error_message(body)
|
|
367
|
+
if msg is None:
|
|
368
|
+
# No structured message anywhere in the body (HTML gateway page, or a
|
|
369
|
+
# kind-only error object). The agent still needs something actionable:
|
|
370
|
+
# enrich the caller's context line with the structured facts in hand —
|
|
371
|
+
# the HTTP status (always) and the wire kind (when present). Never a
|
|
372
|
+
# body dump (pinned design: structured fields only).
|
|
373
|
+
bits = [f"HTTP {status_code}"]
|
|
374
|
+
if kind is not None:
|
|
375
|
+
bits.append(f"kind={kind}")
|
|
376
|
+
msg = f"{fallback or 'request failed'} ({', '.join(bits)})"
|
|
377
|
+
envelope = _error_envelope(body)
|
|
378
|
+
snippet = envelope.get("snippet")
|
|
379
|
+
snippet = snippet if isinstance(snippet, str) and snippet else None
|
|
380
|
+
upstream_sc = envelope.get("upstream_status_code")
|
|
381
|
+
upstream_sc = upstream_sc if isinstance(upstream_sc, int) else None
|
|
382
|
+
url = envelope.get("url")
|
|
383
|
+
url = url if isinstance(url, str) and url else None
|
|
384
|
+
# On a 429, the server's Retry-After hint must survive even when a generated
|
|
385
|
+
# DOMAIN error class matches the body's kind — the domain branch returns
|
|
386
|
+
# before the 429 status branch below. Compute it up front and attach it so
|
|
387
|
+
# the retry loop's getattr(err, "retry_after", None) honors it either way.
|
|
388
|
+
retry_after = _parse_retry_after(headers, body) if status_code == 429 else None
|
|
389
|
+
|
|
390
|
+
def _finish(err: ParseError) -> ParseError:
|
|
391
|
+
err.code = kind
|
|
392
|
+
err.snippet = snippet
|
|
393
|
+
err.upstream_status_code = upstream_sc
|
|
394
|
+
err.url = url
|
|
395
|
+
return err
|
|
396
|
+
|
|
397
|
+
if error_classes and kind is not None:
|
|
398
|
+
cls = error_classes.get(kind)
|
|
399
|
+
if cls is not None:
|
|
400
|
+
obj = cls(status_code, msg, body=body)
|
|
401
|
+
if retry_after is not None:
|
|
402
|
+
obj.retry_after = retry_after
|
|
403
|
+
# Hydrate declared carries from the request params (never the
|
|
404
|
+
# response body — the envelope is executor metadata, not domain
|
|
405
|
+
# values). _CARRY_PARAMS maps attr ident → wire param name and is
|
|
406
|
+
# emitted only for carries that exist in the API's input-param
|
|
407
|
+
# union, so absent mappings stay None by design.
|
|
408
|
+
carry_map = getattr(cls, "_CARRY_PARAMS", None)
|
|
409
|
+
if isinstance(carry_map, dict) and params:
|
|
410
|
+
for attr, src in carry_map.items():
|
|
411
|
+
if src in params and getattr(obj, attr, None) is None:
|
|
412
|
+
setattr(obj, attr, params[src])
|
|
413
|
+
return _finish(obj)
|
|
414
|
+
if status_code == 401:
|
|
415
|
+
return _finish(AuthError(status_code, msg, body=body))
|
|
416
|
+
if status_code == 403:
|
|
417
|
+
return _finish(ForbiddenError(status_code, msg, body=body))
|
|
418
|
+
if status_code == 404:
|
|
419
|
+
return _finish(NotFoundError(status_code, msg, body=body))
|
|
420
|
+
if status_code == 429:
|
|
421
|
+
return _finish(RateLimitError(status_code, msg, body=body, retry_after=retry_after))
|
|
422
|
+
if status_code == 402:
|
|
423
|
+
# Credit / quota exhaustion (executor's paid-tier billing state). Above
|
|
424
|
+
# the 4xx catch-all so it isn't mis-typed as a malformed-param 400;
|
|
425
|
+
# below the registry block so a future kind-carrying 402 still routes to
|
|
426
|
+
# a domain class.
|
|
427
|
+
return _finish(QuotaExceededError(status_code, msg, body=body))
|
|
428
|
+
if 400 <= status_code < 500:
|
|
429
|
+
return _finish(BadRequestError(status_code, msg, body=body))
|
|
430
|
+
if snippet:
|
|
431
|
+
# Surface the upstream payload where the user actually looks — str(e)
|
|
432
|
+
# and uncaught tracebacks. The full raw value stays on .snippet/.body.
|
|
433
|
+
msg = f"{msg} — upstream body (first {_EXCERPT_LIMIT} non-control chars): {_sanitize_excerpt(snippet)}"
|
|
434
|
+
return _finish(UpstreamError(status_code, msg, body=body))
|
|
435
|
+
|
|
436
|
+
|
|
437
|
+
def _parse_retry_after(
|
|
438
|
+
headers: Optional["httpx.Headers"], body: Any
|
|
439
|
+
) -> Optional[int]:
|
|
440
|
+
"""Seconds-to-wait from the ``Retry-After`` header (primary) or the body.
|
|
441
|
+
|
|
442
|
+
Header first (reliable / CORS-exposed). Body fallback covers the flat
|
|
443
|
+
``body['retry_after']``, the executor-nested ``body['error']['retry_after']``,
|
|
444
|
+
and the legacy ``body['detail']['retry_after']``. Returns None when no
|
|
445
|
+
usable signal exists. Never raises."""
|
|
446
|
+
if headers is not None:
|
|
447
|
+
raw = headers.get("Retry-After")
|
|
448
|
+
if raw is not None:
|
|
449
|
+
try:
|
|
450
|
+
return int(raw)
|
|
451
|
+
except (TypeError, ValueError):
|
|
452
|
+
pass # HTTP-date form not handled — fall through to body
|
|
453
|
+
if isinstance(body, dict):
|
|
454
|
+
nested = body.get("error")
|
|
455
|
+
nested = nested if isinstance(nested, dict) else {}
|
|
456
|
+
detail = body.get("detail")
|
|
457
|
+
detail = detail if isinstance(detail, dict) else {}
|
|
458
|
+
ra = body.get("retry_after")
|
|
459
|
+
if ra is None:
|
|
460
|
+
ra = nested.get("retry_after")
|
|
461
|
+
if ra is None:
|
|
462
|
+
ra = detail.get("retry_after")
|
|
463
|
+
try:
|
|
464
|
+
return int(ra) if ra is not None else None
|
|
465
|
+
except (TypeError, ValueError):
|
|
466
|
+
return None
|
|
467
|
+
return None
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _wire_error_kind(body: Any) -> Optional[str]:
|
|
471
|
+
"""The executor's closed wire ``kind`` for an error body, or ``None``.
|
|
472
|
+
|
|
473
|
+
Walks the same shapes ``_build_error``'s registry dispatch always has:
|
|
474
|
+
nested ``body['error']['kind']`` / ``['code']``, then the legacy flat
|
|
475
|
+
``body['error_code']`` / ``body['kind']`` spellings."""
|
|
476
|
+
if not isinstance(body, dict):
|
|
477
|
+
return None
|
|
478
|
+
kind: Any = None
|
|
479
|
+
err = body.get("error")
|
|
480
|
+
if isinstance(err, dict):
|
|
481
|
+
kind = err.get("kind") or err.get("code")
|
|
482
|
+
if kind is None:
|
|
483
|
+
kind = body.get("error_code") or body.get("kind")
|
|
484
|
+
return kind if isinstance(kind, str) else None
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _structured_error_message(body: Any) -> Optional[str]:
|
|
488
|
+
"""The human message from the body's structured string fields, or ``None``
|
|
489
|
+
when the body carries no usable message (non-dict, or kind-only)."""
|
|
490
|
+
if isinstance(body, dict):
|
|
491
|
+
for key in ("detail", "error", "message"):
|
|
492
|
+
v = body.get(key)
|
|
493
|
+
if isinstance(v, str):
|
|
494
|
+
return v
|
|
495
|
+
if isinstance(v, dict):
|
|
496
|
+
m = v.get("message") or v.get("error")
|
|
497
|
+
if isinstance(m, str):
|
|
498
|
+
return m
|
|
499
|
+
return None
|
|
500
|
+
|
|
501
|
+
|
|
502
|
+
def _format_error_message(body: Any, fallback: str = "") -> str:
|
|
503
|
+
return _structured_error_message(body) or fallback or "unknown error"
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
# ---------------------------------------------------------------------------
|
|
507
|
+
# Value coercion — driven by the codegen's per-field type label
|
|
508
|
+
# ---------------------------------------------------------------------------
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
# Non-downcast runtime backstop. When a NON-KEY field declared ``str`` is handed a non-str
|
|
512
|
+
# wire value, downcasting it (``str(False) == 'False'``) changes the observed
|
|
513
|
+
# data shape and can hide a stale/misdeclared schema. Keep such values RAW so an
|
|
514
|
+
# already-shipped or drift-stale package stays non-corrupting without a regen.
|
|
515
|
+
# KEY fields are EXCLUDED (they must keep downcasting so ``'42'`` and ``42``
|
|
516
|
+
# stay ==/hash-equal for dedup). Unconditional — no env opt-out: an in-band
|
|
517
|
+
# escape hatch is an agent-flippable vector that would reintroduce the exact
|
|
518
|
+
# downcast corruption this backstop exists to prevent.
|
|
519
|
+
# This covers ONLY the per-field annotation walk (``_coerce_by_annotation``); the
|
|
520
|
+
# bare-scalar method-RETURN router (``_PRIMITIVE_COERCERS['str']`` via
|
|
521
|
+
# ``coerce()``) is not a field-corruption vector and is left unchanged.
|
|
522
|
+
|
|
523
|
+
# Monotonic clock seam — module-level so tests can monkeypatch wall-clock (the
|
|
524
|
+
# retry budget counts real elapsed time, requests included).
|
|
525
|
+
_monotonic = time.monotonic
|
|
526
|
+
|
|
527
|
+
|
|
528
|
+
def _coerce_bool(value: Any) -> Any:
|
|
529
|
+
if value is None or isinstance(value, bool):
|
|
530
|
+
return value
|
|
531
|
+
if isinstance(value, str):
|
|
532
|
+
folded = value.strip().lower()
|
|
533
|
+
if folded in {"true", "t", "1", "yes", "y", "on"}:
|
|
534
|
+
return True
|
|
535
|
+
if folded in {"false", "f", "0", "no", "n", "off"}:
|
|
536
|
+
return False
|
|
537
|
+
return value
|
|
538
|
+
if isinstance(value, (int, float)) and value in (0, 1):
|
|
539
|
+
return bool(value)
|
|
540
|
+
return value
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
_PRIMITIVE_COERCERS: Dict[str, Callable[[Any], Any]] = {
|
|
544
|
+
"str": lambda v: v if isinstance(v, str) else (str(v) if v is not None else None),
|
|
545
|
+
"int": lambda v: int(v) if v is not None else None,
|
|
546
|
+
"float": lambda v: float(v) if v is not None else None,
|
|
547
|
+
"bool": _coerce_bool,
|
|
548
|
+
"url": str,
|
|
549
|
+
"email": str,
|
|
550
|
+
}
|
|
551
|
+
|
|
552
|
+
|
|
553
|
+
def _coerce_datetime(value: Any) -> Any:
|
|
554
|
+
if value is None or isinstance(value, datetime):
|
|
555
|
+
return value
|
|
556
|
+
# ``bool`` subclasses ``int`` — guard it BEFORE the epoch branch so a wire
|
|
557
|
+
# ``True`` doesn't become ``1970-01-01T00:00:01`` (epoch 1). A bool is never
|
|
558
|
+
# a timestamp; pass it through raw.
|
|
559
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
560
|
+
try:
|
|
561
|
+
return datetime.fromtimestamp(value, tz=timezone.utc)
|
|
562
|
+
except (OSError, OverflowError, ValueError):
|
|
563
|
+
return value
|
|
564
|
+
if isinstance(value, str):
|
|
565
|
+
try:
|
|
566
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
567
|
+
except ValueError:
|
|
568
|
+
return value
|
|
569
|
+
return value
|
|
570
|
+
|
|
571
|
+
|
|
572
|
+
def _coerce_decimal(value: Any) -> Any:
|
|
573
|
+
if value is None or isinstance(value, Decimal):
|
|
574
|
+
return value
|
|
575
|
+
try:
|
|
576
|
+
return Decimal(str(value))
|
|
577
|
+
except (InvalidOperation, TypeError):
|
|
578
|
+
return value
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
# Sentinel: the dict carried no single list to unwrap (zero list fields, or
|
|
582
|
+
# two-or-more NON-EMPTY list fields — genuinely ambiguous). Distinct from a
|
|
583
|
+
# determinable empty result (``[]``). Each call-site decides the disposition.
|
|
584
|
+
_NO_SINGLE_LIST = object()
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _unwrap_single_list(value: Dict[str, Any]) -> Any:
|
|
588
|
+
"""Pick the single list out of an envelope dict, or ``_NO_SINGLE_LIST``.
|
|
589
|
+
|
|
590
|
+
API responses often wrap a list in an envelope, e.g.
|
|
591
|
+
``{"count": 3, "stories": [...]}``. When exactly ONE list-valued field is
|
|
592
|
+
non-empty we return it; an incidental EMPTY list (``{"errors": [],
|
|
593
|
+
"stories": [...]}``) is **ignored when counting** so a stray ``errors: []``
|
|
594
|
+
can't flip the count to 2 and defeat the unwrap. When every list field is
|
|
595
|
+
empty (``{"errors": [], "stories": []}``) we return ``[]`` — a determinable
|
|
596
|
+
empty result, not an ambiguity. Zero list fields, or two-or-more non-empty
|
|
597
|
+
list fields, return ``_NO_SINGLE_LIST`` — the caller decides whether to fail
|
|
598
|
+
loud (a method-return contract boundary) or degrade to raw (a per-field
|
|
599
|
+
walk that must never crash a successful scrape).
|
|
600
|
+
|
|
601
|
+
``value`` is assumed to be a dict (callers guard ``isinstance``). Never
|
|
602
|
+
raises.
|
|
603
|
+
"""
|
|
604
|
+
list_fields = [k for k, v in value.items() if isinstance(v, list)]
|
|
605
|
+
if not list_fields:
|
|
606
|
+
return _NO_SINGLE_LIST
|
|
607
|
+
non_empty = [k for k in list_fields if value[k]]
|
|
608
|
+
if len(non_empty) == 1:
|
|
609
|
+
return value[non_empty[0]]
|
|
610
|
+
if not non_empty:
|
|
611
|
+
# All list fields are empty → an empty result set (determinable).
|
|
612
|
+
return []
|
|
613
|
+
# Two-or-more non-empty lists → genuinely ambiguous.
|
|
614
|
+
return _NO_SINGLE_LIST
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _construct_from_scalar(
|
|
618
|
+
cls: "Type[Resource]", value: Any, *, api: Optional["BaseAPI"], parent: Any
|
|
619
|
+
) -> Any:
|
|
620
|
+
"""Hydrate a constructible resource from a bare scalar matching its
|
|
621
|
+
``keyed_by`` field (``post.subreddit = "smallbusiness"`` →
|
|
622
|
+
``Subreddit(name="smallbusiness")``), or return ``value`` unchanged.
|
|
623
|
+
|
|
624
|
+
The scalar is COERCED to the key field's resolved annotation first (via the
|
|
625
|
+
total/lenient ``_coerce_by_annotation``), so the scalar-path instance and a
|
|
626
|
+
dict-path instance of the same entity agree on the key's TYPE — otherwise a
|
|
627
|
+
wire int key (``42``) on the scalar path vs an int-coerced ``42`` on the
|
|
628
|
+
dict path could diverge and break ``__eq__``/``__hash__`` (which key on the
|
|
629
|
+
field value). Only ``str``/``int`` scalars hydrate; anything else passes
|
|
630
|
+
through. Never raises (coercion is lenient).
|
|
631
|
+
"""
|
|
632
|
+
key = getattr(cls, "_KEY_FIELD", None)
|
|
633
|
+
if not (getattr(cls, "_CONSTRUCTIBLE", False) and key and isinstance(value, (str, int))):
|
|
634
|
+
return value
|
|
635
|
+
key_ann = cls._field_hints().get(key)
|
|
636
|
+
coerced_key = (
|
|
637
|
+
_coerce_by_annotation(value, key_ann, api=api, parent=parent, is_key=True)
|
|
638
|
+
if key_ann is not None
|
|
639
|
+
else value
|
|
640
|
+
)
|
|
641
|
+
return cls(_api=api, _parent=parent, **{key: coerced_key})
|
|
642
|
+
|
|
643
|
+
|
|
644
|
+
def _is_husk(inst: Any) -> bool:
|
|
645
|
+
"""Husk predicate, observed by POST-``_from_payload`` instance
|
|
646
|
+
inspection — never by re-running the wrapped-record unwrap.
|
|
647
|
+
|
|
648
|
+
A husk is a constructed resource where *no declared wire key resolved* (the
|
|
649
|
+
``_resolved_count`` signal ``_from_payload`` set is 0) — this single count
|
|
650
|
+
also subsumes "all declared keys null", since a key whose wire value is null
|
|
651
|
+
is not counted as resolved.
|
|
652
|
+
|
|
653
|
+
**Exempt** when ``_extra`` carries any non-null unmapped key: a record whose
|
|
654
|
+
data landed entirely in ``_extra`` is a mis-DERIVED spec (the fields were
|
|
655
|
+
typed off the wrong shape), NOT a hollow husk — failing loud there would
|
|
656
|
+
discard data the caller can still reach via ``obj._extra``. Total/lenient:
|
|
657
|
+
a missing signal (a non-``_from_payload`` instance) reads as NOT a husk, so
|
|
658
|
+
this can never manufacture a false verdict.
|
|
659
|
+
"""
|
|
660
|
+
if getattr(inst, "_resolved_count", None) != 0:
|
|
661
|
+
return False
|
|
662
|
+
extra = getattr(inst, "_extra", None)
|
|
663
|
+
if isinstance(extra, dict) and any(v is not None for v in extra.values()):
|
|
664
|
+
return False # data lives in _extra → mis-derived spec, not a husk
|
|
665
|
+
return True
|
|
666
|
+
|
|
667
|
+
|
|
668
|
+
def coerce(value: Any, type_label: str, *, resource_classes: Optional[Dict[str, Type[Resource]]] = None,
|
|
669
|
+
api: Optional[BaseAPI] = None, parent: Any = None,
|
|
670
|
+
boundary: bool = False, mode: str = "element") -> Any:
|
|
671
|
+
"""Coerce a raw response value to its declared type.
|
|
672
|
+
|
|
673
|
+
Type labels:
|
|
674
|
+
* ``str``, ``int``, ``float``, ``bool`` — primitive coercion
|
|
675
|
+
* ``datetime`` — int/float epoch OR ISO str → datetime
|
|
676
|
+
* ``Decimal`` — str/int/float → Decimal
|
|
677
|
+
* ``url``, ``email`` — kept as str (semantic only)
|
|
678
|
+
* ``List[X]`` — recurse over each element
|
|
679
|
+
* ``Optional[X]`` — None passes through; else recurse
|
|
680
|
+
* ``Dict[K, V]`` — recurse over values (keys stay str)
|
|
681
|
+
* ``Any`` — pass through unchanged
|
|
682
|
+
* ``<ResourceName>`` — look up in ``resource_classes`` and
|
|
683
|
+
construct an instance via ``_from_payload``
|
|
684
|
+
|
|
685
|
+
``resource_classes`` / ``api`` / ``parent`` are forwarded so resource
|
|
686
|
+
instances can wire back to the owning client.
|
|
687
|
+
|
|
688
|
+
``boundary`` / ``mode`` (return-fail-loud) gate the strict
|
|
689
|
+
return-boundary verdicts. **Both default to the LENIENT setting**
|
|
690
|
+
(``boundary=False``, ``mode='element'``) so an old 3-arg
|
|
691
|
+
``_coerce(value, type_label, ctx)`` call from an already-generated module
|
|
692
|
+
hits the lenient default and behaves exactly as before — this is the
|
|
693
|
+
cross-compat protection for the entire generated fleet. The strict
|
|
694
|
+
husk / scalar verdicts fire ONLY when a *regenerated* module's
|
|
695
|
+
single-resource return passes ``boundary=True, mode='return'`` and the
|
|
696
|
+
declared type resolves to a Resource. Every ``Optional`` / ``List`` /
|
|
697
|
+
``Dict`` recurse flips ``boundary=False`` (a verdict applies to the top
|
|
698
|
+
return only, never to a nested element — per-element disposition is the
|
|
699
|
+
lenient degrade-to-raw).
|
|
700
|
+
"""
|
|
701
|
+
if value is None:
|
|
702
|
+
return None
|
|
703
|
+
|
|
704
|
+
t = type_label.strip()
|
|
705
|
+
# Optional[X] — single-layer unwrap via the shared label grammar (_labels).
|
|
706
|
+
_opt_inner = strip_wrapper(t, "Optional[")
|
|
707
|
+
if _opt_inner is not None:
|
|
708
|
+
return coerce(value, _opt_inner, resource_classes=resource_classes, api=api, parent=parent,
|
|
709
|
+
boundary=False, mode=mode)
|
|
710
|
+
# List[X]
|
|
711
|
+
_list_inner = strip_wrapper(t, "List[")
|
|
712
|
+
if _list_inner is not None:
|
|
713
|
+
inner = _list_inner
|
|
714
|
+
# API responses often wrap the list in an envelope, e.g.
|
|
715
|
+
# ``{"count": 3, "stories": [...]}``. When we expect List[X] and got a
|
|
716
|
+
# dict, unwrap the single (non-empty) list. This is the method-RETURN
|
|
717
|
+
# contract boundary — the caller did ``return _coerce(_resp, 'List[X]',
|
|
718
|
+
# self)`` and will iterate the result, so a dict we can't unwrap to a
|
|
719
|
+
# list would silently iterate dict KEYS. FAIL LOUD instead (a clear
|
|
720
|
+
# error beats silent garbage). The per-FIELD router stays lenient.
|
|
721
|
+
if isinstance(value, dict):
|
|
722
|
+
unwrapped = _unwrap_single_list(value)
|
|
723
|
+
if unwrapped is _NO_SINGLE_LIST:
|
|
724
|
+
raise ParseError(
|
|
725
|
+
0,
|
|
726
|
+
f"expected List[{inner}] but the response was a dict with no "
|
|
727
|
+
f"single list to unwrap (keys: {sorted(value)}); the "
|
|
728
|
+
f"pagination/envelope contract is likely mis-derived",
|
|
729
|
+
body=value,
|
|
730
|
+
)
|
|
731
|
+
value = unwrapped
|
|
732
|
+
if not isinstance(value, list):
|
|
733
|
+
return value
|
|
734
|
+
# Coverage symmetry: a non-paginated List[Resource] recurses at
|
|
735
|
+
# boundary=False — every element gets the SAME lenient degrade-to-raw
|
|
736
|
+
# disposition as a paginated row, never the strict return verdict.
|
|
737
|
+
return [coerce(v, inner, resource_classes=resource_classes, api=api, parent=parent,
|
|
738
|
+
boundary=False, mode=mode) for v in value]
|
|
739
|
+
# Dict[K, V] — only coerce values
|
|
740
|
+
_dict_inner = strip_wrapper(t, "Dict[")
|
|
741
|
+
if _dict_inner is not None:
|
|
742
|
+
inner = _dict_inner
|
|
743
|
+
# First depth-0 comma split (handles Dict[str, List[X]]) — shared
|
|
744
|
+
# splitter; everything past the first comma is the value label.
|
|
745
|
+
parts = split_top(inner)
|
|
746
|
+
v_label = ", ".join(parts[1:]) if len(parts) > 1 else "Any"
|
|
747
|
+
if not isinstance(value, dict):
|
|
748
|
+
return value
|
|
749
|
+
return {k: coerce(v, v_label, resource_classes=resource_classes, api=api, parent=parent,
|
|
750
|
+
boundary=False, mode=mode)
|
|
751
|
+
for k, v in value.items()}
|
|
752
|
+
# Primitives
|
|
753
|
+
if t in _PRIMITIVE_COERCERS:
|
|
754
|
+
try:
|
|
755
|
+
return _PRIMITIVE_COERCERS[t](value)
|
|
756
|
+
except (TypeError, ValueError, OverflowError):
|
|
757
|
+
# OverflowError: ``int(float('inf'))`` (stdlib json.loads admits
|
|
758
|
+
# Infinity/-Infinity/NaN, so a target can return ``{"n": Infinity}``
|
|
759
|
+
# on a declared-int field). Degrade to raw — never raise on a
|
|
760
|
+
# successful scrape.
|
|
761
|
+
return value
|
|
762
|
+
if t == "datetime":
|
|
763
|
+
return _coerce_datetime(value)
|
|
764
|
+
if t == "Decimal":
|
|
765
|
+
return _coerce_decimal(value)
|
|
766
|
+
if t in ("Any", "any"):
|
|
767
|
+
return value
|
|
768
|
+
# Resource type
|
|
769
|
+
if resource_classes and t in resource_classes:
|
|
770
|
+
cls = resource_classes[t]
|
|
771
|
+
# Strict return boundary — fires ONLY for a regenerated module's
|
|
772
|
+
# single-resource return (boundary=True + mode='return'). Anywhere else
|
|
773
|
+
# (lenient default, per-element rows at mode='element', any nested
|
|
774
|
+
# recurse at boundary=False) keeps the historical silent disposition.
|
|
775
|
+
strict = boundary and mode == "return"
|
|
776
|
+
if isinstance(value, dict):
|
|
777
|
+
inst = cls._from_payload(value, api=api, parent=parent)
|
|
778
|
+
if strict and _is_husk(inst):
|
|
779
|
+
# Husk: no declared wire key resolved OR all declared
|
|
780
|
+
# keys null — but EXEMPT when ``_extra`` carries non-null
|
|
781
|
+
# unmapped keys (a mis-DERIVED spec whose data lives in _extra,
|
|
782
|
+
# not a hollow husk — failing loud there would discard
|
|
783
|
+
# retrievable data). Husk-ness is observed by POST-_from_payload
|
|
784
|
+
# instance inspection (the resolved-key signal _from_payload
|
|
785
|
+
# set), never by re-running the unwrap. The verdict lives HERE
|
|
786
|
+
# in coerce(), never in _from_payload (shared with the lenient
|
|
787
|
+
# per-field router → a verdict there would crash the fleet).
|
|
788
|
+
raise ParseError(
|
|
789
|
+
0,
|
|
790
|
+
f"expected {t} but the response resolved no declared field "
|
|
791
|
+
f"(an all-null / zero-key 'husk'); the return-type contract "
|
|
792
|
+
f"is likely mis-derived",
|
|
793
|
+
body=value,
|
|
794
|
+
)
|
|
795
|
+
return inst
|
|
796
|
+
# Constructible resources hydrate from a bare scalar matching the
|
|
797
|
+
# keyed_by field — the key is coerced to its annotation so scalar- and
|
|
798
|
+
# dict-path instances agree on key type (see _construct_from_scalar).
|
|
799
|
+
hydrated = _construct_from_scalar(cls, value, api=api, parent=parent)
|
|
800
|
+
if strict and not isinstance(hydrated, cls):
|
|
801
|
+
# Scalar husk: a bare scalar typed as a non-constructible Resource (no
|
|
802
|
+
# _KEY_FIELD to hydrate from) returned the raw scalar — fail loud.
|
|
803
|
+
# A constructible resource hydrated above (``isinstance`` holds), so
|
|
804
|
+
# only the genuinely non-constructible scalar reaches this raise.
|
|
805
|
+
raise ParseError(
|
|
806
|
+
0,
|
|
807
|
+
f"expected {t} but the response was a bare {type(value).__name__} "
|
|
808
|
+
f"with no key field to construct from; the return-type contract "
|
|
809
|
+
f"is likely mis-derived",
|
|
810
|
+
body=value,
|
|
811
|
+
)
|
|
812
|
+
return hydrated
|
|
813
|
+
# Unknown — pass through. Better to let user see the raw dict than crash.
|
|
814
|
+
return value
|
|
815
|
+
|
|
816
|
+
|
|
817
|
+
# ---------------------------------------------------------------------------
|
|
818
|
+
# Per-field coercion — single-source, driven by the resolved field ANNOTATION
|
|
819
|
+
#
|
|
820
|
+
# This is the per-field coercer (distinct from the ``coerce()`` string-label
|
|
821
|
+
# *return* router above, which generated methods still call). Generated
|
|
822
|
+
# resource classes declare each field's type as a PEP-526 annotation and carry
|
|
823
|
+
# NO parallel ``_FIELDS`` label dict; ``_from_payload`` resolves those
|
|
824
|
+
# annotations to real typing/class OBJECTS (cached per class) and dispatches on
|
|
825
|
+
# them here — never round-tripping a resolved object back through the
|
|
826
|
+
# string-label router (that would no-op and re-create the static/runtime
|
|
827
|
+
# divergence). Total / lenient: an unknown form or un-coercible value passes
|
|
828
|
+
# through raw; it never raises on a successful scrape.
|
|
829
|
+
#
|
|
830
|
+
# WHY TWO DISPATCHERS (do not naively merge): the leaf coercers
|
|
831
|
+
# (_coerce_bool/_coerce_datetime/_coerce_decimal) are ALREADY single-sourced and
|
|
832
|
+
# shared by both. What differs is irreducible and opposite: ``coerce()`` is the
|
|
833
|
+
# RETURN router — it always downcasts ``str`` and FAILS LOUD at the strict boundary
|
|
834
|
+
# (husk/scalar verdicts); this per-field router is LENIENT (degrade-to-raw),
|
|
835
|
+
# honours the str non-downcast rule (``is_key``), and owns the Enum case-fold dispatch.
|
|
836
|
+
# Folding them into one driver would re-encode those divergent dispositions as
|
|
837
|
+
# flags (fail-loud-vs-lenient × is_key × boundary/mode) — more branching, not
|
|
838
|
+
# less — in a path NO emitted-bytes gate can verify. The asymmetries are pinned
|
|
839
|
+
# by tests/test_coercion_asymmetries.py. Keep them two.
|
|
840
|
+
# ---------------------------------------------------------------------------
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
# Both union spellings: ``typing.Union[...]`` (incl. ``Optional[X]``) and the
|
|
844
|
+
# PEP-604 ``A | B`` form, whose origin is ``types.UnionType`` (3.10+). The
|
|
845
|
+
# codegen emits ``Sort | str`` for enum/choice fields, so this is load-bearing.
|
|
846
|
+
_UNION_ORIGINS = (Union, types.UnionType)
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def _coerce_by_annotation(
|
|
850
|
+
value: Any,
|
|
851
|
+
ann: Any,
|
|
852
|
+
*,
|
|
853
|
+
api: Optional[BaseAPI] = None,
|
|
854
|
+
parent: Any = None,
|
|
855
|
+
is_key: bool = False,
|
|
856
|
+
) -> Any:
|
|
857
|
+
"""Coerce ``value`` to the resolved annotation OBJECT ``ann``.
|
|
858
|
+
|
|
859
|
+
Dispatches on ``get_origin`` / ``get_args`` (List/Dict/Optional/Union),
|
|
860
|
+
``isinstance``-against-``Enum``, and the ``datetime``/``Decimal``/``str``/
|
|
861
|
+
``int``/``float``/``bool`` classes. ``Resource`` subclasses construct via
|
|
862
|
+
``_from_payload`` (or constructible-from-scalar). Threads ``api``/``parent``
|
|
863
|
+
into every nested + list/dict element (``parent`` = the owning resource).
|
|
864
|
+
|
|
865
|
+
``is_key`` marks this coercion as a resource's KEY field, threaded
|
|
866
|
+
through the Optional/Union/List/Dict recursion. A key keeps downcasting (so
|
|
867
|
+
``'42'``/``42`` stay ==/hash-equal); a NON-key ``str`` field stops
|
|
868
|
+
downcasting non-strings (the truthiness-inversion backstop). It is NOT
|
|
869
|
+
propagated into a nested resource — that resource recomputes ``is_key`` for
|
|
870
|
+
its own key field.
|
|
871
|
+
|
|
872
|
+
Total/lenient: any unrecognized form or un-coercible value returns ``value``
|
|
873
|
+
unchanged — never raises.
|
|
874
|
+
"""
|
|
875
|
+
if value is None:
|
|
876
|
+
return None
|
|
877
|
+
|
|
878
|
+
# Hot path: resolved resource field annotations are usually concrete scalar
|
|
879
|
+
# classes. Handle those before the slower typing-origin / issubclass probes.
|
|
880
|
+
if ann is str:
|
|
881
|
+
if isinstance(value, str):
|
|
882
|
+
return value
|
|
883
|
+
# Non-downcast backstop: a NON-key str field handed a non-str keeps it RAW (no
|
|
884
|
+
# ``str(False)=='False'`` truthiness inversion). Keys still downcast so
|
|
885
|
+
# ``'42'``/``42`` stay ==/hash-equal. Unconditional — no opt-out.
|
|
886
|
+
if not is_key:
|
|
887
|
+
return value
|
|
888
|
+
return str(value)
|
|
889
|
+
if ann is bool:
|
|
890
|
+
return _coerce_bool(value)
|
|
891
|
+
if ann in (int, float):
|
|
892
|
+
try:
|
|
893
|
+
return ann(value)
|
|
894
|
+
except (TypeError, ValueError, OverflowError):
|
|
895
|
+
# OverflowError: ``int(float('inf'))`` — stdlib json.loads admits
|
|
896
|
+
# Infinity/-Infinity/NaN, so a wire ``Infinity`` on a declared-int
|
|
897
|
+
# field reaches here. Degrade to raw (never raise on a scrape).
|
|
898
|
+
return value
|
|
899
|
+
if ann is datetime:
|
|
900
|
+
return _coerce_datetime(value)
|
|
901
|
+
if ann is Decimal:
|
|
902
|
+
return _coerce_decimal(value)
|
|
903
|
+
|
|
904
|
+
origin = get_origin(ann)
|
|
905
|
+
|
|
906
|
+
# Union / Optional — try each non-None arm; first that changes the value
|
|
907
|
+
# (or matches a resource/enum/special) wins. Optional[X] is X | None.
|
|
908
|
+
if origin in _UNION_ORIGINS:
|
|
909
|
+
args = [a for a in get_args(ann) if a is not type(None)]
|
|
910
|
+
# Single-arm Optional[X] → coerce as X.
|
|
911
|
+
if len(args) == 1:
|
|
912
|
+
return _coerce_by_annotation(value, args[0], api=api, parent=parent, is_key=is_key)
|
|
913
|
+
# Multi-arm union (e.g. ``Sort | str``): prefer an Enum/Resource arm
|
|
914
|
+
# (the precise type) over the bare str/scalar fallback; if none fire,
|
|
915
|
+
# fall through raw.
|
|
916
|
+
for arm in args:
|
|
917
|
+
if _is_enum_type(arm) or _is_resource_type(arm):
|
|
918
|
+
coerced = _coerce_by_annotation(value, arm, api=api, parent=parent, is_key=is_key)
|
|
919
|
+
# Accept the arm when it produced (or already was) an instance of
|
|
920
|
+
# it — an ``isinstance`` check, NOT object identity. (Identity
|
|
921
|
+
# would reject an already-coerced member, e.g. a ``Sort.NEW`` fed
|
|
922
|
+
# back through ``Sort | str``, then stringify it via the str arm.)
|
|
923
|
+
if isinstance(coerced, arm):
|
|
924
|
+
return coerced
|
|
925
|
+
# No precise arm matched — coerce to the first arm (usually str) only if
|
|
926
|
+
# it's a primitive; else pass through.
|
|
927
|
+
for arm in args:
|
|
928
|
+
if arm in (str, int, float, bool, datetime, Decimal):
|
|
929
|
+
return _coerce_by_annotation(value, arm, api=api, parent=parent, is_key=is_key)
|
|
930
|
+
return value
|
|
931
|
+
|
|
932
|
+
# List[X] — envelope-unwrap a single-list dict, then recurse per element.
|
|
933
|
+
# This is the per-FIELD path (reached inside ``_from_payload`` on every
|
|
934
|
+
# ``List[X]`` field). It MUST stay lenient — degrade to raw, never raise —
|
|
935
|
+
# so one malformed nested list can't crash construction of a
|
|
936
|
+
# successfully-scraped resource and discard its other good fields. (Contrast
|
|
937
|
+
# the method-RETURN router ``coerce()``, which fails loud at its boundary.)
|
|
938
|
+
if origin in (list, List):
|
|
939
|
+
(inner,) = get_args(ann) or (Any,)
|
|
940
|
+
if isinstance(value, dict):
|
|
941
|
+
unwrapped = _unwrap_single_list(value)
|
|
942
|
+
if unwrapped is _NO_SINGLE_LIST:
|
|
943
|
+
return value # lenient: keep the raw dict, never raise
|
|
944
|
+
value = unwrapped
|
|
945
|
+
if not isinstance(value, list):
|
|
946
|
+
return value
|
|
947
|
+
return [_coerce_by_annotation(v, inner, api=api, parent=parent, is_key=is_key) for v in value]
|
|
948
|
+
|
|
949
|
+
# Dict[K, V] — coerce values only (keys stay as-is).
|
|
950
|
+
if origin in (dict, Dict):
|
|
951
|
+
args = get_args(ann)
|
|
952
|
+
v_ann = args[1] if len(args) == 2 else Any
|
|
953
|
+
if not isinstance(value, dict):
|
|
954
|
+
return value
|
|
955
|
+
return {k: _coerce_by_annotation(v, v_ann, api=api, parent=parent, is_key=is_key)
|
|
956
|
+
for k, v in value.items()}
|
|
957
|
+
|
|
958
|
+
# Enum — member for an observed value, RAW for an unobserved-but-valid one
|
|
959
|
+
# (never raise). This is the divergence fix: a bare ``type: ContentType``
|
|
960
|
+
# field now coerces to the member at runtime instead of staying a str.
|
|
961
|
+
if _is_enum_type(ann):
|
|
962
|
+
if isinstance(value, ann):
|
|
963
|
+
return value
|
|
964
|
+
try:
|
|
965
|
+
return ann(value) # exact wire-value match
|
|
966
|
+
except (ValueError, KeyError, TypeError):
|
|
967
|
+
pass
|
|
968
|
+
# Case-tolerant fallback (G7): the wire sometimes uppercases an enum
|
|
969
|
+
# value (``FOR_SALE`` for a ``for_sale`` member). Match str(value)
|
|
970
|
+
# against member values case-insensitively before giving up — using the
|
|
971
|
+
# same ``.lower()`` fold the codegen's code-uniqueness gate enforces, so
|
|
972
|
+
# a match is unambiguous. Keep raw on no match (never raise).
|
|
973
|
+
folded = str(value).lower()
|
|
974
|
+
for member in ann:
|
|
975
|
+
if str(member.value).lower() == folded:
|
|
976
|
+
return member
|
|
977
|
+
return value # unobserved-but-valid → keep raw, never raise
|
|
978
|
+
|
|
979
|
+
# Resource — construct from a dict payload, or hydrate constructible from a
|
|
980
|
+
# bare scalar matching the keyed_by field.
|
|
981
|
+
if _is_resource_type(ann):
|
|
982
|
+
if isinstance(value, dict):
|
|
983
|
+
return ann._from_payload(value, api=api, parent=parent)
|
|
984
|
+
return _construct_from_scalar(ann, value, api=api, parent=parent)
|
|
985
|
+
|
|
986
|
+
# Any / unknown annotation form → raw passthrough (total/lenient).
|
|
987
|
+
return value
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def _is_enum_type(ann: Any) -> bool:
|
|
991
|
+
return isinstance(ann, type) and issubclass(ann, Enum)
|
|
992
|
+
|
|
993
|
+
|
|
994
|
+
def _is_resource_type(ann: Any) -> bool:
|
|
995
|
+
return isinstance(ann, type) and issubclass(ann, Resource)
|
|
996
|
+
|
|
997
|
+
|
|
998
|
+
def _resource_payload_meta(cls: "Type[Resource]") -> tuple:
|
|
999
|
+
"""Cached per-class metadata used by ``Resource._from_payload``.
|
|
1000
|
+
|
|
1001
|
+
The generated class's annotations and ``_FIELD_ALIASES`` are fixed at module
|
|
1002
|
+
import. Cache the derived wire-name map and consumed-wire set so high-volume
|
|
1003
|
+
page hydration does not rebuild them for every row.
|
|
1004
|
+
|
|
1005
|
+
This is module-scoped on purpose: every new ``Resource`` member expands
|
|
1006
|
+
codegen's reserved identifier set (``dir(Resource)``), which changes emitted
|
|
1007
|
+
output for schemas that happen to use the same field name.
|
|
1008
|
+
"""
|
|
1009
|
+
cached = cls.__dict__.get("_FIELD_HINTS_CACHE")
|
|
1010
|
+
if (
|
|
1011
|
+
isinstance(cached, tuple)
|
|
1012
|
+
and len(cached) == 2
|
|
1013
|
+
and isinstance(cached[0], dict)
|
|
1014
|
+
and isinstance(cached[1], tuple)
|
|
1015
|
+
):
|
|
1016
|
+
return cached[1]
|
|
1017
|
+
hints = cls._field_hints()
|
|
1018
|
+
aliases = cls._FIELD_ALIASES
|
|
1019
|
+
wire_names = {name: aliases.get(name, name) for name in hints}
|
|
1020
|
+
# Membership-tested only (in _from_payload's wrapper-unwrap probe); a
|
|
1021
|
+
# frozenset dedupes aliases mapping to the same wire key and lets the probe
|
|
1022
|
+
# be a set intersection rather than a per-element scan.
|
|
1023
|
+
wires = frozenset(wire_names.values())
|
|
1024
|
+
consumed = frozenset(hints) | frozenset(aliases.values()) | {"_api", "_parent"}
|
|
1025
|
+
meta = (hints, wire_names, wires, consumed)
|
|
1026
|
+
setattr(cls, "_FIELD_HINTS_CACHE", (hints, meta))
|
|
1027
|
+
return meta
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
# ---------------------------------------------------------------------------
|
|
1031
|
+
# Resource — base for every generated dataclass-like
|
|
1032
|
+
# ---------------------------------------------------------------------------
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
class Resource:
|
|
1036
|
+
"""Base class for every generated resource (Post, Comment, Site, etc.).
|
|
1037
|
+
|
|
1038
|
+
Instances store their fields directly as attributes. ``_api`` is the
|
|
1039
|
+
root client; ``_parent`` is the resource that exposed this one as a
|
|
1040
|
+
sub-resource (for ``$self.parent.<field>`` references during method
|
|
1041
|
+
calls). Fields are set by plain attribute assignment (``_extra`` carries
|
|
1042
|
+
any undeclared wire fields). Instances are mutable by design — no attribute
|
|
1043
|
+
interception is installed, so writes are ordinary.
|
|
1044
|
+
"""
|
|
1045
|
+
|
|
1046
|
+
_RESOURCE_NAME: ClassVar[str] = "Resource"
|
|
1047
|
+
_KEY_FIELD: ClassVar[Optional[str]] = None
|
|
1048
|
+
# Python-name → wire-name map (set by codegen from FieldSpec.alias_from).
|
|
1049
|
+
# Lets a spec say ``created_at`` while the API returns ``created_utc``.
|
|
1050
|
+
_FIELD_ALIASES: ClassVar[Dict[str, str]] = {}
|
|
1051
|
+
# When True, this resource can be instantiated from just its key — used
|
|
1052
|
+
# by the coerce() path when a nested field's payload is a bare scalar
|
|
1053
|
+
# (e.g. ``post.subreddit = "smallbusiness"`` becomes ``Subreddit(name="smallbusiness")``).
|
|
1054
|
+
_CONSTRUCTIBLE: ClassVar[bool] = False
|
|
1055
|
+
# Resource class registry for the owning module. Codegen wires this up
|
|
1056
|
+
# at module import time so the string-label return router (``coerce()``)
|
|
1057
|
+
# can resolve a label like ``Post`` to its class.
|
|
1058
|
+
_RESOURCE_REGISTRY: ClassVar[Dict[str, Type[Resource]]] = {}
|
|
1059
|
+
# Per-class cache for resolved field annotations and derived payload metadata.
|
|
1060
|
+
# Stored directly on each generated subclass (looked up via ``cls.__dict__``),
|
|
1061
|
+
# not in a process-global dict: resolved annotations can reference sibling
|
|
1062
|
+
# resource classes, and a global cache value would pin transient preview
|
|
1063
|
+
# modules alive. A class-local cycle dies with the class/module graph. The
|
|
1064
|
+
# slot name is pre-existing/runtime-reserved; payload metadata deliberately
|
|
1065
|
+
# shares it so adding the optimization does not expand codegen's reserved
|
|
1066
|
+
# member set (which is seeded from dir(Resource)).
|
|
1067
|
+
_FIELD_HINTS_CACHE: ClassVar[Optional[Any]] = None
|
|
1068
|
+
|
|
1069
|
+
@classmethod
|
|
1070
|
+
def _field_hints(cls) -> Dict[str, Any]:
|
|
1071
|
+
"""Resolved ``{field_name: type_object}`` for this class's own
|
|
1072
|
+
annotated fields (the single per-field type source).
|
|
1073
|
+
|
|
1074
|
+
Uses ``get_type_hints(cls)`` with NO explicit ``globalns`` so forward
|
|
1075
|
+
refs resolve in the *generated module's* namespace (where ``Sort`` /
|
|
1076
|
+
``Post`` live). Filters out ClassVars by ORIGIN (``get_origin is
|
|
1077
|
+
ClassVar``) — not by name — because ``BaseAPI`` carries non-underscore
|
|
1078
|
+
ClassVars and the runtime bases carry underscore ClassVars; both must
|
|
1079
|
+
be excluded. Cached per class, resolved lazily. Never raises: a
|
|
1080
|
+
resolution failure yields an empty hint map (the codegen gate makes
|
|
1081
|
+
dangling refs impossible, but stay total).
|
|
1082
|
+
"""
|
|
1083
|
+
cached = cls.__dict__.get("_FIELD_HINTS_CACHE")
|
|
1084
|
+
if isinstance(cached, dict):
|
|
1085
|
+
return cached
|
|
1086
|
+
# The slot is shared with the ``(hints, meta)`` payload-meta form written
|
|
1087
|
+
# by ``_resource_payload_meta``; recognise the SAME shape both readers
|
|
1088
|
+
# write (don't diverge on a looser/stricter predicate).
|
|
1089
|
+
if (
|
|
1090
|
+
isinstance(cached, tuple)
|
|
1091
|
+
and len(cached) == 2
|
|
1092
|
+
and isinstance(cached[0], dict)
|
|
1093
|
+
and isinstance(cached[1], tuple)
|
|
1094
|
+
):
|
|
1095
|
+
return cached[0]
|
|
1096
|
+
try:
|
|
1097
|
+
raw_hints = get_type_hints(cls)
|
|
1098
|
+
except Exception: # noqa: BLE001 — total: never let resolution raise
|
|
1099
|
+
raw_hints = {}
|
|
1100
|
+
# Annotations declared on the runtime bases (Resource/object) are
|
|
1101
|
+
# ClassVars; the per-resource fields are the plain annotations declared
|
|
1102
|
+
# on the generated subclass. Filter ClassVars by origin.
|
|
1103
|
+
#
|
|
1104
|
+
# Flat-inheritance invariant: codegen emits ONLY ``class X(_Resource)``
|
|
1105
|
+
# (single-level — there is no ``extends``/``base`` spec key), so the
|
|
1106
|
+
# own-class ``__annotations__`` filter is correct-by-construction. An
|
|
1107
|
+
# MRO walk would defend a multi-level shape the generator cannot
|
|
1108
|
+
# produce, so we deliberately do NOT walk the MRO. (The codegen asserts
|
|
1109
|
+
# this invariant when it emits each class.)
|
|
1110
|
+
own = getattr(cls, "__annotations__", {})
|
|
1111
|
+
hints: Dict[str, Any] = {}
|
|
1112
|
+
for name, ann in raw_hints.items():
|
|
1113
|
+
if name not in own:
|
|
1114
|
+
continue # inherited base annotation, not a resource field
|
|
1115
|
+
if get_origin(ann) is ClassVar:
|
|
1116
|
+
continue
|
|
1117
|
+
hints[name] = ann
|
|
1118
|
+
setattr(cls, "_FIELD_HINTS_CACHE", hints)
|
|
1119
|
+
return hints
|
|
1120
|
+
|
|
1121
|
+
def __init__(self, *, _api: Optional[BaseAPI], _parent: Any = None, **fields: Any) -> None:
|
|
1122
|
+
# Plain assignment — there is no ``__setattr__``/``__slots__`` override,
|
|
1123
|
+
# so ``object.__setattr__`` was a no-op equal to ``self.x = v`` that
|
|
1124
|
+
# implied a frozen contract the class doesn't enforce.
|
|
1125
|
+
self._api = _api
|
|
1126
|
+
self._parent = _parent
|
|
1127
|
+
field_names = self._field_hints()
|
|
1128
|
+
for name in field_names:
|
|
1129
|
+
setattr(self, name, fields.get(name))
|
|
1130
|
+
# Stash anything passed that isn't a declared field, under _extra, so
|
|
1131
|
+
# the user can access surprise fields via ``obj._extra.get("weird")``
|
|
1132
|
+
# without losing data.
|
|
1133
|
+
self._extra = {k: v for k, v in fields.items() if k not in field_names}
|
|
1134
|
+
# Advisory husk signal: count of declared fields whose RAW wire
|
|
1135
|
+
# value was non-null. SET per-instance by ``_from_payload`` (a classmethod
|
|
1136
|
+
# that builds ``inst`` directly), NOT here — this is an annotation-only
|
|
1137
|
+
# declaration so pyright sees the instance attribute without us actually
|
|
1138
|
+
# creating it. A bare ``self.x: int`` performs no assignment, so instances
|
|
1139
|
+
# NOT built from a payload (e.g. coerce()'s key-only constructions) keep
|
|
1140
|
+
# NO attribute, and the husk read ``getattr(inst, "_resolved_count", None)``
|
|
1141
|
+
# still falls back to ``None`` for them (≠ 0 → not a husk). Declared here
|
|
1142
|
+
# rather than as a class annotation so it stays OUT of ``_field_hints`` /
|
|
1143
|
+
# ``get_type_hints`` and is never mistaken for a coercible wire field.
|
|
1144
|
+
self._resolved_count: int
|
|
1145
|
+
|
|
1146
|
+
@classmethod
|
|
1147
|
+
def _from_payload(
|
|
1148
|
+
cls,
|
|
1149
|
+
payload: Dict[str, Any],
|
|
1150
|
+
*,
|
|
1151
|
+
api: Optional[BaseAPI],
|
|
1152
|
+
parent: Any = None,
|
|
1153
|
+
) -> Resource:
|
|
1154
|
+
"""Construct an instance from a raw response dict, coercing each
|
|
1155
|
+
declared field by its resolved ANNOTATION (single source).
|
|
1156
|
+
``_FIELD_ALIASES`` lets a Python field name read from a different wire
|
|
1157
|
+
key.
|
|
1158
|
+
|
|
1159
|
+
The instance is built first (with raw values), then each field is
|
|
1160
|
+
coerced with ``parent`` set to THIS instance — so a nested resource's
|
|
1161
|
+
``_parent`` is the OWNING resource (active-record navigation), not the
|
|
1162
|
+
grandparent. ``_api`` is threaded all the way down.
|
|
1163
|
+
"""
|
|
1164
|
+
hints, wire_names, wires, consumed = _resource_payload_meta(cls)
|
|
1165
|
+
# Wrapped-record unwrap — the single-RECORD analogue of
|
|
1166
|
+
# ``_unwrap_single_list``. Detail APIs commonly nest the record under a
|
|
1167
|
+
# wrapper key (``{"product": {...}}``, ``{"detail": {...}}``); a spec
|
|
1168
|
+
# whose fields were typed from the INNER record would otherwise hydrate
|
|
1169
|
+
# to an all-None husk (2026-06-09 corpus audit: 46/244 specs). Fires
|
|
1170
|
+
# ONLY when (a) not one declared wire key resolves at the top level —
|
|
1171
|
+
# so any legitimately-flat payload is untouched — and (b) exactly ONE
|
|
1172
|
+
# dict-valued child contains at least one declared wire key (two
|
|
1173
|
+
# matching children = ambiguous, leave raw rather than guess). When a
|
|
1174
|
+
# wrapper is selected, sibling envelope metadata is still forwarded into
|
|
1175
|
+
# _extra; only the structural wrapper key is dropped.
|
|
1176
|
+
if isinstance(payload, dict) and payload and hints:
|
|
1177
|
+
if not (wires & payload.keys()):
|
|
1178
|
+
_cands = [(k, v) for k, v in payload.items()
|
|
1179
|
+
if isinstance(v, dict) and (wires & v.keys())]
|
|
1180
|
+
if len(_cands) == 1:
|
|
1181
|
+
_wrapper, _wrapped = _cands[0]
|
|
1182
|
+
_outer_extra = {k: v for k, v in payload.items() if k != _wrapper}
|
|
1183
|
+
payload = {**_outer_extra, **_wrapped}
|
|
1184
|
+
# Never forward a wire key literally named ``_api``/``_parent`` into
|
|
1185
|
+
# raw_fields: it would collide with the explicit kwargs in
|
|
1186
|
+
# ``cls(_api=..., _parent=..., **raw_fields)`` below and raise "multiple
|
|
1187
|
+
# values for keyword argument". codegen reserves these as field
|
|
1188
|
+
# identifiers, but an UNMAPPED wire key of that name would otherwise slip
|
|
1189
|
+
# through the unknown-field forwarding — drop it.
|
|
1190
|
+
raw_fields: Dict[str, Any] = {}
|
|
1191
|
+
resolved_count = 0
|
|
1192
|
+
for name in hints:
|
|
1193
|
+
raw = payload.get(wire_names[name])
|
|
1194
|
+
raw_fields[name] = raw
|
|
1195
|
+
if raw is not None:
|
|
1196
|
+
resolved_count += 1
|
|
1197
|
+
# Forward unknown fields so _extra has them — but skip wire names that
|
|
1198
|
+
# were already consumed via _FIELD_ALIASES (else we'd both set
|
|
1199
|
+
# ``created_at`` AND leak ``created_utc`` into _extra).
|
|
1200
|
+
for k, v in payload.items():
|
|
1201
|
+
if k not in consumed:
|
|
1202
|
+
raw_fields[k] = v
|
|
1203
|
+
inst = cls(_api=api, _parent=parent, **raw_fields)
|
|
1204
|
+
# Resolved-key signal (husk observation). Count the DECLARED fields
|
|
1205
|
+
# whose post-unwrap raw wire value is non-null — the husk verdict in
|
|
1206
|
+
# coerce() reads this instead of re-running the wrapped-record unwrap.
|
|
1207
|
+
# It is a private instance attribute, set on EVERY constructed instance,
|
|
1208
|
+
# and is purely advisory: _from_payload itself NEVER raises on it (the
|
|
1209
|
+
# verdict lives in coerce(), which is NOT shared with the lenient
|
|
1210
|
+
# per-field router). Computed from the RAW wire values (pre-coercion) so
|
|
1211
|
+
# a lenient coercion that degrades a value to None can't mask a key that
|
|
1212
|
+
# the wire actually resolved.
|
|
1213
|
+
inst._resolved_count = resolved_count
|
|
1214
|
+
# Coerce annotated fields now that the owning instance exists, threading
|
|
1215
|
+
# it as the parent of every nested + list/dict element. Plain assignment
|
|
1216
|
+
# (no __setattr__ override to bypass).
|
|
1217
|
+
key_field = cls._KEY_FIELD
|
|
1218
|
+
for name, ann in hints.items():
|
|
1219
|
+
coerced = _coerce_by_annotation(raw_fields[name], ann, api=api, parent=inst,
|
|
1220
|
+
is_key=(name == key_field))
|
|
1221
|
+
setattr(inst, name, coerced)
|
|
1222
|
+
return inst
|
|
1223
|
+
|
|
1224
|
+
def __repr__(self) -> str:
|
|
1225
|
+
key = self._KEY_FIELD
|
|
1226
|
+
if key and hasattr(self, key):
|
|
1227
|
+
return f"{type(self).__name__}({key}={getattr(self, key)!r})"
|
|
1228
|
+
return f"{type(self).__name__}(...)"
|
|
1229
|
+
|
|
1230
|
+
def _identity_key(self) -> Any:
|
|
1231
|
+
"""The hashable identity value and its hash, or ``None`` when this instance has no
|
|
1232
|
+
usable identity: no ``_KEY_FIELD``; a null key value (a null never
|
|
1233
|
+
asserts identity — two null-keyed instances are NOT the same record);
|
|
1234
|
+
or an unhashable container that leaked into the key field through
|
|
1235
|
+
lenient coercion. ``__eq__`` and ``__hash__`` both route through this
|
|
1236
|
+
single source so they can never disagree, and ``__hash__`` can reuse
|
|
1237
|
+
the hashability probe instead of calling ``hash(val)`` twice."""
|
|
1238
|
+
key = self._KEY_FIELD
|
|
1239
|
+
if not key:
|
|
1240
|
+
return None
|
|
1241
|
+
val = getattr(self, key, None)
|
|
1242
|
+
if val is None:
|
|
1243
|
+
return None
|
|
1244
|
+
try:
|
|
1245
|
+
val_hash = hash(val)
|
|
1246
|
+
except TypeError:
|
|
1247
|
+
return None
|
|
1248
|
+
return val, val_hash
|
|
1249
|
+
|
|
1250
|
+
def __eq__(self, other: Any) -> bool:
|
|
1251
|
+
if not isinstance(other, type(self)):
|
|
1252
|
+
return NotImplemented
|
|
1253
|
+
mine = self._identity_key()
|
|
1254
|
+
theirs = other._identity_key()
|
|
1255
|
+
if mine is None or theirs is None:
|
|
1256
|
+
return self is other
|
|
1257
|
+
return mine[0] == theirs[0]
|
|
1258
|
+
|
|
1259
|
+
def __hash__(self) -> int:
|
|
1260
|
+
identity = self._identity_key()
|
|
1261
|
+
if identity is None:
|
|
1262
|
+
return id(self)
|
|
1263
|
+
return hash((type(self).__name__, identity[1]))
|
|
1264
|
+
|
|
1265
|
+
|
|
1266
|
+
# ---------------------------------------------------------------------------
|
|
1267
|
+
# Collection — typed namespace for sub-resources (`site.subareas`, `reddit.posts`)
|
|
1268
|
+
# ---------------------------------------------------------------------------
|
|
1269
|
+
|
|
1270
|
+
|
|
1271
|
+
class Collection:
|
|
1272
|
+
"""A typed handle on a sub-resource or top-level resource collection.
|
|
1273
|
+
|
|
1274
|
+
The codegen subclasses this per collection so static type checkers see
|
|
1275
|
+
typed ``.get(...)`` / ``.list(...)`` etc. methods. The runtime base
|
|
1276
|
+
just holds the API + parent references.
|
|
1277
|
+
"""
|
|
1278
|
+
|
|
1279
|
+
def __init__(self, _api: BaseAPI, _parent: Any = None) -> None:
|
|
1280
|
+
self._api = _api
|
|
1281
|
+
self._parent = _parent
|
|
1282
|
+
|
|
1283
|
+
|
|
1284
|
+
# ---------------------------------------------------------------------------
|
|
1285
|
+
# Paginator — auto-paging iterator returned by `paginates`-tagged operations
|
|
1286
|
+
# ---------------------------------------------------------------------------
|
|
1287
|
+
|
|
1288
|
+
|
|
1289
|
+
_PaginatorItem = TypeVar("_PaginatorItem")
|
|
1290
|
+
|
|
1291
|
+
|
|
1292
|
+
def _resource_content_projection(it: "Resource") -> Any:
|
|
1293
|
+
"""A STABLE, hashable content signature for a keyless / null-keyed
|
|
1294
|
+
``Resource`` — the runtime fix for the keyless ``_page_signature`` churn.
|
|
1295
|
+
|
|
1296
|
+
A keyless ``Resource`` (``_KEY_FIELD = None``) hashes by ``id(self)``, so the
|
|
1297
|
+
historical signature churned and the content-stall guard NEVER matched a
|
|
1298
|
+
keyless full-page repeat across pages. Two DISTINCT objects carrying the SAME
|
|
1299
|
+
wire content must project EQUAL so the guard can see that repeat; two
|
|
1300
|
+
genuinely-different keyless rows must project differently (no false trigger).
|
|
1301
|
+
|
|
1302
|
+
Content is mined from the resource's declared fields (the single per-class
|
|
1303
|
+
source) + any undeclared wire data in ``_extra``, sorted by name so dict
|
|
1304
|
+
order can't perturb the projection. Each value is taken verbatim when
|
|
1305
|
+
hashable, else rendered to a stable JSON string (``sort_keys``) so an
|
|
1306
|
+
unhashable list/dict field can't break signing; an exotic value falls back
|
|
1307
|
+
to ``repr``. The result is ALWAYS hashable — a keyless Resource therefore
|
|
1308
|
+
always signs (it never forces the whole page to ``None``).
|
|
1309
|
+
|
|
1310
|
+
Scope is deliberately KEYLESS RESOURCES ONLY: a raw dict/list passthrough
|
|
1311
|
+
keeps the historical "unhashable → page unsignable → ``None``" behavior in
|
|
1312
|
+
``_page_signature`` (no content-stall compare), so a heterogeneous
|
|
1313
|
+
keyed-then-raw page stream still terminates at ``max_fetches`` rather than
|
|
1314
|
+
newly raising — preserving cross-compat for every already-generated module."""
|
|
1315
|
+
def _stable(v: Any) -> Any:
|
|
1316
|
+
# A NESTED Resource value must canonicalize by its CONTENT, not be
|
|
1317
|
+
# returned verbatim: a keyless Resource hashes by ``id(self)`` (so the
|
|
1318
|
+
# ``hash(v)`` probe below would succeed and return the churning instance),
|
|
1319
|
+
# which made a keyless value-object field churn across pages and left the
|
|
1320
|
+
# content-stall guard ineffective for nested-value-object shapes (codex
|
|
1321
|
+
# P2). Recurse into its content projection instead. A list/tuple is
|
|
1322
|
+
# canonicalized element-wise so a ``List[Resource]`` value-object field
|
|
1323
|
+
# canonicalizes too; raw dict/list (non-Resource) keeps the historical
|
|
1324
|
+
# json.dumps passthrough below.
|
|
1325
|
+
if isinstance(v, Resource):
|
|
1326
|
+
return _resource_content_projection(v)
|
|
1327
|
+
if isinstance(v, (list, tuple)):
|
|
1328
|
+
return tuple(_stable(x) for x in v)
|
|
1329
|
+
try:
|
|
1330
|
+
hash(v)
|
|
1331
|
+
return v
|
|
1332
|
+
except TypeError:
|
|
1333
|
+
pass
|
|
1334
|
+
try:
|
|
1335
|
+
return json.dumps(v, sort_keys=True, default=str)
|
|
1336
|
+
except (TypeError, ValueError):
|
|
1337
|
+
return repr(v)
|
|
1338
|
+
|
|
1339
|
+
fields = it._field_hints()
|
|
1340
|
+
parts: List[tuple] = [(name, _stable(getattr(it, name, None))) for name in fields]
|
|
1341
|
+
extra = getattr(it, "_extra", None)
|
|
1342
|
+
if isinstance(extra, dict):
|
|
1343
|
+
parts += [(f"_extra:{k}", _stable(v)) for k, v in extra.items()]
|
|
1344
|
+
return ("R", type(it).__name__, tuple(sorted(parts)))
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
def _page_signature(items: Iterable[Any]) -> Optional[tuple]:
|
|
1348
|
+
"""An ordered tuple of per-item identity signals for one page, or ``None``
|
|
1349
|
+
if the page can't be signed.
|
|
1350
|
+
|
|
1351
|
+
Each item's signal is its ``_KEY_FIELD`` value when the item is a keyed
|
|
1352
|
+
``Resource``; a keyless / null-keyed ``Resource`` signs by a STABLE CONTENT
|
|
1353
|
+
PROJECTION (:func:`_resource_content_projection`) instead of the historical
|
|
1354
|
+
``id(self)`` churn — so the content-stall guard can now catch a keyless
|
|
1355
|
+
full-page repeat (two pages of distinct-object-but-same-content keyless rows
|
|
1356
|
+
project EQUAL; genuinely-different keyless content projects differently, no
|
|
1357
|
+
false trigger). A NON-Resource item (raw dict / list / scalar passthrough)
|
|
1358
|
+
keeps the historical rule: hashable → the item itself; unhashable → the
|
|
1359
|
+
whole page is unsignable and we return ``None`` (the caller skips the
|
|
1360
|
+
content-stall compare). That scope keeps a heterogeneous keyed-then-raw
|
|
1361
|
+
stream cross-compatible — a raw-dict page still never signs, so it can't
|
|
1362
|
+
newly raise on an already-generated module.
|
|
1363
|
+
|
|
1364
|
+
Ordered (not a set) so a reordered page reads as different — the strictest
|
|
1365
|
+
same-page signal. For KEYED items the signal is the key only (never full
|
|
1366
|
+
content) so volatile non-key fields (vote counts, timestamps) can't mask a
|
|
1367
|
+
stall; for KEYLESS RESOURCES full content is the only available identity, so
|
|
1368
|
+
the documented residual stands: a keyless feed whose entire page genuinely
|
|
1369
|
+
re-appears within the recent-signature window reads as a stall (the error
|
|
1370
|
+
names the ``max_fetches=`` override).
|
|
1371
|
+
"""
|
|
1372
|
+
sig: List[Any] = []
|
|
1373
|
+
for it in items:
|
|
1374
|
+
kf = getattr(type(it), "_KEY_FIELD", None)
|
|
1375
|
+
if kf:
|
|
1376
|
+
# A KEYED Resource: sign by its key value when present. A NULL key on
|
|
1377
|
+
# this page (key only on detail responses, or an alias mis-mapping)
|
|
1378
|
+
# keeps the historical id-hash behavior — two pages of DISTINCT
|
|
1379
|
+
# null-keyed-on-list records must NOT collapse to one signature and
|
|
1380
|
+
# false-stall (``test_null_keyed_pages_do_not_false_stall``). The
|
|
1381
|
+
# content-projection fix is scoped to GENUINELY KEYLESS resources
|
|
1382
|
+
# below, where id-hash actively hid a real repeat; a keyed-but-
|
|
1383
|
+
# null-on-this-page record is a distinct record whose key simply
|
|
1384
|
+
# isn't on the list page — the cross-compat shape we must not regress.
|
|
1385
|
+
key = getattr(it, kf, None)
|
|
1386
|
+
if key is not None:
|
|
1387
|
+
sig.append(key)
|
|
1388
|
+
continue
|
|
1389
|
+
try:
|
|
1390
|
+
hash(it)
|
|
1391
|
+
except TypeError:
|
|
1392
|
+
return None
|
|
1393
|
+
sig.append(it)
|
|
1394
|
+
continue
|
|
1395
|
+
if isinstance(it, Resource):
|
|
1396
|
+
# Keyless Resource (``_KEY_FIELD is None``): sign by stable content so
|
|
1397
|
+
# a keyless full-page repeat is visible to the stall guard (the
|
|
1398
|
+
# fix — id-hash churned and hid it). Distinct keyless content still
|
|
1399
|
+
# projects differently; the accepted residual is a genuinely
|
|
1400
|
+
# content-identical keyless full page reading as a stall.
|
|
1401
|
+
sig.append(_resource_content_projection(it))
|
|
1402
|
+
continue
|
|
1403
|
+
# Non-Resource passthrough (raw dict/list/scalar) — historical rule:
|
|
1404
|
+
# hashable → the item; unhashable → whole page unsignable (``None``).
|
|
1405
|
+
try:
|
|
1406
|
+
hash(it)
|
|
1407
|
+
except TypeError:
|
|
1408
|
+
return None
|
|
1409
|
+
sig.append(it)
|
|
1410
|
+
return tuple(sig)
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
class Paginator(Generic[_PaginatorItem]):
|
|
1414
|
+
"""Iterator that fetches more pages as the user consumes them.
|
|
1415
|
+
|
|
1416
|
+
Build kwargs contain the original method args; ``_fetch_page`` returns
|
|
1417
|
+
``(items, next_cursor)``. Iteration stops when ``next_cursor`` is None
|
|
1418
|
+
or the user breaks. Cap by passing ``limit=`` at call time — the
|
|
1419
|
+
paginator will short-circuit once that many items have been yielded.
|
|
1420
|
+
|
|
1421
|
+
Generic so codegen can emit ``Paginator[Listing]`` / ``Paginator[Post]``
|
|
1422
|
+
return annotations. Without that, ``for listing in api.listings.search()``
|
|
1423
|
+
widens ``listing`` to ``Any`` and pyright loses all field-access
|
|
1424
|
+
intelligence — defeating the whole point of the typed SDK.
|
|
1425
|
+
|
|
1426
|
+
**Stall safety.** Two disjoint terminators stop a runaway paginator
|
|
1427
|
+
(a server that re-returns the same page because its pagination contract
|
|
1428
|
+
was mis-derived):
|
|
1429
|
+
|
|
1430
|
+
* a **content-stall guard** — measures progress by the *data* (an
|
|
1431
|
+
ordered tuple of per-item keys). When a non-empty page repeats ANY of
|
|
1432
|
+
the last 4 page signatures it raises :class:`PaginationLimitError` —
|
|
1433
|
+
so a same-page stall dies on the 2nd fetch and a short-period cycle
|
|
1434
|
+
(the realistic mis-derived page/offset oscillation) on the 3rd-5th,
|
|
1435
|
+
instead of burning the full billable fetch budget. Cursor-agnostic
|
|
1436
|
+
(the SDK fabricates an advancing cursor for token-less servers, so
|
|
1437
|
+
the cursor is useless as a progress signal); unsignable pages
|
|
1438
|
+
(raw-dict passthrough) are simply invisible to it, never a false
|
|
1439
|
+
trigger. Residual false-positive shape: a burst of ≈k·page_size new
|
|
1440
|
+
items landing between two fetches of an offset-paginated moving feed
|
|
1441
|
+
can make page N+k repeat page N's keys exactly — rare, and the error
|
|
1442
|
+
names the ``max_fetches=`` override an agent can act on.
|
|
1443
|
+
* a **``max_fetches`` iteration bound** — the sole terminator for the
|
|
1444
|
+
cases the content guard cannot see (slow-crawl, unhashable streams,
|
|
1445
|
+
cycles with period > 4). User-overridable; raise it for legitimate
|
|
1446
|
+
deep pagination.
|
|
1447
|
+
|
|
1448
|
+
These cover *disjoint* failure modes — not two layers over one. A
|
|
1449
|
+
previously-hanging paginator now *raises* instead; this terminal outcome
|
|
1450
|
+
applies to already-rendered modules on a runtime bump (they import
|
|
1451
|
+
``Paginator`` from here) without regeneration. The ``max_fetches`` arm
|
|
1452
|
+
also raises on a *legitimately-completing* result set that exceeds the
|
|
1453
|
+
budget (not only hanging ones) — an unbounded ``.list()`` over >1000 pages
|
|
1454
|
+
must pass a larger ``max_fetches=``; the error message names the override.
|
|
1455
|
+
|
|
1456
|
+
**Window exhaustion.** A third terminator covers the upstream's own result
|
|
1457
|
+
window: a mid-stream 4xx (after ≥1 successful page, excluding 429
|
|
1458
|
+
backpressure) means the site refuses deeper pages — e.g. GitHub search
|
|
1459
|
+
serves at most 1000 results regardless of ``total``. That raises
|
|
1460
|
+
:class:`PaginationLimitError` (with the ``UpstreamError`` as ``__cause__``)
|
|
1461
|
+
rather than stopping silently: yielded items are valid, but treating a cap
|
|
1462
|
+
as "the result set ends here" would hand the consumer a wrong-data
|
|
1463
|
+
conclusion. First-fetch errors and 5xx/network failures surface raw.
|
|
1464
|
+
"""
|
|
1465
|
+
|
|
1466
|
+
def __init__(
|
|
1467
|
+
self,
|
|
1468
|
+
fetch_page: Callable[[Optional[Any]], "tuple[List[_PaginatorItem], Optional[Any]]"],
|
|
1469
|
+
*,
|
|
1470
|
+
limit: Optional[int] = None,
|
|
1471
|
+
max_fetches: int = 1000,
|
|
1472
|
+
dedup: bool = False,
|
|
1473
|
+
identity_dedup: bool = False,
|
|
1474
|
+
_api: Optional[Any] = None,
|
|
1475
|
+
) -> None:
|
|
1476
|
+
# Validate here, not in __next__: the constructor is the single
|
|
1477
|
+
# chokepoint every generated op routes through, and a bad kwarg must
|
|
1478
|
+
# fail at the call site that supplied it — not pages later inside
|
|
1479
|
+
# iteration with a bare comparison TypeError. bool is excluded
|
|
1480
|
+
# explicitly (it passes isinstance(int) but `limit=True` is a bug).
|
|
1481
|
+
if limit is not None and (not isinstance(limit, int) or isinstance(limit, bool)):
|
|
1482
|
+
raise TypeError(
|
|
1483
|
+
f"limit must be an int or None, got {limit!r} ({type(limit).__name__})"
|
|
1484
|
+
)
|
|
1485
|
+
if limit is not None and limit < 1:
|
|
1486
|
+
raise ValueError(f"limit must be >= 1, got {limit!r}")
|
|
1487
|
+
if not isinstance(max_fetches, int) or isinstance(max_fetches, bool):
|
|
1488
|
+
raise TypeError(
|
|
1489
|
+
f"max_fetches must be an int, got {max_fetches!r} ({type(max_fetches).__name__})"
|
|
1490
|
+
)
|
|
1491
|
+
if max_fetches < 1:
|
|
1492
|
+
raise ValueError(f"max_fetches must be >= 1, got {max_fetches!r}")
|
|
1493
|
+
# identity_dedup is the raw-page-signature arm layered ON TOP of dedup
|
|
1494
|
+
# (consumed only inside ``if self._dedup:`` in __next__). Passing it with
|
|
1495
|
+
# dedup=False would silently no-op the entire stall-guard, so reject the
|
|
1496
|
+
# combination here rather than let the flag be invisibly inert at runtime.
|
|
1497
|
+
if identity_dedup and not dedup:
|
|
1498
|
+
raise ValueError("identity_dedup=True requires dedup=True")
|
|
1499
|
+
self._fetch_page = fetch_page
|
|
1500
|
+
self._api = _api # owning client, for last_meta passthrough
|
|
1501
|
+
self._limit = limit
|
|
1502
|
+
self._max_fetches = max_fetches
|
|
1503
|
+
# dedup-by-identity: a scheme that can re-serve overlapping items across
|
|
1504
|
+
# pages (time_cursor's inclusive-boundary re-seed; a page/cursor op on a
|
|
1505
|
+
# moving feed). The next request re-seeds and the boundary/overlap items
|
|
1506
|
+
# repeat and would be yielded twice — the content-stall guard only catches
|
|
1507
|
+
# WHOLE-page repeats, not partial overlap. With dedup on, items already
|
|
1508
|
+
# seen (by their _Resource identity / _KEY_FIELD) are dropped, and a
|
|
1509
|
+
# NON-EMPTY page contributing 0 NEW items terminates gracefully —
|
|
1510
|
+
# generalizing the content-stall guard and killing the infinite re-seed.
|
|
1511
|
+
#
|
|
1512
|
+
# F-2 carve-out: a genuinely EMPTY raw page (0 server rows) with a still-
|
|
1513
|
+
# advancing cursor is NOT a 0-new terminus — it is an interstitial gap
|
|
1514
|
+
# that gets RE-FETCHED (matching the non-dedup path), so a single empty
|
|
1515
|
+
# interstitial page no longer silently truncates every page after it. The
|
|
1516
|
+
# 0-new STOP fires only for a NON-empty page whose every row is already
|
|
1517
|
+
# seen (the inclusive-boundary re-seed). See ``__next__``.
|
|
1518
|
+
#
|
|
1519
|
+
# F-3 accepted residual (dedup WITHOUT server cooperation): when the
|
|
1520
|
+
# server's cursor keeps advancing on a NON-empty page whose rows are all
|
|
1521
|
+
# already-seen overlap, the SDK cannot tell that 0-new terminus apart
|
|
1522
|
+
# from a transient overlap that a LATER page would break — both look
|
|
1523
|
+
# identical at the moment of decision (the server gives no end signal).
|
|
1524
|
+
# We take the 0-new STOP (the only contract that terminates the infinite
|
|
1525
|
+
# inclusive-boundary re-seed); a feed that re-serves a fully-overlapping
|
|
1526
|
+
# page mid-stream and only then advances to fresh rows is the documented
|
|
1527
|
+
# residual. It is bounded and safe (it never yields wrong data — it can
|
|
1528
|
+
# only under-read), and a consumer who needs the deeper rows passes a
|
|
1529
|
+
# wider window via a larger ``max_fetches`` on the non-dedup path. Not
|
|
1530
|
+
# over-engineered with a speculative look-ahead.
|
|
1531
|
+
self._dedup = dedup
|
|
1532
|
+
# identity_dedup: the page/cursor identity-dedup arm (the
|
|
1533
|
+
# sample-proven-unique ops) adds the raw-page-signature raise/terminate
|
|
1534
|
+
# rule ON TOP of dedup — BEFORE deduping, a RAW page whose signature
|
|
1535
|
+
# repeats a recent one is a genuine re-served page → raise
|
|
1536
|
+
# PaginationLimitError; a 0-fresh-after-dedup page whose raw signature is
|
|
1537
|
+
# NOVEL is a natural overlap-boundary end → StopIteration. time_cursor
|
|
1538
|
+
# emits ``dedup=True`` ALONE (``identity_dedup`` defaults False), so it
|
|
1539
|
+
# keeps its graceful 0-new stop unchanged — and every ALREADY-GENERATED
|
|
1540
|
+
# time_cursor module (which passes only ``dedup=True``) is behavior- and
|
|
1541
|
+
# byte-stable on a runtime bump (the new flag is opt-in, default-off).
|
|
1542
|
+
self._identity_dedup = identity_dedup
|
|
1543
|
+
self._seen: set = set()
|
|
1544
|
+
self._yielded = 0
|
|
1545
|
+
self._fetches = 0
|
|
1546
|
+
self._cursor: Optional[Any] = None
|
|
1547
|
+
self._buffer: deque[_PaginatorItem] = deque()
|
|
1548
|
+
# Last 4 page signatures (not just the previous one): membership is a
|
|
1549
|
+
# linear ``in`` over ≤4 tuples — signatures may hold unhashable key
|
|
1550
|
+
# values, so a set is not an option (and is not needed at this size).
|
|
1551
|
+
self._recent_sigs: "deque[tuple]" = deque(maxlen=4)
|
|
1552
|
+
self._done = False
|
|
1553
|
+
|
|
1554
|
+
@property
|
|
1555
|
+
def pages_fetched(self) -> int:
|
|
1556
|
+
"""Number of pages fetched from the wire so far."""
|
|
1557
|
+
return self._fetches
|
|
1558
|
+
|
|
1559
|
+
@property
|
|
1560
|
+
def items_yielded(self) -> int:
|
|
1561
|
+
"""Number of items yielded to the caller so far."""
|
|
1562
|
+
return self._yielded
|
|
1563
|
+
|
|
1564
|
+
@property
|
|
1565
|
+
def exhausted(self) -> bool:
|
|
1566
|
+
"""True once iteration has terminated (no more pages)."""
|
|
1567
|
+
return self._done
|
|
1568
|
+
|
|
1569
|
+
@property
|
|
1570
|
+
def last_meta(self) -> Optional["RequestMeta"]:
|
|
1571
|
+
"""Control-loop metadata from the owning client's most recent page."""
|
|
1572
|
+
return getattr(self._api, "last_meta", None)
|
|
1573
|
+
|
|
1574
|
+
def __iter__(self) -> "Paginator[_PaginatorItem]":
|
|
1575
|
+
return self
|
|
1576
|
+
|
|
1577
|
+
def __next__(self) -> _PaginatorItem:
|
|
1578
|
+
if self._limit is not None and self._yielded >= self._limit:
|
|
1579
|
+
raise StopIteration
|
|
1580
|
+
while not self._buffer:
|
|
1581
|
+
if self._done:
|
|
1582
|
+
raise StopIteration
|
|
1583
|
+
if self._fetches >= self._max_fetches:
|
|
1584
|
+
raise PaginationLimitError(
|
|
1585
|
+
f"pagination exceeded max_fetches={self._max_fetches} — likely a "
|
|
1586
|
+
f"mis-derived pagination contract, or raise max_fetches= for "
|
|
1587
|
+
f"legitimate deep pagination",
|
|
1588
|
+
pages_fetched=self._fetches, items_yielded=self._yielded,
|
|
1589
|
+
)
|
|
1590
|
+
try:
|
|
1591
|
+
items, next_cursor = self._fetch_page(self._cursor)
|
|
1592
|
+
except UpstreamError as err:
|
|
1593
|
+
# Window-exhaustion classification. A mid-stream upstream 4xx —
|
|
1594
|
+
# the same params that already succeeded N times, refused one
|
|
1595
|
+
# cursor deeper — is a data boundary (a result-window cap, e.g.
|
|
1596
|
+
# GitHub search caps at 1000), not a transient failure: raise
|
|
1597
|
+
# the typed pagination error with the evidence chained, never
|
|
1598
|
+
# a raw UpstreamError after minutes of valid iteration.
|
|
1599
|
+
# Carve-outs, both deliberate:
|
|
1600
|
+
# * first fetch (_fetches == 0) never classifies — a param/
|
|
1601
|
+
# auth error must surface exactly as on a non-paginated op;
|
|
1602
|
+
# * upstream 429 is backpressure, not a boundary — classify-
|
|
1603
|
+
# ing it would tell the consumer the result set ends here
|
|
1604
|
+
# (the wrong-data failure this design exists to prevent).
|
|
1605
|
+
# The SDK's retry machinery covers only GATEWAY 429s, so
|
|
1606
|
+
# the upstream-429-in-502-envelope surfaces raw.
|
|
1607
|
+
# A genuine page-N-only 4xx is indistinguishable from a window
|
|
1608
|
+
# cap; after N identical-param successes the boundary is the
|
|
1609
|
+
# correct bet, and __cause__ preserves the evidence either way.
|
|
1610
|
+
usc = err.upstream_status_code
|
|
1611
|
+
if (
|
|
1612
|
+
self._fetches >= 1
|
|
1613
|
+
and isinstance(usc, int)
|
|
1614
|
+
and 400 <= usc < 500
|
|
1615
|
+
and usc != 429
|
|
1616
|
+
):
|
|
1617
|
+
raise PaginationLimitError(
|
|
1618
|
+
f"upstream result window exhausted after "
|
|
1619
|
+
f"{self._yielded} items / {self._fetches} pages "
|
|
1620
|
+
f"(upstream HTTP {usc}); items already yielded are "
|
|
1621
|
+
f"valid — catch PaginationLimitError or pass limit= "
|
|
1622
|
+
f"to bound the scan",
|
|
1623
|
+
pages_fetched=self._fetches, items_yielded=self._yielded,
|
|
1624
|
+
) from err
|
|
1625
|
+
raise
|
|
1626
|
+
self._fetches += 1
|
|
1627
|
+
self._buffer = deque(items or [])
|
|
1628
|
+
self._cursor = next_cursor
|
|
1629
|
+
if next_cursor is None:
|
|
1630
|
+
self._done = True
|
|
1631
|
+
if self._dedup:
|
|
1632
|
+
# Whether the server returned an EMPTY page (0 raw rows) — distinct
|
|
1633
|
+
# from a non-empty page that DEDUPS to 0 fresh. An empty raw page
|
|
1634
|
+
# with an advancing cursor is an interstitial gap that must be
|
|
1635
|
+
# RE-FETCHED (the non-dedup path does so via the `while not
|
|
1636
|
+
# self._buffer` loop below); only a non-empty page whose every row
|
|
1637
|
+
# is already seen is the inclusive-boundary 0-new terminus. F-2:
|
|
1638
|
+
# the old dedup arm conflated the two and stopped on the empty
|
|
1639
|
+
# interstitial page, silently dropping every page after it.
|
|
1640
|
+
_raw_empty = not self._buffer
|
|
1641
|
+
if self._identity_dedup:
|
|
1642
|
+
# raw-page-signature raise/terminate (page/cursor identity
|
|
1643
|
+
# dedup only — time_cursor keeps its graceful 0-new stop).
|
|
1644
|
+
# Sign the RAW (pre-dedup) page and reuse the SAME _recent_sigs
|
|
1645
|
+
# the content-stall guard owns: a raw page whose signature
|
|
1646
|
+
# repeats a recent one is a GENUINE re-served page (the cursor
|
|
1647
|
+
# stalled) → raise, exactly as the non-dedup content guard
|
|
1648
|
+
# already does for a repeated non-empty page. A 0-fresh page
|
|
1649
|
+
# whose raw signature is NOVEL is a natural overlap-boundary
|
|
1650
|
+
# end → falls through to the graceful 0-new stop below. (An
|
|
1651
|
+
# EMPTY raw page signs as the empty tuple `()`; guarding the
|
|
1652
|
+
# raise on `_raw_sig` truthiness keeps a bare interstitial
|
|
1653
|
+
# empty page from ever being mis-read as a re-served page.)
|
|
1654
|
+
_raw_sig = _page_signature(self._buffer)
|
|
1655
|
+
if _raw_sig:
|
|
1656
|
+
if _raw_sig in self._recent_sigs:
|
|
1657
|
+
raise PaginationLimitError(
|
|
1658
|
+
"pagination stalled — the server re-served the same "
|
|
1659
|
+
"page of items before dedup; likely a mis-derived "
|
|
1660
|
+
"pagination contract, or raise max_fetches= for "
|
|
1661
|
+
"legitimate deep pagination",
|
|
1662
|
+
pages_fetched=self._fetches, items_yielded=self._yielded,
|
|
1663
|
+
)
|
|
1664
|
+
self._recent_sigs.append(_raw_sig)
|
|
1665
|
+
# Drop items already seen (by _Resource identity / _KEY_FIELD; a
|
|
1666
|
+
# _KEY_FIELD=None resource hashes by id(self) → never collides, so
|
|
1667
|
+
# no data loss). A non-empty page contributing 0 NEW items
|
|
1668
|
+
# terminates gracefully — this supersedes the content-stall check
|
|
1669
|
+
# on the time_cursor path (it handles PARTIAL overlap, which the
|
|
1670
|
+
# whole-page stall guard cannot). max_fetches stays the backstop.
|
|
1671
|
+
fresh: List[_PaginatorItem] = []
|
|
1672
|
+
for it in self._buffer:
|
|
1673
|
+
if isinstance(it, Resource):
|
|
1674
|
+
# Dedup on the identity TOKEN, not the instance — the
|
|
1675
|
+
# set otherwise retains every yielded Resource (full
|
|
1676
|
+
# payload) for the whole iteration. ``_identity_key``
|
|
1677
|
+
# is the single equality source: when it is None (no
|
|
1678
|
+
# key field / null key / unhashable key) the instance
|
|
1679
|
+
# has no identity and membership could never fire
|
|
1680
|
+
# (id-hash vs fresh objects), so skip dedup for it —
|
|
1681
|
+
# observationally identical, retention dropped.
|
|
1682
|
+
idk = it._identity_key()
|
|
1683
|
+
if idk is not None:
|
|
1684
|
+
token = (type(it), idk[0])
|
|
1685
|
+
if token in self._seen:
|
|
1686
|
+
continue
|
|
1687
|
+
self._seen.add(token)
|
|
1688
|
+
fresh.append(it)
|
|
1689
|
+
continue
|
|
1690
|
+
try:
|
|
1691
|
+
if it in self._seen:
|
|
1692
|
+
continue
|
|
1693
|
+
self._seen.add(it)
|
|
1694
|
+
except TypeError:
|
|
1695
|
+
pass # unhashable item → never deduped (fall back to bounds)
|
|
1696
|
+
fresh.append(it)
|
|
1697
|
+
self._buffer = deque(fresh)
|
|
1698
|
+
if not self._buffer:
|
|
1699
|
+
if _raw_empty:
|
|
1700
|
+
# The RAW page was empty — an interstitial gap, NOT the
|
|
1701
|
+
# inclusive-boundary terminus. Re-fetch if the cursor is
|
|
1702
|
+
# still live (the non-dedup behavior); stop only when the
|
|
1703
|
+
# cursor itself ended (`self._done`). Bounded by
|
|
1704
|
+
# max_fetches above, exactly like the non-dedup empty-page
|
|
1705
|
+
# path. This is the F-2 data-loss fix.
|
|
1706
|
+
if self._done:
|
|
1707
|
+
raise StopIteration
|
|
1708
|
+
continue
|
|
1709
|
+
# A NON-EMPTY page that dedups to 0 fresh items: a fully-seen
|
|
1710
|
+
# re-seed (inclusive boundary). Stop — re-fetching would
|
|
1711
|
+
# re-seed the same cursor identically (infinite loop). This is
|
|
1712
|
+
# time_cursor's graceful 0-new terminus.
|
|
1713
|
+
self._done = True
|
|
1714
|
+
raise StopIteration
|
|
1715
|
+
continue # dedup path owns its own termination; skip stall guard
|
|
1716
|
+
if not self._buffer:
|
|
1717
|
+
if self._done:
|
|
1718
|
+
raise StopIteration
|
|
1719
|
+
# else: empty page but more cursors — re-fetch (bounded by
|
|
1720
|
+
# max_fetches above; excluded from the content compare so a
|
|
1721
|
+
# fabricated-cursor empty-page re-fetch isn't misclassified).
|
|
1722
|
+
continue
|
|
1723
|
+
# Content-stall guard — recomputed per page (no cached capability
|
|
1724
|
+
# flag). Only compares when the page is signable; an unsignable
|
|
1725
|
+
# page leaves the window untouched (appending a None would let a
|
|
1726
|
+
# later None == None false-trigger on a legit stream). Matching
|
|
1727
|
+
# ANY of the last 4 signatures catches the same-page stall (lag 1)
|
|
1728
|
+
# AND the short-period cycle (a mis-derived page/offset contract
|
|
1729
|
+
# oscillating between pages) at fetch ~3 instead of burning the
|
|
1730
|
+
# whole max_fetches budget of billable requests.
|
|
1731
|
+
sig = _page_signature(self._buffer)
|
|
1732
|
+
if sig is not None:
|
|
1733
|
+
if sig in self._recent_sigs:
|
|
1734
|
+
raise PaginationLimitError(
|
|
1735
|
+
"pagination stalled — the server re-returned the same page "
|
|
1736
|
+
"of items; likely a mis-derived pagination contract, or raise "
|
|
1737
|
+
"max_fetches= for legitimate deep pagination",
|
|
1738
|
+
pages_fetched=self._fetches, items_yielded=self._yielded,
|
|
1739
|
+
)
|
|
1740
|
+
self._recent_sigs.append(sig)
|
|
1741
|
+
v = self._buffer.popleft()
|
|
1742
|
+
self._yielded += 1
|
|
1743
|
+
return v
|
|
1744
|
+
|
|
1745
|
+
def list(self) -> List[_PaginatorItem]:
|
|
1746
|
+
"""Materialize all pages into a list. Convenient when you know the
|
|
1747
|
+
result set is small. Respects the ``limit`` cap.
|
|
1748
|
+
|
|
1749
|
+
On a :class:`PaginationLimitError` the items collected so far are
|
|
1750
|
+
attached as ``err.partial_items`` before the error propagates — the
|
|
1751
|
+
materialized prefix is valid data and must not vanish with the raise.
|
|
1752
|
+
(Raw ``list(paginator)`` / comprehensions cannot offer this carry;
|
|
1753
|
+
prefer ``.list()``.)"""
|
|
1754
|
+
out: List[_PaginatorItem] = []
|
|
1755
|
+
try:
|
|
1756
|
+
for v in self:
|
|
1757
|
+
out.append(v)
|
|
1758
|
+
except PaginationLimitError as err:
|
|
1759
|
+
err.partial_items = list(out)
|
|
1760
|
+
raise
|
|
1761
|
+
return out
|
|
1762
|
+
|
|
1763
|
+
def first(self) -> Optional[_PaginatorItem]:
|
|
1764
|
+
"""Return the first item or None. Stops paging after that item."""
|
|
1765
|
+
for v in self:
|
|
1766
|
+
return v
|
|
1767
|
+
return None
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
# ---------------------------------------------------------------------------
|
|
1771
|
+
# BaseAPI — root client class
|
|
1772
|
+
# ---------------------------------------------------------------------------
|
|
1773
|
+
|
|
1774
|
+
|
|
1775
|
+
class BaseAPI:
|
|
1776
|
+
"""Base class for every generated root client (Reddit, Craigslist, ...).
|
|
1777
|
+
|
|
1778
|
+
Per-API subclasses set ``SCRAPER_ID`` and expose top-level resource
|
|
1779
|
+
collections / methods. The base handles auth, transport, error
|
|
1780
|
+
classification, and the executor's response-envelope unwrap.
|
|
1781
|
+
"""
|
|
1782
|
+
|
|
1783
|
+
SCRAPER_ID: ClassVar[str] = ""
|
|
1784
|
+
# Pinned snapshot version for a locally-added MARKETPLACE canonical: codegen
|
|
1785
|
+
# bakes this onto the generated client (from the `[tool.parse].marketplace`
|
|
1786
|
+
# entry's recorded version) so every call sends ``API-Snapshot-Version: N``
|
|
1787
|
+
# and the executor serves that exact release — the contract stays frozen even
|
|
1788
|
+
# as the canonical self-heals. None for owned/account APIs (track latest).
|
|
1789
|
+
PINNED_VERSION: ClassVar[Optional[int]] = None
|
|
1790
|
+
# Generated clients override with ``{wire_kind: GeneratedErrorClass}`` for
|
|
1791
|
+
# every ErrorSpec that declared a ``code``; empty here so the base + any
|
|
1792
|
+
# client without declared error codes falls through to the HTTP-status table.
|
|
1793
|
+
_ERROR_CLASSES: ClassVar[Dict[str, type]] = {}
|
|
1794
|
+
|
|
1795
|
+
# Retry defaults: low (an agent must SEE backpressure, not be silently
|
|
1796
|
+
# blocked behind a long backoff chain), and bounded — a retry re-runs a
|
|
1797
|
+
# billable scrape, so only safe (GET/HEAD) requests are ever retried.
|
|
1798
|
+
_DEFAULT_MAX_RETRIES: ClassVar[int] = 1
|
|
1799
|
+
_RETRY_MAX_BACKOFF: ClassVar[float] = 30.0 # per-retry ceiling (s)
|
|
1800
|
+
_RETRY_TOTAL_BUDGET: ClassVar[float] = 60.0 # total-elapsed cap (s)
|
|
1801
|
+
_RETRY_BASE_BACKOFF: ClassVar[float] = 0.5 # exponential base (s)
|
|
1802
|
+
_RETRY_SAFE_METHODS: ClassVar[frozenset] = frozenset({"GET", "HEAD"})
|
|
1803
|
+
|
|
1804
|
+
def __init__(
|
|
1805
|
+
self,
|
|
1806
|
+
api_key: Optional[str] = None,
|
|
1807
|
+
base_url: Optional[str] = None,
|
|
1808
|
+
timeout: float = 60.0,
|
|
1809
|
+
*,
|
|
1810
|
+
max_retries: Optional[int] = None,
|
|
1811
|
+
) -> None:
|
|
1812
|
+
# Store the key (which may be None) WITHOUT raising — keyless
|
|
1813
|
+
# construction is allowed for offline / typed-only introspection
|
|
1814
|
+
# (``Root()`` to read the modeled surface without a key). The
|
|
1815
|
+
# runtime-safety raise moves to the first ``_request`` call.
|
|
1816
|
+
# Route construction through config.resolve() so `parse login`
|
|
1817
|
+
# (which writes ~/.config/parse/credentials) authenticates scripts,
|
|
1818
|
+
# not just the CLI. resolve() chains kwarg → PARSE_API_KEY env →
|
|
1819
|
+
# credentials file → default base_url (the KEY has no default).
|
|
1820
|
+
# Keyless construction still succeeds; the AuthError stays lazy in
|
|
1821
|
+
# _request. Do NOT call require_key() here — wrong type, and it would
|
|
1822
|
+
# break offline/keyless introspection.
|
|
1823
|
+
import httpx # lazy (see plan 003) — a dict lookup after first use
|
|
1824
|
+
|
|
1825
|
+
cfg = _resolve_config(api_key, base_url)
|
|
1826
|
+
self._api_key = cfg.api_key # may be None
|
|
1827
|
+
# OAuth access token from `parse login --web` (host-gated by resolve()).
|
|
1828
|
+
# Used as a Bearer when no api_key is present; the api_key path is
|
|
1829
|
+
# unchanged. Long-running scripts on a web-login token must re-login
|
|
1830
|
+
# after the ~1h token lifetime (the CLI refreshes proactively; the
|
|
1831
|
+
# runtime does not).
|
|
1832
|
+
self._access_token = cfg.access_token # may be None
|
|
1833
|
+
self._base_url = cfg.base_url # already rstrip'd, defaulted
|
|
1834
|
+
self._timeout = timeout
|
|
1835
|
+
# A bare float fans out to all four httpx timeout classes — fine for
|
|
1836
|
+
# read/write/pool (slow scrapes are legitimate), wrong for CONNECT:
|
|
1837
|
+
# TCP connect to the API host either succeeds in milliseconds or the
|
|
1838
|
+
# host is unreachable, and a blackholed route would stall an agent the
|
|
1839
|
+
# full 60s before the (good) network-error message appears. Bound
|
|
1840
|
+
# connect; a per-request ``timeout=`` override REPLACES all four
|
|
1841
|
+
# (httpx semantics), which callers opting in accept.
|
|
1842
|
+
self._client = httpx.Client(
|
|
1843
|
+
timeout=httpx.Timeout(timeout, connect=min(timeout, 10.0))
|
|
1844
|
+
)
|
|
1845
|
+
# Control-loop metadata from the most recent SUCCESSFUL response. Read
|
|
1846
|
+
# after a call to self-pace on credits/quota; on an error read err.meta
|
|
1847
|
+
# instead (last_meta stays on the last success, by design). Semantics
|
|
1848
|
+
# under a BaseAPI shared across threads: last-write-wins. The assignment
|
|
1849
|
+
# is atomic (GIL), but a reader may observe a concurrent request's meta —
|
|
1850
|
+
# for per-call metadata under sharing, use err.meta / the call's result
|
|
1851
|
+
# rather than this convenience attribute.
|
|
1852
|
+
self.last_meta: Optional[RequestMeta] = None
|
|
1853
|
+
# Retry knobs: ``max_retries`` is per-instance (a constructor kwarg);
|
|
1854
|
+
# the backoff ceiling / total budget / base are CLASS consts read via
|
|
1855
|
+
# ``self.`` so a subclass can override them. The dead per-instance
|
|
1856
|
+
# copies of the latter two are gone — nothing exposed a setter, and the
|
|
1857
|
+
# base const was already read off the class, so they were asymmetric
|
|
1858
|
+
# no-ops.
|
|
1859
|
+
self._max_retries = self._DEFAULT_MAX_RETRIES if max_retries is None else max(0, int(max_retries))
|
|
1860
|
+
|
|
1861
|
+
def __enter__(self) -> BaseAPI:
|
|
1862
|
+
return self
|
|
1863
|
+
|
|
1864
|
+
def __exit__(self, *exc: Any) -> None:
|
|
1865
|
+
self.close()
|
|
1866
|
+
|
|
1867
|
+
def close(self) -> None:
|
|
1868
|
+
self._client.close()
|
|
1869
|
+
|
|
1870
|
+
# --- HTTP -------------------------------------------------------------
|
|
1871
|
+
|
|
1872
|
+
def _serialize_param(self, v: Any) -> Any:
|
|
1873
|
+
"""Normalize a Python value into something the wire can carry."""
|
|
1874
|
+
if v is None:
|
|
1875
|
+
return None
|
|
1876
|
+
# Coerce enum members to their wire codes (Enum imported module-top).
|
|
1877
|
+
if isinstance(v, Enum):
|
|
1878
|
+
# If the enum value is itself meaningful (str/int), use it; else fall back to .name
|
|
1879
|
+
inner = v.value
|
|
1880
|
+
return inner if isinstance(inner, (str, int, float, bool)) else v.name
|
|
1881
|
+
if isinstance(v, datetime):
|
|
1882
|
+
return v.isoformat()
|
|
1883
|
+
if isinstance(v, Decimal):
|
|
1884
|
+
return str(v)
|
|
1885
|
+
if isinstance(v, Resource):
|
|
1886
|
+
# Reference by key if available, else raise — won't serialize a whole instance
|
|
1887
|
+
key = type(v)._KEY_FIELD
|
|
1888
|
+
if key and hasattr(v, key):
|
|
1889
|
+
return getattr(v, key)
|
|
1890
|
+
raise TypeError(f"cannot serialize {type(v).__name__} without a keyed_by field")
|
|
1891
|
+
# Recurse into containers so a List[datetime] / List[Decimal] / List[Resource]
|
|
1892
|
+
# (or dict thereof) — shapes codegen itself emits for params — has its
|
|
1893
|
+
# elements normalized too, instead of hard-crashing at json.dumps(v).
|
|
1894
|
+
if isinstance(v, (list, tuple)):
|
|
1895
|
+
return [self._serialize_param(x) for x in v]
|
|
1896
|
+
if isinstance(v, dict):
|
|
1897
|
+
return {k: self._serialize_param(x) for k, x in v.items()}
|
|
1898
|
+
return v
|
|
1899
|
+
|
|
1900
|
+
def _retry_sleep(self, seconds: float) -> None:
|
|
1901
|
+
"""Sleep before a retry. Isolated so tests can record (not wait)."""
|
|
1902
|
+
time.sleep(seconds)
|
|
1903
|
+
|
|
1904
|
+
def _retry_backoff(self, attempt: int, retry_after: Optional[int]) -> float:
|
|
1905
|
+
"""Backoff seconds for ``attempt`` (1-based).
|
|
1906
|
+
|
|
1907
|
+
When the server gives an explicit ``retry_after``, HONOR it as a floor
|
|
1908
|
+
(sleep at least that long) plus a small jitter to de-synchronize retry
|
|
1909
|
+
storms — then cap to the per-retry ceiling (the server could return an
|
|
1910
|
+
absurd value like an hour; the ceiling guarantees we never sleep that
|
|
1911
|
+
long). With no server signal, use exponential backoff with full jitter.
|
|
1912
|
+
"""
|
|
1913
|
+
if retry_after is not None and retry_after > 0:
|
|
1914
|
+
# Honor the server's hint as a floor (capped), + up to 1s jitter.
|
|
1915
|
+
base = min(float(retry_after), self._RETRY_MAX_BACKOFF)
|
|
1916
|
+
jitter = random.uniform(0, min(1.0, max(0.0, self._RETRY_MAX_BACKOFF - base)))
|
|
1917
|
+
return base + jitter
|
|
1918
|
+
# No server signal — exponential backoff with full jitter in [0, target].
|
|
1919
|
+
target = min(self._RETRY_BASE_BACKOFF * (2 ** (attempt - 1)), self._RETRY_MAX_BACKOFF)
|
|
1920
|
+
return random.uniform(0, target) if target > 0 else 0.0
|
|
1921
|
+
|
|
1922
|
+
def _request(
|
|
1923
|
+
self,
|
|
1924
|
+
method: str,
|
|
1925
|
+
endpoint_name: str,
|
|
1926
|
+
params: Dict[str, Any],
|
|
1927
|
+
*,
|
|
1928
|
+
timeout: Optional[float] = None,
|
|
1929
|
+
) -> Any:
|
|
1930
|
+
"""Hit ``/scraper/<SCRAPER_ID>/<endpoint_name>`` with the given params.
|
|
1931
|
+
|
|
1932
|
+
Returns the unwrapped payload (the contents of ``data`` when the
|
|
1933
|
+
scraper used the ``ScraperResult.success`` envelope, else the raw
|
|
1934
|
+
body). Raises a ``ParseError`` subclass for non-2xx responses.
|
|
1935
|
+
|
|
1936
|
+
``timeout`` overrides the client-construction timeout for THIS request
|
|
1937
|
+
only (httpx supports a per-request timeout) — no new client needed.
|
|
1938
|
+
|
|
1939
|
+
On a 429 for a SAFE method (GET/HEAD), retries up to ``max_retries``
|
|
1940
|
+
times with exponential backoff + jitter, honoring the server's
|
|
1941
|
+
``Retry-After`` header (capped to a ceiling + a total-elapsed budget).
|
|
1942
|
+
A POST 429 is NEVER retried — a retry re-runs a billable,
|
|
1943
|
+
non-idempotent scrape. When retries are exhausted the final
|
|
1944
|
+
``RateLimitError`` surfaces unchanged so an agent still sees
|
|
1945
|
+
backpressure.
|
|
1946
|
+
"""
|
|
1947
|
+
import httpx # lazy (see plan 003) — a dict lookup after first use
|
|
1948
|
+
|
|
1949
|
+
# Lazy-auth guard: a keyless client constructs fine (offline
|
|
1950
|
+
# introspection) but a real request needs a key. Raise BEFORE building
|
|
1951
|
+
# headers / touching transport so we never send ``X-API-Key: None``.
|
|
1952
|
+
# The key is RE-DERIVED through the host gate at this header site (not
|
|
1953
|
+
# only at __init__) so the sink stays correct-by-construction even if a
|
|
1954
|
+
# future refactor mutates ``_base_url`` after construction.
|
|
1955
|
+
trusted = _host_is_trusted(self._base_url)
|
|
1956
|
+
key = self._api_key if trusted else None
|
|
1957
|
+
token = self._access_token if trusted else None
|
|
1958
|
+
if not key and not token:
|
|
1959
|
+
if not trusted:
|
|
1960
|
+
raise AuthError(
|
|
1961
|
+
0,
|
|
1962
|
+
f"Parse API requests to {self._base_url!r} cannot carry credentials — "
|
|
1963
|
+
f"it is not a trusted Parse API host. Point base_url at "
|
|
1964
|
+
f"https://{_CANONICAL_HOST} (the default) or a loopback URL for local dev.",
|
|
1965
|
+
)
|
|
1966
|
+
raise AuthError(0, "No Parse credentials found. Set PARSE_API_KEY, run "
|
|
1967
|
+
"`parse login` / `parse login --web`, or pass api_key=.")
|
|
1968
|
+
# Percent-encode both path segments (safe="") so a hostile SCRAPER_ID /
|
|
1969
|
+
# endpoint_name cannot traverse the path (`../../admin`) or smuggle
|
|
1970
|
+
# `?query` / `#fragment` — _request is public, so this holds for every
|
|
1971
|
+
# caller, not just codegen-gated names. A legit id/endpoint
|
|
1972
|
+
# ([A-Za-z0-9_-]) is untouched by quoting.
|
|
1973
|
+
sid = urllib.parse.quote(self.SCRAPER_ID, safe="")
|
|
1974
|
+
ep = urllib.parse.quote(endpoint_name, safe="")
|
|
1975
|
+
url = f"{self._base_url}/scraper/{sid}/{ep}"
|
|
1976
|
+
# X-API-Key when a key is present, else the web-login Bearer token (both
|
|
1977
|
+
# host-gated above). api_key takes priority when both somehow exist.
|
|
1978
|
+
headers = {"X-API-Key": key} if key else {"Authorization": f"Bearer {token}"}
|
|
1979
|
+
# Attribution: lets the backend track SDK-originated usage (vs raw REST/MCP)
|
|
1980
|
+
# and SDK version adoption. Purely informational — never gates the request.
|
|
1981
|
+
headers["X-Parse-Client"] = _client_header()
|
|
1982
|
+
# Version-pin a marketplace canonical: the executor reads this header and
|
|
1983
|
+
# serves the pinned RELEASE snapshot (a positive int — the only shape it
|
|
1984
|
+
# accepts). Only emitted for a pinned client (PINNED_VERSION set), so
|
|
1985
|
+
# owned/account clients are byte-for-byte unchanged.
|
|
1986
|
+
if self.PINNED_VERSION is not None:
|
|
1987
|
+
headers["API-Snapshot-Version"] = str(self.PINNED_VERSION)
|
|
1988
|
+
method_u = method.upper()
|
|
1989
|
+
cleaned = {k: self._serialize_param(v) for k, v in params.items() if v is not None}
|
|
1990
|
+
if method_u in ("GET", "HEAD", "DELETE"):
|
|
1991
|
+
# dict/list params are JSON-encoded into the query string (executor
|
|
1992
|
+
# contract; pinned by test_request_get_encodes_list_and_dict_as_json_query).
|
|
1993
|
+
wire_params: Dict[str, Any] = {}
|
|
1994
|
+
for k, v in cleaned.items():
|
|
1995
|
+
if isinstance(v, (dict, list)):
|
|
1996
|
+
wire_params[k] = json.dumps(v)
|
|
1997
|
+
elif isinstance(v, bool):
|
|
1998
|
+
wire_params[k] = str(v).lower()
|
|
1999
|
+
else:
|
|
2000
|
+
wire_params[k] = v
|
|
2001
|
+
req_kwargs: Dict[str, Any] = {"params": wire_params}
|
|
2002
|
+
else:
|
|
2003
|
+
req_kwargs = {"json": cleaned}
|
|
2004
|
+
if timeout is not None:
|
|
2005
|
+
req_kwargs["timeout"] = timeout
|
|
2006
|
+
|
|
2007
|
+
retryable = method_u in self._RETRY_SAFE_METHODS and self._max_retries > 0
|
|
2008
|
+
started = _monotonic()
|
|
2009
|
+
attempt = 0
|
|
2010
|
+
while True:
|
|
2011
|
+
try:
|
|
2012
|
+
r = self._client.request(method_u, url, headers=headers, **req_kwargs)
|
|
2013
|
+
except httpx.HTTPError as e:
|
|
2014
|
+
# The billing claim is scoped by failure phase: a connect-phase
|
|
2015
|
+
# error provably never reached the server; a read-phase error
|
|
2016
|
+
# (timeout, dropped response) may have executed — and billed —
|
|
2017
|
+
# the scrape, so an agent must not retry on a "not billed"
|
|
2018
|
+
# promise the message can't keep.
|
|
2019
|
+
billed = (
|
|
2020
|
+
"no request was billed"
|
|
2021
|
+
if isinstance(e, (httpx.ConnectError, httpx.ConnectTimeout))
|
|
2022
|
+
else "the request may have reached the server"
|
|
2023
|
+
)
|
|
2024
|
+
raise UpstreamError(
|
|
2025
|
+
0,
|
|
2026
|
+
f"network error calling {endpoint_name!r} at {self._base_url}: {e} "
|
|
2027
|
+
f"— check connectivity/base_url; {billed}.",
|
|
2028
|
+
) from e
|
|
2029
|
+
body = _safe_json(r)
|
|
2030
|
+
meta = RequestMeta.from_headers(r.headers)
|
|
2031
|
+
if r.status_code < 400:
|
|
2032
|
+
self.last_meta = meta
|
|
2033
|
+
return unwrap_scraper_envelope(body) # strip ScraperResult envelope
|
|
2034
|
+
err = _build_error(r.status_code, body, fallback=f"{endpoint_name} failed",
|
|
2035
|
+
error_classes=getattr(self, "_ERROR_CLASSES", None),
|
|
2036
|
+
headers=r.headers, params=params)
|
|
2037
|
+
err.meta = meta
|
|
2038
|
+
# Only a 429 on a safe method is retried; everything else surfaces.
|
|
2039
|
+
if not (retryable and r.status_code == 429 and attempt < self._max_retries):
|
|
2040
|
+
raise err
|
|
2041
|
+
attempt += 1
|
|
2042
|
+
wait = self._retry_backoff(attempt, getattr(err, "retry_after", None))
|
|
2043
|
+
# Total-elapsed cap on WALL CLOCK (request time included, not just
|
|
2044
|
+
# sleeps): if this wait would blow the budget, surface the
|
|
2045
|
+
# backpressure instead of sleeping past it. Read the class const
|
|
2046
|
+
# via ``self.`` so a subclass override flows through.
|
|
2047
|
+
if (_monotonic() - started) + wait > self._RETRY_TOTAL_BUDGET:
|
|
2048
|
+
raise err
|
|
2049
|
+
self._retry_sleep(wait)
|
|
2050
|
+
|
|
2051
|
+
|
|
2052
|
+
def unwrap_scraper_envelope(body: Any) -> Any:
|
|
2053
|
+
"""Strip the ScraperResult envelope: a ``{"status": "success", "data": ...}``
|
|
2054
|
+
body returns ``data`` (so ``items_path`` / field access is relative to the
|
|
2055
|
+
unwrapped body); anything else passes through. SINGLE source of this
|
|
2056
|
+
predicate — the codegen sample-projection (``codegen_reconcile``) imports it
|
|
2057
|
+
so build-time projection unwraps EXACTLY as serving-time runtime does."""
|
|
2058
|
+
if isinstance(body, dict) and "data" in body and body.get("status") == "success":
|
|
2059
|
+
return body["data"]
|
|
2060
|
+
return body
|
|
2061
|
+
|
|
2062
|
+
|
|
2063
|
+
def _walk(d: Any, path: str, default: Any = None) -> Any:
|
|
2064
|
+
"""Resolve a DOTTED response path (``'a.b.c'``) against a nested dict,
|
|
2065
|
+
returning ``default`` the moment a segment is absent or a non-dict is hit.
|
|
2066
|
+
|
|
2067
|
+
The runtime arm of the dotted-pagination-path fix: a paginated ``_fetch_page``
|
|
2068
|
+
reads cursor / has-more / total-pages / page metadata (and a nested
|
|
2069
|
+
``items_path`` parent) off the page body, and a flat ``_resp.get('a.b')`` can
|
|
2070
|
+
never reach a value nested under ``a`` — it silently truncated every
|
|
2071
|
+
dotted-path spec to page 1. FLAT (dot-free) paths never reach here: codegen
|
|
2072
|
+
emits a verbatim ``_resp.get(path)`` for those, so this helper is imported
|
|
2073
|
+
ONLY by modules that actually declare a dotted pagination path (and existing
|
|
2074
|
+
generated modules never reference it — a purely additive runtime symbol)."""
|
|
2075
|
+
for seg in path.split("."):
|
|
2076
|
+
if not isinstance(d, dict) or seg not in d:
|
|
2077
|
+
return default
|
|
2078
|
+
d = d[seg]
|
|
2079
|
+
return d
|
|
2080
|
+
|
|
2081
|
+
|
|
2082
|
+
def _extract_items(resp: Any, items_path: str) -> List[Any]:
|
|
2083
|
+
"""Pull the declared ``items_path`` array out of one paginated page's
|
|
2084
|
+
POST-unwrap response body (the caller has already run
|
|
2085
|
+
``unwrap_scraper_envelope``). The SAMPLE-INDEPENDENT half of the
|
|
2086
|
+
empty-results discriminator — the emitted modules call it directly when the
|
|
2087
|
+
op captured no sample, and it is the runtime safety net every
|
|
2088
|
+
already-generated module inherits on the next import bump.
|
|
2089
|
+
|
|
2090
|
+
Disposition (each a strict safety improvement over the historical
|
|
2091
|
+
``if items_path not in _resp: raise`` that crashed any body lacking the
|
|
2092
|
+
key):
|
|
2093
|
+
|
|
2094
|
+
* ``resp`` is a **list** — a top-level-list page: the value IS the items,
|
|
2095
|
+
return it (the historical guard ``AttributeError``ed / crashed here).
|
|
2096
|
+
* ``resp`` is ``None`` / empty / falsy (envelope-null, ``{}``) — a
|
|
2097
|
+
legitimate zero-result page → ``[]`` (never raise).
|
|
2098
|
+
* ``resp`` is a dict and ``items_path`` is **present + a list** — return
|
|
2099
|
+
it (unchanged behavior).
|
|
2100
|
+
* ``resp`` is a dict and ``items_path`` is **present but NOT a list**
|
|
2101
|
+
(``{jobs: {...}}`` / ``{jobs: 5}``) — the spec's ``items_path`` points at
|
|
2102
|
+
a non-array; today this is silent garbage / a downstream ``TypeError``,
|
|
2103
|
+
so fail loud with an enriched :class:`PaginationError` (mis-derived;
|
|
2104
|
+
sample-INDEPENDENT, unambiguous). This is the cross-compat-safe upgrade.
|
|
2105
|
+
* ``resp`` is a dict and ``items_path`` is **absent** — return ``[]``. No
|
|
2106
|
+
"list under another key → raise" heuristic lives here: an
|
|
2107
|
+
already-generated / no-sample module cannot disambiguate a faceted
|
|
2108
|
+
zero-result (``{results: [], facets: [...]}``) from a mis-derivation, so
|
|
2109
|
+
the runtime arm fails SAFE. The mis-derivation raise lives ONLY in the
|
|
2110
|
+
emitted discriminator, which has the captured sample to decide.
|
|
2111
|
+
"""
|
|
2112
|
+
if isinstance(resp, list):
|
|
2113
|
+
return resp
|
|
2114
|
+
if not isinstance(resp, dict):
|
|
2115
|
+
# envelope-null / None / a non-dict non-list scalar body → zero results.
|
|
2116
|
+
return []
|
|
2117
|
+
if items_path not in resp:
|
|
2118
|
+
return []
|
|
2119
|
+
value = resp.get(items_path)
|
|
2120
|
+
if isinstance(value, list):
|
|
2121
|
+
return value
|
|
2122
|
+
if value is None:
|
|
2123
|
+
# Key present but null — a zero-result page that declared the key. Treat
|
|
2124
|
+
# as empty (the same disposition an absent key gets), never garbage.
|
|
2125
|
+
return []
|
|
2126
|
+
# Key present but a non-list, non-null value: the array the spec promised is
|
|
2127
|
+
# not there. Sample-independent → fail loud (mis-derived items_path).
|
|
2128
|
+
_list_keys = [k for k, v in resp.items() if isinstance(v, list)]
|
|
2129
|
+
raise PaginationError(
|
|
2130
|
+
0,
|
|
2131
|
+
f"items_path {items_path!r} resolved to a {type(value).__name__}, not a "
|
|
2132
|
+
f"list; list-bearing keys present={_list_keys}; spec likely mis-derived; "
|
|
2133
|
+
f"regenerate",
|
|
2134
|
+
body=resp,
|
|
2135
|
+
)
|
|
2136
|
+
|
|
2137
|
+
|
|
2138
|
+
def _safe_json(r: httpx.Response) -> Any:
|
|
2139
|
+
try:
|
|
2140
|
+
return r.json()
|
|
2141
|
+
except Exception:
|
|
2142
|
+
return r.text
|