parse-sdk 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
parse_sdk/_runtime.py ADDED
@@ -0,0 +1,2142 @@
1
+ """Runtime base classes for generated per-API modules.
2
+
3
+ Generated code (``parse_apis/<slug>/__init__.py``) imports from here.
4
+ Anything user-facing lives in ``parse_sdk.__init__`` / ``parse_sdk.errors``
5
+ so users get clean import paths; this file is the implementation surface.
6
+
7
+ The design is "the SDK module's classes are real Python classes, the
8
+ runtime base lives here." A generated class for a Reddit Post looks like:
9
+
10
+ @resource(name="Post", keyed_by="id")
11
+ class Post(Resource):
12
+ id: str
13
+ title: str
14
+ ...
15
+ def refresh(self) -> Post: ...
16
+
17
+ with the methods produced by codegen calling into ``BaseAPI._request``.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ import os
24
+ import random
25
+ import time
26
+ import types
27
+ import urllib.parse
28
+ from collections import deque
29
+ from dataclasses import dataclass
30
+ from datetime import datetime, timezone
31
+ from decimal import Decimal, InvalidOperation
32
+ from enum import Enum
33
+ from typing import (
34
+ Any,
35
+ Callable,
36
+ ClassVar,
37
+ Dict,
38
+ Generic,
39
+ Iterable,
40
+ Iterator,
41
+ List,
42
+ Optional,
43
+ TYPE_CHECKING,
44
+ Type,
45
+ TypeVar,
46
+ Union,
47
+ get_args,
48
+ get_origin,
49
+ get_type_hints,
50
+ )
51
+
52
+ from parse_sdk._labels import split_top, strip_wrapper
53
+ from parse_sdk._sanitize import CONTROL_BYTES as _EXCERPT_UNSAFE
54
+ from parse_sdk.config import CANONICAL_HOST as _CANONICAL_HOST
55
+ from parse_sdk.config import _host_is_trusted
56
+ from parse_sdk.config import resolve as _resolve_config
57
+
58
+ # Identifies SDK-originated calls on the wire so the backend can attribute usage
59
+ # (e.g. PostHog) to the typed SDK + its version. Computed lazily + cached: the
60
+ # version literal lives in ``parse_sdk.__init__``, which imports THIS module, so a
61
+ # top-level import would cycle — by request time the package is fully loaded.
62
+ _CLIENT_HEADER: Optional[str] = None
63
+
64
+
65
+ def _client_header() -> str:
66
+ global _CLIENT_HEADER
67
+ if _CLIENT_HEADER is None:
68
+ try:
69
+ from parse_sdk import __version__ as _v
70
+ except Exception: # pragma: no cover — never block a request on this
71
+ _v = "unknown"
72
+ _CLIENT_HEADER = f"parse-sdk-python/{_v}"
73
+ return _CLIENT_HEADER
74
+
75
+ if TYPE_CHECKING: # avoid runtime import cycle
76
+ # httpx is imported lazily (function-local in BaseAPI.__init__/_request):
77
+ # importing it costs ~32-35 ms, and `import parse_sdk` pulls this module,
78
+ # so every `parse` CLI invocation paid that before any transport ran (see
79
+ # plan 003). Kept here so the httpx annotations below resolve statically.
80
+ import httpx
81
+
82
+
83
+ # ---------------------------------------------------------------------------
84
+ # Request metadata (cost / quota / rate-limit control loop)
85
+ #
86
+ # The executor emits X-Credits-* and X-RateLimit-* headers on every response
87
+ # so callers can self-pace. RequestMeta parses them once into typed Optional
88
+ # fields; surfaced on success via ``client.last_meta`` and on errors via
89
+ # ``err.meta``.
90
+ # ---------------------------------------------------------------------------
91
+
92
+
93
+ @dataclass
94
+ class RequestMeta:
95
+ credits_charged: Optional[int] = None
96
+ credits_remaining: Optional[int] = None
97
+ credits_limit: Optional[int] = None
98
+ rate_limit_limit: Optional[int] = None
99
+ rate_limit_remaining: Optional[int] = None
100
+ rate_limit_reset: Optional[int] = None # epoch seconds
101
+ burst_limit: Optional[int] = None
102
+ refill_per_min: Optional[int] = None
103
+ daily_limit: Optional[int] = None
104
+ daily_remaining: Optional[int] = None
105
+ daily_reset: Optional[int] = None # duration seconds (NOT epoch — unlike rate_limit_reset)
106
+ retry_after: Optional[int] = None
107
+
108
+ @classmethod
109
+ def from_headers(cls, headers: Any) -> "RequestMeta":
110
+ def _i(name: str) -> Optional[int]:
111
+ v = headers.get(name)
112
+ if v is None and isinstance(headers, dict):
113
+ lower = name.lower()
114
+ for k, candidate in headers.items():
115
+ if isinstance(k, str) and k.lower() == lower:
116
+ v = candidate
117
+ break
118
+ if v is None:
119
+ return None
120
+ try:
121
+ return int(v)
122
+ except (TypeError, ValueError):
123
+ return None
124
+
125
+ return cls(
126
+ credits_charged=_i("x-credits-charged"),
127
+ credits_remaining=_i("x-credits-remaining"),
128
+ credits_limit=_i("x-credits-limit"),
129
+ rate_limit_limit=_i("x-ratelimit-limit"),
130
+ rate_limit_remaining=_i("x-ratelimit-remaining"),
131
+ rate_limit_reset=_i("x-ratelimit-reset"),
132
+ burst_limit=_i("x-ratelimit-burst"),
133
+ refill_per_min=_i("x-ratelimit-refill-per-min"),
134
+ daily_limit=_i("x-ratelimit-daily-limit"),
135
+ daily_remaining=_i("x-ratelimit-daily-remaining"),
136
+ daily_reset=_i("x-ratelimit-daily-reset"),
137
+ retry_after=_i("retry-after"),
138
+ )
139
+
140
+
141
+ # ---------------------------------------------------------------------------
142
+ # Error hierarchy
143
+ #
144
+ # All errors inherit from ``ParseError``. Subclasses map to common HTTP
145
+ # semantics; generated per-API modules subclass ``ParseError`` further for
146
+ # domain-specific errors (e.g. ``SubredditPrivate``).
147
+ # ---------------------------------------------------------------------------
148
+
149
+
150
+ class ParseError(RuntimeError):
151
+ """Base class for all SDK-raised errors.
152
+
153
+ Attributes:
154
+ status_code: the PARSE GATEWAY's HTTP status (the response the SDK
155
+ received), or 0 if the request never reached the server.
156
+ When the gateway relays a target-site failure this is the
157
+ gateway's wrapper status (e.g. 502), NOT the target's —
158
+ see ``upstream_status_code`` for that.
159
+ body: parsed JSON body when the server returned one, raw text otherwise.
160
+ code: the wire error ``kind`` from the server's error envelope
161
+ (e.g. ``input_not_found``) when one was sent, else None.
162
+ Stable across APIs — branch on it for portable handling
163
+ even where no domain error class matched.
164
+ snippet: the target site's response-body excerpt relayed by the
165
+ gateway's upstream-error envelope, else None.
166
+ upstream_status_code: the TARGET SITE's HTTP status relayed by the
167
+ gateway's upstream-error envelope (e.g. 404 while
168
+ ``status_code`` is the gateway's 502), else None.
169
+ url: the target-site URL the gateway reported in its error
170
+ envelope (the page the scraper was fetching when it
171
+ failed), else None.
172
+ """
173
+
174
+ def __init__(self, status_code: int, message: str, body: Any = None) -> None:
175
+ super().__init__(message)
176
+ self.status_code = status_code
177
+ self.body = body
178
+ # Control-loop metadata, populated by _request when a response exists.
179
+ # Declared here so `except ParseError as e: e.meta` is always safe —
180
+ # errors raised before a response (keyless auth, network) keep None.
181
+ self.meta: Optional[RequestMeta] = None
182
+ # Envelope-derived fields, populated by _build_error when the server's
183
+ # error body carries them. Declared here (same always-safe contract as
184
+ # ``meta``) so typed consumers can read them on ANY ParseError without
185
+ # hasattr probing.
186
+ self.code: Optional[str] = None
187
+ self.snippet: Optional[str] = None
188
+ self.upstream_status_code: Optional[int] = None
189
+ self.url: Optional[str] = None
190
+
191
+
192
+ class AuthError(ParseError):
193
+ """401 — missing / invalid API key, or session expired."""
194
+
195
+
196
+ class ForbiddenError(ParseError):
197
+ """403 — caller is authenticated but not allowed (rate-limited tier,
198
+ plan gating, etc.)."""
199
+
200
+
201
+ class NotFoundError(ParseError):
202
+ """404 — resource doesn't exist or caller can't see it."""
203
+
204
+
205
+ class BadRequestError(ParseError):
206
+ """400 — request shape was wrong; usually a missing required param
207
+ or a value the upstream rejected."""
208
+
209
+
210
+ class RateLimitError(ParseError):
211
+ """429 — caller exceeded their token bucket or daily cap.
212
+
213
+ ``retry_after`` is the seconds-to-wait suggestion from the server, if any.
214
+ """
215
+
216
+ def __init__(self, status_code: int, message: str, body: Any = None,
217
+ retry_after: Optional[int] = None) -> None:
218
+ super().__init__(status_code, message, body=body)
219
+ self.retry_after = retry_after
220
+
221
+
222
+ class QuotaExceededError(ParseError):
223
+ """402 — the account ran out of credits / quota.
224
+
225
+ The executor returns 402 on paid-tier credit exhaustion (an account-wide
226
+ billing state, distinct from a malformed-param 400). The ``X-Credits-*``
227
+ headers are on ``err.meta``. Subclasses ``ParseError`` directly so a generic
228
+ ``except ParseError`` still catches it."""
229
+
230
+
231
+ class UpstreamError(ParseError):
232
+ """502 / 503 / 504 — the target site was unreachable or returned a
233
+ non-success response that the scraper couldn't recover from.
234
+
235
+ ``status_code`` is the gateway's wrapper status (typically 502);
236
+ ``upstream_status_code`` is the target site's own status when the envelope
237
+ relayed one, and ``snippet`` is the target's response-body excerpt. A
238
+ sanitized, truncated snippet is appended to the message so ``str(e)`` and
239
+ uncaught tracebacks show the upstream payload; the full raw body stays on
240
+ ``.body`` / ``.snippet``."""
241
+
242
+
243
+ class PaginationError(ParseError):
244
+ """The declared ``items_path`` is present in a successful response but does
245
+ NOT resolve to an array (``{jobs: {...}}`` / ``{jobs: 5}``) — the spec's
246
+ ``items_path`` is mis-derived (it points at a scalar/object, not the item
247
+ list), or the site's shape changed. Fail loud rather than yield silent
248
+ garbage.
249
+
250
+ Narrowed: an *absent* ``items_path`` on a successful response
251
+ is no longer a blanket error — a zero-result page that simply omits the key
252
+ (``{}`` / a top-level-list / an envelope-null body) returns ``[]``. The
253
+ raise is reserved for the unambiguous mis-derivation (key present, value not
254
+ a list) plus, in regenerated modules, the sample-proven case (the key was in
255
+ the captured sample but is absent now). See ``_extract_items`` and the
256
+ emitted discriminator."""
257
+
258
+
259
+ class PaginationLimitError(ParseError):
260
+ """A ``Paginator`` could not advance further: stall, fetch budget, or the
261
+ upstream's result window ran out.
262
+
263
+ Raised when a paginated operation re-returns the same page of items (a
264
+ stalled pagination contract — e.g. the server doesn't surface a usable
265
+ next-cursor and the SDK's fabricated cursor is ignored), runs past
266
+ ``max_fetches``, or — mid-iteration, after at least one successful page —
267
+ the upstream refuses deeper pages with a 4xx (a result-window cap, e.g.
268
+ GitHub search stops serving past 1000 results). Carries no HTTP status —
269
+ it's a client-side guard, not a server response — so ``status_code`` is 0;
270
+ on the window-exhaustion path ``__cause__`` is the underlying
271
+ :class:`UpstreamError` (with ``upstream_status_code`` / ``snippet``).
272
+
273
+ Items already yielded before the raise are valid. ``pages_fetched`` /
274
+ ``items_yielded`` say how far iteration got. ``partial_items`` is
275
+ populated ONLY when the failure surfaces through :meth:`Paginator.list`;
276
+ on the direct ``__next__`` path — manual ``for`` iteration, ``list(
277
+ paginator)``, comprehensions — it stays ``None``. That is deliberate: a
278
+ ``for`` consumer already holds each item it was handed, and the streaming
279
+ paginator does not retain yielded items (a documented memory decision), so
280
+ there is nothing to backfill. Prefer ``.list()`` over ``list(paginator)``
281
+ when you need the materialized prefix to survive the raise.
282
+ """
283
+
284
+ def __init__(
285
+ self,
286
+ message: str,
287
+ *,
288
+ pages_fetched: Optional[int] = None,
289
+ items_yielded: Optional[int] = None,
290
+ partial_items: Optional[List[Any]] = None,
291
+ ) -> None:
292
+ super().__init__(0, message, body=None)
293
+ self.pages_fetched = pages_fetched
294
+ self.items_yielded = items_yielded
295
+ self.partial_items = partial_items
296
+
297
+
298
+ # Console-hostile bytes stripped from message-embedded excerpts: C0 controls
299
+ # minus \n\t (so \r IS stripped — a bare carriage return rewrites/hides
300
+ # earlier terminal output even piped), DEL, and C1 (incl. the single-byte CSI
301
+ # \x9b). The class is the single source in ``parse_sdk._sanitize`` (imported as
302
+ # ``_EXCERPT_UNSAFE``), shared with ``_sync``/``cli`` so the two threat models
303
+ # cannot drift. The raw bytes remain available on ``.body`` / ``.snippet``.
304
+ _EXCERPT_LIMIT = 300
305
+
306
+
307
+ def _sanitize_excerpt(text: str, limit: int = _EXCERPT_LIMIT) -> str:
308
+ """Control-strip + hard-truncate an untrusted string for message embedding."""
309
+ cleaned = _EXCERPT_UNSAFE.sub("", text)
310
+ if len(cleaned) > limit:
311
+ return cleaned[:limit] + "…"
312
+ return cleaned
313
+
314
+
315
+ def _error_envelope(body: Any) -> Dict[str, Any]:
316
+ """The server's nested error envelope (``body['error']``) when present.
317
+
318
+ The executor raises ``HTTPException(status, detail={...})`` and its app-level
319
+ handler renames ``detail`` → ``error``, so envelope fields (``kind``,
320
+ ``snippet``, ``upstream_status_code``, ``retry_after``) live one level down.
321
+ Returns ``{}`` for any non-conforming body so callers can ``.get`` freely.
322
+ """
323
+ if isinstance(body, dict):
324
+ env = body.get("error")
325
+ if isinstance(env, dict):
326
+ return env
327
+ return {}
328
+
329
+
330
+ def _build_error(
331
+ status_code: int,
332
+ body: Any,
333
+ fallback: str = "",
334
+ *,
335
+ error_classes: Optional[Dict[str, type]] = None,
336
+ headers: Optional["httpx.Headers"] = None,
337
+ params: Optional[Dict[str, Any]] = None,
338
+ ) -> ParseError:
339
+ """Pick the right exception subclass for an HTTP error response.
340
+
341
+ The executor emits a closed wire ``kind`` (``body['error']['kind']``, e.g.
342
+ ``input_not_found``). When the generated client passes its ``_ERROR_CLASSES``
343
+ registry (``{kind: GeneratedErrorClass}``) and the body's ``kind`` matches,
344
+ the DOMAIN class is raised BEFORE the coarse HTTP-status dispatch — so
345
+ ``except BookNotFound`` fires. The legacy ``body['error_code']`` spelling is
346
+ also honored. Falls through to the status table when there is no registry
347
+ or no match.
348
+
349
+ Every constructed error is finished uniformly: ``code`` (the wire kind),
350
+ ``snippet``, ``upstream_status_code`` and ``url`` are lifted from the
351
+ envelope, and —
352
+ for a matched DOMAIN class — declared ``carries`` are hydrated from the
353
+ caller's REQUEST PARAMS via the class's generated ``_CARRY_PARAMS`` map
354
+ (``{attr_ident: wire_param_name}``). Carries hydrate only when the request
355
+ actually sent the named param and the attribute is still None, so a future
356
+ server-provided carry value is never clobbered. The response body is never
357
+ read for carry values.
358
+
359
+ For 429, ``retry_after`` is read from the ``Retry-After`` *header* first
360
+ (the reliable, CORS-exposed signal the executor's rate_limit_service sets),
361
+ then from the body. The executor nests the body value at
362
+ ``body['error']['retry_after']`` (its custom handler renames detail→error),
363
+ so the body fallback walks BOTH the flat and nested paths.
364
+ """
365
+ kind = _wire_error_kind(body)
366
+ msg = _structured_error_message(body)
367
+ if msg is None:
368
+ # No structured message anywhere in the body (HTML gateway page, or a
369
+ # kind-only error object). The agent still needs something actionable:
370
+ # enrich the caller's context line with the structured facts in hand —
371
+ # the HTTP status (always) and the wire kind (when present). Never a
372
+ # body dump (pinned design: structured fields only).
373
+ bits = [f"HTTP {status_code}"]
374
+ if kind is not None:
375
+ bits.append(f"kind={kind}")
376
+ msg = f"{fallback or 'request failed'} ({', '.join(bits)})"
377
+ envelope = _error_envelope(body)
378
+ snippet = envelope.get("snippet")
379
+ snippet = snippet if isinstance(snippet, str) and snippet else None
380
+ upstream_sc = envelope.get("upstream_status_code")
381
+ upstream_sc = upstream_sc if isinstance(upstream_sc, int) else None
382
+ url = envelope.get("url")
383
+ url = url if isinstance(url, str) and url else None
384
+ # On a 429, the server's Retry-After hint must survive even when a generated
385
+ # DOMAIN error class matches the body's kind — the domain branch returns
386
+ # before the 429 status branch below. Compute it up front and attach it so
387
+ # the retry loop's getattr(err, "retry_after", None) honors it either way.
388
+ retry_after = _parse_retry_after(headers, body) if status_code == 429 else None
389
+
390
+ def _finish(err: ParseError) -> ParseError:
391
+ err.code = kind
392
+ err.snippet = snippet
393
+ err.upstream_status_code = upstream_sc
394
+ err.url = url
395
+ return err
396
+
397
+ if error_classes and kind is not None:
398
+ cls = error_classes.get(kind)
399
+ if cls is not None:
400
+ obj = cls(status_code, msg, body=body)
401
+ if retry_after is not None:
402
+ obj.retry_after = retry_after
403
+ # Hydrate declared carries from the request params (never the
404
+ # response body — the envelope is executor metadata, not domain
405
+ # values). _CARRY_PARAMS maps attr ident → wire param name and is
406
+ # emitted only for carries that exist in the API's input-param
407
+ # union, so absent mappings stay None by design.
408
+ carry_map = getattr(cls, "_CARRY_PARAMS", None)
409
+ if isinstance(carry_map, dict) and params:
410
+ for attr, src in carry_map.items():
411
+ if src in params and getattr(obj, attr, None) is None:
412
+ setattr(obj, attr, params[src])
413
+ return _finish(obj)
414
+ if status_code == 401:
415
+ return _finish(AuthError(status_code, msg, body=body))
416
+ if status_code == 403:
417
+ return _finish(ForbiddenError(status_code, msg, body=body))
418
+ if status_code == 404:
419
+ return _finish(NotFoundError(status_code, msg, body=body))
420
+ if status_code == 429:
421
+ return _finish(RateLimitError(status_code, msg, body=body, retry_after=retry_after))
422
+ if status_code == 402:
423
+ # Credit / quota exhaustion (executor's paid-tier billing state). Above
424
+ # the 4xx catch-all so it isn't mis-typed as a malformed-param 400;
425
+ # below the registry block so a future kind-carrying 402 still routes to
426
+ # a domain class.
427
+ return _finish(QuotaExceededError(status_code, msg, body=body))
428
+ if 400 <= status_code < 500:
429
+ return _finish(BadRequestError(status_code, msg, body=body))
430
+ if snippet:
431
+ # Surface the upstream payload where the user actually looks — str(e)
432
+ # and uncaught tracebacks. The full raw value stays on .snippet/.body.
433
+ msg = f"{msg} — upstream body (first {_EXCERPT_LIMIT} non-control chars): {_sanitize_excerpt(snippet)}"
434
+ return _finish(UpstreamError(status_code, msg, body=body))
435
+
436
+
437
+ def _parse_retry_after(
438
+ headers: Optional["httpx.Headers"], body: Any
439
+ ) -> Optional[int]:
440
+ """Seconds-to-wait from the ``Retry-After`` header (primary) or the body.
441
+
442
+ Header first (reliable / CORS-exposed). Body fallback covers the flat
443
+ ``body['retry_after']``, the executor-nested ``body['error']['retry_after']``,
444
+ and the legacy ``body['detail']['retry_after']``. Returns None when no
445
+ usable signal exists. Never raises."""
446
+ if headers is not None:
447
+ raw = headers.get("Retry-After")
448
+ if raw is not None:
449
+ try:
450
+ return int(raw)
451
+ except (TypeError, ValueError):
452
+ pass # HTTP-date form not handled — fall through to body
453
+ if isinstance(body, dict):
454
+ nested = body.get("error")
455
+ nested = nested if isinstance(nested, dict) else {}
456
+ detail = body.get("detail")
457
+ detail = detail if isinstance(detail, dict) else {}
458
+ ra = body.get("retry_after")
459
+ if ra is None:
460
+ ra = nested.get("retry_after")
461
+ if ra is None:
462
+ ra = detail.get("retry_after")
463
+ try:
464
+ return int(ra) if ra is not None else None
465
+ except (TypeError, ValueError):
466
+ return None
467
+ return None
468
+
469
+
470
+ def _wire_error_kind(body: Any) -> Optional[str]:
471
+ """The executor's closed wire ``kind`` for an error body, or ``None``.
472
+
473
+ Walks the same shapes ``_build_error``'s registry dispatch always has:
474
+ nested ``body['error']['kind']`` / ``['code']``, then the legacy flat
475
+ ``body['error_code']`` / ``body['kind']`` spellings."""
476
+ if not isinstance(body, dict):
477
+ return None
478
+ kind: Any = None
479
+ err = body.get("error")
480
+ if isinstance(err, dict):
481
+ kind = err.get("kind") or err.get("code")
482
+ if kind is None:
483
+ kind = body.get("error_code") or body.get("kind")
484
+ return kind if isinstance(kind, str) else None
485
+
486
+
487
+ def _structured_error_message(body: Any) -> Optional[str]:
488
+ """The human message from the body's structured string fields, or ``None``
489
+ when the body carries no usable message (non-dict, or kind-only)."""
490
+ if isinstance(body, dict):
491
+ for key in ("detail", "error", "message"):
492
+ v = body.get(key)
493
+ if isinstance(v, str):
494
+ return v
495
+ if isinstance(v, dict):
496
+ m = v.get("message") or v.get("error")
497
+ if isinstance(m, str):
498
+ return m
499
+ return None
500
+
501
+
502
+ def _format_error_message(body: Any, fallback: str = "") -> str:
503
+ return _structured_error_message(body) or fallback or "unknown error"
504
+
505
+
506
+ # ---------------------------------------------------------------------------
507
+ # Value coercion — driven by the codegen's per-field type label
508
+ # ---------------------------------------------------------------------------
509
+
510
+
511
+ # Non-downcast runtime backstop. When a NON-KEY field declared ``str`` is handed a non-str
512
+ # wire value, downcasting it (``str(False) == 'False'``) changes the observed
513
+ # data shape and can hide a stale/misdeclared schema. Keep such values RAW so an
514
+ # already-shipped or drift-stale package stays non-corrupting without a regen.
515
+ # KEY fields are EXCLUDED (they must keep downcasting so ``'42'`` and ``42``
516
+ # stay ==/hash-equal for dedup). Unconditional — no env opt-out: an in-band
517
+ # escape hatch is an agent-flippable vector that would reintroduce the exact
518
+ # downcast corruption this backstop exists to prevent.
519
+ # This covers ONLY the per-field annotation walk (``_coerce_by_annotation``); the
520
+ # bare-scalar method-RETURN router (``_PRIMITIVE_COERCERS['str']`` via
521
+ # ``coerce()``) is not a field-corruption vector and is left unchanged.
522
+
523
+ # Monotonic clock seam — module-level so tests can monkeypatch wall-clock (the
524
+ # retry budget counts real elapsed time, requests included).
525
+ _monotonic = time.monotonic
526
+
527
+
528
+ def _coerce_bool(value: Any) -> Any:
529
+ if value is None or isinstance(value, bool):
530
+ return value
531
+ if isinstance(value, str):
532
+ folded = value.strip().lower()
533
+ if folded in {"true", "t", "1", "yes", "y", "on"}:
534
+ return True
535
+ if folded in {"false", "f", "0", "no", "n", "off"}:
536
+ return False
537
+ return value
538
+ if isinstance(value, (int, float)) and value in (0, 1):
539
+ return bool(value)
540
+ return value
541
+
542
+
543
+ _PRIMITIVE_COERCERS: Dict[str, Callable[[Any], Any]] = {
544
+ "str": lambda v: v if isinstance(v, str) else (str(v) if v is not None else None),
545
+ "int": lambda v: int(v) if v is not None else None,
546
+ "float": lambda v: float(v) if v is not None else None,
547
+ "bool": _coerce_bool,
548
+ "url": str,
549
+ "email": str,
550
+ }
551
+
552
+
553
+ def _coerce_datetime(value: Any) -> Any:
554
+ if value is None or isinstance(value, datetime):
555
+ return value
556
+ # ``bool`` subclasses ``int`` — guard it BEFORE the epoch branch so a wire
557
+ # ``True`` doesn't become ``1970-01-01T00:00:01`` (epoch 1). A bool is never
558
+ # a timestamp; pass it through raw.
559
+ if isinstance(value, (int, float)) and not isinstance(value, bool):
560
+ try:
561
+ return datetime.fromtimestamp(value, tz=timezone.utc)
562
+ except (OSError, OverflowError, ValueError):
563
+ return value
564
+ if isinstance(value, str):
565
+ try:
566
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
567
+ except ValueError:
568
+ return value
569
+ return value
570
+
571
+
572
+ def _coerce_decimal(value: Any) -> Any:
573
+ if value is None or isinstance(value, Decimal):
574
+ return value
575
+ try:
576
+ return Decimal(str(value))
577
+ except (InvalidOperation, TypeError):
578
+ return value
579
+
580
+
581
+ # Sentinel: the dict carried no single list to unwrap (zero list fields, or
582
+ # two-or-more NON-EMPTY list fields — genuinely ambiguous). Distinct from a
583
+ # determinable empty result (``[]``). Each call-site decides the disposition.
584
+ _NO_SINGLE_LIST = object()
585
+
586
+
587
+ def _unwrap_single_list(value: Dict[str, Any]) -> Any:
588
+ """Pick the single list out of an envelope dict, or ``_NO_SINGLE_LIST``.
589
+
590
+ API responses often wrap a list in an envelope, e.g.
591
+ ``{"count": 3, "stories": [...]}``. When exactly ONE list-valued field is
592
+ non-empty we return it; an incidental EMPTY list (``{"errors": [],
593
+ "stories": [...]}``) is **ignored when counting** so a stray ``errors: []``
594
+ can't flip the count to 2 and defeat the unwrap. When every list field is
595
+ empty (``{"errors": [], "stories": []}``) we return ``[]`` — a determinable
596
+ empty result, not an ambiguity. Zero list fields, or two-or-more non-empty
597
+ list fields, return ``_NO_SINGLE_LIST`` — the caller decides whether to fail
598
+ loud (a method-return contract boundary) or degrade to raw (a per-field
599
+ walk that must never crash a successful scrape).
600
+
601
+ ``value`` is assumed to be a dict (callers guard ``isinstance``). Never
602
+ raises.
603
+ """
604
+ list_fields = [k for k, v in value.items() if isinstance(v, list)]
605
+ if not list_fields:
606
+ return _NO_SINGLE_LIST
607
+ non_empty = [k for k in list_fields if value[k]]
608
+ if len(non_empty) == 1:
609
+ return value[non_empty[0]]
610
+ if not non_empty:
611
+ # All list fields are empty → an empty result set (determinable).
612
+ return []
613
+ # Two-or-more non-empty lists → genuinely ambiguous.
614
+ return _NO_SINGLE_LIST
615
+
616
+
617
+ def _construct_from_scalar(
618
+ cls: "Type[Resource]", value: Any, *, api: Optional["BaseAPI"], parent: Any
619
+ ) -> Any:
620
+ """Hydrate a constructible resource from a bare scalar matching its
621
+ ``keyed_by`` field (``post.subreddit = "smallbusiness"`` →
622
+ ``Subreddit(name="smallbusiness")``), or return ``value`` unchanged.
623
+
624
+ The scalar is COERCED to the key field's resolved annotation first (via the
625
+ total/lenient ``_coerce_by_annotation``), so the scalar-path instance and a
626
+ dict-path instance of the same entity agree on the key's TYPE — otherwise a
627
+ wire int key (``42``) on the scalar path vs an int-coerced ``42`` on the
628
+ dict path could diverge and break ``__eq__``/``__hash__`` (which key on the
629
+ field value). Only ``str``/``int`` scalars hydrate; anything else passes
630
+ through. Never raises (coercion is lenient).
631
+ """
632
+ key = getattr(cls, "_KEY_FIELD", None)
633
+ if not (getattr(cls, "_CONSTRUCTIBLE", False) and key and isinstance(value, (str, int))):
634
+ return value
635
+ key_ann = cls._field_hints().get(key)
636
+ coerced_key = (
637
+ _coerce_by_annotation(value, key_ann, api=api, parent=parent, is_key=True)
638
+ if key_ann is not None
639
+ else value
640
+ )
641
+ return cls(_api=api, _parent=parent, **{key: coerced_key})
642
+
643
+
644
+ def _is_husk(inst: Any) -> bool:
645
+ """Husk predicate, observed by POST-``_from_payload`` instance
646
+ inspection — never by re-running the wrapped-record unwrap.
647
+
648
+ A husk is a constructed resource where *no declared wire key resolved* (the
649
+ ``_resolved_count`` signal ``_from_payload`` set is 0) — this single count
650
+ also subsumes "all declared keys null", since a key whose wire value is null
651
+ is not counted as resolved.
652
+
653
+ **Exempt** when ``_extra`` carries any non-null unmapped key: a record whose
654
+ data landed entirely in ``_extra`` is a mis-DERIVED spec (the fields were
655
+ typed off the wrong shape), NOT a hollow husk — failing loud there would
656
+ discard data the caller can still reach via ``obj._extra``. Total/lenient:
657
+ a missing signal (a non-``_from_payload`` instance) reads as NOT a husk, so
658
+ this can never manufacture a false verdict.
659
+ """
660
+ if getattr(inst, "_resolved_count", None) != 0:
661
+ return False
662
+ extra = getattr(inst, "_extra", None)
663
+ if isinstance(extra, dict) and any(v is not None for v in extra.values()):
664
+ return False # data lives in _extra → mis-derived spec, not a husk
665
+ return True
666
+
667
+
668
+ def coerce(value: Any, type_label: str, *, resource_classes: Optional[Dict[str, Type[Resource]]] = None,
669
+ api: Optional[BaseAPI] = None, parent: Any = None,
670
+ boundary: bool = False, mode: str = "element") -> Any:
671
+ """Coerce a raw response value to its declared type.
672
+
673
+ Type labels:
674
+ * ``str``, ``int``, ``float``, ``bool`` — primitive coercion
675
+ * ``datetime`` — int/float epoch OR ISO str → datetime
676
+ * ``Decimal`` — str/int/float → Decimal
677
+ * ``url``, ``email`` — kept as str (semantic only)
678
+ * ``List[X]`` — recurse over each element
679
+ * ``Optional[X]`` — None passes through; else recurse
680
+ * ``Dict[K, V]`` — recurse over values (keys stay str)
681
+ * ``Any`` — pass through unchanged
682
+ * ``<ResourceName>`` — look up in ``resource_classes`` and
683
+ construct an instance via ``_from_payload``
684
+
685
+ ``resource_classes`` / ``api`` / ``parent`` are forwarded so resource
686
+ instances can wire back to the owning client.
687
+
688
+ ``boundary`` / ``mode`` (return-fail-loud) gate the strict
689
+ return-boundary verdicts. **Both default to the LENIENT setting**
690
+ (``boundary=False``, ``mode='element'``) so an old 3-arg
691
+ ``_coerce(value, type_label, ctx)`` call from an already-generated module
692
+ hits the lenient default and behaves exactly as before — this is the
693
+ cross-compat protection for the entire generated fleet. The strict
694
+ husk / scalar verdicts fire ONLY when a *regenerated* module's
695
+ single-resource return passes ``boundary=True, mode='return'`` and the
696
+ declared type resolves to a Resource. Every ``Optional`` / ``List`` /
697
+ ``Dict`` recurse flips ``boundary=False`` (a verdict applies to the top
698
+ return only, never to a nested element — per-element disposition is the
699
+ lenient degrade-to-raw).
700
+ """
701
+ if value is None:
702
+ return None
703
+
704
+ t = type_label.strip()
705
+ # Optional[X] — single-layer unwrap via the shared label grammar (_labels).
706
+ _opt_inner = strip_wrapper(t, "Optional[")
707
+ if _opt_inner is not None:
708
+ return coerce(value, _opt_inner, resource_classes=resource_classes, api=api, parent=parent,
709
+ boundary=False, mode=mode)
710
+ # List[X]
711
+ _list_inner = strip_wrapper(t, "List[")
712
+ if _list_inner is not None:
713
+ inner = _list_inner
714
+ # API responses often wrap the list in an envelope, e.g.
715
+ # ``{"count": 3, "stories": [...]}``. When we expect List[X] and got a
716
+ # dict, unwrap the single (non-empty) list. This is the method-RETURN
717
+ # contract boundary — the caller did ``return _coerce(_resp, 'List[X]',
718
+ # self)`` and will iterate the result, so a dict we can't unwrap to a
719
+ # list would silently iterate dict KEYS. FAIL LOUD instead (a clear
720
+ # error beats silent garbage). The per-FIELD router stays lenient.
721
+ if isinstance(value, dict):
722
+ unwrapped = _unwrap_single_list(value)
723
+ if unwrapped is _NO_SINGLE_LIST:
724
+ raise ParseError(
725
+ 0,
726
+ f"expected List[{inner}] but the response was a dict with no "
727
+ f"single list to unwrap (keys: {sorted(value)}); the "
728
+ f"pagination/envelope contract is likely mis-derived",
729
+ body=value,
730
+ )
731
+ value = unwrapped
732
+ if not isinstance(value, list):
733
+ return value
734
+ # Coverage symmetry: a non-paginated List[Resource] recurses at
735
+ # boundary=False — every element gets the SAME lenient degrade-to-raw
736
+ # disposition as a paginated row, never the strict return verdict.
737
+ return [coerce(v, inner, resource_classes=resource_classes, api=api, parent=parent,
738
+ boundary=False, mode=mode) for v in value]
739
+ # Dict[K, V] — only coerce values
740
+ _dict_inner = strip_wrapper(t, "Dict[")
741
+ if _dict_inner is not None:
742
+ inner = _dict_inner
743
+ # First depth-0 comma split (handles Dict[str, List[X]]) — shared
744
+ # splitter; everything past the first comma is the value label.
745
+ parts = split_top(inner)
746
+ v_label = ", ".join(parts[1:]) if len(parts) > 1 else "Any"
747
+ if not isinstance(value, dict):
748
+ return value
749
+ return {k: coerce(v, v_label, resource_classes=resource_classes, api=api, parent=parent,
750
+ boundary=False, mode=mode)
751
+ for k, v in value.items()}
752
+ # Primitives
753
+ if t in _PRIMITIVE_COERCERS:
754
+ try:
755
+ return _PRIMITIVE_COERCERS[t](value)
756
+ except (TypeError, ValueError, OverflowError):
757
+ # OverflowError: ``int(float('inf'))`` (stdlib json.loads admits
758
+ # Infinity/-Infinity/NaN, so a target can return ``{"n": Infinity}``
759
+ # on a declared-int field). Degrade to raw — never raise on a
760
+ # successful scrape.
761
+ return value
762
+ if t == "datetime":
763
+ return _coerce_datetime(value)
764
+ if t == "Decimal":
765
+ return _coerce_decimal(value)
766
+ if t in ("Any", "any"):
767
+ return value
768
+ # Resource type
769
+ if resource_classes and t in resource_classes:
770
+ cls = resource_classes[t]
771
+ # Strict return boundary — fires ONLY for a regenerated module's
772
+ # single-resource return (boundary=True + mode='return'). Anywhere else
773
+ # (lenient default, per-element rows at mode='element', any nested
774
+ # recurse at boundary=False) keeps the historical silent disposition.
775
+ strict = boundary and mode == "return"
776
+ if isinstance(value, dict):
777
+ inst = cls._from_payload(value, api=api, parent=parent)
778
+ if strict and _is_husk(inst):
779
+ # Husk: no declared wire key resolved OR all declared
780
+ # keys null — but EXEMPT when ``_extra`` carries non-null
781
+ # unmapped keys (a mis-DERIVED spec whose data lives in _extra,
782
+ # not a hollow husk — failing loud there would discard
783
+ # retrievable data). Husk-ness is observed by POST-_from_payload
784
+ # instance inspection (the resolved-key signal _from_payload
785
+ # set), never by re-running the unwrap. The verdict lives HERE
786
+ # in coerce(), never in _from_payload (shared with the lenient
787
+ # per-field router → a verdict there would crash the fleet).
788
+ raise ParseError(
789
+ 0,
790
+ f"expected {t} but the response resolved no declared field "
791
+ f"(an all-null / zero-key 'husk'); the return-type contract "
792
+ f"is likely mis-derived",
793
+ body=value,
794
+ )
795
+ return inst
796
+ # Constructible resources hydrate from a bare scalar matching the
797
+ # keyed_by field — the key is coerced to its annotation so scalar- and
798
+ # dict-path instances agree on key type (see _construct_from_scalar).
799
+ hydrated = _construct_from_scalar(cls, value, api=api, parent=parent)
800
+ if strict and not isinstance(hydrated, cls):
801
+ # Scalar husk: a bare scalar typed as a non-constructible Resource (no
802
+ # _KEY_FIELD to hydrate from) returned the raw scalar — fail loud.
803
+ # A constructible resource hydrated above (``isinstance`` holds), so
804
+ # only the genuinely non-constructible scalar reaches this raise.
805
+ raise ParseError(
806
+ 0,
807
+ f"expected {t} but the response was a bare {type(value).__name__} "
808
+ f"with no key field to construct from; the return-type contract "
809
+ f"is likely mis-derived",
810
+ body=value,
811
+ )
812
+ return hydrated
813
+ # Unknown — pass through. Better to let user see the raw dict than crash.
814
+ return value
815
+
816
+
817
+ # ---------------------------------------------------------------------------
818
+ # Per-field coercion — single-source, driven by the resolved field ANNOTATION
819
+ #
820
+ # This is the per-field coercer (distinct from the ``coerce()`` string-label
821
+ # *return* router above, which generated methods still call). Generated
822
+ # resource classes declare each field's type as a PEP-526 annotation and carry
823
+ # NO parallel ``_FIELDS`` label dict; ``_from_payload`` resolves those
824
+ # annotations to real typing/class OBJECTS (cached per class) and dispatches on
825
+ # them here — never round-tripping a resolved object back through the
826
+ # string-label router (that would no-op and re-create the static/runtime
827
+ # divergence). Total / lenient: an unknown form or un-coercible value passes
828
+ # through raw; it never raises on a successful scrape.
829
+ #
830
+ # WHY TWO DISPATCHERS (do not naively merge): the leaf coercers
831
+ # (_coerce_bool/_coerce_datetime/_coerce_decimal) are ALREADY single-sourced and
832
+ # shared by both. What differs is irreducible and opposite: ``coerce()`` is the
833
+ # RETURN router — it always downcasts ``str`` and FAILS LOUD at the strict boundary
834
+ # (husk/scalar verdicts); this per-field router is LENIENT (degrade-to-raw),
835
+ # honours the str non-downcast rule (``is_key``), and owns the Enum case-fold dispatch.
836
+ # Folding them into one driver would re-encode those divergent dispositions as
837
+ # flags (fail-loud-vs-lenient × is_key × boundary/mode) — more branching, not
838
+ # less — in a path NO emitted-bytes gate can verify. The asymmetries are pinned
839
+ # by tests/test_coercion_asymmetries.py. Keep them two.
840
+ # ---------------------------------------------------------------------------
841
+
842
+
843
+ # Both union spellings: ``typing.Union[...]`` (incl. ``Optional[X]``) and the
844
+ # PEP-604 ``A | B`` form, whose origin is ``types.UnionType`` (3.10+). The
845
+ # codegen emits ``Sort | str`` for enum/choice fields, so this is load-bearing.
846
+ _UNION_ORIGINS = (Union, types.UnionType)
847
+
848
+
849
+ def _coerce_by_annotation(
850
+ value: Any,
851
+ ann: Any,
852
+ *,
853
+ api: Optional[BaseAPI] = None,
854
+ parent: Any = None,
855
+ is_key: bool = False,
856
+ ) -> Any:
857
+ """Coerce ``value`` to the resolved annotation OBJECT ``ann``.
858
+
859
+ Dispatches on ``get_origin`` / ``get_args`` (List/Dict/Optional/Union),
860
+ ``isinstance``-against-``Enum``, and the ``datetime``/``Decimal``/``str``/
861
+ ``int``/``float``/``bool`` classes. ``Resource`` subclasses construct via
862
+ ``_from_payload`` (or constructible-from-scalar). Threads ``api``/``parent``
863
+ into every nested + list/dict element (``parent`` = the owning resource).
864
+
865
+ ``is_key`` marks this coercion as a resource's KEY field, threaded
866
+ through the Optional/Union/List/Dict recursion. A key keeps downcasting (so
867
+ ``'42'``/``42`` stay ==/hash-equal); a NON-key ``str`` field stops
868
+ downcasting non-strings (the truthiness-inversion backstop). It is NOT
869
+ propagated into a nested resource — that resource recomputes ``is_key`` for
870
+ its own key field.
871
+
872
+ Total/lenient: any unrecognized form or un-coercible value returns ``value``
873
+ unchanged — never raises.
874
+ """
875
+ if value is None:
876
+ return None
877
+
878
+ # Hot path: resolved resource field annotations are usually concrete scalar
879
+ # classes. Handle those before the slower typing-origin / issubclass probes.
880
+ if ann is str:
881
+ if isinstance(value, str):
882
+ return value
883
+ # Non-downcast backstop: a NON-key str field handed a non-str keeps it RAW (no
884
+ # ``str(False)=='False'`` truthiness inversion). Keys still downcast so
885
+ # ``'42'``/``42`` stay ==/hash-equal. Unconditional — no opt-out.
886
+ if not is_key:
887
+ return value
888
+ return str(value)
889
+ if ann is bool:
890
+ return _coerce_bool(value)
891
+ if ann in (int, float):
892
+ try:
893
+ return ann(value)
894
+ except (TypeError, ValueError, OverflowError):
895
+ # OverflowError: ``int(float('inf'))`` — stdlib json.loads admits
896
+ # Infinity/-Infinity/NaN, so a wire ``Infinity`` on a declared-int
897
+ # field reaches here. Degrade to raw (never raise on a scrape).
898
+ return value
899
+ if ann is datetime:
900
+ return _coerce_datetime(value)
901
+ if ann is Decimal:
902
+ return _coerce_decimal(value)
903
+
904
+ origin = get_origin(ann)
905
+
906
+ # Union / Optional — try each non-None arm; first that changes the value
907
+ # (or matches a resource/enum/special) wins. Optional[X] is X | None.
908
+ if origin in _UNION_ORIGINS:
909
+ args = [a for a in get_args(ann) if a is not type(None)]
910
+ # Single-arm Optional[X] → coerce as X.
911
+ if len(args) == 1:
912
+ return _coerce_by_annotation(value, args[0], api=api, parent=parent, is_key=is_key)
913
+ # Multi-arm union (e.g. ``Sort | str``): prefer an Enum/Resource arm
914
+ # (the precise type) over the bare str/scalar fallback; if none fire,
915
+ # fall through raw.
916
+ for arm in args:
917
+ if _is_enum_type(arm) or _is_resource_type(arm):
918
+ coerced = _coerce_by_annotation(value, arm, api=api, parent=parent, is_key=is_key)
919
+ # Accept the arm when it produced (or already was) an instance of
920
+ # it — an ``isinstance`` check, NOT object identity. (Identity
921
+ # would reject an already-coerced member, e.g. a ``Sort.NEW`` fed
922
+ # back through ``Sort | str``, then stringify it via the str arm.)
923
+ if isinstance(coerced, arm):
924
+ return coerced
925
+ # No precise arm matched — coerce to the first arm (usually str) only if
926
+ # it's a primitive; else pass through.
927
+ for arm in args:
928
+ if arm in (str, int, float, bool, datetime, Decimal):
929
+ return _coerce_by_annotation(value, arm, api=api, parent=parent, is_key=is_key)
930
+ return value
931
+
932
+ # List[X] — envelope-unwrap a single-list dict, then recurse per element.
933
+ # This is the per-FIELD path (reached inside ``_from_payload`` on every
934
+ # ``List[X]`` field). It MUST stay lenient — degrade to raw, never raise —
935
+ # so one malformed nested list can't crash construction of a
936
+ # successfully-scraped resource and discard its other good fields. (Contrast
937
+ # the method-RETURN router ``coerce()``, which fails loud at its boundary.)
938
+ if origin in (list, List):
939
+ (inner,) = get_args(ann) or (Any,)
940
+ if isinstance(value, dict):
941
+ unwrapped = _unwrap_single_list(value)
942
+ if unwrapped is _NO_SINGLE_LIST:
943
+ return value # lenient: keep the raw dict, never raise
944
+ value = unwrapped
945
+ if not isinstance(value, list):
946
+ return value
947
+ return [_coerce_by_annotation(v, inner, api=api, parent=parent, is_key=is_key) for v in value]
948
+
949
+ # Dict[K, V] — coerce values only (keys stay as-is).
950
+ if origin in (dict, Dict):
951
+ args = get_args(ann)
952
+ v_ann = args[1] if len(args) == 2 else Any
953
+ if not isinstance(value, dict):
954
+ return value
955
+ return {k: _coerce_by_annotation(v, v_ann, api=api, parent=parent, is_key=is_key)
956
+ for k, v in value.items()}
957
+
958
+ # Enum — member for an observed value, RAW for an unobserved-but-valid one
959
+ # (never raise). This is the divergence fix: a bare ``type: ContentType``
960
+ # field now coerces to the member at runtime instead of staying a str.
961
+ if _is_enum_type(ann):
962
+ if isinstance(value, ann):
963
+ return value
964
+ try:
965
+ return ann(value) # exact wire-value match
966
+ except (ValueError, KeyError, TypeError):
967
+ pass
968
+ # Case-tolerant fallback (G7): the wire sometimes uppercases an enum
969
+ # value (``FOR_SALE`` for a ``for_sale`` member). Match str(value)
970
+ # against member values case-insensitively before giving up — using the
971
+ # same ``.lower()`` fold the codegen's code-uniqueness gate enforces, so
972
+ # a match is unambiguous. Keep raw on no match (never raise).
973
+ folded = str(value).lower()
974
+ for member in ann:
975
+ if str(member.value).lower() == folded:
976
+ return member
977
+ return value # unobserved-but-valid → keep raw, never raise
978
+
979
+ # Resource — construct from a dict payload, or hydrate constructible from a
980
+ # bare scalar matching the keyed_by field.
981
+ if _is_resource_type(ann):
982
+ if isinstance(value, dict):
983
+ return ann._from_payload(value, api=api, parent=parent)
984
+ return _construct_from_scalar(ann, value, api=api, parent=parent)
985
+
986
+ # Any / unknown annotation form → raw passthrough (total/lenient).
987
+ return value
988
+
989
+
990
+ def _is_enum_type(ann: Any) -> bool:
991
+ return isinstance(ann, type) and issubclass(ann, Enum)
992
+
993
+
994
+ def _is_resource_type(ann: Any) -> bool:
995
+ return isinstance(ann, type) and issubclass(ann, Resource)
996
+
997
+
998
+ def _resource_payload_meta(cls: "Type[Resource]") -> tuple:
999
+ """Cached per-class metadata used by ``Resource._from_payload``.
1000
+
1001
+ The generated class's annotations and ``_FIELD_ALIASES`` are fixed at module
1002
+ import. Cache the derived wire-name map and consumed-wire set so high-volume
1003
+ page hydration does not rebuild them for every row.
1004
+
1005
+ This is module-scoped on purpose: every new ``Resource`` member expands
1006
+ codegen's reserved identifier set (``dir(Resource)``), which changes emitted
1007
+ output for schemas that happen to use the same field name.
1008
+ """
1009
+ cached = cls.__dict__.get("_FIELD_HINTS_CACHE")
1010
+ if (
1011
+ isinstance(cached, tuple)
1012
+ and len(cached) == 2
1013
+ and isinstance(cached[0], dict)
1014
+ and isinstance(cached[1], tuple)
1015
+ ):
1016
+ return cached[1]
1017
+ hints = cls._field_hints()
1018
+ aliases = cls._FIELD_ALIASES
1019
+ wire_names = {name: aliases.get(name, name) for name in hints}
1020
+ # Membership-tested only (in _from_payload's wrapper-unwrap probe); a
1021
+ # frozenset dedupes aliases mapping to the same wire key and lets the probe
1022
+ # be a set intersection rather than a per-element scan.
1023
+ wires = frozenset(wire_names.values())
1024
+ consumed = frozenset(hints) | frozenset(aliases.values()) | {"_api", "_parent"}
1025
+ meta = (hints, wire_names, wires, consumed)
1026
+ setattr(cls, "_FIELD_HINTS_CACHE", (hints, meta))
1027
+ return meta
1028
+
1029
+
1030
+ # ---------------------------------------------------------------------------
1031
+ # Resource — base for every generated dataclass-like
1032
+ # ---------------------------------------------------------------------------
1033
+
1034
+
1035
+ class Resource:
1036
+ """Base class for every generated resource (Post, Comment, Site, etc.).
1037
+
1038
+ Instances store their fields directly as attributes. ``_api`` is the
1039
+ root client; ``_parent`` is the resource that exposed this one as a
1040
+ sub-resource (for ``$self.parent.<field>`` references during method
1041
+ calls). Fields are set by plain attribute assignment (``_extra`` carries
1042
+ any undeclared wire fields). Instances are mutable by design — no attribute
1043
+ interception is installed, so writes are ordinary.
1044
+ """
1045
+
1046
+ _RESOURCE_NAME: ClassVar[str] = "Resource"
1047
+ _KEY_FIELD: ClassVar[Optional[str]] = None
1048
+ # Python-name → wire-name map (set by codegen from FieldSpec.alias_from).
1049
+ # Lets a spec say ``created_at`` while the API returns ``created_utc``.
1050
+ _FIELD_ALIASES: ClassVar[Dict[str, str]] = {}
1051
+ # When True, this resource can be instantiated from just its key — used
1052
+ # by the coerce() path when a nested field's payload is a bare scalar
1053
+ # (e.g. ``post.subreddit = "smallbusiness"`` becomes ``Subreddit(name="smallbusiness")``).
1054
+ _CONSTRUCTIBLE: ClassVar[bool] = False
1055
+ # Resource class registry for the owning module. Codegen wires this up
1056
+ # at module import time so the string-label return router (``coerce()``)
1057
+ # can resolve a label like ``Post`` to its class.
1058
+ _RESOURCE_REGISTRY: ClassVar[Dict[str, Type[Resource]]] = {}
1059
+ # Per-class cache for resolved field annotations and derived payload metadata.
1060
+ # Stored directly on each generated subclass (looked up via ``cls.__dict__``),
1061
+ # not in a process-global dict: resolved annotations can reference sibling
1062
+ # resource classes, and a global cache value would pin transient preview
1063
+ # modules alive. A class-local cycle dies with the class/module graph. The
1064
+ # slot name is pre-existing/runtime-reserved; payload metadata deliberately
1065
+ # shares it so adding the optimization does not expand codegen's reserved
1066
+ # member set (which is seeded from dir(Resource)).
1067
+ _FIELD_HINTS_CACHE: ClassVar[Optional[Any]] = None
1068
+
1069
+ @classmethod
1070
+ def _field_hints(cls) -> Dict[str, Any]:
1071
+ """Resolved ``{field_name: type_object}`` for this class's own
1072
+ annotated fields (the single per-field type source).
1073
+
1074
+ Uses ``get_type_hints(cls)`` with NO explicit ``globalns`` so forward
1075
+ refs resolve in the *generated module's* namespace (where ``Sort`` /
1076
+ ``Post`` live). Filters out ClassVars by ORIGIN (``get_origin is
1077
+ ClassVar``) — not by name — because ``BaseAPI`` carries non-underscore
1078
+ ClassVars and the runtime bases carry underscore ClassVars; both must
1079
+ be excluded. Cached per class, resolved lazily. Never raises: a
1080
+ resolution failure yields an empty hint map (the codegen gate makes
1081
+ dangling refs impossible, but stay total).
1082
+ """
1083
+ cached = cls.__dict__.get("_FIELD_HINTS_CACHE")
1084
+ if isinstance(cached, dict):
1085
+ return cached
1086
+ # The slot is shared with the ``(hints, meta)`` payload-meta form written
1087
+ # by ``_resource_payload_meta``; recognise the SAME shape both readers
1088
+ # write (don't diverge on a looser/stricter predicate).
1089
+ if (
1090
+ isinstance(cached, tuple)
1091
+ and len(cached) == 2
1092
+ and isinstance(cached[0], dict)
1093
+ and isinstance(cached[1], tuple)
1094
+ ):
1095
+ return cached[0]
1096
+ try:
1097
+ raw_hints = get_type_hints(cls)
1098
+ except Exception: # noqa: BLE001 — total: never let resolution raise
1099
+ raw_hints = {}
1100
+ # Annotations declared on the runtime bases (Resource/object) are
1101
+ # ClassVars; the per-resource fields are the plain annotations declared
1102
+ # on the generated subclass. Filter ClassVars by origin.
1103
+ #
1104
+ # Flat-inheritance invariant: codegen emits ONLY ``class X(_Resource)``
1105
+ # (single-level — there is no ``extends``/``base`` spec key), so the
1106
+ # own-class ``__annotations__`` filter is correct-by-construction. An
1107
+ # MRO walk would defend a multi-level shape the generator cannot
1108
+ # produce, so we deliberately do NOT walk the MRO. (The codegen asserts
1109
+ # this invariant when it emits each class.)
1110
+ own = getattr(cls, "__annotations__", {})
1111
+ hints: Dict[str, Any] = {}
1112
+ for name, ann in raw_hints.items():
1113
+ if name not in own:
1114
+ continue # inherited base annotation, not a resource field
1115
+ if get_origin(ann) is ClassVar:
1116
+ continue
1117
+ hints[name] = ann
1118
+ setattr(cls, "_FIELD_HINTS_CACHE", hints)
1119
+ return hints
1120
+
1121
+ def __init__(self, *, _api: Optional[BaseAPI], _parent: Any = None, **fields: Any) -> None:
1122
+ # Plain assignment — there is no ``__setattr__``/``__slots__`` override,
1123
+ # so ``object.__setattr__`` was a no-op equal to ``self.x = v`` that
1124
+ # implied a frozen contract the class doesn't enforce.
1125
+ self._api = _api
1126
+ self._parent = _parent
1127
+ field_names = self._field_hints()
1128
+ for name in field_names:
1129
+ setattr(self, name, fields.get(name))
1130
+ # Stash anything passed that isn't a declared field, under _extra, so
1131
+ # the user can access surprise fields via ``obj._extra.get("weird")``
1132
+ # without losing data.
1133
+ self._extra = {k: v for k, v in fields.items() if k not in field_names}
1134
+ # Advisory husk signal: count of declared fields whose RAW wire
1135
+ # value was non-null. SET per-instance by ``_from_payload`` (a classmethod
1136
+ # that builds ``inst`` directly), NOT here — this is an annotation-only
1137
+ # declaration so pyright sees the instance attribute without us actually
1138
+ # creating it. A bare ``self.x: int`` performs no assignment, so instances
1139
+ # NOT built from a payload (e.g. coerce()'s key-only constructions) keep
1140
+ # NO attribute, and the husk read ``getattr(inst, "_resolved_count", None)``
1141
+ # still falls back to ``None`` for them (≠ 0 → not a husk). Declared here
1142
+ # rather than as a class annotation so it stays OUT of ``_field_hints`` /
1143
+ # ``get_type_hints`` and is never mistaken for a coercible wire field.
1144
+ self._resolved_count: int
1145
+
1146
+ @classmethod
1147
+ def _from_payload(
1148
+ cls,
1149
+ payload: Dict[str, Any],
1150
+ *,
1151
+ api: Optional[BaseAPI],
1152
+ parent: Any = None,
1153
+ ) -> Resource:
1154
+ """Construct an instance from a raw response dict, coercing each
1155
+ declared field by its resolved ANNOTATION (single source).
1156
+ ``_FIELD_ALIASES`` lets a Python field name read from a different wire
1157
+ key.
1158
+
1159
+ The instance is built first (with raw values), then each field is
1160
+ coerced with ``parent`` set to THIS instance — so a nested resource's
1161
+ ``_parent`` is the OWNING resource (active-record navigation), not the
1162
+ grandparent. ``_api`` is threaded all the way down.
1163
+ """
1164
+ hints, wire_names, wires, consumed = _resource_payload_meta(cls)
1165
+ # Wrapped-record unwrap — the single-RECORD analogue of
1166
+ # ``_unwrap_single_list``. Detail APIs commonly nest the record under a
1167
+ # wrapper key (``{"product": {...}}``, ``{"detail": {...}}``); a spec
1168
+ # whose fields were typed from the INNER record would otherwise hydrate
1169
+ # to an all-None husk (2026-06-09 corpus audit: 46/244 specs). Fires
1170
+ # ONLY when (a) not one declared wire key resolves at the top level —
1171
+ # so any legitimately-flat payload is untouched — and (b) exactly ONE
1172
+ # dict-valued child contains at least one declared wire key (two
1173
+ # matching children = ambiguous, leave raw rather than guess). When a
1174
+ # wrapper is selected, sibling envelope metadata is still forwarded into
1175
+ # _extra; only the structural wrapper key is dropped.
1176
+ if isinstance(payload, dict) and payload and hints:
1177
+ if not (wires & payload.keys()):
1178
+ _cands = [(k, v) for k, v in payload.items()
1179
+ if isinstance(v, dict) and (wires & v.keys())]
1180
+ if len(_cands) == 1:
1181
+ _wrapper, _wrapped = _cands[0]
1182
+ _outer_extra = {k: v for k, v in payload.items() if k != _wrapper}
1183
+ payload = {**_outer_extra, **_wrapped}
1184
+ # Never forward a wire key literally named ``_api``/``_parent`` into
1185
+ # raw_fields: it would collide with the explicit kwargs in
1186
+ # ``cls(_api=..., _parent=..., **raw_fields)`` below and raise "multiple
1187
+ # values for keyword argument". codegen reserves these as field
1188
+ # identifiers, but an UNMAPPED wire key of that name would otherwise slip
1189
+ # through the unknown-field forwarding — drop it.
1190
+ raw_fields: Dict[str, Any] = {}
1191
+ resolved_count = 0
1192
+ for name in hints:
1193
+ raw = payload.get(wire_names[name])
1194
+ raw_fields[name] = raw
1195
+ if raw is not None:
1196
+ resolved_count += 1
1197
+ # Forward unknown fields so _extra has them — but skip wire names that
1198
+ # were already consumed via _FIELD_ALIASES (else we'd both set
1199
+ # ``created_at`` AND leak ``created_utc`` into _extra).
1200
+ for k, v in payload.items():
1201
+ if k not in consumed:
1202
+ raw_fields[k] = v
1203
+ inst = cls(_api=api, _parent=parent, **raw_fields)
1204
+ # Resolved-key signal (husk observation). Count the DECLARED fields
1205
+ # whose post-unwrap raw wire value is non-null — the husk verdict in
1206
+ # coerce() reads this instead of re-running the wrapped-record unwrap.
1207
+ # It is a private instance attribute, set on EVERY constructed instance,
1208
+ # and is purely advisory: _from_payload itself NEVER raises on it (the
1209
+ # verdict lives in coerce(), which is NOT shared with the lenient
1210
+ # per-field router). Computed from the RAW wire values (pre-coercion) so
1211
+ # a lenient coercion that degrades a value to None can't mask a key that
1212
+ # the wire actually resolved.
1213
+ inst._resolved_count = resolved_count
1214
+ # Coerce annotated fields now that the owning instance exists, threading
1215
+ # it as the parent of every nested + list/dict element. Plain assignment
1216
+ # (no __setattr__ override to bypass).
1217
+ key_field = cls._KEY_FIELD
1218
+ for name, ann in hints.items():
1219
+ coerced = _coerce_by_annotation(raw_fields[name], ann, api=api, parent=inst,
1220
+ is_key=(name == key_field))
1221
+ setattr(inst, name, coerced)
1222
+ return inst
1223
+
1224
+ def __repr__(self) -> str:
1225
+ key = self._KEY_FIELD
1226
+ if key and hasattr(self, key):
1227
+ return f"{type(self).__name__}({key}={getattr(self, key)!r})"
1228
+ return f"{type(self).__name__}(...)"
1229
+
1230
+ def _identity_key(self) -> Any:
1231
+ """The hashable identity value and its hash, or ``None`` when this instance has no
1232
+ usable identity: no ``_KEY_FIELD``; a null key value (a null never
1233
+ asserts identity — two null-keyed instances are NOT the same record);
1234
+ or an unhashable container that leaked into the key field through
1235
+ lenient coercion. ``__eq__`` and ``__hash__`` both route through this
1236
+ single source so they can never disagree, and ``__hash__`` can reuse
1237
+ the hashability probe instead of calling ``hash(val)`` twice."""
1238
+ key = self._KEY_FIELD
1239
+ if not key:
1240
+ return None
1241
+ val = getattr(self, key, None)
1242
+ if val is None:
1243
+ return None
1244
+ try:
1245
+ val_hash = hash(val)
1246
+ except TypeError:
1247
+ return None
1248
+ return val, val_hash
1249
+
1250
+ def __eq__(self, other: Any) -> bool:
1251
+ if not isinstance(other, type(self)):
1252
+ return NotImplemented
1253
+ mine = self._identity_key()
1254
+ theirs = other._identity_key()
1255
+ if mine is None or theirs is None:
1256
+ return self is other
1257
+ return mine[0] == theirs[0]
1258
+
1259
+ def __hash__(self) -> int:
1260
+ identity = self._identity_key()
1261
+ if identity is None:
1262
+ return id(self)
1263
+ return hash((type(self).__name__, identity[1]))
1264
+
1265
+
1266
+ # ---------------------------------------------------------------------------
1267
+ # Collection — typed namespace for sub-resources (`site.subareas`, `reddit.posts`)
1268
+ # ---------------------------------------------------------------------------
1269
+
1270
+
1271
+ class Collection:
1272
+ """A typed handle on a sub-resource or top-level resource collection.
1273
+
1274
+ The codegen subclasses this per collection so static type checkers see
1275
+ typed ``.get(...)`` / ``.list(...)`` etc. methods. The runtime base
1276
+ just holds the API + parent references.
1277
+ """
1278
+
1279
+ def __init__(self, _api: BaseAPI, _parent: Any = None) -> None:
1280
+ self._api = _api
1281
+ self._parent = _parent
1282
+
1283
+
1284
+ # ---------------------------------------------------------------------------
1285
+ # Paginator — auto-paging iterator returned by `paginates`-tagged operations
1286
+ # ---------------------------------------------------------------------------
1287
+
1288
+
1289
+ _PaginatorItem = TypeVar("_PaginatorItem")
1290
+
1291
+
1292
+ def _resource_content_projection(it: "Resource") -> Any:
1293
+ """A STABLE, hashable content signature for a keyless / null-keyed
1294
+ ``Resource`` — the runtime fix for the keyless ``_page_signature`` churn.
1295
+
1296
+ A keyless ``Resource`` (``_KEY_FIELD = None``) hashes by ``id(self)``, so the
1297
+ historical signature churned and the content-stall guard NEVER matched a
1298
+ keyless full-page repeat across pages. Two DISTINCT objects carrying the SAME
1299
+ wire content must project EQUAL so the guard can see that repeat; two
1300
+ genuinely-different keyless rows must project differently (no false trigger).
1301
+
1302
+ Content is mined from the resource's declared fields (the single per-class
1303
+ source) + any undeclared wire data in ``_extra``, sorted by name so dict
1304
+ order can't perturb the projection. Each value is taken verbatim when
1305
+ hashable, else rendered to a stable JSON string (``sort_keys``) so an
1306
+ unhashable list/dict field can't break signing; an exotic value falls back
1307
+ to ``repr``. The result is ALWAYS hashable — a keyless Resource therefore
1308
+ always signs (it never forces the whole page to ``None``).
1309
+
1310
+ Scope is deliberately KEYLESS RESOURCES ONLY: a raw dict/list passthrough
1311
+ keeps the historical "unhashable → page unsignable → ``None``" behavior in
1312
+ ``_page_signature`` (no content-stall compare), so a heterogeneous
1313
+ keyed-then-raw page stream still terminates at ``max_fetches`` rather than
1314
+ newly raising — preserving cross-compat for every already-generated module."""
1315
+ def _stable(v: Any) -> Any:
1316
+ # A NESTED Resource value must canonicalize by its CONTENT, not be
1317
+ # returned verbatim: a keyless Resource hashes by ``id(self)`` (so the
1318
+ # ``hash(v)`` probe below would succeed and return the churning instance),
1319
+ # which made a keyless value-object field churn across pages and left the
1320
+ # content-stall guard ineffective for nested-value-object shapes (codex
1321
+ # P2). Recurse into its content projection instead. A list/tuple is
1322
+ # canonicalized element-wise so a ``List[Resource]`` value-object field
1323
+ # canonicalizes too; raw dict/list (non-Resource) keeps the historical
1324
+ # json.dumps passthrough below.
1325
+ if isinstance(v, Resource):
1326
+ return _resource_content_projection(v)
1327
+ if isinstance(v, (list, tuple)):
1328
+ return tuple(_stable(x) for x in v)
1329
+ try:
1330
+ hash(v)
1331
+ return v
1332
+ except TypeError:
1333
+ pass
1334
+ try:
1335
+ return json.dumps(v, sort_keys=True, default=str)
1336
+ except (TypeError, ValueError):
1337
+ return repr(v)
1338
+
1339
+ fields = it._field_hints()
1340
+ parts: List[tuple] = [(name, _stable(getattr(it, name, None))) for name in fields]
1341
+ extra = getattr(it, "_extra", None)
1342
+ if isinstance(extra, dict):
1343
+ parts += [(f"_extra:{k}", _stable(v)) for k, v in extra.items()]
1344
+ return ("R", type(it).__name__, tuple(sorted(parts)))
1345
+
1346
+
1347
+ def _page_signature(items: Iterable[Any]) -> Optional[tuple]:
1348
+ """An ordered tuple of per-item identity signals for one page, or ``None``
1349
+ if the page can't be signed.
1350
+
1351
+ Each item's signal is its ``_KEY_FIELD`` value when the item is a keyed
1352
+ ``Resource``; a keyless / null-keyed ``Resource`` signs by a STABLE CONTENT
1353
+ PROJECTION (:func:`_resource_content_projection`) instead of the historical
1354
+ ``id(self)`` churn — so the content-stall guard can now catch a keyless
1355
+ full-page repeat (two pages of distinct-object-but-same-content keyless rows
1356
+ project EQUAL; genuinely-different keyless content projects differently, no
1357
+ false trigger). A NON-Resource item (raw dict / list / scalar passthrough)
1358
+ keeps the historical rule: hashable → the item itself; unhashable → the
1359
+ whole page is unsignable and we return ``None`` (the caller skips the
1360
+ content-stall compare). That scope keeps a heterogeneous keyed-then-raw
1361
+ stream cross-compatible — a raw-dict page still never signs, so it can't
1362
+ newly raise on an already-generated module.
1363
+
1364
+ Ordered (not a set) so a reordered page reads as different — the strictest
1365
+ same-page signal. For KEYED items the signal is the key only (never full
1366
+ content) so volatile non-key fields (vote counts, timestamps) can't mask a
1367
+ stall; for KEYLESS RESOURCES full content is the only available identity, so
1368
+ the documented residual stands: a keyless feed whose entire page genuinely
1369
+ re-appears within the recent-signature window reads as a stall (the error
1370
+ names the ``max_fetches=`` override).
1371
+ """
1372
+ sig: List[Any] = []
1373
+ for it in items:
1374
+ kf = getattr(type(it), "_KEY_FIELD", None)
1375
+ if kf:
1376
+ # A KEYED Resource: sign by its key value when present. A NULL key on
1377
+ # this page (key only on detail responses, or an alias mis-mapping)
1378
+ # keeps the historical id-hash behavior — two pages of DISTINCT
1379
+ # null-keyed-on-list records must NOT collapse to one signature and
1380
+ # false-stall (``test_null_keyed_pages_do_not_false_stall``). The
1381
+ # content-projection fix is scoped to GENUINELY KEYLESS resources
1382
+ # below, where id-hash actively hid a real repeat; a keyed-but-
1383
+ # null-on-this-page record is a distinct record whose key simply
1384
+ # isn't on the list page — the cross-compat shape we must not regress.
1385
+ key = getattr(it, kf, None)
1386
+ if key is not None:
1387
+ sig.append(key)
1388
+ continue
1389
+ try:
1390
+ hash(it)
1391
+ except TypeError:
1392
+ return None
1393
+ sig.append(it)
1394
+ continue
1395
+ if isinstance(it, Resource):
1396
+ # Keyless Resource (``_KEY_FIELD is None``): sign by stable content so
1397
+ # a keyless full-page repeat is visible to the stall guard (the
1398
+ # fix — id-hash churned and hid it). Distinct keyless content still
1399
+ # projects differently; the accepted residual is a genuinely
1400
+ # content-identical keyless full page reading as a stall.
1401
+ sig.append(_resource_content_projection(it))
1402
+ continue
1403
+ # Non-Resource passthrough (raw dict/list/scalar) — historical rule:
1404
+ # hashable → the item; unhashable → whole page unsignable (``None``).
1405
+ try:
1406
+ hash(it)
1407
+ except TypeError:
1408
+ return None
1409
+ sig.append(it)
1410
+ return tuple(sig)
1411
+
1412
+
1413
+ class Paginator(Generic[_PaginatorItem]):
1414
+ """Iterator that fetches more pages as the user consumes them.
1415
+
1416
+ Build kwargs contain the original method args; ``_fetch_page`` returns
1417
+ ``(items, next_cursor)``. Iteration stops when ``next_cursor`` is None
1418
+ or the user breaks. Cap by passing ``limit=`` at call time — the
1419
+ paginator will short-circuit once that many items have been yielded.
1420
+
1421
+ Generic so codegen can emit ``Paginator[Listing]`` / ``Paginator[Post]``
1422
+ return annotations. Without that, ``for listing in api.listings.search()``
1423
+ widens ``listing`` to ``Any`` and pyright loses all field-access
1424
+ intelligence — defeating the whole point of the typed SDK.
1425
+
1426
+ **Stall safety.** Two disjoint terminators stop a runaway paginator
1427
+ (a server that re-returns the same page because its pagination contract
1428
+ was mis-derived):
1429
+
1430
+ * a **content-stall guard** — measures progress by the *data* (an
1431
+ ordered tuple of per-item keys). When a non-empty page repeats ANY of
1432
+ the last 4 page signatures it raises :class:`PaginationLimitError` —
1433
+ so a same-page stall dies on the 2nd fetch and a short-period cycle
1434
+ (the realistic mis-derived page/offset oscillation) on the 3rd-5th,
1435
+ instead of burning the full billable fetch budget. Cursor-agnostic
1436
+ (the SDK fabricates an advancing cursor for token-less servers, so
1437
+ the cursor is useless as a progress signal); unsignable pages
1438
+ (raw-dict passthrough) are simply invisible to it, never a false
1439
+ trigger. Residual false-positive shape: a burst of ≈k·page_size new
1440
+ items landing between two fetches of an offset-paginated moving feed
1441
+ can make page N+k repeat page N's keys exactly — rare, and the error
1442
+ names the ``max_fetches=`` override an agent can act on.
1443
+ * a **``max_fetches`` iteration bound** — the sole terminator for the
1444
+ cases the content guard cannot see (slow-crawl, unhashable streams,
1445
+ cycles with period > 4). User-overridable; raise it for legitimate
1446
+ deep pagination.
1447
+
1448
+ These cover *disjoint* failure modes — not two layers over one. A
1449
+ previously-hanging paginator now *raises* instead; this terminal outcome
1450
+ applies to already-rendered modules on a runtime bump (they import
1451
+ ``Paginator`` from here) without regeneration. The ``max_fetches`` arm
1452
+ also raises on a *legitimately-completing* result set that exceeds the
1453
+ budget (not only hanging ones) — an unbounded ``.list()`` over >1000 pages
1454
+ must pass a larger ``max_fetches=``; the error message names the override.
1455
+
1456
+ **Window exhaustion.** A third terminator covers the upstream's own result
1457
+ window: a mid-stream 4xx (after ≥1 successful page, excluding 429
1458
+ backpressure) means the site refuses deeper pages — e.g. GitHub search
1459
+ serves at most 1000 results regardless of ``total``. That raises
1460
+ :class:`PaginationLimitError` (with the ``UpstreamError`` as ``__cause__``)
1461
+ rather than stopping silently: yielded items are valid, but treating a cap
1462
+ as "the result set ends here" would hand the consumer a wrong-data
1463
+ conclusion. First-fetch errors and 5xx/network failures surface raw.
1464
+ """
1465
+
1466
+ def __init__(
1467
+ self,
1468
+ fetch_page: Callable[[Optional[Any]], "tuple[List[_PaginatorItem], Optional[Any]]"],
1469
+ *,
1470
+ limit: Optional[int] = None,
1471
+ max_fetches: int = 1000,
1472
+ dedup: bool = False,
1473
+ identity_dedup: bool = False,
1474
+ _api: Optional[Any] = None,
1475
+ ) -> None:
1476
+ # Validate here, not in __next__: the constructor is the single
1477
+ # chokepoint every generated op routes through, and a bad kwarg must
1478
+ # fail at the call site that supplied it — not pages later inside
1479
+ # iteration with a bare comparison TypeError. bool is excluded
1480
+ # explicitly (it passes isinstance(int) but `limit=True` is a bug).
1481
+ if limit is not None and (not isinstance(limit, int) or isinstance(limit, bool)):
1482
+ raise TypeError(
1483
+ f"limit must be an int or None, got {limit!r} ({type(limit).__name__})"
1484
+ )
1485
+ if limit is not None and limit < 1:
1486
+ raise ValueError(f"limit must be >= 1, got {limit!r}")
1487
+ if not isinstance(max_fetches, int) or isinstance(max_fetches, bool):
1488
+ raise TypeError(
1489
+ f"max_fetches must be an int, got {max_fetches!r} ({type(max_fetches).__name__})"
1490
+ )
1491
+ if max_fetches < 1:
1492
+ raise ValueError(f"max_fetches must be >= 1, got {max_fetches!r}")
1493
+ # identity_dedup is the raw-page-signature arm layered ON TOP of dedup
1494
+ # (consumed only inside ``if self._dedup:`` in __next__). Passing it with
1495
+ # dedup=False would silently no-op the entire stall-guard, so reject the
1496
+ # combination here rather than let the flag be invisibly inert at runtime.
1497
+ if identity_dedup and not dedup:
1498
+ raise ValueError("identity_dedup=True requires dedup=True")
1499
+ self._fetch_page = fetch_page
1500
+ self._api = _api # owning client, for last_meta passthrough
1501
+ self._limit = limit
1502
+ self._max_fetches = max_fetches
1503
+ # dedup-by-identity: a scheme that can re-serve overlapping items across
1504
+ # pages (time_cursor's inclusive-boundary re-seed; a page/cursor op on a
1505
+ # moving feed). The next request re-seeds and the boundary/overlap items
1506
+ # repeat and would be yielded twice — the content-stall guard only catches
1507
+ # WHOLE-page repeats, not partial overlap. With dedup on, items already
1508
+ # seen (by their _Resource identity / _KEY_FIELD) are dropped, and a
1509
+ # NON-EMPTY page contributing 0 NEW items terminates gracefully —
1510
+ # generalizing the content-stall guard and killing the infinite re-seed.
1511
+ #
1512
+ # F-2 carve-out: a genuinely EMPTY raw page (0 server rows) with a still-
1513
+ # advancing cursor is NOT a 0-new terminus — it is an interstitial gap
1514
+ # that gets RE-FETCHED (matching the non-dedup path), so a single empty
1515
+ # interstitial page no longer silently truncates every page after it. The
1516
+ # 0-new STOP fires only for a NON-empty page whose every row is already
1517
+ # seen (the inclusive-boundary re-seed). See ``__next__``.
1518
+ #
1519
+ # F-3 accepted residual (dedup WITHOUT server cooperation): when the
1520
+ # server's cursor keeps advancing on a NON-empty page whose rows are all
1521
+ # already-seen overlap, the SDK cannot tell that 0-new terminus apart
1522
+ # from a transient overlap that a LATER page would break — both look
1523
+ # identical at the moment of decision (the server gives no end signal).
1524
+ # We take the 0-new STOP (the only contract that terminates the infinite
1525
+ # inclusive-boundary re-seed); a feed that re-serves a fully-overlapping
1526
+ # page mid-stream and only then advances to fresh rows is the documented
1527
+ # residual. It is bounded and safe (it never yields wrong data — it can
1528
+ # only under-read), and a consumer who needs the deeper rows passes a
1529
+ # wider window via a larger ``max_fetches`` on the non-dedup path. Not
1530
+ # over-engineered with a speculative look-ahead.
1531
+ self._dedup = dedup
1532
+ # identity_dedup: the page/cursor identity-dedup arm (the
1533
+ # sample-proven-unique ops) adds the raw-page-signature raise/terminate
1534
+ # rule ON TOP of dedup — BEFORE deduping, a RAW page whose signature
1535
+ # repeats a recent one is a genuine re-served page → raise
1536
+ # PaginationLimitError; a 0-fresh-after-dedup page whose raw signature is
1537
+ # NOVEL is a natural overlap-boundary end → StopIteration. time_cursor
1538
+ # emits ``dedup=True`` ALONE (``identity_dedup`` defaults False), so it
1539
+ # keeps its graceful 0-new stop unchanged — and every ALREADY-GENERATED
1540
+ # time_cursor module (which passes only ``dedup=True``) is behavior- and
1541
+ # byte-stable on a runtime bump (the new flag is opt-in, default-off).
1542
+ self._identity_dedup = identity_dedup
1543
+ self._seen: set = set()
1544
+ self._yielded = 0
1545
+ self._fetches = 0
1546
+ self._cursor: Optional[Any] = None
1547
+ self._buffer: deque[_PaginatorItem] = deque()
1548
+ # Last 4 page signatures (not just the previous one): membership is a
1549
+ # linear ``in`` over ≤4 tuples — signatures may hold unhashable key
1550
+ # values, so a set is not an option (and is not needed at this size).
1551
+ self._recent_sigs: "deque[tuple]" = deque(maxlen=4)
1552
+ self._done = False
1553
+
1554
+ @property
1555
+ def pages_fetched(self) -> int:
1556
+ """Number of pages fetched from the wire so far."""
1557
+ return self._fetches
1558
+
1559
+ @property
1560
+ def items_yielded(self) -> int:
1561
+ """Number of items yielded to the caller so far."""
1562
+ return self._yielded
1563
+
1564
+ @property
1565
+ def exhausted(self) -> bool:
1566
+ """True once iteration has terminated (no more pages)."""
1567
+ return self._done
1568
+
1569
+ @property
1570
+ def last_meta(self) -> Optional["RequestMeta"]:
1571
+ """Control-loop metadata from the owning client's most recent page."""
1572
+ return getattr(self._api, "last_meta", None)
1573
+
1574
+ def __iter__(self) -> "Paginator[_PaginatorItem]":
1575
+ return self
1576
+
1577
+ def __next__(self) -> _PaginatorItem:
1578
+ if self._limit is not None and self._yielded >= self._limit:
1579
+ raise StopIteration
1580
+ while not self._buffer:
1581
+ if self._done:
1582
+ raise StopIteration
1583
+ if self._fetches >= self._max_fetches:
1584
+ raise PaginationLimitError(
1585
+ f"pagination exceeded max_fetches={self._max_fetches} — likely a "
1586
+ f"mis-derived pagination contract, or raise max_fetches= for "
1587
+ f"legitimate deep pagination",
1588
+ pages_fetched=self._fetches, items_yielded=self._yielded,
1589
+ )
1590
+ try:
1591
+ items, next_cursor = self._fetch_page(self._cursor)
1592
+ except UpstreamError as err:
1593
+ # Window-exhaustion classification. A mid-stream upstream 4xx —
1594
+ # the same params that already succeeded N times, refused one
1595
+ # cursor deeper — is a data boundary (a result-window cap, e.g.
1596
+ # GitHub search caps at 1000), not a transient failure: raise
1597
+ # the typed pagination error with the evidence chained, never
1598
+ # a raw UpstreamError after minutes of valid iteration.
1599
+ # Carve-outs, both deliberate:
1600
+ # * first fetch (_fetches == 0) never classifies — a param/
1601
+ # auth error must surface exactly as on a non-paginated op;
1602
+ # * upstream 429 is backpressure, not a boundary — classify-
1603
+ # ing it would tell the consumer the result set ends here
1604
+ # (the wrong-data failure this design exists to prevent).
1605
+ # The SDK's retry machinery covers only GATEWAY 429s, so
1606
+ # the upstream-429-in-502-envelope surfaces raw.
1607
+ # A genuine page-N-only 4xx is indistinguishable from a window
1608
+ # cap; after N identical-param successes the boundary is the
1609
+ # correct bet, and __cause__ preserves the evidence either way.
1610
+ usc = err.upstream_status_code
1611
+ if (
1612
+ self._fetches >= 1
1613
+ and isinstance(usc, int)
1614
+ and 400 <= usc < 500
1615
+ and usc != 429
1616
+ ):
1617
+ raise PaginationLimitError(
1618
+ f"upstream result window exhausted after "
1619
+ f"{self._yielded} items / {self._fetches} pages "
1620
+ f"(upstream HTTP {usc}); items already yielded are "
1621
+ f"valid — catch PaginationLimitError or pass limit= "
1622
+ f"to bound the scan",
1623
+ pages_fetched=self._fetches, items_yielded=self._yielded,
1624
+ ) from err
1625
+ raise
1626
+ self._fetches += 1
1627
+ self._buffer = deque(items or [])
1628
+ self._cursor = next_cursor
1629
+ if next_cursor is None:
1630
+ self._done = True
1631
+ if self._dedup:
1632
+ # Whether the server returned an EMPTY page (0 raw rows) — distinct
1633
+ # from a non-empty page that DEDUPS to 0 fresh. An empty raw page
1634
+ # with an advancing cursor is an interstitial gap that must be
1635
+ # RE-FETCHED (the non-dedup path does so via the `while not
1636
+ # self._buffer` loop below); only a non-empty page whose every row
1637
+ # is already seen is the inclusive-boundary 0-new terminus. F-2:
1638
+ # the old dedup arm conflated the two and stopped on the empty
1639
+ # interstitial page, silently dropping every page after it.
1640
+ _raw_empty = not self._buffer
1641
+ if self._identity_dedup:
1642
+ # raw-page-signature raise/terminate (page/cursor identity
1643
+ # dedup only — time_cursor keeps its graceful 0-new stop).
1644
+ # Sign the RAW (pre-dedup) page and reuse the SAME _recent_sigs
1645
+ # the content-stall guard owns: a raw page whose signature
1646
+ # repeats a recent one is a GENUINE re-served page (the cursor
1647
+ # stalled) → raise, exactly as the non-dedup content guard
1648
+ # already does for a repeated non-empty page. A 0-fresh page
1649
+ # whose raw signature is NOVEL is a natural overlap-boundary
1650
+ # end → falls through to the graceful 0-new stop below. (An
1651
+ # EMPTY raw page signs as the empty tuple `()`; guarding the
1652
+ # raise on `_raw_sig` truthiness keeps a bare interstitial
1653
+ # empty page from ever being mis-read as a re-served page.)
1654
+ _raw_sig = _page_signature(self._buffer)
1655
+ if _raw_sig:
1656
+ if _raw_sig in self._recent_sigs:
1657
+ raise PaginationLimitError(
1658
+ "pagination stalled — the server re-served the same "
1659
+ "page of items before dedup; likely a mis-derived "
1660
+ "pagination contract, or raise max_fetches= for "
1661
+ "legitimate deep pagination",
1662
+ pages_fetched=self._fetches, items_yielded=self._yielded,
1663
+ )
1664
+ self._recent_sigs.append(_raw_sig)
1665
+ # Drop items already seen (by _Resource identity / _KEY_FIELD; a
1666
+ # _KEY_FIELD=None resource hashes by id(self) → never collides, so
1667
+ # no data loss). A non-empty page contributing 0 NEW items
1668
+ # terminates gracefully — this supersedes the content-stall check
1669
+ # on the time_cursor path (it handles PARTIAL overlap, which the
1670
+ # whole-page stall guard cannot). max_fetches stays the backstop.
1671
+ fresh: List[_PaginatorItem] = []
1672
+ for it in self._buffer:
1673
+ if isinstance(it, Resource):
1674
+ # Dedup on the identity TOKEN, not the instance — the
1675
+ # set otherwise retains every yielded Resource (full
1676
+ # payload) for the whole iteration. ``_identity_key``
1677
+ # is the single equality source: when it is None (no
1678
+ # key field / null key / unhashable key) the instance
1679
+ # has no identity and membership could never fire
1680
+ # (id-hash vs fresh objects), so skip dedup for it —
1681
+ # observationally identical, retention dropped.
1682
+ idk = it._identity_key()
1683
+ if idk is not None:
1684
+ token = (type(it), idk[0])
1685
+ if token in self._seen:
1686
+ continue
1687
+ self._seen.add(token)
1688
+ fresh.append(it)
1689
+ continue
1690
+ try:
1691
+ if it in self._seen:
1692
+ continue
1693
+ self._seen.add(it)
1694
+ except TypeError:
1695
+ pass # unhashable item → never deduped (fall back to bounds)
1696
+ fresh.append(it)
1697
+ self._buffer = deque(fresh)
1698
+ if not self._buffer:
1699
+ if _raw_empty:
1700
+ # The RAW page was empty — an interstitial gap, NOT the
1701
+ # inclusive-boundary terminus. Re-fetch if the cursor is
1702
+ # still live (the non-dedup behavior); stop only when the
1703
+ # cursor itself ended (`self._done`). Bounded by
1704
+ # max_fetches above, exactly like the non-dedup empty-page
1705
+ # path. This is the F-2 data-loss fix.
1706
+ if self._done:
1707
+ raise StopIteration
1708
+ continue
1709
+ # A NON-EMPTY page that dedups to 0 fresh items: a fully-seen
1710
+ # re-seed (inclusive boundary). Stop — re-fetching would
1711
+ # re-seed the same cursor identically (infinite loop). This is
1712
+ # time_cursor's graceful 0-new terminus.
1713
+ self._done = True
1714
+ raise StopIteration
1715
+ continue # dedup path owns its own termination; skip stall guard
1716
+ if not self._buffer:
1717
+ if self._done:
1718
+ raise StopIteration
1719
+ # else: empty page but more cursors — re-fetch (bounded by
1720
+ # max_fetches above; excluded from the content compare so a
1721
+ # fabricated-cursor empty-page re-fetch isn't misclassified).
1722
+ continue
1723
+ # Content-stall guard — recomputed per page (no cached capability
1724
+ # flag). Only compares when the page is signable; an unsignable
1725
+ # page leaves the window untouched (appending a None would let a
1726
+ # later None == None false-trigger on a legit stream). Matching
1727
+ # ANY of the last 4 signatures catches the same-page stall (lag 1)
1728
+ # AND the short-period cycle (a mis-derived page/offset contract
1729
+ # oscillating between pages) at fetch ~3 instead of burning the
1730
+ # whole max_fetches budget of billable requests.
1731
+ sig = _page_signature(self._buffer)
1732
+ if sig is not None:
1733
+ if sig in self._recent_sigs:
1734
+ raise PaginationLimitError(
1735
+ "pagination stalled — the server re-returned the same page "
1736
+ "of items; likely a mis-derived pagination contract, or raise "
1737
+ "max_fetches= for legitimate deep pagination",
1738
+ pages_fetched=self._fetches, items_yielded=self._yielded,
1739
+ )
1740
+ self._recent_sigs.append(sig)
1741
+ v = self._buffer.popleft()
1742
+ self._yielded += 1
1743
+ return v
1744
+
1745
+ def list(self) -> List[_PaginatorItem]:
1746
+ """Materialize all pages into a list. Convenient when you know the
1747
+ result set is small. Respects the ``limit`` cap.
1748
+
1749
+ On a :class:`PaginationLimitError` the items collected so far are
1750
+ attached as ``err.partial_items`` before the error propagates — the
1751
+ materialized prefix is valid data and must not vanish with the raise.
1752
+ (Raw ``list(paginator)`` / comprehensions cannot offer this carry;
1753
+ prefer ``.list()``.)"""
1754
+ out: List[_PaginatorItem] = []
1755
+ try:
1756
+ for v in self:
1757
+ out.append(v)
1758
+ except PaginationLimitError as err:
1759
+ err.partial_items = list(out)
1760
+ raise
1761
+ return out
1762
+
1763
+ def first(self) -> Optional[_PaginatorItem]:
1764
+ """Return the first item or None. Stops paging after that item."""
1765
+ for v in self:
1766
+ return v
1767
+ return None
1768
+
1769
+
1770
+ # ---------------------------------------------------------------------------
1771
+ # BaseAPI — root client class
1772
+ # ---------------------------------------------------------------------------
1773
+
1774
+
1775
+ class BaseAPI:
1776
+ """Base class for every generated root client (Reddit, Craigslist, ...).
1777
+
1778
+ Per-API subclasses set ``SCRAPER_ID`` and expose top-level resource
1779
+ collections / methods. The base handles auth, transport, error
1780
+ classification, and the executor's response-envelope unwrap.
1781
+ """
1782
+
1783
+ SCRAPER_ID: ClassVar[str] = ""
1784
+ # Pinned snapshot version for a locally-added MARKETPLACE canonical: codegen
1785
+ # bakes this onto the generated client (from the `[tool.parse].marketplace`
1786
+ # entry's recorded version) so every call sends ``API-Snapshot-Version: N``
1787
+ # and the executor serves that exact release — the contract stays frozen even
1788
+ # as the canonical self-heals. None for owned/account APIs (track latest).
1789
+ PINNED_VERSION: ClassVar[Optional[int]] = None
1790
+ # Generated clients override with ``{wire_kind: GeneratedErrorClass}`` for
1791
+ # every ErrorSpec that declared a ``code``; empty here so the base + any
1792
+ # client without declared error codes falls through to the HTTP-status table.
1793
+ _ERROR_CLASSES: ClassVar[Dict[str, type]] = {}
1794
+
1795
+ # Retry defaults: low (an agent must SEE backpressure, not be silently
1796
+ # blocked behind a long backoff chain), and bounded — a retry re-runs a
1797
+ # billable scrape, so only safe (GET/HEAD) requests are ever retried.
1798
+ _DEFAULT_MAX_RETRIES: ClassVar[int] = 1
1799
+ _RETRY_MAX_BACKOFF: ClassVar[float] = 30.0 # per-retry ceiling (s)
1800
+ _RETRY_TOTAL_BUDGET: ClassVar[float] = 60.0 # total-elapsed cap (s)
1801
+ _RETRY_BASE_BACKOFF: ClassVar[float] = 0.5 # exponential base (s)
1802
+ _RETRY_SAFE_METHODS: ClassVar[frozenset] = frozenset({"GET", "HEAD"})
1803
+
1804
+ def __init__(
1805
+ self,
1806
+ api_key: Optional[str] = None,
1807
+ base_url: Optional[str] = None,
1808
+ timeout: float = 60.0,
1809
+ *,
1810
+ max_retries: Optional[int] = None,
1811
+ ) -> None:
1812
+ # Store the key (which may be None) WITHOUT raising — keyless
1813
+ # construction is allowed for offline / typed-only introspection
1814
+ # (``Root()`` to read the modeled surface without a key). The
1815
+ # runtime-safety raise moves to the first ``_request`` call.
1816
+ # Route construction through config.resolve() so `parse login`
1817
+ # (which writes ~/.config/parse/credentials) authenticates scripts,
1818
+ # not just the CLI. resolve() chains kwarg → PARSE_API_KEY env →
1819
+ # credentials file → default base_url (the KEY has no default).
1820
+ # Keyless construction still succeeds; the AuthError stays lazy in
1821
+ # _request. Do NOT call require_key() here — wrong type, and it would
1822
+ # break offline/keyless introspection.
1823
+ import httpx # lazy (see plan 003) — a dict lookup after first use
1824
+
1825
+ cfg = _resolve_config(api_key, base_url)
1826
+ self._api_key = cfg.api_key # may be None
1827
+ # OAuth access token from `parse login --web` (host-gated by resolve()).
1828
+ # Used as a Bearer when no api_key is present; the api_key path is
1829
+ # unchanged. Long-running scripts on a web-login token must re-login
1830
+ # after the ~1h token lifetime (the CLI refreshes proactively; the
1831
+ # runtime does not).
1832
+ self._access_token = cfg.access_token # may be None
1833
+ self._base_url = cfg.base_url # already rstrip'd, defaulted
1834
+ self._timeout = timeout
1835
+ # A bare float fans out to all four httpx timeout classes — fine for
1836
+ # read/write/pool (slow scrapes are legitimate), wrong for CONNECT:
1837
+ # TCP connect to the API host either succeeds in milliseconds or the
1838
+ # host is unreachable, and a blackholed route would stall an agent the
1839
+ # full 60s before the (good) network-error message appears. Bound
1840
+ # connect; a per-request ``timeout=`` override REPLACES all four
1841
+ # (httpx semantics), which callers opting in accept.
1842
+ self._client = httpx.Client(
1843
+ timeout=httpx.Timeout(timeout, connect=min(timeout, 10.0))
1844
+ )
1845
+ # Control-loop metadata from the most recent SUCCESSFUL response. Read
1846
+ # after a call to self-pace on credits/quota; on an error read err.meta
1847
+ # instead (last_meta stays on the last success, by design). Semantics
1848
+ # under a BaseAPI shared across threads: last-write-wins. The assignment
1849
+ # is atomic (GIL), but a reader may observe a concurrent request's meta —
1850
+ # for per-call metadata under sharing, use err.meta / the call's result
1851
+ # rather than this convenience attribute.
1852
+ self.last_meta: Optional[RequestMeta] = None
1853
+ # Retry knobs: ``max_retries`` is per-instance (a constructor kwarg);
1854
+ # the backoff ceiling / total budget / base are CLASS consts read via
1855
+ # ``self.`` so a subclass can override them. The dead per-instance
1856
+ # copies of the latter two are gone — nothing exposed a setter, and the
1857
+ # base const was already read off the class, so they were asymmetric
1858
+ # no-ops.
1859
+ self._max_retries = self._DEFAULT_MAX_RETRIES if max_retries is None else max(0, int(max_retries))
1860
+
1861
+ def __enter__(self) -> BaseAPI:
1862
+ return self
1863
+
1864
+ def __exit__(self, *exc: Any) -> None:
1865
+ self.close()
1866
+
1867
+ def close(self) -> None:
1868
+ self._client.close()
1869
+
1870
+ # --- HTTP -------------------------------------------------------------
1871
+
1872
+ def _serialize_param(self, v: Any) -> Any:
1873
+ """Normalize a Python value into something the wire can carry."""
1874
+ if v is None:
1875
+ return None
1876
+ # Coerce enum members to their wire codes (Enum imported module-top).
1877
+ if isinstance(v, Enum):
1878
+ # If the enum value is itself meaningful (str/int), use it; else fall back to .name
1879
+ inner = v.value
1880
+ return inner if isinstance(inner, (str, int, float, bool)) else v.name
1881
+ if isinstance(v, datetime):
1882
+ return v.isoformat()
1883
+ if isinstance(v, Decimal):
1884
+ return str(v)
1885
+ if isinstance(v, Resource):
1886
+ # Reference by key if available, else raise — won't serialize a whole instance
1887
+ key = type(v)._KEY_FIELD
1888
+ if key and hasattr(v, key):
1889
+ return getattr(v, key)
1890
+ raise TypeError(f"cannot serialize {type(v).__name__} without a keyed_by field")
1891
+ # Recurse into containers so a List[datetime] / List[Decimal] / List[Resource]
1892
+ # (or dict thereof) — shapes codegen itself emits for params — has its
1893
+ # elements normalized too, instead of hard-crashing at json.dumps(v).
1894
+ if isinstance(v, (list, tuple)):
1895
+ return [self._serialize_param(x) for x in v]
1896
+ if isinstance(v, dict):
1897
+ return {k: self._serialize_param(x) for k, x in v.items()}
1898
+ return v
1899
+
1900
+ def _retry_sleep(self, seconds: float) -> None:
1901
+ """Sleep before a retry. Isolated so tests can record (not wait)."""
1902
+ time.sleep(seconds)
1903
+
1904
+ def _retry_backoff(self, attempt: int, retry_after: Optional[int]) -> float:
1905
+ """Backoff seconds for ``attempt`` (1-based).
1906
+
1907
+ When the server gives an explicit ``retry_after``, HONOR it as a floor
1908
+ (sleep at least that long) plus a small jitter to de-synchronize retry
1909
+ storms — then cap to the per-retry ceiling (the server could return an
1910
+ absurd value like an hour; the ceiling guarantees we never sleep that
1911
+ long). With no server signal, use exponential backoff with full jitter.
1912
+ """
1913
+ if retry_after is not None and retry_after > 0:
1914
+ # Honor the server's hint as a floor (capped), + up to 1s jitter.
1915
+ base = min(float(retry_after), self._RETRY_MAX_BACKOFF)
1916
+ jitter = random.uniform(0, min(1.0, max(0.0, self._RETRY_MAX_BACKOFF - base)))
1917
+ return base + jitter
1918
+ # No server signal — exponential backoff with full jitter in [0, target].
1919
+ target = min(self._RETRY_BASE_BACKOFF * (2 ** (attempt - 1)), self._RETRY_MAX_BACKOFF)
1920
+ return random.uniform(0, target) if target > 0 else 0.0
1921
+
1922
+ def _request(
1923
+ self,
1924
+ method: str,
1925
+ endpoint_name: str,
1926
+ params: Dict[str, Any],
1927
+ *,
1928
+ timeout: Optional[float] = None,
1929
+ ) -> Any:
1930
+ """Hit ``/scraper/<SCRAPER_ID>/<endpoint_name>`` with the given params.
1931
+
1932
+ Returns the unwrapped payload (the contents of ``data`` when the
1933
+ scraper used the ``ScraperResult.success`` envelope, else the raw
1934
+ body). Raises a ``ParseError`` subclass for non-2xx responses.
1935
+
1936
+ ``timeout`` overrides the client-construction timeout for THIS request
1937
+ only (httpx supports a per-request timeout) — no new client needed.
1938
+
1939
+ On a 429 for a SAFE method (GET/HEAD), retries up to ``max_retries``
1940
+ times with exponential backoff + jitter, honoring the server's
1941
+ ``Retry-After`` header (capped to a ceiling + a total-elapsed budget).
1942
+ A POST 429 is NEVER retried — a retry re-runs a billable,
1943
+ non-idempotent scrape. When retries are exhausted the final
1944
+ ``RateLimitError`` surfaces unchanged so an agent still sees
1945
+ backpressure.
1946
+ """
1947
+ import httpx # lazy (see plan 003) — a dict lookup after first use
1948
+
1949
+ # Lazy-auth guard: a keyless client constructs fine (offline
1950
+ # introspection) but a real request needs a key. Raise BEFORE building
1951
+ # headers / touching transport so we never send ``X-API-Key: None``.
1952
+ # The key is RE-DERIVED through the host gate at this header site (not
1953
+ # only at __init__) so the sink stays correct-by-construction even if a
1954
+ # future refactor mutates ``_base_url`` after construction.
1955
+ trusted = _host_is_trusted(self._base_url)
1956
+ key = self._api_key if trusted else None
1957
+ token = self._access_token if trusted else None
1958
+ if not key and not token:
1959
+ if not trusted:
1960
+ raise AuthError(
1961
+ 0,
1962
+ f"Parse API requests to {self._base_url!r} cannot carry credentials — "
1963
+ f"it is not a trusted Parse API host. Point base_url at "
1964
+ f"https://{_CANONICAL_HOST} (the default) or a loopback URL for local dev.",
1965
+ )
1966
+ raise AuthError(0, "No Parse credentials found. Set PARSE_API_KEY, run "
1967
+ "`parse login` / `parse login --web`, or pass api_key=.")
1968
+ # Percent-encode both path segments (safe="") so a hostile SCRAPER_ID /
1969
+ # endpoint_name cannot traverse the path (`../../admin`) or smuggle
1970
+ # `?query` / `#fragment` — _request is public, so this holds for every
1971
+ # caller, not just codegen-gated names. A legit id/endpoint
1972
+ # ([A-Za-z0-9_-]) is untouched by quoting.
1973
+ sid = urllib.parse.quote(self.SCRAPER_ID, safe="")
1974
+ ep = urllib.parse.quote(endpoint_name, safe="")
1975
+ url = f"{self._base_url}/scraper/{sid}/{ep}"
1976
+ # X-API-Key when a key is present, else the web-login Bearer token (both
1977
+ # host-gated above). api_key takes priority when both somehow exist.
1978
+ headers = {"X-API-Key": key} if key else {"Authorization": f"Bearer {token}"}
1979
+ # Attribution: lets the backend track SDK-originated usage (vs raw REST/MCP)
1980
+ # and SDK version adoption. Purely informational — never gates the request.
1981
+ headers["X-Parse-Client"] = _client_header()
1982
+ # Version-pin a marketplace canonical: the executor reads this header and
1983
+ # serves the pinned RELEASE snapshot (a positive int — the only shape it
1984
+ # accepts). Only emitted for a pinned client (PINNED_VERSION set), so
1985
+ # owned/account clients are byte-for-byte unchanged.
1986
+ if self.PINNED_VERSION is not None:
1987
+ headers["API-Snapshot-Version"] = str(self.PINNED_VERSION)
1988
+ method_u = method.upper()
1989
+ cleaned = {k: self._serialize_param(v) for k, v in params.items() if v is not None}
1990
+ if method_u in ("GET", "HEAD", "DELETE"):
1991
+ # dict/list params are JSON-encoded into the query string (executor
1992
+ # contract; pinned by test_request_get_encodes_list_and_dict_as_json_query).
1993
+ wire_params: Dict[str, Any] = {}
1994
+ for k, v in cleaned.items():
1995
+ if isinstance(v, (dict, list)):
1996
+ wire_params[k] = json.dumps(v)
1997
+ elif isinstance(v, bool):
1998
+ wire_params[k] = str(v).lower()
1999
+ else:
2000
+ wire_params[k] = v
2001
+ req_kwargs: Dict[str, Any] = {"params": wire_params}
2002
+ else:
2003
+ req_kwargs = {"json": cleaned}
2004
+ if timeout is not None:
2005
+ req_kwargs["timeout"] = timeout
2006
+
2007
+ retryable = method_u in self._RETRY_SAFE_METHODS and self._max_retries > 0
2008
+ started = _monotonic()
2009
+ attempt = 0
2010
+ while True:
2011
+ try:
2012
+ r = self._client.request(method_u, url, headers=headers, **req_kwargs)
2013
+ except httpx.HTTPError as e:
2014
+ # The billing claim is scoped by failure phase: a connect-phase
2015
+ # error provably never reached the server; a read-phase error
2016
+ # (timeout, dropped response) may have executed — and billed —
2017
+ # the scrape, so an agent must not retry on a "not billed"
2018
+ # promise the message can't keep.
2019
+ billed = (
2020
+ "no request was billed"
2021
+ if isinstance(e, (httpx.ConnectError, httpx.ConnectTimeout))
2022
+ else "the request may have reached the server"
2023
+ )
2024
+ raise UpstreamError(
2025
+ 0,
2026
+ f"network error calling {endpoint_name!r} at {self._base_url}: {e} "
2027
+ f"— check connectivity/base_url; {billed}.",
2028
+ ) from e
2029
+ body = _safe_json(r)
2030
+ meta = RequestMeta.from_headers(r.headers)
2031
+ if r.status_code < 400:
2032
+ self.last_meta = meta
2033
+ return unwrap_scraper_envelope(body) # strip ScraperResult envelope
2034
+ err = _build_error(r.status_code, body, fallback=f"{endpoint_name} failed",
2035
+ error_classes=getattr(self, "_ERROR_CLASSES", None),
2036
+ headers=r.headers, params=params)
2037
+ err.meta = meta
2038
+ # Only a 429 on a safe method is retried; everything else surfaces.
2039
+ if not (retryable and r.status_code == 429 and attempt < self._max_retries):
2040
+ raise err
2041
+ attempt += 1
2042
+ wait = self._retry_backoff(attempt, getattr(err, "retry_after", None))
2043
+ # Total-elapsed cap on WALL CLOCK (request time included, not just
2044
+ # sleeps): if this wait would blow the budget, surface the
2045
+ # backpressure instead of sleeping past it. Read the class const
2046
+ # via ``self.`` so a subclass override flows through.
2047
+ if (_monotonic() - started) + wait > self._RETRY_TOTAL_BUDGET:
2048
+ raise err
2049
+ self._retry_sleep(wait)
2050
+
2051
+
2052
+ def unwrap_scraper_envelope(body: Any) -> Any:
2053
+ """Strip the ScraperResult envelope: a ``{"status": "success", "data": ...}``
2054
+ body returns ``data`` (so ``items_path`` / field access is relative to the
2055
+ unwrapped body); anything else passes through. SINGLE source of this
2056
+ predicate — the codegen sample-projection (``codegen_reconcile``) imports it
2057
+ so build-time projection unwraps EXACTLY as serving-time runtime does."""
2058
+ if isinstance(body, dict) and "data" in body and body.get("status") == "success":
2059
+ return body["data"]
2060
+ return body
2061
+
2062
+
2063
+ def _walk(d: Any, path: str, default: Any = None) -> Any:
2064
+ """Resolve a DOTTED response path (``'a.b.c'``) against a nested dict,
2065
+ returning ``default`` the moment a segment is absent or a non-dict is hit.
2066
+
2067
+ The runtime arm of the dotted-pagination-path fix: a paginated ``_fetch_page``
2068
+ reads cursor / has-more / total-pages / page metadata (and a nested
2069
+ ``items_path`` parent) off the page body, and a flat ``_resp.get('a.b')`` can
2070
+ never reach a value nested under ``a`` — it silently truncated every
2071
+ dotted-path spec to page 1. FLAT (dot-free) paths never reach here: codegen
2072
+ emits a verbatim ``_resp.get(path)`` for those, so this helper is imported
2073
+ ONLY by modules that actually declare a dotted pagination path (and existing
2074
+ generated modules never reference it — a purely additive runtime symbol)."""
2075
+ for seg in path.split("."):
2076
+ if not isinstance(d, dict) or seg not in d:
2077
+ return default
2078
+ d = d[seg]
2079
+ return d
2080
+
2081
+
2082
+ def _extract_items(resp: Any, items_path: str) -> List[Any]:
2083
+ """Pull the declared ``items_path`` array out of one paginated page's
2084
+ POST-unwrap response body (the caller has already run
2085
+ ``unwrap_scraper_envelope``). The SAMPLE-INDEPENDENT half of the
2086
+ empty-results discriminator — the emitted modules call it directly when the
2087
+ op captured no sample, and it is the runtime safety net every
2088
+ already-generated module inherits on the next import bump.
2089
+
2090
+ Disposition (each a strict safety improvement over the historical
2091
+ ``if items_path not in _resp: raise`` that crashed any body lacking the
2092
+ key):
2093
+
2094
+ * ``resp`` is a **list** — a top-level-list page: the value IS the items,
2095
+ return it (the historical guard ``AttributeError``ed / crashed here).
2096
+ * ``resp`` is ``None`` / empty / falsy (envelope-null, ``{}``) — a
2097
+ legitimate zero-result page → ``[]`` (never raise).
2098
+ * ``resp`` is a dict and ``items_path`` is **present + a list** — return
2099
+ it (unchanged behavior).
2100
+ * ``resp`` is a dict and ``items_path`` is **present but NOT a list**
2101
+ (``{jobs: {...}}`` / ``{jobs: 5}``) — the spec's ``items_path`` points at
2102
+ a non-array; today this is silent garbage / a downstream ``TypeError``,
2103
+ so fail loud with an enriched :class:`PaginationError` (mis-derived;
2104
+ sample-INDEPENDENT, unambiguous). This is the cross-compat-safe upgrade.
2105
+ * ``resp`` is a dict and ``items_path`` is **absent** — return ``[]``. No
2106
+ "list under another key → raise" heuristic lives here: an
2107
+ already-generated / no-sample module cannot disambiguate a faceted
2108
+ zero-result (``{results: [], facets: [...]}``) from a mis-derivation, so
2109
+ the runtime arm fails SAFE. The mis-derivation raise lives ONLY in the
2110
+ emitted discriminator, which has the captured sample to decide.
2111
+ """
2112
+ if isinstance(resp, list):
2113
+ return resp
2114
+ if not isinstance(resp, dict):
2115
+ # envelope-null / None / a non-dict non-list scalar body → zero results.
2116
+ return []
2117
+ if items_path not in resp:
2118
+ return []
2119
+ value = resp.get(items_path)
2120
+ if isinstance(value, list):
2121
+ return value
2122
+ if value is None:
2123
+ # Key present but null — a zero-result page that declared the key. Treat
2124
+ # as empty (the same disposition an absent key gets), never garbage.
2125
+ return []
2126
+ # Key present but a non-list, non-null value: the array the spec promised is
2127
+ # not there. Sample-independent → fail loud (mis-derived items_path).
2128
+ _list_keys = [k for k, v in resp.items() if isinstance(v, list)]
2129
+ raise PaginationError(
2130
+ 0,
2131
+ f"items_path {items_path!r} resolved to a {type(value).__name__}, not a "
2132
+ f"list; list-bearing keys present={_list_keys}; spec likely mis-derived; "
2133
+ f"regenerate",
2134
+ body=resp,
2135
+ )
2136
+
2137
+
2138
+ def _safe_json(r: httpx.Response) -> Any:
2139
+ try:
2140
+ return r.json()
2141
+ except Exception:
2142
+ return r.text