mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/harvest/datagov_v4.py

@@ -0,0 +1,798 @@
+"""Strict metadata-only parser for the official Data.gov Catalog API v4 search response."""
+
+from __future__ import annotations
+
+import json
+import re
+from dataclasses import dataclass
+from typing import Any
+from urllib.parse import parse_qs, urlencode, urlsplit
+
+from mostlyright.data_harness.acquisition.http import RetrievalLimits
+from mostlyright.data_harness.acquisition.url_policy import MAX_URL_LENGTH
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.harvest.protocol import (
+    DATAGOV_V4_HARVESTER_ID,
+    DATAGOV_V4_HARVESTER_VERSION,
+    CatalogHarvestError,
+    HarvestCursor,
+    HarvesterDescriptor,
+    HarvestPage,
+    HarvestResponseEvidence,
+    validate_datagov_cursor_text,
+)
+from mostlyright.data_harness.sources.contracts import EvidenceReference, _CanonicalContract
+
+DATAGOV_V4_ENDPOINT = "https://api.gsa.gov/technology/datagov/v4/search"
+DATAGOV_V4_RECORD_VERSION = "harness-datagov-v4-normalized-record.v2"
+DATAGOV_V4_MAX_RESPONSE_BYTES = 16 * 1024 * 1024
+DATAGOV_V4_MAX_RECORD_BYTES = 2 * 1024 * 1024
+DATAGOV_V4_MAX_IDENTIFIER_CHARS = 512
+DATAGOV_V4_MAX_CURSOR = 2_048
+DATAGOV_V4_MAX_JSON_DEPTH = 256
+DATAGOV_V4_MAX_NUMBER_TOKEN = 128
+DATAGOV_V4_LIMITS = RetrievalLimits(
+    max_response_bytes=DATAGOV_V4_MAX_RESPONSE_BYTES,
+    max_aggregate_response_bytes=DATAGOV_V4_MAX_RESPONSE_BYTES,
+    max_redirects=0,
+    max_requests=1,
+    allowed_media_types=("application/json",),
+)
+
+_TOP_LEVEL_FIELDS = frozenset(
+    {
+        "dcat",
+        "description",
+        "distribution_titles",
+        "harvest_record",
+        "harvest_record_raw",
+        "harvest_record_transformed",
+        "has_download",
+        "has_spatial",
+        "identifier",
+        "keyword",
+        "last_harvested_date",
+        "organization",
+        "popularity",
+        "publisher",
+        "slug",
+        "spatial_centroid",
+        "spatial_shape",
+        "theme",
+        "title",
+    }
+)
+# Catalog-ingestion observations the provider rewrites on every re-harvest even when no metadata
+# changed. They stay in raw and normalized evidence but never enter the semantic digest, because a
+# digest that moved with a nightly harvest burst would make two adjacent equal pass maps
+# unsatisfiable. `_score`/`_sort` are per-response ranking values and are not allowlisted at all.
+DATAGOV_V4_VOLATILE_FIELDS = frozenset({"last_harvested_date", "popularity"})
+_VOLATILE_PATHS = frozenset(
+    path for name in DATAGOV_V4_VOLATILE_FIELDS for path in (f"$.{name}", f"$.{name}[*]")
+)
+_DCAT_FIELDS = frozenset(
+    {
+        "@type",
+        "accessLevel",
+        "accrualPeriodicity",
+        "agencyDataSeriesURL",
+        "agencyProgramURL",
+        "analysisUnit",
+        "bureauCode",
+        "categoryDesignation",
+        "collectionInstrument",
+        "contactPoint",
+        "dataQuality",
+        "describedBy",
+        "describedByType",
+        "description",
+        "distribution",
+        "identifier",
+        "isPartOf",
+        "issued",
+        "keyword",
+        "landingPage",
+        "language",
+        "license",
+        "modified",
+        "phone",
+        "programCode",
+        "publisher",
+        "references",
+        "rights",
+        "spatial",
+        "temporal",
+        "theme",
+        "title",
+    }
+)
+_ORGANIZATION_FIELDS = frozenset(
+    {"id", "name", "slug", "organization_type", "aliases", "logo", "description"}
+)
+_DISTRIBUTION_FIELDS = frozenset(
+    {
+        "@type",
+        "accessURL",
+        "describedBy",
+        "describedByType",
+        "description",
+        "downloadURL",
+        "format",
+        "mediaType",
+        "title",
+    }
+)
+_PROVIDER_PATH = re.compile(r'^\$(?:\.[A-Za-z_][A-Za-z0-9_-]*|\[\*\]|\["@type"\])+$')
+_PLAIN_PROVIDER_KEY = re.compile(r"^[A-Za-z_][A-Za-z0-9_-]*$")
+_JSON_NUMBER = re.compile(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?")
+
+
+class _ForeignJsonError(ValueError):
+    pass
+
+
+@dataclass(frozen=True)
+class _ForeignNumber:
+    lexeme: str
+
+
+@dataclass(frozen=True)
+class DatagovProjectedField(_CanonicalContract):
+    path: str
+    value: Any
+    encoding: str = "canonical_json"
+
+    def __post_init__(self) -> None:
+        if (
+            not isinstance(self.path, str)
+            or not _PROVIDER_PATH.fullmatch(self.path)
+            or not _allowed_provider_path(self.path, self.value, self.encoding)
+        ):
+            raise CatalogHarvestError("DATAGOV_PATH", "record.fields.path", "path is invalid")
+        if self.encoding not in {"canonical_json", "typed_json_ast.v1"}:
+            raise CatalogHarvestError("DATAGOV_VALUE", "record.fields.encoding", "is invalid")
+        try:
+            canonical_json_bytes(self.value)
+        except Exception as error:
+            raise CatalogHarvestError(
+                "DATAGOV_VALUE", "record.fields.value", "value is not strict JSON"
+            ) from error
+        if self.encoding == "typed_json_ast.v1":
+            _validate_typed_json_ast(self.value)
+
+    def to_dict(self) -> dict[str, Any]:
+        result = {"path": self.path, "value": self.value}
+        if self.encoding != "canonical_json":
+            result["encoding"] = self.encoding
+        return result
+
+
+@dataclass(frozen=True)
+class DatagovV4Record(_CanonicalContract):
+    record_id: str
+    fields: tuple[DatagovProjectedField, ...]
+    raw_response_sha256: str
+    observations: tuple[DatagovProjectedField, ...] = ()
+    protocol: str = "datagov_v4"
+    schema_version: str = DATAGOV_V4_RECORD_VERSION
+
+    def __post_init__(self) -> None:
+        if self.protocol != "datagov_v4" or self.schema_version != DATAGOV_V4_RECORD_VERSION:
+            raise CatalogHarvestError("VERSION", "record", "unsupported v4 record")
+        if not isinstance(self.record_id, str) or not self.record_id:
+            raise CatalogHarvestError(
+                "DATAGOV_IDENTIFIER", "record.record_id", "identifier must be nonempty"
+            )
+        try:
+            self.record_id.encode("utf-8", errors="strict")
+        except UnicodeEncodeError:
+            raise CatalogHarvestError(
+                "DATAGOV_IDENTIFIER", "record.record_id", "identifier must be valid Unicode"
+            ) from None
+        if len(self.record_id) > DATAGOV_V4_MAX_IDENTIFIER_CHARS:
+            raise CatalogHarvestError(
+                "DATAGOV_IDENTIFIER",
+                "record.record_id",
+                "identifier exceeds the shared provider identity bound",
+            )
+        for character in self.record_id:
+            codepoint = ord(character)
+            if (
+                codepoint < 0x20
+                or 0x7F <= codepoint <= 0x9F
+                or (character.isspace() and character != " ")
+            ):
+                raise CatalogHarvestError(
+                    "DATAGOV_IDENTIFIER",
+                    "record.record_id",
+                    "identifier contains unsafe provider identity text",
+                )
+        if not isinstance(self.fields, tuple) or not self.fields:
+            raise CatalogHarvestError("DATAGOV_FIELDS", "record.fields", "fields are required")
+        if any(not isinstance(field, DatagovProjectedField) for field in self.fields):
+            raise CatalogHarvestError(
+                "DATAGOV_FIELDS", "record.fields", "fields must use the strict contract"
+            )
+        paths = tuple(field.path for field in self.fields)
+        if paths != tuple(sorted(paths)) or len(paths) != len(set(paths)):
+            raise CatalogHarvestError(
+                "DATAGOV_FIELDS", "record.fields", "field paths must be sorted and unique"
+            )
+        if any(path in _VOLATILE_PATHS for path in paths):
+            raise CatalogHarvestError(
+                "DATAGOV_FIELDS",
+                "record.fields",
+                "volatile observation fields may not carry a semantic field path",
+            )
+        if not isinstance(self.observations, tuple) or any(
+            not isinstance(field, DatagovProjectedField) for field in self.observations
+        ):
+            raise CatalogHarvestError(
+                "DATAGOV_OBSERVATIONS",
+                "record.observations",
+                "observations must use the strict contract",
+            )
+        observed_paths = tuple(field.path for field in self.observations)
+        if (
+            observed_paths != tuple(sorted(observed_paths))
+            or len(observed_paths) != len(set(observed_paths))
+            or any(path not in _VOLATILE_PATHS for path in observed_paths)
+        ):
+            raise CatalogHarvestError(
+                "DATAGOV_OBSERVATIONS",
+                "record.observations",
+                "observation paths must be sorted, unique, and volatile",
+            )
+        identifier_fields = tuple(field for field in self.fields if field.path == "$.identifier")
+        if (
+            len(identifier_fields) != 1
+            or identifier_fields[0].encoding != "canonical_json"
+            or identifier_fields[0].value != self.record_id
+        ):
+            raise CatalogHarvestError(
+                "DATAGOV_IDENTIFIER",
+                "record.fields",
+                "exactly one canonical $.identifier must equal record_id",
+            )
+        if len(self.raw_response_sha256) != 64 or any(
+            character not in "0123456789abcdef" for character in self.raw_response_sha256
+        ):
+            raise CatalogHarvestError("DIGEST", "record.raw_response_sha256", "must be SHA-256")
+        try:
+            semantic = self._semantic_dict()
+            observed = self._observed_dict()
+            normalized_size = len(canonical_json_bytes(observed))
+            normalized_record = dict(observed)
+            normalized_record["raw_response_sha256"] = self.raw_response_sha256
+            normalized_record["normalized_sha256"] = canonical_sha256(semantic)
+            canonical_json_bytes(
+                {
+                    "schema_version": "harness-catalog-fill-shard.v1",
+                    "kind": "normalized",
+                    "payload": {
+                        "schema_version": "harness-datagov-v4-normalized-record-segment.v1",
+                        "raw_response_sha256": self.raw_response_sha256,
+                        "first_record": 0,
+                        "records": [normalized_record],
+                    },
+                }
+            )
+        except CanonicalJSONError:
+            raise CatalogHarvestError(
+                "DATAGOV_RECORD_LIMIT",
+                "record",
+                "normalized record exceeds its canonical member contract",
+            ) from None
+        if normalized_size > DATAGOV_V4_MAX_RECORD_BYTES:
+            raise CatalogHarvestError(
+                "DATAGOV_RECORD_LIMIT", "record", "normalized record exceeds its byte bound"
+            )
+
+    def _semantic_dict(self) -> dict[str, Any]:
+        """Exactly the normalized allowlisted metadata fields the pass map converges on."""
+
+        return {
+            "schema_version": self.schema_version,
+            "protocol": self.protocol,
+            "record_id": self.record_id,
+            "fields": [field.to_dict() for field in self.fields],
+        }
+
+    def _observed_dict(self) -> dict[str, Any]:
+        """The semantic record plus retained, non-semantic volatile observation values."""
+
+        result = self._semantic_dict()
+        if self.observations:
+            result["observations"] = [field.to_dict() for field in self.observations]
+        return result
+
+    @property
+    def normalized_sha256(self) -> str:
+        return canonical_sha256(self._semantic_dict())
+
+    @property
+    def digest(self) -> str:
+        return self.normalized_sha256
+
+    def to_dict(self) -> dict[str, Any]:
+        result = self._observed_dict()
+        result["raw_response_sha256"] = self.raw_response_sha256
+        result["normalized_sha256"] = self.normalized_sha256
+        return result
+
+
+class DatagovV4Harvester:
+    descriptor = HarvesterDescriptor(
+        protocol="datagov_v4",
+        harvester_id=DATAGOV_V4_HARVESTER_ID,
+        harvester_version=DATAGOV_V4_HARVESTER_VERSION,
+    )
+
+    def parse(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+    ) -> tuple[DatagovV4Record, ...]:
+        return self.parse_page(
+            payload,
+            uri=uri,
+            observed_at=observed_at,
+            evidence=evidence,
+            response_evidence=None,
+            cursor=None,
+        ).records
+
+    def parse_page(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+        response_evidence: HarvestResponseEvidence | None,
+        cursor: HarvestCursor | None,
+    ) -> HarvestPage:
+        if not isinstance(payload, bytes) or len(payload) > DATAGOV_V4_MAX_RESPONSE_BYTES:
+            raise CatalogHarvestError(
+                "DATAGOV_RESPONSE_LIMIT", "harvest.response", "v4 response exceeds 16 MiB"
+            )
+        if cursor is not None and cursor.protocol != "datagov_v4":
+            raise CatalogHarvestError("HARVEST_CURSOR", "cursor", "protocol mismatch")
+        try:
+            _preflight_json(payload)
+            document = json.loads(
+                payload.decode("utf-8", errors="strict"),
+                object_pairs_hook=_unique_object,
+                parse_float=_bounded_foreign_number,
+                parse_int=_bounded_foreign_integer,
+                parse_constant=_reject_nonfinite,
+            )
+        except (
+            UnicodeDecodeError,
+            json.JSONDecodeError,
+            _ForeignJsonError,
+            RecursionError,
+            ValueError,
+            OverflowError,
+        ):
+            raise CatalogHarvestError(
+                "DATAGOV_JSON", "harvest.response", "v4 response must be strict UTF-8 JSON"
+            ) from None
+        if not isinstance(document, dict) or set(document) - {"results", "sort", "after"}:
+            raise CatalogHarvestError(
+                "DATAGOV_ENVELOPE", "harvest.response", "v4 response envelope is invalid"
+            )
+        if document.get("sort") != _requested_sort(uri) or not isinstance(
+            document.get("results"), list
+        ):
+            raise CatalogHarvestError(
+                "DATAGOV_ENVELOPE", "harvest.response", "v4 sort/results are invalid"
+            )
+        requested_page_size = _requested_page_size(uri)
+        if len(document["results"]) > requested_page_size:
+            raise CatalogHarvestError(
+                "DATAGOV_PAGE_SIZE",
+                "harvest.response.results",
+                "v4 response contains more results than the exact request admitted",
+            )
+        after = document.get("after")
+        if after is not None and (
+            not isinstance(after, str) or not after or len(after) > DATAGOV_V4_MAX_CURSOR
+        ):
+            raise CatalogHarvestError(
+                "HARVEST_CURSOR", "harvest.response.after", "v4 cursor is invalid"
+            )
+        if after is not None:
+            _require_safe_cursor(after)
+        raw_sha256 = sha256_bytes(payload)
+        records: list[DatagovV4Record] = []
+        skipped: list[str] = []
+        for index, item in enumerate(document["results"]):
+            if not isinstance(item, dict):
+                skipped.append(f"index:{index}")
+                continue
+            identifier = item.get("identifier")
+            if not isinstance(identifier, str) or not identifier:
+                skipped.append(f"index:{index}")
+                continue
+            try:
+                fields, observations = _project_record(item)
+                records.append(
+                    DatagovV4Record(
+                        record_id=identifier,
+                        fields=fields,
+                        observations=observations,
+                        raw_response_sha256=raw_sha256,
+                    )
+                )
+            except CatalogHarvestError:
+                skipped.append(f"sha256:{sha256_bytes(identifier.encode('utf-8', 'replace'))}")
+        next_cursor = (
+            None
+            if after is None
+            else HarvestCursor(protocol="datagov_v4", kind="after", value=after)
+        )
+        return HarvestPage(
+            protocol="datagov_v4",
+            records=tuple(records),
+            next_cursor=next_cursor,
+            provider_count=None,
+            count_basis="not_reported",
+            skipped_record_ids=tuple(sorted(skipped)),
+            evidence=evidence,
+            response_evidence=response_evidence,
+        )
+
+
+def _project_record(
+    item: dict[str, Any],
+) -> tuple[tuple[DatagovProjectedField, ...], tuple[DatagovProjectedField, ...]]:
+    """Project the allowlisted union, then split semantic fields from volatile observations."""
+
+    projected: list[DatagovProjectedField] = []
+    base = "$"
+    for name in sorted(set(item) & _TOP_LEVEL_FIELDS):
+        value = item[name]
+        path = f"{base}.{name}"
+        if name == "dcat":
+            if not isinstance(value, dict):
+                raise CatalogHarvestError("DATAGOV_VALUE", path, "dcat must be a closed object")
+            for child in sorted(set(value) & _DCAT_FIELDS):
+                _flatten_allowed(
+                    projected,
+                    _provider_child_path(path, child),
+                    value[child],
+                    child_allowlist=_DISTRIBUTION_FIELDS if child == "distribution" else None,
+                    allow_structured_value=child != "distribution",
+                )
+        elif name == "organization":
+            if not isinstance(value, dict):
+                raise CatalogHarvestError(
+                    "DATAGOV_VALUE", path, "organization must be a closed object"
+                )
+            for child in sorted(set(value) & _ORGANIZATION_FIELDS):
+                _flatten_allowed(
+                    projected,
+                    _provider_child_path(path, child),
+                    value[child],
+                    allow_structured_value=True,
+                )
+        else:
+            _flatten_allowed(projected, path, value, allow_structured_value=True)
+    ordered = sorted(projected, key=lambda field: field.path)
+    return (
+        tuple(field for field in ordered if field.path not in _VOLATILE_PATHS),
+        tuple(field for field in ordered if field.path in _VOLATILE_PATHS),
+    )
+
+
+def _flatten_allowed(
+    target: list[DatagovProjectedField],
+    path: str,
+    value: Any,
+    *,
+    child_allowlist: frozenset[str] | None = None,
+    allow_structured_value: bool = False,
+) -> None:
+    if isinstance(value, dict):
+        if child_allowlist is None:
+            if not allow_structured_value:
+                raise CatalogHarvestError(
+                    "DATAGOV_VALUE", path, "nested objects require a closed field schema"
+                )
+            target.append(_projected_field(path, value))
+            return
+        keys = set(value) & child_allowlist
+        if not keys:
+            target.append(DatagovProjectedField(path, {}))
+            return
+        for key in sorted(keys):
+            _flatten_allowed(target, _provider_child_path(path, key), value[key])
+        return
+    if isinstance(value, list):
+        if not value:
+            target.append(DatagovProjectedField(f"{path}[*]", []))
+            return
+        if child_allowlist is not None:
+            if not all(isinstance(item, dict) for item in value):
+                raise CatalogHarvestError(
+                    "DATAGOV_VALUE", path, "closed object arrays may not mix member types"
+                )
+            keys = set().union(*(set(item) for item in value)) & child_allowlist
+            for key in sorted(keys):
+                values = [item[key] for item in value if key in item]
+                stable_value: Any = values[0] if len(values) == 1 else values
+                target.append(
+                    _projected_field(_provider_child_path(f"{path}[*]", key), stable_value)
+                )
+            return
+        if not allow_structured_value and any(isinstance(item, (dict, list)) for item in value):
+            raise CatalogHarvestError(
+                "DATAGOV_VALUE", path, "nested arrays or objects require a closed field schema"
+            )
+        target.append(_projected_field(f"{path}[*]", value))
+        return
+    target.append(_projected_field(path, value))
+
+
+def _unique_object(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+    result: dict[str, Any] = {}
+    for key, value in pairs:
+        if key in result:
+            raise _ForeignJsonError("duplicate object key")
+        result[key] = value
+    return result
+
+
+def _provider_child_path(path: str, key: str) -> str:
+    if key == "@type":
+        return f'{path}["@type"]'
+    if not _PLAIN_PROVIDER_KEY.fullmatch(key):
+        raise _ForeignJsonError("allowlisted provider key has no canonical path spelling")
+    return f"{path}.{key}"
+
+
+def _allowed_provider_path(path: str, value: Any, encoding: str) -> bool:
+    if path == "$.dcat.distribution":
+        return encoding == "canonical_json" and value == {}
+    if path == "$.dcat.distribution[*]":
+        return encoding == "canonical_json" and value == []
+    for name in _TOP_LEVEL_FIELDS - {"dcat", "organization"}:
+        if path in {f"$.{name}", f"$.{name}[*]"}:
+            return True
+    for name in _ORGANIZATION_FIELDS:
+        if path in {f"$.organization.{name}", f"$.organization.{name}[*]"}:
+            return True
+    for name in _DCAT_FIELDS - {"distribution"}:
+        base = _provider_child_path("$.dcat", name)
+        if path in {base, f"{base}[*]"}:
+            return True
+    for name in _DISTRIBUTION_FIELDS:
+        if path == _provider_child_path("$.dcat.distribution[*]", name):
+            return True
+    return False
+
+
+def _validate_typed_json_ast(value: Any) -> None:
+    pending: list[tuple[Any, int]] = [(value, 1)]
+    visited = 0
+    while pending:
+        current, depth = pending.pop()
+        visited += 1
+        if visited > 1_000_000 or depth > DATAGOV_V4_MAX_JSON_DEPTH:
+            raise CatalogHarvestError(
+                "DATAGOV_VALUE", "record.fields.value", "typed JSON AST exceeds its bound"
+            )
+        if not isinstance(current, dict) or not isinstance(current.get("type"), str):
+            raise CatalogHarvestError(
+                "DATAGOV_VALUE", "record.fields.value", "typed JSON AST node is invalid"
+            )
+        kind = current["type"]
+        if kind == "null":
+            valid = set(current) == {"type"}
+        elif kind == "boolean":
+            valid = set(current) == {"type", "value"} and type(current["value"]) is bool
+        elif kind == "integer":
+            valid = set(current) == {"type", "value"} and type(current["value"]) is int
+        elif kind == "number":
+            lexeme = current.get("lexeme")
+            valid = (
+                set(current) == {"type", "lexeme"}
+                and isinstance(lexeme, str)
+                and len(lexeme) <= DATAGOV_V4_MAX_NUMBER_TOKEN
+                and _JSON_NUMBER.fullmatch(lexeme) is not None
+            )
+        elif kind == "string":
+            valid = set(current) == {"type", "value"} and isinstance(current["value"], str)
+        elif kind == "array":
+            items = current.get("items")
+            valid = set(current) == {"type", "items"} and isinstance(items, list)
+            if valid:
+                pending.extend((item, depth + 1) for item in reversed(items))
+        elif kind == "object":
+            entries = current.get("entries")
+            valid = set(current) == {"type", "entries"} and isinstance(entries, list)
+            if valid:
+                keys: list[str] = []
+                for entry in entries:
+                    if (
+                        not isinstance(entry, dict)
+                        or set(entry) != {"key", "value"}
+                        or not isinstance(entry["key"], str)
+                    ):
+                        valid = False
+                        break
+                    keys.append(entry["key"])
+                    pending.append((entry["value"], depth + 1))
+                valid = valid and keys == sorted(set(keys))
+        else:
+            valid = False
+        if not valid:
+            raise CatalogHarvestError(
+                "DATAGOV_VALUE", "record.fields.value", "typed JSON AST node is invalid"
+            )
+
+
+def _bounded_foreign_integer(value: str) -> int | _ForeignNumber:
+    if len(value) > DATAGOV_V4_MAX_NUMBER_TOKEN:
+        raise _ForeignJsonError("numeric token exceeds bound")
+    parsed = int(value)
+    return parsed if -(1 << 63) <= parsed <= (1 << 63) - 1 else _ForeignNumber(value)
+
+
+def _reject_nonfinite(_value: str) -> None:
+    raise _ForeignJsonError("non-finite number")
+
+
+def _bounded_foreign_number(value: str) -> _ForeignNumber:
+    if len(value) > DATAGOV_V4_MAX_NUMBER_TOKEN:
+        raise _ForeignJsonError("numeric token exceeds bound")
+    return _ForeignNumber(value)
+
+
+def _projected_field(path: str, value: Any) -> DatagovProjectedField:
+    try:
+        if not _contains_foreign_number(value):
+            return DatagovProjectedField(path, value)
+        return DatagovProjectedField(path, _typed_json_ast(value), encoding="typed_json_ast.v1")
+    except (_ForeignJsonError, RecursionError):
+        raise CatalogHarvestError(
+            "DATAGOV_VALUE", path, "provider value exceeds the bounded traversal contract"
+        ) from None
+
+
+def _contains_foreign_number(value: Any) -> bool:
+    pending = [value]
+    visited = 0
+    while pending:
+        current = pending.pop()
+        visited += 1
+        if visited > 1_000_000:
+            raise _ForeignJsonError("foreign JSON value exceeds traversal bound")
+        if isinstance(current, _ForeignNumber):
+            return True
+        if isinstance(current, list):
+            pending.extend(current)
+        elif isinstance(current, dict):
+            pending.extend(current.values())
+    return False
+
+
+def _typed_json_ast(value: Any) -> dict[str, Any]:
+    if value is None:
+        return {"type": "null"}
+    if type(value) is bool:
+        return {"type": "boolean", "value": value}
+    if type(value) is int:
+        return {"type": "integer", "value": value}
+    if isinstance(value, _ForeignNumber):
+        return {"type": "number", "lexeme": value.lexeme}
+    if isinstance(value, str):
+        return {"type": "string", "value": value}
+    if isinstance(value, list):
+        return {"type": "array", "items": [_typed_json_ast(item) for item in value]}
+    if isinstance(value, dict):
+        return {
+            "type": "object",
+            "entries": [
+                {"key": key, "value": _typed_json_ast(value[key])} for key in sorted(value)
+            ],
+        }
+    raise _ForeignJsonError("foreign JSON value type is invalid")
+
+
+def _preflight_json(payload: bytes) -> None:
+    """Bound nesting and numeric lexemes before CPython's decoder touches them."""
+
+    depth = 0
+    in_string = False
+    escaped = False
+    index = 0
+    while index < len(payload):
+        byte = payload[index]
+        if in_string:
+            if escaped:
+                escaped = False
+            elif byte == 0x5C:
+                escaped = True
+            elif byte == 0x22:
+                in_string = False
+            index += 1
+            continue
+        if byte == 0x22:
+            in_string = True
+        elif byte in {0x5B, 0x7B}:
+            depth += 1
+            if depth > DATAGOV_V4_MAX_JSON_DEPTH:
+                raise _ForeignJsonError("JSON nesting exceeds bound")
+        elif byte in {0x5D, 0x7D}:
+            depth -= 1
+            if depth < 0:
+                raise _ForeignJsonError("JSON nesting is invalid")
+        elif byte == 0x2D or 0x30 <= byte <= 0x39:
+            end = index + 1
+            while end < len(payload) and payload[end] in b"0123456789eE+.-":
+                end += 1
+            if end - index > DATAGOV_V4_MAX_NUMBER_TOKEN:
+                raise _ForeignJsonError("numeric token exceeds bound")
+            index = end
+            continue
+        index += 1
+
+
+#: The sorts the v4 cursor contract admits, longest first so the URL bound stays the strictest.
+DATAGOV_V4_SORT_ORDERS = ("last_harvested_date", "relevance")
+
+
+def _requested_sort(uri: str) -> str:
+    query = parse_qs(urlsplit(uri).query, keep_blank_values=True)
+    values = query.get("sort")
+    if values is None:
+        return DATAGOV_V4_SORT_ORDERS[0]
+    if len(values) != 1 or values[0] not in DATAGOV_V4_SORT_ORDERS:
+        raise CatalogHarvestError(
+            "DATAGOV_SORT", "harvest.request.sort", "sort is not one this contract can page"
+        )
+    return values[0]
+
+
+def _requested_page_size(uri: str) -> int:
+    query = parse_qs(urlsplit(uri).query, keep_blank_values=True)
+    values = query.get("per_page")
+    if values is None:
+        return 1_000
+    if len(values) != 1 or not re.fullmatch(r"[1-9][0-9]{0,3}", values[0]):
+        raise CatalogHarvestError(
+            "DATAGOV_PAGE_SIZE", "harvest.request.per_page", "per_page is invalid"
+        )
+    page_size = int(values[0])
+    if page_size > 1_000:
+        raise CatalogHarvestError(
+            "DATAGOV_PAGE_SIZE", "harvest.request.per_page", "per_page exceeds 1000"
+        )
+    return page_size
+
+
+def _require_safe_cursor(value: str) -> None:
+    validate_datagov_cursor_text(value, "harvest.response.after")
+    continuation = f"{DATAGOV_V4_ENDPOINT}?" + urlencode(
+        (
+            ("sort", max(DATAGOV_V4_SORT_ORDERS, key=len)),
+            ("per_page", "1000"),
+            ("after", value),
+        )
+    )
+    if len(continuation) > MAX_URL_LENGTH:
+        raise CatalogHarvestError(
+            "HARVEST_CURSOR",
+            "harvest.response.after",
+            "Data.gov cursor exceeds the canonical continuation URL boundary",
+        )