mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/harvest/protocol.py

@@ -0,0 +1,964 @@
+"""The shared harvest boundary: fetch bounded bytes, admit them, then parse them.
+
+Everything a harvester touches is attacker-influenceable. A public catalogue controls its own
+status line, headers, size and body, and one of the instances this was tested against answers
+HTTP 200 with a bot-wall HTML page. So the discipline here is uniform and not negotiable per
+harvester:
+
+* **Every fetch goes through the hardened retriever.** There is no ``urllib`` or ``requests`` call
+  anywhere in this package, which is what keeps DNS pinning, the egress allowlist and the per-hop
+  public-address check on the path.
+* **Bytes are admitted before they are parsed.** ``admit_public_fact_bytes`` runs on every payload,
+  because this is the moment a stranger's document would become a durable shared fact.
+* **Parsing takes bytes, never a URL.** That is what lets every harvester test run offline against
+  a recorded response, and it puts the untrusted-input boundary in one obvious place.
+* **Caps sit at or below the admission cap**, so an oversized response is refused at the transport
+  rather than downloaded and then rejected. The observed Eurostat structure response is 37,166,239
+  bytes; that is the concrete reason this is enforced rather than advised.
+
+A ``HarvestedRecord`` is deliberately weaker than a ``CatalogEntry``. It holds what a harvester can
+honestly read off a response and nothing more -- no rights adjudication, no liveness observation.
+Turning one into an entry is a separate, explicit step that runs the public-source checks.
+"""
+
+from __future__ import annotations
+
+import re
+from collections.abc import Callable
+from dataclasses import dataclass
+from typing import Any, Protocol
+from urllib.parse import urlsplit
+
+from mostlyright.data_harness.acquisition.http import (
+    DatagovRateObservation,
+    DatagovResponseRefused,
+    DatagovV4Authorization,
+    PinnedHttpsRetriever,
+    RetrievalLimits,
+    RetrievedBytes,
+)
+from mostlyright.data_harness.acquisition.url_policy import AcquisitionSecurityError
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.sources.catalog.admission import (
+    MAX_CATALOG_FACT_BYTES,
+    admit_public_fact_bytes,
+)
+from mostlyright.data_harness.sources.contracts import (
+    DATA_FORMATS,
+    EvidenceReference,
+    SourceContractError,
+    _CanonicalContract,
+)
+
+HARVESTED_RECORD_CONTRACT_VERSION = "harness-source-harvested-record.v1"
+HARVEST_PAGE_CONTRACT_VERSION = "harness-source-harvest-page.v1"
+HARVEST_RESPONSE_EVIDENCE_VERSION = "harness-source-harvest-response-evidence.v1"
+
+HARVEST_PROTOCOLS = ("ckan", "datagov_v4", "stac", "sdmx")
+DATAGOV_V4_HARVESTER_ID = "datagov_v4"
+DATAGOV_V4_HARVESTER_VERSION = "1.0.0"
+DATAGOV_V4_HARVESTER_COORDINATE = f"{DATAGOV_V4_HARVESTER_ID}@{DATAGOV_V4_HARVESTER_VERSION}"
+
+MAX_RECORD_ID = 200
+MAX_RECORD_TEXT = 4_000
+MAX_RECORD_TITLE = 500
+MAX_RECORD_PUBLISHER = 500
+MAX_RECORD_LICENSE_ID = 200
+MAX_RECORD_URI = 2_048
+MAX_RECORD_FORMATS = 16
+
+# Named verbatim so a 401 reads as a scope boundary rather than an invitation to add credentials.
+AUTHENTICATED_ADAPTER_PARKED = (
+    "this endpoint is not open; authenticated source adapters are parked, and their trigger is the "
+    "first real credentialed need plus a broker trust review"
+)
+
+_STATUS_TAIL = re.compile(r"status (\d{3})$")
+# Publisher id characters, deliberately wider than the harness identifier grammar and still
+# closed. The parentheses are there because an SDMX dataflow's natural coordinate is
+# `ESTAT:APRO_CPSH1(1.0)`; no whitespace, quote, angle bracket or control character is admitted.
+_RECORD_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:@/+()-]*$")
+_TIMESTAMP = re.compile(
+    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z$"
+)
+
+
+class CatalogHarvestError(SourceContractError):
+    """A harvest refusal: the endpoint is not open, not what it claimed, or not bounded."""
+
+
+class DatagovFetchRefused(CatalogHarvestError):
+    """A v4 refusal after an HTTP response, carrying only transport accounting."""
+
+    def __init__(
+        self,
+        code: str,
+        path: str,
+        detail: str,
+        *,
+        response_evidence: DatagovRefusedResponseEvidence,
+    ) -> None:
+        self.response_evidence = response_evidence
+        super().__init__(code, path, detail)
+
+
+@dataclass(frozen=True)
+class DatagovRefusedResponseEvidence:
+    """Secret-free accounting for a response whose bytes may not contribute any digest."""
+
+    final_url: str
+    status: int
+    network_bytes: int
+    rate_observation: DatagovRateObservation | None
+    requests: int = 1
+    responses: int = 1
+    content_bytes: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "final_url": self.final_url,
+            "status": self.status,
+            "network_bytes": self.network_bytes,
+            "requests": self.requests,
+            "responses": self.responses,
+            "content_bytes": self.content_bytes,
+            "rate_observation": (
+                self.rate_observation.to_dict() if self.rate_observation is not None else None
+            ),
+        }
+
+
+@dataclass(frozen=True)
+class HarvesterDescriptor:
+    """The immutable identity of one harvester."""
+
+    protocol: str
+    harvester_id: str
+    harvester_version: str
+
+    def __post_init__(self) -> None:
+        if self.protocol not in HARVEST_PROTOCOLS:
+            raise CatalogHarvestError(
+                "ENUM",
+                "harvester.protocol",
+                f"must be one of {list(HARVEST_PROTOCOLS)}",
+            )
+        _identifier(self.harvester_id, "harvester.harvester_id")
+        if not re.fullmatch(r"[1-9][0-9]*\.[0-9]+\.[0-9]+", str(self.harvester_version)):
+            raise CatalogHarvestError(
+                "SEMVER",
+                "harvester.harvester_version",
+                "must be a positive-major semantic version",
+            )
+
+    @property
+    def coordinate(self) -> str:
+        return f"{self.harvester_id}@{self.harvester_version}"
+
+
+@dataclass(frozen=True)
+class HarvestCursor(_CanonicalContract):
+    """A closed protocol continuation coordinate, never an executable request."""
+
+    protocol: str
+    kind: str
+    value: str
+
+    def __post_init__(self) -> None:
+        if self.protocol not in HARVEST_PROTOCOLS:
+            raise CatalogHarvestError("ENUM", "cursor.protocol", "must name a harvest protocol")
+        expected = {
+            "ckan": "offset",
+            "datagov_v4": "after",
+            "stac": "next",
+            "sdmx": "terminal",
+        }[self.protocol]
+        if self.kind != expected:
+            raise CatalogHarvestError(
+                "HARVEST_CURSOR",
+                "cursor.kind",
+                f"{self.protocol} cursors must use {expected!r}",
+            )
+        _text(self.value, "cursor.value", maximum=MAX_RECORD_URI)
+        if self.protocol == "ckan":
+            match = re.fullmatch(r"(0|[1-9][0-9]*):([1-9][0-9]*)", self.value)
+            if match is None or int(match.group(2)) > 1_000:
+                raise CatalogHarvestError(
+                    "HARVEST_CURSOR", "cursor.value", "CKAN cursor must be start:rows"
+                )
+        elif self.protocol == "stac":
+            _https_uri(self.value, "cursor.value")
+        elif self.protocol == "datagov_v4":
+            validate_datagov_cursor_text(self.value, "cursor.value")
+
+    def to_dict(self) -> dict[str, str]:
+        return {"protocol": self.protocol, "kind": self.kind, "value": self.value}
+
+
+@dataclass(frozen=True)
+class HarvestResponseEvidence(_CanonicalContract):
+    """Exact transport work behind one admitted response."""
+
+    final_url: str
+    media_type: str
+    content_sha256: str
+    content_bytes: int
+    requests: int
+    responses: int
+    network_bytes: int
+    transport_evidence_digest: str
+    status: int = 200
+    rate_observation: DatagovRateObservation | None = None
+    schema_version: str = HARVEST_RESPONSE_EVIDENCE_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != HARVEST_RESPONSE_EVIDENCE_VERSION:
+            raise CatalogHarvestError("VERSION", "response.schema_version", "unsupported version")
+        _https_uri(self.final_url, "response.final_url")
+        if not re.fullmatch(r"[0-9a-f]{64}", self.content_sha256):
+            raise CatalogHarvestError("DIGEST", "response.content_sha256", "must be SHA-256")
+        if not re.fullmatch(r"[0-9a-f]{64}", self.transport_evidence_digest):
+            raise CatalogHarvestError(
+                "DIGEST", "response.transport_evidence_digest", "must be SHA-256"
+            )
+        for name, value, minimum in (
+            ("content_bytes", self.content_bytes, 0),
+            ("requests", self.requests, 1),
+            ("responses", self.responses, 1),
+            ("network_bytes", self.network_bytes, 0),
+        ):
+            if type(value) is not int or value < minimum:
+                raise CatalogHarvestError("LIMIT", f"response.{name}", "is outside its bound")
+        if self.content_bytes > self.network_bytes:
+            raise CatalogHarvestError(
+                "HARVEST_TRANSPORT_EVIDENCE",
+                "response.network_bytes",
+                "network bytes may not be smaller than final content bytes",
+            )
+        if type(self.status) is not int or not 100 <= self.status <= 599:
+            raise CatalogHarvestError("HARVEST_STATUS", "response.status", "status is invalid")
+        if self.rate_observation is not None and not isinstance(
+            self.rate_observation, DatagovRateObservation
+        ):
+            raise CatalogHarvestError(
+                "DATAGOV_RATE", "response.rate_observation", "rate observation is invalid"
+            )
+
+    @classmethod
+    def from_retrieval(cls, retrieved: RetrievedBytes) -> HarvestResponseEvidence:
+        requests = sum(max(1, len(hop.peer_attempts)) for hop in retrieved.hops)
+        return cls(
+            final_url=retrieved.final_url,
+            media_type=retrieved.media_type,
+            content_sha256=retrieved.content_sha256,
+            content_bytes=len(retrieved.content),
+            requests=requests,
+            responses=len(retrieved.hops),
+            network_bytes=retrieved.total_response_body_size_bytes,
+            transport_evidence_digest=retrieved.transport_evidence_digest,
+            status=retrieved.hops[-1].status,
+            rate_observation=retrieved.rate_observation,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        result = {
+            "schema_version": self.schema_version,
+            "final_url": self.final_url,
+            "media_type": self.media_type,
+            "content_sha256": self.content_sha256,
+            "content_bytes": self.content_bytes,
+            "requests": self.requests,
+            "responses": self.responses,
+            "network_bytes": self.network_bytes,
+            "transport_evidence_digest": self.transport_evidence_digest,
+        }
+        if self.status != 200 or self.rate_observation is not None:
+            result["status"] = self.status
+            result["rate_observation"] = (
+                self.rate_observation.to_dict() if self.rate_observation is not None else None
+            )
+        return result
+
+
+@dataclass(frozen=True)
+class HarvestPage(_CanonicalContract):
+    """One strict page envelope returned by every supported harvester."""
+
+    protocol: str
+    records: tuple[Any, ...]
+    next_cursor: HarvestCursor | None
+    provider_count: int | None
+    count_basis: str
+    skipped_record_ids: tuple[str, ...]
+    evidence: EvidenceReference
+    response_evidence: HarvestResponseEvidence | None
+    schema_version: str = HARVEST_PAGE_CONTRACT_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != HARVEST_PAGE_CONTRACT_VERSION:
+            raise CatalogHarvestError("VERSION", "page.schema_version", "unsupported version")
+        if self.protocol not in HARVEST_PROTOCOLS:
+            raise CatalogHarvestError("ENUM", "page.protocol", "must name a harvest protocol")
+        if not isinstance(self.records, tuple) or any(
+            not _same_protocol_record(record, self.protocol) for record in self.records
+        ):
+            raise CatalogHarvestError("TYPE", "page.records", "must be same-protocol records")
+        if self.next_cursor is not None and self.next_cursor.protocol != self.protocol:
+            raise CatalogHarvestError("HARVEST_CURSOR", "page.next_cursor", "protocol mismatch")
+        if self.provider_count is not None and (
+            type(self.provider_count) is not int or self.provider_count < 0
+        ):
+            raise CatalogHarvestError("HARVEST_COUNT", "page.provider_count", "must be nonnegative")
+        allowed_basis = {"reported_total", "not_reported", "one_shot_response"}
+        if self.count_basis not in allowed_basis:
+            raise CatalogHarvestError("ENUM", "page.count_basis", "unsupported count basis")
+        if self.count_basis == "reported_total" and self.provider_count is None:
+            raise CatalogHarvestError("HARVEST_COUNT", "page.provider_count", "count is required")
+        if not isinstance(self.skipped_record_ids, tuple):
+            raise CatalogHarvestError("TYPE", "page.skipped_record_ids", "must be a tuple")
+        if tuple(sorted(self.skipped_record_ids)) != self.skipped_record_ids:
+            raise CatalogHarvestError("ORDER", "page.skipped_record_ids", "must be sorted")
+        if not self.records and self.next_cursor is not None:
+            raise CatalogHarvestError(
+                "HARVEST_EMPTY_PAGE", "page.records", "an empty non-terminal page is refused"
+            )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "protocol": self.protocol,
+            "records": [record.to_dict() for record in self.records],
+            "next_cursor": self.next_cursor.to_dict() if self.next_cursor else None,
+            "provider_count": self.provider_count,
+            "count_basis": self.count_basis,
+            "skipped_record_ids": list(self.skipped_record_ids),
+            "evidence": self.evidence.to_dict(),
+            "response_evidence": (
+                self.response_evidence.to_dict() if self.response_evidence is not None else None
+            ),
+        }
+
+
+@dataclass(frozen=True)
+class HarvestedRecord(_CanonicalContract):
+    """One dataset as a public catalogue describes it, read and nothing more."""
+
+    protocol: str
+    record_id: str
+    title: str
+    description: str
+    publisher: str
+    landing_uri: str
+    data_formats: tuple[str, ...]
+    evidence: EvidenceReference
+    license_id: str | None = None
+    license_uri: str | None = None
+    declared_updated_at: str | None = None
+    schema_version: str = HARVESTED_RECORD_CONTRACT_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != HARVESTED_RECORD_CONTRACT_VERSION:
+            raise CatalogHarvestError(
+                "VERSION",
+                "record.schema_version",
+                f"must equal {HARVESTED_RECORD_CONTRACT_VERSION!r}",
+            )
+        if self.protocol not in HARVEST_PROTOCOLS:
+            raise CatalogHarvestError(
+                "ENUM",
+                "record.protocol",
+                f"must be one of {list(HARVEST_PROTOCOLS)}",
+            )
+        # A publisher's dataset id is not ours to shape, so it is bounded and character-checked
+        # rather than forced through the harness identifier regex.
+        record_id = _text(self.record_id, "record.record_id", maximum=MAX_RECORD_ID)
+        if not _RECORD_ID.fullmatch(record_id):
+            raise CatalogHarvestError(
+                "RECORD_ID",
+                "record.record_id",
+                "must contain only unreserved catalogue-id characters",
+            )
+        _text(self.title, "record.title", maximum=MAX_RECORD_TITLE)
+        _text(self.description, "record.description", maximum=MAX_RECORD_TEXT)
+        _text(self.publisher, "record.publisher", maximum=MAX_RECORD_PUBLISHER)
+        _https_uri(self.landing_uri, "record.landing_uri")
+        if not isinstance(self.data_formats, tuple):
+            raise CatalogHarvestError("TYPE", "record.data_formats", "must be an immutable tuple")
+        if len(self.data_formats) > MAX_RECORD_FORMATS:
+            raise CatalogHarvestError(
+                "LIMIT",
+                "record.data_formats",
+                f"must contain at most {MAX_RECORD_FORMATS} formats",
+            )
+        for position, value in enumerate(self.data_formats):
+            if value not in DATA_FORMATS:
+                raise CatalogHarvestError(
+                    "ENUM",
+                    f"record.data_formats[{position}]",
+                    f"must be one of {sorted(DATA_FORMATS)}",
+                )
+        if len(set(self.data_formats)) != len(self.data_formats):
+            raise CatalogHarvestError("DUPLICATE", "record.data_formats", "formats must be unique")
+        if not isinstance(self.evidence, EvidenceReference):
+            raise CatalogHarvestError("TYPE", "record.evidence", "must be an EvidenceReference")
+        if self.license_id is not None:
+            _text(self.license_id, "record.license_id", maximum=MAX_RECORD_LICENSE_ID)
+        if self.license_uri is not None:
+            _https_uri(self.license_uri, "record.license_uri")
+        if self.declared_updated_at is not None:
+            if not _TIMESTAMP.fullmatch(str(self.declared_updated_at)):
+                raise CatalogHarvestError(
+                    "TIMESTAMP",
+                    "record.declared_updated_at",
+                    "must be a canonical RFC 3339 UTC timestamp",
+                )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "protocol": self.protocol,
+            "record_id": self.record_id,
+            "title": self.title,
+            "description": self.description,
+            "publisher": self.publisher,
+            "landing_uri": self.landing_uri,
+            "data_formats": list(self.data_formats),
+            "evidence": self.evidence.to_dict(),
+            "license_id": self.license_id,
+            "license_uri": self.license_uri,
+            "declared_updated_at": self.declared_updated_at,
+        }
+
+    @property
+    def facts_sha256(self) -> str:
+        """The digest of everything the provider *published*, with the observation excluded.
+
+        A rights decision binds to this rather than to ``digest``, and it has to: ``digest``
+        covers the evidence wrapper -- the page URL, the caller's observation timestamp, and
+        the digest of the whole response page -- so a byte-identical record re-observed at a
+        new time, or sharing a page with any unrelated changed record, would void every
+        standing decision and train the operator to re-stamp digests mechanically. Excluding
+        the evidence makes the binding mean what the person meant: these published facts,
+        whoever's page they arrived on, whenever they were fetched.
+        """
+
+        payload = self.to_dict()
+        del payload["evidence"]
+        return canonical_sha256(payload)
+
+
+def declared_timestamp(value: str | None) -> str | None:
+    """A harvester's normalised update time, or ``None`` when it is not one.
+
+    One grammar, checked in one place. Each harvester normalises the spelling its publishers use
+    and hands the result here; anything that does not come out as the canonical RFC 3339 UTC
+    timestamp ``HarvestedRecord`` requires becomes an undeclared update time.
+
+    Returning ``None`` rather than raising is what keeps one record from denying a whole page.
+    ``HarvestedRecord.__post_init__`` raises ``TIMESTAMP`` on a bad value and that exception
+    escapes the enclosing ``parse`` call, so a single publisher field spelled
+    ``2026-07-31T23:59:59+02:00`` -- legal RFC 3339, and not the UTC spelling STAC requires -- or
+    ``not-a-real-timestamp-value`` would refuse every other record in the same response, in a
+    document a stranger controls. An undeclared update time is already a supported state that
+    both harvesters produce for an open-ended or absent value, so degrading this one field on this
+    one record is strictly less lossy than degrading the page.
+
+    A zone-bearing spelling is left undeclared rather than converted. Converting would need a zone
+    arithmetic path this boundary does not otherwise have, and the two protocols harvested here
+    both specify UTC, so an offset is a publisher deviation rather than a case to support. The rule
+    is only as good as what reaches this funnel, which is why :func:`carries_a_zone_offset` exists
+    and why both pre-normalisers call it before they touch the spelling -- see there.
+
+    The record-level check stays where it is. This is the funnel every harvester goes through; that
+    is the backstop for a record built by hand.
+    """
+
+    if value is None:
+        return None
+    return value if _TIMESTAMP.fullmatch(value) else None
+
+
+def carries_a_zone_offset(value: str) -> bool:
+    """True when a timestamp spelling names a UTC offset rather than UTC itself.
+
+    Both pre-normalisers cut a fractional second down to six digits before handing the result to
+    :func:`declared_timestamp`. The offset must be identified first because it rides on the
+    fractional second and would otherwise go with the part that
+    was cut: ``2026-07-31T23:59:59.123456+02:00`` came out as ``2026-07-31T23:59:59.123456Z``, and
+    ``2026-07-31T23:59:59.123+02:00`` -- whose fraction is not six digits, so the whole remainder
+    was dropped -- came out as ``2026-07-31T23:59:59Z``. Both are canonical, both are accepted, and
+    both are a local time relabelled as UTC, by up to fourteen hours. Only the fraction-less
+    spelling was rejected, which is the one every test used.
+
+    That value becomes ``HarvestedRecord.declared_updated_at``, which feeds the health probe's
+    publication delay and the derived update frequency, which the facts gate then decides a
+    source's cadence on. A silently wrong hour is therefore a wrong catalog fact, not a wrong
+    display string, so this is checked before any spelling is normalised rather than after.
+
+    Everything after the date is inspected, because the date's own hyphens are not offsets and
+    every canonical UTC spelling this boundary accepts carries neither a plus nor a minus in its
+    time of day.
+    """
+
+    _, separator, time_of_day = value.partition("T")
+    if not separator:
+        return False
+    return "+" in time_of_day or "-" in time_of_day
+
+
+def bounded_record_text(value: str, *, maximum: int) -> str:
+    """A publisher's free text, cut to the record bound rather than refused at it.
+
+    Same rule as :func:`declared_timestamp`, applied to the four fields it was never applied to.
+    ``HarvestedRecord.__post_init__`` raises ``TEXT`` on an over-length ``title``, ``description``
+    or ``publisher``, and that exception escapes the enclosing ``parse`` call -- so one publisher's
+    long prose refused every other record in the same response, in a document a stranger controls.
+    CKAN's ``notes`` has no server-side length limit and multi-thousand-character descriptions are
+    ordinary on real portals, so this was a first-contact failure rather than an adversarial edge.
+
+    Truncating is the smaller loss, and it is the loss the two other harvesters were already
+    taking on ``description`` before this was stated in one place. A cut description is still the
+    publisher's description; a refused page is every sibling record gone. This is a bound on a
+    descriptive field, never on an identifier: see :func:`usable_record_id` for those.
+
+    Trailing whitespace left by the cut is stripped, because a title ending mid-space is a worse
+    artefact than one ending mid-word, and an all-whitespace tail would leave the field empty --
+    which the record contract refuses. A value that is empty after stripping is returned as-is so
+    the caller's own fallback (``or record_id``, ``or "No description published."``) applies.
+    """
+
+    text = value[:maximum].rstrip()
+    return text if text else value[:maximum]
+
+
+def usable_record_id(value: str | None) -> str | None:
+    """A publisher's dataset id, or ``None`` when it is not one this boundary can carry.
+
+    An identifier is not truncatable. Cutting one to the bound would mint a different dataset's
+    coordinate out of this dataset's, and every citation built on it afterwards would name
+    something the publisher never published -- which is worse than not carrying the record.
+
+    So the degradation for an id is the record, not the field: the caller records it in ``skipped``
+    and keeps the rest of the page, exactly as it already does for a package with no id at all and
+    for one whose landing page is not HTTPS. ``skipped`` is reported in the refusal message when a
+    response yields nothing usable, so a page lost this way is never lost silently.
+    """
+
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    if not text or len(text) > MAX_RECORD_ID or not _RECORD_ID.fullmatch(text):
+        return None
+    return text
+
+
+def usable_https_uri(value: str | None) -> str | None:
+    """An HTTPS URI within the record bound, or ``None``.
+
+    A URI is an identifier too, so it degrades the same way an id does rather than being cut. For
+    ``landing_uri`` the caller skips the record; for ``license_uri``, which is optional, ``None``
+    is the supported state and the licence remains readable from ``license_id``.
+    """
+
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    if not text.startswith("https://") or len(text) <= len("https://"):
+        return None
+    return text if len(text) <= MAX_RECORD_URI else None
+
+
+def usable_license_id(value: str | None) -> str | None:
+    """A licence identifier within the record bound, or ``None``, which reads as unclear rights.
+
+    Not truncated, for the reason ids are not truncated: a cut licence id is a different licence
+    id. Dropping it maps to ``unclear``, which the facts gate escalates to a human rather than
+    guessing at -- the correct outcome for terms this boundary could not read.
+    """
+
+    if not isinstance(value, str):
+        return None
+    text = value.strip()
+    if not text or len(text) > MAX_RECORD_LICENSE_ID:
+        return None
+    return text
+
+
+class SourceHarvester(Protocol):
+    """One endpoint family, read into records. Takes bytes; never fetches."""
+
+    @property
+    def descriptor(self) -> HarvesterDescriptor:
+        """Return the immutable harvester identity."""
+
+    def parse(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+    ) -> tuple[HarvestedRecord, ...]:
+        """Read this exact recorded response into records, or refuse it."""
+
+    def parse_page(
+        self,
+        payload: bytes,
+        *,
+        uri: str,
+        observed_at: str,
+        evidence: EvidenceReference,
+        response_evidence: HarvestResponseEvidence | None,
+        cursor: HarvestCursor | None,
+    ) -> HarvestPage:
+        """Read a response plus its closed continuation coordinate."""
+
+
+class HarvestRegistry:
+    """Allowlist of harvesters, in the shape of the coordinator's adapter allowlist."""
+
+    def __init__(self, harvesters: tuple[SourceHarvester, ...] = ()) -> None:
+        self._harvesters: dict[tuple[str, str], SourceHarvester] = {}
+        for harvester in harvesters:
+            self.register(harvester)
+
+    def register(self, harvester: SourceHarvester) -> None:
+        descriptor = harvester.descriptor
+        if not isinstance(descriptor, HarvesterDescriptor):
+            raise CatalogHarvestError(
+                "HARVESTER_DESCRIPTOR",
+                "harvester.descriptor",
+                "harvester descriptor must use the strict contract",
+            )
+        key = (descriptor.harvester_id, descriptor.harvester_version)
+        if key in self._harvesters:
+            raise CatalogHarvestError(
+                "HARVESTER_DUPLICATE",
+                "registry",
+                f"harvester {descriptor.coordinate} is already registered",
+            )
+        self._harvesters[key] = harvester
+
+    def resolve(self, harvester_id: str, harvester_version: str) -> SourceHarvester:
+        try:
+            return self._harvesters[(harvester_id, harvester_version)]
+        except KeyError:
+            raise CatalogHarvestError(
+                "HARVESTER_UNAVAILABLE",
+                "harvester_id",
+                f"harvester {harvester_id}@{harvester_version} is not allowlisted",
+            ) from None
+
+    def inventory(self) -> tuple[HarvesterDescriptor, ...]:
+        return tuple(self._harvesters[key].descriptor for key in sorted(self._harvesters))
+
+
+def harvest_limits(*media_types: str, max_response_bytes: int) -> RetrievalLimits:
+    """Transport caps for one harvester, never looser than the catalog admission cap."""
+
+    if not media_types:
+        raise CatalogHarvestError(
+            "EMPTY",
+            "media_types",
+            "a harvester must allow at least one media type",
+        )
+    if type(max_response_bytes) is not int or not 1 <= max_response_bytes <= MAX_CATALOG_FACT_BYTES:
+        raise CatalogHarvestError(
+            "HARVEST_RESPONSE_LIMIT",
+            "max_response_bytes",
+            f"must be an integer in [1, {MAX_CATALOG_FACT_BYTES}]",
+        )
+    return RetrievalLimits(
+        max_response_bytes=max_response_bytes,
+        max_redirects=2,
+        max_requests=3,
+        allowed_media_types=tuple(media_types),
+    )
+
+
+def fetch_harvest_payload(
+    *,
+    retriever: PinnedHttpsRetriever,
+    url: str,
+    observed_at: str,
+    limits: RetrievalLimits,
+) -> tuple[bytes, EvidenceReference]:
+    """Fetch one bounded catalogue response and admit it as public fact bytes.
+
+    Returns the exact bytes and the citation for them. The caller parses; this function does not,
+    because a parser that also fetches cannot be tested against a recorded response.
+
+    ``limits`` governs the fetch. It is threaded into the transport through ``retriever.narrowed``
+    rather than merely checked here, because a cap that is validated and then not applied is worse
+    than no cap: ``CKAN_LIMITS`` would read as the control that refuses a bot-wall HTML page while
+    a retriever configured with a looser allowlist accepted it. Narrowing can only tighten what the
+    coordinator configured, so this never widens egress.
+    """
+
+    if limits.max_response_bytes > MAX_CATALOG_FACT_BYTES:
+        raise CatalogHarvestError(
+            "HARVEST_RESPONSE_LIMIT",
+            "limits.max_response_bytes",
+            f"a harvest fetch may not exceed {MAX_CATALOG_FACT_BYTES} bytes",
+        )
+    retrieved = _retrieve_harvest(
+        retriever=retriever,
+        url=url,
+        limits=limits,
+    )
+    evidence = EvidenceReference(
+        uri=retrieved.final_url,
+        observed_at=observed_at,
+        content_sha256=retrieved.content_sha256,
+        media_type=retrieved.media_type,
+    )
+    return retrieved.content, evidence
+
+
+def fetch_harvest_response(
+    *,
+    retriever: PinnedHttpsRetriever,
+    url: str,
+    observed_at: str,
+    limits: RetrievalLimits,
+) -> tuple[bytes, EvidenceReference, HarvestResponseEvidence]:
+    """Fetch like :func:`fetch_harvest_payload`, retaining exact transport accounting."""
+
+    if limits.max_response_bytes > MAX_CATALOG_FACT_BYTES:
+        raise CatalogHarvestError(
+            "HARVEST_RESPONSE_LIMIT",
+            "limits.max_response_bytes",
+            f"a harvest fetch may not exceed {MAX_CATALOG_FACT_BYTES} bytes",
+        )
+    retrieved = _retrieve_harvest(retriever=retriever, url=url, limits=limits)
+    evidence = EvidenceReference(
+        uri=retrieved.final_url,
+        observed_at=observed_at,
+        content_sha256=retrieved.content_sha256,
+        media_type=retrieved.media_type,
+    )
+    return retrieved.content, evidence, HarvestResponseEvidence.from_retrieval(retrieved)
+
+
+def fetch_datagov_v4_response(
+    *,
+    retriever: PinnedHttpsRetriever,
+    url: str,
+    observed_at: str,
+    limits: RetrievalLimits,
+    authorization: DatagovV4Authorization,
+    on_request_started: Callable[[], None] | None = None,
+    on_response_observed: (
+        Callable[[HarvestResponseEvidence | DatagovRefusedResponseEvidence], None] | None
+    ) = None,
+    on_response_received: (
+        Callable[[bytes, EvidenceReference, HarvestResponseEvidence], None] | None
+    ) = None,
+) -> tuple[bytes, EvidenceReference, HarvestResponseEvidence]:
+    """Fetch the exact authenticated v4 response without widening legacy admission caps."""
+
+    if not 1 <= limits.max_response_bytes <= 16 * 1024 * 1024 or limits.max_redirects != 0:
+        raise CatalogHarvestError(
+            "DATAGOV_RESPONSE_LIMIT",
+            "limits",
+            "Data.gov v4 requires the exact 16 MiB, zero-redirect transport boundary",
+        )
+    if not isinstance(authorization, DatagovV4Authorization):
+        raise CatalogHarvestError(
+            "DATAGOV_AUTHORIZATION", "authorization", "strict authorization is required"
+        )
+    bounded = retriever.narrowed(limits)
+    redacted_observation: DatagovRefusedResponseEvidence | None = None
+
+    def observe_transport(
+        status: int, network_bytes: int, rate: DatagovRateObservation | None
+    ) -> None:
+        nonlocal redacted_observation
+        redacted_observation = DatagovRefusedResponseEvidence(
+            final_url=url,
+            status=status,
+            network_bytes=network_bytes,
+            rate_observation=rate,
+        )
+        if on_response_observed is not None:
+            on_response_observed(redacted_observation)
+
+    try:
+        if on_request_started is not None:
+            on_request_started()
+        if isinstance(bounded, PinnedHttpsRetriever):
+            retrieved = bounded.retrieve_datagov_v4(
+                url, authorization, on_response_observed=observe_transport
+            )
+        else:
+            retrieved = bounded.retrieve_datagov_v4(url, authorization)
+    except DatagovResponseRefused as error:
+        redacted = DatagovRefusedResponseEvidence(
+            final_url=url,
+            status=error.status,
+            network_bytes=error.network_bytes,
+            rate_observation=None,
+        )
+        if on_response_observed is not None:
+            on_response_observed(redacted)
+        raise DatagovFetchRefused(
+            error.code,
+            "harvest.response",
+            error.detail,
+            response_evidence=redacted,
+        ) from None
+    except AcquisitionSecurityError as error:
+        if redacted_observation is not None:
+            raise DatagovFetchRefused(
+                error.code,
+                "harvest.response",
+                error.detail,
+                response_evidence=redacted_observation,
+            ) from None
+        raise _translate(error, url) from None
+    response_evidence = HarvestResponseEvidence.from_retrieval(retrieved)
+    redacted = redacted_observation or DatagovRefusedResponseEvidence(
+        final_url=response_evidence.final_url,
+        status=response_evidence.status,
+        network_bytes=response_evidence.network_bytes,
+        rate_observation=response_evidence.rate_observation,
+    )
+    if redacted_observation is None and on_response_observed is not None:
+        on_response_observed(redacted)
+    try:
+        authorization.reject_echoed_response(retrieved.content)
+    except AcquisitionSecurityError as error:
+        raise DatagovFetchRefused(
+            error.code,
+            "harvest.response",
+            error.detail,
+            response_evidence=redacted,
+        ) from None
+    if on_response_observed is not None:
+        on_response_observed(response_evidence)
+    evidence = EvidenceReference(
+        uri=retrieved.final_url,
+        observed_at=observed_at,
+        content_sha256=retrieved.content_sha256,
+        media_type=retrieved.media_type,
+    )
+    if on_response_received is not None:
+        on_response_received(retrieved.content, evidence, response_evidence)
+    return retrieved.content, evidence, response_evidence
+
+
+def _retrieve_harvest(
+    *, retriever: PinnedHttpsRetriever, url: str, limits: RetrievalLimits
+) -> RetrievedBytes:
+    bounded = retriever.narrowed(limits)
+    try:
+        retrieved = bounded.retrieve(url)
+    except AcquisitionSecurityError as error:
+        raise _translate(error, url) from None
+    admit_public_fact_bytes(retrieved.content)
+    return retrieved
+
+
+def _translate(error: AcquisitionSecurityError, url: str) -> CatalogHarvestError:
+    host = _hostname(url)
+    if error.code == "MEDIA_TYPE":
+        # The bot-wall case: a 200 carrying an interstitial HTML page. Naming the host turns a
+        # generic transport error into a durable fact about that instance.
+        return CatalogHarvestError(
+            "HARVEST_MEDIA_TYPE",
+            "harvest.response",
+            f"{host} answered with a media type this harvester does not accept: {error.detail}",
+        )
+    # A cycle survey treats 404 as an absent unpublished run. A catalog harvest still treats that
+    # same response as an endpoint that is not open;
+    # the type split must not bypass the catalog boundary's existing translation.
+    if error.code in {"HTTP_STATUS", "HTTP_NOT_FOUND"}:
+        match = _STATUS_TAIL.search(error.detail)
+        status = int(match.group(1)) if match else None
+        if status in {401, 403}:
+            return CatalogHarvestError(
+                "HARVEST_ENDPOINT_NOT_OPEN",
+                "harvest.response",
+                f"{host} answered HTTP {status}: {AUTHENTICATED_ADAPTER_PARKED}",
+            )
+        return CatalogHarvestError(
+            "HARVEST_ENDPOINT_NOT_OPEN",
+            "harvest.response",
+            f"{host} answered {error.detail}",
+        )
+    if error.code in {"RESPONSE_LIMIT", "CONTENT_LENGTH"}:
+        return CatalogHarvestError(
+            "HARVEST_RESPONSE_LIMIT",
+            "harvest.response",
+            f"{host} answered outside the harvest byte bound: {error.detail}",
+        )
+    # Egress, DNS, URL policy, TLS, concurrency and rate-limit failures are facts about this side
+    # of the connection, not about the catalogue. They propagate untouched.
+    raise error
+
+
+def _hostname(url: str) -> str:
+    return (urlsplit(url).hostname or "the endpoint").lower()
+
+
+def _same_protocol_record(record: Any, protocol: str) -> bool:
+    if isinstance(record, HarvestedRecord):
+        return record.protocol == protocol
+    return (
+        protocol == "datagov_v4"
+        and record.__class__.__name__ == "DatagovV4Record"
+        and record.__class__.__module__.endswith(".harvest.datagov_v4")
+        and getattr(record, "protocol", None) == protocol
+        and callable(getattr(record, "to_dict", None))
+        and isinstance(getattr(record, "digest", None), str)
+    )
+
+
+def _text(value: Any, path: str, *, maximum: int) -> str:
+    if not isinstance(value, str) or not value or len(value) > maximum:
+        raise CatalogHarvestError(
+            "TEXT",
+            path,
+            f"must be a non-empty string no longer than {maximum} characters",
+        )
+    return value
+
+
+def validate_datagov_cursor_text(value: Any, path: str) -> str:
+    """Validate the one closed Unicode/control boundary for v4 continuation text."""
+
+    text = _text(value, path, maximum=MAX_RECORD_URI)
+    try:
+        text.encode("utf-8", errors="strict")
+    except UnicodeEncodeError:
+        raise CatalogHarvestError(
+            "HARVEST_CURSOR", path, "Data.gov cursor must be valid Unicode"
+        ) from None
+    if any(
+        ord(character) < 0x20
+        or 0x7F <= ord(character) <= 0x9F
+        or (character.isspace() and character != " ")
+        for character in text
+    ):
+        raise CatalogHarvestError(
+            "HARVEST_CURSOR", path, "Data.gov cursor contains unsafe control or whitespace text"
+        )
+    return text
+
+
+def _identifier(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=128)
+    if not re.fullmatch(r"[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*", text):
+        raise CatalogHarvestError(
+            "IDENTIFIER",
+            path,
+            "must be a lowercase dotted/dashed/underscored identifier",
+        )
+    return text
+
+
+def _https_uri(value: Any, path: str) -> str:
+    text = _text(value, path, maximum=2_048)
+    if not text.startswith("https://") or len(text) <= len("https://"):
+        raise CatalogHarvestError("HTTPS_URI", path, "must be an HTTPS URI")
+    return text