mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/fill.py

@@ -0,0 +1,3889 @@
+"""Bounded, resumable local fills over the existing catalog harvesters."""
+
+from __future__ import annotations
+
+import re
+import sqlite3
+import tempfile
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass, replace
+from itertools import islice
+from pathlib import Path
+from typing import Any
+from urllib.parse import parse_qsl, urlencode, urlsplit, urlunsplit
+
+from mostlyright.data_harness.acquisition.http import (
+    DATAGOV_V4_AUTHENTICATION_KIND,
+    DatagovRateObservation,
+    DatagovV4Authorization,
+    PeerAttemptError,
+)
+from mostlyright.data_harness.acquisition.url_policy import AcquisitionSecurityError
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.fill_partitions import (
+    dry_run_external_merge_pass,
+    external_merge_pass,
+    plan_partition_runs,
+    read_partition_run,
+    validate_pass_map,
+    write_partition_runs,
+)
+from mostlyright.data_harness.sources.catalog.fill_staging import (
+    SHARD_SCHEMA,
+    CatalogFillRefused,
+    FillStaging,
+    shard_byte_limit,
+)
+from mostlyright.data_harness.sources.catalog.harvest.ckan import (
+    CKAN_LIMITS,
+    CkanHarvester,
+    ckan_licence_rights,
+)
+from mostlyright.data_harness.sources.catalog.harvest.datagov_v4 import (
+    DATAGOV_V4_ENDPOINT,
+    DATAGOV_V4_LIMITS,
+    DatagovProjectedField,
+    DatagovV4Harvester,
+    DatagovV4Record,
+)
+from mostlyright.data_harness.sources.catalog.harvest.protocol import (
+    CatalogHarvestError,
+    DatagovFetchRefused,
+    DatagovRefusedResponseEvidence,
+    HarvestCursor,
+    HarvestedRecord,
+    HarvestPage,
+    HarvestResponseEvidence,
+    SourceHarvester,
+    fetch_datagov_v4_response,
+    fetch_harvest_response,
+)
+from mostlyright.data_harness.sources.catalog.harvest.sdmx import SDMX_LIMITS, SdmxHarvester
+from mostlyright.data_harness.sources.catalog.harvest.stac import (
+    STAC_LIMITS,
+    StacHarvester,
+    stac_licence_rights,
+)
+from mostlyright.data_harness.sources.contracts import EvidenceReference
+
+FILL_CONFIG_SCHEMA = "harness-catalog-fill-config.v1"
+FILL_RECEIPT_SCHEMA = "harness-catalog-fill-receipt.v1"
+DATAGOV_V4_CHECKPOINT_SCHEMA = "harness-datagov-v4-fill-checkpoint.v8"
+
+#: The provider sorts this cursor contract admits.  ``last_harvested_date`` orders by the one
+#: field data.gov rewrites every time it re-harvests a dataset, so a record can move ahead of a
+#: cursor that already passed it and never be seen; ``relevance`` orders by the constant
+#: match-all score, then popularity, then the record's stable UUID.
+DATAGOV_V4_SORT_ORDERS = ("last_harvested_date", "relevance")
+DATAGOV_V4_DEFAULT_SORT = "last_harvested_date"
+DATAGOV_V4_MAX_NORMALIZED_PAGE_BYTES = 64 * 1024 * 1024
+DATAGOV_V4_MAX_NORMALIZED_EXPANSION = 7
+DATAGOV_V4_LOOP_SET_SCHEMA = "harness-datagov-v4-loop-set.v1"
+DATAGOV_V4_LOOP_SET_LEAF_ITEMS = 512
+_V4_VALIDATED_CACHE_LIMIT = 32
+_MAX_SAFE_EPOCH_SECONDS = (1 << 53) - 1
+_V4_POST_RESPONSE_OVERRUN_REASONS = frozenset(
+    {
+        "FILL_RESPONSE_BYTE_LIMIT",
+        "FILL_AGGREGATE_BYTE_LIMIT",
+        "FILL_WALL_TIME_LIMIT",
+    }
+)
+_V4_PREFLIGHT_TERMINAL_REASONS = frozenset(
+    {
+        "FILL_REQUEST_LIMIT",
+        "FILL_RESPONSE_LIMIT",
+        "FILL_PAGE_LIMIT",
+        "FILL_RECORD_LIMIT",
+        "FILL_AGGREGATE_BYTE_LIMIT",
+        "FILL_WALL_TIME_LIMIT",
+    }
+)
+#: Terminal reasons that still seal a final map, so the corpus they name is real.
+_V4_MAP_SEALING_REASONS = frozenset({"FILL_NONCONVERGENT", "FILL_CONFLICTING_IDENTIFIERS"})
+_V4_IDEMPOTENT_TERMINAL_REASONS = (
+    _V4_POST_RESPONSE_OVERRUN_REASONS
+    | _V4_PREFLIGHT_TERMINAL_REASONS
+    | {
+        "FILL_ENDPOINT_DRIFT",
+        "FILL_NONCONVERGENT",
+        "FILL_CONFLICTING_IDENTIFIERS",
+    }
+)
+_V4_EVIDENCE_STREAMS = (
+    "request",
+    "response",
+    "page",
+    "record",
+    "cursor_transition",
+)
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_TIMESTAMP = re.compile(
+    r"^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}(?:\.[0-9]{1,9})?Z$"
+)
+_V4_VALIDATED_CHECKPOINTS: dict[str, tuple[str, tuple[tuple[str, int, int, int, int], ...]]] = {}
+
+
+@dataclass(frozen=True)
+class CatalogFillLimits:
+    max_requests: int
+    max_pages: int
+    max_records: int
+    max_response_bytes: int
+    max_aggregate_bytes: int
+    max_responses: int
+    max_wall_time_ns: int
+
+    def __post_init__(self) -> None:
+        maxima = {
+            "max_requests": 10_000_000,
+            "max_pages": 1_000_000,
+            "max_records": 10_000_000,
+            "max_response_bytes": 64 * 1024 * 1024,
+            "max_aggregate_bytes": 1 << 50,
+            "max_responses": 10_000_000,
+            "max_wall_time_ns": 30 * 24 * 60 * 60 * 1_000_000_000,
+        }
+        for name, maximum in maxima.items():
+            value = getattr(self, name)
+            if type(value) is not int or not 1 <= value <= maximum:
+                raise CatalogFillRefused(
+                    "FILL_LIMIT", f"fill.limits.{name}", f"must be in [1, {maximum}]"
+                )
+
+    def to_dict(self) -> dict[str, int]:
+        return {
+            "max_requests": self.max_requests,
+            "max_pages": self.max_pages,
+            "max_records": self.max_records,
+            "max_response_bytes": self.max_response_bytes,
+            "max_aggregate_bytes": self.max_aggregate_bytes,
+            "max_responses": self.max_responses,
+            "max_wall_time_ns": self.max_wall_time_ns,
+        }
+
+
+@dataclass(frozen=True)
+class CatalogFillConfig:
+    protocol: str
+    endpoint: str
+    observed_at: str
+    page_size: int
+    max_convergence_passes: int
+    limits: CatalogFillLimits
+    authentication_kind: str | None = None
+    minimum_request_interval_ns: int = 0
+    sort: str = DATAGOV_V4_DEFAULT_SORT
+    max_conflicting_identifiers: int = 0
+    harvester_id: str | None = None
+    harvester_version: str = "1.0.0"
+    predecessor_sha256: str | None = None
+    schema_version: str = FILL_CONFIG_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != FILL_CONFIG_SCHEMA:
+            raise CatalogFillRefused("FILL_CONFIG_VERSION", "fill.config", "unsupported version")
+        if self.protocol not in {"ckan", "datagov_v4", "stac", "sdmx"}:
+            raise CatalogFillRefused("FILL_PROTOCOL", "fill.protocol", "unsupported protocol")
+        parsed = urlsplit(self.endpoint)
+        if (
+            parsed.scheme != "https"
+            or not parsed.hostname
+            or parsed.username is not None
+            or parsed.password is not None
+            or parsed.query
+            or parsed.fragment
+        ):
+            raise CatalogFillRefused(
+                "FILL_ENDPOINT",
+                "fill.endpoint",
+                "endpoint must be a credential-free public HTTPS base URL without query or "
+                "fragment",
+            )
+        if not isinstance(self.limits, CatalogFillLimits):
+            raise CatalogFillRefused("FILL_LIMIT", "fill.limits", "strict limits are required")
+        if self.protocol == "datagov_v4":
+            if self.endpoint != DATAGOV_V4_ENDPOINT:
+                raise CatalogFillRefused(
+                    "DATAGOV_ENDPOINT",
+                    "fill.endpoint",
+                    "Data.gov v4 uses the exact official search endpoint",
+                )
+            if self.authentication_kind != DATAGOV_V4_AUTHENTICATION_KIND:
+                raise CatalogFillRefused(
+                    "DATAGOV_AUTHENTICATION",
+                    "fill.authentication_kind",
+                    "Data.gov v4 requires the closed key-file authentication kind",
+                )
+            if self.limits.max_response_bytes != 16 * 1024 * 1024:
+                raise CatalogFillRefused(
+                    "DATAGOV_RESPONSE_LIMIT",
+                    "fill.limits.max_response_bytes",
+                    "Data.gov v4 uses the exact 16 MiB response cap",
+                )
+            if self.sort not in DATAGOV_V4_SORT_ORDERS:
+                raise CatalogFillRefused(
+                    "DATAGOV_SORT",
+                    "fill.sort",
+                    "Data.gov v4 admits exactly the sorts its cursor contract can page",
+                )
+            if (
+                type(self.max_conflicting_identifiers) is not int
+                or self.max_conflicting_identifiers < 0
+            ):
+                raise CatalogFillRefused(
+                    "DATAGOV_CONFLICT_BOUND",
+                    "fill.max_conflicting_identifiers",
+                    "the admitted conflicting-identifier count must be a non-negative integer",
+                )
+        elif (
+            self.authentication_kind is not None
+            or self.minimum_request_interval_ns != 0
+            or self.sort != DATAGOV_V4_DEFAULT_SORT
+            or self.max_conflicting_identifiers != 0
+        ):
+            raise CatalogFillRefused(
+                "FILL_AUTHENTICATION",
+                "fill.authentication_kind",
+                "legacy public protocols remain credential-free",
+            )
+        if (
+            type(self.minimum_request_interval_ns) is not int
+            or not 0 <= self.minimum_request_interval_ns <= 3_600_000_000_000
+        ):
+            raise CatalogFillRefused(
+                "DATAGOV_RATE",
+                "fill.minimum_request_interval_ns",
+                "minimum interval must be in [0, 3600 seconds]",
+            )
+        if type(self.page_size) is not int or not 1 <= self.page_size <= 1_000:
+            raise CatalogFillRefused("FILL_PAGE_SIZE", "fill.page_size", "must be in [1, 1000]")
+        if not isinstance(self.observed_at, str) or not _TIMESTAMP.fullmatch(self.observed_at):
+            raise CatalogFillRefused(
+                "FILL_OBSERVED_AT", "fill.observed_at", "must be a canonical UTC timestamp"
+            )
+        if (
+            type(self.max_convergence_passes) is not int
+            or not 1 <= self.max_convergence_passes <= 100
+        ):
+            raise CatalogFillRefused(
+                "FILL_CONVERGENCE_LIMIT",
+                "fill.max_convergence_passes",
+                "must be in [1, 100]",
+            )
+        expected = _protocol_parts(self.protocol)[1].descriptor.harvester_id
+        if self.harvester_id is not None and self.harvester_id != expected:
+            raise CatalogFillRefused(
+                "FILL_HARVESTER", "fill.harvester_id", "harvester does not match protocol"
+            )
+        if self.harvester_version != "1.0.0":
+            raise CatalogFillRefused(
+                "FILL_HARVESTER", "fill.harvester_version", "harvester version is unavailable"
+            )
+        if self.predecessor_sha256 is not None and not _SHA256.fullmatch(self.predecessor_sha256):
+            raise CatalogFillRefused(
+                "FILL_PREDECESSOR", "fill.predecessor_sha256", "must be SHA-256 or null"
+            )
+
+    @property
+    def coordinate(self) -> str:
+        identifier = self.harvester_id or _protocol_parts(self.protocol)[1].descriptor.harvester_id
+        return f"{identifier}@{self.harvester_version}"
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        result = {
+            "schema_version": self.schema_version,
+            "protocol": self.protocol,
+            "endpoint": self.endpoint,
+            "observed_at": self.observed_at,
+            "page_size": self.page_size,
+            "max_convergence_passes": self.max_convergence_passes,
+            "limits": self.limits.to_dict(),
+            "harvester_id": self.harvester_id
+            or _protocol_parts(self.protocol)[1].descriptor.harvester_id,
+            "harvester_version": self.harvester_version,
+            "predecessor_sha256": self.predecessor_sha256,
+        }
+        if self.protocol == "datagov_v4":
+            result["authentication_kind"] = self.authentication_kind
+            result["minimum_request_interval_ns"] = self.minimum_request_interval_ns
+            result["sort"] = self.sort
+            result["max_conflicting_identifiers"] = self.max_conflicting_identifiers
+        return result
+
+
+@dataclass(frozen=True)
+class CatalogFillReceipt:
+    config_sha256: str
+    endpoint: str
+    harvester_coordinate: str
+    predecessor_sha256: str | None
+    status: str
+    reason_code: str | None
+    requests: int
+    responses: int
+    pages: int
+    records: int
+    unique_records: int
+    network_bytes: int
+    elapsed_ns: int
+    provider_count: int | None
+    count_basis: str | None
+    skipped_records: int
+    flagged_rights: int
+    convergence_passes: int
+    response_evidence_digests: tuple[str, ...]
+    page_shard_digests: tuple[str, ...]
+    flagged_queue_sha256: str
+    rate_observation: dict[str, int | None] | None = None
+    resume_not_before_epoch_seconds: int | None = None
+    evidence_indexes: dict[str, dict[str, Any]] | None = None
+    pass_map_sha256: str | None = None
+    normalized_bytes: int | None = None
+    conflicting_identifiers: int | None = None
+    schema_version: str = FILL_RECEIPT_SCHEMA
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        result = {
+            "schema_version": self.schema_version,
+            "config_sha256": self.config_sha256,
+            "endpoint": self.endpoint,
+            "harvester_coordinate": self.harvester_coordinate,
+            "predecessor_sha256": self.predecessor_sha256,
+            "status": self.status,
+            "reason_code": self.reason_code,
+            "requests": self.requests,
+            "responses": self.responses,
+            "pages": self.pages,
+            "records": self.records,
+            "unique_records": self.unique_records,
+            "network_bytes": self.network_bytes,
+            "elapsed_ns": self.elapsed_ns,
+            "provider_count": self.provider_count,
+            "count_basis": self.count_basis,
+            "skipped_records": self.skipped_records,
+            "flagged_rights": self.flagged_rights,
+            "convergence_passes": self.convergence_passes,
+            "response_evidence_digests": list(self.response_evidence_digests),
+            "page_shard_digests": list(self.page_shard_digests),
+            "flagged_queue_sha256": self.flagged_queue_sha256,
+        }
+        if self.rate_observation is not None or self.resume_not_before_epoch_seconds is not None:
+            result["rate_observation"] = self.rate_observation
+            result["resume_not_before_epoch_seconds"] = self.resume_not_before_epoch_seconds
+        if self.evidence_indexes is not None:
+            result["evidence_indexes"] = self.evidence_indexes
+            result["pass_map_sha256"] = self.pass_map_sha256
+        if self.normalized_bytes is not None:
+            result["normalized_bytes"] = self.normalized_bytes
+        if self.conflicting_identifiers is not None:
+            result["conflicting_identifiers"] = self.conflicting_identifiers
+        return result
+
+
+@dataclass(frozen=True)
+class CatalogFillResult:
+    status: str
+    reason_code: str | None
+    receipt: CatalogFillReceipt
+    checkpoint_sha256: str
+    eligible_staging_sha256: str | None
+
+
+def _run_datagov_v4_fill(
+    *,
+    config: CatalogFillConfig,
+    staging_root: Path,
+    retriever: Any,
+    authorization: DatagovV4Authorization,
+    expected_checkpoint_digest: str | None,
+    clock_ns: Callable[[], int],
+    wall_clock_seconds: Callable[[], int],
+    max_pages_this_invocation: int | None,
+    staging_descriptor: int | None,
+    pacing_sleep: Callable[[float], None] | None,
+) -> CatalogFillResult:
+    """Run v4 through compact index heads and partitioned pass maps only."""
+
+    staging = FillStaging(staging_root, root_descriptor=staging_descriptor)
+    with staging.locked():
+        state = staging.load()
+        if state is None:
+            if expected_checkpoint_digest is not None:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_MISMATCH",
+                    "fill.expected_checkpoint_digest",
+                    "no live checkpoint matches the caller's expected digest",
+                )
+            state = staging.commit(
+                _initial_v4_state(config, staging),
+                expected_generation=-1,
+                expected_checkpoint_digest=None,
+            )
+        else:
+            live_digest = staging.state_digest()
+            journal_predecessor = _v4_journal_predecessor(state)
+            handoff_predecessor = state.get("handoff_predecessor_sha256")
+            if (
+                not isinstance(expected_checkpoint_digest, str)
+                or not _SHA256.fullmatch(expected_checkpoint_digest)
+                or expected_checkpoint_digest
+                not in {live_digest, journal_predecessor, handoff_predecessor}
+            ):
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_MISMATCH",
+                    "fill.expected_checkpoint_digest",
+                    "resume requires the live or journal-authorized predecessor digest",
+                )
+            state = dict(state)
+            if expected_checkpoint_digest == live_digest:
+                state["handoff_predecessor_sha256"] = expected_checkpoint_digest
+            elif state.get("handoff_predecessor_sha256") is None:
+                state["handoff_predecessor_sha256"] = expected_checkpoint_digest
+        _validate_v4_state(state, config)
+        live_digest = staging.state_digest()
+        if _v4_cached_checkpoint_valid(staging, live_digest):
+            _validate_v4_tail_evidence(staging, state, config=config)
+        else:
+            _validate_v4_evidence(staging, state, config=config)
+            _remember_v4_checkpoint(staging, live_digest)
+        if state["reason_code"] in _V4_IDEMPOTENT_TERMINAL_REASONS:
+            return _v4_result(staging, state, config=config, reason=state["reason_code"])
+        if state["inflight_request"] is not None:
+            state, reconciled_reason = _reconcile_v4_inflight(
+                staging, state, config, clock_ns, wall_clock_seconds
+            )
+            if reconciled_reason is not None:
+                return _v4_result(staging, state, config=config, reason=reconciled_reason)
+        if state["completed"]:
+            return _v4_result(staging, state, config=config, reason=None)
+        paced_response_ns: int | None = None
+
+        def pace_in_process(deadline_epoch_seconds: int) -> None:
+            """Wait the politeness interval out here instead of handing it back to the caller.
+
+            The checkpoint has already been committed carrying this deadline, so a process that
+            dies inside this wait resumes exactly where a returned ``DATAGOV_PACING_REQUIRED``
+            would have: the durable journal does not know the difference.  What changes is who
+            waits, and therefore how many times the accumulated evidence tree is re-audited --
+            once per invocation rather than once per page.
+            """
+
+            assert pacing_sleep is not None
+            seconds = _pacing_sleep_seconds(
+                config,
+                deadline_epoch_seconds=deadline_epoch_seconds,
+                now_epoch_seconds=_wall_clock(wall_clock_seconds),
+                anchor_ns=paced_response_ns,
+                now_ns=_clock(clock_ns),
+            )
+            if seconds > 0:
+                pacing_sleep(seconds)
+
+        now = _wall_clock(wall_clock_seconds)
+        wait_until = state["resume_not_before_epoch_seconds"]
+        if state["pending_response"] is None and wait_until is not None and now < wait_until:
+            if pacing_sleep is None or state["wait_reason"] != "DATAGOV_PACING_REQUIRED":
+                return _v4_result(staging, state, config=config, reason=state["wait_reason"])
+            pace_in_process(wait_until)
+        invocation_pages = 0
+        seen_cursors, seen_responses = _v4_pass_loop_coordinates(staging, state)
+        while True:
+            if state["pending_response"] is None:
+                preflight = _preflight_limit(state, config.limits)
+                if preflight is not None:
+                    state = _v4_stop(staging, state, preflight)
+                    return _v4_result(staging, state, config=config, reason=preflight)
+                cursor = _cursor_from_dict(state["next_cursor"])
+                request_url = _request_url(config, cursor)
+                transport_limits = _remaining_transport_limits(
+                    state, config.limits, DATAGOV_V4_LIMITS
+                )
+                authorized_predecessor = staging.state_digest()
+                state = dict(state)
+                state["inflight_request"] = {
+                    "authorized_predecessor_sha256": authorized_predecessor,
+                    "attempt": state["budget"]["requests"] + 1,
+                    "request_url": request_url,
+                    "cursor": cursor.to_dict() if cursor is not None else None,
+                    "started": False,
+                    "started_ns": None,
+                    "response_observed_ns": None,
+                    "response_observed_epoch_seconds": None,
+                    "observed_response": None,
+                }
+                state["reason_code"] = None
+                state = _v4_commit(staging, state)
+
+                def mark_started() -> None:
+                    nonlocal state
+                    inflight = dict(state["inflight_request"])
+                    if not inflight["started"]:
+                        started_ns = _clock(clock_ns)
+                        inflight["started"] = True
+                        inflight["started_ns"] = started_ns
+                        state = dict(state)
+                        state["inflight_request"] = inflight
+                        state = _v4_commit(staging, state)
+
+                def retain_response(
+                    payload: bytes,
+                    evidence: EvidenceReference,
+                    response: HarvestResponseEvidence,
+                    _authorized_predecessor: str = authorized_predecessor,
+                ) -> None:
+                    nonlocal state
+                    if state["pending_response"] is not None:
+                        return
+                    inflight = state["inflight_request"]
+                    if not inflight["started"]:
+                        raise _v4_evidence_corrupt("response preceded request start")
+                    raw_digest = staging.write_blob("raw_response", payload)
+                    elapsed = _v4_elapsed_ns(
+                        inflight["started_ns"], inflight["response_observed_ns"]
+                    )
+                    state = _promote_v4_inflight(
+                        staging,
+                        state,
+                        elapsed_ns=elapsed,
+                        observed_epoch_seconds=inflight["response_observed_epoch_seconds"],
+                    )
+                    state = _append_v4_index(
+                        staging,
+                        state,
+                        "response",
+                        [
+                            {
+                                "request_attempt": inflight["attempt"],
+                                "request_url": inflight["request_url"],
+                                "observed_epoch_seconds": inflight[
+                                    "response_observed_epoch_seconds"
+                                ],
+                                "raw_response_sha256": raw_digest,
+                                "evidence": response.to_dict(),
+                            }
+                        ],
+                    )
+                    budget = dict(state["budget"])
+                    budget["responses"] += response.responses
+                    budget["network_bytes"] += response.network_bytes
+                    budget["elapsed_ns"] += elapsed
+                    state["budget"] = budget
+                    rate = response.rate_observation
+                    state["last_rate_observation"] = rate.to_dict() if rate else None
+                    wait_until, wait_reason = _v4_response_wait(
+                        config,
+                        response.status,
+                        rate,
+                        inflight["response_observed_epoch_seconds"],
+                    )
+                    state["resume_not_before_epoch_seconds"] = wait_until
+                    state["wait_reason"] = wait_reason
+                    state["pending_response"] = {
+                        "authorized_predecessor_sha256": _authorized_predecessor,
+                        "request_url": inflight["request_url"],
+                        "cursor": inflight["cursor"],
+                        "raw_response_sha256": raw_digest,
+                        "evidence": evidence.to_dict(),
+                        "response_evidence": response.to_dict(),
+                    }
+                    state = _v4_commit(staging, state)
+
+                def observe_response(
+                    response: HarvestResponseEvidence | DatagovRefusedResponseEvidence,
+                ) -> None:
+                    nonlocal state, paced_response_ns
+                    inflight = dict(state["inflight_request"])
+                    if not inflight["started"]:
+                        raise _v4_evidence_corrupt("response preceded request start")
+                    if response.requests != 1 or response.responses != 1:
+                        raise CatalogHarvestError(
+                            "DATAGOV_TRANSPORT_SHAPE",
+                            "harvest.response",
+                            "v4 transport evidence exceeded its one-attempt closure",
+                        )
+                    observed_ns = _clock(clock_ns)
+                    observed_epoch_seconds = _wall_clock(wall_clock_seconds)
+                    _v4_elapsed_ns(inflight["started_ns"], observed_ns)
+                    inflight["response_observed_ns"] = observed_ns
+                    inflight["response_observed_epoch_seconds"] = observed_epoch_seconds
+                    # The monotonic instant an in-process pace measures its interval from. The
+                    # durable checkpoint keeps only whole seconds; this keeps the fraction.
+                    paced_response_ns = observed_ns
+                    rate = response.rate_observation
+                    inflight["observed_response"] = {
+                        "status": response.status,
+                        "responses": response.responses,
+                        "content_bytes": response.content_bytes,
+                        "network_bytes": response.network_bytes,
+                        "transport_evidence_digest": getattr(
+                            response, "transport_evidence_digest", None
+                        ),
+                        "rate_observation": rate.to_dict() if rate else None,
+                    }
+                    state = dict(state)
+                    state["inflight_request"] = inflight
+                    state["last_rate_observation"] = rate.to_dict() if rate else None
+                    wait_until, wait_reason = _v4_response_wait(
+                        config, response.status, rate, observed_epoch_seconds
+                    )
+                    state["resume_not_before_epoch_seconds"] = wait_until
+                    state["wait_reason"] = wait_reason
+                    state = _v4_commit(staging, state)
+
+                try:
+                    fetched = fetch_datagov_v4_response(
+                        retriever=retriever,
+                        url=request_url,
+                        observed_at=config.observed_at,
+                        limits=transport_limits,
+                        authorization=authorization,
+                        on_request_started=mark_started,
+                        on_response_observed=observe_response,
+                        on_response_received=retain_response,
+                    )
+                except DatagovFetchRefused as error:
+                    response = error.response_evidence
+                    if not state["inflight_request"]["started"]:
+                        mark_started()
+                    inflight = state["inflight_request"]
+                    elapsed = _v4_elapsed_ns(
+                        inflight["started_ns"], inflight["response_observed_ns"]
+                    )
+                    state = _promote_v4_inflight(
+                        staging,
+                        state,
+                        elapsed_ns=elapsed,
+                        observed_epoch_seconds=inflight["response_observed_epoch_seconds"],
+                    )
+                    state = _append_v4_index(
+                        staging,
+                        state,
+                        "response",
+                        [
+                            {
+                                "request_attempt": inflight["attempt"],
+                                "request_url": inflight["request_url"],
+                                "observed_epoch_seconds": inflight[
+                                    "response_observed_epoch_seconds"
+                                ],
+                                "refusal": {
+                                    "code": error.code,
+                                    "status": response.status,
+                                    "content_bytes": response.content_bytes,
+                                    "network_bytes": response.network_bytes,
+                                    "rate_observation": (
+                                        response.rate_observation.to_dict()
+                                        if response.rate_observation
+                                        else None
+                                    ),
+                                },
+                            }
+                        ],
+                    )
+                    budget = dict(state["budget"])
+                    budget["responses"] += response.responses
+                    budget["network_bytes"] += response.network_bytes
+                    budget["elapsed_ns"] += elapsed
+                    state["budget"] = budget
+                    rate = response.rate_observation
+                    state["last_rate_observation"] = rate.to_dict() if rate else None
+                    wait_until, wait_reason = _v4_response_wait(
+                        config,
+                        response.status,
+                        rate,
+                        inflight["response_observed_epoch_seconds"],
+                    )
+                    state["resume_not_before_epoch_seconds"] = wait_until
+                    state["wait_reason"] = wait_reason
+                    reason = _v4_transport_terminal_reason(error.code, state, config.limits)
+                    state = _v4_stop(staging, state, reason)
+                    return _v4_result(staging, state, config=config, reason=reason)
+                except (CatalogHarvestError, AcquisitionSecurityError) as error:
+                    inflight = state["inflight_request"]
+                    if state["inflight_request"]["started"]:
+                        elapsed = _v4_elapsed_ns(inflight["started_ns"], _clock(clock_ns))
+                        observed_epoch_seconds = _wall_clock(wall_clock_seconds)
+                        state = _promote_v4_inflight(
+                            staging,
+                            state,
+                            elapsed_ns=elapsed,
+                            observed_epoch_seconds=observed_epoch_seconds,
+                        )
+                    else:
+                        state = dict(state)
+                        state["inflight_request"] = None
+                    budget = dict(state["budget"])
+                    if inflight["started"]:
+                        if isinstance(error, PeerAttemptError):
+                            budget["network_bytes"] += error.total_response_body_size_bytes
+                        budget["elapsed_ns"] += elapsed
+                        wait_until = _pacing_wait(config, observed_epoch_seconds)
+                        state["resume_not_before_epoch_seconds"] = wait_until
+                        state["wait_reason"] = (
+                            "DATAGOV_PACING_REQUIRED" if wait_until is not None else None
+                        )
+                    state["budget"] = budget
+                    reason = _v4_transport_terminal_reason(error.code, state, config.limits)
+                    state = _v4_stop(staging, state, reason)
+                    return _v4_result(staging, state, config=config, reason=reason)
+                if state["pending_response"] is None:
+                    retain_response(*fetched)
+                continue
+
+            pending = state["pending_response"]
+            response_wait_until = state["resume_not_before_epoch_seconds"]
+            request_url = pending["request_url"]
+            cursor = _cursor_from_dict(pending["cursor"])
+            raw_digest = pending["raw_response_sha256"]
+            payload = staging.read_blob("raw_response", raw_digest)
+            evidence = _evidence_from_dict(pending["evidence"])
+            response_evidence = _response_evidence_from_dict(pending["response_evidence"])
+            rate = response_evidence.rate_observation
+            late = _late_limit(state, config.limits, response_evidence)
+            if late is not None:
+                state = _clear_v4_pending_and_stop(staging, state, late)
+                return _v4_result(staging, state, config=config, reason=late)
+            try:
+                _validate_response_endpoint(config.endpoint, request_url, response_evidence)
+            except CatalogFillRefused as error:
+                state = _clear_v4_pending_and_stop(staging, state, error.code)
+                return _v4_result(staging, state, config=config, reason=error.code)
+            if response_evidence.status == 429:
+                state = _clear_v4_pending_and_stop(staging, state, "DATAGOV_RATE_LIMITED")
+                return _v4_result(staging, state, config=config, reason="DATAGOV_RATE_LIMITED")
+            if response_evidence.status == 403:
+                state = _clear_v4_pending_and_stop(staging, state, "DATAGOV_AUTHENTICATION_FAILED")
+                return _v4_result(
+                    staging, state, config=config, reason="DATAGOV_AUTHENTICATION_FAILED"
+                )
+            if response_evidence.status != 200:
+                state = _clear_v4_pending_and_stop(staging, state, "DATAGOV_HTTP_STATUS")
+                return _v4_result(staging, state, config=config, reason="DATAGOV_HTTP_STATUS")
+            if rate is None:
+                state = _clear_v4_pending_and_stop(staging, state, "DATAGOV_RATE_METADATA")
+                return _v4_result(staging, state, config=config, reason="DATAGOV_RATE_METADATA")
+
+            try:
+                page = DatagovV4Harvester().parse_page(
+                    payload,
+                    uri=request_url,
+                    observed_at=config.observed_at,
+                    evidence=evidence,
+                    response_evidence=response_evidence,
+                    cursor=cursor,
+                )
+            except CatalogHarvestError as error:
+                state = _clear_v4_pending_and_stop(staging, state, error.code)
+                return _v4_result(staging, state, config=config, reason=error.code)
+            requested_cursor = cursor.to_dict() if cursor is not None else None
+            requested_digest = canonical_sha256(requested_cursor)
+            next_cursor = page.next_cursor.to_dict() if page.next_cursor is not None else None
+            next_digest = canonical_sha256(next_cursor) if next_cursor is not None else None
+            if raw_digest in seen_responses:
+                state = _clear_v4_pending_and_stop(staging, state, "FILL_RESPONSE_LOOP")
+                return _v4_result(staging, state, config=config, reason="FILL_RESPONSE_LOOP")
+            if requested_digest in seen_cursors or (
+                next_cursor is not None
+                and (next_cursor == requested_cursor or next_digest in seen_cursors)
+            ):
+                state = _clear_v4_pending_and_stop(staging, state, "FILL_CURSOR_LOOP")
+                return _v4_result(staging, state, config=config, reason="FILL_CURSOR_LOOP")
+            projected = _projected_page_limit(state, config.limits, page)
+            if projected is not None:
+                state = _clear_v4_pending_and_stop(staging, state, projected)
+                return _v4_result(staging, state, config=config, reason=projected)
+            try:
+                normalized_digest, normalized_bytes = _write_v4_normalized_page(
+                    staging,
+                    raw_response_sha256=raw_digest,
+                    records=[record.to_dict() for record in page.records],
+                    skipped_record_ids=list(page.skipped_record_ids),
+                    maximum_bytes=min(
+                        DATAGOV_V4_MAX_NORMALIZED_PAGE_BYTES,
+                        config.limits.max_aggregate_bytes * DATAGOV_V4_MAX_NORMALIZED_EXPANSION
+                        - state["normalized_bytes"],
+                    ),
+                )
+            except CatalogFillRefused as error:
+                if error.code != "DATAGOV_NORMALIZED_LIMIT":
+                    raise
+                state = _clear_v4_pending_and_stop(staging, state, error.code)
+                return _v4_result(staging, state, config=config, reason=error.code)
+            if (
+                state["normalized_bytes"] + normalized_bytes
+                > config.limits.max_aggregate_bytes * DATAGOV_V4_MAX_NORMALIZED_EXPANSION
+            ):
+                state = _clear_v4_pending_and_stop(staging, state, "DATAGOV_NORMALIZED_LIMIT")
+                return _v4_result(staging, state, config=config, reason="DATAGOV_NORMALIZED_LIMIT")
+            run_descriptors = write_partition_runs(staging, page.records)
+            if run_descriptors:
+                partition_head = staging.append_index(
+                    state["partition_run_index"],
+                    stream="partition_run",
+                    items=run_descriptors,
+                )
+                state["partition_run_index"] = partition_head
+            state = _append_v4_index(
+                staging,
+                state,
+                "page",
+                [
+                    {
+                        "pass": state["pass_number"],
+                        "raw_response_sha256": raw_digest,
+                        "normalized_sha256": normalized_digest,
+                        "record_count": len(page.records),
+                    }
+                ],
+            )
+            if page.records:
+                state = _append_v4_index(
+                    staging,
+                    state,
+                    "record",
+                    [
+                        {
+                            "identifier_sha256": canonical_sha256(record.record_id),
+                            "semantic_sha256": record.digest,
+                            "normalized_page_sha256": normalized_digest,
+                        }
+                        for record in page.records
+                    ],
+                )
+            state = _append_v4_index(
+                staging,
+                state,
+                "cursor_transition",
+                [
+                    {
+                        "request_cursor": cursor.to_dict() if cursor is not None else None,
+                        "next_cursor": (
+                            page.next_cursor.to_dict() if page.next_cursor is not None else None
+                        ),
+                        "normalized_page_sha256": normalized_digest,
+                    }
+                ],
+            )
+            budget = dict(state["budget"])
+            budget["pages"] += 1
+            budget["records"] += len(page.records)
+            state["normalized_bytes"] += normalized_bytes
+            budget["skipped_records"] += len(page.skipped_record_ids)
+            state["budget"] = budget
+            state["next_cursor"] = page.next_cursor.to_dict() if page.next_cursor else None
+            state["reason_code"] = None
+            state["pending_response"] = None
+            seen_cursors.add(requested_digest)
+            seen_responses.add(raw_digest)
+            state["pass_cursor_set_sha256"] = getattr(
+                seen_cursors, "root_sha256", state["pass_cursor_set_sha256"]
+            )
+            state["pass_response_set_sha256"] = getattr(
+                seen_responses, "root_sha256", state["pass_response_set_sha256"]
+            )
+            invocation_pages += 1
+            pacing_wait = response_wait_until
+
+            if page.next_cursor is not None:
+                if pacing_wait is not None:
+                    state["resume_not_before_epoch_seconds"] = pacing_wait
+                    state["wait_reason"] = "DATAGOV_PACING_REQUIRED"
+                state = _v4_commit(staging, state)
+                if max_pages_this_invocation == invocation_pages:
+                    state = _v4_stop(staging, state, "INVOCATION_PAGE_LIMIT")
+                    return _v4_result(staging, state, config=config, reason="INVOCATION_PAGE_LIMIT")
+                if pacing_wait is not None:
+                    if pacing_sleep is None:
+                        state = _v4_stop(staging, state, "DATAGOV_PACING_REQUIRED")
+                        return _v4_result(
+                            staging, state, config=config, reason="DATAGOV_PACING_REQUIRED"
+                        )
+                    pace_in_process(pacing_wait)
+                continue
+
+            run_items = staging.iter_index(
+                state["partition_run_index"],
+                stream="partition_run",
+                start=state["pass_start_run_count"],
+            )
+            pass_map = external_merge_pass(staging, run_items)
+            state["last_pass_manifest_sha256"] = pass_map.manifest_sha256
+            stable = state["previous_pass_map_sha256"] == pass_map.root_sha256
+            state["conflicting_identifiers"] = pass_map.conflicting_identifiers
+            arbitrated = pass_map.conflicting_identifiers <= config.max_conflicting_identifiers
+            if stable and arbitrated:
+                state["completed"] = True
+                state["final_map_sha256"] = pass_map.root_sha256
+                state["unique_records"] = pass_map.unique_records
+                state["resume_not_before_epoch_seconds"] = None
+                state["wait_reason"] = None
+                state = _v4_commit(staging, state)
+                return _v4_result(staging, state, config=config, reason=None)
+            if state["pass_number"] >= config.max_convergence_passes:
+                state["final_map_sha256"] = pass_map.root_sha256
+                state["unique_records"] = pass_map.unique_records
+                state["resume_not_before_epoch_seconds"] = None
+                state["wait_reason"] = None
+                reason = "FILL_CONFLICTING_IDENTIFIERS" if stable else "FILL_NONCONVERGENT"
+                state = _v4_stop(staging, state, reason)
+                return _v4_result(staging, state, config=config, reason=reason)
+            state["previous_pass_map_sha256"] = pass_map.root_sha256
+            state["pass_number"] += 1
+            state["pass_start_run_count"] = state["partition_run_index"]["count"]
+            state["next_cursor"] = None
+            state["unique_records"] = pass_map.unique_records
+            seen_cursors.clear()
+            seen_responses.clear()
+            state["pass_cursor_set_sha256"] = None
+            state["pass_response_set_sha256"] = None
+            if pacing_wait is not None:
+                state["resume_not_before_epoch_seconds"] = pacing_wait
+                state["wait_reason"] = "DATAGOV_PACING_REQUIRED"
+            state = _v4_commit(staging, state)
+            if max_pages_this_invocation == invocation_pages:
+                state = _v4_stop(staging, state, "INVOCATION_PAGE_LIMIT")
+                return _v4_result(staging, state, config=config, reason="INVOCATION_PAGE_LIMIT")
+            if pacing_wait is not None:
+                if pacing_sleep is None:
+                    state = _v4_stop(staging, state, "DATAGOV_PACING_REQUIRED")
+                    return _v4_result(
+                        staging, state, config=config, reason="DATAGOV_PACING_REQUIRED"
+                    )
+                pace_in_process(pacing_wait)
+
+
+def _initial_v4_state(config: CatalogFillConfig, staging: FillStaging) -> dict[str, Any]:
+    state = {
+        "v4_checkpoint_schema_version": DATAGOV_V4_CHECKPOINT_SCHEMA,
+        "config_sha256": config.digest,
+        "endpoint": config.endpoint,
+        "harvester_coordinate": config.coordinate,
+        "predecessor_sha256": config.predecessor_sha256,
+        "next_cursor": None,
+        "budget": {
+            "requests": 0,
+            "responses": 0,
+            "pages": 0,
+            "records": 0,
+            "network_bytes": 0,
+            "elapsed_ns": 0,
+            "skipped_records": 0,
+        },
+        "provider_count": None,
+        "count_basis": "not_reported",
+        "pass_number": 1,
+        "previous_pass_map_sha256": None,
+        "pass_start_run_count": 0,
+        "partition_run_index": staging.empty_index_head(),
+        "evidence_indexes": {stream: staging.empty_index_head() for stream in _V4_EVIDENCE_STREAMS},
+        "last_rate_observation": None,
+        "resume_not_before_epoch_seconds": None,
+        "wait_reason": None,
+        "completed": False,
+        "reason_code": None,
+        "final_map_sha256": None,
+        "last_pass_manifest_sha256": None,
+        "unique_records": 0,
+        "conflicting_identifiers": 0,
+        "normalized_bytes": 0,
+        "inflight_request": None,
+        "pending_response": None,
+        "handoff_predecessor_sha256": None,
+        "validated_tail_sha256": None,
+        "pass_cursor_set_sha256": None,
+        "pass_response_set_sha256": None,
+    }
+    state["validated_tail_sha256"] = _v4_tail_seal(state)
+    return state
+
+
+def _validate_v4_state(state: dict[str, Any], config: CatalogFillConfig) -> None:
+    required = {
+        "v4_checkpoint_schema_version",
+        "config_sha256",
+        "endpoint",
+        "harvester_coordinate",
+        "predecessor_sha256",
+        "next_cursor",
+        "budget",
+        "provider_count",
+        "count_basis",
+        "pass_number",
+        "previous_pass_map_sha256",
+        "pass_start_run_count",
+        "partition_run_index",
+        "evidence_indexes",
+        "last_rate_observation",
+        "resume_not_before_epoch_seconds",
+        "wait_reason",
+        "completed",
+        "reason_code",
+        "final_map_sha256",
+        "last_pass_manifest_sha256",
+        "unique_records",
+        "conflicting_identifiers",
+        "normalized_bytes",
+        "inflight_request",
+        "pending_response",
+        "handoff_predecessor_sha256",
+        "validated_tail_sha256",
+        "pass_cursor_set_sha256",
+        "pass_response_set_sha256",
+        "schema_version",
+        "generation",
+    }
+    if set(state) != required:
+        if state.get("v4_checkpoint_schema_version") != DATAGOV_V4_CHECKPOINT_SCHEMA:
+            raise CatalogFillRefused(
+                "DATAGOV_CHECKPOINT_VERSION",
+                "fill.staging.state",
+                "in-progress v4 checkpoint predates the crash-exact schema",
+            )
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.state", "v4 state keys differ"
+        )
+    if state["v4_checkpoint_schema_version"] != DATAGOV_V4_CHECKPOINT_SCHEMA:
+        raise CatalogFillRefused(
+            "DATAGOV_CHECKPOINT_VERSION",
+            "fill.staging.state",
+            "v4 checkpoint schema is unsupported",
+        )
+    if not _SHA256.fullmatch(state["validated_tail_sha256"] or "") or state[
+        "validated_tail_sha256"
+    ] != _v4_tail_seal(state):
+        raise _v4_evidence_corrupt("validated tail seal differs from checkpoint coordinates")
+    budget = state["budget"]
+    budget_keys = {
+        "requests",
+        "responses",
+        "pages",
+        "records",
+        "network_bytes",
+        "elapsed_ns",
+        "skipped_records",
+    }
+    if (
+        type(state["completed"]) is not bool
+        or type(state["pass_number"]) is not int
+        or not 1 <= state["pass_number"] <= config.max_convergence_passes
+        or type(state["unique_records"]) is not int
+        or state["unique_records"] < 0
+        or type(state["normalized_bytes"]) is not int
+        or state["normalized_bytes"] < 0
+        or state["provider_count"] is not None
+        or state["count_basis"] != "not_reported"
+        or not isinstance(budget, dict)
+        or set(budget) != budget_keys
+        or any(type(budget[name]) is not int or budget[name] < 0 for name in budget_keys)
+        or (state["reason_code"] is not None and not isinstance(state["reason_code"], str))
+        or (state["wait_reason"] is not None and not isinstance(state["wait_reason"], str))
+        or (
+            state["resume_not_before_epoch_seconds"] is not None
+            and (
+                type(state["resume_not_before_epoch_seconds"]) is not int
+                or state["resume_not_before_epoch_seconds"] < 0
+                or state["resume_not_before_epoch_seconds"] > _MAX_SAFE_EPOCH_SECONDS
+                or state["wait_reason"] is None
+            )
+        )
+        or (state["wait_reason"] is not None and state["resume_not_before_epoch_seconds"] is None)
+    ):
+        raise _v4_evidence_corrupt("v4 scalar state is invalid")
+    transport_overrun = (
+        budget["requests"] > config.limits.max_requests
+        or budget["responses"] > config.limits.max_responses
+        or budget["network_bytes"] > config.limits.max_aggregate_bytes
+        or budget["elapsed_ns"] > config.limits.max_wall_time_ns
+    )
+    if (
+        budget["pages"] > config.limits.max_pages
+        or budget["records"] > config.limits.max_records
+        or state["normalized_bytes"]
+        > config.limits.max_aggregate_bytes * DATAGOV_V4_MAX_NORMALIZED_EXPANSION
+        or state["unique_records"] > budget["records"]
+        or (
+            transport_overrun
+            and not (
+                (
+                    state["reason_code"] is None
+                    and not state["completed"]
+                    and state["inflight_request"] is None
+                    and state["pending_response"] is not None
+                )
+                or (
+                    state["reason_code"] in _V4_POST_RESPONSE_OVERRUN_REASONS
+                    and not state["completed"]
+                    and state["inflight_request"] is None
+                    and state["pending_response"] is None
+                )
+            )
+        )
+    ):
+        raise _v4_evidence_corrupt("v4 budget exceeds configuration")
+    if state["reason_code"] in _V4_IDEMPOTENT_TERMINAL_REASONS and (
+        state["inflight_request"] is not None or state["pending_response"] is not None
+    ):
+        raise _v4_evidence_corrupt("terminal stop retains an unresolved journal")
+    rate_value = state["last_rate_observation"]
+    try:
+        if rate_value is not None:
+            if not isinstance(rate_value, dict):
+                raise TypeError
+            DatagovRateObservation(**rate_value)
+    except (AcquisitionSecurityError, TypeError):
+        raise _v4_evidence_corrupt("last rate observation is invalid") from None
+    if state["handoff_predecessor_sha256"] is not None and not _SHA256.fullmatch(
+        state["handoff_predecessor_sha256"]
+    ):
+        raise _v4_evidence_corrupt("handoff predecessor is invalid")
+    for coordinate in (
+        "previous_pass_map_sha256",
+        "final_map_sha256",
+        "last_pass_manifest_sha256",
+        "pass_cursor_set_sha256",
+        "pass_response_set_sha256",
+    ):
+        if state[coordinate] is not None and not _SHA256.fullmatch(state[coordinate]):
+            raise _v4_evidence_corrupt(f"{coordinate} is invalid")
+    if (
+        state["config_sha256"] != config.digest
+        or state["endpoint"] != config.endpoint
+        or state["harvester_coordinate"] != config.coordinate
+        or state["predecessor_sha256"] != config.predecessor_sha256
+    ):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CONFIG", "fill.staging.state", "checkpoint coordinate differs"
+        )
+    next_cursor = _cursor_from_dict(state["next_cursor"])
+    indexes = state["evidence_indexes"]
+    if not isinstance(indexes, dict) or set(indexes) != set(_V4_EVIDENCE_STREAMS):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.indexes", "evidence indexes differ"
+        )
+    if any(
+        not isinstance(indexes[name], dict) or type(indexes[name].get("count")) is not int
+        for name in _V4_EVIDENCE_STREAMS
+    ):
+        raise _v4_evidence_corrupt("evidence index head is invalid")
+    if not isinstance(state["partition_run_index"], dict):
+        raise _v4_evidence_corrupt("partition index head is invalid")
+    expected_counts = {
+        "request": budget["requests"],
+        "response": budget["responses"],
+        "page": budget["pages"],
+        "record": budget["records"],
+        "cursor_transition": budget["pages"],
+    }
+    if any(indexes[name]["count"] != count for name, count in expected_counts.items()):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.indexes", "evidence counts differ"
+        )
+    if state["completed"] and (
+        state["pass_number"] < 2
+        or state["final_map_sha256"] is None
+        or state["final_map_sha256"] != state["previous_pass_map_sha256"]
+        or state["last_pass_manifest_sha256"] is None
+        or state["next_cursor"] is not None
+        or state["inflight_request"] is not None
+        or state["pending_response"] is not None
+        or state["resume_not_before_epoch_seconds"] is not None
+        or state["wait_reason"] is not None
+        or state["reason_code"] is not None
+    ):
+        raise _v4_evidence_corrupt("completed state lacks converged terminal coordinates")
+    if (
+        not state["completed"]
+        and state["final_map_sha256"] is not None
+        and (
+            state["reason_code"] not in _V4_MAP_SEALING_REASONS
+            or state["pass_number"] != config.max_convergence_passes
+            or state["last_pass_manifest_sha256"] is None
+            or state["next_cursor"] is not None
+            or state["inflight_request"] is not None
+            or state["pending_response"] is not None
+            or state["resume_not_before_epoch_seconds"] is not None
+            or state["wait_reason"] is not None
+        )
+    ):
+        raise _v4_evidence_corrupt("map-sealing state lacks terminal max-pass coordinates")
+    if state["reason_code"] in _V4_MAP_SEALING_REASONS and state["final_map_sha256"] is None:
+        raise _v4_evidence_corrupt("map-sealing state lacks its final pass map")
+    if (
+        type(state["pass_start_run_count"]) is not int
+        or not 0 <= state["pass_start_run_count"] <= state["partition_run_index"]["count"]
+    ):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.partition", "pass run offset is invalid"
+        )
+    inflight = state["inflight_request"]
+    pending = state["pending_response"]
+    if inflight is not None and pending is not None:
+        raise _v4_evidence_corrupt("request and response journals overlap")
+    if inflight is not None:
+        if (
+            not isinstance(inflight, dict)
+            or set(inflight)
+            != {
+                "authorized_predecessor_sha256",
+                "attempt",
+                "request_url",
+                "cursor",
+                "started",
+                "started_ns",
+                "response_observed_ns",
+                "response_observed_epoch_seconds",
+                "observed_response",
+            }
+            or not _SHA256.fullmatch(inflight["authorized_predecessor_sha256"] or "")
+            or inflight["attempt"] != budget["requests"] + 1
+            or not _valid_v4_request_url(
+                inflight["request_url"], page_size=config.page_size, sort=config.sort
+            )
+            or type(inflight["started"]) is not bool
+            or (
+                inflight["started"]
+                and (type(inflight["started_ns"]) is not int or inflight["started_ns"] < 0)
+            )
+            or (not inflight["started"] and inflight["started_ns"] is not None)
+            or (
+                inflight["response_observed_ns"] is not None
+                and (
+                    type(inflight["response_observed_ns"]) is not int
+                    or not inflight["started"]
+                    or inflight["response_observed_ns"] < inflight["started_ns"]
+                )
+            )
+            or (
+                inflight["response_observed_epoch_seconds"] is not None
+                and (
+                    type(inflight["response_observed_epoch_seconds"]) is not int
+                    or inflight["response_observed_epoch_seconds"] < 0
+                    or inflight["response_observed_epoch_seconds"] > _MAX_SAFE_EPOCH_SECONDS
+                    or inflight["response_observed_ns"] is None
+                )
+            )
+        ):
+            raise _v4_evidence_corrupt("inflight request journal is invalid")
+        try:
+            inflight_cursor = _cursor_from_dict(inflight["cursor"])
+            _sort, _page_size, request_cursor = _parse_v4_request_url(inflight["request_url"])
+        except (CatalogHarvestError, TypeError, ValueError):
+            raise _v4_evidence_corrupt("inflight cursor is invalid") from None
+        if inflight_cursor != request_cursor or inflight_cursor != next_cursor:
+            raise _v4_evidence_corrupt("inflight cursor differs from request/state")
+        observed = inflight["observed_response"]
+        if (observed is None) != (inflight["response_observed_ns"] is None) or (
+            observed is None
+        ) != (inflight["response_observed_epoch_seconds"] is None):
+            raise _v4_evidence_corrupt("response observation timing differs")
+        if observed is not None and (
+            not isinstance(observed, dict)
+            or set(observed)
+            != {
+                "status",
+                "responses",
+                "content_bytes",
+                "network_bytes",
+                "transport_evidence_digest",
+                "rate_observation",
+            }
+            or type(observed["status"]) is not int
+            or not 100 <= observed["status"] <= 599
+            or observed["responses"] != 1
+            or type(observed["content_bytes"]) is not int
+            or type(observed["network_bytes"]) is not int
+            or observed["content_bytes"] < 0
+            or observed["network_bytes"] < observed["content_bytes"]
+            or (
+                observed["transport_evidence_digest"] is not None
+                and not _SHA256.fullmatch(observed["transport_evidence_digest"])
+            )
+        ):
+            raise _v4_evidence_corrupt("observed response journal is invalid")
+        if observed is not None:
+            try:
+                rate_value = observed["rate_observation"]
+                if rate_value is not None:
+                    if not isinstance(rate_value, dict):
+                        raise TypeError
+                    DatagovRateObservation(**rate_value)
+            except (AcquisitionSecurityError, TypeError):
+                raise _v4_evidence_corrupt("observed response rate is invalid") from None
+    if pending is not None:
+        if (
+            not isinstance(pending, dict)
+            or set(pending)
+            != {
+                "authorized_predecessor_sha256",
+                "request_url",
+                "cursor",
+                "raw_response_sha256",
+                "evidence",
+                "response_evidence",
+            }
+            or not _SHA256.fullmatch(pending["authorized_predecessor_sha256"] or "")
+            or not _SHA256.fullmatch(pending["raw_response_sha256"] or "")
+            or not _valid_v4_request_url(
+                pending["request_url"], page_size=config.page_size, sort=config.sort
+            )
+        ):
+            raise _v4_evidence_corrupt("pending response journal is invalid")
+        try:
+            pending_cursor = _cursor_from_dict(pending["cursor"])
+            _sort, _page_size, request_cursor = _parse_v4_request_url(pending["request_url"])
+            evidence = _evidence_from_dict(pending["evidence"])
+            parsed = _response_evidence_from_dict(pending["response_evidence"])
+        except (CatalogHarvestError, AcquisitionSecurityError, TypeError, ValueError):
+            raise _v4_evidence_corrupt("pending response contract is invalid") from None
+        if (
+            pending_cursor != request_cursor
+            or pending_cursor != next_cursor
+            or evidence.uri != parsed.final_url
+            or evidence.content_sha256 != pending["raw_response_sha256"]
+            or evidence.media_type != "application/json"
+            or parsed.content_sha256 != pending["raw_response_sha256"]
+            or parsed.media_type != "application/json"
+            or parsed.requests != 1
+            or parsed.responses != 1
+            or parsed.network_bytes != parsed.content_bytes
+        ):
+            raise _v4_evidence_corrupt("pending response transport closure differs")
+
+
+def _append_v4_index(
+    staging: FillStaging,
+    state: dict[str, Any],
+    stream: str,
+    items: list[dict[str, Any]],
+) -> dict[str, Any]:
+    if not items:
+        return state
+    next_state = dict(state)
+    indexes = dict(state["evidence_indexes"])
+    indexes[stream] = staging.append_index(indexes[stream], stream=stream, items=items)
+    next_state["evidence_indexes"] = indexes
+    return next_state
+
+
+def _write_v4_normalized_page(
+    staging: FillStaging,
+    *,
+    raw_response_sha256: str,
+    records: list[dict[str, Any]],
+    skipped_record_ids: list[str],
+    maximum_bytes: int = DATAGOV_V4_MAX_NORMALIZED_PAGE_BYTES,
+) -> tuple[str, int]:
+    """Install one bounded page manifest over adaptively split normalized record shards."""
+
+    normalized_bytes = 0
+    planned_segments: list[tuple[dict[str, Any], str]] = []
+
+    def plan_segment(first_record: int, values: list[dict[str, Any]]) -> list[dict[str, Any]]:
+        nonlocal normalized_bytes
+        payload = {
+            "schema_version": "harness-datagov-v4-normalized-record-segment.v1",
+            "raw_response_sha256": raw_response_sha256,
+            "first_record": first_record,
+            "records": values,
+        }
+        try:
+            raw = _v4_normalized_shard_bytes(payload)
+        except CanonicalJSONError:
+            if len(values) <= 1:
+                raise CatalogFillRefused(
+                    "DATAGOV_NORMALIZED_LIMIT",
+                    "fill.normalized",
+                    "one normalized record exceeds the canonical shard contract",
+                ) from None
+            midpoint = len(values) // 2
+            return plan_segment(first_record, values[:midpoint]) + plan_segment(
+                first_record + midpoint, values[midpoint:]
+            )
+        if len(raw) > shard_byte_limit("normalized"):
+            if len(values) <= 1:
+                raise CatalogFillRefused(
+                    "DATAGOV_NORMALIZED_LIMIT",
+                    "fill.normalized",
+                    "one normalized record exceeds the shard byte contract",
+                )
+            midpoint = len(values) // 2
+            return plan_segment(first_record, values[:midpoint]) + plan_segment(
+                first_record + midpoint, values[midpoint:]
+            )
+        if normalized_bytes + len(raw) > maximum_bytes:
+            raise CatalogFillRefused(
+                "DATAGOV_NORMALIZED_LIMIT",
+                "fill.normalized",
+                "normalized page exceeds its aggregate staging byte budget",
+            )
+        digest = sha256_bytes(raw)
+        normalized_bytes += len(raw)
+        planned_segments.append((payload, digest))
+        return [{"sha256": digest, "first_record": first_record, "record_count": len(values)}]
+
+    segments: list[dict[str, Any]] = []
+    for first_record in range(0, len(records), 64):
+        segments.extend(plan_segment(first_record, records[first_record : first_record + 64]))
+    manifest = {
+        "schema_version": "harness-datagov-v4-normalized-page.v2",
+        "raw_response_sha256": raw_response_sha256,
+        "record_count": len(records),
+        "record_segments": segments,
+        "skipped_record_ids": skipped_record_ids,
+    }
+    manifest_raw = _v4_normalized_shard_bytes(manifest)
+    if (
+        len(manifest_raw) > shard_byte_limit("normalized")
+        or normalized_bytes + len(manifest_raw) > maximum_bytes
+    ):
+        raise CatalogFillRefused(
+            "DATAGOV_NORMALIZED_LIMIT",
+            "fill.normalized",
+            "normalized page exceeds its aggregate staging byte budget",
+        )
+    for payload, expected_digest in planned_segments:
+        if staging.write_shard("normalized", payload) != expected_digest:
+            raise _v4_evidence_corrupt("normalized segment write digest differs")
+    digest = staging.write_shard("normalized", manifest)
+    return digest, normalized_bytes + len(manifest_raw)
+
+
+def _v4_normalized_shard_bytes(payload: dict[str, Any]) -> bytes:
+    return canonical_json_bytes(
+        {"schema_version": SHARD_SCHEMA, "kind": "normalized", "payload": payload}
+    )
+
+
+def _v4_normalized_shard_size(payload: dict[str, Any]) -> int:
+    return len(_v4_normalized_shard_bytes(payload))
+
+
+def _v4_tail_seal(state: dict[str, Any]) -> str:
+    """Bind the producer-validated immutable tails without self-reference."""
+
+    excluded = {
+        "generation",
+        "schema_version",
+        "handoff_predecessor_sha256",
+        "validated_tail_sha256",
+    }
+    return canonical_sha256({key: value for key, value in state.items() if key not in excluded})
+
+
+def _v4_cache_key(staging: FillStaging) -> str:
+    return str(staging.root.resolve(strict=False))
+
+
+def _v4_cached_checkpoint_valid(staging: FillStaging, digest: str) -> bool:
+    """Trust an ordinary resume only while the retained object tree is unchanged."""
+
+    key = _v4_cache_key(staging)
+    cached = _V4_VALIDATED_CHECKPOINTS.get(key)
+    if cached is None or cached[0] != digest or cached[1] != staging.integrity_epoch():
+        _V4_VALIDATED_CHECKPOINTS.pop(key, None)
+        return False
+    return True
+
+
+def _remember_v4_checkpoint(staging: FillStaging, digest: str) -> None:
+    """Remember one fully or producer-validated checkpoint for this process only."""
+
+    key = _v4_cache_key(staging)
+    _V4_VALIDATED_CHECKPOINTS.pop(key, None)
+    _V4_VALIDATED_CHECKPOINTS[key] = (digest, staging.integrity_epoch())
+    while len(_V4_VALIDATED_CHECKPOINTS) > _V4_VALIDATED_CACHE_LIMIT:
+        del _V4_VALIDATED_CHECKPOINTS[next(iter(_V4_VALIDATED_CHECKPOINTS))]
+
+
+def _v4_commit(staging: FillStaging, state: dict[str, Any]) -> dict[str, Any]:
+    next_state = dict(state)
+    next_state["validated_tail_sha256"] = _v4_tail_seal(next_state)
+    return staging.commit(
+        next_state,
+        expected_generation=state["generation"],
+        expected_checkpoint_digest=staging.state_digest(),
+    )
+
+
+def _v4_journal_predecessor(state: dict[str, Any]) -> str | None:
+    for name in ("pending_response", "inflight_request"):
+        journal = state.get(name)
+        if isinstance(journal, dict):
+            value = journal.get("authorized_predecessor_sha256")
+            if isinstance(value, str):
+                return value
+    return None
+
+
+def _v4_elapsed_ns(started_ns: Any, ended_ns: Any) -> int:
+    if (
+        type(started_ns) is not int
+        or type(ended_ns) is not int
+        or started_ns < 0
+        or ended_ns < started_ns
+    ):
+        raise CatalogFillRefused(
+            "DATAGOV_CLOCK_DISCONTINUITY",
+            "fill.clock_ns",
+            "monotonic request timing moved backward or changed origin",
+        )
+    return ended_ns - started_ns
+
+
+def _promote_v4_inflight(
+    staging: FillStaging,
+    state: dict[str, Any],
+    *,
+    elapsed_ns: int,
+    observed_epoch_seconds: int,
+) -> dict[str, Any]:
+    inflight = state.get("inflight_request")
+    if not isinstance(inflight, dict) or not inflight.get("started"):
+        raise _v4_evidence_corrupt("started request journal is absent")
+    next_state = _append_v4_index(
+        staging,
+        dict(state),
+        "request",
+        [
+            {
+                "attempt": inflight["attempt"],
+                "request_url": inflight["request_url"],
+                "cursor": inflight["cursor"],
+                "elapsed_ns": elapsed_ns,
+                "observed_epoch_seconds": observed_epoch_seconds,
+            }
+        ],
+    )
+    budget = dict(next_state["budget"])
+    budget["requests"] += 1
+    next_state["budget"] = budget
+    next_state["inflight_request"] = None
+    return next_state
+
+
+def _reconcile_v4_inflight(
+    staging: FillStaging,
+    state: dict[str, Any],
+    config: CatalogFillConfig,
+    clock_ns: Callable[[], int],
+    wall_clock_seconds: Callable[[], int],
+) -> tuple[dict[str, Any], str | None]:
+    inflight = state["inflight_request"]
+    if not inflight["started"]:
+        state = dict(state)
+        state["inflight_request"] = None
+        return _v4_commit(staging, state), None
+    observed = inflight["observed_response"]
+    ended_ns = _clock(clock_ns) if observed is None else inflight["response_observed_ns"]
+    observed_epoch_seconds = (
+        _wall_clock(wall_clock_seconds)
+        if observed is None
+        else inflight["response_observed_epoch_seconds"]
+    )
+    elapsed_ns = _v4_elapsed_ns(inflight["started_ns"], ended_ns)
+    state = _promote_v4_inflight(
+        staging,
+        state,
+        elapsed_ns=elapsed_ns,
+        observed_epoch_seconds=observed_epoch_seconds,
+    )
+    if observed is None:
+        budget = dict(state["budget"])
+        budget["elapsed_ns"] += elapsed_ns
+        state["budget"] = budget
+        state["resume_not_before_epoch_seconds"] = _pacing_wait(config, observed_epoch_seconds)
+        state["wait_reason"] = (
+            "DATAGOV_PACING_REQUIRED"
+            if state["resume_not_before_epoch_seconds"] is not None
+            else None
+        )
+        reason = "DATAGOV_REQUEST_OUTCOME_UNKNOWN"
+        return _v4_stop(staging, state, reason), reason
+    reason = "DATAGOV_RESPONSE_SCAN_INTERRUPTED"
+    redacted = {
+        key: value
+        for key, value in observed.items()
+        if key not in {"responses", "transport_evidence_digest"}
+    }
+    redacted["code"] = reason
+    redacted["content_bytes"] = 0
+    state = _append_v4_index(
+        staging,
+        state,
+        "response",
+        [
+            {
+                "request_attempt": inflight["attempt"],
+                "request_url": inflight["request_url"],
+                "observed_epoch_seconds": observed_epoch_seconds,
+                "refusal": redacted,
+            }
+        ],
+    )
+    budget = dict(state["budget"])
+    budget["responses"] += observed["responses"]
+    budget["network_bytes"] += observed["network_bytes"]
+    budget["elapsed_ns"] += elapsed_ns
+    state["budget"] = budget
+    rate_value = observed["rate_observation"]
+    rate = DatagovRateObservation(**rate_value) if isinstance(rate_value, dict) else None
+    state["last_rate_observation"] = rate_value
+    wait_until, wait_reason = _v4_response_wait(
+        config, observed["status"], rate, observed_epoch_seconds
+    )
+    state["resume_not_before_epoch_seconds"] = wait_until
+    state["wait_reason"] = wait_reason
+    return _v4_stop(staging, state, reason), reason
+
+
+def _clear_v4_pending_and_stop(
+    staging: FillStaging, state: dict[str, Any], reason: str
+) -> dict[str, Any]:
+    state = dict(state)
+    state["pending_response"] = None
+    return _v4_stop(staging, state, reason)
+
+
+def _evidence_from_dict(value: Any) -> EvidenceReference:
+    if not isinstance(value, dict) or set(value) != {
+        "uri",
+        "observed_at",
+        "content_sha256",
+        "media_type",
+    }:
+        raise _v4_evidence_corrupt("pending evidence is invalid")
+    return EvidenceReference(**value)
+
+
+def _response_evidence_from_dict(value: Any) -> HarvestResponseEvidence:
+    if not isinstance(value, dict):
+        raise _v4_evidence_corrupt("pending response evidence is invalid")
+    payload = dict(value)
+    rate_value = payload.pop("rate_observation", None)
+    status = payload.pop("status", 200)
+    if rate_value is not None and not isinstance(rate_value, dict):
+        raise _v4_evidence_corrupt("pending response rate observation is invalid")
+    rate = DatagovRateObservation(**rate_value) if rate_value is not None else None
+    return HarvestResponseEvidence(**payload, status=status, rate_observation=rate)
+
+
+def _v4_stop(staging: FillStaging, state: dict[str, Any], reason: str) -> dict[str, Any]:
+    if state.get("reason_code") == reason:
+        return state
+    next_state = dict(state)
+    next_state["reason_code"] = reason
+    return _v4_commit(staging, next_state)
+
+
+def _v4_result(
+    staging: FillStaging,
+    state: dict[str, Any],
+    *,
+    config: CatalogFillConfig,
+    reason: str | None,
+) -> CatalogFillResult:
+    if state["completed"] or reason in _V4_MAP_SEALING_REASONS:
+        _validate_v4_evidence(staging, state, config=config)
+    receipt = CatalogFillReceipt(
+        config_sha256=state["config_sha256"],
+        endpoint=state["endpoint"],
+        harvester_coordinate=state["harvester_coordinate"],
+        predecessor_sha256=state["predecessor_sha256"],
+        status="complete" if state["completed"] else "incomplete",
+        reason_code=reason,
+        requests=state["budget"]["requests"],
+        responses=state["budget"]["responses"],
+        pages=state["budget"]["pages"],
+        records=state["budget"]["records"],
+        unique_records=state["unique_records"],
+        network_bytes=state["budget"]["network_bytes"],
+        elapsed_ns=state["budget"]["elapsed_ns"],
+        provider_count=None,
+        count_basis="not_reported",
+        skipped_records=state["budget"]["skipped_records"],
+        flagged_rights=0,
+        convergence_passes=state["pass_number"],
+        response_evidence_digests=(),
+        page_shard_digests=(),
+        flagged_queue_sha256=canonical_sha256([]),
+        rate_observation=state["last_rate_observation"],
+        resume_not_before_epoch_seconds=state["resume_not_before_epoch_seconds"],
+        evidence_indexes=state["evidence_indexes"],
+        pass_map_sha256=state["final_map_sha256"] or state["previous_pass_map_sha256"],
+        normalized_bytes=state["normalized_bytes"],
+        conflicting_identifiers=state["conflicting_identifiers"],
+    )
+    digest = staging.state_digest()
+    _remember_v4_checkpoint(staging, digest)
+    return CatalogFillResult(
+        status=receipt.status,
+        reason_code=reason,
+        receipt=receipt,
+        checkpoint_sha256=digest,
+        eligible_staging_sha256=digest if state["completed"] else None,
+    )
+
+
+def _validate_v4_tail_evidence(
+    staging: FillStaging,
+    state: dict[str, Any],
+    *,
+    config: CatalogFillConfig,
+) -> None:
+    """Authenticate only current immutable tails on ordinary resumptions."""
+
+    indexes = state["evidence_indexes"]
+    tails = {
+        stream: staging.index_tail_items(indexes[stream], stream=stream)
+        for stream in _V4_EVIDENCE_STREAMS
+    }
+    staging.index_tail_items(state["partition_run_index"], stream="partition_run")
+    request_tail = tails["request"][-1] if tails["request"] else None
+    if state["budget"]["requests"]:
+        if (
+            not isinstance(request_tail, dict)
+            or request_tail.get("attempt") != state["budget"]["requests"]
+            or not _valid_v4_request_url(
+                request_tail.get("request_url"), page_size=config.page_size, sort=config.sort
+            )
+            or type(request_tail.get("observed_epoch_seconds")) is not int
+            or not 0 <= request_tail["observed_epoch_seconds"] <= _MAX_SAFE_EPOCH_SECONDS
+        ):
+            raise _v4_evidence_corrupt("request tail differs from checkpoint budget")
+    response_tail = tails["response"][-1] if tails["response"] else None
+    if response_tail is not None and (
+        type(response_tail.get("request_attempt")) is not int
+        or not 1 <= response_tail["request_attempt"] <= state["budget"]["requests"]
+        or type(response_tail.get("observed_epoch_seconds")) is not int
+        or not 0 <= response_tail["observed_epoch_seconds"] <= _MAX_SAFE_EPOCH_SECONDS
+    ):
+        raise _v4_evidence_corrupt("response tail is invalid")
+    page_tail = tails["page"][-1] if tails["page"] else None
+    cursor_tail = tails["cursor_transition"][-1] if tails["cursor_transition"] else None
+    if state["budget"]["pages"] and (
+        not isinstance(page_tail, dict)
+        or not isinstance(cursor_tail, dict)
+        or page_tail.get("normalized_sha256") != cursor_tail.get("normalized_page_sha256")
+        or cursor_tail.get("next_cursor") != state["next_cursor"]
+    ):
+        raise _v4_evidence_corrupt("page and cursor tails differ")
+    if page_tail is not None:
+        observed_records = 0
+        try:
+            for _record in iter_v4_normalized_records(
+                staging,
+                page_tail["normalized_sha256"],
+                raw_response_sha256=page_tail["raw_response_sha256"],
+                record_count=page_tail["record_count"],
+            ):
+                observed_records += 1
+        except (CatalogFillRefused, CanonicalJSONError, TypeError, ValueError):
+            raise _v4_evidence_corrupt("normalized page tail is invalid") from None
+        if observed_records != page_tail["record_count"]:
+            raise _v4_evidence_corrupt("normalized page tail count differs")
+        if page_tail.get("pass") == state["pass_number"]:
+            requested_digest = canonical_sha256(cursor_tail["request_cursor"])
+            if not _v4_loop_contains(
+                staging,
+                state["pass_cursor_set_sha256"],
+                "cursor",
+                requested_digest,
+            ) or not _v4_loop_contains(
+                staging,
+                state["pass_response_set_sha256"],
+                "response",
+                page_tail["raw_response_sha256"],
+            ):
+                raise _v4_evidence_corrupt("current pass loop-set omits its page tail")
+    for set_kind, coordinate in (
+        ("cursor", state["pass_cursor_set_sha256"]),
+        ("response", state["pass_response_set_sha256"]),
+    ):
+        if coordinate is not None:
+            _v4_loop_node(staging, coordinate, set_kind=set_kind, depth=0, prefix="")
+    pending = state["pending_response"]
+    if pending is not None:
+        try:
+            response = _response_evidence_from_dict(pending["response_evidence"])
+        except (CatalogHarvestError, AcquisitionSecurityError, TypeError, ValueError):
+            raise _v4_evidence_corrupt("pending response tail is invalid") from None
+        if (
+            not isinstance(response_tail, dict)
+            or response_tail.get("request_attempt") != state["budget"]["requests"]
+            or response_tail.get("request_url") != pending["request_url"]
+            or response_tail.get("raw_response_sha256") != pending["raw_response_sha256"]
+            or response_tail.get("evidence") != response.to_dict()
+        ):
+            raise _v4_evidence_corrupt("pending response differs from response tail")
+    latest_status: int | None = None
+    latest_rate: DatagovRateObservation | None = None
+    latest_epoch: int | None = None
+    latest_attempt = 0
+    latest_full: HarvestResponseEvidence | None = None
+    latest_refusal_code: str | None = None
+    if response_tail is not None:
+        latest_attempt = response_tail["request_attempt"]
+        latest_epoch = response_tail["observed_epoch_seconds"]
+        if "evidence" in response_tail:
+            try:
+                latest_full = _response_evidence_from_dict(response_tail["evidence"])
+            except (CatalogHarvestError, AcquisitionSecurityError, TypeError, ValueError):
+                raise _v4_evidence_corrupt("full response tail is invalid") from None
+            latest_status = latest_full.status
+            latest_rate = latest_full.rate_observation
+        elif "refusal" in response_tail and isinstance(response_tail["refusal"], dict):
+            refusal = response_tail["refusal"]
+            latest_refusal_code = refusal.get("code")
+            latest_status = refusal.get("status")
+            rate_value = refusal.get("rate_observation")
+            try:
+                latest_rate = (
+                    DatagovRateObservation(**rate_value) if rate_value is not None else None
+                )
+            except (AcquisitionSecurityError, TypeError):
+                raise _v4_evidence_corrupt("refusal response tail is invalid") from None
+        else:
+            raise _v4_evidence_corrupt("response tail shape is invalid")
+    inflight = state.get("inflight_request")
+    observed = inflight.get("observed_response") if isinstance(inflight, dict) else None
+    if isinstance(observed, dict):
+        latest_attempt = inflight["attempt"]
+        latest_status = observed["status"]
+        rate_value = observed["rate_observation"]
+        try:
+            latest_rate = DatagovRateObservation(**rate_value) if rate_value is not None else None
+        except (AcquisitionSecurityError, TypeError):
+            raise _v4_evidence_corrupt("inflight response rate is invalid") from None
+        latest_epoch = inflight["response_observed_epoch_seconds"]
+    expected_wait: tuple[int | None, str | None] = (None, None)
+    try:
+        if latest_epoch is not None and latest_attempt == (
+            inflight["attempt"]
+            if isinstance(inflight, dict) and observed is not None
+            else state["budget"]["requests"]
+        ):
+            expected_wait = _v4_response_wait(config, latest_status, latest_rate, latest_epoch)
+        elif request_tail is not None:
+            wait_until = _pacing_wait(config, request_tail["observed_epoch_seconds"])
+            expected_wait = (
+                wait_until,
+                "DATAGOV_PACING_REQUIRED" if wait_until is not None else None,
+            )
+    except CatalogFillRefused:
+        raise _v4_evidence_corrupt("tail observation time cannot produce a safe wait") from None
+    if state["completed"] or state["reason_code"] in _V4_MAP_SEALING_REASONS:
+        expected_wait = (None, None)
+    if (
+        state["resume_not_before_epoch_seconds"],
+        state["wait_reason"],
+    ) != expected_wait:
+        raise _v4_evidence_corrupt("durable wait differs from authenticated tail")
+    reason = state["reason_code"]
+    if reason in (_V4_POST_RESPONSE_OVERRUN_REASONS | _V4_PREFLIGHT_TERMINAL_REASONS):
+        late_reason = (
+            _late_limit(state, config.limits, latest_full) if latest_full is not None else None
+        )
+        prospective_reason = _v4_prospective_terminal_reason(
+            staging,
+            state,
+            config=config,
+            request_item=request_tail,
+            response_item=response_tail,
+            response=latest_full,
+        )
+        refusal_reason = (
+            _v4_transport_terminal_reason(latest_refusal_code, state, config.limits)
+            if isinstance(latest_refusal_code, str)
+            else None
+        )
+        if reason not in {
+            late_reason,
+            _preflight_limit(state, config.limits),
+            prospective_reason,
+            refusal_reason,
+        }:
+            raise _v4_evidence_corrupt("terminal limit reason differs from authenticated tail")
+    if reason == "FILL_ENDPOINT_DRIFT" and (
+        latest_full is None
+        or latest_full.final_url == response_tail.get("request_url")
+        or response_tail.get("request_attempt") != state["budget"]["requests"]
+    ):
+        raise _v4_evidence_corrupt("endpoint drift reason differs from authenticated tail")
+
+
+@dataclass(frozen=True)
+class _V4ValidatedResponseItem:
+    item: dict[str, Any]
+    full: HarvestResponseEvidence | None
+    refusal: dict[str, Any] | None
+    network_bytes: int
+    endpoint_drift: bool
+
+
+def _validate_v4_request_member(
+    item: Any, *, position: int, config: CatalogFillConfig
+) -> dict[str, Any]:
+    if (
+        not isinstance(item, dict)
+        or set(item) != {"attempt", "request_url", "cursor", "elapsed_ns", "observed_epoch_seconds"}
+        or item["attempt"] != position
+        or not _valid_v4_request_url(
+            item["request_url"], page_size=config.page_size, sort=config.sort
+        )
+        or type(item["elapsed_ns"]) is not int
+        or item["elapsed_ns"] < 0
+        or type(item["observed_epoch_seconds"]) is not int
+        or not 0 <= item["observed_epoch_seconds"] <= _MAX_SAFE_EPOCH_SECONDS
+    ):
+        raise _v4_evidence_corrupt("request member is invalid")
+    request_cursor = _cursor_from_dict(item["cursor"])
+    try:
+        _sort, _page_size, url_cursor = _parse_v4_request_url(item["request_url"])
+    except (CatalogHarvestError, TypeError, ValueError):
+        raise _v4_evidence_corrupt("request URL is invalid") from None
+    if request_cursor != url_cursor:
+        raise _v4_evidence_corrupt("request cursor differs from URL")
+    return item
+
+
+def _validate_v4_response_member(
+    staging: FillStaging,
+    item: Any,
+    *,
+    request: dict[str, Any],
+    authenticate_raw: bool,
+) -> _V4ValidatedResponseItem:
+    refusal_keys = {
+        "request_attempt",
+        "request_url",
+        "observed_epoch_seconds",
+        "refusal",
+    }
+    if isinstance(item, dict) and set(item) == refusal_keys:
+        refusal = item["refusal"]
+        if (
+            request["request_url"] != item["request_url"]
+            or item["observed_epoch_seconds"] != request["observed_epoch_seconds"]
+            or not isinstance(refusal, dict)
+            or set(refusal)
+            != {"code", "status", "content_bytes", "network_bytes", "rate_observation"}
+            or not isinstance(refusal["code"], str)
+            or type(refusal["status"]) is not int
+            or not 100 <= refusal["status"] <= 599
+            or refusal["content_bytes"] != 0
+            or type(refusal["network_bytes"]) is not int
+            or refusal["network_bytes"] < 0
+        ):
+            raise _v4_evidence_corrupt("redacted response refusal is invalid")
+        rate_value = refusal["rate_observation"]
+        try:
+            if rate_value is not None:
+                if not isinstance(rate_value, dict):
+                    raise TypeError
+                DatagovRateObservation(**rate_value)
+        except (AcquisitionSecurityError, TypeError):
+            raise _v4_evidence_corrupt("redacted rate observation is invalid") from None
+        return _V4ValidatedResponseItem(
+            item=item,
+            full=None,
+            refusal=refusal,
+            network_bytes=refusal["network_bytes"],
+            endpoint_drift=False,
+        )
+    full_keys = {
+        "request_attempt",
+        "request_url",
+        "observed_epoch_seconds",
+        "raw_response_sha256",
+        "evidence",
+    }
+    if not isinstance(item, dict) or set(item) != full_keys:
+        raise _v4_evidence_corrupt("response member is invalid")
+    raw_digest = item["raw_response_sha256"]
+    evidence = item["evidence"]
+    if (
+        request["request_url"] != item["request_url"]
+        or item["observed_epoch_seconds"] != request["observed_epoch_seconds"]
+        or not _valid_v4_request_url(item["request_url"])
+        or not isinstance(evidence, dict)
+        or not _SHA256.fullmatch(raw_digest or "")
+        or evidence.get("content_sha256") != raw_digest
+    ):
+        raise _v4_evidence_corrupt("response evidence is not bound to exact bytes")
+    try:
+        parsed = _response_evidence_from_dict(evidence)
+    except (CatalogHarvestError, AcquisitionSecurityError, TypeError, ValueError):
+        raise _v4_evidence_corrupt("response evidence is invalid") from None
+    raw_size = len(staging.read_blob("raw_response", raw_digest)) if authenticate_raw else None
+    if (
+        parsed.requests != 1
+        or parsed.responses != 1
+        or parsed.media_type != "application/json"
+        or (
+            raw_size is not None
+            and (parsed.content_bytes != raw_size or parsed.network_bytes != raw_size)
+        )
+    ):
+        raise _v4_evidence_corrupt("response byte/media accounting differs")
+    return _V4ValidatedResponseItem(
+        item=item,
+        full=parsed,
+        refusal=None,
+        network_bytes=parsed.network_bytes,
+        endpoint_drift=parsed.final_url != item["request_url"],
+    )
+
+
+def _iter_v4_request_response_members(
+    staging: FillStaging,
+    indexes: dict[str, Any],
+    *,
+    config: CatalogFillConfig,
+    authenticate_raw: bool = True,
+) -> Iterator[tuple[int, dict[str, Any], _V4ValidatedResponseItem | None]]:
+    responses = iter(staging.iter_index(indexes["response"], stream="response"))
+    response_item = next(responses, None)
+    for position, request_value in enumerate(
+        staging.iter_index(indexes["request"], stream="request"), start=1
+    ):
+        request = _validate_v4_request_member(request_value, position=position, config=config)
+        response: _V4ValidatedResponseItem | None = None
+        if response_item is not None:
+            attempt = (
+                response_item.get("request_attempt") if isinstance(response_item, dict) else None
+            )
+            if type(attempt) is not int or attempt < position:
+                raise _v4_evidence_corrupt("response attempt order differs from requests")
+            if attempt == position:
+                response = _validate_v4_response_member(
+                    staging,
+                    response_item,
+                    request=request,
+                    authenticate_raw=authenticate_raw,
+                )
+                response_item = next(responses, None)
+        yield position, request, response
+    if response_item is not None:
+        raise _v4_evidence_corrupt("response index extends beyond request evidence")
+
+
+def _validate_v4_evidence(
+    staging: FillStaging,
+    state: dict[str, Any],
+    *,
+    config: CatalogFillConfig,
+) -> None:
+    """Authenticate compact chains and bind their ordered members to the fill budget."""
+
+    indexes = state["evidence_indexes"]
+    staging.validate_index(state["partition_run_index"], stream="partition_run")
+    request_count = 0
+    request_elapsed_ns = 0
+    latest_request_epoch_seconds: int | None = None
+    response_count = 0
+    response_network_bytes = 0
+    latest_response_rate: dict[str, Any] | None = None
+    latest_response_status: int | None = None
+    latest_response_epoch_seconds: int | None = None
+    latest_response_attempt = 0
+    latest_full_response: HarvestResponseEvidence | None = None
+    latest_refusal_code: str | None = None
+    latest_response_item: dict[str, Any] | None = None
+    latest_response_request: dict[str, Any] | None = None
+    endpoint_drift_attempt = 0
+    for position, request, response_item in _iter_v4_request_response_members(
+        staging, indexes, config=config
+    ):
+        request_count = position
+        request_elapsed_ns += request["elapsed_ns"]
+        latest_request_epoch_seconds = request["observed_epoch_seconds"]
+        if response_item is None:
+            continue
+        response_count += 1
+        response_network_bytes += response_item.network_bytes
+        latest_response_item = response_item.item
+        latest_response_request = request
+        latest_response_epoch_seconds = response_item.item["observed_epoch_seconds"]
+        latest_response_attempt = position
+        if response_item.refusal is not None:
+            refusal = response_item.refusal
+            latest_response_rate = refusal["rate_observation"]
+            latest_response_status = refusal["status"]
+            latest_refusal_code = refusal["code"]
+            latest_full_response = None
+        else:
+            parsed = response_item.full
+            assert parsed is not None
+            latest_response_rate = (
+                parsed.rate_observation.to_dict() if parsed.rate_observation is not None else None
+            )
+            latest_response_status = parsed.status
+            latest_refusal_code = None
+            latest_full_response = parsed
+            if response_item.endpoint_drift:
+                endpoint_drift_attempt = position
+    if (
+        request_count != state["budget"]["requests"]
+        or request_elapsed_ns != state["budget"]["elapsed_ns"]
+        or response_count != state["budget"]["responses"]
+        or response_network_bytes != state["budget"]["network_bytes"]
+    ):
+        raise _v4_evidence_corrupt("request or response budget differs from evidence")
+    inflight_observed = (
+        state["inflight_request"].get("observed_response")
+        if isinstance(state["inflight_request"], dict)
+        else None
+    )
+    if isinstance(inflight_observed, dict):
+        latest_response_rate = inflight_observed["rate_observation"]
+        latest_response_status = inflight_observed["status"]
+        latest_response_epoch_seconds = state["inflight_request"]["response_observed_epoch_seconds"]
+        latest_response_attempt = state["inflight_request"]["attempt"]
+    if state["last_rate_observation"] != latest_response_rate:
+        raise _v4_evidence_corrupt("last rate observation differs from response evidence")
+    expected_wait: tuple[int | None, str | None] = (None, None)
+    try:
+        if (
+            latest_response_attempt
+            == (
+                state["inflight_request"]["attempt"]
+                if isinstance(state["inflight_request"], dict) and inflight_observed is not None
+                else state["budget"]["requests"]
+            )
+            and latest_response_epoch_seconds is not None
+        ):
+            rate = (
+                DatagovRateObservation(**latest_response_rate)
+                if latest_response_rate is not None
+                else None
+            )
+            expected_wait = _v4_response_wait(
+                config, latest_response_status, rate, latest_response_epoch_seconds
+            )
+        elif latest_request_epoch_seconds is not None:
+            wait_until = _pacing_wait(config, latest_request_epoch_seconds)
+            expected_wait = (
+                wait_until,
+                "DATAGOV_PACING_REQUIRED" if wait_until is not None else None,
+            )
+    except CatalogFillRefused:
+        raise _v4_evidence_corrupt("authenticated observation cannot produce a safe wait") from None
+    if state["completed"] or state["reason_code"] in _V4_MAP_SEALING_REASONS:
+        expected_wait = (None, None)
+    if (
+        state["resume_not_before_epoch_seconds"],
+        state["wait_reason"],
+    ) != expected_wait:
+        raise _v4_evidence_corrupt("durable wait differs from authenticated observation time")
+    if state["reason_code"] in (_V4_POST_RESPONSE_OVERRUN_REASONS | _V4_PREFLIGHT_TERMINAL_REASONS):
+        late_reason = (
+            _late_limit(state, config.limits, latest_full_response)
+            if latest_full_response is not None
+            else None
+        )
+        preflight_reason = _preflight_limit(state, config.limits)
+        prospective_reason = _v4_prospective_terminal_reason(
+            staging,
+            state,
+            config=config,
+            request_item=latest_response_request,
+            response_item=latest_response_item,
+            response=latest_full_response,
+        )
+        refusal_reason = (
+            _v4_transport_terminal_reason(latest_refusal_code, state, config.limits)
+            if latest_refusal_code is not None
+            else None
+        )
+        if state["reason_code"] not in {
+            late_reason,
+            preflight_reason,
+            prospective_reason,
+            refusal_reason,
+        }:
+            raise _v4_evidence_corrupt("terminal limit reason differs from exact evidence")
+    drift_is_latest = (
+        endpoint_drift_attempt == state["budget"]["requests"] and endpoint_drift_attempt > 0
+    )
+    if endpoint_drift_attempt and (
+        not drift_is_latest
+        or (
+            state["reason_code"]
+            not in ({"FILL_ENDPOINT_DRIFT"} | _V4_POST_RESPONSE_OVERRUN_REASONS)
+            and state["pending_response"] is None
+        )
+    ):
+        raise _v4_evidence_corrupt("endpoint drift reason differs from exact response evidence")
+    if not endpoint_drift_attempt and state["reason_code"] == "FILL_ENDPOINT_DRIFT":
+        raise _v4_evidence_corrupt("endpoint drift reason differs from exact response evidence")
+    pending = state["pending_response"]
+    if pending is not None:
+        try:
+            pending_evidence = _evidence_from_dict(pending["evidence"])
+            pending_response = _response_evidence_from_dict(pending["response_evidence"])
+        except (CatalogHarvestError, AcquisitionSecurityError, TypeError, ValueError):
+            raise _v4_evidence_corrupt("pending response contract is invalid") from None
+        if (
+            latest_response_item is None
+            or latest_response_attempt != state["budget"]["requests"]
+            or latest_response_item["request_url"] != pending["request_url"]
+            or latest_response_item["raw_response_sha256"] != pending["raw_response_sha256"]
+            or latest_response_item["evidence"] != pending_response.to_dict()
+            or pending_evidence.uri != pending_response.final_url
+            or pending_evidence.content_sha256 != pending["raw_response_sha256"]
+            or pending_evidence.media_type != "application/json"
+            or pending_response.content_sha256 != pending["raw_response_sha256"]
+        ):
+            raise _v4_evidence_corrupt("pending response differs from authenticated response tail")
+
+    record_items = iter(staging.iter_index(indexes["record"], stream="record"))
+    partition_items = iter(staging.iter_index(state["partition_run_index"], stream="partition_run"))
+    transitions = iter(staging.iter_index(indexes["cursor_transition"], stream="cursor_transition"))
+    successful_responses = (
+        (request, response_item)
+        for _position, request, response_item in _iter_v4_request_response_members(
+            staging, indexes, config=config, authenticate_raw=False
+        )
+        if response_item is not None
+        and response_item.full is not None
+        and response_item.full.status == 200
+        and not response_item.endpoint_drift
+    )
+    observed_pages = 0
+    observed_records = 0
+    observed_skipped = 0
+    observed_normalized_bytes = 0
+    observed_partition_runs = 0
+    expected_pass = 1
+    pass_start_counts = {1: 0}
+    prior_page_terminal = False
+    expected_cursor: dict[str, Any] | None = None
+    current_pass_members = 0
+    try:
+        with tempfile.TemporaryDirectory(prefix="mr-datagov-v4-audit-") as scratch:
+            connection = sqlite3.connect(str(Path(scratch) / "paging.sqlite3"))
+            try:
+                connection.execute("PRAGMA journal_mode=OFF")
+                connection.execute("PRAGMA synchronous=OFF")
+                connection.execute("PRAGMA temp_store=FILE")
+                connection.execute(
+                    "CREATE TABLE seen (kind INTEGER NOT NULL, digest TEXT NOT NULL, "
+                    "PRIMARY KEY (kind, digest)) WITHOUT ROWID"
+                )
+                for item in staging.iter_index(indexes["page"], stream="page"):
+                    if observed_pages and prior_page_terminal:
+                        expected_pass += 1
+                        pass_start_counts[expected_pass] = observed_partition_runs
+                    if (
+                        set(item)
+                        != {"pass", "raw_response_sha256", "normalized_sha256", "record_count"}
+                        or type(item["pass"]) is not int
+                        or item["pass"] != expected_pass
+                        or item["pass"] > state["pass_number"]
+                        or not _SHA256.fullmatch(item["raw_response_sha256"] or "")
+                        or not _SHA256.fullmatch(item["normalized_sha256"] or "")
+                        or type(item["record_count"]) is not int
+                        or item["record_count"] < 0
+                    ):
+                        raise _v4_evidence_corrupt("page member is invalid")
+                    while True:
+                        try:
+                            request, response_item = next(successful_responses)
+                        except StopIteration:
+                            raise _v4_evidence_corrupt(
+                                "page is not an ordered successful response member"
+                            ) from None
+                        if response_item.item["raw_response_sha256"] == item["raw_response_sha256"]:
+                            break
+                    response = response_item.full
+                    if response is None:
+                        raise _v4_evidence_corrupt("successful response evidence is absent")
+                    raw_payload = staging.read_blob("raw_response", item["raw_response_sha256"])
+                    try:
+                        reparsed = DatagovV4Harvester().parse_page(
+                            raw_payload,
+                            uri=request["request_url"],
+                            observed_at=config.observed_at,
+                            evidence=EvidenceReference(
+                                uri=request["request_url"],
+                                observed_at=config.observed_at,
+                                content_sha256=item["raw_response_sha256"],
+                                media_type="application/json",
+                            ),
+                            response_evidence=response,
+                            cursor=_cursor_from_dict(request["cursor"]),
+                        )
+                    except (
+                        CatalogHarvestError,
+                        CatalogFillRefused,
+                        CanonicalJSONError,
+                        TypeError,
+                        ValueError,
+                    ):
+                        raise _v4_evidence_corrupt(
+                            "raw response cannot reproduce normalized page"
+                        ) from None
+                    parsed_next_cursor = (
+                        reparsed.next_cursor.to_dict() if reparsed.next_cursor is not None else None
+                    )
+                    prior_page_terminal = parsed_next_cursor is None
+                    try:
+                        transition = next(transitions)
+                    except StopIteration:
+                        raise _v4_evidence_corrupt("cursor transition membership differs") from None
+                    if (
+                        set(transition)
+                        != {"request_cursor", "next_cursor", "normalized_page_sha256"}
+                        or transition["normalized_page_sha256"] != item["normalized_sha256"]
+                        or transition["request_cursor"] != request["cursor"]
+                        or transition["request_cursor"] != expected_cursor
+                        or transition["next_cursor"] != parsed_next_cursor
+                    ):
+                        raise _v4_evidence_corrupt("cursor transition order differs")
+                    _cursor_from_dict(transition["request_cursor"])
+                    _cursor_from_dict(transition["next_cursor"])
+                    requested_digest = canonical_sha256(transition["request_cursor"])
+                    next_digest = (
+                        canonical_sha256(transition["next_cursor"])
+                        if transition["next_cursor"] is not None
+                        else None
+                    )
+                    cursor_seen = connection.execute(
+                        "SELECT 1 FROM seen WHERE kind=0 AND digest=?", (requested_digest,)
+                    ).fetchone()
+                    response_seen = connection.execute(
+                        "SELECT 1 FROM seen WHERE kind=1 AND digest=?",
+                        (item["raw_response_sha256"],),
+                    ).fetchone()
+                    next_seen = (
+                        connection.execute(
+                            "SELECT 1 FROM seen WHERE kind=0 AND digest=?", (next_digest,)
+                        ).fetchone()
+                        if next_digest is not None
+                        else None
+                    )
+                    if (
+                        cursor_seen is not None
+                        or response_seen is not None
+                        or (
+                            transition["next_cursor"] is not None
+                            and transition["next_cursor"] == transition["request_cursor"]
+                        )
+                        or next_seen is not None
+                    ):
+                        raise _v4_evidence_corrupt("committed page contains a paging loop")
+                    connection.execute(
+                        "INSERT INTO seen(kind,digest) VALUES(0,?)", (requested_digest,)
+                    )
+                    connection.execute(
+                        "INSERT INTO seen(kind,digest) VALUES(1,?)",
+                        (item["raw_response_sha256"],),
+                    )
+                    if item["pass"] == state["pass_number"]:
+                        current_pass_members += 1
+                        if not _v4_loop_contains(
+                            staging,
+                            state["pass_cursor_set_sha256"],
+                            "cursor",
+                            requested_digest,
+                        ) or not _v4_loop_contains(
+                            staging,
+                            state["pass_response_set_sha256"],
+                            "response",
+                            item["raw_response_sha256"],
+                        ):
+                            raise _v4_evidence_corrupt("paging loop-set differs from current pass")
+                    expected_cursor = transition["next_cursor"]
+                    if expected_cursor is None:
+                        connection.execute("DELETE FROM seen")
+
+                    manifest = staging.read_shard("normalized", item["normalized_sha256"])
+                    skipped_ids = manifest.get("skipped_record_ids")
+                    if (
+                        not isinstance(skipped_ids, list)
+                        or any(not isinstance(value, str) for value in skipped_ids)
+                        or skipped_ids != sorted(skipped_ids)
+                    ):
+                        raise _v4_evidence_corrupt("normalized skipped identifiers differ")
+                    observed_skipped += len(skipped_ids)
+                    observed_normalized_bytes += _v4_normalized_shard_size(manifest)
+                    for descriptor in manifest.get("record_segments", []):
+                        if not isinstance(descriptor, dict) or not _SHA256.fullmatch(
+                            descriptor.get("sha256", "")
+                        ):
+                            raise _v4_evidence_corrupt("normalized segment coordinate differs")
+                        observed_normalized_bytes += _v4_normalized_shard_size(
+                            staging.read_shard("normalized", descriptor["sha256"])
+                        )
+                    page_records: list[DatagovV4Record] = []
+                    for record_value in iter_v4_normalized_records(
+                        staging,
+                        item["normalized_sha256"],
+                        raw_response_sha256=item["raw_response_sha256"],
+                        record_count=item["record_count"],
+                    ):
+                        record = _v4_record_from_dict(record_value)
+                        if record.raw_response_sha256 != item["raw_response_sha256"]:
+                            raise _v4_evidence_corrupt("normalized record raw provenance differs")
+                        page_records.append(record)
+                        try:
+                            record_item = next(record_items)
+                        except StopIteration:
+                            raise _v4_evidence_corrupt(
+                                "record index ended before normalized records"
+                            ) from None
+                        if record_item != {
+                            "identifier_sha256": canonical_sha256(record.record_id),
+                            "semantic_sha256": record.digest,
+                            "normalized_page_sha256": item["normalized_sha256"],
+                        }:
+                            raise _v4_evidence_corrupt(
+                                "record index differs from normalized record"
+                            )
+                    if len(page_records) != item["record_count"]:
+                        raise _v4_evidence_corrupt("normalized page membership differs")
+                    if [record.to_dict() for record in page_records] != [
+                        record.to_dict() for record in reparsed.records
+                    ] or tuple(skipped_ids) != reparsed.skipped_record_ids:
+                        raise _v4_evidence_corrupt(
+                            "normalized page differs from authenticated raw response"
+                        )
+                    observed_records += len(page_records)
+                    for partition, entries, conflicts in _expected_v4_page_runs(page_records):
+                        try:
+                            descriptor = next(partition_items)
+                        except StopIteration:
+                            raise _v4_evidence_corrupt(
+                                "partition index ended before page records"
+                            ) from None
+                        actual_partition, actual_entries, actual_conflicts = read_partition_run(
+                            staging, descriptor
+                        )
+                        if (
+                            actual_partition != partition
+                            or actual_entries != entries
+                            or actual_conflicts != conflicts
+                        ):
+                            raise _v4_evidence_corrupt(
+                                "partition run differs from normalized records"
+                            )
+                        observed_partition_runs += 1
+                    observed_pages += 1
+            finally:
+                connection.close()
+    except (OSError, sqlite3.Error) as error:
+        raise CatalogFillRefused(
+            "DATAGOV_EXTERNAL_MERGE",
+            "fill.staging.audit",
+            "bounded paging evidence audit scratch failed",
+        ) from error
+    if next(record_items, None) is not None:
+        raise _v4_evidence_corrupt("record index extends beyond normalized records")
+    if next(partition_items, None) is not None:
+        raise _v4_evidence_corrupt("partition index extends beyond normalized records")
+    if observed_pages == 0:
+        if state["pass_number"] != 1:
+            raise _v4_evidence_corrupt("empty history has an invalid pass number")
+    elif state["pass_number"] == expected_pass + 1 and prior_page_terminal:
+        pass_start_counts[state["pass_number"]] = observed_partition_runs
+    elif state["pass_number"] != expected_pass:
+        raise _v4_evidence_corrupt("state pass number differs from terminal boundaries")
+    if state["pass_start_run_count"] != pass_start_counts[state["pass_number"]]:
+        raise _v4_evidence_corrupt("pass run offset differs from terminal boundaries")
+    if (state["completed"] or state["reason_code"] in _V4_MAP_SEALING_REASONS) and (
+        not prior_page_terminal or state["pass_number"] != expected_pass
+    ):
+        raise _v4_evidence_corrupt("final state is not the terminal current pass")
+    if (state["pass_number"] == 1) != (state["previous_pass_map_sha256"] is None):
+        raise _v4_evidence_corrupt("previous pass map differs from pass state")
+    if (
+        observed_pages != state["budget"]["pages"]
+        or observed_records != state["budget"]["records"]
+        or observed_skipped != state["budget"]["skipped_records"]
+        or observed_normalized_bytes != state["normalized_bytes"]
+    ):
+        raise _v4_evidence_corrupt("page, record, or skipped budget differs from evidence")
+    if next(transitions, None) is not None:
+        raise _v4_evidence_corrupt("cursor transition membership differs")
+    if expected_cursor != state["next_cursor"]:
+        raise _v4_evidence_corrupt("state cursor differs from transition tail")
+    if (
+        _v4_loop_count(staging, state["pass_cursor_set_sha256"], "cursor") != current_pass_members
+        or _v4_loop_count(staging, state["pass_response_set_sha256"], "response")
+        != current_pass_members
+    ):
+        raise _v4_evidence_corrupt("paging loop-set count differs from current pass")
+
+    manifest_sha256 = state["last_pass_manifest_sha256"]
+    map_sha256 = state["final_map_sha256"] or state["previous_pass_map_sha256"]
+    if (manifest_sha256 is None) != (map_sha256 is None):
+        raise _v4_evidence_corrupt("pass manifest and map root differ")
+    if manifest_sha256 is not None:
+        if not _SHA256.fullmatch(manifest_sha256) or not _SHA256.fullmatch(map_sha256):
+            raise _v4_evidence_corrupt("pass manifest coordinate is invalid")
+        result = validate_pass_map(staging, manifest_sha256, map_sha256)
+        if result.unique_records != state["unique_records"]:
+            raise _v4_evidence_corrupt("pass manifest record count differs")
+        if result.conflicting_identifiers != state["conflicting_identifiers"]:
+            raise _v4_evidence_corrupt("pass manifest conflict count differs")
+        if (
+            state["completed"]
+            and result.conflicting_identifiers > config.max_conflicting_identifiers
+        ):
+            raise _v4_evidence_corrupt("completed map retains unadmitted identifier conflicts")
+        run_count = state["partition_run_index"]["count"]
+        if state["final_map_sha256"] is not None:
+            start, stop = pass_start_counts[state["pass_number"]], run_count
+        else:
+            start = pass_start_counts[state["pass_number"] - 1]
+            stop = state["pass_start_run_count"]
+        source_runs = islice(
+            staging.iter_index(state["partition_run_index"], stream="partition_run", start=start),
+            stop - start,
+        )
+        recomputed = dry_run_external_merge_pass(staging, source_runs)
+        if (
+            recomputed.root_sha256 != map_sha256
+            or recomputed.manifest_sha256 != manifest_sha256
+            or recomputed.unique_records != state["unique_records"]
+        ):
+            raise _v4_evidence_corrupt("pass map differs from normalized source runs")
+        if state["final_map_sha256"] is not None:
+            prior_start = pass_start_counts[state["pass_number"] - 1]
+            prior_stop = pass_start_counts[state["pass_number"]]
+            prior_runs = islice(
+                staging.iter_index(
+                    state["partition_run_index"],
+                    stream="partition_run",
+                    start=prior_start,
+                ),
+                prior_stop - prior_start,
+            )
+            prior = dry_run_external_merge_pass(staging, prior_runs)
+            if prior.root_sha256 != state["previous_pass_map_sha256"]:
+                raise _v4_evidence_corrupt("preceding pass map differs from normalized source runs")
+            if state["completed"] and prior.root_sha256 != recomputed.root_sha256:
+                raise _v4_evidence_corrupt("completed passes did not converge")
+
+
+def iter_v4_normalized_records(
+    staging: FillStaging,
+    manifest_sha256: str,
+    *,
+    raw_response_sha256: str,
+    record_count: int,
+) -> Iterator[dict[str, Any]]:
+    """Yield one logical normalized page while authenticating its bounded ordered segments."""
+
+    manifest = staging.read_shard("normalized", manifest_sha256)
+    if (
+        set(manifest)
+        != {
+            "schema_version",
+            "raw_response_sha256",
+            "record_count",
+            "record_segments",
+            "skipped_record_ids",
+        }
+        or manifest["schema_version"] != "harness-datagov-v4-normalized-page.v2"
+        or manifest["raw_response_sha256"] != raw_response_sha256
+        or manifest["record_count"] != record_count
+        or not isinstance(manifest["record_segments"], list)
+        or not isinstance(manifest["skipped_record_ids"], list)
+    ):
+        raise _v4_evidence_corrupt("normalized page manifest differs")
+    next_record = 0
+    for descriptor in manifest["record_segments"]:
+        if (
+            not isinstance(descriptor, dict)
+            or set(descriptor) != {"sha256", "first_record", "record_count"}
+            or not _SHA256.fullmatch(descriptor["sha256"] or "")
+            or descriptor["first_record"] != next_record
+            or type(descriptor["record_count"]) is not int
+            or descriptor["record_count"] < 1
+        ):
+            raise _v4_evidence_corrupt("normalized record segment descriptor differs")
+        segment = staging.read_shard("normalized", descriptor["sha256"])
+        if (
+            set(segment)
+            != {
+                "schema_version",
+                "raw_response_sha256",
+                "first_record",
+                "records",
+            }
+            or segment["schema_version"] != "harness-datagov-v4-normalized-record-segment.v1"
+            or segment["raw_response_sha256"] != raw_response_sha256
+            or segment["first_record"] != next_record
+            or not isinstance(segment["records"], list)
+            or len(segment["records"]) != descriptor["record_count"]
+        ):
+            raise _v4_evidence_corrupt("normalized record segment differs")
+        for record in segment["records"]:
+            if not isinstance(record, dict):
+                raise _v4_evidence_corrupt("normalized record is invalid")
+            yield record
+        next_record += descriptor["record_count"]
+    if next_record != record_count:
+        raise _v4_evidence_corrupt("normalized record segment count differs")
+
+
+def _v4_record_from_dict(value: Any) -> DatagovV4Record:
+    required = {
+        "schema_version",
+        "protocol",
+        "record_id",
+        "fields",
+        "raw_response_sha256",
+        "normalized_sha256",
+    }
+    if (
+        not isinstance(value, dict)
+        or not required <= set(value) <= required | {"observations"}
+        or not isinstance(value["fields"], list)
+        or not isinstance(value.get("observations", []), list)
+    ):
+        raise _v4_evidence_corrupt("normalized record shape differs")
+    try:
+        record = DatagovV4Record(
+            schema_version=value["schema_version"],
+            protocol=value["protocol"],
+            record_id=value["record_id"],
+            fields=_v4_projected_fields(value["fields"]),
+            observations=_v4_projected_fields(value.get("observations", [])),
+            raw_response_sha256=value["raw_response_sha256"],
+        )
+    except (CatalogHarvestError, CanonicalJSONError, TypeError, ValueError):
+        raise _v4_evidence_corrupt("normalized record contract differs") from None
+    if record.to_dict() != value:
+        raise _v4_evidence_corrupt("normalized record digest differs")
+    return record
+
+
+def _v4_projected_fields(values: Any) -> tuple[DatagovProjectedField, ...]:
+    fields: list[DatagovProjectedField] = []
+    for field in values:
+        if not isinstance(field, dict) or not {"path", "value"} <= set(field) <= {
+            "path",
+            "value",
+            "encoding",
+        }:
+            raise TypeError
+        fields.append(
+            DatagovProjectedField(
+                path=field["path"],
+                value=field["value"],
+                encoding=field.get("encoding", "canonical_json"),
+            )
+        )
+    return tuple(fields)
+
+
+def _expected_v4_page_runs(
+    records: list[DatagovV4Record],
+) -> Iterator[tuple[str, list[tuple[str, str]], set[str]]]:
+    yield from plan_partition_runs(records)
+
+
+def _valid_v4_request_url(
+    value: Any, *, page_size: int | None = None, sort: str | None = None
+) -> bool:
+    try:
+        parsed_sort, parsed_page_size, _cursor = _parse_v4_request_url(value)
+    except (CatalogHarvestError, TypeError, ValueError):
+        return False
+    return (page_size is None or parsed_page_size == page_size) and (
+        sort is None or parsed_sort == sort
+    )
+
+
+def _parse_v4_request_url(value: Any) -> tuple[str, int, HarvestCursor | None]:
+    if not isinstance(value, str):
+        raise ValueError("request URL must be text")
+    try:
+        parsed = urlsplit(value)
+        endpoint = urlsplit(DATAGOV_V4_ENDPOINT)
+        port = parsed.port
+        endpoint_port = endpoint.port
+        pairs = parse_qsl(parsed.query, keep_blank_values=True, strict_parsing=True)
+    except ValueError:
+        raise ValueError("request URL cannot be parsed") from None
+    if not (
+        parsed.scheme == endpoint.scheme
+        and parsed.hostname == endpoint.hostname
+        and port == endpoint_port
+        and parsed.path == endpoint.path
+        and parsed.fragment == ""
+        and len(pairs) in {2, 3}
+        and pairs[0][0] == "sort"
+        and pairs[0][1] in DATAGOV_V4_SORT_ORDERS
+        and pairs[1][0] == "per_page"
+    ):
+        raise ValueError("request URL coordinate differs")
+    per_page_text = pairs[1][1]
+    if not re.fullmatch(r"(?:[1-9]|[1-9][0-9]{1,2}|1000)", per_page_text):
+        raise ValueError("request page size differs")
+    cursor: HarvestCursor | None = None
+    if len(pairs) == 3:
+        if pairs[2][0] != "after":
+            raise ValueError("request cursor key differs")
+        cursor = HarvestCursor(protocol="datagov_v4", kind="after", value=pairs[2][1])
+    page_size = int(per_page_text)
+    sort = pairs[0][1]
+    canonical_query = [("sort", sort), ("per_page", per_page_text)]
+    if cursor is not None:
+        canonical_query.append(("after", cursor.value))
+    canonical = f"{DATAGOV_V4_ENDPOINT}?{urlencode(canonical_query)}"
+    if value != canonical:
+        raise ValueError("request URL spelling is not canonical")
+    return sort, page_size, cursor
+
+
+def _v4_evidence_corrupt(detail: str) -> CatalogFillRefused:
+    return CatalogFillRefused("FILL_CHECKPOINT_CORRUPT", "fill.staging.indexes", detail)
+
+
+@dataclass
+class _V4LoopSet:
+    staging: FillStaging
+    set_kind: str
+    root_sha256: str | None
+
+    def __contains__(self, digest: object) -> bool:
+        return isinstance(digest, str) and _v4_loop_contains(
+            self.staging, self.root_sha256, self.set_kind, digest
+        )
+
+    def add(self, digest: str) -> None:
+        self.root_sha256 = _v4_loop_add(self.staging, self.root_sha256, self.set_kind, digest)
+
+    def clear(self) -> None:
+        self.root_sha256 = None
+
+
+def _v4_pass_loop_coordinates(
+    staging: FillStaging, state: dict[str, Any]
+) -> tuple[_V4LoopSet, _V4LoopSet]:
+    return (
+        _V4LoopSet(staging, "cursor", state["pass_cursor_set_sha256"]),
+        _V4LoopSet(staging, "response", state["pass_response_set_sha256"]),
+    )
+
+
+def _v4_loop_node(
+    staging: FillStaging,
+    digest: str,
+    *,
+    set_kind: str,
+    depth: int,
+    prefix: str,
+) -> dict[str, Any]:
+    node = staging.read_shard("loop_set", digest)
+    base_valid = (
+        node.get("schema_version") == DATAGOV_V4_LOOP_SET_SCHEMA
+        and node.get("set_kind") == set_kind
+        and node.get("depth") == depth
+        and node.get("prefix") == prefix
+        and set_kind in {"cursor", "response"}
+        and 0 <= depth <= 64
+    )
+    if not base_valid:
+        raise _v4_evidence_corrupt("paging loop-set node coordinate differs")
+    if set(node) == {"schema_version", "set_kind", "depth", "prefix", "values"}:
+        values = node["values"]
+        if (
+            not isinstance(values, list)
+            or not 1 <= len(values) <= DATAGOV_V4_LOOP_SET_LEAF_ITEMS
+            or values != sorted(set(values))
+            or any(not _SHA256.fullmatch(value) or not value.startswith(prefix) for value in values)
+        ):
+            raise _v4_evidence_corrupt("paging loop-set leaf is invalid")
+        return node
+    if set(node) == {"schema_version", "set_kind", "depth", "prefix", "children"}:
+        children = node["children"]
+        if (
+            depth >= 64
+            or not isinstance(children, list)
+            or not 1 <= len(children) <= 16
+            or any(
+                not isinstance(child, dict)
+                or set(child) != {"nibble", "sha256"}
+                or child["nibble"] not in "0123456789abcdef"
+                or not _SHA256.fullmatch(child["sha256"] or "")
+                for child in children
+            )
+            or [child["nibble"] for child in children]
+            != sorted({child["nibble"] for child in children})
+        ):
+            raise _v4_evidence_corrupt("paging loop-set branch is invalid")
+        return node
+    raise _v4_evidence_corrupt("paging loop-set node shape differs")
+
+
+def _v4_write_loop_leaf(
+    staging: FillStaging,
+    *,
+    set_kind: str,
+    depth: int,
+    prefix: str,
+    values: list[str],
+) -> str:
+    return staging.write_shard(
+        "loop_set",
+        {
+            "schema_version": DATAGOV_V4_LOOP_SET_SCHEMA,
+            "set_kind": set_kind,
+            "depth": depth,
+            "prefix": prefix,
+            "values": values,
+        },
+    )
+
+
+def _v4_write_loop_subtree(
+    staging: FillStaging,
+    *,
+    set_kind: str,
+    depth: int,
+    prefix: str,
+    values: list[str],
+) -> str:
+    if len(values) <= DATAGOV_V4_LOOP_SET_LEAF_ITEMS:
+        return _v4_write_loop_leaf(
+            staging,
+            set_kind=set_kind,
+            depth=depth,
+            prefix=prefix,
+            values=values,
+        )
+    if depth >= 64:
+        raise _v4_evidence_corrupt("paging loop-set leaf cannot split")
+    children = []
+    for nibble in sorted({value[depth] for value in values}):
+        child_values = [value for value in values if value[depth] == nibble]
+        children.append(
+            {
+                "nibble": nibble,
+                "sha256": _v4_write_loop_subtree(
+                    staging,
+                    set_kind=set_kind,
+                    depth=depth + 1,
+                    prefix=prefix + nibble,
+                    values=child_values,
+                ),
+            }
+        )
+    return staging.write_shard(
+        "loop_set",
+        {
+            "schema_version": DATAGOV_V4_LOOP_SET_SCHEMA,
+            "set_kind": set_kind,
+            "depth": depth,
+            "prefix": prefix,
+            "children": children,
+        },
+    )
+
+
+def _v4_loop_add(
+    staging: FillStaging,
+    root_sha256: str | None,
+    set_kind: str,
+    digest: str,
+    *,
+    depth: int = 0,
+    prefix: str = "",
+) -> str:
+    if set_kind not in {"cursor", "response"} or not _SHA256.fullmatch(digest or ""):
+        raise _v4_evidence_corrupt("paging loop-set member is invalid")
+    if root_sha256 is None:
+        return _v4_write_loop_leaf(
+            staging,
+            set_kind=set_kind,
+            depth=depth,
+            prefix=prefix,
+            values=[digest],
+        )
+    node = _v4_loop_node(staging, root_sha256, set_kind=set_kind, depth=depth, prefix=prefix)
+    if "values" in node:
+        values = node["values"]
+        if digest in values:
+            return root_sha256
+        combined = sorted([*values, digest])
+        if len(combined) <= DATAGOV_V4_LOOP_SET_LEAF_ITEMS:
+            return _v4_write_loop_leaf(
+                staging,
+                set_kind=set_kind,
+                depth=depth,
+                prefix=prefix,
+                values=combined,
+            )
+        return _v4_write_loop_subtree(
+            staging,
+            set_kind=set_kind,
+            depth=depth,
+            prefix=prefix,
+            values=combined,
+        )
+    else:
+        children = [dict(child) for child in node["children"]]
+        nibble = digest[depth]
+        for child in children:
+            if child["nibble"] == nibble:
+                child["sha256"] = _v4_loop_add(
+                    staging,
+                    child["sha256"],
+                    set_kind,
+                    digest,
+                    depth=depth + 1,
+                    prefix=prefix + nibble,
+                )
+                break
+        else:
+            children.append(
+                {
+                    "nibble": nibble,
+                    "sha256": _v4_write_loop_leaf(
+                        staging,
+                        set_kind=set_kind,
+                        depth=depth + 1,
+                        prefix=prefix + nibble,
+                        values=[digest],
+                    ),
+                }
+            )
+            children.sort(key=lambda child: child["nibble"])
+    return staging.write_shard(
+        "loop_set",
+        {
+            "schema_version": DATAGOV_V4_LOOP_SET_SCHEMA,
+            "set_kind": set_kind,
+            "depth": depth,
+            "prefix": prefix,
+            "children": children,
+        },
+    )
+
+
+def _v4_loop_contains(
+    staging: FillStaging,
+    root_sha256: str | None,
+    set_kind: str,
+    digest: str,
+) -> bool:
+    if root_sha256 is None:
+        return False
+    depth = 0
+    prefix = ""
+    current = root_sha256
+    while True:
+        node = _v4_loop_node(staging, current, set_kind=set_kind, depth=depth, prefix=prefix)
+        if "values" in node:
+            return digest in node["values"]
+        nibble = digest[depth]
+        child = next((child for child in node["children"] if child["nibble"] == nibble), None)
+        if child is None:
+            return False
+        current = child["sha256"]
+        prefix += nibble
+        depth += 1
+
+
+def _v4_loop_count(staging: FillStaging, root_sha256: str | None, set_kind: str) -> int:
+    if root_sha256 is None:
+        return 0
+    pending = [(root_sha256, 0, "")]
+    count = 0
+    visited = 0
+    while pending:
+        digest, depth, prefix = pending.pop()
+        visited += 1
+        if visited > 2_000_000:
+            raise _v4_evidence_corrupt("paging loop-set exceeds its node bound")
+        node = _v4_loop_node(staging, digest, set_kind=set_kind, depth=depth, prefix=prefix)
+        if "values" in node:
+            count += len(node["values"])
+        else:
+            pending.extend(
+                (child["sha256"], depth + 1, prefix + child["nibble"]) for child in node["children"]
+            )
+    return count
+
+
+def _pacing_wait(config: CatalogFillConfig, observed_epoch_seconds: int) -> int | None:
+    if config.minimum_request_interval_ns == 0:
+        return None
+    seconds = (config.minimum_request_interval_ns + 999_999_999) // 1_000_000_000
+    return _checked_epoch_add(observed_epoch_seconds, seconds)
+
+
+def _pacing_sleep_seconds(
+    config: CatalogFillConfig,
+    *,
+    deadline_epoch_seconds: int,
+    now_epoch_seconds: int,
+    anchor_ns: int | None,
+    now_ns: int,
+) -> float:
+    """How long an in-process pace waits, honouring the later of two independent bounds.
+
+    The wall-clock bound is the durable one: it is the deadline the checkpoint records, so an
+    in-process pace can never resume earlier than a cross-process resume of the same checkpoint
+    would have.  The monotonic bound is the configured interval measured from the response this
+    pace follows.  It is needed because the durable deadline is kept in whole seconds and both
+    ends of it are truncated, which can leave a wall-clock difference nearly a second shorter
+    than the interval it stands for -- a rounding artefact that must never be spent as speed.
+    ``anchor_ns`` is absent only when a fresh process resumes a wait it did not itself observe,
+    and there the durable deadline is the only bound there is evidence for.
+    """
+
+    seconds = float(max(0, deadline_epoch_seconds - now_epoch_seconds))
+    if anchor_ns is None:
+        return seconds
+    remaining_ns = anchor_ns + config.minimum_request_interval_ns - now_ns
+    return max(seconds, max(0, remaining_ns) / 1_000_000_000)
+
+
+def _checked_epoch_add(epoch_seconds: int, delta_seconds: int) -> int:
+    if (
+        type(epoch_seconds) is not int
+        or type(delta_seconds) is not int
+        or not 0 <= epoch_seconds <= _MAX_SAFE_EPOCH_SECONDS
+        or delta_seconds < 0
+        or epoch_seconds > _MAX_SAFE_EPOCH_SECONDS - delta_seconds
+    ):
+        raise CatalogFillRefused(
+            "FILL_CLOCK", "fill.wall_clock", "wall-clock deadline exceeds the safe epoch"
+        )
+    return epoch_seconds + delta_seconds
+
+
+def _v4_response_wait(
+    config: CatalogFillConfig,
+    status: int,
+    rate: DatagovRateObservation | None,
+    observed_epoch_seconds: int,
+) -> tuple[int | None, str | None]:
+    if status == 429:
+        retry_at = (
+            _checked_epoch_add(observed_epoch_seconds, rate.retry_after_seconds)
+            if rate is not None and rate.retry_after_seconds is not None
+            else _checked_epoch_add(observed_epoch_seconds, 3_600)
+        )
+        if rate is not None and rate.reset_epoch_seconds is not None:
+            retry_at = max(retry_at, rate.reset_epoch_seconds)
+        return retry_at, "DATAGOV_RATE_LIMITED"
+    wait_until = _pacing_wait(config, observed_epoch_seconds)
+    return wait_until, "DATAGOV_PACING_REQUIRED" if wait_until is not None else None
+
+
+def run_catalog_fill(
+    *,
+    config: CatalogFillConfig,
+    staging_root: Path,
+    retriever: Any,
+    expected_checkpoint_digest: str | None = None,
+    clock_ns: Callable[[], int] = time.monotonic_ns,
+    max_pages_this_invocation: int | None = None,
+    authorization: DatagovV4Authorization | None = None,
+    wall_clock_seconds: Callable[[], int] = lambda: int(time.time()),
+    staging_descriptor: int | None = None,
+    pacing_sleep: Callable[[float], None] | None = None,
+) -> CatalogFillResult:
+    """Run or resume one fill; only a reconciled stable state becomes eligible.
+
+    ``pacing_sleep`` opts one invocation into waiting the provider's minimum request interval out
+    in this process rather than returning ``DATAGOV_PACING_REQUIRED`` for the caller to wait out
+    and resume.  Absent it the return-and-resume contract is exactly what it always was, which is
+    what the hosted one-shot job depends on.  Present, the interval is unchanged and every page is
+    still committed before the wait, so an interrupted paced sweep resumes from the same journal;
+    what it buys is that the accumulated evidence tree is authenticated once for the invocation
+    instead of once for every page, which is the difference between a sweep whose cost grows with
+    the square of the records it has already staged and one that grows with the records.
+    """
+
+    if not isinstance(config, CatalogFillConfig):
+        raise CatalogFillRefused("FILL_CONFIG", "fill.config", "strict configuration is required")
+    if max_pages_this_invocation is not None and (
+        type(max_pages_this_invocation) is not int or max_pages_this_invocation < 1
+    ):
+        raise CatalogFillRefused(
+            "FILL_INVOCATION_LIMIT", "fill.max_pages_this_invocation", "must be positive"
+        )
+    if pacing_sleep is not None and not callable(pacing_sleep):
+        raise CatalogFillRefused(
+            "FILL_PACING_SLEEP", "fill.pacing_sleep", "in-process pacing needs a callable wait"
+        )
+    if config.protocol == "datagov_v4":
+        if not isinstance(authorization, DatagovV4Authorization):
+            raise CatalogFillRefused(
+                "DATAGOV_AUTHORIZATION", "fill.authorization", "strict authorization is required"
+            )
+    elif authorization is not None:
+        raise CatalogFillRefused(
+            "FILL_AUTHENTICATION",
+            "fill.authorization",
+            "legacy public protocols remain credential-free",
+        )
+    elif pacing_sleep is not None:
+        raise CatalogFillRefused(
+            "FILL_PACING_SLEEP",
+            "fill.pacing_sleep",
+            "only the keyed lane paces, so only it has an interval to wait out",
+        )
+    if config.protocol == "datagov_v4":
+        return _run_datagov_v4_fill(
+            config=config,
+            staging_root=Path(staging_root),
+            retriever=retriever,
+            authorization=authorization,
+            expected_checkpoint_digest=expected_checkpoint_digest,
+            clock_ns=clock_ns,
+            wall_clock_seconds=wall_clock_seconds,
+            max_pages_this_invocation=max_pages_this_invocation,
+            staging_descriptor=staging_descriptor,
+            pacing_sleep=pacing_sleep,
+        )
+    _, harvester, transport_limits = _protocol_parts(config.protocol)
+    if config.limits.max_response_bytes < transport_limits.max_response_bytes:
+        transport_limits = replace(
+            transport_limits, max_response_bytes=config.limits.max_response_bytes
+        )
+    staging = FillStaging(Path(staging_root), root_descriptor=staging_descriptor)
+    with staging.locked():
+        state = staging.load()
+        if state is None:
+            if expected_checkpoint_digest is not None:
+                raise CatalogFillRefused(
+                    "FILL_CHECKPOINT_MISMATCH",
+                    "fill.expected_checkpoint_digest",
+                    "no live checkpoint matches the caller's expected digest",
+                )
+            state = staging.commit(
+                _initial_state(config),
+                expected_generation=-1,
+                expected_checkpoint_digest=None,
+            )
+        elif (
+            not isinstance(expected_checkpoint_digest, str)
+            or not _SHA256.fullmatch(expected_checkpoint_digest)
+            or expected_checkpoint_digest != staging.state_digest()
+        ):
+            raise CatalogFillRefused(
+                "FILL_CHECKPOINT_MISMATCH",
+                "fill.expected_checkpoint_digest",
+                "resume requires the exact caller-retained checkpoint digest",
+            )
+        _validate_state(state, config)
+        if state["completed"]:
+            return _result(staging, state, reason=None)
+        if config.protocol == "datagov_v4":
+            now = _wall_clock(wall_clock_seconds)
+            wait_until = state["resume_not_before_epoch_seconds"]
+            if wait_until is not None and now < wait_until:
+                return _result(staging, state, reason="DATAGOV_RATE_LIMITED")
+        invocation_pages = 0
+        current_pass_map, current_pass_drift = _pass_map(staging, state)
+        seen_cursors, seen_responses = _pass_loop_coordinates(staging, state)
+        while True:
+            preflight = _preflight_limit(state, config.limits)
+            if preflight is not None:
+                state = _stop(staging, state, preflight)
+                return _result(staging, state, reason=preflight)
+
+            cursor = _cursor_from_dict(state["next_cursor"])
+            request_url = _request_url(config, cursor)
+            started = _clock(clock_ns)
+            remaining_transport_limits = _remaining_transport_limits(
+                state, config.limits, transport_limits
+            )
+            if config.protocol == "datagov_v4":
+                payload, evidence, response_evidence = fetch_datagov_v4_response(
+                    retriever=retriever,
+                    url=request_url,
+                    observed_at=config.observed_at,
+                    limits=remaining_transport_limits,
+                    authorization=authorization,
+                )
+            else:
+                payload, evidence, response_evidence = fetch_harvest_response(
+                    retriever=retriever,
+                    url=request_url,
+                    observed_at=config.observed_at,
+                    limits=remaining_transport_limits,
+                )
+            elapsed = _clock(clock_ns) - started
+            _validate_response_endpoint(config.endpoint, request_url, response_evidence)
+            response_digest = staging.write_shard("response", response_evidence.to_dict())
+            state = dict(state)
+            state["response_shards"].append(response_digest)
+            budget = dict(state["budget"])
+            budget["requests"] += response_evidence.requests
+            budget["responses"] += response_evidence.responses
+            budget["network_bytes"] += response_evidence.network_bytes
+            budget["elapsed_ns"] += elapsed
+            state["budget"] = budget
+            if config.protocol == "datagov_v4":
+                rate = response_evidence.rate_observation
+                state["last_rate_observation"] = rate.to_dict() if rate is not None else None
+                if response_evidence.status == 429:
+                    now = _wall_clock(wall_clock_seconds)
+                    retry_at = (
+                        now + rate.retry_after_seconds
+                        if rate is not None and rate.retry_after_seconds is not None
+                        else now + 3_600
+                    )
+                    if rate is not None and rate.reset_epoch_seconds is not None:
+                        retry_at = max(retry_at, rate.reset_epoch_seconds)
+                    state["resume_not_before_epoch_seconds"] = retry_at
+                    state["reason_code"] = "DATAGOV_RATE_LIMITED"
+                    state = _commit(staging, state)
+                    return _result(staging, state, reason="DATAGOV_RATE_LIMITED")
+                if response_evidence.status == 403:
+                    state["resume_not_before_epoch_seconds"] = None
+                    state["reason_code"] = "DATAGOV_AUTHENTICATION_FAILED"
+                    state = _commit(staging, state)
+                    return _result(staging, state, reason="DATAGOV_AUTHENTICATION_FAILED")
+                if response_evidence.status != 200:
+                    state["resume_not_before_epoch_seconds"] = None
+                    state["reason_code"] = "DATAGOV_HTTP_STATUS"
+                    state = _commit(staging, state)
+                    return _result(staging, state, reason="DATAGOV_HTTP_STATUS")
+                state["resume_not_before_epoch_seconds"] = None
+            late = _late_limit(state, config.limits, response_evidence)
+            if late is not None:
+                state = _commit(staging, state)
+                state = _stop(staging, state, late)
+                return _result(staging, state, reason=late)
+
+            page = harvester.parse_page(
+                payload,
+                uri=request_url,
+                observed_at=config.observed_at,
+                evidence=evidence,
+                response_evidence=response_evidence,
+                cursor=cursor,
+            )
+            if page.next_cursor is not None and page.protocol == "stac":
+                _validate_continuation(config.endpoint, page.next_cursor.value)
+            count_drift = _page_count_drift(state, page)
+            id_facts: dict[str, str] = {}
+            drift = current_pass_drift or count_drift
+            for record in page.records:
+                digest = record.digest
+                page_prior = id_facts.get(record.record_id)
+                if page_prior is not None and page_prior != digest:
+                    drift = True
+                    continue
+                id_facts[record.record_id] = digest
+            for identifier, digest in id_facts.items():
+                prior = current_pass_map.get(identifier)
+                if prior is not None and prior != digest:
+                    drift = True
+                elif prior is None:
+                    current_pass_map[identifier] = digest
+            current_pass_drift = drift
+            flagged = _flagged_records(page.records)
+            projected = _projected_page_limit(state, config.limits, page)
+            if projected is not None:
+                state = _commit(staging, state)
+                state = _stop(staging, state, projected)
+                return _result(staging, state, reason=projected)
+            page_payload = {
+                "pass": state["pass_number"],
+                "request_cursor": cursor.to_dict() if cursor is not None else None,
+                "next_cursor": page.next_cursor.to_dict() if page.next_cursor is not None else None,
+                "id_facts": id_facts,
+                "records": [record.to_dict() for record in page.records],
+                "provider_count": page.provider_count,
+                "count_basis": page.count_basis,
+                "skipped_record_ids": list(page.skipped_record_ids),
+                "flagged_rights": flagged,
+                "evidence": page.evidence.to_dict(),
+                "response_evidence_digest": response_digest,
+            }
+            _refuse_loops(seen_cursors, seen_responses, cursor, page)
+            page_digest = staging.write_shard("page", page_payload)
+            state["page_shards"].append(page_digest)
+            budget["pages"] += 1
+            budget["records"] += len(page.records)
+            budget["skipped_records"] += len(page.skipped_record_ids)
+            state["budget"] = budget
+            state["next_cursor"] = page.next_cursor.to_dict() if page.next_cursor else None
+            state["provider_count"] = page.provider_count
+            state["count_basis"] = page.count_basis
+            state["pass_drift"] = drift
+            state["reason_code"] = None
+            state = _commit(staging, state)
+            seen_cursors.add(canonical_sha256(cursor.to_dict() if cursor is not None else None))
+            seen_responses.add(page.evidence.content_sha256)
+            invocation_pages += 1
+            if page.next_cursor is not None:
+                after_page = _preflight_limit(state, config.limits)
+                if after_page is not None:
+                    state = _stop(staging, state, after_page)
+                    return _result(staging, state, reason=after_page)
+                if max_pages_this_invocation == invocation_pages:
+                    reason = (
+                        "INVOCATION_PAGE_LIMIT"
+                        if config.protocol == "datagov_v4"
+                        else "FILL_PAUSED"
+                    )
+                    state = _stop(staging, state, reason)
+                    return _result(staging, state, reason=reason)
+                continue
+
+            pass_digest = canonical_sha256(current_pass_map)
+            processed = _pass_processed_count(staging, state)
+            count_agrees = page.provider_count is None or processed == page.provider_count
+            one_shot = page.count_basis == "one_shot_response"
+            stable = one_shot or state["previous_pass_map_sha256"] == pass_digest
+            if stable and count_agrees and not current_pass_drift:
+                state = dict(state)
+                state["completed"] = True
+                state["reason_code"] = None
+                state["final_map_sha256"] = pass_digest
+                state = _commit(staging, state)
+                return _result(staging, state, reason=None)
+            if state["pass_number"] >= config.max_convergence_passes:
+                reason = "FILL_PROVIDER_COUNT" if not count_agrees else "FILL_NONCONVERGENT"
+                state = _stop(staging, state, reason)
+                return _result(staging, state, reason=reason)
+            state = dict(state)
+            state["previous_pass_map_sha256"] = pass_digest
+            state["pass_number"] += 1
+            state["pass_start_index"] = len(state["page_shards"])
+            state["next_cursor"] = _initial_cursor(config)
+            state["provider_count"] = None
+            state["count_basis"] = None
+            state["pass_drift"] = False
+            state = _commit(staging, state)
+            after_page = _preflight_limit(state, config.limits)
+            if after_page is not None:
+                state = _stop(staging, state, after_page)
+                return _result(staging, state, reason=after_page)
+            if max_pages_this_invocation == invocation_pages:
+                reason = (
+                    "INVOCATION_PAGE_LIMIT" if config.protocol == "datagov_v4" else "FILL_PAUSED"
+                )
+                state = _stop(staging, state, reason)
+                return _result(staging, state, reason=reason)
+            current_pass_map = {}
+            current_pass_drift = False
+            seen_cursors = set()
+            seen_responses = set()
+
+
+def _protocol_parts(protocol: str) -> tuple[str, SourceHarvester, Any]:
+    if protocol == "ckan":
+        return "ckan", CkanHarvester(), CKAN_LIMITS
+    if protocol == "datagov_v4":
+        return "datagov_v4", DatagovV4Harvester(), DATAGOV_V4_LIMITS
+    if protocol == "stac":
+        return "stac", StacHarvester(), STAC_LIMITS
+    if protocol == "sdmx":
+        return "sdmx", SdmxHarvester(), SDMX_LIMITS
+    raise CatalogFillRefused("FILL_PROTOCOL", "fill.protocol", "unsupported protocol")
+
+
+def _initial_state(config: CatalogFillConfig) -> dict[str, Any]:
+    state = {
+        "config_sha256": config.digest,
+        "endpoint": config.endpoint,
+        "harvester_coordinate": config.coordinate,
+        "predecessor_sha256": config.predecessor_sha256,
+        "next_cursor": _initial_cursor(config),
+        "page_shards": [],
+        "response_shards": [],
+        "budget": {
+            "requests": 0,
+            "responses": 0,
+            "pages": 0,
+            "records": 0,
+            "network_bytes": 0,
+            "elapsed_ns": 0,
+            "skipped_records": 0,
+        },
+        "provider_count": None,
+        "count_basis": None,
+        "pass_number": 1,
+        "pass_start_index": 0,
+        "previous_pass_map_sha256": None,
+        "pass_drift": False,
+        "completed": False,
+        "reason_code": None,
+        "final_map_sha256": None,
+    }
+    if config.protocol == "datagov_v4":
+        state["last_rate_observation"] = None
+        state["resume_not_before_epoch_seconds"] = None
+    return state
+
+
+def _initial_cursor(config: CatalogFillConfig) -> dict[str, str] | None:
+    if config.protocol == "ckan":
+        return HarvestCursor(
+            protocol="ckan", kind="offset", value=f"0:{config.page_size}"
+        ).to_dict()
+    return None
+
+
+def _validate_state(state: dict[str, Any], config: CatalogFillConfig) -> None:
+    expected = (
+        state.get("config_sha256") == config.digest
+        and state.get("endpoint") == config.endpoint
+        and state.get("harvester_coordinate") == config.coordinate
+        and state.get("predecessor_sha256") == config.predecessor_sha256
+    )
+    if not expected:
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CONFIG", "fill.staging.state", "checkpoint coordinate differs"
+        )
+    required = {
+        "config_sha256",
+        "endpoint",
+        "harvester_coordinate",
+        "predecessor_sha256",
+        "next_cursor",
+        "page_shards",
+        "response_shards",
+        "budget",
+        "provider_count",
+        "count_basis",
+        "pass_number",
+        "pass_start_index",
+        "previous_pass_map_sha256",
+        "pass_drift",
+        "completed",
+        "reason_code",
+        "final_map_sha256",
+        "schema_version",
+        "generation",
+    }
+    if config.protocol == "datagov_v4":
+        required.update({"last_rate_observation", "resume_not_before_epoch_seconds"})
+    if set(state) != required:
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.state", "state keys differ"
+        )
+    if not isinstance(state["page_shards"], list) or not isinstance(state["response_shards"], list):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.state", "shard lists are invalid"
+        )
+    if type(state["pass_start_index"]) is not int or not 0 <= state["pass_start_index"] <= len(
+        state["page_shards"]
+    ):
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.state", "pass offset is invalid"
+        )
+    _cursor_from_dict(state["next_cursor"])
+
+
+def _cursor_from_dict(value: Any) -> HarvestCursor | None:
+    if value is None:
+        return None
+    if not isinstance(value, dict) or set(value) != {"protocol", "kind", "value"}:
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.cursor", "cursor envelope is invalid"
+        )
+    try:
+        return HarvestCursor(protocol=value["protocol"], kind=value["kind"], value=value["value"])
+    except Exception as error:
+        raise CatalogFillRefused(
+            "FILL_CHECKPOINT_CORRUPT", "fill.staging.cursor", "cursor is invalid"
+        ) from error
+
+
+def _request_url(config: CatalogFillConfig, cursor: HarvestCursor | None) -> str:
+    if config.protocol == "ckan":
+        if cursor is None:
+            raise CatalogFillRefused("FILL_CURSOR", "fill.cursor", "CKAN cursor is required")
+        start, rows = cursor.value.split(":", 1)
+        parts = urlsplit(config.endpoint)
+        query = [
+            (key, value) for key, value in parse_qsl(parts.query) if key not in {"start", "rows"}
+        ]
+        query.extend((("rows", rows), ("start", start)))
+        return urlunsplit((parts.scheme, parts.netloc, parts.path, urlencode(sorted(query)), ""))
+    if config.protocol == "stac" and cursor is not None:
+        return cursor.value
+    if config.protocol == "datagov_v4":
+        if cursor is not None and cursor.protocol != "datagov_v4":
+            raise CatalogFillRefused("FILL_CURSOR", "fill.cursor", "protocol mismatch")
+        query = [("sort", config.sort), ("per_page", str(config.page_size))]
+        if cursor is not None:
+            query.append(("after", cursor.value))
+        return f"{config.endpoint}?{urlencode(query)}"
+    return config.endpoint
+
+
+def _validate_response_endpoint(
+    endpoint: str, requested: str, response: HarvestResponseEvidence
+) -> None:
+    if response.final_url != requested:
+        raise CatalogFillRefused(
+            "FILL_ENDPOINT_DRIFT",
+            "fill.response.final_url",
+            "zero-redirect response URL differs from the exact request",
+        )
+    base = urlsplit(endpoint)
+    requested_parts = urlsplit(requested)
+    for candidate, required_path in (
+        (requested, base.path),
+        (response.final_url, requested_parts.path),
+    ):
+        parsed = urlsplit(candidate)
+        if (
+            parsed.scheme != "https"
+            or parsed.hostname != base.hostname
+            or parsed.port != base.port
+            or parsed.path != required_path
+        ):
+            raise CatalogFillRefused(
+                "FILL_ENDPOINT_DRIFT",
+                "fill.response.final_url",
+                "continuation left endpoint origin",
+            )
+
+
+def _validate_continuation(endpoint: str, continuation: str) -> None:
+    base = urlsplit(endpoint)
+    parsed = urlsplit(continuation)
+    if (
+        parsed.scheme != "https"
+        or parsed.hostname != base.hostname
+        or parsed.port != base.port
+        or parsed.path != base.path
+    ):
+        raise CatalogFillRefused(
+            "FILL_ENDPOINT_DRIFT", "fill.page.next_cursor", "continuation left endpoint origin"
+        )
+
+
+def _page_count_drift(state: dict[str, Any], page: HarvestPage) -> bool:
+    basis = state["count_basis"]
+    count = state["provider_count"]
+    return (basis is not None and basis != page.count_basis) or (
+        count is not None and page.provider_count != count
+    )
+
+
+def _refuse_loops(
+    seen_cursors: set[str],
+    seen_responses: set[str],
+    cursor: HarvestCursor | None,
+    page: HarvestPage,
+) -> None:
+    requested = cursor.to_dict() if cursor is not None else None
+    requested_digest = canonical_sha256(requested)
+    response_digest = page.evidence.content_sha256
+    if requested_digest in seen_cursors:
+        raise CatalogFillRefused("FILL_CURSOR_LOOP", "fill.cursor", "cursor repeated")
+    if response_digest in seen_responses:
+        raise CatalogFillRefused("FILL_RESPONSE_LOOP", "fill.response", "response repeated")
+    if page.next_cursor is not None and page.next_cursor.to_dict() == requested:
+        raise CatalogFillRefused("FILL_CURSOR_LOOP", "fill.cursor", "next cursor did not advance")
+
+
+def _pass_loop_coordinates(
+    staging: FillStaging, state: dict[str, Any]
+) -> tuple[set[str], set[str]]:
+    cursors: set[str] = set()
+    responses: set[str] = set()
+    for digest in state["page_shards"][state["pass_start_index"] :]:
+        shard = staging.read_shard("page", digest)
+        cursors.add(canonical_sha256(shard["request_cursor"]))
+        responses.add(shard["evidence"]["content_sha256"])
+    return cursors, responses
+
+
+def _pass_map(staging: FillStaging, state: dict[str, Any]) -> tuple[dict[str, str], bool]:
+    result: dict[str, str] = {}
+    drift = bool(state["pass_drift"])
+    for digest in state["page_shards"][state["pass_start_index"] :]:
+        shard = staging.read_shard("page", digest)
+        for identifier, facts_digest in shard["id_facts"].items():
+            prior = result.get(identifier)
+            if prior is not None and prior != facts_digest:
+                drift = True
+            else:
+                result[identifier] = facts_digest
+    return result, drift
+
+
+def _pass_processed_count(staging: FillStaging, state: dict[str, Any]) -> int:
+    identifiers: set[str] = set()
+    skipped = 0
+    for digest in state["page_shards"][state["pass_start_index"] :]:
+        shard = staging.read_shard("page", digest)
+        identifiers.update(shard["id_facts"])
+        skipped += len(shard["skipped_record_ids"])
+    return len(identifiers) + skipped
+
+
+def _flagged_records(records: tuple[HarvestedRecord, ...]) -> list[dict[str, Any]]:
+    flagged: list[dict[str, Any]] = []
+    for record in records:
+        if record.protocol == "ckan":
+            status = ckan_licence_rights(record.license_id)[0]
+        elif record.protocol == "stac":
+            status = stac_licence_rights(record.license_id)[0]
+        elif record.protocol == "datagov_v4":
+            status = "unclear"
+        else:
+            status = "unclear"
+        if status in {"unclear", "prohibited"}:
+            flagged.append(
+                {
+                    "protocol": record.protocol,
+                    "record_id": record.record_id,
+                    "record_sha256": record.digest,
+                    "license_id": getattr(record, "license_id", None),
+                    "license_uri": getattr(record, "license_uri", None),
+                    "rights_status": status,
+                    "authority": "human_review_required",
+                }
+            )
+    return sorted(flagged, key=lambda item: (item["protocol"], item["record_id"]))
+
+
+def _preflight_limit(state: dict[str, Any], limits: CatalogFillLimits) -> str | None:
+    budget = state["budget"]
+    checks = (
+        ("requests", limits.max_requests, "FILL_REQUEST_LIMIT"),
+        ("responses", limits.max_responses, "FILL_RESPONSE_LIMIT"),
+        ("pages", limits.max_pages, "FILL_PAGE_LIMIT"),
+        ("records", limits.max_records, "FILL_RECORD_LIMIT"),
+        ("network_bytes", limits.max_aggregate_bytes, "FILL_AGGREGATE_BYTE_LIMIT"),
+        ("elapsed_ns", limits.max_wall_time_ns, "FILL_WALL_TIME_LIMIT"),
+    )
+    for field, maximum, code in checks:
+        if budget[field] >= maximum:
+            return code
+    return None
+
+
+def _remaining_transport_limits(state: dict[str, Any], limits: CatalogFillLimits, transport):
+    """Narrow the next fetch to the aggregate budget still authorized before it starts."""
+
+    budget = state["budget"]
+    remaining_requests = min(
+        limits.max_requests - budget["requests"],
+        limits.max_responses - budget["responses"],
+        transport.max_requests,
+    )
+    remaining_bytes = min(
+        limits.max_aggregate_bytes - budget["network_bytes"],
+        limits.max_response_bytes,
+        transport.max_response_bytes,
+    )
+    remaining_ns = limits.max_wall_time_ns - budget["elapsed_ns"]
+    if remaining_requests < 1 or remaining_bytes < 1 or remaining_ns < 1:
+        raise CatalogFillRefused(
+            "FILL_LIMIT", "fill.limits", "no aggregate transport budget remains"
+        )
+    remaining_seconds = remaining_ns / 1_000_000_000
+    return replace(
+        transport,
+        max_response_bytes=remaining_bytes,
+        max_aggregate_response_bytes=min(
+            limits.max_aggregate_bytes - budget["network_bytes"],
+            transport.max_aggregate_response_bytes,
+        ),
+        max_requests=remaining_requests,
+        max_redirects=min(transport.max_redirects, remaining_requests - 1),
+        connect_timeout_seconds=min(transport.connect_timeout_seconds, remaining_seconds),
+        read_timeout_seconds=min(transport.read_timeout_seconds, remaining_seconds),
+        total_timeout_seconds=min(transport.total_timeout_seconds, remaining_seconds),
+    )
+
+
+def _late_limit(
+    state: dict[str, Any], limits: CatalogFillLimits, response: HarvestResponseEvidence
+) -> str | None:
+    if response.content_bytes > limits.max_response_bytes:
+        return "FILL_RESPONSE_BYTE_LIMIT"
+    budget = state["budget"]
+    checks = (
+        (budget["requests"], limits.max_requests, "FILL_REQUEST_LIMIT"),
+        (budget["responses"], limits.max_responses, "FILL_RESPONSE_LIMIT"),
+        (budget["network_bytes"], limits.max_aggregate_bytes, "FILL_AGGREGATE_BYTE_LIMIT"),
+        (budget["elapsed_ns"], limits.max_wall_time_ns, "FILL_WALL_TIME_LIMIT"),
+    )
+    for value, maximum, code in checks:
+        if value > maximum:
+            return code
+    return None
+
+
+def _v4_transport_terminal_reason(
+    transport_code: str, state: dict[str, Any], limits: CatalogFillLimits
+) -> str:
+    """Normalize real transport cap refusals to durable fill-budget reasons."""
+
+    budget = state["budget"]
+    if budget["network_bytes"] > limits.max_aggregate_bytes:
+        return "FILL_AGGREGATE_BYTE_LIMIT"
+    if budget["elapsed_ns"] > limits.max_wall_time_ns:
+        return "FILL_WALL_TIME_LIMIT"
+    if (
+        transport_code == "AGGREGATE_RESPONSE_LIMIT"
+        and budget["network_bytes"] >= limits.max_aggregate_bytes
+    ):
+        return "FILL_AGGREGATE_BYTE_LIMIT"
+    if transport_code == "TOTAL_TIMEOUT" and budget["elapsed_ns"] >= limits.max_wall_time_ns:
+        return "FILL_WALL_TIME_LIMIT"
+    if transport_code == "RESPONSE_LIMIT":
+        return "FILL_RESPONSE_BYTE_LIMIT"
+    return transport_code
+
+
+def _v4_prospective_terminal_reason(
+    staging: FillStaging,
+    state: dict[str, Any],
+    *,
+    config: CatalogFillConfig,
+    request_item: Any,
+    response_item: Any,
+    response: HarvestResponseEvidence | None,
+) -> str | None:
+    """Reproduce a rejected page/record admission from its authenticated raw tail."""
+
+    if state.get("reason_code") not in {"FILL_PAGE_LIMIT", "FILL_RECORD_LIMIT"}:
+        return None
+    if (
+        response is None
+        or response.status != 200
+        or not isinstance(request_item, dict)
+        or not isinstance(response_item, dict)
+        or response_item.get("request_attempt") != state["budget"]["requests"]
+        or response_item.get("request_url") != request_item.get("request_url")
+        or response.final_url != request_item.get("request_url")
+        or not _SHA256.fullmatch(response_item.get("raw_response_sha256", ""))
+    ):
+        return None
+    raw_digest = response_item["raw_response_sha256"]
+    request_url = request_item["request_url"]
+    try:
+        payload = staging.read_blob("raw_response", raw_digest)
+        page = DatagovV4Harvester().parse_page(
+            payload,
+            uri=request_url,
+            observed_at=config.observed_at,
+            evidence=EvidenceReference(
+                uri=request_url,
+                observed_at=config.observed_at,
+                content_sha256=raw_digest,
+                media_type="application/json",
+            ),
+            response_evidence=response,
+            cursor=_cursor_from_dict(request_item.get("cursor")),
+        )
+    except (CatalogHarvestError, CatalogFillRefused, CanonicalJSONError, TypeError, ValueError):
+        raise _v4_evidence_corrupt("rejected page cannot be reproduced from its raw tail") from None
+    return _projected_page_limit(state, config.limits, page)
+
+
+def _projected_page_limit(
+    state: dict[str, Any], limits: CatalogFillLimits, page: HarvestPage
+) -> str | None:
+    if state["budget"]["pages"] + 1 > limits.max_pages:
+        return "FILL_PAGE_LIMIT"
+    if state["budget"]["records"] + len(page.records) > limits.max_records:
+        return "FILL_RECORD_LIMIT"
+    return None
+
+
+def _stop(staging: FillStaging, state: dict[str, Any], reason: str) -> dict[str, Any]:
+    if state.get("reason_code") == reason:
+        return state
+    next_state = dict(state)
+    next_state["reason_code"] = reason
+    return _commit(staging, next_state)
+
+
+def _commit(
+    staging: FillStaging,
+    next_state: dict[str, Any],
+) -> dict[str, Any]:
+    return staging.commit(
+        next_state,
+        expected_generation=next_state["generation"],
+        expected_checkpoint_digest=staging.state_digest(),
+    )
+
+
+def _result(
+    staging: FillStaging, state: dict[str, Any], *, reason: str | None
+) -> CatalogFillResult:
+    final_map, _ = _pass_map(staging, state)
+    flags: dict[tuple[str, str], dict[str, Any]] = {}
+    for digest in state["page_shards"][state["pass_start_index"] :]:
+        shard = staging.read_shard("page", digest)
+        for item in shard["flagged_rights"]:
+            flags[(item["protocol"], item["record_id"])] = item
+    flagged_queue = [flags[key] for key in sorted(flags)]
+    receipt = CatalogFillReceipt(
+        config_sha256=state["config_sha256"],
+        endpoint=state["endpoint"],
+        harvester_coordinate=state["harvester_coordinate"],
+        predecessor_sha256=state["predecessor_sha256"],
+        status="complete" if state["completed"] else "incomplete",
+        reason_code=reason,
+        requests=state["budget"]["requests"],
+        responses=state["budget"]["responses"],
+        pages=state["budget"]["pages"],
+        records=state["budget"]["records"],
+        unique_records=len(final_map),
+        network_bytes=state["budget"]["network_bytes"],
+        elapsed_ns=state["budget"]["elapsed_ns"],
+        provider_count=state["provider_count"],
+        count_basis=state["count_basis"],
+        skipped_records=state["budget"]["skipped_records"],
+        flagged_rights=len(flagged_queue),
+        convergence_passes=state["pass_number"],
+        response_evidence_digests=tuple(state["response_shards"]),
+        page_shard_digests=tuple(state["page_shards"]),
+        flagged_queue_sha256=canonical_sha256(flagged_queue),
+        rate_observation=state.get("last_rate_observation"),
+        resume_not_before_epoch_seconds=state.get("resume_not_before_epoch_seconds"),
+    )
+    state_digest = staging.state_digest()
+    return CatalogFillResult(
+        status=receipt.status,
+        reason_code=reason,
+        receipt=receipt,
+        checkpoint_sha256=state_digest,
+        eligible_staging_sha256=state_digest if state["completed"] else None,
+    )
+
+
+def _clock(clock_ns: Callable[[], int]) -> int:
+    value = clock_ns()
+    if type(value) is not int or value < 0:
+        raise CatalogFillRefused("FILL_CLOCK", "fill.clock", "clock must return nonnegative int")
+    return value
+
+
+def _wall_clock(clock_seconds: Callable[[], int]) -> int:
+    value = clock_seconds()
+    if type(value) is not int or not 0 <= value <= (1 << 53) - 1:
+        raise CatalogFillRefused(
+            "FILL_CLOCK", "fill.wall_clock", "wall clock must return a nonnegative integer"
+        )
+    return value