mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/streaming_delta.py

@@ -0,0 +1,1097 @@
+"""Exact streaming delta classification over sorted authoring shards.
+
+The v1 classifier in :mod:`delta` reads one whole generation into dictionaries, and it explicitly
+requires the caller to hand it a complete provider-record-to-entry map.  At 548k provider records
+neither is possible: the map alone exceeds what ``canonical`` will serialize, and the entries alone
+exceed the plan's memory budget.  This module replaces that path for provider-scale sweeps while
+:mod:`delta` remains exactly as it is for bounded v1 fills.
+
+The shape of the work is a merge, not a join.
+
+*Two sorted streams, one record of lookahead.*  Authoring shards and identity packs use ascending
+UTF-8 byte order of the exact provider identifier.  Classifying
+one sweep is therefore a linear two-stream merge with one record of lookahead per stream: a current
+record with no predecessor is new, a predecessor with no current record is absent, and a matched
+pair is unchanged or changed depending on one digest comparison.  Nothing is indexed, sorted, or
+held beyond one input shard, one history pack, and bounded output buffers.
+
+*One digest decides.*  The comparison is the semantic classification digest: the entry's facts,
+their provider provenance, and its identity -- deliberately without the managed version link, the
+append-only observations, or the sealed vector instances.  The semantic digest excludes
+``last_harvested_date`` and ``popularity`` from the provider record's semantic digest. A nightly
+re-harvest that rewrites only those values reaches this comparison as *equal*: the sweep appends one
+observation, reuses the entry and its member references, and schedules no encode at all.
+
+*Every provider record lands somewhere.*  A record is new, changed, unchanged, flagged, skipped, or
+failed -- and the six counts sum to exactly the provider record measure, which is checked rather
+than asserted in prose.  ``requests``, ``responses``, ``pages``, and provider records stay four
+separate numbers, because conflating them is how a sweep starts reporting page counts as coverage.
+A skipped record keeps its authoring reason code in the emitted shard and a durable disposition in
+its identity history; it never quietly leaves the denominator.
+
+*A delta is complete or it is nothing.*  Output shards are content-addressed and immutable, the
+successor identity generation is published before the delta manifest, and the delta manifest is
+published last.  A bound reached mid-classification refuses rather than publishing a partial
+manifest, so no consumer can ever read a truncated classification as a whole one.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import time
+from collections.abc import Callable, Iterator
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    canonical_sha256,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.authoring_policy import (
+    AUTHORING_DISPOSITIONS,
+    AuthoringPolicy,
+    resolve_authoring_policy,
+)
+from mostlyright.data_harness.sources.catalog.authoring_shards import (
+    AUTHORING_MANIFEST_SCHEMA,
+    AUTHORING_SHARD_SCHEMA,
+    MANIFEST_FILENAME,
+    MAX_AUTHORING_MANIFEST_BYTES,
+    MAX_AUTHORING_SHARD_ENTRIES,
+    SHARDS_DIRNAME,
+)
+from mostlyright.data_harness.sources.catalog.bounded_io import (
+    BoundedReadFailure,
+    read_bounded_path,
+)
+from mostlyright.data_harness.sources.catalog.coverage import coverage_is_valid
+from mostlyright.data_harness.sources.catalog.entry_v2 import (
+    CatalogEntryV2,
+    catalog_entry_v2_from_dict,
+)
+from mostlyright.data_harness.sources.catalog.generation_receipt import retained_child_directory
+from mostlyright.data_harness.sources.catalog.identity_history import (
+    MAX_IDENTITY_PACK_RECORDS,
+    DurableDirectory,
+    IdentityHistory,
+    IdentityHistoryStore,
+    IdentityHistoryWriter,
+    ObservationEvidence,
+    ProviderIdentity,
+    advance_identity_history,
+    derive_entry_id,
+    observe_identity_absence,
+    record_identity_disposition,
+)
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+DELTA_SHARD_SCHEMA = "harness-catalog-delta-shard.v1"
+DELTA_MANIFEST_SCHEMA = "harness-catalog-delta-manifest.v1"
+DELTA_RECEIPT_SCHEMA = "mr-data-catalog-delta.v2"
+
+DELTA_MANIFEST_FILENAME = "delta-manifest.json"
+DELTA_SHARDS_DIRNAME = "shards"
+DELTA_IDENTITY_DIRNAME = "identity"
+
+# The complete closed partition of one sweep. The first six describe current provider records and
+# sum to the provider record measure; ``absent`` describes predecessor records instead.
+DELTA_CLASSIFICATIONS = (
+    "new",
+    "changed",
+    "unchanged",
+    "absent",
+    "flagged",
+    "skipped",
+    "failed",
+)
+_CURRENT_CLASSIFICATIONS = ("new", "changed", "unchanged", "flagged", "skipped", "failed")
+
+MAX_DELTA_SHARD_ENTRIES = 1_000
+MAX_DELTA_SHARD_BYTES = 8 * 1024 * 1024
+MAX_DELTA_MANIFEST_BYTES = 8 * 1024 * 1024
+MAX_DELTA_INPUT_SHARD_BYTES = 8 * 1024 * 1024
+
+DELTA_ABSENCE_REASON = "DELTA_ABSENT"
+
+_OPEN_MEMBER = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+
+
+class CatalogDeltaStreamRefused(SourceContractError):
+    """A stable refusal of a delta input, coordinate, bound, or durable output."""
+
+
+@dataclass(frozen=True)
+class SweepMeasures:
+    """The four distinct measures of one sweep; none of them stands in for another."""
+
+    requests: int
+    responses: int
+    pages: int
+    provider_records: int
+
+    def __post_init__(self) -> None:
+        for name in ("requests", "responses", "pages", "provider_records"):
+            value = getattr(self, name)
+            if type(value) is not int or value < 0:
+                raise CatalogDeltaStreamRefused(
+                    "DELTA_MEASURE_CONTRACT",
+                    f"measures.{name}",
+                    "must be a non-negative integer",
+                )
+        if self.responses > self.requests:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_MEASURE_CONTRACT",
+                "measures.responses",
+                "a sweep cannot retain more responses than it made requests",
+            )
+        if self.pages > self.responses:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_MEASURE_CONTRACT",
+                "measures.pages",
+                "a sweep cannot normalize more pages than it retained responses",
+            )
+
+    def to_dict(self) -> dict[str, int]:
+        return {
+            "requests": self.requests,
+            "responses": self.responses,
+            "pages": self.pages,
+            "provider_records": self.provider_records,
+        }
+
+
+@dataclass(frozen=True)
+class StreamingDeltaLimits:
+    """Explicit caller-supplied classification bounds; a bound reached is a refusal."""
+
+    max_records: int = 2_000_000
+    max_shard_entries: int = MAX_DELTA_SHARD_ENTRIES
+    max_shard_bytes: int = MAX_DELTA_SHARD_BYTES
+    max_pack_records: int = MAX_IDENTITY_PACK_RECORDS
+    max_disk_bytes: int = 20 * 1024 * 1024 * 1024
+    max_wall_seconds: int = 12 * 60 * 60
+
+    def __post_init__(self) -> None:
+        for name in (
+            "max_records",
+            "max_shard_entries",
+            "max_shard_bytes",
+            "max_pack_records",
+            "max_disk_bytes",
+            "max_wall_seconds",
+        ):
+            value = getattr(self, name)
+            if type(value) is not int or value < 1:
+                raise CatalogDeltaStreamRefused(
+                    "DELTA_LIMIT", f"limits.{name}", "must be a positive integer"
+                )
+        if (
+            self.max_shard_entries > MAX_DELTA_SHARD_ENTRIES
+            or self.max_shard_bytes > MAX_DELTA_SHARD_BYTES
+        ):
+            raise CatalogDeltaStreamRefused(
+                "DELTA_LIMIT",
+                "limits.max_shard_entries",
+                "shard bounds may not exceed the fixed residency contract",
+            )
+
+    def to_dict(self) -> dict[str, int]:
+        return {
+            "max_records": self.max_records,
+            "max_shard_entries": self.max_shard_entries,
+            "max_shard_bytes": self.max_shard_bytes,
+            "max_pack_records": self.max_pack_records,
+            "max_disk_bytes": self.max_disk_bytes,
+            "max_wall_seconds": self.max_wall_seconds,
+        }
+
+
+@dataclass(frozen=True)
+class StreamingDeltaResult:
+    """One complete classification's exact durable outcome."""
+
+    manifest_sha256: str
+    identity_manifest_sha256: str
+    counts: dict[str, int]
+    receipt: dict[str, Any]
+
+
+def semantic_classification_digest(entry: CatalogEntryV2) -> str:
+    """Digest exactly what decides sameness: identity, facts, and their provider provenance.
+
+    The managed version link, the append-only observations, and the sealed vector instances are
+    deliberately excluded. An author always spells a fresh entry as version one, so including the
+    version would make every re-observation look changed; including observations would make every
+    sweep look changed, which is precisely why observations are excluded from the provider digest.
+    """
+
+    if not isinstance(entry, CatalogEntryV2):
+        raise CatalogDeltaStreamRefused("DELTA_INPUT_SHARD", "entry", "must be a CatalogEntryV2")
+    payload = entry.to_dict()
+    for managed in ("entry_version", "previous_semantic_digest", "observations", "vectors"):
+        del payload[managed]
+    return canonical_sha256(payload)
+
+
+def run_streaming_delta(
+    *,
+    authoring_root: Path,
+    expected_authoring_manifest_sha256: str,
+    output_root: Path,
+    output_descriptor: int | None = None,
+    measures: SweepMeasures,
+    observed_at: str,
+    limits: StreamingDeltaLimits | None = None,
+    predecessor_root: Path | None = None,
+    expected_predecessor_manifest_sha256: str | None = None,
+    predecessor_root_descriptor: int | None = None,
+    monotonic: Callable[[], float] = time.monotonic,
+) -> StreamingDeltaResult:
+    """Classify one authored generation against its predecessor identity/history exactly."""
+
+    bounds = StreamingDeltaLimits() if limits is None else limits
+    if not isinstance(bounds, StreamingDeltaLimits):
+        raise CatalogDeltaStreamRefused("DELTA_LIMIT", "limits", "must be StreamingDeltaLimits")
+    if not isinstance(measures, SweepMeasures):
+        raise CatalogDeltaStreamRefused(
+            "DELTA_MEASURE_CONTRACT", "measures", "must be SweepMeasures"
+        )
+    if not _is_digest(expected_authoring_manifest_sha256):
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_DIGEST",
+            "expected_authoring_manifest_sha256",
+            "must be a lowercase SHA-256",
+        )
+    if (predecessor_root is None) != (expected_predecessor_manifest_sha256 is None):
+        raise CatalogDeltaStreamRefused(
+            "DELTA_PREDECESSOR_REQUIRED",
+            "predecessor",
+            "a predecessor is named by both its root and its exact manifest digest, or not at all",
+        )
+    deadline = monotonic() + bounds.max_wall_seconds
+
+    manifest, manifest_sha256 = _read_authoring_manifest(
+        Path(authoring_root), expected_authoring_manifest_sha256
+    )
+    policy = _resolve_policy(manifest)
+    _verify_measures(measures, manifest)
+    current = _AuthoredStream(Path(authoring_root), manifest, bounds)
+
+    predecessor: IdentityHistoryStore | None = None
+    if predecessor_root is not None:
+        predecessor = IdentityHistoryStore.open(
+            Path(predecessor_root),
+            expected_manifest_sha256=expected_predecessor_manifest_sha256,
+            root_descriptor=predecessor_root_descriptor,
+        )
+        if predecessor.provider_id != policy.provider_id:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_PREDECESSOR_PROVIDER",
+                "predecessor.provider_id",
+                "the predecessor generation describes another provider",
+            )
+
+    identity_context = (
+        contextlib.nullcontext(None)
+        if output_descriptor is None
+        else retained_child_directory(
+            output_descriptor,
+            DELTA_IDENTITY_DIRNAME,
+            create=True,
+            label="delta.identity",
+            code_prefix="DELTA",
+            refusal=CatalogDeltaStreamRefused,
+        )
+    )
+    with identity_context as identity_descriptor:
+        identity_writer = IdentityHistoryWriter(
+            Path(output_root) / DELTA_IDENTITY_DIRNAME,
+            provider_id=policy.provider_id,
+            sweep_sha256=manifest_sha256,
+            predecessor_manifest_sha256=expected_predecessor_manifest_sha256,
+            max_pack_records=bounds.max_pack_records,
+            output_descriptor=identity_descriptor,
+        )
+        shards = _DeltaShardWriter(
+            Path(output_root),
+            bounds,
+            sweep_sha256=manifest_sha256,
+            output_descriptor=output_descriptor,
+        )
+        with shards.opened(), identity_writer.opened():
+            state = _merge(
+                current=current,
+                predecessor=predecessor,
+                policy=policy,
+                sweep_sha256=manifest_sha256,
+                observed_at=observed_at,
+                shards=shards,
+                identity_writer=identity_writer,
+                limits=bounds,
+                deadline=deadline,
+                monotonic=monotonic,
+            )
+            shards.flush_all()
+            _verify_denominator(state, manifest)
+            identity_manifest_sha256 = identity_writer.publish()
+            receipt = _receipt(
+                manifest=manifest,
+                manifest_sha256=manifest_sha256,
+                policy=policy,
+                measures=measures,
+                observed_at=observed_at,
+                state=state,
+                shards=shards,
+                limits=bounds,
+                identity_manifest_sha256=identity_manifest_sha256,
+                predecessor_manifest_sha256=expected_predecessor_manifest_sha256,
+            )
+            body = {key: value for key, value in receipt.items() if key != "manifest_sha256"}
+            delta_manifest_sha256 = shards.publish_manifest(
+                {**body, "schema_version": DELTA_MANIFEST_SCHEMA}
+            )
+    return StreamingDeltaResult(
+        manifest_sha256=delta_manifest_sha256,
+        identity_manifest_sha256=identity_manifest_sha256,
+        counts={name: state.counts[name] for name in DELTA_CLASSIFICATIONS},
+        receipt={**receipt, "manifest_sha256": delta_manifest_sha256},
+    )
+
+
+# ------------------------------------------------------------------------------------------
+# Inputs
+# ------------------------------------------------------------------------------------------
+
+
+def _read_authoring_manifest(root: Path, expected_sha256: str) -> tuple[dict[str, Any], str]:
+    raw = _read_bounded(root / MANIFEST_FILENAME, maximum=MAX_AUTHORING_MANIFEST_BYTES)
+    if raw is None:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_MANIFEST",
+            "authoring.manifest",
+            "no readable authored generation manifest at this root",
+        )
+    digest = sha256_bytes(raw)
+    if digest != expected_sha256:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_DIGEST",
+            "authoring.manifest",
+            "the retained authored generation is not the caller's exact manifest",
+        )
+    try:
+        manifest = parse_canonical_json(raw)
+    except CanonicalJSONError as error:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_MANIFEST", "authoring.manifest", "manifest is not canonical"
+        ) from error
+    expected_members = {
+        "schema_version",
+        "policy_id",
+        "policy_sha256",
+        "staging",
+        "coverage",
+        "counts",
+        "layer_text_digests",
+        "shards",
+        "root_sha256",
+    }
+    if (
+        not isinstance(manifest, dict)
+        or set(manifest) != expected_members
+        or manifest["schema_version"] != AUTHORING_MANIFEST_SCHEMA
+        or not isinstance(manifest["staging"], dict)
+        or not isinstance(manifest["counts"], dict)
+        or not isinstance(manifest["shards"], list)
+        or not coverage_is_valid(manifest["coverage"])
+    ):
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_MANIFEST", "authoring.manifest", "manifest contract differs"
+        )
+    body = {key: value for key, value in manifest.items() if key != "root_sha256"}
+    if canonical_sha256(body) != manifest["root_sha256"]:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_MANIFEST", "authoring.manifest", "manifest root digest differs"
+        )
+    for name in ("raw_response_index_count", "normalized_page_index_count", "unique_records"):
+        if type(manifest["staging"].get(name)) is not int:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_MANIFEST",
+                f"authoring.manifest.staging.{name}",
+                "the authored staging coordinate is incomplete",
+            )
+    for name in ("records", *AUTHORING_DISPOSITIONS):
+        if type(manifest["counts"].get(name)) is not int:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_MANIFEST",
+                f"authoring.manifest.counts.{name}",
+                "the authored disposition counts are incomplete",
+            )
+    return manifest, digest
+
+
+def _resolve_policy(manifest: dict[str, Any]) -> AuthoringPolicy:
+    policy = resolve_authoring_policy(manifest["policy_id"])
+    if manifest["policy_sha256"] != policy.digest:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_POLICY",
+            "authoring.manifest.policy_sha256",
+            "the authored generation names another spelling of this policy coordinate",
+        )
+    return policy
+
+
+def _verify_measures(measures: SweepMeasures, manifest: dict[str, Any]) -> None:
+    """Bind the sweep that was harvested to the corpus the authored generation drew from.
+
+    This used to read ``measures.provider_records != manifest["counts"]["records"]`` -- the sweep
+    found exactly as many records as authoring wrote -- which is the same statement as "authoring
+    is exhaustive", asserted rather than recorded. It was true of every generation that could be
+    published, so it looked like an invariant; it was really the *only* coverage the pipeline
+    could express.
+
+    It is now two statements, because it was always two facts. The sweep's unique record count is
+    the corpus the selection chose from, and it must equal what the coverage says it drew from --
+    a coverage cannot claim a corpus its own harvest never found. Separately, what the selection
+    says it took must equal what authoring actually wrote. Between them they say what the single
+    comparison used to say whenever the coverage is exhaustive, and they say something true
+    rather than nothing at all when it is not.
+    """
+
+    staging = manifest["staging"]
+    coverage = manifest["coverage"]
+    if (
+        measures.responses != staging["raw_response_index_count"]
+        or measures.pages != staging["normalized_page_index_count"]
+        or measures.provider_records != coverage["corpus_records"]
+    ):
+        raise CatalogDeltaStreamRefused(
+            "DELTA_MEASURE_MISMATCH",
+            "measures",
+            "responses, pages, and provider records must equal the authored generation's own "
+            "distinct counts",
+        )
+    if (
+        coverage["corpus_records"] != staging["unique_records"]
+        or coverage["selected_records"] != manifest["counts"]["records"]
+    ):
+        raise CatalogDeltaStreamRefused(
+            "DELTA_COVERAGE_MISMATCH",
+            "authoring.manifest.coverage",
+            "the stated coverage must draw from the staging coordinate's corpus and account for "
+            "every authored record",
+        )
+
+
+class _AuthoredStream:
+    """One authored generation as one ascending record stream, one shard resident."""
+
+    def __init__(self, root: Path, manifest: dict[str, Any], limits: StreamingDeltaLimits) -> None:
+        self.root = Path(root)
+        self.manifest = manifest
+        self.limits = limits
+        for descriptor in self.manifest["shards"]:
+            self._validate_descriptor(descriptor)
+        self._iterator = self._records()
+        self._peeked: tuple[bytes, dict[str, Any], str] | None = None
+        self._exhausted = False
+
+    def peek(self) -> tuple[bytes, dict[str, Any], str] | None:
+        if self._peeked is None and not self._exhausted:
+            self._peeked = next(self._iterator, None)
+            if self._peeked is None:
+                self._exhausted = True
+        return self._peeked
+
+    def take(self) -> tuple[bytes, dict[str, Any], str]:
+        item = self.peek()
+        if item is None:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_SHARD", "authoring.shards", "stream exhausted"
+            )
+        self._peeked = None
+        return item
+
+    def _records(self) -> Iterator[tuple[bytes, dict[str, Any], str]]:
+        last: bytes | None = None
+        for descriptor in self.manifest["shards"]:
+            for item in self._shard(descriptor):
+                if (
+                    not isinstance(item, dict)
+                    or not isinstance(item.get("provider_record_id"), str)
+                    or not item["provider_record_id"]
+                    or item.get("disposition") not in AUTHORING_DISPOSITIONS
+                    or not isinstance(item.get("reason_code"), str)
+                    or not isinstance(item.get("entry"), (dict, type(None)))
+                ):
+                    raise CatalogDeltaStreamRefused(
+                        "DELTA_INPUT_SHARD", "authoring.shard.items[]", "authored item differs"
+                    )
+                key = item["provider_record_id"].encode("utf-8")
+                if last is not None and key <= last:
+                    raise CatalogDeltaStreamRefused(
+                        "DELTA_INPUT_ORDER",
+                        "authoring.shards",
+                        "an authored generation must ascend by provider record",
+                    )
+                last = key
+                yield key, item, descriptor["sha256"]
+
+    def _shard(self, descriptor: Any) -> list[Any]:
+        self._validate_descriptor(descriptor)
+        raw = _read_bounded(
+            self.root / SHARDS_DIRNAME / f"{descriptor['sha256']}.json",
+            maximum=MAX_DELTA_INPUT_SHARD_BYTES,
+        )
+        if (
+            raw is None
+            or len(raw) != descriptor["bytes"]
+            or sha256_bytes(raw) != descriptor["sha256"]
+        ):
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_SHARD", "authoring.shard", "a committed authored shard differs"
+            )
+        try:
+            payload = parse_canonical_json(raw)
+        except CanonicalJSONError as error:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_SHARD", "authoring.shard", "shard is not canonical"
+            ) from error
+        if (
+            not isinstance(payload, dict)
+            or payload.get("schema_version") != AUTHORING_SHARD_SCHEMA
+            or not isinstance(payload.get("items"), list)
+            or len(payload["items"]) != descriptor["item_count"]
+        ):
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_SHARD", "authoring.shard", "shard contract differs"
+            )
+        return payload["items"]
+
+    @staticmethod
+    def _validate_descriptor(descriptor: Any) -> None:
+        if (
+            not isinstance(descriptor, dict)
+            or not _is_digest(descriptor.get("sha256"))
+            or type(descriptor.get("bytes")) is not int
+            or not 1 <= descriptor["bytes"] <= MAX_DELTA_INPUT_SHARD_BYTES
+            or type(descriptor.get("item_count")) is not int
+            or not 1 <= descriptor["item_count"] <= MAX_AUTHORING_SHARD_ENTRIES
+        ):
+            raise CatalogDeltaStreamRefused(
+                "DELTA_INPUT_MANIFEST", "authoring.manifest.shards[]", "shard descriptor is invalid"
+            )
+
+
+class _PredecessorStream:
+    """One predecessor identity generation as one ascending record stream, one pack resident."""
+
+    def __init__(self, store: IdentityHistoryStore | None) -> None:
+        self._iterator: Iterator[IdentityHistory] = iter(()) if store is None else store.stream()
+        self._peeked: IdentityHistory | None = None
+        self._exhausted = False
+
+    def peek(self) -> IdentityHistory | None:
+        if self._peeked is None and not self._exhausted:
+            self._peeked = next(self._iterator, None)
+            if self._peeked is None:
+                self._exhausted = True
+        return self._peeked
+
+    def take(self) -> IdentityHistory:
+        item = self.peek()
+        if item is None:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_PREDECESSOR_STREAM", "predecessor", "stream exhausted"
+            )
+        self._peeked = None
+        return item
+
+
+# ------------------------------------------------------------------------------------------
+# The merge
+# ------------------------------------------------------------------------------------------
+
+
+@dataclass
+class _MergeState:
+    counts: dict[str, int]
+    predecessor_records: int = 0
+    reused_entries: int = 0
+    reused_vectors: int = 0
+    scheduled_encodes: int = 0
+
+
+def _merge(
+    *,
+    current: _AuthoredStream,
+    predecessor: IdentityHistoryStore | None,
+    policy: AuthoringPolicy,
+    sweep_sha256: str,
+    observed_at: str,
+    shards: _DeltaShardWriter,
+    identity_writer: IdentityHistoryWriter,
+    limits: StreamingDeltaLimits,
+    deadline: float,
+    monotonic: Callable[[], float],
+) -> _MergeState:
+    previous = _PredecessorStream(predecessor)
+    state = _MergeState(counts=dict.fromkeys(DELTA_CLASSIFICATIONS, 0))
+    classified = 0
+    while True:
+        head = current.peek()
+        retained = previous.peek()
+        if head is None and retained is None:
+            return state
+        if classified >= limits.max_records:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_RECORD_LIMIT",
+                "delta.records",
+                "classification exceeds the caller's record bound; no manifest was published",
+            )
+        if monotonic() >= deadline:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_WALL_LIMIT",
+                "delta.records",
+                "classification exceeds the caller's wall bound; no manifest was published",
+            )
+        classified += 1
+        if retained is None or (head is not None and head[0] < retained.sort_key):
+            _key, item, shard_sha256 = current.take()
+            _classify_current(
+                item,
+                shard_sha256=shard_sha256,
+                retained=None,
+                policy=policy,
+                sweep_sha256=sweep_sha256,
+                observed_at=observed_at,
+                shards=shards,
+                identity_writer=identity_writer,
+                state=state,
+            )
+            continue
+        if head is None or retained.sort_key < head[0]:
+            _classify_absent(
+                previous.take(),
+                sweep_sha256=sweep_sha256,
+                observed_at=observed_at,
+                shards=shards,
+                identity_writer=identity_writer,
+                state=state,
+            )
+            continue
+        _key, item, shard_sha256 = current.take()
+        _classify_current(
+            item,
+            shard_sha256=shard_sha256,
+            retained=previous.take(),
+            policy=policy,
+            sweep_sha256=sweep_sha256,
+            observed_at=observed_at,
+            shards=shards,
+            identity_writer=identity_writer,
+            state=state,
+        )
+        state.predecessor_records += 1
+
+
+def _classify_current(
+    item: dict[str, Any],
+    *,
+    shard_sha256: str,
+    retained: IdentityHistory | None,
+    policy: AuthoringPolicy,
+    sweep_sha256: str,
+    observed_at: str,
+    shards: _DeltaShardWriter,
+    identity_writer: IdentityHistoryWriter,
+    state: _MergeState,
+) -> None:
+    record_id = item["provider_record_id"]
+    entry = _entry(item)
+    identity = _identity(record_id, entry, retained, policy)
+    if item["disposition"] != "authored":
+        history = record_identity_disposition(
+            retained,
+            identity=identity,
+            policy_sha256=policy.digest,
+            harvester_coordinate=policy.harvester_coordinate,
+            disposition=item["disposition"],
+            reason_code=item["reason_code"],
+            observed_at=observed_at,
+            sweep_sha256=sweep_sha256,
+            provider_record_sha256=(
+                None if entry is None else entry.provider_record.provider_record_sha256
+            ),
+        )
+        identity_writer.append(history)
+        state.counts[item["disposition"]] += 1
+        shards.emit(
+            {
+                "provider_record_id": record_id,
+                "classification": item["disposition"],
+                "disposition": item["disposition"],
+                "entry_id": identity.entry_id,
+                "entry_version": None,
+                "semantic_facts_digest": (
+                    None if entry is None else semantic_classification_digest(entry)
+                ),
+                "previous_semantic_digest": None,
+                "provider_record_sha256": (
+                    None if entry is None else entry.provider_record.provider_record_sha256
+                ),
+                "reason_code": item["reason_code"],
+                "reused_vector_sha256s": [],
+                "encode_scheduled": False,
+                "authoring_shard_sha256": shard_sha256,
+            }
+        )
+        return
+
+    if entry is None:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_SHARD",
+            "authoring.shard.items[].entry",
+            "an authored record must carry its entry",
+        )
+    digest = semantic_classification_digest(entry)
+    retained_head = None if retained is None else retained.head
+    history = advance_identity_history(
+        retained,
+        identity=identity,
+        policy_sha256=policy.digest,
+        harvester_coordinate=policy.harvester_coordinate,
+        semantic_facts_digest=digest,
+        provider_record_sha256=entry.provider_record.provider_record_sha256,
+        observation=_observation(entry, digest, sweep_sha256),
+    )
+    identity_writer.append(history)
+    head = history.head
+    assert head is not None
+    if retained_head is None:
+        classification = "new"
+    elif retained_head.semantic_facts_digest == digest:
+        classification = "unchanged"
+    else:
+        classification = "changed"
+    state.counts[classification] += 1
+    if classification == "unchanged":
+        state.reused_entries += 1
+        state.reused_vectors += len(head.vector_sha256s)
+    else:
+        state.scheduled_encodes += 1
+    shards.emit(
+        {
+            "provider_record_id": record_id,
+            "classification": classification,
+            "disposition": "authored",
+            "entry_id": identity.entry_id,
+            "entry_version": head.entry_version,
+            "semantic_facts_digest": head.semantic_facts_digest,
+            "previous_semantic_digest": head.previous_semantic_digest,
+            "provider_record_sha256": head.provider_record_sha256,
+            "reason_code": item["reason_code"],
+            "reused_vector_sha256s": list(head.vector_sha256s)
+            if classification == "unchanged"
+            else [],
+            "encode_scheduled": classification != "unchanged",
+            "authoring_shard_sha256": shard_sha256,
+        }
+    )
+
+
+def _classify_absent(
+    retained: IdentityHistory,
+    *,
+    sweep_sha256: str,
+    observed_at: str,
+    shards: _DeltaShardWriter,
+    identity_writer: IdentityHistoryWriter,
+    state: _MergeState,
+) -> None:
+    head = retained.head
+    history = (
+        retained
+        if head is None
+        else observe_identity_absence(retained, observed_at=observed_at, sweep_sha256=sweep_sha256)
+    )
+    identity_writer.append(history)
+    state.counts["absent"] += 1
+    state.predecessor_records += 1
+    shards.emit(
+        {
+            "provider_record_id": retained.identity.provider_record_id,
+            "classification": "absent",
+            "disposition": None,
+            "entry_id": retained.identity.entry_id,
+            "entry_version": None if head is None else head.entry_version,
+            "semantic_facts_digest": None if head is None else head.semantic_facts_digest,
+            "previous_semantic_digest": None,
+            "provider_record_sha256": None if head is None else head.provider_record_sha256,
+            "reason_code": DELTA_ABSENCE_REASON,
+            "reused_vector_sha256s": [],
+            "encode_scheduled": False,
+            "authoring_shard_sha256": None,
+        }
+    )
+
+
+def _entry(item: dict[str, Any]) -> CatalogEntryV2 | None:
+    if item["entry"] is None:
+        return None
+    try:
+        return catalog_entry_v2_from_dict(item["entry"])
+    except SourceContractError as error:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_SHARD",
+            "authoring.shard.items[].entry",
+            "an authored entry does not rebuild under the v2 contract",
+        ) from error
+
+
+def _identity(
+    record_id: str,
+    entry: CatalogEntryV2 | None,
+    retained: IdentityHistory | None,
+    policy: AuthoringPolicy,
+) -> ProviderIdentity:
+    if retained is not None:
+        identity = retained.identity
+    else:
+        identity = ProviderIdentity(
+            provider_id=policy.provider_id,
+            provider_record_id=record_id,
+            entry_id=derive_entry_id(policy.provider_id, record_id, prefix=policy.entry_id_prefix),
+        )
+    if entry is not None and entry.entry_id != identity.entry_id:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_IDENTITY_DERIVATION",
+            "authoring.shard.items[].entry.entry_id",
+            "an authored entry id must equal this provider record's persisted identity",
+        )
+    if entry is not None and entry.provider_record.provider_record_id != record_id:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_IDENTITY_DERIVATION",
+            "authoring.shard.items[].entry.provider_record",
+            "an authored entry must name the provider record it was authored from",
+        )
+    return identity
+
+
+def _observation(entry: CatalogEntryV2, digest: str, sweep_sha256: str) -> ObservationEvidence:
+    latest = entry.observations[-1]
+    return ObservationEvidence(
+        observed_at=latest.observed_at,
+        semantic_facts_digest=digest,
+        sweep_sha256=sweep_sha256,
+        response_evidence_sha256=latest.response_evidence_sha256,
+        page_evidence_sha256=latest.page_evidence_sha256,
+    )
+
+
+def _verify_denominator(state: _MergeState, manifest: dict[str, Any]) -> None:
+    """Every record this generation *holds* lands in exactly one current classification.
+
+    The denominator is the selected population rather than the swept one. Under an exhaustive
+    coverage the two are the same number and this is the check it has always been. Under a bound
+    they differ by exactly the records the rule did not select, and holding the classification to
+    the swept population would refuse every bounded generation -- not because a record went
+    missing, but because the check was measuring coverage while claiming to measure completeness.
+
+    Completeness of the *classification* is what is load-bearing here: no authored record may be
+    dropped, double-counted, or left unclassified. Coverage is stated in the coverage block, and
+    ``_verify_measures`` has already bound both of its populations to evidence.
+    """
+
+    selected = manifest["coverage"]["selected_records"]
+    classified = sum(state.counts[name] for name in _CURRENT_CLASSIFICATIONS)
+    if classified != selected:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_DENOMINATOR",
+            "delta.counts",
+            "every authored record must land in exactly one current classification; "
+            f"classified={classified}, selected_records={selected}",
+        )
+
+
+# ------------------------------------------------------------------------------------------
+# Outputs
+# ------------------------------------------------------------------------------------------
+
+
+class _DeltaShardWriter:
+    """Immutable content-addressed output shards, one bounded buffer per classification."""
+
+    def __init__(
+        self,
+        root: Path,
+        limits: StreamingDeltaLimits,
+        *,
+        sweep_sha256: str,
+        output_descriptor: int | None = None,
+    ) -> None:
+        self.limits = limits
+        self.sweep_sha256 = sweep_sha256
+        self.shards: list[dict[str, Any]] = []
+        self.chains: dict[str, str | None] = dict.fromkeys(DELTA_CLASSIFICATIONS)
+        self._directory = DurableDirectory(
+            root,
+            member_dirname=DELTA_SHARDS_DIRNAME,
+            code_prefix="DELTA",
+            refusal=CatalogDeltaStreamRefused,
+            root_descriptor=output_descriptor,
+        )
+        self._buffers: dict[str, list[dict[str, Any]]] = {
+            name: [] for name in DELTA_CLASSIFICATIONS
+        }
+        self._buffered_bytes: dict[str, int] = dict.fromkeys(DELTA_CLASSIFICATIONS, 0)
+
+    @contextlib.contextmanager
+    def opened(self) -> Iterator[_DeltaShardWriter]:
+        with self._directory.opened():
+            if (
+                self._directory.read(DELTA_MANIFEST_FILENAME, maximum=MAX_DELTA_MANIFEST_BYTES)
+                is not None
+            ):
+                raise CatalogDeltaStreamRefused(
+                    "DELTA_OUTPUT_EXISTS",
+                    "delta.output",
+                    "a published classification is immutable; classify into a new output root",
+                )
+            yield self
+
+    def emit(self, item: dict[str, Any]) -> None:
+        classification = item["classification"]
+        buffer = self._buffers[classification]
+        size = len(canonical_json_bytes(item))
+        if buffer and (
+            len(buffer) >= self.limits.max_shard_entries
+            or self._buffered_bytes[classification] + size + len(buffer) + 4_096
+            > self.limits.max_shard_bytes
+        ):
+            self.flush(classification)
+            buffer = self._buffers[classification]
+        buffer.append(item)
+        self._buffered_bytes[classification] += size
+        self.chains[classification] = canonical_sha256(
+            {"previous_sha256": self.chains[classification], "item": item}
+        )
+
+    def flush(self, classification: str) -> None:
+        buffer = self._buffers[classification]
+        if not buffer:
+            return
+        payload = {
+            "schema_version": DELTA_SHARD_SCHEMA,
+            "classification": classification,
+            "sweep_sha256": self.sweep_sha256,
+            "first_provider_record_id": buffer[0]["provider_record_id"],
+            "last_provider_record_id": buffer[-1]["provider_record_id"],
+            "items": buffer,
+        }
+        digest, size = self._directory.write_member(payload, maximum=self.limits.max_shard_bytes)
+        self.shards.append(
+            {
+                "sha256": digest,
+                "bytes": size,
+                "classification": classification,
+                "item_count": len(buffer),
+                "first_provider_record_id": payload["first_provider_record_id"],
+                "last_provider_record_id": payload["last_provider_record_id"],
+            }
+        )
+        self._buffers[classification] = []
+        self._buffered_bytes[classification] = 0
+        if self._directory.bytes_written > self.limits.max_disk_bytes:
+            raise CatalogDeltaStreamRefused(
+                "DELTA_DISK_LIMIT", "delta.output", "classification exceeds its local disk bound"
+            )
+
+    def flush_all(self) -> None:
+        for classification in DELTA_CLASSIFICATIONS:
+            self.flush(classification)
+
+    def publish_manifest(self, body: dict[str, Any]) -> str:
+        manifest = {**body, "root_sha256": canonical_sha256(body)}
+        return self._directory.publish(
+            DELTA_MANIFEST_FILENAME, manifest, maximum=MAX_DELTA_MANIFEST_BYTES
+        )
+
+
+def _receipt(
+    *,
+    manifest: dict[str, Any],
+    manifest_sha256: str,
+    policy: AuthoringPolicy,
+    measures: SweepMeasures,
+    observed_at: str,
+    state: _MergeState,
+    shards: _DeltaShardWriter,
+    limits: StreamingDeltaLimits,
+    identity_manifest_sha256: str,
+    predecessor_manifest_sha256: str | None,
+) -> dict[str, Any]:
+    counts = {name: state.counts[name] for name in DELTA_CLASSIFICATIONS}
+    return {
+        "schema_version": DELTA_RECEIPT_SCHEMA,
+        "observed_at": observed_at,
+        "inputs": {
+            "authoring_manifest_sha256": manifest_sha256,
+            "authoring_root_sha256": manifest["root_sha256"],
+            "policy_id": policy.policy_id,
+            "policy_sha256": policy.digest,
+            "provider_id": policy.provider_id,
+            "harvester_coordinate": policy.harvester_coordinate,
+            "staging": manifest["staging"],
+            "coverage": manifest["coverage"],
+            "authored_counts": manifest["counts"],
+            "predecessor_identity_manifest_sha256": predecessor_manifest_sha256,
+        },
+        "measures": measures.to_dict(),
+        "counts": {
+            **counts,
+            "provider_records": measures.provider_records,
+            "predecessor_records": state.predecessor_records,
+        },
+        # Three numbers, because a reader of this receipt alone must be able to tell a
+        # classification that dropped records from a coverage that never selected them.
+        "denominator": {
+            "provider_records": measures.provider_records,
+            "selected_records": manifest["coverage"]["selected_records"],
+            "classified": sum(state.counts[name] for name in _CURRENT_CLASSIFICATIONS),
+        },
+        "reuse": {
+            "reused_entries": state.reused_entries,
+            "reused_vectors": state.reused_vectors,
+            "scheduled_encodes": state.scheduled_encodes,
+        },
+        "outputs": {
+            "identity_manifest_sha256": identity_manifest_sha256,
+            "shard_count": len(shards.shards),
+            "shards": shards.shards,
+            "classification_digests": dict(shards.chains),
+        },
+        "limits": limits.to_dict(),
+        "manifest_sha256": None,
+    }
+
+
+def _read_bounded(path: Path, *, maximum: int) -> bytes | None:
+    try:
+        return read_bounded_path(path, maximum=maximum, missing_ok=True)
+    except BoundedReadFailure as error:
+        raise CatalogDeltaStreamRefused(
+            "DELTA_INPUT_PATH",
+            "delta.input",
+            "input members must be stable bounded single-link regular files",
+        ) from error
+
+
+def _is_digest(value: Any) -> bool:
+    return (
+        isinstance(value, str) and len(value) == 64 and all(c in "0123456789abcdef" for c in value)
+    )