mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/authoring.py

@@ -0,0 +1,1701 @@
+"""Authoring time: compute the four embedding layers once and seal them.
+
+This module is the only place in the catalog where text becomes a vector. It runs when a catalog
+is built, never when one is queried, and the query-side load path (``sealed.py``) computes
+nothing at all. That split is the whole of "models at authoring time only": the runtime path can
+read sealed integers and verify digests, and there is no code on it that could do anything else.
+
+The layer projections below are fixed and documented because they are silently load-bearing --
+changing how an entry is rendered into text changes every vector sealed from it, and a catalog
+whose vectors were computed under an older projection would rank differently while looking
+identical. Treat a projection change as a re-seal of the whole catalog.
+
+Nothing here reads a tenant's data. ``declared_columns`` and ``declared_vocabulary`` hold
+published names only, which is what makes the ``columns`` layer a fact about a public source
+rather than a summary of somebody's rows.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import secrets
+import stat
+import struct
+import tempfile
+from collections.abc import Iterable, Sequence
+from dataclasses import dataclass, replace
+from pathlib import Path
+
+from mostlyright.data_harness.acquisition.url_policy import EgressPolicy
+from mostlyright.data_harness.canonical import (
+    canonical_json_bytes,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.admission import (
+    admit_public_fact_bytes,
+    require_public_source_evidence,
+)
+from mostlyright.data_harness.sources.catalog.contracts import (
+    CATALOG_INDEX_CONTRACT_VERSION,
+    EMBEDDING_LAYERS,
+    CatalogEntry,
+    LayerVectorReference,
+    SourceQuirk,
+    catalog_entry_from_dict,
+)
+from mostlyright.data_harness.sources.catalog.embedding import (
+    BackendDescriptor,
+    EmbeddingBackend,
+)
+from mostlyright.data_harness.sources.catalog.harvest.protocol import HarvestedRecord
+from mostlyright.data_harness.sources.catalog.retrieval_manifest import (
+    LEXICAL_REPRESENTATION,
+    MAX_REPRESENTATION_DESCRIPTOR_BYTES,
+    NEURAL_REPRESENTATION,
+    REPRESENTATION_DESCRIPTOR_FILENAME,
+    RETRIEVAL_MANIFEST_FILENAME,
+    LoadedRetrievalCatalog,
+    LoadedRetrievalRepresentation,
+    RepresentationMember,
+    RetrievalManifestV2,
+    RetrievalRepresentation,
+    backend_descriptor_bytes,
+    backend_descriptor_sha256,
+    load_retrieval_catalog,
+)
+from mostlyright.data_harness.sources.contracts import (
+    RightsEvidence,
+    SourceContractError,
+    SourceObservation,
+)
+
+CATALOG_FILENAME = "catalog.json"
+VECTORS_DIRNAME = "vectors"
+VECTOR_SUFFIX = ".vec"
+MAX_CATALOG_MANIFEST_BYTES = 16 * 1024 * 1024
+
+_ENTRY_ID = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
+
+
+@dataclass(frozen=True)
+class CatalogSealResult:
+    """Exact result of one incremental durable v1 catalog seal."""
+
+    entries: tuple[CatalogEntry, ...]
+    manifest_sha256: str
+    reused_vector_sha256s: tuple[str, ...]
+    created_vector_sha256s: tuple[str, ...]
+    encoder_invocations: int
+
+
+@dataclass(frozen=True)
+class RetrievalSealResult:
+    """Result of one atomic v2 outer-manifest publication."""
+
+    manifest: RetrievalManifestV2
+    manifest_sha256: str
+    lexical_entries: tuple[CatalogEntry, ...]
+    neural_entries: tuple[CatalogEntry, ...] | None
+    lexical_reused_vector_sha256s: tuple[str, ...]
+    lexical_created_vector_sha256s: tuple[str, ...]
+    neural_reused_vector_sha256s: tuple[str, ...]
+    neural_created_vector_sha256s: tuple[str, ...]
+    lexical_encoder_invocations: int
+    neural_encoder_invocations: int
+
+
+def layer_text(entry: CatalogEntry, layer: str) -> str:
+    """Render one entry onto one retrieval layer, deterministically.
+
+    ``description`` is what the source calls itself. ``metadata`` is the facts a searcher would
+    phrase a filter with. ``columns`` is the published names a searcher would name a variable
+    with. ``profiles`` is the shape and cadence a searcher would sanity-check coverage against.
+    """
+
+    if not isinstance(entry, CatalogEntry):
+        raise SourceContractError("TYPE", "entry", "must be a CatalogEntry")
+    if layer not in EMBEDDING_LAYERS:
+        raise SourceContractError(
+            "ENUM",
+            "layer",
+            f"must be one of {list(EMBEDDING_LAYERS)}",
+        )
+    if layer == "description":
+        return f"{entry.title}\n{entry.description}"
+    if layer == "metadata":
+        return " ".join(
+            (
+                entry.publisher,
+                *entry.spatial_scope,
+                *entry.data_formats,
+                entry.access_kind,
+                entry.rights.status,
+                entry.rights.lawful_basis,
+            )
+        )
+    if layer == "columns":
+        parts: list[str] = [*entry.declared_columns, *entry.declared_vocabulary]
+        for quirk in entry.quirks:
+            parts.append(quirk.code)
+            parts.append(quirk.detail)
+        return " ".join(parts)
+    observation = entry.observation
+    row_count = "unknown" if entry.declared_row_count is None else str(entry.declared_row_count)
+    return " ".join(
+        (
+            observation.historical_start,
+            observation.historical_end,
+            observation.live_status,
+            f"update_frequency_seconds {observation.update_frequency_seconds}",
+            f"publication_delay_seconds {observation.publication_delay_seconds}",
+            *entry.data_formats,
+            f"rows {row_count}",
+        )
+    )
+
+
+def seal_catalog(
+    entries: Iterable[CatalogEntry],
+    *,
+    backend: EmbeddingBackend,
+    directory: Path,
+    egress_policy: EgressPolicy,
+    schema_version: str = CATALOG_INDEX_CONTRACT_VERSION,
+) -> tuple[CatalogEntry, ...]:
+    """Encode every layer of every entry, write the sealed catalog, and return the sealed entries.
+
+    This function is the only writer of the shared cross-tenant artifact, so the public-source
+    check runs here too and ``egress_policy`` is required rather than optional. Composition
+    (:func:`compose_catalog_entry`) checks the harvest seam, but an entry can also reach a catalog
+    by being authored or reconstructed from a file, and a check that only one of two doors runs is
+    the "enforced by review" pattern the tenancy rule rejects. Every entry is checked before any
+    byte is written, so a refused entry cannot leave a half-written tree behind.
+
+    Vector files are content-addressed, so two entries whose layer text is identical share one
+    file. The index is sorted by ``(entry_id, entry_version)`` before serialisation, which is what
+    makes a re-seal byte-identical rather than merely equivalent.
+    """
+
+    supplied = tuple(entries)
+    canonical_entries: list[CatalogEntry] = []
+    for index, entry in enumerate(supplied):
+        # Exact type at the boundary, followed by one immutable snapshot of the whole graph. A
+        # nested contract may itself be a subclass with a stateful to_dict; reading the original
+        # graph again would let admission inspect one projection and sealing serialize another.
+        if type(entry) is not CatalogEntry:
+            raise SourceContractError("TYPE", f"entries[{index}]", "must be a CatalogEntry")
+        captured_bytes = canonical_json_bytes(entry.to_dict())
+        admit_public_fact_bytes(captured_bytes)
+        canonical_entry = catalog_entry_from_dict(parse_canonical_json(captured_bytes))
+        if canonical_json_bytes(canonical_entry.to_dict()) != captured_bytes:
+            raise SourceContractError(
+                "CATALOG_ENTRY_PROJECTION",
+                f"entries[{index}]",
+                "the entry's captured projection cannot be rebuilt byte-for-byte as an exact "
+                "CatalogEntry; admission and sealing must inspect the same exact facts",
+            )
+        if canonical_entry.vectors:
+            raise SourceContractError(
+                "CATALOG_ALREADY_SEALED",
+                f"entries[{index}].vectors",
+                f"{canonical_entry.coordinate} already carries sealed vectors; re-sealing must "
+                "be explicit so a stale vector cannot outlive a fact change",
+            )
+        canonical_entries.append(canonical_entry)
+    materialized: Sequence[CatalogEntry] = tuple(canonical_entries)
+    if not isinstance(directory, Path):
+        raise SourceContractError("TYPE", "directory", "must be a Path")
+    for entry in materialized:
+        require_public_source_evidence(entry, egress_policy=egress_policy)
+
+    descriptor = backend.descriptor
+    vector_directory = directory / VECTORS_DIRNAME
+    vector_directory.mkdir(parents=True, exist_ok=True)
+
+    sealed: list[CatalogEntry] = []
+    for entry in materialized:
+        references: list[LayerVectorReference] = []
+        # Taken once, before any vector is written. `facts_digest` excludes `vectors`, so this is
+        # the same value the sealed entry will carry and the same one `SealedEmbeddingIndex.load`
+        # will re-derive -- which is what makes the binding checkable without re-encoding anything.
+        facts_digest = entry.facts_digest
+        for layer in EMBEDDING_LAYERS:
+            vector = backend.encode(layer_text(entry, layer))
+            if len(vector) != descriptor.dimensions:
+                raise SourceContractError(
+                    "EMBEDDING_DIMENSIONS",
+                    f"{entry.coordinate}.{layer}",
+                    f"backend returned {len(vector)} values, expected {descriptor.dimensions}",
+                )
+            packed = struct.pack(f">{descriptor.dimensions}i", *vector)
+            digest = sha256_bytes(packed)
+            _write_vector(vector_directory / f"{digest}{VECTOR_SUFFIX}", packed)
+            references.append(
+                LayerVectorReference(
+                    layer=layer,
+                    backend_id=descriptor.backend_id,
+                    backend_version=descriptor.backend_version,
+                    dimensions=descriptor.dimensions,
+                    quantization=descriptor.quantization,
+                    vector_sha256=digest,
+                    entry_facts_digest=facts_digest,
+                )
+            )
+        sealed.append(replace(entry, vectors=tuple(references)))
+
+    ordered = tuple(sorted(sealed, key=lambda item: (item.entry_id, item.entry_version)))
+    index_bytes = canonical_json_bytes(
+        {
+            "schema_version": schema_version,
+            "entries": [item.to_dict() for item in ordered],
+        }
+    )
+    _atomic_write(directory / CATALOG_FILENAME, index_bytes)
+    return ordered
+
+
+def seal_retrieval_catalog(
+    entries: Iterable[CatalogEntry],
+    *,
+    lexical_backend: EmbeddingBackend,
+    neural_backend: EmbeddingBackend | None,
+    directory: Path,
+    egress_policy: EgressPolicy,
+    fallback_reason: str | None = None,
+    expected_existing_manifest_sha256: str | None = None,
+) -> RetrievalSealResult:
+    """Seal independent v1 representations and publish one explicit v2 head.
+
+    Both representations are derived from one captured tuple of canonical facts.  Their immutable
+    generations are complete and read back before ``retrieval.json`` is replaced, making the
+    outer replacement the sole public linearization point.
+    """
+
+    supplied = tuple(entries)
+    captured = tuple(
+        _capture_entry(entry, index=index, sealed=False) for index, entry in enumerate(supplied)
+    )
+    ordered = tuple(sorted(captured, key=_entry_key))
+    if len({entry.entry_id for entry in ordered}) != len(ordered):
+        raise SourceContractError(
+            "DUPLICATE", "retrieval.entries", "a retrieval catalog has one entry per source"
+        )
+    if neural_backend is None and fallback_reason is None:
+        fallback_reason = "NEURAL_NOT_REQUESTED"
+    if neural_backend is not None and fallback_reason is not None:
+        raise SourceContractError(
+            "RETRIEVAL_FALLBACK",
+            "fallback_reason",
+            "a hybrid seal cannot also claim lexical-only fallback",
+        )
+    for entry in ordered:
+        require_public_source_evidence(entry, egress_policy=egress_policy)
+
+    shared_facts_sha256 = _shared_facts_sha256(ordered)
+    trusted_resume = _load_trusted_resume_catalog(
+        directory=directory,
+        expected_manifest_sha256=expected_existing_manifest_sha256,
+        expected_facts=ordered,
+        lexical_backend=lexical_backend,
+        neural_backend=neural_backend,
+        fallback_reason=fallback_reason,
+    )
+    if trusted_resume is not None:
+        return _result_from_trusted_resume(trusted_resume)
+    lexical_generation = _representation_generation(shared_facts_sha256, lexical_backend.descriptor)
+    lexical_directory = directory / LEXICAL_REPRESENTATION / lexical_generation
+    lexical_result = _seal_representation_generation(
+        reused_entries=(),
+        fresh_entries=ordered,
+        backend=lexical_backend,
+        directory=lexical_directory,
+        predecessor_directory=None,
+        predecessor_manifest_sha256=None,
+        egress_policy=egress_policy,
+    )
+    lexical_representation = _representation_from_seal(
+        root=directory,
+        name=LEXICAL_REPRESENTATION,
+        generation=lexical_generation,
+        descriptor=lexical_backend.descriptor,
+        result=lexical_result,
+    )
+
+    neural_result: CatalogSealResult | None = None
+    representations = [lexical_representation]
+    if neural_backend is not None:
+        neural_generation = _representation_generation(
+            shared_facts_sha256, neural_backend.descriptor
+        )
+        neural_directory = directory / NEURAL_REPRESENTATION / neural_generation
+        neural_result = _seal_representation_generation(
+            reused_entries=(),
+            fresh_entries=ordered,
+            backend=neural_backend,
+            directory=neural_directory,
+            predecessor_directory=None,
+            predecessor_manifest_sha256=None,
+            egress_policy=egress_policy,
+        )
+        representations.append(
+            _representation_from_seal(
+                root=directory,
+                name=NEURAL_REPRESENTATION,
+                generation=neural_generation,
+                descriptor=neural_backend.descriptor,
+                result=neural_result,
+            )
+        )
+
+    manifest = RetrievalManifestV2(
+        shared_facts_sha256=shared_facts_sha256,
+        representations=tuple(representations),
+        fallback_reason=fallback_reason,
+    )
+    # Reparse before publication so a writer bug cannot create a public envelope the reader would
+    # reject. All referenced members have already been durably installed and read back.
+    if RetrievalManifestV2.from_bytes(manifest.canonical_bytes) != manifest:
+        raise SourceContractError(
+            "RETRIEVAL_MANIFEST_READBACK", RETRIEVAL_MANIFEST_FILENAME, "reparse differs"
+        )
+    _publish_retrieval_manifest(directory, manifest.canonical_bytes)
+    return RetrievalSealResult(
+        manifest=manifest,
+        manifest_sha256=manifest.digest,
+        lexical_entries=lexical_result.entries,
+        neural_entries=None if neural_result is None else neural_result.entries,
+        lexical_reused_vector_sha256s=lexical_result.reused_vector_sha256s,
+        lexical_created_vector_sha256s=lexical_result.created_vector_sha256s,
+        neural_reused_vector_sha256s=()
+        if neural_result is None
+        else neural_result.reused_vector_sha256s,
+        neural_created_vector_sha256s=()
+        if neural_result is None
+        else neural_result.created_vector_sha256s,
+        lexical_encoder_invocations=lexical_result.encoder_invocations,
+        neural_encoder_invocations=0
+        if neural_result is None
+        else neural_result.encoder_invocations,
+    )
+
+
+def seal_retrieval_catalog_successor(
+    *,
+    lexical_reused_entries: tuple[CatalogEntry, ...],
+    fresh_entries: tuple[CatalogEntry, ...],
+    lexical_backend: EmbeddingBackend,
+    neural_reused_entries: tuple[CatalogEntry, ...],
+    neural_backend: EmbeddingBackend | None,
+    directory: Path,
+    predecessor_retrieval_directory: Path | None,
+    predecessor_retrieval_manifest_sha256: str | None,
+    egress_policy: EgressPolicy,
+    fallback_reason: str | None = None,
+    expected_existing_manifest_sha256: str | None = None,
+) -> RetrievalSealResult:
+    """Seal a v2 successor while preserving exact predecessor members for unchanged facts."""
+
+    if neural_backend is None:
+        if neural_reused_entries:
+            raise SourceContractError(
+                "RETRIEVAL_NEURAL_REUSE",
+                "retrieval.neural_reused_entries",
+                "neural reuse requires an explicit exact backend",
+            )
+        if fallback_reason is None:
+            fallback_reason = "NEURAL_NOT_REQUESTED"
+    elif fallback_reason is not None:
+        raise SourceContractError(
+            "RETRIEVAL_FALLBACK",
+            "fallback_reason",
+            "a hybrid successor cannot also claim lexical-only fallback",
+        )
+
+    lexical_facts = tuple(sorted((*lexical_reused_entries, *fresh_entries), key=_entry_key))
+    neural_reused_ids = {entry.entry_id for entry in neural_reused_entries}
+    neural_fresh = tuple(
+        replace(entry, vectors=())
+        for entry in lexical_facts
+        if entry.entry_id not in neural_reused_ids
+    )
+    if len(neural_reused_ids) != len(neural_reused_entries):
+        raise SourceContractError(
+            "DUPLICATE", "retrieval.neural_reused_entries", "entry ids must be unique"
+        )
+    lexical_reused_facts = tuple(
+        (entry.coordinate, entry.facts_digest) for entry in lexical_reused_entries
+    )
+    neural_reused_facts = tuple(
+        (entry.coordinate, entry.facts_digest) for entry in neural_reused_entries
+    )
+    if neural_reused_facts and neural_reused_facts != lexical_reused_facts:
+        raise SourceContractError(
+            "RETRIEVAL_SHARED_FACTS",
+            "retrieval.neural_reused_entries",
+            "neural reused entries must exactly match lexical reuse order and facts",
+        )
+    for entry in lexical_facts:
+        require_public_source_evidence(entry, egress_policy=egress_policy)
+    trusted_predecessor = None
+    if lexical_reused_entries or neural_reused_entries:
+        trusted_predecessor = _capture_trusted_retrieval_predecessor(
+            predecessor_directory=predecessor_retrieval_directory,
+            predecessor_manifest_sha256=predecessor_retrieval_manifest_sha256,
+            lexical_reused_entries=lexical_reused_entries,
+            neural_reused_entries=neural_reused_entries,
+            lexical_backend=lexical_backend,
+            neural_backend=neural_backend,
+        )
+    trusted_resume = _load_trusted_resume_catalog(
+        directory=directory,
+        expected_manifest_sha256=expected_existing_manifest_sha256,
+        expected_facts=lexical_facts,
+        lexical_backend=lexical_backend,
+        neural_backend=neural_backend,
+        fallback_reason=fallback_reason,
+    )
+    if trusted_resume is not None:
+        return _result_from_trusted_resume(trusted_resume)
+    shared_facts_sha256 = _shared_facts_sha256(lexical_facts)
+
+    lexical_generation = _representation_generation(shared_facts_sha256, lexical_backend.descriptor)
+    lexical_directory = directory / LEXICAL_REPRESENTATION / lexical_generation
+    if trusted_predecessor is None:
+        lexical_result = _seal_representation_generation(
+            reused_entries=(),
+            fresh_entries=fresh_entries,
+            backend=lexical_backend,
+            directory=lexical_directory,
+            predecessor_directory=None,
+            predecessor_manifest_sha256=None,
+            egress_policy=egress_policy,
+        )
+    else:
+        with tempfile.TemporaryDirectory(prefix="mr-lexical-predecessor-") as snapshot_raw:
+            snapshot_directory, snapshot_digest = _materialize_representation_snapshot(
+                trusted_predecessor.lexical,
+                directory=Path(snapshot_raw),
+            )
+            lexical_result = _seal_representation_generation(
+                reused_entries=lexical_reused_entries,
+                fresh_entries=fresh_entries,
+                backend=lexical_backend,
+                directory=lexical_directory,
+                predecessor_directory=snapshot_directory,
+                predecessor_manifest_sha256=snapshot_digest,
+                egress_policy=egress_policy,
+            )
+    representations = [
+        _representation_from_seal(
+            root=directory,
+            name=LEXICAL_REPRESENTATION,
+            generation=lexical_generation,
+            descriptor=lexical_backend.descriptor,
+            result=lexical_result,
+        )
+    ]
+
+    neural_result: CatalogSealResult | None = None
+    if neural_backend is not None:
+        neural_generation = _representation_generation(
+            shared_facts_sha256, neural_backend.descriptor
+        )
+        neural_directory = directory / NEURAL_REPRESENTATION / neural_generation
+        trusted_neural = None if trusted_predecessor is None else trusted_predecessor.neural
+        if trusted_neural is None:
+            neural_result = _seal_representation_generation(
+                reused_entries=(),
+                fresh_entries=neural_fresh,
+                backend=neural_backend,
+                directory=neural_directory,
+                predecessor_directory=None,
+                predecessor_manifest_sha256=None,
+                egress_policy=egress_policy,
+            )
+        else:
+            with tempfile.TemporaryDirectory(prefix="mr-neural-predecessor-") as snapshot_raw:
+                snapshot_directory, snapshot_digest = _materialize_representation_snapshot(
+                    trusted_neural,
+                    directory=Path(snapshot_raw),
+                )
+                neural_result = _seal_representation_generation(
+                    reused_entries=neural_reused_entries,
+                    fresh_entries=neural_fresh,
+                    backend=neural_backend,
+                    directory=neural_directory,
+                    predecessor_directory=snapshot_directory,
+                    predecessor_manifest_sha256=snapshot_digest,
+                    egress_policy=egress_policy,
+                )
+        if tuple(
+            (entry.coordinate, entry.facts_digest) for entry in neural_result.entries
+        ) != tuple((entry.coordinate, entry.facts_digest) for entry in lexical_result.entries):
+            raise SourceContractError(
+                "RETRIEVAL_SHARED_FACTS",
+                "retrieval.neural",
+                "neural successor differs from lexical canonical facts",
+            )
+        representations.append(
+            _representation_from_seal(
+                root=directory,
+                name=NEURAL_REPRESENTATION,
+                generation=neural_generation,
+                descriptor=neural_backend.descriptor,
+                result=neural_result,
+            )
+        )
+
+    manifest = RetrievalManifestV2(
+        shared_facts_sha256=shared_facts_sha256,
+        representations=tuple(representations),
+        fallback_reason=fallback_reason,
+    )
+    if RetrievalManifestV2.from_bytes(manifest.canonical_bytes) != manifest:
+        raise SourceContractError(
+            "RETRIEVAL_MANIFEST_READBACK", RETRIEVAL_MANIFEST_FILENAME, "reparse differs"
+        )
+    _publish_retrieval_manifest(directory, manifest.canonical_bytes)
+    return RetrievalSealResult(
+        manifest=manifest,
+        manifest_sha256=manifest.digest,
+        lexical_entries=lexical_result.entries,
+        neural_entries=None if neural_result is None else neural_result.entries,
+        lexical_reused_vector_sha256s=lexical_result.reused_vector_sha256s,
+        lexical_created_vector_sha256s=lexical_result.created_vector_sha256s,
+        neural_reused_vector_sha256s=()
+        if neural_result is None
+        else neural_result.reused_vector_sha256s,
+        neural_created_vector_sha256s=()
+        if neural_result is None
+        else neural_result.created_vector_sha256s,
+        lexical_encoder_invocations=lexical_result.encoder_invocations,
+        neural_encoder_invocations=0
+        if neural_result is None
+        else neural_result.encoder_invocations,
+    )
+
+
+def compose_catalog_entry(
+    *,
+    record: HarvestedRecord,
+    rights: RightsEvidence,
+    observation: SourceObservation,
+    spatial_scope: tuple[str, ...],
+    access_kind: str,
+    authentication_required: bool,
+    entry_id: str,
+    egress_policy: EgressPolicy,
+    data_formats: tuple[str, ...] | None = None,
+    quirks: tuple[SourceQuirk, ...] = (),
+    declared_columns: tuple[str, ...] = (),
+    declared_vocabulary: tuple[str, ...] = (),
+    declared_row_count: int | None = None,
+) -> CatalogEntry:
+    """Turn one harvested record into a first-version catalog entry, publicly cited.
+
+    This is the seam where a stranger's document becomes shared knowledge, so it is where the
+    public-source checks actually run. Returning an unchecked entry and expecting the caller to
+    validate it is the "enforced by review" pattern the tenancy rule rejects.
+
+    ``entry_id`` is supplied rather than derived from ``record.record_id``: publisher ids do not
+    satisfy the harness identifier grammar in general, and silently transliterating one would make
+    the ``faostat_qcl@v3`` coordinate unstable across harvests. The author chooses the id; the
+    record is the cited evidence for it.
+
+    ``data_formats`` defaults to what the record declares and exists because some protocols declare
+    nothing. An SDMX structure message describes dataflows, not payloads, so every SDMX record
+    carries ``data_formats=()``; a CKAN package whose resources name only formats outside the closed
+    vocabulary carries the same. Without an override those records could not be composed at all, and
+    the only remaining route into a catalog would be hand-building a ``CatalogEntry`` -- the
+    unchecked route this seam exists to replace. Supplying formats is an author's assertion about
+    what the source really publishes, and the record stays cited as the evidence for the rest of the
+    entry. When neither the record nor the author supplies one, composition refuses and says so.
+    """
+
+    if not isinstance(record, HarvestedRecord):
+        raise SourceContractError("TYPE", "record", "must be a HarvestedRecord")
+    if not isinstance(entry_id, str) or not _ENTRY_ID.fullmatch(entry_id):
+        raise SourceContractError(
+            "CATALOG_ENTRY_ID",
+            "entry_id",
+            f"{entry_id!r} is not a valid entry identifier for record {record.record_id!r}; "
+            "an entry id is chosen by the author, never transliterated from a publisher id",
+        )
+    if data_formats is not None and not isinstance(data_formats, tuple):
+        raise SourceContractError("TYPE", "data_formats", "must be a tuple of format names")
+    resolved_formats = record.data_formats if data_formats is None else data_formats
+    if not resolved_formats:
+        raise SourceContractError(
+            "CATALOG_ENTRY_FORMATS",
+            "data_formats",
+            f"record {record.record_id!r} on protocol {record.protocol!r} declares no payload "
+            "format the harness can read, so the author must supply data_formats explicitly; a "
+            "structure or metadata document describes a source without describing its payload",
+        )
+    entry = CatalogEntry(
+        entry_id=entry_id,
+        entry_version=1,
+        publisher=record.publisher,
+        title=record.title,
+        description=record.description,
+        spatial_scope=spatial_scope,
+        data_formats=resolved_formats,
+        access_kind=access_kind,
+        authentication_required=authentication_required,
+        observation=observation,
+        rights=rights,
+        quirks=quirks,
+        declared_columns=declared_columns,
+        declared_vocabulary=declared_vocabulary,
+        declared_row_count=declared_row_count,
+        evidence=(record.evidence,),
+        vectors=(),
+        previous_digest=None,
+    )
+    require_public_source_evidence(entry, egress_policy=egress_policy)
+    return entry
+
+
+def seal_catalog_successor(
+    *,
+    reused_entries: tuple[CatalogEntry, ...],
+    fresh_entries: tuple[CatalogEntry, ...],
+    backend: EmbeddingBackend,
+    directory: Path,
+    predecessor_directory: Path | None,
+    predecessor_manifest_sha256: str | None,
+    egress_policy: EgressPolicy,
+) -> CatalogSealResult:
+    """Durably publish one complete v1 current-head manifest.
+
+    Existing sealed entries retain their exact vector references and bytes. New and changed
+    unsealed entries are encoded once per layer. Every member is installed through retained root
+    descriptors with exclusive no-follow creation, exact readback, and mode ``0444`` before the
+    manifest is atomically replaced and read back.
+    """
+
+    if not isinstance(directory, Path):
+        raise SourceContractError("TYPE", "directory", "must be a Path")
+    if predecessor_directory is not None and not isinstance(predecessor_directory, Path):
+        raise SourceContractError("TYPE", "predecessor_directory", "must be a Path or None")
+    if not isinstance(reused_entries, tuple) or not isinstance(fresh_entries, tuple):
+        raise SourceContractError(
+            "TYPE", "catalog.successor.entries", "entry collections must be tuples"
+        )
+    captured_reused = tuple(
+        _capture_entry(entry, index=index, sealed=True)
+        for index, entry in enumerate(reused_entries)
+    )
+    captured_fresh = tuple(
+        _capture_entry(entry, index=index, sealed=False)
+        for index, entry in enumerate(fresh_entries)
+    )
+    all_ids = [entry.entry_id for entry in (*captured_reused, *captured_fresh)]
+    if len(set(all_ids)) != len(all_ids):
+        raise SourceContractError(
+            "DUPLICATE", "catalog.entries", "a current-head catalog has one entry per source"
+        )
+    for entry in (*captured_reused, *captured_fresh):
+        require_public_source_evidence(entry, egress_policy=egress_policy)
+
+    _refuse_build_ancestor(directory)
+    root_fd, publication_fds, publication_chain = _open_or_create_catalog_root(directory)
+    source_root_fd = -1
+    vector_fd = -1
+    source_vector_fd = -1
+    try:
+        _verify_publication_chain(publication_chain)
+        _refuse_build_root(root_fd)
+        try:
+            os.mkdir(VECTORS_DIRNAME, mode=0o755, dir_fd=root_fd)
+        except FileExistsError:
+            pass
+        vector_fd = _open_directory_at(root_fd, VECTORS_DIRNAME, "catalog vectors")
+
+        if predecessor_manifest_sha256 is None:
+            if captured_reused:
+                raise SourceContractError(
+                    "CATALOG_PREDECESSOR",
+                    "catalog.predecessor",
+                    "reused entries require an exact predecessor manifest digest",
+                )
+        else:
+            if predecessor_directory is None:
+                raise SourceContractError(
+                    "CATALOG_PREDECESSOR",
+                    "catalog.predecessor",
+                    "an exact predecessor directory is required",
+                )
+            source_root_fd = _open_directory_nofollow(
+                predecessor_directory, "catalog predecessor root"
+            )
+            predecessor_raw = _read_regular_at(
+                source_root_fd,
+                CATALOG_FILENAME,
+                maximum=MAX_CATALOG_MANIFEST_BYTES,
+                label="catalog predecessor manifest",
+            )
+            if sha256_bytes(predecessor_raw) != predecessor_manifest_sha256:
+                raise SourceContractError(
+                    "CATALOG_PREDECESSOR",
+                    "catalog.predecessor",
+                    "predecessor manifest digest differs",
+                )
+            previous = _entries_from_manifest(predecessor_raw)
+            previous_by_id = {entry.entry_id: entry for entry in previous}
+            for entry in captured_reused:
+                observed = previous_by_id.get(entry.entry_id)
+                if observed is None or observed.to_dict() != entry.to_dict():
+                    raise SourceContractError(
+                        "CATALOG_PREDECESSOR",
+                        f"catalog.predecessor.{entry.entry_id}",
+                        "reused entry is not the exact predecessor member",
+                    )
+            source_vector_fd = _open_directory_at(
+                source_root_fd, VECTORS_DIRNAME, "catalog predecessor vectors"
+            )
+
+        descriptor = backend.descriptor
+        payloads: dict[str, bytes] = {}
+        reused_digests: set[str] = set()
+        for entry in captured_reused:
+            for reference in entry.vectors:
+                if (
+                    reference.backend_id != descriptor.backend_id
+                    or reference.backend_version != descriptor.backend_version
+                    or reference.dimensions != descriptor.dimensions
+                    or reference.quantization != descriptor.quantization
+                ):
+                    raise SourceContractError(
+                        "CATALOG_BACKEND_IDENTITY",
+                        f"catalog.entries.{entry.entry_id}",
+                        "reused vectors do not match the selected backend",
+                    )
+                raw = _read_regular_at(
+                    source_vector_fd,
+                    f"{reference.vector_sha256}{VECTOR_SUFFIX}",
+                    maximum=descriptor.dimensions * 4,
+                    label=f"reused vector {reference.vector_sha256}",
+                )
+                if (
+                    len(raw) != descriptor.dimensions * 4
+                    or sha256_bytes(raw) != reference.vector_sha256
+                ):
+                    raise SourceContractError(
+                        "CATALOG_VECTOR_DIGEST",
+                        reference.vector_sha256,
+                        "reused vector bytes differ from their content address",
+                    )
+                prior = payloads.setdefault(reference.vector_sha256, raw)
+                if prior != raw:
+                    raise SourceContractError(
+                        "CATALOG_VECTOR_COLLISION",
+                        reference.vector_sha256,
+                        "one content address resolved to different bytes",
+                    )
+                reused_digests.add(reference.vector_sha256)
+
+        sealed_fresh: list[CatalogEntry] = []
+        encoder_invocations = 0
+        for entry in captured_fresh:
+            facts_digest = entry.facts_digest
+            references: list[LayerVectorReference] = []
+            for layer in EMBEDDING_LAYERS:
+                vector = backend.encode(layer_text(entry, layer))
+                encoder_invocations += 1
+                if len(vector) != descriptor.dimensions:
+                    raise SourceContractError(
+                        "EMBEDDING_DIMENSIONS",
+                        f"{entry.coordinate}.{layer}",
+                        f"backend returned {len(vector)} values, expected {descriptor.dimensions}",
+                    )
+                packed = struct.pack(f">{descriptor.dimensions}i", *vector)
+                digest = sha256_bytes(packed)
+                prior = payloads.setdefault(digest, packed)
+                if prior != packed:
+                    raise SourceContractError(
+                        "CATALOG_VECTOR_COLLISION",
+                        digest,
+                        "one content address resolved to different bytes",
+                    )
+                references.append(
+                    LayerVectorReference(
+                        layer=layer,
+                        backend_id=descriptor.backend_id,
+                        backend_version=descriptor.backend_version,
+                        dimensions=descriptor.dimensions,
+                        quantization=descriptor.quantization,
+                        vector_sha256=digest,
+                        entry_facts_digest=facts_digest,
+                    )
+                )
+            sealed_fresh.append(replace(entry, vectors=tuple(references)))
+
+        ordered = tuple(
+            sorted((*captured_reused, *sealed_fresh), key=lambda item: _entry_key(item))
+        )
+        index_bytes = canonical_json_bytes(
+            {
+                "schema_version": CATALOG_INDEX_CONTRACT_VERSION,
+                "entries": [item.to_dict() for item in ordered],
+            }
+        )
+        # Strict reconstruction before the first output mutation makes the manifest a checked
+        # successor, not merely canonical bytes produced by this process.
+        if _entries_from_manifest(index_bytes) != ordered:
+            raise SourceContractError(
+                "CATALOG_ENTRY_PROJECTION", "catalog.entries", "manifest reconstruction differs"
+            )
+
+        for digest in sorted(payloads):
+            _install_immutable_member(
+                vector_fd,
+                f"{digest}{VECTOR_SUFFIX}",
+                payloads[digest],
+                digest=digest,
+            )
+        _catalog_durability_barrier(vector_fd, "vectors-directory")
+        manifest_digest = sha256_bytes(index_bytes)
+        temporary = f".{CATALOG_FILENAME}.tmp-{secrets.token_hex(16)}"
+        try:
+            manifest_fd = _create_regular_at(root_fd, temporary, mode=0o600)
+            try:
+                _write_all(manifest_fd, index_bytes)
+                _catalog_durability_barrier(manifest_fd, "manifest-temporary")
+                if _read_open_descriptor(manifest_fd, maximum=len(index_bytes)) != index_bytes:
+                    raise SourceContractError(
+                        "CATALOG_MANIFEST_READBACK",
+                        CATALOG_FILENAME,
+                        "temporary manifest readback differs",
+                    )
+            finally:
+                os.close(manifest_fd)
+            os.replace(
+                temporary,
+                CATALOG_FILENAME,
+                src_dir_fd=root_fd,
+                dst_dir_fd=root_fd,
+            )
+            _verify_publication_chain(publication_chain)
+            _catalog_durability_barrier(root_fd, "manifest-replaced")
+            _catalog_durability_barrier(root_fd, "catalog-directory")
+            observed = _read_regular_at(
+                root_fd,
+                CATALOG_FILENAME,
+                maximum=len(index_bytes),
+                label="published catalog manifest",
+            )
+            if observed != index_bytes or sha256_bytes(observed) != manifest_digest:
+                raise SourceContractError(
+                    "CATALOG_MANIFEST_READBACK",
+                    CATALOG_FILENAME,
+                    "published manifest readback differs",
+                )
+            _catalog_durability_barrier(root_fd, "manifest-readback")
+        finally:
+            try:
+                os.unlink(temporary, dir_fd=root_fd)
+            except FileNotFoundError:
+                pass
+        return CatalogSealResult(
+            entries=ordered,
+            manifest_sha256=manifest_digest,
+            reused_vector_sha256s=tuple(sorted(reused_digests)),
+            created_vector_sha256s=tuple(sorted(set(payloads) - reused_digests)),
+            encoder_invocations=encoder_invocations,
+        )
+    finally:
+        for descriptor_to_close in (
+            source_vector_fd,
+            vector_fd,
+            source_root_fd,
+        ):
+            if descriptor_to_close >= 0:
+                os.close(descriptor_to_close)
+        for descriptor_to_close in reversed(publication_fds):
+            os.close(descriptor_to_close)
+
+
+def _capture_entry(entry: CatalogEntry, *, index: int, sealed: bool) -> CatalogEntry:
+    if type(entry) is not CatalogEntry:
+        raise SourceContractError("TYPE", f"entries[{index}]", "must be a CatalogEntry")
+    captured = canonical_json_bytes(entry.to_dict())
+    admit_public_fact_bytes(captured)
+    canonical_entry = catalog_entry_from_dict(parse_canonical_json(captured))
+    if canonical_json_bytes(canonical_entry.to_dict()) != captured:
+        raise SourceContractError(
+            "CATALOG_ENTRY_PROJECTION",
+            f"entries[{index}]",
+            "entry cannot be rebuilt byte-for-byte",
+        )
+    if sealed and not canonical_entry.vectors:
+        raise SourceContractError(
+            "CATALOG_REUSE_UNSEALED", f"entries[{index}]", "reused entry must be sealed"
+        )
+    if not sealed and canonical_entry.vectors:
+        raise SourceContractError(
+            "CATALOG_ALREADY_SEALED", f"entries[{index}]", "fresh entry must be unsealed"
+        )
+    return canonical_entry
+
+
+def _shared_facts_sha256(entries: tuple[CatalogEntry, ...]) -> str:
+    return sha256_bytes(
+        canonical_json_bytes(
+            [
+                {"coordinate": entry.coordinate, "facts_sha256": entry.facts_digest}
+                for entry in entries
+            ]
+        )
+    )
+
+
+def _seal_representation_generation(
+    *,
+    reused_entries: tuple[CatalogEntry, ...],
+    fresh_entries: tuple[CatalogEntry, ...],
+    backend: EmbeddingBackend,
+    directory: Path,
+    predecessor_directory: Path | None,
+    predecessor_manifest_sha256: str | None,
+    egress_policy: EgressPolicy,
+) -> CatalogSealResult:
+    """Seal a new generation; an existing one needs a trusted outer-head resume."""
+
+    _seal_generation_descriptor(directory, backend.descriptor)
+    if (directory / CATALOG_FILENAME).exists():
+        raise SourceContractError(
+            "RETRIEVAL_GENERATION_UNTRUSTED",
+            str(directory),
+            "existing generation requires an explicit authenticated outer manifest digest",
+        )
+    return seal_catalog_successor(
+        reused_entries=reused_entries,
+        fresh_entries=fresh_entries,
+        backend=backend,
+        directory=directory,
+        predecessor_directory=predecessor_directory,
+        predecessor_manifest_sha256=predecessor_manifest_sha256,
+        egress_policy=egress_policy,
+    )
+
+
+def _representation_generation(shared_facts_sha256: str, descriptor: BackendDescriptor) -> str:
+    return sha256_bytes(
+        canonical_json_bytes(
+            {
+                "backend_descriptor_sha256": backend_descriptor_sha256(descriptor),
+                "shared_facts_sha256": shared_facts_sha256,
+            }
+        )
+    )
+
+
+def _capture_trusted_retrieval_predecessor(
+    *,
+    predecessor_directory: Path | None,
+    predecessor_manifest_sha256: str | None,
+    lexical_reused_entries: tuple[CatalogEntry, ...],
+    neural_reused_entries: tuple[CatalogEntry, ...],
+    lexical_backend: EmbeddingBackend,
+    neural_backend: EmbeddingBackend | None,
+) -> LoadedRetrievalCatalog:
+    if predecessor_directory is None or predecessor_manifest_sha256 is None:
+        raise SourceContractError(
+            "CATALOG_PREDECESSOR",
+            "retrieval.predecessor",
+            "representation reuse requires an exact predecessor outer root and manifest digest",
+        )
+    loaded = load_retrieval_catalog(
+        predecessor_directory,
+        expected_manifest_sha256=predecessor_manifest_sha256,
+        neural_backend=neural_backend,
+    )
+    if backend_descriptor_bytes(loaded.lexical.backend.descriptor) != backend_descriptor_bytes(
+        lexical_backend.descriptor
+    ):
+        raise SourceContractError(
+            "RETRIEVAL_BACKEND_MISMATCH",
+            "retrieval.lexical_predecessor",
+            "trusted predecessor uses a different lexical backend descriptor",
+        )
+    previous_lexical = {entry.entry_id: entry for entry in loaded.lexical.entries}
+    for entry in lexical_reused_entries:
+        observed = previous_lexical.get(entry.entry_id)
+        if observed is None or observed.to_dict() != entry.to_dict():
+            raise SourceContractError(
+                "CATALOG_PREDECESSOR",
+                f"retrieval.lexical_predecessor.{entry.entry_id}",
+                "reused lexical entry is not the exact authenticated predecessor member",
+            )
+    if neural_reused_entries and loaded.neural is None:
+        raise SourceContractError(
+            "CATALOG_PREDECESSOR",
+            "retrieval.neural_predecessor",
+            "trusted predecessor has no resolved neural representation",
+        )
+    previous_neural = (
+        {} if loaded.neural is None else {entry.entry_id: entry for entry in loaded.neural.entries}
+    )
+    for entry in neural_reused_entries:
+        observed = previous_neural.get(entry.entry_id)
+        if observed is None or observed.to_dict() != entry.to_dict():
+            raise SourceContractError(
+                "CATALOG_PREDECESSOR",
+                f"retrieval.neural_predecessor.{entry.entry_id}",
+                "reused neural entry is not the exact authenticated predecessor member",
+            )
+    return loaded
+
+
+def _materialize_representation_snapshot(
+    snapshot: LoadedRetrievalRepresentation,
+    *,
+    directory: Path,
+) -> tuple[Path, str]:
+    vectors_directory = directory / VECTORS_DIRNAME
+    vectors_directory.mkdir(parents=True)
+    catalog_raw = canonical_json_bytes(
+        {
+            "entries": [entry.to_dict() for entry in snapshot.entries],
+            "schema_version": CATALOG_INDEX_CONTRACT_VERSION,
+        }
+    )
+    _atomic_write(directory / CATALOG_FILENAME, catalog_raw)
+    written: set[str] = set()
+    for entry in snapshot.entries:
+        for reference in entry.vectors:
+            if reference.vector_sha256 in written:
+                continue
+            vector = snapshot.vectors[(entry.entry_id, reference.layer)]
+            raw = struct.pack(f">{reference.dimensions}i", *vector)
+            if sha256_bytes(raw) != reference.vector_sha256:
+                raise SourceContractError(
+                    "CATALOG_VECTOR_DIGEST",
+                    reference.vector_sha256,
+                    "authenticated snapshot vector no longer matches its content address",
+                )
+            _atomic_write(vectors_directory / f"{reference.vector_sha256}{VECTOR_SUFFIX}", raw)
+            written.add(reference.vector_sha256)
+    return directory, sha256_bytes(catalog_raw)
+
+
+def _load_trusted_resume_catalog(
+    *,
+    directory: Path,
+    expected_manifest_sha256: str | None,
+    expected_facts: tuple[CatalogEntry, ...],
+    lexical_backend: EmbeddingBackend,
+    neural_backend: EmbeddingBackend | None,
+    fallback_reason: str | None,
+) -> LoadedRetrievalCatalog | None:
+    if expected_manifest_sha256 is None:
+        return None
+    loaded = load_retrieval_catalog(
+        directory,
+        expected_manifest_sha256=expected_manifest_sha256,
+        neural_backend=neural_backend,
+    )
+    expected = tuple((entry.coordinate, entry.facts_digest) for entry in expected_facts)
+    observed = tuple((entry.coordinate, entry.facts_digest) for entry in loaded.lexical.entries)
+    if observed != expected:
+        raise SourceContractError(
+            "RETRIEVAL_GENERATION_COLLISION",
+            str(directory),
+            "authenticated existing head differs from the requested canonical facts",
+        )
+    if backend_descriptor_bytes(loaded.lexical.backend.descriptor) != backend_descriptor_bytes(
+        lexical_backend.descriptor
+    ):
+        raise SourceContractError(
+            "RETRIEVAL_BACKEND_MISMATCH",
+            str(directory),
+            "authenticated existing head uses a different lexical backend",
+        )
+    requested_hybrid = neural_backend is not None
+    loaded_hybrid = loaded.manifest.neural is not None
+    if requested_hybrid != loaded_hybrid:
+        raise SourceContractError(
+            "RETRIEVAL_BACKEND_MISMATCH",
+            str(directory),
+            "authenticated existing head has a different representation set",
+        )
+    if loaded.manifest.fallback_reason != fallback_reason:
+        raise SourceContractError(
+            "RETRIEVAL_MANIFEST_FALLBACK",
+            str(directory),
+            "authenticated existing head has a different fallback reason",
+        )
+    if requested_hybrid:
+        assert neural_backend is not None
+        assert loaded.neural is not None
+        if backend_descriptor_bytes(loaded.neural.backend.descriptor) != backend_descriptor_bytes(
+            neural_backend.descriptor
+        ):
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                str(directory),
+                "authenticated existing head uses a different complete neural descriptor",
+            )
+    return loaded
+
+
+def _result_from_trusted_resume(loaded: LoadedRetrievalCatalog) -> RetrievalSealResult:
+    lexical_digests = tuple(
+        sorted(
+            {
+                reference.vector_sha256
+                for entry in loaded.lexical.entries
+                for reference in entry.vectors
+            }
+        )
+    )
+    neural_digests: tuple[str, ...] = ()
+    if loaded.neural is not None:
+        neural_digests = tuple(
+            sorted(
+                {
+                    reference.vector_sha256
+                    for entry in loaded.neural.entries
+                    for reference in entry.vectors
+                }
+            )
+        )
+    return RetrievalSealResult(
+        manifest=loaded.manifest,
+        manifest_sha256=loaded.manifest.digest,
+        lexical_entries=loaded.lexical.entries,
+        neural_entries=None if loaded.neural is None else loaded.neural.entries,
+        lexical_reused_vector_sha256s=lexical_digests,
+        lexical_created_vector_sha256s=(),
+        neural_reused_vector_sha256s=neural_digests,
+        neural_created_vector_sha256s=(),
+        lexical_encoder_invocations=0,
+        neural_encoder_invocations=0,
+    )
+
+
+def _seal_generation_descriptor(directory: Path, descriptor: BackendDescriptor) -> None:
+    raw = backend_descriptor_bytes(descriptor)
+    digest = sha256_bytes(raw)
+    generation_fd, owned, chain = _open_or_create_catalog_root(directory)
+    try:
+        _verify_publication_chain(chain)
+        try:
+            os.stat(CATALOG_FILENAME, dir_fd=generation_fd, follow_symlinks=False)
+            catalog_exists = True
+        except FileNotFoundError:
+            catalog_exists = False
+        try:
+            os.stat(
+                REPRESENTATION_DESCRIPTOR_FILENAME,
+                dir_fd=generation_fd,
+                follow_symlinks=False,
+            )
+            descriptor_exists = True
+        except FileNotFoundError:
+            descriptor_exists = False
+        if catalog_exists and not descriptor_exists:
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                str(directory),
+                "existing generation has no authenticated backend descriptor",
+            )
+        try:
+            _install_immutable_member(
+                generation_fd,
+                REPRESENTATION_DESCRIPTOR_FILENAME,
+                raw,
+                digest=digest,
+            )
+        except SourceContractError as error:
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                str(directory),
+                "existing generation backend descriptor differs",
+            ) from error
+    finally:
+        for owned_fd in reversed(owned):
+            os.close(owned_fd)
+
+
+def _representation_from_seal(
+    *,
+    root: Path,
+    name: str,
+    generation: str,
+    descriptor: BackendDescriptor,
+    result: CatalogSealResult,
+) -> RetrievalRepresentation:
+    root_fd = _open_directory_nofollow(root, "retrieval publication root")
+    name_fd = -1
+    generation_fd = -1
+    vectors_fd = -1
+    try:
+        name_fd = _open_directory_at(root_fd, name, f"{name} representation")
+        generation_fd = _open_directory_at(name_fd, generation, f"{name} generation")
+        catalog_raw = _read_regular_at(
+            generation_fd,
+            CATALOG_FILENAME,
+            maximum=MAX_CATALOG_MANIFEST_BYTES,
+            label=f"{name} catalog manifest",
+        )
+        if sha256_bytes(catalog_raw) != result.manifest_sha256:
+            raise SourceContractError(
+                "RETRIEVAL_MEMBER_READBACK",
+                f"{name}/{generation}/{CATALOG_FILENAME}",
+                "inner manifest differs after durable sealing",
+            )
+        prefix = f"{name}/{generation}"
+        members = [
+            RepresentationMember(
+                path=f"{prefix}/{CATALOG_FILENAME}",
+                size=len(catalog_raw),
+                sha256=result.manifest_sha256,
+            )
+        ]
+        descriptor_raw = _read_regular_at(
+            generation_fd,
+            REPRESENTATION_DESCRIPTOR_FILENAME,
+            maximum=MAX_REPRESENTATION_DESCRIPTOR_BYTES,
+            label=f"{name} backend descriptor",
+        )
+        if descriptor_raw != backend_descriptor_bytes(descriptor):
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                f"{prefix}/{REPRESENTATION_DESCRIPTOR_FILENAME}",
+                "generation descriptor differs after durable sealing",
+            )
+        members.append(
+            RepresentationMember(
+                path=f"{prefix}/{REPRESENTATION_DESCRIPTOR_FILENAME}",
+                size=len(descriptor_raw),
+                sha256=sha256_bytes(descriptor_raw),
+            )
+        )
+        vectors_fd = _open_directory_at(generation_fd, VECTORS_DIRNAME, f"{name} vectors")
+        vector_digests = sorted(
+            {reference.vector_sha256 for entry in result.entries for reference in entry.vectors}
+        )
+        for digest in vector_digests:
+            raw = _read_regular_at(
+                vectors_fd,
+                f"{digest}{VECTOR_SUFFIX}",
+                maximum=descriptor.dimensions * 4,
+                label=f"{name} vector {digest}",
+            )
+            if len(raw) != descriptor.dimensions * 4 or sha256_bytes(raw) != digest:
+                raise SourceContractError(
+                    "RETRIEVAL_MEMBER_READBACK",
+                    f"{prefix}/{VECTORS_DIRNAME}/{digest}{VECTOR_SUFFIX}",
+                    "sealed vector differs after durable sealing",
+                )
+            members.append(
+                RepresentationMember(
+                    path=f"{prefix}/{VECTORS_DIRNAME}/{digest}{VECTOR_SUFFIX}",
+                    size=len(raw),
+                    sha256=digest,
+                )
+            )
+        return RetrievalRepresentation.create(
+            name=name,
+            descriptor=descriptor,
+            members=tuple(members),
+        )
+    finally:
+        for descriptor_to_close in (vectors_fd, generation_fd, name_fd, root_fd):
+            if descriptor_to_close >= 0:
+                os.close(descriptor_to_close)
+
+
+def _publish_retrieval_manifest(directory: Path, raw: bytes) -> None:
+    root_fd, owned, chain = _open_or_create_catalog_root(directory)
+    temporary = f".{RETRIEVAL_MANIFEST_FILENAME}.tmp-{secrets.token_hex(16)}"
+    try:
+        _verify_publication_chain(chain)
+        descriptor = _create_regular_at(root_fd, temporary, mode=0o600)
+        try:
+            _write_all(descriptor, raw)
+            _catalog_durability_barrier(descriptor, "retrieval-manifest-temporary")
+            if _read_open_descriptor(descriptor, maximum=len(raw)) != raw:
+                raise SourceContractError(
+                    "RETRIEVAL_MANIFEST_READBACK",
+                    RETRIEVAL_MANIFEST_FILENAME,
+                    "temporary manifest readback differs",
+                )
+            os.fchmod(descriptor, 0o444)
+        finally:
+            os.close(descriptor)
+        _verify_publication_chain(chain)
+        os.replace(
+            temporary,
+            RETRIEVAL_MANIFEST_FILENAME,
+            src_dir_fd=root_fd,
+            dst_dir_fd=root_fd,
+        )
+        _catalog_durability_barrier(root_fd, "retrieval-manifest-replaced")
+        observed = _read_regular_at(
+            root_fd,
+            RETRIEVAL_MANIFEST_FILENAME,
+            maximum=len(raw),
+            label="published retrieval manifest",
+        )
+        if observed != raw:
+            raise SourceContractError(
+                "RETRIEVAL_MANIFEST_READBACK",
+                RETRIEVAL_MANIFEST_FILENAME,
+                "published manifest readback differs",
+            )
+        _catalog_durability_barrier(root_fd, "retrieval-manifest-readback")
+    finally:
+        try:
+            os.unlink(temporary, dir_fd=root_fd)
+        except FileNotFoundError:
+            pass
+        for descriptor_to_close in reversed(owned):
+            os.close(descriptor_to_close)
+
+
+def _entries_from_manifest(raw: bytes) -> tuple[CatalogEntry, ...]:
+    try:
+        payload = parse_canonical_json(raw)
+    except ValueError as error:
+        raise SourceContractError(
+            "CATALOG_MANIFEST", CATALOG_FILENAME, "manifest is not canonical"
+        ) from error
+    if (
+        not isinstance(payload, dict)
+        or set(payload) != {"schema_version", "entries"}
+        or payload["schema_version"] != CATALOG_INDEX_CONTRACT_VERSION
+        or not isinstance(payload["entries"], list)
+    ):
+        raise SourceContractError(
+            "CATALOG_MANIFEST", CATALOG_FILENAME, "manifest fields or version differ"
+        )
+    entries = tuple(catalog_entry_from_dict(item) for item in payload["entries"])
+    if tuple(sorted(entries, key=_entry_key)) != entries:
+        raise SourceContractError(
+            "CATALOG_MANIFEST", CATALOG_FILENAME, "manifest entries are not sorted"
+        )
+    if len({entry.entry_id for entry in entries}) != len(entries):
+        raise SourceContractError(
+            "CATALOG_MANIFEST", CATALOG_FILENAME, "manifest entry ids are not unique"
+        )
+    return entries
+
+
+def _open_directory_nofollow(path: Path, label: str) -> int:
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags)
+    except OSError as error:
+        raise SourceContractError(
+            "CATALOG_PUBLICATION_ROOT", label, "directory cannot be opened safely"
+        ) from error
+    info = os.fstat(descriptor)
+    if not stat.S_ISDIR(info.st_mode):
+        os.close(descriptor)
+        raise SourceContractError("CATALOG_PUBLICATION_ROOT", label, "must be a directory")
+    return descriptor
+
+
+def _open_or_create_catalog_root(
+    directory: Path,
+) -> tuple[int, list[int], list[tuple[int, str, int, tuple[int, int, int]]]]:
+    """Create/open a catalog only through one retained no-follow ancestor chain."""
+
+    owned: list[int] = []
+    chain: list[tuple[int, str, int, tuple[int, int, int]]] = []
+    try:
+        parent_fd = os.open(
+            "/", os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+        )
+        owned.append(parent_fd)
+        for part in Path(os.path.abspath(directory)).parts[1:]:
+            if _directory_contains_build_manifest(parent_fd):
+                raise SourceContractError(
+                    "CATALOG_PUBLICATION_ROOT",
+                    "catalog.directory",
+                    "a catalog may not be written through or inside a sealed Build root",
+                )
+            try:
+                named = os.stat(part, dir_fd=parent_fd, follow_symlinks=False)
+            except FileNotFoundError:
+                os.mkdir(part, mode=0o755, dir_fd=parent_fd)
+                os.fsync(parent_fd)
+                named = os.stat(part, dir_fd=parent_fd, follow_symlinks=False)
+            child_fd = os.open(
+                part,
+                os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
+                dir_fd=parent_fd,
+            )
+            owned.append(child_fd)
+            opened = os.fstat(child_fd)
+            identity = _directory_identity(opened)
+            if not stat.S_ISDIR(named.st_mode) or _directory_identity(named) != identity:
+                raise SourceContractError(
+                    "CATALOG_PUBLICATION_ROOT",
+                    "catalog.directory",
+                    "named and opened ancestor identities differ",
+                )
+            chain.append((parent_fd, part, child_fd, identity))
+            parent_fd = child_fd
+        _verify_publication_chain(chain)
+        if _directory_contains_build_manifest(parent_fd):
+            raise SourceContractError(
+                "CATALOG_PUBLICATION_ROOT",
+                "catalog.directory",
+                "a catalog may not be written through or inside a sealed Build root",
+            )
+        return parent_fd, owned, chain
+    except BaseException:
+        for descriptor in reversed(owned):
+            os.close(descriptor)
+        raise
+
+
+def _verify_publication_chain(
+    chain: list[tuple[int, str, int, tuple[int, int, int]]],
+) -> None:
+    for parent_fd, name, child_fd, identity in chain:
+        try:
+            named = os.stat(name, dir_fd=parent_fd, follow_symlinks=False)
+            opened = os.fstat(child_fd)
+        except OSError as error:
+            raise SourceContractError(
+                "CATALOG_PUBLICATION_ROOT",
+                "catalog.directory",
+                "retained publication ancestor disappeared",
+            ) from error
+        if _directory_identity(named) != identity or _directory_identity(opened) != identity:
+            raise SourceContractError(
+                "CATALOG_PUBLICATION_ROOT",
+                "catalog.directory",
+                "retained publication ancestor was replaced",
+            )
+
+
+def _directory_identity(info: os.stat_result) -> tuple[int, int, int]:
+    return (info.st_dev, info.st_ino, stat.S_IFMT(info.st_mode))
+
+
+def _open_directory_at(parent_fd: int, name: str, label: str) -> int:
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(name, flags, dir_fd=parent_fd)
+    except OSError as error:
+        raise SourceContractError(
+            "CATALOG_PUBLICATION_ROOT", label, "directory cannot be opened safely"
+        ) from error
+    info = os.fstat(descriptor)
+    if not stat.S_ISDIR(info.st_mode):
+        os.close(descriptor)
+        raise SourceContractError("CATALOG_PUBLICATION_ROOT", label, "must be a directory")
+    return descriptor
+
+
+def _refuse_build_root(root_fd: int) -> None:
+    if _directory_contains_build_manifest(root_fd):
+        raise SourceContractError(
+            "CATALOG_PUBLICATION_ROOT",
+            "catalog.directory",
+            "a catalog may not be written through or inside a sealed Build root",
+        )
+
+
+def _refuse_build_ancestor(directory: Path) -> None:
+    absolute = directory.absolute()
+    for candidate in (absolute, *absolute.parents):
+        try:
+            info = candidate.lstat()
+        except FileNotFoundError:
+            continue
+        if not stat.S_ISDIR(info.st_mode) or stat.S_ISLNK(info.st_mode):
+            raise SourceContractError(
+                "CATALOG_PUBLICATION_ROOT",
+                "catalog.directory",
+                "catalog path ancestors must be real directories",
+            )
+        descriptor = _open_directory_nofollow(candidate, "catalog path ancestor")
+        try:
+            if _directory_contains_build_manifest(descriptor):
+                raise SourceContractError(
+                    "CATALOG_PUBLICATION_ROOT",
+                    "catalog.directory",
+                    "a catalog may not be written through or inside a sealed Build root",
+                )
+        finally:
+            os.close(descriptor)
+
+
+def _directory_contains_build_manifest(root_fd: int) -> bool:
+    try:
+        info = os.stat("manifest.json", dir_fd=root_fd, follow_symlinks=False)
+    except FileNotFoundError:
+        return False
+    if not stat.S_ISREG(info.st_mode) or stat.S_ISLNK(info.st_mode) or info.st_nlink != 1:
+        return False
+    try:
+        raw = _read_regular_at(
+            root_fd,
+            "manifest.json",
+            maximum=MAX_CATALOG_MANIFEST_BYTES,
+            label="possible Build manifest",
+        )
+        payload = parse_canonical_json(raw)
+    except (SourceContractError, ValueError):
+        return False
+    return (
+        isinstance(payload, dict)
+        and isinstance(payload.get("schema_version"), str)
+        and payload["schema_version"].startswith("candidate-manifest.")
+    )
+
+
+def _create_regular_at(parent_fd: int, name: str, *, mode: int) -> int:
+    flags = (
+        os.O_RDWR
+        | os.O_CREAT
+        | os.O_EXCL
+        | getattr(os, "O_NOFOLLOW", 0)
+        | getattr(os, "O_CLOEXEC", 0)
+    )
+    return os.open(name, flags, mode, dir_fd=parent_fd)
+
+
+def _install_immutable_member(parent_fd: int, name: str, raw: bytes, *, digest: str) -> None:
+    created = False
+    try:
+        descriptor = _create_regular_at(parent_fd, name, mode=0o600)
+        created = True
+    except FileExistsError:
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+        try:
+            descriptor = os.open(name, flags, dir_fd=parent_fd)
+        except OSError as error:
+            raise SourceContractError(
+                "CATALOG_VECTOR_COLLISION", name, "existing member cannot be opened safely"
+            ) from error
+    try:
+        before = os.fstat(descriptor)
+        if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
+            raise SourceContractError(
+                "CATALOG_VECTOR_COLLISION", name, "member is not single-link regular"
+            )
+        if created:
+            _write_all(descriptor, raw)
+            os.fsync(descriptor)
+        stable_before = os.fstat(descriptor)
+        observed = _read_open_descriptor(descriptor, maximum=len(raw))
+        after_read = os.fstat(descriptor)
+        if (
+            observed != raw
+            or sha256_bytes(observed) != digest
+            or after_read.st_size != len(raw)
+            or _identity(stable_before) != _identity(after_read)
+        ):
+            raise SourceContractError(
+                "CATALOG_VECTOR_COLLISION", name, "member readback or identity differs"
+            )
+        os.fchmod(descriptor, 0o444)
+        _catalog_durability_barrier(descriptor, "vector-member")
+        sealed = os.fstat(descriptor)
+        if stat.S_IMODE(sealed.st_mode) != 0o444:
+            raise SourceContractError(
+                "CATALOG_VECTOR_MODE", name, "member did not become immutable"
+            )
+    except BaseException:
+        if created:
+            try:
+                os.unlink(name, dir_fd=parent_fd)
+            except FileNotFoundError:
+                pass
+        raise
+    finally:
+        os.close(descriptor)
+
+
+def _read_regular_at(parent_fd: int, name: str, *, maximum: int, label: str) -> bytes:
+    flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+    try:
+        descriptor = os.open(name, flags, dir_fd=parent_fd)
+    except OSError as error:
+        raise SourceContractError(
+            "CATALOG_MEMBER", label, "member cannot be opened safely"
+        ) from error
+    try:
+        before = os.fstat(descriptor)
+        if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or before.st_size > maximum:
+            raise SourceContractError(
+                "CATALOG_MEMBER", label, "member is not bounded single-link regular"
+            )
+        raw = _read_open_descriptor(descriptor, maximum=maximum)
+        after = os.fstat(descriptor)
+        if _identity(before) != _identity(after) or len(raw) != after.st_size:
+            raise SourceContractError("CATALOG_MEMBER", label, "member changed during read")
+        return raw
+    finally:
+        os.close(descriptor)
+
+
+def _read_open_descriptor(descriptor: int, *, maximum: int) -> bytes:
+    os.lseek(descriptor, 0, os.SEEK_SET)
+    observed = bytearray()
+    while len(observed) <= maximum:
+        chunk = os.read(descriptor, min(1024 * 1024, maximum + 1 - len(observed)))
+        if not chunk:
+            break
+        observed.extend(chunk)
+    if len(observed) > maximum:
+        raise SourceContractError("CATALOG_MEMBER", "catalog.member", "member exceeds its bound")
+    return bytes(observed)
+
+
+def _write_all(descriptor: int, raw: bytes) -> None:
+    offset = 0
+    while offset < len(raw):
+        written = os.write(descriptor, raw[offset:])
+        if written < 1:
+            raise SourceContractError(
+                "CATALOG_PUBLICATION", "catalog.member", "write made no progress"
+            )
+        offset += written
+
+
+def _identity(info: os.stat_result) -> tuple[int, int, int, int, int, int]:
+    base = (info.st_dev, info.st_ino, info.st_mode, info.st_nlink, info.st_mtime_ns)
+    return (*base, info.st_size)
+
+
+def _entry_key(entry: CatalogEntry) -> tuple[str, int]:
+    return (entry.entry_id, entry.entry_version)
+
+
+def _catalog_durability_barrier(descriptor: int, boundary: str) -> None:
+    """Persist one publication boundary; the label supports deterministic crash injection."""
+
+    del boundary
+    os.fsync(descriptor)
+
+
+def _write_vector(path: Path, payload: bytes) -> None:
+    if path.exists():
+        existing = path.read_bytes()
+        if existing != payload:
+            # Content addressing must hold. A mismatch means a corrupted tree (or, absurdly, a
+            # sha256 collision). Either way, overwriting would destroy the evidence.
+            raise SourceContractError(
+                "CATALOG_VECTOR_COLLISION",
+                str(path.name),
+                "an existing vector file with this digest holds different bytes",
+            )
+        return
+    _atomic_write(path, payload)
+
+
+def _atomic_write(path: Path, payload: bytes) -> None:
+    temporary = path.with_name(f".{path.name}.partial")
+    temporary.write_bytes(payload)
+    os.replace(temporary, path)