mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/retrieval_manifest.py

@@ -0,0 +1,995 @@
+"""Strict v2 envelope for independently sealed catalog retrieval representations.
+
+The ``harness-source-catalog-index.v1`` artifact remains the representation format.
+This module adds an outer manifest; it does not reinterpret the v1 backend coordinate or its
+``query_safe`` flag.  A lexical representation is always present.  A neural representation is
+optional and, when present, carries the exact explicit local encoder and model-artifact closure
+that the v2 query path is allowed to resolve.
+"""
+
+from __future__ import annotations
+
+import os
+import re
+import stat
+import struct
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    CanonicalJSONError,
+    canonical_json_bytes,
+    parse_canonical_json,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.contracts import (
+    CATALOG_INDEX_CONTRACT_VERSION,
+    EMBEDDING_LAYERS,
+    CatalogEntry,
+    catalog_entry_from_dict,
+)
+from mostlyright.data_harness.sources.catalog.embedding import (
+    BackendDescriptor,
+    EmbeddingBackend,
+    LexicalHashingBackend,
+    PinnedLocalBackendDescriptor,
+)
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+RETRIEVAL_MANIFEST_SCHEMA = "harness-source-catalog-retrieval.v2"
+LEXICAL_REPRESENTATION = "lexical"
+NEURAL_REPRESENTATION = "neural"
+LEXICAL_FALLBACK_REASONS = frozenset(
+    {
+        "NEURAL_NOT_REQUESTED",
+        "NEURAL_RUNTIME_UNAVAILABLE",
+        "NEURAL_TUPLE_UNAVAILABLE",
+        "NEURAL_MODEL_UNAVAILABLE",
+    }
+)
+MAX_REPRESENTATION_MEMBERS = 40_002
+MAX_REPRESENTATION_MEMBER_BYTES = 128 * 1024 * 1024
+RETRIEVAL_MANIFEST_FILENAME = "retrieval.json"
+MAX_RETRIEVAL_MANIFEST_BYTES = 4 * 1024 * 1024
+REPRESENTATION_DESCRIPTOR_FILENAME = "descriptor.json"
+REPRESENTATION_DESCRIPTOR_SCHEMA = "harness-source-catalog-representation-descriptor.v1"
+MAX_REPRESENTATION_DESCRIPTOR_BYTES = 1_024
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+_IDENTIFIER = re.compile(r"^[a-z][a-z0-9]*(?:[._-][a-z0-9]+)*$")
+_SEMVER = re.compile(r"^[1-9][0-9]*\.[0-9]+\.[0-9]+$")
+
+
+class RetrievalManifestError(SourceContractError):
+    """A stable refusal for an invalid or ambiguous v2 retrieval manifest."""
+
+
+@dataclass(frozen=True)
+class RepresentationMember:
+    """One exact regular-file member in a representation closure."""
+
+    path: str
+    size: int
+    sha256: str
+
+    def __post_init__(self) -> None:
+        if (
+            not isinstance(self.path, str)
+            or not self.path
+            or self.path.startswith("/")
+            or "\\" in self.path
+            or any(part in {"", ".", ".."} for part in self.path.split("/"))
+        ):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_MEMBER",
+                "retrieval.representations.members.path",
+                "must be a canonical relative POSIX path",
+            )
+        if type(self.size) is not int or not 0 <= self.size <= MAX_REPRESENTATION_MEMBER_BYTES:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_MEMBER",
+                "retrieval.representations.members.size",
+                f"must be an integer in [0, {MAX_REPRESENTATION_MEMBER_BYTES}]",
+            )
+        _require_digest(
+            self.sha256,
+            "retrieval.representations.members.sha256",
+            "RETRIEVAL_MANIFEST_MEMBER",
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"path": self.path, "sha256": self.sha256, "size": self.size}
+
+
+@dataclass(frozen=True)
+class RetrievalRepresentation:
+    """The exact schema, backend, encoder, and member closure for one representation."""
+
+    name: str
+    schema_version: str
+    backend_id: str
+    backend_version: str
+    dimensions: int
+    quantization: str
+    query_safe: bool
+    backend_coordinate: str
+    query_encoder: str
+    model_pin_sha256: str | None
+    model_artifacts: tuple[tuple[str, str], ...]
+    members: tuple[RepresentationMember, ...]
+    artifact_sha256: str
+
+    def __post_init__(self) -> None:
+        if self.name not in {LEXICAL_REPRESENTATION, NEURAL_REPRESENTATION}:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_REPRESENTATIONS",
+                "retrieval.representations.name",
+                "must be lexical or neural",
+            )
+        if self.schema_version != CATALOG_INDEX_CONTRACT_VERSION:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_SCHEMA",
+                f"retrieval.representations.{self.name}.schema_version",
+                f"must equal {CATALOG_INDEX_CONTRACT_VERSION!r}",
+            )
+        if not isinstance(self.backend_id, str) or _IDENTIFIER.fullmatch(self.backend_id) is None:
+            self._backend_error("backend_id is invalid")
+        if (
+            not isinstance(self.backend_version, str)
+            or _SEMVER.fullmatch(self.backend_version) is None
+        ):
+            self._backend_error("backend_version is invalid")
+        if type(self.dimensions) is not int or not 1 <= self.dimensions <= 4_096:
+            self._backend_error("dimensions are invalid")
+        if not isinstance(self.quantization, str) or not self.quantization:
+            self._backend_error("quantization is invalid")
+        if type(self.query_safe) is not bool:
+            self._backend_error("query_safe must be a boolean")
+        base_coordinate = f"{self.backend_id}@{self.backend_version}"
+        if self.name == LEXICAL_REPRESENTATION:
+            lexical = LexicalHashingBackend().descriptor
+            if (
+                not self.query_safe
+                or self.backend_coordinate != base_coordinate
+                or (
+                    self.backend_id,
+                    self.backend_version,
+                    self.dimensions,
+                    self.quantization,
+                    self.query_safe,
+                )
+                != (
+                    lexical.backend_id,
+                    lexical.backend_version,
+                    lexical.dimensions,
+                    lexical.quantization,
+                    lexical.query_safe,
+                )
+            ):
+                self._backend_error("lexical must retain its exact query-safe v1 coordinate")
+            if (
+                self.query_encoder != base_coordinate
+                or self.model_pin_sha256 is not None
+                or self.model_artifacts
+            ):
+                raise RetrievalManifestError(
+                    "RETRIEVAL_MANIFEST_LEXICAL",
+                    "retrieval.representations.lexical",
+                    "lexical must use its v1 backend and must not carry model artifacts",
+                )
+        else:
+            if self.query_safe or not self.backend_coordinate.startswith(f"{base_coordinate}#"):
+                self._backend_error(
+                    "neural must retain query_safe=false and use an explicit v2 coordinate"
+                )
+            if self.query_encoder != self.backend_coordinate:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_MANIFEST_NEURAL",
+                    "retrieval.representations.neural.query_encoder",
+                    "must name the exact governed local encoder coordinate",
+                )
+            _require_digest(
+                self.model_pin_sha256,
+                "retrieval.representations.neural.model_pin_sha256",
+                "RETRIEVAL_MANIFEST_NEURAL",
+            )
+            _validate_model_artifacts(self.model_artifacts)
+
+        if (
+            not isinstance(self.members, tuple)
+            or not self.members
+            or len(self.members) > MAX_REPRESENTATION_MEMBERS
+            or any(type(member) is not RepresentationMember for member in self.members)
+        ):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_MEMBER",
+                f"retrieval.representations.{self.name}.members",
+                "must be a non-empty bounded tuple of strict members",
+            )
+        paths = tuple(member.path for member in self.members)
+        if paths != tuple(sorted(paths)) or len(set(paths)) != len(paths):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_MEMBER",
+                f"retrieval.representations.{self.name}.members",
+                "must be path-sorted without duplicates",
+            )
+        prefix = f"{self.name}/"
+        catalog_path = next((path for path in paths if path.endswith("/catalog.json")), "")
+        expected_descriptor_path = (
+            f"{catalog_path[: -len('catalog.json')]}{REPRESENTATION_DESCRIPTOR_FILENAME}"
+        )
+        if (
+            any(not path.startswith(prefix) for path in paths)
+            or sum(path.endswith("/catalog.json") for path in paths) != 1
+            or sum(path.endswith(f"/{REPRESENTATION_DESCRIPTOR_FILENAME}") for path in paths) != 1
+            or expected_descriptor_path not in paths
+        ):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_MEMBER",
+                f"retrieval.representations.{self.name}.members",
+                "members may bind only their representation and must include catalog.json and "
+                "descriptor.json once",
+            )
+        expected_artifact = sha256_bytes(
+            canonical_json_bytes([member.to_dict() for member in self.members])
+        )
+        if self.artifact_sha256 != expected_artifact:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_ARTIFACT",
+                f"retrieval.representations.{self.name}.artifact_sha256",
+                "does not bind the exact sorted member closure",
+            )
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        name: str,
+        descriptor: BackendDescriptor,
+        members: tuple[RepresentationMember, ...],
+    ) -> RetrievalRepresentation:
+        if not isinstance(descriptor, BackendDescriptor):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_BACKEND",
+                f"retrieval.representations.{name}.backend",
+                "must use a strict backend descriptor",
+            )
+        ordered = tuple(sorted(members, key=lambda member: member.path))
+        model_artifacts: tuple[tuple[str, str], ...] = ()
+        model_pin_sha256: str | None = None
+        coordinate = descriptor.coordinate
+        if isinstance(descriptor, PinnedLocalBackendDescriptor):
+            model_pin_sha256 = descriptor.pin_sha256
+            model_artifacts = descriptor.artifact_sha256s
+        return cls(
+            name=name,
+            schema_version=CATALOG_INDEX_CONTRACT_VERSION,
+            backend_id=descriptor.backend_id,
+            backend_version=descriptor.backend_version,
+            dimensions=descriptor.dimensions,
+            quantization=descriptor.quantization,
+            query_safe=descriptor.query_safe,
+            backend_coordinate=coordinate,
+            query_encoder=coordinate,
+            model_pin_sha256=model_pin_sha256,
+            model_artifacts=model_artifacts,
+            members=ordered,
+            artifact_sha256=sha256_bytes(
+                canonical_json_bytes([member.to_dict() for member in ordered])
+            ),
+        )
+
+    def _backend_error(self, detail: str) -> None:
+        raise RetrievalManifestError(
+            "RETRIEVAL_MANIFEST_BACKEND",
+            f"retrieval.representations.{self.name}.backend",
+            detail,
+        )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "artifact_sha256": self.artifact_sha256,
+            "backend": {
+                "backend_coordinate": self.backend_coordinate,
+                "backend_id": self.backend_id,
+                "backend_version": self.backend_version,
+                "dimensions": self.dimensions,
+                "quantization": self.quantization,
+                "query_safe": self.query_safe,
+            },
+            "members": [member.to_dict() for member in self.members],
+            "model_pin_sha256": self.model_pin_sha256,
+            "model_artifacts": [
+                {"path": path, "sha256": digest} for path, digest in self.model_artifacts
+            ],
+            "name": self.name,
+            "query_encoder": self.query_encoder,
+            "schema_version": self.schema_version,
+        }
+
+
+@dataclass(frozen=True)
+class RetrievalManifestV2:
+    """One lexical representation and one optional neural representation over shared facts."""
+
+    shared_facts_sha256: str
+    representations: tuple[RetrievalRepresentation, ...]
+    fallback_reason: str | None = None
+    schema_version: str = RETRIEVAL_MANIFEST_SCHEMA
+
+    def __post_init__(self) -> None:
+        if self.schema_version != RETRIEVAL_MANIFEST_SCHEMA:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_VERSION",
+                "retrieval.schema_version",
+                f"must equal {RETRIEVAL_MANIFEST_SCHEMA!r}",
+            )
+        _require_digest(
+            self.shared_facts_sha256,
+            "retrieval.shared_facts_sha256",
+            "RETRIEVAL_MANIFEST_FACTS",
+        )
+        if (
+            not isinstance(self.representations, tuple)
+            or any(type(item) is not RetrievalRepresentation for item in self.representations)
+            or tuple(item.name for item in self.representations)
+            not in {
+                (LEXICAL_REPRESENTATION,),
+                (LEXICAL_REPRESENTATION, NEURAL_REPRESENTATION),
+            }
+        ):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_REPRESENTATIONS",
+                "retrieval.representations",
+                "must contain lexical once followed by at most one neural representation",
+            )
+        if self.neural is None:
+            if self.fallback_reason not in LEXICAL_FALLBACK_REASONS:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_MANIFEST_FALLBACK",
+                    "retrieval.fallback_reason",
+                    "lexical-only v2 requires a typed neural fallback reason",
+                )
+        elif self.fallback_reason is not None:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_FALLBACK",
+                "retrieval.fallback_reason",
+                "a hybrid manifest cannot also claim lexical-only fallback",
+            )
+
+    @property
+    def lexical(self) -> RetrievalRepresentation:
+        return self.representations[0]
+
+    @property
+    def neural(self) -> RetrievalRepresentation | None:
+        return self.representations[1] if len(self.representations) == 2 else None
+
+    @property
+    def canonical_bytes(self) -> bytes:
+        return canonical_json_bytes(self.to_dict())
+
+    @property
+    def digest(self) -> str:
+        return sha256_bytes(self.canonical_bytes)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "fallback_reason": self.fallback_reason,
+            "representations": [item.to_dict() for item in self.representations],
+            "schema_version": self.schema_version,
+            "shared_facts_sha256": self.shared_facts_sha256,
+        }
+
+    @classmethod
+    def from_bytes(cls, raw: bytes) -> RetrievalManifestV2:
+        try:
+            payload = parse_canonical_json(raw)
+        except (CanonicalJSONError, TypeError, ValueError) as error:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_CANONICAL",
+                "retrieval",
+                "manifest bytes must be canonical JSON",
+            ) from error
+        if not isinstance(payload, dict) or set(payload) != {
+            "fallback_reason",
+            "representations",
+            "schema_version",
+            "shared_facts_sha256",
+        }:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_SHAPE",
+                "retrieval",
+                "manifest must contain the exact v2 field set",
+            )
+        raw_representations = payload["representations"]
+        if not isinstance(raw_representations, list):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_REPRESENTATIONS",
+                "retrieval.representations",
+                "must be an array",
+            )
+        representations = tuple(
+            _representation_from_dict(item, index=index)
+            for index, item in enumerate(raw_representations)
+        )
+        try:
+            return cls(
+                schema_version=payload["schema_version"],
+                shared_facts_sha256=payload["shared_facts_sha256"],
+                fallback_reason=payload["fallback_reason"],
+                representations=representations,
+            )
+        except RetrievalManifestError:
+            raise
+        except (TypeError, ValueError) as error:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_SHAPE", "retrieval", "manifest fields are invalid"
+            ) from error
+
+
+def _representation_from_dict(value: Any, *, index: int) -> RetrievalRepresentation:
+    path = f"retrieval.representations[{index}]"
+    exact = {
+        "artifact_sha256",
+        "backend",
+        "members",
+        "model_artifacts",
+        "model_pin_sha256",
+        "name",
+        "query_encoder",
+        "schema_version",
+    }
+    if not isinstance(value, dict) or set(value) != exact:
+        raise RetrievalManifestError(
+            "RETRIEVAL_MANIFEST_REPRESENTATIONS", path, "representation fields differ"
+        )
+    backend = value["backend"]
+    if not isinstance(backend, dict) or set(backend) != {
+        "backend_coordinate",
+        "backend_id",
+        "backend_version",
+        "dimensions",
+        "quantization",
+        "query_safe",
+    }:
+        raise RetrievalManifestError(
+            "RETRIEVAL_MANIFEST_BACKEND", f"{path}.backend", "backend fields differ"
+        )
+    raw_members = value["members"]
+    if not isinstance(raw_members, list):
+        raise RetrievalManifestError(
+            "RETRIEVAL_MANIFEST_MEMBER", f"{path}.members", "must be an array"
+        )
+    members = tuple(_member_from_dict(item, path=f"{path}.members") for item in raw_members)
+    raw_artifacts = value["model_artifacts"]
+    if not isinstance(raw_artifacts, list):
+        raise RetrievalManifestError(
+            "RETRIEVAL_MANIFEST_NEURAL", f"{path}.model_artifacts", "must be an array"
+        )
+    artifacts: list[tuple[str, str]] = []
+    for item in raw_artifacts:
+        if not isinstance(item, dict) or set(item) != {"path", "sha256"}:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_NEURAL",
+                f"{path}.model_artifacts",
+                "artifact fields differ",
+            )
+        artifacts.append((item["path"], item["sha256"]))
+    return RetrievalRepresentation(
+        name=value["name"],
+        schema_version=value["schema_version"],
+        backend_id=backend["backend_id"],
+        backend_version=backend["backend_version"],
+        dimensions=backend["dimensions"],
+        quantization=backend["quantization"],
+        query_safe=backend["query_safe"],
+        backend_coordinate=backend["backend_coordinate"],
+        query_encoder=value["query_encoder"],
+        model_pin_sha256=value["model_pin_sha256"],
+        model_artifacts=tuple(artifacts),
+        members=members,
+        artifact_sha256=value["artifact_sha256"],
+    )
+
+
+def _member_from_dict(value: Any, *, path: str) -> RepresentationMember:
+    if not isinstance(value, dict) or set(value) != {"path", "sha256", "size"}:
+        raise RetrievalManifestError("RETRIEVAL_MANIFEST_MEMBER", path, "member fields differ")
+    return RepresentationMember(path=value["path"], size=value["size"], sha256=value["sha256"])
+
+
+def _validate_model_artifacts(artifacts: tuple[tuple[str, str], ...]) -> None:
+    if not artifacts or tuple(sorted(artifacts)) != artifacts:
+        raise RetrievalManifestError(
+            "RETRIEVAL_MANIFEST_NEURAL",
+            "retrieval.representations.neural.model_artifacts",
+            "must be a non-empty path-sorted artifact closure",
+        )
+    for path, digest in artifacts:
+        if (
+            not isinstance(path, str)
+            or not path
+            or "/" in path
+            or "\\" in path
+            or _SHA256.fullmatch(digest) is None
+        ):
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_NEURAL",
+                "retrieval.representations.neural.model_artifacts",
+                "must contain safe names and lowercase SHA-256 digests",
+            )
+
+
+def _require_digest(value: Any, path: str, code: str) -> None:
+    if not isinstance(value, str) or _SHA256.fullmatch(value) is None:
+        raise RetrievalManifestError(code, path, "must be a lowercase SHA-256 digest")
+
+
+def backend_descriptor_sha256(descriptor: BackendDescriptor) -> str:
+    """Digest the complete backend identity sealed into each v2 generation."""
+
+    model_pin_sha256: str | None = None
+    model_artifacts: tuple[tuple[str, str], ...] = ()
+    if isinstance(descriptor, PinnedLocalBackendDescriptor):
+        model_pin_sha256 = descriptor.pin_sha256
+        model_artifacts = descriptor.artifact_sha256s
+    return _backend_identity_sha256(
+        backend_coordinate=descriptor.coordinate,
+        backend_id=descriptor.backend_id,
+        backend_version=descriptor.backend_version,
+        dimensions=descriptor.dimensions,
+        quantization=descriptor.quantization,
+        query_safe=descriptor.query_safe,
+        model_pin_sha256=model_pin_sha256,
+        model_artifacts=model_artifacts,
+    )
+
+
+def backend_descriptor_bytes(descriptor: BackendDescriptor) -> bytes:
+    """Canonical authenticated v2 generation metadata for one strict descriptor."""
+
+    return _descriptor_metadata_bytes(backend_descriptor_sha256(descriptor))
+
+
+def _backend_identity_sha256(
+    *,
+    backend_coordinate: str,
+    backend_id: str,
+    backend_version: str,
+    dimensions: int,
+    quantization: str,
+    query_safe: bool,
+    model_pin_sha256: str | None,
+    model_artifacts: tuple[tuple[str, str], ...],
+) -> str:
+    return sha256_bytes(
+        canonical_json_bytes(
+            {
+                "backend_coordinate": backend_coordinate,
+                "backend_id": backend_id,
+                "backend_version": backend_version,
+                "dimensions": dimensions,
+                "model_artifacts": [
+                    {"path": path, "sha256": digest} for path, digest in model_artifacts
+                ],
+                "model_pin_sha256": model_pin_sha256,
+                "quantization": quantization,
+                "query_safe": query_safe,
+            }
+        )
+    )
+
+
+def _descriptor_metadata_bytes(descriptor_sha256: str) -> bytes:
+    return canonical_json_bytes(
+        {
+            "backend_descriptor_sha256": descriptor_sha256,
+            "schema_version": REPRESENTATION_DESCRIPTOR_SCHEMA,
+        }
+    )
+
+
+def _representation_descriptor_bytes(representation: RetrievalRepresentation) -> bytes:
+    return _descriptor_metadata_bytes(
+        _backend_identity_sha256(
+            backend_coordinate=representation.backend_coordinate,
+            backend_id=representation.backend_id,
+            backend_version=representation.backend_version,
+            dimensions=representation.dimensions,
+            quantization=representation.quantization,
+            query_safe=representation.query_safe,
+            model_pin_sha256=representation.model_pin_sha256,
+            model_artifacts=representation.model_artifacts,
+        )
+    )
+
+
+@dataclass(frozen=True)
+class LoadedRetrievalRepresentation:
+    """A fully verified sealed representation and its explicitly supplied encoder."""
+
+    entries: tuple[CatalogEntry, ...]
+    vectors: dict[tuple[str, str], tuple[int, ...]]
+    backend: EmbeddingBackend
+
+
+@dataclass(frozen=True)
+class _VerifiedRetrievalRepresentation:
+    """One representation whose inner manifest and vector closure have been authenticated."""
+
+    entries: tuple[CatalogEntry, ...]
+    vectors: dict[tuple[str, str], tuple[int, ...]]
+
+
+@dataclass(frozen=True)
+class LoadedRetrievalCatalog:
+    """The verified v2 envelope, with an optional locally resolved neural representation."""
+
+    manifest: RetrievalManifestV2
+    lexical: LoadedRetrievalRepresentation
+    neural: LoadedRetrievalRepresentation | None
+    fallback_reason: str | None
+
+
+def load_retrieval_catalog(
+    directory: Path,
+    *,
+    expected_manifest_sha256: str,
+    neural_backend: EmbeddingBackend | None = None,
+) -> LoadedRetrievalCatalog:
+    """Verify the complete outer closure before exposing either representation.
+
+    Neural resolution is deliberately explicit.  Omitting the exact local backend never causes
+    ambient model discovery: a hybrid artifact then opens as a typed lexical-only runtime view.
+    """
+
+    if not isinstance(directory, Path):
+        raise RetrievalManifestError("TYPE", "directory", "must be a Path")
+    _require_digest(
+        expected_manifest_sha256,
+        "retrieval.expected_manifest_sha256",
+        "RETRIEVAL_MANIFEST_DIGEST",
+    )
+    root_fd = _open_directory(directory, label="retrieval root")
+    try:
+        raw_manifest = _read_regular(
+            root_fd,
+            (RETRIEVAL_MANIFEST_FILENAME,),
+            maximum=MAX_RETRIEVAL_MANIFEST_BYTES,
+        )
+        if sha256_bytes(raw_manifest) != expected_manifest_sha256:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MANIFEST_DIGEST",
+                RETRIEVAL_MANIFEST_FILENAME,
+                "published manifest differs from the required digest",
+            )
+        manifest = RetrievalManifestV2.from_bytes(raw_manifest)
+        captured: dict[str, bytes] = {}
+        for representation in manifest.representations:
+            for member in representation.members:
+                raw = _read_regular(
+                    root_fd,
+                    tuple(member.path.split("/")),
+                    maximum=member.size,
+                )
+                if len(raw) != member.size or sha256_bytes(raw) != member.sha256:
+                    raise RetrievalManifestError(
+                        "RETRIEVAL_MEMBER_TAMPER",
+                        member.path,
+                        "member size or digest differs from the outer manifest",
+                    )
+                captured[member.path] = raw
+
+        verified_lexical = _verify_representation(manifest.lexical, captured=captured)
+        verified_neural = (
+            None
+            if manifest.neural is None
+            else _verify_representation(manifest.neural, captured=captured)
+        )
+        _require_shared_facts(manifest, verified_lexical, verified_neural)
+
+        lexical = _bind_representation_backend(
+            manifest.lexical,
+            verified=verified_lexical,
+            backend=LexicalHashingBackend(),
+        )
+        neural: LoadedRetrievalRepresentation | None = None
+        fallback_reason = manifest.fallback_reason
+        if manifest.neural is not None and neural_backend is not None:
+            assert verified_neural is not None
+            neural = _bind_representation_backend(
+                manifest.neural,
+                verified=verified_neural,
+                backend=neural_backend,
+            )
+        elif manifest.neural is not None:
+            fallback_reason = "NEURAL_RUNTIME_UNAVAILABLE"
+
+        return LoadedRetrievalCatalog(
+            manifest=manifest,
+            lexical=lexical,
+            neural=neural,
+            fallback_reason=fallback_reason,
+        )
+    finally:
+        os.close(root_fd)
+
+
+def _bind_representation_backend(
+    representation: RetrievalRepresentation,
+    *,
+    verified: _VerifiedRetrievalRepresentation,
+    backend: EmbeddingBackend,
+) -> LoadedRetrievalRepresentation:
+    descriptor = backend.descriptor
+    expected_descriptor = (
+        representation.backend_id,
+        representation.backend_version,
+        representation.dimensions,
+        representation.quantization,
+        representation.query_safe,
+        representation.backend_coordinate,
+    )
+    observed_descriptor = (
+        descriptor.backend_id,
+        descriptor.backend_version,
+        descriptor.dimensions,
+        descriptor.quantization,
+        descriptor.query_safe,
+        descriptor.coordinate,
+    )
+    if observed_descriptor != expected_descriptor:
+        raise RetrievalManifestError(
+            "RETRIEVAL_BACKEND_MISMATCH",
+            f"retrieval.representations.{representation.name}.backend",
+            "the supplied encoder does not match the exact sealed coordinate",
+        )
+    if representation.name == NEURAL_REPRESENTATION:
+        if not isinstance(descriptor, PinnedLocalBackendDescriptor):
+            raise RetrievalManifestError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                "retrieval.representations.neural.backend",
+                "neural loading requires an explicit pinned local descriptor",
+            )
+        if descriptor.artifact_sha256s != representation.model_artifacts:
+            raise RetrievalManifestError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                "retrieval.representations.neural.model_artifacts",
+                "the supplied encoder artifact closure differs",
+            )
+        if descriptor.pin_sha256 != representation.model_pin_sha256:
+            raise RetrievalManifestError(
+                "RETRIEVAL_BACKEND_MISMATCH",
+                "retrieval.representations.neural.model_pin_sha256",
+                "the supplied encoder PIN digest differs",
+            )
+    return LoadedRetrievalRepresentation(
+        entries=verified.entries,
+        vectors=verified.vectors,
+        backend=backend,
+    )
+
+
+def _verify_representation(
+    representation: RetrievalRepresentation,
+    *,
+    captured: dict[str, bytes],
+) -> _VerifiedRetrievalRepresentation:
+
+    catalog_members = [
+        item for item in representation.members if item.path.endswith("/catalog.json")
+    ]
+    catalog_path = catalog_members[0].path
+    try:
+        payload = parse_canonical_json(captured[catalog_path])
+    except (CanonicalJSONError, TypeError, ValueError) as error:
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MANIFEST",
+            catalog_path,
+            "inner v1 manifest is not canonical JSON",
+        ) from error
+    if (
+        not isinstance(payload, dict)
+        or set(payload) != {"entries", "schema_version"}
+        or payload["schema_version"] != CATALOG_INDEX_CONTRACT_VERSION
+        or not isinstance(payload["entries"], list)
+    ):
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MANIFEST",
+            catalog_path,
+            "inner v1 manifest fields differ",
+        )
+    try:
+        entries = tuple(catalog_entry_from_dict(item) for item in payload["entries"])
+    except (SourceContractError, TypeError, ValueError) as error:
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MANIFEST",
+            catalog_path,
+            "inner entries violate the v1 catalog contract",
+        ) from error
+    if tuple(sorted(entries, key=lambda item: (item.entry_id, item.entry_version))) != entries:
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MANIFEST",
+            catalog_path,
+            "inner entries are not in canonical order",
+        )
+    if len({entry.entry_id for entry in entries}) != len(entries):
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MANIFEST",
+            catalog_path,
+            "inner entry ids are not unique",
+        )
+
+    prefix = catalog_path[: -len("catalog.json")]
+    descriptor_path = f"{prefix}{REPRESENTATION_DESCRIPTOR_FILENAME}"
+    try:
+        descriptor_raw = captured[descriptor_path]
+    except KeyError:
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MEMBER",
+            descriptor_path,
+            "representation descriptor is absent from the outer closure",
+        ) from None
+    if descriptor_raw != _representation_descriptor_bytes(representation):
+        raise RetrievalManifestError(
+            "RETRIEVAL_BACKEND_MISMATCH",
+            descriptor_path,
+            "generation descriptor differs from the exact outer backend identity",
+        )
+    required_paths = {catalog_path, descriptor_path}
+    vectors: dict[tuple[str, str], tuple[int, ...]] = {}
+    for entry in entries:
+        if tuple(sorted(reference.layer for reference in entry.vectors)) != tuple(
+            sorted(EMBEDDING_LAYERS)
+        ):
+            raise RetrievalManifestError(
+                "RETRIEVAL_REPRESENTATION_VECTOR",
+                entry.coordinate,
+                "each entry must bind exactly one vector per layer",
+            )
+        for reference in entry.vectors:
+            if reference.entry_facts_digest != entry.facts_digest:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_REPRESENTATION_VECTOR",
+                    f"{entry.coordinate}.{reference.layer}",
+                    "vector is bound to different entry facts",
+                )
+            if (
+                reference.backend_id,
+                reference.backend_version,
+                reference.dimensions,
+                reference.quantization,
+            ) != (
+                representation.backend_id,
+                representation.backend_version,
+                representation.dimensions,
+                representation.quantization,
+            ):
+                raise RetrievalManifestError(
+                    "RETRIEVAL_BACKEND_MISMATCH",
+                    f"{entry.coordinate}.{reference.layer}",
+                    "inner vector coordinate differs from the outer representation",
+                )
+            member_path = f"{prefix}vectors/{reference.vector_sha256}.vec"
+            required_paths.add(member_path)
+            try:
+                vector_raw = captured[member_path]
+            except KeyError:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_REPRESENTATION_MEMBER",
+                    member_path,
+                    "inner manifest references a member absent from the outer closure",
+                ) from None
+            if len(vector_raw) != representation.dimensions * 4:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_REPRESENTATION_VECTOR",
+                    member_path,
+                    "vector byte length differs from its dimensions",
+                )
+            if sha256_bytes(vector_raw) != reference.vector_sha256:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_REPRESENTATION_VECTOR",
+                    member_path,
+                    "vector bytes differ from the digest bound by the inner manifest",
+                )
+            vectors[(entry.entry_id, reference.layer)] = struct.unpack(
+                f">{representation.dimensions}i", vector_raw
+            )
+    declared_paths = {item.path for item in representation.members}
+    if declared_paths != required_paths:
+        raise RetrievalManifestError(
+            "RETRIEVAL_REPRESENTATION_MEMBER",
+            f"retrieval.representations.{representation.name}.members",
+            "outer closure has missing or unreferenced members",
+        )
+    return _VerifiedRetrievalRepresentation(entries=entries, vectors=vectors)
+
+
+def _require_shared_facts(
+    manifest: RetrievalManifestV2,
+    lexical: _VerifiedRetrievalRepresentation,
+    neural: _VerifiedRetrievalRepresentation | None,
+) -> None:
+    facts = tuple((entry.coordinate, entry.facts_digest) for entry in lexical.entries)
+    expected = sha256_bytes(
+        canonical_json_bytes(
+            [{"coordinate": coordinate, "facts_sha256": digest} for coordinate, digest in facts]
+        )
+    )
+    if expected != manifest.shared_facts_sha256:
+        raise RetrievalManifestError(
+            "RETRIEVAL_SHARED_FACTS",
+            "retrieval.shared_facts_sha256",
+            "lexical facts differ from the shared facts binding",
+        )
+    if neural is not None:
+        neural_facts = tuple((entry.coordinate, entry.facts_digest) for entry in neural.entries)
+        if neural_facts != facts:
+            raise RetrievalManifestError(
+                "RETRIEVAL_SHARED_FACTS",
+                "retrieval.representations.neural",
+                "neural and lexical representations do not seal identical facts",
+            )
+
+
+def _open_directory(path: Path, *, label: str, dir_fd: int | None = None) -> int:
+    flags = os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0)
+    try:
+        descriptor = os.open(path, flags, dir_fd=dir_fd)
+    except OSError as error:
+        raise RetrievalManifestError(
+            "RETRIEVAL_MEMBER_TAMPER", label, "directory cannot be opened without following links"
+        ) from error
+    info = os.fstat(descriptor)
+    if not stat.S_ISDIR(info.st_mode):
+        os.close(descriptor)
+        raise RetrievalManifestError("RETRIEVAL_MEMBER_TAMPER", label, "must be a directory")
+    return descriptor
+
+
+def _read_regular(root_fd: int, parts: tuple[str, ...], *, maximum: int) -> bytes:
+    owned: list[int] = []
+    parent_fd = root_fd
+    try:
+        for part in parts[:-1]:
+            child = _open_directory(Path(part), label="/".join(parts), dir_fd=parent_fd)
+            owned.append(child)
+            parent_fd = child
+        flags = os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0) | getattr(os, "O_CLOEXEC", 0)
+        try:
+            descriptor = os.open(parts[-1], flags, dir_fd=parent_fd)
+        except OSError as error:
+            raise RetrievalManifestError(
+                "RETRIEVAL_MEMBER_TAMPER", "/".join(parts), "member cannot be opened safely"
+            ) from error
+        try:
+            before = os.fstat(descriptor)
+            if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1 or before.st_size > maximum:
+                raise RetrievalManifestError(
+                    "RETRIEVAL_MEMBER_TAMPER",
+                    "/".join(parts),
+                    "member is not a bounded single-link regular file",
+                )
+            raw = bytearray()
+            while len(raw) <= maximum:
+                chunk = os.read(descriptor, min(1024 * 1024, maximum + 1 - len(raw)))
+                if not chunk:
+                    break
+                raw.extend(chunk)
+            after = os.fstat(descriptor)
+            if len(raw) > maximum or (before.st_dev, before.st_ino, before.st_size) != (
+                after.st_dev,
+                after.st_ino,
+                after.st_size,
+            ):
+                raise RetrievalManifestError(
+                    "RETRIEVAL_MEMBER_TAMPER", "/".join(parts), "member changed during read"
+                )
+            return bytes(raw)
+        finally:
+            os.close(descriptor)
+    finally:
+        for descriptor in reversed(owned):
+            os.close(descriptor)