mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/query_trace.py

@@ -0,0 +1,1037 @@
+"""A sealed-catalog query whose answer carries its complete ranking and facts trace."""
+
+from __future__ import annotations
+
+import os
+import re
+import stat
+import tempfile
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any
+
+from mostlyright.data_harness.canonical import (
+    canonical_json_bytes,
+    canonical_sha256,
+    sha256_bytes,
+)
+from mostlyright.data_harness.sources.catalog.contracts import EMBEDDING_LAYERS
+from mostlyright.data_harness.sources.catalog.embedding import EmbeddingBackend
+from mostlyright.data_harness.sources.catalog.entry_v2 import (
+    CatalogEntryV2,
+    CatalogEntryV2Decision,
+    CatalogFactFilter,
+    evaluate_catalog_entry_v2,
+)
+from mostlyright.data_harness.sources.catalog.gating import CatalogRequirement, gate_entries
+from mostlyright.data_harness.sources.catalog.packed_retrieval import (
+    PACKED_INCOMPLETE_CODES,
+    CatalogSearchWorkLimits,
+    PackedSearchWork,
+    rank_packed_entries,
+)
+from mostlyright.data_harness.sources.catalog.recommend import (
+    RECOMMENDATION_BUCKETS,
+    SourceRecommendation,
+    recommend_sources,
+)
+from mostlyright.data_harness.sources.catalog.retrieval import _rank_retrieval
+from mostlyright.data_harness.sources.catalog.retrieval_manifest import (
+    LEXICAL_FALLBACK_REASONS,
+    LoadedRetrievalCatalog,
+    load_retrieval_catalog,
+)
+from mostlyright.data_harness.sources.catalog.sealed import CATALOG_FILENAME, SealedEmbeddingIndex
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+CATALOG_QUERY_TRACE_VERSION = "harness-catalog-query-trace.v1"
+CATALOG_QUERY_TRACE_V2_VERSION = "harness-catalog-query-trace.v2"
+CATALOG_PACKED_QUERY_TRACE_VERSION = "harness-catalog-packed-query-trace.v1"
+_DIGEST = re.compile(r"^[0-9a-f]{64}$")
+_MAX_SNAPSHOT_BYTES = 128 * 1024 * 1024
+_MAX_SNAPSHOT_FILES = 40_002
+_OPEN_DIRECTORY = os.O_RDONLY | os.O_DIRECTORY | getattr(os, "O_CLOEXEC", 0)
+_NOFOLLOW = getattr(os, "O_NOFOLLOW", 0)
+
+
+def _read_plain_file(parent_fd: int, name: str, *, remaining: int) -> bytes:
+    """Read one stable, single-link regular file relative to a retained directory."""
+
+    try:
+        descriptor = os.open(
+            name,
+            os.O_RDONLY | _NOFOLLOW | getattr(os, "O_CLOEXEC", 0),
+            dir_fd=parent_fd,
+        )
+    except OSError as error:
+        raise SourceContractError(
+            "CATALOG_SNAPSHOT_MEMBER",
+            name,
+            "cannot open a plain catalog member",
+        ) from error
+    try:
+        before = os.fstat(descriptor)
+        if not stat.S_ISREG(before.st_mode) or before.st_nlink != 1:
+            raise SourceContractError(
+                "CATALOG_SNAPSHOT_MEMBER",
+                name,
+                "catalog members must be single-link regular files",
+            )
+        if before.st_size > remaining:
+            raise SourceContractError(
+                "CATALOG_SNAPSHOT_LIMIT",
+                name,
+                "catalog snapshot exceeds the byte limit",
+            )
+        chunks: list[bytes] = []
+        observed = 0
+        while True:
+            chunk = os.read(descriptor, min(1024 * 1024, remaining - observed + 1))
+            if not chunk:
+                break
+            observed += len(chunk)
+            if observed > remaining:
+                raise SourceContractError(
+                    "CATALOG_SNAPSHOT_LIMIT",
+                    name,
+                    "catalog snapshot exceeds the byte limit",
+                )
+            chunks.append(chunk)
+        after = os.fstat(descriptor)
+
+        def identity(value: os.stat_result) -> tuple[int, int, int, int, int]:
+            return (
+                value.st_dev,
+                value.st_ino,
+                value.st_size,
+                value.st_mtime_ns,
+                value.st_ctime_ns,
+            )
+
+        content = b"".join(chunks)
+        if identity(before) != identity(after) or len(content) != after.st_size:
+            raise SourceContractError(
+                "CATALOG_SNAPSHOT_RACE",
+                name,
+                "catalog member changed while captured",
+            )
+        return content
+    finally:
+        os.close(descriptor)
+
+
+def _capture_catalog(directory: Path) -> tuple[bytes, tuple[tuple[str, bytes], ...]]:
+    """Capture the catalog from one retained root identity without pathname reopens."""
+
+    try:
+        root_fd = os.open(directory, _OPEN_DIRECTORY | _NOFOLLOW)
+    except OSError as error:
+        raise SourceContractError(
+            "CATALOG_SNAPSHOT_ROOT",
+            str(directory),
+            "cannot open a plain catalog directory",
+        ) from error
+    try:
+        catalog = _read_plain_file(root_fd, CATALOG_FILENAME, remaining=_MAX_SNAPSHOT_BYTES)
+        remaining = _MAX_SNAPSHOT_BYTES - len(catalog)
+        try:
+            vectors_fd = os.open("vectors", _OPEN_DIRECTORY | _NOFOLLOW, dir_fd=root_fd)
+        except OSError as error:
+            raise SourceContractError(
+                "CATALOG_SNAPSHOT_MEMBER",
+                "vectors",
+                "cannot open the plain vector directory",
+            ) from error
+        try:
+            entries = sorted(os.scandir(vectors_fd), key=lambda item: item.name)
+            if len(entries) > _MAX_SNAPSHOT_FILES - 1:
+                raise SourceContractError(
+                    "CATALOG_SNAPSHOT_LIMIT",
+                    "vectors",
+                    "catalog snapshot contains too many members",
+                )
+            vectors: list[tuple[str, bytes]] = []
+            for entry in entries:
+                if not re.fullmatch(r"[0-9a-f]{64}\.vec", entry.name):
+                    raise SourceContractError(
+                        "CATALOG_SNAPSHOT_MEMBER",
+                        entry.name,
+                        "vector member name is not canonical",
+                    )
+                content = _read_plain_file(vectors_fd, entry.name, remaining=remaining)
+                remaining -= len(content)
+                vectors.append((entry.name, content))
+        finally:
+            os.close(vectors_fd)
+        root_after = os.fstat(root_fd)
+        if not stat.S_ISDIR(root_after.st_mode):
+            raise SourceContractError(
+                "CATALOG_SNAPSHOT_ROOT",
+                str(directory),
+                "catalog root changed while captured",
+            )
+        return catalog, tuple(vectors)
+    finally:
+        os.close(root_fd)
+
+
+def _load_captured_catalog(
+    catalog: bytes, vectors: tuple[tuple[str, bytes], ...]
+) -> SealedEmbeddingIndex:
+    """Load verified bytes from a private snapshot, never from the caller's live path."""
+
+    with tempfile.TemporaryDirectory(prefix="mostlyright-catalog-snapshot-") as temporary:
+        root = Path(temporary)
+        vector_root = root / "vectors"
+        vector_root.mkdir(mode=0o700)
+        (root / CATALOG_FILENAME).write_bytes(catalog)
+        for name, content in vectors:
+            (vector_root / name).write_bytes(content)
+        return SealedEmbeddingIndex.load(root)
+
+
+@dataclass(frozen=True)
+class TracedCandidate:
+    """One ranked candidate with the exact catalog facts the answer exposes."""
+
+    rank: int
+    entry_coordinate: str
+    entry_digest: str
+    facts_digest: str
+    disposition: str
+    publisher: str
+    title: str
+    data_formats: tuple[str, ...]
+    update_frequency_seconds: int
+    access_kind: str
+    authentication_required: bool
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "rank": self.rank,
+            "entry_coordinate": self.entry_coordinate,
+            "entry_digest": self.entry_digest,
+            "facts_digest": self.facts_digest,
+            "disposition": self.disposition,
+            "publisher": self.publisher,
+            "title": self.title,
+            "data_formats": list(self.data_formats),
+            "update_frequency_seconds": self.update_frequency_seconds,
+            "access_kind": self.access_kind,
+            "authentication_required": self.authentication_required,
+        }
+
+
+@dataclass(frozen=True)
+class CatalogQueryTrace:
+    """Canonical catalog-query evidence that callers may seal and replay."""
+
+    catalog_manifest_sha256: str
+    backend_coordinate: str
+    question_digest: str
+    requirement_digest: str
+    candidates: tuple[TracedCandidate, ...]
+    admitted: tuple[str, ...]
+    escalated: tuple[str, ...]
+    refused: tuple[str, ...]
+    schema_version: str = CATALOG_QUERY_TRACE_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != CATALOG_QUERY_TRACE_VERSION:
+            raise SourceContractError(
+                "VERSION",
+                "trace.schema_version",
+                "unsupported trace version",
+            )
+        for name in ("catalog_manifest_sha256", "question_digest", "requirement_digest"):
+            value = getattr(self, name)
+            if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+                raise SourceContractError("SHA256", f"trace.{name}", "must be a lowercase SHA-256")
+        if not isinstance(self.backend_coordinate, str) or "@" not in self.backend_coordinate:
+            raise SourceContractError(
+                "CATALOG_BACKEND",
+                "trace.backend_coordinate",
+                "must name the exact query backend version",
+            )
+        expected_ranks = tuple(range(1, len(self.candidates) + 1))
+        if tuple(item.rank for item in self.candidates) != expected_ranks:
+            raise SourceContractError(
+                "CATALOG_TRACE_RANK",
+                "trace.candidates",
+                "ranks must be contiguous and ordered from one",
+            )
+
+    @property
+    def digest(self) -> str:
+        return canonical_sha256(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "catalog_manifest_sha256": self.catalog_manifest_sha256,
+            "backend_coordinate": self.backend_coordinate,
+            "question_digest": self.question_digest,
+            "requirement_digest": self.requirement_digest,
+            "candidates": [item.to_dict() for item in self.candidates],
+            "admitted": list(self.admitted),
+            "escalated": list(self.escalated),
+            "refused": list(self.refused),
+        }
+
+
+@dataclass(frozen=True)
+class CatalogQueryTraceV2:
+    """Canonical v2 evidence for one explicitly selected verified representation."""
+
+    retrieval_manifest_sha256: str
+    representation: str
+    representation_artifact_sha256: str
+    backend_coordinate: str
+    fallback_reason: str | None
+    requested_backend: str
+    lexical_fallback_allowed: bool
+    question_digest: str
+    requirement_digest: str
+    candidates: tuple[TracedCandidate, ...]
+    admitted: tuple[str, ...]
+    escalated: tuple[str, ...]
+    refused: tuple[str, ...]
+    schema_version: str = CATALOG_QUERY_TRACE_V2_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != CATALOG_QUERY_TRACE_V2_VERSION:
+            raise SourceContractError(
+                "VERSION", "trace.schema_version", "unsupported v2 trace version"
+            )
+        for name in (
+            "retrieval_manifest_sha256",
+            "representation_artifact_sha256",
+            "question_digest",
+            "requirement_digest",
+        ):
+            value = getattr(self, name)
+            if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+                raise SourceContractError("SHA256", f"trace.{name}", "must be a lowercase SHA-256")
+        if not isinstance(self.representation, str) or self.representation not in {
+            "lexical",
+            "neural",
+        }:
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND", "trace.representation", "must be lexical or neural"
+            )
+        if not isinstance(self.requested_backend, str) or self.requested_backend not in {
+            "lexical",
+            "neural",
+        }:
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND",
+                "trace.requested_backend",
+                "must be lexical or neural",
+            )
+        if type(self.lexical_fallback_allowed) is not bool:
+            raise SourceContractError(
+                "RETRIEVAL_FALLBACK",
+                "trace.lexical_fallback_allowed",
+                "must be a boolean",
+            )
+        if not isinstance(self.backend_coordinate, str) or "@" not in self.backend_coordinate:
+            raise SourceContractError(
+                "CATALOG_BACKEND",
+                "trace.backend_coordinate",
+                "must name the exact query backend version",
+            )
+        if self.fallback_reason is not None and (
+            not isinstance(self.fallback_reason, str)
+            or self.fallback_reason not in LEXICAL_FALLBACK_REASONS
+        ):
+            raise SourceContractError(
+                "RETRIEVAL_FALLBACK",
+                "trace.fallback_reason",
+                "must be a governed lexical fallback code",
+            )
+        if self.requested_backend == "lexical":
+            if self.representation != "lexical":
+                raise SourceContractError(
+                    "RETRIEVAL_BACKEND",
+                    "trace.representation",
+                    "a lexical request cannot produce a neural representation",
+                )
+            if self.fallback_reason is not None:
+                raise SourceContractError(
+                    "RETRIEVAL_FALLBACK",
+                    "trace.fallback_reason",
+                    "an explicit lexical request did not fall back from neural",
+                )
+        elif self.representation == "neural":
+            if self.fallback_reason is not None:
+                raise SourceContractError(
+                    "RETRIEVAL_FALLBACK",
+                    "trace.fallback_reason",
+                    "a neural result cannot also claim lexical fallback",
+                )
+        elif not self.lexical_fallback_allowed or self.fallback_reason is None:
+            raise SourceContractError(
+                "RETRIEVAL_FALLBACK",
+                "trace.fallback_reason",
+                "a neural request may produce lexical only with authorized typed fallback",
+            )
+        if self.representation == "lexical" and self.backend_coordinate != "lexical_hashing@1.0.0":
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND",
+                "trace.backend_coordinate",
+                "a lexical result must name the frozen lexical backend coordinate",
+            )
+        if self.representation == "neural" and (
+            "#" not in self.backend_coordinate or ":" not in self.backend_coordinate
+        ):
+            raise SourceContractError(
+                "RETRIEVAL_BACKEND",
+                "trace.backend_coordinate",
+                "a neural result must name an explicit tuple-qualified backend coordinate",
+            )
+
+        if not isinstance(self.candidates, tuple) or any(
+            type(item) is not TracedCandidate for item in self.candidates
+        ):
+            raise SourceContractError(
+                "CATALOG_TRACE_CANDIDATES",
+                "trace.candidates",
+                "must be an immutable tuple of strict traced candidates",
+            )
+        expected_ranks = tuple(range(1, len(self.candidates) + 1))
+        if tuple(item.rank for item in self.candidates) != expected_ranks:
+            raise SourceContractError(
+                "CATALOG_TRACE_RANK",
+                "trace.candidates",
+                "ranks must be contiguous and ordered from one",
+            )
+        coordinates: list[str] = []
+        dispositions = {disposition for _field, disposition in RECOMMENDATION_BUCKETS}
+        for position, item in enumerate(self.candidates):
+            if (
+                type(item.rank) is not int
+                or not isinstance(item.entry_coordinate, str)
+                or "@" not in item.entry_coordinate
+                or item.entry_coordinate in coordinates
+            ):
+                raise SourceContractError(
+                    "CATALOG_TRACE_CANDIDATES",
+                    f"trace.candidates[{position}].entry_coordinate",
+                    "must be a unique versioned identity",
+                )
+            coordinates.append(item.entry_coordinate)
+            for name in ("entry_digest", "facts_digest"):
+                value = getattr(item, name)
+                if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+                    raise SourceContractError(
+                        "CATALOG_TRACE_CANDIDATES",
+                        f"trace.candidates[{position}].{name}",
+                        "must be a lowercase SHA-256",
+                    )
+            if not isinstance(item.disposition, str) or item.disposition not in dispositions:
+                raise SourceContractError(
+                    "CATALOG_TRACE_CANDIDATES",
+                    f"trace.candidates[{position}].disposition",
+                    "must be a governed facts disposition",
+                )
+            if (
+                not isinstance(item.publisher, str)
+                or not item.publisher
+                or not isinstance(item.title, str)
+                or not item.title
+                or not isinstance(item.data_formats, tuple)
+                or not item.data_formats
+                or any(not isinstance(value, str) or not value for value in item.data_formats)
+                or type(item.update_frequency_seconds) is not int
+                or item.update_frequency_seconds < 0
+                or not isinstance(item.access_kind, str)
+                or not item.access_kind
+                or type(item.authentication_required) is not bool
+            ):
+                raise SourceContractError(
+                    "CATALOG_TRACE_CANDIDATES",
+                    f"trace.candidates[{position}]",
+                    "candidate facts must retain their strict immutable types",
+                )
+        for field, disposition in RECOMMENDATION_BUCKETS:
+            observed = getattr(self, field)
+            if not isinstance(observed, tuple) or any(type(item) is not str for item in observed):
+                raise SourceContractError(
+                    "CATALOG_TRACE_PARTITION",
+                    f"trace.{field}",
+                    "must be an immutable tuple of candidate identities",
+                )
+            expected = tuple(
+                item.entry_coordinate for item in self.candidates if item.disposition == disposition
+            )
+            if observed != expected:
+                raise SourceContractError(
+                    "CATALOG_TRACE_PARTITION",
+                    f"trace.{field}",
+                    "must exactly preserve the ordered candidates with this disposition",
+                )
+
+    @property
+    def canonical_bytes(self) -> bytes:
+        return canonical_json_bytes(self.to_dict())
+
+    @property
+    def digest(self) -> str:
+        return sha256_bytes(self.canonical_bytes)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "retrieval_manifest_sha256": self.retrieval_manifest_sha256,
+            "representation": self.representation,
+            "representation_artifact_sha256": self.representation_artifact_sha256,
+            "backend_coordinate": self.backend_coordinate,
+            "fallback_reason": self.fallback_reason,
+            "requested_backend": self.requested_backend,
+            "lexical_fallback_allowed": self.lexical_fallback_allowed,
+            "question_digest": self.question_digest,
+            "requirement_digest": self.requirement_digest,
+            "candidates": [item.to_dict() for item in self.candidates],
+            "admitted": list(self.admitted),
+            "escalated": list(self.escalated),
+            "refused": list(self.refused),
+        }
+
+
+@dataclass(frozen=True)
+class CatalogQuerySnapshot:
+    """One verified in-memory catalog snapshot and the answer derived from those exact bytes."""
+
+    recommendation: SourceRecommendation
+    trace: CatalogQueryTrace
+    index: SealedEmbeddingIndex
+    catalog_bytes: bytes
+    vector_members: tuple[tuple[str, bytes], ...]
+
+
+def query_catalog_snapshot(
+    *,
+    directory: Path,
+    expected_catalog_manifest_sha256: str,
+    question: str,
+    requirement: CatalogRequirement,
+    limit: int = 5,
+) -> CatalogQuerySnapshot:
+    """Verify and capture a sealed catalog, then answer only from that retained snapshot."""
+
+    if not isinstance(expected_catalog_manifest_sha256, str) or (
+        _DIGEST.fullmatch(expected_catalog_manifest_sha256) is None
+    ):
+        raise SourceContractError(
+            "SHA256",
+            "expected_catalog_manifest_sha256",
+            "must be a lowercase SHA-256",
+        )
+    if not isinstance(directory, Path):
+        raise SourceContractError("TYPE", "directory", "must be a Path")
+    index_path = directory / CATALOG_FILENAME
+    manifest_bytes, vector_members = _capture_catalog(directory)
+    observed_manifest_sha256 = sha256_bytes(manifest_bytes)
+    if observed_manifest_sha256 != expected_catalog_manifest_sha256:
+        raise SourceContractError(
+            "CATALOG_MANIFEST_MISMATCH",
+            str(index_path),
+            "sealed catalog index differs from the expected manifest digest",
+        )
+
+    index = _load_captured_catalog(manifest_bytes, vector_members)
+    recommendation = recommend_sources(
+        index=index,
+        question=question,
+        requirement=requirement,
+        limit=limit,
+    )
+    by_coordinate = {entry.coordinate: entry for entry in index.entries}
+    candidates: list[TracedCandidate] = []
+    for rank, verdict in enumerate(recommendation.candidates, start=1):
+        entry = by_coordinate[verdict.entry_coordinate]
+        candidates.append(
+            TracedCandidate(
+                rank=rank,
+                entry_coordinate=entry.coordinate,
+                entry_digest=entry.digest,
+                facts_digest=entry.facts_digest,
+                disposition=verdict.disposition,
+                publisher=entry.publisher,
+                title=entry.title,
+                data_formats=entry.data_formats,
+                update_frequency_seconds=entry.observation.update_frequency_seconds,
+                access_kind=entry.access_kind,
+                authentication_required=entry.authentication_required,
+            )
+        )
+    trace = CatalogQueryTrace(
+        catalog_manifest_sha256=observed_manifest_sha256,
+        backend_coordinate=index.backend.descriptor.coordinate,
+        question_digest=recommendation.question_digest,
+        requirement_digest=recommendation.requirement_digest,
+        candidates=tuple(candidates),
+        admitted=recommendation.admitted,
+        escalated=recommendation.escalated,
+        refused=recommendation.refused,
+    )
+    return CatalogQuerySnapshot(recommendation, trace, index, manifest_bytes, vector_members)
+
+
+def query_catalog_with_trace(
+    *,
+    directory: Path,
+    expected_catalog_manifest_sha256: str,
+    question: str,
+    requirement: CatalogRequirement,
+    limit: int = 5,
+) -> tuple[SourceRecommendation, CatalogQueryTrace]:
+    """Verify a sealed catalog, answer from it, and return a source-bound trace."""
+
+    snapshot = query_catalog_snapshot(
+        directory=directory,
+        expected_catalog_manifest_sha256=expected_catalog_manifest_sha256,
+        question=question,
+        requirement=requirement,
+        limit=limit,
+    )
+    return snapshot.recommendation, snapshot.trace
+
+
+def query_retrieval_catalog_with_trace(
+    *,
+    directory: Path,
+    expected_retrieval_manifest_sha256: str,
+    neural_backend: EmbeddingBackend | None,
+    backend: str,
+    allow_lexical_fallback: bool,
+    question: str,
+    requirement: CatalogRequirement,
+    limit: int = 5,
+) -> tuple[SourceRecommendation, CatalogQueryTraceV2]:
+    """Query one verified v2 catalog and bind its exact score-free public evidence."""
+
+    loaded = load_retrieval_catalog(
+        directory,
+        expected_manifest_sha256=expected_retrieval_manifest_sha256,
+        neural_backend=neural_backend,
+    )
+    entry_ids, representation, backend_coordinate, fallback_reason = _rank_retrieval(
+        loaded,
+        question,
+        backend=backend,
+        allow_lexical_fallback=allow_lexical_fallback,
+        limit=limit,
+    )
+    recommendation = _recommend_loaded(
+        loaded,
+        entry_ids,
+        question=question,
+        requirement=requirement,
+    )
+    selected_manifest = (
+        loaded.manifest.neural if representation == "neural" else loaded.manifest.lexical
+    )
+    if selected_manifest is None:  # pragma: no cover - guarded by the retrieval result contract
+        raise SourceContractError(
+            "RETRIEVAL_BACKEND", "trace.representation", "selected representation is absent"
+        )
+    entries_by_coordinate = {entry.coordinate: entry for entry in loaded.lexical.entries}
+    trace = CatalogQueryTraceV2(
+        retrieval_manifest_sha256=expected_retrieval_manifest_sha256,
+        representation=representation,
+        representation_artifact_sha256=selected_manifest.artifact_sha256,
+        backend_coordinate=backend_coordinate,
+        fallback_reason=fallback_reason,
+        requested_backend=backend,
+        lexical_fallback_allowed=allow_lexical_fallback,
+        question_digest=recommendation.question_digest,
+        requirement_digest=recommendation.requirement_digest,
+        candidates=_trace_candidates(recommendation, entries_by_coordinate),
+        admitted=recommendation.admitted,
+        escalated=recommendation.escalated,
+        refused=recommendation.refused,
+    )
+    return recommendation, trace
+
+
+def _recommend_loaded(
+    loaded: LoadedRetrievalCatalog,
+    entry_ids: tuple[str, ...],
+    *,
+    question: str,
+    requirement: CatalogRequirement,
+) -> SourceRecommendation:
+    by_id = {entry.entry_id: entry for entry in loaded.lexical.entries}
+    entries = tuple(by_id[entry_id] for entry_id in entry_ids)
+    verdicts = gate_entries(entries, requirement=requirement)
+    sorted_into = {
+        field: tuple(item.entry_coordinate for item in verdicts if item.disposition == disposition)
+        for field, disposition in RECOMMENDATION_BUCKETS
+    }
+    return SourceRecommendation(
+        question_digest=canonical_sha256({"question": question}),
+        requirement_digest=requirement.digest,
+        candidates=verdicts,
+        **sorted_into,
+    )
+
+
+def _trace_candidates(
+    recommendation: SourceRecommendation,
+    entries_by_coordinate: dict[str, Any],
+) -> tuple[TracedCandidate, ...]:
+    candidates: list[TracedCandidate] = []
+    for rank, verdict in enumerate(recommendation.candidates, start=1):
+        entry = entries_by_coordinate[verdict.entry_coordinate]
+        candidates.append(
+            TracedCandidate(
+                rank=rank,
+                entry_coordinate=entry.coordinate,
+                entry_digest=entry.digest,
+                facts_digest=entry.facts_digest,
+                disposition=verdict.disposition,
+                publisher=entry.publisher,
+                title=entry.title,
+                data_formats=entry.data_formats,
+                update_frequency_seconds=entry.observation.update_frequency_seconds,
+                access_kind=entry.access_kind,
+                authentication_required=entry.authentication_required,
+            )
+        )
+    return tuple(candidates)
+
+
+# ----------------------------------------------------------------------------------------------
+# The packed generation's query evidence
+# ----------------------------------------------------------------------------------------------
+
+
+def _packed_disposition(decision: CatalogEntryV2Decision) -> str:
+    """Project the v2 fact authority's verdict onto the three governed answer dispositions.
+
+    ``evaluate_catalog_entry_v2`` is the authority; nothing here re-adjudicates rights. Unclear
+    rights are ``flagged`` there and become ``human_escalation_required`` here, because a human
+    decides them; everything else follows the recommendation the fact authority already reached.
+    """
+
+    if decision.rights_disposition == "flagged":
+        return "human_escalation_required"
+    return "admitted" if decision.recommendation == "use_source" else "refused"
+
+
+@dataclass(frozen=True)
+class PackedTracedCandidate:
+    """One ordered identity with the fact disposition it earned. No ranking number, no key."""
+
+    rank: int
+    entry_coordinate: str
+    entry_digest: str
+    semantic_facts_digest: str
+    disposition: str
+    recommendation: str
+    filters_match: bool
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "rank": self.rank,
+            "entry_coordinate": self.entry_coordinate,
+            "entry_digest": self.entry_digest,
+            "semantic_facts_digest": self.semantic_facts_digest,
+            "disposition": self.disposition,
+            "recommendation": self.recommendation,
+            "filters_match": self.filters_match,
+        }
+
+
+@dataclass(frozen=True)
+class CatalogPackedQueryTrace:
+    """Canonical evidence for one bounded search of one authenticated packed generation.
+
+    It binds what was searched (the head and the receipt coordinate), what was read (the range
+    manifests and every accelerator and pack the search opened), how it was searched (the backend
+    coordinate and the four layers), how much it was allowed to cost and how much it did cost, and
+    what the facts decided about each ordered identity.
+
+    It carries no score, no provider record key and no filesystem path -- an answer's evidence is
+    about identities and facts, and a similarity number in a trace is a similarity number a later
+    reader will eventually treat as a decision.
+    """
+
+    head_sha256: str
+    receipt_coordinate_sha256: str
+    backend_coordinate: str
+    layers: tuple[str, ...]
+    range_manifest_sha256s: tuple[str, ...]
+    vector_member_sha256s: tuple[str, ...]
+    posting_member_sha256s: tuple[str, ...]
+    bound_member_sha256s: tuple[str, ...]
+    work_limits: CatalogSearchWorkLimits
+    work_actuals: PackedSearchWork
+    status: str
+    incomplete_code: str | None
+    question_digest: str
+    requirement_digest: str
+    candidates: tuple[PackedTracedCandidate, ...]
+    admitted: tuple[str, ...]
+    escalated: tuple[str, ...]
+    refused: tuple[str, ...]
+    schema_version: str = CATALOG_PACKED_QUERY_TRACE_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != CATALOG_PACKED_QUERY_TRACE_VERSION:
+            raise SourceContractError(
+                "VERSION", "trace.schema_version", "unsupported packed trace version"
+            )
+        for name in (
+            "head_sha256",
+            "receipt_coordinate_sha256",
+            "question_digest",
+            "requirement_digest",
+        ):
+            value = getattr(self, name)
+            if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+                raise SourceContractError("SHA256", f"trace.{name}", "must be a lowercase SHA-256")
+        if not isinstance(self.backend_coordinate, str) or "@" not in self.backend_coordinate:
+            raise SourceContractError(
+                "CATALOG_BACKEND",
+                "trace.backend_coordinate",
+                "must name the exact query backend version",
+            )
+        if self.layers != EMBEDDING_LAYERS:
+            raise SourceContractError(
+                "CATALOG_TRACE_LAYERS",
+                "trace.layers",
+                f"a packed answer scores exactly the four layers {list(EMBEDDING_LAYERS)}",
+            )
+        for name in (
+            "range_manifest_sha256s",
+            "vector_member_sha256s",
+            "posting_member_sha256s",
+            "bound_member_sha256s",
+        ):
+            values = getattr(self, name)
+            if not isinstance(values, tuple) or any(
+                not isinstance(value, str) or _DIGEST.fullmatch(value) is None for value in values
+            ):
+                raise SourceContractError(
+                    "CATALOG_TRACE_MEMBERS",
+                    f"trace.{name}",
+                    "must be an immutable tuple of lowercase SHA-256 digests",
+                )
+        if not isinstance(self.work_limits, CatalogSearchWorkLimits) or not isinstance(
+            self.work_actuals, PackedSearchWork
+        ):
+            raise SourceContractError(
+                "CATALOG_TRACE_WORK",
+                "trace.work_limits",
+                "a packed answer states the closed work contract and the work it actually did",
+            )
+        if self.status not in {"complete", "incomplete"}:
+            raise SourceContractError(
+                "CATALOG_TRACE_STATUS", "trace.status", "a search is complete or incomplete"
+            )
+        if self.status == "complete":
+            if self.incomplete_code is not None:
+                raise SourceContractError(
+                    "CATALOG_TRACE_STATUS",
+                    "trace.incomplete_code",
+                    "a complete search names no exhausted budget",
+                )
+        elif (
+            self.incomplete_code not in PACKED_INCOMPLETE_CODES
+            or self.candidates
+            or self.admitted
+            or self.escalated
+            or self.refused
+        ):
+            raise SourceContractError(
+                "CATALOG_TRACE_STATUS",
+                "trace.candidates",
+                "an incomplete search names the budget that closed and offers no ranked identity",
+            )
+        if not isinstance(self.candidates, tuple) or any(
+            type(item) is not PackedTracedCandidate for item in self.candidates
+        ):
+            raise SourceContractError(
+                "CATALOG_TRACE_CANDIDATES",
+                "trace.candidates",
+                "must be an immutable tuple of strict packed candidates",
+            )
+        if tuple(item.rank for item in self.candidates) != tuple(
+            range(1, len(self.candidates) + 1)
+        ):
+            raise SourceContractError(
+                "CATALOG_TRACE_RANK",
+                "trace.candidates",
+                "ranks must be contiguous and ordered from one",
+            )
+        coordinates: list[str] = []
+        dispositions = {disposition for _field, disposition in RECOMMENDATION_BUCKETS}
+        for position, item in enumerate(self.candidates):
+            if (
+                not isinstance(item.entry_coordinate, str)
+                or "@" not in item.entry_coordinate
+                or item.entry_coordinate in coordinates
+            ):
+                raise SourceContractError(
+                    "CATALOG_TRACE_CANDIDATES",
+                    f"trace.candidates[{position}].entry_coordinate",
+                    "must be a unique versioned identity",
+                )
+            coordinates.append(item.entry_coordinate)
+            for name in ("entry_digest", "semantic_facts_digest"):
+                value = getattr(item, name)
+                if not isinstance(value, str) or _DIGEST.fullmatch(value) is None:
+                    raise SourceContractError(
+                        "CATALOG_TRACE_CANDIDATES",
+                        f"trace.candidates[{position}].{name}",
+                        "must be a lowercase SHA-256",
+                    )
+            if item.disposition not in dispositions:
+                raise SourceContractError(
+                    "CATALOG_TRACE_CANDIDATES",
+                    f"trace.candidates[{position}].disposition",
+                    "must be a governed facts disposition",
+                )
+        for field, disposition in RECOMMENDATION_BUCKETS:
+            observed = getattr(self, field)
+            if not isinstance(observed, tuple) or any(type(item) is not str for item in observed):
+                raise SourceContractError(
+                    "CATALOG_TRACE_PARTITION",
+                    f"trace.{field}",
+                    "must be an immutable tuple of candidate identities",
+                )
+            expected = tuple(
+                item.entry_coordinate for item in self.candidates if item.disposition == disposition
+            )
+            if observed != expected:
+                raise SourceContractError(
+                    "CATALOG_TRACE_PARTITION",
+                    f"trace.{field}",
+                    "must exactly preserve the ordered candidates with this disposition",
+                )
+
+    @property
+    def canonical_bytes(self) -> bytes:
+        return canonical_json_bytes(self.to_dict())
+
+    @property
+    def digest(self) -> str:
+        return sha256_bytes(self.canonical_bytes)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "head_sha256": self.head_sha256,
+            "receipt_coordinate_sha256": self.receipt_coordinate_sha256,
+            "backend_coordinate": self.backend_coordinate,
+            "layers": list(self.layers),
+            "range_manifest_sha256s": list(self.range_manifest_sha256s),
+            "vector_member_sha256s": list(self.vector_member_sha256s),
+            "posting_member_sha256s": list(self.posting_member_sha256s),
+            "bound_member_sha256s": list(self.bound_member_sha256s),
+            "work_limits": self.work_limits.to_dict(),
+            "work_actuals": self.work_actuals.to_dict(),
+            "status": self.status,
+            "incomplete_code": self.incomplete_code,
+            "question_digest": self.question_digest,
+            "requirement_digest": self.requirement_digest,
+            "candidates": [item.to_dict() for item in self.candidates],
+            "admitted": list(self.admitted),
+            "escalated": list(self.escalated),
+            "refused": list(self.refused),
+        }
+
+
+@dataclass(frozen=True)
+class PackedCatalogAnswer:
+    """One packed answer: its evidence, the fact verdicts behind it, and the entries themselves."""
+
+    status: str
+    incomplete_code: str | None
+    trace: CatalogPackedQueryTrace
+    decisions: tuple[CatalogEntryV2Decision, ...]
+    entries: tuple[CatalogEntryV2, ...]
+
+
+def query_packed_catalog_with_trace(
+    *,
+    directory: Path,
+    expected_head_sha256: str,
+    receipt: Any,
+    backend: EmbeddingBackend,
+    question: str,
+    requirement: CatalogRequirement,
+    work_limits: CatalogSearchWorkLimits,
+    fact_filters: tuple[CatalogFactFilter, ...] = (),
+    limit: int = 5,
+) -> PackedCatalogAnswer:
+    """Rank one authenticated packed generation, then decide each ordered identity on facts.
+
+    The ranking is bounded and exact and knows nothing about facts; the fact authority is
+    ``evaluate_catalog_entry_v2`` and knows nothing about the ranking. ``requirement`` is bound into
+    the evidence so an answer cites what it was asked for, and the structured fact filters the
+    caller supplied are passed to that authority unchanged.
+    """
+
+    if not isinstance(requirement, CatalogRequirement):
+        raise SourceContractError("TYPE", "requirement", "must be a CatalogRequirement")
+    if not isinstance(fact_filters, tuple):
+        raise SourceContractError("TYPE", "fact_filters", "must be an immutable tuple")
+    result = rank_packed_entries(
+        root=directory,
+        expected_head_sha256=expected_head_sha256,
+        receipt=receipt,
+        question=question,
+        backend=backend,
+        limit=limit,
+        work_limits=work_limits,
+    )
+    decisions: list[CatalogEntryV2Decision] = []
+    candidates: list[PackedTracedCandidate] = []
+    for ranked in result.entries:
+        decision = evaluate_catalog_entry_v2(ranked.entry, filters=fact_filters)
+        decisions.append(decision)
+        candidates.append(
+            PackedTracedCandidate(
+                rank=ranked.rank,
+                entry_coordinate=ranked.entry_coordinate,
+                entry_digest=ranked.entry_digest,
+                semantic_facts_digest=ranked.semantic_facts_digest,
+                disposition=_packed_disposition(decision),
+                recommendation=decision.recommendation,
+                filters_match=decision.filters_match,
+            )
+        )
+    sorted_into = {
+        field: tuple(
+            item.entry_coordinate for item in candidates if item.disposition == disposition
+        )
+        for field, disposition in RECOMMENDATION_BUCKETS
+    }
+    trace = CatalogPackedQueryTrace(
+        head_sha256=result.head_sha256,
+        receipt_coordinate_sha256=result.receipt_coordinate_sha256,
+        backend_coordinate=result.backend_coordinate,
+        layers=EMBEDDING_LAYERS,
+        range_manifest_sha256s=result.range_manifest_sha256s,
+        vector_member_sha256s=result.vector_member_sha256s,
+        posting_member_sha256s=result.posting_member_sha256s,
+        bound_member_sha256s=result.bound_member_sha256s,
+        work_limits=result.limits,
+        work_actuals=result.work,
+        status=result.status,
+        incomplete_code=result.incomplete_code,
+        question_digest=result.question_digest,
+        requirement_digest=requirement.digest,
+        candidates=tuple(candidates),
+        **sorted_into,
+    )
+    return PackedCatalogAnswer(
+        status=result.status,
+        incomplete_code=result.incomplete_code,
+        trace=trace,
+        decisions=tuple(decisions),
+        entries=tuple(ranked.entry for ranked in result.entries),
+    )