mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/sealed.py

@@ -0,0 +1,560 @@
+"""Query time: load a sealed catalog, verify every byte of it, and compute nothing.
+
+That sentence is the query-time trust boundary on this side of the line. ``load`` reads a canonical
+index, rebuilds each entry strictly, checks every vector file against the digest carried in the
+entry it belongs to, and resolves an encoder for later query use. It never calls a backend's
+``encode``, opens a socket, or loads a model. If a catalog was sealed by a backend that is not
+safe to run in-process, the load fails rather than degrading into one that would.
+
+Failures are split three ways, in the taxonomy the sealed-artifact replay path already uses,
+because a missing artifact, a drifted artifact and a policy refusal are different operational
+events and reading them as one loses the distinction that matters when something breaks:
+
+* ``CatalogMissError`` -- an artifact the index references is not there.
+* ``CatalogTamperError`` -- an artifact is there but is not what it should be. Three different
+  things count: it disagrees with the index that references it, the index itself disagrees with
+  the backend that will read it, or a sealed vector disagrees with the facts of the entry that
+  references it. The last two are the ones a self-consistent forgery survives, because every check
+  that reads only the artifact agrees with the forgery by construction.
+* ``CatalogRuntimeModelError`` -- the artifacts are fine, but running this catalog at query time
+  would mean running a model at query time.
+"""
+
+from __future__ import annotations
+
+import re
+import struct
+from collections.abc import Mapping
+from dataclasses import dataclass
+from pathlib import Path
+
+from mostlyright.data_harness.canonical import parse_canonical_json, sha256_bytes
+from mostlyright.data_harness.sources.catalog.contracts import (
+    CATALOG_INDEX_CONTRACT_VERSION,
+    EMBEDDING_LAYERS,
+    CatalogEntry,
+    catalog_entry_from_dict,
+)
+from mostlyright.data_harness.sources.catalog.embedding import (
+    DEFAULT_BACKENDS,
+    BackendDescriptor,
+    BackendRegistry,
+    EmbeddingBackend,
+)
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+CATALOG_FILENAME = "catalog.json"
+VECTORS_DIRNAME = "vectors"
+VECTOR_SUFFIX = ".vec"
+MAX_SEALED_ENTRIES = 10_000
+
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+
+class CatalogMissError(SourceContractError):
+    """A sealed artifact the index references is absent."""
+
+
+class CatalogTamperError(SourceContractError):
+    """A sealed artifact is present but does not match what the index says it is."""
+
+
+class CatalogRuntimeModelError(SourceContractError):
+    """The catalog was sealed by a backend that must not run on the query path."""
+
+
+@dataclass(frozen=True)
+class SealedEmbeddingIndex:
+    """A verified catalog: its entries, its integer vectors, and its query-side encoder.
+
+    The query-safe refusal lives on this type and not only in :meth:`load`. ``retrieval`` calls
+    ``index.backend.encode(question)`` on every question, so this is the object that decides whether
+    a model runs on the runtime path. Putting the only check in ``load`` would make this boundary a
+    statement about how callers happen to build one -- an unenforced convention -- and
+    ``SealedEmbeddingIndex(entries=..., vectors=..., backend=...)``
+    is an ordinary public constructor that any caller, or any later convenience path inside this
+    package, can reach without going near ``load``. A check here holds for every construction route
+    there is, which is what lets ``retrieval`` state its precondition as a property of its argument
+    rather than as a provenance claim.
+
+    The same argument reaches the entry/vector binding. ``load`` checks it
+    two ways -- ``entry_facts_digest`` against the entry's own facts, and each vector's bytes
+    against the digest the entry references. This constructor checked neither, so an index built
+    directly out of one entry's entries and another's vectors was accepted in silence and then
+    ranked the wrong source with no error at any point. Both checks now run here, over the same
+    references, which is what makes ``entry_facts_digest`` a binding rather than a field.
+
+    It also reaches the whole backend identity. ``load`` binds all four of ``backend_id``,
+    ``backend_version``, ``dimensions``
+    and ``quantization`` to the backend it resolved; this constructor bound none of them, so an
+    index pairing one query-safe backend's sealed entries with a *different* query-safe backend of
+    the same dimensions and the same quantization constructed in silence and answered the gate-three
+    question with the wrong source. Both routes now run one shared checker,
+    :func:`_require_sealed_vectors_come_from_this_backend`, so neither can enumerate a different
+    rule than the other.
+
+    The checks cost a descriptor read and one digest per sealed vector. They compute no embedding,
+    so they are legal where they stand.
+    """
+
+    entries: tuple[CatalogEntry, ...]
+    vectors: Mapping[tuple[str, str], tuple[int, ...]]
+    backend: EmbeddingBackend
+
+    def __post_init__(self) -> None:
+        if not isinstance(self.entries, tuple) or not self.entries:
+            raise SourceContractError(
+                "TYPE",
+                "index.entries",
+                "must be a non-empty immutable tuple of catalog entries",
+            )
+        for position, entry in enumerate(self.entries):
+            if not isinstance(entry, CatalogEntry):
+                raise SourceContractError(
+                    "TYPE",
+                    f"index.entries[{position}]",
+                    "must be a CatalogEntry",
+                )
+        # One current version per source, refused here and not only in `load`. Two entries sharing
+        # an id make the vector key `(entry_id, layer)` ambiguous, and `recommend_sources` builds
+        # `{entry.entry_id: entry}` -- so the second one wins, the first is dropped from the
+        # answer, and nothing raises. `load` refuses this on the artifact and keeps its own
+        # tamper-taxonomy code for that; this is the same refusal for every other construction
+        # route, by the argument that put the query-safe refusal here.
+        identifiers = [entry.entry_id for entry in self.entries]
+        if len(set(identifiers)) != len(identifiers):
+            raise SourceContractError(
+                "DUPLICATE",
+                "index.entries",
+                "two entries name the same source; an index holds one version of each",
+            )
+        if not isinstance(self.vectors, Mapping):
+            raise SourceContractError(
+                "TYPE",
+                "index.vectors",
+                "must be a mapping of (entry_id, layer) to a sealed integer vector",
+            )
+        descriptor = self.backend.descriptor
+        if not isinstance(descriptor, BackendDescriptor):
+            raise SourceContractError(
+                "EMBEDDING_BACKEND_DESCRIPTOR",
+                "index.backend.descriptor",
+                "backend descriptor must use the strict contract",
+            )
+        if not descriptor.query_safe:
+            raise CatalogRuntimeModelError(
+                "CATALOG_RUNTIME_MODEL",
+                "index.backend",
+                f"{descriptor.coordinate} is not query-safe; an index carrying it would run a "
+                "model on the runtime path the first time it was asked a question",
+            )
+        _require_vectors_the_query_path_can_read(
+            self.vectors, known_entry_ids=frozenset(identifiers), dimensions=descriptor.dimensions
+        )
+        # Runs only once the map is known to be well-formed and complete: it indexes the map by
+        # every reference an entry carries, and it packs each vector back into the sealed
+        # encoding, both of which need the shape checks above to have passed.
+        _require_vectors_bound_to_the_entries_that_reference_them(self.entries, self.vectors)
+        # Runs last, and it is the same function `load` runs against the backend it resolved. It
+        # is what stops a cross-backend index: two backends of identical dimensions and identical
+        # quantization produce vectors that multiply against each other without complaint and rank
+        # a different source first, so nothing above can see the substitution.
+        _require_sealed_vectors_come_from_this_backend(
+            self.entries, descriptor, path="index.entries"
+        )
+
+    @classmethod
+    def load(
+        cls,
+        directory: Path,
+        *,
+        registry: BackendRegistry = DEFAULT_BACKENDS,
+    ) -> SealedEmbeddingIndex:
+        """Read and verify a sealed catalog directory. Computes no embedding."""
+
+        if not isinstance(directory, Path):
+            raise SourceContractError("TYPE", "directory", "must be a Path")
+        index_path = directory / CATALOG_FILENAME
+        try:
+            raw = index_path.read_bytes()
+        except FileNotFoundError:
+            raise CatalogMissError(
+                "CATALOG_INDEX_MISSING",
+                str(index_path),
+                "the sealed catalog index is absent",
+            ) from None
+        # Canonical, not merely parseable: a re-serialised or hand-edited index is refused here,
+        # before a single field of it is trusted.
+        payload = parse_canonical_json(raw)
+        if not isinstance(payload, dict):
+            raise CatalogTamperError(
+                "CATALOG_INDEX_SHAPE",
+                CATALOG_FILENAME,
+                "the sealed catalog index must be an object",
+            )
+        if set(payload) != {"schema_version", "entries"}:
+            raise CatalogTamperError(
+                "CATALOG_INDEX_SHAPE",
+                CATALOG_FILENAME,
+                "must contain exactly ['entries', 'schema_version']",
+            )
+        if payload["schema_version"] != CATALOG_INDEX_CONTRACT_VERSION:
+            raise CatalogTamperError(
+                "VERSION",
+                "catalog.schema_version",
+                f"must equal {CATALOG_INDEX_CONTRACT_VERSION!r}",
+            )
+        raw_entries = payload["entries"]
+        if not isinstance(raw_entries, list) or not raw_entries:
+            raise CatalogTamperError(
+                "CATALOG_INDEX_SHAPE",
+                "catalog.entries",
+                "must be a non-empty array",
+            )
+        if len(raw_entries) > MAX_SEALED_ENTRIES:
+            raise CatalogTamperError(
+                "LIMIT",
+                "catalog.entries",
+                f"must contain at most {MAX_SEALED_ENTRIES} entries",
+            )
+
+        entries = tuple(catalog_entry_from_dict(item) for item in raw_entries)
+        # A sealed catalog holds one current version per source. Allowing two versions of one
+        # entry id would make the vector key ambiguous and silently drop half of them.
+        identifiers = [entry.entry_id for entry in entries]
+        if len(set(identifiers)) != len(identifiers):
+            raise CatalogTamperError(
+                "DUPLICATE",
+                "catalog.entries",
+                "two entries name the same source",
+            )
+
+        vectors: dict[tuple[str, str], tuple[int, ...]] = {}
+        for entry in entries:
+            layers = tuple(reference.layer for reference in entry.vectors)
+            if sorted(layers) != sorted(EMBEDDING_LAYERS):
+                raise CatalogMissError(
+                    "CATALOG_VECTOR_LAYERS",
+                    f"{entry.coordinate}.vectors",
+                    f"a sealed entry must carry one vector per layer {list(EMBEDDING_LAYERS)}",
+                )
+            facts_digest = entry.facts_digest
+            for reference in entry.vectors:
+                # Bind the vector to the facts it was computed from. Every other check in this
+                # loop reads only the artifact, so a catalog assembled from entries carrying some
+                # other entry's vectors satisfies all of them: four layers present, one backend
+                # identity, files present and digest-matching. It would then rank each entry on the
+                # other one's text with no error at any point -- the only silent failure this
+                # module has. `facts_digest` excludes `vectors`, so this is a digest comparison and
+                # not a re-encode; nothing on this path computes an embedding.
+                if reference.entry_facts_digest != facts_digest:
+                    raise CatalogTamperError(
+                        "CATALOG_VECTOR_ENTRY_BINDING",
+                        f"{entry.coordinate}.{reference.layer}",
+                        "the sealed vector was computed from different entry facts than the entry "
+                        "that references it",
+                    )
+                vectors[(entry.entry_id, reference.layer)] = _read_vector(
+                    directory,
+                    digest=reference.vector_sha256,
+                    dimensions=reference.dimensions,
+                    path=f"{entry.coordinate}.{reference.layer}",
+                )
+
+        # The sealed identity is read once, by the shared checker, so this route and the public
+        # constructor cannot enumerate different rules again. Here it names the coordinate the
+        # catalog must be resolved against; below, the same function checks it against what was
+        # resolved.
+        backend_id, backend_version = _require_one_sealed_backend_identity(
+            entries, path="catalog.entries"
+        )[:2]
+        backend = registry.resolve(backend_id, backend_version)
+        # Bind the sealed vector identity to the backend that will read it. The checks above prove
+        # each vector file is internally consistent with the entry that references it, which a
+        # forged index satisfies trivially by declaring its own shorter dimension count -- the
+        # length check and the digest check both agree with the forgery. Only the resolved
+        # descriptor is outside the artifact, so it is the only thing that can catch one.
+        #
+        # Left unbound, a mismatch escapes the load path entirely and surfaces at query time: a
+        # dimension mismatch as a bare ValueError out of the strict `zip` in `retrieval`, which is
+        # outside this module's three-way error taxonomy, and a quantization mismatch not as an
+        # error at all -- two vectors on different scales still multiply, so retrieval would just
+        # rank wrongly and silently. That is the worse of the two, which is why quantization is
+        # bound here as well as the dimension count.
+        #
+        # This runs before the query-safe refusal because it is a statement about the artifact
+        # rather than about policy, and it groups with the integrity checks above. It is the same
+        # function `__post_init__` runs, so `load` grants nothing the constructor does not.
+        _require_sealed_vectors_come_from_this_backend(
+            entries, backend.descriptor, path="catalog.entries"
+        )
+        if not backend.descriptor.query_safe:
+            raise CatalogRuntimeModelError(
+                "CATALOG_RUNTIME_MODEL",
+                "catalog.backend",
+                f"{backend_id}@{backend_version} is not query-safe; a catalog sealed by it "
+                "cannot be queried in-process, because answering a query would mean running a "
+                "model on the runtime path",
+            )
+        return cls(entries=entries, vectors=vectors, backend=backend)
+
+
+def _require_one_sealed_backend_identity(
+    entries: tuple[CatalogEntry, ...],
+    *,
+    path: str,
+) -> tuple[str, str, int, str]:
+    """The single ``(backend_id, backend_version, dimensions, quantization)`` the entries declare.
+
+    An index whose entries disagree about which backend sealed them has no single encoder that
+    could read all of it, so there is nothing to bind and the mixed set is refused here rather
+    than resolved arbitrarily.
+    """
+
+    identities = {
+        (
+            reference.backend_id,
+            reference.backend_version,
+            reference.dimensions,
+            reference.quantization,
+        )
+        for entry in entries
+        for reference in entry.vectors
+    }
+    if len(identities) != 1:
+        raise CatalogTamperError(
+            "CATALOG_MIXED_BACKEND",
+            path,
+            "every sealed vector must come from one backend, version, dimension count and "
+            f"quantization; found {len(identities)}",
+        )
+    return identities.pop()
+
+
+def _require_sealed_vectors_come_from_this_backend(
+    entries: tuple[CatalogEntry, ...],
+    descriptor: BackendDescriptor,
+    *,
+    path: str,
+) -> None:
+    """Refuse an index whose sealed vectors were not produced by the backend that will read them.
+
+    Four fields make up a sealed vector's identity. ``load`` resolves a backend *by* the declared
+    coordinate, so on
+    that route ``backend_id`` and ``backend_version`` agree by construction and only the dimension
+    count and the quantization needed checking. The public constructor takes the backend as an
+    argument, so on that route nothing agreed by construction and nothing was checked: an index
+    built from entries sealed by one query-safe backend, paired with a different query-safe backend
+    of identical dimensions and identical quantization, constructed in silence and then answered
+    the gate-three question with the wrong source. Every other check passed, because every other
+    check reads the artifact against itself -- the digests match, the lengths match, the layers are
+    all present. Only the descriptor of the encoder that will be asked the question is outside the
+    artifact.
+
+    The failure is the silent kind, which is why it is bound at all four fields rather than the two
+    ``load`` happened to need. Vectors from two different embedding schemes still multiply; a
+    mismatch is not an error at query time, it is a confidently wrong answer -- the same argument
+    that put quantization into the binding in the first place.
+
+    Nothing here computes an embedding. It is a comparison of five strings and one integer.
+    """
+
+    backend_id, backend_version, dimensions, quantization = _require_one_sealed_backend_identity(
+        entries, path=path
+    )
+    if (backend_id, backend_version) != (descriptor.backend_id, descriptor.backend_version):
+        raise CatalogTamperError(
+            "CATALOG_BACKEND_IDENTITY",
+            path,
+            f"sealed vectors were produced by {backend_id}@{backend_version} but the backend "
+            f"that would read them is {descriptor.coordinate}",
+        )
+    if descriptor.dimensions != dimensions:
+        raise CatalogTamperError(
+            "CATALOG_VECTOR_DIMENSIONS",
+            path,
+            f"sealed vectors declare {dimensions} dimensions but backend "
+            f"{backend_id}@{backend_version} produces {descriptor.dimensions}",
+        )
+    if descriptor.quantization != quantization:
+        raise CatalogTamperError(
+            "CATALOG_VECTOR_QUANTIZATION",
+            path,
+            f"sealed vectors declare quantization {quantization!r} but backend "
+            f"{backend_id}@{backend_version} produces {descriptor.quantization!r}",
+        )
+
+
+def _require_vectors_bound_to_the_entries_that_reference_them(
+    entries: tuple[CatalogEntry, ...],
+    vectors: Mapping[tuple[str, str], tuple[int, ...]],
+) -> None:
+    """Refuse an index whose vectors are not the vectors its entries reference.
+
+    This is the module's one silent failure, stated on the type rather than only on the load path.
+    Every other check reads the map alone -- keys well-shaped, four layers present, dimensions and
+    integrality right, nothing stray -- and an index assembled from one entry's entries and another
+    entry's vectors satisfies all of them. It then ranks each entry on the other one's text and
+    raises nothing, anywhere. Reproduced on the shipped eval catalog: swapping the four vectors of
+    ``faostat_qcl`` with those of ``sentinel_2_l2a`` constructed cleanly and answered the banana
+    question with ``sentinel_2_l2a``.
+
+    Two things are checked, because either one alone leaves the swap open. ``entry_facts_digest``
+    ties the reference to the facts the vector was computed from, and re-packing the map's vector
+    into its sealed 32-bit encoding ties the *bytes in this map* to the digest the entry carries --
+    without the second, ``entry_facts_digest`` binds a reference to an entry and says nothing about
+    which numbers ended up under that key.
+
+    Reference completeness is required here rather than only in ``load`` for the same reason the
+    map's completeness is: an entry carrying no references at all would be bound by nothing, and
+    ``_require_vectors_the_query_path_can_read`` would still hand ``_best_layer_score`` four
+    vectors to rank it on.
+
+    Nothing here computes an embedding. It is a digest comparison and a struct pack.
+    """
+
+    for entry in entries:
+        layers = tuple(reference.layer for reference in entry.vectors)
+        if sorted(layers) != sorted(EMBEDDING_LAYERS):
+            raise CatalogMissError(
+                "CATALOG_VECTOR_LAYERS",
+                f"{entry.coordinate}.vectors",
+                f"a sealed entry must reference one vector per layer {list(EMBEDDING_LAYERS)}",
+            )
+        facts_digest = entry.facts_digest
+        for reference in entry.vectors:
+            path = f"{entry.coordinate}.{reference.layer}"
+            if reference.entry_facts_digest != facts_digest:
+                raise CatalogTamperError(
+                    "CATALOG_VECTOR_ENTRY_BINDING",
+                    path,
+                    "the sealed vector was computed from different entry facts than the entry "
+                    "that references it",
+                )
+            vector = vectors[(entry.entry_id, reference.layer)]
+            try:
+                sealed = struct.pack(f">{len(vector)}i", *vector)
+            except struct.error:
+                raise CatalogTamperError(
+                    "CATALOG_VECTOR_ENCODING",
+                    path,
+                    "a sealed vector holds a value outside the signed 32-bit encoding sealed "
+                    "vectors are written in, so it cannot be the vector this entry references",
+                ) from None
+            if sha256_bytes(sealed) != reference.vector_sha256:
+                raise CatalogTamperError(
+                    "CATALOG_VECTOR_DIGEST",
+                    path,
+                    "the vector under this key is not the sealed vector this entry references",
+                )
+
+
+def _require_vectors_the_query_path_can_read(
+    vectors: Mapping[tuple[str, str], tuple[int, ...]],
+    *,
+    known_entry_ids: frozenset[str],
+    dimensions: int,
+) -> None:
+    """Refuse a vector map ``retrieval`` could not multiply against a question.
+
+    ``_best_layer_score`` does ``zip(asked, sealed, strict=True)`` and multiplies. Two things break
+    it and neither is caught by anything else once a caller builds an index directly. A wrong
+    dimension count raises a bare ``ValueError`` at query time, outside this module's three-way
+    taxonomy and far from the artifact that caused it. A non-integer element is worse: it does not
+    raise at all, it just ranks on a different scale -- the same silent-wrong failure the
+    quantization binding in :meth:`load` exists to stop.
+
+    ``load`` cannot produce either, because ``struct.unpack`` returns integers of the length it was
+    given and the descriptor binding checks that length. The constructor is public, so the property
+    belongs on the type as well.
+
+    A key naming an entry the index does not carry is refused too. It is dead weight rather than a
+    hazard -- ``_best_layer_score`` looks up by entry id and would never read it -- but a vector for
+    an entry that is not here means the map and the entries came from different places, and that is
+    worth failing on rather than ignoring.
+
+    **Completeness is checked as well as shape**, because checking only what is present validates
+    the wrong half. ``_best_layer_score`` skips a layer it cannot find (``if sealed is None:
+    continue``), so an index missing three of an entry's four layers answers every question without
+    error and ranks on a quarter of the signal. Both ``load`` and the public constructor therefore
+    require each entry to carry one reference per layer.
+    The map must be exactly the cross product of the entries and the four layers.
+    """
+
+    known_layers = frozenset(EMBEDDING_LAYERS)
+    required = {(entry_id, layer) for entry_id in known_entry_ids for layer in EMBEDDING_LAYERS}
+    for key, vector in vectors.items():
+        if not isinstance(key, tuple) or len(key) != 2 or not all(isinstance(p, str) for p in key):
+            raise SourceContractError(
+                "TYPE",
+                "index.vectors",
+                f"every key must be an (entry_id, layer) pair of strings; found {key!r}",
+            )
+        entry_id, layer = key
+        path = f"index.vectors[{entry_id!r}, {layer!r}]"
+        if layer not in known_layers:
+            raise SourceContractError(
+                "CATALOG_VECTOR_LAYERS",
+                path,
+                f"layer must be one of {list(EMBEDDING_LAYERS)}",
+            )
+        if entry_id not in known_entry_ids:
+            raise SourceContractError(
+                "CATALOG_VECTOR_ENTRY_BINDING",
+                path,
+                "a sealed vector names an entry this index does not carry",
+            )
+        if not isinstance(vector, tuple) or any(type(value) is not int for value in vector):
+            raise SourceContractError(
+                "TYPE",
+                path,
+                "a sealed vector must be an immutable tuple of integers",
+            )
+        if len(vector) != dimensions:
+            raise CatalogTamperError(
+                "CATALOG_VECTOR_DIMENSIONS",
+                path,
+                f"sealed vector has {len(vector)} dimensions but the backend reading it produces "
+                f"{dimensions}",
+            )
+    missing = sorted(required - set(vectors))
+    if missing:
+        raise CatalogMissError(
+            "CATALOG_VECTOR_LAYERS",
+            "index.vectors",
+            f"every entry must carry one vector per layer {list(EMBEDDING_LAYERS)}; "
+            f"{len(missing)} are absent, the first being {missing[0]}",
+        )
+
+
+def _read_vector(directory: Path, *, digest: str, dimensions: int, path: str) -> tuple[int, ...]:
+    # Validate the digest before it is used to build a path. The filename is derived from a
+    # checked 64-character hex string and never from anything an artifact supplies verbatim.
+    if not isinstance(digest, str) or not _SHA256.fullmatch(digest):
+        raise CatalogTamperError(
+            "SHA256",
+            path,
+            "vector digest must be a lowercase 64-character SHA-256",
+        )
+    vector_path = directory / VECTORS_DIRNAME / f"{digest}{VECTOR_SUFFIX}"
+    try:
+        raw = vector_path.read_bytes()
+    except FileNotFoundError:
+        raise CatalogMissError(
+            "CATALOG_VECTOR_MISSING",
+            path,
+            f"sealed vector {digest} is absent",
+        ) from None
+    if len(raw) != dimensions * 4:
+        raise CatalogTamperError(
+            "CATALOG_VECTOR_LENGTH",
+            path,
+            f"sealed vector must be {dimensions * 4} bytes, found {len(raw)}",
+        )
+    if sha256_bytes(raw) != digest:
+        raise CatalogTamperError(
+            "CATALOG_VECTOR_DIGEST",
+            path,
+            "sealed vector bytes do not match the digest the entry carries",
+        )
+    return struct.unpack(f">{dimensions}i", raw)