mostlyright-data 0.9.0__py3-none-any.whl

mostlyright/data_harness/sources/catalog/recommend.py

@@ -0,0 +1,171 @@
+"""The agent-facing call: a question in, ranked and gated source recommendations out.
+
+This is the composition root, and the only module in the package that names both retrieval and
+gating. What crosses the boundary is ``tuple[str, ...]``. No number reaches this module from
+``rank_entry_ids``, and none
+could be handed onward to ``gate_entries`` even if one did, because neither ``CatalogEntry`` nor
+``CatalogRequirement`` has a field able to hold one. A composition root reaching both sides is
+what a composition root is for; ``gating`` cannot reach ``retrieval``.
+
+The order is a trust boundary and must not be rearranged: retrieval narrows the field, the facts
+gate decides. The candidate list is never filtered by anything derived from retrieval except
+``limit``, which truncates.
+
+``to_dict`` is canonical-JSON-safe on purpose. One set of facts should be able to drive both a
+sentence a person reads and a machine-readable payload, without a second code path inventing a
+slightly different answer.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+from typing import Any
+
+from mostlyright.data_harness.canonical import canonical_sha256
+from mostlyright.data_harness.sources.catalog.gating import (
+    CATALOG_DISPOSITIONS,
+    CatalogGateVerdict,
+    CatalogRequirement,
+    gate_entries,
+)
+from mostlyright.data_harness.sources.catalog.retrieval import rank_entry_ids
+from mostlyright.data_harness.sources.catalog.sealed import SealedEmbeddingIndex
+from mostlyright.data_harness.sources.contracts import SourceContractError, _CanonicalContract
+
+RECOMMENDATION_CONTRACT_VERSION = "harness-source-recommendation.v1"
+
+# The shape the refusal below names. Checking the length alone made the message a description of
+# a check that was not being run, and every other digest field in this package is checked here.
+_SHA256 = re.compile(r"^[0-9a-f]{64}$")
+
+# Which field on this contract answers for which verdict disposition. Written once and read by
+# both the check below and `recommend_sources`, so the two can never sort into different buckets.
+# `::test_every_gate_disposition_has_a_field_that_answers_for_it` fails if `gating` grows a fourth
+# disposition this table does not name.
+RECOMMENDATION_BUCKETS = (
+    ("admitted", "admitted"),
+    ("escalated", "human_escalation_required"),
+    ("refused", "refused"),
+)
+
+
+@dataclass(frozen=True)
+class SourceRecommendation(_CanonicalContract):
+    """What the catalog can say about one question, with every decision cited.
+
+    ``question_digest`` binds the answer to the exact question without putting a stranger's free
+    text into a digest chain. ``requirement_digest`` binds it to what was asked *for*, which is the
+    other half of the input and the only thing that distinguishes two answers about the same
+    question. It is carried here as well as on each verdict because a question that retrieves
+    nothing is a legitimate answer with no verdicts in it, and that answer would otherwise cite no
+    requirement at all.
+    """
+
+    question_digest: str
+    requirement_digest: str
+    candidates: tuple[CatalogGateVerdict, ...]
+    admitted: tuple[str, ...]
+    escalated: tuple[str, ...]
+    refused: tuple[str, ...]
+    schema_version: str = RECOMMENDATION_CONTRACT_VERSION
+
+    def __post_init__(self) -> None:
+        if self.schema_version != RECOMMENDATION_CONTRACT_VERSION:
+            raise SourceContractError(
+                "VERSION",
+                "recommendation.schema_version",
+                f"must equal {RECOMMENDATION_CONTRACT_VERSION!r}",
+            )
+        for name in ("question_digest", "requirement_digest"):
+            value = getattr(self, name)
+            if not isinstance(value, str) or not _SHA256.fullmatch(value):
+                raise SourceContractError(
+                    "SHA256",
+                    f"recommendation.{name}",
+                    "must be a lowercase 64-character SHA-256",
+                )
+        for name in ("candidates", "admitted", "escalated", "refused"):
+            if not isinstance(getattr(self, name), tuple):
+                raise SourceContractError(
+                    "TYPE",
+                    f"recommendation.{name}",
+                    "must be an immutable tuple",
+                )
+        for position, verdict in enumerate(self.candidates):
+            if not isinstance(verdict, CatalogGateVerdict):
+                raise SourceContractError(
+                    "TYPE",
+                    f"recommendation.candidates[{position}]",
+                    "must be a CatalogGateVerdict",
+                )
+            # One answer, one requirement. A candidate decided against a different requirement
+            # from the one this answer cites would make the citation a decoration.
+            if verdict.requirement_digest != self.requirement_digest:
+                raise SourceContractError(
+                    "REQUIREMENT_BINDING",
+                    f"recommendation.candidates[{position}].requirement_digest",
+                    "a candidate was decided against a different requirement from the one this "
+                    "recommendation cites",
+                )
+        # The three lists must be the verdicts *sorted into their own dispositions*, not merely a
+        # partition of the same coordinates. Checking only that the union matches let a
+        # recommendation name a source under `refused` whose verdict said `admitted` -- two
+        # conflicting answers about one source in the same payload, with nothing raised.
+        expected: dict[str, list[str]] = {name: [] for name in CATALOG_DISPOSITIONS}
+        for verdict in self.candidates:
+            expected[verdict.disposition].append(verdict.entry_coordinate)
+        for name, disposition in RECOMMENDATION_BUCKETS:
+            if sorted(getattr(self, name)) != sorted(expected[disposition]):
+                raise SourceContractError(
+                    "RECOMMENDATION_PARTITION",
+                    f"recommendation.{name}",
+                    f"must name exactly the candidates whose verdict is {disposition!r}, each "
+                    "once; the three lists are the candidates sorted by their own disposition, "
+                    "not an independently supplied partition of them",
+                )
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": self.schema_version,
+            "question_digest": self.question_digest,
+            "requirement_digest": self.requirement_digest,
+            "candidates": [item.to_dict() for item in self.candidates],
+            "admitted": list(self.admitted),
+            "escalated": list(self.escalated),
+            "refused": list(self.refused),
+        }
+
+
+def recommend_sources(
+    *,
+    index: SealedEmbeddingIndex,
+    question: str,
+    requirement: CatalogRequirement,
+    limit: int,
+) -> SourceRecommendation:
+    """Retrieve candidates for this question, then decide each one on facts.
+
+    A question that retrieves nothing yields an empty recommendation rather than an error: "the
+    catalog holds nothing like this" is an answer, and an exception would make a caller unable to
+    tell it from a failure.
+    """
+
+    if not isinstance(index, SealedEmbeddingIndex):
+        raise SourceContractError("TYPE", "index", "must be a SealedEmbeddingIndex")
+    entry_ids = rank_entry_ids(index, question, limit=limit)
+    by_id = {entry.entry_id: entry for entry in index.entries}
+    entries = tuple(by_id[entry_id] for entry_id in entry_ids if entry_id in by_id)
+    verdicts = gate_entries(entries, requirement=requirement)
+    # Sorted through the same table the contract checks against, so the builder and the check
+    # cannot drift into disagreeing about which field answers for which disposition.
+    sorted_into = {
+        field: tuple(item.entry_coordinate for item in verdicts if item.disposition == disposition)
+        for field, disposition in RECOMMENDATION_BUCKETS
+    }
+    return SourceRecommendation(
+        question_digest=canonical_sha256({"question": question}),
+        requirement_digest=requirement.digest,
+        candidates=verdicts,
+        **sorted_into,
+    )

mostlyright/data_harness/sources/catalog/retrieval.py

@@ -0,0 +1,230 @@
+"""Retrieval: find candidate sources. Decide nothing.
+
+This module ranks catalog entries against a question and returns their identifiers. It returns a
+``tuple[str, ...]`` and nothing else, ever. The numbers it computes to produce that order are
+module-private and die at the return statement, so no caller can consult one and no future edit
+can accidentally route one into a decision.
+
+That is a deliberate design choice over the obvious alternative. A ``RankedCandidate`` dataclass
+carrying an unread score field would behave identically today and would quietly break the trust
+boundary: the score would be *available* to the gate, and the separation would rest on
+nobody ever reading it. Making the number impossible to pass is cheaper than policing who reads it.
+
+There is no threshold parameter here, and there must never be one. A threshold that admits or
+rejects is an admission decision made by similarity, which is exactly what the facts gate exists
+to prevent. ``limit`` truncates a list; it cannot admit anything.
+"""
+
+from __future__ import annotations
+
+from mostlyright.data_harness.sources.catalog.contracts import EMBEDDING_LAYERS
+from mostlyright.data_harness.sources.catalog.retrieval_manifest import (
+    LoadedRetrievalCatalog,
+    LoadedRetrievalRepresentation,
+)
+from mostlyright.data_harness.sources.catalog.sealed import SealedEmbeddingIndex
+from mostlyright.data_harness.sources.contracts import SourceContractError
+
+MAX_QUERY_CHARS = 4_096
+MAX_RETRIEVAL_LIMIT = 100
+RETRIEVAL_BACKENDS = frozenset({"lexical", "neural"})
+
+
+def rank_entry_ids(index: SealedEmbeddingIndex, question: str, *, limit: int) -> tuple[str, ...]:
+    """Return at most ``limit`` catalog entry ids, best match first.
+
+    Ties break on ``entry_id`` ascending, so the order is total and reproducible -- an eval that
+    depends on a coin flip is an eval that flakes. A question with no lexical signal returns an
+    empty tuple rather than an arbitrary alphabetical list, so "no signal" stays visibly different
+    from "weak signal".
+    """
+
+    if not isinstance(index, SealedEmbeddingIndex):
+        raise SourceContractError("TYPE", "index", "must be a SealedEmbeddingIndex")
+    if type(limit) is not int or not 1 <= limit <= MAX_RETRIEVAL_LIMIT:
+        raise SourceContractError(
+            "RETRIEVAL_LIMIT",
+            "limit",
+            f"must be an integer in [1, {MAX_RETRIEVAL_LIMIT}]",
+        )
+    if not isinstance(question, str):
+        raise SourceContractError("TYPE", "question", "must be a string")
+    if len(question) > MAX_QUERY_CHARS:
+        raise SourceContractError(
+            "RETRIEVAL_QUERY_LIMIT",
+            "question",
+            f"a question may not exceed {MAX_QUERY_CHARS} characters",
+        )
+
+    # `SealedEmbeddingIndex` cannot hold a backend that is unsafe to run in-process: it refuses one
+    # at construction, not only on the `load` path. That is why the isinstance check above is
+    # sufficient here and this module needs no policy of its own -- the precondition is a property
+    # of the argument's type rather than a claim about where the caller got it from.
+    asked = index.backend.encode(question)
+    if not any(asked):
+        return ()
+
+    ordered: list[tuple[int, str]] = []
+    for entry in index.entries:
+        best = _best_layer_score(index, entry.entry_id, asked)
+        if best <= 0:
+            continue
+        ordered.append((-best, entry.entry_id))
+    ordered.sort()
+    return tuple(entry_id for _negated, entry_id in ordered[:limit])
+
+
+def rank_retrieval_entry_ids(
+    catalog: LoadedRetrievalCatalog,
+    question: str,
+    *,
+    backend: str,
+    allow_lexical_fallback: bool,
+    limit: int,
+) -> tuple[str, ...]:
+    """Order verified v2 entry identities with one explicit representation choice.
+
+    Selecting ``neural`` never discovers a model. The caller must already have supplied an exact
+    backend to ``load_retrieval_catalog``. When it did not, fallback occurs only under the explicit
+    boolean; the v2 trace carries the loader's typed reason. Numerical ranking data stays here.
+    """
+
+    return _rank_retrieval(
+        catalog,
+        question,
+        backend=backend,
+        allow_lexical_fallback=allow_lexical_fallback,
+        limit=limit,
+    )[0]
+
+
+def _rank_retrieval(
+    catalog: LoadedRetrievalCatalog,
+    question: str,
+    *,
+    backend: str,
+    allow_lexical_fallback: bool,
+    limit: int,
+) -> tuple[tuple[str, ...], str, str, str | None]:
+    if not isinstance(catalog, LoadedRetrievalCatalog):
+        raise SourceContractError(
+            "TYPE", "catalog", "must be a fully verified LoadedRetrievalCatalog"
+        )
+    if not isinstance(backend, str) or backend not in RETRIEVAL_BACKENDS:
+        raise SourceContractError(
+            "RETRIEVAL_BACKEND", "backend", "must be either lexical or neural"
+        )
+    if type(allow_lexical_fallback) is not bool:
+        raise SourceContractError("TYPE", "allow_lexical_fallback", "must be a boolean")
+    _validate_request(question, limit)
+
+    selected = catalog.lexical
+    representation = "lexical"
+    fallback_reason: str | None = None
+    if backend == "neural":
+        if catalog.neural is not None:
+            selected = catalog.neural
+            representation = "neural"
+        elif not allow_lexical_fallback:
+            raise SourceContractError(
+                "NEURAL_RETRIEVAL_UNAVAILABLE",
+                "backend",
+                catalog.fallback_reason or "the verified catalog has no neural representation",
+            )
+        else:
+            fallback_reason = catalog.fallback_reason or "NEURAL_RUNTIME_UNAVAILABLE"
+
+    entry_ids = _rank_loaded_entry_ids(selected, question, limit=limit)
+    return (
+        entry_ids,
+        representation,
+        selected.backend.descriptor.coordinate,
+        fallback_reason,
+    )
+
+
+def _validate_request(question: str, limit: int) -> None:
+    if type(limit) is not int or not 1 <= limit <= MAX_RETRIEVAL_LIMIT:
+        raise SourceContractError(
+            "RETRIEVAL_LIMIT",
+            "limit",
+            f"must be an integer in [1, {MAX_RETRIEVAL_LIMIT}]",
+        )
+    if not isinstance(question, str):
+        raise SourceContractError("TYPE", "question", "must be a string")
+    if len(question) > MAX_QUERY_CHARS:
+        raise SourceContractError(
+            "RETRIEVAL_QUERY_LIMIT",
+            "question",
+            f"a question may not exceed {MAX_QUERY_CHARS} characters",
+        )
+
+
+def _rank_loaded_entry_ids(
+    representation: LoadedRetrievalRepresentation,
+    question: str,
+    *,
+    limit: int,
+) -> tuple[str, ...]:
+    if not any(character.isalnum() for character in question):
+        return ()
+    asked = representation.backend.encode(question)
+    dimensions = representation.backend.descriptor.dimensions
+    if (
+        not isinstance(asked, tuple)
+        or len(asked) != dimensions
+        or any(type(value) is not int for value in asked)
+    ):
+        raise SourceContractError(
+            "RETRIEVAL_QUERY_VECTOR",
+            "question",
+            "the selected backend returned a vector outside its exact integer coordinate",
+        )
+    if not any(asked):
+        return ()
+
+    ordered: list[tuple[int, str]] = []
+    for entry in representation.entries:
+        best = _best_loaded_layer_score(representation, entry.entry_id, asked)
+        if best > 0:
+            ordered.append((-best, entry.entry_id))
+    ordered.sort()
+    return tuple(entry_id for _negated, entry_id in ordered[:limit])
+
+
+def _best_layer_score(index: SealedEmbeddingIndex, entry_id: str, asked: tuple[int, ...]) -> int:
+    """The strongest of this entry's four layers against the question. Private on purpose.
+
+    An entry that matches only on its published column names still ranks by that layer, which is
+    what makes the four-layer split worth having.
+    """
+
+    best = 0
+    for layer in EMBEDDING_LAYERS:
+        sealed = index.vectors.get((entry_id, layer))
+        if sealed is None:
+            continue
+        total = 0
+        for left, right in zip(asked, sealed, strict=True):
+            total += left * right
+        best = max(best, total)
+    return best
+
+
+def _best_loaded_layer_score(
+    representation: LoadedRetrievalRepresentation,
+    entry_id: str,
+    asked: tuple[int, ...],
+) -> int:
+    """Return one private v2 ranking number; callers can receive only identities."""
+
+    best = 0
+    for layer in EMBEDDING_LAYERS:
+        sealed = representation.vectors.get((entry_id, layer))
+        if sealed is None:
+            continue
+        total = 0
+        for left, right in zip(asked, sealed, strict=True):
+            total += left * right
+        best = max(best, total)
+    return best