sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/contracts.py

@@ -0,0 +1,514 @@
+"""Versioned frozen-pool reranking contracts.
+
+This module separates a candidate's observable input from reference labels.
+Only :meth:`CandidateRequest.to_dict` may be serialized to a candidate process.
+It builds a fresh allowlisted mapping; it does not serialize a case or reference
+dataclass recursively.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import re
+from dataclasses import dataclass
+from typing import Any, Mapping, Sequence
+
+INTERFACE_VERSION = "sphynx.rag.rerank.v1"
+POOL_SCHEMA_VERSION = "sphynx.rag.rerank.pool.v1"
+REQUEST_SCHEMA_VERSION = "sphynx.rag.rerank.request.v1"
+RESPONSE_SCHEMA_VERSION = "sphynx.rag.rerank.response.v1"
+CASE_SCHEMA_VERSION = "rag-evals.dataset.v2"
+PACK_ID = "rag.rerank.v1"
+SABLE_RERANK_DEV_PACK_ID = "rag.rerank.sable.dev.v1"
+PUBLIC_RERANK_PACK_ID = "rag.rerank.beir.v1"
+
+_ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:-]*$")
+_DATASET_SPLITS = frozenset({"dev", "test", "holdout"})
+_SUITE_TYPES = frozenset(
+    {"core", "customer_acceptance", "production_failure", "adversarial_robustness", "untouched_holdout"}
+)
+
+
+class ContractError(ValueError):
+    """A versioned SDK contract is malformed or does not match its request."""
+
+
+def _id(value: object, field: str) -> str:
+    if not isinstance(value, str) or not _ID.fullmatch(value):
+        raise ContractError(f"{field} must be a nonempty stable identifier")
+    return value
+
+
+def _text(value: object, field: str) -> str:
+    if not isinstance(value, str) or not value.strip() or "\x00" in value:
+        raise ContractError(f"{field} must be nonempty text without NUL bytes")
+    return value
+
+
+def _finite_number(value: object, field: str) -> float:
+    if type(value) not in (int, float):
+        raise ContractError(f"{field} must be a finite JSON number")
+    try:
+        number = float(value)
+    except OverflowError as exc:
+        raise ContractError(f"{field} must be a finite JSON number") from exc
+    if not math.isfinite(number):
+        raise ContractError(f"{field} must be a finite JSON number")
+    return number
+
+
+def _tuple_of_ids(values: object, field: str, *, empty: bool = True) -> tuple[str, ...]:
+    if isinstance(values, (str, bytes)) or not isinstance(values, Sequence):
+        raise ContractError(f"{field} must be a sequence of identifiers")
+    result = tuple(_id(value, f"{field}[{index}]") for index, value in enumerate(values))
+    if not empty and not result:
+        raise ContractError(f"{field} must not be empty")
+    if len(set(result)) != len(result):
+        raise ContractError(f"{field} must contain unique identifiers")
+    return result
+
+
+def canonical_json(value: Any) -> str:
+    """Canonical UTF-8 JSON form for v1 digests and wire messages.
+
+    Object keys are lexical, arrays retain order, whitespace is omitted, Unicode
+    is not ASCII-escaped, and NaN/Infinity are rejected. The caller supplies only
+    JSON data (not dataclasses). The digest hashes the UTF-8 encoded result.
+    """
+
+    try:
+        return json.dumps(value, sort_keys=True, separators=(",", ":"), ensure_ascii=False, allow_nan=False)
+    except (TypeError, ValueError) as exc:
+        raise ContractError(f"value is not canonical JSON: {exc}") from exc
+
+
+@dataclass(frozen=True)
+class CandidateSpec:
+    """A versioned candidate command, invoked by argv without a shell."""
+
+    name: str
+    version: str
+    argv: tuple[str, ...]
+    interface_version: str
+
+    def __post_init__(self) -> None:
+        _id(self.name, "name")
+        _id(self.version, "version")
+        _id(self.interface_version, "interface_version")
+        if isinstance(self.argv, (str, bytes)) or not isinstance(self.argv, Sequence):
+            raise ContractError("argv must be a nonempty sequence of command tokens")
+        argv = tuple(self.argv)
+        if not argv:
+            raise ContractError("argv must not be empty")
+        for index, token in enumerate(argv):
+            _text(token, f"argv[{index}]")
+        object.__setattr__(self, "argv", argv)
+
+
+@dataclass(frozen=True)
+class CorpusVersions:
+    corpus: str
+    index: str
+    chunking: str
+    embedding: str
+
+    def __post_init__(self) -> None:
+        for field in ("corpus", "index", "chunking", "embedding"):
+            _id(getattr(self, field), f"versions.{field}")
+
+    def to_dict(self) -> dict[str, str]:
+        return {
+            "corpus": self.corpus,
+            "index": self.index,
+            "chunking": self.chunking,
+            "embedding": self.embedding,
+        }
+
+
+@dataclass(frozen=True)
+class FrozenCandidate:
+    """An ordered passage in an immutable pool, without grading labels."""
+
+    candidate_id: str
+    document_id: str
+    source_id: str
+    chunk_id: str
+    url: str
+    input: str
+    dense_rank: int
+    dense_score: float
+
+    def __post_init__(self) -> None:
+        for field in ("candidate_id", "document_id", "source_id", "chunk_id"):
+            _id(getattr(self, field), field)
+        _text(self.url, "url")
+        _text(self.input, "input")
+        if type(self.dense_rank) is not int or self.dense_rank < 1:
+            raise ContractError("dense_rank must be a positive integer")
+        _finite_number(self.dense_score, "dense_score")
+
+    def _digest_dict(self) -> dict[str, str | int | float]:
+        return {
+            "candidate_id": self.candidate_id,
+            "document_id": self.document_id,
+            "source_id": self.source_id,
+            "chunk_id": self.chunk_id,
+            "url": self.url,
+            "input": self.input,
+            "dense_rank": self.dense_rank,
+            "dense_score": self.dense_score,
+        }
+
+
+@dataclass(frozen=True)
+class FrozenPoolCase:
+    """The label-free identity and cohort of one frozen-pool test case.
+
+    Cohort fields stay with the case and its digest, not the candidate request.
+    """
+
+    dataset_id: str
+    dataset_version: str
+    case_id: str
+    query_id: str
+    query: str
+    split: str
+    split_group_id: str
+    cluster_id: str
+    suite_type: str
+    versions: CorpusVersions
+    candidates: tuple[FrozenCandidate, ...]
+
+    def __post_init__(self) -> None:
+        for field in ("dataset_id", "dataset_version", "case_id", "query_id"):
+            _id(getattr(self, field), field)
+        _text(self.query, "query")
+        _id(self.split_group_id, "split_group_id")
+        _id(self.cluster_id, "cluster_id")
+        if not isinstance(self.split, str) or self.split not in _DATASET_SPLITS:
+            raise ContractError("split must be dev, test, or holdout")
+        if not isinstance(self.suite_type, str) or self.suite_type not in _SUITE_TYPES:
+            raise ContractError("suite_type must be a supported dataset-v2 suite type")
+        if (self.split == "holdout") != (self.suite_type == "untouched_holdout"):
+            raise ContractError("holdout split and untouched_holdout suite_type must be used together")
+        if not isinstance(self.versions, CorpusVersions):
+            raise ContractError("versions must be CorpusVersions")
+        if isinstance(self.candidates, (str, bytes)) or not isinstance(self.candidates, Sequence):
+            raise ContractError("candidates must be an ordered sequence")
+        candidates = tuple(self.candidates)
+        if not candidates:
+            raise ContractError("candidates must not be empty")
+        if any(not isinstance(candidate, FrozenCandidate) for candidate in candidates):
+            raise ContractError("candidates must contain FrozenCandidate values")
+        ids = [candidate.candidate_id for candidate in candidates]
+        if len(set(ids)) != len(ids):
+            raise ContractError("candidate IDs must be unique within a pool")
+        for index, candidate in enumerate(candidates, start=1):
+            if candidate.dense_rank != index:
+                raise ContractError(f"candidate {candidate.candidate_id} dense_rank must equal pool position {index}")
+        object.__setattr__(self, "candidates", candidates)
+
+    @property
+    def pool_digest(self) -> str:
+        return pool_digest(self)
+
+
+@dataclass(frozen=True)
+class ReferenceCandidate:
+    """Reference judgment. Never pass this type to a candidate request."""
+
+    candidate_id: str
+    relevance_grade: int
+    evidence_group_ids: tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        _id(self.candidate_id, "candidate_id")
+        if type(self.relevance_grade) is not int or not 0 <= self.relevance_grade <= 3:
+            raise ContractError("relevance_grade must be an integer from 0 through 3")
+        object.__setattr__(self, "evidence_group_ids", _tuple_of_ids(self.evidence_group_ids, "evidence_group_ids"))
+
+
+@dataclass(frozen=True)
+class EvidenceGroup:
+    evidence_group_id: str
+    description: str
+    required: bool
+    acceptable_candidate_ids: tuple[str, ...]
+
+    def __post_init__(self) -> None:
+        _id(self.evidence_group_id, "evidence_group_id")
+        _text(self.description, "description")
+        if type(self.required) is not bool:
+            raise ContractError("required must be a boolean")
+        object.__setattr__(
+            self,
+            "acceptable_candidate_ids",
+            _tuple_of_ids(self.acceptable_candidate_ids, "acceptable_candidate_ids"),
+        )
+
+
+@dataclass(frozen=True)
+class CaseReferences:
+    """Protected-by-convention labels; local SDK use cannot make them secret."""
+
+    case_id: str
+    label_authority: str
+    candidates: tuple[ReferenceCandidate, ...]
+    evidence_groups: tuple[EvidenceGroup, ...]
+
+    def __post_init__(self) -> None:
+        _id(self.case_id, "case_id")
+        if self.label_authority not in ("synthetic_authored", "human_adjudicated", "provisional"):
+            raise ContractError("label_authority must be synthetic_authored, human_adjudicated, or provisional")
+        candidates = tuple(self.candidates)
+        groups = tuple(self.evidence_groups)
+        if any(not isinstance(item, ReferenceCandidate) for item in candidates):
+            raise ContractError("reference candidates must be ReferenceCandidate values")
+        if any(not isinstance(item, EvidenceGroup) for item in groups):
+            raise ContractError("evidence_groups must contain EvidenceGroup values")
+        if len({item.candidate_id for item in candidates}) != len(candidates):
+            raise ContractError("reference candidate IDs must be unique")
+        if len({item.evidence_group_id for item in groups}) != len(groups):
+            raise ContractError("evidence-group IDs must be unique")
+        object.__setattr__(self, "candidates", candidates)
+        object.__setattr__(self, "evidence_groups", groups)
+
+
+def pool_digest(case: FrozenPoolCase) -> str:
+    """Hash ordered pool, cohort, source IDs/text, scores, query, and versions.
+
+    Protected reference labels and evidence-group links are intentionally absent.
+    The versioned preimage is canonical JSON encoded as UTF-8; array order matters.
+    """
+
+    if not isinstance(case, FrozenPoolCase):
+        raise ContractError("pool_digest requires FrozenPoolCase")
+    preimage = {
+        "schema_version": POOL_SCHEMA_VERSION,
+        "dataset_id": case.dataset_id,
+        "dataset_version": case.dataset_version,
+        "case_id": case.case_id,
+        "query_id": case.query_id,
+        "query": case.query,
+        "split": case.split,
+        "split_group_id": case.split_group_id,
+        "cluster_id": case.cluster_id,
+        "suite_type": case.suite_type,
+        "versions": case.versions.to_dict(),
+        "candidates": [candidate._digest_dict() for candidate in case.candidates],
+    }
+    return "sha256:" + hashlib.sha256(canonical_json(preimage).encode("utf-8")).hexdigest()
+
+
+@dataclass(frozen=True)
+class RequestCandidate:
+    candidate_id: str
+    input: str
+    dense_rank: int
+    dense_score: float
+
+    def to_dict(self) -> dict[str, str | int | float]:
+        return {
+            "candidate_id": self.candidate_id,
+            "input": self.input,
+            "dense_rank": self.dense_rank,
+            "dense_score": self.dense_score,
+        }
+
+
+@dataclass(frozen=True)
+class CandidateRequest:
+    """The only supported serialization boundary for a candidate process."""
+
+    pack_id: str
+    case_id: str
+    query_id: str
+    query: str
+    versions: CorpusVersions
+    pool_digest: str
+    candidates: tuple[RequestCandidate, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        # Keep the allowlist explicit. Never use dataclasses.asdict(case/references).
+        return {
+            "schema_version": REQUEST_SCHEMA_VERSION,
+            "pack_id": self.pack_id,
+            "case_id": self.case_id,
+            "query_id": self.query_id,
+            "query": self.query,
+            "versions": self.versions.to_dict(),
+            "pool_digest": self.pool_digest,
+            "candidates": [candidate.to_dict() for candidate in self.candidates],
+        }
+
+    def to_json(self) -> str:
+        return canonical_json(self.to_dict())
+
+
+
+def validate_candidate_request(value: object) -> CandidateRequest:
+    """Validate the exact label-free v1 wire shape, without exposing payloads.
+
+    The digest is an opaque identity supplied by the runner: the label-free
+    request omits cohort/source fields needed to recompute the frozen pool.
+    """
+
+    fields = {"schema_version", "pack_id", "case_id", "query_id", "query", "versions", "pool_digest", "candidates"}
+    if not isinstance(value, Mapping) or set(value) != fields:
+        raise ContractError(f"request fields must be exactly {sorted(fields)}")
+    if value["schema_version"] != REQUEST_SCHEMA_VERSION:
+        raise ContractError(f"request schema_version must be {REQUEST_SCHEMA_VERSION}")
+    if value["pack_id"] not in (PACK_ID, SABLE_RERANK_DEV_PACK_ID, PUBLIC_RERANK_PACK_ID):
+        raise ContractError("request pack_id must be a supported v1 reranking pack")
+    case_id = _id(value["case_id"], "request case_id")
+    query_id = _id(value["query_id"], "request query_id")
+    query = _text(value["query"], "request query")
+    digest = value["pool_digest"]
+    if not isinstance(digest, str) or not re.fullmatch(r"sha256:[0-9a-f]{64}", digest):
+        raise ContractError("request pool_digest must be a sha256 digest")
+    versions = value["versions"]
+    version_fields = {"corpus", "index", "chunking", "embedding"}
+    if not isinstance(versions, Mapping) or set(versions) != version_fields:
+        raise ContractError(f"request versions fields must be exactly {sorted(version_fields)}")
+    parsed_versions = CorpusVersions(**versions)
+    raw_candidates = value["candidates"]
+    if not isinstance(raw_candidates, list) or not raw_candidates:
+        raise ContractError("request candidates must be a nonempty JSON array")
+    candidates: list[RequestCandidate] = []
+    seen: set[str] = set()
+    for index, item in enumerate(raw_candidates):
+        prefix = f"request candidates[{index}]"
+        if not isinstance(item, Mapping) or set(item) != {"candidate_id", "input", "dense_rank", "dense_score"}:
+            raise ContractError(f"{prefix} must contain exactly candidate_id, input, dense_rank, dense_score")
+        candidate_id = _id(item["candidate_id"], f"{prefix}.candidate_id")
+        if candidate_id in seen:
+            raise ContractError("request candidate IDs must be unique")
+        seen.add(candidate_id)
+        passage = _text(item["input"], f"{prefix}.input")
+        rank = item["dense_rank"]
+        if type(rank) is not int or rank != index + 1:
+            raise ContractError(f"{prefix}.dense_rank must equal pool position {index + 1}")
+        number = item["dense_score"]
+        _finite_number(number, f"{prefix}.dense_score")
+        candidates.append(RequestCandidate(candidate_id, passage, rank, number))
+    return CandidateRequest(value["pack_id"], case_id, query_id, query, parsed_versions, digest, tuple(candidates))
+
+
+def parse_candidate_request(payload: str | bytes) -> CandidateRequest:
+    """Parse one strict UTF-8 JSON request; reject repeated keys and constants."""
+
+    def unique_fields(pairs: list[tuple[str, Any]]) -> dict[str, Any]:
+        result: dict[str, Any] = {}
+        for key, item in pairs:
+            if key in result:
+                raise ContractError("request JSON contains a duplicate field")
+            result[key] = item
+        return result
+
+    def invalid_constant(value: str) -> None:
+        raise ContractError("request JSON contains a non-finite number")
+
+    if not isinstance(payload, (str, bytes)):
+        raise ContractError("request payload must be UTF-8 JSON text or bytes")
+    try:
+        text = payload.decode("utf-8", errors="strict") if isinstance(payload, bytes) else payload
+        value = json.loads(text, object_pairs_hook=unique_fields, parse_constant=invalid_constant)
+    except ContractError:
+        raise
+    except (UnicodeError, ValueError, RecursionError) as exc:
+        raise ContractError("request must contain one valid UTF-8 JSON object") from exc
+    return validate_candidate_request(value)
+
+
+def build_candidate_request(case: FrozenPoolCase, pack_id: str = PACK_ID) -> CandidateRequest:
+    if not isinstance(case, FrozenPoolCase):
+        raise ContractError("build_candidate_request requires FrozenPoolCase")
+    if pack_id not in (PACK_ID, SABLE_RERANK_DEV_PACK_ID, PUBLIC_RERANK_PACK_ID):
+        raise ContractError(f"request builder does not support {pack_id!r}")
+    return CandidateRequest(
+        pack_id=pack_id,
+        case_id=case.case_id,
+        query_id=case.query_id,
+        query=case.query,
+        versions=case.versions,
+        pool_digest=case.pool_digest,
+        candidates=tuple(
+            RequestCandidate(item.candidate_id, item.input, item.dense_rank, item.dense_score)
+            for item in case.candidates
+        ),
+    )
+
+
+@dataclass(frozen=True)
+class CandidateScore:
+    candidate_id: str
+    ranking_score: float
+
+
+@dataclass(frozen=True)
+class CandidateResponse:
+    query_id: str
+    pool_digest: str
+    scores: tuple[CandidateScore, ...]
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schema_version": RESPONSE_SCHEMA_VERSION,
+            "query_id": self.query_id,
+            "pool_digest": self.pool_digest,
+            "scores": [
+                {"candidate_id": score.candidate_id, "ranking_score": score.ranking_score}
+                for score in self.scores
+            ],
+        }
+
+
+def validate_candidate_response(value: object, request: CandidateRequest) -> CandidateResponse:
+    """Validate exact response shape, full ID coverage, and request identity."""
+
+    if not isinstance(request, CandidateRequest):
+        raise ContractError("request must be CandidateRequest")
+    if not isinstance(value, Mapping):
+        raise ContractError("response must be a JSON object")
+    expected_fields = {"schema_version", "query_id", "pool_digest", "scores"}
+    if set(value) != expected_fields:
+        raise ContractError(f"response fields must be exactly {sorted(expected_fields)}")
+    if value["schema_version"] != RESPONSE_SCHEMA_VERSION:
+        raise ContractError(f"response schema_version must be {RESPONSE_SCHEMA_VERSION}")
+    if value["query_id"] != request.query_id:
+        raise ContractError("response query_id does not match request")
+    if value["pool_digest"] != request.pool_digest:
+        raise ContractError("response pool_digest does not match request")
+    raw_scores = value["scores"]
+    if not isinstance(raw_scores, list):
+        raise ContractError("response scores must be a JSON array")
+    scores: list[CandidateScore] = []
+    seen: set[str] = set()
+    for index, item in enumerate(raw_scores):
+        if not isinstance(item, Mapping) or set(item) != {"candidate_id", "ranking_score"}:
+            raise ContractError(f"scores[{index}] must contain exactly candidate_id and ranking_score")
+        candidate_id = _id(item["candidate_id"], f"scores[{index}].candidate_id")
+        if candidate_id in seen:
+            raise ContractError(f"duplicate score for candidate_id {candidate_id}")
+        seen.add(candidate_id)
+        scores.append(CandidateScore(candidate_id, _finite_number(item["ranking_score"], f"scores[{index}].ranking_score")))
+    expected_ids = {item.candidate_id for item in request.candidates}
+    if seen != expected_ids:
+        raise ContractError(f"response candidate IDs mismatch: missing={sorted(expected_ids - seen)}, extra={sorted(seen - expected_ids)}")
+    return CandidateResponse(query_id=request.query_id, pool_digest=request.pool_digest, scores=tuple(scores))
+
+
+def ranked_candidate_ids(response: CandidateResponse, request: CandidateRequest) -> tuple[str, ...]:
+    """Descending score; ties follow the original frozen-pool order."""
+
+    if response.query_id != request.query_id or response.pool_digest != request.pool_digest:
+        raise ContractError("response identity does not match this request")
+    positions = {item.candidate_id: index for index, item in enumerate(request.candidates)}
+    if len(response.scores) != len(positions) or {item.candidate_id for item in response.scores} != set(positions):
+        raise ContractError("response scores do not cover this request")
+    return tuple(
+        item.candidate_id
+        for item in sorted(response.scores, key=lambda item: (-item.ranking_score, positions[item.candidate_id]))
+    )