sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/benchmarks.py

@@ -0,0 +1,568 @@
+"""Immutable benchmark/label metadata contracts; no execution or data downloads.
+
+Eligibility validates declared metadata, not receipt bytes or human approval.
+The only default release is the existing public illustrative synthetic fixture.
+Legacy evaluation records and comparison digests are not changed by this layer.
+"""
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass, fields
+from datetime import datetime
+import hashlib
+import json
+from pathlib import Path
+import re
+from typing import Any
+
+from .evals import EvaluationContractError, canonical_contract_json
+
+LABEL_SCHEMA = "sphynx.label-set-release.v1"
+BENCHMARK_SCHEMA = "sphynx.benchmark-release.v1"
+CATALOG_SCHEMA = "sphynx.benchmark-catalog.v1"
+ELIGIBILITY_SCHEMA = "sphynx.benchmark-eligibility.v1"
+PROJECTION_VERSION = "sphynx.synthetic-rerank.metadata-projections.v1"
+DEFAULT_BENCHMARK = "synthetic.rerank.fixture@2.0.0"
+DEFAULT_LABELS = "synthetic.rerank.labels@2.0.0"
+LEGACY_BANK = "help-center-evidence-mini@2.0.0"
+_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}\Z")
+_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
+_AUTHORITY = {"synthetic_fixture", "native_public_qrels", "owner_reviewed", "human_adjudicated", "machine_provisional"}
+_PROFILES = {"routing_actions", "passage_qrels", "required_evidence", "passage_qrels_and_evidence_groups", "answer_references", "query_intent"}
+_METRICS = {
+    "rag.invoke.metric.v1": ("routing_actions", "sphynx.rag.invoke.v1"),
+    "rag.rerank.metric.v1": ("passage_qrels_and_evidence_groups", "sphynx.rag.rerank.v1"),
+    "rag.rerank.beir.metric.v1": ("passage_qrels", "sphynx.rag.rerank.v1"),
+    "rag.retrieve.metric.v1": ("required_evidence", "sphynx.rag.retrieve.v1"),
+}
+
+
+class BenchmarkContractError(ValueError):
+    """Stable payload-free diagnostic; metadata errors never expose data bytes."""
+    def __init__(self, code):
+        self.code = code
+        super().__init__(code)
+
+
+def _require(condition, code):
+    if not condition:
+        raise BenchmarkContractError(code)
+
+
+def _id(value):
+    _require(type(value) is str and _ID.fullmatch(value), "invalid_identifier")
+
+
+def _digest(value):
+    _require(type(value) is str and _DIGEST.fullmatch(value), "invalid_digest")
+
+
+def _choice(value, options):
+    _require(type(value) is str and value in options, "unsupported_option")
+
+
+def _count(value, maximum=2**53 - 1):
+    _require(type(value) is int and 0 <= value <= maximum, "invalid_count")
+
+
+def _tuples(values, kind, *, maximum=512):
+    _require(type(values) is tuple and len(values) <= maximum and all(isinstance(v, kind) for v in values), "invalid_collection")
+
+
+def _ids(values):
+    _tuples(values, str)
+    for value in values:
+        _id(value)
+    _require(len(set(values)) == len(values), "duplicate_identifier")
+
+
+def record_digest(value):
+    try:
+        return "sha256:" + hashlib.sha256(canonical_contract_json(value).encode()).hexdigest()
+    except EvaluationContractError:
+        raise BenchmarkContractError("invalid_canonical_metadata") from None
+
+
+class _Record:
+    def to_dict(self):
+        return json.loads(canonical_contract_json(asdict(self)))
+
+    @classmethod
+    def _values(cls, value):
+        _require(type(value) is dict and set(value) == {field.name for field in fields(cls)}, "invalid_record_shape")
+        return dict(value)
+
+
+@dataclass(frozen=True)
+class AssetRef(_Record):
+    ref: str
+    version: str
+    digest: str
+
+    def __post_init__(self):
+        _id(self.ref); _id(self.version); _digest(self.digest)
+
+    @classmethod
+    def from_dict(cls, value):
+        return cls(**cls._values(value))
+
+
+def _asset(value, *, optional=False):
+    _require(isinstance(value, AssetRef) or optional and value is None, "invalid_asset_reference")
+
+
+@dataclass(frozen=True)
+class ReviewReceipt(_Record):
+    receipt: AssetRef
+    reviewer_id: str
+    reviewed_at_utc: str
+    case_ids: tuple[str, ...]
+    kind: str
+    policy_revision: str
+    rubric_revision: str
+
+    def __post_init__(self):
+        _asset(self.receipt); _id(self.reviewer_id); _ids(self.case_ids)
+        _require(bool(self.case_ids), "empty_review_scope")
+        _id(self.policy_revision); _id(self.rubric_revision)
+        _choice(self.kind, {"review", "adjudication"})
+        try:
+            _require(type(self.reviewed_at_utc) is str and re.fullmatch(r"\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(?:\.\d{1,6})?Z", self.reviewed_at_utc), "invalid_review_timestamp")
+            datetime.fromisoformat(self.reviewed_at_utc.replace("Z", "+00:00"))
+        except ValueError:
+            raise BenchmarkContractError("invalid_review_timestamp") from None
+
+    @classmethod
+    def from_dict(cls, value):
+        value = cls._values(value)
+        _require(type(value["case_ids"]) is list, "invalid_collection")
+        value.update(receipt=AssetRef.from_dict(value["receipt"]), case_ids=tuple(value["case_ids"]))
+        return cls(**value)
+
+
+@dataclass(frozen=True)
+class LabelCase(_Record):
+    case_id: str
+    input_digest: str
+    status: str
+
+    def __post_init__(self):
+        _id(self.case_id); _digest(self.input_digest)
+        _choice(self.status, {"judged", "unjudged", "blocked"})
+
+    @classmethod
+    def from_dict(cls, value):
+        return cls(**cls._values(value))
+
+
+@dataclass(frozen=True)
+class LabelLineage(_Record):
+    parent_ref: str
+    parent_digest: str
+    relationship: str
+    reason: str
+    changed_case_count: int
+    evidence: AssetRef | None
+
+    def __post_init__(self):
+        _id(self.parent_ref); _digest(self.parent_digest)
+        _choice(self.relationship, {"revision", "subset"})
+        _choice(self.reason, {"labels_corrected", "policy_changed", "rubric_changed", "subset_selected"})
+        _count(self.changed_case_count); _asset(self.evidence, optional=True)
+        _require(self.changed_case_count == 0 or self.evidence is not None, "lineage_evidence_required")
+        _require(self.reason != "subset_selected" or self.changed_case_count == 0, "changed_labels_need_revision_reason")
+        _require(self.reason != "subset_selected" or self.relationship == "subset", "lineage_subset_reason_mismatch")
+
+    @classmethod
+    def from_dict(cls, value):
+        value = cls._values(value)
+        value["evidence"] = None if value["evidence"] is None else AssetRef.from_dict(value["evidence"])
+        return cls(**value)
+
+
+@dataclass(frozen=True)
+class LabelSetRelease(_Record):
+    schema_version: str
+    ref: str
+    version: str
+    input: AssetRef
+    corpus: AssetRef | None
+    reference: AssetRef
+    label_profile: str
+    policy_revision: str
+    rubric_revision: str
+    authority: str
+    missingness: str
+    cases: tuple[LabelCase, ...]
+    review_receipts: tuple[ReviewReceipt, ...]
+    lineage: tuple[LabelLineage, ...]
+
+    def __post_init__(self):
+        _require(self.schema_version == LABEL_SCHEMA, "unsupported_label_schema")
+        _id(self.ref); _id(self.version); _asset(self.input); _asset(self.corpus, optional=True); _asset(self.reference)
+        _choice(self.label_profile, _PROFILES); _id(self.policy_revision); _id(self.rubric_revision)
+        _choice(self.authority, _AUTHORITY)
+        _choice(self.missingness, {"complete_required", "native_qrels_unjudged", "development_assume_zero"})
+        _tuples(self.cases, LabelCase); _tuples(self.review_receipts, ReviewReceipt); _tuples(self.lineage, LabelLineage, maximum=16)
+        ids = tuple(case.case_id for case in self.cases)
+        _ids(ids); _require(bool(ids), "empty_label_release")
+        _require(len({r.receipt.ref for r in self.review_receipts}) == len(self.review_receipts), "duplicate_review_receipt")
+        _require(len({parent.parent_ref for parent in self.lineage}) == len(self.lineage), "duplicate_lineage_parent")
+        _require(self.authority != "native_public_qrels" or self.label_profile == "passage_qrels", "authority_profile_mismatch")
+        _require(self.missingness != "native_qrels_unjudged" or self.authority == "native_public_qrels", "missingness_authority_mismatch")
+        _require(self.missingness != "development_assume_zero" or self.authority == "machine_provisional", "missingness_authority_mismatch")
+        _require(self.authority in {"owner_reviewed", "human_adjudicated"} or not self.review_receipts, "unexpected_human_receipts")
+        for receipt in self.review_receipts:
+            _require(set(receipt.case_ids) <= set(ids), "review_scope_mismatch")
+            _require(receipt.policy_revision == self.policy_revision and receipt.rubric_revision == self.rubric_revision, "review_policy_mismatch")
+        self.digest
+
+    @property
+    def digest(self):
+        return record_digest(self.to_dict())
+
+    @property
+    def asset_ref(self):
+        return AssetRef(self.ref, self.version, self.digest)
+
+    @classmethod
+    def from_dict(cls, value):
+        value = cls._values(value)
+        for key in ("input", "reference"):
+            value[key] = AssetRef.from_dict(value[key])
+        value["corpus"] = None if value["corpus"] is None else AssetRef.from_dict(value["corpus"])
+        for key, kind in (("cases", LabelCase), ("review_receipts", ReviewReceipt), ("lineage", LabelLineage)):
+            _require(type(value[key]) is list, "invalid_collection")
+            value[key] = tuple(kind.from_dict(item) for item in value[key])
+        return cls(**value)
+
+
+@dataclass(frozen=True)
+class BenchmarkCase(_Record):
+    case_id: str
+    input_digest: str
+    company: str
+    split: str
+    input_form: str
+    form_status: str
+    form_receipt: AssetRef | None
+    length_words: int
+
+    def __post_init__(self):
+        _id(self.case_id); _digest(self.input_digest); _id(self.company); _id(self.split)
+        _require(self.split != "all", "reserved_split_name")
+        _choice(self.input_form, {"single_sentence", "paragraph", "not_applicable", "unclassified"})
+        _choice(self.form_status, {"reviewed", "authored", "unclassified"})
+        _asset(self.form_receipt, optional=True); _count(self.length_words, 1_000_000)
+        _require((self.form_status == "reviewed") == (self.form_receipt is not None), "form_receipt_mismatch")
+        _require((self.input_form == "unclassified") == (self.form_status == "unclassified"), "form_classification_mismatch")
+
+    @classmethod
+    def from_dict(cls, value):
+        value = cls._values(value)
+        value["form_receipt"] = None if value["form_receipt"] is None else AssetRef.from_dict(value["form_receipt"])
+        return cls(**value)
+
+
+@dataclass(frozen=True)
+class BenchmarkRelease(_Record):
+    schema_version: str
+    ref: str
+    version: str
+    evaluation_id: str
+    protocol_revision: str
+    interface_version: str
+    scope: str
+    input: AssetRef
+    corpus: AssetRef | None
+    label_set: AssetRef
+    required_label_profile: str
+    metric_profile: str
+    required_input_form: str
+    intended_use: str
+    cases: tuple[BenchmarkCase, ...]
+
+    def __post_init__(self):
+        _require(self.schema_version == BENCHMARK_SCHEMA, "unsupported_benchmark_schema")
+        for value in (self.ref, self.version, self.evaluation_id, self.protocol_revision, self.interface_version):
+            _id(value)
+        _choice(self.scope, {"component", "pipeline", "complete_system"})
+        _asset(self.input); _asset(self.corpus, optional=True); _asset(self.label_set)
+        _choice(self.required_input_form, {"any", "single_sentence", "paragraph"})
+        _choice(self.intended_use, {"illustrative", "development", "benchmark", "protected"})
+        _require(type(self.metric_profile) is str and self.metric_profile in _METRICS and _METRICS[self.metric_profile] == (self.required_label_profile, self.interface_version), "metric_label_interface_mismatch")
+        _tuples(self.cases, BenchmarkCase); _ids(tuple(case.case_id for case in self.cases))
+        _require(bool(self.cases), "empty_benchmark_release")
+        self.digest
+
+    @property
+    def digest(self):
+        return record_digest(self.to_dict())
+
+    @property
+    def asset_ref(self):
+        return AssetRef(self.ref, self.version, self.digest)
+
+    @classmethod
+    def from_dict(cls, value):
+        value = cls._values(value)
+        for key in ("input", "label_set"):
+            value[key] = AssetRef.from_dict(value[key])
+        value["corpus"] = None if value["corpus"] is None else AssetRef.from_dict(value["corpus"])
+        _require(type(value["cases"]) is list, "invalid_collection")
+        value["cases"] = tuple(BenchmarkCase.from_dict(item) for item in value["cases"])
+        return cls(**value)
+
+
+@dataclass(frozen=True)
+class SliceParameters(_Record):
+    companies: tuple[str, ...] = ()
+    split: str = "all"
+    length_words: tuple[int, int] | None = None
+
+    def __post_init__(self):
+        _ids(self.companies); _id(self.split)
+        if self.length_words is not None:
+            _require(type(self.length_words) is tuple and len(self.length_words) == 2, "invalid_length_range")
+            for value in self.length_words:
+                _count(value, 1_000_000)
+            _require(self.length_words[0] <= self.length_words[1], "invalid_length_range")
+
+    def to_dict(self):
+        return {"companies": list(self.companies), "split": self.split,
+                "length_words": None if self.length_words is None else dict(zip(("minimum", "maximum"), self.length_words))}
+
+    @classmethod
+    def from_dict(cls, value):
+        _require(type(value) is dict and set(value) <= {"companies", "split", "length_words"}, "invalid_slice_parameters")
+        companies, length = value.get("companies", []), value.get("length_words")
+        _require(type(companies) is list, "invalid_collection")
+        if length is not None:
+            _require(type(length) is dict and set(length) == {"minimum", "maximum"}, "invalid_length_range")
+            length = (length["minimum"], length["maximum"])
+        return cls(tuple(companies), value.get("split", "all"), length)
+
+
+@dataclass(frozen=True)
+class BenchmarkEligibility:
+    """A detached compatibility report, not approval, scoring or execution."""
+    _json: str
+
+    def __post_init__(self):
+        try:
+            value = json.loads(self._json)
+        except (ValueError, TypeError):
+            raise BenchmarkContractError("invalid_eligibility_record") from None
+        required = {"schema_version", "benchmark", "label_set", "evaluation_id", "protocol_revision", "interface_version", "scope", "metric_profile", "authority", "missingness", "intended_use", "purpose", "metadata_only", "review_receipts_verified", "semantic_authority_validated", "selected_case_ids", "selected_case_count", "judged_case_count", "unjudged_case_count", "blocked_case_count", "form_blocked_case_count", "status", "blockers", "slice"}
+        _require(type(value) is dict and set(value) == required and value["schema_version"] == ELIGIBILITY_SCHEMA, "invalid_eligibility_record")
+        _require(value["metadata_only"] is True and value["review_receipts_verified"] is False and value["semantic_authority_validated"] is False, "invalid_eligibility_trust_claim")
+        AssetRef.from_dict(value["benchmark"]); AssetRef.from_dict(value["label_set"])
+        for key in ("evaluation_id", "protocol_revision", "interface_version", "metric_profile"):
+            _id(value[key])
+        _choice(value["scope"], {"component", "pipeline", "complete_system"})
+        _choice(value["authority"], _AUTHORITY)
+        _choice(value["missingness"], {"complete_required", "native_qrels_unjudged", "development_assume_zero"})
+        _choice(value["intended_use"], {"illustrative", "development", "benchmark", "protected"})
+        _choice(value["purpose"], {"development_instrumentation", "evaluation"})
+        _choice(value["status"], {"eligible", "blocked"})
+        _require(type(value["selected_case_ids"]) is list and type(value["blockers"]) is list, "invalid_eligibility_record")
+        _ids(tuple(value["selected_case_ids"])); _ids(tuple(value["blockers"]))
+        for key in ("selected_case_count", "judged_case_count", "unjudged_case_count", "blocked_case_count", "form_blocked_case_count"):
+            _count(value[key], 512)
+        _require(0 < value["selected_case_count"] == len(value["selected_case_ids"]) and all(value[key] <= value["selected_case_count"] for key in ("judged_case_count", "unjudged_case_count", "blocked_case_count", "form_blocked_case_count")) and value["judged_case_count"] + value["unjudged_case_count"] <= value["selected_case_count"], "invalid_eligibility_counts")
+        _require((value["status"] == "eligible") == (not value["blockers"]) and value["blockers"] == sorted(value["blockers"]), "invalid_eligibility_status")
+        SliceParameters.from_dict(value["slice"])
+        _require(canonical_contract_json(value) == self._json, "invalid_canonical_metadata")
+
+    def to_dict(self):
+        return json.loads(self._json)
+
+    @property
+    def eligible(self):
+        return self.to_dict()["status"] == "eligible"
+
+
+@dataclass(frozen=True)
+class BenchmarkRegistry:
+    label_sets: tuple[LabelSetRelease, ...]
+    benchmarks: tuple[BenchmarkRelease, ...]
+
+    def __post_init__(self):
+        self.validate()
+
+    def validate(self):
+        _tuples(self.label_sets, LabelSetRelease); _tuples(self.benchmarks, BenchmarkRelease)
+        labels = {record.ref: record for record in self.label_sets}
+        _require(len(labels) == len(self.label_sets), "duplicate_label_release")
+        _require(len({record.ref for record in self.benchmarks}) == len(self.benchmarks), "duplicate_benchmark_release")
+        visiting, visited = set(), set()
+        def visit(ref):
+            _require(ref not in visiting, "lineage_cycle")
+            if ref in visited:
+                return
+            visiting.add(ref)
+            current = labels[ref]
+            current_cases = {case.case_id: case.input_digest for case in current.cases}
+            for link in current.lineage:
+                _require(link.parent_ref in labels, "lineage_parent_missing")
+                parent = labels[link.parent_ref]
+                visit(parent.ref)
+                _require(link.parent_digest == parent.digest, "lineage_parent_digest_mismatch")
+                _require(current.label_profile == parent.label_profile, "lineage_label_profile_mismatch")
+                _require(current.corpus == parent.corpus, "lineage_corpus_mismatch")
+                parent_cases = {case.case_id: case.input_digest for case in parent.cases}
+                if link.relationship == "revision":
+                    _require(current.input == parent.input and current_cases == parent_cases, "lineage_input_mismatch")
+                else:
+                    _require(all(parent_cases.get(key) == value for key, value in current_cases.items()), "lineage_subset_mismatch")
+                _require(link.changed_case_count <= len(current_cases), "lineage_changed_count_invalid")
+                _require(current.policy_revision == parent.policy_revision or link.reason == "policy_changed", "lineage_policy_reason_required")
+                _require(current.rubric_revision == parent.rubric_revision or link.reason in {"rubric_changed", "policy_changed"}, "lineage_rubric_reason_required")
+            visiting.remove(ref); visited.add(ref)
+        for ref in labels:
+            visit(ref)
+        protocols = {}
+        for benchmark in self.benchmarks:
+            _require(benchmark.label_set.ref in labels, "label_release_missing")
+            label = labels[benchmark.label_set.ref]
+            _require(benchmark.label_set == label.asset_ref, "label_release_digest_mismatch")
+            _require((benchmark.input, benchmark.corpus) == (label.input, label.corpus), "benchmark_input_binding_mismatch")
+            _require({c.case_id: c.input_digest for c in benchmark.cases} == {c.case_id: c.input_digest for c in label.cases}, "benchmark_case_binding_mismatch")
+            _require(benchmark.required_label_profile == label.label_profile, "benchmark_label_profile_mismatch")
+            _require(label.authority != "synthetic_fixture" or benchmark.intended_use == "illustrative", "synthetic_authority_not_benchmark")
+            _require(label.authority != "machine_provisional" or benchmark.intended_use == "development", "provisional_authority_not_benchmark")
+            _require(benchmark.intended_use != "protected" or label.authority in {"owner_reviewed", "human_adjudicated"}, "protected_authority_required")
+            key = (benchmark.evaluation_id, benchmark.protocol_revision)
+            boundary = (benchmark.interface_version, benchmark.scope, benchmark.required_label_profile, benchmark.metric_profile, benchmark.required_input_form)
+            _require(key not in protocols or protocols[key] == boundary, "protocol_boundary_mismatch")
+            protocols[key] = boundary
+        return self
+
+    def to_dict(self):
+        return {"schema_version": CATALOG_SCHEMA, "label_sets": [record.to_dict() for record in self.label_sets],
+                "benchmarks": [record.to_dict() for record in self.benchmarks]}
+
+    def resolve(self, benchmark_ref, parameters=None):
+        benchmark = next((b for b in self.benchmarks if b.ref == benchmark_ref), None)
+        _require(benchmark is not None, "benchmark_not_registered")
+        parameters = SliceParameters() if parameters is None else parameters
+        if type(parameters) is dict:
+            parameters = SliceParameters.from_dict(parameters)
+        _require(isinstance(parameters, SliceParameters), "invalid_slice_parameters")
+        _require(set(parameters.companies) <= {case.company for case in benchmark.cases}, "company_not_supported")
+        _require(parameters.split == "all" or parameters.split in {case.split for case in benchmark.cases}, "split_not_supported")
+        selected = [case for case in benchmark.cases if
+                    (not parameters.companies or case.company in parameters.companies)
+                    and (parameters.split == "all" or case.split == parameters.split)
+                    and (parameters.length_words is None or parameters.length_words[0] <= case.length_words <= parameters.length_words[1])]
+        _require(bool(selected), "empty_benchmark_slice")
+        _require(not parameters.companies or {case.company for case in selected} == set(parameters.companies), "unsupported_joint_slice")
+        label = next(record for record in self.label_sets if record.ref == benchmark.label_set.ref)
+        labels = {case.case_id: case for case in label.cases}
+        reviewed = {key for receipt in label.review_receipts for key in receipt.case_ids
+                    if label.authority != "human_adjudicated" or receipt.kind == "adjudication"}
+        blocked, forms, blockers = set(), set(), set()
+        judged = unjudged = 0
+        for case in selected:
+            status = labels[case.case_id].status
+            judged += status == "judged"; unjudged += status == "unjudged"
+            if status == "blocked":
+                blocked.add(case.case_id); blockers.add("blocked_reference_cases")
+            if status == "unjudged" and label.missingness == "complete_required":
+                blockers.add("required_labels_missing")
+            if status == "judged" and label.authority in {"owner_reviewed", "human_adjudicated"} and case.case_id not in reviewed:
+                blocked.add(case.case_id); blockers.add("review_coverage_incomplete")
+            if benchmark.required_input_form != "any" and (case.input_form != benchmark.required_input_form
+                    or case.form_status != "reviewed" and benchmark.intended_use != "illustrative"):
+                forms.add(case.case_id); blockers.add("reviewed_input_form_required")
+        value = {"schema_version": ELIGIBILITY_SCHEMA, "benchmark": benchmark.asset_ref.to_dict(),
+                 "label_set": label.asset_ref.to_dict(), "evaluation_id": benchmark.evaluation_id,
+                 "protocol_revision": benchmark.protocol_revision, "interface_version": benchmark.interface_version,
+                 "scope": benchmark.scope, "metric_profile": benchmark.metric_profile, "authority": label.authority,
+                 "missingness": label.missingness,
+                 "intended_use": benchmark.intended_use,
+                 "purpose": "development_instrumentation" if label.authority == "synthetic_fixture" else "evaluation",
+                 "metadata_only": True, "review_receipts_verified": False, "semantic_authority_validated": False,
+                 "selected_case_ids": [case.case_id for case in selected], "selected_case_count": len(selected),
+                 "judged_case_count": judged, "unjudged_case_count": unjudged, "blocked_case_count": len(blocked),
+                 "form_blocked_case_count": len(forms), "status": "blocked" if blockers else "eligible",
+                 "blockers": sorted(blockers), "slice": parameters.to_dict()}
+        return BenchmarkEligibility(canonical_contract_json(value))
+
+    def validate_legacy(self, resolved_record):
+        """Gate a trusted resolved legacy invocation without rewriting its bytes."""
+        try:
+            from .evals import resolve_eval
+            definition, parameters = resolved_record["definition"], resolved_record["parameters"]
+            expected = resolve_eval("rag.rerank.v1", parameters).to_dict()
+            _require(definition == expected["definition"] and parameters == expected["parameters"]
+                    and resolved_record["artifacts"] == expected["artifacts"],
+                    "legacy_benchmark_binding_mismatch")
+        except (KeyError, TypeError, EvaluationContractError):
+            raise BenchmarkContractError("legacy_benchmark_binding_mismatch") from None
+        eligibility = self.resolve(DEFAULT_BENCHMARK)
+        _require(eligibility.eligible, "legacy_benchmark_not_eligible")
+        return eligibility
+
+
+def synthetic_projection_metadata():
+    """Derived digests only; do not claim independently archived input/label assets."""
+    from .contracts import build_candidate_request, canonical_json
+    from .rag import FIXTURE_SHA256, load_rag_rerank_pack
+    pack = load_rag_rerank_pack()
+    requests = [build_candidate_request(case).to_dict() for case in pack.cases]
+    def projection_digest(value):
+        return "sha256:" + hashlib.sha256(canonical_json(value).encode()).hexdigest()
+    inputs = {"schema_version": "sphynx.synthetic-rerank.input-projection.v1", "cases": requests}
+    corpus = {"schema_version": "sphynx.synthetic-rerank.corpus-projection.v1", "cases": [
+        {"case_id": request["case_id"], "versions": request["versions"], "candidates": [
+            {"candidate_id": candidate["candidate_id"], "input": candidate["input"]} for candidate in request["candidates"]]}
+        for request in requests]}
+    references = {"schema_version": "sphynx.synthetic-rerank.reference-projection.v1", "cases": [
+        asdict(pack.reference_for(case.case_id)) for case in pack.cases]}
+    source = AssetRef(LEGACY_BANK, "2.0.0", "sha256:" + FIXTURE_SHA256)
+    input_ref = AssetRef("synthetic.rerank.input-projection@2.0.0", "2.0.0", projection_digest(inputs))
+    corpus_ref = AssetRef("synthetic.rerank.corpus-projection@2.0.0", "2.0.0", projection_digest(corpus))
+    reference_ref = AssetRef("synthetic.rerank.reference-projection@2.0.0", "2.0.0", projection_digest(references))
+    cases = tuple(BenchmarkCase(case.case_id, projection_digest(request), "synthetic", case.split,
+        "unclassified", "unclassified", None, len(case.query.split())) for case, request in zip(pack.cases, requests))
+    return source, input_ref, corpus_ref, reference_ref, cases
+
+
+def default_registry():
+    """Validate bundled metadata against the actual existing fixture before use."""
+    path = Path(__file__).resolve().parent / "data" / "benchmark-releases.v1.json"
+    try:
+        def unique(pairs):
+            value = {}
+            for key, item in pairs:
+                _require(key not in value, "duplicate_metadata_field")
+                value[key] = item
+            return value
+        raw = json.loads(path.read_bytes(), object_pairs_hook=unique,
+            parse_constant=lambda _: (_ for _ in ()).throw(BenchmarkContractError("invalid_canonical_metadata")))
+        _require(type(raw) is dict and set(raw) == {"schema_version", "source", "projection_version", "metadata_only", "independent_assets_archived", "label_sets", "benchmarks"}, "invalid_catalog_shape")
+        _require(raw["schema_version"] == CATALOG_SCHEMA and raw["projection_version"] == PROJECTION_VERSION
+                 and raw["metadata_only"] is True and raw["independent_assets_archived"] is False,
+                 "invalid_catalog_provenance")
+        source, inputs, corpus, references, cases = synthetic_projection_metadata()
+        _require(raw["source"] == source.to_dict(), "fixture_source_mismatch")
+        _require(type(raw["label_sets"]) is list and type(raw["benchmarks"]) is list
+                 and len(raw["label_sets"]) == len(raw["benchmarks"]) == 1, "unexpected_default_catalog")
+        registry = BenchmarkRegistry(tuple(LabelSetRelease.from_dict(item) for item in raw["label_sets"]),
+                                     tuple(BenchmarkRelease.from_dict(item) for item in raw["benchmarks"]))
+        label, benchmark = registry.label_sets[0], registry.benchmarks[0]
+        _require((label.ref, label.input, label.corpus, label.reference) == (DEFAULT_LABELS, inputs, corpus, references), "fixture_projection_mismatch")
+        _require(benchmark.ref == DEFAULT_BENCHMARK and benchmark.cases == cases and benchmark.evaluation_id == "rag.rerank.v1"
+                 and benchmark.protocol_revision == "rag.rerank.v1" and benchmark.scope == "component"
+                 and benchmark.version == label.version == "2.0.0"
+                 and benchmark.interface_version == "sphynx.rag.rerank.v1" and benchmark.metric_profile == "rag.rerank.metric.v1"
+                 and benchmark.required_label_profile == label.label_profile == "passage_qrels_and_evidence_groups"
+                 and label.policy_revision == "rag.rerank.synthetic-label-policy.v1" and label.rubric_revision == "rag.rerank.metric.v1"
+                 and benchmark.required_input_form == "any" and benchmark.intended_use == "illustrative"
+                 and label.authority == "synthetic_fixture" and label.missingness == "complete_required"
+                 and all(case.status == "judged" for case in label.cases), "fixture_membership_mismatch")
+        return registry
+    except (OSError, ValueError, TypeError, RecursionError) as exc:
+        if isinstance(exc, BenchmarkContractError):
+            raise
+        raise BenchmarkContractError("invalid_benchmark_catalog") from None

sphynx/candidates.py

@@ -0,0 +1,62 @@
+"""Small adapters for candidate scripts using the existing subprocess protocol."""
+
+from __future__ import annotations
+
+import sys
+from contextlib import redirect_stdout
+from typing import Callable
+
+from .contracts import (
+    RESPONSE_SCHEMA_VERSION,
+    ContractError,
+    canonical_json,
+    parse_candidate_request,
+    validate_candidate_response,
+)
+
+
+def serve_reranker(score: Callable[[str, str, float], float]) -> None:
+    """Consume one v1 request and write one fully validated v1 response.
+
+    Call this from a candidate script's ``if __name__ == "__main__"`` block.
+    ``score(query, passage, supplied_score)`` runs once per frozen passage and
+    must return a finite Python int or float (booleans are rejected). Callback
+    prints go to stderr. On invalid input/output or callback failure this raises
+    ``SystemExit(1)`` after a payload-free diagnostic on stderr and writes no
+    response. Timeout, environment, stream limits, and execution remain owned
+    by the subprocess runner. This adapter does not provide OS isolation.
+    """
+
+    try:
+        if not callable(score):
+            raise ContractError("score must be callable")
+        request = parse_candidate_request(getattr(sys.stdin, "buffer", sys.stdin).read())
+    except Exception as exc:
+        message = str(exc) if isinstance(exc, ContractError) else f"could not read request ({type(exc).__name__})"
+        print(f"serve_reranker: {message}", file=sys.stderr)
+        raise SystemExit(1) from None
+
+    scores = []
+    for index, item in enumerate(request.candidates):
+        try:
+            with redirect_stdout(sys.stderr):
+                number = score(request.query, item.input, item.dense_score)
+        except BaseException as exc:
+            # Exception text can contain the query/passage or caller secrets.
+            print(f"serve_reranker: scoring callback failed at passage {index + 1} ({type(exc).__name__})", file=sys.stderr)
+            raise SystemExit(1) from None
+        scores.append({"candidate_id": item.candidate_id, "ranking_score": number})
+    try:
+        response = validate_candidate_response({
+            "schema_version": RESPONSE_SCHEMA_VERSION,
+            "query_id": request.query_id,
+            "pool_digest": request.pool_digest,
+            "scores": scores,
+        }, request)
+        payload = canonical_json(response.to_dict()) + "\n"
+    except Exception as exc:
+        message = str(exc) if isinstance(exc, ContractError) else f"invalid response ({type(exc).__name__})"
+        print(f"serve_reranker: {message}", file=sys.stderr)
+        raise SystemExit(1) from None
+    sys.stdout.write(payload)
+    sys.stdout.flush()