sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/evals.py

@@ -0,0 +1,538 @@
+"""Versioned evaluation-function contracts, without execution or provider calls.
+
+Resolution expands typed defaults and binds operator-registered artifact metadata.
+It does not fetch/verify artifact bytes, authenticate a user, or run an adapter.
+The safe-integer JSON domain gives future Rust implementations explicit hash rules.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import InitVar, dataclass
+from types import MappingProxyType
+from typing import Any, Collection, Mapping
+
+DEFINITION_SCHEMA = "sphynx.eval.definition.v1"
+INVOCATION_SCHEMA = "sphynx.eval.invocation.v1"
+CANONICALIZATION = "sphynx.safe-integer-json.v1"
+_MAX_INTEGER = 2**53 - 1
+_MAX_BYTES = 131_072
+_MISSING = object()
+_ID = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:@/-]{0,255}\Z")
+_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
+_RESOLUTION_TOKEN = object()
+_DECIMAL = re.compile(r"-?(?:0|[1-9][0-9]*)(?:\.[0-9]*[1-9])?\Z")
+
+
+class EvaluationContractError(ValueError):
+    """A definition, parameter, artifact binding, or invocation is invalid."""
+
+
+def canonical_contract_json(value: Any) -> str:
+    """Sorted UTF-8 JSON, safe integers, no floats/surrogates/normalization.
+
+    Decimal quantities can use a string schema with canonical_decimal format.
+    This serializer deliberately does not replace the legacy SDK serializer.
+    """
+    nodes = [0]
+    string_bytes = [0]
+
+    def check(item: Any, depth: int) -> None:
+        nodes[0] += 1
+        if depth > 24 or nodes[0] > 8192:
+            raise EvaluationContractError("contract JSON exceeds structural limits")
+        if item is None or type(item) is bool:
+            return
+        if type(item) is int:
+            if abs(item) > _MAX_INTEGER:
+                raise EvaluationContractError("integer exceeds the interoperable safe-integer range")
+            return
+        if type(item) is str:
+            if len(item) > _MAX_BYTES:
+                raise EvaluationContractError("contract string exceeds the byte limit")
+            try:
+                string_bytes[0] += len(item.encode("utf-8", errors="strict"))
+            except UnicodeEncodeError:
+                raise EvaluationContractError("strings must contain valid Unicode scalar values") from None
+            if string_bytes[0] > _MAX_BYTES:
+                raise EvaluationContractError("contract strings exceed the byte limit")
+            return
+        if isinstance(item, Mapping):
+            for key, child in item.items():
+                if type(key) is not str:
+                    raise EvaluationContractError("JSON object keys must be strings")
+                check(key, depth + 1)
+                check(child, depth + 1)
+            return
+        if isinstance(item, (list, tuple)):
+            for child in item:
+                check(child, depth + 1)
+            return
+        raise EvaluationContractError("contract JSON permits null, booleans, safe integers, strings, arrays, and objects")
+
+    check(value, 0)
+    text = json.dumps(_thaw(value), ensure_ascii=False, sort_keys=True, separators=(",", ":"), allow_nan=False)
+    if len(text.encode("utf-8")) > _MAX_BYTES:
+        raise EvaluationContractError("contract JSON exceeds the byte limit")
+    return text
+
+
+def _thaw(value: Any) -> Any:
+    if isinstance(value, Mapping):
+        return {key: _thaw(child) for key, child in value.items()}
+    if isinstance(value, (tuple, list)):
+        return [_thaw(child) for child in value]
+    return value
+
+
+def _freeze(value: Any) -> Any:
+    if isinstance(value, Mapping):
+        return MappingProxyType({key: _freeze(child) for key, child in value.items()})
+    if isinstance(value, (tuple, list)):
+        return tuple(_freeze(child) for child in value)
+    return value
+
+
+def _copy(value: Any) -> Any:
+    return json.loads(canonical_contract_json(value))
+
+
+def _digest(value: Any) -> str:
+    return "sha256:" + hashlib.sha256(canonical_contract_json(value).encode("utf-8")).hexdigest()
+
+
+def _identifier(value: str, label: str) -> None:
+    if type(value) is not str or not _ID.fullmatch(value):
+        raise EvaluationContractError(f"{label} must be a bounded stable identifier")
+
+
+def _check_pointer(path: str) -> None:
+    if (type(path) is not str or not path.startswith("/") or len(path) > 1024
+            or re.search(r"~(?![01])", path)):
+        raise EvaluationContractError("parameter paths must be bounded RFC 6901 JSON pointers")
+
+
+def _pointer(value: Mapping[str, Any], path: str) -> Any:
+    _check_pointer(path)
+    current: Any = value
+    for part in path[1:].split("/"):
+        key = part.replace("~1", "/").replace("~0", "~")
+        if isinstance(current, Mapping) and key in current:
+            current = current[key]
+        elif isinstance(current, (list, tuple)) and re.fullmatch(r"0|[1-9][0-9]*", key) and int(key) < len(current):
+            current = current[int(key)]
+        else:
+            raise EvaluationContractError(f"parameter path {path!r} does not exist")
+    return current
+
+
+def _schema_check(schema: Mapping[str, Any], depth: int = 0) -> None:
+    if not isinstance(schema, Mapping) or depth > 16:
+        raise EvaluationContractError("parameter schemas must be bounded objects")
+    common = {"type", "title", "description", "default", "enum"}
+    if any(key in schema and type(schema[key]) is not str for key in ("title", "description")):
+        raise EvaluationContractError("schema title and description must be strings")
+    if "oneOf" in schema:
+        if set(schema) - {"oneOf", "default", "title", "description"}:
+            raise EvaluationContractError("unsupported oneOf schema keyword")
+        variants = schema["oneOf"]
+        if not isinstance(variants, (tuple, list)) or not 2 <= len(variants) <= 16:
+            raise EvaluationContractError("oneOf requires 2 to 16 schemas")
+        for variant in variants:
+            _schema_check(variant, depth + 1)
+        return
+    kind = schema.get("type")
+    extras = {"object": {"properties", "required", "additionalProperties"},
+              "array": {"items", "minItems", "maxItems"},
+              "string": {"minLength", "maxLength", "format"},
+              "integer": {"minimum", "maximum"}, "boolean": set(), "null": set()}
+    if type(kind) is not str or kind not in extras or set(schema) - common - extras[kind]:
+        raise EvaluationContractError("unsupported parameter type or schema keyword")
+    if "enum" in schema and (not isinstance(schema["enum"], (tuple, list)) or not schema["enum"]):
+        raise EvaluationContractError("enum must contain options")
+    if "enum" in schema and kind in {"object", "array"}:
+        raise EvaluationContractError("enum is supported for primitive parameters only")
+    if kind == "object":
+        properties = schema.get("properties", {})
+        required = schema.get("required", [])
+        if (not isinstance(properties, Mapping) or schema.get("additionalProperties") is not False
+                or not isinstance(required, (tuple, list))
+                or any(type(key) is not str or key not in properties for key in required)
+                or len(set(required)) != len(required)):
+            raise EvaluationContractError("object schemas require declared properties and additionalProperties=false")
+        for child in properties.values():
+            _schema_check(child, depth + 1)
+    if kind == "array":
+        _schema_check(schema.get("items"), depth + 1)
+    for lower, upper in (("minimum", "maximum"), ("minLength", "maxLength"), ("minItems", "maxItems")):
+        for key in (lower, upper):
+            if key in schema and (type(schema[key]) is not int or (key != "minimum" and key != "maximum" and schema[key] < 0)):
+                raise EvaluationContractError(f"{key} must be an integer bound")
+        if lower in schema and upper in schema and schema[lower] > schema[upper]:
+            raise EvaluationContractError("schema lower bound exceeds upper bound")
+    if "format" in schema and schema["format"] != "canonical_decimal":
+        raise EvaluationContractError("the supported string format is canonical_decimal")
+    if "enum" in schema:
+        plain = {key: value for key, value in schema.items() if key not in {"enum", "default"}}
+        options = schema["enum"]
+        normalized = [_normalize(plain, option, "enum") for option in options]
+        if len({canonical_contract_json(option) for option in normalized}) != len(normalized):
+            raise EvaluationContractError("enum options must be distinct")
+
+
+def _normalize(schema: Mapping[str, Any], value: Any, path: str) -> Any:
+    if "oneOf" in schema:
+        matched = []
+        for variant in schema["oneOf"]:
+            try:
+                matched.append(_normalize(variant, value, path))
+            except EvaluationContractError:
+                pass
+        if len(matched) != 1:
+            raise EvaluationContractError(f"{path} must match exactly one parameter variant")
+        return matched[0]
+    kind = schema["type"]
+    valid = {"object": isinstance(value, Mapping), "array": isinstance(value, (tuple, list)),
+             "string": type(value) is str, "integer": type(value) is int,
+             "boolean": type(value) is bool, "null": value is None}[kind]
+    if not valid:
+        raise EvaluationContractError(f"{path} must be {kind}")
+    if kind == "object":
+        properties = schema.get("properties", {})
+        unknown = set(value) - set(properties)
+        if unknown:
+            raise EvaluationContractError(f"{path} has unsupported fields: {', '.join(sorted(unknown))}")
+        out = {}
+        for key, child in properties.items():
+            raw = value.get(key, _MISSING)
+            if raw is _MISSING:
+                raw = child.get("default", _MISSING)
+            if raw is _MISSING:
+                if key in schema.get("required", ()):
+                    raise EvaluationContractError(f"{path}.{key} is required")
+                continue
+            out[key] = _normalize(child, raw, f"{path}.{key}")
+        value = out
+    elif kind == "array":
+        if not schema.get("minItems", 0) <= len(value) <= schema.get("maxItems", 256):
+            raise EvaluationContractError(f"{path} has an unsupported item count")
+        value = [_normalize(schema["items"], item, f"{path}[{index}]") for index, item in enumerate(value)]
+    elif kind == "integer":
+        if not schema.get("minimum", -_MAX_INTEGER) <= value <= schema.get("maximum", _MAX_INTEGER):
+            raise EvaluationContractError(f"{path} is outside the supported range")
+    elif kind == "string":
+        if not schema.get("minLength", 0) <= len(value) <= schema.get("maxLength", 16384):
+            raise EvaluationContractError(f"{path} has an unsupported length")
+        if schema.get("format") == "canonical_decimal" and (_DECIMAL.fullmatch(value) is None or value == "-0"):
+            raise EvaluationContractError(f"{path} has an unsupported string format")
+    if "enum" in schema and canonical_contract_json(value) not in {canonical_contract_json(item) for item in schema["enum"]}:
+        raise EvaluationContractError(f"{path} is not a supported option")
+    return value
+
+
+@dataclass(frozen=True)
+class ArtifactReference:
+    """Registered metadata only; contains no URI, bytes, labels, or credentials."""
+    ref: str
+    kind: str
+    version: str
+    digest: str
+    public: bool = False
+
+    def __post_init__(self):
+        for key in ("ref", "kind", "version"):
+            _identifier(getattr(self, key), key)
+        if type(self.digest) is not str or not _DIGEST.fullmatch(self.digest) or type(self.public) is not bool:
+            raise EvaluationContractError("artifact requires SHA-256 identity and explicit visibility")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"ref": self.ref, "kind": self.kind, "version": self.version, "digest": self.digest,
+                "metadata_only": True, "bytes_verified": False}
+
+
+@dataclass(frozen=True)
+class ArtifactBinding:
+    parameter_path: str
+    kind: str
+    comparison_kinds: tuple[str, ...] = ("quality", "latency")
+
+    def __post_init__(self):
+        _check_pointer(self.parameter_path)
+        _identifier(self.kind, "artifact kind")
+        if (not isinstance(self.comparison_kinds, tuple) or any(type(k) is not str or k not in {"quality", "latency"} for k in self.comparison_kinds)
+                or len(set(self.comparison_kinds)) != len(self.comparison_kinds)):
+            raise EvaluationContractError("artifact comparison kinds must be unique quality/latency entries")
+
+
+@dataclass(frozen=True)
+class EvaluationDefinition:
+    evaluation_id: str
+    version: str
+    title: str
+    status: str
+    interface_version: str
+    implementation: Mapping[str, Any]
+    parameter_schema: Mapping[str, Any]
+    quality_parameters: tuple[str, ...]
+    latency_parameters: tuple[str, ...]
+    artifact_bindings: tuple[ArtifactBinding, ...] = ()
+    metric_ids: tuple[str, ...] = ()
+
+    def __post_init__(self):
+        for key in ("evaluation_id", "version", "interface_version"):
+            _identifier(getattr(self, key), key)
+        if type(self.status) is not str or self.status not in {"runnable", "contract_only"} or type(self.title) is not str or not self.title:
+            raise EvaluationContractError("definition requires a title and supported status")
+        if (not isinstance(self.implementation, Mapping) or not self.implementation
+                or any(type(value) is not str for value in self.implementation.values())):
+            raise EvaluationContractError("implementation must declare stable implementation revision strings")
+        for key, revision in self.implementation.items():
+            _identifier(key, "implementation component")
+            _identifier(revision, "implementation revision")
+        schema = _copy(self.parameter_schema)
+        _schema_check(schema)
+        if schema.get("type") != "object":
+            raise EvaluationContractError("evaluation parameter root must be an object")
+        if "default" in schema:
+            raise EvaluationContractError("evaluation defaults belong to parameters, not the root object")
+        for paths in (self.quality_parameters, self.latency_parameters):
+            if not isinstance(paths, tuple) or any(type(p) is not str for p in paths) or len(set(paths)) != len(paths):
+                raise EvaluationContractError("comparison parameter paths must be unique JSON pointers")
+            for path in paths:
+                _check_pointer(path)
+        if not isinstance(self.artifact_bindings, tuple) or any(not isinstance(b, ArtifactBinding) for b in self.artifact_bindings):
+            raise EvaluationContractError("artifact bindings must be a tuple of ArtifactBinding")
+        if len({b.parameter_path for b in self.artifact_bindings}) != len(self.artifact_bindings):
+            raise EvaluationContractError("artifact binding paths must be unique")
+        if not isinstance(self.metric_ids, tuple) or any(type(metric) is not str for metric in self.metric_ids) or len(set(self.metric_ids)) != len(self.metric_ids):
+            raise EvaluationContractError("metric IDs must be a unique tuple")
+        for metric in self.metric_ids:
+            _identifier(metric, "metric ID")
+        object.__setattr__(self, "parameter_schema", _freeze(schema))
+        object.__setattr__(self, "implementation", _freeze(_copy(self.implementation)))
+        # Validate all declared defaults, including inactive tagged variants.
+        def defaults(node):
+            if "default" in node:
+                _normalize(node, node["default"], "default")
+            for child in node.get("properties", {}).values():
+                defaults(child)
+            for child in node.get("oneOf", ()):
+                defaults(child)
+            if "items" in node:
+                defaults(node["items"])
+        defaults(schema)
+        canonical_contract_json(self.to_dict())
+
+    @property
+    def runnable(self) -> bool:
+        return self.status == "runnable"
+
+    @property
+    def definition_digest(self) -> str:
+        return _digest(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": DEFINITION_SCHEMA, "canonicalization": CANONICALIZATION,
+                "evaluation_id": self.evaluation_id, "version": self.version, "title": self.title,
+                "status": self.status, "runnable": self.runnable, "interface_version": self.interface_version,
+                "implementation": _thaw(self.implementation), "parameter_schema": _thaw(self.parameter_schema),
+                "comparison": {"quality_parameters": list(self.quality_parameters), "latency_parameters": list(self.latency_parameters),
+                               "candidate_identity_included": False, "reporting_parameters_included": False},
+                "artifact_bindings": [{"parameter_path": b.parameter_path, "kind": b.kind,
+                                       "comparison_kinds": list(b.comparison_kinds)} for b in self.artifact_bindings],
+                "metric_ids": list(self.metric_ids)}
+
+
+@dataclass(frozen=True)
+class ResolvedInvocation:
+    """Immutable resolved test metadata; execution identities are optional bindings."""
+    definition: EvaluationDefinition
+    parameters: Mapping[str, Any]
+    artifacts: tuple[Mapping[str, Any], ...]
+    candidate_identity: Mapping[str, Any] | None = None
+    runtime_identity: Mapping[str, Any] | None = None
+    _registry_token: InitVar[object] = None
+
+    def __post_init__(self, _registry_token):
+        if _registry_token is not _RESOLUTION_TOKEN:
+            raise EvaluationContractError("resolved invocations must be constructed through registry.resolve()")
+        object.__setattr__(self, "parameters", _freeze(_copy(self.parameters)))
+        object.__setattr__(self, "artifacts", tuple(_freeze(_copy(item)) for item in self.artifacts))
+        for key in ("candidate_identity", "runtime_identity"):
+            value = getattr(self, key)
+            if value is not None:
+                if not isinstance(value, Mapping):
+                    raise EvaluationContractError(f"{key} must be an object")
+                object.__setattr__(self, key, _freeze(_copy(value)))
+
+    @property
+    def invocation_digest(self) -> str:
+        return _digest(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": INVOCATION_SCHEMA, "canonicalization": CANONICALIZATION,
+                "definition": {"evaluation_id": self.definition.evaluation_id, "version": self.definition.version,
+                               "digest": self.definition.definition_digest, "implementation": _thaw(self.definition.implementation)},
+                "status": "contract_resolved", "runnable": self.definition.runnable,
+                "parameters": _thaw(self.parameters), "artifacts": _thaw(self.artifacts),
+                "candidate_identity": _thaw(self.candidate_identity), "runtime_identity": _thaw(self.runtime_identity),
+                "runtime_identity_status": "bound" if self.runtime_identity is not None else "unbound",
+                "artifact_bytes_verified": False}
+
+    def comparison_key(self, kind: str = "quality") -> str:
+        if kind not in {"quality", "latency"}:
+            raise EvaluationContractError("comparison kind must be quality or latency")
+        paths = self.definition.quality_parameters
+        if kind == "latency":
+            paths = tuple(dict.fromkeys((*paths, *self.definition.latency_parameters)))
+        return _digest({"schema_version": "sphynx.eval.comparison-cohort.v1", "kind": kind,
+                        "definition_digest": self.definition.definition_digest,
+                        "parameters": {path: _thaw(_pointer(self.parameters, path)) for path in paths},
+                        "artifacts": [_thaw(item) for item in self.artifacts if kind in item["comparison_kinds"]],
+                        "runtime_identity": _thaw(self.runtime_identity) if kind == "latency" else None,
+                        "runtime_identity_status": ("bound" if self.runtime_identity is not None else "unbound") if kind == "latency" else "not_used"})
+
+
+@dataclass(frozen=True)
+class EvaluationRegistry:
+    """Operator-owned registry. Grants must come from trusted authorization code."""
+    definitions: tuple[EvaluationDefinition, ...]
+    artifacts: tuple[ArtifactReference, ...] = ()
+
+    def __post_init__(self):
+        for items, cls, key in ((self.definitions, EvaluationDefinition, "evaluation_id"), (self.artifacts, ArtifactReference, "ref")):
+            if not isinstance(items, tuple) or any(not isinstance(item, cls) for item in items):
+                raise EvaluationContractError("registry entries must be immutable typed tuples")
+            if len({getattr(item, key) for item in items}) != len(items):
+                raise EvaluationContractError("registry identities must be unique")
+
+    def get(self, evaluation_id: str) -> EvaluationDefinition:
+        match = next((d for d in self.definitions if d.evaluation_id == evaluation_id), None)
+        if match is None:
+            raise EvaluationContractError("evaluation definition is not registered")
+        return match
+
+    def resolve(self, evaluation_id: str, parameters: Mapping[str, Any] | None = None, *,
+                granted_artifacts: Collection[str] = (), candidate_identity: Mapping[str, Any] | None = None,
+                runtime_identity: Mapping[str, Any] | None = None) -> ResolvedInvocation:
+        definition = self.get(evaluation_id)
+        if (isinstance(granted_artifacts, (str, bytes)) or not isinstance(granted_artifacts, Collection)
+                or any(type(ref) is not str for ref in granted_artifacts)):
+            raise EvaluationContractError("artifact grants must be an explicit collection of reference strings")
+        if parameters is not None and not isinstance(parameters, Mapping):
+            raise EvaluationContractError("parameters must be an object")
+        values = _normalize(definition.parameter_schema, _copy({} if parameters is None else parameters), "parameters")
+        records = []
+        for binding in sorted(definition.artifact_bindings, key=lambda item: item.parameter_path):
+            ref = _pointer(values, binding.parameter_path)
+            artifact = next((a for a in self.artifacts if a.ref == ref), None)
+            if artifact is None or not (artifact.public or artifact.ref in granted_artifacts):
+                raise EvaluationContractError("requested artifact is unavailable or not authorized")
+            if artifact.kind != binding.kind:
+                raise EvaluationContractError("registered artifact kind differs from parameter binding")
+            records.append({"parameter_path": binding.parameter_path, "comparison_kinds": list(binding.comparison_kinds), **artifact.to_dict()})
+        # All comparison paths must be present in the fully expanded invocation.
+        for path in (*definition.quality_parameters, *definition.latency_parameters):
+            _pointer(values, path)
+        return ResolvedInvocation(definition, values, tuple(records), candidate_identity, runtime_identity, _registry_token=_RESOLUTION_TOKEN)
+
+
+def _object(properties, *, required=(), default=_MISSING):
+    schema = {"type": "object", "properties": properties, "required": list(required), "additionalProperties": False}
+    if default is not _MISSING:
+        schema["default"] = default
+    return schema
+
+
+def _integer(default, minimum, maximum):
+    return {"type": "integer", "default": default, "minimum": minimum, "maximum": maximum}
+
+
+def _reporting():
+    return _object({"format": {"type": "string", "enum": ["markdown", "json"], "default": "markdown"}}, default={})
+
+
+def default_evaluation_registry() -> EvaluationRegistry:
+    # Lazy imports keep this contract layer independent of runner/candidate code.
+    from .metrics import GRADER_RELEASE, METRIC_RELEASE, metric_definitions
+    from .rag import FIXTURE_SHA256
+    bank = "help-center-evidence-mini@2.0.0"
+    legacy = _object({
+        "question_bank": {"type": "string", "enum": [bank], "default": bank},
+        "split": {"type": "string", "enum": ["all"], "default": "all"},
+        "top_k": {"type": "integer", "enum": [5], "default": 5},
+        "relevance_threshold": {"type": "integer", "enum": [2], "default": 2},
+        "runtime": _object({"trials": {"type": "integer", "enum": [1], "default": 1},
+                            "per_case_timeout_seconds": {"type": "integer", "enum": [30], "default": 30}}, default={}),
+        "reporting": _reporting(),
+    })
+    configurable = _copy(legacy)
+    props = configurable["properties"]
+    props["question_bank"] = {"type": "string", "minLength": 1, "maxLength": 256, "default": bank}
+    props["split"] = {"type": "string", "enum": ["all", "dev", "test", "holdout"], "default": "all"}
+    props["top_k"] = _integer(5, 1, 100)
+    props["relevance_threshold"] = _integer(2, 0, 3)
+    props["runtime"] = _object({"trials": _integer(1, 1, 10), "per_case_timeout_seconds": _integer(30, 1, 300)}, default={})
+    quality = ("/question_bank", "/split", "/top_k", "/relevance_threshold", "/runtime")
+    bindings = (ArtifactBinding("/question_bank", "question_bank"),)
+    definitions = (
+        EvaluationDefinition("rag.rerank.v1", "v1", "Six-case illustrative reranking", "runnable",
+                             "sphynx.rag.rerank.v1", {"adapter": "legacy-rag-rerank.v1", "grader": GRADER_RELEASE, "metrics": METRIC_RELEASE},
+                             legacy, quality, ("/runtime",), bindings, tuple(d.metric_id for d in metric_definitions())),
+        EvaluationDefinition("rag.rerank.v2", "v2", "Parameterized reranking contract", "contract_only",
+                             "sphynx.rag.rerank.v1", {"adapter": "not_implemented", "grader": "parameterized-rerank.pending"},
+                             configurable, quality, ("/runtime",), bindings),
+        EvaluationDefinition("audio.similarity.v1", "v1", "Fourier or wavelet audio comparison contract", "contract_only",
+                             "sphynx.audio.similarity.v1", {"adapter": "not_implemented", "grader": "audio-similarity.pending"},
+                             _object({"audio_bank": {"type": "string", "minLength": 1, "maxLength": 256},
+                                      "comparison": {"oneOf": [
+                                          _object({"method": {"type": "string", "enum": ["fourier"]},
+                                                   "fft_size": _integer(1024, 16, 65536),
+                                                   "window": {"type": "string", "enum": ["hann", "rectangular"], "default": "hann"}}, required=("method",)),
+                                          _object({"method": {"type": "string", "enum": ["wavelet"]},
+                                                   "family": {"type": "string", "enum": ["haar", "db2"], "default": "haar"},
+                                                   "levels": _integer(4, 1, 12)}, required=("method",))],
+                                          "default": {"method": "fourier"}},
+                                      "runtime": _object({"trials": _integer(1, 1, 10), "per_case_timeout_seconds": _integer(30, 1, 300)}, default={}),
+                                      "reporting": _reporting()}, required=("audio_bank",)),
+                             ("/audio_bank", "/comparison", "/runtime"), ("/runtime",), (ArtifactBinding("/audio_bank", "audio_bank"),)),
+    )
+    from .public_eval import public_definition, public_artifacts
+    return EvaluationRegistry((*definitions, public_definition()),
+        (ArtifactReference(bank, "question_bank", "2.0.0", "sha256:" + FIXTURE_SHA256, public=True), *public_artifacts()))
+
+
+def list_evals(*, registry: EvaluationRegistry | None = None) -> tuple[EvaluationDefinition, ...]:
+    """Discover installed definitions; does not assert hosted availability."""
+    return (registry or default_evaluation_registry()).definitions
+
+
+def describe_eval(evaluation_id: str, *, registry: EvaluationRegistry | None = None) -> dict[str, Any]:
+    definition = (registry or default_evaluation_registry()).get(evaluation_id)
+    return {**definition.to_dict(), "definition_digest": definition.definition_digest}
+
+
+def resolve_eval(evaluation_id: str, parameters: Mapping[str, Any] | None = None, *,
+                 registry: EvaluationRegistry | None = None, granted_artifacts: Collection[str] = (),
+                 candidate_identity: Mapping[str, Any] | None = None,
+                 runtime_identity: Mapping[str, Any] | None = None) -> ResolvedInvocation:
+    return (registry or default_evaluation_registry()).resolve(
+        evaluation_id, parameters, granted_artifacts=granted_artifacts,
+        candidate_identity=candidate_identity, runtime_identity=runtime_identity)
+
+
+def require_legacy_rerank(evaluation_id: str, parameters: Mapping[str, Any] | None = None) -> ResolvedInvocation:
+    """Fail before process/filesystem work unless the existing adapter supports it."""
+    definition = default_evaluation_registry().get(evaluation_id)
+    if not definition.runnable or definition.evaluation_id != "rag.rerank.v1":
+        raise EvaluationContractError("evaluation has a contract only; its execution adapter is not implemented")
+    resolved = resolve_eval(evaluation_id, parameters)
+    # Keep the legacy record and comparison hashes intact. Admission uses
+    # separate, operator-owned benchmark and label metadata.
+    from .benchmarks import BenchmarkContractError, default_registry
+    try:
+        default_registry().validate_legacy(resolved.to_dict())
+    except BenchmarkContractError as exc:
+        raise EvaluationContractError(str(exc)) from None
+    return resolved

sphynx/evaluation_protocols.py

@@ -0,0 +1,124 @@
+"""Optional domain-neutral protocol identities, never grants or attestations."""
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field, fields
+from types import MappingProxyType
+from typing import Any
+import re
+
+from .workflow_contracts import (
+    WorkflowContractError, _object, _optional_text, _text, _texts, _thaw,
+    contract_digest,
+)
+
+IDENTITY_SCHEMA = "sphynx.versioned-identity.v1"
+PROTOCOL_SCHEMA = "sphynx.evaluation-protocol.v1"
+_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
+
+
+def _envelope(value: Mapping[str, Any], schema: str, cls: type) -> dict[str, Any]:
+    if not isinstance(value, Mapping):
+        raise WorkflowContractError("versioned envelope must be a mapping")
+    result = dict(value)
+    if result.pop("schema_version", None) != schema:
+        raise WorkflowContractError("unsupported " + schema + " envelope version")
+    if set(result) - {item.name for item in fields(cls)}:
+        raise WorkflowContractError("unknown " + schema + " envelope fields")
+    return result
+
+
+@dataclass(frozen=True)
+class VersionedIdentity:
+    """A declared identity. A digest does not authenticate the referenced source."""
+
+    identity_id: str
+    version: str
+    digest: str | None = None
+
+    def __post_init__(self) -> None:
+        _text(self.identity_id, "identity_id")
+        _text(self.version, "version")
+        if self.digest is not None and (
+            type(self.digest) is not str or not _DIGEST.fullmatch(self.digest)
+        ):
+            raise WorkflowContractError("identity digest must be canonical sha256 text")
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": IDENTITY_SCHEMA, "identity_id": self.identity_id,
+                "version": self.version, "digest": self.digest}
+
+    @classmethod
+    def from_dict(cls, value: Mapping[str, Any]) -> VersionedIdentity:
+        try:
+            return cls(**_envelope(value, IDENTITY_SCHEMA, cls))
+        except TypeError as exc:
+            raise WorkflowContractError("invalid versioned identity fields") from exc
+
+
+@dataclass(frozen=True)
+class EvaluationProtocol:
+    """Frozen conditions for an opt-in driver, with an open evaluation unit.
+
+    Population and grouping semantics belong to the protocol author. Bindings
+    identify source/bank/labels/rubric/evaluator/runtime as needed by that driver.
+    They contain permitted identifiers only, never protected questions or grants.
+    PayloadSchema and Dataset still validate and authorize the actual payloads.
+    """
+
+    protocol_id: str
+    version: str
+    implementation_id: str
+    unit: str
+    input_schema_id: str
+    output_schema_id: str
+    population: Mapping[str, Any]
+    bindings: Mapping[str, VersionedIdentity] = field(default_factory=dict)
+    grouping_keys: tuple[str, ...] = ()
+    reset_policy_id: str | None = None
+
+    def __post_init__(self) -> None:
+        for name in ("protocol_id", "version", "implementation_id", "unit",
+                     "input_schema_id", "output_schema_id"):
+            _text(getattr(self, name), name)
+        population = _object(self.population, "protocol population")
+        if not population:
+            raise WorkflowContractError("protocol requires an explicit population rule")
+        object.__setattr__(self, "population", population)
+        if not isinstance(self.bindings, Mapping):
+            raise WorkflowContractError("protocol bindings must be a mapping")
+        bindings = {}
+        for role, identity in self.bindings.items():
+            _text(role, "identity role")
+            if not isinstance(identity, VersionedIdentity):
+                raise WorkflowContractError("protocol bindings require VersionedIdentity values")
+            bindings[role] = identity
+        object.__setattr__(self, "bindings", MappingProxyType(bindings))
+        object.__setattr__(self, "grouping_keys", _texts(self.grouping_keys, "grouping_keys"))
+        _optional_text(self.reset_policy_id, "reset_policy_id")
+
+    @property
+    def protocol_digest(self) -> str:
+        return contract_digest(self.to_dict())
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": PROTOCOL_SCHEMA, "protocol_id": self.protocol_id,
+                "version": self.version, "implementation_id": self.implementation_id,
+                "unit": self.unit, "input_schema_id": self.input_schema_id,
+                "output_schema_id": self.output_schema_id,
+                "population": _thaw(self.population),
+                "bindings": {role: identity.to_dict() for role, identity in self.bindings.items()},
+                "grouping_keys": list(self.grouping_keys), "reset_policy_id": self.reset_policy_id}
+
+    @classmethod
+    def from_dict(cls, value: Mapping[str, Any]) -> EvaluationProtocol:
+        values = _envelope(value, PROTOCOL_SCHEMA, cls)
+        bindings = values.get("bindings", {})
+        if not isinstance(bindings, Mapping):
+            raise WorkflowContractError("protocol bindings must be a mapping")
+        values["bindings"] = {role: VersionedIdentity.from_dict(identity)
+                              for role, identity in bindings.items()}
+        try:
+            return cls(**values)
+        except TypeError as exc:
+            raise WorkflowContractError("invalid evaluation protocol fields") from exc