sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/recorded_scores.py

@@ -0,0 +1,352 @@
+"""Replay a hash-bound snapshot of local reranker outputs.
+
+This adapter measures replay runtime, not model inference latency. Model timing
+belongs to the owner-side score snapshot retained as separate evidence.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from pathlib import Path
+from typing import Any, Mapping
+
+from .contracts import ContractError, canonical_json
+
+RECORDED_SCORE_SCHEMA = "sphynx.recorded.rerank.scores.v1"
+
+
+def replay_rerank_scores(request: Mapping[str, Any], score_path: str | Path,
+                         expected_sha256: str) -> dict[str, Any]:
+    """Return recorded scores only for an exactly matching label-free request."""
+    payload = Path(score_path).read_bytes()
+    if hashlib.sha256(payload).hexdigest() != expected_sha256.removeprefix("sha256:"):
+        raise ContractError("recorded model score snapshot hash differs")
+    try:
+        artifact = json.loads(payload)
+    except (ValueError, UnicodeError) as exc:
+        raise ContractError("invalid recorded model score snapshot") from exc
+    if not isinstance(artifact, dict) or artifact.get("schema_version") != RECORDED_SCORE_SCHEMA:
+        raise ContractError("unsupported recorded model score snapshot")
+    if request.get("schema_version") != "sphynx.rag.rerank.request.v1":
+        raise ContractError("unsupported recorded-score request")
+    rows = artifact.get("cases")
+    if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
+        raise ContractError("recorded model cases are invalid")
+    identifiers = [row.get("case_id") for row in rows]
+    if len(set(identifiers)) != len(identifiers):
+        raise ContractError("recorded model cases contain duplicate IDs")
+    matches = [row for row in rows if row.get("case_id") == request.get("case_id")]
+    if len(matches) != 1:
+        raise ContractError("recorded model case not found")
+    row = matches[0]
+    request_digest = "sha256:" + hashlib.sha256(canonical_json(dict(request)).encode("utf-8")).hexdigest()
+    if row.get("request_sha256") != request_digest:
+        raise ContractError("recorded model request identity differs")
+    scores = row.get("scores")
+    if not isinstance(scores, list) or len(scores) != len(request.get("candidates", [])):
+        raise ContractError("recorded model score coverage differs")
+    expected_ids = [candidate["candidate_id"] for candidate in request["candidates"]]
+    if [item.get("candidate_id") for item in scores] != expected_ids:
+        raise ContractError("recorded model score order or IDs differ")
+    for item in scores:
+        value = item.get("ranking_score")
+        if set(item) != {"candidate_id", "ranking_score"} or isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
+            raise ContractError("recorded model scores must be finite numbers")
+    return {
+        "schema_version": "sphynx.rag.rerank.response.v1",
+        "query_id": request["query_id"], "pool_digest": request["pool_digest"],
+        "scores": scores,
+    }
+# The strict additive interface retains the historical v1 replay API unchanged.
+BOUND_RECORDED_SCORE_SCHEMA = "sphynx.recorded.rerank.scores.v2"
+
+
+def _strict_json(payload: bytes) -> dict[str, Any]:
+    def unique(pairs):
+        result = {}
+        for key, value in pairs:
+            if key in result:
+                raise ContractError("duplicate recorded JSON field")
+            result[key] = value
+        return result
+    def invalid(value):
+        raise ContractError("nonfinite recorded JSON number")
+    try:
+        value = json.loads(payload, object_pairs_hook=unique, parse_constant=invalid)
+    except ContractError:
+        raise
+    except (ValueError, UnicodeError) as exc:
+        raise ContractError("invalid recorded score JSON") from exc
+    if not isinstance(value, dict):
+        raise ContractError("recorded artifact must be an object")
+    return value
+
+
+def _read_bound(path, digest):
+    from .integration_contracts import require_digest
+    require_digest(digest, "recording_sha256")
+    payload = Path(path).read_bytes()
+    if "sha256:" + hashlib.sha256(payload).hexdigest() != digest:
+        raise ContractError("recorded snapshot hash differs")
+    return _strict_json(payload)
+
+
+def _recorded_rows(requests):
+    from .model_adapters import request_pairs
+    result = {}
+    for request in requests:
+        # Detach the entire JSON request, including candidate lists and metadata.
+        request = json.loads(canonical_json(dict(request)))
+        request_pairs(request)
+        case_id = request["case_id"]
+        if case_id in result:
+            raise ContractError("duplicate requested recorded case IDs")
+        result[case_id] = request
+    if not result:
+        raise ContractError("recorded cohort must be nonempty")
+    return result
+
+
+def _validated_scores(scores, request):
+    if not isinstance(scores, list) or len(scores) != len(request["candidates"]):
+        raise ContractError("recorded score coverage differs")
+    expected = [item["candidate_id"] for item in request["candidates"]]
+    if any(not isinstance(item, dict) or set(item) != {"candidate_id", "ranking_score"} for item in scores):
+        raise ContractError("invalid recorded score fields")
+    if [item["candidate_id"] for item in scores] != expected:
+        raise ContractError("recorded candidate order differs")
+    for item in scores:
+        value = item["ranking_score"]
+        if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value):
+            raise ContractError("recorded scores must be finite scalar utilities")
+    return [dict(item) for item in scores]
+
+
+class DatasetBoundRecordedScorer:
+    """Strict complete-cohort replay; no model/provider is imported or executed.
+
+    Construction pins exact score bytes and every ordered label-free request.
+    Each call rechecks bytes against that pin, including after construction.
+    Original inference timing remains provenance; replay timing is lookup only.
+    """
+    def __init__(self, score_path, expected_sha256, *, identity, requests, expected_model):
+        from .integration_contracts import ComparisonIdentity, value_digest, CandidateConfiguration
+        if not isinstance(identity, ComparisonIdentity):
+            raise ContractError("recorded scorer requires ComparisonIdentity")
+        if identity.track != "rerank":
+            raise ContractError("recorded scorer requires the rerank track")
+        self.path = Path(score_path)
+        self.digest = expected_sha256
+        self.identity = identity
+        self._requests = _recorded_rows(requests)
+        self._request_digests = {case_id: value_digest(request) for case_id, request in self._requests.items()}
+        self._expected_model = dict(expected_model)
+        artifact = self._load()
+        self.configuration = CandidateConfiguration("recorded-" + artifact["model"]["id"] + "-v2", "sphynx.recorded.rerank.v2",
+            "sha256:" + hashlib.sha256(Path(__file__).read_bytes()).hexdigest(),
+            {"recording_sha256": expected_sha256, "identity": identity.to_dict(), "model": artifact["model"], "provenance": artifact["provenance"], "request_digests": list(self._request_digests.values()), "timing_scope": "recorded_lookup_only_not_model_inference"}, "recorded")
+
+    def _load(self):
+        from .integration_contracts import value_digest
+        artifact = _read_bound(self.path, self.digest)
+        if set(artifact) != {"schema_version", "identity", "model", "provenance", "cases"} or artifact["schema_version"] != BOUND_RECORDED_SCORE_SCHEMA:
+            raise ContractError("unsupported strict recorded score schema")
+        if artifact["identity"] != self.identity.to_dict():
+            raise ContractError("recorded dataset/cohort/reference/policy identity differs")
+        if artifact["model"] != self._expected_model:
+            raise ContractError("recorded model/checkpoint/tokenizer/truncation identity differs")
+        _validate_model_identity(artifact["model"])
+        provenance = artifact["provenance"]
+        fields = {"source_recording_sha256", "source_pool_sha256", "source_schema", "runtime", "timing_scope"}
+        mapped = isinstance(provenance, dict) and "request_order_mapping" in provenance
+        if mapped:
+            fields.add("request_order_mapping")
+        if not isinstance(provenance, dict) or set(provenance) != fields:
+            raise ContractError("recorded provenance fields differ")
+        if mapped:
+            from .public_rerank import PUBLIC_POOL_RELEASES
+            if (provenance["request_order_mapping"] != "verified_source_vector_to_first_stage_rank_by_exact_candidate_id"
+                or provenance["source_schema"] != "expanded-reranker.model-scores.v2"
+                or provenance["source_pool_sha256"] not in {"sha256:" + release.source_sha256 for release in PUBLIC_POOL_RELEASES}
+                or self.identity.dataset_digest != provenance["source_pool_sha256"]):
+                raise ContractError("invalid recorded release-specific ordering provenance")
+        from .integration_contracts import require_digest
+        require_digest(provenance["source_recording_sha256"], "source_recording_sha256")
+        if provenance["source_pool_sha256"] is not None:
+            require_digest(provenance["source_pool_sha256"], "source_pool_sha256")
+        if not isinstance(provenance["source_schema"], str) or not isinstance(provenance["timing_scope"], str) or not isinstance(provenance["runtime"], dict):
+            raise ContractError("invalid recording timing/runtime provenance")
+        rows = artifact["cases"]
+        if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
+            raise ContractError("invalid recorded cohort rows")
+        if [row.get("case_id") for row in rows] != list(self._requests):
+            raise ContractError("recorded cohort coverage or order differs")
+        for row, request in zip(rows, self._requests.values()):
+            if set(row) != {"case_id", "request_sha256", "scores", "model_inference_ms"} or row["request_sha256"] != value_digest(request):
+                raise ContractError("recorded request identity differs")
+            _validated_scores(row["scores"], request)
+            latency = row["model_inference_ms"]
+            if latency is not None and (isinstance(latency, bool) or not isinstance(latency, (int, float)) or not math.isfinite(latency) or latency < 0):
+                raise ContractError("invalid recorded inference latency")
+        return artifact
+
+    def __call__(self, request):
+        from .integration_contracts import value_digest
+        case_id = request.get("case_id")
+        if case_id not in self._requests or value_digest(request) != self._request_digests[case_id]:
+            raise ContractError("recorded request identity differs")
+        # Exact bytes were fully validated during construction. Rechecking the
+        # pinned bytes is sufficient; repeating cohort validation adds no proof.
+        artifact = _read_bound(self.path, self.digest)
+        row = next(row for row in artifact["cases"] if row["case_id"] == case_id)
+        return {"schema_version": "sphynx.rag.rerank.response.v1", "query_id": request["query_id"], "pool_digest": request["pool_digest"], "scores": _validated_scores(row["scores"], request)}
+
+
+def _validate_model_identity(model):
+    from .model_adapters import get_model_spec
+    from .integration_contracts import require_digest
+    fields = {"id", "checkpoint", "revision", "max_length", "batch_size", "tokenizer_revision", "truncation", "activation", "snapshot_digest", "onnx_sha256"}
+    if not isinstance(model, dict) or set(model) != fields:
+        raise ContractError("recorded model identity fields differ")
+    spec = get_model_spec(model["id"])
+    for name in ("checkpoint", "revision", "max_length", "batch_size"):
+        if model[name] != getattr(spec, name) or (name in ("max_length", "batch_size") and type(model[name]) is not int):
+            raise ContractError("recorded model differs from frozen parameters")
+    if model["tokenizer_revision"] != spec.revision or model["truncation"] != "longest_first" or model["activation"] not in ("identity", "legacy_cross_encoder_default"):
+        raise ContractError("recorded tokenizer/truncation/activation identity differs")
+    for name in ("snapshot_digest", "onnx_sha256"):
+        if model[name] is not None:
+            require_digest(model[name], name)
+    if spec.backend == "onnx_int8" and model["onnx_sha256"] is None:
+        raise ContractError("recorded INT8 graph digest required")
+
+
+def import_recorded_scores(score_path, expected_sha256, *, identity, requests, expected_model, source_pool_path=None, source_pool_sha256=None, source_task_ids=None, source_request_order="source"):
+    """Return a strict v2 artifact from an exact owner-held completed recording.
+
+    Supports the existing SDK v1 and expanded-reranker.model-scores.v2. Expanded
+    inputs require exact original pool bytes and explicit task-ID mapping. No
+    join on text or guessed IDs is performed; every query and ordered passage is
+    checked. Caller saves the result outside source control and pins its bytes.
+    ``snapshot_digest=None`` preserves unknown historical snapshot provenance.
+    ``source_request_order='first_stage_rank'`` is available only for registered
+    revised public pools. It verifies the original score-vector order first,
+    then maps unchanged values by exact candidate ID to the dense request order.
+    """
+    from .integration_contracts import ComparisonIdentity, value_digest
+    if not isinstance(identity, ComparisonIdentity) or identity.track != "rerank":
+        raise ContractError("import requires rerank ComparisonIdentity")
+    if source_request_order not in ("source", "first_stage_rank"):
+        raise ContractError("unsupported recorded source request order")
+    model = dict(expected_model)
+    _validate_model_identity(model)
+    artifact = _read_bound(score_path, expected_sha256)
+    requests = _recorded_rows(requests)
+    source_model = artifact.get("model")
+    if not isinstance(source_model, dict):
+        raise ContractError("source recording model identity missing")
+    for name in ("id", "checkpoint", "revision", "max_length", "batch_size"):
+        if source_model.get(name) != model[name]:
+            raise ContractError("source recorded model identity differs")
+    if model["snapshot_digest"] != source_model.get("snapshot_digest"):
+        raise ContractError("source snapshot digest is absent or differs; historical unknowns must remain null")
+    if model["onnx_sha256"] is not None and source_model.get("onnx_sha256", "").removeprefix("sha256:") != model["onnx_sha256"].removeprefix("sha256:"):
+        raise ContractError("source recorded ONNX graph identity differs")
+    cases = []
+    if artifact.get("schema_version") == RECORDED_SCORE_SCHEMA:
+        if source_request_order != "source":
+            raise ContractError("legacy recording requires unchanged source order")
+        if model["activation"] != "legacy_cross_encoder_default":
+            raise ContractError("SDK v1 recording requires its historical default activation identity")
+        if artifact.get("dataset_digest") != identity.dataset_digest:
+            raise ContractError("source recorded dataset identity differs")
+        rows = artifact.get("cases")
+        if not isinstance(rows, list) or any(not isinstance(row, dict) for row in rows):
+            raise ContractError("invalid source recording rows")
+        if [row.get("case_id") for row in rows] != list(requests) or artifact.get("case_count") != len(rows):
+            raise ContractError("source recording cohort coverage differs")
+        for row, request in zip(rows, requests.values()):
+            if row.get("request_sha256") != value_digest(request):
+                raise ContractError("source recording request identity differs")
+            cases.append({"case_id": request["case_id"], "request_sha256": value_digest(request), "scores": _validated_scores(row.get("scores"), request), "model_inference_ms": row.get("model_inference_ms")})
+        pool_digest = None
+        scope = artifact.get("timing_scope")
+    elif artifact.get("schema_version") == "expanded-reranker.model-scores.v2":
+        if model["activation"] != "identity":
+            raise ContractError("expanded source recording requires raw identity activation")
+        if source_pool_path is None or source_pool_sha256 is None or source_task_ids is None:
+            raise ContractError("expanded recordings require pinned source pool and explicit task-ID mapping")
+        pools = _read_bound(source_pool_path, source_pool_sha256)
+        if source_request_order == "first_stage_rank":
+            from .public_rerank import PUBLIC_POOL_RELEASES, load_public_pool_release
+            release = next((item for item in PUBLIC_POOL_RELEASES
+                            if item.source_sha256 == source_pool_sha256.removeprefix("sha256:")), None)
+            if release is None or identity.dataset_digest != "sha256:" + release.source_sha256:
+                raise ContractError("dense request ordering requires an exact registered public release")
+            load_public_pool_release(source_pool_path, release_id=release.release_id)
+        if artifact.get("pools_sha256") != source_pool_sha256.removeprefix("sha256:"):
+            raise ContractError("source recording pool identity differs")
+        if set(source_task_ids) != set(requests) or len(set(source_task_ids.values())) != len(requests):
+            raise ContractError("source task mapping coverage differs")
+        def unique_rows(rows, field):
+            if not isinstance(rows, list) or any(not isinstance(row, dict) or not isinstance(row.get(field), str) for row in rows):
+                raise ContractError("source rows invalid")
+            result = {row[field]: row for row in rows}
+            if len(result) != len(rows):
+                raise ContractError("source rows contain duplicate IDs")
+            return result
+        pool_rows = unique_rows(pools.get("queries"), "task_id")
+        score_rows = unique_rows(artifact.get("queries"), "task_id")
+        for case_id, request in requests.items():
+            task_id = source_task_ids[case_id]
+            if task_id not in pool_rows or task_id not in score_rows:
+                raise ContractError("source recorded cohort incomplete")
+            pool, row = pool_rows[task_id], score_rows[task_id]
+            source_candidates = pool.get("candidates", [])
+            ordered_candidates = sorted(source_candidates, key=lambda item: item["first_stage_rank"]) if source_request_order == "first_stage_rank" else source_candidates
+            if pool.get("query") != request["query"] or [{"candidate_id": p.get("candidate_id"), "input": p.get("input")} for p in ordered_candidates] != [{"candidate_id": p["candidate_id"], "input": p["input"]} for p in request["candidates"]]:
+                raise ContractError("source pool query/passages differ from request")
+            if source_request_order == "first_stage_rank" and any(
+                candidate["first_stage_rank"] != requested["dense_rank"] or
+                candidate["first_stage_score"] != requested["dense_score"]
+                for candidate, requested in zip(ordered_candidates, request["candidates"])
+            ):
+                raise ContractError("source dense ranks or scores differ from request")
+            split = pool.get("split")
+            completion = artifact.get("completion", {}).get(split, {})
+            split_count = sum(p.get("split") == split for p in pool_rows.values())
+            actual_scored = sum(key in score_rows for key, p in pool_rows.items() if p.get("split") == split)
+            if completion.get("complete") is not True or completion.get("pool_query_count") != split_count or completion.get("scored_query_count") != actual_scored or actual_scored != split_count:
+                raise ContractError("source recording split remains prescreen-only or incomplete")
+            ids, values = row.get("candidate_ids"), row.get("scores")
+            if not isinstance(ids, list) or not isinstance(values, list) or len(ids) != len(values):
+                raise ContractError("source recording candidate coverage differs")
+            if source_request_order == "first_stage_rank":
+                if ids != [candidate["candidate_id"] for candidate in source_candidates]:
+                    raise ContractError("recorded original score-vector order differs from source pool")
+                by_id = dict(zip(ids, values))
+                ids = [candidate["candidate_id"] for candidate in request["candidates"]]
+                values = [by_id[cid] for cid in ids]
+            scores = [{"candidate_id": cid, "ranking_score": value} for cid, value in zip(ids, values)]
+            cases.append({"case_id": case_id, "request_sha256": value_digest(request), "scores": _validated_scores(scores, request), "model_inference_ms": row.get("latency_ms")})
+        pool_digest = source_pool_sha256
+        scope = "source_sequential_per_query_inference_including_tokenization_excluding_model_load; historical_warmup_unknown"
+    else:
+        raise ContractError("unsupported source recorded schema")
+    output = {"schema_version": BOUND_RECORDED_SCORE_SCHEMA, "identity": identity.to_dict(), "model": model,
+        "provenance": {"source_recording_sha256": expected_sha256, "source_pool_sha256": pool_digest, "source_schema": artifact["schema_version"], "runtime": artifact.get("runtime", {}), "timing_scope": scope}, "cases": cases}
+    if source_request_order != "source":
+        output["provenance"]["request_order_mapping"] = "verified_source_vector_to_first_stage_rank_by_exact_candidate_id"
+    # Validate all output values without needing a saved temporary artifact.
+    for row in cases:
+        latency = row["model_inference_ms"]
+        if latency is not None and (isinstance(latency, bool) or not isinstance(latency, (int, float)) or not math.isfinite(latency) or latency < 0):
+            raise ContractError("invalid source recorded inference latency")
+    if not isinstance(scope, str) or not isinstance(output["provenance"]["runtime"], dict):
+        raise ContractError("source timing/runtime provenance missing")
+    return output
+
+
+def create_recorded_scorer(score_path, expected_sha256, *, identity, requests, expected_model):
+    return DatasetBoundRecordedScorer(score_path, expected_sha256, identity=identity, requests=requests, expected_model=expected_model)

sphynx/records.py

@@ -0,0 +1,124 @@
+"""Stable evidence and grading identity contracts for later SDK stages.
+
+These are immutable value types only. Recording, grading, and persistence are
+implemented by separate subsystems, which must preserve these identities.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass
+
+from .contracts import ContractError, _id, _text, _tuple_of_ids
+
+EVIDENCE_SCHEMA_VERSION = "sphynx.evidence.v1"
+EVALUATION_SCHEMA_VERSION = "sphynx.evaluation.v1"
+METRIC_DEFINITION_SCHEMA_VERSION = "sphynx.metric-definition.v1"
+
+_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
+_COVERAGE_STATES = frozenset(
+    {"available", "not_captured", "truncated", "redacted", "corrupt", "expired", "unsupported_schema"}
+)
+
+
+@dataclass(frozen=True)
+class CaptureIdentity:
+    """One attempt at one case; its run and capture IDs must stay distinct."""
+
+    run_id: str
+    capture_id: str
+    case_id: str
+    trial_index: int
+    attempt_index: int
+    pool_digest: str
+    trust_tier: str = "local_unverified"
+    schema_version: str = EVIDENCE_SCHEMA_VERSION
+
+    def __post_init__(self) -> None:
+        for field in ("run_id", "capture_id", "case_id"):
+            _id(getattr(self, field), field)
+        if self.run_id == self.capture_id:
+            raise ContractError("run_id and capture_id must be distinct")
+        for field in ("trial_index", "attempt_index"):
+            value = getattr(self, field)
+            if type(value) is not int or value < 1:
+                raise ContractError(f"{field} must be a positive integer")
+        if not isinstance(self.pool_digest, str) or not _DIGEST.fullmatch(self.pool_digest):
+            raise ContractError("pool_digest must be a sha256: lowercase-hex digest")
+        if self.trust_tier != "local_unverified":
+            raise ContractError("this SDK stage supports only local_unverified trust tier")
+        if self.schema_version != EVIDENCE_SCHEMA_VERSION:
+            raise ContractError(f"capture schema_version must be {EVIDENCE_SCHEMA_VERSION}")
+
+
+@dataclass(frozen=True)
+class EvaluationIdentity:
+    """A grading pass over retained evidence; regrading creates a new ID."""
+
+    run_id: str
+    evaluation_id: str
+    grader_release: str
+    metric_release: str
+    schema_version: str = EVALUATION_SCHEMA_VERSION
+
+    def __post_init__(self) -> None:
+        _id(self.run_id, "run_id")
+        _id(self.evaluation_id, "evaluation_id")
+        if self.run_id == self.evaluation_id:
+            raise ContractError("run_id and evaluation_id must be distinct")
+        _id(self.grader_release, "grader_release")
+        _id(self.metric_release, "metric_release")
+        if self.schema_version != EVALUATION_SCHEMA_VERSION:
+            raise ContractError(f"evaluation schema_version must be {EVALUATION_SCHEMA_VERSION}")
+
+
+@dataclass(frozen=True)
+class EvidenceCapability:
+    """Observed availability, distinct from a metric result or a zero value."""
+
+    capability_id: str
+    capability_schema_version: str
+    state: str
+    observed_count: int
+    reason: str | None = None
+
+    def __post_init__(self) -> None:
+        _id(self.capability_id, "capability_id")
+        _id(self.capability_schema_version, "capability_schema_version")
+        if self.state not in _COVERAGE_STATES:
+            raise ContractError(f"state must be one of {', '.join(sorted(_COVERAGE_STATES))}")
+        if type(self.observed_count) is not int or self.observed_count < 0:
+            raise ContractError("observed_count must be a nonnegative integer")
+        if self.reason is not None:
+            _text(self.reason, "reason")
+        if self.state != "available" and self.reason is None:
+            raise ContractError("unavailable evidence requires a reason")
+
+
+@dataclass(frozen=True)
+class MetricDefinition:
+    """Versioned metric declaration; computation belongs to the grader."""
+
+    metric_id: str
+    release: str
+    unit: str
+    direction: str
+    required_capabilities: tuple[str, ...]
+    primary: bool = False
+    schema_version: str = METRIC_DEFINITION_SCHEMA_VERSION
+
+    def __post_init__(self) -> None:
+        _id(self.metric_id, "metric_id")
+        _id(self.release, "release")
+        _id(self.unit, "unit")
+        if self.direction not in ("higher", "lower"):
+            raise ContractError("direction must be higher or lower")
+        object.__setattr__(
+            self,
+            "required_capabilities",
+            _tuple_of_ids(self.required_capabilities, "required_capabilities"),
+        )
+        if type(self.primary) is not bool:
+            raise ContractError("primary must be a boolean")
+        if self.schema_version != METRIC_DEFINITION_SCHEMA_VERSION:
+            raise ContractError(f"metric definition schema_version must be {METRIC_DEFINITION_SCHEMA_VERSION}")