sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/integration_contracts.py

@@ -0,0 +1,474 @@
+"""Additive integration value contracts; this module never executes candidates.
+
+All digests are canonical ``sha256:`` lowercase hex. Raw-file digests bind exact
+bytes; value digests use the existing canonical_json encoding. Legacy candidate
+responses and pack identities retain their original semantics.
+"""
+from __future__ import annotations
+
+import hashlib
+import re
+from collections.abc import Mapping, Sequence
+from dataclasses import dataclass, fields, is_dataclass
+from pathlib import Path, PurePosixPath
+from types import MappingProxyType
+from typing import Any
+
+from .contracts import ContractError, _finite_number, _id, _text, _tuple_of_ids, canonical_json
+
+CONFIGURATION_SCHEMA = "sphynx.candidate-configuration.v1"
+SCORE_SEMANTICS_SCHEMA = "sphynx.score-semantics.v1"
+RICH_SCORES_SCHEMA = "sphynx.rich-score-capture.v1"
+COMPARISON_SCHEMA = "sphynx.track-comparison-identity.v1"
+TRACK_CAPTURE_SCHEMA = "sphynx.track-capture.v1"
+TRACK_EVALUATION_SCHEMA = "sphynx.track-evaluation.v1"
+_DIGEST = re.compile(r"^sha256:[0-9a-f]{64}$")
+_SECRET_KEYS = frozenset({"api_key", "apikey", "access_token", "refresh_token", "password", "secret", "authorization", "credential", "credentials", "private_key", "credential_value", "bearer_token", "token"})
+
+
+def require_digest(value: object, field: str) -> str:
+    if not isinstance(value, str) or not _DIGEST.fullmatch(value):
+        raise ContractError(f"{field} must be a sha256: lowercase-hex digest")
+    return value
+
+
+def _optional_digest(value: object, field: str) -> None:
+    if value is not None:
+        require_digest(value, field)
+
+
+def _schema(actual: str, expected: str) -> None:
+    if actual != expected:
+        raise ContractError(f"schema_version must be {expected}")
+
+
+def _choice(value: object, choices: tuple[str, ...], field: str) -> None:
+    if not isinstance(value, str) or value not in choices:
+        raise ContractError(f"{field} must be one of {', '.join(choices)}")
+
+
+def _count(value: object, field: str, minimum: int = 0) -> None:
+    if type(value) is not int or value < minimum:
+        raise ContractError(f"{field} must be an integer >= {minimum}")
+
+
+def _sequence(value: object, item_type: type, field: str) -> tuple:
+    if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
+        raise ContractError(f"{field} must be a sequence")
+    result = tuple(value)
+    if any(not isinstance(item, item_type) for item in result):
+        raise ContractError(f"{field} must contain {item_type.__name__} values")
+    return result
+
+
+def _freeze(value: Any, *, reject_secrets: bool = False) -> Any:
+    if isinstance(value, Mapping):
+        result = {}
+        for key, item in value.items():
+            if not isinstance(key, str):
+                raise ContractError("JSON object keys must be strings")
+            normalized = key.lower().replace('-', '_')
+            if reject_secrets and (normalized in _SECRET_KEYS or normalized.endswith(("_api_key", "_access_token", "_password", "_secret", "_private_key"))):
+                raise ContractError("credential values are forbidden; use credential_refs")
+            result[key] = _freeze(item, reject_secrets=reject_secrets)
+        return MappingProxyType(result)
+    if isinstance(value, (list, tuple)):
+        return tuple(_freeze(item, reject_secrets=reject_secrets) for item in value)
+    if value is None or type(value) in (str, bool, int):
+        return value
+    if type(value) is float:
+        _finite_number(value, "JSON value")
+        return value
+    raise ContractError("value must contain only finite JSON data")
+
+
+def _wire(value: Any) -> Any:
+    if is_dataclass(value):
+        return {field.name: _wire(getattr(value, field.name)) for field in fields(value)}
+    if isinstance(value, Mapping):
+        return {key: _wire(item) for key, item in value.items()}
+    if isinstance(value, (tuple, list)):
+        return [_wire(item) for item in value]
+    return value
+
+
+def value_digest(value: Any) -> str:
+    return "sha256:" + hashlib.sha256(canonical_json(_wire(value)).encode("utf-8")).hexdigest()
+
+
+class WireValue:
+    def to_dict(self) -> dict[str, Any]:
+        return _wire(self)
+
+    @classmethod
+    def _checked_fields(cls, value: Mapping[str, Any]) -> dict[str, Any]:
+        if not isinstance(value, Mapping) or set(value) != {field.name for field in fields(cls)}:
+            raise ContractError(f"{cls.__name__} fields differ from the exact schema")
+        return dict(value)
+
+    @classmethod
+    def from_dict(cls, value: Mapping[str, Any]):
+        return cls(**cls._checked_fields(value))
+
+    @property
+    def digest(self) -> str:
+        return value_digest(self)
+
+
+@dataclass(frozen=True)
+class CandidateConfiguration(WireValue):
+    configuration_id: str
+    interface_version: str
+    source_digest: str
+    parameters: Mapping[str, Any]
+    execution_mode: str
+    model_id: str | None = None
+    model_revision: str | None = None
+    model_snapshot_digest: str | None = None
+    credential_refs: tuple[str, ...] = ()
+    schema_version: str = CONFIGURATION_SCHEMA
+
+    def __post_init__(self) -> None:
+        _schema(self.schema_version, CONFIGURATION_SCHEMA)
+        _id(self.configuration_id, "configuration_id")
+        _id(self.interface_version, "interface_version")
+        require_digest(self.source_digest, "source_digest")
+        _choice(self.execution_mode, ("recorded", "trusted_local", "external_command", "live_provider"), "execution_mode")
+        if not isinstance(self.parameters, Mapping):
+            raise ContractError("parameters must be a JSON object")
+        object.__setattr__(self, "parameters", _freeze(self.parameters, reject_secrets=True))
+        refs = _tuple_of_ids(self.credential_refs, "credential_refs")
+        if any(not ref.startswith(("env:", "keyring:", "credential:")) or not ref.split(":", 1)[1] for ref in refs):
+            raise ContractError("credential_refs must use env:, keyring:, or credential: references")
+        object.__setattr__(self, "credential_refs", refs)
+        model = (self.model_id, self.model_revision, self.model_snapshot_digest)
+        if any(item is not None for item in model):
+            if any(item is None for item in model):
+                raise ContractError("model ID, revision and snapshot digest must be supplied together")
+            _text(self.model_id, "model_id")
+            _text(self.model_revision, "model_revision")
+            require_digest(self.model_snapshot_digest, "model_snapshot_digest")
+
+
+@dataclass(frozen=True)
+class ScoreField(WireValue):
+    field_id: str
+    kind: str
+    direction: str
+    unit: str
+    probability_target: str | None = None
+
+    def __post_init__(self) -> None:
+        _id(self.field_id, "field_id")
+        _choice(self.kind, ("utility", "pool_relative", "diagnostic", "probability"), "score kind")
+        _choice(self.direction, ("higher", "lower"), "direction")
+        _id(self.unit, "unit")
+        if self.kind == "probability":
+            _id(self.probability_target, "probability_target")
+            if self.unit != "probability" or self.direction != "higher":
+                raise ContractError("probability requires unit=probability and direction=higher")
+        elif self.probability_target is not None or self.unit == "probability":
+            raise ContractError("only a probability field may declare probability semantics")
+
+
+@dataclass(frozen=True)
+class ScoreSemantics(WireValue):
+    subject_digest: str
+    fields: tuple[ScoreField, ...]
+    fitted_state_digest: str | None = None
+    schema_version: str = SCORE_SEMANTICS_SCHEMA
+
+    def __post_init__(self) -> None:
+        _schema(self.schema_version, SCORE_SEMANTICS_SCHEMA)
+        require_digest(self.subject_digest, "subject_digest")
+        _optional_digest(self.fitted_state_digest, "fitted_state_digest")
+        items = _sequence(self.fields, ScoreField, "fields")
+        if not items or len({item.field_id for item in items}) != len(items):
+            raise ContractError("score fields must be nonempty and unique")
+        object.__setattr__(self, "fields", items)
+
+    @classmethod
+    def from_dict(cls, value):
+        checked = cls._checked_fields(value)
+        if isinstance(checked, dict) and "fields" in checked:
+            checked["fields"] = tuple(ScoreField.from_dict(item) for item in _sequence(checked["fields"], Mapping, "fields"))
+        return super().from_dict(checked)
+
+
+@dataclass(frozen=True)
+class RichScoreRecord(WireValue):
+    case_id: str
+    request_digest: str
+    pool_digest: str
+    candidate_id: str
+    field_id: str
+    value: float | None
+
+    def __post_init__(self) -> None:
+        for name in ("case_id", "candidate_id", "field_id"):
+            _id(getattr(self, name), name)
+        for name in ("request_digest", "pool_digest"):
+            require_digest(getattr(self, name), name)
+        if self.value is not None:
+            object.__setattr__(self, "value", _finite_number(self.value, "score"))
+
+
+@dataclass(frozen=True)
+class RichScoreCapture(WireValue):
+    capture_manifest_digest: str
+    cohort_digest: str
+    reference_digest: str
+    semantics: ScoreSemantics
+    records: tuple[RichScoreRecord, ...]
+    schema_version: str = RICH_SCORES_SCHEMA
+
+    def __post_init__(self) -> None:
+        _schema(self.schema_version, RICH_SCORES_SCHEMA)
+        for name in ("capture_manifest_digest", "cohort_digest", "reference_digest"):
+            require_digest(getattr(self, name), name)
+        if not isinstance(self.semantics, ScoreSemantics):
+            raise ContractError("semantics must be ScoreSemantics")
+        records = _sequence(self.records, RichScoreRecord, "records")
+        definitions = {item.field_id: item for item in self.semantics.fields}
+        seen = set()
+        cases = {}
+        for record in records:
+            key = (record.case_id, record.candidate_id, record.field_id)
+            if key in seen or record.field_id not in definitions:
+                raise ContractError("duplicate score record or undeclared score field")
+            seen.add(key)
+            binding = (record.request_digest, record.pool_digest)
+            if record.case_id in cases and cases[record.case_id] != binding:
+                raise ContractError("scores for one case must bind the same request and pool")
+            cases[record.case_id] = binding
+            if definitions[record.field_id].kind == "probability" and record.value is not None and not 0 <= record.value <= 1:
+                raise ContractError("probability values must lie in [0, 1]")
+        object.__setattr__(self, "records", records)
+
+    @classmethod
+    def from_dict(cls, value):
+        checked = cls._checked_fields(value)
+        if isinstance(checked, dict):
+            if "semantics" in checked:
+                checked["semantics"] = ScoreSemantics.from_dict(checked["semantics"])
+            if "records" in checked:
+                checked["records"] = tuple(RichScoreRecord.from_dict(item) for item in _sequence(checked["records"], Mapping, "records"))
+        return super().from_dict(checked)
+
+
+@dataclass(frozen=True)
+class ComparisonIdentity(WireValue):
+    track: str
+    pack_id: str
+    pack_release: str
+    dataset_schema_version: str
+    dataset_digest: str
+    cohort_digest: str
+    reference_digest: str
+    policy_digest: str
+    metric_release: str
+    grader_release: str
+    subject_digest: str
+    context_condition: str = "none"
+    generator_digest: str | None = None
+    score_semantics_digest: str | None = None
+    fitted_state_digest: str | None = None
+    judgment_policy_digest: str | None = None
+    schema_version: str = COMPARISON_SCHEMA
+
+    def __post_init__(self) -> None:
+        _schema(self.schema_version, COMPARISON_SCHEMA)
+        for name in ("track", "pack_id", "pack_release", "dataset_schema_version", "metric_release", "grader_release"):
+            _id(getattr(self, name), name)
+        for name in ("dataset_digest", "cohort_digest", "reference_digest", "policy_digest", "subject_digest"):
+            require_digest(getattr(self, name), name)
+        for name in ("generator_digest", "score_semantics_digest", "fitted_state_digest", "judgment_policy_digest"):
+            _optional_digest(getattr(self, name), name)
+        _choice(self.context_condition, ("none", "retrieved", "oracle", "empty"), "context_condition")
+        if self.context_condition != "none" and self.generator_digest is None:
+            raise ContractError("context conditions require a pinned generator digest")
+        if self.fitted_state_digest is not None and self.score_semantics_digest is None:
+            raise ContractError("fitted state requires score semantics identity")
+
+
+def require_comparable(baseline: ComparisonIdentity, candidate: ComparisonIdentity, *, context_conditions: bool = False) -> None:
+    """Reject mismatched fixed boundaries; only explicit context diagnostics vary context.
+
+    subject_digest pins the subject boundary and frozen unchanged components.
+    The changed candidate/configuration is captured separately, never silently
+    substituted into a recording. It is not a requirement that two candidate
+    implementations under comparison have identical source code.
+    """
+    if not isinstance(baseline, ComparisonIdentity) or not isinstance(candidate, ComparisonIdentity):
+        raise ContractError("comparison requires ComparisonIdentity values")
+    left, right = baseline.to_dict(), candidate.to_dict()
+    if context_conditions:
+        if baseline.context_condition == "none" or candidate.context_condition == "none":
+            raise ContractError("context diagnostics require explicit generator conditions")
+        left.pop("context_condition")
+        right.pop("context_condition")
+    different = [name for name in left if left[name] != right[name]]
+    if different:
+        raise ContractError("incompatible comparison identity: " + ", ".join(different))
+
+
+@dataclass(frozen=True)
+class ArtifactBinding(WireValue):
+    path: str
+    sha256: str
+    size_bytes: int
+
+    def __post_init__(self) -> None:
+        _text(self.path, "artifact path")
+        path = PurePosixPath(self.path)
+        if path.is_absolute() or "\\" in self.path or ":" in self.path or any(part in ("", ".", "..") for part in self.path.split("/")):
+            raise ContractError("artifact path must be a safe relative POSIX path")
+        require_digest(self.sha256, "artifact sha256")
+        _count(self.size_bytes, "size_bytes")
+
+
+@dataclass(frozen=True)
+class TrackCaseCapture(WireValue):
+    case_id: str
+    trial_index: int
+    attempt_index: int
+    request: ArtifactBinding
+    response: ArtifactBinding | None
+    observed: ArtifactBinding | None
+    failure: str | None
+    runner_latency_ms: float | None
+
+    def __post_init__(self) -> None:
+        _id(self.case_id, "case_id")
+        _count(self.trial_index, "trial_index", 1)
+        _count(self.attempt_index, "attempt_index", 1)
+        if not isinstance(self.request, ArtifactBinding):
+            raise ContractError("request must be ArtifactBinding")
+        for name in ("response", "observed"):
+            item = getattr(self, name)
+            if item is not None and not isinstance(item, ArtifactBinding):
+                raise ContractError(f"{name} must be ArtifactBinding or null")
+        if self.failure is not None:
+            _id(self.failure, "failure")
+        if self.response is None and self.failure is None:
+            raise ContractError("missing response requires an explicit failure state")
+        if self.runner_latency_ms is not None:
+            latency = _finite_number(self.runner_latency_ms, "runner_latency_ms")
+            if latency < 0:
+                raise ContractError("runner latency must be nonnegative")
+            object.__setattr__(self, "runner_latency_ms", latency)
+
+    @classmethod
+    def from_dict(cls, value):
+        checked = cls._checked_fields(value)
+        if isinstance(checked, dict):
+            for name in ("request", "response", "observed"):
+                if name in checked and checked[name] is not None:
+                    checked[name] = ArtifactBinding.from_dict(checked[name])
+        return super().from_dict(checked)
+
+
+@dataclass(frozen=True)
+class TrackCaptureManifest(WireValue):
+    run_id: str
+    identity: ComparisonIdentity
+    candidate: CandidateConfiguration
+    case_ids: tuple[str, ...]
+    captures: tuple[TrackCaseCapture, ...]
+    trust_tier: str = "local_unverified"
+    schema_version: str = TRACK_CAPTURE_SCHEMA
+
+    def __post_init__(self) -> None:
+        _schema(self.schema_version, TRACK_CAPTURE_SCHEMA)
+        _id(self.run_id, "run_id")
+        if not isinstance(self.identity, ComparisonIdentity) or not isinstance(self.candidate, CandidateConfiguration):
+            raise ContractError("capture requires comparison identity and candidate configuration")
+        if self.trust_tier != "local_unverified":
+            raise ContractError("only local_unverified captures are supported")
+        case_ids = _tuple_of_ids(self.case_ids, "case_ids", empty=False)
+        captures = _sequence(self.captures, TrackCaseCapture, "captures")
+        keys = [(item.case_id, item.trial_index, item.attempt_index) for item in captures]
+        if len(set(keys)) != len(keys) or {item.case_id for item in captures} != set(case_ids):
+            raise ContractError("capture coverage must match cohort with unique case/trial/attempt identities")
+        paths = [binding.path for item in captures for binding in (item.request, item.response, item.observed) if binding is not None]
+        if len(set(paths)) != len(paths):
+            raise ContractError("capture artifacts must have unique paths")
+        object.__setattr__(self, "case_ids", case_ids)
+        object.__setattr__(self, "captures", captures)
+
+    @classmethod
+    def from_dict(cls, value):
+        checked = cls._checked_fields(value)
+        if isinstance(checked, dict):
+            if "identity" in checked:
+                checked["identity"] = ComparisonIdentity.from_dict(checked["identity"])
+            if "candidate" in checked:
+                checked["candidate"] = CandidateConfiguration.from_dict(checked["candidate"])
+            if "captures" in checked:
+                checked["captures"] = tuple(TrackCaseCapture.from_dict(item) for item in _sequence(checked["captures"], Mapping, "captures"))
+        return super().from_dict(checked)
+
+
+@dataclass(frozen=True)
+class TrackEvaluationManifest(WireValue):
+    evaluation_id: str
+    run_id: str
+    capture_manifest_digest: str
+    identity: ComparisonIdentity
+    metrics: Mapping[str, Any]
+    judgment_artifact_digest: str | None = None
+    schema_version: str = TRACK_EVALUATION_SCHEMA
+
+    def __post_init__(self) -> None:
+        _schema(self.schema_version, TRACK_EVALUATION_SCHEMA)
+        _id(self.evaluation_id, "evaluation_id")
+        _id(self.run_id, "run_id")
+        if self.evaluation_id == self.run_id:
+            raise ContractError("evaluation_id and run_id must be distinct")
+        require_digest(self.capture_manifest_digest, "capture_manifest_digest")
+        _optional_digest(self.judgment_artifact_digest, "judgment_artifact_digest")
+        if not isinstance(self.identity, ComparisonIdentity) or not isinstance(self.metrics, Mapping):
+            raise ContractError("evaluation requires comparison identity and a metric mapping")
+        object.__setattr__(self, "metrics", _freeze(self.metrics))
+
+    @classmethod
+    def from_dict(cls, value):
+        checked = cls._checked_fields(value)
+        if isinstance(checked, dict) and "identity" in checked:
+            checked["identity"] = ComparisonIdentity.from_dict(checked["identity"])
+        return super().from_dict(checked)
+
+
+@dataclass(frozen=True)
+class TrackRunRef:
+    path: Path
+    manifest_sha256: str
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "path", Path(self.path))
+        require_digest(self.manifest_sha256, "manifest_sha256")
+        _id(self.path.name, "run_id")
+
+    @property
+    def run_id(self) -> str:
+        return self.path.name
+
+
+@dataclass(frozen=True)
+class TrackEvaluationRef:
+    path: Path
+    scorecard_sha256: str
+    run: TrackRunRef
+
+    def __post_init__(self) -> None:
+        object.__setattr__(self, "path", Path(self.path))
+        require_digest(self.scorecard_sha256, "scorecard_sha256")
+        _id(self.path.name, "evaluation_id")
+        if not isinstance(self.run, TrackRunRef):
+            raise ContractError("run must be TrackRunRef")
+        if self.path.parent.name != "evaluations" or self.path.parent.parent != self.run.path:
+            raise ContractError("evaluation must be retained inside its original run/evaluations directory")
+
+    @property
+    def evaluation_id(self) -> str:
+        return self.path.name