sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/evaluator_bindings.py

@@ -0,0 +1,370 @@
+"""Versioned evaluator requirements and explicit report-scoped assessments.
+
+Declarations grant no access. Trusted backends verify native records, authorize
+references and select exact stage evidence before creating grading contexts.
+"""
+from __future__ import annotations
+
+from collections.abc import Mapping
+from dataclasses import dataclass, field
+import math
+import re
+from typing import Any
+
+from .assessment_contracts import (
+    Assessment, EvaluatorContext, _ASSESSMENT_AVAILABILITY, _freeze_json,
+)
+from .evaluation_protocols import VersionedIdentity, _envelope
+from .stage_contracts import StageEvidence, StageSelector
+from .workflow_contracts import (
+    PayloadSchema, WorkflowContractError, _object, _optional_text, _text, _texts,
+    _thaw, contract_digest,
+)
+
+BINDING_SCHEMA = "sphynx.evaluator-binding.v1"
+REQUIREMENT_SCHEMA = "sphynx.evaluator-input.v1"
+REPORT_ASSESSMENT_SCHEMA = "sphynx.report-assessment.v1"
+REPORT_ASSESSMENTS_CAPABILITY = "sphynx.report-assessments.v1"
+_REPORT_CONTEXT_TOKEN = object()
+
+
+def _identity(value: Any, name: str) -> None:
+    if not isinstance(value, VersionedIdentity) or value.digest is None:
+        raise WorkflowContractError(name + " requires an exact version and digest")
+
+
+def _schema_identity(schema: PayloadSchema) -> VersionedIdentity:
+    return VersionedIdentity(schema.schema_id, schema.version, contract_digest(schema.to_dict()))
+
+
+@dataclass(frozen=True)
+class InputRequirement:
+    source: str
+    schema: VersionedIdentity
+    fields: tuple[str, ...] = ()
+    optional: bool = False
+
+    def __post_init__(self) -> None:
+        if self.source not in ("inputs", "outputs", "references"):
+            raise WorkflowContractError("requirement source must be inputs, outputs or references")
+        _identity(self.schema, "input schema")
+        object.__setattr__(self, "fields", _texts(self.fields, "required fields"))
+        if type(self.optional) is not bool:
+            raise WorkflowContractError("optional must be boolean")
+
+    @classmethod
+    def for_schema(cls, source: str, schema: PayloadSchema, *, fields=(), optional=False):
+        if not isinstance(schema, PayloadSchema):
+            raise WorkflowContractError("input requirement needs PayloadSchema")
+        return cls(source, _schema_identity(schema), fields, optional)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": REQUIREMENT_SCHEMA, "source": self.source,
+                "schema": self.schema.to_dict(), "fields": list(self.fields), "optional": self.optional}
+
+    @classmethod
+    def from_dict(cls, value):
+        values = _envelope(value, REQUIREMENT_SCHEMA, cls)
+        if "schema" in values:
+            values["schema"] = VersionedIdentity.from_dict(values["schema"])
+        try:
+            return cls(**values)
+        except TypeError as exc:
+            raise WorkflowContractError("invalid input requirement fields") from exc
+
+
+@dataclass(frozen=True)
+class EvaluatorBinding:
+    binding_id: str
+    version: str
+    evaluator: VersionedIdentity
+    target: str
+    requirements: tuple[InputRequirement, ...] = ()
+    stage_id: str | None = None
+    occurrence: int = 1
+    stage_schema: VersionedIdentity | None = None
+
+    def __post_init__(self) -> None:
+        _text(self.binding_id, "binding_id"); _text(self.version, "binding version")
+        _identity(self.evaluator, "evaluator")
+        if self.target not in ("case", "stage", "report"):
+            raise WorkflowContractError("binding target must be case, stage or report")
+        if not isinstance(self.requirements, (list, tuple)) or any(
+            not isinstance(item, InputRequirement) for item in self.requirements
+        ):
+            raise WorkflowContractError("requirements must contain InputRequirement values")
+        requirements = tuple(self.requirements)
+        if len({item.source for item in requirements}) != len(requirements):
+            raise WorkflowContractError("declare one requirement per input source")
+        if self.target == "report" and any(item.source == "inputs" for item in requirements):
+            raise WorkflowContractError("report targets have no case inputs")
+        object.__setattr__(self, "requirements", requirements)
+        if type(self.occurrence) is not int or self.occurrence < 1:
+            raise WorkflowContractError("stage occurrence must be a positive integer")
+        if self.target == "stage":
+            _text(self.stage_id, "stage_id"); _identity(self.stage_schema, "stage schema")
+        elif self.stage_id is not None or self.stage_schema is not None or self.occurrence != 1:
+            raise WorkflowContractError("only stage bindings accept stage selectors")
+
+    @classmethod
+    def for_evaluator(cls, binding_id: str, version: str, definition, *, target="case", **kwargs):
+        from .workflow_contracts import EvaluatorDefinition
+        if not isinstance(definition, EvaluatorDefinition):
+            raise WorkflowContractError("binding requires EvaluatorDefinition")
+        identity = VersionedIdentity(definition.evaluator_id, definition.version,
+                                     contract_digest(definition.to_dict()))
+        return cls(binding_id, version, identity, target, **kwargs)
+
+    @property
+    def binding_digest(self) -> str:
+        return contract_digest(self.to_dict())
+
+    def selector(self, context: EvaluatorContext) -> StageSelector:
+        if self.target != "stage" or not isinstance(context, EvaluatorContext):
+            raise WorkflowContractError("stage selector requires a case evaluator context")
+        return StageSelector(context.case.case_id, context.metadata["trial"],
+                             context.metadata["attempt"], self.stage_id, self.occurrence)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": BINDING_SCHEMA, "binding_id": self.binding_id, "version": self.version,
+                "evaluator": self.evaluator.to_dict(), "target": self.target,
+                "requirements": [item.to_dict() for item in self.requirements],
+                "stage_id": self.stage_id, "occurrence": self.occurrence,
+                "stage_schema": None if self.stage_schema is None else self.stage_schema.to_dict()}
+
+    @classmethod
+    def from_dict(cls, value):
+        values = _envelope(value, BINDING_SCHEMA, cls)
+        if "evaluator" in values:
+            values["evaluator"] = VersionedIdentity.from_dict(values["evaluator"])
+        if "requirements" in values:
+            raw = values["requirements"]
+            if not isinstance(raw, (list, tuple)):
+                raise WorkflowContractError("requirements must be a sequence")
+            values["requirements"] = tuple(InputRequirement.from_dict(item) for item in raw)
+        if values.get("stage_schema") is not None:
+            values["stage_schema"] = VersionedIdentity.from_dict(values["stage_schema"])
+        try:
+            return cls(**values)
+        except TypeError as exc:
+            raise WorkflowContractError("invalid evaluator binding fields") from exc
+
+
+@dataclass(frozen=True)
+class ReportAssessment:
+    """A report-scoped result. It has no case, trial or candidate-attempt ID."""
+
+    criterion_id: str
+    evaluator_id: str
+    run_id: str
+    grading_id: str
+    kind: str
+    availability: str
+    value: Any = None
+    reason: str | None = None
+    evidence_ids: tuple[str, ...] = ()
+    schema: PayloadSchema | None = field(default=None, repr=False, compare=False)
+
+    def __post_init__(self) -> None:
+        for name in ("criterion_id", "evaluator_id", "run_id", "grading_id", "kind"):
+            _text(getattr(self, name), name)
+        _optional_text(self.reason, "reason")
+        object.__setattr__(self, "evidence_ids", _texts(self.evidence_ids, "evidence_ids"))
+        if type(self.availability) is not str or self.availability not in _ASSESSMENT_AVAILABILITY:
+            raise WorkflowContractError("unsupported report assessment availability")
+        builtin = self.kind in ("score", "assertion", "label")
+        if builtin and self.schema is not None:
+            raise WorkflowContractError("builtin report assessments do not accept a custom schema")
+        if not builtin:
+            if not isinstance(self.schema, PayloadSchema) or (
+                self.kind != "structured" and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:/@+\-]*[.:][A-Za-z0-9_.:/@+\-]+", self.kind)
+            ):
+                raise WorkflowContractError("custom report assessment requires a schema and namespaced kind")
+        if self.availability != "available":
+            if self.value is not None or self.reason is None:
+                raise WorkflowContractError("unavailable report assessment requires reason and no value")
+        elif self.kind == "assertion":
+            if type(self.value) is not bool:
+                raise WorkflowContractError("available report assertion requires boolean")
+        elif self.kind == "score":
+            try:
+                finite = type(self.value) in (int, float) and math.isfinite(self.value)
+            except OverflowError:
+                finite = False
+            if not finite:
+                raise WorkflowContractError("available report score requires finite number")
+        elif self.kind == "label":
+            _text(self.value, "report label")
+        else:
+            object.__setattr__(self, "value", self.schema.validate(self.value))
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"schema_version": REPORT_ASSESSMENT_SCHEMA, "scope": "report",
+                "criterion_id": self.criterion_id, "evaluator_id": self.evaluator_id,
+                "run_id": self.run_id, "grading_id": self.grading_id,
+                "kind": self.kind, "availability": self.availability,
+                "value": _thaw(self.value), "reason": self.reason, "evidence_ids": list(self.evidence_ids),
+                "payload_schema": None if self.schema is None else {
+                    "schema_id": self.schema.schema_id, "version": self.schema.version,
+                    "implementation_id": self.schema.implementation_id}}
+
+
+@dataclass(frozen=True, init=False)
+class ReportEvaluatorContext:
+    """Grader-only report context, deliberately without a general serializer."""
+
+    outputs: Mapping[str, Any] = field(repr=False)
+    references: Mapping[str, Any] = field(repr=False)
+    run_id: str
+    grading_id: str
+    trust_tier: str
+    verified_by: str
+    metadata: Mapping[str, Any] = field(repr=False)
+    evidence_ids: tuple[str, ...]
+
+    def __init__(self, *, outputs, references, run_id, grading_id, trust_tier,
+                 verified_by, metadata=None, evidence_ids=(), _token=None):
+        if _token is not _REPORT_CONTEXT_TOKEN:
+            raise WorkflowContractError("report context must be built inside a verified backend")
+        for name, value in (("run_id", run_id), ("grading_id", grading_id),
+                            ("trust_tier", trust_tier), ("verified_by", verified_by)):
+            _text(value, name); object.__setattr__(self, name, value)
+        object.__setattr__(self, "outputs", _object(outputs, "report outputs"))
+        object.__setattr__(self, "references", _object(references, "report references"))
+        object.__setattr__(self, "metadata", _object({} if metadata is None else metadata, "report metadata"))
+        object.__setattr__(self, "evidence_ids", _texts(evidence_ids, "report evidence_ids"))
+
+
+def _verified_report_context(**kwargs) -> ReportEvaluatorContext:
+    # The trusted caller owns native verification and reference authorization.
+    return ReportEvaluatorContext(**kwargs, _token=_REPORT_CONTEXT_TOKEN)
+
+
+class EvaluatorInputUnavailable(WorkflowContractError):
+    """Required saved input is unavailable; this is not a quality score."""
+
+
+def _registered_schema(registry, identity: VersionedIdentity) -> PayloadSchema:
+    schema = registry.get_schema(identity.identity_id)
+    if _schema_identity(schema) != identity:
+        raise WorkflowContractError("evaluator input schema identity changed")
+    return schema
+
+
+def _binding_dependencies(registry, binding: EvaluatorBinding) -> dict[str, Any]:
+    if not isinstance(binding, EvaluatorBinding):
+        raise WorkflowContractError("grading binding must be EvaluatorBinding")
+    definition = registry.get_evaluator(binding.evaluator.identity_id)
+    current = VersionedIdentity(definition.evaluator_id, definition.version,
+                                contract_digest(definition.to_dict()))
+    if current != binding.evaluator:
+        raise WorkflowContractError("bound evaluator identity changed")
+    schemas = [_registered_schema(registry, requirement.schema).to_dict()
+               for requirement in binding.requirements]
+    if binding.stage_schema is not None:
+        schemas.append(_registered_schema(registry, binding.stage_schema).to_dict())
+    schemas.extend(registry.get_schema(identifier).to_dict() for identifier in definition.assessment_schema_ids)
+    return {"binding": binding.to_dict(), "evaluator": definition.to_dict(), "schemas": schemas}
+
+
+def _validate_custom_result(registry, definition, row) -> None:
+    identity = row.payload_schema if isinstance(row, Assessment) else (
+        None if row.schema is None else {"schema_id": row.schema.schema_id,
+                                         "version": row.schema.version,
+                                         "implementation_id": row.schema.implementation_id})
+    if identity is None:
+        return
+    if identity["schema_id"] not in definition.assessment_schema_ids:
+        raise WorkflowContractError("evaluator returned an undeclared custom assessment schema")
+    schema = registry.get_schema(identity["schema_id"])
+    if any(getattr(schema, key) != identity[key] for key in ("schema_id", "version", "implementation_id")):
+        raise WorkflowContractError("custom assessment schema identity changed")
+    if row.availability == "available":
+        schema.validate(_thaw(row.value))
+
+
+def _dispatch_binding(registry, binding: EvaluatorBinding, context, *, stage=None):
+    """Use only inside grading after native verification and authorization.
+
+    A stage reader must stamp the verified run/trial/attempt into EvidenceView
+    metadata. This checks joins; it does not authenticate an arbitrary caller.
+    """
+    _binding_dependencies(registry, binding)
+    report_target = binding.target == "report"
+    if not isinstance(context, ReportEvaluatorContext if report_target else EvaluatorContext):
+        raise WorkflowContractError("binding target and trusted context differ")
+    if binding.target == "stage":
+        from .assessment_contracts import _verified_evaluator_context
+        selector = binding.selector(context)
+        if not isinstance(stage, StageEvidence) or stage.selector != selector:
+            raise WorkflowContractError("stage evidence selector differs from grading selection")
+        identity = stage.evidence.metadata
+        if (identity.get("run_id"), identity.get("trial"), identity.get("attempt")) != (
+            context.run_id, selector.trial, selector.attempt
+        ):
+            raise WorkflowContractError("stage evidence run/trial/attempt binding differs")
+        # Native artifact schema and permitted projection schema are distinct.
+        if stage.availability == "available":
+            schema = _registered_schema(registry, binding.stage_schema)
+            if any(stage.projection_schema.get(key) != getattr(schema, key)
+                   for key in ("schema_id", "version", "implementation_id")):
+                raise WorkflowContractError("stage projection schema identity differs")
+            outputs = schema.validate(_thaw(stage.projection))
+            if not isinstance(outputs, Mapping):
+                raise WorkflowContractError("case evaluator context requires a mapping stage projection")
+        else:
+            outputs = {}
+        context = _verified_evaluator_context(
+            case=context.case, outputs=outputs, references=context.references,
+            evidence=(stage.evidence,), run_id=context.run_id, grading_id=context.grading_id,
+            trust_tier=context.trust_tier, verified_by=context.verified_by,
+            metadata={**_thaw(context.metadata), "stage_selector": selector.to_dict(),
+                      "binding_availability": {"outputs": stage.availability},
+                      "binding_missing_reason": stage.reason})
+    elif stage is not None:
+        raise WorkflowContractError("non-stage binding does not accept stage evidence")
+
+    missing = {}
+    for requirement in binding.requirements:
+        schema = _registered_schema(registry, requirement.schema)
+        value = context.case.inputs if requirement.source == "inputs" else getattr(context, requirement.source)
+        availability = context.metadata.get("binding_availability", {}).get(requirement.source, "available")
+        absent = [name for name in requirement.fields if name not in value]
+        if availability != "available" or absent:
+            if not requirement.optional:
+                raise EvaluatorInputUnavailable("required evaluator " + requirement.source + " unavailable")
+            missing[requirement.source] = "unavailable" if availability != "available" else "required_fields_absent"
+            continue
+        if requirement.source == "inputs" and context.case.schema_version != schema.schema_id:
+            raise WorkflowContractError("case input schema differs from evaluator requirement")
+        schema.validate(_thaw(value))
+    if missing:
+        if report_target:
+            context = _verified_report_context(outputs=context.outputs, references=context.references,
+                run_id=context.run_id, grading_id=context.grading_id, trust_tier=context.trust_tier,
+                verified_by=context.verified_by, evidence_ids=context.evidence_ids,
+                metadata={**_thaw(context.metadata), "binding_missing": missing})
+        else:
+            from .assessment_contracts import _verified_evaluator_context
+            context = _verified_evaluator_context(case=context.case, outputs=context.outputs,
+                references=context.references, evidence=context.evidence, run_id=context.run_id,
+                grading_id=context.grading_id, trust_tier=context.trust_tier, verified_by=context.verified_by,
+                metadata={**_thaw(context.metadata), "binding_missing": missing})
+    definition = registry.get_evaluator(binding.evaluator.identity_id)
+    rows = tuple(registry.get_evaluator_callable(definition.evaluator_id)(context))
+    evidence_ids = set(context.evidence_ids if report_target else (item.evidence_id for item in context.evidence))
+    for row in rows:
+        if not isinstance(row, ReportAssessment if report_target else Assessment):
+            raise WorkflowContractError("evaluator returned the wrong assessment scope")
+        if (row.evaluator_id, row.run_id, row.grading_id) != (
+            definition.evaluator_id, context.run_id, context.grading_id
+        ) or row.criterion_id not in definition.criterion_ids or not set(row.evidence_ids) <= evidence_ids:
+            raise WorkflowContractError("bound assessment identity or evidence differs")
+        if not report_target and (row.case_id, row.trial, row.attempt) != (
+            context.case.case_id, context.metadata["trial"], context.metadata["attempt"]
+        ):
+            raise WorkflowContractError("bound assessment case/trial/attempt differs")
+        if binding.target == "stage" and stage.evidence.evidence_id not in row.evidence_ids:
+            raise WorkflowContractError("stage assessment must retain its selected stage evidence ID")
+        _validate_custom_result(registry, definition, row)
+    return rows