sable-sphynx 0.7.0.dev0__py3-none-any.whl

sphynx/authoring.py

@@ -0,0 +1,188 @@
+"""Small trusted local callables through the existing lifecycle workflow.
+
+No provider SDK, network request, isolation, source attestation, statistical
+reducer or private acceptance rule is supplied by this convenience module.
+"""
+from __future__ import annotations
+
+from collections.abc import Callable, Mapping
+from dataclasses import dataclass, field, replace
+import inspect
+from typing import Any
+
+from .assessment_contracts import Assessment, CaseView, EvaluatorContext
+from .evaluation_protocols import VersionedIdentity
+from .execution_capabilities import ExecutionPath
+from .lifecycle import AttemptOutput, DriverDescriptor, LifecycleError, RunPolicy
+from .lifecycle_workflow import LifecycleBackend
+from .workflow_contracts import (
+    Criterion, Dataset, DatasetDefinition, EvaluatorDefinition, PayloadSchema,
+    SuiteDefinition, WorkflowContractError, _object, _text, _thaw, contract_digest,
+)
+from .workflow_registry import WorkflowRegistry
+from .workflows import Suite
+
+
+@dataclass(frozen=True)
+class CallableCandidate:
+    """A synchronous JSON-object callable with a declared, unattested identity."""
+
+    identity: VersionedIdentity
+    callback: Callable[[Mapping[str, Any]], Mapping[str, Any]] = field(repr=False, compare=False)
+
+    def __post_init__(self):
+        if not isinstance(self.identity, VersionedIdentity) or not callable(self.callback):
+            raise WorkflowContractError("CallableCandidate requires VersionedIdentity and a callable")
+        if inspect.iscoroutinefunction(self.callback) or inspect.iscoroutinefunction(getattr(self.callback, "__call__", None)):
+            raise WorkflowContractError("CallableCandidate is synchronous; use an explicit asynchronous driver")
+
+
+class _CallableDriver:
+    def __init__(self, output_schema, idempotent):
+        self.output_schema = output_schema
+        self.descriptor = DriverDescriptor("sphynx.authoring.callable.v1",
+            "sphynx.authoring.callable.v1:" + contract_digest(output_schema.to_dict()),
+            ("sphynx.lifecycle.idempotent.v1",) if idempotent else ())
+
+    def candidate_identity(self, candidate):
+        if not isinstance(candidate, CallableCandidate):
+            raise WorkflowContractError("this suite needs a CallableCandidate with an explicit method identity")
+        return {"identity": candidate.identity.to_dict(), "authority": "caller_declared_not_source_attested"}
+
+    def execute(self, candidate, case, context):
+        self.candidate_identity(candidate)
+        output = candidate.callback(_thaw(case.inputs))
+        return AttemptOutput(_thaw(self.output_schema.validate(output)))
+
+
+class _CallableBackend(LifecycleBackend):
+    def __init__(self, *args, idempotent=False, **kwargs):
+        self.idempotent = idempotent
+        super().__init__(*args, **kwargs)
+        path = ExecutionPath(self.descriptor.supported_scopes[0], "local",
+                             capabilities=self.descriptor.capabilities)
+        self.descriptor = replace(self.descriptor,
+            metadata={**_thaw(self.descriptor.metadata), "execution_paths": [path.to_dict()]})
+
+    def _options(self, plan):
+        run, grading = super()._options(plan)
+        if not self.idempotent and (run.trials > 1 or run.candidate_attempts > 1 or plan.parameters.get("resume") is not None):
+            raise LifecycleError("repeats, retries and resume require an explicit idempotent declaration or an advanced driver")
+        if run.reuse != "never" or plan.parameters.get("reuse") is not None:
+            raise LifecycleError("callable authoring does not implement verified reuse; use an explicit driver")
+        return run, grading
+
+
+def _json_object(value):
+    _object(value, "authoring JSON object")
+
+
+def _coverage(record):
+    rows = [row for row in record["rows"] if row["failure"] is None]
+    assessments = [item for row in rows for item in row["output"]["assessments"]]
+    planned = len(record["run"]["definition"]["cases"]) * record["run"]["definition"]["policy"]["trials"]
+    return {"planned_trials": planned, "observed_trials": len(rows), "missing_trials": planned - len(rows),
+            "passing_assertions": sum(item["value"] is True for item in assessments),
+            "failed_assertions": sum(item["value"] is False for item in assessments),
+            "candidate_attempts": len(record["run"]["attempts"])}
+
+
+def evaluation_suite(
+    suite_id: str, *, version: str, implementation_id: str,
+    cases: Mapping[str, Mapping[str, Any]], check: Callable[[EvaluatorContext], bool],
+    references: Mapping[str, Mapping[str, Any]] | None = None,
+    authorize_references: Callable[[str, Any], bool] | None = None, reference_access: Any = None,
+    input_schema: PayloadSchema | None = None, output_schema: PayloadSchema | None = None,
+    reference_schema: PayloadSchema | None = None, scope: str = "component",
+    run_policy: RunPolicy | None = None, idempotent: bool = False,
+) -> Suite:
+    """Build a versioned local assertion suite without hand-registering its parts.
+
+    Reference snapshots are caller-supplied private data. Dataset authorization
+    still precedes reference loading/validation into a grading context. All
+    functions run in the owner's Python process, so use an admitted isolated
+    driver for untrusted candidates or protected production references.
+    """
+    for name, value in (("suite_id", suite_id), ("version", version), ("implementation_id", implementation_id)):
+        _text(value, name)
+    if not callable(check) or inspect.iscoroutinefunction(check) or inspect.iscoroutinefunction(getattr(check, "__call__", None)):
+        raise WorkflowContractError("check must be a synchronous evaluator-context callable")
+    if scope not in ("component", "subsystem", "system") or type(idempotent) is not bool:
+        raise WorkflowContractError("declare component/subsystem/system scope and a boolean idempotent flag")
+    inputs = _object(cases, "authoring cases")
+    if not inputs:
+        raise WorkflowContractError("evaluation_suite requires at least one input case")
+    if references is None:
+        if authorize_references is not None or reference_schema is not None or reference_access is not None:
+            raise WorkflowContractError("reference settings require a reference mapping")
+        refs = None
+    else:
+        if not callable(authorize_references):
+            raise WorkflowContractError("references require explicit authorize_references; no default grant is supplied")
+        refs = _object(references, "private authoring references")
+        if any(not isinstance(value, Mapping) for value in refs.values()):
+            raise WorkflowContractError("each private reference must be a JSON object")
+        if set(refs) != set(inputs):
+            raise WorkflowContractError("reference membership must match the complete case set")
+    registry = WorkflowRegistry()
+    def schema(role, supplied):
+        result = supplied or PayloadSchema(suite_id + "." + role + ".v1", version,
+                                          "sphynx.authoring.json-object.v1", _json_object)
+        if not isinstance(result, PayloadSchema):
+            raise WorkflowContractError(role + " schema must be PayloadSchema")
+        return result
+    input_schema, output_schema = schema("input", input_schema), schema("output", output_schema)
+    reference_schema = None if refs is None else schema("reference", reference_schema)
+    projection_schema = schema("report", None)
+    schemas = {}
+    for value in (input_schema, output_schema, reference_schema, projection_schema):
+        if value is None:
+            continue
+        existing = schemas.get(value.schema_id)
+        if existing is not None and (existing.to_dict() != value.to_dict() or existing.validator is not value.validator):
+            raise WorkflowContractError("one schema ID cannot name different validators/versions")
+        if existing is None:
+            registry.register_schema(value); schemas[value.schema_id] = value
+    rows = tuple(CaseView.validated(identifier, payload, schema=input_schema) for identifier, payload in inputs.items())
+    cohort = contract_digest({"cases": [row.to_dict() for row in rows]})
+    dataset = DatasetDefinition(suite_id + ".dataset", version, suite_id + ".provider", input_schema.schema_id,
+        None if reference_schema is None else reference_schema.schema_id,
+        selection={"membership": "all_declared_cases", "order": [row.case_id for row in rows]},
+        identity={"input_digest": cohort, "authority": "caller_owned_local_inputs"})
+    registry.register_provider(Dataset(dataset, input_schema, lambda selection: iter(rows),
+        reference_schema=reference_schema,
+        reference_loader=None if refs is None else lambda identifier: _thaw(refs[identifier]),
+        authorize_references=authorize_references))
+    criterion_id, evaluator_id = suite_id + ".check", suite_id + ".evaluator"
+    registry.register_criterion(Criterion(criterion_id, version, "The declared check returns True"))
+    def evaluate(context):
+        if context.metadata.get("candidate_failure") is not None:
+            raise WorkflowContractError("cannot check a failed candidate attempt; inspect its execution evidence")
+        value = check(context)
+        if type(value) is not bool:
+            raise WorkflowContractError("authoring check must return exactly True or False; use an explicit evaluator for other outputs")
+        return (Assessment(criterion_id, evaluator_id, context.case.case_id, "assertion", "available", value,
+            evidence_ids=tuple(item.evidence_id for item in context.evidence), run_id=context.run_id,
+            grading_id=context.grading_id, trial=context.metadata["trial"], attempt=context.metadata["attempt"]),)
+    registry.register_evaluator(EvaluatorDefinition(evaluator_id, version, implementation_id,
+        criterion_ids=(criterion_id,)), evaluate)
+    registry.register_projection(projection_schema.schema_id, lambda record: {
+        "schema_version": "sphynx.authoring.report.v1", "suite_id": suite_id,
+        "run_id": record["definition"]["run_id"], "grading_id": record["definition"]["grading_id"],
+        "scope": scope, "mode": "local", "candidate": record["run"]["definition"]["candidate_identity"],
+        "counts": _coverage(record)}, version="v1", implementation_id="sphynx.authoring.coverage.v1")
+    comparison_id = suite_id + ".comparison"
+    registry.register_policy(comparison_id, version="v1", implementation_id="sphynx.authoring.no-comparison.v1",
+        kind="comparison", callback=lambda left, right: {
+            "status": "unavailable", "reason": "No custom comparison policy selected; use the explicit registry path.",
+            "baseline_grading_id": left["definition"]["grading_id"], "candidate_grading_id": right["definition"]["grading_id"]})
+    definition = SuiteDefinition(suite_id, version, suite_id + ".backend", suite_id,
+        configuration={"input_digest": cohort, "output_schema": output_schema.to_dict(),
+                       "idempotent": idempotent, "scope": scope, "authority": "local_unverified"},
+        criterion_ids=(criterion_id,), evaluator_ids=(evaluator_id,), datasets=(dataset,))
+    backend = _CallableBackend(registry, definition, driver=_CallableDriver(output_schema, idempotent),
+        evaluator_id=evaluator_id, projection_schema_id=projection_schema.schema_id,
+        comparison_policy_id=comparison_id, run_policy=run_policy, reference_access=reference_access,
+        scopes=(scope,), idempotent=idempotent)
+    registry.register_backend(backend); registry.register_suite(definition)
+    return Suite.from_registry(suite_id, registry=registry)