sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
|
@@ -0,0 +1,594 @@
|
|
|
1
|
+
"""Domain-neutral contracts for suite backends and their validated payloads.
|
|
2
|
+
|
|
3
|
+
These contracts describe extension boundaries, not an execution engine. A
|
|
4
|
+
backend owns native records, payload validation, reference authorization, and
|
|
5
|
+
comparison policy. Immutable metadata and implementation labels do not verify
|
|
6
|
+
source bytes, authenticate evidence, or provide process isolation.
|
|
7
|
+
"""
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import hashlib
|
|
11
|
+
import json
|
|
12
|
+
import re
|
|
13
|
+
from dataclasses import dataclass, field
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any, Callable, Iterable, Mapping, Protocol, Sequence, TYPE_CHECKING, runtime_checkable
|
|
16
|
+
|
|
17
|
+
from .assessment_contracts import (
|
|
18
|
+
Assessment, AssessmentContractError, CaseView, _freeze_json, _thaw,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
if TYPE_CHECKING:
|
|
22
|
+
from .diagnostics import Diagnostic
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
_DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
class WorkflowContractError(ValueError):
|
|
29
|
+
"""An extension descriptor or workflow binding is malformed."""
|
|
30
|
+
|
|
31
|
+
@property
|
|
32
|
+
def diagnostic(self) -> Diagnostic:
|
|
33
|
+
from .diagnostics import explain_error
|
|
34
|
+
return explain_error(self)
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _text(value: Any, name: str) -> str:
|
|
38
|
+
if type(value) is not str or not value.strip() or "\x00" in value:
|
|
39
|
+
raise WorkflowContractError(f"{name} must be nonempty text without NUL bytes")
|
|
40
|
+
try:
|
|
41
|
+
value.encode("utf-8", errors="strict")
|
|
42
|
+
except UnicodeEncodeError:
|
|
43
|
+
raise WorkflowContractError(f"{name} must contain valid Unicode") from None
|
|
44
|
+
return value
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _optional_text(value: Any, name: str) -> None:
|
|
48
|
+
if value is not None:
|
|
49
|
+
_text(value, name)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _object(value: Any, name: str) -> Mapping[str, Any]:
|
|
53
|
+
if not isinstance(value, Mapping):
|
|
54
|
+
raise WorkflowContractError(f"{name} must be a JSON object")
|
|
55
|
+
try:
|
|
56
|
+
return _freeze_json(value, name)
|
|
57
|
+
except AssessmentContractError as exc:
|
|
58
|
+
raise WorkflowContractError(str(exc)) from None
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def _texts(value: Sequence[str], name: str, *, namespaced: bool = False) -> tuple[str, ...]:
|
|
62
|
+
if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
|
|
63
|
+
raise WorkflowContractError(f"{name} must be a sequence of identifiers")
|
|
64
|
+
items = tuple(_text(item, name) for item in value)
|
|
65
|
+
if len(set(items)) != len(items):
|
|
66
|
+
raise WorkflowContractError(f"{name} must not contain duplicates")
|
|
67
|
+
if namespaced and any("." not in item and ":" not in item for item in items):
|
|
68
|
+
raise WorkflowContractError(f"{name} must use namespaced identifiers")
|
|
69
|
+
return items
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _canonical_json(value: Any) -> str:
|
|
73
|
+
try:
|
|
74
|
+
return json.dumps(_thaw(value), sort_keys=True, separators=(",", ":"),
|
|
75
|
+
ensure_ascii=False, allow_nan=False)
|
|
76
|
+
except (TypeError, ValueError) as exc:
|
|
77
|
+
raise WorkflowContractError(f"value cannot be serialized as canonical JSON: {exc}") from None
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def contract_digest(value: Mapping[str, Any]) -> str:
|
|
81
|
+
"""Hash canonical JSON metadata; this is identity, not authentication."""
|
|
82
|
+
raw = _canonical_json(_object(value, "digest input")).encode("utf-8")
|
|
83
|
+
return "sha256:" + hashlib.sha256(raw).hexdigest()
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True)
|
|
87
|
+
class PayloadSchema:
|
|
88
|
+
"""An exact payload contract backed by an operator-supplied validator.
|
|
89
|
+
|
|
90
|
+
The validator receives a detached JSON value, returns None on success, and
|
|
91
|
+
raises on invalid content. It must not mutate the value. Adapters wrap typed
|
|
92
|
+
validators that return parsed objects. The implementation label is an exact
|
|
93
|
+
operator declaration, not a derived source digest. Registration does not run
|
|
94
|
+
this callback; explicit payload validation does.
|
|
95
|
+
"""
|
|
96
|
+
|
|
97
|
+
schema_id: str
|
|
98
|
+
version: str
|
|
99
|
+
implementation_id: str
|
|
100
|
+
validator: Callable[[Any], None] = field(repr=False, compare=False)
|
|
101
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
102
|
+
|
|
103
|
+
def __post_init__(self) -> None:
|
|
104
|
+
for name in ("schema_id", "version", "implementation_id"):
|
|
105
|
+
_text(getattr(self, name), name)
|
|
106
|
+
if not callable(self.validator):
|
|
107
|
+
raise WorkflowContractError("payload validator must be callable")
|
|
108
|
+
object.__setattr__(self, "metadata", _object(self.metadata, "metadata"))
|
|
109
|
+
|
|
110
|
+
def validate(self, value: Any) -> Any:
|
|
111
|
+
"""Validate content and return a detached immutable JSON snapshot."""
|
|
112
|
+
try:
|
|
113
|
+
frozen = _freeze_json(value, "payload")
|
|
114
|
+
except AssessmentContractError as exc:
|
|
115
|
+
raise WorkflowContractError(str(exc)) from None
|
|
116
|
+
original_json = _canonical_json(frozen)
|
|
117
|
+
detached = _thaw(frozen)
|
|
118
|
+
result = self.validator(detached)
|
|
119
|
+
if result is not None:
|
|
120
|
+
raise WorkflowContractError("payload validator must return None on success")
|
|
121
|
+
try:
|
|
122
|
+
unchanged = _canonical_json(_freeze_json(detached, "validated payload")) == original_json
|
|
123
|
+
except (AssessmentContractError, WorkflowContractError):
|
|
124
|
+
unchanged = False
|
|
125
|
+
if not unchanged:
|
|
126
|
+
raise WorkflowContractError("payload validator must not mutate content")
|
|
127
|
+
return frozen
|
|
128
|
+
|
|
129
|
+
def to_dict(self) -> dict[str, Any]:
|
|
130
|
+
return {"schema_id": self.schema_id, "version": self.version,
|
|
131
|
+
"implementation_id": self.implementation_id, "metadata": _thaw(self.metadata)}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
@dataclass(frozen=True)
|
|
135
|
+
class DatasetDefinition:
|
|
136
|
+
"""Case-provider metadata without reading or materializing its cases.
|
|
137
|
+
|
|
138
|
+
Selection, split, ordering, cohort, and grouping rules are provider-owned.
|
|
139
|
+
Input and reference schema identities stay separate. Providers may iterate
|
|
140
|
+
lazily. None of this metadata grants access to references or external assets.
|
|
141
|
+
"""
|
|
142
|
+
|
|
143
|
+
dataset_id: str
|
|
144
|
+
version: str
|
|
145
|
+
provider_id: str
|
|
146
|
+
input_schema_id: str
|
|
147
|
+
reference_schema_id: str | None = None
|
|
148
|
+
selection: Mapping[str, Any] = field(default_factory=dict)
|
|
149
|
+
identity: Mapping[str, Any] = field(default_factory=dict)
|
|
150
|
+
|
|
151
|
+
def __post_init__(self) -> None:
|
|
152
|
+
for name in ("dataset_id", "version", "provider_id", "input_schema_id"):
|
|
153
|
+
_text(getattr(self, name), name)
|
|
154
|
+
_optional_text(self.reference_schema_id, "reference_schema_id")
|
|
155
|
+
for name in ("selection", "identity"):
|
|
156
|
+
object.__setattr__(self, name, _object(getattr(self, name), name))
|
|
157
|
+
|
|
158
|
+
def to_dict(self) -> dict[str, Any]:
|
|
159
|
+
return {"dataset_id": self.dataset_id, "version": self.version,
|
|
160
|
+
"provider_id": self.provider_id, "input_schema_id": self.input_schema_id,
|
|
161
|
+
"reference_schema_id": self.reference_schema_id,
|
|
162
|
+
"selection": _thaw(self.selection), "identity": _thaw(self.identity)}
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
@runtime_checkable
|
|
166
|
+
class CaseProvider(Protocol):
|
|
167
|
+
"""Input iteration and grader-only reference lookup are different calls.
|
|
168
|
+
|
|
169
|
+
``access`` has backend-defined authorization semantics. The provider must
|
|
170
|
+
validate it; possession of a Python object is not a grant. A candidate must
|
|
171
|
+
receive only its allowlisted CaseView, not the provider or its references.
|
|
172
|
+
"""
|
|
173
|
+
|
|
174
|
+
@property
|
|
175
|
+
def definition(self) -> DatasetDefinition: ...
|
|
176
|
+
|
|
177
|
+
def cases(self, selection: Mapping[str, Any] | None = None) -> Iterable[CaseView]: ...
|
|
178
|
+
|
|
179
|
+
def reference_for(self, case_id: str, *, access: Any) -> Mapping[str, Any]: ...
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass(frozen=True)
|
|
183
|
+
class Dataset:
|
|
184
|
+
"""A schema-bound case provider with lazy, candidate-permitted inputs.
|
|
185
|
+
|
|
186
|
+
``case_source`` is an iterable or a callable receiving the selected immutable
|
|
187
|
+
selection mapping. Use a callable to obtain a fresh iterator for each pass.
|
|
188
|
+
Construction and descriptor inspection do not load cases or references.
|
|
189
|
+
|
|
190
|
+
An optional selection schema validates nondefault selections. Without it,
|
|
191
|
+
only the definition's exact selection is accepted. A backend must bind the
|
|
192
|
+
selected cohort's native identity before execution and comparison.
|
|
193
|
+
|
|
194
|
+
References have a separate schema and loader. The explicit authorization
|
|
195
|
+
callback must return True before that loader is accessed. The callback and
|
|
196
|
+
backend own principal, case membership, and access-policy validation. No
|
|
197
|
+
default object, token shape, or Python type constitutes a reference grant.
|
|
198
|
+
Do not pass this provider to candidate execution; pass its CaseViews only.
|
|
199
|
+
"""
|
|
200
|
+
|
|
201
|
+
definition: DatasetDefinition
|
|
202
|
+
input_schema: PayloadSchema
|
|
203
|
+
case_source: Iterable[CaseView] | Callable[[Mapping[str, Any]], Iterable[CaseView]] = field(
|
|
204
|
+
repr=False, compare=False)
|
|
205
|
+
reference_schema: PayloadSchema | None = field(default=None, repr=False)
|
|
206
|
+
reference_loader: Callable[[str], Mapping[str, Any]] | None = field(
|
|
207
|
+
default=None, repr=False, compare=False)
|
|
208
|
+
authorize_references: Callable[[str, Any], bool] | None = field(
|
|
209
|
+
default=None, repr=False, compare=False)
|
|
210
|
+
selection_schema: PayloadSchema | None = field(default=None, repr=False)
|
|
211
|
+
|
|
212
|
+
def __post_init__(self) -> None:
|
|
213
|
+
if not isinstance(self.definition, DatasetDefinition):
|
|
214
|
+
raise WorkflowContractError("dataset definition must be DatasetDefinition")
|
|
215
|
+
if not isinstance(self.input_schema, PayloadSchema):
|
|
216
|
+
raise WorkflowContractError("dataset input schema must be PayloadSchema")
|
|
217
|
+
if self.input_schema.schema_id != self.definition.input_schema_id:
|
|
218
|
+
raise WorkflowContractError("input schema differs from the dataset definition")
|
|
219
|
+
if not callable(self.case_source) and not isinstance(self.case_source, Iterable):
|
|
220
|
+
raise WorkflowContractError("case source must be an iterable or callable")
|
|
221
|
+
if self.reference_schema is not None:
|
|
222
|
+
if not isinstance(self.reference_schema, PayloadSchema):
|
|
223
|
+
raise WorkflowContractError("reference schema must be PayloadSchema")
|
|
224
|
+
if self.reference_schema.schema_id != self.definition.reference_schema_id:
|
|
225
|
+
raise WorkflowContractError("reference schema differs from the dataset definition")
|
|
226
|
+
if self.reference_loader is not None:
|
|
227
|
+
if not callable(self.reference_loader) or self.reference_schema is None:
|
|
228
|
+
raise WorkflowContractError("reference loader requires a matching payload schema")
|
|
229
|
+
if self.authorize_references is not None and not callable(self.authorize_references):
|
|
230
|
+
raise WorkflowContractError("reference authorization must be callable")
|
|
231
|
+
if self.selection_schema is not None and not isinstance(self.selection_schema, PayloadSchema):
|
|
232
|
+
raise WorkflowContractError("selection schema must be PayloadSchema")
|
|
233
|
+
|
|
234
|
+
def cases(self, selection: Mapping[str, Any] | None = None) -> Iterable[CaseView]:
|
|
235
|
+
"""Validate and yield cases only when the returned iterator is consumed."""
|
|
236
|
+
selected = _object(self.definition.selection if selection is None else selection, "selection")
|
|
237
|
+
if self.selection_schema is not None:
|
|
238
|
+
selected = self.selection_schema.validate(selected)
|
|
239
|
+
if not isinstance(selected, Mapping):
|
|
240
|
+
raise WorkflowContractError("selection schema must validate an object")
|
|
241
|
+
elif _canonical_json(selected) != _canonical_json(self.definition.selection):
|
|
242
|
+
raise WorkflowContractError("nondefault selection requires a selection schema")
|
|
243
|
+
source = self.case_source(selected) if callable(self.case_source) else self.case_source
|
|
244
|
+
try:
|
|
245
|
+
iterator = iter(source)
|
|
246
|
+
except TypeError:
|
|
247
|
+
raise WorkflowContractError("case source must produce an iterable") from None
|
|
248
|
+
seen: set[str] = set()
|
|
249
|
+
for case in iterator:
|
|
250
|
+
if not isinstance(case, CaseView):
|
|
251
|
+
raise WorkflowContractError("case source must yield CaseView values")
|
|
252
|
+
if case.schema_version != self.definition.input_schema_id:
|
|
253
|
+
raise WorkflowContractError("case input schema differs from the dataset definition")
|
|
254
|
+
if case.case_id in seen:
|
|
255
|
+
raise WorkflowContractError("case source contains duplicate case identities")
|
|
256
|
+
seen.add(case.case_id)
|
|
257
|
+
yield CaseView.validated(case.case_id, case.inputs, schema=self.input_schema,
|
|
258
|
+
metadata=case.metadata)
|
|
259
|
+
|
|
260
|
+
def reference_for(self, case_id: str, *, access: Any) -> Mapping[str, Any]:
|
|
261
|
+
"""Authorize first, then load and validate one grader-only reference."""
|
|
262
|
+
_text(case_id, "case_id")
|
|
263
|
+
if self.authorize_references is None:
|
|
264
|
+
raise PermissionError("reference access is not authorized")
|
|
265
|
+
try:
|
|
266
|
+
allowed = self.authorize_references(case_id, access)
|
|
267
|
+
except Exception:
|
|
268
|
+
raise PermissionError("reference access is not authorized") from None
|
|
269
|
+
if allowed is not True:
|
|
270
|
+
raise PermissionError("reference access is not authorized")
|
|
271
|
+
if self.reference_loader is None or self.reference_schema is None:
|
|
272
|
+
raise WorkflowContractError("dataset does not provide reference loading")
|
|
273
|
+
value = self.reference_schema.validate(self.reference_loader(case_id))
|
|
274
|
+
if not isinstance(value, Mapping):
|
|
275
|
+
raise WorkflowContractError("reference schema must validate an object")
|
|
276
|
+
return value
|
|
277
|
+
|
|
278
|
+
def to_dict(self) -> dict[str, Any]:
|
|
279
|
+
"""Return metadata only; omit loaders, authorization, and reference data."""
|
|
280
|
+
return self.definition.to_dict()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
@dataclass(frozen=True)
|
|
284
|
+
class Criterion:
|
|
285
|
+
"""A versioned requirement or dimension, independent of numeric metrics."""
|
|
286
|
+
|
|
287
|
+
criterion_id: str
|
|
288
|
+
version: str
|
|
289
|
+
title: str
|
|
290
|
+
configuration: Mapping[str, Any] = field(default_factory=dict)
|
|
291
|
+
|
|
292
|
+
def __post_init__(self) -> None:
|
|
293
|
+
for name in ("criterion_id", "version", "title"):
|
|
294
|
+
_text(getattr(self, name), name)
|
|
295
|
+
object.__setattr__(self, "configuration", _object(self.configuration, "configuration"))
|
|
296
|
+
|
|
297
|
+
def to_dict(self) -> dict[str, Any]:
|
|
298
|
+
return {"criterion_id": self.criterion_id, "version": self.version,
|
|
299
|
+
"title": self.title, "configuration": _thaw(self.configuration)}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
@dataclass(frozen=True)
|
|
303
|
+
class EvaluatorDefinition:
|
|
304
|
+
"""An evaluator may address several criteria and emit several outputs."""
|
|
305
|
+
|
|
306
|
+
evaluator_id: str
|
|
307
|
+
version: str
|
|
308
|
+
implementation_id: str
|
|
309
|
+
criterion_ids: tuple[str, ...] = ()
|
|
310
|
+
assessment_schema_ids: tuple[str, ...] = ()
|
|
311
|
+
metric_ids: tuple[str, ...] = ()
|
|
312
|
+
required_capabilities: tuple[str, ...] = ()
|
|
313
|
+
configuration: Mapping[str, Any] = field(default_factory=dict)
|
|
314
|
+
|
|
315
|
+
def __post_init__(self) -> None:
|
|
316
|
+
for name in ("evaluator_id", "version", "implementation_id"):
|
|
317
|
+
_text(getattr(self, name), name)
|
|
318
|
+
for name in ("criterion_ids", "assessment_schema_ids", "metric_ids", "required_capabilities"):
|
|
319
|
+
object.__setattr__(self, name, _texts(getattr(self, name), name,
|
|
320
|
+
namespaced=name == "required_capabilities"))
|
|
321
|
+
object.__setattr__(self, "configuration", _object(self.configuration, "configuration"))
|
|
322
|
+
|
|
323
|
+
def to_dict(self) -> dict[str, Any]:
|
|
324
|
+
return {"evaluator_id": self.evaluator_id, "version": self.version,
|
|
325
|
+
"implementation_id": self.implementation_id, "criterion_ids": list(self.criterion_ids),
|
|
326
|
+
"assessment_schema_ids": list(self.assessment_schema_ids), "metric_ids": list(self.metric_ids),
|
|
327
|
+
"required_capabilities": list(self.required_capabilities),
|
|
328
|
+
"configuration": _thaw(self.configuration)}
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
@dataclass(frozen=True)
|
|
332
|
+
class MetricSpec:
|
|
333
|
+
"""Numeric meaning and optional domain policies, without a default reducer.
|
|
334
|
+
|
|
335
|
+
Direction and population semantics belong to the registered domain. Policy
|
|
336
|
+
IDs identify declared implementations; no policy is executed by this type.
|
|
337
|
+
A measurement need not be an assessment of a quality criterion.
|
|
338
|
+
"""
|
|
339
|
+
|
|
340
|
+
metric_id: str
|
|
341
|
+
version: str
|
|
342
|
+
unit: str
|
|
343
|
+
direction: str
|
|
344
|
+
aggregation_policy: str | None = None
|
|
345
|
+
comparison_policy: str | None = None
|
|
346
|
+
population: Mapping[str, Any] = field(default_factory=dict)
|
|
347
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
348
|
+
|
|
349
|
+
def __post_init__(self) -> None:
|
|
350
|
+
for name in ("metric_id", "version", "unit", "direction"):
|
|
351
|
+
_text(getattr(self, name), name)
|
|
352
|
+
for name in ("aggregation_policy", "comparison_policy"):
|
|
353
|
+
_optional_text(getattr(self, name), name)
|
|
354
|
+
for name in ("population", "metadata"):
|
|
355
|
+
object.__setattr__(self, name, _object(getattr(self, name), name))
|
|
356
|
+
|
|
357
|
+
def to_dict(self) -> dict[str, Any]:
|
|
358
|
+
return {"metric_id": self.metric_id, "version": self.version, "unit": self.unit,
|
|
359
|
+
"direction": self.direction, "aggregation_policy": self.aggregation_policy,
|
|
360
|
+
"comparison_policy": self.comparison_policy,
|
|
361
|
+
"population": _thaw(self.population), "metadata": _thaw(self.metadata)}
|
|
362
|
+
|
|
363
|
+
|
|
364
|
+
@dataclass(frozen=True)
|
|
365
|
+
class BackendDescriptor:
|
|
366
|
+
"""Exact backend identity and open, namespaced capability declarations."""
|
|
367
|
+
|
|
368
|
+
backend_id: str
|
|
369
|
+
version: str
|
|
370
|
+
implementation_id: str
|
|
371
|
+
capabilities: tuple[str, ...]
|
|
372
|
+
supported_scopes: tuple[str, ...]
|
|
373
|
+
supported_modes: tuple[str, ...]
|
|
374
|
+
suite_ids: tuple[str, ...] = ()
|
|
375
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
376
|
+
|
|
377
|
+
def __post_init__(self) -> None:
|
|
378
|
+
for name in ("backend_id", "version", "implementation_id"):
|
|
379
|
+
_text(getattr(self, name), name)
|
|
380
|
+
for name in ("capabilities", "supported_scopes", "supported_modes", "suite_ids"):
|
|
381
|
+
object.__setattr__(self, name, _texts(getattr(self, name), name,
|
|
382
|
+
namespaced=name == "capabilities"))
|
|
383
|
+
if not self.supported_scopes or not self.supported_modes:
|
|
384
|
+
raise WorkflowContractError("backend must declare supported scopes and modes")
|
|
385
|
+
object.__setattr__(self, "metadata", _object(self.metadata, "metadata"))
|
|
386
|
+
|
|
387
|
+
@property
|
|
388
|
+
def descriptor_digest(self) -> str:
|
|
389
|
+
return contract_digest(self.to_dict())
|
|
390
|
+
|
|
391
|
+
def to_dict(self) -> dict[str, Any]:
|
|
392
|
+
return {"backend_id": self.backend_id, "version": self.version,
|
|
393
|
+
"implementation_id": self.implementation_id, "capabilities": list(self.capabilities),
|
|
394
|
+
"supported_scopes": list(self.supported_scopes), "supported_modes": list(self.supported_modes),
|
|
395
|
+
"suite_ids": list(self.suite_ids), "metadata": _thaw(self.metadata)}
|
|
396
|
+
|
|
397
|
+
|
|
398
|
+
@dataclass(frozen=True)
|
|
399
|
+
class SuiteDefinition:
|
|
400
|
+
"""Reusable suite metadata; native assets and records remain backend-owned."""
|
|
401
|
+
|
|
402
|
+
suite_id: str
|
|
403
|
+
version: str
|
|
404
|
+
backend_id: str
|
|
405
|
+
title: str
|
|
406
|
+
configuration: Mapping[str, Any] = field(default_factory=dict)
|
|
407
|
+
criterion_ids: tuple[str, ...] = ()
|
|
408
|
+
evaluator_ids: tuple[str, ...] = ()
|
|
409
|
+
metric_ids: tuple[str, ...] = ()
|
|
410
|
+
required_capabilities: tuple[str, ...] = ()
|
|
411
|
+
datasets: tuple[DatasetDefinition, ...] = ()
|
|
412
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
413
|
+
|
|
414
|
+
def __post_init__(self) -> None:
|
|
415
|
+
for name in ("suite_id", "version", "backend_id", "title"):
|
|
416
|
+
_text(getattr(self, name), name)
|
|
417
|
+
for name in ("criterion_ids", "evaluator_ids", "metric_ids", "required_capabilities"):
|
|
418
|
+
object.__setattr__(self, name, _texts(getattr(self, name), name,
|
|
419
|
+
namespaced=name == "required_capabilities"))
|
|
420
|
+
if isinstance(self.datasets, (str, bytes)) or not isinstance(self.datasets, Sequence):
|
|
421
|
+
raise WorkflowContractError("datasets must be a sequence of DatasetDefinition values")
|
|
422
|
+
datasets = tuple(self.datasets)
|
|
423
|
+
if any(not isinstance(item, DatasetDefinition) for item in datasets):
|
|
424
|
+
raise WorkflowContractError("datasets must contain DatasetDefinition values")
|
|
425
|
+
if len({(item.dataset_id, item.version) for item in datasets}) != len(datasets):
|
|
426
|
+
raise WorkflowContractError("suite dataset identities must be unique")
|
|
427
|
+
object.__setattr__(self, "datasets", datasets)
|
|
428
|
+
for name in ("configuration", "metadata"):
|
|
429
|
+
object.__setattr__(self, name, _object(getattr(self, name), name))
|
|
430
|
+
|
|
431
|
+
@property
|
|
432
|
+
def definition_digest(self) -> str:
|
|
433
|
+
return contract_digest(self.to_dict())
|
|
434
|
+
|
|
435
|
+
def to_dict(self) -> dict[str, Any]:
|
|
436
|
+
return {"suite_id": self.suite_id, "version": self.version, "backend_id": self.backend_id,
|
|
437
|
+
"title": self.title, "configuration": _thaw(self.configuration),
|
|
438
|
+
"criterion_ids": list(self.criterion_ids), "evaluator_ids": list(self.evaluator_ids),
|
|
439
|
+
"metric_ids": list(self.metric_ids), "required_capabilities": list(self.required_capabilities),
|
|
440
|
+
"datasets": [item.to_dict() for item in self.datasets], "metadata": _thaw(self.metadata)}
|
|
441
|
+
|
|
442
|
+
|
|
443
|
+
@dataclass(frozen=True)
|
|
444
|
+
class PlanBinding:
|
|
445
|
+
"""Selected settings plus a backend-owned candidate object.
|
|
446
|
+
|
|
447
|
+
The backend must validate and bind the candidate's implementation identity
|
|
448
|
+
during resolution and recheck required facts at admission/launch. Freezing
|
|
449
|
+
this envelope cannot freeze an arbitrary service, function, or native object.
|
|
450
|
+
"""
|
|
451
|
+
|
|
452
|
+
suite: SuiteDefinition
|
|
453
|
+
candidate: Any = field(repr=False, compare=False)
|
|
454
|
+
scope: str = "component"
|
|
455
|
+
mode: str = "local"
|
|
456
|
+
parameters: Mapping[str, Any] = field(default_factory=dict)
|
|
457
|
+
runtime: Mapping[str, Any] = field(default_factory=dict)
|
|
458
|
+
|
|
459
|
+
def __post_init__(self) -> None:
|
|
460
|
+
if not isinstance(self.suite, SuiteDefinition):
|
|
461
|
+
raise WorkflowContractError("plan suite must be SuiteDefinition")
|
|
462
|
+
for name in ("scope", "mode"):
|
|
463
|
+
_text(getattr(self, name), name)
|
|
464
|
+
for name in ("parameters", "runtime"):
|
|
465
|
+
object.__setattr__(self, name, _object(getattr(self, name), name))
|
|
466
|
+
|
|
467
|
+
|
|
468
|
+
@dataclass(frozen=True)
|
|
469
|
+
class ResolvedBinding:
|
|
470
|
+
"""Metadata selected by a backend; resolution does not verify asset bytes."""
|
|
471
|
+
|
|
472
|
+
identity: Mapping[str, Any]
|
|
473
|
+
native_ref: Any = field(default=None, repr=False, compare=False)
|
|
474
|
+
|
|
475
|
+
def __post_init__(self) -> None:
|
|
476
|
+
object.__setattr__(self, "identity", _object(self.identity, "identity"))
|
|
477
|
+
if not self.identity:
|
|
478
|
+
raise WorkflowContractError("resolution requires explicit identity metadata")
|
|
479
|
+
|
|
480
|
+
@property
|
|
481
|
+
def status(self) -> str:
|
|
482
|
+
return "resolved"
|
|
483
|
+
|
|
484
|
+
@property
|
|
485
|
+
def artifact_bytes_verified(self) -> bool:
|
|
486
|
+
return False
|
|
487
|
+
|
|
488
|
+
@property
|
|
489
|
+
def resolution_digest(self) -> str:
|
|
490
|
+
return contract_digest(self.to_dict())
|
|
491
|
+
|
|
492
|
+
def to_dict(self) -> dict[str, Any]:
|
|
493
|
+
return {"status": self.status, "identity": _thaw(self.identity), "artifact_bytes_verified": False}
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
@dataclass(frozen=True)
|
|
497
|
+
class AdmissionBinding:
|
|
498
|
+
"""Backend-verified admission facts without upgrading native authority."""
|
|
499
|
+
|
|
500
|
+
resolution_digest: str
|
|
501
|
+
facts: Mapping[str, Any]
|
|
502
|
+
trust_tier: str
|
|
503
|
+
artifact_bytes_verified: bool
|
|
504
|
+
candidate_source_bytes_verified: bool = False
|
|
505
|
+
native_ref: Any = field(default=None, repr=False, compare=False)
|
|
506
|
+
|
|
507
|
+
def __post_init__(self) -> None:
|
|
508
|
+
if type(self.resolution_digest) is not str or not _DIGEST.fullmatch(self.resolution_digest):
|
|
509
|
+
raise WorkflowContractError("admission requires a canonical resolution digest")
|
|
510
|
+
_text(self.trust_tier, "trust_tier")
|
|
511
|
+
for name in ("artifact_bytes_verified", "candidate_source_bytes_verified"):
|
|
512
|
+
if type(getattr(self, name)) is not bool:
|
|
513
|
+
raise WorkflowContractError(f"{name} must be boolean")
|
|
514
|
+
object.__setattr__(self, "facts", _object(self.facts, "facts"))
|
|
515
|
+
|
|
516
|
+
@property
|
|
517
|
+
def status(self) -> str:
|
|
518
|
+
return "admitted"
|
|
519
|
+
|
|
520
|
+
def to_dict(self) -> dict[str, Any]:
|
|
521
|
+
return {"status": self.status, "resolution_digest": self.resolution_digest,
|
|
522
|
+
"facts": _thaw(self.facts), "trust_tier": self.trust_tier,
|
|
523
|
+
"artifact_bytes_verified": self.artifact_bytes_verified,
|
|
524
|
+
"candidate_source_bytes_verified": self.candidate_source_bytes_verified}
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
@dataclass(frozen=True)
|
|
528
|
+
class ReferenceBinding:
|
|
529
|
+
"""Native record status and identity, separate from evidence verification."""
|
|
530
|
+
|
|
531
|
+
record_id: str | None
|
|
532
|
+
status: str
|
|
533
|
+
trust_tier: str
|
|
534
|
+
run_id: str | None = None
|
|
535
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
536
|
+
|
|
537
|
+
def __post_init__(self) -> None:
|
|
538
|
+
for name in ("record_id", "run_id"):
|
|
539
|
+
_optional_text(getattr(self, name), name)
|
|
540
|
+
for name in ("status", "trust_tier"):
|
|
541
|
+
_text(getattr(self, name), name)
|
|
542
|
+
object.__setattr__(self, "metadata", _object(self.metadata, "metadata"))
|
|
543
|
+
|
|
544
|
+
|
|
545
|
+
@runtime_checkable
|
|
546
|
+
class ReportProjection(Protocol):
|
|
547
|
+
"""Backend-verified permitted presentation; no universal metric layout."""
|
|
548
|
+
|
|
549
|
+
def to_dict(self) -> dict[str, Any]: ...
|
|
550
|
+
|
|
551
|
+
def to_json(self) -> str: ...
|
|
552
|
+
|
|
553
|
+
def to_markdown(self) -> str: ...
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
@runtime_checkable
|
|
557
|
+
class WorkflowBackend(Protocol):
|
|
558
|
+
"""Domain adapter for the shared flow, including native record readers.
|
|
559
|
+
|
|
560
|
+
describe/resolve operate on metadata without candidate execution or asset
|
|
561
|
+
reads. cases may load assets. admit verifies access, bytes and capabilities;
|
|
562
|
+
recheck runs before execute. assess must reuse saved evidence, not execute a
|
|
563
|
+
candidate. report/assessments/compare verify native evidence and permissions.
|
|
564
|
+
Comparison population, aggregation, projection and source authority stay in
|
|
565
|
+
the backend. Recognizing a reference type does not validate its evidence.
|
|
566
|
+
"""
|
|
567
|
+
|
|
568
|
+
@property
|
|
569
|
+
def descriptor(self) -> BackendDescriptor: ...
|
|
570
|
+
|
|
571
|
+
def describe(self, suite_id: str) -> SuiteDefinition: ...
|
|
572
|
+
|
|
573
|
+
def cases(self, suite: SuiteDefinition) -> Iterable[CaseView]: ...
|
|
574
|
+
|
|
575
|
+
def resolve(self, plan: PlanBinding) -> ResolvedBinding: ...
|
|
576
|
+
|
|
577
|
+
def admit(self, plan: PlanBinding, resolved: ResolvedBinding) -> AdmissionBinding: ...
|
|
578
|
+
|
|
579
|
+
def recheck(self, plan: PlanBinding, resolved: ResolvedBinding, admission: AdmissionBinding) -> None: ...
|
|
580
|
+
|
|
581
|
+
def execute(self, plan: PlanBinding, resolved: ResolvedBinding, admission: AdmissionBinding,
|
|
582
|
+
output_dir: str | Path | None) -> Any: ...
|
|
583
|
+
|
|
584
|
+
def accepts_reference(self, native_ref: Any, kind: str) -> bool: ...
|
|
585
|
+
|
|
586
|
+
def reference(self, native_ref: Any, kind: str) -> ReferenceBinding: ...
|
|
587
|
+
|
|
588
|
+
def assess(self, native_run: Any) -> Any: ...
|
|
589
|
+
|
|
590
|
+
def report(self, native_grading: Any) -> ReportProjection: ...
|
|
591
|
+
|
|
592
|
+
def assessments(self, native_grading: Any, case_id: str | None = None) -> Iterable[Assessment]: ...
|
|
593
|
+
|
|
594
|
+
def compare(self, baseline_grading: Any, candidate_grading: Any) -> Any: ...
|