sable-sphynx 0.7.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
  2. sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
  3. sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
  4. sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
  5. sphynx/__init__.py +469 -0
  6. sphynx/__init__.pyi +376 -0
  7. sphynx/_analytical_client.py +77 -0
  8. sphynx/_analytical_values.py +252 -0
  9. sphynx/_audit_client.py +173 -0
  10. sphynx/_comparison_adapters.py +113 -0
  11. sphynx/_componentized_full_rag.py +34 -0
  12. sphynx/_composed_pipeline.py +109 -0
  13. sphynx/_full_rag_grading.py +218 -0
  14. sphynx/_full_rag_trace.py +221 -0
  15. sphynx/_local_eval.py +268 -0
  16. sphynx/_pipeline_core.py +243 -0
  17. sphynx/_pr_client.py +237 -0
  18. sphynx/_rag_diagnostics_calibration.py +285 -0
  19. sphynx/_rag_diagnostics_catalog.py +39 -0
  20. sphynx/_rag_diagnostics_common.py +26 -0
  21. sphynx/_rag_diagnostics_contracts.py +186 -0
  22. sphynx/_rag_diagnostics_engine.py +243 -0
  23. sphynx/_rag_diagnostics_importers.py +242 -0
  24. sphynx/_rag_diagnostics_ranking.py +81 -0
  25. sphynx/_rag_diagnostics_semantic.py +325 -0
  26. sphynx/_rag_ranking.py +149 -0
  27. sphynx/_speech_benchmark_scoring.py +140 -0
  28. sphynx/_speech_benchmark_sources.py +275 -0
  29. sphynx/_speech_complete_scoring.py +218 -0
  30. sphynx/_speech_complete_source.py +118 -0
  31. sphynx/_speech_kido.py +294 -0
  32. sphynx/_speech_scoring.py +77 -0
  33. sphynx/_track_capture.py +213 -0
  34. sphynx/analytical_views.py +515 -0
  35. sphynx/assessment_contracts.py +378 -0
  36. sphynx/assessment_selection.py +57 -0
  37. sphynx/audit_access.py +245 -0
  38. sphynx/audit_contracts.py +204 -0
  39. sphynx/audit_requests.py +180 -0
  40. sphynx/authoring.py +188 -0
  41. sphynx/benchmarks.py +568 -0
  42. sphynx/candidates.py +62 -0
  43. sphynx/catalog.py +272 -0
  44. sphynx/chunking.py +372 -0
  45. sphynx/clickhouse_reader.py +169 -0
  46. sphynx/client.py +486 -0
  47. sphynx/comparison_core.py +149 -0
  48. sphynx/comparison_views.py +223 -0
  49. sphynx/components.py +1661 -0
  50. sphynx/contracts.py +514 -0
  51. sphynx/customer_rerank.py +299 -0
  52. sphynx/customer_retrieval.py +146 -0
  53. sphynx/data/benchmark-releases.v1.json +164 -0
  54. sphynx/data/public-benchmark-release.v1.json +354 -0
  55. sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
  56. sphynx/data/rag-paper-references.v1.json +176 -0
  57. sphynx/data/rag-paper-schemas.v1.json +1069 -0
  58. sphynx/data/rag-rerank-v1.json +507 -0
  59. sphynx/dataset_catalog_data.py +1214 -0
  60. sphynx/dataset_readiness.py +138 -0
  61. sphynx/dataset_v3.py +999 -0
  62. sphynx/demo.py +130 -0
  63. sphynx/demo_candidates/__init__.py +1 -0
  64. sphynx/demo_candidates/dense.py +11 -0
  65. sphynx/demo_candidates/lexical.py +21 -0
  66. sphynx/diagnostics.py +207 -0
  67. sphynx/documentation.py +737 -0
  68. sphynx/documentation_candidate.py +939 -0
  69. sphynx/evals.py +538 -0
  70. sphynx/evaluation_protocols.py +124 -0
  71. sphynx/evaluator_bindings.py +370 -0
  72. sphynx/evidence.py +352 -0
  73. sphynx/execution_capabilities.py +109 -0
  74. sphynx/fast_search.py +285 -0
  75. sphynx/full_rag.py +341 -0
  76. sphynx/hosted/__init__.py +18 -0
  77. sphynx/hosted/benchmark_intake.py +218 -0
  78. sphynx/hosted/contracts.py +361 -0
  79. sphynx/hosted/model_admission.py +130 -0
  80. sphynx/hosted/model_artifacts.py +298 -0
  81. sphynx/hosted/model_contracts.py +63 -0
  82. sphynx/hosted/model_gcs.py +181 -0
  83. sphynx/hosted/model_history.py +84 -0
  84. sphynx/hosted/model_session.py +233 -0
  85. sphynx/hosted/public_suite.py +74 -0
  86. sphynx/hosted/registry.py +97 -0
  87. sphynx/hosted/revisions.py +126 -0
  88. sphynx/hosted/sandbox.py +105 -0
  89. sphynx/hosted/submission.py +287 -0
  90. sphynx/hosted_workflow.py +451 -0
  91. sphynx/http_transport.py +86 -0
  92. sphynx/integrated_rag.py +352 -0
  93. sphynx/integrated_workflow.py +552 -0
  94. sphynx/integration_contracts.py +474 -0
  95. sphynx/invocation.py +791 -0
  96. sphynx/invocation_candidates.py +392 -0
  97. sphynx/invocation_catalog_data.py +40 -0
  98. sphynx/learned_catalog_data.py +30 -0
  99. sphynx/learned_rerank.py +267 -0
  100. sphynx/legacy_full_rag.py +67 -0
  101. sphynx/lifecycle.py +885 -0
  102. sphynx/lifecycle_workflow.py +490 -0
  103. sphynx/metric_views.py +474 -0
  104. sphynx/metrics.py +282 -0
  105. sphynx/model_adapters.py +438 -0
  106. sphynx/model_catalog_data.py +30 -0
  107. sphynx/official_benchmarks.py +385 -0
  108. sphynx/packs.py +258 -0
  109. sphynx/pipeline.py +313 -0
  110. sphynx/pipeline_workflow.py +551 -0
  111. sphynx/pr_evaluations.py +560 -0
  112. sphynx/progress.py +89 -0
  113. sphynx/public_bundle.py +173 -0
  114. sphynx/public_eval.py +62 -0
  115. sphynx/public_rerank.py +355 -0
  116. sphynx/py.typed +0 -0
  117. sphynx/pytest_plugin.py +62 -0
  118. sphynx/question_robustness.py +437 -0
  119. sphynx/rag.py +329 -0
  120. sphynx/rag_diagnostics.py +309 -0
  121. sphynx/rag_papers/__init__.py +29 -0
  122. sphynx/rag_papers/__main__.py +45 -0
  123. sphynx/rag_papers/candidates.py +19 -0
  124. sphynx/rag_papers/contracts.py +282 -0
  125. sphynx/rag_papers/datasets.py +291 -0
  126. sphynx/rag_papers/experiments.py +116 -0
  127. sphynx/rag_papers/judges.py +308 -0
  128. sphynx/rag_papers/registry.py +33 -0
  129. sphynx/rag_papers/runner.py +176 -0
  130. sphynx/rag_papers/sable.py +135 -0
  131. sphynx/rag_papers/schemas.py +12 -0
  132. sphynx/rag_papers/scoring.py +249 -0
  133. sphynx/rag_papers/semantic.py +288 -0
  134. sphynx/rag_papers/trec.py +80 -0
  135. sphynx/recorded_scores.py +352 -0
  136. sphynx/records.py +124 -0
  137. sphynx/reflex_traces.py +193 -0
  138. sphynx/report_rendering.py +71 -0
  139. sphynx/rerank_family_contracts.py +150 -0
  140. sphynx/rerank_workflow.py +424 -0
  141. sphynx/research_candidate.py +190 -0
  142. sphynx/results.py +852 -0
  143. sphynx/retrieval.py +750 -0
  144. sphynx/retrieval_candidates.py +295 -0
  145. sphynx/retrieval_catalog_data.py +19 -0
  146. sphynx/retrieval_recordings.py +125 -0
  147. sphynx/reviewer_audit.py +266 -0
  148. sphynx/routing_policies.py +375 -0
  149. sphynx/runner.py +544 -0
  150. sphynx/sable_rerank.py +489 -0
  151. sphynx/score_reliability.py +393 -0
  152. sphynx/speech_benchmarks.py +204 -0
  153. sphynx/speech_campaigns.py +378 -0
  154. sphynx/speech_complete.py +236 -0
  155. sphynx/speech_contracts.py +225 -0
  156. sphynx/speech_evidence.py +429 -0
  157. sphynx/speech_listening.py +115 -0
  158. sphynx/speech_multilingual.py +71 -0
  159. sphynx/speech_tts.py +479 -0
  160. sphynx/sql/001_sync_v1.sql +30 -0
  161. sphynx/stage_contracts.py +199 -0
  162. sphynx/statistical_catalog_data.py +53 -0
  163. sphynx/statistical_rerank.py +232 -0
  164. sphynx/sync.py +797 -0
  165. sphynx/workbench.py +378 -0
  166. sphynx/workbench_catalog_data.py +13 -0
  167. sphynx/workbench_tracks.py +78 -0
  168. sphynx/workflow_async.py +197 -0
  169. sphynx/workflow_contracts.py +594 -0
  170. sphynx/workflow_decisions.py +211 -0
  171. sphynx/workflow_defaults.py +16 -0
  172. sphynx/workflow_extensions.py +212 -0
  173. sphynx/workflow_registry.py +513 -0
  174. sphynx/workflows.py +792 -0
@@ -0,0 +1,594 @@
1
+ """Domain-neutral contracts for suite backends and their validated payloads.
2
+
3
+ These contracts describe extension boundaries, not an execution engine. A
4
+ backend owns native records, payload validation, reference authorization, and
5
+ comparison policy. Immutable metadata and implementation labels do not verify
6
+ source bytes, authenticate evidence, or provide process isolation.
7
+ """
8
+ from __future__ import annotations
9
+
10
+ import hashlib
11
+ import json
12
+ import re
13
+ from dataclasses import dataclass, field
14
+ from pathlib import Path
15
+ from typing import Any, Callable, Iterable, Mapping, Protocol, Sequence, TYPE_CHECKING, runtime_checkable
16
+
17
+ from .assessment_contracts import (
18
+ Assessment, AssessmentContractError, CaseView, _freeze_json, _thaw,
19
+ )
20
+
21
+ if TYPE_CHECKING:
22
+ from .diagnostics import Diagnostic
23
+
24
+
25
+ _DIGEST = re.compile(r"sha256:[0-9a-f]{64}\Z")
26
+
27
+
28
+ class WorkflowContractError(ValueError):
29
+ """An extension descriptor or workflow binding is malformed."""
30
+
31
+ @property
32
+ def diagnostic(self) -> Diagnostic:
33
+ from .diagnostics import explain_error
34
+ return explain_error(self)
35
+
36
+
37
+ def _text(value: Any, name: str) -> str:
38
+ if type(value) is not str or not value.strip() or "\x00" in value:
39
+ raise WorkflowContractError(f"{name} must be nonempty text without NUL bytes")
40
+ try:
41
+ value.encode("utf-8", errors="strict")
42
+ except UnicodeEncodeError:
43
+ raise WorkflowContractError(f"{name} must contain valid Unicode") from None
44
+ return value
45
+
46
+
47
+ def _optional_text(value: Any, name: str) -> None:
48
+ if value is not None:
49
+ _text(value, name)
50
+
51
+
52
+ def _object(value: Any, name: str) -> Mapping[str, Any]:
53
+ if not isinstance(value, Mapping):
54
+ raise WorkflowContractError(f"{name} must be a JSON object")
55
+ try:
56
+ return _freeze_json(value, name)
57
+ except AssessmentContractError as exc:
58
+ raise WorkflowContractError(str(exc)) from None
59
+
60
+
61
+ def _texts(value: Sequence[str], name: str, *, namespaced: bool = False) -> tuple[str, ...]:
62
+ if isinstance(value, (str, bytes)) or not isinstance(value, Sequence):
63
+ raise WorkflowContractError(f"{name} must be a sequence of identifiers")
64
+ items = tuple(_text(item, name) for item in value)
65
+ if len(set(items)) != len(items):
66
+ raise WorkflowContractError(f"{name} must not contain duplicates")
67
+ if namespaced and any("." not in item and ":" not in item for item in items):
68
+ raise WorkflowContractError(f"{name} must use namespaced identifiers")
69
+ return items
70
+
71
+
72
+ def _canonical_json(value: Any) -> str:
73
+ try:
74
+ return json.dumps(_thaw(value), sort_keys=True, separators=(",", ":"),
75
+ ensure_ascii=False, allow_nan=False)
76
+ except (TypeError, ValueError) as exc:
77
+ raise WorkflowContractError(f"value cannot be serialized as canonical JSON: {exc}") from None
78
+
79
+
80
+ def contract_digest(value: Mapping[str, Any]) -> str:
81
+ """Hash canonical JSON metadata; this is identity, not authentication."""
82
+ raw = _canonical_json(_object(value, "digest input")).encode("utf-8")
83
+ return "sha256:" + hashlib.sha256(raw).hexdigest()
84
+
85
+
86
+ @dataclass(frozen=True)
87
+ class PayloadSchema:
88
+ """An exact payload contract backed by an operator-supplied validator.
89
+
90
+ The validator receives a detached JSON value, returns None on success, and
91
+ raises on invalid content. It must not mutate the value. Adapters wrap typed
92
+ validators that return parsed objects. The implementation label is an exact
93
+ operator declaration, not a derived source digest. Registration does not run
94
+ this callback; explicit payload validation does.
95
+ """
96
+
97
+ schema_id: str
98
+ version: str
99
+ implementation_id: str
100
+ validator: Callable[[Any], None] = field(repr=False, compare=False)
101
+ metadata: Mapping[str, Any] = field(default_factory=dict)
102
+
103
+ def __post_init__(self) -> None:
104
+ for name in ("schema_id", "version", "implementation_id"):
105
+ _text(getattr(self, name), name)
106
+ if not callable(self.validator):
107
+ raise WorkflowContractError("payload validator must be callable")
108
+ object.__setattr__(self, "metadata", _object(self.metadata, "metadata"))
109
+
110
+ def validate(self, value: Any) -> Any:
111
+ """Validate content and return a detached immutable JSON snapshot."""
112
+ try:
113
+ frozen = _freeze_json(value, "payload")
114
+ except AssessmentContractError as exc:
115
+ raise WorkflowContractError(str(exc)) from None
116
+ original_json = _canonical_json(frozen)
117
+ detached = _thaw(frozen)
118
+ result = self.validator(detached)
119
+ if result is not None:
120
+ raise WorkflowContractError("payload validator must return None on success")
121
+ try:
122
+ unchanged = _canonical_json(_freeze_json(detached, "validated payload")) == original_json
123
+ except (AssessmentContractError, WorkflowContractError):
124
+ unchanged = False
125
+ if not unchanged:
126
+ raise WorkflowContractError("payload validator must not mutate content")
127
+ return frozen
128
+
129
+ def to_dict(self) -> dict[str, Any]:
130
+ return {"schema_id": self.schema_id, "version": self.version,
131
+ "implementation_id": self.implementation_id, "metadata": _thaw(self.metadata)}
132
+
133
+
134
+ @dataclass(frozen=True)
135
+ class DatasetDefinition:
136
+ """Case-provider metadata without reading or materializing its cases.
137
+
138
+ Selection, split, ordering, cohort, and grouping rules are provider-owned.
139
+ Input and reference schema identities stay separate. Providers may iterate
140
+ lazily. None of this metadata grants access to references or external assets.
141
+ """
142
+
143
+ dataset_id: str
144
+ version: str
145
+ provider_id: str
146
+ input_schema_id: str
147
+ reference_schema_id: str | None = None
148
+ selection: Mapping[str, Any] = field(default_factory=dict)
149
+ identity: Mapping[str, Any] = field(default_factory=dict)
150
+
151
+ def __post_init__(self) -> None:
152
+ for name in ("dataset_id", "version", "provider_id", "input_schema_id"):
153
+ _text(getattr(self, name), name)
154
+ _optional_text(self.reference_schema_id, "reference_schema_id")
155
+ for name in ("selection", "identity"):
156
+ object.__setattr__(self, name, _object(getattr(self, name), name))
157
+
158
+ def to_dict(self) -> dict[str, Any]:
159
+ return {"dataset_id": self.dataset_id, "version": self.version,
160
+ "provider_id": self.provider_id, "input_schema_id": self.input_schema_id,
161
+ "reference_schema_id": self.reference_schema_id,
162
+ "selection": _thaw(self.selection), "identity": _thaw(self.identity)}
163
+
164
+
165
+ @runtime_checkable
166
+ class CaseProvider(Protocol):
167
+ """Input iteration and grader-only reference lookup are different calls.
168
+
169
+ ``access`` has backend-defined authorization semantics. The provider must
170
+ validate it; possession of a Python object is not a grant. A candidate must
171
+ receive only its allowlisted CaseView, not the provider or its references.
172
+ """
173
+
174
+ @property
175
+ def definition(self) -> DatasetDefinition: ...
176
+
177
+ def cases(self, selection: Mapping[str, Any] | None = None) -> Iterable[CaseView]: ...
178
+
179
+ def reference_for(self, case_id: str, *, access: Any) -> Mapping[str, Any]: ...
180
+
181
+
182
+ @dataclass(frozen=True)
183
+ class Dataset:
184
+ """A schema-bound case provider with lazy, candidate-permitted inputs.
185
+
186
+ ``case_source`` is an iterable or a callable receiving the selected immutable
187
+ selection mapping. Use a callable to obtain a fresh iterator for each pass.
188
+ Construction and descriptor inspection do not load cases or references.
189
+
190
+ An optional selection schema validates nondefault selections. Without it,
191
+ only the definition's exact selection is accepted. A backend must bind the
192
+ selected cohort's native identity before execution and comparison.
193
+
194
+ References have a separate schema and loader. The explicit authorization
195
+ callback must return True before that loader is accessed. The callback and
196
+ backend own principal, case membership, and access-policy validation. No
197
+ default object, token shape, or Python type constitutes a reference grant.
198
+ Do not pass this provider to candidate execution; pass its CaseViews only.
199
+ """
200
+
201
+ definition: DatasetDefinition
202
+ input_schema: PayloadSchema
203
+ case_source: Iterable[CaseView] | Callable[[Mapping[str, Any]], Iterable[CaseView]] = field(
204
+ repr=False, compare=False)
205
+ reference_schema: PayloadSchema | None = field(default=None, repr=False)
206
+ reference_loader: Callable[[str], Mapping[str, Any]] | None = field(
207
+ default=None, repr=False, compare=False)
208
+ authorize_references: Callable[[str, Any], bool] | None = field(
209
+ default=None, repr=False, compare=False)
210
+ selection_schema: PayloadSchema | None = field(default=None, repr=False)
211
+
212
+ def __post_init__(self) -> None:
213
+ if not isinstance(self.definition, DatasetDefinition):
214
+ raise WorkflowContractError("dataset definition must be DatasetDefinition")
215
+ if not isinstance(self.input_schema, PayloadSchema):
216
+ raise WorkflowContractError("dataset input schema must be PayloadSchema")
217
+ if self.input_schema.schema_id != self.definition.input_schema_id:
218
+ raise WorkflowContractError("input schema differs from the dataset definition")
219
+ if not callable(self.case_source) and not isinstance(self.case_source, Iterable):
220
+ raise WorkflowContractError("case source must be an iterable or callable")
221
+ if self.reference_schema is not None:
222
+ if not isinstance(self.reference_schema, PayloadSchema):
223
+ raise WorkflowContractError("reference schema must be PayloadSchema")
224
+ if self.reference_schema.schema_id != self.definition.reference_schema_id:
225
+ raise WorkflowContractError("reference schema differs from the dataset definition")
226
+ if self.reference_loader is not None:
227
+ if not callable(self.reference_loader) or self.reference_schema is None:
228
+ raise WorkflowContractError("reference loader requires a matching payload schema")
229
+ if self.authorize_references is not None and not callable(self.authorize_references):
230
+ raise WorkflowContractError("reference authorization must be callable")
231
+ if self.selection_schema is not None and not isinstance(self.selection_schema, PayloadSchema):
232
+ raise WorkflowContractError("selection schema must be PayloadSchema")
233
+
234
+ def cases(self, selection: Mapping[str, Any] | None = None) -> Iterable[CaseView]:
235
+ """Validate and yield cases only when the returned iterator is consumed."""
236
+ selected = _object(self.definition.selection if selection is None else selection, "selection")
237
+ if self.selection_schema is not None:
238
+ selected = self.selection_schema.validate(selected)
239
+ if not isinstance(selected, Mapping):
240
+ raise WorkflowContractError("selection schema must validate an object")
241
+ elif _canonical_json(selected) != _canonical_json(self.definition.selection):
242
+ raise WorkflowContractError("nondefault selection requires a selection schema")
243
+ source = self.case_source(selected) if callable(self.case_source) else self.case_source
244
+ try:
245
+ iterator = iter(source)
246
+ except TypeError:
247
+ raise WorkflowContractError("case source must produce an iterable") from None
248
+ seen: set[str] = set()
249
+ for case in iterator:
250
+ if not isinstance(case, CaseView):
251
+ raise WorkflowContractError("case source must yield CaseView values")
252
+ if case.schema_version != self.definition.input_schema_id:
253
+ raise WorkflowContractError("case input schema differs from the dataset definition")
254
+ if case.case_id in seen:
255
+ raise WorkflowContractError("case source contains duplicate case identities")
256
+ seen.add(case.case_id)
257
+ yield CaseView.validated(case.case_id, case.inputs, schema=self.input_schema,
258
+ metadata=case.metadata)
259
+
260
+ def reference_for(self, case_id: str, *, access: Any) -> Mapping[str, Any]:
261
+ """Authorize first, then load and validate one grader-only reference."""
262
+ _text(case_id, "case_id")
263
+ if self.authorize_references is None:
264
+ raise PermissionError("reference access is not authorized")
265
+ try:
266
+ allowed = self.authorize_references(case_id, access)
267
+ except Exception:
268
+ raise PermissionError("reference access is not authorized") from None
269
+ if allowed is not True:
270
+ raise PermissionError("reference access is not authorized")
271
+ if self.reference_loader is None or self.reference_schema is None:
272
+ raise WorkflowContractError("dataset does not provide reference loading")
273
+ value = self.reference_schema.validate(self.reference_loader(case_id))
274
+ if not isinstance(value, Mapping):
275
+ raise WorkflowContractError("reference schema must validate an object")
276
+ return value
277
+
278
+ def to_dict(self) -> dict[str, Any]:
279
+ """Return metadata only; omit loaders, authorization, and reference data."""
280
+ return self.definition.to_dict()
281
+
282
+
283
+ @dataclass(frozen=True)
284
+ class Criterion:
285
+ """A versioned requirement or dimension, independent of numeric metrics."""
286
+
287
+ criterion_id: str
288
+ version: str
289
+ title: str
290
+ configuration: Mapping[str, Any] = field(default_factory=dict)
291
+
292
+ def __post_init__(self) -> None:
293
+ for name in ("criterion_id", "version", "title"):
294
+ _text(getattr(self, name), name)
295
+ object.__setattr__(self, "configuration", _object(self.configuration, "configuration"))
296
+
297
+ def to_dict(self) -> dict[str, Any]:
298
+ return {"criterion_id": self.criterion_id, "version": self.version,
299
+ "title": self.title, "configuration": _thaw(self.configuration)}
300
+
301
+
302
+ @dataclass(frozen=True)
303
+ class EvaluatorDefinition:
304
+ """An evaluator may address several criteria and emit several outputs."""
305
+
306
+ evaluator_id: str
307
+ version: str
308
+ implementation_id: str
309
+ criterion_ids: tuple[str, ...] = ()
310
+ assessment_schema_ids: tuple[str, ...] = ()
311
+ metric_ids: tuple[str, ...] = ()
312
+ required_capabilities: tuple[str, ...] = ()
313
+ configuration: Mapping[str, Any] = field(default_factory=dict)
314
+
315
+ def __post_init__(self) -> None:
316
+ for name in ("evaluator_id", "version", "implementation_id"):
317
+ _text(getattr(self, name), name)
318
+ for name in ("criterion_ids", "assessment_schema_ids", "metric_ids", "required_capabilities"):
319
+ object.__setattr__(self, name, _texts(getattr(self, name), name,
320
+ namespaced=name == "required_capabilities"))
321
+ object.__setattr__(self, "configuration", _object(self.configuration, "configuration"))
322
+
323
+ def to_dict(self) -> dict[str, Any]:
324
+ return {"evaluator_id": self.evaluator_id, "version": self.version,
325
+ "implementation_id": self.implementation_id, "criterion_ids": list(self.criterion_ids),
326
+ "assessment_schema_ids": list(self.assessment_schema_ids), "metric_ids": list(self.metric_ids),
327
+ "required_capabilities": list(self.required_capabilities),
328
+ "configuration": _thaw(self.configuration)}
329
+
330
+
331
+ @dataclass(frozen=True)
332
+ class MetricSpec:
333
+ """Numeric meaning and optional domain policies, without a default reducer.
334
+
335
+ Direction and population semantics belong to the registered domain. Policy
336
+ IDs identify declared implementations; no policy is executed by this type.
337
+ A measurement need not be an assessment of a quality criterion.
338
+ """
339
+
340
+ metric_id: str
341
+ version: str
342
+ unit: str
343
+ direction: str
344
+ aggregation_policy: str | None = None
345
+ comparison_policy: str | None = None
346
+ population: Mapping[str, Any] = field(default_factory=dict)
347
+ metadata: Mapping[str, Any] = field(default_factory=dict)
348
+
349
+ def __post_init__(self) -> None:
350
+ for name in ("metric_id", "version", "unit", "direction"):
351
+ _text(getattr(self, name), name)
352
+ for name in ("aggregation_policy", "comparison_policy"):
353
+ _optional_text(getattr(self, name), name)
354
+ for name in ("population", "metadata"):
355
+ object.__setattr__(self, name, _object(getattr(self, name), name))
356
+
357
+ def to_dict(self) -> dict[str, Any]:
358
+ return {"metric_id": self.metric_id, "version": self.version, "unit": self.unit,
359
+ "direction": self.direction, "aggregation_policy": self.aggregation_policy,
360
+ "comparison_policy": self.comparison_policy,
361
+ "population": _thaw(self.population), "metadata": _thaw(self.metadata)}
362
+
363
+
364
+ @dataclass(frozen=True)
365
+ class BackendDescriptor:
366
+ """Exact backend identity and open, namespaced capability declarations."""
367
+
368
+ backend_id: str
369
+ version: str
370
+ implementation_id: str
371
+ capabilities: tuple[str, ...]
372
+ supported_scopes: tuple[str, ...]
373
+ supported_modes: tuple[str, ...]
374
+ suite_ids: tuple[str, ...] = ()
375
+ metadata: Mapping[str, Any] = field(default_factory=dict)
376
+
377
+ def __post_init__(self) -> None:
378
+ for name in ("backend_id", "version", "implementation_id"):
379
+ _text(getattr(self, name), name)
380
+ for name in ("capabilities", "supported_scopes", "supported_modes", "suite_ids"):
381
+ object.__setattr__(self, name, _texts(getattr(self, name), name,
382
+ namespaced=name == "capabilities"))
383
+ if not self.supported_scopes or not self.supported_modes:
384
+ raise WorkflowContractError("backend must declare supported scopes and modes")
385
+ object.__setattr__(self, "metadata", _object(self.metadata, "metadata"))
386
+
387
+ @property
388
+ def descriptor_digest(self) -> str:
389
+ return contract_digest(self.to_dict())
390
+
391
+ def to_dict(self) -> dict[str, Any]:
392
+ return {"backend_id": self.backend_id, "version": self.version,
393
+ "implementation_id": self.implementation_id, "capabilities": list(self.capabilities),
394
+ "supported_scopes": list(self.supported_scopes), "supported_modes": list(self.supported_modes),
395
+ "suite_ids": list(self.suite_ids), "metadata": _thaw(self.metadata)}
396
+
397
+
398
+ @dataclass(frozen=True)
399
+ class SuiteDefinition:
400
+ """Reusable suite metadata; native assets and records remain backend-owned."""
401
+
402
+ suite_id: str
403
+ version: str
404
+ backend_id: str
405
+ title: str
406
+ configuration: Mapping[str, Any] = field(default_factory=dict)
407
+ criterion_ids: tuple[str, ...] = ()
408
+ evaluator_ids: tuple[str, ...] = ()
409
+ metric_ids: tuple[str, ...] = ()
410
+ required_capabilities: tuple[str, ...] = ()
411
+ datasets: tuple[DatasetDefinition, ...] = ()
412
+ metadata: Mapping[str, Any] = field(default_factory=dict)
413
+
414
+ def __post_init__(self) -> None:
415
+ for name in ("suite_id", "version", "backend_id", "title"):
416
+ _text(getattr(self, name), name)
417
+ for name in ("criterion_ids", "evaluator_ids", "metric_ids", "required_capabilities"):
418
+ object.__setattr__(self, name, _texts(getattr(self, name), name,
419
+ namespaced=name == "required_capabilities"))
420
+ if isinstance(self.datasets, (str, bytes)) or not isinstance(self.datasets, Sequence):
421
+ raise WorkflowContractError("datasets must be a sequence of DatasetDefinition values")
422
+ datasets = tuple(self.datasets)
423
+ if any(not isinstance(item, DatasetDefinition) for item in datasets):
424
+ raise WorkflowContractError("datasets must contain DatasetDefinition values")
425
+ if len({(item.dataset_id, item.version) for item in datasets}) != len(datasets):
426
+ raise WorkflowContractError("suite dataset identities must be unique")
427
+ object.__setattr__(self, "datasets", datasets)
428
+ for name in ("configuration", "metadata"):
429
+ object.__setattr__(self, name, _object(getattr(self, name), name))
430
+
431
+ @property
432
+ def definition_digest(self) -> str:
433
+ return contract_digest(self.to_dict())
434
+
435
+ def to_dict(self) -> dict[str, Any]:
436
+ return {"suite_id": self.suite_id, "version": self.version, "backend_id": self.backend_id,
437
+ "title": self.title, "configuration": _thaw(self.configuration),
438
+ "criterion_ids": list(self.criterion_ids), "evaluator_ids": list(self.evaluator_ids),
439
+ "metric_ids": list(self.metric_ids), "required_capabilities": list(self.required_capabilities),
440
+ "datasets": [item.to_dict() for item in self.datasets], "metadata": _thaw(self.metadata)}
441
+
442
+
443
+ @dataclass(frozen=True)
444
+ class PlanBinding:
445
+ """Selected settings plus a backend-owned candidate object.
446
+
447
+ The backend must validate and bind the candidate's implementation identity
448
+ during resolution and recheck required facts at admission/launch. Freezing
449
+ this envelope cannot freeze an arbitrary service, function, or native object.
450
+ """
451
+
452
+ suite: SuiteDefinition
453
+ candidate: Any = field(repr=False, compare=False)
454
+ scope: str = "component"
455
+ mode: str = "local"
456
+ parameters: Mapping[str, Any] = field(default_factory=dict)
457
+ runtime: Mapping[str, Any] = field(default_factory=dict)
458
+
459
+ def __post_init__(self) -> None:
460
+ if not isinstance(self.suite, SuiteDefinition):
461
+ raise WorkflowContractError("plan suite must be SuiteDefinition")
462
+ for name in ("scope", "mode"):
463
+ _text(getattr(self, name), name)
464
+ for name in ("parameters", "runtime"):
465
+ object.__setattr__(self, name, _object(getattr(self, name), name))
466
+
467
+
468
+ @dataclass(frozen=True)
469
+ class ResolvedBinding:
470
+ """Metadata selected by a backend; resolution does not verify asset bytes."""
471
+
472
+ identity: Mapping[str, Any]
473
+ native_ref: Any = field(default=None, repr=False, compare=False)
474
+
475
+ def __post_init__(self) -> None:
476
+ object.__setattr__(self, "identity", _object(self.identity, "identity"))
477
+ if not self.identity:
478
+ raise WorkflowContractError("resolution requires explicit identity metadata")
479
+
480
+ @property
481
+ def status(self) -> str:
482
+ return "resolved"
483
+
484
+ @property
485
+ def artifact_bytes_verified(self) -> bool:
486
+ return False
487
+
488
+ @property
489
+ def resolution_digest(self) -> str:
490
+ return contract_digest(self.to_dict())
491
+
492
+ def to_dict(self) -> dict[str, Any]:
493
+ return {"status": self.status, "identity": _thaw(self.identity), "artifact_bytes_verified": False}
494
+
495
+
496
+ @dataclass(frozen=True)
497
+ class AdmissionBinding:
498
+ """Backend-verified admission facts without upgrading native authority."""
499
+
500
+ resolution_digest: str
501
+ facts: Mapping[str, Any]
502
+ trust_tier: str
503
+ artifact_bytes_verified: bool
504
+ candidate_source_bytes_verified: bool = False
505
+ native_ref: Any = field(default=None, repr=False, compare=False)
506
+
507
+ def __post_init__(self) -> None:
508
+ if type(self.resolution_digest) is not str or not _DIGEST.fullmatch(self.resolution_digest):
509
+ raise WorkflowContractError("admission requires a canonical resolution digest")
510
+ _text(self.trust_tier, "trust_tier")
511
+ for name in ("artifact_bytes_verified", "candidate_source_bytes_verified"):
512
+ if type(getattr(self, name)) is not bool:
513
+ raise WorkflowContractError(f"{name} must be boolean")
514
+ object.__setattr__(self, "facts", _object(self.facts, "facts"))
515
+
516
+ @property
517
+ def status(self) -> str:
518
+ return "admitted"
519
+
520
+ def to_dict(self) -> dict[str, Any]:
521
+ return {"status": self.status, "resolution_digest": self.resolution_digest,
522
+ "facts": _thaw(self.facts), "trust_tier": self.trust_tier,
523
+ "artifact_bytes_verified": self.artifact_bytes_verified,
524
+ "candidate_source_bytes_verified": self.candidate_source_bytes_verified}
525
+
526
+
527
+ @dataclass(frozen=True)
528
+ class ReferenceBinding:
529
+ """Native record status and identity, separate from evidence verification."""
530
+
531
+ record_id: str | None
532
+ status: str
533
+ trust_tier: str
534
+ run_id: str | None = None
535
+ metadata: Mapping[str, Any] = field(default_factory=dict)
536
+
537
+ def __post_init__(self) -> None:
538
+ for name in ("record_id", "run_id"):
539
+ _optional_text(getattr(self, name), name)
540
+ for name in ("status", "trust_tier"):
541
+ _text(getattr(self, name), name)
542
+ object.__setattr__(self, "metadata", _object(self.metadata, "metadata"))
543
+
544
+
545
+ @runtime_checkable
546
+ class ReportProjection(Protocol):
547
+ """Backend-verified permitted presentation; no universal metric layout."""
548
+
549
+ def to_dict(self) -> dict[str, Any]: ...
550
+
551
+ def to_json(self) -> str: ...
552
+
553
+ def to_markdown(self) -> str: ...
554
+
555
+
556
+ @runtime_checkable
557
+ class WorkflowBackend(Protocol):
558
+ """Domain adapter for the shared flow, including native record readers.
559
+
560
+ describe/resolve operate on metadata without candidate execution or asset
561
+ reads. cases may load assets. admit verifies access, bytes and capabilities;
562
+ recheck runs before execute. assess must reuse saved evidence, not execute a
563
+ candidate. report/assessments/compare verify native evidence and permissions.
564
+ Comparison population, aggregation, projection and source authority stay in
565
+ the backend. Recognizing a reference type does not validate its evidence.
566
+ """
567
+
568
+ @property
569
+ def descriptor(self) -> BackendDescriptor: ...
570
+
571
+ def describe(self, suite_id: str) -> SuiteDefinition: ...
572
+
573
+ def cases(self, suite: SuiteDefinition) -> Iterable[CaseView]: ...
574
+
575
+ def resolve(self, plan: PlanBinding) -> ResolvedBinding: ...
576
+
577
+ def admit(self, plan: PlanBinding, resolved: ResolvedBinding) -> AdmissionBinding: ...
578
+
579
+ def recheck(self, plan: PlanBinding, resolved: ResolvedBinding, admission: AdmissionBinding) -> None: ...
580
+
581
+ def execute(self, plan: PlanBinding, resolved: ResolvedBinding, admission: AdmissionBinding,
582
+ output_dir: str | Path | None) -> Any: ...
583
+
584
+ def accepts_reference(self, native_ref: Any, kind: str) -> bool: ...
585
+
586
+ def reference(self, native_ref: Any, kind: str) -> ReferenceBinding: ...
587
+
588
+ def assess(self, native_run: Any) -> Any: ...
589
+
590
+ def report(self, native_grading: Any) -> ReportProjection: ...
591
+
592
+ def assessments(self, native_grading: Any, case_id: str | None = None) -> Iterable[Assessment]: ...
593
+
594
+ def compare(self, baseline_grading: Any, candidate_grading: Any) -> Any: ...