sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
sphynx/workflows.py
ADDED
|
@@ -0,0 +1,792 @@
|
|
|
1
|
+
"""Domain-neutral Suite -> Plan -> Run -> Report orchestration.
|
|
2
|
+
|
|
3
|
+
Registered backends own candidate protocols, datasets, admission, native records,
|
|
4
|
+
evaluators, report shapes and comparison policy. This module does not load domain
|
|
5
|
+
assets, interpret metric layouts, authenticate evidence, or create a sandbox.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
import time
|
|
11
|
+
import uuid
|
|
12
|
+
|
|
13
|
+
from collections.abc import Iterable, Mapping
|
|
14
|
+
from dataclasses import InitVar, dataclass, field
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Any, TYPE_CHECKING
|
|
17
|
+
|
|
18
|
+
from .assessment_contracts import Assessment, CaseView
|
|
19
|
+
from .assessment_selection import AssessmentSelector
|
|
20
|
+
from .evaluator_bindings import ReportAssessment
|
|
21
|
+
from .workflow_contracts import (
|
|
22
|
+
AdmissionBinding, DatasetDefinition, PlanBinding, ReferenceBinding,
|
|
23
|
+
ReportProjection, ResolvedBinding, SuiteDefinition, WorkflowBackend,
|
|
24
|
+
WorkflowContractError, _text, _thaw, contract_digest,
|
|
25
|
+
)
|
|
26
|
+
from .workflow_defaults import default_workflow_registry
|
|
27
|
+
from .workflow_decisions import Verdict
|
|
28
|
+
from .workflow_registry import WorkflowRegistry
|
|
29
|
+
from .workflow_async import (
|
|
30
|
+
AsyncWorkflowBackend, GradingSnapshot, SubmissionError, SubmissionIntent,
|
|
31
|
+
SubmissionSnapshot,
|
|
32
|
+
)
|
|
33
|
+
from .stage_contracts import StageEvidence, StageEvidenceBackend, StageSelector
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
if TYPE_CHECKING:
|
|
37
|
+
from .comparison_views import ComparisonView
|
|
38
|
+
from .diagnostics import Diagnostic
|
|
39
|
+
from .progress import ProgressView
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
# Keep the public error import while registry errors remain catchable through it.
|
|
43
|
+
WorkflowError = WorkflowContractError
|
|
44
|
+
_ADMISSION_TOKEN = object()
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class WorkflowTimeoutError(TimeoutError):
|
|
48
|
+
"""Waiting ended; the handle still identifies the original remote work."""
|
|
49
|
+
|
|
50
|
+
def __init__(self, message: str, handle: Any) -> None:
|
|
51
|
+
self.handle = handle
|
|
52
|
+
super().__init__(message)
|
|
53
|
+
|
|
54
|
+
@property
|
|
55
|
+
def diagnostic(self) -> Diagnostic:
|
|
56
|
+
from .diagnostics import explain_error
|
|
57
|
+
return explain_error(self)
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
class PendingWorkError(WorkflowError):
|
|
61
|
+
"""Asynchronous work failed; its native identity remains on handle."""
|
|
62
|
+
|
|
63
|
+
def __init__(self, message: str, handle: Any) -> None:
|
|
64
|
+
self.handle = handle
|
|
65
|
+
super().__init__(message)
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class GradingRequestError(WorkflowError):
|
|
69
|
+
"""An uncertain grading POST retains its run and recovery key."""
|
|
70
|
+
|
|
71
|
+
def __init__(self, message: str, *, run_id: str, idempotency_key: str) -> None:
|
|
72
|
+
self.run_id, self.idempotency_key = run_id, idempotency_key
|
|
73
|
+
super().__init__(message)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _async_backend(backend: WorkflowBackend, capability: str) -> AsyncWorkflowBackend:
|
|
77
|
+
if capability not in backend.descriptor.capabilities or not isinstance(backend, AsyncWorkflowBackend):
|
|
78
|
+
raise WorkflowError("backend does not support " + capability)
|
|
79
|
+
return backend
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _wait(handle: Any, ready: Any, *, timeout: float, poll_interval: float,
|
|
83
|
+
on_progress: Any = None) -> Any:
|
|
84
|
+
for value, name in ((timeout, "timeout"), (poll_interval, "poll_interval")):
|
|
85
|
+
if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value <= 0:
|
|
86
|
+
raise WorkflowError(name + " must be finite and positive")
|
|
87
|
+
if on_progress is not None and not callable(on_progress):
|
|
88
|
+
raise WorkflowError("on_progress must be callable")
|
|
89
|
+
deadline = time.monotonic() + timeout
|
|
90
|
+
current = handle
|
|
91
|
+
while True:
|
|
92
|
+
if on_progress is not None:
|
|
93
|
+
on_progress(current.snapshot)
|
|
94
|
+
if current.snapshot.failed:
|
|
95
|
+
raise PendingWorkError(current.snapshot.reason or "asynchronous work failed", current)
|
|
96
|
+
result = ready(current)
|
|
97
|
+
if result is not None:
|
|
98
|
+
return result
|
|
99
|
+
remaining = deadline - time.monotonic()
|
|
100
|
+
if remaining <= 0:
|
|
101
|
+
raise WorkflowTimeoutError("wait timed out; retain the handle and resume polling", current)
|
|
102
|
+
time.sleep(min(poll_interval, remaining))
|
|
103
|
+
current = current.refresh()
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _stages(backend: WorkflowBackend, native_ref: Any, selector: StageSelector) -> tuple[StageEvidence, ...]:
|
|
107
|
+
if not isinstance(selector, StageSelector):
|
|
108
|
+
raise WorkflowError("stage evidence requires an exact StageSelector")
|
|
109
|
+
if "sphynx.stage-evidence.v1" not in backend.descriptor.capabilities or not isinstance(backend, StageEvidenceBackend):
|
|
110
|
+
raise WorkflowError("backend does not support stage evidence")
|
|
111
|
+
rows = backend.stage_evidence(native_ref, selector)
|
|
112
|
+
if not isinstance(rows, tuple) or any(not isinstance(row, StageEvidence) for row in rows):
|
|
113
|
+
raise WorkflowError("stage evidence backend must return a tuple of StageEvidence")
|
|
114
|
+
if any(row.selector != selector for row in rows):
|
|
115
|
+
raise WorkflowError("backend returned stage evidence outside the exact selection")
|
|
116
|
+
# The backend authorizes this projection and retains its native visibility
|
|
117
|
+
# label. A common selector never grants access or invents a global ACL.
|
|
118
|
+
return rows
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def _registry(value: WorkflowRegistry | None) -> WorkflowRegistry:
|
|
122
|
+
if value is None:
|
|
123
|
+
return default_workflow_registry()
|
|
124
|
+
if not isinstance(value, WorkflowRegistry):
|
|
125
|
+
raise WorkflowError("registry must be WorkflowRegistry")
|
|
126
|
+
return value
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _reference_backend(native_ref: Any, kind: str, registry: WorkflowRegistry | None,
|
|
130
|
+
backend_id: str | None) -> tuple[WorkflowRegistry, str, WorkflowBackend]:
|
|
131
|
+
selected = _registry(registry)
|
|
132
|
+
if backend_id is not None:
|
|
133
|
+
backend = selected.get_backend(backend_id)
|
|
134
|
+
if not backend.accepts_reference(native_ref, kind):
|
|
135
|
+
raise WorkflowError(f"backend does not recognize the native {kind} reference")
|
|
136
|
+
return selected, backend_id, backend
|
|
137
|
+
matches = []
|
|
138
|
+
for descriptor in selected.list_backends():
|
|
139
|
+
backend = selected.get_backend(descriptor.backend_id)
|
|
140
|
+
if backend.accepts_reference(native_ref, kind):
|
|
141
|
+
matches.append((descriptor.backend_id, backend))
|
|
142
|
+
if len(matches) != 1:
|
|
143
|
+
raise WorkflowError(
|
|
144
|
+
f"native {kind} reference has no unique registered backend; select backend_id explicitly"
|
|
145
|
+
)
|
|
146
|
+
identifier, backend = matches[0]
|
|
147
|
+
return selected, identifier, backend
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _reference(backend: WorkflowBackend, native_ref: Any, kind: str) -> ReferenceBinding:
|
|
151
|
+
binding = backend.reference(native_ref, kind)
|
|
152
|
+
if not isinstance(binding, ReferenceBinding):
|
|
153
|
+
raise WorkflowError("backend reference reader must return ReferenceBinding")
|
|
154
|
+
if binding.record_id is None:
|
|
155
|
+
reason = binding.metadata.get("reason")
|
|
156
|
+
raise WorkflowError(reason if isinstance(reason, str) and reason else "no readable native record identity")
|
|
157
|
+
return binding
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _case_views(rows: Iterable[CaseView]) -> Iterable[CaseView]:
|
|
161
|
+
for row in rows:
|
|
162
|
+
if not isinstance(row, CaseView):
|
|
163
|
+
raise WorkflowError("backend case access must produce CaseView values")
|
|
164
|
+
yield row
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _assessment_views(rows: Iterable[Assessment], case_id: str | None = None) -> Iterable[Assessment]:
|
|
168
|
+
for row in rows:
|
|
169
|
+
if not isinstance(row, Assessment):
|
|
170
|
+
raise WorkflowError("backend assessment access must produce Assessment values")
|
|
171
|
+
if case_id is not None and row.case_id != case_id:
|
|
172
|
+
raise WorkflowError("backend returned assessments outside the selected case")
|
|
173
|
+
yield row
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
@dataclass(frozen=True)
|
|
177
|
+
class Suite:
|
|
178
|
+
"""Reusable registered cases, criteria, and execution requirements.
|
|
179
|
+
|
|
180
|
+
A native pack can be selected through ``from_pack`` when its adapter is
|
|
181
|
+
registered. ``from_registry`` selects custom extensions through the same
|
|
182
|
+
flow. Registration metadata does not establish execution admission.
|
|
183
|
+
"""
|
|
184
|
+
|
|
185
|
+
suite_id: str
|
|
186
|
+
registry: WorkflowRegistry | None = field(default=None, repr=False, compare=False)
|
|
187
|
+
_definition: SuiteDefinition = field(init=False, repr=False)
|
|
188
|
+
|
|
189
|
+
def __post_init__(self) -> None:
|
|
190
|
+
registry = _registry(self.registry)
|
|
191
|
+
definition, _ = registry.resolve_suite(self.suite_id)
|
|
192
|
+
object.__setattr__(self, "registry", registry)
|
|
193
|
+
object.__setattr__(self, "_definition", definition)
|
|
194
|
+
|
|
195
|
+
@classmethod
|
|
196
|
+
def from_pack(cls, pack_id: str, *, registry: WorkflowRegistry | None = None) -> Suite:
|
|
197
|
+
"""Select a registered native adapter through its existing pack ID."""
|
|
198
|
+
return cls(pack_id, registry=registry)
|
|
199
|
+
|
|
200
|
+
@classmethod
|
|
201
|
+
def from_registry(cls, suite_id: str, *, registry: WorkflowRegistry) -> Suite:
|
|
202
|
+
"""Select a suite from explicit operator-provided extension bindings."""
|
|
203
|
+
return cls(suite_id, registry=registry)
|
|
204
|
+
|
|
205
|
+
def _registered(self, *, scope: str | None = None,
|
|
206
|
+
mode: str | None = None) -> tuple[SuiteDefinition, WorkflowBackend]:
|
|
207
|
+
assert self.registry is not None
|
|
208
|
+
return self.registry.verify_suite(self._definition, scope=scope, mode=mode)
|
|
209
|
+
|
|
210
|
+
@property
|
|
211
|
+
def definition(self) -> SuiteDefinition:
|
|
212
|
+
return self._registered()[0]
|
|
213
|
+
|
|
214
|
+
@property
|
|
215
|
+
def backend_id(self) -> str:
|
|
216
|
+
return self.definition.backend_id
|
|
217
|
+
|
|
218
|
+
@property
|
|
219
|
+
def criterion_ids(self) -> tuple[str, ...]:
|
|
220
|
+
return self.definition.criterion_ids
|
|
221
|
+
|
|
222
|
+
@property
|
|
223
|
+
def metric_ids(self) -> tuple[str, ...]:
|
|
224
|
+
return self.definition.metric_ids
|
|
225
|
+
|
|
226
|
+
@property
|
|
227
|
+
def evaluator_ids(self) -> tuple[str, ...]:
|
|
228
|
+
return self.definition.evaluator_ids
|
|
229
|
+
|
|
230
|
+
@property
|
|
231
|
+
def datasets(self) -> tuple[DatasetDefinition, ...]:
|
|
232
|
+
return self.definition.datasets
|
|
233
|
+
|
|
234
|
+
def cases(self) -> Iterable[CaseView]:
|
|
235
|
+
"""Read candidate-permitted cases; preserve a backend's lazy iteration."""
|
|
236
|
+
definition, backend = self._registered()
|
|
237
|
+
rows = backend.cases(definition)
|
|
238
|
+
if isinstance(rows, tuple):
|
|
239
|
+
if any(not isinstance(row, CaseView) for row in rows):
|
|
240
|
+
raise WorkflowError("backend case access must produce CaseView values")
|
|
241
|
+
return rows
|
|
242
|
+
try:
|
|
243
|
+
iter(rows)
|
|
244
|
+
except TypeError:
|
|
245
|
+
raise WorkflowError("backend case access must return an iterable") from None
|
|
246
|
+
return _case_views(rows)
|
|
247
|
+
|
|
248
|
+
def plan(self, candidate: Any, *, scope: str = "component", mode: str = "local",
|
|
249
|
+
parameters: Mapping[str, Any] | None = None,
|
|
250
|
+
runtime: Mapping[str, Any] | None = None) -> Plan:
|
|
251
|
+
"""Select a candidate and supported conditions; the backend validates them."""
|
|
252
|
+
return Plan(self, candidate, scope=scope, mode=mode,
|
|
253
|
+
parameters={} if parameters is None else parameters,
|
|
254
|
+
runtime={} if runtime is None else runtime)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@dataclass(frozen=True)
|
|
258
|
+
class Plan:
|
|
259
|
+
"""Selected suite, opaque candidate, conditions, and backend parameters."""
|
|
260
|
+
|
|
261
|
+
suite: Suite
|
|
262
|
+
candidate: Any = field(repr=False, compare=False)
|
|
263
|
+
scope: str = "component"
|
|
264
|
+
mode: str = "local"
|
|
265
|
+
parameters: Mapping[str, Any] = field(default_factory=dict)
|
|
266
|
+
runtime: Mapping[str, Any] = field(default_factory=dict)
|
|
267
|
+
|
|
268
|
+
def __post_init__(self) -> None:
|
|
269
|
+
if not isinstance(self.suite, Suite):
|
|
270
|
+
raise WorkflowError("plan requires a Suite")
|
|
271
|
+
definition, _ = self.suite._registered(scope=self.scope, mode=self.mode)
|
|
272
|
+
selected = PlanBinding(definition, self.candidate, self.scope, self.mode,
|
|
273
|
+
self.parameters, self.runtime)
|
|
274
|
+
object.__setattr__(self, "parameters", selected.parameters)
|
|
275
|
+
object.__setattr__(self, "runtime", selected.runtime)
|
|
276
|
+
|
|
277
|
+
def _selected(self) -> tuple[PlanBinding, WorkflowBackend]:
|
|
278
|
+
definition, backend = self.suite._registered(scope=self.scope, mode=self.mode)
|
|
279
|
+
return PlanBinding(definition, self.candidate, self.scope, self.mode,
|
|
280
|
+
self.parameters, self.runtime), backend
|
|
281
|
+
|
|
282
|
+
def resolve(self) -> ResolvedPlan:
|
|
283
|
+
"""Bind backend metadata without asset reads or candidate execution."""
|
|
284
|
+
selected, backend = self._selected()
|
|
285
|
+
return ResolvedPlan(self, backend.resolve(selected))
|
|
286
|
+
|
|
287
|
+
def run(self, output_dir: str | Path | None = None) -> Run:
|
|
288
|
+
"""Resolve, admit, and execute. Saved assessment remains separate."""
|
|
289
|
+
from .execution_capabilities import validate_execution_requirement
|
|
290
|
+
_, backend = self._selected()
|
|
291
|
+
validate_execution_requirement(backend.descriptor, scope=self.scope, mode=self.mode,
|
|
292
|
+
operation="execute")
|
|
293
|
+
return self.resolve().admit().execute(output_dir)
|
|
294
|
+
|
|
295
|
+
def submission_intent(self, *, idempotency_key: str | None = None) -> SubmissionIntent:
|
|
296
|
+
"""Retain the recovery identity before a hosted submission attempt."""
|
|
297
|
+
_, backend = self._selected()
|
|
298
|
+
_async_backend(backend, "sphynx.async-submission.v1")
|
|
299
|
+
return SubmissionIntent(uuid.uuid4().hex if idempotency_key is None else idempotency_key,
|
|
300
|
+
self.resolve().binding.identity)
|
|
301
|
+
|
|
302
|
+
def submit(self, *, intent: SubmissionIntent | None = None,
|
|
303
|
+
idempotency_key: str | None = None) -> Submission:
|
|
304
|
+
"""Submit once. Retain intent to recover an uncertain POST response."""
|
|
305
|
+
if intent is not None and idempotency_key is not None:
|
|
306
|
+
raise WorkflowError("provide intent or idempotency_key, not both")
|
|
307
|
+
from .execution_capabilities import validate_execution_requirement
|
|
308
|
+
_, backend = self._selected()
|
|
309
|
+
validate_execution_requirement(backend.descriptor, scope=self.scope, mode=self.mode,
|
|
310
|
+
operation="submit")
|
|
311
|
+
resolved = self.resolve()
|
|
312
|
+
selected, backend = resolved._verified()
|
|
313
|
+
asynchronous = _async_backend(backend, "sphynx.async-submission.v1")
|
|
314
|
+
intent = intent if intent is not None else SubmissionIntent(
|
|
315
|
+
uuid.uuid4().hex if idempotency_key is None else idempotency_key, resolved.binding.identity)
|
|
316
|
+
if not isinstance(intent, SubmissionIntent) or contract_digest(intent.request_identity) != contract_digest(resolved.binding.identity):
|
|
317
|
+
raise WorkflowError("submission intent does not match the resolved plan")
|
|
318
|
+
admission = resolved.admit()
|
|
319
|
+
backend.recheck(selected, resolved.binding, admission.binding)
|
|
320
|
+
try:
|
|
321
|
+
native = asynchronous.submit(selected, resolved.binding, admission.binding, intent.idempotency_key)
|
|
322
|
+
handle = Submission(native, registry=self.suite.registry, backend_id=self.suite.backend_id)
|
|
323
|
+
if (handle.snapshot.idempotency_key != intent.idempotency_key or
|
|
324
|
+
contract_digest(handle.snapshot.request_identity) != contract_digest(intent.request_identity)):
|
|
325
|
+
raise WorkflowError("submission response differs from its retained intent")
|
|
326
|
+
return handle
|
|
327
|
+
except SubmissionError:
|
|
328
|
+
raise
|
|
329
|
+
except Exception as exc:
|
|
330
|
+
raise SubmissionError(str(exc), intent=intent) from exc
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
@dataclass(frozen=True)
|
|
334
|
+
class ResolvedPlan:
|
|
335
|
+
"""Backend resolution metadata, separate from execution admission."""
|
|
336
|
+
|
|
337
|
+
plan: Plan
|
|
338
|
+
binding: ResolvedBinding
|
|
339
|
+
|
|
340
|
+
def __post_init__(self) -> None:
|
|
341
|
+
if not isinstance(self.plan, Plan) or not isinstance(self.binding, ResolvedBinding):
|
|
342
|
+
raise WorkflowError("resolved plan requires a Plan and ResolvedBinding")
|
|
343
|
+
self._verified()
|
|
344
|
+
|
|
345
|
+
def _verified(self) -> tuple[PlanBinding, WorkflowBackend]:
|
|
346
|
+
selected, backend = self.plan._selected()
|
|
347
|
+
current = backend.resolve(selected)
|
|
348
|
+
if not isinstance(current, ResolvedBinding):
|
|
349
|
+
raise WorkflowError("backend resolution must return ResolvedBinding")
|
|
350
|
+
if "evaluation_protocol" in selected.suite.metadata:
|
|
351
|
+
from .evaluation_protocols import EvaluationProtocol
|
|
352
|
+
protocol = EvaluationProtocol.from_dict(selected.suite.metadata["evaluation_protocol"])
|
|
353
|
+
if current.identity.get("evaluation_protocol_digest") != protocol.protocol_digest:
|
|
354
|
+
raise WorkflowError("backend resolution must bind the evaluation protocol digest")
|
|
355
|
+
if current.resolution_digest != self.binding.resolution_digest:
|
|
356
|
+
raise WorkflowError("resolved settings or candidate identity changed; resolve the plan again")
|
|
357
|
+
return selected, backend
|
|
358
|
+
|
|
359
|
+
@property
|
|
360
|
+
def native_invocation(self) -> Any:
|
|
361
|
+
"""Compatibility access to the backend's opaque native resolution."""
|
|
362
|
+
return self.binding.native_ref
|
|
363
|
+
|
|
364
|
+
@property
|
|
365
|
+
def status(self) -> str:
|
|
366
|
+
return self.binding.status
|
|
367
|
+
|
|
368
|
+
@property
|
|
369
|
+
def artifact_bytes_verified(self) -> bool:
|
|
370
|
+
return self.binding.artifact_bytes_verified
|
|
371
|
+
|
|
372
|
+
@property
|
|
373
|
+
def plan_digest(self) -> str:
|
|
374
|
+
return contract_digest(self.to_dict())
|
|
375
|
+
|
|
376
|
+
def to_dict(self) -> dict[str, Any]:
|
|
377
|
+
definition, backend = self.plan.suite._registered(scope=self.plan.scope, mode=self.plan.mode)
|
|
378
|
+
return {"schema_version": "sphynx.plan-view.v2", "status": self.status,
|
|
379
|
+
"suite": definition.to_dict(), "backend": backend.descriptor.to_dict(),
|
|
380
|
+
"scope": self.plan.scope, "mode": self.plan.mode,
|
|
381
|
+
"parameters": _thaw(self.plan.parameters), "runtime": _thaw(self.plan.runtime),
|
|
382
|
+
"resolution": self.binding.to_dict(), "artifact_bytes_verified": False}
|
|
383
|
+
|
|
384
|
+
def admit(self) -> Admission:
|
|
385
|
+
"""Apply backend-owned identity, asset, access and capability checks."""
|
|
386
|
+
selected, backend = self._verified()
|
|
387
|
+
admitted = backend.admit(selected, self.binding)
|
|
388
|
+
return Admission(self, admitted, _token=_ADMISSION_TOKEN)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
@dataclass(frozen=True)
|
|
392
|
+
class Admission:
|
|
393
|
+
"""Backend admission facts, rechecked before candidate execution."""
|
|
394
|
+
|
|
395
|
+
resolved: ResolvedPlan
|
|
396
|
+
binding: AdmissionBinding
|
|
397
|
+
_token: InitVar[object] = None
|
|
398
|
+
|
|
399
|
+
def __post_init__(self, _token: object) -> None:
|
|
400
|
+
if _token is not _ADMISSION_TOKEN:
|
|
401
|
+
raise WorkflowError("create admission through resolved_plan.admit()")
|
|
402
|
+
if not isinstance(self.resolved, ResolvedPlan) or not isinstance(self.binding, AdmissionBinding):
|
|
403
|
+
raise WorkflowError("admission requires a ResolvedPlan and AdmissionBinding")
|
|
404
|
+
if self.binding.resolution_digest != self.resolved.binding.resolution_digest:
|
|
405
|
+
raise WorkflowError("admission does not match the resolved plan")
|
|
406
|
+
|
|
407
|
+
def __getattr__(self, name: str) -> Any:
|
|
408
|
+
# Native convenience names remain available without baking their fields
|
|
409
|
+
# into the common schema. The fact's meaning stays backend-owned.
|
|
410
|
+
binding = object.__getattribute__(self, "binding")
|
|
411
|
+
if name in binding.facts:
|
|
412
|
+
return binding.facts[name]
|
|
413
|
+
raise AttributeError(name)
|
|
414
|
+
|
|
415
|
+
@property
|
|
416
|
+
def facts(self) -> Mapping[str, Any]:
|
|
417
|
+
return self.binding.facts
|
|
418
|
+
|
|
419
|
+
@property
|
|
420
|
+
def status(self) -> str:
|
|
421
|
+
return self.binding.status
|
|
422
|
+
|
|
423
|
+
@property
|
|
424
|
+
def trust_tier(self) -> str:
|
|
425
|
+
return self.binding.trust_tier
|
|
426
|
+
|
|
427
|
+
@property
|
|
428
|
+
def artifact_bytes_verified(self) -> bool:
|
|
429
|
+
return self.binding.artifact_bytes_verified
|
|
430
|
+
|
|
431
|
+
@property
|
|
432
|
+
def candidate_source_bytes_verified(self) -> bool:
|
|
433
|
+
return self.binding.candidate_source_bytes_verified
|
|
434
|
+
|
|
435
|
+
def to_dict(self) -> dict[str, Any]:
|
|
436
|
+
return {"schema_version": "sphynx.admission-view.v2",
|
|
437
|
+
"plan_digest": self.resolved.plan_digest, **self.binding.to_dict()}
|
|
438
|
+
|
|
439
|
+
def execute(self, output_dir: str | Path | None = None) -> Run:
|
|
440
|
+
selected, backend = self.resolved._verified()
|
|
441
|
+
from .execution_capabilities import validate_execution_requirement
|
|
442
|
+
validate_execution_requirement(backend.descriptor, scope=selected.scope, mode=selected.mode,
|
|
443
|
+
operation="execute")
|
|
444
|
+
backend.recheck(selected, self.resolved.binding, self.binding)
|
|
445
|
+
native = backend.execute(selected, self.resolved.binding, self.binding, output_dir)
|
|
446
|
+
return Run(native, registry=self.resolved.plan.suite.registry,
|
|
447
|
+
backend_id=self.resolved.plan.suite.backend_id)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
@dataclass(frozen=True)
|
|
451
|
+
class Run:
|
|
452
|
+
"""Opaque native execution evidence bound to its registered backend."""
|
|
453
|
+
|
|
454
|
+
native_ref: Any = field(repr=False)
|
|
455
|
+
registry: WorkflowRegistry | None = field(default=None, repr=False, compare=False)
|
|
456
|
+
backend_id: str | None = None
|
|
457
|
+
|
|
458
|
+
def __post_init__(self) -> None:
|
|
459
|
+
registry, identifier, _ = _reference_backend(self.native_ref, "run", self.registry, self.backend_id)
|
|
460
|
+
object.__setattr__(self, "registry", registry)
|
|
461
|
+
object.__setattr__(self, "backend_id", identifier)
|
|
462
|
+
|
|
463
|
+
def _backend(self) -> WorkflowBackend:
|
|
464
|
+
assert self.registry is not None and self.backend_id is not None
|
|
465
|
+
backend = self.registry.get_backend(self.backend_id)
|
|
466
|
+
if not backend.accepts_reference(self.native_ref, "run"):
|
|
467
|
+
raise WorkflowError("backend no longer recognizes the native run reference")
|
|
468
|
+
return backend
|
|
469
|
+
|
|
470
|
+
@property
|
|
471
|
+
def run_id(self) -> str:
|
|
472
|
+
binding = _reference(self._backend(), self.native_ref, "run")
|
|
473
|
+
assert binding.record_id is not None
|
|
474
|
+
return binding.run_id if binding.run_id is not None else binding.record_id
|
|
475
|
+
|
|
476
|
+
@property
|
|
477
|
+
def status(self) -> str:
|
|
478
|
+
return _reference(self._backend(), self.native_ref, "run").status
|
|
479
|
+
|
|
480
|
+
@property
|
|
481
|
+
def trust_tier(self) -> str:
|
|
482
|
+
return _reference(self._backend(), self.native_ref, "run").trust_tier
|
|
483
|
+
|
|
484
|
+
def assess(self) -> Report:
|
|
485
|
+
"""Assess saved evidence without executing the candidate again."""
|
|
486
|
+
native = self._backend().assess(self.native_ref)
|
|
487
|
+
return Report(native, registry=self.registry, backend_id=self.backend_id)
|
|
488
|
+
|
|
489
|
+
def start_grading(self, *, idempotency_key: str | None = None) -> GradingPass:
|
|
490
|
+
"""Request a new grading pass over saved evidence without execution."""
|
|
491
|
+
backend = _async_backend(self._backend(), "sphynx.async-grading.v1")
|
|
492
|
+
key = uuid.uuid4().hex if idempotency_key is None else idempotency_key
|
|
493
|
+
_text(key, "idempotency_key")
|
|
494
|
+
run_id = self.run_id
|
|
495
|
+
try:
|
|
496
|
+
native = backend.start_grading(self.native_ref, key)
|
|
497
|
+
result = GradingPass(native, registry=self.registry, backend_id=self.backend_id)
|
|
498
|
+
if result.snapshot.idempotency_key != key or result.snapshot.run_id != run_id:
|
|
499
|
+
raise WorkflowError("new grading pass differs from the requested run or recovery key")
|
|
500
|
+
return result
|
|
501
|
+
except Exception as exc:
|
|
502
|
+
raise GradingRequestError(str(exc), run_id=run_id, idempotency_key=key) from exc
|
|
503
|
+
|
|
504
|
+
def stage_evidence(self, selector: StageSelector) -> tuple[StageEvidence, ...]:
|
|
505
|
+
"""Read verified permitted evidence for exactly one stage instance."""
|
|
506
|
+
return _stages(self._backend(), self.native_ref, selector)
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
@dataclass(frozen=True)
|
|
510
|
+
class Report:
|
|
511
|
+
"""Backend-verified permitted presentation and selected assessments.
|
|
512
|
+
|
|
513
|
+
The backend owns its report layout and visibility rules. ``metrics`` and
|
|
514
|
+
``counts`` are optional mapping-shaped summary conveniences. Other layouts
|
|
515
|
+
remain unchanged through the full projection. No verdict is inferred.
|
|
516
|
+
"""
|
|
517
|
+
|
|
518
|
+
native_ref: Any = field(repr=False)
|
|
519
|
+
registry: WorkflowRegistry | None = field(default=None, repr=False, compare=False)
|
|
520
|
+
backend_id: str | None = None
|
|
521
|
+
|
|
522
|
+
def __post_init__(self) -> None:
|
|
523
|
+
registry, identifier, backend = _reference_backend(self.native_ref, "grading", self.registry, self.backend_id)
|
|
524
|
+
_reference(backend, self.native_ref, "grading")
|
|
525
|
+
object.__setattr__(self, "registry", registry)
|
|
526
|
+
object.__setattr__(self, "backend_id", identifier)
|
|
527
|
+
|
|
528
|
+
def _backend(self) -> WorkflowBackend:
|
|
529
|
+
assert self.registry is not None and self.backend_id is not None
|
|
530
|
+
backend = self.registry.get_backend(self.backend_id)
|
|
531
|
+
if not backend.accepts_reference(self.native_ref, "grading"):
|
|
532
|
+
raise WorkflowError("backend no longer recognizes the native grading reference")
|
|
533
|
+
return backend
|
|
534
|
+
|
|
535
|
+
def _projection(self) -> ReportProjection:
|
|
536
|
+
projection = self._backend().report(self.native_ref)
|
|
537
|
+
if not isinstance(projection, ReportProjection):
|
|
538
|
+
raise WorkflowError("backend report must implement ReportProjection")
|
|
539
|
+
return projection
|
|
540
|
+
|
|
541
|
+
@property
|
|
542
|
+
def grading_id(self) -> str:
|
|
543
|
+
binding = _reference(self._backend(), self.native_ref, "grading")
|
|
544
|
+
assert binding.record_id is not None
|
|
545
|
+
return binding.record_id
|
|
546
|
+
|
|
547
|
+
@property
|
|
548
|
+
def run_id(self) -> str | None:
|
|
549
|
+
return _reference(self._backend(), self.native_ref, "grading").run_id
|
|
550
|
+
|
|
551
|
+
@property
|
|
552
|
+
def status(self) -> str:
|
|
553
|
+
return _reference(self._backend(), self.native_ref, "grading").status
|
|
554
|
+
|
|
555
|
+
@property
|
|
556
|
+
def trust_tier(self) -> str:
|
|
557
|
+
return _reference(self._backend(), self.native_ref, "grading").trust_tier
|
|
558
|
+
|
|
559
|
+
@property
|
|
560
|
+
def metrics(self) -> Mapping[str, Any]:
|
|
561
|
+
return self._summary_mapping("metrics")
|
|
562
|
+
|
|
563
|
+
@property
|
|
564
|
+
def counts(self) -> Mapping[str, Any]:
|
|
565
|
+
return self._summary_mapping("counts")
|
|
566
|
+
|
|
567
|
+
def _summary_mapping(self, key: str) -> Mapping[str, Any]:
|
|
568
|
+
data = self.to_dict()
|
|
569
|
+
if key not in data:
|
|
570
|
+
return {}
|
|
571
|
+
value = data[key]
|
|
572
|
+
if not isinstance(value, Mapping):
|
|
573
|
+
raise WorkflowError(
|
|
574
|
+
f"backend {key} summary is not a mapping; inspect the complete report"
|
|
575
|
+
)
|
|
576
|
+
return value
|
|
577
|
+
|
|
578
|
+
def to_dict(self) -> dict[str, Any]:
|
|
579
|
+
data = self._projection().to_dict()
|
|
580
|
+
if not isinstance(data, dict):
|
|
581
|
+
raise WorkflowError("backend report projection must return a dictionary")
|
|
582
|
+
return data
|
|
583
|
+
|
|
584
|
+
def to_json(self) -> str:
|
|
585
|
+
value = self._projection().to_json()
|
|
586
|
+
if not isinstance(value, str):
|
|
587
|
+
raise WorkflowError("backend report JSON projection must return text")
|
|
588
|
+
return value
|
|
589
|
+
|
|
590
|
+
def to_markdown(self) -> str:
|
|
591
|
+
value = self._projection().to_markdown()
|
|
592
|
+
if not isinstance(value, str):
|
|
593
|
+
raise WorkflowError("backend report Markdown projection must return text")
|
|
594
|
+
return value
|
|
595
|
+
|
|
596
|
+
def decide(self, policy_id: str, *, baseline: Report | None = None,
|
|
597
|
+
parameters: Mapping[str, Any] | None = None) -> Verdict:
|
|
598
|
+
"""Apply an explicit registered policy to permitted saved report data."""
|
|
599
|
+
from .workflow_decisions import decide_report
|
|
600
|
+
|
|
601
|
+
return decide_report(self, policy_id, baseline=baseline, parameters=parameters)
|
|
602
|
+
|
|
603
|
+
def report_assessments(self) -> tuple[ReportAssessment, ...]:
|
|
604
|
+
"""Read saved report-scoped assessments; never execute a candidate."""
|
|
605
|
+
backend = self._backend()
|
|
606
|
+
reader = getattr(backend, "report_assessments", None)
|
|
607
|
+
if not callable(reader):
|
|
608
|
+
raise WorkflowError("backend does not support report-scoped assessment reads")
|
|
609
|
+
expected = _reference(backend, self.native_ref, "grading")
|
|
610
|
+
rows = tuple(reader(self.native_ref))
|
|
611
|
+
if any(not isinstance(row, ReportAssessment) or row.run_id != expected.run_id
|
|
612
|
+
or row.grading_id != expected.record_id for row in rows):
|
|
613
|
+
raise WorkflowError("backend returned invalid report-scoped assessment identity")
|
|
614
|
+
return rows
|
|
615
|
+
|
|
616
|
+
def assessments(self, case_id: str | None = None) -> Iterable[Assessment]:
|
|
617
|
+
"""Read typed assessments while retaining a backend's lazy access."""
|
|
618
|
+
if case_id is not None:
|
|
619
|
+
_text(case_id, "case_id")
|
|
620
|
+
values = self._backend().assessments(self.native_ref, case_id)
|
|
621
|
+
if isinstance(values, tuple):
|
|
622
|
+
if any(not isinstance(value, Assessment) for value in values):
|
|
623
|
+
raise WorkflowError("backend assessment access must produce Assessment values")
|
|
624
|
+
if case_id is not None and any(value.case_id != case_id for value in values):
|
|
625
|
+
raise WorkflowError("backend returned assessments outside the selected case")
|
|
626
|
+
return values
|
|
627
|
+
try:
|
|
628
|
+
iter(values)
|
|
629
|
+
except TypeError:
|
|
630
|
+
raise WorkflowError("backend assessment access must return an iterable") from None
|
|
631
|
+
return _assessment_views(values, case_id)
|
|
632
|
+
|
|
633
|
+
def select_assessments(self, selector: AssessmentSelector) -> Iterable[Assessment]:
|
|
634
|
+
"""Filter permitted assessments lazily, retaining exact case/trial identity."""
|
|
635
|
+
if not isinstance(selector, AssessmentSelector):
|
|
636
|
+
raise WorkflowError("selector must be AssessmentSelector")
|
|
637
|
+
return selector.select(self.assessments(case_id=selector.case_id))
|
|
638
|
+
|
|
639
|
+
def compare(self, baseline: Report) -> Any:
|
|
640
|
+
"""Apply the shared backend's comparison policy to baseline then candidate."""
|
|
641
|
+
if not isinstance(baseline, Report):
|
|
642
|
+
raise WorkflowError("baseline must be Report")
|
|
643
|
+
candidate_backend, baseline_backend = self._backend(), baseline._backend()
|
|
644
|
+
if candidate_backend.descriptor.descriptor_digest != baseline_backend.descriptor.descriptor_digest:
|
|
645
|
+
raise WorkflowError("reports use different backend bindings")
|
|
646
|
+
return candidate_backend.compare(baseline.native_ref, self.native_ref)
|
|
647
|
+
|
|
648
|
+
def compare_view(self, baseline: Report) -> ComparisonView:
|
|
649
|
+
"""Read an optional typed native comparison; never run or regrade work."""
|
|
650
|
+
from .comparison_views import ComparisonView
|
|
651
|
+
if not isinstance(baseline, Report):
|
|
652
|
+
raise WorkflowError("baseline must be Report")
|
|
653
|
+
backend, other = self._backend(), baseline._backend()
|
|
654
|
+
if backend.descriptor.descriptor_digest != other.descriptor.descriptor_digest:
|
|
655
|
+
raise WorkflowError("reports use different backend bindings")
|
|
656
|
+
reader = getattr(backend, "compare_view", None)
|
|
657
|
+
if not callable(reader):
|
|
658
|
+
raise WorkflowError("backend has no comparison view adapter; use compare() for its native result")
|
|
659
|
+
left = _reference(other, baseline.native_ref, "grading")
|
|
660
|
+
right = _reference(backend, self.native_ref, "grading")
|
|
661
|
+
view = reader(baseline.native_ref, self.native_ref)
|
|
662
|
+
if not isinstance(view, ComparisonView):
|
|
663
|
+
raise WorkflowError("backend comparison view must be ComparisonView")
|
|
664
|
+
if contract_digest(view.backend) != backend.descriptor.descriptor_digest:
|
|
665
|
+
raise WorkflowError("comparison view uses a different backend binding")
|
|
666
|
+
for source, expected in ((view.baseline, left), (view.candidate, right)):
|
|
667
|
+
if any(source.get(key) != getattr(expected, native_key) for key, native_key in
|
|
668
|
+
(("grading_id", "record_id"), ("run_id", "run_id"), ("trust_tier", "trust_tier"), ("status", "status"))):
|
|
669
|
+
raise WorkflowError("comparison view refers to a different report identity/status")
|
|
670
|
+
return view
|
|
671
|
+
|
|
672
|
+
def stage_evidence(self, selector: StageSelector) -> tuple[StageEvidence, ...]:
|
|
673
|
+
"""Select stage evidence through this report's verified native binding."""
|
|
674
|
+
return _stages(self._backend(), self.native_ref, selector)
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
@dataclass(frozen=True)
|
|
678
|
+
class Submission:
|
|
679
|
+
"""An immutable view of submitted coordination work. Refresh returns a view."""
|
|
680
|
+
|
|
681
|
+
native_ref: Any = field(repr=False)
|
|
682
|
+
registry: WorkflowRegistry = field(repr=False, compare=False)
|
|
683
|
+
backend_id: str
|
|
684
|
+
snapshot: SubmissionSnapshot = field(init=False)
|
|
685
|
+
|
|
686
|
+
def __post_init__(self) -> None:
|
|
687
|
+
backend = _async_backend(self.registry.get_backend(self.backend_id), "sphynx.async-submission.v1")
|
|
688
|
+
snapshot = backend.submission_snapshot(self.native_ref)
|
|
689
|
+
if not isinstance(snapshot, SubmissionSnapshot):
|
|
690
|
+
raise WorkflowError("backend must provide SubmissionSnapshot")
|
|
691
|
+
object.__setattr__(self, "snapshot", snapshot)
|
|
692
|
+
|
|
693
|
+
@property
|
|
694
|
+
def submission_id(self) -> str:
|
|
695
|
+
return self.snapshot.submission_id
|
|
696
|
+
|
|
697
|
+
@property
|
|
698
|
+
def progress(self) -> ProgressView:
|
|
699
|
+
return self.snapshot.progress
|
|
700
|
+
|
|
701
|
+
@property
|
|
702
|
+
def status(self) -> str:
|
|
703
|
+
return self.snapshot.status
|
|
704
|
+
|
|
705
|
+
@property
|
|
706
|
+
def intent(self) -> SubmissionIntent:
|
|
707
|
+
return SubmissionIntent(self.snapshot.idempotency_key, self.snapshot.request_identity)
|
|
708
|
+
|
|
709
|
+
def refresh(self) -> Submission:
|
|
710
|
+
backend = _async_backend(self.registry.get_backend(self.backend_id), "sphynx.async-submission.v1")
|
|
711
|
+
updated = Submission(backend.refresh_submission(self.native_ref), self.registry, self.backend_id)
|
|
712
|
+
old, new = self.snapshot, updated.snapshot
|
|
713
|
+
if (old.submission_id != new.submission_id or old.idempotency_key != new.idempotency_key or
|
|
714
|
+
contract_digest(old.request_identity) != contract_digest(new.request_identity) or
|
|
715
|
+
any(getattr(old, name) is not None and getattr(old, name) != getattr(new, name)
|
|
716
|
+
for name in ("run_id", "grading_id")) or
|
|
717
|
+
(old.complete and not new.complete) or (old.failed and not new.failed)):
|
|
718
|
+
raise WorkflowError("submission identity or terminal state changed during polling")
|
|
719
|
+
return updated
|
|
720
|
+
|
|
721
|
+
def wait_for_run(self, *, timeout: float = 900, poll_interval: float = 2,
|
|
722
|
+
on_progress: Any = None) -> Run:
|
|
723
|
+
def ready(handle):
|
|
724
|
+
if handle.snapshot.native_run is None:
|
|
725
|
+
return None
|
|
726
|
+
run = Run(handle.snapshot.native_run, registry=self.registry, backend_id=self.backend_id)
|
|
727
|
+
if run.run_id != handle.snapshot.run_id:
|
|
728
|
+
raise WorkflowError("submission links a different native run")
|
|
729
|
+
return run
|
|
730
|
+
return _wait(self, ready, timeout=timeout, poll_interval=poll_interval, on_progress=on_progress)
|
|
731
|
+
|
|
732
|
+
def wait_for_report(self, *, timeout: float = 900, poll_interval: float = 2,
|
|
733
|
+
on_progress: Any = None) -> Report:
|
|
734
|
+
def ready(handle):
|
|
735
|
+
if handle.snapshot.native_grading is None:
|
|
736
|
+
return None
|
|
737
|
+
report = Report(handle.snapshot.native_grading, registry=self.registry, backend_id=self.backend_id)
|
|
738
|
+
if report.grading_id != handle.snapshot.grading_id or report.run_id != handle.snapshot.run_id:
|
|
739
|
+
raise WorkflowError("submission links a different native grading pass")
|
|
740
|
+
return report
|
|
741
|
+
return _wait(self, ready, timeout=timeout, poll_interval=poll_interval, on_progress=on_progress)
|
|
742
|
+
|
|
743
|
+
|
|
744
|
+
@dataclass(frozen=True)
|
|
745
|
+
class GradingPass:
|
|
746
|
+
"""A pending saved-evidence grading pass, separate from its final report."""
|
|
747
|
+
|
|
748
|
+
native_ref: Any = field(repr=False)
|
|
749
|
+
registry: WorkflowRegistry = field(repr=False, compare=False)
|
|
750
|
+
backend_id: str
|
|
751
|
+
snapshot: GradingSnapshot = field(init=False)
|
|
752
|
+
|
|
753
|
+
def __post_init__(self) -> None:
|
|
754
|
+
backend = _async_backend(self.registry.get_backend(self.backend_id), "sphynx.async-grading.v1")
|
|
755
|
+
snapshot = backend.grading_snapshot(self.native_ref)
|
|
756
|
+
if not isinstance(snapshot, GradingSnapshot):
|
|
757
|
+
raise WorkflowError("backend must provide GradingSnapshot")
|
|
758
|
+
object.__setattr__(self, "snapshot", snapshot)
|
|
759
|
+
|
|
760
|
+
@property
|
|
761
|
+
def grading_id(self) -> str:
|
|
762
|
+
return self.snapshot.grading_id
|
|
763
|
+
|
|
764
|
+
@property
|
|
765
|
+
def progress(self) -> ProgressView:
|
|
766
|
+
return self.snapshot.progress
|
|
767
|
+
|
|
768
|
+
@property
|
|
769
|
+
def status(self) -> str:
|
|
770
|
+
return self.snapshot.status
|
|
771
|
+
|
|
772
|
+
def refresh(self) -> GradingPass:
|
|
773
|
+
backend = _async_backend(self.registry.get_backend(self.backend_id), "sphynx.async-grading.v1")
|
|
774
|
+
updated = GradingPass(backend.refresh_grading(self.native_ref), self.registry, self.backend_id)
|
|
775
|
+
old, new = self.snapshot, updated.snapshot
|
|
776
|
+
if (old.grading_id != new.grading_id or old.run_id != new.run_id or
|
|
777
|
+
old.idempotency_key != new.idempotency_key or
|
|
778
|
+
contract_digest(old.request_identity) != contract_digest(new.request_identity) or
|
|
779
|
+
(old.complete and not new.complete) or (old.failed and not new.failed)):
|
|
780
|
+
raise WorkflowError("grading identity or terminal state changed during polling")
|
|
781
|
+
return updated
|
|
782
|
+
|
|
783
|
+
def wait_for_report(self, *, timeout: float = 900, poll_interval: float = 2,
|
|
784
|
+
on_progress: Any = None) -> Report:
|
|
785
|
+
def ready(handle):
|
|
786
|
+
if handle.snapshot.completed_ref is None:
|
|
787
|
+
return None
|
|
788
|
+
report = Report(handle.snapshot.completed_ref, registry=self.registry, backend_id=self.backend_id)
|
|
789
|
+
if report.grading_id != handle.grading_id or report.run_id != handle.snapshot.run_id:
|
|
790
|
+
raise WorkflowError("completed grading links a different report")
|
|
791
|
+
return report
|
|
792
|
+
return _wait(self, ready, timeout=timeout, poll_interval=poll_interval, on_progress=on_progress)
|