sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
|
@@ -0,0 +1,490 @@
|
|
|
1
|
+
"""Registered lifecycle drivers through the unchanged common workflow protocol.
|
|
2
|
+
|
|
3
|
+
Providers validate candidate-permitted cases. Evaluators, reducers, report
|
|
4
|
+
projections and comparisons are explicit operator registrations. No domain,
|
|
5
|
+
numeric reducer, quality threshold or protected-reference grant is implicit.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Mapping
|
|
10
|
+
import json
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from .assessment_contracts import Assessment, CaseView, EvidenceView, _verified_evaluator_context
|
|
14
|
+
from .lifecycle import (
|
|
15
|
+
GradingPolicy, LifecycleError, LifecycleGradingRef, LifecycleJournal,
|
|
16
|
+
LifecycleRunRef, RunPolicy, _driver_admission, _object, _raw, _read_json, _require,
|
|
17
|
+
_thaw, assess_lifecycle, execute_lifecycle,
|
|
18
|
+
verify_lifecycle_grading, verify_lifecycle_run,
|
|
19
|
+
)
|
|
20
|
+
from .workflow_contracts import (
|
|
21
|
+
AdmissionBinding, BackendDescriptor, PlanBinding, ReferenceBinding, ResolvedBinding,
|
|
22
|
+
ReportProjection, SuiteDefinition, WorkflowContractError, contract_digest,
|
|
23
|
+
)
|
|
24
|
+
from .workflow_registry import WorkflowRegistry
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
class _ComparisonMismatch(LifecycleError, WorkflowContractError):
|
|
28
|
+
"""Native comparison conditions differ; not an evidence/access failure."""
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def _comparison_require(condition, message):
|
|
32
|
+
if not condition:
|
|
33
|
+
raise _ComparisonMismatch(message)
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class LifecycleProjection:
|
|
37
|
+
"""A permitted explicit projection, without a compulsory numeric layout."""
|
|
38
|
+
def __init__(self, value, *, context=None):
|
|
39
|
+
self._value = _object(value, "report projection")
|
|
40
|
+
self._context = _object(context or {}, "report presentation context")
|
|
41
|
+
|
|
42
|
+
def to_dict(self):
|
|
43
|
+
return _thaw(self._value)
|
|
44
|
+
|
|
45
|
+
def to_json(self):
|
|
46
|
+
return _raw(self._value).decode("utf-8")
|
|
47
|
+
|
|
48
|
+
def to_markdown(self):
|
|
49
|
+
from .report_rendering import lifecycle_markdown
|
|
50
|
+
return lifecycle_markdown(self.to_dict(), _thaw(self._context))
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
class LifecycleBackend:
|
|
54
|
+
"""An operator-configured WorkflowBackend for versioned lifecycle records.
|
|
55
|
+
|
|
56
|
+
Plan parameters can select run_policy, grading_policy and serialized
|
|
57
|
+
resume/reuse references. References are local owner-readable records. Only
|
|
58
|
+
an explicit admission_verifier can claim external asset/source verification.
|
|
59
|
+
"""
|
|
60
|
+
def __init__(self, registry, suite, *, driver, evaluator_id, projection_schema_id,
|
|
61
|
+
comparison_policy_id, run_policy=None, grading_policy=None,
|
|
62
|
+
reference_access=None, admission_verifier=None, scopes=("component",),
|
|
63
|
+
modes=("local",), capabilities=(), evaluator_binding=None, report_binding=None,
|
|
64
|
+
include_emitted_evidence=False):
|
|
65
|
+
_require(isinstance(registry, WorkflowRegistry) and isinstance(suite, SuiteDefinition),
|
|
66
|
+
"lifecycle backend requires registry and suite")
|
|
67
|
+
_require(evaluator_id in suite.evaluator_ids, "lifecycle evaluator must be declared by the suite")
|
|
68
|
+
self.registry, self.suite, self.driver = registry, suite, driver
|
|
69
|
+
self.evaluator_id, self.projection_schema_id = evaluator_id, projection_schema_id
|
|
70
|
+
self.comparison_policy_id = comparison_policy_id
|
|
71
|
+
self.run_policy, self.grading_policy = run_policy or RunPolicy(), grading_policy or GradingPolicy()
|
|
72
|
+
self.reference_access, self.admission_verifier = reference_access, admission_verifier
|
|
73
|
+
_require(type(include_emitted_evidence) is bool, "include_emitted_evidence must be boolean")
|
|
74
|
+
self.include_emitted_evidence = include_emitted_evidence
|
|
75
|
+
self.evaluator_binding, self.report_binding = evaluator_binding, report_binding
|
|
76
|
+
if evaluator_binding is not None or report_binding is not None:
|
|
77
|
+
from .evaluator_bindings import _binding_dependencies
|
|
78
|
+
if evaluator_binding is not None:
|
|
79
|
+
_binding_dependencies(registry, evaluator_binding)
|
|
80
|
+
_require(evaluator_binding.target in ("case", "stage")
|
|
81
|
+
and evaluator_binding.evaluator.identity_id == evaluator_id,
|
|
82
|
+
"lifecycle case/stage binding must match the selected evaluator")
|
|
83
|
+
if evaluator_binding.target == "stage":
|
|
84
|
+
_require(callable(getattr(self, "evaluator_stage_evidence", None)),
|
|
85
|
+
"stage binding requires an explicit verified stage reader")
|
|
86
|
+
if report_binding is not None:
|
|
87
|
+
_binding_dependencies(registry, report_binding)
|
|
88
|
+
_require(report_binding.target == "report"
|
|
89
|
+
and report_binding.evaluator.identity_id in suite.evaluator_ids,
|
|
90
|
+
"report binding must select a declared report evaluator")
|
|
91
|
+
self._validate_report_policy(self.grading_policy)
|
|
92
|
+
_driver_admission(driver, self.run_policy)
|
|
93
|
+
self.cancellation = None
|
|
94
|
+
self.descriptor = BackendDescriptor(suite.backend_id, "v1", "sphynx.lifecycle.backend.v1",
|
|
95
|
+
tuple(dict.fromkeys(("sphynx.lifecycle.run.v1", "sphynx.saved-assessment.v1",
|
|
96
|
+
*driver.descriptor.capabilities, *capabilities))), tuple(scopes), tuple(modes),
|
|
97
|
+
suite_ids=(suite.suite_id,), metadata={"trust_tier": "local_unverified", "driver": driver.descriptor.to_dict()})
|
|
98
|
+
|
|
99
|
+
def describe(self, suite_id):
|
|
100
|
+
_require(suite_id == self.suite.suite_id, "unknown lifecycle suite")
|
|
101
|
+
return self.suite
|
|
102
|
+
|
|
103
|
+
def _validate_report_policy(self, policy):
|
|
104
|
+
if self.report_binding is not None:
|
|
105
|
+
_require(isinstance(policy, GradingPolicy) and policy == GradingPolicy(),
|
|
106
|
+
"report bindings currently require default grading policy; report retries/budgets are unsupported")
|
|
107
|
+
|
|
108
|
+
def _options(self, plan):
|
|
109
|
+
_require(isinstance(plan, PlanBinding) and plan.suite.definition_digest == self.suite.definition_digest,
|
|
110
|
+
"lifecycle suite definition changed")
|
|
111
|
+
_require(plan.scope in self.descriptor.supported_scopes and plan.mode in self.descriptor.supported_modes,
|
|
112
|
+
"unsupported lifecycle scope/mode")
|
|
113
|
+
_require(set(plan.parameters) <= {"run_policy", "grading_policy", "resume", "reuse"},
|
|
114
|
+
"unsupported lifecycle parameters")
|
|
115
|
+
if plan.runtime:
|
|
116
|
+
validate = getattr(self.driver, "validate_runtime", None)
|
|
117
|
+
_require(callable(validate) and validate(plan.runtime) is None, "driver does not support runtime overrides")
|
|
118
|
+
run = RunPolicy.from_dict(plan.parameters["run_policy"]) if "run_policy" in plan.parameters else self.run_policy
|
|
119
|
+
grading = GradingPolicy.from_dict(plan.parameters["grading_policy"]) if "grading_policy" in plan.parameters else self.grading_policy
|
|
120
|
+
self._validate_report_policy(grading)
|
|
121
|
+
_driver_admission(self.driver, run)
|
|
122
|
+
if run.reducer_id is not None:
|
|
123
|
+
self.registry.get_policy_callable(run.reducer_id, kind="aggregation")
|
|
124
|
+
return run, grading
|
|
125
|
+
|
|
126
|
+
def _source(self):
|
|
127
|
+
seen = set()
|
|
128
|
+
for dataset in self.suite.datasets:
|
|
129
|
+
provider = self.registry.get_provider(dataset.provider_id)
|
|
130
|
+
schema = self.registry.get_schema(dataset.input_schema_id)
|
|
131
|
+
for case in provider.cases(dataset.selection):
|
|
132
|
+
_require(isinstance(case, CaseView) and case.schema_version == schema.schema_id,
|
|
133
|
+
"provider case input schema differs")
|
|
134
|
+
_require(case.case_id not in seen, "duplicate case across lifecycle providers")
|
|
135
|
+
seen.add(case.case_id)
|
|
136
|
+
yield CaseView.validated(case.case_id, case.inputs, schema=schema, metadata=case.metadata), dataset
|
|
137
|
+
|
|
138
|
+
def cases(self, suite):
|
|
139
|
+
_require(suite.definition_digest == self.suite.definition_digest, "lifecycle suite differs")
|
|
140
|
+
return (case for case, _ in self._source())
|
|
141
|
+
|
|
142
|
+
def _execution_dependencies(self):
|
|
143
|
+
return {"datasets": [d.to_dict() for d in self.suite.datasets],
|
|
144
|
+
"input_schemas": [self.registry.get_schema(d.input_schema_id).to_dict() for d in self.suite.datasets]}
|
|
145
|
+
|
|
146
|
+
def _grading_dependencies(self, reducer_id=None):
|
|
147
|
+
evaluator = self.registry.get_evaluator(self.evaluator_id)
|
|
148
|
+
dependencies = {"evaluator": evaluator.to_dict(),
|
|
149
|
+
"assessment_schemas": [self.registry.get_schema(s).to_dict() for s in evaluator.assessment_schema_ids],
|
|
150
|
+
"reference_schemas": [self.registry.get_schema(d.reference_schema_id).to_dict()
|
|
151
|
+
for d in self.suite.datasets if d.reference_schema_id is not None],
|
|
152
|
+
"criteria": [self.registry.get_criterion(c).to_dict() for c in self.suite.criterion_ids],
|
|
153
|
+
"metrics": [self.registry.get_metric(c).to_dict() for c in self.suite.metric_ids],
|
|
154
|
+
"report_schema": self.registry.get_schema(self.projection_schema_id).to_dict(),
|
|
155
|
+
"projection": self.registry.get_projection(self.projection_schema_id).to_dict(),
|
|
156
|
+
"comparison": self.registry.get_policy(self.comparison_policy_id, kind="comparison").to_dict(),
|
|
157
|
+
"reducer": None if reducer_id is None else self.registry.get_policy(reducer_id, kind="aggregation").to_dict()}
|
|
158
|
+
if self.evaluator_binding is not None or self.report_binding is not None:
|
|
159
|
+
from .evaluator_bindings import _binding_dependencies
|
|
160
|
+
for name in ("evaluator_binding", "report_binding"):
|
|
161
|
+
binding = getattr(self, name)
|
|
162
|
+
if binding is not None:
|
|
163
|
+
dependencies[name] = _binding_dependencies(self.registry, binding)
|
|
164
|
+
if self.include_emitted_evidence:
|
|
165
|
+
dependencies["emitted_evidence_policy"] = "verified-attempt-artifacts.v1"
|
|
166
|
+
return dependencies
|
|
167
|
+
|
|
168
|
+
def resolve(self, plan):
|
|
169
|
+
run, grading = self._options(plan)
|
|
170
|
+
return ResolvedBinding({"backend": self.descriptor.to_dict(), "suite": self.suite.to_dict(),
|
|
171
|
+
"candidate": _thaw(_object(self.driver.candidate_identity(plan.candidate), "candidate identity")),
|
|
172
|
+
"scope": plan.scope, "mode": plan.mode, "runtime": _thaw(plan.runtime),
|
|
173
|
+
"run_policy": run.to_dict(), "grading_policy": grading.to_dict(),
|
|
174
|
+
"execution_dependencies": self._execution_dependencies(),
|
|
175
|
+
"grading_dependencies": self._grading_dependencies(run.reducer_id),
|
|
176
|
+
"resume": _thaw(plan.parameters.get("resume")), "reuse": _thaw(plan.parameters.get("reuse"))})
|
|
177
|
+
|
|
178
|
+
def admit(self, plan, resolved):
|
|
179
|
+
_require(self.resolve(plan).resolution_digest == resolved.resolution_digest, "lifecycle resolution changed")
|
|
180
|
+
cases = tuple(self.cases(plan.suite))
|
|
181
|
+
_require(bool(cases), "lifecycle provider has no selected cases")
|
|
182
|
+
facts = {"cohort_sha256": contract_digest({"cases": [c.to_dict() for c in cases]}),
|
|
183
|
+
"case_count": len(cases), "execution_dependencies": self._execution_dependencies()}
|
|
184
|
+
verified = {} if self.admission_verifier is None else self.admission_verifier(plan, cases)
|
|
185
|
+
verified = _thaw(_object(verified, "admission verification"))
|
|
186
|
+
facts["verification"] = verified
|
|
187
|
+
return AdmissionBinding(resolved.resolution_digest, facts, "local_unverified",
|
|
188
|
+
verified.get("artifact_bytes_verified", False),
|
|
189
|
+
verified.get("candidate_source_bytes_verified", False))
|
|
190
|
+
|
|
191
|
+
def recheck(self, plan, resolved, admission):
|
|
192
|
+
current = self.admit(plan, resolved)
|
|
193
|
+
_require(current.to_dict() == admission.to_dict(), "lifecycle assets/cohort/admission changed")
|
|
194
|
+
|
|
195
|
+
@staticmethod
|
|
196
|
+
def _run_ref(value):
|
|
197
|
+
if value is None:
|
|
198
|
+
return None
|
|
199
|
+
_require(isinstance(value, Mapping) and set(value) <= {"path", "definition_sha256", "head_sha256"}
|
|
200
|
+
and {"path", "definition_sha256"} <= set(value), "invalid serialized lifecycle reference")
|
|
201
|
+
return LifecycleRunRef(Path(value["path"]), value["definition_sha256"], value.get("head_sha256"))
|
|
202
|
+
|
|
203
|
+
def execute(self, plan, resolved, admission, output_dir):
|
|
204
|
+
self.recheck(plan, resolved, admission)
|
|
205
|
+
run, grading = self._options(plan)
|
|
206
|
+
execution_identity = {key: _thaw(resolved.identity[key]) for key in
|
|
207
|
+
("backend", "suite", "scope", "mode", "runtime", "execution_dependencies")}
|
|
208
|
+
verification = admission.facts["verification"]
|
|
209
|
+
execution_identity["admission_fixed"] = _thaw(verification.get("fixed", {}))
|
|
210
|
+
execution_identity["admission_candidate"] = _thaw(verification.get("candidate", {}))
|
|
211
|
+
return execute_lifecycle(self.driver, plan.candidate, self.cases(plan.suite), output_dir or ".sphynx/lifecycle",
|
|
212
|
+
identity=execution_identity, policy=run, grading_policy=grading, cancellation=self.cancellation,
|
|
213
|
+
resume=self._run_ref(plan.parameters.get("resume")), reuse=self._run_ref(plan.parameters.get("reuse")),
|
|
214
|
+
recheck=lambda: self.recheck(plan, resolved, admission))
|
|
215
|
+
|
|
216
|
+
def accepts_reference(self, native_ref, kind):
|
|
217
|
+
ref = native_ref.run if isinstance(native_ref, LifecycleGradingRef) else native_ref
|
|
218
|
+
if kind == "run" and not isinstance(native_ref, LifecycleRunRef) or kind == "grading" and not isinstance(native_ref, LifecycleGradingRef):
|
|
219
|
+
return False
|
|
220
|
+
if not isinstance(ref, LifecycleRunRef):
|
|
221
|
+
return False
|
|
222
|
+
try:
|
|
223
|
+
definition, _ = _read_json(ref.path / "definition.json")
|
|
224
|
+
return definition["identity"]["backend"]["backend_id"] == self.descriptor.backend_id
|
|
225
|
+
except (LifecycleError, KeyError, TypeError):
|
|
226
|
+
return False
|
|
227
|
+
|
|
228
|
+
def _run(self, ref):
|
|
229
|
+
_require(self.accepts_reference(ref, "run"), "foreign lifecycle run")
|
|
230
|
+
record = verify_lifecycle_run(ref)
|
|
231
|
+
identity = record["definition"]["identity"]
|
|
232
|
+
_require(identity["backend"] == self.descriptor.to_dict() and identity["suite"] == self.suite.to_dict()
|
|
233
|
+
and identity["execution_dependencies"] == self._execution_dependencies(),
|
|
234
|
+
"lifecycle execution definitions changed")
|
|
235
|
+
return record
|
|
236
|
+
|
|
237
|
+
def reference(self, native_ref, kind):
|
|
238
|
+
if kind == "run":
|
|
239
|
+
record = self._run(native_ref)
|
|
240
|
+
return ReferenceBinding(native_ref.run_id, record["status"], "local_unverified", native_ref.run_id,
|
|
241
|
+
{"cleanup_status": record["cleanup_status"]})
|
|
242
|
+
record = self._grading(native_ref)
|
|
243
|
+
successes = {(r["case_id"], r["trial"]) for r in record["rows"] if r["failure"] is None}
|
|
244
|
+
planned = len(record["run"]["definition"]["cases"]) * record["run"]["definition"]["policy"]["trials"]
|
|
245
|
+
return ReferenceBinding(native_ref.grading_id, "completed" if len(successes) == planned and not record.get("report_assessment_failure") else "incomplete",
|
|
246
|
+
"local_unverified", native_ref.run.run_id)
|
|
247
|
+
|
|
248
|
+
def _references(self, run_record):
|
|
249
|
+
expected = {case["case_id"]: case for case in run_record["definition"]["cases"]}
|
|
250
|
+
references = {}
|
|
251
|
+
for case, dataset in self._source():
|
|
252
|
+
_require(case.case_id in expected and case.to_dict() == expected[case.case_id],
|
|
253
|
+
"provider case changed since lifecycle execution")
|
|
254
|
+
references[case.case_id] = ({} if dataset.reference_schema_id is None else
|
|
255
|
+
self.registry.get_provider(dataset.provider_id).reference_for(case.case_id, access=self.reference_access))
|
|
256
|
+
_require(set(references) == set(expected), "grading provider cohort differs")
|
|
257
|
+
return references
|
|
258
|
+
|
|
259
|
+
def assess(self, native_run):
|
|
260
|
+
return self.assess_with_policy(native_run)
|
|
261
|
+
|
|
262
|
+
def assess_with_policy(self, native_run, policy=None):
|
|
263
|
+
run = self._run(native_run)
|
|
264
|
+
if self.report_binding is not None:
|
|
265
|
+
default = run["definition"].get("default_grading_policy")
|
|
266
|
+
self._validate_report_policy(policy if policy is not None else
|
|
267
|
+
GradingPolicy.from_dict(default) if default is not None else GradingPolicy())
|
|
268
|
+
references = self._references(run)
|
|
269
|
+
reducer_id = run["definition"]["policy"]["reducer_id"]
|
|
270
|
+
identity = {"grading_dependencies": self._grading_dependencies(reducer_id),
|
|
271
|
+
"reference_hashes": {case: contract_digest(ref) for case, ref in references.items()}}
|
|
272
|
+
evaluator = self.registry.get_evaluator_callable(self.evaluator_id)
|
|
273
|
+
emitted = {}
|
|
274
|
+
if self.include_emitted_evidence:
|
|
275
|
+
for event in run["events"]:
|
|
276
|
+
data = event["data"]
|
|
277
|
+
if event["event"] == "artifact":
|
|
278
|
+
emitted.setdefault((data["case_id"], data["trial"], data["attempt"]), []).append(data["binding"])
|
|
279
|
+
def evaluate(case, attempt, grading):
|
|
280
|
+
evidence = []
|
|
281
|
+
bindings = dict(attempt["artifacts"])
|
|
282
|
+
if attempt.get("output_binding"):
|
|
283
|
+
bindings["outputs"] = attempt["output_binding"]
|
|
284
|
+
selected_bindings = list(bindings.items())
|
|
285
|
+
selected_bindings.extend((binding["role"], binding) for binding in
|
|
286
|
+
emitted.get((case.case_id, grading.trial, grading.attempt), ()))
|
|
287
|
+
seen = set()
|
|
288
|
+
for role, binding in selected_bindings:
|
|
289
|
+
if self.include_emitted_evidence and binding["path"] in seen:
|
|
290
|
+
continue
|
|
291
|
+
seen.add(binding["path"])
|
|
292
|
+
evidence.append(EvidenceView(native_run.run_id + ":" + binding["path"], binding["sha256"],
|
|
293
|
+
"application/json" if role == "outputs" else "application/octet-stream", role,
|
|
294
|
+
self.driver.descriptor.driver_id, "local_owner", case_id=case.case_id,
|
|
295
|
+
metadata={"trial": grading.trial, "attempt": grading.attempt}))
|
|
296
|
+
context = _verified_evaluator_context(case=case, outputs=attempt["outputs"],
|
|
297
|
+
references=references[case.case_id], evidence=tuple(evidence), run_id=grading.run_id,
|
|
298
|
+
grading_id=grading.grading_id, trust_tier="local_unverified", verified_by=self.descriptor.backend_id,
|
|
299
|
+
metadata={"trial": grading.trial, "attempt": grading.attempt, "grading_attempt": grading.grading_attempt,
|
|
300
|
+
"candidate_failure": attempt["failure"], "reused": attempt.get("reused", False)})
|
|
301
|
+
if self.evaluator_binding is None:
|
|
302
|
+
rows = tuple(evaluator(context))
|
|
303
|
+
else:
|
|
304
|
+
from .evaluator_bindings import _dispatch_binding
|
|
305
|
+
stage = (self.evaluator_stage_evidence(native_run, self.evaluator_binding.selector(context))
|
|
306
|
+
if self.evaluator_binding.target == "stage" else None)
|
|
307
|
+
rows = _dispatch_binding(self.registry, self.evaluator_binding, context, stage=stage)
|
|
308
|
+
allowed_evidence = {view.evidence_id for view in evidence}
|
|
309
|
+
for row in rows:
|
|
310
|
+
_require(isinstance(row, Assessment) and row.case_id == case.case_id
|
|
311
|
+
and row.criterion_id in self.suite.criterion_ids and row.evaluator_id == self.evaluator_id
|
|
312
|
+
and row.run_id == grading.run_id and row.grading_id == grading.grading_id
|
|
313
|
+
and row.trial == grading.trial and row.attempt == grading.attempt
|
|
314
|
+
and set(row.evidence_ids) <= allowed_evidence, "evaluator assessment identity or evidence differs")
|
|
315
|
+
if row.payload_schema is not None:
|
|
316
|
+
_require(row.payload_schema["schema_id"] in self.registry.get_evaluator(self.evaluator_id).assessment_schema_ids,
|
|
317
|
+
"evaluator emitted an undeclared assessment schema")
|
|
318
|
+
self._assessment(row.to_dict()) # Validate against the registered, not caller-substituted schema.
|
|
319
|
+
return {"assessments": [row.to_dict() for row in rows]}
|
|
320
|
+
grading = assess_lifecycle(native_run, evaluate, identity=identity, policy=policy)
|
|
321
|
+
record = verify_lifecycle_grading(grading)
|
|
322
|
+
journal = LifecycleJournal(grading.path, grading.definition_sha256)
|
|
323
|
+
with journal.writer():
|
|
324
|
+
if reducer_id is not None:
|
|
325
|
+
value = self.registry.get_policy_callable(reducer_id, kind="aggregation")(record)
|
|
326
|
+
binding = journal.artifact("analysis", _raw(_object(value, "analysis projection")))
|
|
327
|
+
journal.append("analysis", {"policy_id": reducer_id, "binding": binding})
|
|
328
|
+
record["analysis"] = _thaw(_object(value, "analysis projection"))
|
|
329
|
+
projection = self.registry.get_projection_callable(self.projection_schema_id)(record)
|
|
330
|
+
value = projection.to_dict() if isinstance(projection, ReportProjection) else projection
|
|
331
|
+
value = self.registry.get_schema(self.projection_schema_id).validate(value)
|
|
332
|
+
binding = journal.artifact("report", _raw(value))
|
|
333
|
+
head = journal.append("projection", {"schema_id": self.projection_schema_id, "binding": binding})
|
|
334
|
+
if self.report_binding is not None:
|
|
335
|
+
from .evaluator_bindings import _dispatch_binding, _verified_report_context
|
|
336
|
+
# Report grading is one attempt per saved pass. A later regrade
|
|
337
|
+
# is distinct; do not retry quality outcomes or rerun candidates.
|
|
338
|
+
context = _verified_report_context(outputs=_thaw(value), references=_thaw(references),
|
|
339
|
+
run_id=grading.run.run_id, grading_id=grading.grading_id,
|
|
340
|
+
trust_tier="local_unverified", verified_by=self.descriptor.backend_id,
|
|
341
|
+
evidence_ids=(grading.run.run_id + ":" + binding["path"],))
|
|
342
|
+
try:
|
|
343
|
+
rows = _dispatch_binding(self.registry, self.report_binding, context)
|
|
344
|
+
payload = {"binding_digest": self.report_binding.binding_digest,
|
|
345
|
+
"rows": [row.to_dict() for row in rows]}
|
|
346
|
+
report_rows = journal.artifact("report_assessments", _raw(payload))
|
|
347
|
+
head = journal.append("report_assessments", {"binding": report_rows})
|
|
348
|
+
except Exception as exc:
|
|
349
|
+
head = journal.append("report_assessment_failure", {
|
|
350
|
+
"binding_digest": self.report_binding.binding_digest,
|
|
351
|
+
"failure": "evaluator_error", "error_type": type(exc).__name__})
|
|
352
|
+
return LifecycleGradingRef(grading.path, grading.definition_sha256, grading.run, head)
|
|
353
|
+
|
|
354
|
+
def _grading(self, ref):
|
|
355
|
+
_require(self.accepts_reference(ref, "grading"), "foreign lifecycle grading pass")
|
|
356
|
+
self._run(ref.run)
|
|
357
|
+
record = verify_lifecycle_grading(ref)
|
|
358
|
+
reducer_id = record["run"]["definition"]["policy"]["reducer_id"]
|
|
359
|
+
_require(record["definition"]["identity"]["grading_dependencies"] == self._grading_dependencies(reducer_id),
|
|
360
|
+
"grading definitions changed; retain the matching registered versions")
|
|
361
|
+
journal = LifecycleJournal(ref.path, ref.definition_sha256)
|
|
362
|
+
for event in record["events"]:
|
|
363
|
+
if event["event"] in ("analysis", "projection"):
|
|
364
|
+
value = json.loads(journal.read_artifact(event["data"]["binding"]))
|
|
365
|
+
record[event["event"]] = value
|
|
366
|
+
if event["event"] == "report_assessments":
|
|
367
|
+
value = json.loads(journal.read_artifact(event["data"]["binding"]))
|
|
368
|
+
_require(self.report_binding is not None
|
|
369
|
+
and value.get("binding_digest") == self.report_binding.binding_digest
|
|
370
|
+
and set(value) == {"binding_digest", "rows"} and type(value["rows"]) is list,
|
|
371
|
+
"saved report assessment binding differs")
|
|
372
|
+
record["report_assessments"] = value["rows"]
|
|
373
|
+
if event["event"] == "report_assessment_failure":
|
|
374
|
+
_require(self.report_binding is not None and event["data"].get("binding_digest")
|
|
375
|
+
== self.report_binding.binding_digest, "saved report failure binding differs")
|
|
376
|
+
record["report_assessment_failure"] = event["data"]["failure"]
|
|
377
|
+
return record
|
|
378
|
+
|
|
379
|
+
def report(self, native_grading):
|
|
380
|
+
record = self._grading(native_grading)
|
|
381
|
+
_require("projection" in record, "saved lifecycle report projection is missing")
|
|
382
|
+
run = record["run"]["definition"]
|
|
383
|
+
identity = run["identity"]
|
|
384
|
+
planned = len(run["cases"]) * run["policy"]["trials"]
|
|
385
|
+
observed = len({(row["case_id"], row["trial"]) for row in record["rows"] if row["failure"] is None})
|
|
386
|
+
# Do not add cases, references, commands or arbitrary driver metadata to
|
|
387
|
+
# presentation. The native projection and its serialized bytes stay exact.
|
|
388
|
+
candidate = run["candidate_identity"]
|
|
389
|
+
nested = candidate.get("identity")
|
|
390
|
+
candidate = {key: value for key, value in candidate.items() if key in
|
|
391
|
+
("authority", "candidate_id", "name", "version", "revision", "head_sha", "commit_sha")
|
|
392
|
+
and type(value) is str}
|
|
393
|
+
if isinstance(nested, Mapping):
|
|
394
|
+
candidate["identity"] = {key: value for key, value in nested.items()
|
|
395
|
+
if key in ("schema_version", "identity_id", "version", "digest") and type(value) is str}
|
|
396
|
+
evaluator = self.registry.get_evaluator(self.evaluator_id)
|
|
397
|
+
context = {"suite_id": self.suite.suite_id, "suite_version": self.suite.version,
|
|
398
|
+
"run_id": native_grading.run.run_id, "grading_id": native_grading.grading_id,
|
|
399
|
+
"scope": identity["scope"], "mode": identity["mode"],
|
|
400
|
+
"status": "completed" if observed == planned and not record.get("report_assessment_failure") else "incomplete",
|
|
401
|
+
"trust_tier": "local_unverified", "candidate": candidate,
|
|
402
|
+
"evaluator": {"id": evaluator.evaluator_id, "version": evaluator.version,
|
|
403
|
+
"implementation_id": evaluator.implementation_id},
|
|
404
|
+
"execution_policy": run["policy"],
|
|
405
|
+
"coverage": {"required_trials": planned, "observed_trials": observed,
|
|
406
|
+
"missing_trials": planned - observed},
|
|
407
|
+
"metric_definitions": {name: {"unit": self.registry.get_metric(name).unit,
|
|
408
|
+
"direction": self.registry.get_metric(name).direction}
|
|
409
|
+
for name in self.suite.metric_ids}}
|
|
410
|
+
return LifecycleProjection(self.registry.get_schema(self.projection_schema_id).validate(record["projection"]), context=context)
|
|
411
|
+
|
|
412
|
+
def _assessment(self, data):
|
|
413
|
+
data = dict(data); data.pop("schema_version", None)
|
|
414
|
+
payload = data.pop("payload_schema", None)
|
|
415
|
+
if payload is None:
|
|
416
|
+
return Assessment(**data)
|
|
417
|
+
schema = self.registry.get_schema(payload["schema_id"])
|
|
418
|
+
_require(all(getattr(schema, key) == payload[key] for key in ("schema_id", "version", "implementation_id")),
|
|
419
|
+
"saved assessment schema binding differs")
|
|
420
|
+
return Assessment.structured(schema=schema, **data)
|
|
421
|
+
|
|
422
|
+
def assessments(self, native_grading, case_id=None):
|
|
423
|
+
record = self._grading(native_grading)
|
|
424
|
+
return tuple(self._assessment(data) for row in record["rows"] if row["failure"] is None
|
|
425
|
+
for data in row["output"].get("assessments", []) if case_id is None or data["case_id"] == case_id)
|
|
426
|
+
|
|
427
|
+
def report_assessments(self, native_grading):
|
|
428
|
+
from .evaluator_bindings import ReportAssessment, REPORT_ASSESSMENT_SCHEMA
|
|
429
|
+
record = self._grading(native_grading)
|
|
430
|
+
_require(not record.get("report_assessment_failure"), "saved report evaluator failed")
|
|
431
|
+
rows = []
|
|
432
|
+
for raw in record.get("report_assessments", []):
|
|
433
|
+
data = dict(raw)
|
|
434
|
+
_require(data.pop("schema_version", None) == REPORT_ASSESSMENT_SCHEMA
|
|
435
|
+
and data.pop("scope", None) == "report", "invalid report assessment envelope")
|
|
436
|
+
payload = data.pop("payload_schema", None)
|
|
437
|
+
if payload is not None:
|
|
438
|
+
schema = self.registry.get_schema(payload["schema_id"])
|
|
439
|
+
_require(set(payload) == {"schema_id", "version", "implementation_id"}
|
|
440
|
+
and all(getattr(schema, key) == payload[key] for key in payload),
|
|
441
|
+
"saved report assessment schema differs")
|
|
442
|
+
data["schema"] = schema
|
|
443
|
+
row = ReportAssessment(**data)
|
|
444
|
+
_require(row.run_id == native_grading.run.run_id and row.grading_id == native_grading.grading_id
|
|
445
|
+
and self.report_binding is not None
|
|
446
|
+
and row.evaluator_id == self.report_binding.evaluator.identity_id,
|
|
447
|
+
"saved report assessment identity differs")
|
|
448
|
+
rows.append(row)
|
|
449
|
+
return tuple(rows)
|
|
450
|
+
|
|
451
|
+
def compare(self, baseline_grading, candidate_grading):
|
|
452
|
+
baseline, candidate = self._grading(baseline_grading), self._grading(candidate_grading)
|
|
453
|
+
self._comparison_conditions(baseline, candidate)
|
|
454
|
+
return self.registry.get_policy_callable(self.comparison_policy_id, kind="comparison")(baseline, candidate)
|
|
455
|
+
|
|
456
|
+
@staticmethod
|
|
457
|
+
def _comparison_conditions(baseline, candidate):
|
|
458
|
+
left, right = baseline["run"]["definition"], candidate["run"]["definition"]
|
|
459
|
+
fixed_left = {k: v for k, v in left["identity"].items() if k != "admission_candidate"}
|
|
460
|
+
fixed_right = {k: v for k, v in right["identity"].items() if k != "admission_candidate"}
|
|
461
|
+
_comparison_require(contract_digest({"identity": fixed_left, "cases": left["cases"],
|
|
462
|
+
"policy": {k: v for k, v in left["policy"].items() if k != "reuse"}}) ==
|
|
463
|
+
contract_digest({"identity": fixed_right, "cases": right["cases"],
|
|
464
|
+
"policy": {k: v for k, v in right["policy"].items() if k != "reuse"}}),
|
|
465
|
+
"incompatible lifecycle execution/comparison conditions")
|
|
466
|
+
_comparison_require(baseline["definition"]["identity"] == candidate["definition"]["identity"],
|
|
467
|
+
"incompatible grading/reference/analysis conditions")
|
|
468
|
+
_comparison_require(baseline["definition"]["policy"] == candidate["definition"]["policy"],
|
|
469
|
+
"incompatible grading retry/budget conditions")
|
|
470
|
+
|
|
471
|
+
|
|
472
|
+
def compare_view(self, baseline_grading, candidate_grading):
|
|
473
|
+
from ._comparison_adapters import _sources, _view
|
|
474
|
+
baseline, candidate = self._grading(baseline_grading), self._grading(candidate_grading)
|
|
475
|
+
conditions = [{"execution_identity": row["run"]["definition"]["identity"],
|
|
476
|
+
"execution_policy": row["run"]["definition"]["policy"],
|
|
477
|
+
"grading_identity": row["definition"]["identity"],
|
|
478
|
+
"grading_policy": row["definition"]["policy"]} for row in (baseline, candidate)]
|
|
479
|
+
sources = _sources(self, baseline_grading, candidate_grading, conditions=conditions)
|
|
480
|
+
try:
|
|
481
|
+
self._comparison_conditions(baseline, candidate)
|
|
482
|
+
except _ComparisonMismatch as exc:
|
|
483
|
+
return _view(sources, {"policy_id": self.comparison_policy_id, "refusal": str(exc)},
|
|
484
|
+
status="incompatible", reason=str(exc))
|
|
485
|
+
projector = getattr(self, "project_comparison", None)
|
|
486
|
+
if not callable(projector):
|
|
487
|
+
return _view(sources, {}, status="unavailable",
|
|
488
|
+
reason="No common view adapter is registered for this native comparison policy; use compare().")
|
|
489
|
+
native = self.registry.get_policy_callable(self.comparison_policy_id, kind="comparison")(baseline, candidate)
|
|
490
|
+
return projector(sources, native)
|