sable-sphynx 0.7.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- sable_sphynx-0.7.0.dev0.dist-info/METADATA +152 -0
- sable_sphynx-0.7.0.dev0.dist-info/RECORD +174 -0
- sable_sphynx-0.7.0.dev0.dist-info/WHEEL +5 -0
- sable_sphynx-0.7.0.dev0.dist-info/top_level.txt +1 -0
- sphynx/__init__.py +469 -0
- sphynx/__init__.pyi +376 -0
- sphynx/_analytical_client.py +77 -0
- sphynx/_analytical_values.py +252 -0
- sphynx/_audit_client.py +173 -0
- sphynx/_comparison_adapters.py +113 -0
- sphynx/_componentized_full_rag.py +34 -0
- sphynx/_composed_pipeline.py +109 -0
- sphynx/_full_rag_grading.py +218 -0
- sphynx/_full_rag_trace.py +221 -0
- sphynx/_local_eval.py +268 -0
- sphynx/_pipeline_core.py +243 -0
- sphynx/_pr_client.py +237 -0
- sphynx/_rag_diagnostics_calibration.py +285 -0
- sphynx/_rag_diagnostics_catalog.py +39 -0
- sphynx/_rag_diagnostics_common.py +26 -0
- sphynx/_rag_diagnostics_contracts.py +186 -0
- sphynx/_rag_diagnostics_engine.py +243 -0
- sphynx/_rag_diagnostics_importers.py +242 -0
- sphynx/_rag_diagnostics_ranking.py +81 -0
- sphynx/_rag_diagnostics_semantic.py +325 -0
- sphynx/_rag_ranking.py +149 -0
- sphynx/_speech_benchmark_scoring.py +140 -0
- sphynx/_speech_benchmark_sources.py +275 -0
- sphynx/_speech_complete_scoring.py +218 -0
- sphynx/_speech_complete_source.py +118 -0
- sphynx/_speech_kido.py +294 -0
- sphynx/_speech_scoring.py +77 -0
- sphynx/_track_capture.py +213 -0
- sphynx/analytical_views.py +515 -0
- sphynx/assessment_contracts.py +378 -0
- sphynx/assessment_selection.py +57 -0
- sphynx/audit_access.py +245 -0
- sphynx/audit_contracts.py +204 -0
- sphynx/audit_requests.py +180 -0
- sphynx/authoring.py +188 -0
- sphynx/benchmarks.py +568 -0
- sphynx/candidates.py +62 -0
- sphynx/catalog.py +272 -0
- sphynx/chunking.py +372 -0
- sphynx/clickhouse_reader.py +169 -0
- sphynx/client.py +486 -0
- sphynx/comparison_core.py +149 -0
- sphynx/comparison_views.py +223 -0
- sphynx/components.py +1661 -0
- sphynx/contracts.py +514 -0
- sphynx/customer_rerank.py +299 -0
- sphynx/customer_retrieval.py +146 -0
- sphynx/data/benchmark-releases.v1.json +164 -0
- sphynx/data/public-benchmark-release.v1.json +354 -0
- sphynx/data/rag-diagnostics-synthetic.v1.json +6982 -0
- sphynx/data/rag-paper-references.v1.json +176 -0
- sphynx/data/rag-paper-schemas.v1.json +1069 -0
- sphynx/data/rag-rerank-v1.json +507 -0
- sphynx/dataset_catalog_data.py +1214 -0
- sphynx/dataset_readiness.py +138 -0
- sphynx/dataset_v3.py +999 -0
- sphynx/demo.py +130 -0
- sphynx/demo_candidates/__init__.py +1 -0
- sphynx/demo_candidates/dense.py +11 -0
- sphynx/demo_candidates/lexical.py +21 -0
- sphynx/diagnostics.py +207 -0
- sphynx/documentation.py +737 -0
- sphynx/documentation_candidate.py +939 -0
- sphynx/evals.py +538 -0
- sphynx/evaluation_protocols.py +124 -0
- sphynx/evaluator_bindings.py +370 -0
- sphynx/evidence.py +352 -0
- sphynx/execution_capabilities.py +109 -0
- sphynx/fast_search.py +285 -0
- sphynx/full_rag.py +341 -0
- sphynx/hosted/__init__.py +18 -0
- sphynx/hosted/benchmark_intake.py +218 -0
- sphynx/hosted/contracts.py +361 -0
- sphynx/hosted/model_admission.py +130 -0
- sphynx/hosted/model_artifacts.py +298 -0
- sphynx/hosted/model_contracts.py +63 -0
- sphynx/hosted/model_gcs.py +181 -0
- sphynx/hosted/model_history.py +84 -0
- sphynx/hosted/model_session.py +233 -0
- sphynx/hosted/public_suite.py +74 -0
- sphynx/hosted/registry.py +97 -0
- sphynx/hosted/revisions.py +126 -0
- sphynx/hosted/sandbox.py +105 -0
- sphynx/hosted/submission.py +287 -0
- sphynx/hosted_workflow.py +451 -0
- sphynx/http_transport.py +86 -0
- sphynx/integrated_rag.py +352 -0
- sphynx/integrated_workflow.py +552 -0
- sphynx/integration_contracts.py +474 -0
- sphynx/invocation.py +791 -0
- sphynx/invocation_candidates.py +392 -0
- sphynx/invocation_catalog_data.py +40 -0
- sphynx/learned_catalog_data.py +30 -0
- sphynx/learned_rerank.py +267 -0
- sphynx/legacy_full_rag.py +67 -0
- sphynx/lifecycle.py +885 -0
- sphynx/lifecycle_workflow.py +490 -0
- sphynx/metric_views.py +474 -0
- sphynx/metrics.py +282 -0
- sphynx/model_adapters.py +438 -0
- sphynx/model_catalog_data.py +30 -0
- sphynx/official_benchmarks.py +385 -0
- sphynx/packs.py +258 -0
- sphynx/pipeline.py +313 -0
- sphynx/pipeline_workflow.py +551 -0
- sphynx/pr_evaluations.py +560 -0
- sphynx/progress.py +89 -0
- sphynx/public_bundle.py +173 -0
- sphynx/public_eval.py +62 -0
- sphynx/public_rerank.py +355 -0
- sphynx/py.typed +0 -0
- sphynx/pytest_plugin.py +62 -0
- sphynx/question_robustness.py +437 -0
- sphynx/rag.py +329 -0
- sphynx/rag_diagnostics.py +309 -0
- sphynx/rag_papers/__init__.py +29 -0
- sphynx/rag_papers/__main__.py +45 -0
- sphynx/rag_papers/candidates.py +19 -0
- sphynx/rag_papers/contracts.py +282 -0
- sphynx/rag_papers/datasets.py +291 -0
- sphynx/rag_papers/experiments.py +116 -0
- sphynx/rag_papers/judges.py +308 -0
- sphynx/rag_papers/registry.py +33 -0
- sphynx/rag_papers/runner.py +176 -0
- sphynx/rag_papers/sable.py +135 -0
- sphynx/rag_papers/schemas.py +12 -0
- sphynx/rag_papers/scoring.py +249 -0
- sphynx/rag_papers/semantic.py +288 -0
- sphynx/rag_papers/trec.py +80 -0
- sphynx/recorded_scores.py +352 -0
- sphynx/records.py +124 -0
- sphynx/reflex_traces.py +193 -0
- sphynx/report_rendering.py +71 -0
- sphynx/rerank_family_contracts.py +150 -0
- sphynx/rerank_workflow.py +424 -0
- sphynx/research_candidate.py +190 -0
- sphynx/results.py +852 -0
- sphynx/retrieval.py +750 -0
- sphynx/retrieval_candidates.py +295 -0
- sphynx/retrieval_catalog_data.py +19 -0
- sphynx/retrieval_recordings.py +125 -0
- sphynx/reviewer_audit.py +266 -0
- sphynx/routing_policies.py +375 -0
- sphynx/runner.py +544 -0
- sphynx/sable_rerank.py +489 -0
- sphynx/score_reliability.py +393 -0
- sphynx/speech_benchmarks.py +204 -0
- sphynx/speech_campaigns.py +378 -0
- sphynx/speech_complete.py +236 -0
- sphynx/speech_contracts.py +225 -0
- sphynx/speech_evidence.py +429 -0
- sphynx/speech_listening.py +115 -0
- sphynx/speech_multilingual.py +71 -0
- sphynx/speech_tts.py +479 -0
- sphynx/sql/001_sync_v1.sql +30 -0
- sphynx/stage_contracts.py +199 -0
- sphynx/statistical_catalog_data.py +53 -0
- sphynx/statistical_rerank.py +232 -0
- sphynx/sync.py +797 -0
- sphynx/workbench.py +378 -0
- sphynx/workbench_catalog_data.py +13 -0
- sphynx/workbench_tracks.py +78 -0
- sphynx/workflow_async.py +197 -0
- sphynx/workflow_contracts.py +594 -0
- sphynx/workflow_decisions.py +211 -0
- sphynx/workflow_defaults.py +16 -0
- sphynx/workflow_extensions.py +212 -0
- sphynx/workflow_registry.py +513 -0
- sphynx/workflows.py +792 -0
sphynx/lifecycle.py
ADDED
|
@@ -0,0 +1,885 @@
|
|
|
1
|
+
"""Versioned local lifecycle records; native fixed-attempt records stay unchanged.
|
|
2
|
+
|
|
3
|
+
Drivers own domain execution and enforcement. This coordinator does not make
|
|
4
|
+
local evidence authenticated, sandbox Python callables, or choose quality-best
|
|
5
|
+
outputs. A declared reducer belongs to the backend, never to this journal.
|
|
6
|
+
"""
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
from collections.abc import Mapping, Sequence
|
|
10
|
+
from contextlib import contextmanager
|
|
11
|
+
from dataclasses import dataclass, field
|
|
12
|
+
from datetime import datetime, timezone
|
|
13
|
+
import hashlib
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import os
|
|
17
|
+
from pathlib import Path, PurePosixPath
|
|
18
|
+
import re
|
|
19
|
+
import threading
|
|
20
|
+
import time
|
|
21
|
+
from typing import Any, Callable, Protocol
|
|
22
|
+
from uuid import uuid4
|
|
23
|
+
|
|
24
|
+
from .assessment_contracts import CaseView
|
|
25
|
+
from .workflow_contracts import WorkflowContractError, _object, _thaw, contract_digest
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
RUN_SCHEMA = "sphynx.lifecycle.run.v1"
|
|
29
|
+
JOURNAL_SCHEMA = "sphynx.lifecycle.event.v1"
|
|
30
|
+
GRADING_SCHEMA = "sphynx.lifecycle.grading.v1"
|
|
31
|
+
_HASH = re.compile(r"sha256:[0-9a-f]{64}\Z")
|
|
32
|
+
_ENTRY = re.compile(r"([0-9]{8})\.([0-9a-f]{64})\.json\Z")
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class LifecycleError(WorkflowContractError):
|
|
36
|
+
"""Invalid policy, interrupted execution, incompatible reuse or corrupt bytes."""
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _require(value, message):
|
|
40
|
+
if not value:
|
|
41
|
+
raise LifecycleError(message)
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _text(value, name):
|
|
45
|
+
_require(type(value) is str and bool(value.strip()), name + " must be text")
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _positive(value, name, *, optional=False):
|
|
49
|
+
if optional and value is None:
|
|
50
|
+
return
|
|
51
|
+
_require(type(value) is int and value >= 1, name + " must be a positive integer")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def _seconds(value, name):
|
|
55
|
+
_require(value is None or type(value) in (int, float) and math.isfinite(value) and value > 0,
|
|
56
|
+
name + " must be positive finite seconds or None")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _raw(value):
|
|
60
|
+
return json.dumps(_thaw(value), sort_keys=True, separators=(",", ":"),
|
|
61
|
+
ensure_ascii=False, allow_nan=False).encode("utf-8")
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _sha(value):
|
|
65
|
+
return "sha256:" + hashlib.sha256(value).hexdigest()
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def _read_json(path):
|
|
69
|
+
def pairs(rows):
|
|
70
|
+
result = {}
|
|
71
|
+
for key, value in rows:
|
|
72
|
+
_require(key not in result, "duplicate lifecycle JSON key")
|
|
73
|
+
result[key] = value
|
|
74
|
+
return result
|
|
75
|
+
try:
|
|
76
|
+
raw = path.read_bytes()
|
|
77
|
+
value = json.loads(raw, object_pairs_hook=pairs,
|
|
78
|
+
parse_constant=lambda _: (_ for _ in ()).throw(LifecycleError("nonfinite JSON")))
|
|
79
|
+
_raw(value)
|
|
80
|
+
return value, raw
|
|
81
|
+
except (OSError, ValueError, TypeError, UnicodeError) as exc:
|
|
82
|
+
raise LifecycleError("unreadable lifecycle JSON") from exc
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def _safe(root, relative):
|
|
86
|
+
_require(type(relative) is str and "\\" not in relative and ":" not in relative,
|
|
87
|
+
"unsafe lifecycle artifact path")
|
|
88
|
+
p = PurePosixPath(relative)
|
|
89
|
+
_require(not p.is_absolute() and all(x not in ("", ".", "..") for x in relative.split("/")),
|
|
90
|
+
"unsafe lifecycle artifact path")
|
|
91
|
+
root = Path(root).absolute()
|
|
92
|
+
_require(not root.is_symlink(), "symlinked lifecycle root")
|
|
93
|
+
result = root.joinpath(*p.parts)
|
|
94
|
+
_require(result.resolve().is_relative_to(root.resolve()) and not any(
|
|
95
|
+
parent.is_symlink() for parent in (result, *result.parents) if parent != root.parent),
|
|
96
|
+
"symlinked or escaped lifecycle artifact")
|
|
97
|
+
return result
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
@dataclass(frozen=True)
|
|
101
|
+
class RunLimits:
|
|
102
|
+
"""Output limits concern retained bytes, not process memory or OS isolation."""
|
|
103
|
+
max_total_attempts: int | None = None
|
|
104
|
+
max_wall_seconds: float | None = None
|
|
105
|
+
per_attempt_seconds: float | None = None
|
|
106
|
+
max_output_bytes: int | None = None
|
|
107
|
+
max_tokens: int | None = None
|
|
108
|
+
max_cost: float | None = None
|
|
109
|
+
enforcement: str = "hard"
|
|
110
|
+
|
|
111
|
+
def __post_init__(self):
|
|
112
|
+
for name in ("max_total_attempts", "max_output_bytes", "max_tokens"):
|
|
113
|
+
_positive(getattr(self, name), name, optional=True)
|
|
114
|
+
for name in ("max_wall_seconds", "per_attempt_seconds", "max_cost"):
|
|
115
|
+
_seconds(getattr(self, name), name)
|
|
116
|
+
_require(self.enforcement in ("hard", "cooperative"), "unknown deadline enforcement")
|
|
117
|
+
|
|
118
|
+
def to_dict(self):
|
|
119
|
+
return {name: getattr(self, name) for name in self.__dataclass_fields__}
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass(frozen=True)
|
|
123
|
+
class RunPolicy:
|
|
124
|
+
trials: int = 1
|
|
125
|
+
candidate_attempts: int = 1
|
|
126
|
+
retry_on: tuple[str, ...] = ("candidate_error", "timeout")
|
|
127
|
+
selection: str = "first_success"
|
|
128
|
+
seeds: tuple[int, ...] = ()
|
|
129
|
+
limits: RunLimits = field(default_factory=RunLimits)
|
|
130
|
+
reducer_id: str | None = None
|
|
131
|
+
reuse: str = "never"
|
|
132
|
+
|
|
133
|
+
def __post_init__(self):
|
|
134
|
+
_positive(self.trials, "trials"); _positive(self.candidate_attempts, "candidate_attempts")
|
|
135
|
+
_require(self.selection == "first_success", "only execution-first-success selection is supported")
|
|
136
|
+
_require(isinstance(self.limits, RunLimits), "limits must be RunLimits")
|
|
137
|
+
_require(isinstance(self.retry_on, (tuple, list)) and all(type(v) is str and v for v in self.retry_on),
|
|
138
|
+
"retry_on must contain failure codes")
|
|
139
|
+
_require(not {"success", "quality", "failed_assertion"}.intersection(self.retry_on),
|
|
140
|
+
"quality outcomes cannot trigger candidate retries")
|
|
141
|
+
object.__setattr__(self, "retry_on", tuple(self.retry_on))
|
|
142
|
+
_require(isinstance(self.seeds, (tuple, list)) and (not self.seeds or len(self.seeds) == self.trials)
|
|
143
|
+
and all(type(v) is int and v >= 0 for v in self.seeds), "seeds must bind every trial")
|
|
144
|
+
object.__setattr__(self, "seeds", tuple(self.seeds))
|
|
145
|
+
_require(self.reuse in ("never", "verified"), "reuse must be never or verified")
|
|
146
|
+
if self.reducer_id is not None:
|
|
147
|
+
_text(self.reducer_id, "reducer_id")
|
|
148
|
+
|
|
149
|
+
def to_dict(self):
|
|
150
|
+
return {"schema_version": "sphynx.lifecycle.policy.v1", "trials": self.trials,
|
|
151
|
+
"candidate_attempts": self.candidate_attempts, "retry_on": list(self.retry_on),
|
|
152
|
+
"selection": self.selection, "seeds": list(self.seeds), "limits": self.limits.to_dict(),
|
|
153
|
+
"reducer_id": self.reducer_id, "reuse": self.reuse}
|
|
154
|
+
|
|
155
|
+
@classmethod
|
|
156
|
+
def from_dict(cls, value):
|
|
157
|
+
value = dict(value)
|
|
158
|
+
if "schema_version" in value:
|
|
159
|
+
_require(value.pop("schema_version") == "sphynx.lifecycle.policy.v1", "unsupported run policy")
|
|
160
|
+
try:
|
|
161
|
+
if "limits" in value and isinstance(value["limits"], Mapping):
|
|
162
|
+
value["limits"] = RunLimits(**value["limits"])
|
|
163
|
+
return cls(**value)
|
|
164
|
+
except TypeError as exc:
|
|
165
|
+
raise LifecycleError("unknown lifecycle policy fields") from exc
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
@dataclass(frozen=True)
|
|
169
|
+
class GradingPolicy:
|
|
170
|
+
attempts: int = 1
|
|
171
|
+
retry_on: tuple[str, ...] = ("evaluator_error",)
|
|
172
|
+
max_total_attempts: int | None = None
|
|
173
|
+
max_wall_seconds: float | None = None
|
|
174
|
+
enforcement: str = "between_calls"
|
|
175
|
+
|
|
176
|
+
def __post_init__(self):
|
|
177
|
+
_positive(self.attempts, "grading attempts")
|
|
178
|
+
_positive(self.max_total_attempts, "grading max_total_attempts", optional=True)
|
|
179
|
+
_seconds(self.max_wall_seconds, "grading max_wall_seconds")
|
|
180
|
+
_require(self.enforcement == "between_calls", "grader deadline enforcement is between calls")
|
|
181
|
+
_require(isinstance(self.retry_on, (tuple, list)) and all(type(v) is str and v for v in self.retry_on),
|
|
182
|
+
"grading retry_on must contain failure codes")
|
|
183
|
+
object.__setattr__(self, "retry_on", tuple(self.retry_on))
|
|
184
|
+
|
|
185
|
+
def to_dict(self):
|
|
186
|
+
return {"schema_version": "sphynx.lifecycle.grading-policy.v1", "attempts": self.attempts,
|
|
187
|
+
"retry_on": list(self.retry_on), "max_total_attempts": self.max_total_attempts,
|
|
188
|
+
"max_wall_seconds": self.max_wall_seconds, "enforcement": self.enforcement}
|
|
189
|
+
|
|
190
|
+
@classmethod
|
|
191
|
+
def from_dict(cls, value):
|
|
192
|
+
value = dict(value)
|
|
193
|
+
if "schema_version" in value:
|
|
194
|
+
_require(value.pop("schema_version") == "sphynx.lifecycle.grading-policy.v1", "unsupported grading policy")
|
|
195
|
+
try:
|
|
196
|
+
return cls(**value)
|
|
197
|
+
except TypeError as exc:
|
|
198
|
+
raise LifecycleError("unknown grading policy fields") from exc
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
@dataclass(frozen=True)
|
|
202
|
+
class DriverDescriptor:
|
|
203
|
+
driver_id: str
|
|
204
|
+
implementation_id: str
|
|
205
|
+
capabilities: tuple[str, ...] = ()
|
|
206
|
+
|
|
207
|
+
def __post_init__(self):
|
|
208
|
+
_text(self.driver_id, "driver_id"); _text(self.implementation_id, "implementation_id")
|
|
209
|
+
_require(isinstance(self.capabilities, (tuple, list)) and all(
|
|
210
|
+
type(x) is str and ("." in x or ":" in x) for x in self.capabilities), "driver capabilities must be namespaced")
|
|
211
|
+
object.__setattr__(self, "capabilities", tuple(self.capabilities))
|
|
212
|
+
|
|
213
|
+
def to_dict(self):
|
|
214
|
+
return {"driver_id": self.driver_id, "implementation_id": self.implementation_id,
|
|
215
|
+
"capabilities": list(self.capabilities)}
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
class CancellationToken:
|
|
219
|
+
"""A request, not evidence that an external operation has stopped."""
|
|
220
|
+
def __init__(self):
|
|
221
|
+
self._event = threading.Event()
|
|
222
|
+
self.reason = "operator_cancelled"
|
|
223
|
+
|
|
224
|
+
def cancel(self, reason="operator_cancelled"):
|
|
225
|
+
_text(reason, "cancellation reason")
|
|
226
|
+
self.reason = reason
|
|
227
|
+
self._event.set()
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def requested(self):
|
|
231
|
+
return self._event.is_set()
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
@dataclass(frozen=True)
|
|
235
|
+
class DriverContext:
|
|
236
|
+
run_id: str
|
|
237
|
+
case_id: str | None
|
|
238
|
+
trial: int
|
|
239
|
+
attempt: int
|
|
240
|
+
seed: int | None
|
|
241
|
+
prepared: Mapping[str, Any]
|
|
242
|
+
deadline: float | None
|
|
243
|
+
cancellation: CancellationToken
|
|
244
|
+
emit: Callable[[str, bytes], Mapping[str, Any]] = field(repr=False)
|
|
245
|
+
limits: RunLimits = field(default_factory=RunLimits)
|
|
246
|
+
remaining: Mapping[str, Any] = field(default_factory=dict)
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
@dataclass(frozen=True)
|
|
250
|
+
class AttemptOutput:
|
|
251
|
+
outputs: Mapping[str, Any] = field(default_factory=dict)
|
|
252
|
+
artifacts: Mapping[str, bytes] = field(default_factory=dict)
|
|
253
|
+
failure: str | None = None
|
|
254
|
+
usage: Mapping[str, Any] = field(default_factory=dict)
|
|
255
|
+
metadata: Mapping[str, Any] = field(default_factory=dict)
|
|
256
|
+
|
|
257
|
+
def __post_init__(self):
|
|
258
|
+
for name in ("outputs", "usage", "metadata"):
|
|
259
|
+
object.__setattr__(self, name, _object(getattr(self, name), name))
|
|
260
|
+
_require(isinstance(self.artifacts, Mapping) and all(type(k) is str and k and type(v) is bytes
|
|
261
|
+
for k, v in self.artifacts.items()), "artifacts must map roles to bytes")
|
|
262
|
+
object.__setattr__(self, "artifacts", dict(self.artifacts))
|
|
263
|
+
if self.failure is not None:
|
|
264
|
+
_text(self.failure, "failure")
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
class DriverFailure(Exception):
|
|
268
|
+
"""A driver failure that retains observed partial outputs/artifacts."""
|
|
269
|
+
def __init__(self, failure="candidate_error", *, partial=None):
|
|
270
|
+
_text(failure, "failure")
|
|
271
|
+
self.failure = failure
|
|
272
|
+
self.partial = partial or AttemptOutput(failure=failure)
|
|
273
|
+
super().__init__(failure)
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
class LifecycleDriver(Protocol):
|
|
277
|
+
descriptor: DriverDescriptor
|
|
278
|
+
def candidate_identity(self, candidate: Any) -> Mapping[str, Any]: ...
|
|
279
|
+
def execute(self, candidate: Any, case: CaseView, context: DriverContext) -> AttemptOutput: ...
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
@dataclass(frozen=True)
|
|
283
|
+
class LifecycleRunRef:
|
|
284
|
+
path: Path
|
|
285
|
+
definition_sha256: str
|
|
286
|
+
head_sha256: str | None = None
|
|
287
|
+
|
|
288
|
+
def __post_init__(self):
|
|
289
|
+
object.__setattr__(self, "path", Path(self.path))
|
|
290
|
+
_require(type(self.definition_sha256) is str and bool(_HASH.fullmatch(self.definition_sha256)), "invalid lifecycle definition digest")
|
|
291
|
+
_require(self.head_sha256 is None or type(self.head_sha256) is str and bool(_HASH.fullmatch(self.head_sha256)), "invalid lifecycle head digest")
|
|
292
|
+
|
|
293
|
+
@property
|
|
294
|
+
def run_id(self):
|
|
295
|
+
return self.path.name
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
@dataclass(frozen=True)
|
|
299
|
+
class LifecycleGradingRef:
|
|
300
|
+
path: Path
|
|
301
|
+
definition_sha256: str
|
|
302
|
+
run: LifecycleRunRef
|
|
303
|
+
head_sha256: str | None = None
|
|
304
|
+
|
|
305
|
+
def __post_init__(self):
|
|
306
|
+
object.__setattr__(self, "path", Path(self.path))
|
|
307
|
+
_require(isinstance(self.run, LifecycleRunRef), "grading requires its native lifecycle run")
|
|
308
|
+
_require(self.path.parent.name == "gradings" and self.path.parent.parent == self.run.path,
|
|
309
|
+
"grading must remain inside its original run")
|
|
310
|
+
_require(type(self.definition_sha256) is str and bool(_HASH.fullmatch(self.definition_sha256)),
|
|
311
|
+
"invalid grading definition digest")
|
|
312
|
+
_require(self.head_sha256 is None or type(self.head_sha256) is str and bool(_HASH.fullmatch(self.head_sha256)),
|
|
313
|
+
"invalid grading journal head")
|
|
314
|
+
|
|
315
|
+
@property
|
|
316
|
+
def grading_id(self):
|
|
317
|
+
return self.path.name
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
class LifecycleJournal:
|
|
321
|
+
"""Hash-linked immutable events and artifacts with an OS-released writer lock."""
|
|
322
|
+
def __init__(self, path, definition_sha256):
|
|
323
|
+
self.path = Path(path)
|
|
324
|
+
self.definition_sha256 = definition_sha256
|
|
325
|
+
|
|
326
|
+
@classmethod
|
|
327
|
+
def create(cls, path, definition):
|
|
328
|
+
path = Path(path)
|
|
329
|
+
_require(not path.exists(), "lifecycle output already exists")
|
|
330
|
+
path.mkdir(parents=True)
|
|
331
|
+
raw = _raw(definition)
|
|
332
|
+
with _safe(path, "definition.json").open("xb") as stream:
|
|
333
|
+
stream.write(raw); stream.flush(); os.fsync(stream.fileno())
|
|
334
|
+
_safe(path, "journal").mkdir()
|
|
335
|
+
return cls(path, _sha(raw))
|
|
336
|
+
|
|
337
|
+
@contextmanager
|
|
338
|
+
def writer(self):
|
|
339
|
+
path = _safe(self.path, ".writer.lock")
|
|
340
|
+
stream = path.open("a+b")
|
|
341
|
+
try:
|
|
342
|
+
stream.seek(0, 2)
|
|
343
|
+
if stream.tell() == 0:
|
|
344
|
+
stream.write(b"0"); stream.flush()
|
|
345
|
+
stream.seek(0)
|
|
346
|
+
try:
|
|
347
|
+
if os.name == "nt":
|
|
348
|
+
import msvcrt
|
|
349
|
+
msvcrt.locking(stream.fileno(), msvcrt.LK_NBLCK, 1)
|
|
350
|
+
else:
|
|
351
|
+
import fcntl
|
|
352
|
+
fcntl.flock(stream.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
353
|
+
except OSError as exc:
|
|
354
|
+
raise LifecycleError("another lifecycle writer is active") from exc
|
|
355
|
+
yield self
|
|
356
|
+
finally:
|
|
357
|
+
stream.close() # The OS also releases this lock after process death.
|
|
358
|
+
|
|
359
|
+
def read(self, head=None):
|
|
360
|
+
definition, raw = _read_json(_safe(self.path, "definition.json"))
|
|
361
|
+
_require(_sha(raw) == self.definition_sha256, "lifecycle definition changed")
|
|
362
|
+
if head == self.definition_sha256:
|
|
363
|
+
return definition, [], self.definition_sha256
|
|
364
|
+
events, previous, found = [], self.definition_sha256, head is None
|
|
365
|
+
names = sorted(_safe(self.path, "journal").iterdir())
|
|
366
|
+
expected = 1
|
|
367
|
+
for path in names:
|
|
368
|
+
if path.name.startswith(".pending-"):
|
|
369
|
+
continue
|
|
370
|
+
match = _ENTRY.fullmatch(path.name)
|
|
371
|
+
_require(match is not None and int(match[1]) == expected, "lifecycle journal sequence differs")
|
|
372
|
+
event, raw = _read_json(_safe(self.path, "journal/" + path.name))
|
|
373
|
+
actual = _sha(raw)
|
|
374
|
+
_require(actual == "sha256:" + match[2] and event.get("schema_version") == JOURNAL_SCHEMA
|
|
375
|
+
and event.get("sequence") == expected and event.get("previous") == previous,
|
|
376
|
+
"lifecycle event digest or predecessor differs")
|
|
377
|
+
events.append(event); previous = actual; expected += 1
|
|
378
|
+
if actual == head:
|
|
379
|
+
found = True
|
|
380
|
+
break
|
|
381
|
+
_require(found, "saved lifecycle journal head is absent")
|
|
382
|
+
return definition, events, previous
|
|
383
|
+
|
|
384
|
+
def append(self, event, data):
|
|
385
|
+
_, events, previous = self.read()
|
|
386
|
+
row = {"schema_version": JOURNAL_SCHEMA, "sequence": len(events) + 1,
|
|
387
|
+
"previous": previous, "event": event, "data": _thaw(_object(data, "event data")),
|
|
388
|
+
"wall_time_utc": datetime.now(timezone.utc).isoformat()}
|
|
389
|
+
raw = _raw(row)
|
|
390
|
+
temporary = _safe(self.path, "journal/.pending-" + uuid4().hex)
|
|
391
|
+
with temporary.open("xb") as stream:
|
|
392
|
+
stream.write(raw); stream.flush(); os.fsync(stream.fileno())
|
|
393
|
+
destination = _safe(self.path, f"journal/{len(events)+1:08d}.{_sha(raw)[7:]}.json")
|
|
394
|
+
_require(not destination.exists(), "lifecycle event already exists")
|
|
395
|
+
os.replace(temporary, destination)
|
|
396
|
+
return _sha(raw)
|
|
397
|
+
|
|
398
|
+
def artifact(self, role, data):
|
|
399
|
+
_text(role, "artifact role"); _require(type(data) is bytes, "artifact data must be bytes")
|
|
400
|
+
relative = "artifacts/" + uuid4().hex + ".bin"
|
|
401
|
+
path = _safe(self.path, relative); path.parent.mkdir(exist_ok=True)
|
|
402
|
+
with path.open("xb") as stream:
|
|
403
|
+
stream.write(data); stream.flush(); os.fsync(stream.fileno())
|
|
404
|
+
return {"path": relative, "sha256": _sha(data), "size_bytes": len(data), "role": role}
|
|
405
|
+
|
|
406
|
+
def read_artifact(self, binding):
|
|
407
|
+
path = _safe(self.path, binding["path"])
|
|
408
|
+
try:
|
|
409
|
+
raw = path.read_bytes()
|
|
410
|
+
except OSError as exc:
|
|
411
|
+
raise LifecycleError("lifecycle artifact is missing") from exc
|
|
412
|
+
_require(len(raw) == binding["size_bytes"] and _sha(raw) == binding["sha256"],
|
|
413
|
+
"lifecycle artifact bytes changed")
|
|
414
|
+
return raw
|
|
415
|
+
|
|
416
|
+
|
|
417
|
+
def verify_lifecycle_run(ref):
|
|
418
|
+
_require(isinstance(ref, LifecycleRunRef), "expected LifecycleRunRef")
|
|
419
|
+
journal = LifecycleJournal(ref.path, ref.definition_sha256)
|
|
420
|
+
definition, events, head = journal.read(ref.head_sha256)
|
|
421
|
+
_require(definition.get("schema_version") == RUN_SCHEMA and definition.get("run_id") == ref.run_id,
|
|
422
|
+
"unsupported or mismatched lifecycle run")
|
|
423
|
+
policy = RunPolicy.from_dict(definition["policy"])
|
|
424
|
+
cases = definition.get("cases", [])
|
|
425
|
+
_require(cases and len({c["case_id"] for c in cases}) == len(cases), "lifecycle cohort differs")
|
|
426
|
+
allowed = {c["case_id"] for c in cases}; started = {}; attempts = []
|
|
427
|
+
for event in events:
|
|
428
|
+
data = event["data"]
|
|
429
|
+
if event["event"] == "artifact":
|
|
430
|
+
journal.read_artifact(data["binding"])
|
|
431
|
+
if event["event"] == "candidate_started":
|
|
432
|
+
key = (data["case_id"], data["trial"], data["attempt"])
|
|
433
|
+
_require(type(key[1]) is int and type(key[2]) is int and key not in started
|
|
434
|
+
and key[0] in allowed and 1 <= key[1] <= policy.trials
|
|
435
|
+
and 1 <= key[2] <= policy.candidate_attempts, "candidate attempt identity differs")
|
|
436
|
+
previous = [row for row in attempts if (row["case_id"], row["trial"]) == key[:2]]
|
|
437
|
+
_require(key[2] == len(previous) + 1 and all(row["failure"] is not None for row in previous),
|
|
438
|
+
"candidate retry sequence or first-success rule differs")
|
|
439
|
+
started[key] = False
|
|
440
|
+
if event["event"] == "candidate_finished":
|
|
441
|
+
key = (data["case_id"], data["trial"], data["attempt"])
|
|
442
|
+
_require(key in started and not started[key], "candidate completion lacks a unique start")
|
|
443
|
+
started[key] = True
|
|
444
|
+
_require(data.get("failure") is None or type(data["failure"]) is str, "invalid failure code")
|
|
445
|
+
for binding in data.get("artifacts", {}).values():
|
|
446
|
+
journal.read_artifact(binding)
|
|
447
|
+
outputs = json.loads(journal.read_artifact(data["outputs"])) if data.get("outputs") else {}
|
|
448
|
+
outputs = _thaw(_object(outputs, "verified outputs"))
|
|
449
|
+
attempts.append({**data, "outputs": outputs, "output_binding": data.get("outputs")})
|
|
450
|
+
terminal_event = next((e for e in reversed(events) if e["event"] in
|
|
451
|
+
("run_finished", "run_interrupted", "run_resumed")), None)
|
|
452
|
+
status = ("running" if terminal_event is None or terminal_event["event"] == "run_resumed" else
|
|
453
|
+
"interrupted" if terminal_event["event"] == "run_interrupted" else terminal_event["data"]["status"])
|
|
454
|
+
cleanup = next((e["data"]["status"] for e in reversed(events) if e["event"] == "cleanup"), None)
|
|
455
|
+
return {"definition": definition, "events": events, "head_sha256": head,
|
|
456
|
+
"attempts": attempts, "unfinished": [key for key, finished in started.items() if not finished],
|
|
457
|
+
"status": status, "cleanup_status": cleanup}
|
|
458
|
+
|
|
459
|
+
|
|
460
|
+
def open_lifecycle_run(path, *, expected_definition_sha256=None, expected_head_sha256=None):
|
|
461
|
+
"""Open local bytes; computing a digest does not authenticate their owner."""
|
|
462
|
+
path = Path(path)
|
|
463
|
+
_, raw = _read_json(_safe(path, "definition.json"))
|
|
464
|
+
digest = expected_definition_sha256 or _sha(raw)
|
|
465
|
+
ref = LifecycleRunRef(path, digest, expected_head_sha256)
|
|
466
|
+
verified = verify_lifecycle_run(ref)
|
|
467
|
+
return LifecycleRunRef(path, digest, verified["head_sha256"])
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _driver_admission(driver, policy):
|
|
471
|
+
_require(isinstance(getattr(driver, "descriptor", None), DriverDescriptor), "driver descriptor is required")
|
|
472
|
+
_require(callable(getattr(driver, "candidate_identity", None)) and callable(getattr(driver, "execute", None)),
|
|
473
|
+
"driver identity and execution methods are required")
|
|
474
|
+
caps = driver.descriptor.capabilities
|
|
475
|
+
limits = policy.limits
|
|
476
|
+
if limits.per_attempt_seconds is not None or limits.max_wall_seconds is not None:
|
|
477
|
+
required = "sphynx.lifecycle.deadline." + limits.enforcement + ".v1"
|
|
478
|
+
_require(required in caps, "driver cannot enforce the requested deadline policy")
|
|
479
|
+
for field, capability in (("max_tokens", "sphynx.lifecycle.tokens.v1"),
|
|
480
|
+
("max_cost", "sphynx.lifecycle.cost.v1")):
|
|
481
|
+
_require(getattr(limits, field) is None or capability in caps,
|
|
482
|
+
"driver cannot enforce " + field)
|
|
483
|
+
_require(not policy.seeds or "sphynx.lifecycle.seeds.v1" in caps,
|
|
484
|
+
"driver has no declared seed binding")
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def _case_rows(cases):
|
|
488
|
+
rows = tuple(cases)
|
|
489
|
+
_require(rows and all(isinstance(case, CaseView) for case in rows), "lifecycle needs CaseViews")
|
|
490
|
+
_require(len({case.case_id for case in rows}) == len(rows), "duplicate lifecycle case")
|
|
491
|
+
return rows
|
|
492
|
+
|
|
493
|
+
|
|
494
|
+
def _selection(record):
|
|
495
|
+
"""Select execution-first-success, otherwise retain the final failed attempt."""
|
|
496
|
+
result = {}
|
|
497
|
+
for row in record["attempts"]:
|
|
498
|
+
key = (row["case_id"], row["trial"])
|
|
499
|
+
if key not in result or result[key]["failure"] is not None:
|
|
500
|
+
result[key] = row
|
|
501
|
+
return result
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _reuse_identity(definition):
|
|
505
|
+
value = {key: definition[key] for key in ("identity", "candidate_identity", "driver", "cases", "policy")}
|
|
506
|
+
value = json.loads(_raw(value))
|
|
507
|
+
value["policy"].pop("reuse", None)
|
|
508
|
+
return contract_digest(value)
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
class _LifecycleStop(Exception):
|
|
512
|
+
pass
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _preparation_identity(value):
|
|
516
|
+
return {key: val for key, val in value.items() if key != "usage"}
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def execute_lifecycle(driver, candidate, cases, output_dir, *, identity, policy=None,
|
|
520
|
+
cancellation=None, resume=None, reuse=None, grading_policy=None, recheck=None):
|
|
521
|
+
"""Execute declared trials and failure retries, retaining every committed try.
|
|
522
|
+
|
|
523
|
+
Deadlines apply to elapsed run wall time, including preparation and idle time
|
|
524
|
+
before resume. Hard enforcement requires a driver capability. Cooperative
|
|
525
|
+
execution can overrun a callback; this is recorded rather than called hard.
|
|
526
|
+
"""
|
|
527
|
+
policy = policy or RunPolicy()
|
|
528
|
+
_require(isinstance(policy, RunPolicy), "policy must be RunPolicy")
|
|
529
|
+
_driver_admission(driver, policy)
|
|
530
|
+
cases = _case_rows(cases)
|
|
531
|
+
token = cancellation or CancellationToken()
|
|
532
|
+
_require(isinstance(token, CancellationToken), "invalid cancellation token")
|
|
533
|
+
candidate_identity = _thaw(_object(driver.candidate_identity(candidate), "candidate identity"))
|
|
534
|
+
requested = {"identity": _thaw(_object(identity, "execution identity")), "policy": policy.to_dict(),
|
|
535
|
+
"candidate_identity": candidate_identity, "driver": driver.descriptor.to_dict(),
|
|
536
|
+
"cases": [case.to_dict() for case in cases]}
|
|
537
|
+
reuse_record = None
|
|
538
|
+
if reuse is not None:
|
|
539
|
+
_require(policy.reuse == "verified" and "sphynx.lifecycle.reuse.v1" in driver.descriptor.capabilities,
|
|
540
|
+
"compatible reuse is not enabled by this policy/driver")
|
|
541
|
+
reuse_record = verify_lifecycle_run(reuse)
|
|
542
|
+
_require(not reuse_record["unfinished"], "cannot reuse an interrupted attempt")
|
|
543
|
+
_require(_reuse_identity(requested) == _reuse_identity(reuse_record["definition"]),
|
|
544
|
+
"incompatible lifecycle reuse identity")
|
|
545
|
+
authorize = getattr(driver, "validate_reuse", None)
|
|
546
|
+
_require(callable(authorize) and authorize(candidate, reuse_record, requested["identity"]) is True,
|
|
547
|
+
"driver did not authorize freshness and reuse")
|
|
548
|
+
if resume is None:
|
|
549
|
+
run_id = "run-" + uuid4().hex
|
|
550
|
+
definition = {"schema_version": RUN_SCHEMA, "run_id": run_id, **requested,
|
|
551
|
+
"created_utc": datetime.now(timezone.utc).isoformat(),
|
|
552
|
+
"default_grading_policy": None if grading_policy is None else grading_policy.to_dict()}
|
|
553
|
+
journal = LifecycleJournal.create(Path(output_dir) / "runs" / run_id, definition)
|
|
554
|
+
else:
|
|
555
|
+
_require(isinstance(resume, LifecycleRunRef), "resume requires LifecycleRunRef")
|
|
556
|
+
journal = LifecycleJournal(resume.path, resume.definition_sha256)
|
|
557
|
+
definition, _, head = journal.read()
|
|
558
|
+
_require(resume.head_sha256 is None or resume.head_sha256 == head, "resume reference is not the current head")
|
|
559
|
+
_require(all(definition.get(key) == value for key, value in requested.items()),
|
|
560
|
+
"incompatible lifecycle resume identity")
|
|
561
|
+
run_id = resume.run_id
|
|
562
|
+
ref = lambda: LifecycleRunRef(journal.path, journal.definition_sha256, journal.read()[2])
|
|
563
|
+
initial_elapsed = (datetime.now(timezone.utc) - datetime.fromisoformat(definition["created_utc"])).total_seconds()
|
|
564
|
+
_require(initial_elapsed >= 0, "run clock moved backwards")
|
|
565
|
+
segment_start = time.monotonic()
|
|
566
|
+
prepared = {}
|
|
567
|
+
cleanup_needed = False
|
|
568
|
+
used = {"tokens": 0, "cost": 0.0}
|
|
569
|
+
emitted = {}
|
|
570
|
+
|
|
571
|
+
def elapsed():
|
|
572
|
+
return initial_elapsed + time.monotonic() - segment_start
|
|
573
|
+
|
|
574
|
+
def context(case_id=None, trial=0, attempt=0):
|
|
575
|
+
wall = None if policy.limits.max_wall_seconds is None else policy.limits.max_wall_seconds - elapsed()
|
|
576
|
+
seconds = policy.limits.per_attempt_seconds
|
|
577
|
+
if wall is not None:
|
|
578
|
+
seconds = wall if seconds is None else min(seconds, wall)
|
|
579
|
+
deadline = None if seconds is None else time.monotonic() + max(0.0, seconds)
|
|
580
|
+
def emit(role, data):
|
|
581
|
+
_require(type(data) is bytes, "streamed artifacts must be bytes")
|
|
582
|
+
key = (case_id, trial, attempt)
|
|
583
|
+
original_size = len(data)
|
|
584
|
+
maximum = policy.limits.max_output_bytes
|
|
585
|
+
if maximum is not None:
|
|
586
|
+
data = data[:max(0, maximum - emitted.get(key, 0))]
|
|
587
|
+
emitted[key] = emitted.get(key, 0) + len(data)
|
|
588
|
+
binding = journal.artifact(role, data)
|
|
589
|
+
journal.append("artifact", {"case_id": case_id, "trial": trial, "attempt": attempt,
|
|
590
|
+
"binding": binding, "truncated": len(data) != original_size,
|
|
591
|
+
"observed_bytes": original_size})
|
|
592
|
+
return binding
|
|
593
|
+
remaining = {name: None if maximum is None else max(0, maximum - used[name])
|
|
594
|
+
for name, maximum in (("tokens", policy.limits.max_tokens), ("cost", policy.limits.max_cost))}
|
|
595
|
+
return DriverContext(run_id, case_id, trial, attempt,
|
|
596
|
+
policy.seeds[trial - 1] if trial and policy.seeds else None,
|
|
597
|
+
_object(prepared, "preparation"), deadline, token, emit, policy.limits,
|
|
598
|
+
_object(remaining, "remaining budget"))
|
|
599
|
+
|
|
600
|
+
def consume(usage):
|
|
601
|
+
for name in used:
|
|
602
|
+
value = usage.get(name, 0)
|
|
603
|
+
_require(type(value) in (int, float) and math.isfinite(value) and value >= 0,
|
|
604
|
+
"driver usage must be finite and nonnegative")
|
|
605
|
+
_require(name != "tokens" or type(value) is int, "token usage must be integer")
|
|
606
|
+
used[name] += value
|
|
607
|
+
|
|
608
|
+
def budget_reason(record):
|
|
609
|
+
if token.requested:
|
|
610
|
+
return token.reason
|
|
611
|
+
started = sum(event["event"] == "candidate_started" and not event["data"].get("reused", False)
|
|
612
|
+
for event in record["events"])
|
|
613
|
+
if policy.limits.max_total_attempts is not None and started >= policy.limits.max_total_attempts:
|
|
614
|
+
return "attempt_budget_exhausted"
|
|
615
|
+
if policy.limits.max_wall_seconds is not None and elapsed() >= policy.limits.max_wall_seconds:
|
|
616
|
+
return "wall_budget_exhausted"
|
|
617
|
+
for name, maximum in (("tokens", policy.limits.max_tokens), ("cost", policy.limits.max_cost)):
|
|
618
|
+
if maximum is not None and used[name] >= maximum:
|
|
619
|
+
return name + "_budget_exhausted"
|
|
620
|
+
return None
|
|
621
|
+
|
|
622
|
+
with journal.writer():
|
|
623
|
+
if resume is not None:
|
|
624
|
+
_require(resume.head_sha256 is None or resume.head_sha256 == journal.read()[2],
|
|
625
|
+
"resume reference is not the current head")
|
|
626
|
+
record = verify_lifecycle_run(ref())
|
|
627
|
+
for row in record["attempts"]:
|
|
628
|
+
if not row.get("reused", False):
|
|
629
|
+
consume(row.get("usage", {}))
|
|
630
|
+
for event in record["events"]:
|
|
631
|
+
if event["event"] == "prepared":
|
|
632
|
+
consume(event["data"]["value"].get("usage", {}))
|
|
633
|
+
# A started but uncommitted operation is uncertain, never a free retry.
|
|
634
|
+
interrupted = bool(record["unfinished"])
|
|
635
|
+
for case_id, trial, attempt in record["unfinished"]:
|
|
636
|
+
start = next(e["data"] for e in record["events"] if e["event"] == "candidate_started"
|
|
637
|
+
and (e["data"]["case_id"], e["data"]["trial"], e["data"]["attempt"]) == (case_id, trial, attempt))
|
|
638
|
+
reserved = {name: value for name, value in start.get("reserved", {}).items() if value is not None}
|
|
639
|
+
journal.append("candidate_finished", {"case_id": case_id, "trial": trial, "attempt": attempt,
|
|
640
|
+
"failure": "interrupted", "outputs": None, "artifacts": {}, "usage": reserved,
|
|
641
|
+
"elapsed_seconds": None, "metadata": {"side_effects": "unknown",
|
|
642
|
+
"usage_authority": "reserved_unknown_after_interruption"}, "reused": False})
|
|
643
|
+
consume(reserved)
|
|
644
|
+
record = verify_lifecycle_run(ref())
|
|
645
|
+
if interrupted and not ("sphynx.lifecycle.idempotent.v1" in driver.descriptor.capabilities
|
|
646
|
+
or "sphynx.lifecycle.reset.v1" in driver.descriptor.capabilities
|
|
647
|
+
and callable(getattr(driver, "reset", None))):
|
|
648
|
+
journal.append("run_finished", {"status": "interrupted", "reason": "driver_cannot_recover_uncertain_side_effects"})
|
|
649
|
+
return ref()
|
|
650
|
+
previous_preparation = next((e["data"]["value"] for e in record["events"] if e["event"] == "prepared"), None)
|
|
651
|
+
if record["unfinished"]:
|
|
652
|
+
raise LifecycleError("unfinished lifecycle attempt remains")
|
|
653
|
+
selected = _selection(record)
|
|
654
|
+
def done(case, trial):
|
|
655
|
+
row = selected.get((case.case_id, trial))
|
|
656
|
+
return row is not None and (row["failure"] is None or row["attempt"] >= policy.candidate_attempts
|
|
657
|
+
or row["failure"] not in policy.retry_on)
|
|
658
|
+
if all(done(case, trial) for case in cases for trial in range(1, policy.trials + 1)):
|
|
659
|
+
if record["status"] in ("running", "interrupted"):
|
|
660
|
+
journal.append("run_finished", {"status": "completed_with_failures" if any(
|
|
661
|
+
row["failure"] is not None for row in selected.values()) else "completed", "reason": None})
|
|
662
|
+
return ref()
|
|
663
|
+
reason = budget_reason(record)
|
|
664
|
+
if reason:
|
|
665
|
+
journal.append("run_finished", {"status": "cancelled" if token.requested else "limited", "reason": reason})
|
|
666
|
+
return ref()
|
|
667
|
+
try:
|
|
668
|
+
if resume is not None:
|
|
669
|
+
journal.append("run_resumed", {"previous_head": record["head_sha256"]})
|
|
670
|
+
cleanup_needed = True
|
|
671
|
+
prepare = getattr(driver, "prepare", None)
|
|
672
|
+
begin = time.monotonic()
|
|
673
|
+
value = {} if prepare is None else prepare(candidate, context())
|
|
674
|
+
prepared = _thaw(_object(value, "prepared identity"))
|
|
675
|
+
if previous_preparation is not None:
|
|
676
|
+
_require(_preparation_identity(prepared) == _preparation_identity(previous_preparation),
|
|
677
|
+
"prepared environment/corpus changed before resume")
|
|
678
|
+
if reuse_record is not None:
|
|
679
|
+
source_preparation = next((e["data"]["value"] for e in reuse_record["events"]
|
|
680
|
+
if e["event"] == "prepared"), {})
|
|
681
|
+
_require(_preparation_identity(prepared) == _preparation_identity(source_preparation),
|
|
682
|
+
"prepared environment/corpus changed before reuse")
|
|
683
|
+
journal.append("prepared", {"value": prepared, "elapsed_seconds": time.monotonic() - begin})
|
|
684
|
+
consume(prepared.get("usage", {}))
|
|
685
|
+
for case in cases:
|
|
686
|
+
for trial in range(1, policy.trials + 1):
|
|
687
|
+
record = verify_lifecycle_run(ref()); selected = _selection(record)
|
|
688
|
+
if done(case, trial):
|
|
689
|
+
continue
|
|
690
|
+
# Reuse all prior failures as well as the selected success.
|
|
691
|
+
reusable = [] if reuse_record is None else [r for r in reuse_record["attempts"]
|
|
692
|
+
if (r["case_id"], r["trial"]) == (case.case_id, trial)]
|
|
693
|
+
if reusable and reusable[-1]["failure"] is None:
|
|
694
|
+
source = LifecycleJournal(reuse.path, reuse.definition_sha256)
|
|
695
|
+
for old in reusable:
|
|
696
|
+
key = {"case_id": case.case_id, "trial": trial, "attempt": old["attempt"]}
|
|
697
|
+
journal.append("candidate_started", {**key, "reused": True})
|
|
698
|
+
artifacts = {role: journal.artifact(role, source.read_artifact(binding))
|
|
699
|
+
for role, binding in old["artifacts"].items()}
|
|
700
|
+
outputs = journal.artifact("outputs", _raw(old["outputs"]))
|
|
701
|
+
journal.append("candidate_finished", {**key, "failure": old["failure"],
|
|
702
|
+
"outputs": outputs, "artifacts": artifacts, "usage": {}, "elapsed_seconds": old.get("elapsed_seconds"),
|
|
703
|
+
"reused": True, "metadata": {"source_run": reuse.run_id, "source_head": reuse_record["head_sha256"],
|
|
704
|
+
"source_attempt": old["attempt"], "timing_authority": "reused_saved_observation"}})
|
|
705
|
+
continue
|
|
706
|
+
previous = [r for r in record["attempts"] if (r["case_id"], r["trial"]) == (case.case_id, trial)]
|
|
707
|
+
for attempt in range(len(previous) + 1, policy.candidate_attempts + 1):
|
|
708
|
+
record = verify_lifecycle_run(ref())
|
|
709
|
+
reason = budget_reason(record)
|
|
710
|
+
if reason:
|
|
711
|
+
journal.append("run_finished", {"status": "cancelled" if token.requested else "limited", "reason": reason})
|
|
712
|
+
raise _LifecycleStop()
|
|
713
|
+
_require(_thaw(_object(driver.candidate_identity(candidate), "candidate identity")) == candidate_identity,
|
|
714
|
+
"candidate identity changed before execution")
|
|
715
|
+
if recheck is not None:
|
|
716
|
+
recheck()
|
|
717
|
+
key = {"case_id": case.case_id, "trial": trial, "attempt": attempt}
|
|
718
|
+
ctx = context(case.case_id, trial, attempt)
|
|
719
|
+
journal.append("candidate_started", {**key, "reused": False, "reserved": _thaw(ctx.remaining)})
|
|
720
|
+
begin = time.monotonic()
|
|
721
|
+
try:
|
|
722
|
+
reset = getattr(driver, "reset", None)
|
|
723
|
+
if reset is not None:
|
|
724
|
+
try:
|
|
725
|
+
reset(candidate, case, ctx)
|
|
726
|
+
except Exception as exc:
|
|
727
|
+
raise DriverFailure("environment_error", partial=AttemptOutput(
|
|
728
|
+
failure="environment_error", metadata={"error_type": type(exc).__name__})) from exc
|
|
729
|
+
output = driver.execute(candidate, case, ctx)
|
|
730
|
+
_require(isinstance(output, AttemptOutput), "driver must return AttemptOutput")
|
|
731
|
+
except DriverFailure as exc:
|
|
732
|
+
output = AttemptOutput(exc.partial.outputs, exc.partial.artifacts, exc.failure,
|
|
733
|
+
exc.partial.usage, exc.partial.metadata)
|
|
734
|
+
except Exception as exc:
|
|
735
|
+
output = AttemptOutput(failure="candidate_error", metadata={"error_type": type(exc).__name__})
|
|
736
|
+
duration = time.monotonic() - begin
|
|
737
|
+
failure = output.failure
|
|
738
|
+
metadata = _thaw(output.metadata)
|
|
739
|
+
if token.requested:
|
|
740
|
+
metadata["cancellation_requested"] = True
|
|
741
|
+
if metadata.get("cancellation_acknowledged") is True:
|
|
742
|
+
failure = "cancelled"
|
|
743
|
+
if ctx.deadline is not None and time.monotonic() > ctx.deadline:
|
|
744
|
+
failure = "timeout"
|
|
745
|
+
encoded = _raw(output.outputs)
|
|
746
|
+
streamed = emitted.get((case.case_id, trial, attempt), 0)
|
|
747
|
+
retained = streamed + len(encoded) + sum(len(raw) for raw in output.artifacts.values())
|
|
748
|
+
truncated = policy.limits.max_output_bytes is not None and retained > policy.limits.max_output_bytes
|
|
749
|
+
if truncated:
|
|
750
|
+
failure = "output_limit"
|
|
751
|
+
raw = (encoded + b"\n" + b"".join(output.artifacts.values()))[:max(0, policy.limits.max_output_bytes-streamed)]
|
|
752
|
+
artifacts = {"truncated_output": journal.artifact("truncated_output", raw)}
|
|
753
|
+
else:
|
|
754
|
+
artifacts = {role: journal.artifact(role, raw) for role, raw in output.artifacts.items()}
|
|
755
|
+
outputs = None if truncated else journal.artifact("outputs", encoded)
|
|
756
|
+
usage = _thaw(output.usage)
|
|
757
|
+
for name, maximum in (("tokens", policy.limits.max_tokens), ("cost", policy.limits.max_cost)):
|
|
758
|
+
if maximum is not None and name not in usage:
|
|
759
|
+
usage[name] = ctx.remaining[name]
|
|
760
|
+
metadata["usage_authority"] = "reserved_unreported"
|
|
761
|
+
failure = "usage_unavailable"
|
|
762
|
+
consume(usage)
|
|
763
|
+
for name, maximum in (("tokens", policy.limits.max_tokens), ("cost", policy.limits.max_cost)):
|
|
764
|
+
if maximum is not None and used[name] > maximum:
|
|
765
|
+
failure = name + "_budget_exceeded"
|
|
766
|
+
journal.append("candidate_finished", {**key, "failure": failure, "outputs": outputs,
|
|
767
|
+
"artifacts": artifacts, "usage": usage, "elapsed_seconds": duration,
|
|
768
|
+
"metadata": metadata, "reused": False})
|
|
769
|
+
if failure is None or failure not in policy.retry_on:
|
|
770
|
+
break
|
|
771
|
+
record = verify_lifecycle_run(ref())
|
|
772
|
+
failure_count = sum(row["failure"] is not None for row in _selection(record).values())
|
|
773
|
+
journal.append("run_finished", {"status": "cancelled" if token.requested else
|
|
774
|
+
"completed_with_failures" if failure_count else "completed",
|
|
775
|
+
"reason": token.reason if token.requested else None})
|
|
776
|
+
except _LifecycleStop:
|
|
777
|
+
pass
|
|
778
|
+
except BaseException as exc:
|
|
779
|
+
journal.append("run_interrupted", {"error_type": type(exc).__name__})
|
|
780
|
+
raise
|
|
781
|
+
finally:
|
|
782
|
+
if cleanup_needed:
|
|
783
|
+
cleanup = getattr(driver, "cleanup", None)
|
|
784
|
+
try:
|
|
785
|
+
value = {} if cleanup is None else cleanup(candidate, context())
|
|
786
|
+
journal.append("cleanup", {"status": "completed", "value": {} if value is None else value})
|
|
787
|
+
except Exception as exc:
|
|
788
|
+
journal.append("cleanup", {"status": "failed", "error_type": type(exc).__name__})
|
|
789
|
+
return ref()
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
@dataclass(frozen=True)
|
|
793
|
+
class GradingContext:
|
|
794
|
+
run_id: str
|
|
795
|
+
grading_id: str
|
|
796
|
+
case_id: str
|
|
797
|
+
trial: int
|
|
798
|
+
attempt: int
|
|
799
|
+
grading_attempt: int
|
|
800
|
+
|
|
801
|
+
|
|
802
|
+
def assess_lifecycle(run, evaluator, *, identity, policy=None, cancellation=None):
|
|
803
|
+
"""Apply evaluator retries to one verified immutable run snapshot only."""
|
|
804
|
+
verified = verify_lifecycle_run(run)
|
|
805
|
+
default = verified["definition"].get("default_grading_policy")
|
|
806
|
+
policy = policy or (GradingPolicy.from_dict(default) if default is not None else GradingPolicy())
|
|
807
|
+
_require(isinstance(policy, GradingPolicy) and callable(evaluator), "invalid grading policy/evaluator")
|
|
808
|
+
_require(not verified["unfinished"], "cannot grade uncommitted execution")
|
|
809
|
+
run = LifecycleRunRef(run.path, run.definition_sha256, verified["head_sha256"])
|
|
810
|
+
grading_id = "grading-" + uuid4().hex
|
|
811
|
+
definition = {"schema_version": GRADING_SCHEMA, "grading_id": grading_id, "run_id": run.run_id,
|
|
812
|
+
"run_definition_sha256": run.definition_sha256, "run_head_sha256": run.head_sha256,
|
|
813
|
+
"identity": _thaw(_object(identity, "grading identity")), "policy": policy.to_dict()}
|
|
814
|
+
journal = LifecycleJournal.create(run.path / "gradings" / grading_id, definition)
|
|
815
|
+
token = cancellation or CancellationToken()
|
|
816
|
+
selected = _selection(verified); count = 0; begin = time.monotonic()
|
|
817
|
+
with journal.writer():
|
|
818
|
+
for raw_case in verified["definition"]["cases"]:
|
|
819
|
+
case = CaseView(**raw_case)
|
|
820
|
+
for trial in range(1, verified["definition"]["policy"]["trials"] + 1):
|
|
821
|
+
row = selected.get((case.case_id, trial))
|
|
822
|
+
if row is None:
|
|
823
|
+
journal.append("grading_unavailable", {"case_id": case.case_id, "trial": trial,
|
|
824
|
+
"reason": "candidate_not_started"})
|
|
825
|
+
continue
|
|
826
|
+
for attempt in range(1, policy.attempts + 1):
|
|
827
|
+
if token.requested or policy.max_total_attempts is not None and count >= policy.max_total_attempts or (
|
|
828
|
+
policy.max_wall_seconds is not None and time.monotonic() - begin >= policy.max_wall_seconds
|
|
829
|
+
):
|
|
830
|
+
journal.append("grading_limited", {"case_id": case.case_id, "trial": trial,
|
|
831
|
+
"reason": token.reason if token.requested else "grading_budget_exhausted"})
|
|
832
|
+
break
|
|
833
|
+
context = GradingContext(run.run_id, grading_id, case.case_id, trial, row["attempt"], attempt)
|
|
834
|
+
key = {"case_id": case.case_id, "trial": trial, "candidate_attempt": row["attempt"], "grading_attempt": attempt}
|
|
835
|
+
journal.append("grading_started", key); count += 1
|
|
836
|
+
try:
|
|
837
|
+
output = evaluator(case, row, context)
|
|
838
|
+
output = _object(output, "evaluator output")
|
|
839
|
+
failure = None
|
|
840
|
+
except DriverFailure as exc:
|
|
841
|
+
output, failure = exc.partial.outputs, exc.failure
|
|
842
|
+
except Exception as exc:
|
|
843
|
+
output, failure = {"error_type": type(exc).__name__}, "evaluator_error"
|
|
844
|
+
binding = journal.artifact("assessment", _raw(output))
|
|
845
|
+
journal.append("grading_finished", {**key, "failure": failure, "output": binding})
|
|
846
|
+
if policy.max_wall_seconds is not None and time.monotonic() - begin > policy.max_wall_seconds:
|
|
847
|
+
journal.append("grading_deadline_exceeded", {**key, "enforcement": "between_calls"})
|
|
848
|
+
if failure is None or failure not in policy.retry_on:
|
|
849
|
+
break
|
|
850
|
+
head = journal.append("grading_complete", {"attempt_count": count})
|
|
851
|
+
return LifecycleGradingRef(journal.path, journal.definition_sha256, run, head)
|
|
852
|
+
|
|
853
|
+
|
|
854
|
+
def verify_lifecycle_grading(ref):
|
|
855
|
+
_require(isinstance(ref, LifecycleGradingRef), "expected LifecycleGradingRef")
|
|
856
|
+
run = verify_lifecycle_run(ref.run)
|
|
857
|
+
journal = LifecycleJournal(ref.path, ref.definition_sha256)
|
|
858
|
+
definition, events, head = journal.read(ref.head_sha256)
|
|
859
|
+
_require(definition.get("schema_version") == GRADING_SCHEMA and definition.get("grading_id") == ref.grading_id
|
|
860
|
+
and definition.get("run_id") == ref.run.run_id
|
|
861
|
+
and definition.get("run_definition_sha256") == ref.run.definition_sha256
|
|
862
|
+
and definition.get("run_head_sha256") == run["head_sha256"], "grading/source snapshot differs")
|
|
863
|
+
policy = GradingPolicy.from_dict(definition["policy"])
|
|
864
|
+
selected = _selection(run)
|
|
865
|
+
started = set(); completed = set(); rows = []
|
|
866
|
+
for event in events:
|
|
867
|
+
data = event["data"]
|
|
868
|
+
if event["event"] == "grading_started":
|
|
869
|
+
key = (data["case_id"], data["trial"], data["candidate_attempt"], data["grading_attempt"])
|
|
870
|
+
source = selected.get(key[:2])
|
|
871
|
+
earlier = [row for row in rows if (row["case_id"], row["trial"]) == key[:2]]
|
|
872
|
+
_require(key not in started and key not in completed and source is not None
|
|
873
|
+
and key[2] == source["attempt"] and type(key[3]) is int
|
|
874
|
+
and 1 <= key[3] <= policy.attempts and key[3] == len(earlier) + 1
|
|
875
|
+
and all(row["failure"] is not None for row in earlier), "invalid or duplicate grading attempt")
|
|
876
|
+
started.add(key)
|
|
877
|
+
if event["event"] == "grading_finished":
|
|
878
|
+
key = (data["case_id"], data["trial"], data["candidate_attempt"], data["grading_attempt"])
|
|
879
|
+
_require(key in started, "grading completion lacks start")
|
|
880
|
+
started.remove(key)
|
|
881
|
+
completed.add(key)
|
|
882
|
+
output = _thaw(_object(json.loads(journal.read_artifact(data["output"])), "verified evaluator output"))
|
|
883
|
+
rows.append({**data, "output": output})
|
|
884
|
+
_require(not started, "grading attempt remains uncommitted")
|
|
885
|
+
return {"definition": definition, "run": run, "events": events, "rows": rows, "head_sha256": head}
|