dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
"""Validated immutable read model for one persisted run archive."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
import json
|
|
7
|
+
import math
|
|
8
|
+
import os
|
|
9
|
+
from collections.abc import Callable, Mapping
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path, PurePosixPath
|
|
12
|
+
from types import MappingProxyType
|
|
13
|
+
from typing import cast
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
from safetensors import SafetensorError
|
|
17
|
+
from safetensors.numpy import (
|
|
18
|
+
load as _load, # pyright: ignore[reportUnknownVariableType]
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
from dromeus.errors import RunError
|
|
22
|
+
from dromeus.manifests.canonical import canonical_hash, parse_sealed_json
|
|
23
|
+
from dromeus.manifests.models import SealedManifest, Sha256
|
|
24
|
+
|
|
25
|
+
ARCHIVE_VERSION = 2
|
|
26
|
+
_STATE_FIELDS = frozenset(
|
|
27
|
+
{
|
|
28
|
+
"archive_version",
|
|
29
|
+
"manifest_hash",
|
|
30
|
+
"committed_round",
|
|
31
|
+
"state_checksum",
|
|
32
|
+
"algorithm_state",
|
|
33
|
+
"prepared_commit",
|
|
34
|
+
"schedule_history",
|
|
35
|
+
"metrics",
|
|
36
|
+
"transfer_diagnostics",
|
|
37
|
+
"consensus",
|
|
38
|
+
"terminal",
|
|
39
|
+
}
|
|
40
|
+
)
|
|
41
|
+
_LOAD_SAFETENSORS = cast(
|
|
42
|
+
Callable[[bytes], dict[str, np.ndarray]],
|
|
43
|
+
_load,
|
|
44
|
+
)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
class RunArchiveError(RunError, ValueError):
|
|
48
|
+
"""A persisted run archive is unsafe, inconsistent, or corrupt."""
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
JsonRecord = Mapping[str, object]
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
@dataclass(frozen=True, slots=True)
|
|
55
|
+
class CheckpointRecord:
|
|
56
|
+
relative_path: str
|
|
57
|
+
sha256: Sha256
|
|
58
|
+
|
|
59
|
+
def as_json(self) -> dict[str, str]:
|
|
60
|
+
return {"path": self.relative_path, "sha256": self.sha256}
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True, slots=True)
|
|
64
|
+
class PreparedCommitState:
|
|
65
|
+
round_id: int
|
|
66
|
+
state_checksum: Sha256
|
|
67
|
+
algorithm_state: CheckpointRecord
|
|
68
|
+
schedule: JsonRecord
|
|
69
|
+
metrics: JsonRecord | None
|
|
70
|
+
transfer_diagnostics: JsonRecord | None
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@dataclass(frozen=True, slots=True)
|
|
74
|
+
class ConsensusState:
|
|
75
|
+
round_id: int
|
|
76
|
+
normalized_rms: float
|
|
77
|
+
sketch_count: int
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@dataclass(frozen=True, slots=True)
|
|
81
|
+
class TerminalState:
|
|
82
|
+
result: str
|
|
83
|
+
diagnostics: JsonRecord
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@dataclass(frozen=True, slots=True)
|
|
87
|
+
class ArchiveState:
|
|
88
|
+
archive_version: int
|
|
89
|
+
manifest_hash: Sha256
|
|
90
|
+
committed_round: int
|
|
91
|
+
state_checksum: Sha256 | None
|
|
92
|
+
algorithm_state: CheckpointRecord | None
|
|
93
|
+
prepared_commit: PreparedCommitState | None
|
|
94
|
+
schedule_history: tuple[JsonRecord, ...]
|
|
95
|
+
metrics: tuple[JsonRecord, ...]
|
|
96
|
+
transfer_diagnostics: tuple[JsonRecord, ...]
|
|
97
|
+
consensus: tuple[ConsensusState, ...]
|
|
98
|
+
terminal: TerminalState | None
|
|
99
|
+
|
|
100
|
+
def checkpoint_records(self) -> tuple[CheckpointRecord, ...]:
|
|
101
|
+
records: list[CheckpointRecord] = []
|
|
102
|
+
if self.algorithm_state is not None:
|
|
103
|
+
records.append(self.algorithm_state)
|
|
104
|
+
if self.prepared_commit is not None:
|
|
105
|
+
records.append(self.prepared_commit.algorithm_state)
|
|
106
|
+
return tuple(records)
|
|
107
|
+
|
|
108
|
+
def as_json(self) -> dict[str, object]:
|
|
109
|
+
return {
|
|
110
|
+
"archive_version": ARCHIVE_VERSION,
|
|
111
|
+
"manifest_hash": self.manifest_hash,
|
|
112
|
+
"committed_round": self.committed_round,
|
|
113
|
+
"state_checksum": self.state_checksum,
|
|
114
|
+
"algorithm_state": (
|
|
115
|
+
self.algorithm_state.as_json()
|
|
116
|
+
if self.algorithm_state is not None
|
|
117
|
+
else None
|
|
118
|
+
),
|
|
119
|
+
"prepared_commit": (
|
|
120
|
+
_prepared_as_json(self.prepared_commit)
|
|
121
|
+
if self.prepared_commit is not None
|
|
122
|
+
else None
|
|
123
|
+
),
|
|
124
|
+
"schedule_history": [
|
|
125
|
+
_thaw_json(record) for record in self.schedule_history
|
|
126
|
+
],
|
|
127
|
+
"metrics": [_thaw_json(record) for record in self.metrics],
|
|
128
|
+
"transfer_diagnostics": [
|
|
129
|
+
_thaw_json(record) for record in self.transfer_diagnostics
|
|
130
|
+
],
|
|
131
|
+
"consensus": [
|
|
132
|
+
{
|
|
133
|
+
"round_id": record.round_id,
|
|
134
|
+
"normalized_rms": record.normalized_rms,
|
|
135
|
+
"sketch_count": record.sketch_count,
|
|
136
|
+
}
|
|
137
|
+
for record in self.consensus
|
|
138
|
+
],
|
|
139
|
+
"terminal": (
|
|
140
|
+
{
|
|
141
|
+
"result": self.terminal.result,
|
|
142
|
+
"diagnostics": _thaw_json(self.terminal.diagnostics),
|
|
143
|
+
}
|
|
144
|
+
if self.terminal is not None
|
|
145
|
+
else None
|
|
146
|
+
),
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
@dataclass(frozen=True, slots=True)
|
|
151
|
+
class CheckpointRef:
|
|
152
|
+
"""Archive-bound checkpoint reference that cannot bypass validation."""
|
|
153
|
+
|
|
154
|
+
_root: Path
|
|
155
|
+
relative_path: str
|
|
156
|
+
recorded_sha256: Sha256
|
|
157
|
+
|
|
158
|
+
def verify(self) -> None:
|
|
159
|
+
self._verified_bytes()
|
|
160
|
+
|
|
161
|
+
def _verified_bytes(self) -> bytes:
|
|
162
|
+
data = _read_safe_file(self._root, self.relative_path)
|
|
163
|
+
if hashlib.sha256(data).hexdigest() != self.recorded_sha256:
|
|
164
|
+
raise RunArchiveError(f"checkpoint hash mismatch: {self.relative_path}")
|
|
165
|
+
return data
|
|
166
|
+
|
|
167
|
+
def load_tensors(self) -> dict[str, np.ndarray]:
|
|
168
|
+
data = self._verified_bytes()
|
|
169
|
+
try:
|
|
170
|
+
return {
|
|
171
|
+
name: value.copy(order="C")
|
|
172
|
+
for name, value in _LOAD_SAFETENSORS(data).items()
|
|
173
|
+
}
|
|
174
|
+
except (SafetensorError, ValueError, TypeError) as error:
|
|
175
|
+
raise RunArchiveError(
|
|
176
|
+
f"invalid checkpoint: {self.relative_path}"
|
|
177
|
+
) from error
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
@dataclass(frozen=True, slots=True)
|
|
181
|
+
class RunArchive:
|
|
182
|
+
"""Validated manifest, state, and safe checkpoint references."""
|
|
183
|
+
|
|
184
|
+
root: Path
|
|
185
|
+
manifest: SealedManifest
|
|
186
|
+
manifest_hash: Sha256
|
|
187
|
+
state: ArchiveState
|
|
188
|
+
algorithm_state: CheckpointRef | None
|
|
189
|
+
|
|
190
|
+
@classmethod
|
|
191
|
+
def open(cls, root: Path) -> RunArchive:
|
|
192
|
+
try:
|
|
193
|
+
resolved_root = root.resolve(strict=True)
|
|
194
|
+
except OSError as error:
|
|
195
|
+
raise RunArchiveError(f"run archive does not exist: {root}") from error
|
|
196
|
+
if not resolved_root.is_dir():
|
|
197
|
+
raise RunArchiveError(f"run archive is not a directory: {root}")
|
|
198
|
+
try:
|
|
199
|
+
manifest = parse_sealed_json(
|
|
200
|
+
_read_safe_file(resolved_root, "manifest.json")
|
|
201
|
+
)
|
|
202
|
+
except (ValueError, TypeError) as error:
|
|
203
|
+
raise RunArchiveError(f"invalid run manifest: {root}") from error
|
|
204
|
+
state = load_archive_state(resolved_root / "state.json")
|
|
205
|
+
manifest_hash = canonical_hash(manifest)
|
|
206
|
+
if state.manifest_hash != manifest_hash:
|
|
207
|
+
raise RunArchiveError(f"manifest hash mismatch in {root}")
|
|
208
|
+
_validate_state_consistency(state)
|
|
209
|
+
records = state.checkpoint_records()
|
|
210
|
+
for record in records:
|
|
211
|
+
_safe_file_path(resolved_root, record.relative_path)
|
|
212
|
+
return cls(
|
|
213
|
+
root=resolved_root,
|
|
214
|
+
manifest=manifest,
|
|
215
|
+
manifest_hash=manifest_hash,
|
|
216
|
+
state=state,
|
|
217
|
+
algorithm_state=(
|
|
218
|
+
_bind_checkpoint(resolved_root, state.algorithm_state)
|
|
219
|
+
if state.algorithm_state is not None
|
|
220
|
+
else None
|
|
221
|
+
),
|
|
222
|
+
)
|
|
223
|
+
|
|
224
|
+
@property
|
|
225
|
+
def archive_version(self) -> int:
|
|
226
|
+
return self.state.archive_version
|
|
227
|
+
|
|
228
|
+
@property
|
|
229
|
+
def terminal(self) -> TerminalState | None:
|
|
230
|
+
return self.state.terminal
|
|
231
|
+
|
|
232
|
+
@property
|
|
233
|
+
def prepared_commit(self) -> PreparedCommitState | None:
|
|
234
|
+
return self.state.prepared_commit
|
|
235
|
+
|
|
236
|
+
def require_complete(self, expected_rounds: int) -> RunArchive:
|
|
237
|
+
"""Require exactly ``expected_rounds`` committed rounds."""
|
|
238
|
+
if expected_rounds <= 0:
|
|
239
|
+
raise ValueError("expected_rounds must be positive")
|
|
240
|
+
if (
|
|
241
|
+
self.terminal is None
|
|
242
|
+
or self.terminal.result != "complete"
|
|
243
|
+
or self.state.committed_round != expected_rounds - 1
|
|
244
|
+
or self.prepared_commit is not None
|
|
245
|
+
):
|
|
246
|
+
raise RunArchiveError(
|
|
247
|
+
f"run archive is not complete through {expected_rounds} rounds"
|
|
248
|
+
)
|
|
249
|
+
return self
|
|
250
|
+
|
|
251
|
+
def verify_checkpoint_integrity(self) -> None:
|
|
252
|
+
seen: set[tuple[str, str]] = set()
|
|
253
|
+
for record in self.state.checkpoint_records():
|
|
254
|
+
key = (record.relative_path, record.sha256)
|
|
255
|
+
if key in seen:
|
|
256
|
+
continue
|
|
257
|
+
seen.add(key)
|
|
258
|
+
_bind_checkpoint(self.root, record).verify()
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def load_archive_state(path: Path) -> ArchiveState:
|
|
262
|
+
try:
|
|
263
|
+
root = path.parent.resolve(strict=True)
|
|
264
|
+
value = cast(
|
|
265
|
+
object,
|
|
266
|
+
json.loads(
|
|
267
|
+
_read_safe_file(root, path.name),
|
|
268
|
+
parse_constant=_reject_json_constant,
|
|
269
|
+
),
|
|
270
|
+
)
|
|
271
|
+
except (OSError, ValueError) as error:
|
|
272
|
+
raise RunArchiveError(f"run state is unreadable: {path}") from error
|
|
273
|
+
record = _mapping(value, "run state")
|
|
274
|
+
version = _integer(record.get("archive_version"), "archive_version")
|
|
275
|
+
if version != ARCHIVE_VERSION:
|
|
276
|
+
raise RunArchiveError(f"unsupported archive version: {version}")
|
|
277
|
+
if frozenset(record) != _STATE_FIELDS:
|
|
278
|
+
raise RunArchiveError("run state fields are invalid")
|
|
279
|
+
state = ArchiveState(
|
|
280
|
+
archive_version=version,
|
|
281
|
+
manifest_hash=_sha256(record.get("manifest_hash"), "manifest_hash"),
|
|
282
|
+
committed_round=_integer(
|
|
283
|
+
record.get("committed_round"), "committed_round", minimum=-1
|
|
284
|
+
),
|
|
285
|
+
state_checksum=_optional_sha256(record.get("state_checksum"), "state_checksum"),
|
|
286
|
+
algorithm_state=_optional_checkpoint(
|
|
287
|
+
record.get("algorithm_state"), "algorithm_state"
|
|
288
|
+
),
|
|
289
|
+
prepared_commit=_optional_prepared(record.get("prepared_commit")),
|
|
290
|
+
schedule_history=_record_list(
|
|
291
|
+
record.get("schedule_history"), "schedule_history"
|
|
292
|
+
),
|
|
293
|
+
metrics=_record_list(record.get("metrics"), "metrics"),
|
|
294
|
+
transfer_diagnostics=_record_list(
|
|
295
|
+
record.get("transfer_diagnostics"), "transfer_diagnostics"
|
|
296
|
+
),
|
|
297
|
+
consensus=_consensus_list(record.get("consensus")),
|
|
298
|
+
terminal=_optional_terminal(record.get("terminal")),
|
|
299
|
+
)
|
|
300
|
+
_validate_state_consistency(state)
|
|
301
|
+
return state
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _prepared_as_json(value: PreparedCommitState) -> dict[str, object]:
|
|
305
|
+
return {
|
|
306
|
+
"round_id": value.round_id,
|
|
307
|
+
"state_checksum": value.state_checksum,
|
|
308
|
+
"algorithm_state": value.algorithm_state.as_json(),
|
|
309
|
+
"schedule": _thaw_json(value.schedule),
|
|
310
|
+
"metrics": _thaw_json(value.metrics) if value.metrics is not None else None,
|
|
311
|
+
"transfer_diagnostics": (
|
|
312
|
+
_thaw_json(value.transfer_diagnostics)
|
|
313
|
+
if value.transfer_diagnostics is not None
|
|
314
|
+
else None
|
|
315
|
+
),
|
|
316
|
+
}
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _optional_prepared(value: object) -> PreparedCommitState | None:
|
|
320
|
+
if value is None:
|
|
321
|
+
return None
|
|
322
|
+
record = _mapping(value, "prepared_commit")
|
|
323
|
+
expected = {
|
|
324
|
+
"round_id",
|
|
325
|
+
"state_checksum",
|
|
326
|
+
"algorithm_state",
|
|
327
|
+
"schedule",
|
|
328
|
+
"metrics",
|
|
329
|
+
"transfer_diagnostics",
|
|
330
|
+
}
|
|
331
|
+
if set(record) != expected:
|
|
332
|
+
raise RunArchiveError("prepared_commit fields are invalid")
|
|
333
|
+
return PreparedCommitState(
|
|
334
|
+
round_id=_integer(record["round_id"], "prepared round", minimum=0),
|
|
335
|
+
state_checksum=_sha256(record["state_checksum"], "prepared state_checksum"),
|
|
336
|
+
algorithm_state=_checkpoint(
|
|
337
|
+
record["algorithm_state"], "prepared algorithm_state"
|
|
338
|
+
),
|
|
339
|
+
schedule=_immutable_record(record["schedule"], "prepared schedule"),
|
|
340
|
+
metrics=_optional_record(record["metrics"], "prepared metrics"),
|
|
341
|
+
transfer_diagnostics=_optional_record(
|
|
342
|
+
record["transfer_diagnostics"], "prepared transfer_diagnostics"
|
|
343
|
+
),
|
|
344
|
+
)
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _optional_checkpoint(value: object, label: str) -> CheckpointRecord | None:
|
|
348
|
+
return None if value is None else _checkpoint(value, label)
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _checkpoint(value: object, label: str) -> CheckpointRecord:
|
|
352
|
+
record = _mapping(value, label)
|
|
353
|
+
if set(record) != {"path", "sha256"}:
|
|
354
|
+
raise RunArchiveError(f"{label} fields are invalid")
|
|
355
|
+
relative_path = record["path"]
|
|
356
|
+
if not isinstance(relative_path, str):
|
|
357
|
+
raise RunArchiveError(f"{label} path is invalid")
|
|
358
|
+
digest = _sha256(record["sha256"], f"{label} sha256")
|
|
359
|
+
_validate_relative_path(relative_path)
|
|
360
|
+
return CheckpointRecord(relative_path=relative_path, sha256=digest)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _record_list(value: object, label: str) -> tuple[JsonRecord, ...]:
|
|
364
|
+
if not isinstance(value, list):
|
|
365
|
+
raise RunArchiveError(f"{label} must be a list")
|
|
366
|
+
return tuple(
|
|
367
|
+
_immutable_record(item, f"{label} entry") for item in cast(list[object], value)
|
|
368
|
+
)
|
|
369
|
+
|
|
370
|
+
|
|
371
|
+
def _consensus_list(value: object) -> tuple[ConsensusState, ...]:
|
|
372
|
+
if not isinstance(value, list):
|
|
373
|
+
raise RunArchiveError("consensus must be a list")
|
|
374
|
+
result: list[ConsensusState] = []
|
|
375
|
+
seen: set[int] = set()
|
|
376
|
+
for item in cast(list[object], value):
|
|
377
|
+
record = _mapping(item, "consensus entry")
|
|
378
|
+
if set(record) != {"round_id", "normalized_rms", "sketch_count"}:
|
|
379
|
+
raise RunArchiveError("consensus entry fields are invalid")
|
|
380
|
+
round_id = _integer(record["round_id"], "consensus round", minimum=0)
|
|
381
|
+
distance = record["normalized_rms"]
|
|
382
|
+
if (
|
|
383
|
+
isinstance(distance, bool)
|
|
384
|
+
or not isinstance(distance, (int, float))
|
|
385
|
+
or not math.isfinite(distance)
|
|
386
|
+
or distance < 0
|
|
387
|
+
):
|
|
388
|
+
raise RunArchiveError("consensus distance is invalid")
|
|
389
|
+
sketch_count = _integer(
|
|
390
|
+
record["sketch_count"], "consensus sketch_count", minimum=1
|
|
391
|
+
)
|
|
392
|
+
if round_id in seen:
|
|
393
|
+
raise RunArchiveError("consensus rounds must be unique")
|
|
394
|
+
seen.add(round_id)
|
|
395
|
+
result.append(
|
|
396
|
+
ConsensusState(
|
|
397
|
+
round_id=round_id,
|
|
398
|
+
normalized_rms=float(distance),
|
|
399
|
+
sketch_count=sketch_count,
|
|
400
|
+
)
|
|
401
|
+
)
|
|
402
|
+
return tuple(result)
|
|
403
|
+
|
|
404
|
+
|
|
405
|
+
def _optional_terminal(value: object) -> TerminalState | None:
|
|
406
|
+
if value is None:
|
|
407
|
+
return None
|
|
408
|
+
record = _mapping(value, "terminal")
|
|
409
|
+
if set(record) != {"result", "diagnostics"}:
|
|
410
|
+
raise RunArchiveError("terminal fields are invalid")
|
|
411
|
+
result = record["result"]
|
|
412
|
+
if not isinstance(result, str) or not result:
|
|
413
|
+
raise RunArchiveError("terminal result is invalid")
|
|
414
|
+
return TerminalState(
|
|
415
|
+
result=result,
|
|
416
|
+
diagnostics=_immutable_record(record["diagnostics"], "terminal diagnostics"),
|
|
417
|
+
)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def _validate_state_consistency(state: ArchiveState) -> None:
|
|
421
|
+
if state.committed_round == -1:
|
|
422
|
+
if (
|
|
423
|
+
state.state_checksum is not None
|
|
424
|
+
or state.algorithm_state is not None
|
|
425
|
+
or state.schedule_history
|
|
426
|
+
):
|
|
427
|
+
raise RunArchiveError("empty archive contains committed state")
|
|
428
|
+
elif (
|
|
429
|
+
state.state_checksum is None
|
|
430
|
+
or state.algorithm_state is None
|
|
431
|
+
or len(state.schedule_history) != state.committed_round + 1
|
|
432
|
+
):
|
|
433
|
+
raise RunArchiveError("committed archive state is incomplete")
|
|
434
|
+
if (
|
|
435
|
+
state.prepared_commit is not None
|
|
436
|
+
and state.prepared_commit.round_id <= state.committed_round
|
|
437
|
+
):
|
|
438
|
+
raise RunArchiveError("prepared commit does not advance the archive")
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def _bind_checkpoint(root: Path, record: CheckpointRecord) -> CheckpointRef:
|
|
442
|
+
_safe_file_path(root, record.relative_path)
|
|
443
|
+
return CheckpointRef(
|
|
444
|
+
_root=root,
|
|
445
|
+
relative_path=record.relative_path,
|
|
446
|
+
recorded_sha256=record.sha256,
|
|
447
|
+
)
|
|
448
|
+
|
|
449
|
+
|
|
450
|
+
def _read_safe_file(root: Path, relative_path: str) -> bytes:
|
|
451
|
+
path = _safe_file_path(root, relative_path)
|
|
452
|
+
flags = os.O_RDONLY
|
|
453
|
+
if hasattr(os, "O_NOFOLLOW"):
|
|
454
|
+
flags |= os.O_NOFOLLOW
|
|
455
|
+
try:
|
|
456
|
+
descriptor = os.open(path, flags)
|
|
457
|
+
except OSError as error:
|
|
458
|
+
raise RunArchiveError(f"cannot open archive file: {relative_path}") from error
|
|
459
|
+
try:
|
|
460
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
461
|
+
return handle.read()
|
|
462
|
+
except OSError as error:
|
|
463
|
+
raise RunArchiveError(f"cannot read archive file: {relative_path}") from error
|
|
464
|
+
|
|
465
|
+
|
|
466
|
+
def _safe_file_path(root: Path, relative_path: str) -> Path:
|
|
467
|
+
_validate_relative_path(relative_path)
|
|
468
|
+
path = root
|
|
469
|
+
try:
|
|
470
|
+
for part in PurePosixPath(relative_path).parts:
|
|
471
|
+
path = path / part
|
|
472
|
+
if path.is_symlink():
|
|
473
|
+
raise RunArchiveError(
|
|
474
|
+
f"archive path contains a symlink: {relative_path}"
|
|
475
|
+
)
|
|
476
|
+
resolved = path.resolve(strict=True)
|
|
477
|
+
except OSError as error:
|
|
478
|
+
raise RunArchiveError(f"archive file is missing: {relative_path}") from error
|
|
479
|
+
if not resolved.is_relative_to(root) or not resolved.is_file():
|
|
480
|
+
raise RunArchiveError(f"archive path is unsafe: {relative_path}")
|
|
481
|
+
return resolved
|
|
482
|
+
|
|
483
|
+
|
|
484
|
+
def _validate_relative_path(value: str) -> None:
|
|
485
|
+
path = PurePosixPath(value)
|
|
486
|
+
if (
|
|
487
|
+
not value
|
|
488
|
+
or path.is_absolute()
|
|
489
|
+
or any(part in {"", ".", ".."} for part in path.parts)
|
|
490
|
+
or "\\" in value
|
|
491
|
+
):
|
|
492
|
+
raise RunArchiveError(f"archive path is unsafe: {value}")
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _mapping(value: object, label: str) -> dict[str, object]:
|
|
496
|
+
if not isinstance(value, dict):
|
|
497
|
+
raise RunArchiveError(f"{label} must be a JSON object")
|
|
498
|
+
entries = cast(dict[object, object], value)
|
|
499
|
+
if not all(isinstance(key, str) for key in entries):
|
|
500
|
+
raise RunArchiveError(f"{label} must be a JSON object")
|
|
501
|
+
return cast(dict[str, object], entries)
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _immutable_record(value: object, label: str) -> JsonRecord:
|
|
505
|
+
record = _mapping(value, label)
|
|
506
|
+
return cast(JsonRecord, _freeze_json(record))
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _optional_record(value: object, label: str) -> JsonRecord | None:
|
|
510
|
+
return None if value is None else _immutable_record(value, label)
|
|
511
|
+
|
|
512
|
+
|
|
513
|
+
def _freeze_json(value: object) -> object:
|
|
514
|
+
if isinstance(value, dict):
|
|
515
|
+
entries = cast(dict[object, object], value)
|
|
516
|
+
if not all(isinstance(key, str) for key in entries):
|
|
517
|
+
raise RunArchiveError("JSON object keys must be strings")
|
|
518
|
+
return MappingProxyType(
|
|
519
|
+
{cast(str, key): _freeze_json(item) for key, item in entries.items()}
|
|
520
|
+
)
|
|
521
|
+
if isinstance(value, list):
|
|
522
|
+
return tuple(_freeze_json(item) for item in cast(list[object], value))
|
|
523
|
+
if value is None or isinstance(value, (str, bool, int)):
|
|
524
|
+
return value
|
|
525
|
+
if isinstance(value, float) and math.isfinite(value):
|
|
526
|
+
return value
|
|
527
|
+
raise RunArchiveError("archive contains an invalid JSON value")
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def _thaw_json(value: object) -> object:
|
|
531
|
+
if isinstance(value, Mapping):
|
|
532
|
+
entries = cast(Mapping[object, object], value)
|
|
533
|
+
return {cast(str, key): _thaw_json(item) for key, item in entries.items()}
|
|
534
|
+
if isinstance(value, tuple):
|
|
535
|
+
return [_thaw_json(item) for item in cast(tuple[object, ...], value)]
|
|
536
|
+
return value
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _integer(value: object, label: str, *, minimum: int | None = None) -> int:
|
|
540
|
+
if isinstance(value, bool) or not isinstance(value, int):
|
|
541
|
+
raise RunArchiveError(f"{label} must be an integer")
|
|
542
|
+
if minimum is not None and value < minimum:
|
|
543
|
+
raise RunArchiveError(f"{label} is below its minimum")
|
|
544
|
+
return value
|
|
545
|
+
|
|
546
|
+
|
|
547
|
+
def _sha256(value: object, label: str) -> Sha256:
|
|
548
|
+
if (
|
|
549
|
+
not isinstance(value, str)
|
|
550
|
+
or len(value) != 64
|
|
551
|
+
or any(character not in "0123456789abcdef" for character in value)
|
|
552
|
+
):
|
|
553
|
+
raise RunArchiveError(f"{label} must be a SHA-256 digest")
|
|
554
|
+
return value
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def _optional_sha256(value: object, label: str) -> Sha256 | None:
|
|
558
|
+
return None if value is None else _sha256(value, label)
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
def _reject_json_constant(value: str) -> object:
|
|
562
|
+
raise ValueError(f"invalid JSON constant: {value}")
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
__all__ = [
|
|
566
|
+
"ARCHIVE_VERSION",
|
|
567
|
+
"ArchiveState",
|
|
568
|
+
"CheckpointRecord",
|
|
569
|
+
"CheckpointRef",
|
|
570
|
+
"ConsensusState",
|
|
571
|
+
"PreparedCommitState",
|
|
572
|
+
"RunArchive",
|
|
573
|
+
"RunArchiveError",
|
|
574
|
+
"TerminalState",
|
|
575
|
+
"load_archive_state",
|
|
576
|
+
]
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
"""Crash-safe local file replacement shared by run state, ledgers and node files."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import uuid
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
def atomic_write_bytes(path: Path, data: bytes, *, mode: int = 0o666) -> None:
|
|
11
|
+
"""Durably replace ``path`` with ``data`` or leave the old file in place.
|
|
12
|
+
|
|
13
|
+
Writes a uniquely named sibling temp file (created exclusively with ``mode``,
|
|
14
|
+
subject to the umask), fsyncs it, renames it over ``path`` and fsyncs the
|
|
15
|
+
directory. Raises ``OSError`` on failure after removing the temp file.
|
|
16
|
+
"""
|
|
17
|
+
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
18
|
+
try:
|
|
19
|
+
descriptor = os.open(temporary, os.O_CREAT | os.O_EXCL | os.O_WRONLY, mode)
|
|
20
|
+
with os.fdopen(descriptor, "wb") as handle:
|
|
21
|
+
handle.write(data)
|
|
22
|
+
handle.flush()
|
|
23
|
+
os.fsync(handle.fileno())
|
|
24
|
+
os.replace(temporary, path)
|
|
25
|
+
fsync_directory(path.parent)
|
|
26
|
+
finally:
|
|
27
|
+
temporary.unlink(missing_ok=True)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def fsync_directory(path: Path) -> None:
|
|
31
|
+
"""Persist directory entries; skipped where directories cannot be opened."""
|
|
32
|
+
try:
|
|
33
|
+
descriptor = os.open(path, os.O_RDONLY)
|
|
34
|
+
except OSError:
|
|
35
|
+
return
|
|
36
|
+
try:
|
|
37
|
+
os.fsync(descriptor)
|
|
38
|
+
finally:
|
|
39
|
+
os.close(descriptor)
|