dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
"""Local, non-rollbackable full-horizon privacy reservations.
|
|
2
|
+
|
|
3
|
+
The trusted operator must preserve this file and compose external dataset uses.
|
|
4
|
+
An explicitly created new lineage is not evidence that data has never been used.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import fcntl
|
|
10
|
+
import hashlib
|
|
11
|
+
import math
|
|
12
|
+
import os
|
|
13
|
+
from collections.abc import Callable, Generator
|
|
14
|
+
from contextlib import contextmanager
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
from typing import Literal
|
|
17
|
+
|
|
18
|
+
from dromeus.errors import PrivacyError
|
|
19
|
+
from dromeus.manifests.canonical import canonical_json
|
|
20
|
+
from dromeus.manifests.models import DomainModel, PrivacyReservation, Sha256
|
|
21
|
+
from dromeus.persistence.files import atomic_write_bytes
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
class _LedgerState(DomainModel):
|
|
25
|
+
version: Literal[1] = 1
|
|
26
|
+
lineage_hash: Sha256
|
|
27
|
+
reservations: tuple[PrivacyReservation, ...] = ()
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class PrivacyLedger:
|
|
31
|
+
"""Atomic, locked storage independent of run/model checkpoints."""
|
|
32
|
+
|
|
33
|
+
def __init__(self, path: Path | str, *, lineage: str) -> None:
|
|
34
|
+
if not lineage.strip():
|
|
35
|
+
raise PrivacyError("local dataset lineage is required")
|
|
36
|
+
self.path = Path(path)
|
|
37
|
+
self._lineage_hash = hashlib.sha256(lineage.encode()).hexdigest()
|
|
38
|
+
|
|
39
|
+
@classmethod
|
|
40
|
+
def create(cls, path: Path | str, *, lineage: str) -> PrivacyLedger:
|
|
41
|
+
"""Explicit bootstrap for a new controlled lineage; never overwrite."""
|
|
42
|
+
ledger = cls(path, lineage=lineage)
|
|
43
|
+
ledger.path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
44
|
+
with ledger._locked():
|
|
45
|
+
if ledger.path.exists() or ledger.path.is_symlink():
|
|
46
|
+
raise PrivacyError("privacy ledger already exists")
|
|
47
|
+
ledger._write(_LedgerState(lineage_hash=ledger._lineage_hash))
|
|
48
|
+
return ledger
|
|
49
|
+
|
|
50
|
+
@contextmanager
|
|
51
|
+
def _locked(self) -> Generator[None]:
|
|
52
|
+
lock_path = self.path.with_name(self.path.name + ".lock")
|
|
53
|
+
descriptor = os.open(lock_path, os.O_CREAT | os.O_RDWR | os.O_NOFOLLOW, 0o600)
|
|
54
|
+
try:
|
|
55
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX)
|
|
56
|
+
yield
|
|
57
|
+
finally:
|
|
58
|
+
fcntl.flock(descriptor, fcntl.LOCK_UN)
|
|
59
|
+
os.close(descriptor)
|
|
60
|
+
|
|
61
|
+
def _read(self) -> _LedgerState:
|
|
62
|
+
if not self.path.is_file() or self.path.is_symlink():
|
|
63
|
+
raise PrivacyError("privacy ledger missing or unsafe")
|
|
64
|
+
try:
|
|
65
|
+
state = _LedgerState.model_validate_json(self.path.read_bytes())
|
|
66
|
+
except Exception:
|
|
67
|
+
raise PrivacyError("privacy ledger corrupt") from None
|
|
68
|
+
if state.lineage_hash != self._lineage_hash:
|
|
69
|
+
raise PrivacyError("privacy ledger lineage mismatch")
|
|
70
|
+
ids = [r.run_id for r in state.reservations]
|
|
71
|
+
if len(ids) != len(set(ids)):
|
|
72
|
+
raise PrivacyError("privacy ledger has duplicate reservations")
|
|
73
|
+
return state
|
|
74
|
+
|
|
75
|
+
def _write(self, state: _LedgerState) -> None:
|
|
76
|
+
atomic_write_bytes(self.path, canonical_json(state), mode=0o600)
|
|
77
|
+
|
|
78
|
+
def reservations(self) -> tuple[PrivacyReservation, ...]:
|
|
79
|
+
with self._locked():
|
|
80
|
+
return self._read().reservations
|
|
81
|
+
|
|
82
|
+
def reserve(
|
|
83
|
+
self,
|
|
84
|
+
reservation: PrivacyReservation,
|
|
85
|
+
*,
|
|
86
|
+
epsilon_limit: float | None,
|
|
87
|
+
forecast: Callable[[tuple[PrivacyReservation, ...]], float],
|
|
88
|
+
) -> None:
|
|
89
|
+
"""Forecast the composed history, persist, then permit output."""
|
|
90
|
+
with self._locked():
|
|
91
|
+
state = self._read()
|
|
92
|
+
if any(r.run_id == reservation.run_id for r in state.reservations):
|
|
93
|
+
raise PrivacyError("run already reserved; use a new run identity")
|
|
94
|
+
history = (*state.reservations, reservation)
|
|
95
|
+
expenditure = forecast(history)
|
|
96
|
+
if not math.isfinite(expenditure) or expenditure < 0:
|
|
97
|
+
raise PrivacyError("invalid privacy budget forecast")
|
|
98
|
+
if epsilon_limit is not None and expenditure > epsilon_limit:
|
|
99
|
+
raise PrivacyError("composed privacy budget exceeds limit")
|
|
100
|
+
self._write(state.model_copy(update={"reservations": history}))
|
|
101
|
+
|
|
102
|
+
def check(self, reservation: PrivacyReservation) -> None:
|
|
103
|
+
if reservation not in self.reservations():
|
|
104
|
+
raise PrivacyError("privacy reservation missing or changed")
|
|
@@ -0,0 +1,352 @@
|
|
|
1
|
+
"""Atomic run-state storage."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import os
|
|
6
|
+
import uuid
|
|
7
|
+
from collections.abc import Mapping
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from threading import RLock
|
|
10
|
+
from typing import cast
|
|
11
|
+
|
|
12
|
+
import numpy as np
|
|
13
|
+
from safetensors.numpy import save_file # pyright: ignore[reportUnknownVariableType]
|
|
14
|
+
|
|
15
|
+
from dromeus.errors import RunError
|
|
16
|
+
from dromeus.manifests.canonical import (
|
|
17
|
+
canonical_hash,
|
|
18
|
+
canonical_json,
|
|
19
|
+
canonical_json_bytes,
|
|
20
|
+
file_sha256,
|
|
21
|
+
)
|
|
22
|
+
from dromeus.manifests.models import SealedManifest, Sha256
|
|
23
|
+
from dromeus.persistence.archive import (
|
|
24
|
+
ARCHIVE_VERSION,
|
|
25
|
+
ArchiveState,
|
|
26
|
+
RunArchiveError,
|
|
27
|
+
load_archive_state,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.persistence.files import atomic_write_bytes, fsync_directory
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
class RunStoreError(RunError):
|
|
33
|
+
"""A durable run-state operation could not be completed safely."""
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
JsonRecord = Mapping[str, object]
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class RunStore:
|
|
40
|
+
"""Persist audit state with one crash-safe committed checkpoint."""
|
|
41
|
+
|
|
42
|
+
def __init__(self, root: Path) -> None:
|
|
43
|
+
self._restricted = False
|
|
44
|
+
self._root = root
|
|
45
|
+
self._checkpoint_root = root / "checkpoints"
|
|
46
|
+
self._root.mkdir(parents=True, exist_ok=True)
|
|
47
|
+
self._checkpoint_root.mkdir(parents=True, exist_ok=True)
|
|
48
|
+
self._manifest_path = root / "manifest.json"
|
|
49
|
+
self._state_path = root / "state.json"
|
|
50
|
+
self._lock = RLock()
|
|
51
|
+
|
|
52
|
+
@property
|
|
53
|
+
def root(self) -> Path:
|
|
54
|
+
return self._root
|
|
55
|
+
|
|
56
|
+
def initialize(self, manifest: SealedManifest) -> Sha256:
|
|
57
|
+
"""Write the sealed manifest once and initialize current archive state."""
|
|
58
|
+
with self._lock:
|
|
59
|
+
if manifest.privacy is not None:
|
|
60
|
+
if self._root.is_symlink() or self._checkpoint_root.is_symlink():
|
|
61
|
+
raise RunStoreError(
|
|
62
|
+
"private checkpoint directories must not be symlinks"
|
|
63
|
+
)
|
|
64
|
+
self._restricted = True
|
|
65
|
+
self._root.chmod(0o700)
|
|
66
|
+
self._checkpoint_root.chmod(0o700)
|
|
67
|
+
manifest_bytes = canonical_json(manifest)
|
|
68
|
+
manifest_hash = canonical_hash(manifest)
|
|
69
|
+
if self._manifest_path.exists():
|
|
70
|
+
if self._manifest_path.read_bytes() != manifest_bytes:
|
|
71
|
+
raise RunStoreError("run store manifest does not match")
|
|
72
|
+
else:
|
|
73
|
+
_atomic_write(self._manifest_path, manifest_bytes)
|
|
74
|
+
|
|
75
|
+
if self._state_path.exists():
|
|
76
|
+
state = self.load_state()
|
|
77
|
+
if state.manifest_hash != manifest_hash:
|
|
78
|
+
raise RunStoreError("run store state manifest does not match")
|
|
79
|
+
self._remove_unreferenced_checkpoints(state)
|
|
80
|
+
else:
|
|
81
|
+
_atomic_write_json(self._state_path, _initial_state(manifest_hash))
|
|
82
|
+
return manifest_hash
|
|
83
|
+
|
|
84
|
+
def load_state(self) -> ArchiveState:
|
|
85
|
+
"""Return validated immutable archive state."""
|
|
86
|
+
with self._lock:
|
|
87
|
+
if not self._state_path.is_file():
|
|
88
|
+
raise RunStoreError("run store has not been initialized")
|
|
89
|
+
try:
|
|
90
|
+
return load_archive_state(self._state_path)
|
|
91
|
+
except RunArchiveError as error:
|
|
92
|
+
raise RunStoreError(str(error)) from error
|
|
93
|
+
|
|
94
|
+
def persist_prepared_commit(
|
|
95
|
+
self,
|
|
96
|
+
*,
|
|
97
|
+
committed_round: int,
|
|
98
|
+
algorithm_state: Mapping[str, np.ndarray],
|
|
99
|
+
state_checksum: str,
|
|
100
|
+
schedule: JsonRecord,
|
|
101
|
+
metrics: JsonRecord | None = None,
|
|
102
|
+
transfer_diagnostics: JsonRecord | None = None,
|
|
103
|
+
) -> ArchiveState:
|
|
104
|
+
"""Persist round artifacts without exposing the round as committed."""
|
|
105
|
+
with self._lock:
|
|
106
|
+
if committed_round < 0:
|
|
107
|
+
raise ValueError("committed_round must be non-negative")
|
|
108
|
+
state = self._writable_state()
|
|
109
|
+
if committed_round <= state.committed_round:
|
|
110
|
+
if (
|
|
111
|
+
committed_round == state.committed_round
|
|
112
|
+
and state.state_checksum == state_checksum
|
|
113
|
+
):
|
|
114
|
+
return state
|
|
115
|
+
raise RunStoreError("committed round must advance monotonically")
|
|
116
|
+
prepared = state.prepared_commit
|
|
117
|
+
if prepared is not None:
|
|
118
|
+
if (
|
|
119
|
+
prepared.round_id == committed_round
|
|
120
|
+
and prepared.state_checksum == state_checksum
|
|
121
|
+
):
|
|
122
|
+
return state
|
|
123
|
+
raise RunStoreError("another prepared commit already exists")
|
|
124
|
+
|
|
125
|
+
committed_name = f"committed-round-{committed_round:06d}.safetensors"
|
|
126
|
+
committed_hash = _atomic_save_tensors(
|
|
127
|
+
self._checkpoint_root / committed_name,
|
|
128
|
+
algorithm_state,
|
|
129
|
+
restricted=self._restricted,
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
next_state = state.as_json()
|
|
133
|
+
next_state["prepared_commit"] = {
|
|
134
|
+
"round_id": committed_round,
|
|
135
|
+
"state_checksum": state_checksum,
|
|
136
|
+
"algorithm_state": _checkpoint_json(committed_name, committed_hash),
|
|
137
|
+
"schedule": dict(schedule),
|
|
138
|
+
"metrics": dict(metrics) if metrics is not None else None,
|
|
139
|
+
"transfer_diagnostics": (
|
|
140
|
+
dict(transfer_diagnostics)
|
|
141
|
+
if transfer_diagnostics is not None
|
|
142
|
+
else None
|
|
143
|
+
),
|
|
144
|
+
}
|
|
145
|
+
try:
|
|
146
|
+
_atomic_write_json(self._state_path, next_state)
|
|
147
|
+
except Exception:
|
|
148
|
+
_remove_file(self._checkpoint_root / committed_name)
|
|
149
|
+
raise
|
|
150
|
+
return self.load_state()
|
|
151
|
+
|
|
152
|
+
def confirm_prepared_commit(
|
|
153
|
+
self,
|
|
154
|
+
*,
|
|
155
|
+
committed_round: int,
|
|
156
|
+
state_checksum: str,
|
|
157
|
+
) -> ArchiveState:
|
|
158
|
+
"""Expose a prepared round after the peer confirms its commit."""
|
|
159
|
+
with self._lock:
|
|
160
|
+
if committed_round < 0:
|
|
161
|
+
raise ValueError("committed_round must be non-negative")
|
|
162
|
+
state = self._writable_state()
|
|
163
|
+
if committed_round <= state.committed_round:
|
|
164
|
+
if (
|
|
165
|
+
committed_round == state.committed_round
|
|
166
|
+
and state.state_checksum == state_checksum
|
|
167
|
+
):
|
|
168
|
+
return state
|
|
169
|
+
raise RunStoreError("committed round must advance monotonically")
|
|
170
|
+
prepared = state.prepared_commit
|
|
171
|
+
if (
|
|
172
|
+
prepared is None
|
|
173
|
+
or prepared.round_id != committed_round
|
|
174
|
+
or prepared.state_checksum != state_checksum
|
|
175
|
+
):
|
|
176
|
+
raise RunStoreError("prepared commit does not match confirmation")
|
|
177
|
+
|
|
178
|
+
previous_checkpoint = state.algorithm_state
|
|
179
|
+
next_state = state.as_json()
|
|
180
|
+
prepared_json = cast(dict[str, object], next_state["prepared_commit"])
|
|
181
|
+
next_state.update(
|
|
182
|
+
{
|
|
183
|
+
"committed_round": committed_round,
|
|
184
|
+
"state_checksum": state_checksum,
|
|
185
|
+
"algorithm_state": prepared.algorithm_state.as_json(),
|
|
186
|
+
"prepared_commit": None,
|
|
187
|
+
}
|
|
188
|
+
)
|
|
189
|
+
cast(list[object], next_state["schedule_history"]).append(
|
|
190
|
+
prepared_json["schedule"]
|
|
191
|
+
)
|
|
192
|
+
if prepared.metrics is not None:
|
|
193
|
+
cast(list[object], next_state["metrics"]).append(
|
|
194
|
+
prepared_json["metrics"]
|
|
195
|
+
)
|
|
196
|
+
if prepared.transfer_diagnostics is not None:
|
|
197
|
+
cast(list[object], next_state["transfer_diagnostics"]).append(
|
|
198
|
+
prepared_json["transfer_diagnostics"]
|
|
199
|
+
)
|
|
200
|
+
_atomic_write_json(self._state_path, next_state)
|
|
201
|
+
if previous_checkpoint is not None:
|
|
202
|
+
_remove_file(self._root / previous_checkpoint.relative_path)
|
|
203
|
+
return self.load_state()
|
|
204
|
+
|
|
205
|
+
def record_consensus(
|
|
206
|
+
self,
|
|
207
|
+
*,
|
|
208
|
+
round_id: int,
|
|
209
|
+
normalized_rms: float,
|
|
210
|
+
sketch_count: int,
|
|
211
|
+
) -> ArchiveState:
|
|
212
|
+
"""Append one completed live consensus result atomically and idempotently."""
|
|
213
|
+
if round_id < 0:
|
|
214
|
+
raise ValueError("round_id must be non-negative")
|
|
215
|
+
if not np.isfinite(normalized_rms) or normalized_rms < 0:
|
|
216
|
+
raise ValueError("normalized_rms must be finite and non-negative")
|
|
217
|
+
if sketch_count <= 0:
|
|
218
|
+
raise ValueError("sketch_count must be positive")
|
|
219
|
+
with self._lock:
|
|
220
|
+
state = self._writable_state()
|
|
221
|
+
for existing in state.consensus:
|
|
222
|
+
if existing.round_id != round_id:
|
|
223
|
+
continue
|
|
224
|
+
if (
|
|
225
|
+
existing.normalized_rms != normalized_rms
|
|
226
|
+
or existing.sketch_count != sketch_count
|
|
227
|
+
):
|
|
228
|
+
raise RunStoreError("consensus result already recorded")
|
|
229
|
+
return state
|
|
230
|
+
next_state = state.as_json()
|
|
231
|
+
cast(list[object], next_state["consensus"]).append(
|
|
232
|
+
{
|
|
233
|
+
"round_id": round_id,
|
|
234
|
+
"normalized_rms": float(normalized_rms),
|
|
235
|
+
"sketch_count": sketch_count,
|
|
236
|
+
}
|
|
237
|
+
)
|
|
238
|
+
_atomic_write_json(self._state_path, next_state)
|
|
239
|
+
return self.load_state()
|
|
240
|
+
|
|
241
|
+
def record_terminal(
|
|
242
|
+
self, result: str, diagnostics: JsonRecord | None = None
|
|
243
|
+
) -> None:
|
|
244
|
+
"""Record one terminal result; identical duplicate calls are idempotent."""
|
|
245
|
+
with self._lock:
|
|
246
|
+
if not result:
|
|
247
|
+
raise ValueError("terminal result must not be empty")
|
|
248
|
+
state = self._writable_state()
|
|
249
|
+
terminal = {"result": result, "diagnostics": dict(diagnostics or {})}
|
|
250
|
+
if state.terminal is not None:
|
|
251
|
+
existing = {
|
|
252
|
+
"result": state.terminal.result,
|
|
253
|
+
"diagnostics": dict(state.terminal.diagnostics),
|
|
254
|
+
}
|
|
255
|
+
if existing == terminal:
|
|
256
|
+
return
|
|
257
|
+
raise RunStoreError("terminal result already recorded")
|
|
258
|
+
next_state = state.as_json()
|
|
259
|
+
discarded = (
|
|
260
|
+
state.prepared_commit.algorithm_state
|
|
261
|
+
if result != "complete" and state.prepared_commit is not None
|
|
262
|
+
else None
|
|
263
|
+
)
|
|
264
|
+
if discarded is not None:
|
|
265
|
+
next_state["prepared_commit"] = None
|
|
266
|
+
next_state["terminal"] = terminal
|
|
267
|
+
_atomic_write_json(self._state_path, next_state)
|
|
268
|
+
if discarded is not None:
|
|
269
|
+
_remove_file(self._root / discarded.relative_path)
|
|
270
|
+
|
|
271
|
+
def _writable_state(self) -> ArchiveState:
|
|
272
|
+
return self.load_state()
|
|
273
|
+
|
|
274
|
+
def _remove_unreferenced_checkpoints(self, state: ArchiveState) -> None:
|
|
275
|
+
referenced = {record.relative_path for record in state.checkpoint_records()}
|
|
276
|
+
for path in self._checkpoint_root.glob("*.safetensors"):
|
|
277
|
+
relative_path = path.relative_to(self._root).as_posix()
|
|
278
|
+
if relative_path not in referenced:
|
|
279
|
+
_remove_file(path)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _initial_state(manifest_hash: Sha256) -> dict[str, object]:
|
|
283
|
+
return {
|
|
284
|
+
"archive_version": ARCHIVE_VERSION,
|
|
285
|
+
"manifest_hash": manifest_hash,
|
|
286
|
+
"committed_round": -1,
|
|
287
|
+
"state_checksum": None,
|
|
288
|
+
"algorithm_state": None,
|
|
289
|
+
"prepared_commit": None,
|
|
290
|
+
"schedule_history": [],
|
|
291
|
+
"metrics": [],
|
|
292
|
+
"transfer_diagnostics": [],
|
|
293
|
+
"consensus": [],
|
|
294
|
+
"terminal": None,
|
|
295
|
+
}
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
def _checkpoint_json(name: str, digest: Sha256) -> dict[str, str]:
|
|
299
|
+
return {"path": f"checkpoints/{name}", "sha256": digest}
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _atomic_save_tensors(
|
|
303
|
+
path: Path, tensors: Mapping[str, np.ndarray], *, restricted: bool = False
|
|
304
|
+
) -> Sha256:
|
|
305
|
+
if not tensors:
|
|
306
|
+
raise ValueError("checkpoint must contain at least one tensor")
|
|
307
|
+
temporary = path.with_name(f".{path.name}.{uuid.uuid4().hex}.tmp")
|
|
308
|
+
try:
|
|
309
|
+
save_file(
|
|
310
|
+
{name: value.copy(order="C") for name, value in tensors.items()},
|
|
311
|
+
str(temporary),
|
|
312
|
+
)
|
|
313
|
+
if restricted:
|
|
314
|
+
temporary.chmod(0o600)
|
|
315
|
+
_fsync_file(temporary)
|
|
316
|
+
os.replace(temporary, path)
|
|
317
|
+
fsync_directory(path.parent)
|
|
318
|
+
return file_sha256(path)
|
|
319
|
+
except (OSError, ValueError, TypeError) as error:
|
|
320
|
+
temporary.unlink(missing_ok=True)
|
|
321
|
+
raise RunStoreError(f"could not persist checkpoint {path.name}") from error
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _atomic_write_json(path: Path, value: object) -> None:
|
|
325
|
+
try:
|
|
326
|
+
data = canonical_json_bytes(value)
|
|
327
|
+
except (TypeError, ValueError) as error:
|
|
328
|
+
raise RunStoreError("run state is not JSON serializable") from error
|
|
329
|
+
_atomic_write(path, data)
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _atomic_write(path: Path, data: bytes) -> None:
|
|
333
|
+
try:
|
|
334
|
+
atomic_write_bytes(path, data)
|
|
335
|
+
except OSError as error:
|
|
336
|
+
raise RunStoreError(f"could not persist {path.name}") from error
|
|
337
|
+
|
|
338
|
+
|
|
339
|
+
def _fsync_file(path: Path) -> None:
|
|
340
|
+
with path.open("rb") as handle:
|
|
341
|
+
os.fsync(handle.fileno())
|
|
342
|
+
|
|
343
|
+
|
|
344
|
+
def _remove_file(path: Path) -> None:
|
|
345
|
+
try:
|
|
346
|
+
path.unlink(missing_ok=True)
|
|
347
|
+
fsync_directory(path.parent)
|
|
348
|
+
except OSError:
|
|
349
|
+
pass
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
__all__ = ["RunStore", "RunStoreError"]
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
"""Validated Dromeus wire protocol."""
|
|
2
|
+
|
|
3
|
+
from dromeus.protocol.codec import (
|
|
4
|
+
EnvelopeError,
|
|
5
|
+
ProtocolDecodeError,
|
|
6
|
+
decode_envelope,
|
|
7
|
+
decode_message,
|
|
8
|
+
encode_envelope,
|
|
9
|
+
encode_message,
|
|
10
|
+
)
|
|
11
|
+
from dromeus.protocol.models import Envelope, MessageType, create_envelope
|
|
12
|
+
from dromeus.protocol.version import PROTOCOL_VERSION
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"Envelope",
|
|
16
|
+
"EnvelopeError",
|
|
17
|
+
"MessageType",
|
|
18
|
+
"PROTOCOL_VERSION",
|
|
19
|
+
"ProtocolDecodeError",
|
|
20
|
+
"create_envelope",
|
|
21
|
+
"decode_envelope",
|
|
22
|
+
"decode_message",
|
|
23
|
+
"encode_envelope",
|
|
24
|
+
"encode_message",
|
|
25
|
+
]
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
"""Bounded MessagePack encoding for Dromeus protocol models."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from typing import cast
|
|
6
|
+
|
|
7
|
+
import msgpack # pyright: ignore[reportMissingTypeStubs]
|
|
8
|
+
from pydantic import ValidationError
|
|
9
|
+
|
|
10
|
+
from dromeus.errors import RunError
|
|
11
|
+
from dromeus.protocol.models import DomainModel, Envelope
|
|
12
|
+
|
|
13
|
+
DEFAULT_MAX_PAYLOAD_BYTES = 8 * 1024 * 1024
|
|
14
|
+
_ENVELOPE_OVERHEAD_BYTES = 4096
|
|
15
|
+
_MAX_ARRAY_LENGTH = 1024
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class ProtocolDecodeError(RunError, ValueError):
|
|
19
|
+
"""Encoded protocol bytes are malformed or unsupported."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class EnvelopeError(ProtocolDecodeError):
|
|
23
|
+
"""The wire envelope is malformed or unauthorised."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def encode_message(model: DomainModel) -> bytes:
|
|
27
|
+
value = model.model_dump(mode="python")
|
|
28
|
+
return cast(bytes, msgpack.packb(value)) # pyright: ignore[reportUnknownMemberType]
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def decode_message[T: DomainModel](
|
|
32
|
+
data: bytes,
|
|
33
|
+
model_type: type[T],
|
|
34
|
+
*,
|
|
35
|
+
max_bytes: int,
|
|
36
|
+
) -> T:
|
|
37
|
+
try:
|
|
38
|
+
return model_type.model_validate(_unpack(data, max_bytes=max_bytes))
|
|
39
|
+
except (ValueError, ValidationError, msgpack.UnpackException) as error:
|
|
40
|
+
raise ProtocolDecodeError("invalid protocol payload") from error
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def encode_envelope(envelope: Envelope) -> bytes:
|
|
44
|
+
value = envelope.model_dump(mode="python", exclude_none=True)
|
|
45
|
+
return cast(bytes, msgpack.packb(value)) # pyright: ignore[reportUnknownMemberType]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def decode_envelope(
|
|
49
|
+
data: bytes,
|
|
50
|
+
*,
|
|
51
|
+
authenticated_sender: str,
|
|
52
|
+
participant_keys: frozenset[str] | None,
|
|
53
|
+
max_payload_bytes: int = DEFAULT_MAX_PAYLOAD_BYTES,
|
|
54
|
+
) -> Envelope:
|
|
55
|
+
envelope = _decode_envelope(data, max_payload_bytes=max_payload_bytes)
|
|
56
|
+
if envelope.sender_public_key != authenticated_sender:
|
|
57
|
+
raise EnvelopeError("authenticated sender does not match envelope")
|
|
58
|
+
if (
|
|
59
|
+
participant_keys is not None
|
|
60
|
+
and envelope.sender_public_key not in participant_keys
|
|
61
|
+
):
|
|
62
|
+
raise EnvelopeError("sender is not a sealed participant")
|
|
63
|
+
return envelope
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def decode_envelope_sender(
|
|
67
|
+
data: bytes,
|
|
68
|
+
*,
|
|
69
|
+
max_payload_bytes: int = DEFAULT_MAX_PAYLOAD_BYTES,
|
|
70
|
+
) -> str:
|
|
71
|
+
return _decode_envelope(data, max_payload_bytes=max_payload_bytes).sender_public_key
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _decode_envelope(data: bytes, *, max_payload_bytes: int) -> Envelope:
|
|
75
|
+
if max_payload_bytes < 0:
|
|
76
|
+
raise ValueError("max payload bytes must not be negative")
|
|
77
|
+
try:
|
|
78
|
+
envelope = Envelope.model_validate(
|
|
79
|
+
_unpack(
|
|
80
|
+
data,
|
|
81
|
+
max_bytes=max_payload_bytes + _ENVELOPE_OVERHEAD_BYTES,
|
|
82
|
+
max_bin_len=max_payload_bytes,
|
|
83
|
+
)
|
|
84
|
+
)
|
|
85
|
+
except (ValueError, ValidationError, msgpack.UnpackException) as error:
|
|
86
|
+
raise EnvelopeError("invalid envelope") from error
|
|
87
|
+
if envelope.payload_length > max_payload_bytes:
|
|
88
|
+
raise EnvelopeError("payload exceeds size limit")
|
|
89
|
+
return envelope
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _unpack(
|
|
93
|
+
data: bytes,
|
|
94
|
+
*,
|
|
95
|
+
max_bytes: int,
|
|
96
|
+
max_bin_len: int | None = None,
|
|
97
|
+
) -> object:
|
|
98
|
+
if max_bytes < 0:
|
|
99
|
+
raise ValueError("max bytes must not be negative")
|
|
100
|
+
if len(data) > max_bytes:
|
|
101
|
+
raise ProtocolDecodeError("encoded payload exceeds size limit")
|
|
102
|
+
binary_limit = max_bytes if max_bin_len is None else max_bin_len
|
|
103
|
+
return cast(
|
|
104
|
+
object,
|
|
105
|
+
msgpack.unpackb( # pyright: ignore[reportUnknownMemberType]
|
|
106
|
+
data,
|
|
107
|
+
raw=False,
|
|
108
|
+
strict_map_key=True,
|
|
109
|
+
max_bin_len=binary_limit,
|
|
110
|
+
max_str_len=4096,
|
|
111
|
+
max_array_len=_MAX_ARRAY_LENGTH,
|
|
112
|
+
max_map_len=32,
|
|
113
|
+
max_ext_len=0,
|
|
114
|
+
),
|
|
115
|
+
)
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
__all__ = [
|
|
119
|
+
"DEFAULT_MAX_PAYLOAD_BYTES",
|
|
120
|
+
"EnvelopeError",
|
|
121
|
+
"ProtocolDecodeError",
|
|
122
|
+
"decode_envelope",
|
|
123
|
+
"decode_envelope_sender",
|
|
124
|
+
"decode_message",
|
|
125
|
+
"encode_envelope",
|
|
126
|
+
"encode_message",
|
|
127
|
+
]
|