dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/node.py
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
"""Run one Dromeus node: the public `Node` and its machine-local config."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import inspect
|
|
7
|
+
import json
|
|
8
|
+
import time
|
|
9
|
+
from collections.abc import Awaitable, Callable
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from datetime import UTC, datetime, timedelta
|
|
12
|
+
from enum import StrEnum
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Annotated, Self, cast
|
|
15
|
+
from urllib.parse import urlparse
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
import yaml
|
|
19
|
+
from pydantic import (
|
|
20
|
+
BaseModel,
|
|
21
|
+
ConfigDict,
|
|
22
|
+
Field,
|
|
23
|
+
ValidationError,
|
|
24
|
+
field_validator,
|
|
25
|
+
model_validator,
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
from dromeus.application import LocalData, check_environment, load_draft, prepare_task
|
|
29
|
+
from dromeus.errors import ConfigError, FormationError
|
|
30
|
+
from dromeus.manifests.canonical import (
|
|
31
|
+
canonical_json,
|
|
32
|
+
)
|
|
33
|
+
from dromeus.manifests.models import (
|
|
34
|
+
DraftRunSpec,
|
|
35
|
+
Invitation,
|
|
36
|
+
SealedManifestExpectation,
|
|
37
|
+
)
|
|
38
|
+
from dromeus.membership.formation import FormationResult, create_invitation
|
|
39
|
+
from dromeus.persistence.files import atomic_write_bytes
|
|
40
|
+
from dromeus.runtime import (
|
|
41
|
+
FailureConfig,
|
|
42
|
+
InitiatorFormation,
|
|
43
|
+
NodeRunResult,
|
|
44
|
+
NodeRuntime,
|
|
45
|
+
ParticipantFormation,
|
|
46
|
+
TrainingConfig,
|
|
47
|
+
WorkloadFactory,
|
|
48
|
+
private_preparation_for_node,
|
|
49
|
+
)
|
|
50
|
+
from dromeus.telemetry.events import (
|
|
51
|
+
EventSink,
|
|
52
|
+
JsonlEventSink,
|
|
53
|
+
emit_event,
|
|
54
|
+
event_log_path,
|
|
55
|
+
)
|
|
56
|
+
from dromeus.telemetry.metrics import JsonlMetricsPublisher
|
|
57
|
+
from dromeus.training.task import TrainingTask
|
|
58
|
+
from dromeus.transport.axl import AXLBridgeConfig, AXLTransport
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
class NodeRole(StrEnum):
|
|
62
|
+
INITIATOR = "initiator"
|
|
63
|
+
PARTICIPANT = "participant"
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
class NodeConfig(BaseModel):
|
|
67
|
+
"""Validated machine-local inputs for one non-interactive node."""
|
|
68
|
+
|
|
69
|
+
model_config = ConfigDict(extra="forbid", frozen=True)
|
|
70
|
+
|
|
71
|
+
role: NodeRole
|
|
72
|
+
draft_path: Path
|
|
73
|
+
axl_bridge_url: Annotated[str, Field(min_length=1)]
|
|
74
|
+
run_root: Path
|
|
75
|
+
invitation_path: Path
|
|
76
|
+
bootstrap_uri: Annotated[str, Field(min_length=1)]
|
|
77
|
+
invitation_timeout_seconds: Annotated[float, Field(gt=0)] = 300.0
|
|
78
|
+
enrollment_window_seconds: Annotated[float, Field(gt=0)] | None = None
|
|
79
|
+
manifest_expectation: SealedManifestExpectation | None = None
|
|
80
|
+
|
|
81
|
+
@model_validator(mode="after")
|
|
82
|
+
def enrollment_window_is_initiator_only(self) -> Self:
|
|
83
|
+
if (
|
|
84
|
+
self.enrollment_window_seconds is not None
|
|
85
|
+
and self.role is not NodeRole.INITIATOR
|
|
86
|
+
):
|
|
87
|
+
raise ValueError("enrollment_window_seconds applies to the initiator only")
|
|
88
|
+
return self
|
|
89
|
+
|
|
90
|
+
@field_validator("axl_bridge_url")
|
|
91
|
+
@classmethod
|
|
92
|
+
def bridge_must_be_loopback(cls, value: str) -> str:
|
|
93
|
+
parsed = urlparse(value)
|
|
94
|
+
if parsed.scheme != "http" or parsed.hostname not in {
|
|
95
|
+
"127.0.0.1",
|
|
96
|
+
"::1",
|
|
97
|
+
"localhost",
|
|
98
|
+
}:
|
|
99
|
+
raise ValueError("AXL bridge URL must use HTTP on loopback")
|
|
100
|
+
return value.rstrip("/")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
TrainingDecorator = Callable[
|
|
104
|
+
[FormationResult, str, TrainingConfig],
|
|
105
|
+
TrainingConfig,
|
|
106
|
+
]
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
def load_node_config(path: Path) -> NodeConfig:
|
|
110
|
+
"""Load one closed node configuration from YAML."""
|
|
111
|
+
try:
|
|
112
|
+
value = cast(object, yaml.safe_load(path.read_text(encoding="utf-8")))
|
|
113
|
+
return NodeConfig.model_validate(value)
|
|
114
|
+
except (OSError, yaml.YAMLError, ValidationError) as error:
|
|
115
|
+
raise ConfigError(f"invalid node config {path}: {error}") from error
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
@dataclass(frozen=True)
|
|
119
|
+
class RunResult:
|
|
120
|
+
"""What a node reports after the group finishes every round."""
|
|
121
|
+
|
|
122
|
+
run_id: str
|
|
123
|
+
manifest_hash: str
|
|
124
|
+
committed_rounds: int
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
class Node:
|
|
128
|
+
"""One participant in a run, set up from its node config file.
|
|
129
|
+
|
|
130
|
+
The config names this machine's AXL bridge, role, draft and run directory.
|
|
131
|
+
Create it with `Node.from_file`, then `await node.run(task, local=...)`.
|
|
132
|
+
"""
|
|
133
|
+
|
|
134
|
+
def __init__(self, config: NodeConfig) -> None:
|
|
135
|
+
self._config = config
|
|
136
|
+
|
|
137
|
+
@classmethod
|
|
138
|
+
def from_file(cls, path: Path | str) -> Node:
|
|
139
|
+
return cls(load_node_config(Path(path)))
|
|
140
|
+
|
|
141
|
+
async def run(self, task: TrainingTask, *, local: LocalData) -> RunResult:
|
|
142
|
+
"""Join the group, train every round, and return when the run completes.
|
|
143
|
+
|
|
144
|
+
Before contacting AXL, checks the draft against this task and against
|
|
145
|
+
the installed Dromeus and PyTorch versions.
|
|
146
|
+
"""
|
|
147
|
+
result = await run_node(
|
|
148
|
+
self._config,
|
|
149
|
+
prepare_training=lambda draft: prepare_task(
|
|
150
|
+
draft=draft, task=task, local=local
|
|
151
|
+
),
|
|
152
|
+
)
|
|
153
|
+
return RunResult(
|
|
154
|
+
run_id=result.formation.manifest.run_id,
|
|
155
|
+
manifest_hash=result.formation.manifest_hash,
|
|
156
|
+
committed_rounds=len(result.commits),
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
async def run_node(
|
|
161
|
+
config: NodeConfig,
|
|
162
|
+
*,
|
|
163
|
+
prepare_training: WorkloadFactory,
|
|
164
|
+
training_decorator: TrainingDecorator | None = None,
|
|
165
|
+
on_ready: Callable[[FormationResult, str, EventSink], None | Awaitable[None]]
|
|
166
|
+
| None = None,
|
|
167
|
+
) -> NodeRunResult:
|
|
168
|
+
"""Run a developer-supplied workload; this module never selects models or data."""
|
|
169
|
+
draft, event_sink = await asyncio.to_thread(
|
|
170
|
+
_prepare_node_start,
|
|
171
|
+
config,
|
|
172
|
+
)
|
|
173
|
+
check_environment(draft)
|
|
174
|
+
prepared_training = await asyncio.to_thread(prepare_training, draft)
|
|
175
|
+
prepared_training.validate_draft(draft)
|
|
176
|
+
private_trainer = private_preparation_for_node(draft, prepared_training)
|
|
177
|
+
local_tensor_schema = prepared_training.tensor_schema
|
|
178
|
+
transport = AXLTransport(AXLBridgeConfig(base_url=config.axl_bridge_url))
|
|
179
|
+
local_key = await transport.local_public_key()
|
|
180
|
+
failure = FailureConfig.for_run_root(config.run_root)
|
|
181
|
+
runtime = NodeRuntime(
|
|
182
|
+
transport=transport,
|
|
183
|
+
draft=draft,
|
|
184
|
+
environment=draft.environment,
|
|
185
|
+
dataset=draft.dataset,
|
|
186
|
+
artifact_root=config.run_root / "formation-artifacts",
|
|
187
|
+
event_sink=event_sink,
|
|
188
|
+
failure=failure,
|
|
189
|
+
local_tensor_schema=local_tensor_schema,
|
|
190
|
+
private_trainer=private_trainer,
|
|
191
|
+
)
|
|
192
|
+
event_sink = runtime.event_sink or event_sink
|
|
193
|
+
|
|
194
|
+
if config.role is NodeRole.INITIATOR:
|
|
195
|
+
checkpoint = await asyncio.to_thread(
|
|
196
|
+
prepared_training.create_initial_checkpoint,
|
|
197
|
+
config.run_root / "initial.safetensors",
|
|
198
|
+
)
|
|
199
|
+
expires_at = (
|
|
200
|
+
None
|
|
201
|
+
if config.enrollment_window_seconds is None
|
|
202
|
+
else datetime.now(UTC)
|
|
203
|
+
+ timedelta(seconds=config.enrollment_window_seconds)
|
|
204
|
+
)
|
|
205
|
+
invitation = create_invitation(
|
|
206
|
+
draft=draft,
|
|
207
|
+
initiator_public_key=local_key,
|
|
208
|
+
bootstrap_uri=config.bootstrap_uri,
|
|
209
|
+
enrollment_expires_at=expires_at,
|
|
210
|
+
)
|
|
211
|
+
await asyncio.to_thread(_write_invitation, config.invitation_path, invitation)
|
|
212
|
+
formation = InitiatorFormation(
|
|
213
|
+
bootstrap_uri=config.bootstrap_uri,
|
|
214
|
+
checkpoint_path=checkpoint.path,
|
|
215
|
+
tensor_schema=checkpoint.tensor_schema,
|
|
216
|
+
enrollment_expires_at=expires_at,
|
|
217
|
+
)
|
|
218
|
+
else:
|
|
219
|
+
invitation = await _read_invitation(
|
|
220
|
+
config.invitation_path,
|
|
221
|
+
timeout_seconds=config.invitation_timeout_seconds,
|
|
222
|
+
)
|
|
223
|
+
if invitation.bootstrap_uri != config.bootstrap_uri:
|
|
224
|
+
raise ConfigError("invitation bootstrap URI does not match node config")
|
|
225
|
+
formation = ParticipantFormation(invitation=invitation)
|
|
226
|
+
|
|
227
|
+
run_store_root = failure.run_store.root
|
|
228
|
+
|
|
229
|
+
def build_training(result: FormationResult) -> TrainingConfig:
|
|
230
|
+
metrics = JsonlMetricsPublisher(
|
|
231
|
+
sink=event_sink,
|
|
232
|
+
run_id=result.manifest.run_id,
|
|
233
|
+
manifest_hash=result.manifest_hash,
|
|
234
|
+
node_id=local_key,
|
|
235
|
+
)
|
|
236
|
+
training = prepared_training.build_config(
|
|
237
|
+
result=result,
|
|
238
|
+
local_public_key=local_key,
|
|
239
|
+
run_root=config.run_root,
|
|
240
|
+
metrics_publisher=metrics,
|
|
241
|
+
)
|
|
242
|
+
if training_decorator is not None:
|
|
243
|
+
return training_decorator(result, local_key, training)
|
|
244
|
+
return training
|
|
245
|
+
|
|
246
|
+
async def record_ready(result: FormationResult) -> None:
|
|
247
|
+
await _write_topology_snapshot(
|
|
248
|
+
transport,
|
|
249
|
+
run_store_root / "topology-ready.json",
|
|
250
|
+
)
|
|
251
|
+
await asyncio.to_thread(
|
|
252
|
+
emit_event,
|
|
253
|
+
"node_ready",
|
|
254
|
+
run_id=result.manifest.run_id,
|
|
255
|
+
manifest_hash=result.manifest_hash,
|
|
256
|
+
node_id=local_key,
|
|
257
|
+
sink=event_sink,
|
|
258
|
+
)
|
|
259
|
+
if on_ready is not None:
|
|
260
|
+
notification = on_ready(result, local_key, event_sink)
|
|
261
|
+
if inspect.isawaitable(notification):
|
|
262
|
+
await notification
|
|
263
|
+
|
|
264
|
+
async def record_complete(result: NodeRunResult) -> None:
|
|
265
|
+
await _write_topology_snapshot(
|
|
266
|
+
transport,
|
|
267
|
+
run_store_root / "topology-complete.json",
|
|
268
|
+
)
|
|
269
|
+
await asyncio.to_thread(
|
|
270
|
+
emit_event,
|
|
271
|
+
"node_complete",
|
|
272
|
+
run_id=result.formation.manifest.run_id,
|
|
273
|
+
manifest_hash=result.formation.manifest_hash,
|
|
274
|
+
node_id=local_key,
|
|
275
|
+
sink=event_sink,
|
|
276
|
+
committed_rounds=len(result.commits),
|
|
277
|
+
)
|
|
278
|
+
|
|
279
|
+
return await runtime.run_to_completion(
|
|
280
|
+
formation=formation,
|
|
281
|
+
training_factory=build_training,
|
|
282
|
+
manifest_expectation=config.manifest_expectation,
|
|
283
|
+
ready_hook=record_ready,
|
|
284
|
+
completion_hook=record_complete,
|
|
285
|
+
)
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _write_invitation(path: Path, invitation: Invitation) -> None:
|
|
289
|
+
_atomic_write(path, canonical_json(invitation))
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def _prepare_node_start(
|
|
293
|
+
config: NodeConfig,
|
|
294
|
+
) -> tuple[DraftRunSpec, JsonlEventSink]:
|
|
295
|
+
draft = load_draft(config.draft_path)
|
|
296
|
+
config.run_root.mkdir(parents=True, exist_ok=True)
|
|
297
|
+
event_sink = JsonlEventSink(event_log_path(config.run_root))
|
|
298
|
+
emit_event(
|
|
299
|
+
"node_start",
|
|
300
|
+
run_id=draft.run_id,
|
|
301
|
+
sink=event_sink,
|
|
302
|
+
role=config.role,
|
|
303
|
+
pytorch_build=torch.__version__, # drafts pin the release, not the build
|
|
304
|
+
)
|
|
305
|
+
return draft, event_sink
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
async def _read_invitation(path: Path, *, timeout_seconds: float) -> Invitation:
|
|
309
|
+
"""Wait for a complete invitation file.
|
|
310
|
+
|
|
311
|
+
Users deliver the invitation by copying it, so the file can exist before all
|
|
312
|
+
of its bytes arrive. An empty or unparseable file means "not yet".
|
|
313
|
+
"""
|
|
314
|
+
deadline = time.monotonic() + timeout_seconds
|
|
315
|
+
problem = "file does not exist"
|
|
316
|
+
while time.monotonic() < deadline:
|
|
317
|
+
try:
|
|
318
|
+
data = await asyncio.to_thread(path.read_bytes)
|
|
319
|
+
return Invitation.model_validate_json(data)
|
|
320
|
+
except FileNotFoundError:
|
|
321
|
+
problem = "file does not exist"
|
|
322
|
+
except ValidationError as error:
|
|
323
|
+
problem = f"file is incomplete or invalid: {error.errors()[0]['msg']}"
|
|
324
|
+
await asyncio.sleep(0.2)
|
|
325
|
+
raise FormationError(f"timed out waiting for invitation {path}: {problem}")
|
|
326
|
+
|
|
327
|
+
|
|
328
|
+
async def _write_topology_snapshot(transport: AXLTransport, path: Path) -> None:
|
|
329
|
+
topology = await transport.topology()
|
|
330
|
+
payload = (
|
|
331
|
+
json.dumps(topology, allow_nan=False, sort_keys=True, indent=2) + "\n"
|
|
332
|
+
).encode()
|
|
333
|
+
await asyncio.to_thread(_atomic_write, path, payload)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def _atomic_write(path: Path, payload: bytes) -> None:
|
|
337
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
338
|
+
atomic_write_bytes(path, payload)
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"""Durable run state."""
|
|
2
|
+
|
|
3
|
+
from dromeus.persistence.archive import (
|
|
4
|
+
ARCHIVE_VERSION,
|
|
5
|
+
ArchiveState,
|
|
6
|
+
CheckpointRef,
|
|
7
|
+
RunArchive,
|
|
8
|
+
RunArchiveError,
|
|
9
|
+
)
|
|
10
|
+
from dromeus.persistence.run_store import RunStore, RunStoreError
|
|
11
|
+
|
|
12
|
+
__all__ = [
|
|
13
|
+
"ARCHIVE_VERSION",
|
|
14
|
+
"ArchiveState",
|
|
15
|
+
"CheckpointRef",
|
|
16
|
+
"RunArchive",
|
|
17
|
+
"RunArchiveError",
|
|
18
|
+
"RunStore",
|
|
19
|
+
"RunStoreError",
|
|
20
|
+
]
|