dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/node.py ADDED
@@ -0,0 +1,338 @@
1
+ """Run one Dromeus node: the public `Node` and its machine-local config."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import inspect
7
+ import json
8
+ import time
9
+ from collections.abc import Awaitable, Callable
10
+ from dataclasses import dataclass
11
+ from datetime import UTC, datetime, timedelta
12
+ from enum import StrEnum
13
+ from pathlib import Path
14
+ from typing import Annotated, Self, cast
15
+ from urllib.parse import urlparse
16
+
17
+ import torch
18
+ import yaml
19
+ from pydantic import (
20
+ BaseModel,
21
+ ConfigDict,
22
+ Field,
23
+ ValidationError,
24
+ field_validator,
25
+ model_validator,
26
+ )
27
+
28
+ from dromeus.application import LocalData, check_environment, load_draft, prepare_task
29
+ from dromeus.errors import ConfigError, FormationError
30
+ from dromeus.manifests.canonical import (
31
+ canonical_json,
32
+ )
33
+ from dromeus.manifests.models import (
34
+ DraftRunSpec,
35
+ Invitation,
36
+ SealedManifestExpectation,
37
+ )
38
+ from dromeus.membership.formation import FormationResult, create_invitation
39
+ from dromeus.persistence.files import atomic_write_bytes
40
+ from dromeus.runtime import (
41
+ FailureConfig,
42
+ InitiatorFormation,
43
+ NodeRunResult,
44
+ NodeRuntime,
45
+ ParticipantFormation,
46
+ TrainingConfig,
47
+ WorkloadFactory,
48
+ private_preparation_for_node,
49
+ )
50
+ from dromeus.telemetry.events import (
51
+ EventSink,
52
+ JsonlEventSink,
53
+ emit_event,
54
+ event_log_path,
55
+ )
56
+ from dromeus.telemetry.metrics import JsonlMetricsPublisher
57
+ from dromeus.training.task import TrainingTask
58
+ from dromeus.transport.axl import AXLBridgeConfig, AXLTransport
59
+
60
+
61
+ class NodeRole(StrEnum):
62
+ INITIATOR = "initiator"
63
+ PARTICIPANT = "participant"
64
+
65
+
66
+ class NodeConfig(BaseModel):
67
+ """Validated machine-local inputs for one non-interactive node."""
68
+
69
+ model_config = ConfigDict(extra="forbid", frozen=True)
70
+
71
+ role: NodeRole
72
+ draft_path: Path
73
+ axl_bridge_url: Annotated[str, Field(min_length=1)]
74
+ run_root: Path
75
+ invitation_path: Path
76
+ bootstrap_uri: Annotated[str, Field(min_length=1)]
77
+ invitation_timeout_seconds: Annotated[float, Field(gt=0)] = 300.0
78
+ enrollment_window_seconds: Annotated[float, Field(gt=0)] | None = None
79
+ manifest_expectation: SealedManifestExpectation | None = None
80
+
81
+ @model_validator(mode="after")
82
+ def enrollment_window_is_initiator_only(self) -> Self:
83
+ if (
84
+ self.enrollment_window_seconds is not None
85
+ and self.role is not NodeRole.INITIATOR
86
+ ):
87
+ raise ValueError("enrollment_window_seconds applies to the initiator only")
88
+ return self
89
+
90
+ @field_validator("axl_bridge_url")
91
+ @classmethod
92
+ def bridge_must_be_loopback(cls, value: str) -> str:
93
+ parsed = urlparse(value)
94
+ if parsed.scheme != "http" or parsed.hostname not in {
95
+ "127.0.0.1",
96
+ "::1",
97
+ "localhost",
98
+ }:
99
+ raise ValueError("AXL bridge URL must use HTTP on loopback")
100
+ return value.rstrip("/")
101
+
102
+
103
+ TrainingDecorator = Callable[
104
+ [FormationResult, str, TrainingConfig],
105
+ TrainingConfig,
106
+ ]
107
+
108
+
109
+ def load_node_config(path: Path) -> NodeConfig:
110
+ """Load one closed node configuration from YAML."""
111
+ try:
112
+ value = cast(object, yaml.safe_load(path.read_text(encoding="utf-8")))
113
+ return NodeConfig.model_validate(value)
114
+ except (OSError, yaml.YAMLError, ValidationError) as error:
115
+ raise ConfigError(f"invalid node config {path}: {error}") from error
116
+
117
+
118
+ @dataclass(frozen=True)
119
+ class RunResult:
120
+ """What a node reports after the group finishes every round."""
121
+
122
+ run_id: str
123
+ manifest_hash: str
124
+ committed_rounds: int
125
+
126
+
127
+ class Node:
128
+ """One participant in a run, set up from its node config file.
129
+
130
+ The config names this machine's AXL bridge, role, draft and run directory.
131
+ Create it with `Node.from_file`, then `await node.run(task, local=...)`.
132
+ """
133
+
134
+ def __init__(self, config: NodeConfig) -> None:
135
+ self._config = config
136
+
137
+ @classmethod
138
+ def from_file(cls, path: Path | str) -> Node:
139
+ return cls(load_node_config(Path(path)))
140
+
141
+ async def run(self, task: TrainingTask, *, local: LocalData) -> RunResult:
142
+ """Join the group, train every round, and return when the run completes.
143
+
144
+ Before contacting AXL, checks the draft against this task and against
145
+ the installed Dromeus and PyTorch versions.
146
+ """
147
+ result = await run_node(
148
+ self._config,
149
+ prepare_training=lambda draft: prepare_task(
150
+ draft=draft, task=task, local=local
151
+ ),
152
+ )
153
+ return RunResult(
154
+ run_id=result.formation.manifest.run_id,
155
+ manifest_hash=result.formation.manifest_hash,
156
+ committed_rounds=len(result.commits),
157
+ )
158
+
159
+
160
+ async def run_node(
161
+ config: NodeConfig,
162
+ *,
163
+ prepare_training: WorkloadFactory,
164
+ training_decorator: TrainingDecorator | None = None,
165
+ on_ready: Callable[[FormationResult, str, EventSink], None | Awaitable[None]]
166
+ | None = None,
167
+ ) -> NodeRunResult:
168
+ """Run a developer-supplied workload; this module never selects models or data."""
169
+ draft, event_sink = await asyncio.to_thread(
170
+ _prepare_node_start,
171
+ config,
172
+ )
173
+ check_environment(draft)
174
+ prepared_training = await asyncio.to_thread(prepare_training, draft)
175
+ prepared_training.validate_draft(draft)
176
+ private_trainer = private_preparation_for_node(draft, prepared_training)
177
+ local_tensor_schema = prepared_training.tensor_schema
178
+ transport = AXLTransport(AXLBridgeConfig(base_url=config.axl_bridge_url))
179
+ local_key = await transport.local_public_key()
180
+ failure = FailureConfig.for_run_root(config.run_root)
181
+ runtime = NodeRuntime(
182
+ transport=transport,
183
+ draft=draft,
184
+ environment=draft.environment,
185
+ dataset=draft.dataset,
186
+ artifact_root=config.run_root / "formation-artifacts",
187
+ event_sink=event_sink,
188
+ failure=failure,
189
+ local_tensor_schema=local_tensor_schema,
190
+ private_trainer=private_trainer,
191
+ )
192
+ event_sink = runtime.event_sink or event_sink
193
+
194
+ if config.role is NodeRole.INITIATOR:
195
+ checkpoint = await asyncio.to_thread(
196
+ prepared_training.create_initial_checkpoint,
197
+ config.run_root / "initial.safetensors",
198
+ )
199
+ expires_at = (
200
+ None
201
+ if config.enrollment_window_seconds is None
202
+ else datetime.now(UTC)
203
+ + timedelta(seconds=config.enrollment_window_seconds)
204
+ )
205
+ invitation = create_invitation(
206
+ draft=draft,
207
+ initiator_public_key=local_key,
208
+ bootstrap_uri=config.bootstrap_uri,
209
+ enrollment_expires_at=expires_at,
210
+ )
211
+ await asyncio.to_thread(_write_invitation, config.invitation_path, invitation)
212
+ formation = InitiatorFormation(
213
+ bootstrap_uri=config.bootstrap_uri,
214
+ checkpoint_path=checkpoint.path,
215
+ tensor_schema=checkpoint.tensor_schema,
216
+ enrollment_expires_at=expires_at,
217
+ )
218
+ else:
219
+ invitation = await _read_invitation(
220
+ config.invitation_path,
221
+ timeout_seconds=config.invitation_timeout_seconds,
222
+ )
223
+ if invitation.bootstrap_uri != config.bootstrap_uri:
224
+ raise ConfigError("invitation bootstrap URI does not match node config")
225
+ formation = ParticipantFormation(invitation=invitation)
226
+
227
+ run_store_root = failure.run_store.root
228
+
229
+ def build_training(result: FormationResult) -> TrainingConfig:
230
+ metrics = JsonlMetricsPublisher(
231
+ sink=event_sink,
232
+ run_id=result.manifest.run_id,
233
+ manifest_hash=result.manifest_hash,
234
+ node_id=local_key,
235
+ )
236
+ training = prepared_training.build_config(
237
+ result=result,
238
+ local_public_key=local_key,
239
+ run_root=config.run_root,
240
+ metrics_publisher=metrics,
241
+ )
242
+ if training_decorator is not None:
243
+ return training_decorator(result, local_key, training)
244
+ return training
245
+
246
+ async def record_ready(result: FormationResult) -> None:
247
+ await _write_topology_snapshot(
248
+ transport,
249
+ run_store_root / "topology-ready.json",
250
+ )
251
+ await asyncio.to_thread(
252
+ emit_event,
253
+ "node_ready",
254
+ run_id=result.manifest.run_id,
255
+ manifest_hash=result.manifest_hash,
256
+ node_id=local_key,
257
+ sink=event_sink,
258
+ )
259
+ if on_ready is not None:
260
+ notification = on_ready(result, local_key, event_sink)
261
+ if inspect.isawaitable(notification):
262
+ await notification
263
+
264
+ async def record_complete(result: NodeRunResult) -> None:
265
+ await _write_topology_snapshot(
266
+ transport,
267
+ run_store_root / "topology-complete.json",
268
+ )
269
+ await asyncio.to_thread(
270
+ emit_event,
271
+ "node_complete",
272
+ run_id=result.formation.manifest.run_id,
273
+ manifest_hash=result.formation.manifest_hash,
274
+ node_id=local_key,
275
+ sink=event_sink,
276
+ committed_rounds=len(result.commits),
277
+ )
278
+
279
+ return await runtime.run_to_completion(
280
+ formation=formation,
281
+ training_factory=build_training,
282
+ manifest_expectation=config.manifest_expectation,
283
+ ready_hook=record_ready,
284
+ completion_hook=record_complete,
285
+ )
286
+
287
+
288
+ def _write_invitation(path: Path, invitation: Invitation) -> None:
289
+ _atomic_write(path, canonical_json(invitation))
290
+
291
+
292
+ def _prepare_node_start(
293
+ config: NodeConfig,
294
+ ) -> tuple[DraftRunSpec, JsonlEventSink]:
295
+ draft = load_draft(config.draft_path)
296
+ config.run_root.mkdir(parents=True, exist_ok=True)
297
+ event_sink = JsonlEventSink(event_log_path(config.run_root))
298
+ emit_event(
299
+ "node_start",
300
+ run_id=draft.run_id,
301
+ sink=event_sink,
302
+ role=config.role,
303
+ pytorch_build=torch.__version__, # drafts pin the release, not the build
304
+ )
305
+ return draft, event_sink
306
+
307
+
308
+ async def _read_invitation(path: Path, *, timeout_seconds: float) -> Invitation:
309
+ """Wait for a complete invitation file.
310
+
311
+ Users deliver the invitation by copying it, so the file can exist before all
312
+ of its bytes arrive. An empty or unparseable file means "not yet".
313
+ """
314
+ deadline = time.monotonic() + timeout_seconds
315
+ problem = "file does not exist"
316
+ while time.monotonic() < deadline:
317
+ try:
318
+ data = await asyncio.to_thread(path.read_bytes)
319
+ return Invitation.model_validate_json(data)
320
+ except FileNotFoundError:
321
+ problem = "file does not exist"
322
+ except ValidationError as error:
323
+ problem = f"file is incomplete or invalid: {error.errors()[0]['msg']}"
324
+ await asyncio.sleep(0.2)
325
+ raise FormationError(f"timed out waiting for invitation {path}: {problem}")
326
+
327
+
328
+ async def _write_topology_snapshot(transport: AXLTransport, path: Path) -> None:
329
+ topology = await transport.topology()
330
+ payload = (
331
+ json.dumps(topology, allow_nan=False, sort_keys=True, indent=2) + "\n"
332
+ ).encode()
333
+ await asyncio.to_thread(_atomic_write, path, payload)
334
+
335
+
336
+ def _atomic_write(path: Path, payload: bytes) -> None:
337
+ path.parent.mkdir(parents=True, exist_ok=True)
338
+ atomic_write_bytes(path, payload)
@@ -0,0 +1,20 @@
1
+ """Durable run state."""
2
+
3
+ from dromeus.persistence.archive import (
4
+ ARCHIVE_VERSION,
5
+ ArchiveState,
6
+ CheckpointRef,
7
+ RunArchive,
8
+ RunArchiveError,
9
+ )
10
+ from dromeus.persistence.run_store import RunStore, RunStoreError
11
+
12
+ __all__ = [
13
+ "ARCHIVE_VERSION",
14
+ "ArchiveState",
15
+ "CheckpointRef",
16
+ "RunArchive",
17
+ "RunArchiveError",
18
+ "RunStore",
19
+ "RunStoreError",
20
+ ]