dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/cli.py
ADDED
|
@@ -0,0 +1,341 @@
|
|
|
1
|
+
"""The `dromeus` command: a thin shell over the library.
|
|
2
|
+
|
|
3
|
+
Each command parses its arguments, makes one library call and prints the
|
|
4
|
+
result. Errors print as one line and map to a stable exit code (`ExitCode`);
|
|
5
|
+
`--debug` adds the traceback.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import argparse
|
|
11
|
+
import asyncio
|
|
12
|
+
import importlib
|
|
13
|
+
import os
|
|
14
|
+
import sys
|
|
15
|
+
import traceback
|
|
16
|
+
from collections.abc import Callable, Sequence
|
|
17
|
+
from enum import IntEnum
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import TYPE_CHECKING, cast
|
|
20
|
+
|
|
21
|
+
from dromeus.errors import (
|
|
22
|
+
ConfigError,
|
|
23
|
+
DraftError,
|
|
24
|
+
FormationError,
|
|
25
|
+
PrivacyError,
|
|
26
|
+
RunError,
|
|
27
|
+
TaskError,
|
|
28
|
+
one_line,
|
|
29
|
+
)
|
|
30
|
+
|
|
31
|
+
if TYPE_CHECKING:
|
|
32
|
+
from dromeus.application import LocalData
|
|
33
|
+
from dromeus.telemetry.progress import Progress
|
|
34
|
+
from dromeus.training.task import TrainingTask
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
class ExitCode(IntEnum):
|
|
38
|
+
"""Process exit status of every `dromeus` command."""
|
|
39
|
+
|
|
40
|
+
OK = 0
|
|
41
|
+
ERROR = 1
|
|
42
|
+
"""Unexpected failure: a bug, or an exception from the task's own code."""
|
|
43
|
+
USAGE = 2
|
|
44
|
+
"""The command line is malformed."""
|
|
45
|
+
INVALID_INPUT = 3
|
|
46
|
+
"""A node config, draft, task or privacy setting is wrong."""
|
|
47
|
+
FORMATION_FAILED = 4
|
|
48
|
+
"""The group did not form: invitation, enrollment or readiness failed."""
|
|
49
|
+
RUN_FAILED = 5
|
|
50
|
+
"""The run failed after formation."""
|
|
51
|
+
NOT_READY = 6
|
|
52
|
+
"""`dromeus doctor` found at least one failing check."""
|
|
53
|
+
INTERRUPTED = 130
|
|
54
|
+
"""Stopped with Ctrl-C."""
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def main(argv: Sequence[str] | None = None) -> int:
|
|
58
|
+
parser = _parser()
|
|
59
|
+
args = parser.parse_args(argv)
|
|
60
|
+
command = cast(Callable[[argparse.Namespace], ExitCode], args.command)
|
|
61
|
+
debug = bool(getattr(args, "debug", False))
|
|
62
|
+
try:
|
|
63
|
+
return int(command(args))
|
|
64
|
+
except KeyboardInterrupt:
|
|
65
|
+
_print_error("interrupted")
|
|
66
|
+
return ExitCode.INTERRUPTED
|
|
67
|
+
except Exception as error:
|
|
68
|
+
if debug:
|
|
69
|
+
traceback.print_exc()
|
|
70
|
+
code = exit_code_for(error)
|
|
71
|
+
message = one_line(str(error)) or type(error).__name__
|
|
72
|
+
if code is ExitCode.ERROR:
|
|
73
|
+
message = f"unexpected {type(error).__name__}: {message}"
|
|
74
|
+
if not debug:
|
|
75
|
+
message += " (rerun with --debug for the traceback)"
|
|
76
|
+
_print_error(message)
|
|
77
|
+
return code
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def exit_code_for(error: BaseException) -> ExitCode:
|
|
81
|
+
"""Map a failure to its documented exit code."""
|
|
82
|
+
if isinstance(error, FormationError):
|
|
83
|
+
return ExitCode.FORMATION_FAILED
|
|
84
|
+
if isinstance(error, RunError):
|
|
85
|
+
return ExitCode.RUN_FAILED
|
|
86
|
+
if isinstance(error, ConfigError | DraftError | TaskError | PrivacyError):
|
|
87
|
+
return ExitCode.INVALID_INPUT
|
|
88
|
+
return ExitCode.ERROR
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def _parser() -> argparse.ArgumentParser:
|
|
92
|
+
common = argparse.ArgumentParser(add_help=False)
|
|
93
|
+
common.add_argument(
|
|
94
|
+
"--debug",
|
|
95
|
+
action="store_true",
|
|
96
|
+
default=argparse.SUPPRESS, # so `dromeus --debug run` and `run --debug` agree
|
|
97
|
+
help="print full tracebacks on errors",
|
|
98
|
+
)
|
|
99
|
+
parser = argparse.ArgumentParser(
|
|
100
|
+
prog="dromeus",
|
|
101
|
+
description="Decentralized federated learning over Gensyn AXL.",
|
|
102
|
+
parents=[common],
|
|
103
|
+
)
|
|
104
|
+
parser.add_argument("--version", action="version", version=_version())
|
|
105
|
+
commands = parser.add_subparsers(title="commands", required=True)
|
|
106
|
+
task_help = (
|
|
107
|
+
"module:callable returning the TrainingTask, or (TrainingTask, LocalData); "
|
|
108
|
+
"the module is imported from the current directory"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
run = commands.add_parser(
|
|
112
|
+
"run", parents=[common], help="join the group and train every round"
|
|
113
|
+
)
|
|
114
|
+
run.add_argument("--config", required=True, type=Path, help="node config YAML")
|
|
115
|
+
run.add_argument(
|
|
116
|
+
"--task",
|
|
117
|
+
required=True,
|
|
118
|
+
help="module:callable returning (TrainingTask, LocalData) for this node",
|
|
119
|
+
)
|
|
120
|
+
run.set_defaults(command=_run)
|
|
121
|
+
|
|
122
|
+
check = commands.add_parser(
|
|
123
|
+
"check",
|
|
124
|
+
parents=[common],
|
|
125
|
+
help="check a draft against the task and this machine, offline",
|
|
126
|
+
)
|
|
127
|
+
check.add_argument("--draft", required=True, type=Path, help="shared draft YAML")
|
|
128
|
+
check.add_argument("--task", required=True, help=task_help)
|
|
129
|
+
check.set_defaults(command=_check)
|
|
130
|
+
|
|
131
|
+
draft = commands.add_parser(
|
|
132
|
+
"draft",
|
|
133
|
+
parents=[common],
|
|
134
|
+
help="write the group's shared draft for a task",
|
|
135
|
+
description=(
|
|
136
|
+
"Write the shared draft with default NoLoCo, transport and no privacy. "
|
|
137
|
+
"For privacy, transport or divergence settings call make_draft from "
|
|
138
|
+
"Python."
|
|
139
|
+
),
|
|
140
|
+
)
|
|
141
|
+
draft.add_argument("output", type=Path, help="draft YAML to write")
|
|
142
|
+
draft.add_argument("--task", required=True, help=task_help)
|
|
143
|
+
draft.add_argument("--run-id", required=True)
|
|
144
|
+
draft.add_argument("--participants", required=True, type=int)
|
|
145
|
+
draft.add_argument("--rounds", required=True, type=int)
|
|
146
|
+
draft.add_argument("--local-steps", required=True, type=int)
|
|
147
|
+
draft.add_argument(
|
|
148
|
+
"--axl-version", required=True, help="AXL revision every node runs"
|
|
149
|
+
)
|
|
150
|
+
draft.add_argument("--seed", type=int, default=0)
|
|
151
|
+
draft.set_defaults(command=_draft)
|
|
152
|
+
|
|
153
|
+
doctor = commands.add_parser(
|
|
154
|
+
"doctor",
|
|
155
|
+
parents=[common],
|
|
156
|
+
help="check that this machine is ready to run a node",
|
|
157
|
+
)
|
|
158
|
+
doctor.add_argument("--config", required=True, type=Path, help="node config YAML")
|
|
159
|
+
doctor.add_argument("--task", help=f"also check the draft against it; {task_help}")
|
|
160
|
+
doctor.set_defaults(command=_doctor)
|
|
161
|
+
|
|
162
|
+
watch = commands.add_parser(
|
|
163
|
+
"watch",
|
|
164
|
+
parents=[common],
|
|
165
|
+
help="follow a node's rounds, loss and consensus distance",
|
|
166
|
+
)
|
|
167
|
+
watch.add_argument(
|
|
168
|
+
"--run-root", required=True, type=Path, help="run_root from the node config"
|
|
169
|
+
)
|
|
170
|
+
watch.add_argument(
|
|
171
|
+
"--once", action="store_true", help="print what is logged so far and exit"
|
|
172
|
+
)
|
|
173
|
+
watch.set_defaults(command=_watch)
|
|
174
|
+
return parser
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
def _run(args: argparse.Namespace) -> ExitCode:
|
|
178
|
+
from dromeus.node import Node, load_node_config
|
|
179
|
+
|
|
180
|
+
config = load_node_config(cast(Path, args.config))
|
|
181
|
+
task, local = load_task(cast(str, args.task))
|
|
182
|
+
if local is None:
|
|
183
|
+
raise ConfigError(
|
|
184
|
+
f"{args.task} must return (TrainingTask, LocalData) to run a node"
|
|
185
|
+
)
|
|
186
|
+
_print_error(f"running; follow with: dromeus watch --run-root {config.run_root}")
|
|
187
|
+
result = asyncio.run(Node(config).run(task, local=local))
|
|
188
|
+
print(
|
|
189
|
+
f"run {result.run_id} complete: {result.committed_rounds} rounds committed, "
|
|
190
|
+
f"manifest {result.manifest_hash}"
|
|
191
|
+
)
|
|
192
|
+
return ExitCode.OK
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
def _check(args: argparse.Namespace) -> ExitCode:
|
|
196
|
+
from dromeus.application import check_draft, load_draft
|
|
197
|
+
from dromeus.manifests.canonical import canonical_hash
|
|
198
|
+
|
|
199
|
+
path = cast(Path, args.draft)
|
|
200
|
+
task, _ = load_task(cast(str, args.task))
|
|
201
|
+
check_draft(path, task)
|
|
202
|
+
draft = load_draft(path)
|
|
203
|
+
print(f"draft {path} matches this task and machine")
|
|
204
|
+
print(" model, task and loss definitions: match")
|
|
205
|
+
print(
|
|
206
|
+
f" Dromeus {draft.environment.dromeus_version}, "
|
|
207
|
+
f"PyTorch {draft.environment.pytorch_version}: installed"
|
|
208
|
+
)
|
|
209
|
+
print(f" draft hash: {canonical_hash(draft)}")
|
|
210
|
+
return ExitCode.OK
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _draft(args: argparse.Namespace) -> ExitCode:
|
|
214
|
+
from dromeus.application import make_draft
|
|
215
|
+
|
|
216
|
+
task, _ = load_task(cast(str, args.task))
|
|
217
|
+
path = cast(Path, args.output)
|
|
218
|
+
digest = make_draft(
|
|
219
|
+
task,
|
|
220
|
+
path,
|
|
221
|
+
run_id=cast(str, args.run_id),
|
|
222
|
+
participants=cast(int, args.participants),
|
|
223
|
+
rounds=cast(int, args.rounds),
|
|
224
|
+
local_steps=cast(int, args.local_steps),
|
|
225
|
+
axl_version=cast(str, args.axl_version),
|
|
226
|
+
seed=cast(int, args.seed),
|
|
227
|
+
)
|
|
228
|
+
print(f"wrote {path}; draft hash {digest}")
|
|
229
|
+
return ExitCode.OK
|
|
230
|
+
|
|
231
|
+
|
|
232
|
+
def _doctor(args: argparse.Namespace) -> ExitCode:
|
|
233
|
+
from dromeus.diagnostics import Status, diagnose
|
|
234
|
+
|
|
235
|
+
reference = cast(str | None, args.task)
|
|
236
|
+
task = None if reference is None else load_task(reference)[0]
|
|
237
|
+
findings = asyncio.run(diagnose(cast(Path, args.config), task=task))
|
|
238
|
+
width = max(len(finding.check) for finding in findings)
|
|
239
|
+
for finding in findings:
|
|
240
|
+
print(f"{finding.status.value:<5} {finding.check:<{width}} {finding.detail}")
|
|
241
|
+
if finding.fix is not None:
|
|
242
|
+
print(f"{'':<5} {'':<{width}} fix: {finding.fix}")
|
|
243
|
+
if any(finding.status is Status.FAIL for finding in findings):
|
|
244
|
+
return ExitCode.NOT_READY
|
|
245
|
+
return ExitCode.OK
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _watch(args: argparse.Namespace) -> ExitCode:
|
|
249
|
+
from dromeus.telemetry.events import event_log_path
|
|
250
|
+
from dromeus.telemetry.progress import RunFailed, read_progress
|
|
251
|
+
|
|
252
|
+
path = event_log_path(cast(Path, args.run_root))
|
|
253
|
+
code = ExitCode.OK
|
|
254
|
+
for item in read_progress(path, follow=not args.once):
|
|
255
|
+
print(describe_progress(item), flush=True)
|
|
256
|
+
if isinstance(item, RunFailed):
|
|
257
|
+
code = ExitCode.RUN_FAILED
|
|
258
|
+
return code
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def describe_progress(item: Progress) -> str:
|
|
262
|
+
"""One terminal line for one progress item."""
|
|
263
|
+
from dromeus.telemetry.progress import (
|
|
264
|
+
ConsensusMeasured,
|
|
265
|
+
NodeReady,
|
|
266
|
+
NodeStarted,
|
|
267
|
+
RoundCommitted,
|
|
268
|
+
RunCompleted,
|
|
269
|
+
)
|
|
270
|
+
|
|
271
|
+
match item:
|
|
272
|
+
case NodeStarted():
|
|
273
|
+
return f"node started: run {item.run_id} as {item.role}"
|
|
274
|
+
case NodeReady():
|
|
275
|
+
return f"group formed: manifest {item.manifest_hash[:16]}"
|
|
276
|
+
case RoundCommitted():
|
|
277
|
+
loss = "-" if item.local_loss is None else f"{item.local_loss:.4g}"
|
|
278
|
+
metrics = "".join(
|
|
279
|
+
f" {name} {value:.4g}" for name, value in sorted(item.metrics.items())
|
|
280
|
+
)
|
|
281
|
+
return (
|
|
282
|
+
f"round {item.round_id:>5} loss {loss}{metrics} "
|
|
283
|
+
f"{item.round_seconds:.1f}s peer {item.peer_id[:8]}"
|
|
284
|
+
)
|
|
285
|
+
case ConsensusMeasured():
|
|
286
|
+
return (
|
|
287
|
+
f"round {item.round_id:>5} consensus distance "
|
|
288
|
+
f"{item.normalized_rms:.4g}"
|
|
289
|
+
)
|
|
290
|
+
case RunCompleted():
|
|
291
|
+
return f"run complete: {item.committed_rounds} rounds committed"
|
|
292
|
+
case _:
|
|
293
|
+
where = "" if item.round_id is None else f" in round {item.round_id}"
|
|
294
|
+
return f"run failed{where}: {item.error}"
|
|
295
|
+
|
|
296
|
+
|
|
297
|
+
def load_task(reference: str) -> tuple[TrainingTask, LocalData | None]:
|
|
298
|
+
"""Call a `module:callable` that returns a task, or a task and local data.
|
|
299
|
+
|
|
300
|
+
The current directory is importable, as with `python -m`.
|
|
301
|
+
"""
|
|
302
|
+
from dromeus.application import LocalData
|
|
303
|
+
from dromeus.training.task import TrainingTask
|
|
304
|
+
|
|
305
|
+
module_name, separator, name = reference.partition(":")
|
|
306
|
+
if not separator or not module_name or not name:
|
|
307
|
+
raise ConfigError(f"task must be given as module:callable, got {reference!r}")
|
|
308
|
+
if os.getcwd() not in sys.path:
|
|
309
|
+
sys.path.insert(0, os.getcwd())
|
|
310
|
+
try:
|
|
311
|
+
module = importlib.import_module(module_name)
|
|
312
|
+
except ModuleNotFoundError as error:
|
|
313
|
+
if error.name is not None and module_name.startswith(error.name):
|
|
314
|
+
raise ConfigError(
|
|
315
|
+
f"cannot import {module_name!r} from {os.getcwd()}: {error}"
|
|
316
|
+
) from error
|
|
317
|
+
raise
|
|
318
|
+
setup = getattr(module, name, None)
|
|
319
|
+
if not callable(setup):
|
|
320
|
+
raise ConfigError(f"{reference} is not a callable in {module_name}")
|
|
321
|
+
value: object = setup()
|
|
322
|
+
if isinstance(value, TrainingTask):
|
|
323
|
+
return value, None
|
|
324
|
+
match value:
|
|
325
|
+
case (TrainingTask() as task, LocalData() as local):
|
|
326
|
+
return task, local
|
|
327
|
+
case _:
|
|
328
|
+
raise ConfigError(
|
|
329
|
+
f"{reference} must return a TrainingTask or "
|
|
330
|
+
f"(TrainingTask, LocalData), not {type(value).__name__}"
|
|
331
|
+
)
|
|
332
|
+
|
|
333
|
+
|
|
334
|
+
def _print_error(message: str) -> None:
|
|
335
|
+
print(f"dromeus: {message}", file=sys.stderr, flush=True)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _version() -> str:
|
|
339
|
+
from importlib.metadata import version
|
|
340
|
+
|
|
341
|
+
return f"dromeus {version('dromeus')}"
|
dromeus/diagnostics.py
ADDED
|
@@ -0,0 +1,303 @@
|
|
|
1
|
+
"""Check whether this machine is ready to run a node, without training.
|
|
2
|
+
|
|
3
|
+
`dromeus doctor` prints the findings from `diagnose`. Each check looks at one
|
|
4
|
+
thing (the node config, the draft, the AXL bridge, peer links, the invitation,
|
|
5
|
+
the clock) and records what it saw. A finding that is not OK carries one line
|
|
6
|
+
telling the user what to do. Nothing here joins a group or trains a model.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import shutil
|
|
12
|
+
import subprocess
|
|
13
|
+
import sys
|
|
14
|
+
from collections.abc import Callable
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
from datetime import UTC, datetime
|
|
17
|
+
from enum import StrEnum
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import cast
|
|
20
|
+
|
|
21
|
+
import yaml
|
|
22
|
+
from pydantic import ValidationError
|
|
23
|
+
|
|
24
|
+
from dromeus.application import check_environment, load_draft, require_task_matches
|
|
25
|
+
from dromeus.errors import ConfigError, DraftError, DromeusError, TaskError, one_line
|
|
26
|
+
from dromeus.manifests.canonical import canonical_hash
|
|
27
|
+
from dromeus.manifests.models import DraftRunSpec, Invitation
|
|
28
|
+
from dromeus.node import NodeConfig, NodeRole, load_node_config
|
|
29
|
+
from dromeus.training.task import TrainingTask
|
|
30
|
+
from dromeus.transport.axl import AXLBridgeConfig, AXLTransport
|
|
31
|
+
from dromeus.transport.interface import TransportError
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class Status(StrEnum):
|
|
35
|
+
OK = "ok"
|
|
36
|
+
WARN = "warn"
|
|
37
|
+
FAIL = "fail"
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
@dataclass(frozen=True, slots=True)
|
|
41
|
+
class Finding:
|
|
42
|
+
"""What one check observed, and what to do about it when it is not OK."""
|
|
43
|
+
|
|
44
|
+
check: str
|
|
45
|
+
status: Status
|
|
46
|
+
detail: str
|
|
47
|
+
fix: str | None = None
|
|
48
|
+
|
|
49
|
+
def __post_init__(self) -> None:
|
|
50
|
+
if self.status is not Status.OK and not self.fix:
|
|
51
|
+
raise ValueError(f"{self.check} finding is not OK but has no fix")
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
CONFIG_FIX = (
|
|
55
|
+
"Correct the node config fields named in the error: role, draft_path, "
|
|
56
|
+
"axl_bridge_url, run_root, invitation_path, bootstrap_uri."
|
|
57
|
+
)
|
|
58
|
+
DRAFT_FIX = (
|
|
59
|
+
"Point draft_path at the group's draft file, and install the Dromeus and "
|
|
60
|
+
"PyTorch versions it names."
|
|
61
|
+
)
|
|
62
|
+
TASK_FIX = (
|
|
63
|
+
"The draft was made from a different task definition; regenerate it with "
|
|
64
|
+
"make_draft from this task, or use the group's task code."
|
|
65
|
+
)
|
|
66
|
+
BRIDGE_FIX = (
|
|
67
|
+
"Start the AXL node on this machine and make sure its API port matches "
|
|
68
|
+
"axl_bridge_url in the node config."
|
|
69
|
+
)
|
|
70
|
+
NODE_KEY_FIX = (
|
|
71
|
+
"Restart the AXL node on this machine; the bridge did not report its public key."
|
|
72
|
+
)
|
|
73
|
+
PEERS_FIX = (
|
|
74
|
+
"Check the Peers list and bootstrap address in the AXL config, and that this "
|
|
75
|
+
"machine can reach them outbound."
|
|
76
|
+
)
|
|
77
|
+
INVITATION_MISSING_FIX = (
|
|
78
|
+
"Copy the initiator's invitation file to invitation_path; the node waits "
|
|
79
|
+
"invitation_timeout_seconds for it."
|
|
80
|
+
)
|
|
81
|
+
INVITATION_INVALID_FIX = (
|
|
82
|
+
"Copy the initiator's invitation file again, completely, to invitation_path."
|
|
83
|
+
)
|
|
84
|
+
INVITATION_BOOTSTRAP_FIX = (
|
|
85
|
+
"Set bootstrap_uri in the node config to the initiator's bootstrap URI, or ask "
|
|
86
|
+
"for an invitation made for this one."
|
|
87
|
+
)
|
|
88
|
+
INVITATION_DRAFT_FIX = (
|
|
89
|
+
"Use the same draft file as the initiator, or ask the initiator for an "
|
|
90
|
+
"invitation made from yours."
|
|
91
|
+
)
|
|
92
|
+
INVITATION_EXPIRED_FIX = (
|
|
93
|
+
"Ask the initiator for a fresh invitation, or check this machine's clock."
|
|
94
|
+
)
|
|
95
|
+
CLOCK_FAIL_FIX = (
|
|
96
|
+
"Enable NTP on this machine, for example `timedatectl set-ntp true` or chrony."
|
|
97
|
+
)
|
|
98
|
+
CLOCK_WARN_FIX = (
|
|
99
|
+
"Make sure NTP is enabled; invitation expiry compares clocks across machines."
|
|
100
|
+
)
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def system_clock_synchronized() -> bool | None:
|
|
104
|
+
"""Whether systemd reports the clock as NTP-synchronized.
|
|
105
|
+
|
|
106
|
+
Returns None on other platforms, without timedatectl, or on any error.
|
|
107
|
+
"""
|
|
108
|
+
if sys.platform != "linux" or shutil.which("timedatectl") is None:
|
|
109
|
+
return None
|
|
110
|
+
try:
|
|
111
|
+
result = subprocess.run(
|
|
112
|
+
["timedatectl", "show", "-p", "NTPSynchronized", "--value"],
|
|
113
|
+
capture_output=True,
|
|
114
|
+
text=True,
|
|
115
|
+
timeout=5,
|
|
116
|
+
check=False,
|
|
117
|
+
)
|
|
118
|
+
except (OSError, subprocess.SubprocessError):
|
|
119
|
+
return None
|
|
120
|
+
answer = result.stdout.strip()
|
|
121
|
+
if answer == "yes":
|
|
122
|
+
return True
|
|
123
|
+
if answer == "no":
|
|
124
|
+
return False
|
|
125
|
+
return None
|
|
126
|
+
|
|
127
|
+
|
|
128
|
+
async def diagnose(
|
|
129
|
+
config_path: Path,
|
|
130
|
+
*,
|
|
131
|
+
task: TrainingTask | None = None,
|
|
132
|
+
clock_synchronized: Callable[[], bool | None] = system_clock_synchronized,
|
|
133
|
+
) -> list[Finding]:
|
|
134
|
+
"""Run the checks in order and return one finding per check that ran.
|
|
135
|
+
|
|
136
|
+
A bad config stops everything. A bad draft skips the checks that need it,
|
|
137
|
+
and an unreachable bridge skips the node key and peer checks.
|
|
138
|
+
"""
|
|
139
|
+
try:
|
|
140
|
+
config = load_node_config(config_path)
|
|
141
|
+
except ConfigError as error:
|
|
142
|
+
return [Finding("config", Status.FAIL, one_line(str(error)), CONFIG_FIX)]
|
|
143
|
+
findings = [
|
|
144
|
+
Finding("config", Status.OK, f"{config_path} is valid; role {config.role}")
|
|
145
|
+
]
|
|
146
|
+
|
|
147
|
+
draft_finding, draft = _check_draft(config.draft_path)
|
|
148
|
+
findings.append(draft_finding)
|
|
149
|
+
if draft is not None and task is not None:
|
|
150
|
+
findings.append(_check_task(draft, task))
|
|
151
|
+
|
|
152
|
+
bridge_finding, topology = await _check_bridge(config.axl_bridge_url)
|
|
153
|
+
findings.append(bridge_finding)
|
|
154
|
+
if topology is not None:
|
|
155
|
+
findings.append(_check_node_key(topology))
|
|
156
|
+
findings.append(_check_peers(topology))
|
|
157
|
+
|
|
158
|
+
if draft is not None and config.role is NodeRole.PARTICIPANT:
|
|
159
|
+
findings.append(_check_invitation(config, draft))
|
|
160
|
+
|
|
161
|
+
findings.append(_check_clock(clock_synchronized))
|
|
162
|
+
return findings
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _check_draft(draft_path: Path) -> tuple[Finding, DraftRunSpec | None]:
|
|
166
|
+
try:
|
|
167
|
+
draft = load_draft(draft_path)
|
|
168
|
+
check_environment(draft)
|
|
169
|
+
except (DromeusError, ValueError, yaml.YAMLError) as error:
|
|
170
|
+
return Finding("draft", Status.FAIL, one_line(str(error)), DRAFT_FIX), None
|
|
171
|
+
detail = (
|
|
172
|
+
f"run {draft.run_id}: {draft.participant_count} participants, "
|
|
173
|
+
f"{draft.round_count} rounds"
|
|
174
|
+
)
|
|
175
|
+
return Finding("draft", Status.OK, detail), draft
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def _check_task(draft: DraftRunSpec, task: TrainingTask) -> Finding:
|
|
179
|
+
try:
|
|
180
|
+
require_task_matches(draft, task)
|
|
181
|
+
except DraftError as error:
|
|
182
|
+
return Finding("task", Status.FAIL, one_line(str(error)), TASK_FIX)
|
|
183
|
+
except TaskError as error:
|
|
184
|
+
return Finding(
|
|
185
|
+
"task",
|
|
186
|
+
Status.FAIL,
|
|
187
|
+
one_line(str(error)),
|
|
188
|
+
"Fix the task definition in the --task module so it can be described.",
|
|
189
|
+
)
|
|
190
|
+
return Finding("task", Status.OK, "draft matches this task")
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
async def _check_bridge(base_url: str) -> tuple[Finding, dict[str, object] | None]:
|
|
194
|
+
transport = AXLTransport(AXLBridgeConfig(base_url=base_url))
|
|
195
|
+
try:
|
|
196
|
+
topology = await transport.topology()
|
|
197
|
+
except (TransportError, OSError, ValueError) as error:
|
|
198
|
+
detail = f"cannot reach the AXL bridge at {base_url} ({one_line(str(error))})"
|
|
199
|
+
return Finding("axl bridge", Status.FAIL, detail, BRIDGE_FIX), None
|
|
200
|
+
return Finding("axl bridge", Status.OK, f"reachable at {base_url}"), topology
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _check_node_key(topology: dict[str, object]) -> Finding:
|
|
204
|
+
key = topology.get("our_public_key")
|
|
205
|
+
if isinstance(key, str) and key:
|
|
206
|
+
return Finding("node key", Status.OK, f"public key {key[:16]}...")
|
|
207
|
+
return Finding(
|
|
208
|
+
"node key",
|
|
209
|
+
Status.FAIL,
|
|
210
|
+
"the AXL bridge did not report this node's public key",
|
|
211
|
+
NODE_KEY_FIX,
|
|
212
|
+
)
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def _check_peers(topology: dict[str, object]) -> Finding:
|
|
216
|
+
peers = topology.get("peers")
|
|
217
|
+
entries: list[object] = cast(list[object], peers) if isinstance(peers, list) else []
|
|
218
|
+
# AXL may list several links to one peer (one per direction); count peers.
|
|
219
|
+
up = {key for entry in entries if (key := _up_peer_key(entry)) is not None}
|
|
220
|
+
if not up:
|
|
221
|
+
return Finding(
|
|
222
|
+
"peers", Status.FAIL, f"no peer link up ({len(entries)} listed)", PEERS_FIX
|
|
223
|
+
)
|
|
224
|
+
return Finding("peers", Status.OK, f"connected to {len(up)} peer(s)")
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
def _up_peer_key(entry: object) -> str | None:
|
|
228
|
+
if not isinstance(entry, dict):
|
|
229
|
+
return None
|
|
230
|
+
peer = cast(dict[str, object], entry)
|
|
231
|
+
key = peer.get("public_key")
|
|
232
|
+
return key if peer.get("up") is True and isinstance(key, str) and key else None
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _check_invitation(config: NodeConfig, draft: DraftRunSpec) -> Finding:
|
|
236
|
+
path = config.invitation_path
|
|
237
|
+
try:
|
|
238
|
+
data = path.read_bytes()
|
|
239
|
+
except FileNotFoundError:
|
|
240
|
+
return Finding(
|
|
241
|
+
"invitation",
|
|
242
|
+
Status.WARN,
|
|
243
|
+
f"not delivered yet: {path} does not exist",
|
|
244
|
+
INVITATION_MISSING_FIX,
|
|
245
|
+
)
|
|
246
|
+
except OSError as error:
|
|
247
|
+
return Finding(
|
|
248
|
+
"invitation",
|
|
249
|
+
Status.FAIL,
|
|
250
|
+
f"cannot read {path}: {one_line(str(error))}",
|
|
251
|
+
INVITATION_INVALID_FIX,
|
|
252
|
+
)
|
|
253
|
+
try:
|
|
254
|
+
invitation = Invitation.model_validate_json(data)
|
|
255
|
+
except ValidationError as error:
|
|
256
|
+
return Finding(
|
|
257
|
+
"invitation",
|
|
258
|
+
Status.FAIL,
|
|
259
|
+
f"{path} is incomplete or invalid: {one_line(str(error))}",
|
|
260
|
+
INVITATION_INVALID_FIX,
|
|
261
|
+
)
|
|
262
|
+
if invitation.bootstrap_uri != config.bootstrap_uri:
|
|
263
|
+
return Finding(
|
|
264
|
+
"invitation",
|
|
265
|
+
Status.FAIL,
|
|
266
|
+
f"invitation is for bootstrap {invitation.bootstrap_uri}; "
|
|
267
|
+
f"node config says {config.bootstrap_uri}",
|
|
268
|
+
INVITATION_BOOTSTRAP_FIX,
|
|
269
|
+
)
|
|
270
|
+
if invitation.draft_hash != canonical_hash(draft):
|
|
271
|
+
return Finding(
|
|
272
|
+
"invitation",
|
|
273
|
+
Status.FAIL,
|
|
274
|
+
"invitation was made from a different draft than draft_path",
|
|
275
|
+
INVITATION_DRAFT_FIX,
|
|
276
|
+
)
|
|
277
|
+
now = datetime.now(UTC)
|
|
278
|
+
expires_at = invitation.enrollment_expires_at
|
|
279
|
+
if expires_at is not None and expires_at <= now:
|
|
280
|
+
return Finding(
|
|
281
|
+
"invitation",
|
|
282
|
+
Status.FAIL,
|
|
283
|
+
f"invitation expired at {expires_at.isoformat(timespec='seconds')}; "
|
|
284
|
+
f"this machine's UTC time is {now.isoformat(timespec='seconds')}",
|
|
285
|
+
INVITATION_EXPIRED_FIX,
|
|
286
|
+
)
|
|
287
|
+
return Finding("invitation", Status.OK, f"invitation for run {invitation.run_id}")
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
def _check_clock(clock_synchronized: Callable[[], bool | None]) -> Finding:
|
|
291
|
+
synchronized = clock_synchronized()
|
|
292
|
+
if synchronized is True:
|
|
293
|
+
return Finding("clock", Status.OK, "system clock is synchronized")
|
|
294
|
+
if synchronized is False:
|
|
295
|
+
return Finding(
|
|
296
|
+
"clock", Status.FAIL, "system clock is not synchronized", CLOCK_FAIL_FIX
|
|
297
|
+
)
|
|
298
|
+
return Finding(
|
|
299
|
+
"clock",
|
|
300
|
+
Status.WARN,
|
|
301
|
+
"could not tell whether the clock is synchronized",
|
|
302
|
+
CLOCK_WARN_FIX,
|
|
303
|
+
)
|
dromeus/errors.py
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
"""Errors raised by Dromeus.
|
|
2
|
+
|
|
3
|
+
Catch `DromeusError` for any library failure. Input errors are also `ValueError`
|
|
4
|
+
and run failures are also `RuntimeError`, following Python convention.
|
|
5
|
+
"""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
class DromeusError(Exception):
|
|
11
|
+
"""Base class for every error Dromeus raises on purpose."""
|
|
12
|
+
|
|
13
|
+
|
|
14
|
+
class ConfigError(DromeusError, ValueError):
|
|
15
|
+
"""A node configuration or local setting is invalid."""
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class TaskError(DromeusError, ValueError):
|
|
19
|
+
"""A training task, its model, batches, loss or optimizer is not admissible."""
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class DraftError(DromeusError, ValueError):
|
|
23
|
+
"""A draft is malformed or does not match the local task or environment."""
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class PrivacyError(DromeusError, ValueError):
|
|
27
|
+
"""A privacy policy, ledger, budget or private execution check failed."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
class FormationError(DromeusError, RuntimeError):
|
|
31
|
+
"""The group could not form: invitation, membership or readiness failed."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class RunError(DromeusError, RuntimeError):
|
|
35
|
+
"""The run failed after formation: peers, transport, rounds or persistence."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def one_line(message: str) -> str:
|
|
39
|
+
"""Collapse a multi-line error message, such as pydantic's, to one line."""
|
|
40
|
+
lines = (line.strip() for line in message.splitlines())
|
|
41
|
+
return "; ".join(
|
|
42
|
+
line
|
|
43
|
+
for line in lines
|
|
44
|
+
if line and not line.startswith("For further information visit")
|
|
45
|
+
)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
__all__ = [
|
|
49
|
+
"ConfigError",
|
|
50
|
+
"DraftError",
|
|
51
|
+
"DromeusError",
|
|
52
|
+
"FormationError",
|
|
53
|
+
"PrivacyError",
|
|
54
|
+
"RunError",
|
|
55
|
+
"TaskError",
|
|
56
|
+
"one_line",
|
|
57
|
+
]
|