dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/cli.py ADDED
@@ -0,0 +1,341 @@
1
+ """The `dromeus` command: a thin shell over the library.
2
+
3
+ Each command parses its arguments, makes one library call and prints the
4
+ result. Errors print as one line and map to a stable exit code (`ExitCode`);
5
+ `--debug` adds the traceback.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import argparse
11
+ import asyncio
12
+ import importlib
13
+ import os
14
+ import sys
15
+ import traceback
16
+ from collections.abc import Callable, Sequence
17
+ from enum import IntEnum
18
+ from pathlib import Path
19
+ from typing import TYPE_CHECKING, cast
20
+
21
+ from dromeus.errors import (
22
+ ConfigError,
23
+ DraftError,
24
+ FormationError,
25
+ PrivacyError,
26
+ RunError,
27
+ TaskError,
28
+ one_line,
29
+ )
30
+
31
+ if TYPE_CHECKING:
32
+ from dromeus.application import LocalData
33
+ from dromeus.telemetry.progress import Progress
34
+ from dromeus.training.task import TrainingTask
35
+
36
+
37
+ class ExitCode(IntEnum):
38
+ """Process exit status of every `dromeus` command."""
39
+
40
+ OK = 0
41
+ ERROR = 1
42
+ """Unexpected failure: a bug, or an exception from the task's own code."""
43
+ USAGE = 2
44
+ """The command line is malformed."""
45
+ INVALID_INPUT = 3
46
+ """A node config, draft, task or privacy setting is wrong."""
47
+ FORMATION_FAILED = 4
48
+ """The group did not form: invitation, enrollment or readiness failed."""
49
+ RUN_FAILED = 5
50
+ """The run failed after formation."""
51
+ NOT_READY = 6
52
+ """`dromeus doctor` found at least one failing check."""
53
+ INTERRUPTED = 130
54
+ """Stopped with Ctrl-C."""
55
+
56
+
57
+ def main(argv: Sequence[str] | None = None) -> int:
58
+ parser = _parser()
59
+ args = parser.parse_args(argv)
60
+ command = cast(Callable[[argparse.Namespace], ExitCode], args.command)
61
+ debug = bool(getattr(args, "debug", False))
62
+ try:
63
+ return int(command(args))
64
+ except KeyboardInterrupt:
65
+ _print_error("interrupted")
66
+ return ExitCode.INTERRUPTED
67
+ except Exception as error:
68
+ if debug:
69
+ traceback.print_exc()
70
+ code = exit_code_for(error)
71
+ message = one_line(str(error)) or type(error).__name__
72
+ if code is ExitCode.ERROR:
73
+ message = f"unexpected {type(error).__name__}: {message}"
74
+ if not debug:
75
+ message += " (rerun with --debug for the traceback)"
76
+ _print_error(message)
77
+ return code
78
+
79
+
80
+ def exit_code_for(error: BaseException) -> ExitCode:
81
+ """Map a failure to its documented exit code."""
82
+ if isinstance(error, FormationError):
83
+ return ExitCode.FORMATION_FAILED
84
+ if isinstance(error, RunError):
85
+ return ExitCode.RUN_FAILED
86
+ if isinstance(error, ConfigError | DraftError | TaskError | PrivacyError):
87
+ return ExitCode.INVALID_INPUT
88
+ return ExitCode.ERROR
89
+
90
+
91
+ def _parser() -> argparse.ArgumentParser:
92
+ common = argparse.ArgumentParser(add_help=False)
93
+ common.add_argument(
94
+ "--debug",
95
+ action="store_true",
96
+ default=argparse.SUPPRESS, # so `dromeus --debug run` and `run --debug` agree
97
+ help="print full tracebacks on errors",
98
+ )
99
+ parser = argparse.ArgumentParser(
100
+ prog="dromeus",
101
+ description="Decentralized federated learning over Gensyn AXL.",
102
+ parents=[common],
103
+ )
104
+ parser.add_argument("--version", action="version", version=_version())
105
+ commands = parser.add_subparsers(title="commands", required=True)
106
+ task_help = (
107
+ "module:callable returning the TrainingTask, or (TrainingTask, LocalData); "
108
+ "the module is imported from the current directory"
109
+ )
110
+
111
+ run = commands.add_parser(
112
+ "run", parents=[common], help="join the group and train every round"
113
+ )
114
+ run.add_argument("--config", required=True, type=Path, help="node config YAML")
115
+ run.add_argument(
116
+ "--task",
117
+ required=True,
118
+ help="module:callable returning (TrainingTask, LocalData) for this node",
119
+ )
120
+ run.set_defaults(command=_run)
121
+
122
+ check = commands.add_parser(
123
+ "check",
124
+ parents=[common],
125
+ help="check a draft against the task and this machine, offline",
126
+ )
127
+ check.add_argument("--draft", required=True, type=Path, help="shared draft YAML")
128
+ check.add_argument("--task", required=True, help=task_help)
129
+ check.set_defaults(command=_check)
130
+
131
+ draft = commands.add_parser(
132
+ "draft",
133
+ parents=[common],
134
+ help="write the group's shared draft for a task",
135
+ description=(
136
+ "Write the shared draft with default NoLoCo, transport and no privacy. "
137
+ "For privacy, transport or divergence settings call make_draft from "
138
+ "Python."
139
+ ),
140
+ )
141
+ draft.add_argument("output", type=Path, help="draft YAML to write")
142
+ draft.add_argument("--task", required=True, help=task_help)
143
+ draft.add_argument("--run-id", required=True)
144
+ draft.add_argument("--participants", required=True, type=int)
145
+ draft.add_argument("--rounds", required=True, type=int)
146
+ draft.add_argument("--local-steps", required=True, type=int)
147
+ draft.add_argument(
148
+ "--axl-version", required=True, help="AXL revision every node runs"
149
+ )
150
+ draft.add_argument("--seed", type=int, default=0)
151
+ draft.set_defaults(command=_draft)
152
+
153
+ doctor = commands.add_parser(
154
+ "doctor",
155
+ parents=[common],
156
+ help="check that this machine is ready to run a node",
157
+ )
158
+ doctor.add_argument("--config", required=True, type=Path, help="node config YAML")
159
+ doctor.add_argument("--task", help=f"also check the draft against it; {task_help}")
160
+ doctor.set_defaults(command=_doctor)
161
+
162
+ watch = commands.add_parser(
163
+ "watch",
164
+ parents=[common],
165
+ help="follow a node's rounds, loss and consensus distance",
166
+ )
167
+ watch.add_argument(
168
+ "--run-root", required=True, type=Path, help="run_root from the node config"
169
+ )
170
+ watch.add_argument(
171
+ "--once", action="store_true", help="print what is logged so far and exit"
172
+ )
173
+ watch.set_defaults(command=_watch)
174
+ return parser
175
+
176
+
177
+ def _run(args: argparse.Namespace) -> ExitCode:
178
+ from dromeus.node import Node, load_node_config
179
+
180
+ config = load_node_config(cast(Path, args.config))
181
+ task, local = load_task(cast(str, args.task))
182
+ if local is None:
183
+ raise ConfigError(
184
+ f"{args.task} must return (TrainingTask, LocalData) to run a node"
185
+ )
186
+ _print_error(f"running; follow with: dromeus watch --run-root {config.run_root}")
187
+ result = asyncio.run(Node(config).run(task, local=local))
188
+ print(
189
+ f"run {result.run_id} complete: {result.committed_rounds} rounds committed, "
190
+ f"manifest {result.manifest_hash}"
191
+ )
192
+ return ExitCode.OK
193
+
194
+
195
+ def _check(args: argparse.Namespace) -> ExitCode:
196
+ from dromeus.application import check_draft, load_draft
197
+ from dromeus.manifests.canonical import canonical_hash
198
+
199
+ path = cast(Path, args.draft)
200
+ task, _ = load_task(cast(str, args.task))
201
+ check_draft(path, task)
202
+ draft = load_draft(path)
203
+ print(f"draft {path} matches this task and machine")
204
+ print(" model, task and loss definitions: match")
205
+ print(
206
+ f" Dromeus {draft.environment.dromeus_version}, "
207
+ f"PyTorch {draft.environment.pytorch_version}: installed"
208
+ )
209
+ print(f" draft hash: {canonical_hash(draft)}")
210
+ return ExitCode.OK
211
+
212
+
213
+ def _draft(args: argparse.Namespace) -> ExitCode:
214
+ from dromeus.application import make_draft
215
+
216
+ task, _ = load_task(cast(str, args.task))
217
+ path = cast(Path, args.output)
218
+ digest = make_draft(
219
+ task,
220
+ path,
221
+ run_id=cast(str, args.run_id),
222
+ participants=cast(int, args.participants),
223
+ rounds=cast(int, args.rounds),
224
+ local_steps=cast(int, args.local_steps),
225
+ axl_version=cast(str, args.axl_version),
226
+ seed=cast(int, args.seed),
227
+ )
228
+ print(f"wrote {path}; draft hash {digest}")
229
+ return ExitCode.OK
230
+
231
+
232
+ def _doctor(args: argparse.Namespace) -> ExitCode:
233
+ from dromeus.diagnostics import Status, diagnose
234
+
235
+ reference = cast(str | None, args.task)
236
+ task = None if reference is None else load_task(reference)[0]
237
+ findings = asyncio.run(diagnose(cast(Path, args.config), task=task))
238
+ width = max(len(finding.check) for finding in findings)
239
+ for finding in findings:
240
+ print(f"{finding.status.value:<5} {finding.check:<{width}} {finding.detail}")
241
+ if finding.fix is not None:
242
+ print(f"{'':<5} {'':<{width}} fix: {finding.fix}")
243
+ if any(finding.status is Status.FAIL for finding in findings):
244
+ return ExitCode.NOT_READY
245
+ return ExitCode.OK
246
+
247
+
248
+ def _watch(args: argparse.Namespace) -> ExitCode:
249
+ from dromeus.telemetry.events import event_log_path
250
+ from dromeus.telemetry.progress import RunFailed, read_progress
251
+
252
+ path = event_log_path(cast(Path, args.run_root))
253
+ code = ExitCode.OK
254
+ for item in read_progress(path, follow=not args.once):
255
+ print(describe_progress(item), flush=True)
256
+ if isinstance(item, RunFailed):
257
+ code = ExitCode.RUN_FAILED
258
+ return code
259
+
260
+
261
+ def describe_progress(item: Progress) -> str:
262
+ """One terminal line for one progress item."""
263
+ from dromeus.telemetry.progress import (
264
+ ConsensusMeasured,
265
+ NodeReady,
266
+ NodeStarted,
267
+ RoundCommitted,
268
+ RunCompleted,
269
+ )
270
+
271
+ match item:
272
+ case NodeStarted():
273
+ return f"node started: run {item.run_id} as {item.role}"
274
+ case NodeReady():
275
+ return f"group formed: manifest {item.manifest_hash[:16]}"
276
+ case RoundCommitted():
277
+ loss = "-" if item.local_loss is None else f"{item.local_loss:.4g}"
278
+ metrics = "".join(
279
+ f" {name} {value:.4g}" for name, value in sorted(item.metrics.items())
280
+ )
281
+ return (
282
+ f"round {item.round_id:>5} loss {loss}{metrics} "
283
+ f"{item.round_seconds:.1f}s peer {item.peer_id[:8]}"
284
+ )
285
+ case ConsensusMeasured():
286
+ return (
287
+ f"round {item.round_id:>5} consensus distance "
288
+ f"{item.normalized_rms:.4g}"
289
+ )
290
+ case RunCompleted():
291
+ return f"run complete: {item.committed_rounds} rounds committed"
292
+ case _:
293
+ where = "" if item.round_id is None else f" in round {item.round_id}"
294
+ return f"run failed{where}: {item.error}"
295
+
296
+
297
+ def load_task(reference: str) -> tuple[TrainingTask, LocalData | None]:
298
+ """Call a `module:callable` that returns a task, or a task and local data.
299
+
300
+ The current directory is importable, as with `python -m`.
301
+ """
302
+ from dromeus.application import LocalData
303
+ from dromeus.training.task import TrainingTask
304
+
305
+ module_name, separator, name = reference.partition(":")
306
+ if not separator or not module_name or not name:
307
+ raise ConfigError(f"task must be given as module:callable, got {reference!r}")
308
+ if os.getcwd() not in sys.path:
309
+ sys.path.insert(0, os.getcwd())
310
+ try:
311
+ module = importlib.import_module(module_name)
312
+ except ModuleNotFoundError as error:
313
+ if error.name is not None and module_name.startswith(error.name):
314
+ raise ConfigError(
315
+ f"cannot import {module_name!r} from {os.getcwd()}: {error}"
316
+ ) from error
317
+ raise
318
+ setup = getattr(module, name, None)
319
+ if not callable(setup):
320
+ raise ConfigError(f"{reference} is not a callable in {module_name}")
321
+ value: object = setup()
322
+ if isinstance(value, TrainingTask):
323
+ return value, None
324
+ match value:
325
+ case (TrainingTask() as task, LocalData() as local):
326
+ return task, local
327
+ case _:
328
+ raise ConfigError(
329
+ f"{reference} must return a TrainingTask or "
330
+ f"(TrainingTask, LocalData), not {type(value).__name__}"
331
+ )
332
+
333
+
334
+ def _print_error(message: str) -> None:
335
+ print(f"dromeus: {message}", file=sys.stderr, flush=True)
336
+
337
+
338
+ def _version() -> str:
339
+ from importlib.metadata import version
340
+
341
+ return f"dromeus {version('dromeus')}"
dromeus/diagnostics.py ADDED
@@ -0,0 +1,303 @@
1
+ """Check whether this machine is ready to run a node, without training.
2
+
3
+ `dromeus doctor` prints the findings from `diagnose`. Each check looks at one
4
+ thing (the node config, the draft, the AXL bridge, peer links, the invitation,
5
+ the clock) and records what it saw. A finding that is not OK carries one line
6
+ telling the user what to do. Nothing here joins a group or trains a model.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import shutil
12
+ import subprocess
13
+ import sys
14
+ from collections.abc import Callable
15
+ from dataclasses import dataclass
16
+ from datetime import UTC, datetime
17
+ from enum import StrEnum
18
+ from pathlib import Path
19
+ from typing import cast
20
+
21
+ import yaml
22
+ from pydantic import ValidationError
23
+
24
+ from dromeus.application import check_environment, load_draft, require_task_matches
25
+ from dromeus.errors import ConfigError, DraftError, DromeusError, TaskError, one_line
26
+ from dromeus.manifests.canonical import canonical_hash
27
+ from dromeus.manifests.models import DraftRunSpec, Invitation
28
+ from dromeus.node import NodeConfig, NodeRole, load_node_config
29
+ from dromeus.training.task import TrainingTask
30
+ from dromeus.transport.axl import AXLBridgeConfig, AXLTransport
31
+ from dromeus.transport.interface import TransportError
32
+
33
+
34
+ class Status(StrEnum):
35
+ OK = "ok"
36
+ WARN = "warn"
37
+ FAIL = "fail"
38
+
39
+
40
+ @dataclass(frozen=True, slots=True)
41
+ class Finding:
42
+ """What one check observed, and what to do about it when it is not OK."""
43
+
44
+ check: str
45
+ status: Status
46
+ detail: str
47
+ fix: str | None = None
48
+
49
+ def __post_init__(self) -> None:
50
+ if self.status is not Status.OK and not self.fix:
51
+ raise ValueError(f"{self.check} finding is not OK but has no fix")
52
+
53
+
54
+ CONFIG_FIX = (
55
+ "Correct the node config fields named in the error: role, draft_path, "
56
+ "axl_bridge_url, run_root, invitation_path, bootstrap_uri."
57
+ )
58
+ DRAFT_FIX = (
59
+ "Point draft_path at the group's draft file, and install the Dromeus and "
60
+ "PyTorch versions it names."
61
+ )
62
+ TASK_FIX = (
63
+ "The draft was made from a different task definition; regenerate it with "
64
+ "make_draft from this task, or use the group's task code."
65
+ )
66
+ BRIDGE_FIX = (
67
+ "Start the AXL node on this machine and make sure its API port matches "
68
+ "axl_bridge_url in the node config."
69
+ )
70
+ NODE_KEY_FIX = (
71
+ "Restart the AXL node on this machine; the bridge did not report its public key."
72
+ )
73
+ PEERS_FIX = (
74
+ "Check the Peers list and bootstrap address in the AXL config, and that this "
75
+ "machine can reach them outbound."
76
+ )
77
+ INVITATION_MISSING_FIX = (
78
+ "Copy the initiator's invitation file to invitation_path; the node waits "
79
+ "invitation_timeout_seconds for it."
80
+ )
81
+ INVITATION_INVALID_FIX = (
82
+ "Copy the initiator's invitation file again, completely, to invitation_path."
83
+ )
84
+ INVITATION_BOOTSTRAP_FIX = (
85
+ "Set bootstrap_uri in the node config to the initiator's bootstrap URI, or ask "
86
+ "for an invitation made for this one."
87
+ )
88
+ INVITATION_DRAFT_FIX = (
89
+ "Use the same draft file as the initiator, or ask the initiator for an "
90
+ "invitation made from yours."
91
+ )
92
+ INVITATION_EXPIRED_FIX = (
93
+ "Ask the initiator for a fresh invitation, or check this machine's clock."
94
+ )
95
+ CLOCK_FAIL_FIX = (
96
+ "Enable NTP on this machine, for example `timedatectl set-ntp true` or chrony."
97
+ )
98
+ CLOCK_WARN_FIX = (
99
+ "Make sure NTP is enabled; invitation expiry compares clocks across machines."
100
+ )
101
+
102
+
103
+ def system_clock_synchronized() -> bool | None:
104
+ """Whether systemd reports the clock as NTP-synchronized.
105
+
106
+ Returns None on other platforms, without timedatectl, or on any error.
107
+ """
108
+ if sys.platform != "linux" or shutil.which("timedatectl") is None:
109
+ return None
110
+ try:
111
+ result = subprocess.run(
112
+ ["timedatectl", "show", "-p", "NTPSynchronized", "--value"],
113
+ capture_output=True,
114
+ text=True,
115
+ timeout=5,
116
+ check=False,
117
+ )
118
+ except (OSError, subprocess.SubprocessError):
119
+ return None
120
+ answer = result.stdout.strip()
121
+ if answer == "yes":
122
+ return True
123
+ if answer == "no":
124
+ return False
125
+ return None
126
+
127
+
128
+ async def diagnose(
129
+ config_path: Path,
130
+ *,
131
+ task: TrainingTask | None = None,
132
+ clock_synchronized: Callable[[], bool | None] = system_clock_synchronized,
133
+ ) -> list[Finding]:
134
+ """Run the checks in order and return one finding per check that ran.
135
+
136
+ A bad config stops everything. A bad draft skips the checks that need it,
137
+ and an unreachable bridge skips the node key and peer checks.
138
+ """
139
+ try:
140
+ config = load_node_config(config_path)
141
+ except ConfigError as error:
142
+ return [Finding("config", Status.FAIL, one_line(str(error)), CONFIG_FIX)]
143
+ findings = [
144
+ Finding("config", Status.OK, f"{config_path} is valid; role {config.role}")
145
+ ]
146
+
147
+ draft_finding, draft = _check_draft(config.draft_path)
148
+ findings.append(draft_finding)
149
+ if draft is not None and task is not None:
150
+ findings.append(_check_task(draft, task))
151
+
152
+ bridge_finding, topology = await _check_bridge(config.axl_bridge_url)
153
+ findings.append(bridge_finding)
154
+ if topology is not None:
155
+ findings.append(_check_node_key(topology))
156
+ findings.append(_check_peers(topology))
157
+
158
+ if draft is not None and config.role is NodeRole.PARTICIPANT:
159
+ findings.append(_check_invitation(config, draft))
160
+
161
+ findings.append(_check_clock(clock_synchronized))
162
+ return findings
163
+
164
+
165
+ def _check_draft(draft_path: Path) -> tuple[Finding, DraftRunSpec | None]:
166
+ try:
167
+ draft = load_draft(draft_path)
168
+ check_environment(draft)
169
+ except (DromeusError, ValueError, yaml.YAMLError) as error:
170
+ return Finding("draft", Status.FAIL, one_line(str(error)), DRAFT_FIX), None
171
+ detail = (
172
+ f"run {draft.run_id}: {draft.participant_count} participants, "
173
+ f"{draft.round_count} rounds"
174
+ )
175
+ return Finding("draft", Status.OK, detail), draft
176
+
177
+
178
+ def _check_task(draft: DraftRunSpec, task: TrainingTask) -> Finding:
179
+ try:
180
+ require_task_matches(draft, task)
181
+ except DraftError as error:
182
+ return Finding("task", Status.FAIL, one_line(str(error)), TASK_FIX)
183
+ except TaskError as error:
184
+ return Finding(
185
+ "task",
186
+ Status.FAIL,
187
+ one_line(str(error)),
188
+ "Fix the task definition in the --task module so it can be described.",
189
+ )
190
+ return Finding("task", Status.OK, "draft matches this task")
191
+
192
+
193
+ async def _check_bridge(base_url: str) -> tuple[Finding, dict[str, object] | None]:
194
+ transport = AXLTransport(AXLBridgeConfig(base_url=base_url))
195
+ try:
196
+ topology = await transport.topology()
197
+ except (TransportError, OSError, ValueError) as error:
198
+ detail = f"cannot reach the AXL bridge at {base_url} ({one_line(str(error))})"
199
+ return Finding("axl bridge", Status.FAIL, detail, BRIDGE_FIX), None
200
+ return Finding("axl bridge", Status.OK, f"reachable at {base_url}"), topology
201
+
202
+
203
+ def _check_node_key(topology: dict[str, object]) -> Finding:
204
+ key = topology.get("our_public_key")
205
+ if isinstance(key, str) and key:
206
+ return Finding("node key", Status.OK, f"public key {key[:16]}...")
207
+ return Finding(
208
+ "node key",
209
+ Status.FAIL,
210
+ "the AXL bridge did not report this node's public key",
211
+ NODE_KEY_FIX,
212
+ )
213
+
214
+
215
+ def _check_peers(topology: dict[str, object]) -> Finding:
216
+ peers = topology.get("peers")
217
+ entries: list[object] = cast(list[object], peers) if isinstance(peers, list) else []
218
+ # AXL may list several links to one peer (one per direction); count peers.
219
+ up = {key for entry in entries if (key := _up_peer_key(entry)) is not None}
220
+ if not up:
221
+ return Finding(
222
+ "peers", Status.FAIL, f"no peer link up ({len(entries)} listed)", PEERS_FIX
223
+ )
224
+ return Finding("peers", Status.OK, f"connected to {len(up)} peer(s)")
225
+
226
+
227
+ def _up_peer_key(entry: object) -> str | None:
228
+ if not isinstance(entry, dict):
229
+ return None
230
+ peer = cast(dict[str, object], entry)
231
+ key = peer.get("public_key")
232
+ return key if peer.get("up") is True and isinstance(key, str) and key else None
233
+
234
+
235
+ def _check_invitation(config: NodeConfig, draft: DraftRunSpec) -> Finding:
236
+ path = config.invitation_path
237
+ try:
238
+ data = path.read_bytes()
239
+ except FileNotFoundError:
240
+ return Finding(
241
+ "invitation",
242
+ Status.WARN,
243
+ f"not delivered yet: {path} does not exist",
244
+ INVITATION_MISSING_FIX,
245
+ )
246
+ except OSError as error:
247
+ return Finding(
248
+ "invitation",
249
+ Status.FAIL,
250
+ f"cannot read {path}: {one_line(str(error))}",
251
+ INVITATION_INVALID_FIX,
252
+ )
253
+ try:
254
+ invitation = Invitation.model_validate_json(data)
255
+ except ValidationError as error:
256
+ return Finding(
257
+ "invitation",
258
+ Status.FAIL,
259
+ f"{path} is incomplete or invalid: {one_line(str(error))}",
260
+ INVITATION_INVALID_FIX,
261
+ )
262
+ if invitation.bootstrap_uri != config.bootstrap_uri:
263
+ return Finding(
264
+ "invitation",
265
+ Status.FAIL,
266
+ f"invitation is for bootstrap {invitation.bootstrap_uri}; "
267
+ f"node config says {config.bootstrap_uri}",
268
+ INVITATION_BOOTSTRAP_FIX,
269
+ )
270
+ if invitation.draft_hash != canonical_hash(draft):
271
+ return Finding(
272
+ "invitation",
273
+ Status.FAIL,
274
+ "invitation was made from a different draft than draft_path",
275
+ INVITATION_DRAFT_FIX,
276
+ )
277
+ now = datetime.now(UTC)
278
+ expires_at = invitation.enrollment_expires_at
279
+ if expires_at is not None and expires_at <= now:
280
+ return Finding(
281
+ "invitation",
282
+ Status.FAIL,
283
+ f"invitation expired at {expires_at.isoformat(timespec='seconds')}; "
284
+ f"this machine's UTC time is {now.isoformat(timespec='seconds')}",
285
+ INVITATION_EXPIRED_FIX,
286
+ )
287
+ return Finding("invitation", Status.OK, f"invitation for run {invitation.run_id}")
288
+
289
+
290
+ def _check_clock(clock_synchronized: Callable[[], bool | None]) -> Finding:
291
+ synchronized = clock_synchronized()
292
+ if synchronized is True:
293
+ return Finding("clock", Status.OK, "system clock is synchronized")
294
+ if synchronized is False:
295
+ return Finding(
296
+ "clock", Status.FAIL, "system clock is not synchronized", CLOCK_FAIL_FIX
297
+ )
298
+ return Finding(
299
+ "clock",
300
+ Status.WARN,
301
+ "could not tell whether the clock is synchronized",
302
+ CLOCK_WARN_FIX,
303
+ )
dromeus/errors.py ADDED
@@ -0,0 +1,57 @@
1
+ """Errors raised by Dromeus.
2
+
3
+ Catch `DromeusError` for any library failure. Input errors are also `ValueError`
4
+ and run failures are also `RuntimeError`, following Python convention.
5
+ """
6
+
7
+ from __future__ import annotations
8
+
9
+
10
+ class DromeusError(Exception):
11
+ """Base class for every error Dromeus raises on purpose."""
12
+
13
+
14
+ class ConfigError(DromeusError, ValueError):
15
+ """A node configuration or local setting is invalid."""
16
+
17
+
18
+ class TaskError(DromeusError, ValueError):
19
+ """A training task, its model, batches, loss or optimizer is not admissible."""
20
+
21
+
22
+ class DraftError(DromeusError, ValueError):
23
+ """A draft is malformed or does not match the local task or environment."""
24
+
25
+
26
+ class PrivacyError(DromeusError, ValueError):
27
+ """A privacy policy, ledger, budget or private execution check failed."""
28
+
29
+
30
+ class FormationError(DromeusError, RuntimeError):
31
+ """The group could not form: invitation, membership or readiness failed."""
32
+
33
+
34
+ class RunError(DromeusError, RuntimeError):
35
+ """The run failed after formation: peers, transport, rounds or persistence."""
36
+
37
+
38
+ def one_line(message: str) -> str:
39
+ """Collapse a multi-line error message, such as pydantic's, to one line."""
40
+ lines = (line.strip() for line in message.splitlines())
41
+ return "; ".join(
42
+ line
43
+ for line in lines
44
+ if line and not line.startswith("For further information visit")
45
+ )
46
+
47
+
48
+ __all__ = [
49
+ "ConfigError",
50
+ "DraftError",
51
+ "DromeusError",
52
+ "FormationError",
53
+ "PrivacyError",
54
+ "RunError",
55
+ "TaskError",
56
+ "one_line",
57
+ ]