dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,255 @@
1
+ """Progress items read from one node's live JSONL event log.
2
+
3
+ `read_progress` follows the log a node writes under its run root and turns the
4
+ few events a live terminal view needs (start, readiness, committed rounds,
5
+ consensus distance, completion or failure) into typed items. Every other event
6
+ is ignored. The CLI only formats these items; the parsing rules live here.
7
+ """
8
+
9
+ from __future__ import annotations
10
+
11
+ import json
12
+ import time
13
+ from collections.abc import Iterator, Mapping
14
+ from dataclasses import dataclass
15
+ from io import BufferedReader
16
+ from pathlib import Path
17
+ from typing import cast
18
+
19
+ from dromeus.errors import ConfigError
20
+
21
+ _CHUNK_BYTES = 64 * 1024
22
+ _ROUND_SECONDS_FIELDS = (
23
+ "local_compute_seconds",
24
+ "peer_wait_seconds",
25
+ "transfer_seconds",
26
+ "mixing_seconds",
27
+ "evaluation_seconds",
28
+ )
29
+
30
+
31
+ @dataclass(frozen=True, slots=True)
32
+ class NodeStarted:
33
+ run_id: str
34
+ role: str
35
+
36
+
37
+ @dataclass(frozen=True, slots=True)
38
+ class NodeReady:
39
+ run_id: str
40
+ manifest_hash: str
41
+
42
+
43
+ @dataclass(frozen=True, slots=True)
44
+ class RoundCommitted:
45
+ round_id: int
46
+ peer_id: str
47
+ local_loss: float | None
48
+ metrics: dict[str, float]
49
+ round_seconds: float
50
+ encoded_bytes: int | None
51
+
52
+
53
+ @dataclass(frozen=True, slots=True)
54
+ class ConsensusMeasured:
55
+ round_id: int
56
+ normalized_rms: float
57
+
58
+
59
+ @dataclass(frozen=True, slots=True)
60
+ class RunCompleted:
61
+ committed_rounds: int
62
+
63
+
64
+ @dataclass(frozen=True, slots=True)
65
+ class RunFailed:
66
+ error: str
67
+ round_id: int | None
68
+
69
+
70
+ type Progress = (
71
+ NodeStarted
72
+ | NodeReady
73
+ | RoundCommitted
74
+ | ConsensusMeasured
75
+ | RunCompleted
76
+ | RunFailed
77
+ )
78
+
79
+
80
+ class _Malformed(Exception):
81
+ """A record is missing or mistypes a field its progress item needs."""
82
+
83
+
84
+ def parse_progress(record: Mapping[str, object]) -> Progress | None:
85
+ """Map one decoded log record to a progress item, or None if it is not one."""
86
+ event = record.get("event")
87
+ try:
88
+ if event == "node_start":
89
+ return NodeStarted(
90
+ run_id=_str(record, "run_id"),
91
+ role=_str(record, "role"),
92
+ )
93
+ if event == "node_ready":
94
+ return NodeReady(
95
+ run_id=_str(record, "run_id"),
96
+ manifest_hash=_str(record, "manifest_hash"),
97
+ )
98
+ if event == "round_metrics":
99
+ return _round_committed(record)
100
+ if event == "consensus_observation":
101
+ return ConsensusMeasured(
102
+ round_id=_int(record, "round_id"),
103
+ normalized_rms=_float(record, "normalized_rms"),
104
+ )
105
+ if event == "node_complete":
106
+ return RunCompleted(committed_rounds=_int(record, "committed_rounds"))
107
+ if event in ("run_failed", "formation_failed"):
108
+ return RunFailed(
109
+ error=_str(record, "error"),
110
+ round_id=_optional_int(record, "round_id"),
111
+ )
112
+ except _Malformed:
113
+ return None
114
+ return None
115
+
116
+
117
+ def read_progress(
118
+ path: Path,
119
+ *,
120
+ follow: bool,
121
+ poll_seconds: float = 0.5,
122
+ ) -> Iterator[Progress]:
123
+ """Yield progress items from a node's JSONL log, in log order.
124
+
125
+ With `follow=False` the existing file is read to its end. With `follow=True`
126
+ the file may appear later and is polled like `tail -f`. The iterator stops
127
+ right after a terminal item (`RunCompleted` or `RunFailed`). A missing file
128
+ raises `ConfigError` immediately when `follow=False`.
129
+ """
130
+ if not follow and not path.is_file():
131
+ raise _missing_log(path)
132
+ return _stream(path, follow=follow, poll_seconds=poll_seconds)
133
+
134
+
135
+ def _stream(path: Path, *, follow: bool, poll_seconds: float) -> Iterator[Progress]:
136
+ with _open_log(path, follow=follow, poll_seconds=poll_seconds) as handle:
137
+ pending = b""
138
+ while True:
139
+ chunk = handle.read(_CHUNK_BYTES)
140
+ if not chunk:
141
+ if not follow:
142
+ return
143
+ time.sleep(poll_seconds)
144
+ continue
145
+ # Newline bytes never occur inside a UTF-8 sequence, so each complete
146
+ # line decodes on its own even when a chunk ends mid-character.
147
+ *lines, pending = (pending + chunk).split(b"\n")
148
+ for line in lines:
149
+ item = _parse_line(line)
150
+ if item is None:
151
+ continue
152
+ yield item
153
+ if isinstance(item, RunCompleted | RunFailed):
154
+ return
155
+
156
+
157
+ def _open_log(path: Path, *, follow: bool, poll_seconds: float) -> BufferedReader:
158
+ while True:
159
+ try:
160
+ return path.open("rb")
161
+ except FileNotFoundError:
162
+ if not follow:
163
+ raise _missing_log(path) from None
164
+ time.sleep(poll_seconds)
165
+
166
+
167
+ def _parse_line(line: bytes) -> Progress | None:
168
+ try:
169
+ value = cast(object, json.loads(line.decode("utf-8")))
170
+ except (ValueError, RecursionError):
171
+ return None
172
+ if not isinstance(value, dict):
173
+ return None
174
+ return parse_progress(cast(dict[str, object], value))
175
+
176
+
177
+ def _missing_log(path: Path) -> ConfigError:
178
+ return ConfigError(
179
+ f"node event log not found: {path}; check that the run root is right "
180
+ "and that the node has started"
181
+ )
182
+
183
+
184
+ def _round_committed(record: Mapping[str, object]) -> RoundCommitted:
185
+ round_seconds = 0.0
186
+ for key in _ROUND_SECONDS_FIELDS:
187
+ round_seconds += _float(record, key)
188
+ return RoundCommitted(
189
+ round_id=_int(record, "round_id"),
190
+ peer_id=_str(record, "peer_id"),
191
+ local_loss=_optional_float(record, "local_loss"),
192
+ metrics=_metrics(record, "evaluation_metrics"),
193
+ round_seconds=round_seconds,
194
+ encoded_bytes=_optional_int(record, "encoded_artifact_bytes"),
195
+ )
196
+
197
+
198
+ def _str(record: Mapping[str, object], key: str) -> str:
199
+ value = record.get(key)
200
+ if not isinstance(value, str):
201
+ raise _Malformed(key)
202
+ return value
203
+
204
+
205
+ def _int(record: Mapping[str, object], key: str) -> int:
206
+ value = record.get(key)
207
+ if isinstance(value, bool) or not isinstance(value, int):
208
+ raise _Malformed(key)
209
+ return value
210
+
211
+
212
+ def _float(record: Mapping[str, object], key: str) -> float:
213
+ value = record.get(key)
214
+ if isinstance(value, bool) or not isinstance(value, int | float):
215
+ raise _Malformed(key)
216
+ return float(value)
217
+
218
+
219
+ def _optional_int(record: Mapping[str, object], key: str) -> int | None:
220
+ if record.get(key) is None:
221
+ return None
222
+ return _int(record, key)
223
+
224
+
225
+ def _optional_float(record: Mapping[str, object], key: str) -> float | None:
226
+ if record.get(key) is None:
227
+ return None
228
+ return _float(record, key)
229
+
230
+
231
+ def _metrics(record: Mapping[str, object], key: str) -> dict[str, float]:
232
+ value = record.get(key)
233
+ if not isinstance(value, dict):
234
+ raise _Malformed(key)
235
+ metrics: dict[str, float] = {}
236
+ for name, number in cast(dict[object, object], value).items():
237
+ if not isinstance(name, str) or isinstance(number, bool):
238
+ raise _Malformed(key)
239
+ if not isinstance(number, int | float):
240
+ raise _Malformed(key)
241
+ metrics[name] = float(number)
242
+ return metrics
243
+
244
+
245
+ __all__ = [
246
+ "ConsensusMeasured",
247
+ "NodeReady",
248
+ "NodeStarted",
249
+ "Progress",
250
+ "RoundCommitted",
251
+ "RunCompleted",
252
+ "RunFailed",
253
+ "parse_progress",
254
+ "read_progress",
255
+ ]
@@ -0,0 +1 @@
1
+ """Local model training."""
@@ -0,0 +1,60 @@
1
+ """Trainer interface."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Mapping
7
+ from dataclasses import dataclass
8
+ from types import MappingProxyType
9
+ from typing import Protocol, runtime_checkable
10
+
11
+ import numpy as np
12
+
13
+ from dromeus.errors import TaskError
14
+
15
+
16
+ @dataclass(frozen=True, slots=True)
17
+ class EvaluationResult:
18
+ """Named finite metrics with application-defined meaning and units."""
19
+
20
+ metrics: Mapping[str, float]
21
+
22
+ def __post_init__(self) -> None:
23
+ if any(
24
+ not name.strip() or not math.isfinite(value)
25
+ for name, value in self.metrics.items()
26
+ ):
27
+ raise TaskError("evaluation metrics need nonblank names and finite values")
28
+ object.__setattr__(self, "metrics", MappingProxyType(dict(self.metrics)))
29
+
30
+
31
+ class WeightTrainer(Protocol):
32
+ """Owns model, optimizer, and local training state."""
33
+
34
+ def train_local_steps(self, step_count: int) -> None: ...
35
+
36
+ def weights(self) -> dict[str, np.ndarray]:
37
+ """Return a copy of current model weights."""
38
+ ...
39
+
40
+ def load_weights(self, weights: dict[str, np.ndarray]) -> None:
41
+ """Replace current model weights."""
42
+ ...
43
+
44
+ @property
45
+ def local_loss(self) -> float | None:
46
+ """Return the latest local loss, or none before training."""
47
+ ...
48
+
49
+ def evaluate(self) -> EvaluationResult | None:
50
+ """Return named evaluation metrics, or none when evaluation is disabled."""
51
+ ...
52
+
53
+
54
+ @runtime_checkable
55
+ class CheckpointTrainer(WeightTrainer, Protocol):
56
+ """Trainer whose complete durable state is tensor-serializable."""
57
+
58
+ def checkpoint_tensors(self) -> dict[str, np.ndarray]: ...
59
+
60
+ def load_checkpoint_tensors(self, state: dict[str, np.ndarray]) -> None: ...
@@ -0,0 +1,111 @@
1
+ """Shared exchangeable model-state and tensor-schema utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from typing import Literal
6
+
7
+ import torch
8
+ from torch import Tensor, nn
9
+
10
+ from dromeus.errors import TaskError
11
+ from dromeus.manifests.models import Tensor as TensorSpec
12
+ from dromeus.manifests.models import TensorSchema
13
+
14
+ _TORCH_TO_SCHEMA_DTYPE: dict[
15
+ torch.dtype,
16
+ Literal["float16", "float32", "float64"],
17
+ ] = {
18
+ torch.float16: "float16",
19
+ torch.float32: "float32",
20
+ torch.float64: "float64",
21
+ }
22
+
23
+
24
+ def floating_model_state(
25
+ model: nn.Module, names: tuple[str, ...] | None = None
26
+ ) -> dict[str, Tensor]:
27
+ """Return exchangeable floating parameters and model buffers."""
28
+ if names is None:
29
+ return {
30
+ name: value
31
+ for name, value in model.state_dict().items()
32
+ if value.is_floating_point()
33
+ }
34
+ # The exchange inventory is registered parameters, never an application
35
+ # override's serialization output or mutable auxiliary model state.
36
+ state = dict(model.named_parameters(remove_duplicate=False))
37
+ if not names or len(set(names)) != len(names) or not set(names) <= state.keys():
38
+ raise TaskError("invalid declared exchange state")
39
+ return {name: state[name] for name in names}
40
+
41
+
42
+ def tensor_schema_for_model(
43
+ model: nn.Module, names: tuple[str, ...] | None = None
44
+ ) -> TensorSchema:
45
+ """Derive the canonical wire schema from exchangeable model state."""
46
+ tensors: list[TensorSpec] = []
47
+ for name, value in floating_model_state(model, names).items():
48
+ dtype = _TORCH_TO_SCHEMA_DTYPE.get(value.dtype)
49
+ if dtype is None:
50
+ raise TaskError(f"unsupported model state dtype: {value.dtype}")
51
+ tensors.append(
52
+ TensorSpec(
53
+ name=name,
54
+ dtype=dtype,
55
+ shape=tuple(value.shape),
56
+ )
57
+ )
58
+ return TensorSchema(tensors=tuple(tensors))
59
+
60
+
61
+ type _StateLayout = tuple[int, int, torch.dtype, tuple[int, ...], tuple[int, ...]]
62
+
63
+
64
+ def model_state_alias_groups(model: nn.Module) -> tuple[tuple[str, ...], ...]:
65
+ """Identify exact aliases and reject unsupported overlapping write layouts.
66
+
67
+ Byte spans are conservative for strided views: ambiguous overlaps reject.
68
+ Ordinary transposes, exact ties, and disjoint spans remain supported.
69
+ """
70
+ devices: dict[str, list[tuple[_StateLayout, str]]] = {}
71
+ for name, value in model.state_dict().items():
72
+ if value.numel() == 0:
73
+ continue
74
+ shape, strides = tuple(value.shape), tuple(value.stride())
75
+ span = 1
76
+ for stride, size in sorted(zip(strides, shape, strict=True)):
77
+ if size <= 1:
78
+ continue
79
+ if stride < span:
80
+ raise TaskError(f"unsupported overlapping model state: {name}")
81
+ span += (size - 1) * stride
82
+ start = (
83
+ value.untyped_storage().data_ptr()
84
+ + value.storage_offset() * value.element_size()
85
+ )
86
+ end = start + span * value.element_size()
87
+ layout = (start, end, value.dtype, shape, strides)
88
+ devices.setdefault(str(value.device), []).append((layout, name))
89
+ aliases: list[tuple[str, ...]] = []
90
+ for values in devices.values():
91
+ previous: _StateLayout | None = None
92
+ group: list[str] = []
93
+ for layout, name in sorted(
94
+ values, key=lambda item: (item[0][0], item[0][1], item[1])
95
+ ):
96
+ if previous is not None and layout[0] < previous[1]:
97
+ if layout != previous:
98
+ raise TaskError(
99
+ f"unsupported overlapping model state: {group[0]} and {name}"
100
+ )
101
+ group.append(name)
102
+ else:
103
+ if len(group) > 1:
104
+ aliases.append(tuple(sorted(group)))
105
+ previous, group = layout, [name]
106
+ if len(group) > 1:
107
+ aliases.append(tuple(sorted(group)))
108
+ return tuple(sorted(aliases))
109
+
110
+
111
+ __all__ = ["floating_model_state", "tensor_schema_for_model"]