dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
dromeus/gossip/engine.py
ADDED
|
@@ -0,0 +1,378 @@
|
|
|
1
|
+
"""Event-driven local training and pairwise commit orchestration."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import inspect
|
|
7
|
+
import time
|
|
8
|
+
from collections.abc import Awaitable, Callable
|
|
9
|
+
|
|
10
|
+
from dromeus.algorithms.base import (
|
|
11
|
+
AlgorithmObservations,
|
|
12
|
+
UpdateBundle,
|
|
13
|
+
checksum_tensors,
|
|
14
|
+
)
|
|
15
|
+
from dromeus.gossip._async import run_blocking as _run_blocking
|
|
16
|
+
from dromeus.gossip.interfaces import (
|
|
17
|
+
CommitCallback,
|
|
18
|
+
ConsensusPublisher,
|
|
19
|
+
EvaluationCallback,
|
|
20
|
+
EvaluationMetrics,
|
|
21
|
+
FailureBroadcaster,
|
|
22
|
+
GossipAlgorithm,
|
|
23
|
+
PairCommitError,
|
|
24
|
+
PairExchangeResult,
|
|
25
|
+
PairTransport,
|
|
26
|
+
RoundCommit,
|
|
27
|
+
RunFailure,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.gossip.peer_scheduler import PeerScheduler
|
|
30
|
+
from dromeus.manifests.models import PublicKey, RoundId, TransportLimits
|
|
31
|
+
from dromeus.telemetry.metrics import MetricsPublisher, RoundTiming
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
class GossipEngine:
|
|
35
|
+
"""Run fixed-round local training without a group-wide barrier."""
|
|
36
|
+
|
|
37
|
+
def __init__(
|
|
38
|
+
self,
|
|
39
|
+
*,
|
|
40
|
+
local_public_key: PublicKey,
|
|
41
|
+
round_count: int,
|
|
42
|
+
scheduler: PeerScheduler,
|
|
43
|
+
algorithm: GossipAlgorithm,
|
|
44
|
+
transport: PairTransport,
|
|
45
|
+
commit_callback: CommitCallback,
|
|
46
|
+
confirm_callback: CommitCallback | None = None,
|
|
47
|
+
timeout_seconds: float | None = None,
|
|
48
|
+
transport_limits: TransportLimits | None = None,
|
|
49
|
+
failure_callback: Callable[[RunFailure], None | Awaitable[None]] | None = None,
|
|
50
|
+
failure_broadcaster: FailureBroadcaster | None = None,
|
|
51
|
+
consensus_publisher: ConsensusPublisher | None = None,
|
|
52
|
+
evaluation_interval: int = 5,
|
|
53
|
+
evaluation_callback: EvaluationCallback | None = None,
|
|
54
|
+
metrics_publisher: MetricsPublisher | None = None,
|
|
55
|
+
redact_failures: bool = False,
|
|
56
|
+
) -> None:
|
|
57
|
+
if round_count <= 0:
|
|
58
|
+
raise ValueError("round_count must be positive")
|
|
59
|
+
if timeout_seconds is not None and timeout_seconds <= 0:
|
|
60
|
+
raise ValueError("timeout_seconds must be positive")
|
|
61
|
+
if timeout_seconds is not None and transport_limits is not None:
|
|
62
|
+
raise ValueError("pass timeout_seconds or transport_limits, not both")
|
|
63
|
+
if evaluation_interval <= 0:
|
|
64
|
+
raise ValueError("evaluation_interval must be positive")
|
|
65
|
+
self._redact_failures = redact_failures
|
|
66
|
+
self._local_public_key = local_public_key
|
|
67
|
+
self._round_count = round_count
|
|
68
|
+
self._scheduler = scheduler
|
|
69
|
+
self._algorithm = algorithm
|
|
70
|
+
self._transport = transport
|
|
71
|
+
self._commit_callback = commit_callback
|
|
72
|
+
self._confirm_callback = confirm_callback
|
|
73
|
+
self._timeout_seconds = (
|
|
74
|
+
timeout_seconds
|
|
75
|
+
if timeout_seconds is not None
|
|
76
|
+
else (
|
|
77
|
+
max(
|
|
78
|
+
transport_limits.transfer_lifetime_limit,
|
|
79
|
+
transport_limits.retry_timeout_seconds
|
|
80
|
+
* (transport_limits.max_retries + 4),
|
|
81
|
+
)
|
|
82
|
+
+ transport_limits.retry_timeout_seconds
|
|
83
|
+
if transport_limits is not None
|
|
84
|
+
else None
|
|
85
|
+
)
|
|
86
|
+
)
|
|
87
|
+
self._failure_callback = failure_callback
|
|
88
|
+
self._failure_broadcaster = failure_broadcaster
|
|
89
|
+
self._consensus_publisher = consensus_publisher
|
|
90
|
+
self._evaluation_interval = evaluation_interval
|
|
91
|
+
self._evaluation_callback = evaluation_callback
|
|
92
|
+
self._metrics_publisher = metrics_publisher
|
|
93
|
+
self._current_round = 0
|
|
94
|
+
self._commits: list[RoundCommit] = []
|
|
95
|
+
self._failure: RunFailure | None = None
|
|
96
|
+
|
|
97
|
+
@property
|
|
98
|
+
def current_round(self) -> RoundId:
|
|
99
|
+
return self._current_round
|
|
100
|
+
|
|
101
|
+
@property
|
|
102
|
+
def commits(self) -> tuple[RoundCommit, ...]:
|
|
103
|
+
return tuple(self._commits)
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def failure(self) -> RunFailure | None:
|
|
107
|
+
return self._failure
|
|
108
|
+
|
|
109
|
+
async def run(self) -> tuple[RoundCommit, ...]:
|
|
110
|
+
"""Train and commit every manifest round in order."""
|
|
111
|
+
while self._current_round < self._round_count:
|
|
112
|
+
await self.run_round(self._current_round)
|
|
113
|
+
return self.commits
|
|
114
|
+
|
|
115
|
+
async def run_round(self, round_id: RoundId) -> RoundCommit:
|
|
116
|
+
"""Complete one scheduled pair exchange and commit."""
|
|
117
|
+
if self._failure is not None:
|
|
118
|
+
raise PairCommitError("run has already failed")
|
|
119
|
+
if round_id != self._current_round:
|
|
120
|
+
raise PairCommitError(
|
|
121
|
+
f"round {round_id} is not current; expected {self._current_round}"
|
|
122
|
+
)
|
|
123
|
+
try:
|
|
124
|
+
return await self._run_round(round_id)
|
|
125
|
+
except asyncio.CancelledError:
|
|
126
|
+
raise
|
|
127
|
+
except TimeoutError as error:
|
|
128
|
+
failure = PairCommitError("pair round deadline exceeded")
|
|
129
|
+
await self._record_failure(round_id, failure)
|
|
130
|
+
raise failure from error
|
|
131
|
+
except Exception as error:
|
|
132
|
+
await self._record_failure(round_id, error)
|
|
133
|
+
if self._redact_failures:
|
|
134
|
+
raise PairCommitError("private run failed") from None
|
|
135
|
+
raise
|
|
136
|
+
|
|
137
|
+
async def _run_round(self, round_id: RoundId) -> RoundCommit:
|
|
138
|
+
pairing = self._scheduler.schedule(round_id)
|
|
139
|
+
try:
|
|
140
|
+
peer = pairing.peer_for(self._local_public_key)
|
|
141
|
+
except KeyError as error:
|
|
142
|
+
raise PairCommitError(str(error)) from error
|
|
143
|
+
|
|
144
|
+
local_bundle: UpdateBundle | None = None
|
|
145
|
+
peer_bundle: UpdateBundle | None = None
|
|
146
|
+
try:
|
|
147
|
+
local_started = time.perf_counter()
|
|
148
|
+
await _run_blocking(self._algorithm.pre_local, round_id)
|
|
149
|
+
await _run_blocking(self._algorithm.local_training)
|
|
150
|
+
bundle_task = asyncio.create_task(
|
|
151
|
+
asyncio.to_thread(self._algorithm.post_local_bundle)
|
|
152
|
+
)
|
|
153
|
+
try:
|
|
154
|
+
materialized = await asyncio.shield(bundle_task)
|
|
155
|
+
except asyncio.CancelledError:
|
|
156
|
+
local_bundle = await bundle_task
|
|
157
|
+
raise
|
|
158
|
+
local_bundle = materialized
|
|
159
|
+
if materialized.metadata.round_id != round_id:
|
|
160
|
+
raise PairCommitError("local update round does not match current round")
|
|
161
|
+
local_compute_seconds = time.perf_counter() - local_started
|
|
162
|
+
|
|
163
|
+
peer_wait_started = time.perf_counter()
|
|
164
|
+
remote_digest = await self._with_pair_timeout(
|
|
165
|
+
self._transport.exchange_update_ready(
|
|
166
|
+
peer=peer,
|
|
167
|
+
round_id=round_id,
|
|
168
|
+
bundle_checksum=materialized.digest,
|
|
169
|
+
)
|
|
170
|
+
)
|
|
171
|
+
peer_wait_seconds = time.perf_counter() - peer_wait_started
|
|
172
|
+
|
|
173
|
+
transfer_started = time.perf_counter()
|
|
174
|
+
exchange = await self._with_pair_timeout(
|
|
175
|
+
self._transport.exchange_update(
|
|
176
|
+
peer=peer,
|
|
177
|
+
round_id=round_id,
|
|
178
|
+
bundle=materialized,
|
|
179
|
+
)
|
|
180
|
+
)
|
|
181
|
+
peer_bundle = exchange.bundle
|
|
182
|
+
transfer_seconds = time.perf_counter() - transfer_started
|
|
183
|
+
if peer_bundle.metadata.round_id != round_id:
|
|
184
|
+
raise PairCommitError("peer update round does not match current round")
|
|
185
|
+
try:
|
|
186
|
+
peer_update = await _run_blocking(
|
|
187
|
+
self._algorithm.validate_peer, peer_bundle
|
|
188
|
+
)
|
|
189
|
+
except (ValueError, TypeError) as error:
|
|
190
|
+
raise PairCommitError("peer update validation failed") from error
|
|
191
|
+
if remote_digest != peer_bundle.digest:
|
|
192
|
+
raise PairCommitError("peer UPDATE_READY bundle digest mismatch")
|
|
193
|
+
|
|
194
|
+
mixing_started = time.perf_counter()
|
|
195
|
+
try:
|
|
196
|
+
post_mix = await _run_blocking(
|
|
197
|
+
self._algorithm.peer_apply,
|
|
198
|
+
peer_update,
|
|
199
|
+
)
|
|
200
|
+
except (ValueError, TypeError) as error:
|
|
201
|
+
raise PairCommitError("peer update application failed") from error
|
|
202
|
+
mixing_seconds = time.perf_counter() - mixing_started
|
|
203
|
+
state_checksum = await _run_blocking(checksum_tensors, post_mix.weights)
|
|
204
|
+
observations = await self._capture_algorithm_observations()
|
|
205
|
+
commit = RoundCommit(
|
|
206
|
+
round_id=round_id,
|
|
207
|
+
peer_public_key=peer,
|
|
208
|
+
local_bundle_digest=materialized.digest,
|
|
209
|
+
peer_bundle_digest=peer_bundle.digest,
|
|
210
|
+
state_checksum=state_checksum,
|
|
211
|
+
local_loss=observations.local_loss,
|
|
212
|
+
error_feedback_residual_l2_norm=(
|
|
213
|
+
observations.error_feedback_residual_l2_norm
|
|
214
|
+
),
|
|
215
|
+
error_feedback_signal_l2_norm=observations.error_feedback_signal_l2_norm,
|
|
216
|
+
error_feedback_residual_to_signal_ratio=(
|
|
217
|
+
observations.error_feedback_residual_to_signal_ratio
|
|
218
|
+
),
|
|
219
|
+
transfer_id=exchange.transfer_id,
|
|
220
|
+
transfer_retries=exchange.retry_count,
|
|
221
|
+
encoded_artifact_bytes=sum(
|
|
222
|
+
artifact.size_bytes for artifact in materialized.metadata.artifacts
|
|
223
|
+
),
|
|
224
|
+
)
|
|
225
|
+
result = await _run_blocking(self._commit_callback, commit)
|
|
226
|
+
if inspect.isawaitable(result):
|
|
227
|
+
await result
|
|
228
|
+
commit_wait_started = time.perf_counter()
|
|
229
|
+
await self._with_pair_timeout(
|
|
230
|
+
self._transport.exchange_round_committed(
|
|
231
|
+
peer=peer,
|
|
232
|
+
round_id=round_id,
|
|
233
|
+
state_checksum=state_checksum,
|
|
234
|
+
)
|
|
235
|
+
)
|
|
236
|
+
if self._confirm_callback is not None:
|
|
237
|
+
result = await _run_blocking(self._confirm_callback, commit)
|
|
238
|
+
if inspect.isawaitable(result):
|
|
239
|
+
await result
|
|
240
|
+
peer_wait_seconds += time.perf_counter() - commit_wait_started
|
|
241
|
+
|
|
242
|
+
self._commits.append(commit)
|
|
243
|
+
self._current_round += 1
|
|
244
|
+
if self._consensus_publisher is not None:
|
|
245
|
+
try:
|
|
246
|
+
self._consensus_publisher.submit(
|
|
247
|
+
round_id=round_id,
|
|
248
|
+
weights=post_mix.weights,
|
|
249
|
+
)
|
|
250
|
+
except Exception:
|
|
251
|
+
pass
|
|
252
|
+
evaluation_started = time.perf_counter()
|
|
253
|
+
evaluation = await self._evaluate_if_due(round_id)
|
|
254
|
+
evaluation_seconds = time.perf_counter() - evaluation_started
|
|
255
|
+
if self._metrics_publisher is not None:
|
|
256
|
+
timing = RoundTiming(
|
|
257
|
+
round_id=round_id,
|
|
258
|
+
peer_id=peer,
|
|
259
|
+
local_compute_seconds=local_compute_seconds,
|
|
260
|
+
peer_wait_seconds=peer_wait_seconds,
|
|
261
|
+
transfer_seconds=transfer_seconds,
|
|
262
|
+
mixing_seconds=mixing_seconds,
|
|
263
|
+
evaluation_seconds=evaluation_seconds,
|
|
264
|
+
retries=commit.transfer_retries,
|
|
265
|
+
local_loss=commit.local_loss,
|
|
266
|
+
error_feedback_residual_l2_norm=(
|
|
267
|
+
commit.error_feedback_residual_l2_norm
|
|
268
|
+
),
|
|
269
|
+
error_feedback_signal_l2_norm=commit.error_feedback_signal_l2_norm,
|
|
270
|
+
error_feedback_residual_to_signal_ratio=(
|
|
271
|
+
commit.error_feedback_residual_to_signal_ratio
|
|
272
|
+
),
|
|
273
|
+
evaluation_metrics=evaluation.metrics
|
|
274
|
+
if evaluation is not None
|
|
275
|
+
else None,
|
|
276
|
+
transfer_id=commit.transfer_id,
|
|
277
|
+
encoded_artifact_bytes=commit.encoded_artifact_bytes,
|
|
278
|
+
)
|
|
279
|
+
try:
|
|
280
|
+
self._metrics_publisher.submit(timing)
|
|
281
|
+
except Exception:
|
|
282
|
+
pass
|
|
283
|
+
return commit
|
|
284
|
+
finally:
|
|
285
|
+
try:
|
|
286
|
+
if peer_bundle is not None:
|
|
287
|
+
await _run_blocking(self._algorithm.release_bundle, peer_bundle)
|
|
288
|
+
finally:
|
|
289
|
+
if local_bundle is not None:
|
|
290
|
+
await _run_blocking(self._algorithm.release_bundle, local_bundle)
|
|
291
|
+
|
|
292
|
+
async def _capture_algorithm_observations(self) -> AlgorithmObservations:
|
|
293
|
+
try:
|
|
294
|
+
return await _run_blocking(self._algorithm.observations)
|
|
295
|
+
except Exception:
|
|
296
|
+
return AlgorithmObservations()
|
|
297
|
+
|
|
298
|
+
async def _with_pair_timeout[T](self, operation: Awaitable[T]) -> T:
|
|
299
|
+
if self._timeout_seconds is None:
|
|
300
|
+
return await operation
|
|
301
|
+
return await asyncio.wait_for(operation, timeout=self._timeout_seconds)
|
|
302
|
+
|
|
303
|
+
async def _evaluate_if_due(self, round_id: RoundId) -> EvaluationMetrics | None:
|
|
304
|
+
completed_round = round_id + 1
|
|
305
|
+
if (
|
|
306
|
+
completed_round % self._evaluation_interval != 0
|
|
307
|
+
and completed_round != self._round_count
|
|
308
|
+
):
|
|
309
|
+
return None
|
|
310
|
+
try:
|
|
311
|
+
evaluation = await _run_blocking(self._algorithm.evaluate)
|
|
312
|
+
except Exception:
|
|
313
|
+
return None
|
|
314
|
+
if evaluation is None:
|
|
315
|
+
return None
|
|
316
|
+
metrics = EvaluationMetrics(
|
|
317
|
+
round_id=round_id,
|
|
318
|
+
metrics=evaluation.metrics,
|
|
319
|
+
)
|
|
320
|
+
if self._evaluation_callback is not None:
|
|
321
|
+
try:
|
|
322
|
+
result = self._evaluation_callback(metrics)
|
|
323
|
+
if inspect.isawaitable(result):
|
|
324
|
+
await result
|
|
325
|
+
except Exception:
|
|
326
|
+
pass
|
|
327
|
+
return metrics
|
|
328
|
+
|
|
329
|
+
async def _record_failure(self, round_id: RoundId, error: Exception) -> None:
|
|
330
|
+
if self._failure is not None:
|
|
331
|
+
return
|
|
332
|
+
failure = RunFailure(
|
|
333
|
+
round_id=round_id,
|
|
334
|
+
error_type=type(error).__name__,
|
|
335
|
+
reason=(
|
|
336
|
+
"private run failed"
|
|
337
|
+
if self._redact_failures
|
|
338
|
+
else str(error)[:1024] or "pair round failed"
|
|
339
|
+
),
|
|
340
|
+
)
|
|
341
|
+
self._failure = failure
|
|
342
|
+
if self._failure_callback is not None:
|
|
343
|
+
try:
|
|
344
|
+
result = self._failure_callback(failure)
|
|
345
|
+
if inspect.isawaitable(result):
|
|
346
|
+
await result
|
|
347
|
+
except Exception:
|
|
348
|
+
pass
|
|
349
|
+
if self._metrics_publisher is not None:
|
|
350
|
+
try:
|
|
351
|
+
self._metrics_publisher.submit_failure(
|
|
352
|
+
round_id=failure.round_id,
|
|
353
|
+
error_type=failure.error_type,
|
|
354
|
+
reason=failure.reason,
|
|
355
|
+
)
|
|
356
|
+
except Exception:
|
|
357
|
+
pass
|
|
358
|
+
if self._failure_broadcaster is not None:
|
|
359
|
+
try:
|
|
360
|
+
await self._failure_broadcaster.broadcast_run_failed(failure)
|
|
361
|
+
except Exception:
|
|
362
|
+
pass
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
__all__ = [
|
|
366
|
+
"CommitCallback",
|
|
367
|
+
"ConsensusPublisher",
|
|
368
|
+
"EvaluationCallback",
|
|
369
|
+
"EvaluationMetrics",
|
|
370
|
+
"FailureBroadcaster",
|
|
371
|
+
"GossipAlgorithm",
|
|
372
|
+
"GossipEngine",
|
|
373
|
+
"PairCommitError",
|
|
374
|
+
"PairExchangeResult",
|
|
375
|
+
"PairTransport",
|
|
376
|
+
"RoundCommit",
|
|
377
|
+
"RunFailure",
|
|
378
|
+
]
|
|
@@ -0,0 +1,158 @@
|
|
|
1
|
+
"""Transport-independent contracts for gossip orchestration and adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Awaitable, Callable, Mapping
|
|
6
|
+
from dataclasses import dataclass
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import Protocol
|
|
9
|
+
|
|
10
|
+
import numpy as np
|
|
11
|
+
|
|
12
|
+
from dromeus.algorithms.base import (
|
|
13
|
+
AlgorithmObservations,
|
|
14
|
+
AlgorithmSnapshot,
|
|
15
|
+
AlgorithmUpdate,
|
|
16
|
+
UpdateBundle,
|
|
17
|
+
)
|
|
18
|
+
from dromeus.errors import RunError
|
|
19
|
+
from dromeus.manifests.models import PublicKey, RoundId
|
|
20
|
+
from dromeus.training.base import EvaluationResult
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
class PairCommitError(RunError):
|
|
24
|
+
"""A peer update or pair commit could not be completed safely."""
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
@dataclass(frozen=True, slots=True)
|
|
28
|
+
class RunFailure:
|
|
29
|
+
"""Terminal failure evidence for persistence and control-plane reporting."""
|
|
30
|
+
|
|
31
|
+
round_id: RoundId
|
|
32
|
+
error_type: str
|
|
33
|
+
reason: str
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
@dataclass(frozen=True, slots=True)
|
|
37
|
+
class EvaluationMetrics:
|
|
38
|
+
"""One node's deterministic evaluation result for a committed round."""
|
|
39
|
+
|
|
40
|
+
round_id: RoundId
|
|
41
|
+
metrics: Mapping[str, float]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
@dataclass(frozen=True, slots=True)
|
|
45
|
+
class PairExchangeResult:
|
|
46
|
+
"""Peer bundle plus immutable diagnostics for its transfer."""
|
|
47
|
+
|
|
48
|
+
bundle: UpdateBundle
|
|
49
|
+
transfer_id: str | None = None
|
|
50
|
+
retry_count: int = 0
|
|
51
|
+
|
|
52
|
+
def __post_init__(self) -> None:
|
|
53
|
+
if self.transfer_id is not None and not self.transfer_id:
|
|
54
|
+
raise ValueError("transfer_id must be non-empty when present")
|
|
55
|
+
if self.retry_count < 0:
|
|
56
|
+
raise ValueError("retry_count must be non-negative")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
@dataclass(frozen=True, slots=True)
|
|
60
|
+
class ConsensusBroadcastResult:
|
|
61
|
+
"""Best-effort consensus telemetry broadcast accounting."""
|
|
62
|
+
|
|
63
|
+
payload_bytes: int
|
|
64
|
+
recipient_count: int
|
|
65
|
+
successful_recipient_count: int
|
|
66
|
+
retry_count: int
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
class FailureBroadcaster(Protocol):
|
|
70
|
+
async def broadcast_run_failed(self, failure: RunFailure) -> None: ...
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
class ConsensusPublisher(Protocol):
|
|
74
|
+
def submit(
|
|
75
|
+
self, *, round_id: RoundId, weights: Mapping[str, np.ndarray]
|
|
76
|
+
) -> bool: ...
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
EvaluationCallback = Callable[[EvaluationMetrics], None | Awaitable[None]]
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class GossipAlgorithm(Protocol):
|
|
83
|
+
def configure_bundle_codec(
|
|
84
|
+
self,
|
|
85
|
+
*,
|
|
86
|
+
artifact_root: Path,
|
|
87
|
+
run_id: str,
|
|
88
|
+
manifest_hash: str,
|
|
89
|
+
sender_public_key: str,
|
|
90
|
+
algorithm_id: str,
|
|
91
|
+
) -> None: ...
|
|
92
|
+
|
|
93
|
+
def pre_local(self, round_id: RoundId) -> None: ...
|
|
94
|
+
|
|
95
|
+
def local_training(self) -> None: ...
|
|
96
|
+
|
|
97
|
+
def post_local_bundle(self) -> UpdateBundle: ...
|
|
98
|
+
|
|
99
|
+
def validate_peer(self, peer_bundle: UpdateBundle) -> AlgorithmUpdate: ...
|
|
100
|
+
|
|
101
|
+
def peer_apply(self, peer_update: AlgorithmUpdate) -> AlgorithmSnapshot: ...
|
|
102
|
+
|
|
103
|
+
def release_bundle(self, bundle: UpdateBundle) -> None: ...
|
|
104
|
+
|
|
105
|
+
def checkpoint_tensors(self) -> dict[str, np.ndarray]: ...
|
|
106
|
+
|
|
107
|
+
def observations(self) -> AlgorithmObservations: ...
|
|
108
|
+
|
|
109
|
+
def evaluate(self) -> EvaluationResult | None: ...
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
class PairTransport(Protocol):
|
|
113
|
+
"""Transport seam for one peer's update and commit handshake."""
|
|
114
|
+
|
|
115
|
+
async def exchange_update(
|
|
116
|
+
self,
|
|
117
|
+
*,
|
|
118
|
+
peer: PublicKey,
|
|
119
|
+
round_id: RoundId,
|
|
120
|
+
bundle: UpdateBundle,
|
|
121
|
+
) -> PairExchangeResult: ...
|
|
122
|
+
|
|
123
|
+
async def exchange_update_ready(
|
|
124
|
+
self,
|
|
125
|
+
*,
|
|
126
|
+
peer: PublicKey,
|
|
127
|
+
round_id: RoundId,
|
|
128
|
+
bundle_checksum: str,
|
|
129
|
+
) -> str: ...
|
|
130
|
+
|
|
131
|
+
async def exchange_round_committed(
|
|
132
|
+
self,
|
|
133
|
+
*,
|
|
134
|
+
peer: PublicKey,
|
|
135
|
+
round_id: RoundId,
|
|
136
|
+
state_checksum: str,
|
|
137
|
+
) -> None: ...
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
@dataclass(frozen=True, slots=True)
|
|
141
|
+
class RoundCommit:
|
|
142
|
+
"""Evidence passed to the atomic persistence seam after peer validation."""
|
|
143
|
+
|
|
144
|
+
round_id: RoundId
|
|
145
|
+
peer_public_key: PublicKey
|
|
146
|
+
local_bundle_digest: str
|
|
147
|
+
peer_bundle_digest: str
|
|
148
|
+
state_checksum: str
|
|
149
|
+
local_loss: float | None = None
|
|
150
|
+
error_feedback_residual_l2_norm: float | None = None
|
|
151
|
+
error_feedback_signal_l2_norm: float | None = None
|
|
152
|
+
error_feedback_residual_to_signal_ratio: float | None = None
|
|
153
|
+
transfer_id: str | None = None
|
|
154
|
+
transfer_retries: int = 0
|
|
155
|
+
encoded_artifact_bytes: int | None = None
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
CommitCallback = Callable[[RoundCommit], None | Awaitable[None]]
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
"""Deterministic random perfect matchings for fixed membership."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import hashlib
|
|
6
|
+
from collections.abc import Sequence
|
|
7
|
+
from dataclasses import dataclass
|
|
8
|
+
from typing import cast
|
|
9
|
+
|
|
10
|
+
from dromeus.manifests.models import (
|
|
11
|
+
MAX_PARTICIPANT_COUNT,
|
|
12
|
+
Participant,
|
|
13
|
+
ParticipantCountError,
|
|
14
|
+
PublicKey,
|
|
15
|
+
RoundId,
|
|
16
|
+
)
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass(frozen=True, slots=True)
|
|
20
|
+
class Pairing:
|
|
21
|
+
"""One round's disjoint peer pairs."""
|
|
22
|
+
|
|
23
|
+
round_id: RoundId
|
|
24
|
+
pairs: tuple[tuple[PublicKey, PublicKey], ...]
|
|
25
|
+
|
|
26
|
+
@property
|
|
27
|
+
def peers(self) -> tuple[PublicKey, ...]:
|
|
28
|
+
return tuple(peer for pair in self.pairs for peer in pair)
|
|
29
|
+
|
|
30
|
+
def peer_for(self, local_public_key: PublicKey) -> PublicKey:
|
|
31
|
+
for left, right in self.pairs:
|
|
32
|
+
if left == local_public_key:
|
|
33
|
+
return right
|
|
34
|
+
if right == local_public_key:
|
|
35
|
+
return left
|
|
36
|
+
raise KeyError(f"unknown peer: {local_public_key}")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
class PeerScheduler:
|
|
40
|
+
"""Derive identical pairings from membership, seed, and round number."""
|
|
41
|
+
|
|
42
|
+
def __init__(
|
|
43
|
+
self,
|
|
44
|
+
participants: Sequence[Participant | PublicKey],
|
|
45
|
+
*,
|
|
46
|
+
seed: int,
|
|
47
|
+
) -> None:
|
|
48
|
+
self._members = self._normalise_members(participants)
|
|
49
|
+
self._seed = seed
|
|
50
|
+
|
|
51
|
+
def schedule(self, round_id: RoundId) -> Pairing:
|
|
52
|
+
"""Return the deterministic perfect matching for one round."""
|
|
53
|
+
if round_id < 0:
|
|
54
|
+
raise ValueError("round_id must be non-negative")
|
|
55
|
+
ordered = sorted(
|
|
56
|
+
self._members,
|
|
57
|
+
key=lambda key: (
|
|
58
|
+
hashlib.sha256(
|
|
59
|
+
f"{self._seed}\x00{round_id}\x00{key}".encode()
|
|
60
|
+
).digest(),
|
|
61
|
+
key,
|
|
62
|
+
),
|
|
63
|
+
)
|
|
64
|
+
pairs: list[tuple[PublicKey, PublicKey]] = []
|
|
65
|
+
for index in range(0, len(ordered), 2):
|
|
66
|
+
left, right = sorted((ordered[index], ordered[index + 1]))
|
|
67
|
+
pairs.append((left, right))
|
|
68
|
+
return Pairing(round_id=round_id, pairs=tuple(pairs))
|
|
69
|
+
|
|
70
|
+
@staticmethod
|
|
71
|
+
def _normalise_members(
|
|
72
|
+
participants: Sequence[Participant | PublicKey],
|
|
73
|
+
) -> tuple[PublicKey, ...]:
|
|
74
|
+
if not participants:
|
|
75
|
+
raise ParticipantCountError(
|
|
76
|
+
"scheduler participant count must be between 2 and 16 inclusive"
|
|
77
|
+
)
|
|
78
|
+
first = participants[0]
|
|
79
|
+
if isinstance(first, Participant):
|
|
80
|
+
if not all(
|
|
81
|
+
isinstance(participant, Participant) for participant in participants
|
|
82
|
+
):
|
|
83
|
+
raise ValueError(
|
|
84
|
+
"participants must all be Participant models or public keys"
|
|
85
|
+
)
|
|
86
|
+
models = cast(Sequence[Participant], participants)
|
|
87
|
+
if len({participant.node_index for participant in models}) != len(models):
|
|
88
|
+
raise ValueError("participant node indices must be unique")
|
|
89
|
+
members: tuple[PublicKey, ...] = tuple(
|
|
90
|
+
participant.public_key
|
|
91
|
+
for participant in sorted(models, key=lambda item: item.node_index)
|
|
92
|
+
)
|
|
93
|
+
elif any(isinstance(participant, Participant) for participant in participants):
|
|
94
|
+
raise ValueError(
|
|
95
|
+
"participants must all be Participant models or public keys"
|
|
96
|
+
)
|
|
97
|
+
else:
|
|
98
|
+
members = tuple(
|
|
99
|
+
cast(PublicKey, participant) for participant in participants
|
|
100
|
+
)
|
|
101
|
+
if any(not member for member in members):
|
|
102
|
+
raise ValueError("participant public keys must not be empty")
|
|
103
|
+
if len(set(members)) != len(members):
|
|
104
|
+
raise ValueError("participant public keys must be unique")
|
|
105
|
+
count = len(members)
|
|
106
|
+
# The pair scheduler also serves two-peer engine seams; manifests enforce
|
|
107
|
+
# the production run-group minimum of four participants.
|
|
108
|
+
if count < 2 or count > MAX_PARTICIPANT_COUNT:
|
|
109
|
+
raise ParticipantCountError(
|
|
110
|
+
"scheduler participant count must be between 2 and 16 inclusive"
|
|
111
|
+
)
|
|
112
|
+
if count % 2:
|
|
113
|
+
raise ParticipantCountError("participant count must be even")
|
|
114
|
+
return members
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
__all__ = ["Pairing", "ParticipantCountError", "PeerScheduler"]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""Run manifest models and encoding."""
|