dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,29 @@
1
+ """Pairwise gossip runtime."""
2
+
3
+ from dromeus.gossip.axl import AXLFailureBroadcaster, AXLPairTransport
4
+ from dromeus.gossip.engine import GossipEngine
5
+ from dromeus.gossip.interfaces import (
6
+ ConsensusBroadcastResult,
7
+ ConsensusPublisher,
8
+ FailureBroadcaster,
9
+ PairCommitError,
10
+ PairExchangeResult,
11
+ PairTransport,
12
+ RunFailure,
13
+ )
14
+ from dromeus.gossip.peer_scheduler import Pairing, PeerScheduler
15
+
16
+ __all__ = [
17
+ "AXLFailureBroadcaster",
18
+ "AXLPairTransport",
19
+ "ConsensusBroadcastResult",
20
+ "ConsensusPublisher",
21
+ "FailureBroadcaster",
22
+ "GossipEngine",
23
+ "PairCommitError",
24
+ "PairExchangeResult",
25
+ "PairTransport",
26
+ "Pairing",
27
+ "PeerScheduler",
28
+ "RunFailure",
29
+ ]
@@ -0,0 +1,15 @@
1
+ """Cancellation-safe offloading shared by gossip orchestration and adapters."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ from collections.abc import Callable
7
+
8
+
9
+ async def run_blocking[T](callback: Callable[..., T], *args: object) -> T:
10
+ task = asyncio.create_task(asyncio.to_thread(callback, *args))
11
+ try:
12
+ return await asyncio.shield(task)
13
+ except asyncio.CancelledError:
14
+ await task
15
+ raise
dromeus/gossip/axl.py ADDED
@@ -0,0 +1,497 @@
1
+ """AXL pair exchange and broadcasting over reliable opaque artifact transfer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import math
7
+ from pathlib import Path
8
+
9
+ import numpy as np
10
+
11
+ from dromeus.algorithms.base import MaterializedArtifact, UpdateBundle
12
+ from dromeus.gossip._async import run_blocking as _run_blocking
13
+ from dromeus.gossip.interfaces import (
14
+ ConsensusBroadcastResult,
15
+ PairCommitError,
16
+ PairExchangeResult,
17
+ RunFailure,
18
+ )
19
+ from dromeus.manifests.canonical import (
20
+ materialize_bundle_metadata,
21
+ parse_bundle_metadata,
22
+ )
23
+ from dromeus.manifests.models import (
24
+ MessageId,
25
+ OpaqueUpdateBundleMetadata,
26
+ PublicKey,
27
+ RoundId,
28
+ )
29
+ from dromeus.protocol.codec import decode_message, encode_message
30
+ from dromeus.protocol.models import (
31
+ Envelope,
32
+ MessageType,
33
+ PairCommitMessage,
34
+ RunFailedMessage,
35
+ )
36
+ from dromeus.telemetry.consensus import encode_sketch
37
+ from dromeus.transport.envelope_sender import EnvelopeSender
38
+ from dromeus.transport.outbound_scheduler import Priority, SendTiming
39
+ from dromeus.transport.receiver import MessageChannel, Receiver
40
+ from dromeus.transport.transfer import ArtifactReceipt, TransferError, TransferManager
41
+
42
+
43
+ def decode_run_failure(payload: bytes) -> RunFailure:
44
+ """Decode validated terminal failure evidence from a control envelope."""
45
+ message = decode_message(
46
+ payload,
47
+ RunFailedMessage,
48
+ max_bytes=4096,
49
+ )
50
+ return RunFailure(
51
+ round_id=message.round_id,
52
+ error_type=message.error_type,
53
+ reason=message.reason,
54
+ )
55
+
56
+
57
+ class AXLFailureBroadcaster:
58
+ """Failure-only control broadcaster with no artifact filesystem dependency."""
59
+
60
+ def __init__(
61
+ self,
62
+ *,
63
+ sender: EnvelopeSender,
64
+ participant_keys: frozenset[PublicKey],
65
+ ) -> None:
66
+ self._sender = sender
67
+ self._participant_keys = participant_keys
68
+
69
+ async def broadcast_run_failed(self, failure: RunFailure) -> None:
70
+ """Best-effort failure broadcast using manifest-bounded sender retries."""
71
+ peers = self._participant_keys - {self._sender.local_public_key}
72
+ if not peers:
73
+ return
74
+ payload = encode_message(
75
+ RunFailedMessage(
76
+ round_id=failure.round_id,
77
+ error_type=failure.error_type[:128],
78
+ reason=failure.reason[:1024],
79
+ )
80
+ )
81
+
82
+ local_key = self._sender.local_public_key
83
+ await asyncio.gather(
84
+ *(
85
+ self._sender.send(
86
+ peer,
87
+ message_type=MessageType.RUN_FAILED,
88
+ message_id=f"run-failed-{failure.round_id}-{local_key[:8]}",
89
+ payload=payload,
90
+ priority=Priority.CONTROL,
91
+ round_id=failure.round_id,
92
+ correlation_id=f"run-failure-{failure.round_id}",
93
+ )
94
+ for peer in peers
95
+ ),
96
+ return_exceptions=True,
97
+ )
98
+
99
+
100
+ class AXLPairTransport:
101
+ """Pair transport backed by reliable opaque artifact transfer."""
102
+
103
+ def __init__(
104
+ self,
105
+ *,
106
+ sender: EnvelopeSender,
107
+ receiver: Receiver,
108
+ transfer_manager: TransferManager,
109
+ metadata_root: Path,
110
+ participant_keys: frozenset[PublicKey] | None = None,
111
+ ) -> None:
112
+ self._sender = sender
113
+ self._transport_limits = sender.transport_limits
114
+ self._receiver = receiver
115
+ self._transfer_manager = transfer_manager
116
+ self._metadata_root = metadata_root
117
+ self._participant_keys = participant_keys or frozenset()
118
+ self._failure_broadcaster = AXLFailureBroadcaster(
119
+ sender=sender,
120
+ participant_keys=self._participant_keys,
121
+ )
122
+ self._ready_cache: dict[RoundId, str] = {}
123
+ self._committed_rounds: dict[RoundId, str] = {}
124
+
125
+ async def broadcast_run_failed(self, failure: RunFailure) -> None:
126
+ await self._failure_broadcaster.broadcast_run_failed(failure)
127
+
128
+ async def broadcast_consensus_sketch(
129
+ self, *, round_id: RoundId, sketch: np.ndarray
130
+ ) -> ConsensusBroadcastResult:
131
+ """Best-effort low-priority broadcast of one FP32 consensus sketch."""
132
+ peers = self._participant_keys - {self._sender.local_public_key}
133
+ payload = encode_sketch(sketch)
134
+ if not peers:
135
+ return ConsensusBroadcastResult(
136
+ payload_bytes=len(payload),
137
+ recipient_count=0,
138
+ successful_recipient_count=0,
139
+ retry_count=0,
140
+ )
141
+
142
+ async def send(peer: PublicKey) -> SendTiming:
143
+ return await self._sender.send(
144
+ peer,
145
+ message_type=MessageType.CONSENSUS_SKETCH,
146
+ message_id=(
147
+ f"consensus-sketch-{round_id}-"
148
+ f"{self._sender.local_public_key[:8]}"
149
+ ),
150
+ payload=payload,
151
+ priority=Priority.TELEMETRY,
152
+ round_id=round_id,
153
+ correlation_id=f"consensus-round-{round_id}",
154
+ )
155
+
156
+ results = await asyncio.gather(
157
+ *(send(peer) for peer in peers),
158
+ return_exceptions=True,
159
+ )
160
+ timings = [result for result in results if isinstance(result, SendTiming)]
161
+ return ConsensusBroadcastResult(
162
+ payload_bytes=len(payload),
163
+ recipient_count=len(peers),
164
+ successful_recipient_count=len(timings),
165
+ retry_count=sum(timing.retry_count for timing in timings),
166
+ )
167
+
168
+ async def exchange_update(
169
+ self,
170
+ *,
171
+ peer: PublicKey,
172
+ round_id: RoundId,
173
+ bundle: UpdateBundle,
174
+ ) -> PairExchangeResult:
175
+ self._validate_bundle(
176
+ bundle.metadata,
177
+ sender=self._sender.local_public_key,
178
+ round_id=round_id,
179
+ )
180
+ await _run_blocking(
181
+ bundle.validate_materialized,
182
+ self._transport_limits.max_update_bundle_bytes,
183
+ )
184
+ receipts: list[ArtifactReceipt] = []
185
+ metadata_receipt: ArtifactReceipt | None = None
186
+ claimed = False
187
+ carrier_task = asyncio.create_task(
188
+ asyncio.to_thread(
189
+ materialize_bundle_metadata,
190
+ bundle.metadata,
191
+ self._metadata_root,
192
+ )
193
+ )
194
+ try:
195
+ metadata_carrier = await asyncio.shield(carrier_task)
196
+ except asyncio.CancelledError:
197
+ metadata_carrier = await carrier_task
198
+ await _run_blocking(metadata_carrier.path.unlink, True)
199
+ raise
200
+ try:
201
+ retry_count = 0
202
+ transfer_id = await self._transfer_manager.send_artifact(
203
+ destination=peer,
204
+ artifact_name="update-bundle-metadata",
205
+ artifact_path=metadata_carrier.path,
206
+ codec_id="safetensors",
207
+ tensor_schema=metadata_carrier.tensor_schema,
208
+ round_id=round_id,
209
+ )
210
+ timing = self._transfer_manager.last_timing
211
+ retry_count += timing.retry_count if timing is not None else 0
212
+ metadata_receipt = await self._next_peer_receipt(
213
+ peer=peer, round_id=round_id
214
+ )
215
+ if metadata_receipt.artifact_name != "update-bundle-metadata":
216
+ raise PairCommitError("peer update metadata artifact is missing")
217
+ peer_metadata = await _run_blocking(
218
+ parse_bundle_metadata, metadata_receipt.path
219
+ )
220
+ self._validate_bundle(peer_metadata, sender=peer, round_id=round_id)
221
+ await self._transfer_manager.release_receipt(metadata_receipt)
222
+ metadata_receipt = None
223
+
224
+ for artifact, materialized in zip(
225
+ bundle.metadata.artifacts,
226
+ bundle.artifacts,
227
+ strict=True,
228
+ ):
229
+ transfer_id = (
230
+ await self._transfer_manager.send_artifact(
231
+ destination=peer,
232
+ artifact_name=artifact.name,
233
+ artifact_path=materialized.path,
234
+ codec_id=materialized.transfer_codec_id,
235
+ tensor_schema=materialized.transfer_schema,
236
+ round_id=round_id,
237
+ )
238
+ )
239
+ timing = self._transfer_manager.last_timing
240
+ retry_count += timing.retry_count if timing is not None else 0
241
+ for _ in peer_metadata.artifacts:
242
+ receipt = await self._next_peer_receipt(
243
+ peer=peer, round_id=round_id
244
+ )
245
+ receipts.append(receipt)
246
+ receipt_by_name = {receipt.artifact_name: receipt for receipt in receipts}
247
+ if len(receipt_by_name) != len(receipts):
248
+ raise PairCommitError("peer update contains duplicate artifacts")
249
+ for artifact in peer_metadata.artifacts:
250
+ receipt = receipt_by_name.get(artifact.name)
251
+ if (
252
+ receipt is None
253
+ or receipt.size_bytes != artifact.size_bytes
254
+ or receipt.sha256 != artifact.sha256
255
+ ):
256
+ raise PairCommitError("peer update artifact metadata mismatch")
257
+ peer_bundle = UpdateBundle(
258
+ metadata=peer_metadata,
259
+ artifacts=tuple(
260
+ MaterializedArtifact(
261
+ path=receipt_by_name[artifact.name].path,
262
+ transfer_codec_id=receipt_by_name[
263
+ artifact.name
264
+ ].codec_id,
265
+ transfer_schema=receipt_by_name[
266
+ artifact.name
267
+ ].tensor_schema,
268
+ )
269
+ for artifact in peer_metadata.artifacts
270
+ ),
271
+ )
272
+ await _run_blocking(
273
+ peer_bundle.validate_materialized,
274
+ self._transport_limits.max_update_bundle_bytes,
275
+ )
276
+ for receipt in receipts:
277
+ self._transfer_manager.claim_receipt(receipt)
278
+ claimed = True
279
+ return PairExchangeResult(
280
+ bundle=peer_bundle,
281
+ transfer_id=transfer_id,
282
+ retry_count=retry_count,
283
+ )
284
+ except (
285
+ OSError,
286
+ ValueError,
287
+ TypeError,
288
+ TransferError,
289
+ ) as error:
290
+ raise PairCommitError("peer update transfer failed") from error
291
+ finally:
292
+ if not claimed:
293
+ await self._transfer_manager.discard_round_transfers(
294
+ sender=peer, round_id=round_id
295
+ )
296
+ for receipt in receipts:
297
+ await self._transfer_manager.release_receipt(receipt)
298
+ if metadata_receipt is not None:
299
+ await self._transfer_manager.release_receipt(metadata_receipt)
300
+ await _run_blocking(metadata_carrier.path.unlink, True)
301
+
302
+ async def _next_peer_receipt(
303
+ self, *, peer: PublicKey, round_id: RoundId
304
+ ) -> ArtifactReceipt:
305
+ receipt = await self._transfer_manager.next_artifact(
306
+ timeout_seconds=self._timeout_seconds
307
+ )
308
+ if receipt.sender_public_key != peer or receipt.round_id != round_id:
309
+ await self._transfer_manager.release_receipt(receipt)
310
+ raise PairCommitError("received unexpected peer update artifact")
311
+ return receipt
312
+
313
+ def _validate_bundle(
314
+ self,
315
+ metadata: OpaqueUpdateBundleMetadata,
316
+ *,
317
+ sender: PublicKey,
318
+ round_id: RoundId,
319
+ ) -> None:
320
+ if (
321
+ metadata.run_id != self._sender.run_id
322
+ or metadata.manifest_hash != self._sender.manifest_hash
323
+ or metadata.algorithm_id != self._sender.algorithm_id
324
+ or metadata.sender_public_key != sender
325
+ or metadata.round_id != round_id
326
+ ):
327
+ raise PairCommitError("update bundle context mismatch")
328
+ if (
329
+ sum(artifact.size_bytes for artifact in metadata.artifacts)
330
+ > self._transport_limits.max_update_bundle_bytes
331
+ ):
332
+ raise PairCommitError("update bundle exceeds manifest payload limit")
333
+
334
+ async def exchange_update_ready(
335
+ self,
336
+ *,
337
+ peer: PublicKey,
338
+ round_id: RoundId,
339
+ bundle_checksum: str,
340
+ ) -> str:
341
+ cached = self._ready_cache.get(round_id)
342
+ if cached is not None:
343
+ return cached
344
+ envelope = await self._exchange_pair_message(
345
+ destination=peer,
346
+ message_type=MessageType.UPDATE_READY,
347
+ message_id=f"update-ready-{round_id}",
348
+ payload=PairCommitMessage(round_id=round_id, checksum=bundle_checksum),
349
+ round_id=round_id,
350
+ exchange_timeout_seconds=(
351
+ self._transport_limits.transfer_lifetime_limit
352
+ ),
353
+ )
354
+ message = decode_message(
355
+ envelope.payload,
356
+ PairCommitMessage,
357
+ max_bytes=self._transport_limits.max_payload_bytes,
358
+ )
359
+ if message.round_id != round_id:
360
+ raise PairCommitError("peer UPDATE_READY round mismatch")
361
+ self._ready_cache[round_id] = message.checksum
362
+ return message.checksum
363
+
364
+ async def exchange_round_committed(
365
+ self,
366
+ *,
367
+ peer: PublicKey,
368
+ round_id: RoundId,
369
+ state_checksum: str,
370
+ ) -> None:
371
+ committed_checksum = self._committed_rounds.get(round_id)
372
+ if committed_checksum is not None:
373
+ if committed_checksum != state_checksum:
374
+ raise PairCommitError("duplicate ROUND_COMMITTED checksum mismatch")
375
+ return
376
+ envelope = await self._exchange_pair_message(
377
+ destination=peer,
378
+ message_type=MessageType.ROUND_COMMITTED,
379
+ message_id=f"round-committed-{round_id}",
380
+ payload=PairCommitMessage(round_id=round_id, checksum=state_checksum),
381
+ round_id=round_id,
382
+ )
383
+ message = decode_message(
384
+ envelope.payload,
385
+ PairCommitMessage,
386
+ max_bytes=self._transport_limits.max_payload_bytes,
387
+ )
388
+ if message.round_id != round_id:
389
+ raise PairCommitError("peer ROUND_COMMITTED round mismatch")
390
+ self._receiver.set_current_round(round_id + 1)
391
+ await self._receiver.advance_round(round_id + 1)
392
+ self._committed_rounds[round_id] = state_checksum
393
+
394
+ @property
395
+ def _timeout_seconds(self) -> float:
396
+ return max(
397
+ self._transport_limits.transfer_lifetime_limit,
398
+ self._transport_limits.retry_timeout_seconds
399
+ * (self._transport_limits.max_retries + 4),
400
+ )
401
+
402
+ async def _send_pair_message(
403
+ self,
404
+ *,
405
+ destination: PublicKey,
406
+ message_type: MessageType,
407
+ message_id: MessageId,
408
+ payload: PairCommitMessage,
409
+ round_id: RoundId,
410
+ ) -> None:
411
+ await self._sender.send(
412
+ destination,
413
+ message_type=message_type,
414
+ message_id=message_id,
415
+ payload=encode_message(payload),
416
+ priority=Priority.CONTROL,
417
+ round_id=round_id,
418
+ correlation_id=f"pair-round-{round_id}",
419
+ )
420
+
421
+ async def _exchange_pair_message(
422
+ self,
423
+ *,
424
+ destination: PublicKey,
425
+ message_type: MessageType,
426
+ message_id: MessageId,
427
+ payload: PairCommitMessage,
428
+ round_id: RoundId,
429
+ exchange_timeout_seconds: float | None = None,
430
+ ) -> Envelope:
431
+ attempts = (
432
+ math.ceil(
433
+ exchange_timeout_seconds
434
+ / self._transport_limits.retry_timeout_seconds
435
+ )
436
+ if exchange_timeout_seconds is not None
437
+ else self._transport_limits.max_retries + 1
438
+ )
439
+ for attempt in range(attempts):
440
+ await self._send_pair_message(
441
+ destination=destination,
442
+ message_type=message_type,
443
+ message_id=message_id,
444
+ payload=payload,
445
+ round_id=round_id,
446
+ )
447
+ try:
448
+ envelope = await self._receive_pair_message(
449
+ peer=destination,
450
+ message_type=message_type,
451
+ round_id=round_id,
452
+ timeout_seconds=self._transport_limits.retry_timeout_seconds,
453
+ )
454
+ except PairCommitError as error:
455
+ if (
456
+ not isinstance(error.__cause__, TimeoutError)
457
+ or attempt + 1 >= attempts
458
+ ):
459
+ raise
460
+ continue
461
+ for _ in range(self._transport_limits.max_retries):
462
+ await self._send_pair_message(
463
+ destination=destination,
464
+ message_type=message_type,
465
+ message_id=message_id,
466
+ payload=payload,
467
+ round_id=round_id,
468
+ )
469
+ return envelope
470
+ raise PairCommitError("pair commit deadline exceeded")
471
+
472
+ async def _receive_pair_message(
473
+ self,
474
+ *,
475
+ peer: PublicKey,
476
+ message_type: MessageType,
477
+ round_id: RoundId,
478
+ timeout_seconds: float | None = None,
479
+ ) -> Envelope:
480
+ try:
481
+ envelope = await self._receiver.receive(
482
+ MessageChannel.PAIR_COMMIT,
483
+ timeout_seconds=(
484
+ timeout_seconds
485
+ if timeout_seconds is not None
486
+ else self._timeout_seconds
487
+ ),
488
+ )
489
+ except TimeoutError as error:
490
+ raise PairCommitError("pair commit deadline exceeded") from error
491
+ if (
492
+ envelope.sender_public_key != peer
493
+ or envelope.message_type is not message_type
494
+ or envelope.round_id != round_id
495
+ ):
496
+ raise PairCommitError("received unexpected pair commit message")
497
+ return envelope