dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,854 @@
1
+ """Reliable artifact transfer."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import hashlib
7
+ import math
8
+ import time
9
+ import uuid
10
+ from dataclasses import dataclass
11
+ from pathlib import Path
12
+
13
+ from dromeus.errors import RunError
14
+ from dromeus.manifests.canonical import file_sha256
15
+ from dromeus.manifests.models import (
16
+ Identifier,
17
+ MessageId,
18
+ PublicKey,
19
+ RoundId,
20
+ Sha256,
21
+ TensorSchema,
22
+ TransferId,
23
+ )
24
+ from dromeus.protocol.codec import (
25
+ ProtocolDecodeError,
26
+ decode_message,
27
+ encode_message,
28
+ )
29
+ from dromeus.protocol.models import (
30
+ Chunk,
31
+ ChunkAck,
32
+ Envelope,
33
+ MessageType,
34
+ TransferBegin,
35
+ )
36
+ from dromeus.telemetry.events import EventSink, emit_event
37
+ from dromeus.telemetry.evidence import (
38
+ TransferMessageSentEvidence,
39
+ append_evidence,
40
+ )
41
+ from dromeus.transport.envelope_sender import EnvelopeSender
42
+ from dromeus.transport.outbound_scheduler import Priority, SendTiming
43
+ from dromeus.transport.receiver import MessageChannel, Receiver
44
+
45
+
46
+ class TransferError(RunError):
47
+ """Artifact transfer failed terminally."""
48
+
49
+
50
+ @dataclass(frozen=True)
51
+ class ArtifactReceipt:
52
+ transfer_id: TransferId
53
+ sender_public_key: PublicKey
54
+ artifact_name: Identifier
55
+ path: Path
56
+ sha256: Sha256
57
+ size_bytes: int
58
+ round_id: RoundId | None
59
+ codec_id: Identifier
60
+ tensor_schema: TensorSchema
61
+
62
+
63
+ @dataclass(frozen=True)
64
+ class TransferTiming:
65
+ """Observable duration and retry count for one outbound artifact."""
66
+
67
+ elapsed_seconds: float
68
+ retry_count: int
69
+
70
+
71
+ @dataclass
72
+ class _Reservation:
73
+ size_bytes: int
74
+ final_path: Path | None = None
75
+
76
+
77
+ class _ArtifactStore:
78
+ """Private transfer storage with exact per-transfer reservations."""
79
+
80
+ def __init__(self, root: Path, *, max_bytes: int = 128 * 1024 * 1024) -> None:
81
+ if max_bytes <= 0:
82
+ raise ValueError("max_bytes must be positive")
83
+ self._root = root
84
+ self._tmp_root = root / ".tmp"
85
+ self._root.mkdir(parents=True, exist_ok=True)
86
+ self._tmp_root.mkdir(parents=True, exist_ok=True)
87
+ self._max_bytes = max_bytes
88
+ self._reserved_bytes = 0
89
+ self._reservations: dict[TransferId, _Reservation] = {}
90
+
91
+ def reserve(self, transfer_id: TransferId, size_bytes: int) -> None:
92
+ if transfer_id in self._reservations:
93
+ raise TransferError("transfer capacity already reserved")
94
+ if self._reserved_bytes + size_bytes > self._max_bytes:
95
+ raise TransferError("artifact store capacity exceeded")
96
+ temp = self._temp_path(transfer_id)
97
+ try:
98
+ temp.unlink(missing_ok=True)
99
+ temp.touch(exist_ok=False)
100
+ except OSError as error:
101
+ raise TransferError("failed to create transfer artifact") from error
102
+ self._reservations[transfer_id] = _Reservation(size_bytes=size_bytes)
103
+ self._reserved_bytes += size_bytes
104
+
105
+ def append(self, transfer_id: TransferId, data: bytes) -> None:
106
+ reservation = self._reservation(transfer_id)
107
+ if reservation.final_path is not None:
108
+ raise TransferError("cannot append to finalized transfer")
109
+ try:
110
+ with self._temp_path(transfer_id).open("ab") as handle:
111
+ handle.write(data)
112
+ except OSError as error:
113
+ raise TransferError("failed to write transfer artifact") from error
114
+
115
+ def write_chunk(
116
+ self,
117
+ transfer_id: TransferId,
118
+ *,
119
+ offset: int,
120
+ data: bytes,
121
+ ) -> None:
122
+ reservation = self._reservation(transfer_id)
123
+ if reservation.final_path is not None:
124
+ raise TransferError("cannot write to finalized transfer")
125
+ if offset < 0 or offset + len(data) > reservation.size_bytes:
126
+ raise TransferError("chunk write exceeds transfer reservation")
127
+ try:
128
+ with self._temp_path(transfer_id).open("r+b") as handle:
129
+ handle.seek(offset)
130
+ handle.write(data)
131
+ except OSError as error:
132
+ raise TransferError("failed to write transfer chunk") from error
133
+
134
+ def finalize(
135
+ self,
136
+ transfer_id: TransferId,
137
+ artifact_name: Identifier,
138
+ expected_size_bytes: int,
139
+ expected_sha256: Sha256,
140
+ ) -> tuple[Path, Sha256]:
141
+ reservation = self._reservation(transfer_id)
142
+ if reservation.size_bytes != expected_size_bytes:
143
+ raise TransferError("transfer reservation size mismatch")
144
+ temp = self._temp_path(transfer_id)
145
+ try:
146
+ if temp.stat().st_size != expected_size_bytes:
147
+ raise TransferError("final artifact size mismatch")
148
+ digest = file_sha256(temp)
149
+ if digest != expected_sha256:
150
+ raise TransferError("final artifact checksum mismatch")
151
+ final = self._final_path(transfer_id, artifact_name)
152
+ if final.exists():
153
+ raise TransferError("final artifact already exists")
154
+ temp.replace(final)
155
+ except OSError as error:
156
+ raise TransferError("failed to finalize transfer artifact") from error
157
+ reservation.final_path = final
158
+ return final, digest
159
+
160
+ def commit(self, transfer_id: TransferId) -> None:
161
+ reservation = self._reservation(transfer_id)
162
+ if reservation.final_path is None:
163
+ raise TransferError("cannot commit unfinished transfer")
164
+
165
+ def claim(self, transfer_id: TransferId) -> None:
166
+ reservation = self._reservation(transfer_id)
167
+ if reservation.final_path is None:
168
+ raise TransferError("cannot claim unfinished transfer")
169
+ self._release(transfer_id)
170
+
171
+ def abort(self, transfer_id: TransferId) -> None:
172
+ reservation = self._reservation(transfer_id)
173
+ try:
174
+ self._temp_path(transfer_id).unlink(missing_ok=True)
175
+ if reservation.final_path is not None:
176
+ reservation.final_path.unlink(missing_ok=True)
177
+ except OSError as error:
178
+ raise TransferError("failed to remove transfer artifact") from error
179
+ self._release(transfer_id)
180
+
181
+ def _reservation(self, transfer_id: TransferId) -> _Reservation:
182
+ reservation = self._reservations.get(transfer_id)
183
+ if reservation is None:
184
+ raise TransferError("transfer capacity is not reserved")
185
+ return reservation
186
+
187
+ def _release(self, transfer_id: TransferId) -> None:
188
+ reservation = self._reservations.pop(transfer_id, None)
189
+ if reservation is None:
190
+ raise TransferError("transfer capacity already released")
191
+ remaining = self._reserved_bytes - reservation.size_bytes
192
+ if remaining < 0:
193
+ raise TransferError("transfer capacity accounting underflow")
194
+ self._reserved_bytes = remaining
195
+
196
+ def _temp_path(self, transfer_id: TransferId) -> Path:
197
+ return self._tmp_root / f"{transfer_id}.part"
198
+
199
+ def _final_path(
200
+ self, transfer_id: TransferId, artifact_name: Identifier
201
+ ) -> Path:
202
+ safe_name = artifact_name.replace("/", "_")
203
+ return self._root / f"{transfer_id}-{safe_name}.bin"
204
+
205
+
206
+ @dataclass
207
+ class _IncomingTransfer:
208
+ sender_public_key: PublicKey
209
+ begin: TransferBegin
210
+ round_id: RoundId | None
211
+ written_chunk_hashes: dict[int, Sha256]
212
+ started_at: float
213
+
214
+
215
+ @dataclass(frozen=True)
216
+ class _FinishedTransfer:
217
+ """Kept for one transfer lifetime so retried messages are acknowledged."""
218
+
219
+ sender_public_key: PublicKey
220
+ begin: TransferBegin
221
+ chunk_hashes: dict[int, Sha256]
222
+ finished_at: float
223
+
224
+
225
+ def _artifact_metadata(path: Path) -> tuple[int, Sha256]:
226
+ try:
227
+ size_bytes = path.stat().st_size
228
+ except OSError as error:
229
+ raise TransferError("cannot inspect transfer artifact") from error
230
+ if size_bytes <= 0:
231
+ raise TransferError("transfer artifact must not be empty")
232
+ return size_bytes, file_sha256(path)
233
+
234
+
235
+ def _read_chunk(path: Path, *, index: int, chunk_size: int) -> bytes:
236
+ try:
237
+ with path.open("rb") as handle:
238
+ handle.seek(index * chunk_size)
239
+ return handle.read(chunk_size)
240
+ except OSError as error:
241
+ raise TransferError("cannot read transfer artifact chunk") from error
242
+
243
+
244
+ class TransferManager:
245
+ """Bounded multi-chunk transfer with sliding-window selective retry."""
246
+
247
+ def __init__(
248
+ self,
249
+ *,
250
+ sender: EnvelopeSender,
251
+ receiver: Receiver,
252
+ artifact_root: Path,
253
+ max_inflight_bytes: int = 128 * 1024 * 1024,
254
+ event_sink: EventSink | None = None,
255
+ ) -> None:
256
+ self._sender = sender
257
+ self._transport_limits = sender.transport_limits
258
+ self._receiver = receiver
259
+ self._artifact_store = _ArtifactStore(
260
+ artifact_root,
261
+ max_bytes=self._transport_limits.artifact_store_capacity_limit,
262
+ )
263
+ self._max_incoming_reservation_bytes = max_inflight_bytes
264
+ self._incoming_reserved_bytes = 0
265
+ self._event_sink = event_sink
266
+ self._incoming: dict[TransferId, _IncomingTransfer] = {}
267
+ self._completed: dict[TransferId, ArtifactReceipt] = {}
268
+ self._finished: dict[TransferId, _FinishedTransfer] = {}
269
+ self._completed_queue: asyncio.Queue[ArtifactReceipt] = asyncio.Queue(
270
+ maxsize=64
271
+ )
272
+ self._discarded_round_transfers: set[tuple[PublicKey, RoundId]] = set()
273
+ self._ack_waiters: dict[tuple[TransferId, int], asyncio.Future[ChunkAck]] = {}
274
+ self._stop = asyncio.Event()
275
+ self._transfer_task: asyncio.Task[None] | None = None
276
+ self._ack_task: asyncio.Task[None] | None = None
277
+ self._last_timing: TransferTiming | None = None
278
+
279
+ @property
280
+ def last_timing(self) -> TransferTiming | None:
281
+ return self._last_timing
282
+
283
+ async def start(self) -> None:
284
+ if self._transfer_task is None:
285
+ self._transfer_task = asyncio.create_task(
286
+ self._transfer_loop(), name="dromeus-transfer-loop"
287
+ )
288
+ self._ack_task = asyncio.create_task(
289
+ self._ack_loop(), name="dromeus-ack-loop"
290
+ )
291
+
292
+ async def stop(self) -> None:
293
+ self._stop.set()
294
+ tasks = tuple(
295
+ task
296
+ for task in (self._transfer_task, self._ack_task)
297
+ if task is not None
298
+ )
299
+ for task in tasks:
300
+ if not task.done():
301
+ task.cancel()
302
+ await asyncio.gather(*tasks, return_exceptions=True)
303
+ for transfer_id in tuple(self._incoming):
304
+ self._abort_incoming(transfer_id)
305
+ for waiter in self._ack_waiters.values():
306
+ if not waiter.done():
307
+ waiter.cancel()
308
+ self._ack_waiters.clear()
309
+ for transfer_id in tuple(self._completed):
310
+ await asyncio.to_thread(self._artifact_store.abort, transfer_id)
311
+ self._completed.clear()
312
+ self._finished.clear()
313
+ while True:
314
+ try:
315
+ self._completed_queue.get_nowait()
316
+ except asyncio.QueueEmpty:
317
+ break
318
+
319
+ async def send_artifact(
320
+ self,
321
+ *,
322
+ destination: PublicKey,
323
+ artifact_name: Identifier,
324
+ artifact_path: Path,
325
+ codec_id: Identifier,
326
+ tensor_schema: TensorSchema,
327
+ round_id: RoundId | None = None,
328
+ ) -> TransferId:
329
+ started = time.perf_counter()
330
+ retry_count = 0
331
+ try:
332
+ payload_size, total_sha256 = await asyncio.to_thread(
333
+ _artifact_metadata, artifact_path
334
+ )
335
+ if payload_size > self._transport_limits.artifact_size_limit:
336
+ raise TransferError("transfer exceeds manifest artifact limit")
337
+ chunk_size = self._transport_limits.effective_chunk_size
338
+ chunk_count = math.ceil(payload_size / chunk_size)
339
+ transfer_id = str(uuid.uuid4())
340
+ begin = TransferBegin(
341
+ transfer_id=transfer_id,
342
+ artifact_name=artifact_name,
343
+ total_size_bytes=payload_size,
344
+ total_sha256=total_sha256,
345
+ chunk_count=chunk_count,
346
+ codec_id=codec_id,
347
+ tensor_schema=tensor_schema,
348
+ )
349
+ begin_timing = await self._send_message(
350
+ destination=destination,
351
+ message_type=MessageType.TRANSFER_BEGIN,
352
+ message_id=f"{transfer_id}-begin",
353
+ correlation_id=transfer_id,
354
+ payload=encode_message(begin),
355
+ round_id=round_id,
356
+ priority=Priority.CONTROL,
357
+ )
358
+ retry_count += begin_timing.retry_count
359
+ semaphore = asyncio.Semaphore(
360
+ self._transport_limits.effective_window_size
361
+ )
362
+ chunk_tasks = tuple(
363
+ asyncio.create_task(
364
+ self._send_chunk_with_ack(
365
+ semaphore=semaphore,
366
+ destination=destination,
367
+ artifact_path=artifact_path,
368
+ transfer_id=transfer_id,
369
+ begin=begin,
370
+ chunk_index=chunk_index,
371
+ chunk_size=chunk_size,
372
+ round_id=round_id,
373
+ ),
374
+ name=f"dromeus-transfer-chunk-{chunk_index}",
375
+ )
376
+ for chunk_index in range(chunk_count)
377
+ )
378
+ try:
379
+ chunk_retries = await asyncio.gather(*chunk_tasks)
380
+ except BaseException:
381
+ for task in chunk_tasks:
382
+ if not task.done():
383
+ task.cancel()
384
+ await asyncio.gather(*chunk_tasks, return_exceptions=True)
385
+ raise
386
+ # The receiver acknowledges the last chunk only after verifying and
387
+ # materializing the whole artifact, so every ack means delivered.
388
+ retry_count += sum(chunk_retries)
389
+ self._last_timing = TransferTiming(
390
+ elapsed_seconds=time.perf_counter() - started,
391
+ retry_count=retry_count,
392
+ )
393
+ return transfer_id
394
+ except BaseException:
395
+ self._last_timing = TransferTiming(
396
+ elapsed_seconds=time.perf_counter() - started,
397
+ retry_count=retry_count,
398
+ )
399
+ raise
400
+
401
+ async def _send_chunk_with_ack(
402
+ self,
403
+ *,
404
+ semaphore: asyncio.Semaphore,
405
+ destination: PublicKey,
406
+ artifact_path: Path,
407
+ transfer_id: TransferId,
408
+ begin: TransferBegin,
409
+ chunk_index: int,
410
+ chunk_size: int,
411
+ round_id: RoundId | None,
412
+ ) -> int:
413
+ async with semaphore:
414
+ chunk_bytes = await asyncio.to_thread(
415
+ _read_chunk,
416
+ artifact_path,
417
+ index=chunk_index,
418
+ chunk_size=chunk_size,
419
+ )
420
+ if not chunk_bytes:
421
+ raise TransferError("transfer artifact changed during chunking")
422
+ chunk = Chunk(
423
+ transfer_id=transfer_id,
424
+ chunk_index=chunk_index,
425
+ chunk_count=begin.chunk_count,
426
+ chunk_sha256=hashlib.sha256(chunk_bytes).hexdigest(),
427
+ data=chunk_bytes,
428
+ )
429
+ retry_count = 0
430
+ attempts = self._transport_limits.max_retries + 1
431
+ for attempt in range(attempts):
432
+ ack_key = (transfer_id, chunk_index)
433
+ loop = asyncio.get_running_loop()
434
+ ack_future: asyncio.Future[ChunkAck] = loop.create_future()
435
+ self._ack_waiters[ack_key] = ack_future
436
+ try:
437
+ if attempt > 0:
438
+ retry_count += 1
439
+ begin_timing = await self._send_message(
440
+ destination=destination,
441
+ message_type=MessageType.TRANSFER_BEGIN,
442
+ message_id=f"{transfer_id}-begin",
443
+ correlation_id=transfer_id,
444
+ payload=encode_message(begin),
445
+ round_id=round_id,
446
+ priority=Priority.CONTROL,
447
+ )
448
+ retry_count += begin_timing.retry_count
449
+ chunk_timing = await self._send_message(
450
+ destination=destination,
451
+ message_type=MessageType.CHUNK,
452
+ message_id=f"{transfer_id}-chunk-{chunk_index}",
453
+ correlation_id=transfer_id,
454
+ payload=encode_message(chunk),
455
+ round_id=round_id,
456
+ priority=Priority.DATA,
457
+ )
458
+ retry_count += chunk_timing.retry_count
459
+ ack = await asyncio.wait_for(
460
+ asyncio.shield(ack_future),
461
+ timeout=self._transport_limits.retry_timeout_seconds,
462
+ )
463
+ except TimeoutError:
464
+ if self._ack_waiters.get(ack_key) is ack_future:
465
+ self._ack_waiters.pop(ack_key, None)
466
+ ack_future.cancel()
467
+ continue
468
+ except BaseException:
469
+ if self._ack_waiters.get(ack_key) is ack_future:
470
+ self._ack_waiters.pop(ack_key, None)
471
+ ack_future.cancel()
472
+ raise
473
+ if ack.chunk_sha256 != chunk.chunk_sha256:
474
+ raise TransferError("chunk acknowledgement checksum mismatch")
475
+ return retry_count
476
+ raise TransferError("chunk acknowledgement retries exhausted")
477
+
478
+ async def next_artifact(self, *, timeout_seconds: float) -> ArtifactReceipt:
479
+ while True:
480
+ receipt = await asyncio.wait_for(
481
+ self._completed_queue.get(), timeout=timeout_seconds
482
+ )
483
+ if (
484
+ receipt.round_id is not None
485
+ and (receipt.sender_public_key, receipt.round_id)
486
+ in self._discarded_round_transfers
487
+ ):
488
+ await self.release_receipt(receipt)
489
+ continue
490
+ return receipt
491
+
492
+ async def release_receipt(self, receipt: ArtifactReceipt) -> None:
493
+ """Remove one materialized receipt and its artifact."""
494
+ await asyncio.to_thread(receipt.path.unlink, missing_ok=True)
495
+ self.claim_receipt(receipt)
496
+
497
+ def claim_receipt(self, receipt: ArtifactReceipt) -> None:
498
+ """Transfer ownership of one materialized artifact to its consumer."""
499
+ completed = self._completed.pop(receipt.transfer_id, None)
500
+ if completed is not None:
501
+ self._artifact_store.claim(receipt.transfer_id)
502
+ retained: list[ArtifactReceipt] = []
503
+ while True:
504
+ try:
505
+ queued = self._completed_queue.get_nowait()
506
+ except asyncio.QueueEmpty:
507
+ break
508
+ if queued.transfer_id != receipt.transfer_id:
509
+ retained.append(queued)
510
+ for queued in retained:
511
+ self._completed_queue.put_nowait(queued)
512
+
513
+ async def discard_round_transfers(
514
+ self, *, sender: PublicKey, round_id: RoundId
515
+ ) -> None:
516
+ """Reject and remove all transfer state for one failed peer round."""
517
+ self._discarded_round_transfers.add((sender, round_id))
518
+ for transfer_id, incoming in tuple(self._incoming.items()):
519
+ if incoming.sender_public_key == sender and incoming.round_id == round_id:
520
+ self._abort_incoming(transfer_id)
521
+ for receipt in tuple(self._completed.values()):
522
+ if receipt.sender_public_key == sender and receipt.round_id == round_id:
523
+ await self.release_receipt(receipt)
524
+ retained: list[ArtifactReceipt] = []
525
+ while True:
526
+ try:
527
+ receipt = self._completed_queue.get_nowait()
528
+ except asyncio.QueueEmpty:
529
+ break
530
+ if receipt.sender_public_key == sender and receipt.round_id == round_id:
531
+ await self.release_receipt(receipt)
532
+ else:
533
+ retained.append(receipt)
534
+ for receipt in retained:
535
+ self._completed_queue.put_nowait(receipt)
536
+
537
+ async def _transfer_loop(self) -> None:
538
+ while not self._stop.is_set():
539
+ try:
540
+ envelope = await self._receiver.receive(
541
+ MessageChannel.TRANSFER, timeout_seconds=0.1
542
+ )
543
+ except TimeoutError:
544
+ self._expire_incoming()
545
+ continue
546
+ self._expire_incoming()
547
+ try:
548
+ await self._handle_transfer(envelope)
549
+ except (TransferError, ValueError, ProtocolDecodeError) as error:
550
+ emit_event(
551
+ "transfer_message_rejected",
552
+ run_id=envelope.run_id,
553
+ message_id=envelope.message_id,
554
+ transfer_id=envelope.correlation_id,
555
+ peer_id=envelope.sender_public_key,
556
+ round_id=envelope.round_id,
557
+ error=str(error),
558
+ sink=self._event_sink,
559
+ )
560
+ continue
561
+
562
+ async def _ack_loop(self) -> None:
563
+ while not self._stop.is_set():
564
+ try:
565
+ envelope = await self._receiver.receive(
566
+ MessageChannel.ACKNOWLEDGMENT, timeout_seconds=0.1
567
+ )
568
+ except TimeoutError:
569
+ continue
570
+ try:
571
+ ack = decode_message(
572
+ envelope.payload,
573
+ ChunkAck,
574
+ max_bytes=self._transport_limits.message_payload_limit,
575
+ )
576
+ waiter = self._ack_waiters.pop((ack.transfer_id, ack.chunk_index), None)
577
+ if waiter is not None and not waiter.done():
578
+ waiter.set_result(ack)
579
+ except (ValueError, ProtocolDecodeError) as error:
580
+ emit_event(
581
+ "transfer_ack_rejected",
582
+ run_id=envelope.run_id,
583
+ message_id=envelope.message_id,
584
+ transfer_id=envelope.correlation_id,
585
+ peer_id=envelope.sender_public_key,
586
+ round_id=envelope.round_id,
587
+ error=str(error),
588
+ sink=self._event_sink,
589
+ )
590
+
591
+ async def _handle_transfer(self, envelope: Envelope) -> None:
592
+ if envelope.message_type is MessageType.TRANSFER_BEGIN:
593
+ begin = decode_message(
594
+ envelope.payload,
595
+ TransferBegin,
596
+ max_bytes=self._transport_limits.message_payload_limit,
597
+ )
598
+ if (
599
+ envelope.round_id is not None
600
+ and (envelope.sender_public_key, envelope.round_id)
601
+ in self._discarded_round_transfers
602
+ ):
603
+ raise TransferError("round transfers were discarded")
604
+ if begin.total_size_bytes > self._transport_limits.artifact_size_limit:
605
+ raise TransferError("transfer exceeds manifest artifact limit")
606
+ expected_chunk_count = math.ceil(
607
+ begin.total_size_bytes
608
+ / self._transport_limits.effective_chunk_size
609
+ )
610
+ if begin.chunk_count != expected_chunk_count:
611
+ raise TransferError("transfer chunk count does not match manifest")
612
+ incoming = self._incoming.get(begin.transfer_id)
613
+ if incoming is not None:
614
+ if (
615
+ incoming.sender_public_key != envelope.sender_public_key
616
+ or incoming.round_id != envelope.round_id
617
+ or incoming.begin != begin
618
+ ):
619
+ raise TransferError("conflicting duplicate transfer begin")
620
+ return
621
+ finished = self._finished.get(begin.transfer_id)
622
+ if finished is not None:
623
+ if (
624
+ finished.sender_public_key != envelope.sender_public_key
625
+ or finished.begin != begin
626
+ ):
627
+ raise TransferError("conflicting duplicate transfer begin")
628
+ return
629
+ if len(self._incoming) >= self._transport_limits.concurrent_transfer_limit:
630
+ raise TransferError("concurrent transfer limit exceeded")
631
+ if (
632
+ self._incoming_reserved_bytes + begin.total_size_bytes
633
+ > self._max_incoming_reservation_bytes
634
+ ):
635
+ raise TransferError("local incoming reservation limit exceeded")
636
+ self._artifact_store.reserve(
637
+ begin.transfer_id, begin.total_size_bytes
638
+ )
639
+ self._incoming_reserved_bytes += begin.total_size_bytes
640
+ self._incoming[begin.transfer_id] = _IncomingTransfer(
641
+ sender_public_key=envelope.sender_public_key,
642
+ begin=begin,
643
+ round_id=envelope.round_id,
644
+ written_chunk_hashes={},
645
+ started_at=time.monotonic(),
646
+ )
647
+ return
648
+ if envelope.message_type is MessageType.CHUNK:
649
+ chunk = decode_message(
650
+ envelope.payload,
651
+ Chunk,
652
+ max_bytes=self._transport_limits.message_payload_limit,
653
+ )
654
+ incoming = self._incoming.get(chunk.transfer_id)
655
+ if incoming is None:
656
+ finished = self._finished.get(chunk.transfer_id)
657
+ if finished is None:
658
+ raise TransferError("received chunk without transfer begin")
659
+ if (
660
+ finished.sender_public_key != envelope.sender_public_key
661
+ or finished.chunk_hashes.get(chunk.chunk_index)
662
+ != chunk.chunk_sha256
663
+ ):
664
+ raise TransferError("conflicting duplicate transfer chunk")
665
+ await self._acknowledge_chunk(
666
+ envelope.sender_public_key, chunk, envelope.round_id
667
+ )
668
+ return
669
+ if envelope.sender_public_key != incoming.sender_public_key:
670
+ raise TransferError("transfer sender changed")
671
+ if (
672
+ chunk.chunk_count != incoming.begin.chunk_count
673
+ or chunk.chunk_index >= chunk.chunk_count
674
+ ):
675
+ self._abort_incoming(chunk.transfer_id)
676
+ raise TransferError("invalid transfer chunk index or count")
677
+ expected_size = min(
678
+ self._transport_limits.effective_chunk_size,
679
+ incoming.begin.total_size_bytes
680
+ - (
681
+ chunk.chunk_index
682
+ * self._transport_limits.effective_chunk_size
683
+ ),
684
+ )
685
+ if len(chunk.data) != expected_size:
686
+ self._abort_incoming(chunk.transfer_id)
687
+ raise TransferError("transfer chunk size does not match manifest")
688
+ written_hash = incoming.written_chunk_hashes.get(chunk.chunk_index)
689
+ if written_hash is not None:
690
+ if written_hash != chunk.chunk_sha256:
691
+ self._abort_incoming(chunk.transfer_id)
692
+ raise TransferError("conflicting duplicate transfer chunk")
693
+ await self._acknowledge_chunk(
694
+ envelope.sender_public_key, chunk, envelope.round_id
695
+ )
696
+ return
697
+ if hashlib.sha256(chunk.data).hexdigest() != chunk.chunk_sha256:
698
+ raise TransferError("chunk checksum mismatch")
699
+ append_task = asyncio.create_task(
700
+ asyncio.to_thread(
701
+ self._artifact_store.write_chunk,
702
+ chunk.transfer_id,
703
+ offset=(
704
+ chunk.chunk_index
705
+ * self._transport_limits.effective_chunk_size
706
+ ),
707
+ data=chunk.data,
708
+ )
709
+ )
710
+ try:
711
+ await asyncio.shield(append_task)
712
+ except asyncio.CancelledError:
713
+ try:
714
+ await append_task
715
+ finally:
716
+ self._abort_incoming(chunk.transfer_id)
717
+ raise
718
+ except TransferError:
719
+ self._abort_incoming(chunk.transfer_id)
720
+ raise
721
+ incoming.written_chunk_hashes[chunk.chunk_index] = chunk.chunk_sha256
722
+ if len(incoming.written_chunk_hashes) == incoming.begin.chunk_count:
723
+ await self._finish_incoming(chunk.transfer_id, incoming)
724
+ await self._acknowledge_chunk(
725
+ envelope.sender_public_key, chunk, envelope.round_id
726
+ )
727
+ return
728
+ raise TransferError(f"unexpected transfer message {envelope.message_type}")
729
+
730
+ async def _finish_incoming(
731
+ self, transfer_id: TransferId, incoming: _IncomingTransfer
732
+ ) -> None:
733
+ finalize_task = asyncio.create_task(
734
+ asyncio.to_thread(
735
+ self._artifact_store.finalize,
736
+ transfer_id,
737
+ incoming.begin.artifact_name,
738
+ incoming.begin.total_size_bytes,
739
+ incoming.begin.total_sha256,
740
+ )
741
+ )
742
+ try:
743
+ final_path, digest = await asyncio.shield(finalize_task)
744
+ except asyncio.CancelledError:
745
+ try:
746
+ await finalize_task
747
+ finally:
748
+ self._abort_incoming(transfer_id)
749
+ raise
750
+ except TransferError:
751
+ self._abort_incoming(transfer_id)
752
+ raise
753
+ receipt = ArtifactReceipt(
754
+ transfer_id=transfer_id,
755
+ sender_public_key=incoming.sender_public_key,
756
+ artifact_name=incoming.begin.artifact_name,
757
+ path=final_path,
758
+ sha256=digest,
759
+ size_bytes=incoming.begin.total_size_bytes,
760
+ round_id=incoming.round_id,
761
+ codec_id=incoming.begin.codec_id,
762
+ tensor_schema=incoming.begin.tensor_schema,
763
+ )
764
+ self._artifact_store.commit(transfer_id)
765
+ self._incoming.pop(transfer_id, None)
766
+ self._incoming_reserved_bytes -= incoming.begin.total_size_bytes
767
+ self._finished[transfer_id] = _FinishedTransfer(
768
+ sender_public_key=incoming.sender_public_key,
769
+ begin=incoming.begin,
770
+ chunk_hashes=dict(incoming.written_chunk_hashes),
771
+ finished_at=time.monotonic(),
772
+ )
773
+ self._completed[transfer_id] = receipt
774
+ await self._completed_queue.put(receipt)
775
+
776
+ def _expire_incoming(self) -> None:
777
+ lifetime = self._transport_limits.transfer_lifetime_limit
778
+ now = time.monotonic()
779
+ for transfer_id, incoming in tuple(self._incoming.items()):
780
+ if now - incoming.started_at >= lifetime:
781
+ self._abort_incoming(transfer_id)
782
+ for transfer_id, finished in tuple(self._finished.items()):
783
+ if now - finished.finished_at >= lifetime:
784
+ del self._finished[transfer_id]
785
+
786
+ def _abort_incoming(self, transfer_id: TransferId) -> None:
787
+ incoming = self._incoming.pop(transfer_id, None)
788
+ if incoming is not None:
789
+ self._incoming_reserved_bytes -= incoming.begin.total_size_bytes
790
+ if self._incoming_reserved_bytes < 0:
791
+ raise TransferError("incoming reservation accounting underflow")
792
+ self._artifact_store.abort(transfer_id)
793
+
794
+ async def _acknowledge_chunk(
795
+ self,
796
+ destination: PublicKey,
797
+ chunk: Chunk,
798
+ round_id: RoundId | None,
799
+ ) -> None:
800
+ await self._send_message(
801
+ destination=destination,
802
+ message_type=MessageType.CHUNK_ACK,
803
+ message_id=f"{chunk.transfer_id}-ack-{chunk.chunk_index}",
804
+ correlation_id=chunk.transfer_id,
805
+ payload=encode_message(
806
+ ChunkAck(
807
+ transfer_id=chunk.transfer_id,
808
+ chunk_index=chunk.chunk_index,
809
+ chunk_sha256=chunk.chunk_sha256,
810
+ )
811
+ ),
812
+ round_id=round_id,
813
+ priority=Priority.ACK,
814
+ )
815
+
816
+ async def _send_message(
817
+ self,
818
+ *,
819
+ destination: PublicKey,
820
+ message_type: MessageType,
821
+ message_id: MessageId,
822
+ correlation_id: MessageId | None,
823
+ payload: bytes,
824
+ round_id: RoundId | None,
825
+ priority: Priority,
826
+ ) -> SendTiming:
827
+ timing = await self._sender.send(
828
+ destination,
829
+ message_type=message_type,
830
+ message_id=message_id,
831
+ payload=payload,
832
+ priority=priority,
833
+ round_id=round_id,
834
+ correlation_id=correlation_id,
835
+ )
836
+ append_evidence(
837
+ self._event_sink,
838
+ TransferMessageSentEvidence(
839
+ run_id=self._sender.run_id,
840
+ manifest_hash=self._sender.manifest_hash,
841
+ node_id=self._sender.local_public_key,
842
+ message_id=message_id,
843
+ transfer_id=correlation_id,
844
+ peer_id=destination,
845
+ round_id=round_id,
846
+ message_type=message_type.value,
847
+ payload_bytes=len(payload),
848
+ queue_seconds=float(timing.queue_seconds),
849
+ send_seconds=float(timing.send_seconds),
850
+ retry_count=timing.retry_count,
851
+ completion_seconds=float(timing.completion_seconds),
852
+ ),
853
+ )
854
+ return timing