dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,854 @@
|
|
|
1
|
+
"""Reliable artifact transfer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import math
|
|
8
|
+
import time
|
|
9
|
+
import uuid
|
|
10
|
+
from dataclasses import dataclass
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
|
|
13
|
+
from dromeus.errors import RunError
|
|
14
|
+
from dromeus.manifests.canonical import file_sha256
|
|
15
|
+
from dromeus.manifests.models import (
|
|
16
|
+
Identifier,
|
|
17
|
+
MessageId,
|
|
18
|
+
PublicKey,
|
|
19
|
+
RoundId,
|
|
20
|
+
Sha256,
|
|
21
|
+
TensorSchema,
|
|
22
|
+
TransferId,
|
|
23
|
+
)
|
|
24
|
+
from dromeus.protocol.codec import (
|
|
25
|
+
ProtocolDecodeError,
|
|
26
|
+
decode_message,
|
|
27
|
+
encode_message,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.protocol.models import (
|
|
30
|
+
Chunk,
|
|
31
|
+
ChunkAck,
|
|
32
|
+
Envelope,
|
|
33
|
+
MessageType,
|
|
34
|
+
TransferBegin,
|
|
35
|
+
)
|
|
36
|
+
from dromeus.telemetry.events import EventSink, emit_event
|
|
37
|
+
from dromeus.telemetry.evidence import (
|
|
38
|
+
TransferMessageSentEvidence,
|
|
39
|
+
append_evidence,
|
|
40
|
+
)
|
|
41
|
+
from dromeus.transport.envelope_sender import EnvelopeSender
|
|
42
|
+
from dromeus.transport.outbound_scheduler import Priority, SendTiming
|
|
43
|
+
from dromeus.transport.receiver import MessageChannel, Receiver
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class TransferError(RunError):
|
|
47
|
+
"""Artifact transfer failed terminally."""
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
@dataclass(frozen=True)
|
|
51
|
+
class ArtifactReceipt:
|
|
52
|
+
transfer_id: TransferId
|
|
53
|
+
sender_public_key: PublicKey
|
|
54
|
+
artifact_name: Identifier
|
|
55
|
+
path: Path
|
|
56
|
+
sha256: Sha256
|
|
57
|
+
size_bytes: int
|
|
58
|
+
round_id: RoundId | None
|
|
59
|
+
codec_id: Identifier
|
|
60
|
+
tensor_schema: TensorSchema
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
@dataclass(frozen=True)
|
|
64
|
+
class TransferTiming:
|
|
65
|
+
"""Observable duration and retry count for one outbound artifact."""
|
|
66
|
+
|
|
67
|
+
elapsed_seconds: float
|
|
68
|
+
retry_count: int
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass
|
|
72
|
+
class _Reservation:
|
|
73
|
+
size_bytes: int
|
|
74
|
+
final_path: Path | None = None
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
class _ArtifactStore:
|
|
78
|
+
"""Private transfer storage with exact per-transfer reservations."""
|
|
79
|
+
|
|
80
|
+
def __init__(self, root: Path, *, max_bytes: int = 128 * 1024 * 1024) -> None:
|
|
81
|
+
if max_bytes <= 0:
|
|
82
|
+
raise ValueError("max_bytes must be positive")
|
|
83
|
+
self._root = root
|
|
84
|
+
self._tmp_root = root / ".tmp"
|
|
85
|
+
self._root.mkdir(parents=True, exist_ok=True)
|
|
86
|
+
self._tmp_root.mkdir(parents=True, exist_ok=True)
|
|
87
|
+
self._max_bytes = max_bytes
|
|
88
|
+
self._reserved_bytes = 0
|
|
89
|
+
self._reservations: dict[TransferId, _Reservation] = {}
|
|
90
|
+
|
|
91
|
+
def reserve(self, transfer_id: TransferId, size_bytes: int) -> None:
|
|
92
|
+
if transfer_id in self._reservations:
|
|
93
|
+
raise TransferError("transfer capacity already reserved")
|
|
94
|
+
if self._reserved_bytes + size_bytes > self._max_bytes:
|
|
95
|
+
raise TransferError("artifact store capacity exceeded")
|
|
96
|
+
temp = self._temp_path(transfer_id)
|
|
97
|
+
try:
|
|
98
|
+
temp.unlink(missing_ok=True)
|
|
99
|
+
temp.touch(exist_ok=False)
|
|
100
|
+
except OSError as error:
|
|
101
|
+
raise TransferError("failed to create transfer artifact") from error
|
|
102
|
+
self._reservations[transfer_id] = _Reservation(size_bytes=size_bytes)
|
|
103
|
+
self._reserved_bytes += size_bytes
|
|
104
|
+
|
|
105
|
+
def append(self, transfer_id: TransferId, data: bytes) -> None:
|
|
106
|
+
reservation = self._reservation(transfer_id)
|
|
107
|
+
if reservation.final_path is not None:
|
|
108
|
+
raise TransferError("cannot append to finalized transfer")
|
|
109
|
+
try:
|
|
110
|
+
with self._temp_path(transfer_id).open("ab") as handle:
|
|
111
|
+
handle.write(data)
|
|
112
|
+
except OSError as error:
|
|
113
|
+
raise TransferError("failed to write transfer artifact") from error
|
|
114
|
+
|
|
115
|
+
def write_chunk(
|
|
116
|
+
self,
|
|
117
|
+
transfer_id: TransferId,
|
|
118
|
+
*,
|
|
119
|
+
offset: int,
|
|
120
|
+
data: bytes,
|
|
121
|
+
) -> None:
|
|
122
|
+
reservation = self._reservation(transfer_id)
|
|
123
|
+
if reservation.final_path is not None:
|
|
124
|
+
raise TransferError("cannot write to finalized transfer")
|
|
125
|
+
if offset < 0 or offset + len(data) > reservation.size_bytes:
|
|
126
|
+
raise TransferError("chunk write exceeds transfer reservation")
|
|
127
|
+
try:
|
|
128
|
+
with self._temp_path(transfer_id).open("r+b") as handle:
|
|
129
|
+
handle.seek(offset)
|
|
130
|
+
handle.write(data)
|
|
131
|
+
except OSError as error:
|
|
132
|
+
raise TransferError("failed to write transfer chunk") from error
|
|
133
|
+
|
|
134
|
+
def finalize(
|
|
135
|
+
self,
|
|
136
|
+
transfer_id: TransferId,
|
|
137
|
+
artifact_name: Identifier,
|
|
138
|
+
expected_size_bytes: int,
|
|
139
|
+
expected_sha256: Sha256,
|
|
140
|
+
) -> tuple[Path, Sha256]:
|
|
141
|
+
reservation = self._reservation(transfer_id)
|
|
142
|
+
if reservation.size_bytes != expected_size_bytes:
|
|
143
|
+
raise TransferError("transfer reservation size mismatch")
|
|
144
|
+
temp = self._temp_path(transfer_id)
|
|
145
|
+
try:
|
|
146
|
+
if temp.stat().st_size != expected_size_bytes:
|
|
147
|
+
raise TransferError("final artifact size mismatch")
|
|
148
|
+
digest = file_sha256(temp)
|
|
149
|
+
if digest != expected_sha256:
|
|
150
|
+
raise TransferError("final artifact checksum mismatch")
|
|
151
|
+
final = self._final_path(transfer_id, artifact_name)
|
|
152
|
+
if final.exists():
|
|
153
|
+
raise TransferError("final artifact already exists")
|
|
154
|
+
temp.replace(final)
|
|
155
|
+
except OSError as error:
|
|
156
|
+
raise TransferError("failed to finalize transfer artifact") from error
|
|
157
|
+
reservation.final_path = final
|
|
158
|
+
return final, digest
|
|
159
|
+
|
|
160
|
+
def commit(self, transfer_id: TransferId) -> None:
|
|
161
|
+
reservation = self._reservation(transfer_id)
|
|
162
|
+
if reservation.final_path is None:
|
|
163
|
+
raise TransferError("cannot commit unfinished transfer")
|
|
164
|
+
|
|
165
|
+
def claim(self, transfer_id: TransferId) -> None:
|
|
166
|
+
reservation = self._reservation(transfer_id)
|
|
167
|
+
if reservation.final_path is None:
|
|
168
|
+
raise TransferError("cannot claim unfinished transfer")
|
|
169
|
+
self._release(transfer_id)
|
|
170
|
+
|
|
171
|
+
def abort(self, transfer_id: TransferId) -> None:
|
|
172
|
+
reservation = self._reservation(transfer_id)
|
|
173
|
+
try:
|
|
174
|
+
self._temp_path(transfer_id).unlink(missing_ok=True)
|
|
175
|
+
if reservation.final_path is not None:
|
|
176
|
+
reservation.final_path.unlink(missing_ok=True)
|
|
177
|
+
except OSError as error:
|
|
178
|
+
raise TransferError("failed to remove transfer artifact") from error
|
|
179
|
+
self._release(transfer_id)
|
|
180
|
+
|
|
181
|
+
def _reservation(self, transfer_id: TransferId) -> _Reservation:
|
|
182
|
+
reservation = self._reservations.get(transfer_id)
|
|
183
|
+
if reservation is None:
|
|
184
|
+
raise TransferError("transfer capacity is not reserved")
|
|
185
|
+
return reservation
|
|
186
|
+
|
|
187
|
+
def _release(self, transfer_id: TransferId) -> None:
|
|
188
|
+
reservation = self._reservations.pop(transfer_id, None)
|
|
189
|
+
if reservation is None:
|
|
190
|
+
raise TransferError("transfer capacity already released")
|
|
191
|
+
remaining = self._reserved_bytes - reservation.size_bytes
|
|
192
|
+
if remaining < 0:
|
|
193
|
+
raise TransferError("transfer capacity accounting underflow")
|
|
194
|
+
self._reserved_bytes = remaining
|
|
195
|
+
|
|
196
|
+
def _temp_path(self, transfer_id: TransferId) -> Path:
|
|
197
|
+
return self._tmp_root / f"{transfer_id}.part"
|
|
198
|
+
|
|
199
|
+
def _final_path(
|
|
200
|
+
self, transfer_id: TransferId, artifact_name: Identifier
|
|
201
|
+
) -> Path:
|
|
202
|
+
safe_name = artifact_name.replace("/", "_")
|
|
203
|
+
return self._root / f"{transfer_id}-{safe_name}.bin"
|
|
204
|
+
|
|
205
|
+
|
|
206
|
+
@dataclass
|
|
207
|
+
class _IncomingTransfer:
|
|
208
|
+
sender_public_key: PublicKey
|
|
209
|
+
begin: TransferBegin
|
|
210
|
+
round_id: RoundId | None
|
|
211
|
+
written_chunk_hashes: dict[int, Sha256]
|
|
212
|
+
started_at: float
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
@dataclass(frozen=True)
|
|
216
|
+
class _FinishedTransfer:
|
|
217
|
+
"""Kept for one transfer lifetime so retried messages are acknowledged."""
|
|
218
|
+
|
|
219
|
+
sender_public_key: PublicKey
|
|
220
|
+
begin: TransferBegin
|
|
221
|
+
chunk_hashes: dict[int, Sha256]
|
|
222
|
+
finished_at: float
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def _artifact_metadata(path: Path) -> tuple[int, Sha256]:
|
|
226
|
+
try:
|
|
227
|
+
size_bytes = path.stat().st_size
|
|
228
|
+
except OSError as error:
|
|
229
|
+
raise TransferError("cannot inspect transfer artifact") from error
|
|
230
|
+
if size_bytes <= 0:
|
|
231
|
+
raise TransferError("transfer artifact must not be empty")
|
|
232
|
+
return size_bytes, file_sha256(path)
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def _read_chunk(path: Path, *, index: int, chunk_size: int) -> bytes:
|
|
236
|
+
try:
|
|
237
|
+
with path.open("rb") as handle:
|
|
238
|
+
handle.seek(index * chunk_size)
|
|
239
|
+
return handle.read(chunk_size)
|
|
240
|
+
except OSError as error:
|
|
241
|
+
raise TransferError("cannot read transfer artifact chunk") from error
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
class TransferManager:
|
|
245
|
+
"""Bounded multi-chunk transfer with sliding-window selective retry."""
|
|
246
|
+
|
|
247
|
+
def __init__(
|
|
248
|
+
self,
|
|
249
|
+
*,
|
|
250
|
+
sender: EnvelopeSender,
|
|
251
|
+
receiver: Receiver,
|
|
252
|
+
artifact_root: Path,
|
|
253
|
+
max_inflight_bytes: int = 128 * 1024 * 1024,
|
|
254
|
+
event_sink: EventSink | None = None,
|
|
255
|
+
) -> None:
|
|
256
|
+
self._sender = sender
|
|
257
|
+
self._transport_limits = sender.transport_limits
|
|
258
|
+
self._receiver = receiver
|
|
259
|
+
self._artifact_store = _ArtifactStore(
|
|
260
|
+
artifact_root,
|
|
261
|
+
max_bytes=self._transport_limits.artifact_store_capacity_limit,
|
|
262
|
+
)
|
|
263
|
+
self._max_incoming_reservation_bytes = max_inflight_bytes
|
|
264
|
+
self._incoming_reserved_bytes = 0
|
|
265
|
+
self._event_sink = event_sink
|
|
266
|
+
self._incoming: dict[TransferId, _IncomingTransfer] = {}
|
|
267
|
+
self._completed: dict[TransferId, ArtifactReceipt] = {}
|
|
268
|
+
self._finished: dict[TransferId, _FinishedTransfer] = {}
|
|
269
|
+
self._completed_queue: asyncio.Queue[ArtifactReceipt] = asyncio.Queue(
|
|
270
|
+
maxsize=64
|
|
271
|
+
)
|
|
272
|
+
self._discarded_round_transfers: set[tuple[PublicKey, RoundId]] = set()
|
|
273
|
+
self._ack_waiters: dict[tuple[TransferId, int], asyncio.Future[ChunkAck]] = {}
|
|
274
|
+
self._stop = asyncio.Event()
|
|
275
|
+
self._transfer_task: asyncio.Task[None] | None = None
|
|
276
|
+
self._ack_task: asyncio.Task[None] | None = None
|
|
277
|
+
self._last_timing: TransferTiming | None = None
|
|
278
|
+
|
|
279
|
+
@property
|
|
280
|
+
def last_timing(self) -> TransferTiming | None:
|
|
281
|
+
return self._last_timing
|
|
282
|
+
|
|
283
|
+
async def start(self) -> None:
|
|
284
|
+
if self._transfer_task is None:
|
|
285
|
+
self._transfer_task = asyncio.create_task(
|
|
286
|
+
self._transfer_loop(), name="dromeus-transfer-loop"
|
|
287
|
+
)
|
|
288
|
+
self._ack_task = asyncio.create_task(
|
|
289
|
+
self._ack_loop(), name="dromeus-ack-loop"
|
|
290
|
+
)
|
|
291
|
+
|
|
292
|
+
async def stop(self) -> None:
|
|
293
|
+
self._stop.set()
|
|
294
|
+
tasks = tuple(
|
|
295
|
+
task
|
|
296
|
+
for task in (self._transfer_task, self._ack_task)
|
|
297
|
+
if task is not None
|
|
298
|
+
)
|
|
299
|
+
for task in tasks:
|
|
300
|
+
if not task.done():
|
|
301
|
+
task.cancel()
|
|
302
|
+
await asyncio.gather(*tasks, return_exceptions=True)
|
|
303
|
+
for transfer_id in tuple(self._incoming):
|
|
304
|
+
self._abort_incoming(transfer_id)
|
|
305
|
+
for waiter in self._ack_waiters.values():
|
|
306
|
+
if not waiter.done():
|
|
307
|
+
waiter.cancel()
|
|
308
|
+
self._ack_waiters.clear()
|
|
309
|
+
for transfer_id in tuple(self._completed):
|
|
310
|
+
await asyncio.to_thread(self._artifact_store.abort, transfer_id)
|
|
311
|
+
self._completed.clear()
|
|
312
|
+
self._finished.clear()
|
|
313
|
+
while True:
|
|
314
|
+
try:
|
|
315
|
+
self._completed_queue.get_nowait()
|
|
316
|
+
except asyncio.QueueEmpty:
|
|
317
|
+
break
|
|
318
|
+
|
|
319
|
+
async def send_artifact(
|
|
320
|
+
self,
|
|
321
|
+
*,
|
|
322
|
+
destination: PublicKey,
|
|
323
|
+
artifact_name: Identifier,
|
|
324
|
+
artifact_path: Path,
|
|
325
|
+
codec_id: Identifier,
|
|
326
|
+
tensor_schema: TensorSchema,
|
|
327
|
+
round_id: RoundId | None = None,
|
|
328
|
+
) -> TransferId:
|
|
329
|
+
started = time.perf_counter()
|
|
330
|
+
retry_count = 0
|
|
331
|
+
try:
|
|
332
|
+
payload_size, total_sha256 = await asyncio.to_thread(
|
|
333
|
+
_artifact_metadata, artifact_path
|
|
334
|
+
)
|
|
335
|
+
if payload_size > self._transport_limits.artifact_size_limit:
|
|
336
|
+
raise TransferError("transfer exceeds manifest artifact limit")
|
|
337
|
+
chunk_size = self._transport_limits.effective_chunk_size
|
|
338
|
+
chunk_count = math.ceil(payload_size / chunk_size)
|
|
339
|
+
transfer_id = str(uuid.uuid4())
|
|
340
|
+
begin = TransferBegin(
|
|
341
|
+
transfer_id=transfer_id,
|
|
342
|
+
artifact_name=artifact_name,
|
|
343
|
+
total_size_bytes=payload_size,
|
|
344
|
+
total_sha256=total_sha256,
|
|
345
|
+
chunk_count=chunk_count,
|
|
346
|
+
codec_id=codec_id,
|
|
347
|
+
tensor_schema=tensor_schema,
|
|
348
|
+
)
|
|
349
|
+
begin_timing = await self._send_message(
|
|
350
|
+
destination=destination,
|
|
351
|
+
message_type=MessageType.TRANSFER_BEGIN,
|
|
352
|
+
message_id=f"{transfer_id}-begin",
|
|
353
|
+
correlation_id=transfer_id,
|
|
354
|
+
payload=encode_message(begin),
|
|
355
|
+
round_id=round_id,
|
|
356
|
+
priority=Priority.CONTROL,
|
|
357
|
+
)
|
|
358
|
+
retry_count += begin_timing.retry_count
|
|
359
|
+
semaphore = asyncio.Semaphore(
|
|
360
|
+
self._transport_limits.effective_window_size
|
|
361
|
+
)
|
|
362
|
+
chunk_tasks = tuple(
|
|
363
|
+
asyncio.create_task(
|
|
364
|
+
self._send_chunk_with_ack(
|
|
365
|
+
semaphore=semaphore,
|
|
366
|
+
destination=destination,
|
|
367
|
+
artifact_path=artifact_path,
|
|
368
|
+
transfer_id=transfer_id,
|
|
369
|
+
begin=begin,
|
|
370
|
+
chunk_index=chunk_index,
|
|
371
|
+
chunk_size=chunk_size,
|
|
372
|
+
round_id=round_id,
|
|
373
|
+
),
|
|
374
|
+
name=f"dromeus-transfer-chunk-{chunk_index}",
|
|
375
|
+
)
|
|
376
|
+
for chunk_index in range(chunk_count)
|
|
377
|
+
)
|
|
378
|
+
try:
|
|
379
|
+
chunk_retries = await asyncio.gather(*chunk_tasks)
|
|
380
|
+
except BaseException:
|
|
381
|
+
for task in chunk_tasks:
|
|
382
|
+
if not task.done():
|
|
383
|
+
task.cancel()
|
|
384
|
+
await asyncio.gather(*chunk_tasks, return_exceptions=True)
|
|
385
|
+
raise
|
|
386
|
+
# The receiver acknowledges the last chunk only after verifying and
|
|
387
|
+
# materializing the whole artifact, so every ack means delivered.
|
|
388
|
+
retry_count += sum(chunk_retries)
|
|
389
|
+
self._last_timing = TransferTiming(
|
|
390
|
+
elapsed_seconds=time.perf_counter() - started,
|
|
391
|
+
retry_count=retry_count,
|
|
392
|
+
)
|
|
393
|
+
return transfer_id
|
|
394
|
+
except BaseException:
|
|
395
|
+
self._last_timing = TransferTiming(
|
|
396
|
+
elapsed_seconds=time.perf_counter() - started,
|
|
397
|
+
retry_count=retry_count,
|
|
398
|
+
)
|
|
399
|
+
raise
|
|
400
|
+
|
|
401
|
+
async def _send_chunk_with_ack(
|
|
402
|
+
self,
|
|
403
|
+
*,
|
|
404
|
+
semaphore: asyncio.Semaphore,
|
|
405
|
+
destination: PublicKey,
|
|
406
|
+
artifact_path: Path,
|
|
407
|
+
transfer_id: TransferId,
|
|
408
|
+
begin: TransferBegin,
|
|
409
|
+
chunk_index: int,
|
|
410
|
+
chunk_size: int,
|
|
411
|
+
round_id: RoundId | None,
|
|
412
|
+
) -> int:
|
|
413
|
+
async with semaphore:
|
|
414
|
+
chunk_bytes = await asyncio.to_thread(
|
|
415
|
+
_read_chunk,
|
|
416
|
+
artifact_path,
|
|
417
|
+
index=chunk_index,
|
|
418
|
+
chunk_size=chunk_size,
|
|
419
|
+
)
|
|
420
|
+
if not chunk_bytes:
|
|
421
|
+
raise TransferError("transfer artifact changed during chunking")
|
|
422
|
+
chunk = Chunk(
|
|
423
|
+
transfer_id=transfer_id,
|
|
424
|
+
chunk_index=chunk_index,
|
|
425
|
+
chunk_count=begin.chunk_count,
|
|
426
|
+
chunk_sha256=hashlib.sha256(chunk_bytes).hexdigest(),
|
|
427
|
+
data=chunk_bytes,
|
|
428
|
+
)
|
|
429
|
+
retry_count = 0
|
|
430
|
+
attempts = self._transport_limits.max_retries + 1
|
|
431
|
+
for attempt in range(attempts):
|
|
432
|
+
ack_key = (transfer_id, chunk_index)
|
|
433
|
+
loop = asyncio.get_running_loop()
|
|
434
|
+
ack_future: asyncio.Future[ChunkAck] = loop.create_future()
|
|
435
|
+
self._ack_waiters[ack_key] = ack_future
|
|
436
|
+
try:
|
|
437
|
+
if attempt > 0:
|
|
438
|
+
retry_count += 1
|
|
439
|
+
begin_timing = await self._send_message(
|
|
440
|
+
destination=destination,
|
|
441
|
+
message_type=MessageType.TRANSFER_BEGIN,
|
|
442
|
+
message_id=f"{transfer_id}-begin",
|
|
443
|
+
correlation_id=transfer_id,
|
|
444
|
+
payload=encode_message(begin),
|
|
445
|
+
round_id=round_id,
|
|
446
|
+
priority=Priority.CONTROL,
|
|
447
|
+
)
|
|
448
|
+
retry_count += begin_timing.retry_count
|
|
449
|
+
chunk_timing = await self._send_message(
|
|
450
|
+
destination=destination,
|
|
451
|
+
message_type=MessageType.CHUNK,
|
|
452
|
+
message_id=f"{transfer_id}-chunk-{chunk_index}",
|
|
453
|
+
correlation_id=transfer_id,
|
|
454
|
+
payload=encode_message(chunk),
|
|
455
|
+
round_id=round_id,
|
|
456
|
+
priority=Priority.DATA,
|
|
457
|
+
)
|
|
458
|
+
retry_count += chunk_timing.retry_count
|
|
459
|
+
ack = await asyncio.wait_for(
|
|
460
|
+
asyncio.shield(ack_future),
|
|
461
|
+
timeout=self._transport_limits.retry_timeout_seconds,
|
|
462
|
+
)
|
|
463
|
+
except TimeoutError:
|
|
464
|
+
if self._ack_waiters.get(ack_key) is ack_future:
|
|
465
|
+
self._ack_waiters.pop(ack_key, None)
|
|
466
|
+
ack_future.cancel()
|
|
467
|
+
continue
|
|
468
|
+
except BaseException:
|
|
469
|
+
if self._ack_waiters.get(ack_key) is ack_future:
|
|
470
|
+
self._ack_waiters.pop(ack_key, None)
|
|
471
|
+
ack_future.cancel()
|
|
472
|
+
raise
|
|
473
|
+
if ack.chunk_sha256 != chunk.chunk_sha256:
|
|
474
|
+
raise TransferError("chunk acknowledgement checksum mismatch")
|
|
475
|
+
return retry_count
|
|
476
|
+
raise TransferError("chunk acknowledgement retries exhausted")
|
|
477
|
+
|
|
478
|
+
async def next_artifact(self, *, timeout_seconds: float) -> ArtifactReceipt:
|
|
479
|
+
while True:
|
|
480
|
+
receipt = await asyncio.wait_for(
|
|
481
|
+
self._completed_queue.get(), timeout=timeout_seconds
|
|
482
|
+
)
|
|
483
|
+
if (
|
|
484
|
+
receipt.round_id is not None
|
|
485
|
+
and (receipt.sender_public_key, receipt.round_id)
|
|
486
|
+
in self._discarded_round_transfers
|
|
487
|
+
):
|
|
488
|
+
await self.release_receipt(receipt)
|
|
489
|
+
continue
|
|
490
|
+
return receipt
|
|
491
|
+
|
|
492
|
+
async def release_receipt(self, receipt: ArtifactReceipt) -> None:
|
|
493
|
+
"""Remove one materialized receipt and its artifact."""
|
|
494
|
+
await asyncio.to_thread(receipt.path.unlink, missing_ok=True)
|
|
495
|
+
self.claim_receipt(receipt)
|
|
496
|
+
|
|
497
|
+
def claim_receipt(self, receipt: ArtifactReceipt) -> None:
|
|
498
|
+
"""Transfer ownership of one materialized artifact to its consumer."""
|
|
499
|
+
completed = self._completed.pop(receipt.transfer_id, None)
|
|
500
|
+
if completed is not None:
|
|
501
|
+
self._artifact_store.claim(receipt.transfer_id)
|
|
502
|
+
retained: list[ArtifactReceipt] = []
|
|
503
|
+
while True:
|
|
504
|
+
try:
|
|
505
|
+
queued = self._completed_queue.get_nowait()
|
|
506
|
+
except asyncio.QueueEmpty:
|
|
507
|
+
break
|
|
508
|
+
if queued.transfer_id != receipt.transfer_id:
|
|
509
|
+
retained.append(queued)
|
|
510
|
+
for queued in retained:
|
|
511
|
+
self._completed_queue.put_nowait(queued)
|
|
512
|
+
|
|
513
|
+
async def discard_round_transfers(
|
|
514
|
+
self, *, sender: PublicKey, round_id: RoundId
|
|
515
|
+
) -> None:
|
|
516
|
+
"""Reject and remove all transfer state for one failed peer round."""
|
|
517
|
+
self._discarded_round_transfers.add((sender, round_id))
|
|
518
|
+
for transfer_id, incoming in tuple(self._incoming.items()):
|
|
519
|
+
if incoming.sender_public_key == sender and incoming.round_id == round_id:
|
|
520
|
+
self._abort_incoming(transfer_id)
|
|
521
|
+
for receipt in tuple(self._completed.values()):
|
|
522
|
+
if receipt.sender_public_key == sender and receipt.round_id == round_id:
|
|
523
|
+
await self.release_receipt(receipt)
|
|
524
|
+
retained: list[ArtifactReceipt] = []
|
|
525
|
+
while True:
|
|
526
|
+
try:
|
|
527
|
+
receipt = self._completed_queue.get_nowait()
|
|
528
|
+
except asyncio.QueueEmpty:
|
|
529
|
+
break
|
|
530
|
+
if receipt.sender_public_key == sender and receipt.round_id == round_id:
|
|
531
|
+
await self.release_receipt(receipt)
|
|
532
|
+
else:
|
|
533
|
+
retained.append(receipt)
|
|
534
|
+
for receipt in retained:
|
|
535
|
+
self._completed_queue.put_nowait(receipt)
|
|
536
|
+
|
|
537
|
+
async def _transfer_loop(self) -> None:
|
|
538
|
+
while not self._stop.is_set():
|
|
539
|
+
try:
|
|
540
|
+
envelope = await self._receiver.receive(
|
|
541
|
+
MessageChannel.TRANSFER, timeout_seconds=0.1
|
|
542
|
+
)
|
|
543
|
+
except TimeoutError:
|
|
544
|
+
self._expire_incoming()
|
|
545
|
+
continue
|
|
546
|
+
self._expire_incoming()
|
|
547
|
+
try:
|
|
548
|
+
await self._handle_transfer(envelope)
|
|
549
|
+
except (TransferError, ValueError, ProtocolDecodeError) as error:
|
|
550
|
+
emit_event(
|
|
551
|
+
"transfer_message_rejected",
|
|
552
|
+
run_id=envelope.run_id,
|
|
553
|
+
message_id=envelope.message_id,
|
|
554
|
+
transfer_id=envelope.correlation_id,
|
|
555
|
+
peer_id=envelope.sender_public_key,
|
|
556
|
+
round_id=envelope.round_id,
|
|
557
|
+
error=str(error),
|
|
558
|
+
sink=self._event_sink,
|
|
559
|
+
)
|
|
560
|
+
continue
|
|
561
|
+
|
|
562
|
+
async def _ack_loop(self) -> None:
|
|
563
|
+
while not self._stop.is_set():
|
|
564
|
+
try:
|
|
565
|
+
envelope = await self._receiver.receive(
|
|
566
|
+
MessageChannel.ACKNOWLEDGMENT, timeout_seconds=0.1
|
|
567
|
+
)
|
|
568
|
+
except TimeoutError:
|
|
569
|
+
continue
|
|
570
|
+
try:
|
|
571
|
+
ack = decode_message(
|
|
572
|
+
envelope.payload,
|
|
573
|
+
ChunkAck,
|
|
574
|
+
max_bytes=self._transport_limits.message_payload_limit,
|
|
575
|
+
)
|
|
576
|
+
waiter = self._ack_waiters.pop((ack.transfer_id, ack.chunk_index), None)
|
|
577
|
+
if waiter is not None and not waiter.done():
|
|
578
|
+
waiter.set_result(ack)
|
|
579
|
+
except (ValueError, ProtocolDecodeError) as error:
|
|
580
|
+
emit_event(
|
|
581
|
+
"transfer_ack_rejected",
|
|
582
|
+
run_id=envelope.run_id,
|
|
583
|
+
message_id=envelope.message_id,
|
|
584
|
+
transfer_id=envelope.correlation_id,
|
|
585
|
+
peer_id=envelope.sender_public_key,
|
|
586
|
+
round_id=envelope.round_id,
|
|
587
|
+
error=str(error),
|
|
588
|
+
sink=self._event_sink,
|
|
589
|
+
)
|
|
590
|
+
|
|
591
|
+
async def _handle_transfer(self, envelope: Envelope) -> None:
|
|
592
|
+
if envelope.message_type is MessageType.TRANSFER_BEGIN:
|
|
593
|
+
begin = decode_message(
|
|
594
|
+
envelope.payload,
|
|
595
|
+
TransferBegin,
|
|
596
|
+
max_bytes=self._transport_limits.message_payload_limit,
|
|
597
|
+
)
|
|
598
|
+
if (
|
|
599
|
+
envelope.round_id is not None
|
|
600
|
+
and (envelope.sender_public_key, envelope.round_id)
|
|
601
|
+
in self._discarded_round_transfers
|
|
602
|
+
):
|
|
603
|
+
raise TransferError("round transfers were discarded")
|
|
604
|
+
if begin.total_size_bytes > self._transport_limits.artifact_size_limit:
|
|
605
|
+
raise TransferError("transfer exceeds manifest artifact limit")
|
|
606
|
+
expected_chunk_count = math.ceil(
|
|
607
|
+
begin.total_size_bytes
|
|
608
|
+
/ self._transport_limits.effective_chunk_size
|
|
609
|
+
)
|
|
610
|
+
if begin.chunk_count != expected_chunk_count:
|
|
611
|
+
raise TransferError("transfer chunk count does not match manifest")
|
|
612
|
+
incoming = self._incoming.get(begin.transfer_id)
|
|
613
|
+
if incoming is not None:
|
|
614
|
+
if (
|
|
615
|
+
incoming.sender_public_key != envelope.sender_public_key
|
|
616
|
+
or incoming.round_id != envelope.round_id
|
|
617
|
+
or incoming.begin != begin
|
|
618
|
+
):
|
|
619
|
+
raise TransferError("conflicting duplicate transfer begin")
|
|
620
|
+
return
|
|
621
|
+
finished = self._finished.get(begin.transfer_id)
|
|
622
|
+
if finished is not None:
|
|
623
|
+
if (
|
|
624
|
+
finished.sender_public_key != envelope.sender_public_key
|
|
625
|
+
or finished.begin != begin
|
|
626
|
+
):
|
|
627
|
+
raise TransferError("conflicting duplicate transfer begin")
|
|
628
|
+
return
|
|
629
|
+
if len(self._incoming) >= self._transport_limits.concurrent_transfer_limit:
|
|
630
|
+
raise TransferError("concurrent transfer limit exceeded")
|
|
631
|
+
if (
|
|
632
|
+
self._incoming_reserved_bytes + begin.total_size_bytes
|
|
633
|
+
> self._max_incoming_reservation_bytes
|
|
634
|
+
):
|
|
635
|
+
raise TransferError("local incoming reservation limit exceeded")
|
|
636
|
+
self._artifact_store.reserve(
|
|
637
|
+
begin.transfer_id, begin.total_size_bytes
|
|
638
|
+
)
|
|
639
|
+
self._incoming_reserved_bytes += begin.total_size_bytes
|
|
640
|
+
self._incoming[begin.transfer_id] = _IncomingTransfer(
|
|
641
|
+
sender_public_key=envelope.sender_public_key,
|
|
642
|
+
begin=begin,
|
|
643
|
+
round_id=envelope.round_id,
|
|
644
|
+
written_chunk_hashes={},
|
|
645
|
+
started_at=time.monotonic(),
|
|
646
|
+
)
|
|
647
|
+
return
|
|
648
|
+
if envelope.message_type is MessageType.CHUNK:
|
|
649
|
+
chunk = decode_message(
|
|
650
|
+
envelope.payload,
|
|
651
|
+
Chunk,
|
|
652
|
+
max_bytes=self._transport_limits.message_payload_limit,
|
|
653
|
+
)
|
|
654
|
+
incoming = self._incoming.get(chunk.transfer_id)
|
|
655
|
+
if incoming is None:
|
|
656
|
+
finished = self._finished.get(chunk.transfer_id)
|
|
657
|
+
if finished is None:
|
|
658
|
+
raise TransferError("received chunk without transfer begin")
|
|
659
|
+
if (
|
|
660
|
+
finished.sender_public_key != envelope.sender_public_key
|
|
661
|
+
or finished.chunk_hashes.get(chunk.chunk_index)
|
|
662
|
+
!= chunk.chunk_sha256
|
|
663
|
+
):
|
|
664
|
+
raise TransferError("conflicting duplicate transfer chunk")
|
|
665
|
+
await self._acknowledge_chunk(
|
|
666
|
+
envelope.sender_public_key, chunk, envelope.round_id
|
|
667
|
+
)
|
|
668
|
+
return
|
|
669
|
+
if envelope.sender_public_key != incoming.sender_public_key:
|
|
670
|
+
raise TransferError("transfer sender changed")
|
|
671
|
+
if (
|
|
672
|
+
chunk.chunk_count != incoming.begin.chunk_count
|
|
673
|
+
or chunk.chunk_index >= chunk.chunk_count
|
|
674
|
+
):
|
|
675
|
+
self._abort_incoming(chunk.transfer_id)
|
|
676
|
+
raise TransferError("invalid transfer chunk index or count")
|
|
677
|
+
expected_size = min(
|
|
678
|
+
self._transport_limits.effective_chunk_size,
|
|
679
|
+
incoming.begin.total_size_bytes
|
|
680
|
+
- (
|
|
681
|
+
chunk.chunk_index
|
|
682
|
+
* self._transport_limits.effective_chunk_size
|
|
683
|
+
),
|
|
684
|
+
)
|
|
685
|
+
if len(chunk.data) != expected_size:
|
|
686
|
+
self._abort_incoming(chunk.transfer_id)
|
|
687
|
+
raise TransferError("transfer chunk size does not match manifest")
|
|
688
|
+
written_hash = incoming.written_chunk_hashes.get(chunk.chunk_index)
|
|
689
|
+
if written_hash is not None:
|
|
690
|
+
if written_hash != chunk.chunk_sha256:
|
|
691
|
+
self._abort_incoming(chunk.transfer_id)
|
|
692
|
+
raise TransferError("conflicting duplicate transfer chunk")
|
|
693
|
+
await self._acknowledge_chunk(
|
|
694
|
+
envelope.sender_public_key, chunk, envelope.round_id
|
|
695
|
+
)
|
|
696
|
+
return
|
|
697
|
+
if hashlib.sha256(chunk.data).hexdigest() != chunk.chunk_sha256:
|
|
698
|
+
raise TransferError("chunk checksum mismatch")
|
|
699
|
+
append_task = asyncio.create_task(
|
|
700
|
+
asyncio.to_thread(
|
|
701
|
+
self._artifact_store.write_chunk,
|
|
702
|
+
chunk.transfer_id,
|
|
703
|
+
offset=(
|
|
704
|
+
chunk.chunk_index
|
|
705
|
+
* self._transport_limits.effective_chunk_size
|
|
706
|
+
),
|
|
707
|
+
data=chunk.data,
|
|
708
|
+
)
|
|
709
|
+
)
|
|
710
|
+
try:
|
|
711
|
+
await asyncio.shield(append_task)
|
|
712
|
+
except asyncio.CancelledError:
|
|
713
|
+
try:
|
|
714
|
+
await append_task
|
|
715
|
+
finally:
|
|
716
|
+
self._abort_incoming(chunk.transfer_id)
|
|
717
|
+
raise
|
|
718
|
+
except TransferError:
|
|
719
|
+
self._abort_incoming(chunk.transfer_id)
|
|
720
|
+
raise
|
|
721
|
+
incoming.written_chunk_hashes[chunk.chunk_index] = chunk.chunk_sha256
|
|
722
|
+
if len(incoming.written_chunk_hashes) == incoming.begin.chunk_count:
|
|
723
|
+
await self._finish_incoming(chunk.transfer_id, incoming)
|
|
724
|
+
await self._acknowledge_chunk(
|
|
725
|
+
envelope.sender_public_key, chunk, envelope.round_id
|
|
726
|
+
)
|
|
727
|
+
return
|
|
728
|
+
raise TransferError(f"unexpected transfer message {envelope.message_type}")
|
|
729
|
+
|
|
730
|
+
async def _finish_incoming(
|
|
731
|
+
self, transfer_id: TransferId, incoming: _IncomingTransfer
|
|
732
|
+
) -> None:
|
|
733
|
+
finalize_task = asyncio.create_task(
|
|
734
|
+
asyncio.to_thread(
|
|
735
|
+
self._artifact_store.finalize,
|
|
736
|
+
transfer_id,
|
|
737
|
+
incoming.begin.artifact_name,
|
|
738
|
+
incoming.begin.total_size_bytes,
|
|
739
|
+
incoming.begin.total_sha256,
|
|
740
|
+
)
|
|
741
|
+
)
|
|
742
|
+
try:
|
|
743
|
+
final_path, digest = await asyncio.shield(finalize_task)
|
|
744
|
+
except asyncio.CancelledError:
|
|
745
|
+
try:
|
|
746
|
+
await finalize_task
|
|
747
|
+
finally:
|
|
748
|
+
self._abort_incoming(transfer_id)
|
|
749
|
+
raise
|
|
750
|
+
except TransferError:
|
|
751
|
+
self._abort_incoming(transfer_id)
|
|
752
|
+
raise
|
|
753
|
+
receipt = ArtifactReceipt(
|
|
754
|
+
transfer_id=transfer_id,
|
|
755
|
+
sender_public_key=incoming.sender_public_key,
|
|
756
|
+
artifact_name=incoming.begin.artifact_name,
|
|
757
|
+
path=final_path,
|
|
758
|
+
sha256=digest,
|
|
759
|
+
size_bytes=incoming.begin.total_size_bytes,
|
|
760
|
+
round_id=incoming.round_id,
|
|
761
|
+
codec_id=incoming.begin.codec_id,
|
|
762
|
+
tensor_schema=incoming.begin.tensor_schema,
|
|
763
|
+
)
|
|
764
|
+
self._artifact_store.commit(transfer_id)
|
|
765
|
+
self._incoming.pop(transfer_id, None)
|
|
766
|
+
self._incoming_reserved_bytes -= incoming.begin.total_size_bytes
|
|
767
|
+
self._finished[transfer_id] = _FinishedTransfer(
|
|
768
|
+
sender_public_key=incoming.sender_public_key,
|
|
769
|
+
begin=incoming.begin,
|
|
770
|
+
chunk_hashes=dict(incoming.written_chunk_hashes),
|
|
771
|
+
finished_at=time.monotonic(),
|
|
772
|
+
)
|
|
773
|
+
self._completed[transfer_id] = receipt
|
|
774
|
+
await self._completed_queue.put(receipt)
|
|
775
|
+
|
|
776
|
+
def _expire_incoming(self) -> None:
|
|
777
|
+
lifetime = self._transport_limits.transfer_lifetime_limit
|
|
778
|
+
now = time.monotonic()
|
|
779
|
+
for transfer_id, incoming in tuple(self._incoming.items()):
|
|
780
|
+
if now - incoming.started_at >= lifetime:
|
|
781
|
+
self._abort_incoming(transfer_id)
|
|
782
|
+
for transfer_id, finished in tuple(self._finished.items()):
|
|
783
|
+
if now - finished.finished_at >= lifetime:
|
|
784
|
+
del self._finished[transfer_id]
|
|
785
|
+
|
|
786
|
+
def _abort_incoming(self, transfer_id: TransferId) -> None:
|
|
787
|
+
incoming = self._incoming.pop(transfer_id, None)
|
|
788
|
+
if incoming is not None:
|
|
789
|
+
self._incoming_reserved_bytes -= incoming.begin.total_size_bytes
|
|
790
|
+
if self._incoming_reserved_bytes < 0:
|
|
791
|
+
raise TransferError("incoming reservation accounting underflow")
|
|
792
|
+
self._artifact_store.abort(transfer_id)
|
|
793
|
+
|
|
794
|
+
async def _acknowledge_chunk(
|
|
795
|
+
self,
|
|
796
|
+
destination: PublicKey,
|
|
797
|
+
chunk: Chunk,
|
|
798
|
+
round_id: RoundId | None,
|
|
799
|
+
) -> None:
|
|
800
|
+
await self._send_message(
|
|
801
|
+
destination=destination,
|
|
802
|
+
message_type=MessageType.CHUNK_ACK,
|
|
803
|
+
message_id=f"{chunk.transfer_id}-ack-{chunk.chunk_index}",
|
|
804
|
+
correlation_id=chunk.transfer_id,
|
|
805
|
+
payload=encode_message(
|
|
806
|
+
ChunkAck(
|
|
807
|
+
transfer_id=chunk.transfer_id,
|
|
808
|
+
chunk_index=chunk.chunk_index,
|
|
809
|
+
chunk_sha256=chunk.chunk_sha256,
|
|
810
|
+
)
|
|
811
|
+
),
|
|
812
|
+
round_id=round_id,
|
|
813
|
+
priority=Priority.ACK,
|
|
814
|
+
)
|
|
815
|
+
|
|
816
|
+
async def _send_message(
|
|
817
|
+
self,
|
|
818
|
+
*,
|
|
819
|
+
destination: PublicKey,
|
|
820
|
+
message_type: MessageType,
|
|
821
|
+
message_id: MessageId,
|
|
822
|
+
correlation_id: MessageId | None,
|
|
823
|
+
payload: bytes,
|
|
824
|
+
round_id: RoundId | None,
|
|
825
|
+
priority: Priority,
|
|
826
|
+
) -> SendTiming:
|
|
827
|
+
timing = await self._sender.send(
|
|
828
|
+
destination,
|
|
829
|
+
message_type=message_type,
|
|
830
|
+
message_id=message_id,
|
|
831
|
+
payload=payload,
|
|
832
|
+
priority=priority,
|
|
833
|
+
round_id=round_id,
|
|
834
|
+
correlation_id=correlation_id,
|
|
835
|
+
)
|
|
836
|
+
append_evidence(
|
|
837
|
+
self._event_sink,
|
|
838
|
+
TransferMessageSentEvidence(
|
|
839
|
+
run_id=self._sender.run_id,
|
|
840
|
+
manifest_hash=self._sender.manifest_hash,
|
|
841
|
+
node_id=self._sender.local_public_key,
|
|
842
|
+
message_id=message_id,
|
|
843
|
+
transfer_id=correlation_id,
|
|
844
|
+
peer_id=destination,
|
|
845
|
+
round_id=round_id,
|
|
846
|
+
message_type=message_type.value,
|
|
847
|
+
payload_bytes=len(payload),
|
|
848
|
+
queue_seconds=float(timing.queue_seconds),
|
|
849
|
+
send_seconds=float(timing.send_seconds),
|
|
850
|
+
retry_count=timing.retry_count,
|
|
851
|
+
completion_seconds=float(timing.completion_seconds),
|
|
852
|
+
),
|
|
853
|
+
)
|
|
854
|
+
return timing
|