dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"""Pairwise gossip runtime."""
|
|
2
|
+
|
|
3
|
+
from dromeus.gossip.axl import AXLFailureBroadcaster, AXLPairTransport
|
|
4
|
+
from dromeus.gossip.engine import GossipEngine
|
|
5
|
+
from dromeus.gossip.interfaces import (
|
|
6
|
+
ConsensusBroadcastResult,
|
|
7
|
+
ConsensusPublisher,
|
|
8
|
+
FailureBroadcaster,
|
|
9
|
+
PairCommitError,
|
|
10
|
+
PairExchangeResult,
|
|
11
|
+
PairTransport,
|
|
12
|
+
RunFailure,
|
|
13
|
+
)
|
|
14
|
+
from dromeus.gossip.peer_scheduler import Pairing, PeerScheduler
|
|
15
|
+
|
|
16
|
+
__all__ = [
|
|
17
|
+
"AXLFailureBroadcaster",
|
|
18
|
+
"AXLPairTransport",
|
|
19
|
+
"ConsensusBroadcastResult",
|
|
20
|
+
"ConsensusPublisher",
|
|
21
|
+
"FailureBroadcaster",
|
|
22
|
+
"GossipEngine",
|
|
23
|
+
"PairCommitError",
|
|
24
|
+
"PairExchangeResult",
|
|
25
|
+
"PairTransport",
|
|
26
|
+
"Pairing",
|
|
27
|
+
"PeerScheduler",
|
|
28
|
+
"RunFailure",
|
|
29
|
+
]
|
dromeus/gossip/_async.py
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
"""Cancellation-safe offloading shared by gossip orchestration and adapters."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
async def run_blocking[T](callback: Callable[..., T], *args: object) -> T:
|
|
10
|
+
task = asyncio.create_task(asyncio.to_thread(callback, *args))
|
|
11
|
+
try:
|
|
12
|
+
return await asyncio.shield(task)
|
|
13
|
+
except asyncio.CancelledError:
|
|
14
|
+
await task
|
|
15
|
+
raise
|
dromeus/gossip/axl.py
ADDED
|
@@ -0,0 +1,497 @@
|
|
|
1
|
+
"""AXL pair exchange and broadcasting over reliable opaque artifact transfer."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import math
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
|
|
9
|
+
import numpy as np
|
|
10
|
+
|
|
11
|
+
from dromeus.algorithms.base import MaterializedArtifact, UpdateBundle
|
|
12
|
+
from dromeus.gossip._async import run_blocking as _run_blocking
|
|
13
|
+
from dromeus.gossip.interfaces import (
|
|
14
|
+
ConsensusBroadcastResult,
|
|
15
|
+
PairCommitError,
|
|
16
|
+
PairExchangeResult,
|
|
17
|
+
RunFailure,
|
|
18
|
+
)
|
|
19
|
+
from dromeus.manifests.canonical import (
|
|
20
|
+
materialize_bundle_metadata,
|
|
21
|
+
parse_bundle_metadata,
|
|
22
|
+
)
|
|
23
|
+
from dromeus.manifests.models import (
|
|
24
|
+
MessageId,
|
|
25
|
+
OpaqueUpdateBundleMetadata,
|
|
26
|
+
PublicKey,
|
|
27
|
+
RoundId,
|
|
28
|
+
)
|
|
29
|
+
from dromeus.protocol.codec import decode_message, encode_message
|
|
30
|
+
from dromeus.protocol.models import (
|
|
31
|
+
Envelope,
|
|
32
|
+
MessageType,
|
|
33
|
+
PairCommitMessage,
|
|
34
|
+
RunFailedMessage,
|
|
35
|
+
)
|
|
36
|
+
from dromeus.telemetry.consensus import encode_sketch
|
|
37
|
+
from dromeus.transport.envelope_sender import EnvelopeSender
|
|
38
|
+
from dromeus.transport.outbound_scheduler import Priority, SendTiming
|
|
39
|
+
from dromeus.transport.receiver import MessageChannel, Receiver
|
|
40
|
+
from dromeus.transport.transfer import ArtifactReceipt, TransferError, TransferManager
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def decode_run_failure(payload: bytes) -> RunFailure:
|
|
44
|
+
"""Decode validated terminal failure evidence from a control envelope."""
|
|
45
|
+
message = decode_message(
|
|
46
|
+
payload,
|
|
47
|
+
RunFailedMessage,
|
|
48
|
+
max_bytes=4096,
|
|
49
|
+
)
|
|
50
|
+
return RunFailure(
|
|
51
|
+
round_id=message.round_id,
|
|
52
|
+
error_type=message.error_type,
|
|
53
|
+
reason=message.reason,
|
|
54
|
+
)
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
class AXLFailureBroadcaster:
|
|
58
|
+
"""Failure-only control broadcaster with no artifact filesystem dependency."""
|
|
59
|
+
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
*,
|
|
63
|
+
sender: EnvelopeSender,
|
|
64
|
+
participant_keys: frozenset[PublicKey],
|
|
65
|
+
) -> None:
|
|
66
|
+
self._sender = sender
|
|
67
|
+
self._participant_keys = participant_keys
|
|
68
|
+
|
|
69
|
+
async def broadcast_run_failed(self, failure: RunFailure) -> None:
|
|
70
|
+
"""Best-effort failure broadcast using manifest-bounded sender retries."""
|
|
71
|
+
peers = self._participant_keys - {self._sender.local_public_key}
|
|
72
|
+
if not peers:
|
|
73
|
+
return
|
|
74
|
+
payload = encode_message(
|
|
75
|
+
RunFailedMessage(
|
|
76
|
+
round_id=failure.round_id,
|
|
77
|
+
error_type=failure.error_type[:128],
|
|
78
|
+
reason=failure.reason[:1024],
|
|
79
|
+
)
|
|
80
|
+
)
|
|
81
|
+
|
|
82
|
+
local_key = self._sender.local_public_key
|
|
83
|
+
await asyncio.gather(
|
|
84
|
+
*(
|
|
85
|
+
self._sender.send(
|
|
86
|
+
peer,
|
|
87
|
+
message_type=MessageType.RUN_FAILED,
|
|
88
|
+
message_id=f"run-failed-{failure.round_id}-{local_key[:8]}",
|
|
89
|
+
payload=payload,
|
|
90
|
+
priority=Priority.CONTROL,
|
|
91
|
+
round_id=failure.round_id,
|
|
92
|
+
correlation_id=f"run-failure-{failure.round_id}",
|
|
93
|
+
)
|
|
94
|
+
for peer in peers
|
|
95
|
+
),
|
|
96
|
+
return_exceptions=True,
|
|
97
|
+
)
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
class AXLPairTransport:
|
|
101
|
+
"""Pair transport backed by reliable opaque artifact transfer."""
|
|
102
|
+
|
|
103
|
+
def __init__(
|
|
104
|
+
self,
|
|
105
|
+
*,
|
|
106
|
+
sender: EnvelopeSender,
|
|
107
|
+
receiver: Receiver,
|
|
108
|
+
transfer_manager: TransferManager,
|
|
109
|
+
metadata_root: Path,
|
|
110
|
+
participant_keys: frozenset[PublicKey] | None = None,
|
|
111
|
+
) -> None:
|
|
112
|
+
self._sender = sender
|
|
113
|
+
self._transport_limits = sender.transport_limits
|
|
114
|
+
self._receiver = receiver
|
|
115
|
+
self._transfer_manager = transfer_manager
|
|
116
|
+
self._metadata_root = metadata_root
|
|
117
|
+
self._participant_keys = participant_keys or frozenset()
|
|
118
|
+
self._failure_broadcaster = AXLFailureBroadcaster(
|
|
119
|
+
sender=sender,
|
|
120
|
+
participant_keys=self._participant_keys,
|
|
121
|
+
)
|
|
122
|
+
self._ready_cache: dict[RoundId, str] = {}
|
|
123
|
+
self._committed_rounds: dict[RoundId, str] = {}
|
|
124
|
+
|
|
125
|
+
async def broadcast_run_failed(self, failure: RunFailure) -> None:
|
|
126
|
+
await self._failure_broadcaster.broadcast_run_failed(failure)
|
|
127
|
+
|
|
128
|
+
async def broadcast_consensus_sketch(
|
|
129
|
+
self, *, round_id: RoundId, sketch: np.ndarray
|
|
130
|
+
) -> ConsensusBroadcastResult:
|
|
131
|
+
"""Best-effort low-priority broadcast of one FP32 consensus sketch."""
|
|
132
|
+
peers = self._participant_keys - {self._sender.local_public_key}
|
|
133
|
+
payload = encode_sketch(sketch)
|
|
134
|
+
if not peers:
|
|
135
|
+
return ConsensusBroadcastResult(
|
|
136
|
+
payload_bytes=len(payload),
|
|
137
|
+
recipient_count=0,
|
|
138
|
+
successful_recipient_count=0,
|
|
139
|
+
retry_count=0,
|
|
140
|
+
)
|
|
141
|
+
|
|
142
|
+
async def send(peer: PublicKey) -> SendTiming:
|
|
143
|
+
return await self._sender.send(
|
|
144
|
+
peer,
|
|
145
|
+
message_type=MessageType.CONSENSUS_SKETCH,
|
|
146
|
+
message_id=(
|
|
147
|
+
f"consensus-sketch-{round_id}-"
|
|
148
|
+
f"{self._sender.local_public_key[:8]}"
|
|
149
|
+
),
|
|
150
|
+
payload=payload,
|
|
151
|
+
priority=Priority.TELEMETRY,
|
|
152
|
+
round_id=round_id,
|
|
153
|
+
correlation_id=f"consensus-round-{round_id}",
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
results = await asyncio.gather(
|
|
157
|
+
*(send(peer) for peer in peers),
|
|
158
|
+
return_exceptions=True,
|
|
159
|
+
)
|
|
160
|
+
timings = [result for result in results if isinstance(result, SendTiming)]
|
|
161
|
+
return ConsensusBroadcastResult(
|
|
162
|
+
payload_bytes=len(payload),
|
|
163
|
+
recipient_count=len(peers),
|
|
164
|
+
successful_recipient_count=len(timings),
|
|
165
|
+
retry_count=sum(timing.retry_count for timing in timings),
|
|
166
|
+
)
|
|
167
|
+
|
|
168
|
+
async def exchange_update(
|
|
169
|
+
self,
|
|
170
|
+
*,
|
|
171
|
+
peer: PublicKey,
|
|
172
|
+
round_id: RoundId,
|
|
173
|
+
bundle: UpdateBundle,
|
|
174
|
+
) -> PairExchangeResult:
|
|
175
|
+
self._validate_bundle(
|
|
176
|
+
bundle.metadata,
|
|
177
|
+
sender=self._sender.local_public_key,
|
|
178
|
+
round_id=round_id,
|
|
179
|
+
)
|
|
180
|
+
await _run_blocking(
|
|
181
|
+
bundle.validate_materialized,
|
|
182
|
+
self._transport_limits.max_update_bundle_bytes,
|
|
183
|
+
)
|
|
184
|
+
receipts: list[ArtifactReceipt] = []
|
|
185
|
+
metadata_receipt: ArtifactReceipt | None = None
|
|
186
|
+
claimed = False
|
|
187
|
+
carrier_task = asyncio.create_task(
|
|
188
|
+
asyncio.to_thread(
|
|
189
|
+
materialize_bundle_metadata,
|
|
190
|
+
bundle.metadata,
|
|
191
|
+
self._metadata_root,
|
|
192
|
+
)
|
|
193
|
+
)
|
|
194
|
+
try:
|
|
195
|
+
metadata_carrier = await asyncio.shield(carrier_task)
|
|
196
|
+
except asyncio.CancelledError:
|
|
197
|
+
metadata_carrier = await carrier_task
|
|
198
|
+
await _run_blocking(metadata_carrier.path.unlink, True)
|
|
199
|
+
raise
|
|
200
|
+
try:
|
|
201
|
+
retry_count = 0
|
|
202
|
+
transfer_id = await self._transfer_manager.send_artifact(
|
|
203
|
+
destination=peer,
|
|
204
|
+
artifact_name="update-bundle-metadata",
|
|
205
|
+
artifact_path=metadata_carrier.path,
|
|
206
|
+
codec_id="safetensors",
|
|
207
|
+
tensor_schema=metadata_carrier.tensor_schema,
|
|
208
|
+
round_id=round_id,
|
|
209
|
+
)
|
|
210
|
+
timing = self._transfer_manager.last_timing
|
|
211
|
+
retry_count += timing.retry_count if timing is not None else 0
|
|
212
|
+
metadata_receipt = await self._next_peer_receipt(
|
|
213
|
+
peer=peer, round_id=round_id
|
|
214
|
+
)
|
|
215
|
+
if metadata_receipt.artifact_name != "update-bundle-metadata":
|
|
216
|
+
raise PairCommitError("peer update metadata artifact is missing")
|
|
217
|
+
peer_metadata = await _run_blocking(
|
|
218
|
+
parse_bundle_metadata, metadata_receipt.path
|
|
219
|
+
)
|
|
220
|
+
self._validate_bundle(peer_metadata, sender=peer, round_id=round_id)
|
|
221
|
+
await self._transfer_manager.release_receipt(metadata_receipt)
|
|
222
|
+
metadata_receipt = None
|
|
223
|
+
|
|
224
|
+
for artifact, materialized in zip(
|
|
225
|
+
bundle.metadata.artifacts,
|
|
226
|
+
bundle.artifacts,
|
|
227
|
+
strict=True,
|
|
228
|
+
):
|
|
229
|
+
transfer_id = (
|
|
230
|
+
await self._transfer_manager.send_artifact(
|
|
231
|
+
destination=peer,
|
|
232
|
+
artifact_name=artifact.name,
|
|
233
|
+
artifact_path=materialized.path,
|
|
234
|
+
codec_id=materialized.transfer_codec_id,
|
|
235
|
+
tensor_schema=materialized.transfer_schema,
|
|
236
|
+
round_id=round_id,
|
|
237
|
+
)
|
|
238
|
+
)
|
|
239
|
+
timing = self._transfer_manager.last_timing
|
|
240
|
+
retry_count += timing.retry_count if timing is not None else 0
|
|
241
|
+
for _ in peer_metadata.artifacts:
|
|
242
|
+
receipt = await self._next_peer_receipt(
|
|
243
|
+
peer=peer, round_id=round_id
|
|
244
|
+
)
|
|
245
|
+
receipts.append(receipt)
|
|
246
|
+
receipt_by_name = {receipt.artifact_name: receipt for receipt in receipts}
|
|
247
|
+
if len(receipt_by_name) != len(receipts):
|
|
248
|
+
raise PairCommitError("peer update contains duplicate artifacts")
|
|
249
|
+
for artifact in peer_metadata.artifacts:
|
|
250
|
+
receipt = receipt_by_name.get(artifact.name)
|
|
251
|
+
if (
|
|
252
|
+
receipt is None
|
|
253
|
+
or receipt.size_bytes != artifact.size_bytes
|
|
254
|
+
or receipt.sha256 != artifact.sha256
|
|
255
|
+
):
|
|
256
|
+
raise PairCommitError("peer update artifact metadata mismatch")
|
|
257
|
+
peer_bundle = UpdateBundle(
|
|
258
|
+
metadata=peer_metadata,
|
|
259
|
+
artifacts=tuple(
|
|
260
|
+
MaterializedArtifact(
|
|
261
|
+
path=receipt_by_name[artifact.name].path,
|
|
262
|
+
transfer_codec_id=receipt_by_name[
|
|
263
|
+
artifact.name
|
|
264
|
+
].codec_id,
|
|
265
|
+
transfer_schema=receipt_by_name[
|
|
266
|
+
artifact.name
|
|
267
|
+
].tensor_schema,
|
|
268
|
+
)
|
|
269
|
+
for artifact in peer_metadata.artifacts
|
|
270
|
+
),
|
|
271
|
+
)
|
|
272
|
+
await _run_blocking(
|
|
273
|
+
peer_bundle.validate_materialized,
|
|
274
|
+
self._transport_limits.max_update_bundle_bytes,
|
|
275
|
+
)
|
|
276
|
+
for receipt in receipts:
|
|
277
|
+
self._transfer_manager.claim_receipt(receipt)
|
|
278
|
+
claimed = True
|
|
279
|
+
return PairExchangeResult(
|
|
280
|
+
bundle=peer_bundle,
|
|
281
|
+
transfer_id=transfer_id,
|
|
282
|
+
retry_count=retry_count,
|
|
283
|
+
)
|
|
284
|
+
except (
|
|
285
|
+
OSError,
|
|
286
|
+
ValueError,
|
|
287
|
+
TypeError,
|
|
288
|
+
TransferError,
|
|
289
|
+
) as error:
|
|
290
|
+
raise PairCommitError("peer update transfer failed") from error
|
|
291
|
+
finally:
|
|
292
|
+
if not claimed:
|
|
293
|
+
await self._transfer_manager.discard_round_transfers(
|
|
294
|
+
sender=peer, round_id=round_id
|
|
295
|
+
)
|
|
296
|
+
for receipt in receipts:
|
|
297
|
+
await self._transfer_manager.release_receipt(receipt)
|
|
298
|
+
if metadata_receipt is not None:
|
|
299
|
+
await self._transfer_manager.release_receipt(metadata_receipt)
|
|
300
|
+
await _run_blocking(metadata_carrier.path.unlink, True)
|
|
301
|
+
|
|
302
|
+
async def _next_peer_receipt(
|
|
303
|
+
self, *, peer: PublicKey, round_id: RoundId
|
|
304
|
+
) -> ArtifactReceipt:
|
|
305
|
+
receipt = await self._transfer_manager.next_artifact(
|
|
306
|
+
timeout_seconds=self._timeout_seconds
|
|
307
|
+
)
|
|
308
|
+
if receipt.sender_public_key != peer or receipt.round_id != round_id:
|
|
309
|
+
await self._transfer_manager.release_receipt(receipt)
|
|
310
|
+
raise PairCommitError("received unexpected peer update artifact")
|
|
311
|
+
return receipt
|
|
312
|
+
|
|
313
|
+
def _validate_bundle(
|
|
314
|
+
self,
|
|
315
|
+
metadata: OpaqueUpdateBundleMetadata,
|
|
316
|
+
*,
|
|
317
|
+
sender: PublicKey,
|
|
318
|
+
round_id: RoundId,
|
|
319
|
+
) -> None:
|
|
320
|
+
if (
|
|
321
|
+
metadata.run_id != self._sender.run_id
|
|
322
|
+
or metadata.manifest_hash != self._sender.manifest_hash
|
|
323
|
+
or metadata.algorithm_id != self._sender.algorithm_id
|
|
324
|
+
or metadata.sender_public_key != sender
|
|
325
|
+
or metadata.round_id != round_id
|
|
326
|
+
):
|
|
327
|
+
raise PairCommitError("update bundle context mismatch")
|
|
328
|
+
if (
|
|
329
|
+
sum(artifact.size_bytes for artifact in metadata.artifacts)
|
|
330
|
+
> self._transport_limits.max_update_bundle_bytes
|
|
331
|
+
):
|
|
332
|
+
raise PairCommitError("update bundle exceeds manifest payload limit")
|
|
333
|
+
|
|
334
|
+
async def exchange_update_ready(
|
|
335
|
+
self,
|
|
336
|
+
*,
|
|
337
|
+
peer: PublicKey,
|
|
338
|
+
round_id: RoundId,
|
|
339
|
+
bundle_checksum: str,
|
|
340
|
+
) -> str:
|
|
341
|
+
cached = self._ready_cache.get(round_id)
|
|
342
|
+
if cached is not None:
|
|
343
|
+
return cached
|
|
344
|
+
envelope = await self._exchange_pair_message(
|
|
345
|
+
destination=peer,
|
|
346
|
+
message_type=MessageType.UPDATE_READY,
|
|
347
|
+
message_id=f"update-ready-{round_id}",
|
|
348
|
+
payload=PairCommitMessage(round_id=round_id, checksum=bundle_checksum),
|
|
349
|
+
round_id=round_id,
|
|
350
|
+
exchange_timeout_seconds=(
|
|
351
|
+
self._transport_limits.transfer_lifetime_limit
|
|
352
|
+
),
|
|
353
|
+
)
|
|
354
|
+
message = decode_message(
|
|
355
|
+
envelope.payload,
|
|
356
|
+
PairCommitMessage,
|
|
357
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
358
|
+
)
|
|
359
|
+
if message.round_id != round_id:
|
|
360
|
+
raise PairCommitError("peer UPDATE_READY round mismatch")
|
|
361
|
+
self._ready_cache[round_id] = message.checksum
|
|
362
|
+
return message.checksum
|
|
363
|
+
|
|
364
|
+
async def exchange_round_committed(
|
|
365
|
+
self,
|
|
366
|
+
*,
|
|
367
|
+
peer: PublicKey,
|
|
368
|
+
round_id: RoundId,
|
|
369
|
+
state_checksum: str,
|
|
370
|
+
) -> None:
|
|
371
|
+
committed_checksum = self._committed_rounds.get(round_id)
|
|
372
|
+
if committed_checksum is not None:
|
|
373
|
+
if committed_checksum != state_checksum:
|
|
374
|
+
raise PairCommitError("duplicate ROUND_COMMITTED checksum mismatch")
|
|
375
|
+
return
|
|
376
|
+
envelope = await self._exchange_pair_message(
|
|
377
|
+
destination=peer,
|
|
378
|
+
message_type=MessageType.ROUND_COMMITTED,
|
|
379
|
+
message_id=f"round-committed-{round_id}",
|
|
380
|
+
payload=PairCommitMessage(round_id=round_id, checksum=state_checksum),
|
|
381
|
+
round_id=round_id,
|
|
382
|
+
)
|
|
383
|
+
message = decode_message(
|
|
384
|
+
envelope.payload,
|
|
385
|
+
PairCommitMessage,
|
|
386
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
387
|
+
)
|
|
388
|
+
if message.round_id != round_id:
|
|
389
|
+
raise PairCommitError("peer ROUND_COMMITTED round mismatch")
|
|
390
|
+
self._receiver.set_current_round(round_id + 1)
|
|
391
|
+
await self._receiver.advance_round(round_id + 1)
|
|
392
|
+
self._committed_rounds[round_id] = state_checksum
|
|
393
|
+
|
|
394
|
+
@property
|
|
395
|
+
def _timeout_seconds(self) -> float:
|
|
396
|
+
return max(
|
|
397
|
+
self._transport_limits.transfer_lifetime_limit,
|
|
398
|
+
self._transport_limits.retry_timeout_seconds
|
|
399
|
+
* (self._transport_limits.max_retries + 4),
|
|
400
|
+
)
|
|
401
|
+
|
|
402
|
+
async def _send_pair_message(
|
|
403
|
+
self,
|
|
404
|
+
*,
|
|
405
|
+
destination: PublicKey,
|
|
406
|
+
message_type: MessageType,
|
|
407
|
+
message_id: MessageId,
|
|
408
|
+
payload: PairCommitMessage,
|
|
409
|
+
round_id: RoundId,
|
|
410
|
+
) -> None:
|
|
411
|
+
await self._sender.send(
|
|
412
|
+
destination,
|
|
413
|
+
message_type=message_type,
|
|
414
|
+
message_id=message_id,
|
|
415
|
+
payload=encode_message(payload),
|
|
416
|
+
priority=Priority.CONTROL,
|
|
417
|
+
round_id=round_id,
|
|
418
|
+
correlation_id=f"pair-round-{round_id}",
|
|
419
|
+
)
|
|
420
|
+
|
|
421
|
+
async def _exchange_pair_message(
|
|
422
|
+
self,
|
|
423
|
+
*,
|
|
424
|
+
destination: PublicKey,
|
|
425
|
+
message_type: MessageType,
|
|
426
|
+
message_id: MessageId,
|
|
427
|
+
payload: PairCommitMessage,
|
|
428
|
+
round_id: RoundId,
|
|
429
|
+
exchange_timeout_seconds: float | None = None,
|
|
430
|
+
) -> Envelope:
|
|
431
|
+
attempts = (
|
|
432
|
+
math.ceil(
|
|
433
|
+
exchange_timeout_seconds
|
|
434
|
+
/ self._transport_limits.retry_timeout_seconds
|
|
435
|
+
)
|
|
436
|
+
if exchange_timeout_seconds is not None
|
|
437
|
+
else self._transport_limits.max_retries + 1
|
|
438
|
+
)
|
|
439
|
+
for attempt in range(attempts):
|
|
440
|
+
await self._send_pair_message(
|
|
441
|
+
destination=destination,
|
|
442
|
+
message_type=message_type,
|
|
443
|
+
message_id=message_id,
|
|
444
|
+
payload=payload,
|
|
445
|
+
round_id=round_id,
|
|
446
|
+
)
|
|
447
|
+
try:
|
|
448
|
+
envelope = await self._receive_pair_message(
|
|
449
|
+
peer=destination,
|
|
450
|
+
message_type=message_type,
|
|
451
|
+
round_id=round_id,
|
|
452
|
+
timeout_seconds=self._transport_limits.retry_timeout_seconds,
|
|
453
|
+
)
|
|
454
|
+
except PairCommitError as error:
|
|
455
|
+
if (
|
|
456
|
+
not isinstance(error.__cause__, TimeoutError)
|
|
457
|
+
or attempt + 1 >= attempts
|
|
458
|
+
):
|
|
459
|
+
raise
|
|
460
|
+
continue
|
|
461
|
+
for _ in range(self._transport_limits.max_retries):
|
|
462
|
+
await self._send_pair_message(
|
|
463
|
+
destination=destination,
|
|
464
|
+
message_type=message_type,
|
|
465
|
+
message_id=message_id,
|
|
466
|
+
payload=payload,
|
|
467
|
+
round_id=round_id,
|
|
468
|
+
)
|
|
469
|
+
return envelope
|
|
470
|
+
raise PairCommitError("pair commit deadline exceeded")
|
|
471
|
+
|
|
472
|
+
async def _receive_pair_message(
|
|
473
|
+
self,
|
|
474
|
+
*,
|
|
475
|
+
peer: PublicKey,
|
|
476
|
+
message_type: MessageType,
|
|
477
|
+
round_id: RoundId,
|
|
478
|
+
timeout_seconds: float | None = None,
|
|
479
|
+
) -> Envelope:
|
|
480
|
+
try:
|
|
481
|
+
envelope = await self._receiver.receive(
|
|
482
|
+
MessageChannel.PAIR_COMMIT,
|
|
483
|
+
timeout_seconds=(
|
|
484
|
+
timeout_seconds
|
|
485
|
+
if timeout_seconds is not None
|
|
486
|
+
else self._timeout_seconds
|
|
487
|
+
),
|
|
488
|
+
)
|
|
489
|
+
except TimeoutError as error:
|
|
490
|
+
raise PairCommitError("pair commit deadline exceeded") from error
|
|
491
|
+
if (
|
|
492
|
+
envelope.sender_public_key != peer
|
|
493
|
+
or envelope.message_type is not message_type
|
|
494
|
+
or envelope.round_id != round_id
|
|
495
|
+
):
|
|
496
|
+
raise PairCommitError("received unexpected pair commit message")
|
|
497
|
+
return envelope
|