dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,576 @@
|
|
|
1
|
+
"""Membership formation protocol."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from dataclasses import dataclass
|
|
6
|
+
from datetime import UTC, datetime
|
|
7
|
+
from pathlib import Path
|
|
8
|
+
from typing import NoReturn
|
|
9
|
+
|
|
10
|
+
from dromeus.errors import FormationError
|
|
11
|
+
from dromeus.manifests.canonical import (
|
|
12
|
+
canonical_hash,
|
|
13
|
+
canonical_json,
|
|
14
|
+
file_sha256,
|
|
15
|
+
parse_sealed_json,
|
|
16
|
+
validate_sealed_draft,
|
|
17
|
+
)
|
|
18
|
+
from dromeus.manifests.models import (
|
|
19
|
+
ApplicationTaskContract,
|
|
20
|
+
DraftRunSpec,
|
|
21
|
+
EnvironmentFingerprint,
|
|
22
|
+
Invitation,
|
|
23
|
+
Participant,
|
|
24
|
+
PublicKey,
|
|
25
|
+
SealedManifest,
|
|
26
|
+
Sha256,
|
|
27
|
+
TensorSchema,
|
|
28
|
+
TransportLimits,
|
|
29
|
+
)
|
|
30
|
+
from dromeus.protocol.codec import decode_message, encode_message
|
|
31
|
+
from dromeus.protocol.models import (
|
|
32
|
+
Envelope,
|
|
33
|
+
JoinAccepted,
|
|
34
|
+
JoinRequest,
|
|
35
|
+
MessageId,
|
|
36
|
+
MessageType,
|
|
37
|
+
ReadyMessage,
|
|
38
|
+
StartMessage,
|
|
39
|
+
)
|
|
40
|
+
from dromeus.telemetry.events import EventSink, emit_event
|
|
41
|
+
from dromeus.transport.envelope_sender import UNSEALED_MANIFEST_HASH, EnvelopeSender
|
|
42
|
+
from dromeus.transport.interface import AsyncTransport
|
|
43
|
+
from dromeus.transport.outbound_scheduler import OutboundScheduler, Priority
|
|
44
|
+
from dromeus.transport.receiver import MessageChannel, Receiver, ReceiverPolicy
|
|
45
|
+
from dromeus.transport.transfer import TransferManager
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class ReadyValidationError(FormationError, ValueError):
|
|
49
|
+
"""The local node cannot enter the READY barrier."""
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def validate_ready(
|
|
53
|
+
*,
|
|
54
|
+
manifest: SealedManifest,
|
|
55
|
+
local_public_key: PublicKey,
|
|
56
|
+
environment: EnvironmentFingerprint,
|
|
57
|
+
dataset: ApplicationTaskContract,
|
|
58
|
+
checkpoint_hash: Sha256,
|
|
59
|
+
local_tensor_schema: TensorSchema | None = None,
|
|
60
|
+
) -> None:
|
|
61
|
+
if local_public_key not in {
|
|
62
|
+
participant.public_key for participant in manifest.participants
|
|
63
|
+
}:
|
|
64
|
+
raise ReadyValidationError("local public key is not a sealed participant")
|
|
65
|
+
if environment != manifest.environment:
|
|
66
|
+
raise ReadyValidationError("environment fingerprint does not match manifest")
|
|
67
|
+
if dataset != manifest.dataset:
|
|
68
|
+
raise ReadyValidationError("dataset contract does not match manifest")
|
|
69
|
+
if checkpoint_hash != manifest.initial_checkpoint_hash:
|
|
70
|
+
raise ReadyValidationError("checkpoint hash does not match manifest")
|
|
71
|
+
if (
|
|
72
|
+
local_tensor_schema is not None
|
|
73
|
+
and local_tensor_schema != manifest.tensor_schema
|
|
74
|
+
):
|
|
75
|
+
raise ReadyValidationError("local model tensor schema does not match manifest")
|
|
76
|
+
try:
|
|
77
|
+
validate_sealed_draft(manifest)
|
|
78
|
+
except ValueError as error:
|
|
79
|
+
raise ReadyValidationError(str(error)) from error
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def create_invitation(
|
|
83
|
+
*,
|
|
84
|
+
draft: DraftRunSpec,
|
|
85
|
+
initiator_public_key: PublicKey,
|
|
86
|
+
bootstrap_uri: str,
|
|
87
|
+
enrollment_expires_at: datetime | None = None,
|
|
88
|
+
) -> Invitation:
|
|
89
|
+
return Invitation(
|
|
90
|
+
run_id=draft.run_id,
|
|
91
|
+
initiator_public_key=initiator_public_key,
|
|
92
|
+
bootstrap_uri=bootstrap_uri,
|
|
93
|
+
draft_hash=canonical_hash(draft),
|
|
94
|
+
expected_participant_count=draft.participant_count,
|
|
95
|
+
enrollment_expires_at=enrollment_expires_at,
|
|
96
|
+
)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def seal_manifest(
|
|
100
|
+
*,
|
|
101
|
+
draft: DraftRunSpec,
|
|
102
|
+
participant_keys: set[PublicKey],
|
|
103
|
+
initial_checkpoint_hash: Sha256,
|
|
104
|
+
tensor_schema: TensorSchema,
|
|
105
|
+
) -> SealedManifest:
|
|
106
|
+
expected_count = draft.participant_count
|
|
107
|
+
if len(participant_keys) != expected_count:
|
|
108
|
+
raise FormationError("participant key count does not match declared membership")
|
|
109
|
+
ordered_keys = tuple(sorted(participant_keys))
|
|
110
|
+
participants = tuple(
|
|
111
|
+
Participant(public_key=public_key, node_index=node_index)
|
|
112
|
+
for node_index, public_key in enumerate(ordered_keys)
|
|
113
|
+
)
|
|
114
|
+
data = draft.model_dump(mode="python")
|
|
115
|
+
data.update(
|
|
116
|
+
{
|
|
117
|
+
"draft_hash": canonical_hash(draft),
|
|
118
|
+
"participants": participants,
|
|
119
|
+
"initial_checkpoint_hash": initial_checkpoint_hash,
|
|
120
|
+
"tensor_schema": tensor_schema,
|
|
121
|
+
}
|
|
122
|
+
)
|
|
123
|
+
return SealedManifest.model_validate(data)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
@dataclass(frozen=True)
|
|
127
|
+
class FormationResult:
|
|
128
|
+
manifest: SealedManifest
|
|
129
|
+
manifest_hash: Sha256
|
|
130
|
+
checkpoint_path: Path
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True, slots=True)
|
|
134
|
+
class FormationServices:
|
|
135
|
+
"""Validated services retained for post-formation runtime work."""
|
|
136
|
+
|
|
137
|
+
receiver: Receiver
|
|
138
|
+
sender: EnvelopeSender
|
|
139
|
+
transfer_manager: TransferManager
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
class FormationProtocol:
|
|
143
|
+
"""End-to-end fixed-membership formation over receiver/sender/transfer."""
|
|
144
|
+
|
|
145
|
+
def __init__(
|
|
146
|
+
self,
|
|
147
|
+
*,
|
|
148
|
+
transport: AsyncTransport,
|
|
149
|
+
draft: DraftRunSpec,
|
|
150
|
+
environment: EnvironmentFingerprint,
|
|
151
|
+
dataset: ApplicationTaskContract,
|
|
152
|
+
transport_limits: TransportLimits,
|
|
153
|
+
artifact_root: Path,
|
|
154
|
+
local_tensor_schema: TensorSchema | None = None,
|
|
155
|
+
event_sink: EventSink | None = None,
|
|
156
|
+
) -> None:
|
|
157
|
+
self._transport = transport
|
|
158
|
+
self._draft = draft
|
|
159
|
+
self._environment = environment
|
|
160
|
+
self._dataset = dataset
|
|
161
|
+
self._local_tensor_schema = local_tensor_schema
|
|
162
|
+
self._transport_limits = transport_limits
|
|
163
|
+
self._artifact_root = artifact_root
|
|
164
|
+
self._event_sink = event_sink
|
|
165
|
+
self._receiver = Receiver(
|
|
166
|
+
transport,
|
|
167
|
+
ReceiverPolicy(
|
|
168
|
+
run_id=draft.run_id,
|
|
169
|
+
algorithm_id=draft.algorithm_id,
|
|
170
|
+
max_payload_bytes=transport_limits.message_payload_limit,
|
|
171
|
+
),
|
|
172
|
+
event_sink=event_sink,
|
|
173
|
+
)
|
|
174
|
+
self._scheduler = OutboundScheduler(
|
|
175
|
+
transport,
|
|
176
|
+
per_peer_in_flight=transport_limits.effective_window_size,
|
|
177
|
+
)
|
|
178
|
+
self._transfer_manager: TransferManager | None = None
|
|
179
|
+
self._sealed_sender: EnvelopeSender | None = None
|
|
180
|
+
self._formation_complete = False
|
|
181
|
+
|
|
182
|
+
@property
|
|
183
|
+
def services(self) -> FormationServices:
|
|
184
|
+
"""Return transport services after sealed formation completes."""
|
|
185
|
+
if (
|
|
186
|
+
not self._formation_complete
|
|
187
|
+
or self._transfer_manager is None
|
|
188
|
+
or self._sealed_sender is None
|
|
189
|
+
):
|
|
190
|
+
raise FormationError("formation has not completed")
|
|
191
|
+
return FormationServices(
|
|
192
|
+
receiver=self._receiver,
|
|
193
|
+
sender=self._sealed_sender,
|
|
194
|
+
transfer_manager=self._transfer_manager,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
async def start(self) -> None:
|
|
198
|
+
await self._receiver.start()
|
|
199
|
+
await self._scheduler.start()
|
|
200
|
+
|
|
201
|
+
async def stop(self) -> None:
|
|
202
|
+
self._formation_complete = False
|
|
203
|
+
if self._transfer_manager is not None:
|
|
204
|
+
await self._transfer_manager.stop()
|
|
205
|
+
await self._scheduler.stop()
|
|
206
|
+
await self._receiver.stop()
|
|
207
|
+
|
|
208
|
+
async def initiate(
|
|
209
|
+
self,
|
|
210
|
+
*,
|
|
211
|
+
bootstrap_uri: str,
|
|
212
|
+
checkpoint_path: Path,
|
|
213
|
+
tensor_schema: TensorSchema,
|
|
214
|
+
enrollment_expires_at: datetime | None = None,
|
|
215
|
+
) -> FormationResult:
|
|
216
|
+
"""Admit joins until the group is full; refuse joins after enrollment closes."""
|
|
217
|
+
local_key = await self._transport.local_public_key()
|
|
218
|
+
emit_event(
|
|
219
|
+
"formation_started",
|
|
220
|
+
run_id=self._draft.run_id,
|
|
221
|
+
peer_id=local_key,
|
|
222
|
+
role="initiator",
|
|
223
|
+
sink=self._event_sink,
|
|
224
|
+
)
|
|
225
|
+
invitation = create_invitation(
|
|
226
|
+
draft=self._draft,
|
|
227
|
+
initiator_public_key=local_key,
|
|
228
|
+
bootstrap_uri=bootstrap_uri,
|
|
229
|
+
enrollment_expires_at=enrollment_expires_at,
|
|
230
|
+
)
|
|
231
|
+
participant_keys = {local_key}
|
|
232
|
+
participant_count = self._draft.participant_count
|
|
233
|
+
while len(participant_keys) < participant_count:
|
|
234
|
+
envelope = await self._next_join(
|
|
235
|
+
enrollment_expires_at, joined=len(participant_keys)
|
|
236
|
+
)
|
|
237
|
+
if envelope.message_type is not MessageType.JOIN_REQUEST:
|
|
238
|
+
continue
|
|
239
|
+
join = decode_message(
|
|
240
|
+
envelope.payload,
|
|
241
|
+
JoinRequest,
|
|
242
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
243
|
+
)
|
|
244
|
+
if join.draft_hash != invitation.draft_hash:
|
|
245
|
+
continue
|
|
246
|
+
if envelope.sender_public_key in participant_keys:
|
|
247
|
+
continue
|
|
248
|
+
participant_keys.add(envelope.sender_public_key)
|
|
249
|
+
await self._send_control(
|
|
250
|
+
destination=envelope.sender_public_key,
|
|
251
|
+
message_type=MessageType.JOIN_ACCEPTED,
|
|
252
|
+
message_id=f"join-accepted-{len(participant_keys)}",
|
|
253
|
+
payload=encode_message(JoinAccepted(draft_hash=invitation.draft_hash)),
|
|
254
|
+
)
|
|
255
|
+
checkpoint_hash = file_sha256(checkpoint_path)
|
|
256
|
+
manifest = seal_manifest(
|
|
257
|
+
draft=self._draft,
|
|
258
|
+
participant_keys=participant_keys,
|
|
259
|
+
initial_checkpoint_hash=checkpoint_hash,
|
|
260
|
+
tensor_schema=tensor_schema,
|
|
261
|
+
)
|
|
262
|
+
manifest_hash = canonical_hash(manifest)
|
|
263
|
+
self._receiver.configure_sealed_run(
|
|
264
|
+
participant_keys=frozenset(participant_keys),
|
|
265
|
+
manifest_hash=manifest_hash,
|
|
266
|
+
)
|
|
267
|
+
await self._broadcast_manifest(manifest, participant_keys - {local_key})
|
|
268
|
+
transfer = await self._create_transfer_manager(
|
|
269
|
+
local_key, manifest, manifest_hash
|
|
270
|
+
)
|
|
271
|
+
for peer in participant_keys - {local_key}:
|
|
272
|
+
await transfer.send_artifact(
|
|
273
|
+
destination=peer,
|
|
274
|
+
artifact_name="initial-checkpoint",
|
|
275
|
+
artifact_path=checkpoint_path,
|
|
276
|
+
codec_id=self._draft.codec_id,
|
|
277
|
+
tensor_schema=tensor_schema,
|
|
278
|
+
)
|
|
279
|
+
validate_ready(
|
|
280
|
+
manifest=manifest,
|
|
281
|
+
local_public_key=local_key,
|
|
282
|
+
environment=self._environment,
|
|
283
|
+
dataset=self._dataset,
|
|
284
|
+
checkpoint_hash=checkpoint_hash,
|
|
285
|
+
local_tensor_schema=self._local_tensor_schema,
|
|
286
|
+
)
|
|
287
|
+
ready_keys = {local_key}
|
|
288
|
+
while len(ready_keys) < len(manifest.participants):
|
|
289
|
+
envelope = await self._next_control()
|
|
290
|
+
if envelope.message_type is not MessageType.READY:
|
|
291
|
+
continue
|
|
292
|
+
ready = decode_message(
|
|
293
|
+
envelope.payload,
|
|
294
|
+
ReadyMessage,
|
|
295
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
296
|
+
)
|
|
297
|
+
if ready.manifest_hash != manifest_hash:
|
|
298
|
+
continue
|
|
299
|
+
ready_keys.add(envelope.sender_public_key)
|
|
300
|
+
for peer in participant_keys - {local_key}:
|
|
301
|
+
await self._send_control(
|
|
302
|
+
destination=peer,
|
|
303
|
+
message_type=MessageType.START,
|
|
304
|
+
message_id=f"start-{peer[:8]}",
|
|
305
|
+
payload=encode_message(StartMessage(manifest_hash=manifest_hash)),
|
|
306
|
+
manifest_hash=manifest_hash,
|
|
307
|
+
)
|
|
308
|
+
started_keys = {local_key}
|
|
309
|
+
while len(started_keys) < len(manifest.participants):
|
|
310
|
+
envelope = await self._next_control()
|
|
311
|
+
if envelope.message_type is not MessageType.START_ACK:
|
|
312
|
+
continue
|
|
313
|
+
start = decode_message(
|
|
314
|
+
envelope.payload,
|
|
315
|
+
StartMessage,
|
|
316
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
317
|
+
)
|
|
318
|
+
if start.manifest_hash != manifest_hash:
|
|
319
|
+
continue
|
|
320
|
+
started_keys.add(envelope.sender_public_key)
|
|
321
|
+
result = FormationResult(
|
|
322
|
+
manifest=manifest,
|
|
323
|
+
manifest_hash=manifest_hash,
|
|
324
|
+
checkpoint_path=checkpoint_path,
|
|
325
|
+
)
|
|
326
|
+
emit_event(
|
|
327
|
+
"formation_completed",
|
|
328
|
+
run_id=manifest.run_id,
|
|
329
|
+
peer_id=local_key,
|
|
330
|
+
manifest_hash=manifest_hash,
|
|
331
|
+
sink=self._event_sink,
|
|
332
|
+
)
|
|
333
|
+
self._formation_complete = True
|
|
334
|
+
return result
|
|
335
|
+
|
|
336
|
+
async def join(
|
|
337
|
+
self,
|
|
338
|
+
*,
|
|
339
|
+
invitation: Invitation,
|
|
340
|
+
) -> FormationResult:
|
|
341
|
+
if (
|
|
342
|
+
invitation.enrollment_expires_at is not None
|
|
343
|
+
and invitation.enrollment_expires_at <= datetime.now(UTC)
|
|
344
|
+
):
|
|
345
|
+
raise FormationError("invitation has expired")
|
|
346
|
+
if invitation.expected_participant_count != self._draft.participant_count:
|
|
347
|
+
raise FormationError("invitation participant count does not match draft")
|
|
348
|
+
local_key = await self._transport.local_public_key()
|
|
349
|
+
emit_event(
|
|
350
|
+
"formation_started",
|
|
351
|
+
run_id=self._draft.run_id,
|
|
352
|
+
peer_id=local_key,
|
|
353
|
+
role="participant",
|
|
354
|
+
sink=self._event_sink,
|
|
355
|
+
)
|
|
356
|
+
await self._send_control(
|
|
357
|
+
destination=invitation.initiator_public_key,
|
|
358
|
+
message_type=MessageType.JOIN_REQUEST,
|
|
359
|
+
message_id=f"join-{local_key[:8]}",
|
|
360
|
+
payload=encode_message(JoinRequest(draft_hash=invitation.draft_hash)),
|
|
361
|
+
)
|
|
362
|
+
accepted = False
|
|
363
|
+
manifest: SealedManifest | None = None
|
|
364
|
+
manifest_hash: Sha256 | None = None
|
|
365
|
+
while manifest is None:
|
|
366
|
+
envelope = await self._next_control()
|
|
367
|
+
if envelope.message_type is MessageType.JOIN_ACCEPTED:
|
|
368
|
+
join = decode_message(
|
|
369
|
+
envelope.payload,
|
|
370
|
+
JoinAccepted,
|
|
371
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
372
|
+
)
|
|
373
|
+
if join.draft_hash == invitation.draft_hash:
|
|
374
|
+
accepted = True
|
|
375
|
+
continue
|
|
376
|
+
if envelope.message_type is not MessageType.MANIFEST_SEALED:
|
|
377
|
+
continue
|
|
378
|
+
manifest = parse_sealed_json(envelope.payload)
|
|
379
|
+
try:
|
|
380
|
+
validate_sealed_draft(
|
|
381
|
+
manifest, expected_draft_hash=canonical_hash(self._draft)
|
|
382
|
+
)
|
|
383
|
+
except ValueError as error:
|
|
384
|
+
raise FormationError(str(error)) from error
|
|
385
|
+
manifest_hash = canonical_hash(manifest)
|
|
386
|
+
if not accepted:
|
|
387
|
+
raise FormationError("initiator never accepted join request")
|
|
388
|
+
if local_key not in {
|
|
389
|
+
participant.public_key for participant in manifest.participants
|
|
390
|
+
}:
|
|
391
|
+
raise FormationError("local node was not sealed into the manifest")
|
|
392
|
+
assert manifest_hash is not None
|
|
393
|
+
self._receiver.configure_sealed_run(
|
|
394
|
+
participant_keys=frozenset(
|
|
395
|
+
participant.public_key for participant in manifest.participants
|
|
396
|
+
),
|
|
397
|
+
manifest_hash=manifest_hash,
|
|
398
|
+
)
|
|
399
|
+
transfer = await self._create_transfer_manager(
|
|
400
|
+
local_key, manifest, manifest_hash
|
|
401
|
+
)
|
|
402
|
+
try:
|
|
403
|
+
checkpoint = await transfer.next_artifact(
|
|
404
|
+
timeout_seconds=self._formation_timeout_seconds
|
|
405
|
+
)
|
|
406
|
+
except TimeoutError as error:
|
|
407
|
+
raise FormationError("timed out waiting for initial checkpoint") from error
|
|
408
|
+
validate_ready(
|
|
409
|
+
manifest=manifest,
|
|
410
|
+
local_public_key=local_key,
|
|
411
|
+
environment=self._environment,
|
|
412
|
+
dataset=self._dataset,
|
|
413
|
+
checkpoint_hash=checkpoint.sha256,
|
|
414
|
+
local_tensor_schema=self._local_tensor_schema,
|
|
415
|
+
)
|
|
416
|
+
await self._send_control(
|
|
417
|
+
destination=invitation.initiator_public_key,
|
|
418
|
+
message_type=MessageType.READY,
|
|
419
|
+
message_id=f"ready-{local_key[:8]}",
|
|
420
|
+
payload=encode_message(ReadyMessage(manifest_hash=manifest_hash)),
|
|
421
|
+
manifest_hash=manifest_hash,
|
|
422
|
+
)
|
|
423
|
+
while True:
|
|
424
|
+
envelope = await self._next_control()
|
|
425
|
+
if envelope.message_type is not MessageType.START:
|
|
426
|
+
continue
|
|
427
|
+
start = decode_message(
|
|
428
|
+
envelope.payload,
|
|
429
|
+
StartMessage,
|
|
430
|
+
max_bytes=self._transport_limits.max_payload_bytes,
|
|
431
|
+
)
|
|
432
|
+
if start.manifest_hash != manifest_hash:
|
|
433
|
+
continue
|
|
434
|
+
break
|
|
435
|
+
await self._send_control(
|
|
436
|
+
destination=invitation.initiator_public_key,
|
|
437
|
+
message_type=MessageType.START_ACK,
|
|
438
|
+
message_id=f"start-ack-{local_key[:8]}",
|
|
439
|
+
payload=encode_message(StartMessage(manifest_hash=manifest_hash)),
|
|
440
|
+
manifest_hash=manifest_hash,
|
|
441
|
+
)
|
|
442
|
+
transfer.claim_receipt(checkpoint)
|
|
443
|
+
result = FormationResult(
|
|
444
|
+
manifest=manifest,
|
|
445
|
+
manifest_hash=manifest_hash,
|
|
446
|
+
checkpoint_path=checkpoint.path,
|
|
447
|
+
)
|
|
448
|
+
emit_event(
|
|
449
|
+
"formation_completed",
|
|
450
|
+
run_id=manifest.run_id,
|
|
451
|
+
peer_id=local_key,
|
|
452
|
+
manifest_hash=manifest_hash,
|
|
453
|
+
sink=self._event_sink,
|
|
454
|
+
)
|
|
455
|
+
self._formation_complete = True
|
|
456
|
+
return result
|
|
457
|
+
|
|
458
|
+
async def _broadcast_manifest(
|
|
459
|
+
self, manifest: SealedManifest, peers: set[PublicKey]
|
|
460
|
+
) -> None:
|
|
461
|
+
payload = canonical_json(manifest)
|
|
462
|
+
manifest_hash = canonical_hash(manifest)
|
|
463
|
+
for peer in peers:
|
|
464
|
+
await self._send_control(
|
|
465
|
+
destination=peer,
|
|
466
|
+
message_type=MessageType.MANIFEST_SEALED,
|
|
467
|
+
message_id=f"manifest-{peer[:8]}",
|
|
468
|
+
payload=payload,
|
|
469
|
+
manifest_hash=manifest_hash,
|
|
470
|
+
)
|
|
471
|
+
|
|
472
|
+
async def _create_transfer_manager(
|
|
473
|
+
self,
|
|
474
|
+
local_public_key: PublicKey,
|
|
475
|
+
manifest: SealedManifest,
|
|
476
|
+
manifest_hash: Sha256,
|
|
477
|
+
) -> TransferManager:
|
|
478
|
+
sender = EnvelopeSender(
|
|
479
|
+
scheduler=self._scheduler,
|
|
480
|
+
local_public_key=local_public_key,
|
|
481
|
+
run_id=manifest.run_id,
|
|
482
|
+
manifest_hash=manifest_hash,
|
|
483
|
+
algorithm_id=manifest.algorithm_id,
|
|
484
|
+
transport_limits=manifest.transport,
|
|
485
|
+
)
|
|
486
|
+
manager = TransferManager(
|
|
487
|
+
sender=sender,
|
|
488
|
+
receiver=self._receiver,
|
|
489
|
+
artifact_root=self._artifact_root,
|
|
490
|
+
max_inflight_bytes=manifest.transport.artifact_store_capacity_limit,
|
|
491
|
+
event_sink=self._event_sink,
|
|
492
|
+
)
|
|
493
|
+
await manager.start()
|
|
494
|
+
self._transfer_manager = manager
|
|
495
|
+
self._sealed_sender = sender
|
|
496
|
+
return manager
|
|
497
|
+
|
|
498
|
+
async def _next_control(self, timeout_seconds: float | None = None) -> Envelope:
|
|
499
|
+
try:
|
|
500
|
+
return await self._receiver.receive(
|
|
501
|
+
MessageChannel.CONTROL,
|
|
502
|
+
timeout_seconds=self._formation_timeout_seconds
|
|
503
|
+
if timeout_seconds is None
|
|
504
|
+
else timeout_seconds,
|
|
505
|
+
)
|
|
506
|
+
except TimeoutError as error:
|
|
507
|
+
self._fail_formation("formation control deadline exceeded", error)
|
|
508
|
+
|
|
509
|
+
async def _next_join(
|
|
510
|
+
self, enrollment_expires_at: datetime | None, *, joined: int
|
|
511
|
+
) -> Envelope:
|
|
512
|
+
"""Next control message while enrolling; fail once enrollment has closed."""
|
|
513
|
+
if enrollment_expires_at is None:
|
|
514
|
+
return await self._next_control()
|
|
515
|
+
remaining = (enrollment_expires_at - datetime.now(UTC)).total_seconds()
|
|
516
|
+
closed = (
|
|
517
|
+
f"enrollment closed at {enrollment_expires_at.isoformat()} with "
|
|
518
|
+
f"{joined} of {self._draft.participant_count} participants joined"
|
|
519
|
+
)
|
|
520
|
+
if remaining <= 0:
|
|
521
|
+
self._fail_formation(closed)
|
|
522
|
+
if remaining >= self._formation_timeout_seconds:
|
|
523
|
+
return await self._next_control()
|
|
524
|
+
try:
|
|
525
|
+
return await self._receiver.receive(
|
|
526
|
+
MessageChannel.CONTROL, timeout_seconds=remaining
|
|
527
|
+
)
|
|
528
|
+
except TimeoutError as error:
|
|
529
|
+
self._fail_formation(closed, error)
|
|
530
|
+
|
|
531
|
+
def _fail_formation(
|
|
532
|
+
self, message: str, cause: BaseException | None = None
|
|
533
|
+
) -> NoReturn:
|
|
534
|
+
emit_event(
|
|
535
|
+
"formation_failed",
|
|
536
|
+
run_id=self._draft.run_id,
|
|
537
|
+
error=message,
|
|
538
|
+
sink=self._event_sink,
|
|
539
|
+
)
|
|
540
|
+
raise FormationError(message) from cause
|
|
541
|
+
|
|
542
|
+
@property
|
|
543
|
+
def _formation_timeout_seconds(self) -> float:
|
|
544
|
+
retry_budget = self._transport_limits.retry_timeout_seconds * (
|
|
545
|
+
self._transport_limits.max_retries + 4
|
|
546
|
+
)
|
|
547
|
+
transfer_budget = (
|
|
548
|
+
self._transport_limits.transfer_lifetime_limit
|
|
549
|
+
* self._draft.participant_count
|
|
550
|
+
)
|
|
551
|
+
return max(retry_budget, transfer_budget)
|
|
552
|
+
|
|
553
|
+
async def _send_control(
|
|
554
|
+
self,
|
|
555
|
+
*,
|
|
556
|
+
destination: PublicKey,
|
|
557
|
+
message_type: MessageType,
|
|
558
|
+
message_id: MessageId,
|
|
559
|
+
payload: bytes,
|
|
560
|
+
manifest_hash: Sha256 | None = None,
|
|
561
|
+
) -> None:
|
|
562
|
+
sender = EnvelopeSender(
|
|
563
|
+
scheduler=self._scheduler,
|
|
564
|
+
local_public_key=await self._transport.local_public_key(),
|
|
565
|
+
run_id=self._draft.run_id,
|
|
566
|
+
manifest_hash=manifest_hash or UNSEALED_MANIFEST_HASH,
|
|
567
|
+
algorithm_id=self._draft.algorithm_id,
|
|
568
|
+
transport_limits=self._transport_limits,
|
|
569
|
+
)
|
|
570
|
+
await sender.send(
|
|
571
|
+
destination,
|
|
572
|
+
message_type=message_type,
|
|
573
|
+
message_id=message_id,
|
|
574
|
+
payload=payload,
|
|
575
|
+
priority=Priority.CONTROL,
|
|
576
|
+
)
|