dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,326 @@
|
|
|
1
|
+
"""Outbound message scheduling."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import heapq
|
|
7
|
+
import time
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from enum import IntEnum
|
|
10
|
+
|
|
11
|
+
from dromeus.manifests.models import PublicKey
|
|
12
|
+
from dromeus.transport.interface import AsyncTransport
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class Priority(IntEnum):
|
|
16
|
+
ACK = 0
|
|
17
|
+
CONTROL = 10
|
|
18
|
+
DATA = 20
|
|
19
|
+
TELEMETRY = 30
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass(frozen=True)
|
|
23
|
+
class SendTiming:
|
|
24
|
+
queue_seconds: float
|
|
25
|
+
send_seconds: float
|
|
26
|
+
retry_count: int
|
|
27
|
+
completion_seconds: float
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(order=True)
|
|
31
|
+
class _ScheduledSend:
|
|
32
|
+
priority: int
|
|
33
|
+
sequence: int
|
|
34
|
+
destination: PublicKey = field(compare=False)
|
|
35
|
+
payload: bytes = field(compare=False)
|
|
36
|
+
retries: int = field(compare=False)
|
|
37
|
+
retry_delay_seconds: float = field(compare=False)
|
|
38
|
+
result: asyncio.Future[SendTiming] = field(compare=False)
|
|
39
|
+
enqueued_at: float = field(compare=False)
|
|
40
|
+
admission: asyncio.Semaphore = field(compare=False)
|
|
41
|
+
retry_count: int = field(default=0, compare=False)
|
|
42
|
+
first_attempt_at: float | None = field(default=None, compare=False)
|
|
43
|
+
send_seconds: float = field(default=0.0, compare=False)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def is_control(self) -> bool:
|
|
47
|
+
return self.priority <= Priority.CONTROL
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class OutboundScheduler:
|
|
51
|
+
"""Bounded priority sender with independent per-peer control and bulk lanes."""
|
|
52
|
+
|
|
53
|
+
def __init__(
|
|
54
|
+
self,
|
|
55
|
+
transport: AsyncTransport,
|
|
56
|
+
*,
|
|
57
|
+
max_queue_size: int = 256,
|
|
58
|
+
max_control_queue_size: int = 64,
|
|
59
|
+
per_peer_in_flight: int = 1,
|
|
60
|
+
) -> None:
|
|
61
|
+
if max_queue_size <= 0:
|
|
62
|
+
raise ValueError("max_queue_size must be positive")
|
|
63
|
+
if max_control_queue_size <= 0:
|
|
64
|
+
raise ValueError("max_control_queue_size must be positive")
|
|
65
|
+
if per_peer_in_flight <= 0:
|
|
66
|
+
raise ValueError("per_peer_in_flight must be positive")
|
|
67
|
+
self._transport = transport
|
|
68
|
+
self._queue = asyncio.PriorityQueue[_ScheduledSend]()
|
|
69
|
+
self._control_admission = asyncio.Semaphore(max_control_queue_size)
|
|
70
|
+
self._bulk_admission = asyncio.Semaphore(max_queue_size)
|
|
71
|
+
self._per_peer_limit = per_peer_in_flight
|
|
72
|
+
self._task: asyncio.Task[None] | None = None
|
|
73
|
+
self._stop = asyncio.Event()
|
|
74
|
+
self._wake = asyncio.Event()
|
|
75
|
+
self._sequence = 0
|
|
76
|
+
self._delay_sequence = 0
|
|
77
|
+
self._delayed: list[tuple[float, int, _ScheduledSend]] = []
|
|
78
|
+
self._peer_control_active: set[PublicKey] = set()
|
|
79
|
+
self._peer_bulk_active: dict[PublicKey, int] = {}
|
|
80
|
+
self._active_tasks: set[asyncio.Task[None]] = set()
|
|
81
|
+
self._admission_waiters: set[asyncio.Future[None]] = set()
|
|
82
|
+
|
|
83
|
+
async def start(self) -> None:
|
|
84
|
+
if self._stop.is_set():
|
|
85
|
+
raise RuntimeError("outbound scheduler is stopped")
|
|
86
|
+
if self._task is None:
|
|
87
|
+
self._task = asyncio.create_task(self._run(), name="dromeus-send")
|
|
88
|
+
|
|
89
|
+
async def stop(self) -> None:
|
|
90
|
+
self._stop.set()
|
|
91
|
+
self._wake.set()
|
|
92
|
+
if self._admission_waiters:
|
|
93
|
+
await asyncio.gather(*tuple(self._admission_waiters))
|
|
94
|
+
if self._task is not None:
|
|
95
|
+
await self._task
|
|
96
|
+
else:
|
|
97
|
+
self._fail_queued()
|
|
98
|
+
if self._active_tasks:
|
|
99
|
+
await asyncio.gather(*tuple(self._active_tasks))
|
|
100
|
+
|
|
101
|
+
async def send(
|
|
102
|
+
self,
|
|
103
|
+
destination: PublicKey,
|
|
104
|
+
payload: bytes,
|
|
105
|
+
*,
|
|
106
|
+
priority: Priority,
|
|
107
|
+
retries: int = 0,
|
|
108
|
+
retry_delay_seconds: float = 0.1,
|
|
109
|
+
) -> SendTiming:
|
|
110
|
+
if self._stop.is_set():
|
|
111
|
+
raise RuntimeError("outbound scheduler is stopped")
|
|
112
|
+
loop = asyncio.get_running_loop()
|
|
113
|
+
admission = (
|
|
114
|
+
self._control_admission
|
|
115
|
+
if priority <= Priority.CONTROL
|
|
116
|
+
else self._bulk_admission
|
|
117
|
+
)
|
|
118
|
+
if admission.locked():
|
|
119
|
+
admission_waiter: asyncio.Future[None] = loop.create_future()
|
|
120
|
+
self._admission_waiters.add(admission_waiter)
|
|
121
|
+
try:
|
|
122
|
+
await self._acquire_admission(admission)
|
|
123
|
+
finally:
|
|
124
|
+
self._admission_waiters.discard(admission_waiter)
|
|
125
|
+
admission_waiter.set_result(None)
|
|
126
|
+
else:
|
|
127
|
+
await admission.acquire()
|
|
128
|
+
enqueued_at = time.monotonic()
|
|
129
|
+
future: asyncio.Future[SendTiming] = loop.create_future()
|
|
130
|
+
self._sequence += 1
|
|
131
|
+
scheduled = _ScheduledSend(
|
|
132
|
+
priority=int(priority),
|
|
133
|
+
sequence=self._sequence,
|
|
134
|
+
destination=destination,
|
|
135
|
+
payload=payload,
|
|
136
|
+
retries=retries,
|
|
137
|
+
retry_delay_seconds=retry_delay_seconds,
|
|
138
|
+
result=future,
|
|
139
|
+
enqueued_at=enqueued_at,
|
|
140
|
+
admission=admission,
|
|
141
|
+
)
|
|
142
|
+
future.add_done_callback(lambda _: self._wake.set())
|
|
143
|
+
await self._queue.put(scheduled)
|
|
144
|
+
self._wake.set()
|
|
145
|
+
return await future
|
|
146
|
+
|
|
147
|
+
async def _acquire_admission(self, admission: asyncio.Semaphore) -> None:
|
|
148
|
+
acquire = asyncio.create_task(admission.acquire())
|
|
149
|
+
stopping = asyncio.create_task(self._stop.wait())
|
|
150
|
+
acquired = False
|
|
151
|
+
try:
|
|
152
|
+
done, _ = await asyncio.wait(
|
|
153
|
+
(acquire, stopping), return_when=asyncio.FIRST_COMPLETED
|
|
154
|
+
)
|
|
155
|
+
if stopping in done:
|
|
156
|
+
raise RuntimeError("outbound scheduler is stopped")
|
|
157
|
+
acquired = acquire.result()
|
|
158
|
+
if self._stop.is_set():
|
|
159
|
+
raise RuntimeError("outbound scheduler is stopped")
|
|
160
|
+
except BaseException:
|
|
161
|
+
if (
|
|
162
|
+
not acquired
|
|
163
|
+
and acquire.done()
|
|
164
|
+
and not acquire.cancelled()
|
|
165
|
+
and acquire.exception() is None
|
|
166
|
+
):
|
|
167
|
+
acquired = acquire.result()
|
|
168
|
+
if acquired:
|
|
169
|
+
admission.release()
|
|
170
|
+
raise
|
|
171
|
+
finally:
|
|
172
|
+
for task in (acquire, stopping):
|
|
173
|
+
if not task.done():
|
|
174
|
+
task.cancel()
|
|
175
|
+
|
|
176
|
+
async def _run(self) -> None:
|
|
177
|
+
while True:
|
|
178
|
+
self._wake.clear()
|
|
179
|
+
if self._stop.is_set():
|
|
180
|
+
self._fail_queued()
|
|
181
|
+
return
|
|
182
|
+
self._promote_delayed()
|
|
183
|
+
self._dispatch_ready()
|
|
184
|
+
timeout = (
|
|
185
|
+
max(0.0, self._delayed[0][0] - time.monotonic())
|
|
186
|
+
if self._delayed
|
|
187
|
+
else None
|
|
188
|
+
)
|
|
189
|
+
try:
|
|
190
|
+
await asyncio.wait_for(self._wake.wait(), timeout=timeout)
|
|
191
|
+
except TimeoutError:
|
|
192
|
+
pass
|
|
193
|
+
|
|
194
|
+
def _fail_queued(self) -> None:
|
|
195
|
+
while not self._queue.empty():
|
|
196
|
+
scheduled = self._queue.get_nowait()
|
|
197
|
+
self._queue.task_done()
|
|
198
|
+
if not scheduled.result.done():
|
|
199
|
+
scheduled.result.set_exception(
|
|
200
|
+
RuntimeError("outbound scheduler is stopped")
|
|
201
|
+
)
|
|
202
|
+
scheduled.admission.release()
|
|
203
|
+
while self._delayed:
|
|
204
|
+
_, _, scheduled = heapq.heappop(self._delayed)
|
|
205
|
+
if not scheduled.result.done():
|
|
206
|
+
scheduled.result.set_exception(
|
|
207
|
+
RuntimeError("outbound scheduler is stopped")
|
|
208
|
+
)
|
|
209
|
+
scheduled.admission.release()
|
|
210
|
+
|
|
211
|
+
def _promote_delayed(self) -> None:
|
|
212
|
+
pending: list[tuple[float, int, _ScheduledSend]] = []
|
|
213
|
+
now = time.monotonic()
|
|
214
|
+
while self._delayed:
|
|
215
|
+
ready_at, delay_sequence, scheduled = heapq.heappop(self._delayed)
|
|
216
|
+
if scheduled.result.done():
|
|
217
|
+
scheduled.admission.release()
|
|
218
|
+
elif ready_at <= now:
|
|
219
|
+
self._sequence += 1
|
|
220
|
+
scheduled.sequence = self._sequence
|
|
221
|
+
self._queue.put_nowait(scheduled)
|
|
222
|
+
else:
|
|
223
|
+
pending.append((ready_at, delay_sequence, scheduled))
|
|
224
|
+
self._delayed = pending
|
|
225
|
+
heapq.heapify(self._delayed)
|
|
226
|
+
|
|
227
|
+
def _dispatch_ready(self) -> None:
|
|
228
|
+
blocked: list[_ScheduledSend] = []
|
|
229
|
+
for _ in range(self._queue.qsize()):
|
|
230
|
+
scheduled = self._queue.get_nowait()
|
|
231
|
+
self._queue.task_done()
|
|
232
|
+
if scheduled.result.done():
|
|
233
|
+
scheduled.admission.release()
|
|
234
|
+
continue
|
|
235
|
+
if not self._lane_available(scheduled):
|
|
236
|
+
blocked.append(scheduled)
|
|
237
|
+
continue
|
|
238
|
+
self._reserve_lane(scheduled)
|
|
239
|
+
task = asyncio.create_task(
|
|
240
|
+
self._run_attempt(scheduled), name="dromeus-send-attempt"
|
|
241
|
+
)
|
|
242
|
+
self._active_tasks.add(task)
|
|
243
|
+
task.add_done_callback(self._active_tasks.discard)
|
|
244
|
+
for scheduled in blocked:
|
|
245
|
+
self._queue.put_nowait(scheduled)
|
|
246
|
+
|
|
247
|
+
def _lane_available(self, scheduled: _ScheduledSend) -> bool:
|
|
248
|
+
if scheduled.is_control:
|
|
249
|
+
return scheduled.destination not in self._peer_control_active
|
|
250
|
+
return (
|
|
251
|
+
self._peer_bulk_active.get(scheduled.destination, 0)
|
|
252
|
+
< self._per_peer_limit
|
|
253
|
+
)
|
|
254
|
+
|
|
255
|
+
def _reserve_lane(self, scheduled: _ScheduledSend) -> None:
|
|
256
|
+
if scheduled.is_control:
|
|
257
|
+
self._peer_control_active.add(scheduled.destination)
|
|
258
|
+
return
|
|
259
|
+
self._peer_bulk_active[scheduled.destination] = (
|
|
260
|
+
self._peer_bulk_active.get(scheduled.destination, 0) + 1
|
|
261
|
+
)
|
|
262
|
+
|
|
263
|
+
def _release_lane(self, scheduled: _ScheduledSend) -> None:
|
|
264
|
+
if scheduled.is_control:
|
|
265
|
+
self._peer_control_active.remove(scheduled.destination)
|
|
266
|
+
else:
|
|
267
|
+
remaining = self._peer_bulk_active[scheduled.destination] - 1
|
|
268
|
+
if remaining:
|
|
269
|
+
self._peer_bulk_active[scheduled.destination] = remaining
|
|
270
|
+
else:
|
|
271
|
+
del self._peer_bulk_active[scheduled.destination]
|
|
272
|
+
self._wake.set()
|
|
273
|
+
|
|
274
|
+
async def _run_attempt(self, scheduled: _ScheduledSend) -> None:
|
|
275
|
+
try:
|
|
276
|
+
should_retry = await self._attempt(scheduled)
|
|
277
|
+
finally:
|
|
278
|
+
self._release_lane(scheduled)
|
|
279
|
+
if should_retry:
|
|
280
|
+
if not self._stop.is_set() and not scheduled.result.done():
|
|
281
|
+
self._delay_sequence += 1
|
|
282
|
+
heapq.heappush(
|
|
283
|
+
self._delayed,
|
|
284
|
+
(
|
|
285
|
+
time.monotonic() + scheduled.retry_delay_seconds,
|
|
286
|
+
self._delay_sequence,
|
|
287
|
+
scheduled,
|
|
288
|
+
),
|
|
289
|
+
)
|
|
290
|
+
self._wake.set()
|
|
291
|
+
return
|
|
292
|
+
if not scheduled.result.done():
|
|
293
|
+
scheduled.result.set_exception(
|
|
294
|
+
RuntimeError("outbound scheduler is stopped")
|
|
295
|
+
)
|
|
296
|
+
scheduled.admission.release()
|
|
297
|
+
|
|
298
|
+
async def _attempt(self, scheduled: _ScheduledSend) -> bool:
|
|
299
|
+
if scheduled.result.done():
|
|
300
|
+
return False
|
|
301
|
+
send_started = time.monotonic()
|
|
302
|
+
if scheduled.first_attempt_at is None:
|
|
303
|
+
scheduled.first_attempt_at = send_started
|
|
304
|
+
try:
|
|
305
|
+
await self._transport.send(scheduled.destination, scheduled.payload)
|
|
306
|
+
except Exception as error:
|
|
307
|
+
scheduled.send_seconds += time.monotonic() - send_started
|
|
308
|
+
if scheduled.result.done():
|
|
309
|
+
return False
|
|
310
|
+
if self._stop.is_set() or scheduled.retry_count >= scheduled.retries:
|
|
311
|
+
scheduled.result.set_exception(error)
|
|
312
|
+
return False
|
|
313
|
+
scheduled.retry_count += 1
|
|
314
|
+
return True
|
|
315
|
+
completed = time.monotonic()
|
|
316
|
+
scheduled.send_seconds += completed - send_started
|
|
317
|
+
if not scheduled.result.done():
|
|
318
|
+
scheduled.result.set_result(
|
|
319
|
+
SendTiming(
|
|
320
|
+
queue_seconds=scheduled.first_attempt_at - scheduled.enqueued_at,
|
|
321
|
+
send_seconds=scheduled.send_seconds,
|
|
322
|
+
retry_count=scheduled.retry_count,
|
|
323
|
+
completion_seconds=completed - scheduled.enqueued_at,
|
|
324
|
+
)
|
|
325
|
+
)
|
|
326
|
+
return False
|
|
@@ -0,0 +1,345 @@
|
|
|
1
|
+
"""Inbound message routing."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
from collections.abc import Callable
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from enum import StrEnum
|
|
9
|
+
|
|
10
|
+
from dromeus.errors import RunError
|
|
11
|
+
from dromeus.manifests.models import (
|
|
12
|
+
AlgorithmId,
|
|
13
|
+
MessageId,
|
|
14
|
+
PublicKey,
|
|
15
|
+
RoundId,
|
|
16
|
+
RunId,
|
|
17
|
+
Sha256,
|
|
18
|
+
)
|
|
19
|
+
from dromeus.protocol.codec import EnvelopeError, decode_envelope
|
|
20
|
+
from dromeus.protocol.models import Envelope, MessageType
|
|
21
|
+
from dromeus.telemetry.events import EventSink, emit_event
|
|
22
|
+
from dromeus.transport.interface import AsyncTransport, TransportError
|
|
23
|
+
|
|
24
|
+
CONTROL_TYPES = {
|
|
25
|
+
MessageType.JOIN_REQUEST,
|
|
26
|
+
MessageType.JOIN_ACCEPTED,
|
|
27
|
+
MessageType.MANIFEST_SEALED,
|
|
28
|
+
MessageType.READY,
|
|
29
|
+
MessageType.START,
|
|
30
|
+
MessageType.START_ACK,
|
|
31
|
+
MessageType.RUN_FAILED,
|
|
32
|
+
MessageType.RUN_COMPLETE,
|
|
33
|
+
}
|
|
34
|
+
TRANSFER_TYPES = {
|
|
35
|
+
MessageType.TRANSFER_BEGIN,
|
|
36
|
+
MessageType.CHUNK,
|
|
37
|
+
}
|
|
38
|
+
ACK_TYPES = {MessageType.CHUNK_ACK}
|
|
39
|
+
TELEMETRY_TYPES = {MessageType.CONSENSUS_SKETCH}
|
|
40
|
+
ROUND_TRACKED_TYPES = TRANSFER_TYPES | {
|
|
41
|
+
MessageType.UPDATE_READY,
|
|
42
|
+
MessageType.ROUND_COMMITTED,
|
|
43
|
+
MessageType.CONSENSUS_SKETCH,
|
|
44
|
+
}
|
|
45
|
+
IDEMPOTENT_TYPES = {MessageType.UPDATE_READY, MessageType.ROUND_COMMITTED}
|
|
46
|
+
MAX_FUTURE_ROUND_MESSAGES = 64
|
|
47
|
+
MAX_TELEMETRY_ROUND_AGE = 64
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
class ReceiverError(RunError):
|
|
51
|
+
"""Inbound traffic violated routing policy."""
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
class MessageChannel(StrEnum):
|
|
55
|
+
CONTROL = "control"
|
|
56
|
+
TRANSFER = "transfer"
|
|
57
|
+
ACKNOWLEDGMENT = "acknowledgment"
|
|
58
|
+
TELEMETRY = "telemetry"
|
|
59
|
+
PAIR_COMMIT = "pair_commit"
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
@dataclass
|
|
63
|
+
class ReceiverPolicy:
|
|
64
|
+
run_id: RunId | None = None
|
|
65
|
+
manifest_hash: Sha256 | None = None
|
|
66
|
+
algorithm_id: AlgorithmId | None = None
|
|
67
|
+
participant_keys: frozenset[PublicKey] | None = None
|
|
68
|
+
current_round: Callable[[], int] = lambda: 0
|
|
69
|
+
max_payload_bytes: int = 8 * 1024 * 1024
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass
|
|
73
|
+
class ReceiverStats:
|
|
74
|
+
"""Valid/rejected traffic and valid telemetry discarded on saturation."""
|
|
75
|
+
|
|
76
|
+
accepted_messages: int = 0
|
|
77
|
+
rejected_messages: int = 0
|
|
78
|
+
dropped_telemetry_messages: int = 0
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class Receiver:
|
|
83
|
+
transport: AsyncTransport
|
|
84
|
+
_policy: ReceiverPolicy = field(default_factory=ReceiverPolicy)
|
|
85
|
+
stats: ReceiverStats = field(default_factory=ReceiverStats)
|
|
86
|
+
event_sink: EventSink | None = None
|
|
87
|
+
|
|
88
|
+
def __post_init__(self) -> None:
|
|
89
|
+
self._queues = {
|
|
90
|
+
channel: asyncio.Queue[Envelope](maxsize=64) for channel in MessageChannel
|
|
91
|
+
}
|
|
92
|
+
# Values are the message round for round-tracked types, else None.
|
|
93
|
+
# Round-tracked entries are pruned once the round window rejects them.
|
|
94
|
+
self._seen_messages: dict[MessageId, RoundId | None] = {}
|
|
95
|
+
self._seen_idempotent_messages: dict[
|
|
96
|
+
tuple[PublicKey, MessageId], RoundId | None
|
|
97
|
+
] = {}
|
|
98
|
+
self._future_round_messages: list[Envelope] = []
|
|
99
|
+
self._future_round_transfer_by_sender: dict[PublicKey, MessageId] = {}
|
|
100
|
+
self._task: asyncio.Task[None] | None = None
|
|
101
|
+
self._stop = asyncio.Event()
|
|
102
|
+
|
|
103
|
+
async def start(self) -> None:
|
|
104
|
+
if self._task is None:
|
|
105
|
+
self._task = asyncio.create_task(self._run(), name="dromeus-recv")
|
|
106
|
+
|
|
107
|
+
async def stop(self) -> None:
|
|
108
|
+
self._stop.set()
|
|
109
|
+
if self._task is not None:
|
|
110
|
+
await self._task
|
|
111
|
+
|
|
112
|
+
async def _run(self) -> None:
|
|
113
|
+
while not self._stop.is_set():
|
|
114
|
+
try:
|
|
115
|
+
inbound = await self.transport.recv(timeout_seconds=0.1)
|
|
116
|
+
except TransportError as error:
|
|
117
|
+
emit_event(
|
|
118
|
+
"transport_receive_failed",
|
|
119
|
+
run_id=self._policy.run_id,
|
|
120
|
+
error=str(error),
|
|
121
|
+
sink=self.event_sink,
|
|
122
|
+
)
|
|
123
|
+
continue
|
|
124
|
+
if inbound is None:
|
|
125
|
+
continue
|
|
126
|
+
try:
|
|
127
|
+
envelope = decode_envelope(
|
|
128
|
+
inbound.payload,
|
|
129
|
+
authenticated_sender=inbound.sender_public_key,
|
|
130
|
+
participant_keys=self._policy.participant_keys,
|
|
131
|
+
max_payload_bytes=self._policy.max_payload_bytes,
|
|
132
|
+
)
|
|
133
|
+
should_route = self._validate_envelope(envelope)
|
|
134
|
+
except (EnvelopeError, ReceiverError, TransportError) as error:
|
|
135
|
+
self.stats.rejected_messages += 1
|
|
136
|
+
emit_event(
|
|
137
|
+
"message_rejected",
|
|
138
|
+
run_id=self._policy.run_id,
|
|
139
|
+
peer_id=inbound.sender_public_key,
|
|
140
|
+
error=str(error),
|
|
141
|
+
sink=self.event_sink,
|
|
142
|
+
)
|
|
143
|
+
continue
|
|
144
|
+
if should_route:
|
|
145
|
+
should_route = await self._route(envelope)
|
|
146
|
+
self.stats.accepted_messages += 1
|
|
147
|
+
emit_event(
|
|
148
|
+
"message_received",
|
|
149
|
+
run_id=envelope.run_id,
|
|
150
|
+
message_id=envelope.message_id,
|
|
151
|
+
transfer_id=envelope.correlation_id,
|
|
152
|
+
peer_id=envelope.sender_public_key,
|
|
153
|
+
round_id=envelope.round_id,
|
|
154
|
+
message_type=envelope.message_type,
|
|
155
|
+
routed=should_route,
|
|
156
|
+
sink=self.event_sink,
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
def _validate_envelope(self, envelope: Envelope) -> bool:
|
|
160
|
+
if self._policy.run_id is not None and envelope.run_id != self._policy.run_id:
|
|
161
|
+
raise ReceiverError("unexpected run id")
|
|
162
|
+
if (
|
|
163
|
+
self._policy.manifest_hash is not None
|
|
164
|
+
and envelope.manifest_hash != self._policy.manifest_hash
|
|
165
|
+
):
|
|
166
|
+
raise ReceiverError("unexpected manifest hash")
|
|
167
|
+
if (
|
|
168
|
+
self._policy.algorithm_id is not None
|
|
169
|
+
and envelope.algorithm_id != self._policy.algorithm_id
|
|
170
|
+
):
|
|
171
|
+
raise ReceiverError("unexpected algorithm id")
|
|
172
|
+
tracked_round = (
|
|
173
|
+
envelope.round_id
|
|
174
|
+
if envelope.message_type in ROUND_TRACKED_TYPES
|
|
175
|
+
else None
|
|
176
|
+
)
|
|
177
|
+
idempotent_key: tuple[PublicKey, MessageId] | None = None
|
|
178
|
+
if envelope.message_type in IDEMPOTENT_TYPES:
|
|
179
|
+
idempotent_key = (envelope.sender_public_key, envelope.message_id)
|
|
180
|
+
if idempotent_key in self._seen_idempotent_messages:
|
|
181
|
+
return False
|
|
182
|
+
elif envelope.message_type not in TRANSFER_TYPES | ACK_TYPES:
|
|
183
|
+
if envelope.message_id in self._seen_messages:
|
|
184
|
+
raise ReceiverError("replayed message id")
|
|
185
|
+
self._seen_messages[envelope.message_id] = tracked_round
|
|
186
|
+
if (
|
|
187
|
+
envelope.message_type in ROUND_TRACKED_TYPES
|
|
188
|
+
and envelope.round_id is not None
|
|
189
|
+
):
|
|
190
|
+
should_route = self._validate_round_window(envelope)
|
|
191
|
+
if idempotent_key is not None:
|
|
192
|
+
self._seen_idempotent_messages[idempotent_key] = tracked_round
|
|
193
|
+
return should_route
|
|
194
|
+
if idempotent_key is not None:
|
|
195
|
+
self._seen_idempotent_messages[idempotent_key] = tracked_round
|
|
196
|
+
return True
|
|
197
|
+
|
|
198
|
+
def _validate_round_window(self, envelope: Envelope) -> bool:
|
|
199
|
+
current_round = self._policy.current_round()
|
|
200
|
+
assert envelope.round_id is not None
|
|
201
|
+
if envelope.message_type is MessageType.CONSENSUS_SKETCH:
|
|
202
|
+
if envelope.round_id <= current_round:
|
|
203
|
+
if current_round - envelope.round_id > MAX_TELEMETRY_ROUND_AGE:
|
|
204
|
+
raise ReceiverError("stale telemetry message")
|
|
205
|
+
return True
|
|
206
|
+
if envelope.round_id < current_round:
|
|
207
|
+
raise ReceiverError("stale message")
|
|
208
|
+
if envelope.round_id > current_round + 1:
|
|
209
|
+
raise ReceiverError("message too far in future")
|
|
210
|
+
if envelope.round_id == current_round + 1:
|
|
211
|
+
if any(
|
|
212
|
+
buffered.sender_public_key == envelope.sender_public_key
|
|
213
|
+
and buffered.message_id == envelope.message_id
|
|
214
|
+
for buffered in self._future_round_messages
|
|
215
|
+
):
|
|
216
|
+
return False
|
|
217
|
+
if len(self._future_round_messages) >= MAX_FUTURE_ROUND_MESSAGES:
|
|
218
|
+
raise ReceiverError("future-round message buffer is full")
|
|
219
|
+
if envelope.message_type in TRANSFER_TYPES:
|
|
220
|
+
transfer_id = envelope.correlation_id
|
|
221
|
+
if transfer_id is None:
|
|
222
|
+
raise ReceiverError("future-round transfer has no correlation id")
|
|
223
|
+
existing_transfer = self._future_round_transfer_by_sender.get(
|
|
224
|
+
envelope.sender_public_key
|
|
225
|
+
)
|
|
226
|
+
if existing_transfer is not None and existing_transfer != transfer_id:
|
|
227
|
+
raise ReceiverError("too many future-round updates from sender")
|
|
228
|
+
self._future_round_transfer_by_sender[envelope.sender_public_key] = (
|
|
229
|
+
transfer_id
|
|
230
|
+
)
|
|
231
|
+
self._future_round_messages.append(envelope)
|
|
232
|
+
return False
|
|
233
|
+
return True
|
|
234
|
+
|
|
235
|
+
async def _route(self, envelope: Envelope) -> bool:
|
|
236
|
+
if envelope.message_type in CONTROL_TYPES:
|
|
237
|
+
return await self._enqueue_required(MessageChannel.CONTROL, envelope)
|
|
238
|
+
if envelope.message_type in TRANSFER_TYPES:
|
|
239
|
+
return await self._enqueue_required(MessageChannel.TRANSFER, envelope)
|
|
240
|
+
if envelope.message_type in ACK_TYPES:
|
|
241
|
+
return await self._enqueue_required(MessageChannel.ACKNOWLEDGMENT, envelope)
|
|
242
|
+
if envelope.message_type in TELEMETRY_TYPES:
|
|
243
|
+
# Drop incoming telemetry rather than stopping the sole transport reader.
|
|
244
|
+
try:
|
|
245
|
+
self._queues[MessageChannel.TELEMETRY].put_nowait(envelope)
|
|
246
|
+
except asyncio.QueueFull:
|
|
247
|
+
self.stats.dropped_telemetry_messages += 1
|
|
248
|
+
emit_event(
|
|
249
|
+
"telemetry_message_dropped",
|
|
250
|
+
run_id=envelope.run_id,
|
|
251
|
+
message_id=envelope.message_id,
|
|
252
|
+
peer_id=envelope.sender_public_key,
|
|
253
|
+
round_id=envelope.round_id,
|
|
254
|
+
reason="queue_full",
|
|
255
|
+
dropped_telemetry_messages=self.stats.dropped_telemetry_messages,
|
|
256
|
+
sink=self.event_sink,
|
|
257
|
+
)
|
|
258
|
+
return False
|
|
259
|
+
return True
|
|
260
|
+
return await self._enqueue_required(MessageChannel.PAIR_COMMIT, envelope)
|
|
261
|
+
|
|
262
|
+
async def _enqueue_required(
|
|
263
|
+
self, channel: MessageChannel, envelope: Envelope
|
|
264
|
+
) -> bool:
|
|
265
|
+
"""Keep protocol backpressure, allowing shutdown to end a blocked put."""
|
|
266
|
+
queue = self._queues[channel]
|
|
267
|
+
try:
|
|
268
|
+
queue.put_nowait(envelope)
|
|
269
|
+
return True
|
|
270
|
+
except asyncio.QueueFull:
|
|
271
|
+
pass
|
|
272
|
+
admission = asyncio.create_task(queue.put(envelope))
|
|
273
|
+
stopping = asyncio.create_task(self._stop.wait())
|
|
274
|
+
try:
|
|
275
|
+
done, _ = await asyncio.wait(
|
|
276
|
+
(admission, stopping), return_when=asyncio.FIRST_COMPLETED
|
|
277
|
+
)
|
|
278
|
+
if admission in done:
|
|
279
|
+
admission.result()
|
|
280
|
+
return True
|
|
281
|
+
return False
|
|
282
|
+
finally:
|
|
283
|
+
for task in (admission, stopping):
|
|
284
|
+
if not task.done():
|
|
285
|
+
task.cancel()
|
|
286
|
+
await asyncio.gather(admission, stopping, return_exceptions=True)
|
|
287
|
+
|
|
288
|
+
async def receive(
|
|
289
|
+
self,
|
|
290
|
+
channel: MessageChannel,
|
|
291
|
+
*,
|
|
292
|
+
timeout_seconds: float | None = None,
|
|
293
|
+
) -> Envelope:
|
|
294
|
+
"""Return the next validated message for a protocol channel."""
|
|
295
|
+
queue = self._queues[channel]
|
|
296
|
+
if timeout_seconds is None:
|
|
297
|
+
return await queue.get()
|
|
298
|
+
return await asyncio.wait_for(queue.get(), timeout=timeout_seconds)
|
|
299
|
+
|
|
300
|
+
def configure_sealed_run(
|
|
301
|
+
self,
|
|
302
|
+
*,
|
|
303
|
+
manifest_hash: Sha256,
|
|
304
|
+
participant_keys: frozenset[PublicKey],
|
|
305
|
+
) -> None:
|
|
306
|
+
self._policy.manifest_hash = manifest_hash
|
|
307
|
+
self._policy.participant_keys = participant_keys
|
|
308
|
+
|
|
309
|
+
def set_current_round(self, round_id: RoundId) -> None:
|
|
310
|
+
"""Advance round validation after a pair commit is durable."""
|
|
311
|
+
self._policy.current_round = lambda: round_id
|
|
312
|
+
# Every round-tracked type is rejected once older than the telemetry age.
|
|
313
|
+
oldest_routable = round_id - MAX_TELEMETRY_ROUND_AGE
|
|
314
|
+
_forget_rounds_before(self._seen_messages, oldest_routable)
|
|
315
|
+
_forget_rounds_before(self._seen_idempotent_messages, oldest_routable)
|
|
316
|
+
|
|
317
|
+
async def advance_round(self, next_round: RoundId) -> None:
|
|
318
|
+
ready = [
|
|
319
|
+
envelope
|
|
320
|
+
for envelope in self._future_round_messages
|
|
321
|
+
if envelope.round_id == next_round
|
|
322
|
+
]
|
|
323
|
+
self._future_round_messages = [
|
|
324
|
+
envelope
|
|
325
|
+
for envelope in self._future_round_messages
|
|
326
|
+
if envelope.round_id != next_round
|
|
327
|
+
]
|
|
328
|
+
self._future_round_transfer_by_sender = {
|
|
329
|
+
envelope.sender_public_key: envelope.correlation_id
|
|
330
|
+
for envelope in self._future_round_messages
|
|
331
|
+
if envelope.message_type in TRANSFER_TYPES
|
|
332
|
+
and envelope.correlation_id is not None
|
|
333
|
+
}
|
|
334
|
+
for envelope in ready:
|
|
335
|
+
await self._route(envelope)
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _forget_rounds_before[K](seen: dict[K, RoundId | None], round_id: int) -> None:
|
|
339
|
+
expired = [
|
|
340
|
+
key
|
|
341
|
+
for key, seen_round in seen.items()
|
|
342
|
+
if seen_round is not None and seen_round < round_id
|
|
343
|
+
]
|
|
344
|
+
for key in expired:
|
|
345
|
+
del seen[key]
|