dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,909 @@
1
+ """Deterministic live and exact consensus distance calculations."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import asyncio
6
+ import hashlib
7
+ import inspect
8
+ import math
9
+ from collections import deque
10
+ from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
11
+ from dataclasses import dataclass
12
+ from threading import RLock
13
+ from typing import Literal
14
+
15
+ import numpy as np
16
+
17
+ from dromeus.errors import RunError
18
+ from dromeus.manifests.models import ConsensusSketchMessage, PublicKey, RoundId
19
+
20
+ COUNT_SKETCH_SIZE = 4096
21
+ # Keep integer draw boundaries aligned across named tensors. NumPy's small
22
+ # integer draws can pack several values into a generator word.
23
+ _COUNT_SKETCH_CHUNK_VALUES = 65536
24
+
25
+
26
+ class ConsensusSketchError(RunError, ValueError):
27
+ """A consensus sketch was malformed or arrived from an invalid member."""
28
+
29
+
30
+ @dataclass(frozen=True, slots=True)
31
+ class MemberDistance:
32
+ """A sealed member's distance from the complete group's mean sketch."""
33
+
34
+ public_key: PublicKey
35
+ absolute_distance: float
36
+ normalized_distance: float
37
+
38
+ def __post_init__(self) -> None:
39
+ if not self.public_key or any(
40
+ not math.isfinite(value) or value < 0
41
+ for value in (self.absolute_distance, self.normalized_distance)
42
+ ):
43
+ raise ConsensusSketchError("invalid member distance")
44
+
45
+
46
+ @dataclass(frozen=True, slots=True)
47
+ class ConsensusObservation:
48
+ """One complete round's group spread, scale and per-member distances."""
49
+
50
+ round_id: RoundId
51
+ normalized_rms: float
52
+ sketch_count: int
53
+ absolute_rms_spread: float
54
+ mean_sketch_norm: float
55
+ denominator_degenerate: bool
56
+ member_distances: tuple[MemberDistance, ...] = ()
57
+
58
+ def __post_init__(self) -> None:
59
+ if (
60
+ self.round_id < 0
61
+ or self.sketch_count < 1
62
+ or any(
63
+ not math.isfinite(v) or v < 0
64
+ for v in (
65
+ self.normalized_rms,
66
+ self.absolute_rms_spread,
67
+ self.mean_sketch_norm,
68
+ )
69
+ )
70
+ ):
71
+ raise ConsensusSketchError("invalid consensus observation")
72
+ if self.denominator_degenerate != (self.mean_sketch_norm <= 1e-12):
73
+ raise ConsensusSketchError("inconsistent denominator scale flag")
74
+ if not math.isclose(
75
+ self.normalized_rms,
76
+ self.absolute_rms_spread / (self.mean_sketch_norm + 1e-12),
77
+ rel_tol=1e-9,
78
+ abs_tol=1e-12,
79
+ ):
80
+ raise ConsensusSketchError("inconsistent consensus scale measurements")
81
+ if self.member_distances:
82
+ if (
83
+ len(self.member_distances) != self.sketch_count
84
+ or len({m.public_key for m in self.member_distances})
85
+ != self.sketch_count
86
+ or any(
87
+ not math.isclose(
88
+ m.normalized_distance,
89
+ m.absolute_distance / (self.mean_sketch_norm + 1e-12),
90
+ rel_tol=1e-9,
91
+ abs_tol=1e-12,
92
+ )
93
+ for m in self.member_distances
94
+ )
95
+ or not math.isclose(
96
+ math.hypot(*(m.absolute_distance for m in self.member_distances))
97
+ / math.sqrt(self.sketch_count),
98
+ self.absolute_rms_spread,
99
+ rel_tol=1e-9,
100
+ abs_tol=1e-12,
101
+ )
102
+ ):
103
+ raise ConsensusSketchError("inconsistent member distances")
104
+
105
+
106
+ def consensus_observation(
107
+ sketches: Sequence[np.ndarray],
108
+ *,
109
+ round_id: int,
110
+ participant_keys: Sequence[PublicKey] | None = None,
111
+ ) -> ConsensusObservation:
112
+ """Measure spread and mean norm with a 1e-12 denominator stabilizer."""
113
+ if not sketches:
114
+ raise ConsensusSketchError("at least one sketch is required")
115
+ first = np.asarray(sketches[0])
116
+ if first.ndim != 1 or not first.size:
117
+ raise ConsensusSketchError("sketch must be a nonempty vector")
118
+ values = np.stack([_validate_sketch(x, size=first.size) for x in sketches]).astype(
119
+ np.float64
120
+ )
121
+ return _measure_consensus(
122
+ values, round_id=round_id, participant_keys=participant_keys
123
+ )
124
+
125
+
126
+ def exact_consensus_observation(
127
+ models: Sequence[Mapping[str, np.ndarray]], *, round_id: int
128
+ ) -> ConsensusObservation:
129
+ """Use the same scale definitions for exact public model snapshots."""
130
+ if not models or any(model.keys() != models[0].keys() for model in models):
131
+ raise ValueError("exact models must share their named schema")
132
+ if any(
133
+ model[name].shape != models[0][name].shape for model in models for name in model
134
+ ):
135
+ raise ValueError("exact model shapes differ")
136
+ values = np.stack([_flatten_weights(model) for model in models]).astype(np.float64)
137
+ return _measure_consensus(values, round_id=round_id)
138
+
139
+
140
+ def _measure_consensus(
141
+ values: np.ndarray,
142
+ *,
143
+ round_id: int,
144
+ participant_keys: Sequence[PublicKey] | None = None,
145
+ ) -> ConsensusObservation:
146
+ if participant_keys is not None and (
147
+ len(participant_keys) != len(values)
148
+ or len(set(participant_keys)) != len(values)
149
+ or any(not key for key in participant_keys)
150
+ ):
151
+ raise ConsensusSketchError("participant keys must match sketches and be unique")
152
+ mean = values.mean(axis=0)
153
+ squared_distances = np.sum((values - mean) ** 2, axis=1)
154
+ spread = float(np.sqrt(np.mean(squared_distances)))
155
+ norm = float(np.linalg.norm(mean))
156
+ members = (
157
+ tuple(
158
+ sorted(
159
+ (
160
+ MemberDistance(
161
+ key, float(distance), float(distance) / (norm + 1e-12)
162
+ )
163
+ for key, distance in zip(
164
+ participant_keys, np.sqrt(squared_distances), strict=True
165
+ )
166
+ ),
167
+ key=lambda member: member.public_key,
168
+ )
169
+ )
170
+ if participant_keys is not None
171
+ else ()
172
+ )
173
+ return ConsensusObservation(
174
+ round_id=round_id,
175
+ normalized_rms=spread / (norm + 1e-12),
176
+ sketch_count=len(values),
177
+ absolute_rms_spread=spread,
178
+ mean_sketch_norm=norm,
179
+ denominator_degenerate=norm <= 1e-12,
180
+ member_distances=members,
181
+ )
182
+
183
+
184
+ SketchReceiver = Callable[[float], Awaitable[ConsensusSketchMessage]]
185
+ SketchPublisher = Callable[[RoundId, np.ndarray], Awaitable[None]]
186
+ DistanceCallback = Callable[[ConsensusObservation], None | Awaitable[None]]
187
+
188
+
189
+ def count_sketch(
190
+ weights: Mapping[str, np.ndarray],
191
+ *,
192
+ seed: int,
193
+ size: int = COUNT_SKETCH_SIZE,
194
+ ) -> np.ndarray:
195
+ """Return a deterministic signed CountSketch of named finite tensors."""
196
+ if size <= 0:
197
+ raise ValueError("sketch size must be positive")
198
+ arrays = _validated_sketch_arrays(weights)
199
+ element_count = sum(value.size for value in arrays)
200
+ normalised_seed = _normalise_seed(seed)
201
+ bucket_rng = np.random.default_rng(normalised_seed)
202
+ sign_rng = np.random.default_rng(normalised_seed)
203
+ # The projection draws ALL buckets before ANY signs. Replay that
204
+ # first segment in bounded chunks for the sign stream instead of changing
205
+ # the seed/distribution or assuming a raw bit-generator advance count.
206
+ for start in range(0, element_count, _COUNT_SKETCH_CHUNK_VALUES):
207
+ sign_rng.integers(
208
+ 0, size, size=min(_COUNT_SKETCH_CHUNK_VALUES, element_count - start)
209
+ )
210
+ sketch = np.zeros(size, dtype=np.float32)
211
+ for values in _sketch_value_chunks(arrays):
212
+ buckets = bucket_rng.integers(0, size, size=values.size)
213
+ signs = np.where(
214
+ sign_rng.integers(0, 2, size=values.size, dtype=np.int8) == 0,
215
+ -1.0,
216
+ 1.0,
217
+ )
218
+ np.add.at(sketch, buckets, values * signs)
219
+ return sketch
220
+
221
+
222
+ def _validated_sketch_arrays(
223
+ weights: Mapping[str, np.ndarray],
224
+ ) -> tuple[np.ndarray, ...]:
225
+ """Validate in projection name order without allocating model-sized copies."""
226
+ if not weights:
227
+ raise ValueError("weights must contain at least one tensor")
228
+ arrays: list[np.ndarray] = []
229
+ for name in sorted(weights):
230
+ value = np.asarray(weights[name])
231
+ if not np.issubdtype(value.dtype, np.floating):
232
+ raise ValueError(f"tensor {name} must be floating point")
233
+ for start in range(0, value.size, _COUNT_SKETCH_CHUNK_VALUES):
234
+ if not np.isfinite(
235
+ value.flat[start : start + _COUNT_SKETCH_CHUNK_VALUES]
236
+ ).all():
237
+ raise ValueError(f"tensor {name} contains non-finite values")
238
+ arrays.append(value)
239
+ return tuple(arrays)
240
+
241
+
242
+ def _sketch_value_chunks(arrays: Sequence[np.ndarray]) -> Iterator[np.ndarray]:
243
+ """Flatten globally in C order, keeping per-name RNG boundaries."""
244
+ buffer = np.empty(_COUNT_SKETCH_CHUNK_VALUES, dtype=np.float64)
245
+ used = 0
246
+ for value in arrays:
247
+ position = 0
248
+ while position < value.size:
249
+ count = min(_COUNT_SKETCH_CHUNK_VALUES - used, value.size - position)
250
+ buffer[used : used + count] = value.flat[position : position + count]
251
+ used += count
252
+ position += count
253
+ if used == _COUNT_SKETCH_CHUNK_VALUES:
254
+ yield buffer
255
+ used = 0
256
+ if used:
257
+ yield buffer[:used]
258
+
259
+
260
+ def encode_sketch(sketch: np.ndarray, *, size: int = COUNT_SKETCH_SIZE) -> bytes:
261
+ """Encode exactly one FP32 sketch without a JSON or MessagePack expansion."""
262
+ value = _validate_sketch(sketch, size=size)
263
+ return value.astype("<f4", copy=False).tobytes()
264
+
265
+
266
+ def decode_sketch(data: bytes, *, size: int = COUNT_SKETCH_SIZE) -> np.ndarray:
267
+ """Decode a bounded FP32 sketch payload."""
268
+ if len(data) != size * np.dtype(np.float32).itemsize:
269
+ raise ConsensusSketchError("sketch payload has an unexpected size")
270
+ value = np.frombuffer(data, dtype="<f4").copy()
271
+ return _validate_sketch(value, size=size)
272
+
273
+
274
+ class ConsensusSketchBuffer:
275
+ """Bounded round-keyed sketch buffer that never waits on training."""
276
+
277
+ def __init__(
278
+ self,
279
+ *,
280
+ participant_keys: Sequence[PublicKey],
281
+ size: int = COUNT_SKETCH_SIZE,
282
+ max_pending_rounds: int = 64,
283
+ ) -> None:
284
+ if not participant_keys or len(set(participant_keys)) != len(participant_keys):
285
+ raise ValueError("participant keys must be non-empty and unique")
286
+ if size <= 0:
287
+ raise ValueError("sketch size must be positive")
288
+ if max_pending_rounds <= 0:
289
+ raise ValueError("max_pending_rounds must be positive")
290
+ self._participant_keys = frozenset(participant_keys)
291
+ self._size = size
292
+ self._max_pending_rounds = max_pending_rounds
293
+ self._lock = RLock()
294
+ self._pending: dict[RoundId, dict[PublicKey, np.ndarray]] = {}
295
+ self._results: dict[RoundId, ConsensusObservation] = {}
296
+ self._dropped = 0
297
+
298
+ def add(
299
+ self,
300
+ *,
301
+ round_id: RoundId,
302
+ sender_public_key: PublicKey,
303
+ sketch: np.ndarray,
304
+ ) -> ConsensusObservation | None:
305
+ with self._lock:
306
+ if sender_public_key not in self._participant_keys:
307
+ raise ConsensusSketchError("sender is not a sealed participant")
308
+ value = _validate_sketch(sketch, size=self._size)
309
+ existing_result = self._results.get(round_id)
310
+ if existing_result is not None:
311
+ existing = self._pending[round_id][sender_public_key]
312
+ if not np.array_equal(existing, value):
313
+ raise ConsensusSketchError("duplicate sketch does not match")
314
+ return existing_result
315
+ if (
316
+ round_id not in self._pending
317
+ and len(self.pending_rounds()) >= self._max_pending_rounds
318
+ ):
319
+ oldest_round = min(self.pending_rounds())
320
+ self._pending.pop(oldest_round, None)
321
+ self._dropped += 1
322
+ round_sketches = self._pending.setdefault(round_id, {})
323
+ existing = round_sketches.get(sender_public_key)
324
+ if existing is not None:
325
+ if not np.array_equal(existing, value):
326
+ raise ConsensusSketchError("duplicate sketch does not match")
327
+ else:
328
+ round_sketches[sender_public_key] = value
329
+ if len(round_sketches) != len(self._participant_keys):
330
+ return None
331
+ result = consensus_observation(
332
+ [round_sketches[key] for key in sorted(round_sketches)],
333
+ round_id=round_id,
334
+ participant_keys=sorted(round_sketches),
335
+ )
336
+ self._results[round_id] = result
337
+ completed_rounds = sorted(self._results)
338
+ for old_round in completed_rounds[: -self._max_pending_rounds]:
339
+ self._results.pop(old_round, None)
340
+ self._pending.pop(old_round, None)
341
+ return result
342
+
343
+ def result(self, round_id: RoundId) -> ConsensusObservation | None:
344
+ with self._lock:
345
+ return self._results.get(round_id)
346
+
347
+ @property
348
+ def dropped(self) -> int:
349
+ """Return incomplete rounds evicted to keep the buffer bounded."""
350
+ return self._dropped
351
+
352
+ def pending_rounds(self) -> tuple[RoundId, ...]:
353
+ with self._lock:
354
+ return tuple(
355
+ sorted(
356
+ round_id
357
+ for round_id in self._pending
358
+ if round_id not in self._results
359
+ )
360
+ )
361
+
362
+
363
+ @dataclass(frozen=True, slots=True)
364
+ class ConsensusCompletionResult:
365
+ """Local final-round evidence; never an all-peer delivery acknowledgement."""
366
+
367
+ round_id: RoundId
368
+ status: Literal["complete", "timeout", "dropped", "stopped"]
369
+ received_member_keys: tuple[PublicKey, ...]
370
+ report_completed: bool
371
+ dropped: int
372
+
373
+
374
+ class ConsensusSketchPublisher:
375
+ """Bounded background sketch computation and publication queue."""
376
+
377
+ def __init__(
378
+ self,
379
+ *,
380
+ seed: int,
381
+ publish: Callable[[RoundId, np.ndarray], Awaitable[None]],
382
+ size: int = COUNT_SKETCH_SIZE,
383
+ max_queue_size: int = 64,
384
+ on_sketch: Callable[[RoundId, np.ndarray], None | Awaitable[None]]
385
+ | None = None,
386
+ ) -> None:
387
+ if max_queue_size <= 0:
388
+ raise ValueError("max_queue_size must be positive")
389
+ self._seed = seed
390
+ self._publish = publish
391
+ self._on_sketch = on_sketch
392
+ self._size = size
393
+ self._dropped = 0
394
+ self._queue: asyncio.Queue[tuple[RoundId, Mapping[str, np.ndarray]]] = (
395
+ asyncio.Queue(maxsize=max_queue_size)
396
+ )
397
+ self._stop = asyncio.Event()
398
+ self._task: asyncio.Task[None] | None = None
399
+ self._inflight = False
400
+ self._thread_call_active = False
401
+
402
+ def submit(self, *, round_id: RoundId, weights: Mapping[str, np.ndarray]) -> bool:
403
+ """Queue a model snapshot, returning false when telemetry is full."""
404
+ if self._queue.full():
405
+ self._dropped += 1
406
+ return False
407
+ try:
408
+ self._queue.put_nowait((round_id, weights))
409
+ except asyncio.QueueFull:
410
+ self._dropped += 1
411
+ return False
412
+ return True
413
+
414
+ async def start(self) -> None:
415
+ if self._task is None:
416
+ self._task = asyncio.create_task(self._run(), name="dromeus-consensus")
417
+
418
+ async def stop(self, *, timeout_seconds: float | None = None) -> bool:
419
+ if timeout_seconds is not None and timeout_seconds <= 0:
420
+ raise ValueError("stop timeout must be positive")
421
+ self._stop.set()
422
+ task = self._task
423
+ if task is None:
424
+ return True
425
+ try:
426
+ if timeout_seconds is None:
427
+ await task
428
+ else:
429
+ await asyncio.wait_for(task, timeout=timeout_seconds)
430
+ return True
431
+ except TimeoutError:
432
+ self._dropped += 1 + self._queue.qsize()
433
+ return False
434
+ finally:
435
+ self._task = None
436
+
437
+ @property
438
+ def pending(self) -> int:
439
+ return self._queue.qsize()
440
+
441
+ @property
442
+ def dropped(self) -> int:
443
+ """Return the number of snapshots rejected by the bounded queue."""
444
+ return self._dropped
445
+
446
+ async def wait_idle(self) -> None:
447
+ """Wait for queued AND in-flight publication without stopping its worker.
448
+
449
+ The caller must first finish submissions. Cancellation affects this wait,
450
+ not the background projection thread or publisher task.
451
+ """
452
+ await self._queue.join()
453
+
454
+ async def finish(self) -> None:
455
+ """Finish accepted publication without cascading caller cancellation."""
456
+ await self.wait_idle()
457
+ self._stop.set()
458
+ if self._task is not None:
459
+ await asyncio.shield(self._task)
460
+
461
+ def abort(self) -> tuple[asyncio.Task[None] | None, bool]:
462
+ """Cancel async publication once; native projection may continue."""
463
+ thread_may_continue = self._thread_call_active
464
+ self._dropped += self.pending + int(self._inflight)
465
+ self._stop.set()
466
+ while not self._queue.empty():
467
+ self._queue.get_nowait()
468
+ self._queue.task_done()
469
+ if self._task is not None and not self._task.done():
470
+ self._task.cancel()
471
+ return self._task, thread_may_continue
472
+
473
+ async def _run(self) -> None:
474
+ while not self._stop.is_set() or not self._queue.empty():
475
+ try:
476
+ round_id, weights = await asyncio.wait_for(
477
+ self._queue.get(), timeout=0.1
478
+ )
479
+ except TimeoutError:
480
+ continue
481
+ self._inflight = True
482
+ try:
483
+ self._thread_call_active = True
484
+ snapshot = await asyncio.to_thread(_copy_weights, weights)
485
+ sketch = await asyncio.to_thread(
486
+ count_sketch,
487
+ snapshot,
488
+ seed=self._seed,
489
+ size=self._size,
490
+ )
491
+ self._thread_call_active = False
492
+ if self._on_sketch is not None:
493
+ try:
494
+ result = self._on_sketch(round_id, sketch)
495
+ if inspect.isawaitable(result):
496
+ await result
497
+ except Exception:
498
+ self._dropped += 1
499
+ pass
500
+ try:
501
+ await self._publish(round_id, sketch)
502
+ except Exception:
503
+ self._dropped += 1
504
+ pass
505
+ except Exception:
506
+ self._dropped += 1
507
+ pass
508
+ finally:
509
+ # A live publisher can wait indefinitely for the next item.
510
+ # Do not keep the last model and defensive copy in its frame.
511
+ self._inflight = False
512
+ self._thread_call_active = False
513
+ weights = None
514
+ snapshot = None
515
+ self._queue.task_done()
516
+
517
+
518
+ class LiveConsensusTelemetry:
519
+ """Publish local sketches and consume remote sketches off training path."""
520
+
521
+ def __init__(
522
+ self,
523
+ *,
524
+ local_public_key: PublicKey,
525
+ participant_keys: Sequence[PublicKey],
526
+ seed: int,
527
+ receive: SketchReceiver,
528
+ publish: SketchPublisher,
529
+ on_distance: DistanceCallback | None = None,
530
+ size: int = COUNT_SKETCH_SIZE,
531
+ max_pending_rounds: int = 64,
532
+ max_queue_size: int = 64,
533
+ drain_timeout_seconds: float = 5.0,
534
+ ) -> None:
535
+ if drain_timeout_seconds <= 0:
536
+ raise ValueError("drain timeout must be positive")
537
+ self._local_public_key = local_public_key
538
+ self._receive = receive
539
+ self._on_distance = on_distance
540
+ self._buffer = ConsensusSketchBuffer(
541
+ participant_keys=participant_keys,
542
+ size=size,
543
+ max_pending_rounds=max_pending_rounds,
544
+ )
545
+ self._publisher = ConsensusSketchPublisher(
546
+ seed=seed,
547
+ publish=publish,
548
+ size=size,
549
+ max_queue_size=max_queue_size,
550
+ on_sketch=self._accept_local,
551
+ )
552
+ self._size = size
553
+ self._drain_timeout_seconds = drain_timeout_seconds
554
+ self._submitted_rounds: deque[RoundId] = deque()
555
+ self._reported_rounds: set[RoundId] = set()
556
+ self._completed_reports: set[RoundId] = set()
557
+ self._reported_round_order: deque[RoundId] = deque()
558
+ self._max_pending_rounds = max_pending_rounds
559
+ self._max_reported_rounds = max_pending_rounds
560
+ self._distance_queue: asyncio.Queue[ConsensusObservation] = asyncio.Queue(
561
+ maxsize=max_pending_rounds
562
+ )
563
+ self._dropped_distance_reports = 0
564
+ self._stop = asyncio.Event()
565
+ self._report_stop = asyncio.Event()
566
+ self._task: asyncio.Task[None] | None = None
567
+ self._report_task: asyncio.Task[None] | None = None
568
+ self._report_inflight = False
569
+ self._shutdown_waiter: asyncio.Task[None] | None = None
570
+ self._shutdown_tasks: set[asyncio.Task[None]] = set()
571
+ self._shutdown_incomplete = False
572
+ self._projection_thread_may_continue = False
573
+
574
+ @property
575
+ def pending(self) -> int:
576
+ """Return queued local sketches awaiting background publication."""
577
+ return self._publisher.pending
578
+
579
+ def submit(self, *, round_id: RoundId, weights: Mapping[str, np.ndarray]) -> bool:
580
+ """Queue one committed local model without waiting on telemetry."""
581
+ submitted = self._publisher.submit(round_id=round_id, weights=weights)
582
+ if submitted:
583
+ self._submitted_rounds.append(round_id)
584
+ while len(self._submitted_rounds) > self._max_pending_rounds:
585
+ self._submitted_rounds.popleft()
586
+ return submitted
587
+
588
+ def result(self, round_id: RoundId) -> ConsensusObservation | None:
589
+ """Return completed live consensus for one round, when available."""
590
+ return self._buffer.result(round_id)
591
+
592
+ @property
593
+ def dropped(self) -> int:
594
+ """Return local sketch and distance-report drops."""
595
+ return (
596
+ self._publisher.dropped
597
+ + self._buffer.dropped
598
+ + self._dropped_distance_reports
599
+ )
600
+
601
+ async def start(self) -> None:
602
+ if self._task is None:
603
+ self._stop.clear()
604
+ self._report_stop.clear()
605
+ self._task = asyncio.create_task(
606
+ self._consume(), name="dromeus-consensus-recv"
607
+ )
608
+ self._report_task = asyncio.create_task(
609
+ self._report_distances(), name="dromeus-consensus-report"
610
+ )
611
+ await self._publisher.start()
612
+
613
+ @property
614
+ def shutdown_incomplete(self) -> bool:
615
+ """Whether opted-in cleanup exhausted its deadline or was cancelled."""
616
+ return self._shutdown_incomplete
617
+
618
+ @property
619
+ def pending_shutdown_tasks(self) -> int:
620
+ """Unfinished asyncio tasks, NOT proof that native threads have stopped."""
621
+ return sum(not task.done() for task in self._shutdown_tasks)
622
+
623
+ @property
624
+ def projection_thread_may_continue(self) -> bool:
625
+ """Conservative flag: cancelling to_thread cannot kill its native thread."""
626
+ return self._projection_thread_may_continue
627
+
628
+ async def stop(self, *, deadline: float | None = None) -> None:
629
+ if self._shutdown_incomplete:
630
+ return # Preserve bounded/idempotent shutdown after an opted-in abort.
631
+ if deadline is None:
632
+ await self._stop_without_deadline()
633
+ return
634
+ if not math.isfinite(deadline):
635
+ raise ValueError("finite absolute shutdown deadline required")
636
+ if self._shutdown_incomplete:
637
+ return # Idempotent; never repeatedly cancel an uncooperative callback.
638
+ if self._task is None:
639
+ return
640
+ if self._shutdown_waiter is None:
641
+ self._shutdown_waiter = asyncio.create_task(
642
+ self._finish_without_cancelling_workers(), name="consensus-finish"
643
+ )
644
+ waiter = self._shutdown_waiter
645
+ try:
646
+ done, _ = await asyncio.wait(
647
+ {waiter}, timeout=max(0.0, deadline - asyncio.get_running_loop().time())
648
+ )
649
+ if waiter in done:
650
+ waiter.result()
651
+ return
652
+ except BaseException:
653
+ self._abort_bounded_shutdown()
654
+ raise
655
+ self._abort_bounded_shutdown()
656
+
657
+ async def _finish_without_cancelling_workers(self) -> None:
658
+ await self._publisher.finish()
659
+ await self._drain_submitted()
660
+ self._stop.set()
661
+ if self._task is not None:
662
+ await asyncio.shield(self._task)
663
+ self._report_stop.set()
664
+ await self._distance_queue.join()
665
+ if self._report_task is not None:
666
+ await asyncio.shield(self._report_task)
667
+ self._task = None
668
+ self._report_task = None
669
+
670
+ def _abort_bounded_shutdown(self) -> None:
671
+ if self._shutdown_incomplete:
672
+ return
673
+ self._shutdown_incomplete = True
674
+ publisher_task, self._projection_thread_may_continue = self._publisher.abort()
675
+ self._dropped_distance_reports += self._distance_queue.qsize() + int(
676
+ self._report_inflight
677
+ )
678
+ self._stop.set()
679
+ self._report_stop.set()
680
+ self._submitted_rounds.clear()
681
+ # Release queued models/reports. In-flight frames are released by their
682
+ # cooperative finally blocks; an ignoring callback stays referenced.
683
+ while not self._distance_queue.empty():
684
+ self._distance_queue.get_nowait()
685
+ self._distance_queue.task_done()
686
+ for task in (
687
+ self._shutdown_waiter,
688
+ publisher_task,
689
+ self._task,
690
+ self._report_task,
691
+ ):
692
+ if task is None or task.done():
693
+ continue
694
+ self._shutdown_tasks.add(task)
695
+ task.add_done_callback(self._observe_shutdown_task)
696
+ if task is not publisher_task:
697
+ task.cancel()
698
+
699
+ def _observe_shutdown_task(self, task: asyncio.Task[None]) -> None:
700
+ if not task.cancelled():
701
+ task.exception() # Consume failures without an unbounded await.
702
+ self._shutdown_tasks.discard(task)
703
+
704
+ async def _stop_without_deadline(self) -> None:
705
+ """Default shutdown: drain and report without an absolute deadline."""
706
+ if self._task is None:
707
+ return
708
+ publisher_drained = await self._publisher.stop(
709
+ timeout_seconds=self._drain_timeout_seconds
710
+ )
711
+ if publisher_drained:
712
+ try:
713
+ await asyncio.wait_for(
714
+ self._drain_submitted(), timeout=self._drain_timeout_seconds
715
+ )
716
+ except TimeoutError:
717
+ pass
718
+ else:
719
+ self._submitted_rounds.clear()
720
+ self._stop.set()
721
+ await self._task
722
+ self._report_stop.set()
723
+ if self._report_task is not None:
724
+ await self._report_task
725
+ self._task = None
726
+ self._report_task = None
727
+
728
+ async def wait_complete(
729
+ self, round_id: RoundId, *, deadline: float
730
+ ) -> ConsensusCompletionResult:
731
+ """Observe final publication, all local members and the report callback.
732
+
733
+ This is opt-in, does not close receivers, and does not assert remote
734
+ receipt of our outgoing sketch. Use a separate application barrier for
735
+ that. The engine must have finished submitting before calling this API.
736
+ """
737
+ if round_id < 0 or not math.isfinite(deadline):
738
+ raise ValueError("valid final round and absolute deadline required")
739
+
740
+ def result(
741
+ status: Literal["complete", "timeout", "dropped", "stopped"],
742
+ ) -> ConsensusCompletionResult:
743
+ observation = self.result(round_id)
744
+ keys = (
745
+ tuple(member.public_key for member in observation.member_distances)
746
+ if isinstance(observation, ConsensusObservation)
747
+ else ()
748
+ )
749
+ return ConsensusCompletionResult(
750
+ round_id,
751
+ status,
752
+ keys,
753
+ round_id in self._completed_reports,
754
+ self.dropped,
755
+ )
756
+
757
+ if asyncio.get_running_loop().time() >= deadline:
758
+ return result("timeout")
759
+ if self._task is None or self._task.done():
760
+ return result("stopped")
761
+ try:
762
+ async with asyncio.timeout_at(deadline):
763
+ await self._publisher.wait_idle()
764
+ while self.result(round_id) is None:
765
+ if self.dropped:
766
+ return result("dropped")
767
+ if self._task.done():
768
+ return result("stopped")
769
+ await asyncio.sleep(0.01)
770
+ await self._distance_queue.join()
771
+ # A result can precede its callback's queue admission. Checking
772
+ # the completed callback marker closes that scheduling seam.
773
+ while round_id not in self._completed_reports:
774
+ if self.dropped:
775
+ return result("dropped")
776
+ if self._report_task is None or self._report_task.done():
777
+ return result("stopped")
778
+ await asyncio.sleep(0.01)
779
+ return result("dropped" if self.dropped else "complete")
780
+ except TimeoutError:
781
+ return result("timeout")
782
+
783
+ async def _accept_local(self, round_id: RoundId, sketch: np.ndarray) -> None:
784
+ await self._accept(
785
+ ConsensusSketchMessage(
786
+ sender_public_key=self._local_public_key,
787
+ round_id=round_id,
788
+ payload=encode_sketch(sketch, size=self._size),
789
+ ),
790
+ sketch,
791
+ )
792
+
793
+ async def _consume(self) -> None:
794
+ while not self._stop.is_set():
795
+ try:
796
+ message = await self._receive(0.1)
797
+ except TimeoutError:
798
+ continue
799
+ try:
800
+ sketch = decode_sketch(message.payload, size=self._size)
801
+ await self._accept(message, sketch)
802
+ except (ConsensusSketchError, TypeError, ValueError):
803
+ continue
804
+
805
+ async def _accept(
806
+ self, message: ConsensusSketchMessage, sketch: np.ndarray
807
+ ) -> None:
808
+ try:
809
+ distance = await asyncio.to_thread(
810
+ self._buffer.add,
811
+ round_id=message.round_id,
812
+ sender_public_key=message.sender_public_key,
813
+ sketch=sketch,
814
+ )
815
+ except (ConsensusSketchError, TypeError, ValueError):
816
+ return
817
+ if distance is None or distance.round_id in self._reported_rounds:
818
+ return
819
+ try:
820
+ self._submitted_rounds.remove(distance.round_id)
821
+ except ValueError:
822
+ pass
823
+ try:
824
+ self._distance_queue.put_nowait(distance)
825
+ except asyncio.QueueFull:
826
+ self._dropped_distance_reports += 1
827
+ return
828
+ self._reported_rounds.add(distance.round_id)
829
+ self._reported_round_order.append(distance.round_id)
830
+ while len(self._reported_round_order) > self._max_reported_rounds:
831
+ expired = self._reported_round_order.popleft()
832
+ self._reported_rounds.remove(expired)
833
+ self._completed_reports.discard(expired)
834
+
835
+ async def _report_distances(self) -> None:
836
+ while not self._report_stop.is_set() or not self._distance_queue.empty():
837
+ try:
838
+ distance = await asyncio.wait_for(
839
+ self._distance_queue.get(), timeout=0.1
840
+ )
841
+ except TimeoutError:
842
+ continue
843
+ self._report_inflight = True
844
+ try:
845
+ if self._on_distance is not None:
846
+ result = self._on_distance(distance)
847
+ if inspect.isawaitable(result):
848
+ await result
849
+ if distance.round_id in self._reported_rounds:
850
+ self._completed_reports.add(distance.round_id)
851
+ except Exception:
852
+ self._dropped_distance_reports += 1
853
+ pass
854
+ finally:
855
+ self._report_inflight = False
856
+ self._distance_queue.task_done()
857
+
858
+ async def _drain_submitted(self) -> None:
859
+ while self._submitted_rounds:
860
+ await asyncio.sleep(0.01)
861
+
862
+
863
+ def _copy_weights(weights: Mapping[str, np.ndarray]) -> dict[str, np.ndarray]:
864
+ return {name: np.ascontiguousarray(value).copy() for name, value in weights.items()}
865
+
866
+
867
+ def _flatten_weights(weights: Mapping[str, np.ndarray]) -> np.ndarray:
868
+ if not weights:
869
+ raise ValueError("weights must contain at least one tensor")
870
+ flattened: list[np.ndarray] = []
871
+ for name in sorted(weights):
872
+ value = np.asarray(weights[name])
873
+ if not np.issubdtype(value.dtype, np.floating):
874
+ raise ValueError(f"tensor {name} must be floating point")
875
+ if not np.isfinite(value).all():
876
+ raise ValueError(f"tensor {name} contains non-finite values")
877
+ flattened.append(value.astype(np.float64, copy=False).reshape(-1))
878
+ return np.concatenate(flattened)
879
+
880
+
881
+ def _validate_sketch(
882
+ sketch: np.ndarray, *, size: int = COUNT_SKETCH_SIZE
883
+ ) -> np.ndarray:
884
+ value = np.asarray(sketch)
885
+ if value.shape != (size,) or value.dtype != np.float32:
886
+ raise ConsensusSketchError("sketch must be a one-dimensional FP32 vector")
887
+ if not np.isfinite(value).all():
888
+ raise ConsensusSketchError("sketch contains non-finite values")
889
+ return value.copy()
890
+
891
+
892
+ def _normalise_seed(seed: int) -> int:
893
+ digest = hashlib.sha256(str(seed).encode()).digest()
894
+ return int.from_bytes(digest[:8], "little")
895
+
896
+
897
+ __all__ = [
898
+ "COUNT_SKETCH_SIZE",
899
+ "ConsensusObservation",
900
+ "ConsensusSketchBuffer",
901
+ "ConsensusSketchError",
902
+ "ConsensusSketchMessage",
903
+ "ConsensusSketchPublisher",
904
+ "LiveConsensusTelemetry",
905
+ "MemberDistance",
906
+ "count_sketch",
907
+ "decode_sketch",
908
+ "encode_sketch",
909
+ ]