dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,909 @@
|
|
|
1
|
+
"""Deterministic live and exact consensus distance calculations."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import asyncio
|
|
6
|
+
import hashlib
|
|
7
|
+
import inspect
|
|
8
|
+
import math
|
|
9
|
+
from collections import deque
|
|
10
|
+
from collections.abc import Awaitable, Callable, Iterator, Mapping, Sequence
|
|
11
|
+
from dataclasses import dataclass
|
|
12
|
+
from threading import RLock
|
|
13
|
+
from typing import Literal
|
|
14
|
+
|
|
15
|
+
import numpy as np
|
|
16
|
+
|
|
17
|
+
from dromeus.errors import RunError
|
|
18
|
+
from dromeus.manifests.models import ConsensusSketchMessage, PublicKey, RoundId
|
|
19
|
+
|
|
20
|
+
COUNT_SKETCH_SIZE = 4096
|
|
21
|
+
# Keep integer draw boundaries aligned across named tensors. NumPy's small
|
|
22
|
+
# integer draws can pack several values into a generator word.
|
|
23
|
+
_COUNT_SKETCH_CHUNK_VALUES = 65536
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
class ConsensusSketchError(RunError, ValueError):
|
|
27
|
+
"""A consensus sketch was malformed or arrived from an invalid member."""
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass(frozen=True, slots=True)
|
|
31
|
+
class MemberDistance:
|
|
32
|
+
"""A sealed member's distance from the complete group's mean sketch."""
|
|
33
|
+
|
|
34
|
+
public_key: PublicKey
|
|
35
|
+
absolute_distance: float
|
|
36
|
+
normalized_distance: float
|
|
37
|
+
|
|
38
|
+
def __post_init__(self) -> None:
|
|
39
|
+
if not self.public_key or any(
|
|
40
|
+
not math.isfinite(value) or value < 0
|
|
41
|
+
for value in (self.absolute_distance, self.normalized_distance)
|
|
42
|
+
):
|
|
43
|
+
raise ConsensusSketchError("invalid member distance")
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@dataclass(frozen=True, slots=True)
|
|
47
|
+
class ConsensusObservation:
|
|
48
|
+
"""One complete round's group spread, scale and per-member distances."""
|
|
49
|
+
|
|
50
|
+
round_id: RoundId
|
|
51
|
+
normalized_rms: float
|
|
52
|
+
sketch_count: int
|
|
53
|
+
absolute_rms_spread: float
|
|
54
|
+
mean_sketch_norm: float
|
|
55
|
+
denominator_degenerate: bool
|
|
56
|
+
member_distances: tuple[MemberDistance, ...] = ()
|
|
57
|
+
|
|
58
|
+
def __post_init__(self) -> None:
|
|
59
|
+
if (
|
|
60
|
+
self.round_id < 0
|
|
61
|
+
or self.sketch_count < 1
|
|
62
|
+
or any(
|
|
63
|
+
not math.isfinite(v) or v < 0
|
|
64
|
+
for v in (
|
|
65
|
+
self.normalized_rms,
|
|
66
|
+
self.absolute_rms_spread,
|
|
67
|
+
self.mean_sketch_norm,
|
|
68
|
+
)
|
|
69
|
+
)
|
|
70
|
+
):
|
|
71
|
+
raise ConsensusSketchError("invalid consensus observation")
|
|
72
|
+
if self.denominator_degenerate != (self.mean_sketch_norm <= 1e-12):
|
|
73
|
+
raise ConsensusSketchError("inconsistent denominator scale flag")
|
|
74
|
+
if not math.isclose(
|
|
75
|
+
self.normalized_rms,
|
|
76
|
+
self.absolute_rms_spread / (self.mean_sketch_norm + 1e-12),
|
|
77
|
+
rel_tol=1e-9,
|
|
78
|
+
abs_tol=1e-12,
|
|
79
|
+
):
|
|
80
|
+
raise ConsensusSketchError("inconsistent consensus scale measurements")
|
|
81
|
+
if self.member_distances:
|
|
82
|
+
if (
|
|
83
|
+
len(self.member_distances) != self.sketch_count
|
|
84
|
+
or len({m.public_key for m in self.member_distances})
|
|
85
|
+
!= self.sketch_count
|
|
86
|
+
or any(
|
|
87
|
+
not math.isclose(
|
|
88
|
+
m.normalized_distance,
|
|
89
|
+
m.absolute_distance / (self.mean_sketch_norm + 1e-12),
|
|
90
|
+
rel_tol=1e-9,
|
|
91
|
+
abs_tol=1e-12,
|
|
92
|
+
)
|
|
93
|
+
for m in self.member_distances
|
|
94
|
+
)
|
|
95
|
+
or not math.isclose(
|
|
96
|
+
math.hypot(*(m.absolute_distance for m in self.member_distances))
|
|
97
|
+
/ math.sqrt(self.sketch_count),
|
|
98
|
+
self.absolute_rms_spread,
|
|
99
|
+
rel_tol=1e-9,
|
|
100
|
+
abs_tol=1e-12,
|
|
101
|
+
)
|
|
102
|
+
):
|
|
103
|
+
raise ConsensusSketchError("inconsistent member distances")
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def consensus_observation(
|
|
107
|
+
sketches: Sequence[np.ndarray],
|
|
108
|
+
*,
|
|
109
|
+
round_id: int,
|
|
110
|
+
participant_keys: Sequence[PublicKey] | None = None,
|
|
111
|
+
) -> ConsensusObservation:
|
|
112
|
+
"""Measure spread and mean norm with a 1e-12 denominator stabilizer."""
|
|
113
|
+
if not sketches:
|
|
114
|
+
raise ConsensusSketchError("at least one sketch is required")
|
|
115
|
+
first = np.asarray(sketches[0])
|
|
116
|
+
if first.ndim != 1 or not first.size:
|
|
117
|
+
raise ConsensusSketchError("sketch must be a nonempty vector")
|
|
118
|
+
values = np.stack([_validate_sketch(x, size=first.size) for x in sketches]).astype(
|
|
119
|
+
np.float64
|
|
120
|
+
)
|
|
121
|
+
return _measure_consensus(
|
|
122
|
+
values, round_id=round_id, participant_keys=participant_keys
|
|
123
|
+
)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def exact_consensus_observation(
|
|
127
|
+
models: Sequence[Mapping[str, np.ndarray]], *, round_id: int
|
|
128
|
+
) -> ConsensusObservation:
|
|
129
|
+
"""Use the same scale definitions for exact public model snapshots."""
|
|
130
|
+
if not models or any(model.keys() != models[0].keys() for model in models):
|
|
131
|
+
raise ValueError("exact models must share their named schema")
|
|
132
|
+
if any(
|
|
133
|
+
model[name].shape != models[0][name].shape for model in models for name in model
|
|
134
|
+
):
|
|
135
|
+
raise ValueError("exact model shapes differ")
|
|
136
|
+
values = np.stack([_flatten_weights(model) for model in models]).astype(np.float64)
|
|
137
|
+
return _measure_consensus(values, round_id=round_id)
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _measure_consensus(
|
|
141
|
+
values: np.ndarray,
|
|
142
|
+
*,
|
|
143
|
+
round_id: int,
|
|
144
|
+
participant_keys: Sequence[PublicKey] | None = None,
|
|
145
|
+
) -> ConsensusObservation:
|
|
146
|
+
if participant_keys is not None and (
|
|
147
|
+
len(participant_keys) != len(values)
|
|
148
|
+
or len(set(participant_keys)) != len(values)
|
|
149
|
+
or any(not key for key in participant_keys)
|
|
150
|
+
):
|
|
151
|
+
raise ConsensusSketchError("participant keys must match sketches and be unique")
|
|
152
|
+
mean = values.mean(axis=0)
|
|
153
|
+
squared_distances = np.sum((values - mean) ** 2, axis=1)
|
|
154
|
+
spread = float(np.sqrt(np.mean(squared_distances)))
|
|
155
|
+
norm = float(np.linalg.norm(mean))
|
|
156
|
+
members = (
|
|
157
|
+
tuple(
|
|
158
|
+
sorted(
|
|
159
|
+
(
|
|
160
|
+
MemberDistance(
|
|
161
|
+
key, float(distance), float(distance) / (norm + 1e-12)
|
|
162
|
+
)
|
|
163
|
+
for key, distance in zip(
|
|
164
|
+
participant_keys, np.sqrt(squared_distances), strict=True
|
|
165
|
+
)
|
|
166
|
+
),
|
|
167
|
+
key=lambda member: member.public_key,
|
|
168
|
+
)
|
|
169
|
+
)
|
|
170
|
+
if participant_keys is not None
|
|
171
|
+
else ()
|
|
172
|
+
)
|
|
173
|
+
return ConsensusObservation(
|
|
174
|
+
round_id=round_id,
|
|
175
|
+
normalized_rms=spread / (norm + 1e-12),
|
|
176
|
+
sketch_count=len(values),
|
|
177
|
+
absolute_rms_spread=spread,
|
|
178
|
+
mean_sketch_norm=norm,
|
|
179
|
+
denominator_degenerate=norm <= 1e-12,
|
|
180
|
+
member_distances=members,
|
|
181
|
+
)
|
|
182
|
+
|
|
183
|
+
|
|
184
|
+
SketchReceiver = Callable[[float], Awaitable[ConsensusSketchMessage]]
|
|
185
|
+
SketchPublisher = Callable[[RoundId, np.ndarray], Awaitable[None]]
|
|
186
|
+
DistanceCallback = Callable[[ConsensusObservation], None | Awaitable[None]]
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
def count_sketch(
|
|
190
|
+
weights: Mapping[str, np.ndarray],
|
|
191
|
+
*,
|
|
192
|
+
seed: int,
|
|
193
|
+
size: int = COUNT_SKETCH_SIZE,
|
|
194
|
+
) -> np.ndarray:
|
|
195
|
+
"""Return a deterministic signed CountSketch of named finite tensors."""
|
|
196
|
+
if size <= 0:
|
|
197
|
+
raise ValueError("sketch size must be positive")
|
|
198
|
+
arrays = _validated_sketch_arrays(weights)
|
|
199
|
+
element_count = sum(value.size for value in arrays)
|
|
200
|
+
normalised_seed = _normalise_seed(seed)
|
|
201
|
+
bucket_rng = np.random.default_rng(normalised_seed)
|
|
202
|
+
sign_rng = np.random.default_rng(normalised_seed)
|
|
203
|
+
# The projection draws ALL buckets before ANY signs. Replay that
|
|
204
|
+
# first segment in bounded chunks for the sign stream instead of changing
|
|
205
|
+
# the seed/distribution or assuming a raw bit-generator advance count.
|
|
206
|
+
for start in range(0, element_count, _COUNT_SKETCH_CHUNK_VALUES):
|
|
207
|
+
sign_rng.integers(
|
|
208
|
+
0, size, size=min(_COUNT_SKETCH_CHUNK_VALUES, element_count - start)
|
|
209
|
+
)
|
|
210
|
+
sketch = np.zeros(size, dtype=np.float32)
|
|
211
|
+
for values in _sketch_value_chunks(arrays):
|
|
212
|
+
buckets = bucket_rng.integers(0, size, size=values.size)
|
|
213
|
+
signs = np.where(
|
|
214
|
+
sign_rng.integers(0, 2, size=values.size, dtype=np.int8) == 0,
|
|
215
|
+
-1.0,
|
|
216
|
+
1.0,
|
|
217
|
+
)
|
|
218
|
+
np.add.at(sketch, buckets, values * signs)
|
|
219
|
+
return sketch
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def _validated_sketch_arrays(
|
|
223
|
+
weights: Mapping[str, np.ndarray],
|
|
224
|
+
) -> tuple[np.ndarray, ...]:
|
|
225
|
+
"""Validate in projection name order without allocating model-sized copies."""
|
|
226
|
+
if not weights:
|
|
227
|
+
raise ValueError("weights must contain at least one tensor")
|
|
228
|
+
arrays: list[np.ndarray] = []
|
|
229
|
+
for name in sorted(weights):
|
|
230
|
+
value = np.asarray(weights[name])
|
|
231
|
+
if not np.issubdtype(value.dtype, np.floating):
|
|
232
|
+
raise ValueError(f"tensor {name} must be floating point")
|
|
233
|
+
for start in range(0, value.size, _COUNT_SKETCH_CHUNK_VALUES):
|
|
234
|
+
if not np.isfinite(
|
|
235
|
+
value.flat[start : start + _COUNT_SKETCH_CHUNK_VALUES]
|
|
236
|
+
).all():
|
|
237
|
+
raise ValueError(f"tensor {name} contains non-finite values")
|
|
238
|
+
arrays.append(value)
|
|
239
|
+
return tuple(arrays)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def _sketch_value_chunks(arrays: Sequence[np.ndarray]) -> Iterator[np.ndarray]:
|
|
243
|
+
"""Flatten globally in C order, keeping per-name RNG boundaries."""
|
|
244
|
+
buffer = np.empty(_COUNT_SKETCH_CHUNK_VALUES, dtype=np.float64)
|
|
245
|
+
used = 0
|
|
246
|
+
for value in arrays:
|
|
247
|
+
position = 0
|
|
248
|
+
while position < value.size:
|
|
249
|
+
count = min(_COUNT_SKETCH_CHUNK_VALUES - used, value.size - position)
|
|
250
|
+
buffer[used : used + count] = value.flat[position : position + count]
|
|
251
|
+
used += count
|
|
252
|
+
position += count
|
|
253
|
+
if used == _COUNT_SKETCH_CHUNK_VALUES:
|
|
254
|
+
yield buffer
|
|
255
|
+
used = 0
|
|
256
|
+
if used:
|
|
257
|
+
yield buffer[:used]
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
def encode_sketch(sketch: np.ndarray, *, size: int = COUNT_SKETCH_SIZE) -> bytes:
|
|
261
|
+
"""Encode exactly one FP32 sketch without a JSON or MessagePack expansion."""
|
|
262
|
+
value = _validate_sketch(sketch, size=size)
|
|
263
|
+
return value.astype("<f4", copy=False).tobytes()
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def decode_sketch(data: bytes, *, size: int = COUNT_SKETCH_SIZE) -> np.ndarray:
|
|
267
|
+
"""Decode a bounded FP32 sketch payload."""
|
|
268
|
+
if len(data) != size * np.dtype(np.float32).itemsize:
|
|
269
|
+
raise ConsensusSketchError("sketch payload has an unexpected size")
|
|
270
|
+
value = np.frombuffer(data, dtype="<f4").copy()
|
|
271
|
+
return _validate_sketch(value, size=size)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
class ConsensusSketchBuffer:
|
|
275
|
+
"""Bounded round-keyed sketch buffer that never waits on training."""
|
|
276
|
+
|
|
277
|
+
def __init__(
|
|
278
|
+
self,
|
|
279
|
+
*,
|
|
280
|
+
participant_keys: Sequence[PublicKey],
|
|
281
|
+
size: int = COUNT_SKETCH_SIZE,
|
|
282
|
+
max_pending_rounds: int = 64,
|
|
283
|
+
) -> None:
|
|
284
|
+
if not participant_keys or len(set(participant_keys)) != len(participant_keys):
|
|
285
|
+
raise ValueError("participant keys must be non-empty and unique")
|
|
286
|
+
if size <= 0:
|
|
287
|
+
raise ValueError("sketch size must be positive")
|
|
288
|
+
if max_pending_rounds <= 0:
|
|
289
|
+
raise ValueError("max_pending_rounds must be positive")
|
|
290
|
+
self._participant_keys = frozenset(participant_keys)
|
|
291
|
+
self._size = size
|
|
292
|
+
self._max_pending_rounds = max_pending_rounds
|
|
293
|
+
self._lock = RLock()
|
|
294
|
+
self._pending: dict[RoundId, dict[PublicKey, np.ndarray]] = {}
|
|
295
|
+
self._results: dict[RoundId, ConsensusObservation] = {}
|
|
296
|
+
self._dropped = 0
|
|
297
|
+
|
|
298
|
+
def add(
|
|
299
|
+
self,
|
|
300
|
+
*,
|
|
301
|
+
round_id: RoundId,
|
|
302
|
+
sender_public_key: PublicKey,
|
|
303
|
+
sketch: np.ndarray,
|
|
304
|
+
) -> ConsensusObservation | None:
|
|
305
|
+
with self._lock:
|
|
306
|
+
if sender_public_key not in self._participant_keys:
|
|
307
|
+
raise ConsensusSketchError("sender is not a sealed participant")
|
|
308
|
+
value = _validate_sketch(sketch, size=self._size)
|
|
309
|
+
existing_result = self._results.get(round_id)
|
|
310
|
+
if existing_result is not None:
|
|
311
|
+
existing = self._pending[round_id][sender_public_key]
|
|
312
|
+
if not np.array_equal(existing, value):
|
|
313
|
+
raise ConsensusSketchError("duplicate sketch does not match")
|
|
314
|
+
return existing_result
|
|
315
|
+
if (
|
|
316
|
+
round_id not in self._pending
|
|
317
|
+
and len(self.pending_rounds()) >= self._max_pending_rounds
|
|
318
|
+
):
|
|
319
|
+
oldest_round = min(self.pending_rounds())
|
|
320
|
+
self._pending.pop(oldest_round, None)
|
|
321
|
+
self._dropped += 1
|
|
322
|
+
round_sketches = self._pending.setdefault(round_id, {})
|
|
323
|
+
existing = round_sketches.get(sender_public_key)
|
|
324
|
+
if existing is not None:
|
|
325
|
+
if not np.array_equal(existing, value):
|
|
326
|
+
raise ConsensusSketchError("duplicate sketch does not match")
|
|
327
|
+
else:
|
|
328
|
+
round_sketches[sender_public_key] = value
|
|
329
|
+
if len(round_sketches) != len(self._participant_keys):
|
|
330
|
+
return None
|
|
331
|
+
result = consensus_observation(
|
|
332
|
+
[round_sketches[key] for key in sorted(round_sketches)],
|
|
333
|
+
round_id=round_id,
|
|
334
|
+
participant_keys=sorted(round_sketches),
|
|
335
|
+
)
|
|
336
|
+
self._results[round_id] = result
|
|
337
|
+
completed_rounds = sorted(self._results)
|
|
338
|
+
for old_round in completed_rounds[: -self._max_pending_rounds]:
|
|
339
|
+
self._results.pop(old_round, None)
|
|
340
|
+
self._pending.pop(old_round, None)
|
|
341
|
+
return result
|
|
342
|
+
|
|
343
|
+
def result(self, round_id: RoundId) -> ConsensusObservation | None:
|
|
344
|
+
with self._lock:
|
|
345
|
+
return self._results.get(round_id)
|
|
346
|
+
|
|
347
|
+
@property
|
|
348
|
+
def dropped(self) -> int:
|
|
349
|
+
"""Return incomplete rounds evicted to keep the buffer bounded."""
|
|
350
|
+
return self._dropped
|
|
351
|
+
|
|
352
|
+
def pending_rounds(self) -> tuple[RoundId, ...]:
|
|
353
|
+
with self._lock:
|
|
354
|
+
return tuple(
|
|
355
|
+
sorted(
|
|
356
|
+
round_id
|
|
357
|
+
for round_id in self._pending
|
|
358
|
+
if round_id not in self._results
|
|
359
|
+
)
|
|
360
|
+
)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
@dataclass(frozen=True, slots=True)
|
|
364
|
+
class ConsensusCompletionResult:
|
|
365
|
+
"""Local final-round evidence; never an all-peer delivery acknowledgement."""
|
|
366
|
+
|
|
367
|
+
round_id: RoundId
|
|
368
|
+
status: Literal["complete", "timeout", "dropped", "stopped"]
|
|
369
|
+
received_member_keys: tuple[PublicKey, ...]
|
|
370
|
+
report_completed: bool
|
|
371
|
+
dropped: int
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
class ConsensusSketchPublisher:
|
|
375
|
+
"""Bounded background sketch computation and publication queue."""
|
|
376
|
+
|
|
377
|
+
def __init__(
|
|
378
|
+
self,
|
|
379
|
+
*,
|
|
380
|
+
seed: int,
|
|
381
|
+
publish: Callable[[RoundId, np.ndarray], Awaitable[None]],
|
|
382
|
+
size: int = COUNT_SKETCH_SIZE,
|
|
383
|
+
max_queue_size: int = 64,
|
|
384
|
+
on_sketch: Callable[[RoundId, np.ndarray], None | Awaitable[None]]
|
|
385
|
+
| None = None,
|
|
386
|
+
) -> None:
|
|
387
|
+
if max_queue_size <= 0:
|
|
388
|
+
raise ValueError("max_queue_size must be positive")
|
|
389
|
+
self._seed = seed
|
|
390
|
+
self._publish = publish
|
|
391
|
+
self._on_sketch = on_sketch
|
|
392
|
+
self._size = size
|
|
393
|
+
self._dropped = 0
|
|
394
|
+
self._queue: asyncio.Queue[tuple[RoundId, Mapping[str, np.ndarray]]] = (
|
|
395
|
+
asyncio.Queue(maxsize=max_queue_size)
|
|
396
|
+
)
|
|
397
|
+
self._stop = asyncio.Event()
|
|
398
|
+
self._task: asyncio.Task[None] | None = None
|
|
399
|
+
self._inflight = False
|
|
400
|
+
self._thread_call_active = False
|
|
401
|
+
|
|
402
|
+
def submit(self, *, round_id: RoundId, weights: Mapping[str, np.ndarray]) -> bool:
|
|
403
|
+
"""Queue a model snapshot, returning false when telemetry is full."""
|
|
404
|
+
if self._queue.full():
|
|
405
|
+
self._dropped += 1
|
|
406
|
+
return False
|
|
407
|
+
try:
|
|
408
|
+
self._queue.put_nowait((round_id, weights))
|
|
409
|
+
except asyncio.QueueFull:
|
|
410
|
+
self._dropped += 1
|
|
411
|
+
return False
|
|
412
|
+
return True
|
|
413
|
+
|
|
414
|
+
async def start(self) -> None:
|
|
415
|
+
if self._task is None:
|
|
416
|
+
self._task = asyncio.create_task(self._run(), name="dromeus-consensus")
|
|
417
|
+
|
|
418
|
+
async def stop(self, *, timeout_seconds: float | None = None) -> bool:
|
|
419
|
+
if timeout_seconds is not None and timeout_seconds <= 0:
|
|
420
|
+
raise ValueError("stop timeout must be positive")
|
|
421
|
+
self._stop.set()
|
|
422
|
+
task = self._task
|
|
423
|
+
if task is None:
|
|
424
|
+
return True
|
|
425
|
+
try:
|
|
426
|
+
if timeout_seconds is None:
|
|
427
|
+
await task
|
|
428
|
+
else:
|
|
429
|
+
await asyncio.wait_for(task, timeout=timeout_seconds)
|
|
430
|
+
return True
|
|
431
|
+
except TimeoutError:
|
|
432
|
+
self._dropped += 1 + self._queue.qsize()
|
|
433
|
+
return False
|
|
434
|
+
finally:
|
|
435
|
+
self._task = None
|
|
436
|
+
|
|
437
|
+
@property
|
|
438
|
+
def pending(self) -> int:
|
|
439
|
+
return self._queue.qsize()
|
|
440
|
+
|
|
441
|
+
@property
|
|
442
|
+
def dropped(self) -> int:
|
|
443
|
+
"""Return the number of snapshots rejected by the bounded queue."""
|
|
444
|
+
return self._dropped
|
|
445
|
+
|
|
446
|
+
async def wait_idle(self) -> None:
|
|
447
|
+
"""Wait for queued AND in-flight publication without stopping its worker.
|
|
448
|
+
|
|
449
|
+
The caller must first finish submissions. Cancellation affects this wait,
|
|
450
|
+
not the background projection thread or publisher task.
|
|
451
|
+
"""
|
|
452
|
+
await self._queue.join()
|
|
453
|
+
|
|
454
|
+
async def finish(self) -> None:
|
|
455
|
+
"""Finish accepted publication without cascading caller cancellation."""
|
|
456
|
+
await self.wait_idle()
|
|
457
|
+
self._stop.set()
|
|
458
|
+
if self._task is not None:
|
|
459
|
+
await asyncio.shield(self._task)
|
|
460
|
+
|
|
461
|
+
def abort(self) -> tuple[asyncio.Task[None] | None, bool]:
|
|
462
|
+
"""Cancel async publication once; native projection may continue."""
|
|
463
|
+
thread_may_continue = self._thread_call_active
|
|
464
|
+
self._dropped += self.pending + int(self._inflight)
|
|
465
|
+
self._stop.set()
|
|
466
|
+
while not self._queue.empty():
|
|
467
|
+
self._queue.get_nowait()
|
|
468
|
+
self._queue.task_done()
|
|
469
|
+
if self._task is not None and not self._task.done():
|
|
470
|
+
self._task.cancel()
|
|
471
|
+
return self._task, thread_may_continue
|
|
472
|
+
|
|
473
|
+
async def _run(self) -> None:
|
|
474
|
+
while not self._stop.is_set() or not self._queue.empty():
|
|
475
|
+
try:
|
|
476
|
+
round_id, weights = await asyncio.wait_for(
|
|
477
|
+
self._queue.get(), timeout=0.1
|
|
478
|
+
)
|
|
479
|
+
except TimeoutError:
|
|
480
|
+
continue
|
|
481
|
+
self._inflight = True
|
|
482
|
+
try:
|
|
483
|
+
self._thread_call_active = True
|
|
484
|
+
snapshot = await asyncio.to_thread(_copy_weights, weights)
|
|
485
|
+
sketch = await asyncio.to_thread(
|
|
486
|
+
count_sketch,
|
|
487
|
+
snapshot,
|
|
488
|
+
seed=self._seed,
|
|
489
|
+
size=self._size,
|
|
490
|
+
)
|
|
491
|
+
self._thread_call_active = False
|
|
492
|
+
if self._on_sketch is not None:
|
|
493
|
+
try:
|
|
494
|
+
result = self._on_sketch(round_id, sketch)
|
|
495
|
+
if inspect.isawaitable(result):
|
|
496
|
+
await result
|
|
497
|
+
except Exception:
|
|
498
|
+
self._dropped += 1
|
|
499
|
+
pass
|
|
500
|
+
try:
|
|
501
|
+
await self._publish(round_id, sketch)
|
|
502
|
+
except Exception:
|
|
503
|
+
self._dropped += 1
|
|
504
|
+
pass
|
|
505
|
+
except Exception:
|
|
506
|
+
self._dropped += 1
|
|
507
|
+
pass
|
|
508
|
+
finally:
|
|
509
|
+
# A live publisher can wait indefinitely for the next item.
|
|
510
|
+
# Do not keep the last model and defensive copy in its frame.
|
|
511
|
+
self._inflight = False
|
|
512
|
+
self._thread_call_active = False
|
|
513
|
+
weights = None
|
|
514
|
+
snapshot = None
|
|
515
|
+
self._queue.task_done()
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
class LiveConsensusTelemetry:
|
|
519
|
+
"""Publish local sketches and consume remote sketches off training path."""
|
|
520
|
+
|
|
521
|
+
def __init__(
|
|
522
|
+
self,
|
|
523
|
+
*,
|
|
524
|
+
local_public_key: PublicKey,
|
|
525
|
+
participant_keys: Sequence[PublicKey],
|
|
526
|
+
seed: int,
|
|
527
|
+
receive: SketchReceiver,
|
|
528
|
+
publish: SketchPublisher,
|
|
529
|
+
on_distance: DistanceCallback | None = None,
|
|
530
|
+
size: int = COUNT_SKETCH_SIZE,
|
|
531
|
+
max_pending_rounds: int = 64,
|
|
532
|
+
max_queue_size: int = 64,
|
|
533
|
+
drain_timeout_seconds: float = 5.0,
|
|
534
|
+
) -> None:
|
|
535
|
+
if drain_timeout_seconds <= 0:
|
|
536
|
+
raise ValueError("drain timeout must be positive")
|
|
537
|
+
self._local_public_key = local_public_key
|
|
538
|
+
self._receive = receive
|
|
539
|
+
self._on_distance = on_distance
|
|
540
|
+
self._buffer = ConsensusSketchBuffer(
|
|
541
|
+
participant_keys=participant_keys,
|
|
542
|
+
size=size,
|
|
543
|
+
max_pending_rounds=max_pending_rounds,
|
|
544
|
+
)
|
|
545
|
+
self._publisher = ConsensusSketchPublisher(
|
|
546
|
+
seed=seed,
|
|
547
|
+
publish=publish,
|
|
548
|
+
size=size,
|
|
549
|
+
max_queue_size=max_queue_size,
|
|
550
|
+
on_sketch=self._accept_local,
|
|
551
|
+
)
|
|
552
|
+
self._size = size
|
|
553
|
+
self._drain_timeout_seconds = drain_timeout_seconds
|
|
554
|
+
self._submitted_rounds: deque[RoundId] = deque()
|
|
555
|
+
self._reported_rounds: set[RoundId] = set()
|
|
556
|
+
self._completed_reports: set[RoundId] = set()
|
|
557
|
+
self._reported_round_order: deque[RoundId] = deque()
|
|
558
|
+
self._max_pending_rounds = max_pending_rounds
|
|
559
|
+
self._max_reported_rounds = max_pending_rounds
|
|
560
|
+
self._distance_queue: asyncio.Queue[ConsensusObservation] = asyncio.Queue(
|
|
561
|
+
maxsize=max_pending_rounds
|
|
562
|
+
)
|
|
563
|
+
self._dropped_distance_reports = 0
|
|
564
|
+
self._stop = asyncio.Event()
|
|
565
|
+
self._report_stop = asyncio.Event()
|
|
566
|
+
self._task: asyncio.Task[None] | None = None
|
|
567
|
+
self._report_task: asyncio.Task[None] | None = None
|
|
568
|
+
self._report_inflight = False
|
|
569
|
+
self._shutdown_waiter: asyncio.Task[None] | None = None
|
|
570
|
+
self._shutdown_tasks: set[asyncio.Task[None]] = set()
|
|
571
|
+
self._shutdown_incomplete = False
|
|
572
|
+
self._projection_thread_may_continue = False
|
|
573
|
+
|
|
574
|
+
@property
|
|
575
|
+
def pending(self) -> int:
|
|
576
|
+
"""Return queued local sketches awaiting background publication."""
|
|
577
|
+
return self._publisher.pending
|
|
578
|
+
|
|
579
|
+
def submit(self, *, round_id: RoundId, weights: Mapping[str, np.ndarray]) -> bool:
|
|
580
|
+
"""Queue one committed local model without waiting on telemetry."""
|
|
581
|
+
submitted = self._publisher.submit(round_id=round_id, weights=weights)
|
|
582
|
+
if submitted:
|
|
583
|
+
self._submitted_rounds.append(round_id)
|
|
584
|
+
while len(self._submitted_rounds) > self._max_pending_rounds:
|
|
585
|
+
self._submitted_rounds.popleft()
|
|
586
|
+
return submitted
|
|
587
|
+
|
|
588
|
+
def result(self, round_id: RoundId) -> ConsensusObservation | None:
|
|
589
|
+
"""Return completed live consensus for one round, when available."""
|
|
590
|
+
return self._buffer.result(round_id)
|
|
591
|
+
|
|
592
|
+
@property
|
|
593
|
+
def dropped(self) -> int:
|
|
594
|
+
"""Return local sketch and distance-report drops."""
|
|
595
|
+
return (
|
|
596
|
+
self._publisher.dropped
|
|
597
|
+
+ self._buffer.dropped
|
|
598
|
+
+ self._dropped_distance_reports
|
|
599
|
+
)
|
|
600
|
+
|
|
601
|
+
async def start(self) -> None:
|
|
602
|
+
if self._task is None:
|
|
603
|
+
self._stop.clear()
|
|
604
|
+
self._report_stop.clear()
|
|
605
|
+
self._task = asyncio.create_task(
|
|
606
|
+
self._consume(), name="dromeus-consensus-recv"
|
|
607
|
+
)
|
|
608
|
+
self._report_task = asyncio.create_task(
|
|
609
|
+
self._report_distances(), name="dromeus-consensus-report"
|
|
610
|
+
)
|
|
611
|
+
await self._publisher.start()
|
|
612
|
+
|
|
613
|
+
@property
|
|
614
|
+
def shutdown_incomplete(self) -> bool:
|
|
615
|
+
"""Whether opted-in cleanup exhausted its deadline or was cancelled."""
|
|
616
|
+
return self._shutdown_incomplete
|
|
617
|
+
|
|
618
|
+
@property
|
|
619
|
+
def pending_shutdown_tasks(self) -> int:
|
|
620
|
+
"""Unfinished asyncio tasks, NOT proof that native threads have stopped."""
|
|
621
|
+
return sum(not task.done() for task in self._shutdown_tasks)
|
|
622
|
+
|
|
623
|
+
@property
|
|
624
|
+
def projection_thread_may_continue(self) -> bool:
|
|
625
|
+
"""Conservative flag: cancelling to_thread cannot kill its native thread."""
|
|
626
|
+
return self._projection_thread_may_continue
|
|
627
|
+
|
|
628
|
+
async def stop(self, *, deadline: float | None = None) -> None:
|
|
629
|
+
if self._shutdown_incomplete:
|
|
630
|
+
return # Preserve bounded/idempotent shutdown after an opted-in abort.
|
|
631
|
+
if deadline is None:
|
|
632
|
+
await self._stop_without_deadline()
|
|
633
|
+
return
|
|
634
|
+
if not math.isfinite(deadline):
|
|
635
|
+
raise ValueError("finite absolute shutdown deadline required")
|
|
636
|
+
if self._shutdown_incomplete:
|
|
637
|
+
return # Idempotent; never repeatedly cancel an uncooperative callback.
|
|
638
|
+
if self._task is None:
|
|
639
|
+
return
|
|
640
|
+
if self._shutdown_waiter is None:
|
|
641
|
+
self._shutdown_waiter = asyncio.create_task(
|
|
642
|
+
self._finish_without_cancelling_workers(), name="consensus-finish"
|
|
643
|
+
)
|
|
644
|
+
waiter = self._shutdown_waiter
|
|
645
|
+
try:
|
|
646
|
+
done, _ = await asyncio.wait(
|
|
647
|
+
{waiter}, timeout=max(0.0, deadline - asyncio.get_running_loop().time())
|
|
648
|
+
)
|
|
649
|
+
if waiter in done:
|
|
650
|
+
waiter.result()
|
|
651
|
+
return
|
|
652
|
+
except BaseException:
|
|
653
|
+
self._abort_bounded_shutdown()
|
|
654
|
+
raise
|
|
655
|
+
self._abort_bounded_shutdown()
|
|
656
|
+
|
|
657
|
+
async def _finish_without_cancelling_workers(self) -> None:
|
|
658
|
+
await self._publisher.finish()
|
|
659
|
+
await self._drain_submitted()
|
|
660
|
+
self._stop.set()
|
|
661
|
+
if self._task is not None:
|
|
662
|
+
await asyncio.shield(self._task)
|
|
663
|
+
self._report_stop.set()
|
|
664
|
+
await self._distance_queue.join()
|
|
665
|
+
if self._report_task is not None:
|
|
666
|
+
await asyncio.shield(self._report_task)
|
|
667
|
+
self._task = None
|
|
668
|
+
self._report_task = None
|
|
669
|
+
|
|
670
|
+
def _abort_bounded_shutdown(self) -> None:
|
|
671
|
+
if self._shutdown_incomplete:
|
|
672
|
+
return
|
|
673
|
+
self._shutdown_incomplete = True
|
|
674
|
+
publisher_task, self._projection_thread_may_continue = self._publisher.abort()
|
|
675
|
+
self._dropped_distance_reports += self._distance_queue.qsize() + int(
|
|
676
|
+
self._report_inflight
|
|
677
|
+
)
|
|
678
|
+
self._stop.set()
|
|
679
|
+
self._report_stop.set()
|
|
680
|
+
self._submitted_rounds.clear()
|
|
681
|
+
# Release queued models/reports. In-flight frames are released by their
|
|
682
|
+
# cooperative finally blocks; an ignoring callback stays referenced.
|
|
683
|
+
while not self._distance_queue.empty():
|
|
684
|
+
self._distance_queue.get_nowait()
|
|
685
|
+
self._distance_queue.task_done()
|
|
686
|
+
for task in (
|
|
687
|
+
self._shutdown_waiter,
|
|
688
|
+
publisher_task,
|
|
689
|
+
self._task,
|
|
690
|
+
self._report_task,
|
|
691
|
+
):
|
|
692
|
+
if task is None or task.done():
|
|
693
|
+
continue
|
|
694
|
+
self._shutdown_tasks.add(task)
|
|
695
|
+
task.add_done_callback(self._observe_shutdown_task)
|
|
696
|
+
if task is not publisher_task:
|
|
697
|
+
task.cancel()
|
|
698
|
+
|
|
699
|
+
def _observe_shutdown_task(self, task: asyncio.Task[None]) -> None:
|
|
700
|
+
if not task.cancelled():
|
|
701
|
+
task.exception() # Consume failures without an unbounded await.
|
|
702
|
+
self._shutdown_tasks.discard(task)
|
|
703
|
+
|
|
704
|
+
async def _stop_without_deadline(self) -> None:
|
|
705
|
+
"""Default shutdown: drain and report without an absolute deadline."""
|
|
706
|
+
if self._task is None:
|
|
707
|
+
return
|
|
708
|
+
publisher_drained = await self._publisher.stop(
|
|
709
|
+
timeout_seconds=self._drain_timeout_seconds
|
|
710
|
+
)
|
|
711
|
+
if publisher_drained:
|
|
712
|
+
try:
|
|
713
|
+
await asyncio.wait_for(
|
|
714
|
+
self._drain_submitted(), timeout=self._drain_timeout_seconds
|
|
715
|
+
)
|
|
716
|
+
except TimeoutError:
|
|
717
|
+
pass
|
|
718
|
+
else:
|
|
719
|
+
self._submitted_rounds.clear()
|
|
720
|
+
self._stop.set()
|
|
721
|
+
await self._task
|
|
722
|
+
self._report_stop.set()
|
|
723
|
+
if self._report_task is not None:
|
|
724
|
+
await self._report_task
|
|
725
|
+
self._task = None
|
|
726
|
+
self._report_task = None
|
|
727
|
+
|
|
728
|
+
async def wait_complete(
|
|
729
|
+
self, round_id: RoundId, *, deadline: float
|
|
730
|
+
) -> ConsensusCompletionResult:
|
|
731
|
+
"""Observe final publication, all local members and the report callback.
|
|
732
|
+
|
|
733
|
+
This is opt-in, does not close receivers, and does not assert remote
|
|
734
|
+
receipt of our outgoing sketch. Use a separate application barrier for
|
|
735
|
+
that. The engine must have finished submitting before calling this API.
|
|
736
|
+
"""
|
|
737
|
+
if round_id < 0 or not math.isfinite(deadline):
|
|
738
|
+
raise ValueError("valid final round and absolute deadline required")
|
|
739
|
+
|
|
740
|
+
def result(
|
|
741
|
+
status: Literal["complete", "timeout", "dropped", "stopped"],
|
|
742
|
+
) -> ConsensusCompletionResult:
|
|
743
|
+
observation = self.result(round_id)
|
|
744
|
+
keys = (
|
|
745
|
+
tuple(member.public_key for member in observation.member_distances)
|
|
746
|
+
if isinstance(observation, ConsensusObservation)
|
|
747
|
+
else ()
|
|
748
|
+
)
|
|
749
|
+
return ConsensusCompletionResult(
|
|
750
|
+
round_id,
|
|
751
|
+
status,
|
|
752
|
+
keys,
|
|
753
|
+
round_id in self._completed_reports,
|
|
754
|
+
self.dropped,
|
|
755
|
+
)
|
|
756
|
+
|
|
757
|
+
if asyncio.get_running_loop().time() >= deadline:
|
|
758
|
+
return result("timeout")
|
|
759
|
+
if self._task is None or self._task.done():
|
|
760
|
+
return result("stopped")
|
|
761
|
+
try:
|
|
762
|
+
async with asyncio.timeout_at(deadline):
|
|
763
|
+
await self._publisher.wait_idle()
|
|
764
|
+
while self.result(round_id) is None:
|
|
765
|
+
if self.dropped:
|
|
766
|
+
return result("dropped")
|
|
767
|
+
if self._task.done():
|
|
768
|
+
return result("stopped")
|
|
769
|
+
await asyncio.sleep(0.01)
|
|
770
|
+
await self._distance_queue.join()
|
|
771
|
+
# A result can precede its callback's queue admission. Checking
|
|
772
|
+
# the completed callback marker closes that scheduling seam.
|
|
773
|
+
while round_id not in self._completed_reports:
|
|
774
|
+
if self.dropped:
|
|
775
|
+
return result("dropped")
|
|
776
|
+
if self._report_task is None or self._report_task.done():
|
|
777
|
+
return result("stopped")
|
|
778
|
+
await asyncio.sleep(0.01)
|
|
779
|
+
return result("dropped" if self.dropped else "complete")
|
|
780
|
+
except TimeoutError:
|
|
781
|
+
return result("timeout")
|
|
782
|
+
|
|
783
|
+
async def _accept_local(self, round_id: RoundId, sketch: np.ndarray) -> None:
|
|
784
|
+
await self._accept(
|
|
785
|
+
ConsensusSketchMessage(
|
|
786
|
+
sender_public_key=self._local_public_key,
|
|
787
|
+
round_id=round_id,
|
|
788
|
+
payload=encode_sketch(sketch, size=self._size),
|
|
789
|
+
),
|
|
790
|
+
sketch,
|
|
791
|
+
)
|
|
792
|
+
|
|
793
|
+
async def _consume(self) -> None:
|
|
794
|
+
while not self._stop.is_set():
|
|
795
|
+
try:
|
|
796
|
+
message = await self._receive(0.1)
|
|
797
|
+
except TimeoutError:
|
|
798
|
+
continue
|
|
799
|
+
try:
|
|
800
|
+
sketch = decode_sketch(message.payload, size=self._size)
|
|
801
|
+
await self._accept(message, sketch)
|
|
802
|
+
except (ConsensusSketchError, TypeError, ValueError):
|
|
803
|
+
continue
|
|
804
|
+
|
|
805
|
+
async def _accept(
|
|
806
|
+
self, message: ConsensusSketchMessage, sketch: np.ndarray
|
|
807
|
+
) -> None:
|
|
808
|
+
try:
|
|
809
|
+
distance = await asyncio.to_thread(
|
|
810
|
+
self._buffer.add,
|
|
811
|
+
round_id=message.round_id,
|
|
812
|
+
sender_public_key=message.sender_public_key,
|
|
813
|
+
sketch=sketch,
|
|
814
|
+
)
|
|
815
|
+
except (ConsensusSketchError, TypeError, ValueError):
|
|
816
|
+
return
|
|
817
|
+
if distance is None or distance.round_id in self._reported_rounds:
|
|
818
|
+
return
|
|
819
|
+
try:
|
|
820
|
+
self._submitted_rounds.remove(distance.round_id)
|
|
821
|
+
except ValueError:
|
|
822
|
+
pass
|
|
823
|
+
try:
|
|
824
|
+
self._distance_queue.put_nowait(distance)
|
|
825
|
+
except asyncio.QueueFull:
|
|
826
|
+
self._dropped_distance_reports += 1
|
|
827
|
+
return
|
|
828
|
+
self._reported_rounds.add(distance.round_id)
|
|
829
|
+
self._reported_round_order.append(distance.round_id)
|
|
830
|
+
while len(self._reported_round_order) > self._max_reported_rounds:
|
|
831
|
+
expired = self._reported_round_order.popleft()
|
|
832
|
+
self._reported_rounds.remove(expired)
|
|
833
|
+
self._completed_reports.discard(expired)
|
|
834
|
+
|
|
835
|
+
async def _report_distances(self) -> None:
|
|
836
|
+
while not self._report_stop.is_set() or not self._distance_queue.empty():
|
|
837
|
+
try:
|
|
838
|
+
distance = await asyncio.wait_for(
|
|
839
|
+
self._distance_queue.get(), timeout=0.1
|
|
840
|
+
)
|
|
841
|
+
except TimeoutError:
|
|
842
|
+
continue
|
|
843
|
+
self._report_inflight = True
|
|
844
|
+
try:
|
|
845
|
+
if self._on_distance is not None:
|
|
846
|
+
result = self._on_distance(distance)
|
|
847
|
+
if inspect.isawaitable(result):
|
|
848
|
+
await result
|
|
849
|
+
if distance.round_id in self._reported_rounds:
|
|
850
|
+
self._completed_reports.add(distance.round_id)
|
|
851
|
+
except Exception:
|
|
852
|
+
self._dropped_distance_reports += 1
|
|
853
|
+
pass
|
|
854
|
+
finally:
|
|
855
|
+
self._report_inflight = False
|
|
856
|
+
self._distance_queue.task_done()
|
|
857
|
+
|
|
858
|
+
async def _drain_submitted(self) -> None:
|
|
859
|
+
while self._submitted_rounds:
|
|
860
|
+
await asyncio.sleep(0.01)
|
|
861
|
+
|
|
862
|
+
|
|
863
|
+
def _copy_weights(weights: Mapping[str, np.ndarray]) -> dict[str, np.ndarray]:
|
|
864
|
+
return {name: np.ascontiguousarray(value).copy() for name, value in weights.items()}
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def _flatten_weights(weights: Mapping[str, np.ndarray]) -> np.ndarray:
|
|
868
|
+
if not weights:
|
|
869
|
+
raise ValueError("weights must contain at least one tensor")
|
|
870
|
+
flattened: list[np.ndarray] = []
|
|
871
|
+
for name in sorted(weights):
|
|
872
|
+
value = np.asarray(weights[name])
|
|
873
|
+
if not np.issubdtype(value.dtype, np.floating):
|
|
874
|
+
raise ValueError(f"tensor {name} must be floating point")
|
|
875
|
+
if not np.isfinite(value).all():
|
|
876
|
+
raise ValueError(f"tensor {name} contains non-finite values")
|
|
877
|
+
flattened.append(value.astype(np.float64, copy=False).reshape(-1))
|
|
878
|
+
return np.concatenate(flattened)
|
|
879
|
+
|
|
880
|
+
|
|
881
|
+
def _validate_sketch(
|
|
882
|
+
sketch: np.ndarray, *, size: int = COUNT_SKETCH_SIZE
|
|
883
|
+
) -> np.ndarray:
|
|
884
|
+
value = np.asarray(sketch)
|
|
885
|
+
if value.shape != (size,) or value.dtype != np.float32:
|
|
886
|
+
raise ConsensusSketchError("sketch must be a one-dimensional FP32 vector")
|
|
887
|
+
if not np.isfinite(value).all():
|
|
888
|
+
raise ConsensusSketchError("sketch contains non-finite values")
|
|
889
|
+
return value.copy()
|
|
890
|
+
|
|
891
|
+
|
|
892
|
+
def _normalise_seed(seed: int) -> int:
|
|
893
|
+
digest = hashlib.sha256(str(seed).encode()).digest()
|
|
894
|
+
return int.from_bytes(digest[:8], "little")
|
|
895
|
+
|
|
896
|
+
|
|
897
|
+
__all__ = [
|
|
898
|
+
"COUNT_SKETCH_SIZE",
|
|
899
|
+
"ConsensusObservation",
|
|
900
|
+
"ConsensusSketchBuffer",
|
|
901
|
+
"ConsensusSketchError",
|
|
902
|
+
"ConsensusSketchMessage",
|
|
903
|
+
"ConsensusSketchPublisher",
|
|
904
|
+
"LiveConsensusTelemetry",
|
|
905
|
+
"MemberDistance",
|
|
906
|
+
"count_sketch",
|
|
907
|
+
"decode_sketch",
|
|
908
|
+
"encode_sketch",
|
|
909
|
+
]
|