dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,623 @@
|
|
|
1
|
+
"""Update codecs and their serializable local state."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Callable, Mapping
|
|
7
|
+
from dataclasses import dataclass, field
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
from types import MappingProxyType
|
|
10
|
+
from typing import Protocol, cast, runtime_checkable
|
|
11
|
+
from uuid import uuid4
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
from safetensors.numpy import (
|
|
15
|
+
load_file as _load_file, # pyright: ignore[reportUnknownVariableType]
|
|
16
|
+
)
|
|
17
|
+
from safetensors.numpy import (
|
|
18
|
+
save_file as _save_file, # pyright: ignore[reportUnknownVariableType]
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
from dromeus.algorithms.base import MaterializedArtifact, UpdateBundle
|
|
22
|
+
from dromeus.manifests.canonical import canonical_hash, file_sha256
|
|
23
|
+
from dromeus.manifests.models import (
|
|
24
|
+
AlgorithmId,
|
|
25
|
+
OpaqueArtifactMetadata,
|
|
26
|
+
OpaqueUpdateBundleMetadata,
|
|
27
|
+
PublicKey,
|
|
28
|
+
RoundId,
|
|
29
|
+
RunId,
|
|
30
|
+
Sha256,
|
|
31
|
+
TensorSchema,
|
|
32
|
+
UpdateCodecBinding,
|
|
33
|
+
)
|
|
34
|
+
from dromeus.manifests.models import Tensor as TensorSpec
|
|
35
|
+
|
|
36
|
+
_LoadSafetensors = Callable[[str], dict[str, np.ndarray]]
|
|
37
|
+
_SaveSafetensors = Callable[[dict[str, np.ndarray], str], None]
|
|
38
|
+
load_safetensors = cast(_LoadSafetensors, _load_file)
|
|
39
|
+
save_safetensors = cast(_SaveSafetensors, _save_file)
|
|
40
|
+
|
|
41
|
+
TensorMap = dict[str, np.ndarray]
|
|
42
|
+
NamedTensorMap = dict[str, TensorMap]
|
|
43
|
+
StateMap = Mapping[str, object]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
@runtime_checkable
|
|
47
|
+
class UpdateCodec(Protocol):
|
|
48
|
+
"""Encode/decode an algorithm update without owning transport concerns."""
|
|
49
|
+
|
|
50
|
+
@property
|
|
51
|
+
def codec_id(self) -> str: ...
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def lossy(self) -> bool: ...
|
|
55
|
+
|
|
56
|
+
def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
|
|
57
|
+
"""Resolve the wire schema, rejecting an incompatible logical schema."""
|
|
58
|
+
...
|
|
59
|
+
|
|
60
|
+
def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap: ...
|
|
61
|
+
|
|
62
|
+
def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap: ...
|
|
63
|
+
|
|
64
|
+
def state_dict(self) -> dict[str, object]: ...
|
|
65
|
+
|
|
66
|
+
def load_state_dict(self, state: StateMap) -> None: ...
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
@dataclass(frozen=True, slots=True)
|
|
70
|
+
class CodecDescription:
|
|
71
|
+
"""Validated capabilities bound to an algorithm's logical schema."""
|
|
72
|
+
|
|
73
|
+
codec_id: str
|
|
74
|
+
lossy: bool
|
|
75
|
+
encoded_schema: TensorSchema
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def describe_update_codec(
|
|
79
|
+
codec: object, logical_schema: TensorSchema
|
|
80
|
+
) -> CodecDescription:
|
|
81
|
+
"""Require complete capabilities before an algorithm can use a codec."""
|
|
82
|
+
if not isinstance(codec, UpdateCodec):
|
|
83
|
+
raise TypeError("codec must implement the complete UpdateCodec interface")
|
|
84
|
+
codec_id = cast(object, codec.codec_id)
|
|
85
|
+
lossy = cast(object, codec.lossy)
|
|
86
|
+
encoded_schema = cast(object, codec.encoded_schema_for(logical_schema))
|
|
87
|
+
if not isinstance(codec_id, str) or not codec_id:
|
|
88
|
+
raise TypeError("codec ID must be a nonempty string")
|
|
89
|
+
if not isinstance(lossy, bool):
|
|
90
|
+
raise TypeError("codec lossy marker must be boolean")
|
|
91
|
+
if not isinstance(encoded_schema, TensorSchema):
|
|
92
|
+
raise TypeError("codec must resolve an encoded tensor schema")
|
|
93
|
+
return CodecDescription(codec_id, lossy, encoded_schema)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class NamedUpdateBundleCodec(Protocol):
|
|
97
|
+
"""Materialize independently encoded named tensor artifacts."""
|
|
98
|
+
|
|
99
|
+
def encode(
|
|
100
|
+
self,
|
|
101
|
+
*,
|
|
102
|
+
round_id: RoundId,
|
|
103
|
+
artifacts: Mapping[str, Mapping[str, np.ndarray]],
|
|
104
|
+
codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
|
|
105
|
+
) -> UpdateBundle: ...
|
|
106
|
+
|
|
107
|
+
def decode(
|
|
108
|
+
self,
|
|
109
|
+
bundle: UpdateBundle,
|
|
110
|
+
codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
|
|
111
|
+
) -> NamedTensorMap: ...
|
|
112
|
+
|
|
113
|
+
def release(self, bundle: UpdateBundle) -> None: ...
|
|
114
|
+
|
|
115
|
+
def state_dict(self) -> dict[str, object]: ...
|
|
116
|
+
|
|
117
|
+
def load_state_dict(self, state: StateMap) -> None: ...
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
@dataclass(frozen=True, slots=True)
|
|
121
|
+
class IdentityCodec:
|
|
122
|
+
"""Preserve named tensors unchanged and keep no codec state."""
|
|
123
|
+
|
|
124
|
+
_codec_id: str = "safetensors"
|
|
125
|
+
|
|
126
|
+
@property
|
|
127
|
+
def codec_id(self) -> str:
|
|
128
|
+
return self._codec_id
|
|
129
|
+
|
|
130
|
+
@property
|
|
131
|
+
def lossy(self) -> bool:
|
|
132
|
+
return False
|
|
133
|
+
|
|
134
|
+
def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
|
|
135
|
+
"""Identity uses the algorithm's unchanged logical tensor schema."""
|
|
136
|
+
return logical_schema
|
|
137
|
+
|
|
138
|
+
def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
139
|
+
return _copy_tensors(tensors)
|
|
140
|
+
|
|
141
|
+
def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
142
|
+
return _copy_tensors(tensors)
|
|
143
|
+
|
|
144
|
+
def state_dict(self) -> dict[str, object]:
|
|
145
|
+
return {}
|
|
146
|
+
|
|
147
|
+
def load_state_dict(self, state: StateMap) -> None:
|
|
148
|
+
if state:
|
|
149
|
+
raise ValueError("identity codec has no state")
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
@dataclass(frozen=True, slots=True)
|
|
153
|
+
class DenseInt8Codec:
|
|
154
|
+
"""Per-tensor symmetric signed-int8 quantization with explicit metadata."""
|
|
155
|
+
|
|
156
|
+
logical_schema: TensorSchema
|
|
157
|
+
_encoded_schema: TensorSchema = field(init=False, repr=False)
|
|
158
|
+
|
|
159
|
+
def __post_init__(self) -> None:
|
|
160
|
+
_validate_lossy_logical_schema(self.logical_schema)
|
|
161
|
+
object.__setattr__(
|
|
162
|
+
self,
|
|
163
|
+
"_encoded_schema",
|
|
164
|
+
_dense_encoded_schema(self.logical_schema),
|
|
165
|
+
)
|
|
166
|
+
|
|
167
|
+
@property
|
|
168
|
+
def codec_id(self) -> str:
|
|
169
|
+
return "dense-int8"
|
|
170
|
+
|
|
171
|
+
@property
|
|
172
|
+
def encoded_schema(self) -> TensorSchema:
|
|
173
|
+
return self._encoded_schema
|
|
174
|
+
|
|
175
|
+
@property
|
|
176
|
+
def lossy(self) -> bool:
|
|
177
|
+
return True
|
|
178
|
+
|
|
179
|
+
def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
|
|
180
|
+
_validate_codec_logical_schema(logical_schema, self.logical_schema)
|
|
181
|
+
return self.encoded_schema
|
|
182
|
+
|
|
183
|
+
def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
184
|
+
validate_tensor_map(tensors, self.logical_schema)
|
|
185
|
+
encoded: TensorMap = {}
|
|
186
|
+
for spec in self.logical_schema.tensors:
|
|
187
|
+
quantized, scale = _quantize_symmetric(tensors[spec.name])
|
|
188
|
+
encoded[f"{spec.name}.__q"] = quantized
|
|
189
|
+
encoded[f"{spec.name}.__scale"] = np.array(
|
|
190
|
+
[scale], dtype="<f4"
|
|
191
|
+
)
|
|
192
|
+
encoded[f"{spec.name}.__zero_point"] = np.array(
|
|
193
|
+
[0], dtype="<i4"
|
|
194
|
+
)
|
|
195
|
+
validate_tensor_map(encoded, self.encoded_schema)
|
|
196
|
+
return encoded
|
|
197
|
+
|
|
198
|
+
def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
199
|
+
validate_tensor_map(tensors, self.encoded_schema)
|
|
200
|
+
decoded: TensorMap = {}
|
|
201
|
+
for spec in self.logical_schema.tensors:
|
|
202
|
+
scale = _validated_quantization_metadata(tensors, spec.name)
|
|
203
|
+
quantized = np.asarray(tensors[f"{spec.name}.__q"], dtype=np.int8)
|
|
204
|
+
decoded[spec.name] = (
|
|
205
|
+
quantized.astype(np.float32) * np.float32(scale)
|
|
206
|
+
).reshape(spec.shape)
|
|
207
|
+
validate_tensor_map(decoded, self.logical_schema)
|
|
208
|
+
return decoded
|
|
209
|
+
|
|
210
|
+
def state_dict(self) -> dict[str, object]:
|
|
211
|
+
return {}
|
|
212
|
+
|
|
213
|
+
def load_state_dict(self, state: StateMap) -> None:
|
|
214
|
+
if state:
|
|
215
|
+
raise ValueError("dense int8 codec has no state")
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
@dataclass(frozen=True, slots=True)
|
|
219
|
+
class BitmapTopKInt8Codec:
|
|
220
|
+
"""Deterministic top-k int8 values with one-bit membership indices."""
|
|
221
|
+
|
|
222
|
+
logical_schema: TensorSchema
|
|
223
|
+
top_k_fraction: float
|
|
224
|
+
_encoded_schema: TensorSchema = field(init=False, repr=False)
|
|
225
|
+
|
|
226
|
+
def __post_init__(self) -> None:
|
|
227
|
+
_validate_lossy_logical_schema(self.logical_schema)
|
|
228
|
+
if not 0.0 < self.top_k_fraction <= 0.5 or not math.isfinite(
|
|
229
|
+
self.top_k_fraction
|
|
230
|
+
):
|
|
231
|
+
raise ValueError("bitmap top-k fraction must be finite in (0, 0.5]")
|
|
232
|
+
object.__setattr__(
|
|
233
|
+
self,
|
|
234
|
+
"_encoded_schema",
|
|
235
|
+
_bitmap_topk_encoded_schema(
|
|
236
|
+
self.logical_schema,
|
|
237
|
+
self.top_k_fraction,
|
|
238
|
+
),
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
@property
|
|
242
|
+
def codec_id(self) -> str:
|
|
243
|
+
return "topk-bitmap-int8"
|
|
244
|
+
|
|
245
|
+
@property
|
|
246
|
+
def encoded_schema(self) -> TensorSchema:
|
|
247
|
+
return self._encoded_schema
|
|
248
|
+
|
|
249
|
+
@property
|
|
250
|
+
def lossy(self) -> bool:
|
|
251
|
+
return True
|
|
252
|
+
|
|
253
|
+
def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
|
|
254
|
+
_validate_codec_logical_schema(logical_schema, self.logical_schema)
|
|
255
|
+
return self.encoded_schema
|
|
256
|
+
|
|
257
|
+
def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
258
|
+
validate_tensor_map(tensors, self.logical_schema)
|
|
259
|
+
encoded: TensorMap = {}
|
|
260
|
+
for spec in self.logical_schema.tensors:
|
|
261
|
+
flattened = np.asarray(tensors[spec.name], dtype=np.float32).reshape(-1)
|
|
262
|
+
indices = _deterministic_topk_indices(
|
|
263
|
+
flattened,
|
|
264
|
+
_top_k_count(flattened.size, self.top_k_fraction),
|
|
265
|
+
)
|
|
266
|
+
quantized, scale = _quantize_symmetric(flattened[indices])
|
|
267
|
+
membership = np.zeros(flattened.size, dtype=np.uint8)
|
|
268
|
+
membership[indices] = np.uint8(1)
|
|
269
|
+
bitmap = np.packbits(membership, bitorder="little").view(np.int8)
|
|
270
|
+
encoded[f"{spec.name}.__bitmap"] = np.ascontiguousarray(bitmap)
|
|
271
|
+
encoded[f"{spec.name}.__values"] = quantized
|
|
272
|
+
encoded[f"{spec.name}.__scale"] = np.array([scale], dtype="<f4")
|
|
273
|
+
encoded[f"{spec.name}.__zero_point"] = np.array([0], dtype="<i4")
|
|
274
|
+
validate_tensor_map(encoded, self.encoded_schema)
|
|
275
|
+
return encoded
|
|
276
|
+
|
|
277
|
+
def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
278
|
+
validate_tensor_map(tensors, self.encoded_schema)
|
|
279
|
+
decoded: TensorMap = {}
|
|
280
|
+
for spec in self.logical_schema.tensors:
|
|
281
|
+
element_count = math.prod(spec.shape)
|
|
282
|
+
bitmap = np.asarray(
|
|
283
|
+
tensors[f"{spec.name}.__bitmap"],
|
|
284
|
+
dtype=np.int8,
|
|
285
|
+
).view(np.uint8)
|
|
286
|
+
bits = np.unpackbits(bitmap, bitorder="little")
|
|
287
|
+
if np.any(bits[element_count:]):
|
|
288
|
+
raise ValueError("bitmap top-k padding bits must be zero")
|
|
289
|
+
indices = np.flatnonzero(bits[:element_count])
|
|
290
|
+
expected_count = _top_k_count(element_count, self.top_k_fraction)
|
|
291
|
+
if indices.size != expected_count:
|
|
292
|
+
raise ValueError("bitmap top-k selected count does not match schema")
|
|
293
|
+
scale = _validated_quantization_metadata(tensors, spec.name)
|
|
294
|
+
quantized = np.asarray(
|
|
295
|
+
tensors[f"{spec.name}.__values"],
|
|
296
|
+
dtype=np.int8,
|
|
297
|
+
)
|
|
298
|
+
value = np.zeros(element_count, dtype=np.float32)
|
|
299
|
+
value[indices] = quantized.astype(np.float32) * np.float32(scale)
|
|
300
|
+
decoded[spec.name] = value.reshape(spec.shape)
|
|
301
|
+
validate_tensor_map(decoded, self.logical_schema)
|
|
302
|
+
return decoded
|
|
303
|
+
|
|
304
|
+
def state_dict(self) -> dict[str, object]:
|
|
305
|
+
return {}
|
|
306
|
+
|
|
307
|
+
def load_state_dict(self, state: StateMap) -> None:
|
|
308
|
+
if state:
|
|
309
|
+
raise ValueError("bitmap top-k int8 codec has no state")
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
def _validate_codec_logical_schema(
|
|
313
|
+
actual: TensorSchema, expected: TensorSchema
|
|
314
|
+
) -> None:
|
|
315
|
+
if actual != expected:
|
|
316
|
+
raise ValueError("codec logical schema does not match algorithm")
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _validate_lossy_logical_schema(schema: TensorSchema) -> None:
|
|
320
|
+
if any(tensor.dtype != "float32" for tensor in schema.tensors):
|
|
321
|
+
raise ValueError("lossy codecs require FP32 logical tensors")
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _dense_encoded_schema(schema: TensorSchema) -> TensorSchema:
|
|
325
|
+
tensors: list[TensorSpec] = []
|
|
326
|
+
for spec in schema.tensors:
|
|
327
|
+
tensors.extend(
|
|
328
|
+
(
|
|
329
|
+
TensorSpec(name=f"{spec.name}.__q", dtype="int8", shape=spec.shape),
|
|
330
|
+
TensorSpec(
|
|
331
|
+
name=f"{spec.name}.__scale", dtype="float32", shape=(1,)
|
|
332
|
+
),
|
|
333
|
+
TensorSpec(
|
|
334
|
+
name=f"{spec.name}.__zero_point", dtype="int32", shape=(1,)
|
|
335
|
+
),
|
|
336
|
+
)
|
|
337
|
+
)
|
|
338
|
+
return TensorSchema(tensors=tuple(sorted(tensors, key=lambda item: item.name)))
|
|
339
|
+
|
|
340
|
+
|
|
341
|
+
def _bitmap_topk_encoded_schema(
|
|
342
|
+
schema: TensorSchema,
|
|
343
|
+
top_k_fraction: float,
|
|
344
|
+
) -> TensorSchema:
|
|
345
|
+
tensors: list[TensorSpec] = []
|
|
346
|
+
for spec in schema.tensors:
|
|
347
|
+
element_count = math.prod(spec.shape)
|
|
348
|
+
tensors.extend(
|
|
349
|
+
(
|
|
350
|
+
TensorSpec(
|
|
351
|
+
name=f"{spec.name}.__bitmap",
|
|
352
|
+
dtype="int8",
|
|
353
|
+
shape=(math.ceil(element_count / 8),),
|
|
354
|
+
),
|
|
355
|
+
TensorSpec(
|
|
356
|
+
name=f"{spec.name}.__values",
|
|
357
|
+
dtype="int8",
|
|
358
|
+
shape=(_top_k_count(element_count, top_k_fraction),),
|
|
359
|
+
),
|
|
360
|
+
TensorSpec(
|
|
361
|
+
name=f"{spec.name}.__scale",
|
|
362
|
+
dtype="float32",
|
|
363
|
+
shape=(1,),
|
|
364
|
+
),
|
|
365
|
+
TensorSpec(
|
|
366
|
+
name=f"{spec.name}.__zero_point",
|
|
367
|
+
dtype="int32",
|
|
368
|
+
shape=(1,),
|
|
369
|
+
),
|
|
370
|
+
)
|
|
371
|
+
)
|
|
372
|
+
return TensorSchema(tensors=tuple(sorted(tensors, key=lambda item: item.name)))
|
|
373
|
+
|
|
374
|
+
|
|
375
|
+
def _top_k_count(element_count: int, fraction: float) -> int:
|
|
376
|
+
return min(element_count, max(1, math.ceil(element_count * fraction)))
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _deterministic_topk_indices(values: np.ndarray, k: int) -> np.ndarray:
|
|
380
|
+
magnitudes = np.abs(values)
|
|
381
|
+
if k == values.size:
|
|
382
|
+
return np.arange(values.size, dtype=np.int64)
|
|
383
|
+
threshold = np.partition(magnitudes, values.size - k)[values.size - k]
|
|
384
|
+
greater = np.flatnonzero(magnitudes > threshold)
|
|
385
|
+
equal = np.flatnonzero(magnitudes == threshold)
|
|
386
|
+
needed = k - greater.size
|
|
387
|
+
selected = np.concatenate((greater, equal[:needed]))
|
|
388
|
+
return np.sort(selected).astype(np.int64, copy=False)
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _quantize_symmetric(value: np.ndarray) -> tuple[np.ndarray, float]:
|
|
392
|
+
source = np.asarray(value, dtype=np.float32)
|
|
393
|
+
maximum = float(np.max(np.abs(source))) if source.size else 0.0
|
|
394
|
+
scale = maximum / 127.0 if maximum > 0.0 else 1.0
|
|
395
|
+
if np.float32(scale) == np.float32(0.0):
|
|
396
|
+
scale = float(np.nextafter(np.float32(0.0), np.float32(1.0)))
|
|
397
|
+
quantized = np.clip(np.rint(source / np.float32(scale)), -127, 127).astype(
|
|
398
|
+
np.int8
|
|
399
|
+
)
|
|
400
|
+
return np.ascontiguousarray(quantized), scale
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def _validated_quantization_metadata(
|
|
404
|
+
tensors: Mapping[str, np.ndarray], name: str
|
|
405
|
+
) -> float:
|
|
406
|
+
scale = float(np.asarray(tensors[f"{name}.__scale"])[0])
|
|
407
|
+
if not math.isfinite(scale) or scale <= 0.0:
|
|
408
|
+
raise ValueError("quantization scale must be positive and finite")
|
|
409
|
+
zero_point = int(np.asarray(tensors[f"{name}.__zero_point"])[0])
|
|
410
|
+
if zero_point != 0:
|
|
411
|
+
raise ValueError("symmetric quantization zero point must equal zero")
|
|
412
|
+
return scale
|
|
413
|
+
|
|
414
|
+
|
|
415
|
+
def _copy_tensors(tensors: Mapping[str, np.ndarray]) -> TensorMap:
|
|
416
|
+
return {
|
|
417
|
+
name: np.ascontiguousarray(value).copy() for name, value in tensors.items()
|
|
418
|
+
}
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
@dataclass(frozen=True, slots=True)
|
|
422
|
+
class NamedSafetensorsUpdateBundleCodec:
|
|
423
|
+
"""Materialize each named update artifact as one safetensors file."""
|
|
424
|
+
|
|
425
|
+
artifact_root: Path
|
|
426
|
+
run_id: RunId
|
|
427
|
+
manifest_hash: Sha256
|
|
428
|
+
sender_public_key: PublicKey
|
|
429
|
+
algorithm_id: AlgorithmId
|
|
430
|
+
artifact_schemas: Mapping[str, TensorSchema]
|
|
431
|
+
|
|
432
|
+
def __post_init__(self) -> None:
|
|
433
|
+
schemas = dict(self.artifact_schemas)
|
|
434
|
+
if not 1 <= len(schemas) <= 16:
|
|
435
|
+
raise ValueError("named bundle codec requires 1 through 16 artifacts")
|
|
436
|
+
object.__setattr__(self, "artifact_schemas", MappingProxyType(schemas))
|
|
437
|
+
|
|
438
|
+
def encode(
|
|
439
|
+
self,
|
|
440
|
+
*,
|
|
441
|
+
round_id: RoundId,
|
|
442
|
+
artifacts: Mapping[str, Mapping[str, np.ndarray]],
|
|
443
|
+
codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
|
|
444
|
+
) -> UpdateBundle:
|
|
445
|
+
names = tuple(sorted(self.artifact_schemas))
|
|
446
|
+
if set(artifacts) != set(names):
|
|
447
|
+
raise ValueError("named update artifacts do not match codec")
|
|
448
|
+
bindings = self._bindings(codec_bindings)
|
|
449
|
+
self.artifact_root.mkdir(parents=True, exist_ok=True)
|
|
450
|
+
paths: list[Path] = []
|
|
451
|
+
metadata: list[OpaqueArtifactMetadata] = []
|
|
452
|
+
materialized: list[MaterializedArtifact] = []
|
|
453
|
+
try:
|
|
454
|
+
for name in names:
|
|
455
|
+
values = _copy_tensors(artifacts[name])
|
|
456
|
+
schema = self.artifact_schemas[name]
|
|
457
|
+
validate_tensor_map(values, schema)
|
|
458
|
+
path = (
|
|
459
|
+
self.artifact_root
|
|
460
|
+
/ f"round-{round_id}-{name}-{uuid4().hex}.safetensors"
|
|
461
|
+
)
|
|
462
|
+
paths.append(path)
|
|
463
|
+
save_safetensors(values, str(path))
|
|
464
|
+
binding = bindings[name]
|
|
465
|
+
metadata.append(
|
|
466
|
+
OpaqueArtifactMetadata(
|
|
467
|
+
name=name,
|
|
468
|
+
size_bytes=path.stat().st_size,
|
|
469
|
+
sha256=file_sha256(path),
|
|
470
|
+
codec_id=binding.codec_id,
|
|
471
|
+
logical_schema_hash=canonical_hash(
|
|
472
|
+
binding.logical_schema
|
|
473
|
+
),
|
|
474
|
+
encoded_schema_hash=canonical_hash(schema),
|
|
475
|
+
)
|
|
476
|
+
)
|
|
477
|
+
materialized.append(
|
|
478
|
+
MaterializedArtifact(
|
|
479
|
+
path=path,
|
|
480
|
+
transfer_codec_id="safetensors",
|
|
481
|
+
transfer_schema=schema,
|
|
482
|
+
)
|
|
483
|
+
)
|
|
484
|
+
return UpdateBundle(
|
|
485
|
+
metadata=OpaqueUpdateBundleMetadata(
|
|
486
|
+
run_id=self.run_id,
|
|
487
|
+
manifest_hash=self.manifest_hash,
|
|
488
|
+
sender_public_key=self.sender_public_key,
|
|
489
|
+
algorithm_id=self.algorithm_id,
|
|
490
|
+
round_id=round_id,
|
|
491
|
+
artifacts=tuple(metadata),
|
|
492
|
+
),
|
|
493
|
+
artifacts=tuple(materialized),
|
|
494
|
+
)
|
|
495
|
+
except BaseException:
|
|
496
|
+
for path in paths:
|
|
497
|
+
path.unlink(missing_ok=True)
|
|
498
|
+
raise
|
|
499
|
+
|
|
500
|
+
def decode(
|
|
501
|
+
self,
|
|
502
|
+
bundle: UpdateBundle,
|
|
503
|
+
codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
|
|
504
|
+
) -> NamedTensorMap:
|
|
505
|
+
metadata = bundle.metadata
|
|
506
|
+
if metadata.run_id != self.run_id:
|
|
507
|
+
raise ValueError("bundle run does not match codec")
|
|
508
|
+
if metadata.manifest_hash != self.manifest_hash:
|
|
509
|
+
raise ValueError("bundle manifest does not match codec")
|
|
510
|
+
if metadata.algorithm_id != self.algorithm_id:
|
|
511
|
+
raise ValueError("bundle algorithm does not match codec")
|
|
512
|
+
names = set(self.artifact_schemas)
|
|
513
|
+
metadata_by_name = {
|
|
514
|
+
artifact.name: artifact for artifact in metadata.artifacts
|
|
515
|
+
}
|
|
516
|
+
materialized_by_name = {
|
|
517
|
+
item.name: artifact
|
|
518
|
+
for item, artifact in zip(
|
|
519
|
+
metadata.artifacts, bundle.artifacts, strict=True
|
|
520
|
+
)
|
|
521
|
+
}
|
|
522
|
+
if (
|
|
523
|
+
set(metadata_by_name) != names
|
|
524
|
+
or len(metadata_by_name) != len(metadata.artifacts)
|
|
525
|
+
or set(materialized_by_name) != names
|
|
526
|
+
):
|
|
527
|
+
raise ValueError("named update artifacts do not match codec")
|
|
528
|
+
bindings = self._bindings(codec_bindings)
|
|
529
|
+
decoded: NamedTensorMap = {}
|
|
530
|
+
for name in sorted(names):
|
|
531
|
+
item = metadata_by_name[name]
|
|
532
|
+
artifact = materialized_by_name[name]
|
|
533
|
+
schema = self.artifact_schemas[name]
|
|
534
|
+
binding = bindings[name]
|
|
535
|
+
if (
|
|
536
|
+
item.codec_id != binding.codec_id
|
|
537
|
+
or item.logical_schema_hash
|
|
538
|
+
!= canonical_hash(binding.logical_schema)
|
|
539
|
+
or item.encoded_schema_hash != canonical_hash(schema)
|
|
540
|
+
):
|
|
541
|
+
raise ValueError("named update artifact binding does not match codec")
|
|
542
|
+
if artifact.path.stat().st_size != item.size_bytes:
|
|
543
|
+
raise ValueError("bundle artifact size mismatch")
|
|
544
|
+
if file_sha256(artifact.path) != item.sha256:
|
|
545
|
+
raise ValueError("bundle artifact checksum mismatch")
|
|
546
|
+
values = {
|
|
547
|
+
tensor_name: np.ascontiguousarray(value)
|
|
548
|
+
for tensor_name, value in load_safetensors(
|
|
549
|
+
str(artifact.path)
|
|
550
|
+
).items()
|
|
551
|
+
}
|
|
552
|
+
validate_tensor_map(values, schema)
|
|
553
|
+
decoded[name] = values
|
|
554
|
+
return decoded
|
|
555
|
+
|
|
556
|
+
def release(self, bundle: UpdateBundle) -> None:
|
|
557
|
+
for artifact in bundle.artifacts:
|
|
558
|
+
artifact.path.unlink(missing_ok=True)
|
|
559
|
+
|
|
560
|
+
def state_dict(self) -> dict[str, object]:
|
|
561
|
+
return {}
|
|
562
|
+
|
|
563
|
+
def load_state_dict(self, state: StateMap) -> None:
|
|
564
|
+
if state:
|
|
565
|
+
raise ValueError("named safetensors bundle codec has no state")
|
|
566
|
+
|
|
567
|
+
def _bindings(
|
|
568
|
+
self,
|
|
569
|
+
bindings: Mapping[str, UpdateCodecBinding] | None,
|
|
570
|
+
) -> dict[str, UpdateCodecBinding]:
|
|
571
|
+
if bindings is None:
|
|
572
|
+
return {
|
|
573
|
+
name: _resolve_codec_binding(None, schema)
|
|
574
|
+
for name, schema in self.artifact_schemas.items()
|
|
575
|
+
}
|
|
576
|
+
if set(bindings) != set(self.artifact_schemas):
|
|
577
|
+
raise ValueError("named update codec bindings do not match artifacts")
|
|
578
|
+
return dict(bindings)
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _resolve_codec_binding(
|
|
582
|
+
binding: UpdateCodecBinding | None,
|
|
583
|
+
encoded_schema: TensorSchema,
|
|
584
|
+
) -> UpdateCodecBinding:
|
|
585
|
+
if binding is not None:
|
|
586
|
+
return binding
|
|
587
|
+
return UpdateCodecBinding(
|
|
588
|
+
codec_id="safetensors",
|
|
589
|
+
logical_schema=encoded_schema,
|
|
590
|
+
)
|
|
591
|
+
|
|
592
|
+
|
|
593
|
+
def validate_tensor_map(
|
|
594
|
+
tensors: Mapping[str, np.ndarray], schema: TensorSchema
|
|
595
|
+
) -> None:
|
|
596
|
+
expected = {tensor.name: tensor for tensor in schema.tensors}
|
|
597
|
+
if set(tensors) != set(expected):
|
|
598
|
+
raise ValueError("tensor names do not match schema")
|
|
599
|
+
for name, value in tensors.items():
|
|
600
|
+
tensor = np.ascontiguousarray(value)
|
|
601
|
+
spec = expected[name]
|
|
602
|
+
if str(tensor.dtype) != spec.dtype:
|
|
603
|
+
raise ValueError(f"tensor {name} dtype does not match schema")
|
|
604
|
+
if tensor.shape != spec.shape:
|
|
605
|
+
raise ValueError(f"tensor {name} shape does not match schema")
|
|
606
|
+
if not np.isfinite(tensor).all():
|
|
607
|
+
raise ValueError(f"tensor {name} contains non-finite values")
|
|
608
|
+
|
|
609
|
+
__all__ = [
|
|
610
|
+
"BitmapTopKInt8Codec",
|
|
611
|
+
"CodecDescription",
|
|
612
|
+
"DenseInt8Codec",
|
|
613
|
+
"IdentityCodec",
|
|
614
|
+
"NamedSafetensorsUpdateBundleCodec",
|
|
615
|
+
"NamedTensorMap",
|
|
616
|
+
"NamedUpdateBundleCodec",
|
|
617
|
+
"StateMap",
|
|
618
|
+
"TensorMap",
|
|
619
|
+
"UpdateCodecBinding",
|
|
620
|
+
"UpdateCodec",
|
|
621
|
+
"describe_update_codec",
|
|
622
|
+
"validate_tensor_map",
|
|
623
|
+
]
|