dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,623 @@
1
+ """Update codecs and their serializable local state."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Callable, Mapping
7
+ from dataclasses import dataclass, field
8
+ from pathlib import Path
9
+ from types import MappingProxyType
10
+ from typing import Protocol, cast, runtime_checkable
11
+ from uuid import uuid4
12
+
13
+ import numpy as np
14
+ from safetensors.numpy import (
15
+ load_file as _load_file, # pyright: ignore[reportUnknownVariableType]
16
+ )
17
+ from safetensors.numpy import (
18
+ save_file as _save_file, # pyright: ignore[reportUnknownVariableType]
19
+ )
20
+
21
+ from dromeus.algorithms.base import MaterializedArtifact, UpdateBundle
22
+ from dromeus.manifests.canonical import canonical_hash, file_sha256
23
+ from dromeus.manifests.models import (
24
+ AlgorithmId,
25
+ OpaqueArtifactMetadata,
26
+ OpaqueUpdateBundleMetadata,
27
+ PublicKey,
28
+ RoundId,
29
+ RunId,
30
+ Sha256,
31
+ TensorSchema,
32
+ UpdateCodecBinding,
33
+ )
34
+ from dromeus.manifests.models import Tensor as TensorSpec
35
+
36
+ _LoadSafetensors = Callable[[str], dict[str, np.ndarray]]
37
+ _SaveSafetensors = Callable[[dict[str, np.ndarray], str], None]
38
+ load_safetensors = cast(_LoadSafetensors, _load_file)
39
+ save_safetensors = cast(_SaveSafetensors, _save_file)
40
+
41
+ TensorMap = dict[str, np.ndarray]
42
+ NamedTensorMap = dict[str, TensorMap]
43
+ StateMap = Mapping[str, object]
44
+
45
+
46
+ @runtime_checkable
47
+ class UpdateCodec(Protocol):
48
+ """Encode/decode an algorithm update without owning transport concerns."""
49
+
50
+ @property
51
+ def codec_id(self) -> str: ...
52
+
53
+ @property
54
+ def lossy(self) -> bool: ...
55
+
56
+ def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
57
+ """Resolve the wire schema, rejecting an incompatible logical schema."""
58
+ ...
59
+
60
+ def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap: ...
61
+
62
+ def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap: ...
63
+
64
+ def state_dict(self) -> dict[str, object]: ...
65
+
66
+ def load_state_dict(self, state: StateMap) -> None: ...
67
+
68
+
69
+ @dataclass(frozen=True, slots=True)
70
+ class CodecDescription:
71
+ """Validated capabilities bound to an algorithm's logical schema."""
72
+
73
+ codec_id: str
74
+ lossy: bool
75
+ encoded_schema: TensorSchema
76
+
77
+
78
+ def describe_update_codec(
79
+ codec: object, logical_schema: TensorSchema
80
+ ) -> CodecDescription:
81
+ """Require complete capabilities before an algorithm can use a codec."""
82
+ if not isinstance(codec, UpdateCodec):
83
+ raise TypeError("codec must implement the complete UpdateCodec interface")
84
+ codec_id = cast(object, codec.codec_id)
85
+ lossy = cast(object, codec.lossy)
86
+ encoded_schema = cast(object, codec.encoded_schema_for(logical_schema))
87
+ if not isinstance(codec_id, str) or not codec_id:
88
+ raise TypeError("codec ID must be a nonempty string")
89
+ if not isinstance(lossy, bool):
90
+ raise TypeError("codec lossy marker must be boolean")
91
+ if not isinstance(encoded_schema, TensorSchema):
92
+ raise TypeError("codec must resolve an encoded tensor schema")
93
+ return CodecDescription(codec_id, lossy, encoded_schema)
94
+
95
+
96
+ class NamedUpdateBundleCodec(Protocol):
97
+ """Materialize independently encoded named tensor artifacts."""
98
+
99
+ def encode(
100
+ self,
101
+ *,
102
+ round_id: RoundId,
103
+ artifacts: Mapping[str, Mapping[str, np.ndarray]],
104
+ codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
105
+ ) -> UpdateBundle: ...
106
+
107
+ def decode(
108
+ self,
109
+ bundle: UpdateBundle,
110
+ codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
111
+ ) -> NamedTensorMap: ...
112
+
113
+ def release(self, bundle: UpdateBundle) -> None: ...
114
+
115
+ def state_dict(self) -> dict[str, object]: ...
116
+
117
+ def load_state_dict(self, state: StateMap) -> None: ...
118
+
119
+
120
+ @dataclass(frozen=True, slots=True)
121
+ class IdentityCodec:
122
+ """Preserve named tensors unchanged and keep no codec state."""
123
+
124
+ _codec_id: str = "safetensors"
125
+
126
+ @property
127
+ def codec_id(self) -> str:
128
+ return self._codec_id
129
+
130
+ @property
131
+ def lossy(self) -> bool:
132
+ return False
133
+
134
+ def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
135
+ """Identity uses the algorithm's unchanged logical tensor schema."""
136
+ return logical_schema
137
+
138
+ def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
139
+ return _copy_tensors(tensors)
140
+
141
+ def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
142
+ return _copy_tensors(tensors)
143
+
144
+ def state_dict(self) -> dict[str, object]:
145
+ return {}
146
+
147
+ def load_state_dict(self, state: StateMap) -> None:
148
+ if state:
149
+ raise ValueError("identity codec has no state")
150
+
151
+
152
+ @dataclass(frozen=True, slots=True)
153
+ class DenseInt8Codec:
154
+ """Per-tensor symmetric signed-int8 quantization with explicit metadata."""
155
+
156
+ logical_schema: TensorSchema
157
+ _encoded_schema: TensorSchema = field(init=False, repr=False)
158
+
159
+ def __post_init__(self) -> None:
160
+ _validate_lossy_logical_schema(self.logical_schema)
161
+ object.__setattr__(
162
+ self,
163
+ "_encoded_schema",
164
+ _dense_encoded_schema(self.logical_schema),
165
+ )
166
+
167
+ @property
168
+ def codec_id(self) -> str:
169
+ return "dense-int8"
170
+
171
+ @property
172
+ def encoded_schema(self) -> TensorSchema:
173
+ return self._encoded_schema
174
+
175
+ @property
176
+ def lossy(self) -> bool:
177
+ return True
178
+
179
+ def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
180
+ _validate_codec_logical_schema(logical_schema, self.logical_schema)
181
+ return self.encoded_schema
182
+
183
+ def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
184
+ validate_tensor_map(tensors, self.logical_schema)
185
+ encoded: TensorMap = {}
186
+ for spec in self.logical_schema.tensors:
187
+ quantized, scale = _quantize_symmetric(tensors[spec.name])
188
+ encoded[f"{spec.name}.__q"] = quantized
189
+ encoded[f"{spec.name}.__scale"] = np.array(
190
+ [scale], dtype="<f4"
191
+ )
192
+ encoded[f"{spec.name}.__zero_point"] = np.array(
193
+ [0], dtype="<i4"
194
+ )
195
+ validate_tensor_map(encoded, self.encoded_schema)
196
+ return encoded
197
+
198
+ def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
199
+ validate_tensor_map(tensors, self.encoded_schema)
200
+ decoded: TensorMap = {}
201
+ for spec in self.logical_schema.tensors:
202
+ scale = _validated_quantization_metadata(tensors, spec.name)
203
+ quantized = np.asarray(tensors[f"{spec.name}.__q"], dtype=np.int8)
204
+ decoded[spec.name] = (
205
+ quantized.astype(np.float32) * np.float32(scale)
206
+ ).reshape(spec.shape)
207
+ validate_tensor_map(decoded, self.logical_schema)
208
+ return decoded
209
+
210
+ def state_dict(self) -> dict[str, object]:
211
+ return {}
212
+
213
+ def load_state_dict(self, state: StateMap) -> None:
214
+ if state:
215
+ raise ValueError("dense int8 codec has no state")
216
+
217
+
218
+ @dataclass(frozen=True, slots=True)
219
+ class BitmapTopKInt8Codec:
220
+ """Deterministic top-k int8 values with one-bit membership indices."""
221
+
222
+ logical_schema: TensorSchema
223
+ top_k_fraction: float
224
+ _encoded_schema: TensorSchema = field(init=False, repr=False)
225
+
226
+ def __post_init__(self) -> None:
227
+ _validate_lossy_logical_schema(self.logical_schema)
228
+ if not 0.0 < self.top_k_fraction <= 0.5 or not math.isfinite(
229
+ self.top_k_fraction
230
+ ):
231
+ raise ValueError("bitmap top-k fraction must be finite in (0, 0.5]")
232
+ object.__setattr__(
233
+ self,
234
+ "_encoded_schema",
235
+ _bitmap_topk_encoded_schema(
236
+ self.logical_schema,
237
+ self.top_k_fraction,
238
+ ),
239
+ )
240
+
241
+ @property
242
+ def codec_id(self) -> str:
243
+ return "topk-bitmap-int8"
244
+
245
+ @property
246
+ def encoded_schema(self) -> TensorSchema:
247
+ return self._encoded_schema
248
+
249
+ @property
250
+ def lossy(self) -> bool:
251
+ return True
252
+
253
+ def encoded_schema_for(self, logical_schema: TensorSchema) -> TensorSchema:
254
+ _validate_codec_logical_schema(logical_schema, self.logical_schema)
255
+ return self.encoded_schema
256
+
257
+ def encode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
258
+ validate_tensor_map(tensors, self.logical_schema)
259
+ encoded: TensorMap = {}
260
+ for spec in self.logical_schema.tensors:
261
+ flattened = np.asarray(tensors[spec.name], dtype=np.float32).reshape(-1)
262
+ indices = _deterministic_topk_indices(
263
+ flattened,
264
+ _top_k_count(flattened.size, self.top_k_fraction),
265
+ )
266
+ quantized, scale = _quantize_symmetric(flattened[indices])
267
+ membership = np.zeros(flattened.size, dtype=np.uint8)
268
+ membership[indices] = np.uint8(1)
269
+ bitmap = np.packbits(membership, bitorder="little").view(np.int8)
270
+ encoded[f"{spec.name}.__bitmap"] = np.ascontiguousarray(bitmap)
271
+ encoded[f"{spec.name}.__values"] = quantized
272
+ encoded[f"{spec.name}.__scale"] = np.array([scale], dtype="<f4")
273
+ encoded[f"{spec.name}.__zero_point"] = np.array([0], dtype="<i4")
274
+ validate_tensor_map(encoded, self.encoded_schema)
275
+ return encoded
276
+
277
+ def decode(self, tensors: Mapping[str, np.ndarray]) -> TensorMap:
278
+ validate_tensor_map(tensors, self.encoded_schema)
279
+ decoded: TensorMap = {}
280
+ for spec in self.logical_schema.tensors:
281
+ element_count = math.prod(spec.shape)
282
+ bitmap = np.asarray(
283
+ tensors[f"{spec.name}.__bitmap"],
284
+ dtype=np.int8,
285
+ ).view(np.uint8)
286
+ bits = np.unpackbits(bitmap, bitorder="little")
287
+ if np.any(bits[element_count:]):
288
+ raise ValueError("bitmap top-k padding bits must be zero")
289
+ indices = np.flatnonzero(bits[:element_count])
290
+ expected_count = _top_k_count(element_count, self.top_k_fraction)
291
+ if indices.size != expected_count:
292
+ raise ValueError("bitmap top-k selected count does not match schema")
293
+ scale = _validated_quantization_metadata(tensors, spec.name)
294
+ quantized = np.asarray(
295
+ tensors[f"{spec.name}.__values"],
296
+ dtype=np.int8,
297
+ )
298
+ value = np.zeros(element_count, dtype=np.float32)
299
+ value[indices] = quantized.astype(np.float32) * np.float32(scale)
300
+ decoded[spec.name] = value.reshape(spec.shape)
301
+ validate_tensor_map(decoded, self.logical_schema)
302
+ return decoded
303
+
304
+ def state_dict(self) -> dict[str, object]:
305
+ return {}
306
+
307
+ def load_state_dict(self, state: StateMap) -> None:
308
+ if state:
309
+ raise ValueError("bitmap top-k int8 codec has no state")
310
+
311
+
312
+ def _validate_codec_logical_schema(
313
+ actual: TensorSchema, expected: TensorSchema
314
+ ) -> None:
315
+ if actual != expected:
316
+ raise ValueError("codec logical schema does not match algorithm")
317
+
318
+
319
+ def _validate_lossy_logical_schema(schema: TensorSchema) -> None:
320
+ if any(tensor.dtype != "float32" for tensor in schema.tensors):
321
+ raise ValueError("lossy codecs require FP32 logical tensors")
322
+
323
+
324
+ def _dense_encoded_schema(schema: TensorSchema) -> TensorSchema:
325
+ tensors: list[TensorSpec] = []
326
+ for spec in schema.tensors:
327
+ tensors.extend(
328
+ (
329
+ TensorSpec(name=f"{spec.name}.__q", dtype="int8", shape=spec.shape),
330
+ TensorSpec(
331
+ name=f"{spec.name}.__scale", dtype="float32", shape=(1,)
332
+ ),
333
+ TensorSpec(
334
+ name=f"{spec.name}.__zero_point", dtype="int32", shape=(1,)
335
+ ),
336
+ )
337
+ )
338
+ return TensorSchema(tensors=tuple(sorted(tensors, key=lambda item: item.name)))
339
+
340
+
341
+ def _bitmap_topk_encoded_schema(
342
+ schema: TensorSchema,
343
+ top_k_fraction: float,
344
+ ) -> TensorSchema:
345
+ tensors: list[TensorSpec] = []
346
+ for spec in schema.tensors:
347
+ element_count = math.prod(spec.shape)
348
+ tensors.extend(
349
+ (
350
+ TensorSpec(
351
+ name=f"{spec.name}.__bitmap",
352
+ dtype="int8",
353
+ shape=(math.ceil(element_count / 8),),
354
+ ),
355
+ TensorSpec(
356
+ name=f"{spec.name}.__values",
357
+ dtype="int8",
358
+ shape=(_top_k_count(element_count, top_k_fraction),),
359
+ ),
360
+ TensorSpec(
361
+ name=f"{spec.name}.__scale",
362
+ dtype="float32",
363
+ shape=(1,),
364
+ ),
365
+ TensorSpec(
366
+ name=f"{spec.name}.__zero_point",
367
+ dtype="int32",
368
+ shape=(1,),
369
+ ),
370
+ )
371
+ )
372
+ return TensorSchema(tensors=tuple(sorted(tensors, key=lambda item: item.name)))
373
+
374
+
375
+ def _top_k_count(element_count: int, fraction: float) -> int:
376
+ return min(element_count, max(1, math.ceil(element_count * fraction)))
377
+
378
+
379
+ def _deterministic_topk_indices(values: np.ndarray, k: int) -> np.ndarray:
380
+ magnitudes = np.abs(values)
381
+ if k == values.size:
382
+ return np.arange(values.size, dtype=np.int64)
383
+ threshold = np.partition(magnitudes, values.size - k)[values.size - k]
384
+ greater = np.flatnonzero(magnitudes > threshold)
385
+ equal = np.flatnonzero(magnitudes == threshold)
386
+ needed = k - greater.size
387
+ selected = np.concatenate((greater, equal[:needed]))
388
+ return np.sort(selected).astype(np.int64, copy=False)
389
+
390
+
391
+ def _quantize_symmetric(value: np.ndarray) -> tuple[np.ndarray, float]:
392
+ source = np.asarray(value, dtype=np.float32)
393
+ maximum = float(np.max(np.abs(source))) if source.size else 0.0
394
+ scale = maximum / 127.0 if maximum > 0.0 else 1.0
395
+ if np.float32(scale) == np.float32(0.0):
396
+ scale = float(np.nextafter(np.float32(0.0), np.float32(1.0)))
397
+ quantized = np.clip(np.rint(source / np.float32(scale)), -127, 127).astype(
398
+ np.int8
399
+ )
400
+ return np.ascontiguousarray(quantized), scale
401
+
402
+
403
+ def _validated_quantization_metadata(
404
+ tensors: Mapping[str, np.ndarray], name: str
405
+ ) -> float:
406
+ scale = float(np.asarray(tensors[f"{name}.__scale"])[0])
407
+ if not math.isfinite(scale) or scale <= 0.0:
408
+ raise ValueError("quantization scale must be positive and finite")
409
+ zero_point = int(np.asarray(tensors[f"{name}.__zero_point"])[0])
410
+ if zero_point != 0:
411
+ raise ValueError("symmetric quantization zero point must equal zero")
412
+ return scale
413
+
414
+
415
+ def _copy_tensors(tensors: Mapping[str, np.ndarray]) -> TensorMap:
416
+ return {
417
+ name: np.ascontiguousarray(value).copy() for name, value in tensors.items()
418
+ }
419
+
420
+
421
+ @dataclass(frozen=True, slots=True)
422
+ class NamedSafetensorsUpdateBundleCodec:
423
+ """Materialize each named update artifact as one safetensors file."""
424
+
425
+ artifact_root: Path
426
+ run_id: RunId
427
+ manifest_hash: Sha256
428
+ sender_public_key: PublicKey
429
+ algorithm_id: AlgorithmId
430
+ artifact_schemas: Mapping[str, TensorSchema]
431
+
432
+ def __post_init__(self) -> None:
433
+ schemas = dict(self.artifact_schemas)
434
+ if not 1 <= len(schemas) <= 16:
435
+ raise ValueError("named bundle codec requires 1 through 16 artifacts")
436
+ object.__setattr__(self, "artifact_schemas", MappingProxyType(schemas))
437
+
438
+ def encode(
439
+ self,
440
+ *,
441
+ round_id: RoundId,
442
+ artifacts: Mapping[str, Mapping[str, np.ndarray]],
443
+ codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
444
+ ) -> UpdateBundle:
445
+ names = tuple(sorted(self.artifact_schemas))
446
+ if set(artifacts) != set(names):
447
+ raise ValueError("named update artifacts do not match codec")
448
+ bindings = self._bindings(codec_bindings)
449
+ self.artifact_root.mkdir(parents=True, exist_ok=True)
450
+ paths: list[Path] = []
451
+ metadata: list[OpaqueArtifactMetadata] = []
452
+ materialized: list[MaterializedArtifact] = []
453
+ try:
454
+ for name in names:
455
+ values = _copy_tensors(artifacts[name])
456
+ schema = self.artifact_schemas[name]
457
+ validate_tensor_map(values, schema)
458
+ path = (
459
+ self.artifact_root
460
+ / f"round-{round_id}-{name}-{uuid4().hex}.safetensors"
461
+ )
462
+ paths.append(path)
463
+ save_safetensors(values, str(path))
464
+ binding = bindings[name]
465
+ metadata.append(
466
+ OpaqueArtifactMetadata(
467
+ name=name,
468
+ size_bytes=path.stat().st_size,
469
+ sha256=file_sha256(path),
470
+ codec_id=binding.codec_id,
471
+ logical_schema_hash=canonical_hash(
472
+ binding.logical_schema
473
+ ),
474
+ encoded_schema_hash=canonical_hash(schema),
475
+ )
476
+ )
477
+ materialized.append(
478
+ MaterializedArtifact(
479
+ path=path,
480
+ transfer_codec_id="safetensors",
481
+ transfer_schema=schema,
482
+ )
483
+ )
484
+ return UpdateBundle(
485
+ metadata=OpaqueUpdateBundleMetadata(
486
+ run_id=self.run_id,
487
+ manifest_hash=self.manifest_hash,
488
+ sender_public_key=self.sender_public_key,
489
+ algorithm_id=self.algorithm_id,
490
+ round_id=round_id,
491
+ artifacts=tuple(metadata),
492
+ ),
493
+ artifacts=tuple(materialized),
494
+ )
495
+ except BaseException:
496
+ for path in paths:
497
+ path.unlink(missing_ok=True)
498
+ raise
499
+
500
+ def decode(
501
+ self,
502
+ bundle: UpdateBundle,
503
+ codec_bindings: Mapping[str, UpdateCodecBinding] | None = None,
504
+ ) -> NamedTensorMap:
505
+ metadata = bundle.metadata
506
+ if metadata.run_id != self.run_id:
507
+ raise ValueError("bundle run does not match codec")
508
+ if metadata.manifest_hash != self.manifest_hash:
509
+ raise ValueError("bundle manifest does not match codec")
510
+ if metadata.algorithm_id != self.algorithm_id:
511
+ raise ValueError("bundle algorithm does not match codec")
512
+ names = set(self.artifact_schemas)
513
+ metadata_by_name = {
514
+ artifact.name: artifact for artifact in metadata.artifacts
515
+ }
516
+ materialized_by_name = {
517
+ item.name: artifact
518
+ for item, artifact in zip(
519
+ metadata.artifacts, bundle.artifacts, strict=True
520
+ )
521
+ }
522
+ if (
523
+ set(metadata_by_name) != names
524
+ or len(metadata_by_name) != len(metadata.artifacts)
525
+ or set(materialized_by_name) != names
526
+ ):
527
+ raise ValueError("named update artifacts do not match codec")
528
+ bindings = self._bindings(codec_bindings)
529
+ decoded: NamedTensorMap = {}
530
+ for name in sorted(names):
531
+ item = metadata_by_name[name]
532
+ artifact = materialized_by_name[name]
533
+ schema = self.artifact_schemas[name]
534
+ binding = bindings[name]
535
+ if (
536
+ item.codec_id != binding.codec_id
537
+ or item.logical_schema_hash
538
+ != canonical_hash(binding.logical_schema)
539
+ or item.encoded_schema_hash != canonical_hash(schema)
540
+ ):
541
+ raise ValueError("named update artifact binding does not match codec")
542
+ if artifact.path.stat().st_size != item.size_bytes:
543
+ raise ValueError("bundle artifact size mismatch")
544
+ if file_sha256(artifact.path) != item.sha256:
545
+ raise ValueError("bundle artifact checksum mismatch")
546
+ values = {
547
+ tensor_name: np.ascontiguousarray(value)
548
+ for tensor_name, value in load_safetensors(
549
+ str(artifact.path)
550
+ ).items()
551
+ }
552
+ validate_tensor_map(values, schema)
553
+ decoded[name] = values
554
+ return decoded
555
+
556
+ def release(self, bundle: UpdateBundle) -> None:
557
+ for artifact in bundle.artifacts:
558
+ artifact.path.unlink(missing_ok=True)
559
+
560
+ def state_dict(self) -> dict[str, object]:
561
+ return {}
562
+
563
+ def load_state_dict(self, state: StateMap) -> None:
564
+ if state:
565
+ raise ValueError("named safetensors bundle codec has no state")
566
+
567
+ def _bindings(
568
+ self,
569
+ bindings: Mapping[str, UpdateCodecBinding] | None,
570
+ ) -> dict[str, UpdateCodecBinding]:
571
+ if bindings is None:
572
+ return {
573
+ name: _resolve_codec_binding(None, schema)
574
+ for name, schema in self.artifact_schemas.items()
575
+ }
576
+ if set(bindings) != set(self.artifact_schemas):
577
+ raise ValueError("named update codec bindings do not match artifacts")
578
+ return dict(bindings)
579
+
580
+
581
+ def _resolve_codec_binding(
582
+ binding: UpdateCodecBinding | None,
583
+ encoded_schema: TensorSchema,
584
+ ) -> UpdateCodecBinding:
585
+ if binding is not None:
586
+ return binding
587
+ return UpdateCodecBinding(
588
+ codec_id="safetensors",
589
+ logical_schema=encoded_schema,
590
+ )
591
+
592
+
593
+ def validate_tensor_map(
594
+ tensors: Mapping[str, np.ndarray], schema: TensorSchema
595
+ ) -> None:
596
+ expected = {tensor.name: tensor for tensor in schema.tensors}
597
+ if set(tensors) != set(expected):
598
+ raise ValueError("tensor names do not match schema")
599
+ for name, value in tensors.items():
600
+ tensor = np.ascontiguousarray(value)
601
+ spec = expected[name]
602
+ if str(tensor.dtype) != spec.dtype:
603
+ raise ValueError(f"tensor {name} dtype does not match schema")
604
+ if tensor.shape != spec.shape:
605
+ raise ValueError(f"tensor {name} shape does not match schema")
606
+ if not np.isfinite(tensor).all():
607
+ raise ValueError(f"tensor {name} contains non-finite values")
608
+
609
+ __all__ = [
610
+ "BitmapTopKInt8Codec",
611
+ "CodecDescription",
612
+ "DenseInt8Codec",
613
+ "IdentityCodec",
614
+ "NamedSafetensorsUpdateBundleCodec",
615
+ "NamedTensorMap",
616
+ "NamedUpdateBundleCodec",
617
+ "StateMap",
618
+ "TensorMap",
619
+ "UpdateCodecBinding",
620
+ "UpdateCodec",
621
+ "describe_update_codec",
622
+ "validate_tensor_map",
623
+ ]