dromeus 0.4.0.dev1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (69) hide show
  1. dromeus/__init__.py +109 -0
  2. dromeus/__main__.py +6 -0
  3. dromeus/algorithms/__init__.py +31 -0
  4. dromeus/algorithms/base.py +188 -0
  5. dromeus/algorithms/codec.py +623 -0
  6. dromeus/algorithms/noloco.py +486 -0
  7. dromeus/application.py +523 -0
  8. dromeus/cli.py +341 -0
  9. dromeus/diagnostics.py +303 -0
  10. dromeus/errors.py +57 -0
  11. dromeus/gossip/__init__.py +29 -0
  12. dromeus/gossip/_async.py +15 -0
  13. dromeus/gossip/axl.py +497 -0
  14. dromeus/gossip/engine.py +378 -0
  15. dromeus/gossip/interfaces.py +158 -0
  16. dromeus/gossip/peer_scheduler.py +117 -0
  17. dromeus/manifests/__init__.py +1 -0
  18. dromeus/manifests/canonical.py +157 -0
  19. dromeus/manifests/models.py +694 -0
  20. dromeus/membership/__init__.py +1 -0
  21. dromeus/membership/formation.py +576 -0
  22. dromeus/node.py +338 -0
  23. dromeus/persistence/__init__.py +20 -0
  24. dromeus/persistence/archive.py +576 -0
  25. dromeus/persistence/files.py +39 -0
  26. dromeus/persistence/privacy_ledger.py +104 -0
  27. dromeus/persistence/run_store.py +352 -0
  28. dromeus/protocol/__init__.py +25 -0
  29. dromeus/protocol/codec.py +127 -0
  30. dromeus/protocol/models.py +199 -0
  31. dromeus/protocol/version.py +5 -0
  32. dromeus/py.typed +1 -0
  33. dromeus/runtime.py +1081 -0
  34. dromeus/telemetry/__init__.py +61 -0
  35. dromeus/telemetry/consensus.py +909 -0
  36. dromeus/telemetry/divergence.py +321 -0
  37. dromeus/telemetry/divergence_monitor.py +97 -0
  38. dromeus/telemetry/events.py +214 -0
  39. dromeus/telemetry/evidence.py +429 -0
  40. dromeus/telemetry/metrics.py +200 -0
  41. dromeus/telemetry/progress.py +255 -0
  42. dromeus/training/__init__.py +1 -0
  43. dromeus/training/base.py +60 -0
  44. dromeus/training/model_state.py +111 -0
  45. dromeus/training/native_gradients.py +289 -0
  46. dromeus/training/optimizer_factory.py +426 -0
  47. dromeus/training/ordinary_task.py +408 -0
  48. dromeus/training/privacy.py +96 -0
  49. dromeus/training/private_data.py +96 -0
  50. dromeus/training/private_model.py +414 -0
  51. dromeus/training/private_state.py +97 -0
  52. dromeus/training/private_trainer.py +534 -0
  53. dromeus/training/private_workload.py +217 -0
  54. dromeus/training/state.py +57 -0
  55. dromeus/training/task.py +464 -0
  56. dromeus/training/trainer.py +294 -0
  57. dromeus/training/versions.py +15 -0
  58. dromeus/transport/__init__.py +1 -0
  59. dromeus/transport/axl.py +168 -0
  60. dromeus/transport/envelope_sender.py +73 -0
  61. dromeus/transport/interface.py +31 -0
  62. dromeus/transport/outbound_scheduler.py +326 -0
  63. dromeus/transport/receiver.py +345 -0
  64. dromeus/transport/transfer.py +854 -0
  65. dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
  66. dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
  67. dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
  68. dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
  69. dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,486 @@
1
+ """NoLoCo pairwise outer optimization."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Mapping
7
+ from copy import deepcopy
8
+ from dataclasses import dataclass, field
9
+ from pathlib import Path
10
+ from types import MappingProxyType
11
+ from typing import Any, cast
12
+
13
+ import numpy as np
14
+
15
+ from dromeus.algorithms.base import (
16
+ AlgorithmObservations,
17
+ AlgorithmSnapshot,
18
+ AlgorithmUpdate,
19
+ NamedValidatedUpdate,
20
+ UpdateBundle,
21
+ checksum_artifacts,
22
+ )
23
+ from dromeus.algorithms.codec import (
24
+ CodecDescription,
25
+ IdentityCodec,
26
+ NamedSafetensorsUpdateBundleCodec,
27
+ NamedUpdateBundleCodec,
28
+ UpdateCodec,
29
+ describe_update_codec,
30
+ validate_tensor_map,
31
+ )
32
+ from dromeus.manifests.models import (
33
+ NoLoCoConfig,
34
+ RoundId,
35
+ TensorSchema,
36
+ UpdateCodecBinding,
37
+ )
38
+ from dromeus.training.base import CheckpointTrainer, EvaluationResult, WeightTrainer
39
+
40
+ _ARTIFACT_NAMES = ("outer_gradient", "slow_weights")
41
+ _STATE_PREFIX = "noloco."
42
+ _PHASES = {
43
+ "created": 0,
44
+ "pre-local": 1,
45
+ "post-local": 2,
46
+ "bundled": 3,
47
+ "post-outer": 4,
48
+ }
49
+
50
+
51
+ def _identity_codecs() -> dict[str, UpdateCodec]:
52
+ return {name: IdentityCodec("identity") for name in _ARTIFACT_NAMES}
53
+
54
+
55
+ @dataclass
56
+ class NoLoCoAlgorithm:
57
+ """Own NoLoCo slow weights, outer momentum, and opaque bundle lifecycle.
58
+
59
+ Paper ``phi`` is ``_slow_weights``, post-local ``theta`` is the trainer's fast
60
+ weights, and Dromeus names the descent-oriented outer gradient ``g = phi -
61
+ theta``. ``alpha``, ``beta``, and ``gamma`` map directly to ``config``. The
62
+ identity path subtracts the averaged gradients and slow-weight correction. This
63
+ is algebraically equivalent to the pinned executable upstream convention, which
64
+ adds ``beta * mean(theta - phi)`` and effectively uses ``gamma = beta``. It does
65
+ not follow the divergent sign printed in paper Equation 2.
66
+ """
67
+
68
+ trainer: WeightTrainer
69
+ tensor_schema: TensorSchema
70
+ config: NoLoCoConfig
71
+ bundle_codec: NamedUpdateBundleCodec | None = None
72
+ artifact_codecs: Mapping[str, UpdateCodec] = field(default_factory=_identity_codecs)
73
+ manifest_codec_ids: Mapping[str, str] | None = None
74
+ _round_id: RoundId = 0
75
+ _phase: str = "created"
76
+ _slow_weights: dict[str, np.ndarray] = field(init=False, repr=False)
77
+ _outer_momentum: dict[str, np.ndarray] = field(init=False, repr=False)
78
+ _error_feedback_residual: dict[str, np.ndarray] = field(init=False, repr=False)
79
+ _completed_outer_steps: int = field(default=0, init=False, repr=False)
80
+ _local_artifacts: dict[str, dict[str, np.ndarray]] | None = field(
81
+ default=None, init=False, repr=False
82
+ )
83
+ _codec_descriptions: Mapping[str, CodecDescription] = field(init=False, repr=False)
84
+
85
+ def __post_init__(self) -> None:
86
+ if set(self.artifact_codecs) != set(_ARTIFACT_NAMES):
87
+ raise ValueError("NoLoCo codecs must cover both named artifacts")
88
+ self.artifact_codecs = MappingProxyType(dict(self.artifact_codecs))
89
+ self._codec_descriptions = MappingProxyType(
90
+ {
91
+ name: describe_update_codec(codec, self.tensor_schema)
92
+ for name, codec in self.artifact_codecs.items()
93
+ }
94
+ )
95
+ if self.manifest_codec_ids is not None:
96
+ actual = {
97
+ name: description.codec_id
98
+ for name, description in self._codec_descriptions.items()
99
+ }
100
+ if dict(self.manifest_codec_ids) != actual:
101
+ raise ValueError("NoLoCo codecs do not match manifest")
102
+ weights = self.trainer.weights()
103
+ self._validate_tensors(weights)
104
+ self._slow_weights = self._copy_tensors(weights)
105
+ self._outer_momentum = {
106
+ name: np.zeros_like(value, dtype=np.float32)
107
+ for name, value in weights.items()
108
+ }
109
+ self._error_feedback_residual = {
110
+ name: np.zeros_like(value, dtype=np.float32)
111
+ for name, value in weights.items()
112
+ }
113
+
114
+ def pre_local(self, round_id: RoundId) -> None:
115
+ if self._phase == "created":
116
+ weights = self.trainer.weights()
117
+ self._validate_tensors(weights)
118
+ self._slow_weights = self._copy_tensors(weights)
119
+ self._outer_momentum = {
120
+ name: np.zeros_like(value, dtype=np.float32)
121
+ for name, value in weights.items()
122
+ }
123
+ self._error_feedback_residual = {
124
+ name: np.zeros_like(value, dtype=np.float32)
125
+ for name, value in weights.items()
126
+ }
127
+ self._round_id = round_id
128
+ self.trainer.load_weights(self._copy_tensors(self._slow_weights))
129
+ self._local_artifacts = None
130
+ self._phase = "pre-local"
131
+
132
+ def local_training(self) -> None:
133
+ self.trainer.train_local_steps(self.config.inner_steps)
134
+ self._phase = "post-local"
135
+
136
+ def post_local_bundle(self) -> UpdateBundle:
137
+ if self.bundle_codec is None:
138
+ raise RuntimeError("update bundle codec is not configured")
139
+ fast_weights = self.trainer.weights()
140
+ self._validate_tensors(fast_weights)
141
+ logical = {
142
+ "outer_gradient": {
143
+ name: (
144
+ self._slow_weights[name].astype(np.float32)
145
+ - fast_weights[name].astype(np.float32)
146
+ + (
147
+ self._error_feedback_residual[name]
148
+ if self._codec_descriptions["outer_gradient"].lossy
149
+ else np.float32(0.0)
150
+ )
151
+ ).astype(np.float32)
152
+ for name in self._slow_weights
153
+ },
154
+ "slow_weights": self._copy_tensors(self._slow_weights),
155
+ }
156
+ encoded = {
157
+ name: self.artifact_codecs[name].encode(logical[name])
158
+ for name in _ARTIFACT_NAMES
159
+ }
160
+ bundle = self.bundle_codec.encode(
161
+ round_id=self._round_id,
162
+ artifacts=encoded,
163
+ codec_bindings=self._codec_bindings(),
164
+ )
165
+ try:
166
+ local = {
167
+ name: self.artifact_codecs[name].decode(encoded[name])
168
+ for name in _ARTIFACT_NAMES
169
+ }
170
+ self._validate_artifacts(local)
171
+ except BaseException:
172
+ self.bundle_codec.release(bundle)
173
+ raise
174
+ self._local_artifacts = {
175
+ name: self._copy_tensors(tensors) for name, tensors in local.items()
176
+ }
177
+ self._error_feedback_residual = {
178
+ name: (
179
+ logical["outer_gradient"][name] - local["outer_gradient"][name]
180
+ if self._codec_descriptions["outer_gradient"].lossy
181
+ else np.zeros_like(value, dtype=np.float32)
182
+ ).astype(np.float32)
183
+ for name, value in self._slow_weights.items()
184
+ }
185
+ self._phase = "bundled"
186
+ return bundle
187
+
188
+ def validate_peer(self, peer_bundle: UpdateBundle) -> NamedValidatedUpdate:
189
+ if self.bundle_codec is None:
190
+ raise RuntimeError("update bundle codec is not configured")
191
+ if peer_bundle.metadata.round_id != self._round_id:
192
+ raise ValueError("peer bundle round does not match current round")
193
+ encoded = self.bundle_codec.decode(
194
+ peer_bundle, codec_bindings=self._codec_bindings()
195
+ )
196
+ artifacts = {
197
+ name: self.artifact_codecs[name].decode(encoded[name])
198
+ for name in _ARTIFACT_NAMES
199
+ }
200
+ self._validate_artifacts(artifacts)
201
+ return NamedValidatedUpdate(
202
+ round_id=self._round_id,
203
+ artifacts=artifacts,
204
+ checksum=checksum_artifacts(artifacts),
205
+ )
206
+
207
+ def peer_apply(self, peer_update: AlgorithmUpdate) -> AlgorithmSnapshot:
208
+ if not isinstance(peer_update, NamedValidatedUpdate):
209
+ raise TypeError("NoLoCo requires named validated artifacts")
210
+ if peer_update.round_id != self._round_id:
211
+ raise ValueError("peer update round does not match current round")
212
+ self._validate_artifacts(peer_update.artifacts)
213
+ if checksum_artifacts(peer_update.artifacts) != peer_update.checksum:
214
+ raise ValueError("peer update checksum mismatch")
215
+ if self._local_artifacts is None:
216
+ raise RuntimeError("local update bundle has not been built")
217
+ local_gradient = self._local_artifacts["outer_gradient"]
218
+ local_slow = self._local_artifacts["slow_weights"]
219
+ peer_gradient = peer_update.artifacts["outer_gradient"]
220
+ peer_slow = peer_update.artifacts["slow_weights"]
221
+ next_momentum: dict[str, np.ndarray] = {}
222
+ next_slow: dict[str, np.ndarray] = {}
223
+ for name in self._slow_weights:
224
+ momentum = (
225
+ np.float32(self.config.alpha) * self._outer_momentum[name]
226
+ - np.float32(self.config.beta / 2.0)
227
+ * (local_gradient[name] + peer_gradient[name])
228
+ - np.float32(self.config.gamma / 2.0)
229
+ * (local_slow[name] - peer_slow[name])
230
+ ).astype(np.float32)
231
+ next_momentum[name] = momentum
232
+ next_slow[name] = (
233
+ self._slow_weights[name].astype(np.float32) + momentum
234
+ ).astype(np.float32)
235
+ self._outer_momentum = next_momentum
236
+ self._slow_weights = next_slow
237
+ self._completed_outer_steps += 1
238
+ self.trainer.load_weights(self._copy_tensors(next_slow))
239
+ self._phase = "post-outer"
240
+ return self.snapshot()
241
+
242
+ def release_bundle(self, bundle: UpdateBundle) -> None:
243
+ if self.bundle_codec is None:
244
+ raise RuntimeError("update bundle codec is not configured")
245
+ self.bundle_codec.release(bundle)
246
+
247
+ def configure_bundle_codec(
248
+ self,
249
+ *,
250
+ artifact_root: Path,
251
+ run_id: str,
252
+ manifest_hash: str,
253
+ sender_public_key: str,
254
+ algorithm_id: str,
255
+ ) -> None:
256
+ """Bind formed run context without exposing codec details to runtime."""
257
+ if self.bundle_codec is None:
258
+ self.bundle_codec = NamedSafetensorsUpdateBundleCodec(
259
+ artifact_root=artifact_root,
260
+ run_id=run_id,
261
+ manifest_hash=manifest_hash,
262
+ sender_public_key=sender_public_key,
263
+ algorithm_id=algorithm_id,
264
+ artifact_schemas={
265
+ name: description.encoded_schema
266
+ for name, description in self._codec_descriptions.items()
267
+ },
268
+ )
269
+
270
+ def snapshot(self) -> AlgorithmSnapshot:
271
+ return AlgorithmSnapshot(
272
+ round_id=self._round_id,
273
+ phase=self._phase,
274
+ weights=self._slow_weights,
275
+ )
276
+
277
+ def evaluate(self) -> EvaluationResult | None:
278
+ return self.trainer.evaluate()
279
+
280
+ def observations(self) -> AlgorithmObservations:
281
+ observations = AlgorithmObservations(local_loss=self.trainer.local_loss)
282
+ if self._local_artifacts is None:
283
+ return observations
284
+ residual_norm = _tensor_l2_norm(self._error_feedback_residual)
285
+ signal = {
286
+ name: (
287
+ self._local_artifacts["outer_gradient"][name]
288
+ + self._error_feedback_residual[name]
289
+ ).astype(np.float32)
290
+ for name in self._error_feedback_residual
291
+ }
292
+ signal_norm = _tensor_l2_norm(signal)
293
+ ratio = residual_norm / signal_norm if signal_norm > 0.0 else 0.0
294
+ return AlgorithmObservations(
295
+ local_loss=observations.local_loss,
296
+ error_feedback_residual_l2_norm=residual_norm,
297
+ error_feedback_signal_l2_norm=signal_norm,
298
+ error_feedback_residual_to_signal_ratio=ratio,
299
+ )
300
+
301
+ def checkpoint_tensors(self) -> dict[str, np.ndarray]:
302
+ """Return complete durable state in the versioned flat namespace."""
303
+ if not isinstance(self.trainer, CheckpointTrainer):
304
+ raise TypeError("NoLoCo trainer must expose checkpoint state")
305
+ state: dict[str, np.ndarray] = {
306
+ f"{_STATE_PREFIX}schema_version": np.array([1], dtype=np.int64),
307
+ f"{_STATE_PREFIX}round_id": np.array([self._round_id], dtype=np.int64),
308
+ f"{_STATE_PREFIX}completed_outer_steps": np.array(
309
+ [self._completed_outer_steps], dtype=np.int64
310
+ ),
311
+ f"{_STATE_PREFIX}phase": np.array([_PHASES[self._phase]], dtype=np.int64),
312
+ }
313
+ for name, value in self._slow_weights.items():
314
+ state[f"{_STATE_PREFIX}slow_weights.{name}"] = value.copy()
315
+ for name, value in self._outer_momentum.items():
316
+ state[f"{_STATE_PREFIX}outer_momentum.{name}"] = value.copy()
317
+ for name, value in self._error_feedback_residual.items():
318
+ state[f"{_STATE_PREFIX}error_feedback_residual.{name}"] = value.copy()
319
+ for name, value in self.trainer.checkpoint_tensors().items():
320
+ state[f"{_STATE_PREFIX}trainer.{name}"] = value.copy(order="C")
321
+ for artifact_name, codec in self.artifact_codecs.items():
322
+ for name, value in codec.state_dict().items():
323
+ if not isinstance(value, np.ndarray):
324
+ raise TypeError("NoLoCo codec checkpoint state must be tensors")
325
+ state[f"{_STATE_PREFIX}codec.{artifact_name}.{name}"] = (
326
+ np.ascontiguousarray(cast(Any, value)).copy()
327
+ )
328
+ return state
329
+
330
+ def load_checkpoint_tensors(self, state: Mapping[str, np.ndarray]) -> None:
331
+ """Restore a validated checkpoint produced by `checkpoint_tensors`."""
332
+ if not isinstance(self.trainer, CheckpointTrainer):
333
+ raise TypeError("NoLoCo trainer must expose checkpoint state")
334
+ version = self._counter(state, "schema_version")
335
+ if version != 1:
336
+ raise ValueError("unsupported NoLoCo checkpoint version")
337
+ round_id = self._counter(state, "round_id")
338
+ completed_outer_steps = self._counter(state, "completed_outer_steps")
339
+ phase_value = self._counter(state, "phase")
340
+ phase_by_value = {value: name for name, value in _PHASES.items()}
341
+ if phase_value not in phase_by_value:
342
+ raise ValueError("NoLoCo checkpoint phase is invalid")
343
+ names = {tensor.name for tensor in self.tensor_schema.tensors}
344
+ slow_weights = self._checkpoint_group(state, "slow_weights", names)
345
+ outer_momentum = self._checkpoint_group(state, "outer_momentum", names)
346
+ residual = self._checkpoint_group(state, "error_feedback_residual", names)
347
+ self._validate_tensors(slow_weights)
348
+ self._validate_tensors(outer_momentum)
349
+ self._validate_tensors(residual)
350
+ trainer_prefix = f"{_STATE_PREFIX}trainer."
351
+ trainer_state = {
352
+ name.removeprefix(trainer_prefix): value.copy(order="C")
353
+ for name, value in state.items()
354
+ if name.startswith(trainer_prefix)
355
+ }
356
+ if not trainer_state:
357
+ raise ValueError("NoLoCo trainer checkpoint state is missing")
358
+ previous_trainer = {
359
+ name: value.copy(order="C")
360
+ for name, value in self.trainer.checkpoint_tensors().items()
361
+ }
362
+ previous_codecs = {
363
+ name: deepcopy(codec.state_dict())
364
+ for name, codec in self.artifact_codecs.items()
365
+ }
366
+ try:
367
+ self.trainer.load_checkpoint_tensors(trainer_state)
368
+ restored_weights = self.trainer.weights()
369
+ self._validate_tensors(restored_weights)
370
+ if any(
371
+ not np.array_equal(restored_weights[name], slow_weights[name])
372
+ for name in names
373
+ ):
374
+ raise ValueError("NoLoCo trainer and slow weights do not match")
375
+ for artifact_name, codec in self.artifact_codecs.items():
376
+ codec_prefix = f"{_STATE_PREFIX}codec.{artifact_name}."
377
+ codec.load_state_dict(
378
+ {
379
+ name.removeprefix(codec_prefix): value.copy()
380
+ for name, value in state.items()
381
+ if name.startswith(codec_prefix)
382
+ }
383
+ )
384
+ except Exception as error:
385
+ rollback_errors: list[Exception] = []
386
+ try:
387
+ self.trainer.load_checkpoint_tensors(previous_trainer)
388
+ except Exception as rollback_error:
389
+ rollback_errors.append(rollback_error)
390
+ for name, codec in self.artifact_codecs.items():
391
+ try:
392
+ codec.load_state_dict(previous_codecs[name])
393
+ except Exception as rollback_error:
394
+ rollback_errors.append(rollback_error)
395
+ if rollback_errors:
396
+ raise RuntimeError(
397
+ "NoLoCo checkpoint rollback failed"
398
+ ) from ExceptionGroup(
399
+ "restore and rollback errors", [error, *rollback_errors]
400
+ )
401
+ raise
402
+ self._round_id = round_id
403
+ self._completed_outer_steps = completed_outer_steps
404
+ self._phase = phase_by_value[phase_value]
405
+ self._slow_weights = slow_weights
406
+ self._outer_momentum = outer_momentum
407
+ self._error_feedback_residual = residual
408
+ self._local_artifacts = None
409
+
410
+ def state_dict(self) -> dict[str, object]:
411
+ return dict(self.checkpoint_tensors())
412
+
413
+ def load_state_dict(self, state: Mapping[str, object]) -> None:
414
+ if not all(isinstance(value, np.ndarray) for value in state.values()):
415
+ raise ValueError("NoLoCo algorithm state must contain only tensors")
416
+ tensor_state: dict[str, np.ndarray] = {
417
+ name: cast(np.ndarray, value) for name, value in state.items()
418
+ }
419
+ self.load_checkpoint_tensors(tensor_state)
420
+
421
+ def _validate_artifacts(
422
+ self, artifacts: Mapping[str, Mapping[str, np.ndarray]]
423
+ ) -> None:
424
+ if set(artifacts) != set(_ARTIFACT_NAMES):
425
+ raise ValueError("NoLoCo update requires both named artifacts")
426
+ for tensors in artifacts.values():
427
+ self._validate_tensors(tensors)
428
+
429
+ def _validate_tensors(self, tensors: Mapping[str, np.ndarray]) -> None:
430
+ validate_tensor_map(tensors, self.tensor_schema)
431
+ for value in tensors.values():
432
+ if value.dtype != np.float32:
433
+ raise ValueError("NoLoCo requires FP32 logical tensors")
434
+
435
+ def _codec_bindings(self) -> dict[str, UpdateCodecBinding]:
436
+ return {
437
+ name: UpdateCodecBinding(
438
+ codec_id=description.codec_id,
439
+ logical_schema=self.tensor_schema,
440
+ )
441
+ for name, description in self._codec_descriptions.items()
442
+ }
443
+
444
+ @staticmethod
445
+ def _counter(state: Mapping[str, np.ndarray], name: str) -> int:
446
+ value = np.asarray(state.get(f"{_STATE_PREFIX}{name}"))
447
+ if value.dtype != np.int64 or value.shape != (1,) or int(value[0]) < 0:
448
+ raise ValueError(f"NoLoCo checkpoint {name} is invalid")
449
+ return int(value[0])
450
+
451
+ @staticmethod
452
+ def _checkpoint_group(
453
+ state: Mapping[str, np.ndarray],
454
+ group: str,
455
+ names: set[str],
456
+ ) -> dict[str, np.ndarray]:
457
+ prefix = f"{_STATE_PREFIX}{group}."
458
+ values = {
459
+ name.removeprefix(prefix): np.ascontiguousarray(value).copy()
460
+ for name, value in state.items()
461
+ if name.startswith(prefix)
462
+ }
463
+ if set(values) != names:
464
+ raise ValueError(f"NoLoCo checkpoint {group} is incomplete")
465
+ return values
466
+
467
+ @staticmethod
468
+ def _copy_tensors(
469
+ tensors: Mapping[str, np.ndarray],
470
+ ) -> dict[str, np.ndarray]:
471
+ return {
472
+ name: np.ascontiguousarray(value).copy() for name, value in tensors.items()
473
+ }
474
+
475
+
476
+ def _tensor_l2_norm(tensors: Mapping[str, np.ndarray]) -> float:
477
+ total = 0.0
478
+ for value in tensors.values():
479
+ flattened = np.asarray(value, dtype=np.float32).reshape(-1)
480
+ for start in range(0, flattened.size, 1_048_576):
481
+ chunk = flattened[start : start + 1_048_576].astype(np.float64, copy=False)
482
+ total += float(np.sum(np.square(chunk), dtype=np.float64))
483
+ return math.sqrt(total)
484
+
485
+
486
+ __all__ = ["NoLoCoAlgorithm"]