dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,486 @@
|
|
|
1
|
+
"""NoLoCo pairwise outer optimization."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Mapping
|
|
7
|
+
from copy import deepcopy
|
|
8
|
+
from dataclasses import dataclass, field
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from types import MappingProxyType
|
|
11
|
+
from typing import Any, cast
|
|
12
|
+
|
|
13
|
+
import numpy as np
|
|
14
|
+
|
|
15
|
+
from dromeus.algorithms.base import (
|
|
16
|
+
AlgorithmObservations,
|
|
17
|
+
AlgorithmSnapshot,
|
|
18
|
+
AlgorithmUpdate,
|
|
19
|
+
NamedValidatedUpdate,
|
|
20
|
+
UpdateBundle,
|
|
21
|
+
checksum_artifacts,
|
|
22
|
+
)
|
|
23
|
+
from dromeus.algorithms.codec import (
|
|
24
|
+
CodecDescription,
|
|
25
|
+
IdentityCodec,
|
|
26
|
+
NamedSafetensorsUpdateBundleCodec,
|
|
27
|
+
NamedUpdateBundleCodec,
|
|
28
|
+
UpdateCodec,
|
|
29
|
+
describe_update_codec,
|
|
30
|
+
validate_tensor_map,
|
|
31
|
+
)
|
|
32
|
+
from dromeus.manifests.models import (
|
|
33
|
+
NoLoCoConfig,
|
|
34
|
+
RoundId,
|
|
35
|
+
TensorSchema,
|
|
36
|
+
UpdateCodecBinding,
|
|
37
|
+
)
|
|
38
|
+
from dromeus.training.base import CheckpointTrainer, EvaluationResult, WeightTrainer
|
|
39
|
+
|
|
40
|
+
_ARTIFACT_NAMES = ("outer_gradient", "slow_weights")
|
|
41
|
+
_STATE_PREFIX = "noloco."
|
|
42
|
+
_PHASES = {
|
|
43
|
+
"created": 0,
|
|
44
|
+
"pre-local": 1,
|
|
45
|
+
"post-local": 2,
|
|
46
|
+
"bundled": 3,
|
|
47
|
+
"post-outer": 4,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _identity_codecs() -> dict[str, UpdateCodec]:
|
|
52
|
+
return {name: IdentityCodec("identity") for name in _ARTIFACT_NAMES}
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
@dataclass
|
|
56
|
+
class NoLoCoAlgorithm:
|
|
57
|
+
"""Own NoLoCo slow weights, outer momentum, and opaque bundle lifecycle.
|
|
58
|
+
|
|
59
|
+
Paper ``phi`` is ``_slow_weights``, post-local ``theta`` is the trainer's fast
|
|
60
|
+
weights, and Dromeus names the descent-oriented outer gradient ``g = phi -
|
|
61
|
+
theta``. ``alpha``, ``beta``, and ``gamma`` map directly to ``config``. The
|
|
62
|
+
identity path subtracts the averaged gradients and slow-weight correction. This
|
|
63
|
+
is algebraically equivalent to the pinned executable upstream convention, which
|
|
64
|
+
adds ``beta * mean(theta - phi)`` and effectively uses ``gamma = beta``. It does
|
|
65
|
+
not follow the divergent sign printed in paper Equation 2.
|
|
66
|
+
"""
|
|
67
|
+
|
|
68
|
+
trainer: WeightTrainer
|
|
69
|
+
tensor_schema: TensorSchema
|
|
70
|
+
config: NoLoCoConfig
|
|
71
|
+
bundle_codec: NamedUpdateBundleCodec | None = None
|
|
72
|
+
artifact_codecs: Mapping[str, UpdateCodec] = field(default_factory=_identity_codecs)
|
|
73
|
+
manifest_codec_ids: Mapping[str, str] | None = None
|
|
74
|
+
_round_id: RoundId = 0
|
|
75
|
+
_phase: str = "created"
|
|
76
|
+
_slow_weights: dict[str, np.ndarray] = field(init=False, repr=False)
|
|
77
|
+
_outer_momentum: dict[str, np.ndarray] = field(init=False, repr=False)
|
|
78
|
+
_error_feedback_residual: dict[str, np.ndarray] = field(init=False, repr=False)
|
|
79
|
+
_completed_outer_steps: int = field(default=0, init=False, repr=False)
|
|
80
|
+
_local_artifacts: dict[str, dict[str, np.ndarray]] | None = field(
|
|
81
|
+
default=None, init=False, repr=False
|
|
82
|
+
)
|
|
83
|
+
_codec_descriptions: Mapping[str, CodecDescription] = field(init=False, repr=False)
|
|
84
|
+
|
|
85
|
+
def __post_init__(self) -> None:
|
|
86
|
+
if set(self.artifact_codecs) != set(_ARTIFACT_NAMES):
|
|
87
|
+
raise ValueError("NoLoCo codecs must cover both named artifacts")
|
|
88
|
+
self.artifact_codecs = MappingProxyType(dict(self.artifact_codecs))
|
|
89
|
+
self._codec_descriptions = MappingProxyType(
|
|
90
|
+
{
|
|
91
|
+
name: describe_update_codec(codec, self.tensor_schema)
|
|
92
|
+
for name, codec in self.artifact_codecs.items()
|
|
93
|
+
}
|
|
94
|
+
)
|
|
95
|
+
if self.manifest_codec_ids is not None:
|
|
96
|
+
actual = {
|
|
97
|
+
name: description.codec_id
|
|
98
|
+
for name, description in self._codec_descriptions.items()
|
|
99
|
+
}
|
|
100
|
+
if dict(self.manifest_codec_ids) != actual:
|
|
101
|
+
raise ValueError("NoLoCo codecs do not match manifest")
|
|
102
|
+
weights = self.trainer.weights()
|
|
103
|
+
self._validate_tensors(weights)
|
|
104
|
+
self._slow_weights = self._copy_tensors(weights)
|
|
105
|
+
self._outer_momentum = {
|
|
106
|
+
name: np.zeros_like(value, dtype=np.float32)
|
|
107
|
+
for name, value in weights.items()
|
|
108
|
+
}
|
|
109
|
+
self._error_feedback_residual = {
|
|
110
|
+
name: np.zeros_like(value, dtype=np.float32)
|
|
111
|
+
for name, value in weights.items()
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
def pre_local(self, round_id: RoundId) -> None:
|
|
115
|
+
if self._phase == "created":
|
|
116
|
+
weights = self.trainer.weights()
|
|
117
|
+
self._validate_tensors(weights)
|
|
118
|
+
self._slow_weights = self._copy_tensors(weights)
|
|
119
|
+
self._outer_momentum = {
|
|
120
|
+
name: np.zeros_like(value, dtype=np.float32)
|
|
121
|
+
for name, value in weights.items()
|
|
122
|
+
}
|
|
123
|
+
self._error_feedback_residual = {
|
|
124
|
+
name: np.zeros_like(value, dtype=np.float32)
|
|
125
|
+
for name, value in weights.items()
|
|
126
|
+
}
|
|
127
|
+
self._round_id = round_id
|
|
128
|
+
self.trainer.load_weights(self._copy_tensors(self._slow_weights))
|
|
129
|
+
self._local_artifacts = None
|
|
130
|
+
self._phase = "pre-local"
|
|
131
|
+
|
|
132
|
+
def local_training(self) -> None:
|
|
133
|
+
self.trainer.train_local_steps(self.config.inner_steps)
|
|
134
|
+
self._phase = "post-local"
|
|
135
|
+
|
|
136
|
+
def post_local_bundle(self) -> UpdateBundle:
|
|
137
|
+
if self.bundle_codec is None:
|
|
138
|
+
raise RuntimeError("update bundle codec is not configured")
|
|
139
|
+
fast_weights = self.trainer.weights()
|
|
140
|
+
self._validate_tensors(fast_weights)
|
|
141
|
+
logical = {
|
|
142
|
+
"outer_gradient": {
|
|
143
|
+
name: (
|
|
144
|
+
self._slow_weights[name].astype(np.float32)
|
|
145
|
+
- fast_weights[name].astype(np.float32)
|
|
146
|
+
+ (
|
|
147
|
+
self._error_feedback_residual[name]
|
|
148
|
+
if self._codec_descriptions["outer_gradient"].lossy
|
|
149
|
+
else np.float32(0.0)
|
|
150
|
+
)
|
|
151
|
+
).astype(np.float32)
|
|
152
|
+
for name in self._slow_weights
|
|
153
|
+
},
|
|
154
|
+
"slow_weights": self._copy_tensors(self._slow_weights),
|
|
155
|
+
}
|
|
156
|
+
encoded = {
|
|
157
|
+
name: self.artifact_codecs[name].encode(logical[name])
|
|
158
|
+
for name in _ARTIFACT_NAMES
|
|
159
|
+
}
|
|
160
|
+
bundle = self.bundle_codec.encode(
|
|
161
|
+
round_id=self._round_id,
|
|
162
|
+
artifacts=encoded,
|
|
163
|
+
codec_bindings=self._codec_bindings(),
|
|
164
|
+
)
|
|
165
|
+
try:
|
|
166
|
+
local = {
|
|
167
|
+
name: self.artifact_codecs[name].decode(encoded[name])
|
|
168
|
+
for name in _ARTIFACT_NAMES
|
|
169
|
+
}
|
|
170
|
+
self._validate_artifacts(local)
|
|
171
|
+
except BaseException:
|
|
172
|
+
self.bundle_codec.release(bundle)
|
|
173
|
+
raise
|
|
174
|
+
self._local_artifacts = {
|
|
175
|
+
name: self._copy_tensors(tensors) for name, tensors in local.items()
|
|
176
|
+
}
|
|
177
|
+
self._error_feedback_residual = {
|
|
178
|
+
name: (
|
|
179
|
+
logical["outer_gradient"][name] - local["outer_gradient"][name]
|
|
180
|
+
if self._codec_descriptions["outer_gradient"].lossy
|
|
181
|
+
else np.zeros_like(value, dtype=np.float32)
|
|
182
|
+
).astype(np.float32)
|
|
183
|
+
for name, value in self._slow_weights.items()
|
|
184
|
+
}
|
|
185
|
+
self._phase = "bundled"
|
|
186
|
+
return bundle
|
|
187
|
+
|
|
188
|
+
def validate_peer(self, peer_bundle: UpdateBundle) -> NamedValidatedUpdate:
|
|
189
|
+
if self.bundle_codec is None:
|
|
190
|
+
raise RuntimeError("update bundle codec is not configured")
|
|
191
|
+
if peer_bundle.metadata.round_id != self._round_id:
|
|
192
|
+
raise ValueError("peer bundle round does not match current round")
|
|
193
|
+
encoded = self.bundle_codec.decode(
|
|
194
|
+
peer_bundle, codec_bindings=self._codec_bindings()
|
|
195
|
+
)
|
|
196
|
+
artifacts = {
|
|
197
|
+
name: self.artifact_codecs[name].decode(encoded[name])
|
|
198
|
+
for name in _ARTIFACT_NAMES
|
|
199
|
+
}
|
|
200
|
+
self._validate_artifacts(artifacts)
|
|
201
|
+
return NamedValidatedUpdate(
|
|
202
|
+
round_id=self._round_id,
|
|
203
|
+
artifacts=artifacts,
|
|
204
|
+
checksum=checksum_artifacts(artifacts),
|
|
205
|
+
)
|
|
206
|
+
|
|
207
|
+
def peer_apply(self, peer_update: AlgorithmUpdate) -> AlgorithmSnapshot:
|
|
208
|
+
if not isinstance(peer_update, NamedValidatedUpdate):
|
|
209
|
+
raise TypeError("NoLoCo requires named validated artifacts")
|
|
210
|
+
if peer_update.round_id != self._round_id:
|
|
211
|
+
raise ValueError("peer update round does not match current round")
|
|
212
|
+
self._validate_artifacts(peer_update.artifacts)
|
|
213
|
+
if checksum_artifacts(peer_update.artifacts) != peer_update.checksum:
|
|
214
|
+
raise ValueError("peer update checksum mismatch")
|
|
215
|
+
if self._local_artifacts is None:
|
|
216
|
+
raise RuntimeError("local update bundle has not been built")
|
|
217
|
+
local_gradient = self._local_artifacts["outer_gradient"]
|
|
218
|
+
local_slow = self._local_artifacts["slow_weights"]
|
|
219
|
+
peer_gradient = peer_update.artifacts["outer_gradient"]
|
|
220
|
+
peer_slow = peer_update.artifacts["slow_weights"]
|
|
221
|
+
next_momentum: dict[str, np.ndarray] = {}
|
|
222
|
+
next_slow: dict[str, np.ndarray] = {}
|
|
223
|
+
for name in self._slow_weights:
|
|
224
|
+
momentum = (
|
|
225
|
+
np.float32(self.config.alpha) * self._outer_momentum[name]
|
|
226
|
+
- np.float32(self.config.beta / 2.0)
|
|
227
|
+
* (local_gradient[name] + peer_gradient[name])
|
|
228
|
+
- np.float32(self.config.gamma / 2.0)
|
|
229
|
+
* (local_slow[name] - peer_slow[name])
|
|
230
|
+
).astype(np.float32)
|
|
231
|
+
next_momentum[name] = momentum
|
|
232
|
+
next_slow[name] = (
|
|
233
|
+
self._slow_weights[name].astype(np.float32) + momentum
|
|
234
|
+
).astype(np.float32)
|
|
235
|
+
self._outer_momentum = next_momentum
|
|
236
|
+
self._slow_weights = next_slow
|
|
237
|
+
self._completed_outer_steps += 1
|
|
238
|
+
self.trainer.load_weights(self._copy_tensors(next_slow))
|
|
239
|
+
self._phase = "post-outer"
|
|
240
|
+
return self.snapshot()
|
|
241
|
+
|
|
242
|
+
def release_bundle(self, bundle: UpdateBundle) -> None:
|
|
243
|
+
if self.bundle_codec is None:
|
|
244
|
+
raise RuntimeError("update bundle codec is not configured")
|
|
245
|
+
self.bundle_codec.release(bundle)
|
|
246
|
+
|
|
247
|
+
def configure_bundle_codec(
|
|
248
|
+
self,
|
|
249
|
+
*,
|
|
250
|
+
artifact_root: Path,
|
|
251
|
+
run_id: str,
|
|
252
|
+
manifest_hash: str,
|
|
253
|
+
sender_public_key: str,
|
|
254
|
+
algorithm_id: str,
|
|
255
|
+
) -> None:
|
|
256
|
+
"""Bind formed run context without exposing codec details to runtime."""
|
|
257
|
+
if self.bundle_codec is None:
|
|
258
|
+
self.bundle_codec = NamedSafetensorsUpdateBundleCodec(
|
|
259
|
+
artifact_root=artifact_root,
|
|
260
|
+
run_id=run_id,
|
|
261
|
+
manifest_hash=manifest_hash,
|
|
262
|
+
sender_public_key=sender_public_key,
|
|
263
|
+
algorithm_id=algorithm_id,
|
|
264
|
+
artifact_schemas={
|
|
265
|
+
name: description.encoded_schema
|
|
266
|
+
for name, description in self._codec_descriptions.items()
|
|
267
|
+
},
|
|
268
|
+
)
|
|
269
|
+
|
|
270
|
+
def snapshot(self) -> AlgorithmSnapshot:
|
|
271
|
+
return AlgorithmSnapshot(
|
|
272
|
+
round_id=self._round_id,
|
|
273
|
+
phase=self._phase,
|
|
274
|
+
weights=self._slow_weights,
|
|
275
|
+
)
|
|
276
|
+
|
|
277
|
+
def evaluate(self) -> EvaluationResult | None:
|
|
278
|
+
return self.trainer.evaluate()
|
|
279
|
+
|
|
280
|
+
def observations(self) -> AlgorithmObservations:
|
|
281
|
+
observations = AlgorithmObservations(local_loss=self.trainer.local_loss)
|
|
282
|
+
if self._local_artifacts is None:
|
|
283
|
+
return observations
|
|
284
|
+
residual_norm = _tensor_l2_norm(self._error_feedback_residual)
|
|
285
|
+
signal = {
|
|
286
|
+
name: (
|
|
287
|
+
self._local_artifacts["outer_gradient"][name]
|
|
288
|
+
+ self._error_feedback_residual[name]
|
|
289
|
+
).astype(np.float32)
|
|
290
|
+
for name in self._error_feedback_residual
|
|
291
|
+
}
|
|
292
|
+
signal_norm = _tensor_l2_norm(signal)
|
|
293
|
+
ratio = residual_norm / signal_norm if signal_norm > 0.0 else 0.0
|
|
294
|
+
return AlgorithmObservations(
|
|
295
|
+
local_loss=observations.local_loss,
|
|
296
|
+
error_feedback_residual_l2_norm=residual_norm,
|
|
297
|
+
error_feedback_signal_l2_norm=signal_norm,
|
|
298
|
+
error_feedback_residual_to_signal_ratio=ratio,
|
|
299
|
+
)
|
|
300
|
+
|
|
301
|
+
def checkpoint_tensors(self) -> dict[str, np.ndarray]:
|
|
302
|
+
"""Return complete durable state in the versioned flat namespace."""
|
|
303
|
+
if not isinstance(self.trainer, CheckpointTrainer):
|
|
304
|
+
raise TypeError("NoLoCo trainer must expose checkpoint state")
|
|
305
|
+
state: dict[str, np.ndarray] = {
|
|
306
|
+
f"{_STATE_PREFIX}schema_version": np.array([1], dtype=np.int64),
|
|
307
|
+
f"{_STATE_PREFIX}round_id": np.array([self._round_id], dtype=np.int64),
|
|
308
|
+
f"{_STATE_PREFIX}completed_outer_steps": np.array(
|
|
309
|
+
[self._completed_outer_steps], dtype=np.int64
|
|
310
|
+
),
|
|
311
|
+
f"{_STATE_PREFIX}phase": np.array([_PHASES[self._phase]], dtype=np.int64),
|
|
312
|
+
}
|
|
313
|
+
for name, value in self._slow_weights.items():
|
|
314
|
+
state[f"{_STATE_PREFIX}slow_weights.{name}"] = value.copy()
|
|
315
|
+
for name, value in self._outer_momentum.items():
|
|
316
|
+
state[f"{_STATE_PREFIX}outer_momentum.{name}"] = value.copy()
|
|
317
|
+
for name, value in self._error_feedback_residual.items():
|
|
318
|
+
state[f"{_STATE_PREFIX}error_feedback_residual.{name}"] = value.copy()
|
|
319
|
+
for name, value in self.trainer.checkpoint_tensors().items():
|
|
320
|
+
state[f"{_STATE_PREFIX}trainer.{name}"] = value.copy(order="C")
|
|
321
|
+
for artifact_name, codec in self.artifact_codecs.items():
|
|
322
|
+
for name, value in codec.state_dict().items():
|
|
323
|
+
if not isinstance(value, np.ndarray):
|
|
324
|
+
raise TypeError("NoLoCo codec checkpoint state must be tensors")
|
|
325
|
+
state[f"{_STATE_PREFIX}codec.{artifact_name}.{name}"] = (
|
|
326
|
+
np.ascontiguousarray(cast(Any, value)).copy()
|
|
327
|
+
)
|
|
328
|
+
return state
|
|
329
|
+
|
|
330
|
+
def load_checkpoint_tensors(self, state: Mapping[str, np.ndarray]) -> None:
|
|
331
|
+
"""Restore a validated checkpoint produced by `checkpoint_tensors`."""
|
|
332
|
+
if not isinstance(self.trainer, CheckpointTrainer):
|
|
333
|
+
raise TypeError("NoLoCo trainer must expose checkpoint state")
|
|
334
|
+
version = self._counter(state, "schema_version")
|
|
335
|
+
if version != 1:
|
|
336
|
+
raise ValueError("unsupported NoLoCo checkpoint version")
|
|
337
|
+
round_id = self._counter(state, "round_id")
|
|
338
|
+
completed_outer_steps = self._counter(state, "completed_outer_steps")
|
|
339
|
+
phase_value = self._counter(state, "phase")
|
|
340
|
+
phase_by_value = {value: name for name, value in _PHASES.items()}
|
|
341
|
+
if phase_value not in phase_by_value:
|
|
342
|
+
raise ValueError("NoLoCo checkpoint phase is invalid")
|
|
343
|
+
names = {tensor.name for tensor in self.tensor_schema.tensors}
|
|
344
|
+
slow_weights = self._checkpoint_group(state, "slow_weights", names)
|
|
345
|
+
outer_momentum = self._checkpoint_group(state, "outer_momentum", names)
|
|
346
|
+
residual = self._checkpoint_group(state, "error_feedback_residual", names)
|
|
347
|
+
self._validate_tensors(slow_weights)
|
|
348
|
+
self._validate_tensors(outer_momentum)
|
|
349
|
+
self._validate_tensors(residual)
|
|
350
|
+
trainer_prefix = f"{_STATE_PREFIX}trainer."
|
|
351
|
+
trainer_state = {
|
|
352
|
+
name.removeprefix(trainer_prefix): value.copy(order="C")
|
|
353
|
+
for name, value in state.items()
|
|
354
|
+
if name.startswith(trainer_prefix)
|
|
355
|
+
}
|
|
356
|
+
if not trainer_state:
|
|
357
|
+
raise ValueError("NoLoCo trainer checkpoint state is missing")
|
|
358
|
+
previous_trainer = {
|
|
359
|
+
name: value.copy(order="C")
|
|
360
|
+
for name, value in self.trainer.checkpoint_tensors().items()
|
|
361
|
+
}
|
|
362
|
+
previous_codecs = {
|
|
363
|
+
name: deepcopy(codec.state_dict())
|
|
364
|
+
for name, codec in self.artifact_codecs.items()
|
|
365
|
+
}
|
|
366
|
+
try:
|
|
367
|
+
self.trainer.load_checkpoint_tensors(trainer_state)
|
|
368
|
+
restored_weights = self.trainer.weights()
|
|
369
|
+
self._validate_tensors(restored_weights)
|
|
370
|
+
if any(
|
|
371
|
+
not np.array_equal(restored_weights[name], slow_weights[name])
|
|
372
|
+
for name in names
|
|
373
|
+
):
|
|
374
|
+
raise ValueError("NoLoCo trainer and slow weights do not match")
|
|
375
|
+
for artifact_name, codec in self.artifact_codecs.items():
|
|
376
|
+
codec_prefix = f"{_STATE_PREFIX}codec.{artifact_name}."
|
|
377
|
+
codec.load_state_dict(
|
|
378
|
+
{
|
|
379
|
+
name.removeprefix(codec_prefix): value.copy()
|
|
380
|
+
for name, value in state.items()
|
|
381
|
+
if name.startswith(codec_prefix)
|
|
382
|
+
}
|
|
383
|
+
)
|
|
384
|
+
except Exception as error:
|
|
385
|
+
rollback_errors: list[Exception] = []
|
|
386
|
+
try:
|
|
387
|
+
self.trainer.load_checkpoint_tensors(previous_trainer)
|
|
388
|
+
except Exception as rollback_error:
|
|
389
|
+
rollback_errors.append(rollback_error)
|
|
390
|
+
for name, codec in self.artifact_codecs.items():
|
|
391
|
+
try:
|
|
392
|
+
codec.load_state_dict(previous_codecs[name])
|
|
393
|
+
except Exception as rollback_error:
|
|
394
|
+
rollback_errors.append(rollback_error)
|
|
395
|
+
if rollback_errors:
|
|
396
|
+
raise RuntimeError(
|
|
397
|
+
"NoLoCo checkpoint rollback failed"
|
|
398
|
+
) from ExceptionGroup(
|
|
399
|
+
"restore and rollback errors", [error, *rollback_errors]
|
|
400
|
+
)
|
|
401
|
+
raise
|
|
402
|
+
self._round_id = round_id
|
|
403
|
+
self._completed_outer_steps = completed_outer_steps
|
|
404
|
+
self._phase = phase_by_value[phase_value]
|
|
405
|
+
self._slow_weights = slow_weights
|
|
406
|
+
self._outer_momentum = outer_momentum
|
|
407
|
+
self._error_feedback_residual = residual
|
|
408
|
+
self._local_artifacts = None
|
|
409
|
+
|
|
410
|
+
def state_dict(self) -> dict[str, object]:
|
|
411
|
+
return dict(self.checkpoint_tensors())
|
|
412
|
+
|
|
413
|
+
def load_state_dict(self, state: Mapping[str, object]) -> None:
|
|
414
|
+
if not all(isinstance(value, np.ndarray) for value in state.values()):
|
|
415
|
+
raise ValueError("NoLoCo algorithm state must contain only tensors")
|
|
416
|
+
tensor_state: dict[str, np.ndarray] = {
|
|
417
|
+
name: cast(np.ndarray, value) for name, value in state.items()
|
|
418
|
+
}
|
|
419
|
+
self.load_checkpoint_tensors(tensor_state)
|
|
420
|
+
|
|
421
|
+
def _validate_artifacts(
|
|
422
|
+
self, artifacts: Mapping[str, Mapping[str, np.ndarray]]
|
|
423
|
+
) -> None:
|
|
424
|
+
if set(artifacts) != set(_ARTIFACT_NAMES):
|
|
425
|
+
raise ValueError("NoLoCo update requires both named artifacts")
|
|
426
|
+
for tensors in artifacts.values():
|
|
427
|
+
self._validate_tensors(tensors)
|
|
428
|
+
|
|
429
|
+
def _validate_tensors(self, tensors: Mapping[str, np.ndarray]) -> None:
|
|
430
|
+
validate_tensor_map(tensors, self.tensor_schema)
|
|
431
|
+
for value in tensors.values():
|
|
432
|
+
if value.dtype != np.float32:
|
|
433
|
+
raise ValueError("NoLoCo requires FP32 logical tensors")
|
|
434
|
+
|
|
435
|
+
def _codec_bindings(self) -> dict[str, UpdateCodecBinding]:
|
|
436
|
+
return {
|
|
437
|
+
name: UpdateCodecBinding(
|
|
438
|
+
codec_id=description.codec_id,
|
|
439
|
+
logical_schema=self.tensor_schema,
|
|
440
|
+
)
|
|
441
|
+
for name, description in self._codec_descriptions.items()
|
|
442
|
+
}
|
|
443
|
+
|
|
444
|
+
@staticmethod
|
|
445
|
+
def _counter(state: Mapping[str, np.ndarray], name: str) -> int:
|
|
446
|
+
value = np.asarray(state.get(f"{_STATE_PREFIX}{name}"))
|
|
447
|
+
if value.dtype != np.int64 or value.shape != (1,) or int(value[0]) < 0:
|
|
448
|
+
raise ValueError(f"NoLoCo checkpoint {name} is invalid")
|
|
449
|
+
return int(value[0])
|
|
450
|
+
|
|
451
|
+
@staticmethod
|
|
452
|
+
def _checkpoint_group(
|
|
453
|
+
state: Mapping[str, np.ndarray],
|
|
454
|
+
group: str,
|
|
455
|
+
names: set[str],
|
|
456
|
+
) -> dict[str, np.ndarray]:
|
|
457
|
+
prefix = f"{_STATE_PREFIX}{group}."
|
|
458
|
+
values = {
|
|
459
|
+
name.removeprefix(prefix): np.ascontiguousarray(value).copy()
|
|
460
|
+
for name, value in state.items()
|
|
461
|
+
if name.startswith(prefix)
|
|
462
|
+
}
|
|
463
|
+
if set(values) != names:
|
|
464
|
+
raise ValueError(f"NoLoCo checkpoint {group} is incomplete")
|
|
465
|
+
return values
|
|
466
|
+
|
|
467
|
+
@staticmethod
|
|
468
|
+
def _copy_tensors(
|
|
469
|
+
tensors: Mapping[str, np.ndarray],
|
|
470
|
+
) -> dict[str, np.ndarray]:
|
|
471
|
+
return {
|
|
472
|
+
name: np.ascontiguousarray(value).copy() for name, value in tensors.items()
|
|
473
|
+
}
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
def _tensor_l2_norm(tensors: Mapping[str, np.ndarray]) -> float:
|
|
477
|
+
total = 0.0
|
|
478
|
+
for value in tensors.values():
|
|
479
|
+
flattened = np.asarray(value, dtype=np.float32).reshape(-1)
|
|
480
|
+
for start in range(0, flattened.size, 1_048_576):
|
|
481
|
+
chunk = flattened[start : start + 1_048_576].astype(np.float64, copy=False)
|
|
482
|
+
total += float(np.sum(np.square(chunk), dtype=np.float64))
|
|
483
|
+
return math.sqrt(total)
|
|
484
|
+
|
|
485
|
+
|
|
486
|
+
__all__ = ["NoLoCoAlgorithm"]
|