dromeus 0.4.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dromeus/__init__.py +109 -0
- dromeus/__main__.py +6 -0
- dromeus/algorithms/__init__.py +31 -0
- dromeus/algorithms/base.py +188 -0
- dromeus/algorithms/codec.py +623 -0
- dromeus/algorithms/noloco.py +486 -0
- dromeus/application.py +523 -0
- dromeus/cli.py +341 -0
- dromeus/diagnostics.py +303 -0
- dromeus/errors.py +57 -0
- dromeus/gossip/__init__.py +29 -0
- dromeus/gossip/_async.py +15 -0
- dromeus/gossip/axl.py +497 -0
- dromeus/gossip/engine.py +378 -0
- dromeus/gossip/interfaces.py +158 -0
- dromeus/gossip/peer_scheduler.py +117 -0
- dromeus/manifests/__init__.py +1 -0
- dromeus/manifests/canonical.py +157 -0
- dromeus/manifests/models.py +694 -0
- dromeus/membership/__init__.py +1 -0
- dromeus/membership/formation.py +576 -0
- dromeus/node.py +338 -0
- dromeus/persistence/__init__.py +20 -0
- dromeus/persistence/archive.py +576 -0
- dromeus/persistence/files.py +39 -0
- dromeus/persistence/privacy_ledger.py +104 -0
- dromeus/persistence/run_store.py +352 -0
- dromeus/protocol/__init__.py +25 -0
- dromeus/protocol/codec.py +127 -0
- dromeus/protocol/models.py +199 -0
- dromeus/protocol/version.py +5 -0
- dromeus/py.typed +1 -0
- dromeus/runtime.py +1081 -0
- dromeus/telemetry/__init__.py +61 -0
- dromeus/telemetry/consensus.py +909 -0
- dromeus/telemetry/divergence.py +321 -0
- dromeus/telemetry/divergence_monitor.py +97 -0
- dromeus/telemetry/events.py +214 -0
- dromeus/telemetry/evidence.py +429 -0
- dromeus/telemetry/metrics.py +200 -0
- dromeus/telemetry/progress.py +255 -0
- dromeus/training/__init__.py +1 -0
- dromeus/training/base.py +60 -0
- dromeus/training/model_state.py +111 -0
- dromeus/training/native_gradients.py +289 -0
- dromeus/training/optimizer_factory.py +426 -0
- dromeus/training/ordinary_task.py +408 -0
- dromeus/training/privacy.py +96 -0
- dromeus/training/private_data.py +96 -0
- dromeus/training/private_model.py +414 -0
- dromeus/training/private_state.py +97 -0
- dromeus/training/private_trainer.py +534 -0
- dromeus/training/private_workload.py +217 -0
- dromeus/training/state.py +57 -0
- dromeus/training/task.py +464 -0
- dromeus/training/trainer.py +294 -0
- dromeus/training/versions.py +15 -0
- dromeus/transport/__init__.py +1 -0
- dromeus/transport/axl.py +168 -0
- dromeus/transport/envelope_sender.py +73 -0
- dromeus/transport/interface.py +31 -0
- dromeus/transport/outbound_scheduler.py +326 -0
- dromeus/transport/receiver.py +345 -0
- dromeus/transport/transfer.py +854 -0
- dromeus-0.4.0.dev1.dist-info/METADATA +150 -0
- dromeus-0.4.0.dev1.dist-info/RECORD +69 -0
- dromeus-0.4.0.dev1.dist-info/WHEEL +4 -0
- dromeus-0.4.0.dev1.dist-info/entry_points.txt +2 -0
- dromeus-0.4.0.dev1.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,426 @@
|
|
|
1
|
+
"""Construction and state validation for built-in and group-selected optimizers.
|
|
2
|
+
|
|
3
|
+
Factories and validators are trusted application code. Their implementation hash
|
|
4
|
+
must bind the optimizer, factory and state validator. Optimizers may consume only
|
|
5
|
+
sanitized gradients, model parameters, public settings and their own numeric state.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import inspect
|
|
11
|
+
import json
|
|
12
|
+
import math
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass
|
|
15
|
+
from typing import Any, cast
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
from torch import nn
|
|
19
|
+
|
|
20
|
+
from dromeus.errors import TaskError
|
|
21
|
+
from dromeus.manifests.models import (
|
|
22
|
+
FactoryOptimizerSpec,
|
|
23
|
+
OptimizerParameterGroup,
|
|
24
|
+
PrivateOptimizerSpec,
|
|
25
|
+
)
|
|
26
|
+
from dromeus.training.private_state import decode_state, encode_state
|
|
27
|
+
from dromeus.training.versions import torch_release
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def build_private_optimizer(
|
|
31
|
+
model: nn.Module, spec: PrivateOptimizerSpec | FactoryOptimizerSpec
|
|
32
|
+
) -> torch.optim.Optimizer:
|
|
33
|
+
"""Build the declared ordinary update rule, before optional Opacus wrapping."""
|
|
34
|
+
if isinstance(spec, FactoryOptimizerSpec):
|
|
35
|
+
raise TaskError("factory optimizer requires a local optimizer registration")
|
|
36
|
+
parameters = [p for p in model.parameters() if p.requires_grad]
|
|
37
|
+
common: dict[str, Any] = dict(
|
|
38
|
+
lr=spec.learning_rate,
|
|
39
|
+
weight_decay=spec.weight_decay,
|
|
40
|
+
foreach=False,
|
|
41
|
+
fused=False,
|
|
42
|
+
differentiable=False,
|
|
43
|
+
maximize=False,
|
|
44
|
+
)
|
|
45
|
+
if spec.name == "sgd":
|
|
46
|
+
return torch.optim.SGD(
|
|
47
|
+
parameters,
|
|
48
|
+
momentum=spec.momentum,
|
|
49
|
+
dampening=spec.dampening,
|
|
50
|
+
nesterov=spec.nesterov,
|
|
51
|
+
**common,
|
|
52
|
+
)
|
|
53
|
+
factory = torch.optim.Adam if spec.name == "adam" else torch.optim.AdamW
|
|
54
|
+
return factory(
|
|
55
|
+
parameters,
|
|
56
|
+
betas=spec.betas,
|
|
57
|
+
eps=spec.numerical_epsilon,
|
|
58
|
+
amsgrad=False,
|
|
59
|
+
capturable=False,
|
|
60
|
+
**common,
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def validate_builtin_optimizer_checkpoint(
|
|
65
|
+
state: dict[str, Any],
|
|
66
|
+
*,
|
|
67
|
+
spec: PrivateOptimizerSpec,
|
|
68
|
+
parameters: tuple[torch.Tensor, ...],
|
|
69
|
+
initial_groups: list[dict[str, Any]],
|
|
70
|
+
logical_steps: int,
|
|
71
|
+
horizon: int,
|
|
72
|
+
schedule: str,
|
|
73
|
+
) -> None:
|
|
74
|
+
"""Validate built-in state against the frozen recipe without mutating it.
|
|
75
|
+
|
|
76
|
+
Parameters retain model order and groups come from the initial state_dict,
|
|
77
|
+
before scheduling. Factory-defined state uses its registered validator.
|
|
78
|
+
"""
|
|
79
|
+
if set(state) != {"state", "param_groups"}:
|
|
80
|
+
raise TaskError("optimizer checkpoint fields mismatch")
|
|
81
|
+
groups = state["param_groups"]
|
|
82
|
+
if len(groups) != len(initial_groups):
|
|
83
|
+
raise TaskError("optimizer checkpoint groups mismatch")
|
|
84
|
+
for group, initial in zip(groups, initial_groups, strict=True):
|
|
85
|
+
expected = dict(initial)
|
|
86
|
+
if schedule == "linear_decay":
|
|
87
|
+
expected["lr"] *= 1 - logical_steps / horizon
|
|
88
|
+
if group != expected:
|
|
89
|
+
raise TaskError("optimizer checkpoint options or schedule mismatch")
|
|
90
|
+
fields: set[str] = (
|
|
91
|
+
({"momentum_buffer"} if spec.momentum else set())
|
|
92
|
+
if spec.name == "sgd"
|
|
93
|
+
else {"step", "exp_avg", "exp_avg_sq"}
|
|
94
|
+
)
|
|
95
|
+
ids: set[int] = set(range(len(parameters))) if logical_steps and fields else set()
|
|
96
|
+
if set(state["state"]) != ids:
|
|
97
|
+
raise TaskError("optimizer checkpoint state coverage mismatch")
|
|
98
|
+
for index in ids:
|
|
99
|
+
values = state["state"][index]
|
|
100
|
+
if set(values) != fields:
|
|
101
|
+
raise TaskError("optimizer checkpoint state fields mismatch")
|
|
102
|
+
for name, tensor in values.items():
|
|
103
|
+
shape = () if name == "step" else parameters[index].shape
|
|
104
|
+
if (
|
|
105
|
+
not isinstance(tensor, torch.Tensor)
|
|
106
|
+
or tensor.shape != shape
|
|
107
|
+
or tensor.dtype != torch.float32
|
|
108
|
+
):
|
|
109
|
+
raise TaskError("optimizer checkpoint tensor mismatch")
|
|
110
|
+
if name == "step" and float(tensor) != logical_steps:
|
|
111
|
+
raise TaskError("optimizer checkpoint step mismatch")
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
def _json_option(value: Any, depth: int = 0) -> Any:
|
|
115
|
+
if depth > 32:
|
|
116
|
+
raise TaskError("optimizer option nesting exceeds bound")
|
|
117
|
+
if value is None or type(value) in (str, bool, int):
|
|
118
|
+
return value
|
|
119
|
+
if type(value) is float and math.isfinite(value):
|
|
120
|
+
return value
|
|
121
|
+
if isinstance(value, (tuple, list)):
|
|
122
|
+
return [_json_option(item, depth + 1) for item in cast(list[Any], value)]
|
|
123
|
+
if isinstance(value, dict) and all(
|
|
124
|
+
type(k) is str for k in cast(dict[Any, Any], value)
|
|
125
|
+
):
|
|
126
|
+
return {
|
|
127
|
+
key: _json_option(item, depth + 1)
|
|
128
|
+
for key, item in cast(dict[str, Any], value).items()
|
|
129
|
+
}
|
|
130
|
+
raise TaskError("optimizer options must contain finite JSON values")
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
def _canonical_options(value: Any) -> str:
|
|
134
|
+
return json.dumps(_json_option(value), sort_keys=True, separators=(",", ":"))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _implementation_id(optimizer_type: type[torch.optim.Optimizer]) -> str:
|
|
138
|
+
return f"{optimizer_type.__module__}.{optimizer_type.__qualname__}"
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
@dataclass(frozen=True)
|
|
142
|
+
class _ActualGroup:
|
|
143
|
+
parameter_names: tuple[str, ...]
|
|
144
|
+
options: dict[str, Any]
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def _groups(
|
|
148
|
+
optimizer: torch.optim.Optimizer, model: nn.Module
|
|
149
|
+
) -> tuple[_ActualGroup, ...]:
|
|
150
|
+
named = tuple((name, p) for name, p in model.named_parameters() if p.requires_grad)
|
|
151
|
+
names = {id(parameter): name for name, parameter in named}
|
|
152
|
+
seen: set[int] = set()
|
|
153
|
+
result: list[_ActualGroup] = []
|
|
154
|
+
for group in optimizer.param_groups:
|
|
155
|
+
members: list[str] = []
|
|
156
|
+
for parameter in group["params"]:
|
|
157
|
+
identity = id(parameter)
|
|
158
|
+
if identity not in names or identity in seen:
|
|
159
|
+
raise TaskError(
|
|
160
|
+
"optimizer parameters must cover the model exactly once"
|
|
161
|
+
)
|
|
162
|
+
seen.add(identity)
|
|
163
|
+
members.append(names[identity])
|
|
164
|
+
options = _json_option({k: v for k, v in group.items() if k != "params"})
|
|
165
|
+
result.append(_ActualGroup(parameter_names=tuple(members), options=options))
|
|
166
|
+
if seen != set(names):
|
|
167
|
+
raise TaskError("optimizer parameters must cover the model exactly once")
|
|
168
|
+
return tuple(result)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _numeric_copy(state: dict[str, Any]) -> dict[str, Any]:
|
|
172
|
+
"""Reject nonnumeric payloads before giving a detached copy to trusted code."""
|
|
173
|
+
|
|
174
|
+
def bounded(value: Any, depth: int = 0) -> None:
|
|
175
|
+
if depth > 32:
|
|
176
|
+
raise TaskError("optimizer state nesting exceeds bound")
|
|
177
|
+
if isinstance(value, dict):
|
|
178
|
+
for key, item in cast(dict[Any, Any], value).items():
|
|
179
|
+
bounded(key, depth + 1)
|
|
180
|
+
bounded(item, depth + 1)
|
|
181
|
+
elif isinstance(value, (tuple, list)):
|
|
182
|
+
for item in cast(list[Any], value):
|
|
183
|
+
bounded(item, depth + 1)
|
|
184
|
+
|
|
185
|
+
bounded(state)
|
|
186
|
+
return cast(dict[str, Any], decode_state(encode_state(state)))
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
@dataclass(frozen=True)
|
|
190
|
+
class OptimizerRegistration:
|
|
191
|
+
"""Attested optimizer construction and optimizer-specific restore validation.
|
|
192
|
+
|
|
193
|
+
``validate_state`` receives a detached numeric state dictionary, parameters in
|
|
194
|
+
optimizer group order, and the completed logical step count. It must reject
|
|
195
|
+
invalid field layouts, tensor shapes/dtypes and counters. Public eager state
|
|
196
|
+
at step zero is allowed; prior private training state is not public initialization.
|
|
197
|
+
"""
|
|
198
|
+
|
|
199
|
+
optimizer_type: type[torch.optim.Optimizer]
|
|
200
|
+
implementation_hash: str
|
|
201
|
+
create: Callable[[nn.Module], torch.optim.Optimizer]
|
|
202
|
+
validate_state: Callable[[dict[str, Any], tuple[torch.Tensor, ...], int], None]
|
|
203
|
+
|
|
204
|
+
def _validate_type(self, optimizer: torch.optim.Optimizer) -> None:
|
|
205
|
+
if not issubclass(cast(Any, self.optimizer_type), torch.optim.Optimizer) or (
|
|
206
|
+
type(optimizer) is not self.optimizer_type
|
|
207
|
+
):
|
|
208
|
+
raise TaskError("optimizer factory returned a different class")
|
|
209
|
+
try:
|
|
210
|
+
inspect.signature(optimizer.step).bind()
|
|
211
|
+
except (TypeError, ValueError):
|
|
212
|
+
raise TaskError(
|
|
213
|
+
"optimizer step must work without closures or arguments"
|
|
214
|
+
) from None
|
|
215
|
+
|
|
216
|
+
def describe(self, model: nn.Module) -> FactoryOptimizerSpec:
|
|
217
|
+
"""Describe a fresh optimizer constructed against a public model."""
|
|
218
|
+
optimizer = self.create(model)
|
|
219
|
+
self._validate_type(optimizer)
|
|
220
|
+
spec = FactoryOptimizerSpec(
|
|
221
|
+
implementation_id=_implementation_id(self.optimizer_type),
|
|
222
|
+
implementation_hash=self.implementation_hash,
|
|
223
|
+
torch_version=torch_release(),
|
|
224
|
+
groups=tuple(
|
|
225
|
+
OptimizerParameterGroup(
|
|
226
|
+
parameter_names=g.parameter_names, options=g.options
|
|
227
|
+
)
|
|
228
|
+
for g in _groups(optimizer, model)
|
|
229
|
+
),
|
|
230
|
+
)
|
|
231
|
+
self.validate_checkpoint(
|
|
232
|
+
optimizer,
|
|
233
|
+
model,
|
|
234
|
+
spec,
|
|
235
|
+
optimizer.state_dict(),
|
|
236
|
+
logical_steps=0,
|
|
237
|
+
horizon=1,
|
|
238
|
+
schedule="constant",
|
|
239
|
+
)
|
|
240
|
+
return spec
|
|
241
|
+
|
|
242
|
+
def build(
|
|
243
|
+
self, model: nn.Module, spec: FactoryOptimizerSpec
|
|
244
|
+
) -> torch.optim.Optimizer:
|
|
245
|
+
"""Construct and validate a fresh public optimizer before privacy spending."""
|
|
246
|
+
optimizer = self.create(model)
|
|
247
|
+
self.check(
|
|
248
|
+
optimizer, model, spec, logical_steps=0, horizon=1, schedule="constant"
|
|
249
|
+
)
|
|
250
|
+
self.validate_checkpoint(
|
|
251
|
+
optimizer,
|
|
252
|
+
model,
|
|
253
|
+
spec,
|
|
254
|
+
optimizer.state_dict(),
|
|
255
|
+
logical_steps=0,
|
|
256
|
+
horizon=1,
|
|
257
|
+
schedule="constant",
|
|
258
|
+
)
|
|
259
|
+
return optimizer
|
|
260
|
+
|
|
261
|
+
def check(
|
|
262
|
+
self,
|
|
263
|
+
optimizer: torch.optim.Optimizer,
|
|
264
|
+
model: nn.Module,
|
|
265
|
+
spec: FactoryOptimizerSpec,
|
|
266
|
+
*,
|
|
267
|
+
logical_steps: int,
|
|
268
|
+
horizon: int,
|
|
269
|
+
schedule: str,
|
|
270
|
+
) -> None:
|
|
271
|
+
"""Reject changed optimizer identity, parameter mapping, options or rates."""
|
|
272
|
+
self._validate_type(optimizer)
|
|
273
|
+
if (
|
|
274
|
+
spec.implementation_id != _implementation_id(self.optimizer_type)
|
|
275
|
+
or spec.implementation_hash != self.implementation_hash
|
|
276
|
+
or spec.torch_version != torch_release()
|
|
277
|
+
):
|
|
278
|
+
raise TaskError(
|
|
279
|
+
"optimizer implementation differs from shared specification"
|
|
280
|
+
)
|
|
281
|
+
actual = _groups(optimizer, model)
|
|
282
|
+
if len(actual) != len(spec.groups):
|
|
283
|
+
raise TaskError("optimizer parameter groups changed")
|
|
284
|
+
for group, expected in zip(actual, spec.groups, strict=True):
|
|
285
|
+
rate = _rate(expected, logical_steps, horizon, schedule)
|
|
286
|
+
if (
|
|
287
|
+
group.parameter_names != expected.parameter_names
|
|
288
|
+
or not _matching_options(group.options, expected, rate)
|
|
289
|
+
):
|
|
290
|
+
raise TaskError(
|
|
291
|
+
"optimizer groups or options differ from shared specification"
|
|
292
|
+
)
|
|
293
|
+
|
|
294
|
+
def validate_checkpoint(
|
|
295
|
+
self,
|
|
296
|
+
optimizer: torch.optim.Optimizer,
|
|
297
|
+
model: nn.Module,
|
|
298
|
+
spec: FactoryOptimizerSpec,
|
|
299
|
+
state: dict[str, Any],
|
|
300
|
+
*,
|
|
301
|
+
logical_steps: int,
|
|
302
|
+
horizon: int,
|
|
303
|
+
schedule: str,
|
|
304
|
+
) -> None:
|
|
305
|
+
"""Validate restore candidates without mutating the live optimizer."""
|
|
306
|
+
self._validate_type(optimizer)
|
|
307
|
+
if (
|
|
308
|
+
spec.implementation_id != _implementation_id(self.optimizer_type)
|
|
309
|
+
or spec.implementation_hash != self.implementation_hash
|
|
310
|
+
or spec.torch_version != torch_release()
|
|
311
|
+
):
|
|
312
|
+
raise TaskError("optimizer checkpoint implementation mismatch")
|
|
313
|
+
actual = _groups(optimizer, model)
|
|
314
|
+
if tuple(g.parameter_names for g in actual) != tuple(
|
|
315
|
+
g.parameter_names for g in spec.groups
|
|
316
|
+
):
|
|
317
|
+
raise TaskError("optimizer checkpoint model mapping mismatch")
|
|
318
|
+
candidate = _numeric_copy(state)
|
|
319
|
+
if set(candidate) != {"state", "param_groups"}:
|
|
320
|
+
raise TaskError("invalid optimizer checkpoint fields")
|
|
321
|
+
groups = candidate["param_groups"]
|
|
322
|
+
reference = optimizer.state_dict()["param_groups"]
|
|
323
|
+
if not isinstance(groups, list) or len(cast(list[Any], groups)) != len(
|
|
324
|
+
spec.groups
|
|
325
|
+
):
|
|
326
|
+
raise TaskError("optimizer checkpoint group count mismatch")
|
|
327
|
+
known: set[int] = set()
|
|
328
|
+
for group, current, expected in zip(
|
|
329
|
+
cast(list[Any], groups), reference, spec.groups, strict=True
|
|
330
|
+
):
|
|
331
|
+
if not isinstance(group, dict) or set(cast(dict[str, Any], group)) != set(
|
|
332
|
+
current
|
|
333
|
+
):
|
|
334
|
+
raise TaskError("optimizer checkpoint group fields mismatch")
|
|
335
|
+
group = cast(dict[str, Any], group)
|
|
336
|
+
ids = group["params"]
|
|
337
|
+
if (
|
|
338
|
+
not isinstance(ids, list)
|
|
339
|
+
or any(type(index) is not int for index in cast(list[Any], ids))
|
|
340
|
+
or ids != current["params"]
|
|
341
|
+
):
|
|
342
|
+
raise TaskError("optimizer checkpoint parameter mapping mismatch")
|
|
343
|
+
known.update(cast(list[int], ids))
|
|
344
|
+
rate = _rate(expected, logical_steps, horizon, schedule)
|
|
345
|
+
if not _matching_options(
|
|
346
|
+
{k: v for k, v in group.items() if k != "params"}, expected, rate
|
|
347
|
+
):
|
|
348
|
+
raise TaskError("optimizer checkpoint options or schedule mismatch")
|
|
349
|
+
saved_state = candidate["state"]
|
|
350
|
+
if not isinstance(saved_state, dict) or any(
|
|
351
|
+
type(index) is not int or index not in known
|
|
352
|
+
for index in cast(dict[Any, Any], saved_state)
|
|
353
|
+
):
|
|
354
|
+
raise TaskError("optimizer checkpoint state references unknown parameters")
|
|
355
|
+
parameters = tuple(
|
|
356
|
+
parameter.detach().clone()
|
|
357
|
+
for group in optimizer.param_groups
|
|
358
|
+
for parameter in group["params"]
|
|
359
|
+
)
|
|
360
|
+
self.validate_state(candidate, parameters, logical_steps)
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def _matching_options(
|
|
364
|
+
actual: dict[str, Any], expected: OptimizerParameterGroup, rate: float
|
|
365
|
+
) -> bool:
|
|
366
|
+
current_rate = actual.get("lr")
|
|
367
|
+
return (
|
|
368
|
+
type(current_rate) in (int, float)
|
|
369
|
+
and math.isfinite(cast(float, current_rate))
|
|
370
|
+
and current_rate == rate
|
|
371
|
+
and _canonical_options({k: v for k, v in actual.items() if k != "lr"})
|
|
372
|
+
== _canonical_options({k: v for k, v in expected.options.items() if k != "lr"})
|
|
373
|
+
)
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def _rate(
|
|
377
|
+
group: OptimizerParameterGroup, steps: int, horizon: int, schedule: str
|
|
378
|
+
) -> float:
|
|
379
|
+
if type(steps) is not int or not 0 <= steps <= horizon or horizon <= 0:
|
|
380
|
+
raise TaskError("invalid optimizer schedule counters")
|
|
381
|
+
if schedule not in {"constant", "linear_decay"}:
|
|
382
|
+
raise TaskError("unsupported optimizer schedule")
|
|
383
|
+
rate = cast(float, group.options["lr"])
|
|
384
|
+
return rate * (1 - steps / horizon) if schedule == "linear_decay" else rate
|
|
385
|
+
|
|
386
|
+
|
|
387
|
+
class SanitizedOptimizer(torch.optim.Optimizer):
|
|
388
|
+
"""Expose sanitized gradients to the selected optimizer's update only.
|
|
389
|
+
|
|
390
|
+
Opacus owns clipping/noise/accumulation. Its private parameter attributes are
|
|
391
|
+
temporarily hidden during the update and restored even when the update fails.
|
|
392
|
+
This is a lifecycle guard for trusted code, not a malicious-code sandbox.
|
|
393
|
+
"""
|
|
394
|
+
|
|
395
|
+
def __init__(self, optimizer: torch.optim.Optimizer) -> None:
|
|
396
|
+
self.original_optimizer = optimizer
|
|
397
|
+
super().__init__(optimizer.param_groups, optimizer.defaults)
|
|
398
|
+
self.param_groups = optimizer.param_groups
|
|
399
|
+
self.state = optimizer.state
|
|
400
|
+
|
|
401
|
+
def step(self, closure: Callable[[], float] | None = None) -> Any:
|
|
402
|
+
if closure is not None:
|
|
403
|
+
raise TaskError("private optimizers cannot evaluate closures")
|
|
404
|
+
hidden: list[tuple[torch.Tensor, str, Any]] = []
|
|
405
|
+
try:
|
|
406
|
+
for group in self.param_groups:
|
|
407
|
+
for parameter in group["params"]:
|
|
408
|
+
for name in ("grad_sample", "summed_grad"):
|
|
409
|
+
if hasattr(parameter, name):
|
|
410
|
+
hidden.append((parameter, name, getattr(parameter, name)))
|
|
411
|
+
delattr(parameter, name)
|
|
412
|
+
return self.original_optimizer.step() # pyright: ignore[reportUnknownMemberType]
|
|
413
|
+
finally:
|
|
414
|
+
for parameter, name, value in hidden:
|
|
415
|
+
setattr(parameter, name, value)
|
|
416
|
+
|
|
417
|
+
def zero_grad(self, set_to_none: bool = True) -> None:
|
|
418
|
+
torch.optim.Optimizer.zero_grad(self, set_to_none=set_to_none)
|
|
419
|
+
|
|
420
|
+
def state_dict(self) -> dict[str, Any]:
|
|
421
|
+
return self.original_optimizer.state_dict()
|
|
422
|
+
|
|
423
|
+
def load_state_dict(self, state_dict: dict[str, Any]) -> None:
|
|
424
|
+
self.original_optimizer.load_state_dict(state_dict)
|
|
425
|
+
self.param_groups = self.original_optimizer.param_groups
|
|
426
|
+
self.state = self.original_optimizer.state
|