tabpackage 0.0.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tabpackage/__init__.py +18 -0
- tabpackage/_training.py +3380 -0
- tabpackage/ensemble.py +263 -0
- tabpackage/metrics.py +253 -0
- tabpackage/nn.py +2110 -0
- tabpackage/optim.py +943 -0
- tabpackage/sklearn/__init__.py +18 -0
- tabpackage/sklearn/_config.py +561 -0
- tabpackage/sklearn/_data.py +307 -0
- tabpackage/sklearn/_estimators.py +1210 -0
- tabpackage/utils/__init__.py +1 -0
- tabpackage/utils/torch.py +166 -0
- tabpackage-0.0.1.dist-info/METADATA +398 -0
- tabpackage-0.0.1.dist-info/RECORD +15 -0
- tabpackage-0.0.1.dist-info/WHEEL +4 -0
tabpackage/optim.py
ADDED
|
@@ -0,0 +1,943 @@
|
|
|
1
|
+
"""Optimizers, including the "optimizer pack" pattern.
|
|
2
|
+
|
|
3
|
+
An optimizer pack is applied to the members of a module pack from `tabpackage.nn` in
|
|
4
|
+
parallel, with independent optimizer states and potentially different hyperparameters.
|
|
5
|
+
The base class for optimizer packs is `OptimizerPack`.
|
|
6
|
+
|
|
7
|
+
Note:
|
|
8
|
+
Vanilla non-pack optimizers can be used to optimize module packs when all pack
|
|
9
|
+
members have the same optimization hyperparameters. This is why in many places here
|
|
10
|
+
and in other files, the type annotation for optimizers is `Optimizer` rather than
|
|
11
|
+
`OptimizerPack`, and utilities like those for removing or resetting pack members are
|
|
12
|
+
free functions rather than methods of `OptimizerPack`.
|
|
13
|
+
|
|
14
|
+
Using vanilla optimizers in the case of shared hyperparameters can accelerate
|
|
15
|
+
training due to their efficient implementation in PyTorch, particularly when
|
|
16
|
+
`foreach` or the currently experimental `fused` implementation is available, as with
|
|
17
|
+
AdamW. See the official PyTorch documentation for details.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
import functools
|
|
21
|
+
import math
|
|
22
|
+
import typing
|
|
23
|
+
import warnings
|
|
24
|
+
from collections.abc import Callable
|
|
25
|
+
from typing import Any, Final, TypeAlias, TypeVar
|
|
26
|
+
|
|
27
|
+
import torch
|
|
28
|
+
import torch.nn as nn # noqa: PLR0402
|
|
29
|
+
from torch import Tensor
|
|
30
|
+
from torch.optim.optimizer import ParamsT
|
|
31
|
+
|
|
32
|
+
import tabpackage.nn
|
|
33
|
+
from tabpackage.nn import PACK_DIM, ParameterPack
|
|
34
|
+
|
|
35
|
+
_TORCH_IS_OLDER_THAN_2_13 = (
|
|
36
|
+
torch.torch_version.TorchVersion(torch.__version__) < '2.13.0'
|
|
37
|
+
)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
ScalarGroupValue: TypeAlias = bool | int | float | str | bytes
|
|
41
|
+
NumericScalarGroupValue: TypeAlias = bool | int | float
|
|
42
|
+
NumericGroupValueList: TypeAlias = list[bool] | list[int] | list[float]
|
|
43
|
+
|
|
44
|
+
_T = TypeVar('_T')
|
|
45
|
+
_ScalarGroupValueT = TypeVar('_ScalarGroupValueT', bound=ScalarGroupValue)
|
|
46
|
+
_NumericScalarGroupValueT = TypeVar(
|
|
47
|
+
'_NumericScalarGroupValueT', bound=NumericScalarGroupValue
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def _check_lr(value: float) -> None:
|
|
52
|
+
if not math.isfinite(value) or value < 0.0:
|
|
53
|
+
raise ValueError('The learning rate must be finite and non-negative')
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def _check_momentum(value: float) -> None:
|
|
57
|
+
if not math.isfinite(value) or value < 0.0 or value >= 1.0:
|
|
58
|
+
raise ValueError(
|
|
59
|
+
'Momentum and moving-average decay coefficients must be finite and in the'
|
|
60
|
+
' range [0, 1)'
|
|
61
|
+
)
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def _check_weight_decay(value: float) -> None:
|
|
65
|
+
if not math.isfinite(value) or value < 0.0:
|
|
66
|
+
raise ValueError('The weight decay must be finite and non-negative')
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _check_eps(value: float) -> None:
|
|
70
|
+
if not math.isfinite(value) or value <= 0.0:
|
|
71
|
+
raise ValueError('The epsilon value must be finite and positive')
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
@typing.overload
|
|
75
|
+
def _make_group_value(
|
|
76
|
+
value: _ScalarGroupValueT, pack_size: int, device: torch.device | None
|
|
77
|
+
) -> _ScalarGroupValueT: ...
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@typing.overload
|
|
81
|
+
def _make_group_value(
|
|
82
|
+
value: list[_NumericScalarGroupValueT], pack_size: int, device: torch.device | None
|
|
83
|
+
) -> Tensor: ...
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@typing.overload
|
|
87
|
+
def _make_group_value(
|
|
88
|
+
value: Tensor, pack_size: int, device: torch.device | None
|
|
89
|
+
) -> Tensor: ...
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _make_group_value(value, pack_size: int, device: torch.device | None):
|
|
93
|
+
if value is None or isinstance(value, ScalarGroupValue):
|
|
94
|
+
return value
|
|
95
|
+
elif isinstance(value, list):
|
|
96
|
+
if len(value) != pack_size:
|
|
97
|
+
raise ValueError(
|
|
98
|
+
'The size of the hyperparameter list must match the pack size'
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
return torch.tensor(value, device=device)
|
|
102
|
+
elif isinstance(value, Tensor):
|
|
103
|
+
if value.ndim == 0:
|
|
104
|
+
raise ValueError(
|
|
105
|
+
'Tensors can be used only to specify per-member hyperparameters; use a'
|
|
106
|
+
' Python scalar to share one value across all members'
|
|
107
|
+
)
|
|
108
|
+
if len(value) != pack_size:
|
|
109
|
+
raise ValueError(
|
|
110
|
+
'The first dimension of a hyperparameter tensor must match the pack'
|
|
111
|
+
' size'
|
|
112
|
+
)
|
|
113
|
+
|
|
114
|
+
# Copy tensors so groups do not share mutable hyperparameter storage.
|
|
115
|
+
return value.to(device, copy=True)
|
|
116
|
+
else:
|
|
117
|
+
raise TypeError(f'Unsupported value type: {type(value)}')
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _get_device(optimizer: torch.optim.Optimizer) -> torch.device | None:
|
|
121
|
+
for group in optimizer.param_groups:
|
|
122
|
+
for p in group['params']:
|
|
123
|
+
return p.device
|
|
124
|
+
return None
|
|
125
|
+
|
|
126
|
+
|
|
127
|
+
def _get_pack_size(optimizer: torch.optim.Optimizer) -> int | None:
|
|
128
|
+
for group in optimizer.param_groups:
|
|
129
|
+
for p in group['params']:
|
|
130
|
+
if isinstance(p, ParameterPack):
|
|
131
|
+
return p.shape[PACK_DIM]
|
|
132
|
+
return None
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _make_defaults(
|
|
136
|
+
keys_values_checks: list[
|
|
137
|
+
tuple[str, ScalarGroupValue | NumericGroupValueList, Callable[..., None]]
|
|
138
|
+
],
|
|
139
|
+
*,
|
|
140
|
+
pack_size: int,
|
|
141
|
+
) -> dict[str, Any]:
|
|
142
|
+
defaults = {}
|
|
143
|
+
for key, value, check_fn in keys_values_checks:
|
|
144
|
+
if isinstance(value, list):
|
|
145
|
+
if len(value) != pack_size:
|
|
146
|
+
raise ValueError(
|
|
147
|
+
'The size of each hyperparameter list must match the pack size'
|
|
148
|
+
)
|
|
149
|
+
|
|
150
|
+
for x in value:
|
|
151
|
+
check_fn(x)
|
|
152
|
+
else:
|
|
153
|
+
check_fn(value)
|
|
154
|
+
defaults[key] = value
|
|
155
|
+
return defaults
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
class OptimizerPack(torch.optim.Optimizer):
|
|
159
|
+
"""The base class for optimizer packs."""
|
|
160
|
+
|
|
161
|
+
def __init__(
|
|
162
|
+
self, params: ParamsT, defaults: dict[str, Any], *, pack_size: int
|
|
163
|
+
) -> None:
|
|
164
|
+
super().__init__(params, defaults)
|
|
165
|
+
for key, value in list(self.defaults.items()):
|
|
166
|
+
self.defaults[key] = _make_group_value(value, pack_size, _get_device(self))
|
|
167
|
+
|
|
168
|
+
def add_param_group(self, param_group):
|
|
169
|
+
output = super().add_param_group(param_group)
|
|
170
|
+
group = self.param_groups[-1]
|
|
171
|
+
|
|
172
|
+
group_params = group['params']
|
|
173
|
+
if not group_params:
|
|
174
|
+
# Otherwise, it will be impossible to infer the device and pack size below.
|
|
175
|
+
raise ValueError('Empty parameter groups are not supported')
|
|
176
|
+
|
|
177
|
+
device = None
|
|
178
|
+
pack_size = None
|
|
179
|
+
for p in group['params']:
|
|
180
|
+
if not isinstance(p, ParameterPack):
|
|
181
|
+
raise TypeError('For now, only parameter packs are supported')
|
|
182
|
+
|
|
183
|
+
if device is None:
|
|
184
|
+
device = p.device
|
|
185
|
+
pack_size = p.shape[PACK_DIM]
|
|
186
|
+
assert device is not None
|
|
187
|
+
assert pack_size is not None
|
|
188
|
+
|
|
189
|
+
for key, value in list(group.items()):
|
|
190
|
+
if key != 'params':
|
|
191
|
+
group[key] = _make_group_value(value, pack_size, device)
|
|
192
|
+
|
|
193
|
+
return output
|
|
194
|
+
|
|
195
|
+
def _replace_impl(
|
|
196
|
+
self,
|
|
197
|
+
*,
|
|
198
|
+
pack_idx: Tensor,
|
|
199
|
+
keys_values_checks: list[
|
|
200
|
+
tuple[str, NumericGroupValueList | None, Callable[..., None]]
|
|
201
|
+
],
|
|
202
|
+
) -> None:
|
|
203
|
+
tabpackage.nn.check_pack_idx(pack_idx, _get_pack_size(self))
|
|
204
|
+
|
|
205
|
+
for _, value, check_fn in keys_values_checks:
|
|
206
|
+
if value is not None:
|
|
207
|
+
if len(value) != len(pack_idx):
|
|
208
|
+
raise ValueError(
|
|
209
|
+
'Each replacement hyperparameter list must contain one value'
|
|
210
|
+
' per selected pack member'
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
for x in value:
|
|
214
|
+
check_fn(x)
|
|
215
|
+
|
|
216
|
+
for key, value, _ in keys_values_checks:
|
|
217
|
+
if value is None:
|
|
218
|
+
continue
|
|
219
|
+
|
|
220
|
+
value_tensor = None
|
|
221
|
+
replaced = False
|
|
222
|
+
for group in self.param_groups:
|
|
223
|
+
current_value = group[key]
|
|
224
|
+
if isinstance(current_value, Tensor):
|
|
225
|
+
if value_tensor is None:
|
|
226
|
+
value_tensor = torch.tensor(
|
|
227
|
+
value,
|
|
228
|
+
dtype=current_value.dtype,
|
|
229
|
+
device=current_value.device,
|
|
230
|
+
)
|
|
231
|
+
current_value[pack_idx] = value_tensor
|
|
232
|
+
replaced = True
|
|
233
|
+
|
|
234
|
+
if not replaced:
|
|
235
|
+
raise RuntimeError(f'Replacement is not possible for "{key}"')
|
|
236
|
+
|
|
237
|
+
optimizer_pack_reset_state(self, pack_idx=pack_idx)
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _make_weight_decay_multiplier(
|
|
241
|
+
*, lr: float | Tensor, weight_decay: float | Tensor
|
|
242
|
+
) -> float | Tensor | None:
|
|
243
|
+
return (
|
|
244
|
+
None
|
|
245
|
+
if (
|
|
246
|
+
(isinstance(lr, float) and lr == 0.0)
|
|
247
|
+
or (isinstance(weight_decay, float) and weight_decay == 0.0)
|
|
248
|
+
)
|
|
249
|
+
else (1 - lr * weight_decay)
|
|
250
|
+
)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
@typing.overload
|
|
254
|
+
def _maybe_unsqueeze(value: Tensor, *, p: Tensor) -> Tensor: ...
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
@typing.overload
|
|
258
|
+
def _maybe_unsqueeze(value: _T, *, p: Tensor) -> _T: ...
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
def _maybe_unsqueeze(value, *, p):
|
|
262
|
+
return (
|
|
263
|
+
value[(slice(None), *((None,) * (p.ndim - 1)))]
|
|
264
|
+
if isinstance(value, Tensor)
|
|
265
|
+
else value
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
class AdamWPack(OptimizerPack):
|
|
270
|
+
"""A pack of AdamW optimizers."""
|
|
271
|
+
|
|
272
|
+
def __init__(
|
|
273
|
+
self,
|
|
274
|
+
params: ParamsT,
|
|
275
|
+
*,
|
|
276
|
+
lr: float | list[float],
|
|
277
|
+
# Unlike in `torch.optim.AdamW`, `beta1` and `beta2` are separate arguments,
|
|
278
|
+
# making it easy to specify different values for each pack member by passing
|
|
279
|
+
# lists, as with all other scalar hyperparameters.
|
|
280
|
+
beta1: float | list[float] = 0.9,
|
|
281
|
+
beta2: float | list[float] = 0.999,
|
|
282
|
+
eps: float | list[float] = 1e-8,
|
|
283
|
+
weight_decay: float | list[float],
|
|
284
|
+
pack_size: int,
|
|
285
|
+
support_replacement: bool = False,
|
|
286
|
+
):
|
|
287
|
+
"""
|
|
288
|
+
Args:
|
|
289
|
+
support_replacement: if `True`, the optimizer supports selective replacement
|
|
290
|
+
of current members with new ones. In this case, the underlying step
|
|
291
|
+
counter is not an integer scalar anymore, but a tensor with per-member
|
|
292
|
+
steps, which makes the optimization step a bit slower, and changes the
|
|
293
|
+
results numerically (even if replacements never occur).
|
|
294
|
+
"""
|
|
295
|
+
if pack_size <= 0:
|
|
296
|
+
raise ValueError('The pack size must be positive')
|
|
297
|
+
|
|
298
|
+
super().__init__(
|
|
299
|
+
params,
|
|
300
|
+
_make_defaults(
|
|
301
|
+
[
|
|
302
|
+
('lr', lr, _check_lr),
|
|
303
|
+
('beta1', beta1, _check_momentum),
|
|
304
|
+
('beta2', beta2, _check_momentum),
|
|
305
|
+
('eps', eps, _check_eps),
|
|
306
|
+
('weight_decay', weight_decay, _check_weight_decay),
|
|
307
|
+
],
|
|
308
|
+
pack_size=pack_size,
|
|
309
|
+
),
|
|
310
|
+
pack_size=pack_size,
|
|
311
|
+
)
|
|
312
|
+
|
|
313
|
+
self._support_replacement: Final = support_replacement
|
|
314
|
+
|
|
315
|
+
def replace(
|
|
316
|
+
self,
|
|
317
|
+
*,
|
|
318
|
+
pack_idx: Tensor,
|
|
319
|
+
lr: list[float] | None = None,
|
|
320
|
+
beta1: list[float] | None = None,
|
|
321
|
+
beta2: list[float] | None = None,
|
|
322
|
+
eps: list[float] | None = None,
|
|
323
|
+
weight_decay: list[float] | None = None,
|
|
324
|
+
) -> None:
|
|
325
|
+
"""Replace selected members with new ones.
|
|
326
|
+
|
|
327
|
+
The method resets the state of the selected optimizers and optionally changes
|
|
328
|
+
their hyperparameters.
|
|
329
|
+
"""
|
|
330
|
+
if not self._support_replacement:
|
|
331
|
+
raise RuntimeError(
|
|
332
|
+
'Replacing optimizer pack members requires constructing the optimizer'
|
|
333
|
+
' with `support_replacement=True`'
|
|
334
|
+
)
|
|
335
|
+
|
|
336
|
+
self._replace_impl(
|
|
337
|
+
pack_idx=pack_idx,
|
|
338
|
+
keys_values_checks=[
|
|
339
|
+
('lr', lr, _check_lr),
|
|
340
|
+
('beta1', beta1, _check_momentum),
|
|
341
|
+
('beta2', beta2, _check_momentum),
|
|
342
|
+
('eps', eps, _check_eps),
|
|
343
|
+
('weight_decay', weight_decay, _check_weight_decay),
|
|
344
|
+
],
|
|
345
|
+
)
|
|
346
|
+
|
|
347
|
+
@torch.no_grad()
|
|
348
|
+
def step(self, closure: Callable[[], Tensor] | None = None) -> Tensor | None: # ty: ignore[invalid-method-override]
|
|
349
|
+
"""Perform a single optimization step for all pack members."""
|
|
350
|
+
loss = None
|
|
351
|
+
if closure is not None:
|
|
352
|
+
loss = closure()
|
|
353
|
+
|
|
354
|
+
for group in self.param_groups:
|
|
355
|
+
lr = group['lr']
|
|
356
|
+
beta1 = group['beta1']
|
|
357
|
+
beta2 = group['beta2']
|
|
358
|
+
eps = group['eps']
|
|
359
|
+
|
|
360
|
+
weight_decay_multiplier = _make_weight_decay_multiplier(
|
|
361
|
+
lr=lr, weight_decay=group['weight_decay']
|
|
362
|
+
)
|
|
363
|
+
|
|
364
|
+
for p in group['params']:
|
|
365
|
+
assert isinstance(p, ParameterPack), (
|
|
366
|
+
'For now, only parameter packs are supported'
|
|
367
|
+
)
|
|
368
|
+
if p.grad is None:
|
|
369
|
+
continue
|
|
370
|
+
|
|
371
|
+
grad = p.grad.data
|
|
372
|
+
if grad.is_sparse:
|
|
373
|
+
raise RuntimeError('Sparse gradients are not supported')
|
|
374
|
+
|
|
375
|
+
maybe_unsqueeze = functools.partial(_maybe_unsqueeze, p=p)
|
|
376
|
+
|
|
377
|
+
state = self.state[p]
|
|
378
|
+
if len(state) == 0:
|
|
379
|
+
# Initialize the state.
|
|
380
|
+
state['step'] = (
|
|
381
|
+
torch.zeros(
|
|
382
|
+
p.shape[PACK_DIM],
|
|
383
|
+
dtype=torch.int64,
|
|
384
|
+
device=p.device,
|
|
385
|
+
)
|
|
386
|
+
if self._support_replacement
|
|
387
|
+
else 0
|
|
388
|
+
)
|
|
389
|
+
state['exp_avg'] = torch.zeros_like(p.data)
|
|
390
|
+
state['exp_avg_sq'] = torch.zeros_like(p.data)
|
|
391
|
+
exp_avg = state['exp_avg']
|
|
392
|
+
exp_avg_sq = state['exp_avg_sq']
|
|
393
|
+
|
|
394
|
+
state['step'] += 1
|
|
395
|
+
|
|
396
|
+
if weight_decay_multiplier is not None:
|
|
397
|
+
p.mul_(maybe_unsqueeze(weight_decay_multiplier))
|
|
398
|
+
|
|
399
|
+
# Update biased first moment estimate.
|
|
400
|
+
exp_avg.lerp_(grad, maybe_unsqueeze(1 - beta1))
|
|
401
|
+
|
|
402
|
+
# Update biased second raw moment estimate.
|
|
403
|
+
if isinstance(beta2, float):
|
|
404
|
+
exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
|
|
405
|
+
else:
|
|
406
|
+
exp_avg_sq.lerp_(grad.square(), maybe_unsqueeze(1 - beta2))
|
|
407
|
+
|
|
408
|
+
# Perform the bias correction.
|
|
409
|
+
bias_correction1 = 1 - beta1 ** state['step']
|
|
410
|
+
bias_correction2 = 1 - beta2 ** state['step']
|
|
411
|
+
|
|
412
|
+
step_size = lr / bias_correction1
|
|
413
|
+
|
|
414
|
+
if isinstance(bias_correction2, float):
|
|
415
|
+
denom = exp_avg_sq.sqrt() / bias_correction2**0.5
|
|
416
|
+
else:
|
|
417
|
+
denom = exp_avg_sq.sqrt().div_(
|
|
418
|
+
maybe_unsqueeze(bias_correction2**0.5)
|
|
419
|
+
)
|
|
420
|
+
denom.add_(maybe_unsqueeze(eps))
|
|
421
|
+
|
|
422
|
+
if isinstance(step_size, float):
|
|
423
|
+
p.addcdiv_(exp_avg, denom, value=-step_size)
|
|
424
|
+
else:
|
|
425
|
+
p.add_((exp_avg / denom).mul_(maybe_unsqueeze(step_size)), alpha=-1)
|
|
426
|
+
|
|
427
|
+
return loss
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
def compute_muon_update_scale_pack(module: tabpackage.nn.LinearPack) -> Tensor | None:
|
|
431
|
+
"""Compute Muon update scaling coefficients for a pack of linear layers."""
|
|
432
|
+
if module.in_features is None and module.out_features is None:
|
|
433
|
+
return None
|
|
434
|
+
|
|
435
|
+
if module.in_features is None:
|
|
436
|
+
assert module.out_features is not None
|
|
437
|
+
pack_size = module.out_features.shape[PACK_DIM]
|
|
438
|
+
device = module.out_features.device
|
|
439
|
+
else:
|
|
440
|
+
pack_size = module.in_features.shape[PACK_DIM]
|
|
441
|
+
device = module.in_features.device
|
|
442
|
+
|
|
443
|
+
out_f = (
|
|
444
|
+
torch.full((pack_size,), float(module.max_out_features), device=device)
|
|
445
|
+
if module.out_features is None
|
|
446
|
+
else module.out_features.float()
|
|
447
|
+
)
|
|
448
|
+
in_f = (
|
|
449
|
+
torch.full((pack_size,), float(module.max_in_features), device=device)
|
|
450
|
+
if module.in_features is None
|
|
451
|
+
else module.in_features.float()
|
|
452
|
+
)
|
|
453
|
+
return (out_f / in_f).clamp_(min=1.0).sqrt_()
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
# The following function is a modified version of this function:
|
|
457
|
+
# https://github.com/KellerJordan/Muon/blob/f98f1cacc0263b04290753e32be8d498c1efc806/muon.py#L5
|
|
458
|
+
#
|
|
459
|
+
# The modification consists in using `torch.baddbmm` for better efficiency. This
|
|
460
|
+
# requires `G` to be three-dimensional.
|
|
461
|
+
#
|
|
462
|
+
# fmt: off
|
|
463
|
+
# ruff: disable[E501]
|
|
464
|
+
def _zeropower_via_newtonschulz5(G, steps: int):
|
|
465
|
+
"""
|
|
466
|
+
Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
|
|
467
|
+
quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
|
|
468
|
+
of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
|
|
469
|
+
zero even beyond the point where the iteration no longer converges all the way to one everywhere
|
|
470
|
+
on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
|
|
471
|
+
where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
|
|
472
|
+
performance at all relative to UV^T, where USV^T = G is the SVD.
|
|
473
|
+
"""
|
|
474
|
+
assert G.ndim == 3 # batched Muon implementation by @scottjmaddox, and put into practice in the record by @YouJiacheng
|
|
475
|
+
a, b, c = (3.4445, -4.7750, 2.0315)
|
|
476
|
+
X = G.bfloat16()
|
|
477
|
+
if G.size(-2) > G.size(-1):
|
|
478
|
+
X = X.mT
|
|
479
|
+
|
|
480
|
+
# Ensure spectral norm is at most 1
|
|
481
|
+
X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)
|
|
482
|
+
# Perform the NS iterations
|
|
483
|
+
for _ in range(steps):
|
|
484
|
+
A = X @ X.mT
|
|
485
|
+
B = torch.baddbmm(A, A, A, beta=b, alpha=c) # quintic computation strategy adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
|
|
486
|
+
X = torch.baddbmm(X, B, X, beta=a)
|
|
487
|
+
|
|
488
|
+
if G.size(-2) > G.size(-1):
|
|
489
|
+
X = X.mT
|
|
490
|
+
return X
|
|
491
|
+
# ruff: enable[E501]
|
|
492
|
+
# fmt: on
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
class MuonAdamWPack(OptimizerPack):
|
|
496
|
+
"""A pack of optimizers combining Muon and AdamW.
|
|
497
|
+
|
|
498
|
+
Muon is intended for weights (excluding biases) of backbones' hidden linear layers,
|
|
499
|
+
and AdamW for everything else. Parameter groups optimized with Muon must be created
|
|
500
|
+
with `make_muon_parameter_group` (see the note below for details).
|
|
501
|
+
|
|
502
|
+
Note:
|
|
503
|
+
For parameter packs optimized with Muon, the spectral correction coefficients
|
|
504
|
+
must be provided explicitly, because they depend on the actual per-model
|
|
505
|
+
parameter shapes, which cannot be inferred from parameter packs alone. As a
|
|
506
|
+
consequence, parameter packs with different per-model shapes cannot be
|
|
507
|
+
optimized with Muon within one parameter group, because they will have different
|
|
508
|
+
coefficients. To make the optimizer easier to use correctly, all Muon parameter
|
|
509
|
+
groups are currently required to have no more than one parameter.
|
|
510
|
+
|
|
511
|
+
"""
|
|
512
|
+
|
|
513
|
+
@staticmethod
|
|
514
|
+
def make_muon_parameter_group(module: tabpackage.nn.LinearPack) -> dict[str, Any]:
|
|
515
|
+
"""Create a single-parameter group to be optimized with Muon."""
|
|
516
|
+
if (
|
|
517
|
+
module.max_in_features > module.max_out_features
|
|
518
|
+
and module.max_out_features < 16
|
|
519
|
+
):
|
|
520
|
+
warnings.warn(
|
|
521
|
+
f'The pack of linear layers has shape {module.weight.shape}, which'
|
|
522
|
+
' looks more like an output layer (should not be optimized with Muon)'
|
|
523
|
+
' than a backbone layer (can be optimized with Muon). Is this'
|
|
524
|
+
' intentional?',
|
|
525
|
+
stacklevel=2,
|
|
526
|
+
)
|
|
527
|
+
|
|
528
|
+
return {
|
|
529
|
+
'params': [module.weight],
|
|
530
|
+
'muon': True,
|
|
531
|
+
'muon_update_scale': compute_muon_update_scale_pack(module),
|
|
532
|
+
}
|
|
533
|
+
|
|
534
|
+
def __init__(
|
|
535
|
+
self,
|
|
536
|
+
params: ParamsT,
|
|
537
|
+
*,
|
|
538
|
+
adamw_lr: float | list[float],
|
|
539
|
+
adamw_beta1: float | list[float] = 0.9,
|
|
540
|
+
adamw_beta2: float | list[float] = 0.999,
|
|
541
|
+
adamw_eps: float | list[float] = 1e-8,
|
|
542
|
+
weight_decay: float | list[float],
|
|
543
|
+
muon_lr: float | list[float],
|
|
544
|
+
muon_momentum: float | list[float] = 0.95,
|
|
545
|
+
muon_newton_schulz_steps: int = 5,
|
|
546
|
+
muon_nesterov: bool = True,
|
|
547
|
+
muon_skip_zero_updates: bool = False,
|
|
548
|
+
pack_size: int,
|
|
549
|
+
support_replacement: bool = False,
|
|
550
|
+
# In practice, the following argument should usually be `True` for better
|
|
551
|
+
# efficiency unless the original parameter gradients must be preserved.
|
|
552
|
+
allow_inplace_grad_mutation: bool,
|
|
553
|
+
compile_muon_newton_schulz: bool = False,
|
|
554
|
+
):
|
|
555
|
+
"""
|
|
556
|
+
Args:
|
|
557
|
+
muon_skip_zero_updates: if `True`, the members with all-zero updates are
|
|
558
|
+
excluded from the Newton-Schulz iterations. On accelerators, this costs
|
|
559
|
+
a device-to-host synchronization per step for each Muon parameter group
|
|
560
|
+
with a gradient. This can speed up training of large packs of
|
|
561
|
+
depth-varying MLPs.
|
|
562
|
+
support_replacement: same as in `AdamWPack`.
|
|
563
|
+
"""
|
|
564
|
+
if pack_size <= 0:
|
|
565
|
+
raise ValueError('The pack size must be positive')
|
|
566
|
+
|
|
567
|
+
if (
|
|
568
|
+
compile_muon_newton_schulz
|
|
569
|
+
and torch.torch_version.TorchVersion(torch.__version__) < '2.12.0'
|
|
570
|
+
):
|
|
571
|
+
raise RuntimeError(
|
|
572
|
+
'Compiling the Newton-Schulz iteration for Muon requires PyTorch 2.12.0'
|
|
573
|
+
' or newer'
|
|
574
|
+
)
|
|
575
|
+
|
|
576
|
+
super().__init__(
|
|
577
|
+
params,
|
|
578
|
+
{
|
|
579
|
+
'muon': False,
|
|
580
|
+
'muon_newton_schulz_steps': muon_newton_schulz_steps,
|
|
581
|
+
'muon_nesterov': muon_nesterov,
|
|
582
|
+
'muon_skip_zero_updates': muon_skip_zero_updates,
|
|
583
|
+
**_make_defaults(
|
|
584
|
+
[
|
|
585
|
+
('adamw_lr', adamw_lr, _check_lr),
|
|
586
|
+
('adamw_beta1', adamw_beta1, _check_momentum),
|
|
587
|
+
('adamw_beta2', adamw_beta2, _check_momentum),
|
|
588
|
+
('adamw_eps', adamw_eps, _check_eps),
|
|
589
|
+
('weight_decay', weight_decay, _check_weight_decay),
|
|
590
|
+
('muon_lr', muon_lr, _check_lr),
|
|
591
|
+
('muon_momentum', muon_momentum, _check_momentum),
|
|
592
|
+
],
|
|
593
|
+
pack_size=pack_size,
|
|
594
|
+
),
|
|
595
|
+
},
|
|
596
|
+
pack_size=pack_size,
|
|
597
|
+
)
|
|
598
|
+
|
|
599
|
+
self._support_replacement: Final = support_replacement
|
|
600
|
+
self._allow_inplace_grad_mutation: Final = allow_inplace_grad_mutation
|
|
601
|
+
self._zeropower_via_newtonschulz5 = (
|
|
602
|
+
torch.compile(
|
|
603
|
+
_zeropower_via_newtonschulz5,
|
|
604
|
+
fullgraph=True,
|
|
605
|
+
# Emulate eager precision casts to reduce numerical differences from
|
|
606
|
+
# the non-compiled variant.
|
|
607
|
+
options={'emulate_precision_casts': True},
|
|
608
|
+
)
|
|
609
|
+
if compile_muon_newton_schulz
|
|
610
|
+
else _zeropower_via_newtonschulz5
|
|
611
|
+
)
|
|
612
|
+
|
|
613
|
+
def add_param_group(self, param_group):
|
|
614
|
+
output = super().add_param_group(param_group)
|
|
615
|
+
group = self.param_groups[-1]
|
|
616
|
+
if group['muon']:
|
|
617
|
+
if 'muon_update_scale' not in group:
|
|
618
|
+
raise ValueError(
|
|
619
|
+
'The just added parameter group is configured to be optimized with'
|
|
620
|
+
' Muon. In this case, it must have the "muon_update_scale" key'
|
|
621
|
+
' with the value computed with the'
|
|
622
|
+
' `compute_muon_update_scale_pack` function.'
|
|
623
|
+
)
|
|
624
|
+
if len(group['params']) > 1:
|
|
625
|
+
raise ValueError(
|
|
626
|
+
'The just added parameter group configured to be optimized with'
|
|
627
|
+
' Muon has more than one parameter, which is not allowed'
|
|
628
|
+
)
|
|
629
|
+
return output
|
|
630
|
+
|
|
631
|
+
def replace(
|
|
632
|
+
self,
|
|
633
|
+
*,
|
|
634
|
+
pack_idx: Tensor,
|
|
635
|
+
adamw_lr: list[float] | None = None,
|
|
636
|
+
adamw_beta1: list[float] | None = None,
|
|
637
|
+
adamw_beta2: list[float] | None = None,
|
|
638
|
+
adamw_eps: list[float] | None = None,
|
|
639
|
+
weight_decay: list[float] | None = None,
|
|
640
|
+
muon_lr: list[float] | None = None,
|
|
641
|
+
muon_momentum: list[float] | None = None,
|
|
642
|
+
module: nn.Module,
|
|
643
|
+
) -> None:
|
|
644
|
+
"""Replace selected members with new ones.
|
|
645
|
+
|
|
646
|
+
See `AdamWPack.replace` for additiona details.
|
|
647
|
+
|
|
648
|
+
Important:
|
|
649
|
+
`module` must undergo its member replacement _before_ this method is called,
|
|
650
|
+
because it is used to recompute the new Muon update scales.
|
|
651
|
+
"""
|
|
652
|
+
if not self._support_replacement:
|
|
653
|
+
raise RuntimeError(
|
|
654
|
+
'Replacing optimizer pack members requires constructing the optimizer'
|
|
655
|
+
' with `support_replacement=True`'
|
|
656
|
+
)
|
|
657
|
+
|
|
658
|
+
self._replace_impl(
|
|
659
|
+
pack_idx=pack_idx,
|
|
660
|
+
keys_values_checks=[
|
|
661
|
+
('adamw_lr', adamw_lr, _check_lr),
|
|
662
|
+
('adamw_beta1', adamw_beta1, _check_momentum),
|
|
663
|
+
('adamw_beta2', adamw_beta2, _check_momentum),
|
|
664
|
+
('adamw_eps', adamw_eps, _check_eps),
|
|
665
|
+
('weight_decay', weight_decay, _check_weight_decay),
|
|
666
|
+
('muon_lr', muon_lr, _check_lr),
|
|
667
|
+
('muon_momentum', muon_momentum, _check_momentum),
|
|
668
|
+
],
|
|
669
|
+
)
|
|
670
|
+
|
|
671
|
+
muon_parameter_to_module = {
|
|
672
|
+
x.weight: x
|
|
673
|
+
for x in module.modules()
|
|
674
|
+
if isinstance(x, tabpackage.nn.LinearPack)
|
|
675
|
+
}
|
|
676
|
+
for group in self.param_groups:
|
|
677
|
+
if group['muon']:
|
|
678
|
+
group['muon_update_scale'] = compute_muon_update_scale_pack(
|
|
679
|
+
muon_parameter_to_module[group['params'][0]]
|
|
680
|
+
)
|
|
681
|
+
|
|
682
|
+
def _step_muon(self, group: dict[str, Any]) -> None:
|
|
683
|
+
lr = group['muon_lr']
|
|
684
|
+
momentum = group['muon_momentum']
|
|
685
|
+
weight_decay = group['weight_decay']
|
|
686
|
+
newton_schulz_steps = group['muon_newton_schulz_steps']
|
|
687
|
+
nesterov = group['muon_nesterov']
|
|
688
|
+
skip_zero_updates = group['muon_skip_zero_updates']
|
|
689
|
+
update_scale = group['muon_update_scale']
|
|
690
|
+
|
|
691
|
+
weight_decay_multiplier = _make_weight_decay_multiplier(
|
|
692
|
+
lr=lr, weight_decay=weight_decay
|
|
693
|
+
)
|
|
694
|
+
|
|
695
|
+
for p in group['params']:
|
|
696
|
+
if p.ndim != 3: # 1 pack dimension + 2 layer dimensions.
|
|
697
|
+
raise ValueError(
|
|
698
|
+
'Parameters optimized with Muon must have three dimensions'
|
|
699
|
+
)
|
|
700
|
+
|
|
701
|
+
assert isinstance(p, ParameterPack), (
|
|
702
|
+
'For now, only parameter packs are supported'
|
|
703
|
+
)
|
|
704
|
+
if p.grad is None:
|
|
705
|
+
continue
|
|
706
|
+
|
|
707
|
+
grad = p.grad.data
|
|
708
|
+
if grad.is_sparse:
|
|
709
|
+
raise RuntimeError('Sparse gradients are not supported')
|
|
710
|
+
|
|
711
|
+
maybe_unsqueeze = functools.partial(_maybe_unsqueeze, p=p)
|
|
712
|
+
|
|
713
|
+
state = self.state[p]
|
|
714
|
+
if len(state) == 0:
|
|
715
|
+
# Initialize the state.
|
|
716
|
+
state['muon_momentum_buffer'] = torch.zeros_like(p)
|
|
717
|
+
momentum_buffer: Tensor = state['muon_momentum_buffer']
|
|
718
|
+
|
|
719
|
+
if weight_decay_multiplier is not None:
|
|
720
|
+
p.mul_(maybe_unsqueeze(weight_decay_multiplier))
|
|
721
|
+
|
|
722
|
+
momentum_buffer.lerp_(grad, maybe_unsqueeze(1 - momentum))
|
|
723
|
+
update = (
|
|
724
|
+
(grad.lerp_ if self._allow_inplace_grad_mutation else grad.lerp)(
|
|
725
|
+
momentum_buffer, maybe_unsqueeze(momentum)
|
|
726
|
+
)
|
|
727
|
+
if nesterov
|
|
728
|
+
else momentum_buffer
|
|
729
|
+
)
|
|
730
|
+
if skip_zero_updates:
|
|
731
|
+
# All-zero updates do not contribute to the optimization step, and
|
|
732
|
+
# therefore can be safely skipped without participating in the
|
|
733
|
+
# expensive Newton-Schulz iterations. The cost of applying this
|
|
734
|
+
# heuristic is the device-to-host synchronization coming from
|
|
735
|
+
# `torch.nonzero`. It could be avoided by incorporating additional
|
|
736
|
+
# knowledge about the optimized module into the optimizer's state,
|
|
737
|
+
# however, that would make the code significantly more complicated and
|
|
738
|
+
# brittle. The current simple approach already works well.
|
|
739
|
+
assert PACK_DIM == 0
|
|
740
|
+
active_pack_idx = torch.nonzero(update.flatten(1).any(1))[:, 0]
|
|
741
|
+
if len(active_pack_idx) == 0:
|
|
742
|
+
# All members have all-zero updates, and weight decay and momentum
|
|
743
|
+
# updates have already been applied.
|
|
744
|
+
continue
|
|
745
|
+
else:
|
|
746
|
+
active_pack_idx = None
|
|
747
|
+
if active_pack_idx is None or len(active_pack_idx) == len(update):
|
|
748
|
+
update = self._zeropower_via_newtonschulz5(
|
|
749
|
+
update, steps=newton_schulz_steps
|
|
750
|
+
)
|
|
751
|
+
else:
|
|
752
|
+
active_update = self._zeropower_via_newtonschulz5(
|
|
753
|
+
update.index_select(0, active_pack_idx), steps=newton_schulz_steps
|
|
754
|
+
)
|
|
755
|
+
update = torch.zeros_like(update, dtype=active_update.dtype)
|
|
756
|
+
if update.device.type == 'mps' and _TORCH_IS_OLDER_THAN_2_13:
|
|
757
|
+
# The `index_copy_` operation on MPS:
|
|
758
|
+
# - Fallbacks on CPU prior to PyTorch 2.8.
|
|
759
|
+
# - Is inefficient prior to PyTorch 2.13.
|
|
760
|
+
update[active_pack_idx] = active_update
|
|
761
|
+
else:
|
|
762
|
+
update.index_copy_(0, active_pack_idx, active_update)
|
|
763
|
+
|
|
764
|
+
scaled_lr = lr * (
|
|
765
|
+
max(1, grad.size(-2) / grad.size(-1)) ** 0.5
|
|
766
|
+
if update_scale is None
|
|
767
|
+
else update_scale
|
|
768
|
+
)
|
|
769
|
+
|
|
770
|
+
assert update.shape == p.shape
|
|
771
|
+
if isinstance(scaled_lr, float):
|
|
772
|
+
p.add_(update, alpha=-scaled_lr)
|
|
773
|
+
else:
|
|
774
|
+
p.addcmul_(update, maybe_unsqueeze(scaled_lr), value=-1)
|
|
775
|
+
|
|
776
|
+
def _step_adamw(self, group: dict[str, Any]) -> None:
|
|
777
|
+
lr = group['adamw_lr']
|
|
778
|
+
beta1 = group['adamw_beta1']
|
|
779
|
+
beta2 = group['adamw_beta2']
|
|
780
|
+
eps = group['adamw_eps']
|
|
781
|
+
weight_decay = group['weight_decay']
|
|
782
|
+
|
|
783
|
+
weight_decay_multiplier = _make_weight_decay_multiplier(
|
|
784
|
+
lr=lr, weight_decay=weight_decay
|
|
785
|
+
)
|
|
786
|
+
|
|
787
|
+
for p in group['params']:
|
|
788
|
+
assert isinstance(p, ParameterPack), (
|
|
789
|
+
'For now, only parameter packs are supported'
|
|
790
|
+
)
|
|
791
|
+
if p.grad is None:
|
|
792
|
+
continue
|
|
793
|
+
|
|
794
|
+
grad = p.grad.data
|
|
795
|
+
if grad.is_sparse:
|
|
796
|
+
raise RuntimeError('Sparse gradients are not supported')
|
|
797
|
+
|
|
798
|
+
maybe_unsqueeze = functools.partial(_maybe_unsqueeze, p=p)
|
|
799
|
+
|
|
800
|
+
state = self.state[p]
|
|
801
|
+
if len(state) == 0:
|
|
802
|
+
# Initialize the state.
|
|
803
|
+
state['step'] = (
|
|
804
|
+
torch.zeros(
|
|
805
|
+
p.shape[PACK_DIM],
|
|
806
|
+
dtype=torch.int64,
|
|
807
|
+
device=p.device,
|
|
808
|
+
)
|
|
809
|
+
if self._support_replacement
|
|
810
|
+
else 0
|
|
811
|
+
)
|
|
812
|
+
state['exp_avg'] = torch.zeros_like(p.data)
|
|
813
|
+
state['exp_avg_sq'] = torch.zeros_like(p.data)
|
|
814
|
+
exp_avg = state['exp_avg']
|
|
815
|
+
exp_avg_sq = state['exp_avg_sq']
|
|
816
|
+
|
|
817
|
+
state['step'] += 1
|
|
818
|
+
|
|
819
|
+
if weight_decay_multiplier is not None:
|
|
820
|
+
p.mul_(maybe_unsqueeze(weight_decay_multiplier))
|
|
821
|
+
|
|
822
|
+
# Update biased first moment estimate.
|
|
823
|
+
exp_avg.lerp_(grad, maybe_unsqueeze(1 - beta1))
|
|
824
|
+
|
|
825
|
+
# Update biased second raw moment estimate.
|
|
826
|
+
if isinstance(beta2, float):
|
|
827
|
+
exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
|
|
828
|
+
else:
|
|
829
|
+
exp_avg_sq.lerp_(grad.square(), maybe_unsqueeze(1 - beta2))
|
|
830
|
+
|
|
831
|
+
# Perform the bias correction.
|
|
832
|
+
bias_correction1 = 1 - beta1 ** state['step']
|
|
833
|
+
bias_correction2 = 1 - beta2 ** state['step']
|
|
834
|
+
|
|
835
|
+
step_size = lr / bias_correction1
|
|
836
|
+
|
|
837
|
+
if isinstance(bias_correction2, float):
|
|
838
|
+
denom = exp_avg_sq.sqrt() / bias_correction2**0.5
|
|
839
|
+
else:
|
|
840
|
+
denom = exp_avg_sq.sqrt().div_(maybe_unsqueeze(bias_correction2**0.5))
|
|
841
|
+
denom.add_(maybe_unsqueeze(eps))
|
|
842
|
+
|
|
843
|
+
if isinstance(step_size, float):
|
|
844
|
+
p.addcdiv_(exp_avg, denom, value=-step_size)
|
|
845
|
+
else:
|
|
846
|
+
p.add_((exp_avg / denom).mul_(maybe_unsqueeze(step_size)), alpha=-1)
|
|
847
|
+
|
|
848
|
+
@torch.no_grad()
|
|
849
|
+
def step(self, closure: Callable[[], Tensor] | None = None) -> Tensor | None: # ty: ignore[invalid-method-override]
|
|
850
|
+
"""Perform a single optimization step for all pack members."""
|
|
851
|
+
loss = None
|
|
852
|
+
if closure is not None:
|
|
853
|
+
loss = closure()
|
|
854
|
+
|
|
855
|
+
for group in self.param_groups:
|
|
856
|
+
if group['muon']:
|
|
857
|
+
self._step_muon(group)
|
|
858
|
+
else:
|
|
859
|
+
self._step_adamw(group)
|
|
860
|
+
|
|
861
|
+
return loss
|
|
862
|
+
|
|
863
|
+
|
|
864
|
+
# >>> Optimizer utilities
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def optimizer_pack_remove(
|
|
868
|
+
optimizer: torch.optim.Optimizer,
|
|
869
|
+
*,
|
|
870
|
+
pack_idx: Tensor,
|
|
871
|
+
old_to_new_parameters: dict[ParameterPack, ParameterPack],
|
|
872
|
+
) -> None:
|
|
873
|
+
"""Remove some members from an optimizer pack."""
|
|
874
|
+
if not old_to_new_parameters:
|
|
875
|
+
raise ValueError(
|
|
876
|
+
'The mapping from old parameters to new parameters must not be empty'
|
|
877
|
+
)
|
|
878
|
+
|
|
879
|
+
pack_size = _get_pack_size(optimizer)
|
|
880
|
+
if pack_size is None:
|
|
881
|
+
raise ValueError('Failed to infer the pack size')
|
|
882
|
+
|
|
883
|
+
tabpackage.nn.check_pack_idx(pack_idx, pack_size)
|
|
884
|
+
|
|
885
|
+
keep_pack_idx = tabpackage.nn.make_complementary_pack_idx(
|
|
886
|
+
pack_idx, pack_size=pack_size
|
|
887
|
+
)
|
|
888
|
+
|
|
889
|
+
for group in optimizer.param_groups:
|
|
890
|
+
for key, value in list(group.items()):
|
|
891
|
+
if isinstance(value, Tensor) and value.ndim > 0:
|
|
892
|
+
group[key] = value[keep_pack_idx]
|
|
893
|
+
del key, value
|
|
894
|
+
|
|
895
|
+
for i, p in list(enumerate(group['params'])):
|
|
896
|
+
if isinstance(p, ParameterPack):
|
|
897
|
+
state = optimizer.state.pop(p, None)
|
|
898
|
+
# `state` may be missing even if the optimizer has already been used.
|
|
899
|
+
# For example, in `MLPBackbonePack`, some blocks may remain unused
|
|
900
|
+
# (and therefore have no corresponding optimizer state) if the maximum
|
|
901
|
+
# allowed number of blocks is never reached.
|
|
902
|
+
if state is not None:
|
|
903
|
+
for key, value in list(state.items()):
|
|
904
|
+
if isinstance(value, Tensor) and value.ndim > 0:
|
|
905
|
+
state[key] = value[keep_pack_idx]
|
|
906
|
+
del key, value
|
|
907
|
+
p_new = old_to_new_parameters[p]
|
|
908
|
+
group['params'][i] = p_new
|
|
909
|
+
if state is not None:
|
|
910
|
+
optimizer.state[p_new] = state
|
|
911
|
+
|
|
912
|
+
|
|
913
|
+
def optimizer_pack_reset_state(
|
|
914
|
+
optimizer: torch.optim.Optimizer, *, pack_idx: Tensor
|
|
915
|
+
) -> None:
|
|
916
|
+
"""Zero state tensors of specified pack members."""
|
|
917
|
+
tabpackage.nn.check_pack_idx(pack_idx, _get_pack_size(optimizer))
|
|
918
|
+
|
|
919
|
+
for group in optimizer.param_groups:
|
|
920
|
+
for p in group['params']:
|
|
921
|
+
if isinstance(p, ParameterPack):
|
|
922
|
+
state = optimizer.state.get(p)
|
|
923
|
+
if state is not None:
|
|
924
|
+
for value in state.values():
|
|
925
|
+
if isinstance(value, Tensor) and value.ndim > 0:
|
|
926
|
+
value[pack_idx] = 0
|
|
927
|
+
|
|
928
|
+
|
|
929
|
+
def should_skip_weight_decay(
|
|
930
|
+
module_name: str, module: nn.Module, parameter_name: str, parameter: nn.Parameter
|
|
931
|
+
):
|
|
932
|
+
"""Determine if weight decay should be disabled for a given parameter."""
|
|
933
|
+
del module_name
|
|
934
|
+
return (
|
|
935
|
+
# Weight decay is disabled for bias-like parameters, normalization layers, and
|
|
936
|
+
# the first submodules of feature embeddings.
|
|
937
|
+
parameter_name.endswith('bias')
|
|
938
|
+
or isinstance(module, tabpackage.nn.LinearEmbeddingsPack)
|
|
939
|
+
or (
|
|
940
|
+
isinstance(module, tabpackage.nn.CosineEmbeddingsPack)
|
|
941
|
+
and parameter is module.weight
|
|
942
|
+
)
|
|
943
|
+
)
|