tabpackage 0.0.1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
tabpackage/optim.py ADDED
@@ -0,0 +1,943 @@
1
+ """Optimizers, including the "optimizer pack" pattern.
2
+
3
+ An optimizer pack is applied to the members of a module pack from `tabpackage.nn` in
4
+ parallel, with independent optimizer states and potentially different hyperparameters.
5
+ The base class for optimizer packs is `OptimizerPack`.
6
+
7
+ Note:
8
+ Vanilla non-pack optimizers can be used to optimize module packs when all pack
9
+ members have the same optimization hyperparameters. This is why in many places here
10
+ and in other files, the type annotation for optimizers is `Optimizer` rather than
11
+ `OptimizerPack`, and utilities like those for removing or resetting pack members are
12
+ free functions rather than methods of `OptimizerPack`.
13
+
14
+ Using vanilla optimizers in the case of shared hyperparameters can accelerate
15
+ training due to their efficient implementation in PyTorch, particularly when
16
+ `foreach` or the currently experimental `fused` implementation is available, as with
17
+ AdamW. See the official PyTorch documentation for details.
18
+ """
19
+
20
+ import functools
21
+ import math
22
+ import typing
23
+ import warnings
24
+ from collections.abc import Callable
25
+ from typing import Any, Final, TypeAlias, TypeVar
26
+
27
+ import torch
28
+ import torch.nn as nn # noqa: PLR0402
29
+ from torch import Tensor
30
+ from torch.optim.optimizer import ParamsT
31
+
32
+ import tabpackage.nn
33
+ from tabpackage.nn import PACK_DIM, ParameterPack
34
+
35
+ _TORCH_IS_OLDER_THAN_2_13 = (
36
+ torch.torch_version.TorchVersion(torch.__version__) < '2.13.0'
37
+ )
38
+
39
+
40
+ ScalarGroupValue: TypeAlias = bool | int | float | str | bytes
41
+ NumericScalarGroupValue: TypeAlias = bool | int | float
42
+ NumericGroupValueList: TypeAlias = list[bool] | list[int] | list[float]
43
+
44
+ _T = TypeVar('_T')
45
+ _ScalarGroupValueT = TypeVar('_ScalarGroupValueT', bound=ScalarGroupValue)
46
+ _NumericScalarGroupValueT = TypeVar(
47
+ '_NumericScalarGroupValueT', bound=NumericScalarGroupValue
48
+ )
49
+
50
+
51
+ def _check_lr(value: float) -> None:
52
+ if not math.isfinite(value) or value < 0.0:
53
+ raise ValueError('The learning rate must be finite and non-negative')
54
+
55
+
56
+ def _check_momentum(value: float) -> None:
57
+ if not math.isfinite(value) or value < 0.0 or value >= 1.0:
58
+ raise ValueError(
59
+ 'Momentum and moving-average decay coefficients must be finite and in the'
60
+ ' range [0, 1)'
61
+ )
62
+
63
+
64
+ def _check_weight_decay(value: float) -> None:
65
+ if not math.isfinite(value) or value < 0.0:
66
+ raise ValueError('The weight decay must be finite and non-negative')
67
+
68
+
69
+ def _check_eps(value: float) -> None:
70
+ if not math.isfinite(value) or value <= 0.0:
71
+ raise ValueError('The epsilon value must be finite and positive')
72
+
73
+
74
+ @typing.overload
75
+ def _make_group_value(
76
+ value: _ScalarGroupValueT, pack_size: int, device: torch.device | None
77
+ ) -> _ScalarGroupValueT: ...
78
+
79
+
80
+ @typing.overload
81
+ def _make_group_value(
82
+ value: list[_NumericScalarGroupValueT], pack_size: int, device: torch.device | None
83
+ ) -> Tensor: ...
84
+
85
+
86
+ @typing.overload
87
+ def _make_group_value(
88
+ value: Tensor, pack_size: int, device: torch.device | None
89
+ ) -> Tensor: ...
90
+
91
+
92
+ def _make_group_value(value, pack_size: int, device: torch.device | None):
93
+ if value is None or isinstance(value, ScalarGroupValue):
94
+ return value
95
+ elif isinstance(value, list):
96
+ if len(value) != pack_size:
97
+ raise ValueError(
98
+ 'The size of the hyperparameter list must match the pack size'
99
+ )
100
+
101
+ return torch.tensor(value, device=device)
102
+ elif isinstance(value, Tensor):
103
+ if value.ndim == 0:
104
+ raise ValueError(
105
+ 'Tensors can be used only to specify per-member hyperparameters; use a'
106
+ ' Python scalar to share one value across all members'
107
+ )
108
+ if len(value) != pack_size:
109
+ raise ValueError(
110
+ 'The first dimension of a hyperparameter tensor must match the pack'
111
+ ' size'
112
+ )
113
+
114
+ # Copy tensors so groups do not share mutable hyperparameter storage.
115
+ return value.to(device, copy=True)
116
+ else:
117
+ raise TypeError(f'Unsupported value type: {type(value)}')
118
+
119
+
120
+ def _get_device(optimizer: torch.optim.Optimizer) -> torch.device | None:
121
+ for group in optimizer.param_groups:
122
+ for p in group['params']:
123
+ return p.device
124
+ return None
125
+
126
+
127
+ def _get_pack_size(optimizer: torch.optim.Optimizer) -> int | None:
128
+ for group in optimizer.param_groups:
129
+ for p in group['params']:
130
+ if isinstance(p, ParameterPack):
131
+ return p.shape[PACK_DIM]
132
+ return None
133
+
134
+
135
+ def _make_defaults(
136
+ keys_values_checks: list[
137
+ tuple[str, ScalarGroupValue | NumericGroupValueList, Callable[..., None]]
138
+ ],
139
+ *,
140
+ pack_size: int,
141
+ ) -> dict[str, Any]:
142
+ defaults = {}
143
+ for key, value, check_fn in keys_values_checks:
144
+ if isinstance(value, list):
145
+ if len(value) != pack_size:
146
+ raise ValueError(
147
+ 'The size of each hyperparameter list must match the pack size'
148
+ )
149
+
150
+ for x in value:
151
+ check_fn(x)
152
+ else:
153
+ check_fn(value)
154
+ defaults[key] = value
155
+ return defaults
156
+
157
+
158
+ class OptimizerPack(torch.optim.Optimizer):
159
+ """The base class for optimizer packs."""
160
+
161
+ def __init__(
162
+ self, params: ParamsT, defaults: dict[str, Any], *, pack_size: int
163
+ ) -> None:
164
+ super().__init__(params, defaults)
165
+ for key, value in list(self.defaults.items()):
166
+ self.defaults[key] = _make_group_value(value, pack_size, _get_device(self))
167
+
168
+ def add_param_group(self, param_group):
169
+ output = super().add_param_group(param_group)
170
+ group = self.param_groups[-1]
171
+
172
+ group_params = group['params']
173
+ if not group_params:
174
+ # Otherwise, it will be impossible to infer the device and pack size below.
175
+ raise ValueError('Empty parameter groups are not supported')
176
+
177
+ device = None
178
+ pack_size = None
179
+ for p in group['params']:
180
+ if not isinstance(p, ParameterPack):
181
+ raise TypeError('For now, only parameter packs are supported')
182
+
183
+ if device is None:
184
+ device = p.device
185
+ pack_size = p.shape[PACK_DIM]
186
+ assert device is not None
187
+ assert pack_size is not None
188
+
189
+ for key, value in list(group.items()):
190
+ if key != 'params':
191
+ group[key] = _make_group_value(value, pack_size, device)
192
+
193
+ return output
194
+
195
+ def _replace_impl(
196
+ self,
197
+ *,
198
+ pack_idx: Tensor,
199
+ keys_values_checks: list[
200
+ tuple[str, NumericGroupValueList | None, Callable[..., None]]
201
+ ],
202
+ ) -> None:
203
+ tabpackage.nn.check_pack_idx(pack_idx, _get_pack_size(self))
204
+
205
+ for _, value, check_fn in keys_values_checks:
206
+ if value is not None:
207
+ if len(value) != len(pack_idx):
208
+ raise ValueError(
209
+ 'Each replacement hyperparameter list must contain one value'
210
+ ' per selected pack member'
211
+ )
212
+
213
+ for x in value:
214
+ check_fn(x)
215
+
216
+ for key, value, _ in keys_values_checks:
217
+ if value is None:
218
+ continue
219
+
220
+ value_tensor = None
221
+ replaced = False
222
+ for group in self.param_groups:
223
+ current_value = group[key]
224
+ if isinstance(current_value, Tensor):
225
+ if value_tensor is None:
226
+ value_tensor = torch.tensor(
227
+ value,
228
+ dtype=current_value.dtype,
229
+ device=current_value.device,
230
+ )
231
+ current_value[pack_idx] = value_tensor
232
+ replaced = True
233
+
234
+ if not replaced:
235
+ raise RuntimeError(f'Replacement is not possible for "{key}"')
236
+
237
+ optimizer_pack_reset_state(self, pack_idx=pack_idx)
238
+
239
+
240
+ def _make_weight_decay_multiplier(
241
+ *, lr: float | Tensor, weight_decay: float | Tensor
242
+ ) -> float | Tensor | None:
243
+ return (
244
+ None
245
+ if (
246
+ (isinstance(lr, float) and lr == 0.0)
247
+ or (isinstance(weight_decay, float) and weight_decay == 0.0)
248
+ )
249
+ else (1 - lr * weight_decay)
250
+ )
251
+
252
+
253
+ @typing.overload
254
+ def _maybe_unsqueeze(value: Tensor, *, p: Tensor) -> Tensor: ...
255
+
256
+
257
+ @typing.overload
258
+ def _maybe_unsqueeze(value: _T, *, p: Tensor) -> _T: ...
259
+
260
+
261
+ def _maybe_unsqueeze(value, *, p):
262
+ return (
263
+ value[(slice(None), *((None,) * (p.ndim - 1)))]
264
+ if isinstance(value, Tensor)
265
+ else value
266
+ )
267
+
268
+
269
+ class AdamWPack(OptimizerPack):
270
+ """A pack of AdamW optimizers."""
271
+
272
+ def __init__(
273
+ self,
274
+ params: ParamsT,
275
+ *,
276
+ lr: float | list[float],
277
+ # Unlike in `torch.optim.AdamW`, `beta1` and `beta2` are separate arguments,
278
+ # making it easy to specify different values for each pack member by passing
279
+ # lists, as with all other scalar hyperparameters.
280
+ beta1: float | list[float] = 0.9,
281
+ beta2: float | list[float] = 0.999,
282
+ eps: float | list[float] = 1e-8,
283
+ weight_decay: float | list[float],
284
+ pack_size: int,
285
+ support_replacement: bool = False,
286
+ ):
287
+ """
288
+ Args:
289
+ support_replacement: if `True`, the optimizer supports selective replacement
290
+ of current members with new ones. In this case, the underlying step
291
+ counter is not an integer scalar anymore, but a tensor with per-member
292
+ steps, which makes the optimization step a bit slower, and changes the
293
+ results numerically (even if replacements never occur).
294
+ """
295
+ if pack_size <= 0:
296
+ raise ValueError('The pack size must be positive')
297
+
298
+ super().__init__(
299
+ params,
300
+ _make_defaults(
301
+ [
302
+ ('lr', lr, _check_lr),
303
+ ('beta1', beta1, _check_momentum),
304
+ ('beta2', beta2, _check_momentum),
305
+ ('eps', eps, _check_eps),
306
+ ('weight_decay', weight_decay, _check_weight_decay),
307
+ ],
308
+ pack_size=pack_size,
309
+ ),
310
+ pack_size=pack_size,
311
+ )
312
+
313
+ self._support_replacement: Final = support_replacement
314
+
315
+ def replace(
316
+ self,
317
+ *,
318
+ pack_idx: Tensor,
319
+ lr: list[float] | None = None,
320
+ beta1: list[float] | None = None,
321
+ beta2: list[float] | None = None,
322
+ eps: list[float] | None = None,
323
+ weight_decay: list[float] | None = None,
324
+ ) -> None:
325
+ """Replace selected members with new ones.
326
+
327
+ The method resets the state of the selected optimizers and optionally changes
328
+ their hyperparameters.
329
+ """
330
+ if not self._support_replacement:
331
+ raise RuntimeError(
332
+ 'Replacing optimizer pack members requires constructing the optimizer'
333
+ ' with `support_replacement=True`'
334
+ )
335
+
336
+ self._replace_impl(
337
+ pack_idx=pack_idx,
338
+ keys_values_checks=[
339
+ ('lr', lr, _check_lr),
340
+ ('beta1', beta1, _check_momentum),
341
+ ('beta2', beta2, _check_momentum),
342
+ ('eps', eps, _check_eps),
343
+ ('weight_decay', weight_decay, _check_weight_decay),
344
+ ],
345
+ )
346
+
347
+ @torch.no_grad()
348
+ def step(self, closure: Callable[[], Tensor] | None = None) -> Tensor | None: # ty: ignore[invalid-method-override]
349
+ """Perform a single optimization step for all pack members."""
350
+ loss = None
351
+ if closure is not None:
352
+ loss = closure()
353
+
354
+ for group in self.param_groups:
355
+ lr = group['lr']
356
+ beta1 = group['beta1']
357
+ beta2 = group['beta2']
358
+ eps = group['eps']
359
+
360
+ weight_decay_multiplier = _make_weight_decay_multiplier(
361
+ lr=lr, weight_decay=group['weight_decay']
362
+ )
363
+
364
+ for p in group['params']:
365
+ assert isinstance(p, ParameterPack), (
366
+ 'For now, only parameter packs are supported'
367
+ )
368
+ if p.grad is None:
369
+ continue
370
+
371
+ grad = p.grad.data
372
+ if grad.is_sparse:
373
+ raise RuntimeError('Sparse gradients are not supported')
374
+
375
+ maybe_unsqueeze = functools.partial(_maybe_unsqueeze, p=p)
376
+
377
+ state = self.state[p]
378
+ if len(state) == 0:
379
+ # Initialize the state.
380
+ state['step'] = (
381
+ torch.zeros(
382
+ p.shape[PACK_DIM],
383
+ dtype=torch.int64,
384
+ device=p.device,
385
+ )
386
+ if self._support_replacement
387
+ else 0
388
+ )
389
+ state['exp_avg'] = torch.zeros_like(p.data)
390
+ state['exp_avg_sq'] = torch.zeros_like(p.data)
391
+ exp_avg = state['exp_avg']
392
+ exp_avg_sq = state['exp_avg_sq']
393
+
394
+ state['step'] += 1
395
+
396
+ if weight_decay_multiplier is not None:
397
+ p.mul_(maybe_unsqueeze(weight_decay_multiplier))
398
+
399
+ # Update biased first moment estimate.
400
+ exp_avg.lerp_(grad, maybe_unsqueeze(1 - beta1))
401
+
402
+ # Update biased second raw moment estimate.
403
+ if isinstance(beta2, float):
404
+ exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
405
+ else:
406
+ exp_avg_sq.lerp_(grad.square(), maybe_unsqueeze(1 - beta2))
407
+
408
+ # Perform the bias correction.
409
+ bias_correction1 = 1 - beta1 ** state['step']
410
+ bias_correction2 = 1 - beta2 ** state['step']
411
+
412
+ step_size = lr / bias_correction1
413
+
414
+ if isinstance(bias_correction2, float):
415
+ denom = exp_avg_sq.sqrt() / bias_correction2**0.5
416
+ else:
417
+ denom = exp_avg_sq.sqrt().div_(
418
+ maybe_unsqueeze(bias_correction2**0.5)
419
+ )
420
+ denom.add_(maybe_unsqueeze(eps))
421
+
422
+ if isinstance(step_size, float):
423
+ p.addcdiv_(exp_avg, denom, value=-step_size)
424
+ else:
425
+ p.add_((exp_avg / denom).mul_(maybe_unsqueeze(step_size)), alpha=-1)
426
+
427
+ return loss
428
+
429
+
430
+ def compute_muon_update_scale_pack(module: tabpackage.nn.LinearPack) -> Tensor | None:
431
+ """Compute Muon update scaling coefficients for a pack of linear layers."""
432
+ if module.in_features is None and module.out_features is None:
433
+ return None
434
+
435
+ if module.in_features is None:
436
+ assert module.out_features is not None
437
+ pack_size = module.out_features.shape[PACK_DIM]
438
+ device = module.out_features.device
439
+ else:
440
+ pack_size = module.in_features.shape[PACK_DIM]
441
+ device = module.in_features.device
442
+
443
+ out_f = (
444
+ torch.full((pack_size,), float(module.max_out_features), device=device)
445
+ if module.out_features is None
446
+ else module.out_features.float()
447
+ )
448
+ in_f = (
449
+ torch.full((pack_size,), float(module.max_in_features), device=device)
450
+ if module.in_features is None
451
+ else module.in_features.float()
452
+ )
453
+ return (out_f / in_f).clamp_(min=1.0).sqrt_()
454
+
455
+
456
+ # The following function is a modified version of this function:
457
+ # https://github.com/KellerJordan/Muon/blob/f98f1cacc0263b04290753e32be8d498c1efc806/muon.py#L5
458
+ #
459
+ # The modification consists in using `torch.baddbmm` for better efficiency. This
460
+ # requires `G` to be three-dimensional.
461
+ #
462
+ # fmt: off
463
+ # ruff: disable[E501]
464
+ def _zeropower_via_newtonschulz5(G, steps: int):
465
+ """
466
+ Newton-Schulz iteration to compute the zeroth power / orthogonalization of G. We opt to use a
467
+ quintic iteration whose coefficients are selected to maximize the slope at zero. For the purpose
468
+ of minimizing steps, it turns out to be empirically effective to keep increasing the slope at
469
+ zero even beyond the point where the iteration no longer converges all the way to one everywhere
470
+ on the interval. This iteration therefore does not produce UV^T but rather something like US'V^T
471
+ where S' is diagonal with S_{ii}' ~ Uniform(0.5, 1.5), which turns out not to hurt model
472
+ performance at all relative to UV^T, where USV^T = G is the SVD.
473
+ """
474
+ assert G.ndim == 3 # batched Muon implementation by @scottjmaddox, and put into practice in the record by @YouJiacheng
475
+ a, b, c = (3.4445, -4.7750, 2.0315)
476
+ X = G.bfloat16()
477
+ if G.size(-2) > G.size(-1):
478
+ X = X.mT
479
+
480
+ # Ensure spectral norm is at most 1
481
+ X = X / (X.norm(dim=(-2, -1), keepdim=True) + 1e-7)
482
+ # Perform the NS iterations
483
+ for _ in range(steps):
484
+ A = X @ X.mT
485
+ B = torch.baddbmm(A, A, A, beta=b, alpha=c) # quintic computation strategy adapted from suggestion by @jxbz, @leloykun, and @YouJiacheng
486
+ X = torch.baddbmm(X, B, X, beta=a)
487
+
488
+ if G.size(-2) > G.size(-1):
489
+ X = X.mT
490
+ return X
491
+ # ruff: enable[E501]
492
+ # fmt: on
493
+
494
+
495
+ class MuonAdamWPack(OptimizerPack):
496
+ """A pack of optimizers combining Muon and AdamW.
497
+
498
+ Muon is intended for weights (excluding biases) of backbones' hidden linear layers,
499
+ and AdamW for everything else. Parameter groups optimized with Muon must be created
500
+ with `make_muon_parameter_group` (see the note below for details).
501
+
502
+ Note:
503
+ For parameter packs optimized with Muon, the spectral correction coefficients
504
+ must be provided explicitly, because they depend on the actual per-model
505
+ parameter shapes, which cannot be inferred from parameter packs alone. As a
506
+ consequence, parameter packs with different per-model shapes cannot be
507
+ optimized with Muon within one parameter group, because they will have different
508
+ coefficients. To make the optimizer easier to use correctly, all Muon parameter
509
+ groups are currently required to have no more than one parameter.
510
+
511
+ """
512
+
513
+ @staticmethod
514
+ def make_muon_parameter_group(module: tabpackage.nn.LinearPack) -> dict[str, Any]:
515
+ """Create a single-parameter group to be optimized with Muon."""
516
+ if (
517
+ module.max_in_features > module.max_out_features
518
+ and module.max_out_features < 16
519
+ ):
520
+ warnings.warn(
521
+ f'The pack of linear layers has shape {module.weight.shape}, which'
522
+ ' looks more like an output layer (should not be optimized with Muon)'
523
+ ' than a backbone layer (can be optimized with Muon). Is this'
524
+ ' intentional?',
525
+ stacklevel=2,
526
+ )
527
+
528
+ return {
529
+ 'params': [module.weight],
530
+ 'muon': True,
531
+ 'muon_update_scale': compute_muon_update_scale_pack(module),
532
+ }
533
+
534
+ def __init__(
535
+ self,
536
+ params: ParamsT,
537
+ *,
538
+ adamw_lr: float | list[float],
539
+ adamw_beta1: float | list[float] = 0.9,
540
+ adamw_beta2: float | list[float] = 0.999,
541
+ adamw_eps: float | list[float] = 1e-8,
542
+ weight_decay: float | list[float],
543
+ muon_lr: float | list[float],
544
+ muon_momentum: float | list[float] = 0.95,
545
+ muon_newton_schulz_steps: int = 5,
546
+ muon_nesterov: bool = True,
547
+ muon_skip_zero_updates: bool = False,
548
+ pack_size: int,
549
+ support_replacement: bool = False,
550
+ # In practice, the following argument should usually be `True` for better
551
+ # efficiency unless the original parameter gradients must be preserved.
552
+ allow_inplace_grad_mutation: bool,
553
+ compile_muon_newton_schulz: bool = False,
554
+ ):
555
+ """
556
+ Args:
557
+ muon_skip_zero_updates: if `True`, the members with all-zero updates are
558
+ excluded from the Newton-Schulz iterations. On accelerators, this costs
559
+ a device-to-host synchronization per step for each Muon parameter group
560
+ with a gradient. This can speed up training of large packs of
561
+ depth-varying MLPs.
562
+ support_replacement: same as in `AdamWPack`.
563
+ """
564
+ if pack_size <= 0:
565
+ raise ValueError('The pack size must be positive')
566
+
567
+ if (
568
+ compile_muon_newton_schulz
569
+ and torch.torch_version.TorchVersion(torch.__version__) < '2.12.0'
570
+ ):
571
+ raise RuntimeError(
572
+ 'Compiling the Newton-Schulz iteration for Muon requires PyTorch 2.12.0'
573
+ ' or newer'
574
+ )
575
+
576
+ super().__init__(
577
+ params,
578
+ {
579
+ 'muon': False,
580
+ 'muon_newton_schulz_steps': muon_newton_schulz_steps,
581
+ 'muon_nesterov': muon_nesterov,
582
+ 'muon_skip_zero_updates': muon_skip_zero_updates,
583
+ **_make_defaults(
584
+ [
585
+ ('adamw_lr', adamw_lr, _check_lr),
586
+ ('adamw_beta1', adamw_beta1, _check_momentum),
587
+ ('adamw_beta2', adamw_beta2, _check_momentum),
588
+ ('adamw_eps', adamw_eps, _check_eps),
589
+ ('weight_decay', weight_decay, _check_weight_decay),
590
+ ('muon_lr', muon_lr, _check_lr),
591
+ ('muon_momentum', muon_momentum, _check_momentum),
592
+ ],
593
+ pack_size=pack_size,
594
+ ),
595
+ },
596
+ pack_size=pack_size,
597
+ )
598
+
599
+ self._support_replacement: Final = support_replacement
600
+ self._allow_inplace_grad_mutation: Final = allow_inplace_grad_mutation
601
+ self._zeropower_via_newtonschulz5 = (
602
+ torch.compile(
603
+ _zeropower_via_newtonschulz5,
604
+ fullgraph=True,
605
+ # Emulate eager precision casts to reduce numerical differences from
606
+ # the non-compiled variant.
607
+ options={'emulate_precision_casts': True},
608
+ )
609
+ if compile_muon_newton_schulz
610
+ else _zeropower_via_newtonschulz5
611
+ )
612
+
613
+ def add_param_group(self, param_group):
614
+ output = super().add_param_group(param_group)
615
+ group = self.param_groups[-1]
616
+ if group['muon']:
617
+ if 'muon_update_scale' not in group:
618
+ raise ValueError(
619
+ 'The just added parameter group is configured to be optimized with'
620
+ ' Muon. In this case, it must have the "muon_update_scale" key'
621
+ ' with the value computed with the'
622
+ ' `compute_muon_update_scale_pack` function.'
623
+ )
624
+ if len(group['params']) > 1:
625
+ raise ValueError(
626
+ 'The just added parameter group configured to be optimized with'
627
+ ' Muon has more than one parameter, which is not allowed'
628
+ )
629
+ return output
630
+
631
+ def replace(
632
+ self,
633
+ *,
634
+ pack_idx: Tensor,
635
+ adamw_lr: list[float] | None = None,
636
+ adamw_beta1: list[float] | None = None,
637
+ adamw_beta2: list[float] | None = None,
638
+ adamw_eps: list[float] | None = None,
639
+ weight_decay: list[float] | None = None,
640
+ muon_lr: list[float] | None = None,
641
+ muon_momentum: list[float] | None = None,
642
+ module: nn.Module,
643
+ ) -> None:
644
+ """Replace selected members with new ones.
645
+
646
+ See `AdamWPack.replace` for additiona details.
647
+
648
+ Important:
649
+ `module` must undergo its member replacement _before_ this method is called,
650
+ because it is used to recompute the new Muon update scales.
651
+ """
652
+ if not self._support_replacement:
653
+ raise RuntimeError(
654
+ 'Replacing optimizer pack members requires constructing the optimizer'
655
+ ' with `support_replacement=True`'
656
+ )
657
+
658
+ self._replace_impl(
659
+ pack_idx=pack_idx,
660
+ keys_values_checks=[
661
+ ('adamw_lr', adamw_lr, _check_lr),
662
+ ('adamw_beta1', adamw_beta1, _check_momentum),
663
+ ('adamw_beta2', adamw_beta2, _check_momentum),
664
+ ('adamw_eps', adamw_eps, _check_eps),
665
+ ('weight_decay', weight_decay, _check_weight_decay),
666
+ ('muon_lr', muon_lr, _check_lr),
667
+ ('muon_momentum', muon_momentum, _check_momentum),
668
+ ],
669
+ )
670
+
671
+ muon_parameter_to_module = {
672
+ x.weight: x
673
+ for x in module.modules()
674
+ if isinstance(x, tabpackage.nn.LinearPack)
675
+ }
676
+ for group in self.param_groups:
677
+ if group['muon']:
678
+ group['muon_update_scale'] = compute_muon_update_scale_pack(
679
+ muon_parameter_to_module[group['params'][0]]
680
+ )
681
+
682
+ def _step_muon(self, group: dict[str, Any]) -> None:
683
+ lr = group['muon_lr']
684
+ momentum = group['muon_momentum']
685
+ weight_decay = group['weight_decay']
686
+ newton_schulz_steps = group['muon_newton_schulz_steps']
687
+ nesterov = group['muon_nesterov']
688
+ skip_zero_updates = group['muon_skip_zero_updates']
689
+ update_scale = group['muon_update_scale']
690
+
691
+ weight_decay_multiplier = _make_weight_decay_multiplier(
692
+ lr=lr, weight_decay=weight_decay
693
+ )
694
+
695
+ for p in group['params']:
696
+ if p.ndim != 3: # 1 pack dimension + 2 layer dimensions.
697
+ raise ValueError(
698
+ 'Parameters optimized with Muon must have three dimensions'
699
+ )
700
+
701
+ assert isinstance(p, ParameterPack), (
702
+ 'For now, only parameter packs are supported'
703
+ )
704
+ if p.grad is None:
705
+ continue
706
+
707
+ grad = p.grad.data
708
+ if grad.is_sparse:
709
+ raise RuntimeError('Sparse gradients are not supported')
710
+
711
+ maybe_unsqueeze = functools.partial(_maybe_unsqueeze, p=p)
712
+
713
+ state = self.state[p]
714
+ if len(state) == 0:
715
+ # Initialize the state.
716
+ state['muon_momentum_buffer'] = torch.zeros_like(p)
717
+ momentum_buffer: Tensor = state['muon_momentum_buffer']
718
+
719
+ if weight_decay_multiplier is not None:
720
+ p.mul_(maybe_unsqueeze(weight_decay_multiplier))
721
+
722
+ momentum_buffer.lerp_(grad, maybe_unsqueeze(1 - momentum))
723
+ update = (
724
+ (grad.lerp_ if self._allow_inplace_grad_mutation else grad.lerp)(
725
+ momentum_buffer, maybe_unsqueeze(momentum)
726
+ )
727
+ if nesterov
728
+ else momentum_buffer
729
+ )
730
+ if skip_zero_updates:
731
+ # All-zero updates do not contribute to the optimization step, and
732
+ # therefore can be safely skipped without participating in the
733
+ # expensive Newton-Schulz iterations. The cost of applying this
734
+ # heuristic is the device-to-host synchronization coming from
735
+ # `torch.nonzero`. It could be avoided by incorporating additional
736
+ # knowledge about the optimized module into the optimizer's state,
737
+ # however, that would make the code significantly more complicated and
738
+ # brittle. The current simple approach already works well.
739
+ assert PACK_DIM == 0
740
+ active_pack_idx = torch.nonzero(update.flatten(1).any(1))[:, 0]
741
+ if len(active_pack_idx) == 0:
742
+ # All members have all-zero updates, and weight decay and momentum
743
+ # updates have already been applied.
744
+ continue
745
+ else:
746
+ active_pack_idx = None
747
+ if active_pack_idx is None or len(active_pack_idx) == len(update):
748
+ update = self._zeropower_via_newtonschulz5(
749
+ update, steps=newton_schulz_steps
750
+ )
751
+ else:
752
+ active_update = self._zeropower_via_newtonschulz5(
753
+ update.index_select(0, active_pack_idx), steps=newton_schulz_steps
754
+ )
755
+ update = torch.zeros_like(update, dtype=active_update.dtype)
756
+ if update.device.type == 'mps' and _TORCH_IS_OLDER_THAN_2_13:
757
+ # The `index_copy_` operation on MPS:
758
+ # - Fallbacks on CPU prior to PyTorch 2.8.
759
+ # - Is inefficient prior to PyTorch 2.13.
760
+ update[active_pack_idx] = active_update
761
+ else:
762
+ update.index_copy_(0, active_pack_idx, active_update)
763
+
764
+ scaled_lr = lr * (
765
+ max(1, grad.size(-2) / grad.size(-1)) ** 0.5
766
+ if update_scale is None
767
+ else update_scale
768
+ )
769
+
770
+ assert update.shape == p.shape
771
+ if isinstance(scaled_lr, float):
772
+ p.add_(update, alpha=-scaled_lr)
773
+ else:
774
+ p.addcmul_(update, maybe_unsqueeze(scaled_lr), value=-1)
775
+
776
+ def _step_adamw(self, group: dict[str, Any]) -> None:
777
+ lr = group['adamw_lr']
778
+ beta1 = group['adamw_beta1']
779
+ beta2 = group['adamw_beta2']
780
+ eps = group['adamw_eps']
781
+ weight_decay = group['weight_decay']
782
+
783
+ weight_decay_multiplier = _make_weight_decay_multiplier(
784
+ lr=lr, weight_decay=weight_decay
785
+ )
786
+
787
+ for p in group['params']:
788
+ assert isinstance(p, ParameterPack), (
789
+ 'For now, only parameter packs are supported'
790
+ )
791
+ if p.grad is None:
792
+ continue
793
+
794
+ grad = p.grad.data
795
+ if grad.is_sparse:
796
+ raise RuntimeError('Sparse gradients are not supported')
797
+
798
+ maybe_unsqueeze = functools.partial(_maybe_unsqueeze, p=p)
799
+
800
+ state = self.state[p]
801
+ if len(state) == 0:
802
+ # Initialize the state.
803
+ state['step'] = (
804
+ torch.zeros(
805
+ p.shape[PACK_DIM],
806
+ dtype=torch.int64,
807
+ device=p.device,
808
+ )
809
+ if self._support_replacement
810
+ else 0
811
+ )
812
+ state['exp_avg'] = torch.zeros_like(p.data)
813
+ state['exp_avg_sq'] = torch.zeros_like(p.data)
814
+ exp_avg = state['exp_avg']
815
+ exp_avg_sq = state['exp_avg_sq']
816
+
817
+ state['step'] += 1
818
+
819
+ if weight_decay_multiplier is not None:
820
+ p.mul_(maybe_unsqueeze(weight_decay_multiplier))
821
+
822
+ # Update biased first moment estimate.
823
+ exp_avg.lerp_(grad, maybe_unsqueeze(1 - beta1))
824
+
825
+ # Update biased second raw moment estimate.
826
+ if isinstance(beta2, float):
827
+ exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
828
+ else:
829
+ exp_avg_sq.lerp_(grad.square(), maybe_unsqueeze(1 - beta2))
830
+
831
+ # Perform the bias correction.
832
+ bias_correction1 = 1 - beta1 ** state['step']
833
+ bias_correction2 = 1 - beta2 ** state['step']
834
+
835
+ step_size = lr / bias_correction1
836
+
837
+ if isinstance(bias_correction2, float):
838
+ denom = exp_avg_sq.sqrt() / bias_correction2**0.5
839
+ else:
840
+ denom = exp_avg_sq.sqrt().div_(maybe_unsqueeze(bias_correction2**0.5))
841
+ denom.add_(maybe_unsqueeze(eps))
842
+
843
+ if isinstance(step_size, float):
844
+ p.addcdiv_(exp_avg, denom, value=-step_size)
845
+ else:
846
+ p.add_((exp_avg / denom).mul_(maybe_unsqueeze(step_size)), alpha=-1)
847
+
848
+ @torch.no_grad()
849
+ def step(self, closure: Callable[[], Tensor] | None = None) -> Tensor | None: # ty: ignore[invalid-method-override]
850
+ """Perform a single optimization step for all pack members."""
851
+ loss = None
852
+ if closure is not None:
853
+ loss = closure()
854
+
855
+ for group in self.param_groups:
856
+ if group['muon']:
857
+ self._step_muon(group)
858
+ else:
859
+ self._step_adamw(group)
860
+
861
+ return loss
862
+
863
+
864
+ # >>> Optimizer utilities
865
+
866
+
867
+ def optimizer_pack_remove(
868
+ optimizer: torch.optim.Optimizer,
869
+ *,
870
+ pack_idx: Tensor,
871
+ old_to_new_parameters: dict[ParameterPack, ParameterPack],
872
+ ) -> None:
873
+ """Remove some members from an optimizer pack."""
874
+ if not old_to_new_parameters:
875
+ raise ValueError(
876
+ 'The mapping from old parameters to new parameters must not be empty'
877
+ )
878
+
879
+ pack_size = _get_pack_size(optimizer)
880
+ if pack_size is None:
881
+ raise ValueError('Failed to infer the pack size')
882
+
883
+ tabpackage.nn.check_pack_idx(pack_idx, pack_size)
884
+
885
+ keep_pack_idx = tabpackage.nn.make_complementary_pack_idx(
886
+ pack_idx, pack_size=pack_size
887
+ )
888
+
889
+ for group in optimizer.param_groups:
890
+ for key, value in list(group.items()):
891
+ if isinstance(value, Tensor) and value.ndim > 0:
892
+ group[key] = value[keep_pack_idx]
893
+ del key, value
894
+
895
+ for i, p in list(enumerate(group['params'])):
896
+ if isinstance(p, ParameterPack):
897
+ state = optimizer.state.pop(p, None)
898
+ # `state` may be missing even if the optimizer has already been used.
899
+ # For example, in `MLPBackbonePack`, some blocks may remain unused
900
+ # (and therefore have no corresponding optimizer state) if the maximum
901
+ # allowed number of blocks is never reached.
902
+ if state is not None:
903
+ for key, value in list(state.items()):
904
+ if isinstance(value, Tensor) and value.ndim > 0:
905
+ state[key] = value[keep_pack_idx]
906
+ del key, value
907
+ p_new = old_to_new_parameters[p]
908
+ group['params'][i] = p_new
909
+ if state is not None:
910
+ optimizer.state[p_new] = state
911
+
912
+
913
+ def optimizer_pack_reset_state(
914
+ optimizer: torch.optim.Optimizer, *, pack_idx: Tensor
915
+ ) -> None:
916
+ """Zero state tensors of specified pack members."""
917
+ tabpackage.nn.check_pack_idx(pack_idx, _get_pack_size(optimizer))
918
+
919
+ for group in optimizer.param_groups:
920
+ for p in group['params']:
921
+ if isinstance(p, ParameterPack):
922
+ state = optimizer.state.get(p)
923
+ if state is not None:
924
+ for value in state.values():
925
+ if isinstance(value, Tensor) and value.ndim > 0:
926
+ value[pack_idx] = 0
927
+
928
+
929
+ def should_skip_weight_decay(
930
+ module_name: str, module: nn.Module, parameter_name: str, parameter: nn.Parameter
931
+ ):
932
+ """Determine if weight decay should be disabled for a given parameter."""
933
+ del module_name
934
+ return (
935
+ # Weight decay is disabled for bias-like parameters, normalization layers, and
936
+ # the first submodules of feature embeddings.
937
+ parameter_name.endswith('bias')
938
+ or isinstance(module, tabpackage.nn.LinearEmbeddingsPack)
939
+ or (
940
+ isinstance(module, tabpackage.nn.CosineEmbeddingsPack)
941
+ and parameter is module.weight
942
+ )
943
+ )