braintools 0.0.1__py2.py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,1098 @@
1
+ # -*- coding: utf-8 -*-
2
+
3
+ import copy
4
+ import functools
5
+ from typing import Union, Dict, Optional, Tuple, Any, TypeVar
6
+
7
+ import braincore as bc
8
+ import jax
9
+ import jax.numpy as jnp
10
+ from brainpy import check
11
+
12
+ from ._lr_scheduler import make_schedule, LRScheduler
13
+
14
+ __all__ = [
15
+ 'to_same_dict_tree',
16
+ 'OptimState',
17
+ 'Optimizer',
18
+ 'SGD',
19
+ 'Momentum',
20
+ 'MomentumNesterov',
21
+ 'Adagrad',
22
+ 'Adadelta',
23
+ 'RMSProp',
24
+ 'Adam',
25
+ 'LARS',
26
+ 'Adan',
27
+ 'AdamW',
28
+ ]
29
+
30
+ T = TypeVar('T')
31
+
32
+
33
+ def cast(value: Any, dtype: Any) -> jax.Array:
34
+ if isinstance(value, jax.Array):
35
+ return value.astype(dtype)
36
+ return jnp.asarray(value, dtype=dtype)
37
+
38
+
39
+ def fcast(value: T, dtype: Any = None) -> jax.Array:
40
+ return cast(value, dtype=dtype or bc.environ.dftype())
41
+
42
+
43
+ def to_same_dict_tree(*dicts: dict):
44
+ """
45
+ Convert multiple dictionaries to the same tree structure.
46
+
47
+ Parameters
48
+ ----------
49
+ *dicts: dict
50
+ The dictionaries to be converted.
51
+
52
+ Returns
53
+ -------
54
+ dict
55
+ The converted dictionary.
56
+ """
57
+ if len(dicts):
58
+ dicts = tuple(copy.copy(d) for d in dicts)
59
+
60
+ # all keys
61
+ all_keys = tuple(set(d.keys()) for d in dicts)
62
+ for keys in all_keys[1:]:
63
+ if len(all_keys[0].difference(keys)) > 0:
64
+ raise ValueError('Dictionary does not match.')
65
+
66
+ # flatten to normal python dict
67
+ r = [dict() for _ in range(len(dicts))]
68
+ for i in range(len(dicts)):
69
+ for k in all_keys[0]:
70
+ v = dicts[i][k]
71
+ if isinstance(v, bc.State):
72
+ r[i][k] = v.value
73
+ else:
74
+ r[i][k] = v
75
+ if len(dicts) == 1:
76
+ return r[0]
77
+ else:
78
+ return tuple(r)
79
+
80
+
81
+ def _sgd(pv, gd, weight_decay, lr=None):
82
+ if weight_decay is None:
83
+ if lr is None:
84
+ return pv - gd
85
+ else:
86
+ return pv - lr * gd
87
+ else:
88
+ if lr is None:
89
+ return (1 - weight_decay) * pv - gd
90
+ else:
91
+ return (1 - weight_decay) * pv - lr * gd
92
+
93
+
94
+ class OptimState(bc.LongTermState):
95
+ """
96
+ The state for optimizer.
97
+ """
98
+ pass
99
+
100
+
101
+ class Optimizer(bc.Module):
102
+ """Base Optimizer Class.
103
+
104
+ Parameters
105
+ ----------
106
+ lr: float, LRScheduler
107
+ learning rate.
108
+ """
109
+
110
+ lr: LRScheduler # learning rate
111
+ weight_states: bc.StateStack # states to train, invisible to ``.states()``
112
+
113
+ def __init__(
114
+ self,
115
+ lr: Union[float, LRScheduler, bc.State],
116
+ name: Optional[str] = None
117
+ ):
118
+ super().__init__(name=name)
119
+ self.lr: LRScheduler = make_schedule(lr)
120
+ self.weight_states = bc.StateStack()
121
+
122
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
123
+ raise NotImplementedError
124
+
125
+ def __repr__(self):
126
+ return f"{self.__class__.__name__}(lr={self.lr}{self.extra_repr()})"
127
+
128
+ def extra_repr(self) -> str:
129
+ return ''
130
+
131
+ def update(self, grads: dict):
132
+ raise NotImplementedError
133
+
134
+
135
+ class _WeightDecayOptimizer(Optimizer):
136
+ def __init__(
137
+ self,
138
+ lr: Union[float, LRScheduler, bc.State],
139
+ weight_decay: Optional[float] = None,
140
+ name: Optional[str] = None
141
+ ):
142
+ super().__init__(lr=lr, name=name)
143
+ self.lr: LRScheduler = make_schedule(lr)
144
+ weight_decay = check.is_float(weight_decay, min_bound=0., max_bound=1., allow_none=True)
145
+ self.weight_decay = (fcast(weight_decay) if weight_decay is not None else None)
146
+
147
+ def extra_repr(self) -> str:
148
+ return ''
149
+
150
+ def __repr__(self):
151
+ return f"{self.__class__.__name__}(lr={self.lr}, weight_decay={self.weight_decay}{self.extra_repr()})"
152
+
153
+
154
+ class SGD(_WeightDecayOptimizer):
155
+ r"""Stochastic gradient descent optimizer.
156
+
157
+ SGD performs a parameter update for training examples :math:`x` and label
158
+ :math:`y`:
159
+
160
+ .. math::
161
+
162
+ \theta = \theta - \eta \cdot \nabla_\theta J(\theta; x; y)
163
+
164
+
165
+ Parameters
166
+ ----------
167
+ lr: float, LRScheduler
168
+ learning rate.
169
+
170
+ """
171
+
172
+ def __init__(
173
+ self,
174
+ lr: Union[float, LRScheduler, bc.State],
175
+ weight_decay: Optional[float] = None,
176
+ name: Optional[str] = None
177
+ ):
178
+ super(SGD, self).__init__(lr=lr,
179
+ weight_decay=weight_decay,
180
+ name=name)
181
+
182
+ def register_trainable_weights(self, states: Optional[Dict[str, bc.State]] = None):
183
+ states = dict() if states is None else states
184
+ assert isinstance(states, dict), '"states" must be a dict of braincore.State.'
185
+ for k, v in states.items():
186
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
187
+ self.weight_states.add_unique_elem(k, v)
188
+
189
+ def update(self, grads: dict):
190
+ lr = self.lr()
191
+ weight_values, grad_values = to_same_dict_tree(self.weight_states, grads)
192
+ updates = jax.tree.map(functools.partial(_sgd, lr=lr, weight_decay=self.weight_decay),
193
+ weight_values,
194
+ grad_values)
195
+ self.weight_states.assign_values(updates)
196
+ self.lr.step_call()
197
+
198
+
199
+ class Momentum(_WeightDecayOptimizer):
200
+ r"""Momentum optimizer.
201
+
202
+ Momentum [1]_ is a method that helps accelerate SGD in the relevant direction
203
+ and dampens oscillations. It does this by adding a fraction :math:`\gamma`
204
+ of the update vector of the past time step to the current update vector:
205
+
206
+ .. math::
207
+
208
+ \begin{align}
209
+ \begin{split}
210
+ v_t &= \gamma v_{t-1} + \eta \nabla_\theta J( \theta) \\
211
+ \theta &= \theta - v_t
212
+ \end{split}
213
+ \end{align}
214
+
215
+ Parameters
216
+ ----------
217
+ lr: float, LRScheduler
218
+ learning rate.
219
+
220
+ References
221
+ ----------
222
+
223
+ .. [1] Qian, N. (1999). On the momentum term in gradient descent learning
224
+ algorithms. Neural Networks : The Official Journal of the International
225
+ Neural Network Society, 12(1), 145–151. http://doi.org/10.1016/S0893-6080(98)00116-6
226
+
227
+ """
228
+
229
+ def __init__(
230
+ self,
231
+ lr: Union[float, LRScheduler, bc.State],
232
+ momentum: float = 0.9,
233
+ weight_decay: Optional[float] = None,
234
+ name: Optional[str] = None
235
+ ):
236
+ super(Momentum, self).__init__(lr=lr, weight_decay=weight_decay, name=name)
237
+ self.momentum = fcast(momentum)
238
+ self.momentum_states = bc.visible_state_dict()
239
+
240
+ def extra_repr(self) -> str:
241
+ return f", momentum={self.momentum}"
242
+
243
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
244
+ train_states = dict() if train_states is None else train_states
245
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
246
+
247
+ for k, v in train_states.items():
248
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
249
+ self.weight_states.add_unique_elem(k, v)
250
+ self.momentum_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
251
+
252
+ def update(self, grads: dict):
253
+ lr = self.lr()
254
+ states_values, grad_values, momentum_values = to_same_dict_tree(self.weight_states, grads, self.momentum_states)
255
+ momentum_values = jax.tree.map(lambda vv, gg: self.momentum * vv - lr * gg,
256
+ momentum_values,
257
+ grad_values)
258
+ new_weight_values = jax.tree.map(functools.partial(_sgd, lr=lr, weight_decay=self.weight_decay),
259
+ states_values,
260
+ momentum_values)
261
+ self.momentum_states.assign_values(momentum_values)
262
+ self.weight_states.assign_values(new_weight_values)
263
+ self.lr.step_call()
264
+
265
+
266
+ class MomentumNesterov(_WeightDecayOptimizer):
267
+ r"""Nesterov accelerated gradient optimizer [2]_.
268
+
269
+ .. math::
270
+
271
+ \begin{align}
272
+ \begin{split}
273
+ v_t &= \gamma v_{t-1} + \eta \nabla_\theta J( \theta - \gamma v_{t-1} ) \\
274
+ \theta &= \theta - v_t
275
+ \end{split}
276
+ \end{align}
277
+
278
+ Parameters
279
+ ----------
280
+ lr: float, LRScheduler
281
+ learning rate.
282
+
283
+ References
284
+ ----------
285
+ .. [2] Nesterov, Y. (1983). A method for unconstrained convex minimization problem with the rate of convergence o(1/k2). Doklady ANSSSR (translated as Soviet.Math.Docl.), vol. 269, pp. 543– 547.
286
+
287
+ """
288
+
289
+ def __init__(
290
+ self,
291
+ lr: Union[float, LRScheduler, bc.State],
292
+ weight_decay: Optional[float] = None,
293
+ momentum: float = 0.9,
294
+ name: Optional[str] = None
295
+ ):
296
+ super(MomentumNesterov, self).__init__(lr=lr, weight_decay=weight_decay, name=name)
297
+
298
+ self.momentum = fcast(momentum)
299
+ self.momentum_states = bc.visible_state_dict()
300
+
301
+ def extra_repr(self) -> str:
302
+ return f", momentum={self.momentum}"
303
+
304
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
305
+ train_states = dict() if train_states is None else train_states
306
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
307
+ for k, v in train_states.items():
308
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
309
+ self.weight_states.add_unique_elem(k, v)
310
+ self.momentum_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
311
+
312
+ def update(self, grads: dict):
313
+ lr = self.lr()
314
+ states_values, grad_values, momentum_values = to_same_dict_tree(self.weight_states, grads, self.momentum_states)
315
+ momentum_values = jax.tree.map(lambda mv, gv: self.momentum * mv - lr * gv,
316
+ momentum_values,
317
+ grad_values)
318
+ weight_values = jax.tree.map(functools.partial(_sgd, lr=lr, weight_decay=self.weight_decay),
319
+ states_values,
320
+ momentum_values)
321
+ self.weight_states.assign_values(weight_values)
322
+ self.momentum_states.assign_values(momentum_values)
323
+ self.lr.step_call()
324
+
325
+
326
+ class Adagrad(_WeightDecayOptimizer):
327
+ r"""Optimizer that implements the Adagrad algorithm.
328
+
329
+ Adagrad [3]_ is an optimizer with parameter-specific learning rates, which are
330
+ adapted relative to how frequently a parameter gets updated during training.
331
+ The more updates a parameter receives, the smaller the updates.
332
+
333
+ .. math::
334
+
335
+ \theta_{t+1} = \theta_{t} - \dfrac{\eta}{\sqrt{G_{t} + \epsilon}} \odot g_{t}
336
+
337
+ where :math:`G(t)` contains the sum of the squares of the past gradients
338
+
339
+ One of Adagrad's main benefits is that it eliminates the need to manually tune
340
+ the learning rate. Most implementations use a default value of 0.01 and leave it at that.
341
+ Adagrad's main weakness is its accumulation of the squared gradients in the denominator:
342
+ Since every added term is positive, the accumulated sum keeps growing during training.
343
+ This in turn causes the learning rate to shrink and eventually become infinitesimally
344
+ small, at which point the algorithm is no longer able to acquire additional knowledge.
345
+
346
+ Parameters
347
+ ----------
348
+ lr: float, LRScheduler
349
+ learning rate.
350
+
351
+ References
352
+ ----------
353
+ .. [3] Duchi, J., Hazan, E., & Singer, Y. (2011). Adaptive Subgradient Methods for Online Learning and Stochastic Optimization. Journal of Machine Learning Research, 12, 2121–2159. Retrieved from http://jmlr.org/papers/v12/duchi11a.html
354
+
355
+ """
356
+
357
+ def __init__(
358
+ self,
359
+ lr: Union[float, LRScheduler, bc.State],
360
+ weight_decay: Optional[float] = None,
361
+ epsilon: float = 1e-6,
362
+ name: Optional[str] = None
363
+ ):
364
+ super(Adagrad, self).__init__(lr=lr, weight_decay=weight_decay, name=name)
365
+ self.epsilon = fcast(epsilon)
366
+ self.cache_states = bc.visible_state_dict()
367
+
368
+ def extra_repr(self) -> str:
369
+ return f", epsilon={self.epsilon}"
370
+
371
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
372
+ train_states = dict() if train_states is None else train_states
373
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
374
+ for k, v in train_states.items():
375
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
376
+ self.weight_states.add_unique_elem(k, v)
377
+ self.cache_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
378
+
379
+ def update(self, grads: dict):
380
+ lr = self.lr()
381
+ cache_values, grad_values, weight_values = to_same_dict_tree(self.cache_states, grads, self.weight_states)
382
+ cache_values = jax.tree.map(lambda cv, gv: cv + gv ** 2, cache_values, grad_values)
383
+ updates = jax.tree.map(lambda cv, gv: lr * gv / jnp.sqrt(cv + self.epsilon), cache_values, grad_values)
384
+ weight_values = jax.tree.map(functools.partial(_sgd, weight_decay=self.weight_decay),
385
+ weight_values,
386
+ updates)
387
+ self.cache_states.assign_values(cache_values)
388
+ self.weight_states.assign_values(weight_values)
389
+ self.lr.step_call()
390
+
391
+
392
+ class Adadelta(_WeightDecayOptimizer):
393
+ r"""Optimizer that implements the Adadelta algorithm.
394
+
395
+ Adadelta [4]_ optimization is a stochastic gradient descent method that is based
396
+ on adaptive learning rate per dimension to address two drawbacks:
397
+
398
+ - The continual decay of learning rates throughout training.
399
+ - The need for a manually selected global learning rate.
400
+
401
+ Adadelta is a more robust extension of Adagrad that adapts learning rates based on
402
+ a moving window of gradient updates, instead of accumulating all past gradients.
403
+ This way, Adadelta continues learning even when many updates have been done. Compared
404
+ to Adagrad, in the original version of Adadelta you don't have to set an initial
405
+ learning rate.
406
+
407
+ .. math::
408
+
409
+ \boldsymbol{s}_t \leftarrow \rho \boldsymbol{s}_{t-1} + (1 - \rho) \boldsymbol{g}_t \odot \boldsymbol{g}_t, \\
410
+ \boldsymbol{g}_t' \leftarrow \sqrt{\frac{\Delta\boldsymbol{x}_{t-1} + \epsilon}{\boldsymbol{s}_t + \epsilon}} \odot \boldsymbol{g}_t, \\
411
+ \boldsymbol{x}_t \leftarrow \boldsymbol{x}_{t-1} - \boldsymbol{g}'_t, \\
412
+ \Delta\boldsymbol{x}_t \leftarrow \rho \Delta\boldsymbol{x}_{t-1} + (1 - \rho) \boldsymbol{g}'_t \odot \boldsymbol{g}'_t.
413
+
414
+ :math:`\rho` should be between 0 and 1. A value of rho close to 1 will decay the
415
+ moving average slowly and a value close to 0 will decay the moving average fast.
416
+
417
+ :math:`\rho` = 0.95 and :math:`\epsilon`=1e-6 are suggested in the paper and reported
418
+ to work for multiple datasets (MNIST, speech).
419
+
420
+ In the paper, no learning rate is considered (so learning_rate=1.0). Probably best to
421
+ keep it at this value. epsilon is important for the very first update (so the
422
+ numerator does not become 0).
423
+
424
+ Parameters
425
+ ----------
426
+ lr: float, LRScheduler
427
+ learning rate.
428
+
429
+ References
430
+ ----------
431
+ .. [4] Zeiler, M. D. (2012). ADADELTA: An Adaptive Learning Rate Method. Retrieved from http://arxiv.org/abs/1212.5701
432
+
433
+ """
434
+
435
+ def __init__(
436
+ self,
437
+ lr: Union[float, LRScheduler, bc.State] = 0.01,
438
+ weight_decay: Optional[float] = None,
439
+ epsilon: float = 1e-6,
440
+ rho: float = 0.95,
441
+ name: Optional[str] = None
442
+ ):
443
+ super(Adadelta, self).__init__(lr=lr, weight_decay=weight_decay, name=name)
444
+
445
+ self.epsilon = fcast(epsilon)
446
+ self.rho = fcast(rho)
447
+ self.cache_states = bc.visible_state_dict()
448
+ self.delta_states = bc.visible_state_dict()
449
+
450
+ def extra_repr(self) -> str:
451
+ return f", epsilon={self.epsilon}, rho={self.rho}"
452
+
453
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
454
+ train_states = dict() if train_states is None else train_states
455
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
456
+ for k, v in train_states.items():
457
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
458
+ self.weight_states.add_unique_elem(k, v)
459
+ self.cache_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
460
+ self.delta_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
461
+
462
+ def update(self, grads: dict):
463
+ weight_values, grad_values, cache_values, delta_values = to_same_dict_tree(
464
+ self.weight_states, grads, self.cache_states, self.delta_states)
465
+ cache_values = jax.tree.map(lambda cv, gv: self.rho * cv + (1 - self.rho) * gv ** 2, cache_values, grad_values)
466
+ updates = jax.tree.map(lambda gv, dv, cv: gv * jnp.sqrt(dv + self.epsilon) / jnp.sqrt(cv + self.epsilon),
467
+ grad_values, delta_values, cache_values)
468
+ delta_values = jax.tree.map(lambda dv, upd: self.rho * dv + (1 - self.rho) * upd ** 2, delta_values, updates)
469
+ weight_values = jax.tree.map(functools.partial(_sgd, weight_decay=self.weight_decay),
470
+ weight_values,
471
+ updates)
472
+ self.weight_states.assign_values(weight_values)
473
+ self.delta_states.assign_values(delta_values)
474
+ self.cache_states.assign_values(cache_values)
475
+ self.lr.step_call()
476
+
477
+
478
+ class RMSProp(_WeightDecayOptimizer):
479
+ r"""Optimizer that implements the RMSprop algorithm.
480
+
481
+ RMSprop [5]_ and Adadelta have both been developed independently around the same time
482
+ stemming from the need to resolve Adagrad's radically diminishing learning rates.
483
+
484
+ The gist of RMSprop is to:
485
+
486
+ - Maintain a moving (discounted) average of the square of gradients
487
+ - Divide the gradient by the root of this average
488
+
489
+ .. math::
490
+
491
+ \begin{split}c_t &= \rho c_{t-1} + (1-\rho)*g^2\\
492
+ p_t &= \frac{\eta}{\sqrt{c_t + \epsilon}} * g \end{split}
493
+
494
+ The centered version additionally maintains a moving average of the gradients,
495
+ and uses that average to estimate the variance.
496
+
497
+ Parameters
498
+ ----------
499
+ lr: float, LRScheduler
500
+ learning rate.
501
+
502
+ References
503
+ ----------
504
+ .. [5] Tieleman, T. and Hinton, G. (2012):
505
+ Neural Networks for Machine Learning, Lecture 6.5 - rmsprop.
506
+ Coursera. http://www.youtube.com/watch?v=O3sxAc4hxZU (formula @5:20)
507
+ """
508
+
509
+ def __init__(
510
+ self,
511
+ lr: Union[float, LRScheduler, bc.State],
512
+ weight_decay: Optional[float] = None,
513
+ epsilon: float = 1e-6,
514
+ rho: float = 0.9,
515
+ name: Optional[str] = None
516
+ ):
517
+ super(RMSProp, self).__init__(lr=lr, weight_decay=weight_decay, name=name)
518
+
519
+ self.epsilon = fcast(epsilon)
520
+ self.rho = fcast(rho)
521
+ self.cache_states = bc.visible_state_dict()
522
+
523
+ def extra_repr(self) -> str:
524
+ return f", epsilon={self.epsilon}, rho={self.rho}"
525
+
526
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
527
+ train_states = dict() if train_states is None else train_states
528
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
529
+ for k, v in train_states.items():
530
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
531
+ self.weight_states.add_unique_elem(k, v)
532
+ self.cache_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
533
+
534
+ def update(self, grads: dict):
535
+ lr = self.lr()
536
+ weight_values, grad_values, cache_values = to_same_dict_tree(self.weight_states, grads, self.cache_states)
537
+ cache_values = jax.tree.map(lambda cv, gv: self.rho * cv + (1 - self.rho) * gv ** 2, cache_values, grad_values)
538
+ update = jax.tree.map(lambda gv, cv: lr * gv / jnp.sqrt(cv + self.epsilon), grad_values, cache_values)
539
+ weight_values = jax.tree.map(functools.partial(_sgd, weight_decay=self.weight_decay),
540
+ weight_values,
541
+ update)
542
+ self.weight_states.assign_values(weight_values)
543
+ self.cache_states.assign_values(cache_values)
544
+ self.lr.step_call()
545
+
546
+
547
+ class Adam(_WeightDecayOptimizer):
548
+ """Optimizer that implements the Adam algorithm.
549
+
550
+ Adam [6]_ - a stochastic gradient descent method (SGD) that computes
551
+ individual adaptive learning rates for different parameters from estimates of
552
+ first- and second-order moments of the gradients.
553
+
554
+ Parameters
555
+ ----------
556
+ lr: float, LRScheduler
557
+ learning rate.
558
+ beta1: optional, float
559
+ A positive scalar value for beta_1, the exponential decay rate
560
+ for the first moment estimates (default 0.9).
561
+ beta2: optional, float
562
+ A positive scalar value for beta_2, the exponential decay rate
563
+ for the second moment estimates (default 0.999).
564
+ eps: optional, float
565
+ A positive scalar value for epsilon, a small constant for
566
+ numerical stability (default 1e-8).
567
+ name : optional, str
568
+ The optimizer name.
569
+
570
+ References
571
+ ----------
572
+ .. [6] Kingma, D. P., & Ba, J. (2014). Adam: A method for stochastic optimization. arXiv preprint arXiv:1412.6980.
573
+ """
574
+
575
+ def __init__(
576
+ self,
577
+ lr: Union[float, bc.State, LRScheduler],
578
+ beta1: float = 0.9,
579
+ beta2: float = 0.999,
580
+ eps: float = 1e-8,
581
+ weight_decay: Optional[float] = None,
582
+ name: Optional[str] = None
583
+ ):
584
+ super(Adam, self).__init__(lr=lr,
585
+ weight_decay=weight_decay,
586
+ name=name)
587
+
588
+ self.beta1 = fcast(beta1)
589
+ self.beta2 = fcast(beta2)
590
+ self.eps = fcast(eps)
591
+ self.m1_states = bc.visible_state_dict()
592
+ self.m2_states = bc.visible_state_dict()
593
+
594
+ def extra_repr(self) -> str:
595
+ return f", beta1={self.beta1}, beta2={self.beta2}, eps={self.eps}"
596
+
597
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
598
+ train_states = dict() if train_states is None else train_states
599
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
600
+
601
+ for k, v in train_states.items():
602
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
603
+ self.weight_states.add_unique_elem(k, v)
604
+ self.m1_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
605
+ self.m2_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
606
+
607
+ def update(self, grads: dict):
608
+ lr = self.lr()
609
+ lr = lr / (1 - self.beta1 ** (self.lr.last_epoch.value + 2))
610
+ lr = lr * jnp.sqrt(1 - self.beta2 ** (self.lr.last_epoch.value + 2))
611
+ weight_values, grad_values, m1_values, m2_values = to_same_dict_tree(
612
+ self.weight_states, grads, self.m1_states, self.m2_states)
613
+ m1_values = jax.tree.map(lambda m1, gv: self.beta1 * m1 + (1 - self.beta1) * gv, m1_values, grad_values)
614
+ m2_values = jax.tree.map(lambda m2, gv: self.beta2 * m2 + (1 - self.beta2) * gv ** 2, m2_values, grad_values)
615
+ update = jax.tree.map(lambda m1, m2: lr * m1 / (jnp.sqrt(m2) + self.eps), m1_values, m2_values)
616
+ weight_values = jax.tree.map(functools.partial(_sgd, weight_decay=self.weight_decay),
617
+ weight_values,
618
+ update)
619
+ self.weight_states.assign_values(weight_values)
620
+ self.m1_states.assign_values(m1_values)
621
+ self.m2_states.assign_values(m2_values)
622
+ self.lr.step_call()
623
+
624
+
625
+ class LARS(_WeightDecayOptimizer):
626
+ r"""Layer-wise adaptive rate scaling (LARS) optimizer [1]_.
627
+
628
+ Layer-wise Adaptive Rate Scaling, or LARS, is a large batch
629
+ optimization technique. There are two notable differences
630
+ between LARS and other adaptive algorithms such as `Adam` or `RMSProp`:
631
+ first, LARS uses a separate learning rate for each layer and not for
632
+ each weight. And second, the magnitude of the update is controlled
633
+ with respect to the weight norm for better control of training speed.
634
+
635
+ .. math::
636
+
637
+ m_{t} = \beta_{1}m_{t-1} + \left(1-\beta_{1}\right)\left(g_{t} + \lambda{x_{t}}\right) \\
638
+ x_{t+1}^{\left(i\right)} = x_{t}^{\left(i\right)} - \eta_{t}\frac{\phi\left(|| x_{t}^{\left(i\right)} ||\right)}{|| m_{t}^{\left(i\right)} || }m_{t}^{\left(i\right)}
639
+
640
+ Parameters
641
+ ----------
642
+ lr: float, LRScheduler
643
+ learning rate.
644
+ momentum: float
645
+ coefficient used for the moving average of the gradient.
646
+ weight_decay: float
647
+ weight decay coefficient.
648
+ tc: float
649
+ trust coefficient eta ( < 1) for trust ratio computation.
650
+ eps: float
651
+ epsilon used for trust ratio computation.
652
+
653
+ References
654
+ ----------
655
+ .. [1] You, Yang, Igor Gitman and Boris Ginsburg. “Large Batch Training of Convolutional Networks.” arXiv: Computer Vision and Pattern Recognition (2017): n. pag.
656
+ """
657
+
658
+ def __init__(
659
+ self,
660
+ lr: Union[float, LRScheduler, bc.State],
661
+ momentum: float = 0.9,
662
+ weight_decay: float = 1e-4,
663
+ tc: float = 1e-3,
664
+ eps: float = 1e-5,
665
+ name: Optional[str] = None
666
+ ):
667
+ super(LARS, self).__init__(lr=lr,
668
+ weight_decay=weight_decay,
669
+ name=name)
670
+ assert self.weight_decay is None, 'LARS does not support weight decay.'
671
+
672
+ self.momentum = fcast(momentum)
673
+ self.tc = fcast(tc)
674
+ self.eps = fcast(eps)
675
+ self.momentum_states = bc.visible_state_dict()
676
+
677
+ def extra_repr(self) -> str:
678
+ return f", momentum={self.momentum}, tc={self.tc}, eps={self.eps}"
679
+
680
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
681
+ train_states = dict() if train_states is None else train_states
682
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
683
+ for k, v in train_states.items():
684
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
685
+ self.weight_states.add_unique_elem(k, v)
686
+ self.momentum_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
687
+
688
+ def update(self, grads: dict):
689
+ lr = self.lr()
690
+ weight_values, grad_values, momentum_values = to_same_dict_tree(self.weight_states, grads, self.momentum_states)
691
+
692
+ def _lars_update(pv, gv, mv):
693
+ p_norm = jnp.linalg.norm(pv)
694
+ g_norm = jnp.linalg.norm(gv)
695
+ trust_ratio = self.tc * p_norm / (g_norm + self.weight_decay * p_norm + self.eps)
696
+ local_lr = lr * jnp.maximum(jnp.logical_or(p_norm == 0, g_norm == 0), trust_ratio)
697
+ mv = self.momentum * mv + local_lr * (gv + self.weight_decay * pv)
698
+ return mv
699
+
700
+ momentum_values = jax.tree.map(_lars_update, weight_values, grad_values, momentum_values)
701
+ weight_values = jax.tree.map(lambda pv, mv: pv - mv, weight_values, momentum_values)
702
+ self.weight_states.assign_values(weight_values)
703
+ self.momentum_states.assign_values(momentum_values)
704
+ self.lr.step_call()
705
+
706
+
707
+ class Adan(_WeightDecayOptimizer):
708
+ r"""Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models [1]_.
709
+
710
+ .. math::
711
+
712
+ \begin{equation}
713
+ \begin{aligned}
714
+ & \mathbf{m}_k=\left(1-\beta_1\right) \mathbf{m}_{k-1}+\beta_1 \mathbf{g}_k \\
715
+ & \mathbf{v}_k=\left(1-\beta_2\right) \mathbf{v}_{k-1}+\beta_2\left(\mathbf{g}_k-\mathbf{g}_{k-1}\right) \\
716
+ & \mathbf{n}_k=\left(1-\beta_3\right) \mathbf{n}_{k-1}+\beta_3\left[\mathbf{g}_k+\left(1-\beta_2\right)\left(\mathbf{g}_k-\mathbf{g}_{k-1}\right)\right]^2 \\
717
+ & \boldsymbol{\eta}_k=\eta /\left(\sqrt{\mathbf{n}_k+\varepsilon}\right) \\
718
+ & \boldsymbol{\theta}_{k+1}=\left(1+\lambda_k \eta\right)^{-1}\left[\boldsymbol{\theta}_k-\boldsymbol{\eta}_k \circ\left(\mathbf{m}_k+\left(1-\beta_2\right) \mathbf{v}_k\right)\right] \\
719
+ \end{aligned}
720
+ \end{equation}
721
+
722
+ Parameters
723
+ ----------
724
+ lr: float, LRScheduler
725
+ learning rate. Can be much higher than Adam, up to 5-10x. (default: 1e-3)
726
+ betas : tuple
727
+ Coefficients used for computing running averages of gradient and its norm. (default: (0.02, 0.08, 0.01))
728
+ eps : float
729
+ The term added to the denominator to improve numerical stability. (default: 1e-8)
730
+ weight_decay : float
731
+ decoupled weight decay (L2 penalty) (default: 0)
732
+ no_prox: bool
733
+ how to perform the decoupled weight decay (default: False).
734
+ It determines the update rule of parameters with weight decay.
735
+ By default, Adan updates the parameters in the way presented in Algorithm 1 in the paper:
736
+
737
+ .. math::
738
+ \boldsymbol{\theta}_{k+1} = ( 1+\lambda \eta)^{-1}\left[\boldsymbol{\theta}_k - \boldsymbol{\eta}_k \circ (\mathbf{m}_k+(1-{\color{blue}\beta_2})\mathbf{v}k)\right],
739
+
740
+ But one also can update the parameter like Adamw:
741
+
742
+ .. math::
743
+ \boldsymbol{\theta}_{k+1} = ( 1-\lambda \eta)\boldsymbol{\theta}_k - \boldsymbol{\eta}_k \circ (\mathbf{m}_k+(1-{\color{blue}\beta_2})\mathbf{v}_k).
744
+
745
+ References
746
+ ----------
747
+ .. [1] Xie, Xingyu, Pan Zhou, Huan Li, Zhouchen Lin and Shuicheng Yan.
748
+ “Adan: Adaptive Nesterov Momentum Algorithm for Faster Optimizing
749
+ Deep Models.” ArXiv abs/2208.06677 (2022): n. pag.
750
+ """
751
+
752
+ def __init__(
753
+ self,
754
+ lr: Union[float, LRScheduler, bc.State] = 1e-3,
755
+ betas: Tuple[float, float, float] = (0.02, 0.08, 0.01),
756
+ eps: float = 1e-8,
757
+ weight_decay: float = 0.02,
758
+ no_prox: bool = False,
759
+ name: Optional[str] = None,
760
+ ):
761
+ super(Adan, self).__init__(lr=lr, weight_decay=weight_decay, name=name)
762
+
763
+ assert len(betas) == 3
764
+ if eps < 0.:
765
+ raise ValueError("Invalid epsilon value: {}".format(eps))
766
+ if not 0.0 <= betas[0] < 1.0:
767
+ raise ValueError("Invalid beta parameter at index 0: {}".format(betas[0]))
768
+ if not 0.0 <= betas[1] < 1.0:
769
+ raise ValueError("Invalid beta parameter at index 1: {}".format(betas[1]))
770
+ if not 0.0 <= betas[2] < 1.0:
771
+ raise ValueError("Invalid beta parameter at index 2: {}".format(betas[2]))
772
+
773
+ self.betas = fcast(jnp.asarray(betas))
774
+ self.eps = fcast(eps)
775
+ self.no_prox = no_prox
776
+ self.exp_avg_states = bc.visible_state_dict()
777
+ self.exp_avg_sq_states = bc.visible_state_dict()
778
+ self.exp_avg_diff_states = bc.visible_state_dict()
779
+ self.pre_grad_states = bc.visible_state_dict()
780
+
781
+ def extra_repr(self) -> str:
782
+ return f", betas={self.betas}, eps={self.eps}, weight_decay={self.weight_decay}, no_prox={self.no_prox}"
783
+
784
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
785
+ train_states = dict() if train_states is None else train_states
786
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
787
+ for k, v in train_states.items():
788
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
789
+ self.weight_states.add_unique_elem(k, v)
790
+ self.exp_avg_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
791
+ self.exp_avg_sq_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
792
+ self.exp_avg_diff_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
793
+ self.pre_grad_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
794
+
795
+ def update(self, grads: dict):
796
+ lr = self.lr()
797
+ step = self.lr.last_epoch.value + 1
798
+ correct_m = 1 / (1 - (1 - self.betas[0]) ** (step + 1))
799
+ correct_v = 1 / (1 - (1 - self.betas[1]) ** (step + 1))
800
+ correct_n = 1 / (1 - (1 - self.betas[2]) ** (step + 1))
801
+ m_values, n_values, v_values, pre_g_values, weight_values, grad_values = to_same_dict_tree(
802
+ self.exp_avg_states, self.exp_avg_diff_states, self.exp_avg_sq_states, self.pre_grad_states,
803
+ self.weight_states, grads)
804
+
805
+ def _adan_update(m, n, v, pre_g, g, p):
806
+ m = m * (1 - self.betas[0]) + self.betas[0] * g
807
+ gd = g - pre_g
808
+ v = v * (1 - self.betas[1]) + self.betas[1] * gd
809
+ n = n * (1 - self.betas[2]) + self.betas[2] * (g + (1 - self.betas[1]) * gd) ** 2
810
+ weighted_step_size = lr / (jnp.sqrt(n * correct_n) + self.eps)
811
+ if self.no_prox:
812
+ p = (p * (1 - self.weight_decay * lr) -
813
+ weighted_step_size * (m * correct_m + (1 - self.betas[1]) * v * correct_v))
814
+ else:
815
+ p = ((p - weighted_step_size * (m * correct_m + (1 - self.betas[1]) * v * correct_v)) /
816
+ (1 + self.weight_decay * lr))
817
+ return m, n, v, p
818
+
819
+ m_values, n_values, v_values, weight_values = jax.tree.map(
820
+ _adan_update, m_values, n_values, v_values, pre_g_values, grad_values, weight_values)
821
+ self.exp_avg_states.assign_values(m_values)
822
+ self.exp_avg_diff_states.assign_values(n_values)
823
+ self.exp_avg_sq_states.assign_values(v_values)
824
+ self.weight_states.assign_values(weight_values)
825
+ self.lr.step_call()
826
+
827
+
828
+ class AdamW(_WeightDecayOptimizer):
829
+ r"""Adam with weight decay regularization [1]_.
830
+
831
+ AdamW uses weight decay to regularize learning towards small weights, as
832
+ this leads to better generalization. In SGD you can also use L2 regularization
833
+ to implement this as an additive loss term, however L2 regularization
834
+ does not behave as intended for adaptive gradient algorithms such as Adam.
835
+
836
+ .. math::
837
+
838
+ \begin{aligned}
839
+ &\rule{110mm}{0.4pt} \\
840
+ &\textbf{input} : \gamma \text{(lr)}, \: \beta_1, \beta_2
841
+ \text{(betas)}, \: \theta_0 \text{(params)}, \: f(\theta) \text{(objective)},
842
+ \: \epsilon \text{ (epsilon)} \\
843
+ &\hspace{13mm} \lambda \text{(weight decay)}, \: \textit{amsgrad},
844
+ \: \textit{maximize} \\
845
+ &\textbf{initialize} : m_0 \leftarrow 0 \text{ (first moment)}, v_0 \leftarrow 0
846
+ \text{ ( second moment)}, \: \widehat{v_0}^{max}\leftarrow 0 \\[-1.ex]
847
+ &\rule{110mm}{0.4pt} \\
848
+ &\textbf{for} \: t=1 \: \textbf{to} \: \ldots \: \textbf{do} \\
849
+
850
+ &\hspace{5mm}\textbf{if} \: \textit{maximize}: \\
851
+ &\hspace{10mm}g_t \leftarrow -\nabla_{\theta} f_t (\theta_{t-1}) \\
852
+ &\hspace{5mm}\textbf{else} \\
853
+ &\hspace{10mm}g_t \leftarrow \nabla_{\theta} f_t (\theta_{t-1}) \\
854
+ &\hspace{5mm} \theta_t \leftarrow \theta_{t-1} - \gamma \lambda \theta_{t-1} \\
855
+ &\hspace{5mm}m_t \leftarrow \beta_1 m_{t-1} + (1 - \beta_1) g_t \\
856
+ &\hspace{5mm}v_t \leftarrow \beta_2 v_{t-1} + (1-\beta_2) g^2_t \\
857
+ &\hspace{5mm}\widehat{m_t} \leftarrow m_t/\big(1-\beta_1^t \big) \\
858
+ &\hspace{5mm}\widehat{v_t} \leftarrow v_t/\big(1-\beta_2^t \big) \\
859
+ &\hspace{5mm}\textbf{if} \: amsgrad \\
860
+ &\hspace{10mm}\widehat{v_t}^{max} \leftarrow \mathrm{max}(\widehat{v_t}^{max},
861
+ \widehat{v_t}) \\
862
+ &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
863
+ \big(\sqrt{\widehat{v_t}^{max}} + \epsilon \big) \\
864
+ &\hspace{5mm}\textbf{else} \\
865
+ &\hspace{10mm}\theta_t \leftarrow \theta_t - \gamma \widehat{m_t}/
866
+ \big(\sqrt{\widehat{v_t}} + \epsilon \big) \\
867
+ &\rule{110mm}{0.4pt} \\[-1.ex]
868
+ &\bf{return} \: \theta_t \\[-1.ex]
869
+ &\rule{110mm}{0.4pt} \\[-1.ex]
870
+ \end{aligned}
871
+
872
+
873
+ Parameters
874
+ ----------
875
+ lr: float, LRScheduler
876
+ learning rate.
877
+ beta1: optional, float
878
+ A positive scalar value for beta_1, the exponential decay rate
879
+ for the first moment estimates. Generally close to 1.
880
+ beta2: optional, float
881
+ A positive scalar value for beta_2, the exponential decay rate
882
+ for the second moment estimates. Generally close to 1.
883
+ eps: optional, float
884
+ A positive scalar value for epsilon, a small constant for
885
+ numerical stability.
886
+ weight_decay: float
887
+ Strength of the weight decay regularization. Note that this
888
+ weight decay is multiplied with the learning rate.
889
+ amsgrad: bool
890
+ whether to use the AMSGrad variant of this algorithm
891
+ from the paper `On the Convergence of Adam and Beyond`.
892
+ name : optional, str
893
+ The optimizer name.
894
+
895
+ References
896
+ ----------
897
+ .. [1] Loshchilov, Ilya and Frank Hutter. “Decoupled Weight Decay Regularization.” International Conference on Learning Representations (2019).
898
+
899
+ """
900
+
901
+ def __init__(
902
+ self,
903
+ lr: Union[float, LRScheduler, bc.State],
904
+ beta1: float = 0.9,
905
+ beta2: float = 0.999,
906
+ eps: float = 1e-8,
907
+ weight_decay: float = 1e-2,
908
+ amsgrad: bool = False,
909
+ name: Optional[str] = None,
910
+ ):
911
+ super(AdamW, self).__init__(lr=lr,
912
+ weight_decay=weight_decay,
913
+ name=name)
914
+
915
+ if eps < 0.:
916
+ raise ValueError("Invalid epsilon value: {}".format(eps))
917
+ if not 0.0 <= beta1 < 1.0:
918
+ raise ValueError("Invalid beta parameter at index 0: {}".format(beta1))
919
+ if not 0.0 <= beta2 < 1.0:
920
+ raise ValueError("Invalid beta parameter at index 1: {}".format(beta2))
921
+ if weight_decay < 0.:
922
+ raise ValueError("Invalid weight_decay value: {}".format(weight_decay))
923
+
924
+ self.beta1 = fcast(beta1)
925
+ self.beta2 = fcast(beta2)
926
+ self.eps = fcast(eps)
927
+ self.amsgrad = amsgrad
928
+ self.m1_states = bc.visible_state_dict()
929
+ self.m2_states = bc.visible_state_dict()
930
+ if self.amsgrad:
931
+ self.vmax_states = bc.visible_state_dict()
932
+
933
+ def extra_repr(self) -> str:
934
+ return (f", beta1={self.beta1}, beta2={self.beta2}, eps={self.eps}"
935
+ f", weight_decay={self.weight_decay}, amsgrad={self.amsgrad}")
936
+
937
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
938
+ train_states = dict() if train_states is None else train_states
939
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
940
+ for k, v in train_states.items():
941
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
942
+ self.weight_states.add_unique_elem(k, v)
943
+ self.m1_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
944
+ self.m2_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
945
+ if self.amsgrad:
946
+ self.vmax_states[k] = OptimState(bc.math.tree_zeros_like(v.value))
947
+
948
+ def update(self, grads: dict):
949
+ lr_old = self.lr()
950
+ step = self.lr.last_epoch.value + 2
951
+ bias_correction1 = 1 - self.beta1 ** step
952
+ bias_correction2 = 1 - self.beta2 ** step
953
+ lr = lr_old * jnp.sqrt(bias_correction2) / bias_correction1
954
+
955
+ def _adamw_update(p, m, v, g, vmax=None):
956
+ if self.weight_decay != 0:
957
+ p *= (1 - lr_old * self.weight_decay)
958
+ m = self.beta1 * m + (1 - self.beta1) * g
959
+ v = self.beta2 * v + (1 - self.beta2) * g ** 2
960
+ if self.amsgrad:
961
+ vmax = jnp.maximum(vmax, v)
962
+ denom = jnp.sqrt(vmax) + self.eps
963
+ return p - lr * m / denom, m, v, vmax
964
+ else:
965
+ denom = jnp.sqrt(v.value) + self.eps
966
+ return p - lr * m / denom, m, v
967
+
968
+ if self.amsgrad:
969
+ weight_values, m1_values, m2_values, vmax_values = to_same_dict_tree(
970
+ self.weight_states, self.m1_states, self.m2_states, self.vmax_states)
971
+ weight_values, m1_values, m2_values, vmax_values = jax.tree.map(
972
+ _adamw_update, weight_values, m1_values, m2_values, grads, vmax_values)
973
+ self.vmax_states.assign_values(vmax_values)
974
+ else:
975
+ weight_values, m1_values, m2_values = to_same_dict_tree(self.weight_states, self.m1_states, self.m2_states)
976
+ weight_values, m1_values, m2_values = jax.tree.map(
977
+ _adamw_update, weight_values, m1_values, m2_values, grads)
978
+ self.weight_states.assign_values(weight_values)
979
+ self.m1_states.assign_values(m1_values)
980
+ self.m2_states.assign_values(m2_values)
981
+ self.lr.step_call()
982
+
983
+
984
+ class SM3(_WeightDecayOptimizer):
985
+ """SM3 algorithm [1]_.
986
+
987
+ The 'Square-root of Minima of Sums of Maxima of Squared-gradients Method'
988
+ (SM3) algorithm is a memory-efficient adaptive optimization algorithm similar
989
+ to Adam and Adagrad with greatly reduced memory usage for history tensors.
990
+ For an `n x m` matrix, Adam and Adagrad use `O(nm)` memory for history
991
+ tensors, while SM3 uses `O(n+m)` due to the chosen cover. In general, a tensor
992
+ of shape `(n_1, n_2, ..., n_k)` optimized using Adam will use `O(prod n_i)`
993
+ memory for storage tensors, while the optimization using SM3 will use
994
+ `O(sum n_i)` memory. Despite storing fewer parameters, this optimization
995
+ algorithm manages to be comparably effective.
996
+
997
+ This advantage drastically shrinks when `momentum > 0`. The momentum is
998
+ tracked using a tensor of the same shape as the tensor being optimized. With
999
+ momentum, SM3 will use just over half as much memory as Adam, and a bit more
1000
+ than Adagrad.
1001
+
1002
+ Parameters
1003
+ ----------
1004
+ lr: float, LRScheduler
1005
+ learning rate.
1006
+ momentum: float
1007
+ coefficient used to scale prior updates
1008
+ before adding. This drastically increases memory usage if
1009
+ `momentum > 0.0`. (default: 0.0)
1010
+ beta: float
1011
+ coefficient used for exponential moving averages (default: 0.0)
1012
+ eps: float
1013
+ Term added to square-root in denominator to
1014
+ improve numerical stability (default: 1e-30).
1015
+
1016
+ References
1017
+ ----------
1018
+ .. [1] Anil, Rohan, Vineet Gupta, Tomer Koren and Yoram Singer. “Memory Efficient Adaptive Optimization.” Neural Information Processing Systems (2019).
1019
+
1020
+ """
1021
+
1022
+ def __init__(
1023
+ self,
1024
+ lr: Union[float, LRScheduler, bc.State],
1025
+ beta: float = 0.,
1026
+ momentum: float = 0.,
1027
+ eps: float = 1e-30,
1028
+ weight_decay: Optional[float] = None,
1029
+ name: Optional[str] = None,
1030
+ ):
1031
+ super(SM3, self).__init__(lr=lr,
1032
+ weight_decay=weight_decay,
1033
+ name=name)
1034
+
1035
+ if not 0.0 <= momentum < 1.0:
1036
+ raise ValueError("Invalid momentum: {0}".format(momentum))
1037
+ if not 0.0 <= beta < 1.0:
1038
+ raise ValueError("Invalid beta: {0}".format(beta))
1039
+ if not 0.0 <= eps:
1040
+ raise ValueError("Invalid eps: {0}".format(eps))
1041
+
1042
+ self.eps = fcast(eps)
1043
+ self.beta = fcast(beta)
1044
+ self.momentum = fcast(momentum)
1045
+ self.memory_states = bc.visible_state_dict()
1046
+
1047
+ def extra_repr(self) -> str:
1048
+ return f", beta={self.beta}, momentum={self.momentum}, eps={self.eps}"
1049
+
1050
+ def register_trainable_weights(self, train_states: Optional[Dict[str, bc.State]] = None):
1051
+ train_states = dict() if train_states is None else train_states
1052
+ assert isinstance(train_states, dict), '"states" must be a dict of braincore.State.'
1053
+ for k, v in train_states.items():
1054
+ assert isinstance(v, bc.State), f'"{k}" must be an instance of braincore.State.'
1055
+ self.weight_states.add_unique_elem(k, v)
1056
+ rank, ndim, dtype = v.value.shape, v.value.ndim, v.value.dtype
1057
+ for i in range(ndim):
1058
+ shape = [1] * ndim
1059
+ shape[i] = rank[i]
1060
+ self.memory_states[f'{k}_m{i}'] = bc.State(jnp.zeros(shape, dtype=dtype))
1061
+ if self.momentum > 0.:
1062
+ self.memory_states[f'{k}_mbuffer'] = bc.State(jnp.zeros_like(v.value))
1063
+
1064
+ def update(self, grads: dict):
1065
+ lr = self.lr()
1066
+
1067
+ for k, p in self.weight_states.items():
1068
+ g = grads[k]
1069
+ ndim = p.ndim
1070
+ update = self.memory_states[f'{k}_m0'].value
1071
+ for i in range(1, ndim):
1072
+ update = jnp.minimum(update, self.memory_states[f'{k}_m{i}'].value)
1073
+ if self.beta > 0.:
1074
+ update *= self.beta
1075
+ update += g * g * (1 - self.beta)
1076
+ # Computes max along all dimensions except the given dim.
1077
+ # If tensor is a scalar, it returns tensor.
1078
+ for i in range(ndim):
1079
+ result = update
1080
+ for j in range(ndim):
1081
+ if i != j:
1082
+ result = jnp.max(result, axis=j, keepdim=True)
1083
+ acc = self.memory_states[f'{k}_m{i}'].value
1084
+ if self.beta > 0.:
1085
+ acc.value = jnp.maximum(acc, result)
1086
+ else:
1087
+ # No need to compare - nu_max is bigger because of grad ** 2
1088
+ acc.value = result
1089
+ update = g / jnp.sqrt(update + self.eps)
1090
+ if self.momentum > 0.:
1091
+ m_buffer = self.memory_states[f'{k}_mbuffer'].value
1092
+ update = update * (1. - self.momentum) + m_buffer * self.momentum
1093
+ m_buffer.value = update
1094
+ if self.weight_decay is None:
1095
+ p.value -= lr * update
1096
+ else:
1097
+ p.value = (1 - self.weight_decay) * p - lr * update
1098
+ self.lr.step_call()