pytorch-optimizer 3.8.0__tar.gz → 3.8.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/PKG-INFO +133 -127
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/README.md +126 -123
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pyproject.toml +39 -12
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/__init__.py +3 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/base/exception.py +8 -6
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/base/optimizer.py +103 -94
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/base/scheduler.py +12 -10
- pytorch_optimizer-3.8.2/pytorch_optimizer/base/type.py +27 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/bi_tempered.py +50 -41
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/cross_entropy.py +5 -4
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/dice.py +23 -20
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/f1.py +4 -3
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/focal.py +24 -20
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/jaccard.py +21 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/ldam.py +6 -5
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/lovasz.py +7 -5
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/tversky.py +5 -4
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/__init__.py +9 -5
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/chebyshev.py +32 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/cosine_anealing.py +10 -9
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/experimental/deberta_v3_lr_scheduler.py +14 -13
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/linear_warmup.py +6 -5
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/proportion.py +16 -8
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/rex.py +6 -5
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/wsd.py +23 -23
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/__init__.py +10 -7
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/a2grad.py +16 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adabelief.py +22 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adabound.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adadelta.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adafactor.py +30 -29
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adagc.py +22 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adahessian.py +24 -23
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adai.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adalite.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adam_mini.py +21 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adamax.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adamc.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adamg.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adamod.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adamp.py +52 -49
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adams.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adamw.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adan.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adanorm.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adapnm.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adashift.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adasmooth.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/ademamix.py +40 -38
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/adopt.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/agc.py +7 -6
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/aggmo.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/aida.py +24 -23
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/alig.py +15 -14
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/amos.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/apollo.py +37 -35
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/avagrad.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/came.py +21 -20
- pytorch_optimizer-3.8.2/pytorch_optimizer/optimizer/conda.py +146 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/dadapt.py +92 -87
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/demo.py +32 -25
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/diffgrad.py +22 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/emonavi.py +129 -111
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/exadam.py +19 -23
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/experimental/ranger25.py +33 -32
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/fadam.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/fira.py +16 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/focus.py +16 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/fp16.py +45 -40
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/fromage.py +14 -13
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/ftrl.py +16 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/galore.py +16 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/galore_utils.py +77 -76
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/gradient_centralization.py +5 -3
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/grams.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/gravity.py +14 -13
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/grokfast.py +48 -50
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/kate.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/lamb.py +27 -26
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/laprop.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/lars.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/lion.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/lomo.py +27 -25
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/lookahead.py +16 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/madgrad.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/mars.py +28 -28
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/msvag.py +12 -11
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/muon.py +275 -71
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/nero.py +16 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/novograd.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/orthograd.py +10 -9
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/padam.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/pcgrad.py +17 -15
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/pid.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/pnm.py +18 -17
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/prodigy.py +28 -27
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/psgd.py +31 -30
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/psgd_utils.py +12 -11
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/qhadam.py +19 -18
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/qhm.py +17 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/racs.py +43 -41
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/radam.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/ranger.py +22 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/ranger21.py +50 -49
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/rotograd.py +47 -43
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/sam.py +342 -184
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/schedulefree.py +94 -93
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/scion.py +55 -49
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/sgd.py +101 -95
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/shampoo.py +63 -74
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/shampoo_utils.py +121 -96
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/sm3.py +18 -16
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/snsm.py +30 -30
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/soap.py +30 -30
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/sophia.py +22 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/spam.py +53 -46
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/splus.py +27 -26
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/srmm.py +12 -11
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/swats.py +20 -19
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/tam.py +24 -23
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/tiger.py +14 -13
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/trac.py +26 -24
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/utils.py +76 -66
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/yogi.py +17 -16
- pytorch_optimizer-3.8.0/pytorch_optimizer/base/type.py +0 -21
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/LICENSE +0 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/base/__init__.py +0 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/loss/__init__.py +0 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/lr_scheduler/experimental/__init__.py +0 -0
- {pytorch_optimizer-3.8.0 → pytorch_optimizer-3.8.2}/pytorch_optimizer/optimizer/experimental/__init__.py +0 -0
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
Metadata-Version: 2.
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
2
|
Name: pytorch_optimizer
|
|
3
|
-
Version: 3.8.
|
|
3
|
+
Version: 3.8.2
|
|
4
4
|
Summary: optimizer & lr scheduler & objective function collections in PyTorch
|
|
5
5
|
License: Apache-2.0
|
|
6
|
-
|
|
6
|
+
License-File: LICENSE
|
|
7
|
+
Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,Alice,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaGC,AdaMax,AdaMuon,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,Simplified-AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,CAME,Conda,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,EmoFact,EmoLynx,EmoNavi,EmoNeco,EmoZeal,EXAdam,FAdam,Fira,FOCUS,FriendlySAM,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,PSGD,QHAdam,QHM,RACS,RAdam,Ranger,Ranger21,RotoGrad,SAM,GCSAM,LookSAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SCION,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,StableSPAM,SPlus,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,VSGD,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
|
|
7
8
|
Author: kozistr
|
|
8
9
|
Author-email: kozistr@gmail.com
|
|
9
10
|
Maintainer: kozistr
|
|
@@ -22,6 +23,7 @@ Classifier: Programming Language :: Python :: 3.10
|
|
|
22
23
|
Classifier: Programming Language :: Python :: 3.11
|
|
23
24
|
Classifier: Programming Language :: Python :: 3.12
|
|
24
25
|
Classifier: Programming Language :: Python :: 3.13
|
|
26
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
25
27
|
Classifier: Topic :: Scientific/Engineering
|
|
26
28
|
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
27
29
|
Classifier: Topic :: Software Development
|
|
@@ -29,8 +31,9 @@ Classifier: Topic :: Software Development :: Libraries
|
|
|
29
31
|
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
30
32
|
Requires-Dist: numpy (<=1.24.4) ; python_version < "3.9"
|
|
31
33
|
Requires-Dist: numpy (>1.24.4) ; python_version >= "3.9"
|
|
32
|
-
Requires-Dist: torch (>=1.10) ; python_version >= "3.
|
|
34
|
+
Requires-Dist: torch (>=1.10) ; python_version >= "3.10"
|
|
33
35
|
Requires-Dist: torch (>=1.10,<2.5) ; python_version < "3.9"
|
|
36
|
+
Requires-Dist: torch (>=1.10,<2.9) ; python_version == "3.9"
|
|
34
37
|
Project-URL: Documentation, https://pytorch-optimizers.readthedocs.io/en/latest
|
|
35
38
|
Project-URL: Homepage, https://github.com/kozistr/pytorch_optimizer
|
|
36
39
|
Project-URL: Repository, https://github.com/kozistr/pytorch_optimizer
|
|
@@ -38,17 +41,17 @@ Description-Content-Type: text/markdown
|
|
|
38
41
|
|
|
39
42
|
# pytorch-optimizer
|
|
40
43
|
|
|
41
|
-
| |
|
|
42
|
-
|
|
43
|
-
| Build |  [](https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest)
|
|
44
|
-
| Quality | [](https://codecov.io/gh/kozistr/pytorch_optimizer) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/pr_codeql.yml)  [](https://github.com/charliermarsh/ruff) |
|
|
45
|
-
| Package | [](https://badge.fury.io/py/pytorch-optimizer) [](https://pypi.python.org/pypi/pytorch-optimizer/)
|
|
46
|
-
| Status | [](https://pepy.tech/project/pytorch-optimizer) [](https://pepy.tech/project/pytorch-optimizer)
|
|
47
|
-
| License | [](https://opensource.org/licenses/Apache-2.0)
|
|
44
|
+
| | |
|
|
45
|
+
|---------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
46
|
+
| Build |  [](https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest) |
|
|
47
|
+
| Quality | [](https://codecov.io/gh/kozistr/pytorch_optimizer) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/pr_codeql.yml)  [](https://github.com/charliermarsh/ruff) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/static_analysis.yml) |
|
|
48
|
+
| Package | [](https://badge.fury.io/py/pytorch-optimizer) [](https://pypi.python.org/pypi/pytorch-optimizer/) |
|
|
49
|
+
| Status | [](https://pepy.tech/project/pytorch-optimizer) [](https://pepy.tech/project/pytorch-optimizer) |
|
|
50
|
+
| License | [](https://opensource.org/licenses/Apache-2.0) |
|
|
48
51
|
|
|
49
52
|
## The reasons why you use `pytorch-optimizer`.
|
|
50
53
|
|
|
51
|
-
* Wide range of supported optimizers. Currently, **
|
|
54
|
+
* Wide range of supported optimizers. Currently, **129 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
|
|
52
55
|
* Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
|
|
53
56
|
* Easy to use, clean, and tested codes
|
|
54
57
|
* Active maintenance
|
|
@@ -143,121 +146,124 @@ get_supported_optimizers(['adam*', 'ranger*'])
|
|
|
143
146
|
# ['adamax', 'adamg', 'adammini', 'adamod', 'adamp', 'adams', 'adamw', 'ranger', 'ranger21']
|
|
144
147
|
```
|
|
145
148
|
|
|
146
|
-
| Optimizer | Description | Official Code | Paper
|
|
147
|
-
|
|
148
|
-
| AdaBelief | *Adapting Step-sizes by the Belief in Observed Gradients* | [github](https://github.com/juntang-zhuang/Adabelief-Optimizer) | <https://arxiv.org/abs/2010.07468>
|
|
149
|
-
| AdaBound | *Adaptive Gradient Methods with Dynamic Bound of Learning Rate* | [github](https://github.com/Luolc/AdaBound/blob/master/adabound/adabound.py) | <https://openreview.net/forum?id=Bkg3g2R9FX>
|
|
150
|
-
| AdaHessian | *An Adaptive Second Order Optimizer for Machine Learning* | [github](https://github.com/amirgholami/adahessian) | <https://arxiv.org/abs/2006.00719>
|
|
151
|
-
| AdamD | *Improved bias-correction in Adam* | | <https://arxiv.org/abs/2110.10828>
|
|
152
|
-
| AdamP | *Slowing Down the Slowdown for Momentum Optimizers on Scale-invariant Weights* | [github](https://github.com/clovaai/AdamP) | <https://arxiv.org/abs/2006.08217>
|
|
153
|
-
| diffGrad | *An Optimization Method for Convolutional Neural Networks* | [github](https://github.com/shivram1987/diffGrad) | <https://arxiv.org/abs/1909.11015v3>
|
|
154
|
-
| MADGRAD | *A Momentumized, Adaptive, Dual Averaged Gradient Method for Stochastic* | [github](https://github.com/facebookresearch/madgrad) | <https://arxiv.org/abs/2101.11075>
|
|
155
|
-
| RAdam | *On the Variance of the Adaptive Learning Rate and Beyond* | [github](https://github.com/LiyuanLucasLiu/RAdam) | <https://arxiv.org/abs/1908.03265>
|
|
156
|
-
| Ranger | *a synergistic optimizer combining RAdam and LookAhead, and now GC in one optimizer* | [github](https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer) | <https://bit.ly/3zyspC3>
|
|
157
|
-
| Ranger21 | *a synergistic deep learning optimizer* | [github](https://github.com/lessw2020/Ranger21) | <https://arxiv.org/abs/2106.13731>
|
|
158
|
-
| Lamb | *Large Batch Optimization for Deep Learning* | [github](https://github.com/cybertronai/pytorch-lamb) | <https://arxiv.org/abs/1904.00962>
|
|
159
|
-
| Shampoo | *Preconditioned Stochastic Tensor Optimization* | [github](https://github.com/moskomule/shampoo.pytorch) | <https://arxiv.org/abs/1802.09568>
|
|
160
|
-
| Nero | *Learning by Turning: Neural Architecture Aware Optimisation* | [github](https://github.com/jxbz/nero) | <https://arxiv.org/abs/2102.07227>
|
|
161
|
-
| Adan | *Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models* | [github](https://github.com/sail-sg/Adan) | <https://arxiv.org/abs/2208.06677>
|
|
162
|
-
| Adai | *Disentangling the Effects of Adaptive Learning Rate and Momentum* | [github](https://github.com/zeke-xie/adaptive-inertia-adai) | <https://arxiv.org/abs/2006.15815>
|
|
163
|
-
| SAM | *Sharpness-Aware Minimization* | [github](https://github.com/davda54/sam) | <https://arxiv.org/abs/2010.01412>
|
|
164
|
-
| ASAM | *Adaptive Sharpness-Aware Minimization* | [github](https://github.com/davda54/sam) | <https://arxiv.org/abs/2102.11600>
|
|
165
|
-
| GSAM | *Surrogate Gap Guided Sharpness-Aware Minimization* | [github](https://github.com/juntang-zhuang/GSAM) | <https://openreview.net/pdf?id=edONMAnhLu->
|
|
166
|
-
| D-Adaptation | *Learning-Rate-Free Learning by D-Adaptation* | [github](https://github.com/facebookresearch/dadaptation) | <https://arxiv.org/abs/2301.07733>
|
|
167
|
-
| AdaFactor | *Adaptive Learning Rates with Sublinear Memory Cost* | [github](https://github.com/DeadAt0m/adafactor-pytorch) | <https://arxiv.org/abs/1804.04235>
|
|
168
|
-
| Apollo | *An Adaptive Parameter-wise Diagonal Quasi-Newton Method for Nonconvex Stochastic Optimization* | [github](https://github.com/XuezheMax/apollo) | <https://arxiv.org/abs/2009.13586>
|
|
169
|
-
| NovoGrad | *Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks* | [github](https://github.com/lonePatient/NovoGrad-pytorch) | <https://arxiv.org/abs/1905.11286>
|
|
170
|
-
| Lion | *Symbolic Discovery of Optimization Algorithms* | [github](https://github.com/google/automl/tree/master/lion) | <https://arxiv.org/abs/2302.06675>
|
|
171
|
-
| Ali-G | *Adaptive Learning Rates for Interpolation with Gradients* | [github](https://github.com/oval-group/ali-g) | <https://arxiv.org/abs/1906.05661>
|
|
172
|
-
| SM3 | *Memory-Efficient Adaptive Optimization* | [github](https://github.com/google-research/google-research/tree/master/sm3) | <https://arxiv.org/abs/1901.11150>
|
|
173
|
-
| AdaNorm | *Adaptive Gradient Norm Correction based Optimizer for CNNs* | [github](https://github.com/shivram1987/AdaNorm) | <https://arxiv.org/abs/2210.06364>
|
|
174
|
-
| RotoGrad | *Gradient Homogenization in Multitask Learning* | [github](https://github.com/adrianjav/rotograd) | <https://openreview.net/pdf?id=T8wHz4rnuGL>
|
|
175
|
-
| A2Grad | *Optimal Adaptive and Accelerated Stochastic Gradient Descent* | [github](https://github.com/severilov/A2Grad_optimizer) | <https://arxiv.org/abs/1810.00553>
|
|
176
|
-
| AccSGD | *Accelerating Stochastic Gradient Descent For Least Squares Regression* | [github](https://github.com/rahulkidambi/AccSGD) | <https://arxiv.org/abs/1704.08227>
|
|
177
|
-
| SGDW | *Decoupled Weight Decay Regularization* | [github](https://github.com/loshchil/AdamW-and-SGDW) | <https://arxiv.org/abs/1711.05101>
|
|
178
|
-
| ASGD | *Adaptive Gradient Descent without Descent* | [github](https://github.com/ymalitsky/adaptive_GD) | <https://arxiv.org/abs/1910.09529>
|
|
179
|
-
| Yogi | *Adaptive Methods for Nonconvex Optimization* | | [NIPS 2018](https://papers.nips.cc/paper/8186-adaptive-methods-for-nonconvex-optimization)
|
|
180
|
-
| SWATS | *Improving Generalization Performance by Switching from Adam to SGD* | | <https://arxiv.org/abs/1712.07628>
|
|
181
|
-
| Fromage | *On the distance between two neural networks and the stability of learning* | [github](https://github.com/jxbz/fromage) | <https://arxiv.org/abs/2002.03432>
|
|
182
|
-
| MSVAG | *Dissecting Adam: The Sign, Magnitude and Variance of Stochastic Gradients* | [github](https://github.com/lballes/msvag) | <https://arxiv.org/abs/1705.07774>
|
|
183
|
-
| AdaMod | *An Adaptive and Momental Bound Method for Stochastic Learning* | [github](https://github.com/lancopku/AdaMod) | <https://arxiv.org/abs/1910.12249>
|
|
184
|
-
| AggMo | *Aggregated Momentum: Stability Through Passive Damping* | [github](https://github.com/AtheMathmo/AggMo) | <https://arxiv.org/abs/1804.00325>
|
|
185
|
-
| QHAdam | *Quasi-hyperbolic momentum and Adam for deep learning* | [github](https://github.com/facebookresearch/qhoptim) | <https://arxiv.org/abs/1810.06801>
|
|
186
|
-
| PID | *A PID Controller Approach for Stochastic Optimization of Deep Networks* | [github](https://github.com/tensorboy/PIDOptimizer) | [CVPR 18](http://www4.comp.polyu.edu.hk/~cslzhang/paper/CVPR18_PID.pdf)
|
|
187
|
-
| Gravity | *a Kinematic Approach on Optimization in Deep Learning* | [github](https://github.com/dariush-bahrami/gravity.optimizer) | <https://arxiv.org/abs/2101.09192>
|
|
188
|
-
| AdaSmooth | *An Adaptive Learning Rate Method based on Effective Ratio* | | <https://arxiv.org/abs/2204.00825v1>
|
|
189
|
-
| SRMM | *Stochastic regularized majorization-minimization with weakly convex and multi-convex surrogates* | [github](https://github.com/HanbaekLyu/SRMM) | <https://arxiv.org/abs/2201.01652>
|
|
190
|
-
| AvaGrad | *Domain-independent Dominance of Adaptive Methods* | [github](https://github.com/lolemacs/avagrad) | <https://arxiv.org/abs/1912.01823>
|
|
191
|
-
| PCGrad | *Gradient Surgery for Multi-Task Learning* | [github](https://github.com/tianheyu927/PCGrad) | <https://arxiv.org/abs/2001.06782>
|
|
192
|
-
| AMSGrad | *On the Convergence of Adam and Beyond* | | <https://openreview.net/pdf?id=ryQu7f-RZ>
|
|
193
|
-
| Lookahead | *k steps forward, 1 step back* | [github](https://github.com/pytorch/examples/tree/main/imagenet) | <https://arxiv.org/abs/1907.08610>
|
|
194
|
-
| PNM | *Manipulating Stochastic Gradient Noise to Improve Generalization* | [github](https://github.com/zeke-xie/Positive-Negative-Momentum) | <https://arxiv.org/abs/2103.17182>
|
|
195
|
-
| GC | *Gradient Centralization* | [github](https://github.com/Yonghongwei/Gradient-Centralization) | <https://arxiv.org/abs/2004.01461>
|
|
196
|
-
| AGC | *Adaptive Gradient Clipping* | [github](https://github.com/deepmind/deepmind-research/tree/master/nfnets) | <https://arxiv.org/abs/2102.06171>
|
|
197
|
-
| Stable WD | *Understanding and Scheduling Weight Decay* | [github](https://github.com/zeke-xie/stable-weight-decay-regularization) | <https://arxiv.org/abs/2011.11152>
|
|
198
|
-
| Softplus T | *Calibrating the Adaptive Learning Rate to Improve Convergence of ADAM* | | <https://arxiv.org/abs/1908.00700>
|
|
199
|
-
| Un-tuned w/u | *On the adequacy of untuned warmup for adaptive optimization* | | <https://arxiv.org/abs/1910.04209>
|
|
200
|
-
| Norm Loss | *An efficient yet effective regularization method for deep neural networks* | | <https://arxiv.org/abs/2103.06583>
|
|
201
|
-
| AdaShift | *Decorrelation and Convergence of Adaptive Learning Rate Methods* | [github](https://github.com/MichaelKonobeev/adashift) | <https://arxiv.org/abs/1810.00143v4>
|
|
202
|
-
| AdaDelta | *An Adaptive Learning Rate Method* | | <https://arxiv.org/abs/1212.5701v1>
|
|
203
|
-
| Amos | *An Adam-style Optimizer with Adaptive Weight Decay towards Model-Oriented Scale* | [github](https://github.com/google-research/jestimator) | <https://arxiv.org/abs/2210.11693>
|
|
204
|
-
| SignSGD | *Compressed Optimisation for Non-Convex Problems* | [github](https://github.com/jxbz/signSGD) | <https://arxiv.org/abs/1802.04434>
|
|
205
|
-
| Sophia | *A Scalable Stochastic Second-order Optimizer for Language Model Pre-training* | [github](https://github.com/Liuhong99/Sophia) | <https://arxiv.org/abs/2305.14342>
|
|
206
|
-
| Prodigy | *An Expeditiously Adaptive Parameter-Free Learner* | [github](https://github.com/konstmish/prodigy) | <https://arxiv.org/abs/2306.06101>
|
|
207
|
-
| PAdam | *Closing the Generalization Gap of Adaptive Gradient Methods in Training Deep Neural Networks* | [github](https://github.com/uclaml/Padam) | <https://arxiv.org/abs/1806.06763>
|
|
208
|
-
| LOMO | *Full Parameter Fine-tuning for Large Language Models with Limited Resources* | [github](https://github.com/OpenLMLab/LOMO) | <https://arxiv.org/abs/2306.09782>
|
|
209
|
-
| AdaLOMO | *Low-memory Optimization with Adaptive Learning Rate* | [github](https://github.com/OpenLMLab/LOMO) | <https://arxiv.org/abs/2310.10195>
|
|
210
|
-
| Tiger | *A Tight-fisted Optimizer, an optimizer that is extremely budget-conscious* | [github](https://github.com/bojone/tiger) |
|
|
211
|
-
| CAME | *Confidence-guided Adaptive Memory Efficient Optimization* | [github](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/CAME) | <https://aclanthology.org/2023.acl-long.243/>
|
|
212
|
-
| WSAM | *Sharpness-Aware Minimization Revisited: Weighted Sharpness as a Regularization Term* | [github](https://github.com/intelligent-machine-learning/dlrover/blob/master/atorch/atorch/optimizers/wsam.py) | <https://arxiv.org/abs/2305.15817>
|
|
213
|
-
| Aida | *A DNN Optimizer that Improves over AdaBelief by Suppression of the Adaptive Stepsize Range* | [github](https://github.com/guoqiang-zhang-x/Aida-Optimizer) | <https://arxiv.org/abs/2203.13273>
|
|
214
|
-
| GaLore | *Memory-Efficient LLM Training by Gradient Low-Rank Projection* | [github](https://github.com/jiaweizzhao/GaLore) | <https://arxiv.org/abs/2403.03507>
|
|
215
|
-
| Adalite | *Adalite optimizer* | [github](https://github.com/VatsaDev/adalite) | <https://github.com/VatsaDev/adalite>
|
|
216
|
-
| bSAM | *SAM as an Optimal Relaxation of Bayes* | [github](https://github.com/team-approx-bayes/bayesian-sam) | <https://arxiv.org/abs/2210.01620>
|
|
217
|
-
| Schedule-Free | *Schedule-Free Optimizers* | [github](https://github.com/facebookresearch/schedule_free) | <https://github.com/facebookresearch/schedule_free>
|
|
218
|
-
| FAdam | *Adam is a natural gradient optimizer using diagonal empirical Fisher information* | [github](https://github.com/lessw2020/fadam_pytorch) | <https://arxiv.org/abs/2405.12807>
|
|
219
|
-
| Grokfast | *Accelerated Grokking by Amplifying Slow Gradients* | [github](https://github.com/ironjr/grokfast) | <https://arxiv.org/abs/2405.20233>
|
|
220
|
-
| Kate | *Remove that Square Root: A New Efficient Scale-Invariant Version of AdaGrad* | [github](https://github.com/nazya/KATE) | <https://arxiv.org/abs/2403.02648>
|
|
221
|
-
| StableAdamW | *Stable and low-precision training for large-scale vision-language models* | | <https://arxiv.org/abs/2304.13013>
|
|
222
|
-
| AdamMini | *Use Fewer Learning Rates To Gain More* | [github](https://github.com/zyushun/Adam-mini) | <https://arxiv.org/abs/2406.16793>
|
|
223
|
-
| TRAC | *Adaptive Parameter-free Optimization* | [github](https://github.com/ComputationalRobotics/TRAC) | <https://arxiv.org/abs/2405.16642>
|
|
224
|
-
| AdamG | *Towards Stability of Parameter-free Optimization* | | <https://arxiv.org/abs/2405.04376>
|
|
225
|
-
| AdEMAMix | *Better, Faster, Older* | [github](https://github.com/nanowell/AdEMAMix-Optimizer-Pytorch) | <https://arxiv.org/abs/2409.03137>
|
|
226
|
-
| SOAP | *Improving and Stabilizing Shampoo using Adam* | [github](https://github.com/nikhilvyas/SOAP) | <https://arxiv.org/abs/2409.11321>
|
|
227
|
-
| ADOPT | *Modified Adam Can Converge with Any β2 with the Optimal Rate* | [github](https://github.com/iShohei220/adopt) | <https://arxiv.org/abs/2411.02853>
|
|
228
|
-
| FTRL | *Follow The Regularized Leader* | | <https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/41159.pdf>
|
|
229
|
-
| Cautious | *Improving Training with One Line of Code* | [github](https://github.com/kyleliang919/C-Optim) | <https://arxiv.org/pdf/2411.16085v1>
|
|
230
|
-
| DeMo | *Decoupled Momentum Optimization* | [github](https://github.com/bloc97/DeMo) | <https://arxiv.org/abs/2411.19870>
|
|
231
|
-
| MicroAdam | *Accurate Adaptive Optimization with Low Space Overhead and Provable Convergence* | [github](https://github.com/IST-DASLab/MicroAdam) | <https://arxiv.org/abs/2405.15593>
|
|
232
|
-
| Muon | *MomentUm Orthogonalized by Newton-schulz* | [github](https://github.com/KellerJordan/Muon) | <https://x.com/kellerjordan0/status/1842300916864844014>
|
|
233
|
-
| LaProp | *Separating Momentum and Adaptivity in Adam* | [github](https://github.com/Z-T-WANG/LaProp-Optimizer) | <https://arxiv.org/abs/2002.04839>
|
|
234
|
-
| APOLLO | *SGD-like Memory, AdamW-level Performance* | [github](https://github.com/zhuhanqing/APOLLO) | <https://arxiv.org/abs/2412.05270>
|
|
235
|
-
| MARS | *Unleashing the Power of Variance Reduction for Training Large Models* | [github](https://github.com/AGI-Arena/MARS) | <https://arxiv.org/abs/2411.10438>
|
|
236
|
-
| SGDSaI | *No More Adam: Learning Rate Scaling at Initialization is All You Need* | [github](https://github.com/AnonymousAlethiometer/SGD_SaI) | <https://arxiv.org/abs/2411.10438>
|
|
237
|
-
| Grams | *Gradient Descent with Adaptive Momentum Scaling* | | <https://arxiv.org/abs/2412.17107>
|
|
238
|
-
| OrthoGrad | *Grokking at the Edge of Numerical Stability* | [github](https://github.com/LucasPrietoAl/grokking-at-the-edge-of-numerical-stability) | <https://arxiv.org/abs/2501.04697>
|
|
239
|
-
| Adam-ATAN2 | *Scaling Exponents Across Parameterizations and Optimizers* | | <https://arxiv.org/abs/2407.05872>
|
|
240
|
-
| SPAM | *Spike-Aware Adam with Momentum Reset for Stable LLM Training* | [github](https://github.com/TianjinYellow/SPAM-Optimizer) | <https://arxiv.org/abs/2501.06842>
|
|
241
|
-
| TAM | *Torque-Aware Momentum* | | <https://arxiv.org/abs/2412.18790>
|
|
242
|
-
| FOCUS | *First Order Concentrated Updating Scheme* | [github](https://github.com/liuyz0/FOCUS) | <https://arxiv.org/abs/2501.12243>
|
|
243
|
-
| PSGD | *Preconditioned Stochastic Gradient Descent* | [github](https://github.com/lixilinx/psgd_torch) | <https://arxiv.org/abs/1512.04202>
|
|
244
|
-
| EXAdam | *The Power of Adaptive Cross-Moments* | [github](https://github.com/AhmedMostafa16/EXAdam) | <https://arxiv.org/abs/2412.20302>
|
|
245
|
-
| GCSAM | *Gradient Centralized Sharpness Aware Minimization* | [github](https://github.com/mhassann22/GCSAM) | <https://arxiv.org/abs/2501.11584>
|
|
246
|
-
| LookSAM | *Towards Efficient and Scalable Sharpness-Aware Minimization* | [github](https://github.com/rollovd/LookSAM) | <https://arxiv.org/abs/2203.02714>
|
|
247
|
-
| SCION | *Training Deep Learning Models with Norm-Constrained LMOs* | [github](https://github.com/LIONS-EPFL/scion) | <https://arxiv.org/abs/2502.07529>
|
|
248
|
-
| COSMOS | *SOAP with Muon* | [github](https://github.com/lliu606/COSMOS) |
|
|
249
|
-
| StableSPAM | *How to Train in 4-Bit More Stably than 16-Bit Adam* | [github](https://github.com/TianjinYellow/StableSPAM) | <https://arxiv.org/abs/2502.17055>
|
|
250
|
-
| AdaGC | *Improving Training Stability for Large Language Model Pretraining* | | <https://arxiv.org/abs/2502.11034>
|
|
251
|
-
| Simplified-Ademamix | *Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD Variants* | [github](https://github.com/DepenM/Simplified-AdEMAMix/) | <https://arxiv.org/abs/2502.02431>
|
|
252
|
-
| Fira | *Can We Achieve Full-rank Training of LLMs Under Low-rank Constraint?* | [github](https://github.com/xichen-fy/Fira) | <https://arxiv.org/abs/2410.01623>
|
|
253
|
-
| RACS & Alice | *Towards Efficient Optimizer Design for LLM via Structured Fisher Approximation with a Low-Rank Extension* | | <https://arxiv.org/pdf/2502.07752>
|
|
254
|
-
| VSGD | *Variational Stochastic Gradient Descent for Deep Neural Networks* | [github](https://github.com/generativeai-tue/vsgd) | <https://openreview.net/forum?id=xu4ATNjcdy>
|
|
255
|
-
| SNSM | *Subset-Norm and Subspace-Momentum: Faster Memory-Efficient Adaptive Optimization with Convergence Guarantees* | [github](https://github.com/timmytonga/sn-sm) | <https://arxiv.org/abs/2411.07120>
|
|
256
|
-
| AdamC | *Why Gradients Rapidly Increase Near the End of Training* | | <https://arxiv.org/abs/2506.02285>
|
|
257
|
-
| AdaMuon | *Adaptive Muon Optimizer* | | <https://arxiv.org/abs/2507.11005v1>
|
|
258
|
-
| SPlus | *A Stable Whitening Optimizer for Efficient Neural Network Training* | [github](https://github.com/kvfrans/splus) | <https://arxiv.org/abs/2506.07254>
|
|
259
|
-
| EmoNavi | *An emotion-driven optimizer that feels loss and navigates accordingly* | [github](https://github.com/muooon/EmoNavi) |
|
|
260
|
-
| Refined Schedule-Free | *Through the River: Understanding the Benefit of Schedule-Free Methods for Language Model Training* | | <https://arxiv.org/abs/2507.09846>
|
|
149
|
+
| Optimizer | Description | Official Code | Paper | Citation |
|
|
150
|
+
|-----------------------|----------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------|----------------------------------------------------------------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------|
|
|
151
|
+
| AdaBelief | *Adapting Step-sizes by the Belief in Observed Gradients* | [github](https://github.com/juntang-zhuang/Adabelief-Optimizer) | <https://arxiv.org/abs/2010.07468> | [cite](https://ui.adsabs.harvard.edu/abs/2020arXiv201007468Z/exportcitation) |
|
|
152
|
+
| AdaBound | *Adaptive Gradient Methods with Dynamic Bound of Learning Rate* | [github](https://github.com/Luolc/AdaBound/blob/master/adabound/adabound.py) | <https://openreview.net/forum?id=Bkg3g2R9FX> | [cite](https://github.com/Luolc/AdaBound#citing) |
|
|
153
|
+
| AdaHessian | *An Adaptive Second Order Optimizer for Machine Learning* | [github](https://github.com/amirgholami/adahessian) | <https://arxiv.org/abs/2006.00719> | [cite](https://github.com/amirgholami/adahessian#citation) |
|
|
154
|
+
| AdamD | *Improved bias-correction in Adam* | | <https://arxiv.org/abs/2110.10828> | [cite](https://ui.adsabs.harvard.edu/abs/2021arXiv211010828S/exportcitation) |
|
|
155
|
+
| AdamP | *Slowing Down the Slowdown for Momentum Optimizers on Scale-invariant Weights* | [github](https://github.com/clovaai/AdamP) | <https://arxiv.org/abs/2006.08217> | [cite](https://github.com/clovaai/AdamP#how-to-cite) |
|
|
156
|
+
| diffGrad | *An Optimization Method for Convolutional Neural Networks* | [github](https://github.com/shivram1987/diffGrad) | <https://arxiv.org/abs/1909.11015v3> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190911015D/exportcitation) |
|
|
157
|
+
| MADGRAD | *A Momentumized, Adaptive, Dual Averaged Gradient Method for Stochastic* | [github](https://github.com/facebookresearch/madgrad) | <https://arxiv.org/abs/2101.11075> | [cite](https://github.com/facebookresearch/madgrad#tech-report) |
|
|
158
|
+
| RAdam | *On the Variance of the Adaptive Learning Rate and Beyond* | [github](https://github.com/LiyuanLucasLiu/RAdam) | <https://arxiv.org/abs/1908.03265> | [cite](https://github.com/LiyuanLucasLiu/RAdam#citation) |
|
|
159
|
+
| Ranger | *a synergistic optimizer combining RAdam and LookAhead, and now GC in one optimizer* | [github](https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer) | <https://bit.ly/3zyspC3> | [cite](https://github.com/lessw2020/Ranger-Deep-Learning-Optimizer#citing-this-work) |
|
|
160
|
+
| Ranger21 | *a synergistic deep learning optimizer* | [github](https://github.com/lessw2020/Ranger21) | <https://arxiv.org/abs/2106.13731> | [cite](https://github.com/lessw2020/Ranger21#referencing-this-work) |
|
|
161
|
+
| Lamb | *Large Batch Optimization for Deep Learning* | [github](https://github.com/cybertronai/pytorch-lamb) | <https://arxiv.org/abs/1904.00962> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190400962Y/exportcitation) |
|
|
162
|
+
| Shampoo | *Preconditioned Stochastic Tensor Optimization* | [github](https://github.com/moskomule/shampoo.pytorch) | <https://arxiv.org/abs/1802.09568> | [cite](https://ui.adsabs.harvard.edu/abs/2018arXiv180209568G/exportcitation) |
|
|
163
|
+
| Nero | *Learning by Turning: Neural Architecture Aware Optimisation* | [github](https://github.com/jxbz/nero) | <https://arxiv.org/abs/2102.07227> | [cite](https://ui.adsabs.harvard.edu/abs/2021arXiv210207227L/exportcitation) |
|
|
164
|
+
| Adan | *Adaptive Nesterov Momentum Algorithm for Faster Optimizing Deep Models* | [github](https://github.com/sail-sg/Adan) | <https://arxiv.org/abs/2208.06677> | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220806677X/exportcitation) |
|
|
165
|
+
| Adai | *Disentangling the Effects of Adaptive Learning Rate and Momentum* | [github](https://github.com/zeke-xie/adaptive-inertia-adai) | <https://arxiv.org/abs/2006.15815> | [cite](https://github.com/zeke-xie/adaptive-inertia-adai#citing) |
|
|
166
|
+
| SAM | *Sharpness-Aware Minimization* | [github](https://github.com/davda54/sam) | <https://arxiv.org/abs/2010.01412> | [cite](https://ui.adsabs.harvard.edu/abs/2020arXiv201001412F/exportcitation) |
|
|
167
|
+
| ASAM | *Adaptive Sharpness-Aware Minimization* | [github](https://github.com/davda54/sam) | <https://arxiv.org/abs/2102.11600> | [cite](https://ui.adsabs.harvard.edu/abs/2021arXiv210211600K/exportcitation) |
|
|
168
|
+
| GSAM | *Surrogate Gap Guided Sharpness-Aware Minimization* | [github](https://github.com/juntang-zhuang/GSAM) | <https://openreview.net/pdf?id=edONMAnhLu-> | [cite](https://github.com/juntang-zhuang/GSAM#citation) |
|
|
169
|
+
| D-Adaptation | *Learning-Rate-Free Learning by D-Adaptation* | [github](https://github.com/facebookresearch/dadaptation) | <https://arxiv.org/abs/2301.07733> | [cite](https://ui.adsabs.harvard.edu/abs/2023arXiv230107733D/exportcitation) |
|
|
170
|
+
| AdaFactor | *Adaptive Learning Rates with Sublinear Memory Cost* | [github](https://github.com/DeadAt0m/adafactor-pytorch) | <https://arxiv.org/abs/1804.04235> | [cite](https://ui.adsabs.harvard.edu/abs/2018arXiv180404235S/exportcitation) |
|
|
171
|
+
| Apollo | *An Adaptive Parameter-wise Diagonal Quasi-Newton Method for Nonconvex Stochastic Optimization* | [github](https://github.com/XuezheMax/apollo) | <https://arxiv.org/abs/2009.13586> | [cite](https://ui.adsabs.harvard.edu/abs/2020arXiv200913586M/exportcitation) |
|
|
172
|
+
| NovoGrad | *Stochastic Gradient Methods with Layer-wise Adaptive Moments for Training of Deep Networks* | [github](https://github.com/lonePatient/NovoGrad-pytorch) | <https://arxiv.org/abs/1905.11286> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190511286G/exportcitation) |
|
|
173
|
+
| Lion | *Symbolic Discovery of Optimization Algorithms* | [github](https://github.com/google/automl/tree/master/lion) | <https://arxiv.org/abs/2302.06675> | [cite](https://github.com/google/automl/tree/master/lion#citation) |
|
|
174
|
+
| Ali-G | *Adaptive Learning Rates for Interpolation with Gradients* | [github](https://github.com/oval-group/ali-g) | <https://arxiv.org/abs/1906.05661> | [cite](https://github.com/oval-group/ali-g#adaptive-learning-rates-for-interpolation-with-gradients) |
|
|
175
|
+
| SM3 | *Memory-Efficient Adaptive Optimization* | [github](https://github.com/google-research/google-research/tree/master/sm3) | <https://arxiv.org/abs/1901.11150> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190111150A/exportcitation) |
|
|
176
|
+
| AdaNorm | *Adaptive Gradient Norm Correction based Optimizer for CNNs* | [github](https://github.com/shivram1987/AdaNorm) | <https://arxiv.org/abs/2210.06364> | [cite](https://github.com/shivram1987/AdaNorm/tree/main#citation) |
|
|
177
|
+
| RotoGrad | *Gradient Homogenization in Multitask Learning* | [github](https://github.com/adrianjav/rotograd) | <https://openreview.net/pdf?id=T8wHz4rnuGL> | [cite](https://github.com/adrianjav/rotograd#citing) |
|
|
178
|
+
| A2Grad | *Optimal Adaptive and Accelerated Stochastic Gradient Descent* | [github](https://github.com/severilov/A2Grad_optimizer) | <https://arxiv.org/abs/1810.00553> | [cite](https://ui.adsabs.harvard.edu/abs/2018arXiv181000553D/exportcitation) |
|
|
179
|
+
| AccSGD | *Accelerating Stochastic Gradient Descent For Least Squares Regression* | [github](https://github.com/rahulkidambi/AccSGD) | <https://arxiv.org/abs/1704.08227> | [cite](https://github.com/rahulkidambi/AccSGD#citation) |
|
|
180
|
+
| SGDW | *Decoupled Weight Decay Regularization* | [github](https://github.com/loshchil/AdamW-and-SGDW) | <https://arxiv.org/abs/1711.05101> | [cite](https://github.com/loshchil/AdamW-and-SGDW#contact) |
|
|
181
|
+
| ASGD | *Adaptive Gradient Descent without Descent* | [github](https://github.com/ymalitsky/adaptive_GD) | <https://arxiv.org/abs/1910.09529> | [cite](https://github.com/ymalitsky/adaptive_GD#reference) |
|
|
182
|
+
| Yogi | *Adaptive Methods for Nonconvex Optimization* | | [NIPS 2018](https://papers.nips.cc/paper/8186-adaptive-methods-for-nonconvex-optimization) | [cite](https://proceedings.neurips.cc/paper_files/paper/2018/hash/90365351ccc7437a1309dc64e4db32a3-Abstract.html) |
|
|
183
|
+
| SWATS | *Improving Generalization Performance by Switching from Adam to SGD* | | <https://arxiv.org/abs/1712.07628> | [cite](https://ui.adsabs.harvard.edu/abs/2017arXiv171207628S/exportcitation) |
|
|
184
|
+
| Fromage | *On the distance between two neural networks and the stability of learning* | [github](https://github.com/jxbz/fromage) | <https://arxiv.org/abs/2002.03432> | [cite](https://github.com/jxbz/fromage#citation) |
|
|
185
|
+
| MSVAG | *Dissecting Adam: The Sign, Magnitude and Variance of Stochastic Gradients* | [github](https://github.com/lballes/msvag) | <https://arxiv.org/abs/1705.07774> | [cite](https://github.com/lballes/msvag#citation) |
|
|
186
|
+
| AdaMod | *An Adaptive and Momental Bound Method for Stochastic Learning* | [github](https://github.com/lancopku/AdaMod) | <https://arxiv.org/abs/1910.12249> | [cite](https://github.com/lancopku/AdaMod#citation) |
|
|
187
|
+
| AggMo | *Aggregated Momentum: Stability Through Passive Damping* | [github](https://github.com/AtheMathmo/AggMo) | <https://arxiv.org/abs/1804.00325> | [cite](https://ui.adsabs.harvard.edu/abs/2018arXiv180400325L/exportcitation) |
|
|
188
|
+
| QHAdam | *Quasi-hyperbolic momentum and Adam for deep learning* | [github](https://github.com/facebookresearch/qhoptim) | <https://arxiv.org/abs/1810.06801> | [cite](https://github.com/facebookresearch/qhoptim#reference) |
|
|
189
|
+
| PID | *A PID Controller Approach for Stochastic Optimization of Deep Networks* | [github](https://github.com/tensorboy/PIDOptimizer) | [CVPR 18](http://www4.comp.polyu.edu.hk/~cslzhang/paper/CVPR18_PID.pdf) | [cite](https://github.com/tensorboy/PIDOptimizer#citation) |
|
|
190
|
+
| Gravity | *a Kinematic Approach on Optimization in Deep Learning* | [github](https://github.com/dariush-bahrami/gravity.optimizer) | <https://arxiv.org/abs/2101.09192> | [cite](https://ui.adsabs.harvard.edu/abs/2021arXiv210109192B/exportcitation) |
|
|
191
|
+
| AdaSmooth | *An Adaptive Learning Rate Method based on Effective Ratio* | | <https://arxiv.org/abs/2204.00825v1> | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220400825L/exportcitation) |
|
|
192
|
+
| SRMM | *Stochastic regularized majorization-minimization with weakly convex and multi-convex surrogates* | [github](https://github.com/HanbaekLyu/SRMM) | <https://arxiv.org/abs/2201.01652> | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220101652L/exportcitation) |
|
|
193
|
+
| AvaGrad | *Domain-independent Dominance of Adaptive Methods* | [github](https://github.com/lolemacs/avagrad) | <https://arxiv.org/abs/1912.01823> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv191201823S/exportcitation) |
|
|
194
|
+
| PCGrad | *Gradient Surgery for Multi-Task Learning* | [github](https://github.com/tianheyu927/PCGrad) | <https://arxiv.org/abs/2001.06782> | [cite](https://github.com/tianheyu927/PCGrad#reference) |
|
|
195
|
+
| AMSGrad | *On the Convergence of Adam and Beyond* | | <https://openreview.net/pdf?id=ryQu7f-RZ> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190409237R/exportcitation) |
|
|
196
|
+
| Lookahead | *k steps forward, 1 step back* | [github](https://github.com/pytorch/examples/tree/main/imagenet) | <https://arxiv.org/abs/1907.08610> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190708610Z/exportcitation) |
|
|
197
|
+
| PNM | *Manipulating Stochastic Gradient Noise to Improve Generalization* | [github](https://github.com/zeke-xie/Positive-Negative-Momentum) | <https://arxiv.org/abs/2103.17182> | [cite](https://github.com/zeke-xie/Positive-Negative-Momentum#citing) |
|
|
198
|
+
| GC | *Gradient Centralization* | [github](https://github.com/Yonghongwei/Gradient-Centralization) | <https://arxiv.org/abs/2004.01461> | [cite](https://github.com/Yonghongwei/Gradient-Centralization#citation) |
|
|
199
|
+
| AGC | *Adaptive Gradient Clipping* | [github](https://github.com/deepmind/deepmind-research/tree/master/nfnets) | <https://arxiv.org/abs/2102.06171> | [cite](https://ui.adsabs.harvard.edu/abs/2021arXiv210206171B/exportcitation) |
|
|
200
|
+
| Stable WD | *Understanding and Scheduling Weight Decay* | [github](https://github.com/zeke-xie/stable-weight-decay-regularization) | <https://arxiv.org/abs/2011.11152> | [cite](https://ui.adsabs.harvard.edu/abs/2020arXiv201111152X/exportcitation) |
|
|
201
|
+
| Softplus T | *Calibrating the Adaptive Learning Rate to Improve Convergence of ADAM* | | <https://arxiv.org/abs/1908.00700> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv190800700T/exportcitation) |
|
|
202
|
+
| Un-tuned w/u | *On the adequacy of untuned warmup for adaptive optimization* | | <https://arxiv.org/abs/1910.04209> | [cite](https://ui.adsabs.harvard.edu/abs/2019arXiv191004209M/exportcitation) |
|
|
203
|
+
| Norm Loss | *An efficient yet effective regularization method for deep neural networks* | | <https://arxiv.org/abs/2103.06583> | [cite](https://ui.adsabs.harvard.edu/abs/2021arXiv210306583G/exportcitation) |
|
|
204
|
+
| AdaShift | *Decorrelation and Convergence of Adaptive Learning Rate Methods* | [github](https://github.com/MichaelKonobeev/adashift) | <https://arxiv.org/abs/1810.00143v4> | [cite](https://ui.adsabs.harvard.edu/abs/2018arXiv181000143Z/exportcitation) |
|
|
205
|
+
| AdaDelta | *An Adaptive Learning Rate Method* | | <https://arxiv.org/abs/1212.5701v1> | [cite](https://ui.adsabs.harvard.edu/abs/2012arXiv1212.5701Z/exportcitation) |
|
|
206
|
+
| Amos | *An Adam-style Optimizer with Adaptive Weight Decay towards Model-Oriented Scale* | [github](https://github.com/google-research/jestimator) | <https://arxiv.org/abs/2210.11693> | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv221011693T/exportcitation) |
|
|
207
|
+
| SignSGD | *Compressed Optimisation for Non-Convex Problems* | [github](https://github.com/jxbz/signSGD) | <https://arxiv.org/abs/1802.04434> | [cite](https://ui.adsabs.harvard.edu/abs/2018arXiv180204434B/exportcitation) |
|
|
208
|
+
| Sophia | *A Scalable Stochastic Second-order Optimizer for Language Model Pre-training* | [github](https://github.com/Liuhong99/Sophia) | <https://arxiv.org/abs/2305.14342> | [cite](https://github.com/Liuhong99/Sophia) |
|
|
209
|
+
| Prodigy | *An Expeditiously Adaptive Parameter-Free Learner* | [github](https://github.com/konstmish/prodigy) | <https://arxiv.org/abs/2306.06101> | [cite](https://github.com/konstmish/prodigy#how-to-cite) |
|
|
210
|
+
| PAdam | *Closing the Generalization Gap of Adaptive Gradient Methods in Training Deep Neural Networks* | [github](https://github.com/uclaml/Padam) | <https://arxiv.org/abs/1806.06763> | [cite](https://github.com/uclaml/Padam#citation) |
|
|
211
|
+
| LOMO | *Full Parameter Fine-tuning for Large Language Models with Limited Resources* | [github](https://github.com/OpenLMLab/LOMO) | <https://arxiv.org/abs/2306.09782> | [cite](https://github.com/OpenLMLab/LOMO#citation) |
|
|
212
|
+
| AdaLOMO | *Low-memory Optimization with Adaptive Learning Rate* | [github](https://github.com/OpenLMLab/LOMO) | <https://arxiv.org/abs/2310.10195> | [cite](https://github.com/OpenLMLab/LOMO#citation) |
|
|
213
|
+
| Tiger | *A Tight-fisted Optimizer, an optimizer that is extremely budget-conscious* | [github](https://github.com/bojone/tiger) | | [cite](https://github.com/bojone/tiger/blob/main/README_en.md#citation) |
|
|
214
|
+
| CAME | *Confidence-guided Adaptive Memory Efficient Optimization* | [github](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/CAME) | <https://aclanthology.org/2023.acl-long.243/> | [cite](https://github.com/huawei-noah/Pretrained-Language-Model/tree/master/CAME#citation) |
|
|
215
|
+
| WSAM | *Sharpness-Aware Minimization Revisited: Weighted Sharpness as a Regularization Term* | [github](https://github.com/intelligent-machine-learning/dlrover/blob/master/atorch/atorch/optimizers/wsam.py) | <https://arxiv.org/abs/2305.15817> | [cite](https://github.com/intelligent-machine-learning/dlrover) |
|
|
216
|
+
| Aida | *A DNN Optimizer that Improves over AdaBelief by Suppression of the Adaptive Stepsize Range* | [github](https://github.com/guoqiang-zhang-x/Aida-Optimizer) | <https://arxiv.org/abs/2203.13273> | [cite](https://github.com/guoqiang-zhang-x/Aida-Optimizer?tab=readme-ov-file#1-brief-description-of-aida) |
|
|
217
|
+
| GaLore | *Memory-Efficient LLM Training by Gradient Low-Rank Projection* | [github](https://github.com/jiaweizzhao/GaLore) | <https://arxiv.org/abs/2403.03507> | [cite](https://github.com/jiaweizzhao/GaLore/tree/master?tab=readme-ov-file#citation) |
|
|
218
|
+
| Adalite | *Adalite optimizer* | [github](https://github.com/VatsaDev/adalite) | <https://github.com/VatsaDev/adalite> | [cite](https://github.com/VatsaDev/adalite) |
|
|
219
|
+
| bSAM | *SAM as an Optimal Relaxation of Bayes* | [github](https://github.com/team-approx-bayes/bayesian-sam) | <https://arxiv.org/abs/2210.01620> | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv221001620M/exportcitation) |
|
|
220
|
+
| Schedule-Free | *Schedule-Free Optimizers* | [github](https://github.com/facebookresearch/schedule_free) | <https://github.com/facebookresearch/schedule_free> | [cite](https://github.com/facebookresearch/schedule_free) |
|
|
221
|
+
| FAdam | *Adam is a natural gradient optimizer using diagonal empirical Fisher information* | [github](https://github.com/lessw2020/fadam_pytorch) | <https://arxiv.org/abs/2405.12807> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240512807H/exportcitation) |
|
|
222
|
+
| Grokfast | *Accelerated Grokking by Amplifying Slow Gradients* | [github](https://github.com/ironjr/grokfast) | <https://arxiv.org/abs/2405.20233> | [cite](https://github.com/ironjr/grokfast?tab=readme-ov-file#citation) |
|
|
223
|
+
| Kate | *Remove that Square Root: A New Efficient Scale-Invariant Version of AdaGrad* | [github](https://github.com/nazya/KATE) | <https://arxiv.org/abs/2403.02648> | [cite](https://github.com/nazya/KATE?tab=readme-ov-file#remove-that-square-root-a-new-efficient-scale-invariant-version-of-adagrad) |
|
|
224
|
+
| StableAdamW | *Stable and low-precision training for large-scale vision-language models* | | <https://arxiv.org/abs/2304.13013> | [cite](https://ui.adsabs.harvard.edu/abs/2023arXiv230413013W/exportcitation) |
|
|
225
|
+
| AdamMini | *Use Fewer Learning Rates To Gain More* | [github](https://github.com/zyushun/Adam-mini) | <https://arxiv.org/abs/2406.16793> | [cite](https://github.com/zyushun/Adam-mini?tab=readme-ov-file#citation) |
|
|
226
|
+
| TRAC | *Adaptive Parameter-free Optimization* | [github](https://github.com/ComputationalRobotics/TRAC) | <https://arxiv.org/abs/2405.16642> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240516642M/exportcitation) |
|
|
227
|
+
| AdamG | *Towards Stability of Parameter-free Optimization* | | <https://arxiv.org/abs/2405.04376> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240504376P/exportcitation) |
|
|
228
|
+
| AdEMAMix | *Better, Faster, Older* | [github](https://github.com/nanowell/AdEMAMix-Optimizer-Pytorch) | <https://arxiv.org/abs/2409.03137> | [cite](https://github.com/nanowell/AdEMAMix-Optimizer-Pytorch?tab=readme-ov-file#reference) |
|
|
229
|
+
| SOAP | *Improving and Stabilizing Shampoo using Adam* | [github](https://github.com/nikhilvyas/SOAP) | <https://arxiv.org/abs/2409.11321> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240911321V/exportcitation) |
|
|
230
|
+
| ADOPT | *Modified Adam Can Converge with Any β2 with the Optimal Rate* | [github](https://github.com/iShohei220/adopt) | <https://arxiv.org/abs/2411.02853> | [cite](https://github.com/iShohei220/adopt?tab=readme-ov-file#citation) |
|
|
231
|
+
| FTRL | *Follow The Regularized Leader* | | <https://static.googleusercontent.com/media/research.google.com/en//pubs/archive/41159.pdf> | |
|
|
232
|
+
| Cautious | *Improving Training with One Line of Code* | [github](https://github.com/kyleliang919/C-Optim) | <https://arxiv.org/pdf/2411.16085v1> | [cite](https://github.com/kyleliang919/C-Optim?tab=readme-ov-file#citation) |
|
|
233
|
+
| DeMo | *Decoupled Momentum Optimization* | [github](https://github.com/bloc97/DeMo) | <https://arxiv.org/abs/2411.19870> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv241119870P/exportcitation) |
|
|
234
|
+
| MicroAdam | *Accurate Adaptive Optimization with Low Space Overhead and Provable Convergence* | [github](https://github.com/IST-DASLab/MicroAdam) | <https://arxiv.org/abs/2405.15593> | [cite](https://github.com/IST-DASLab/MicroAdam?tab=readme-ov-file#citing) |
|
|
235
|
+
| Muon | *MomentUm Orthogonalized by Newton-schulz* | [github](https://github.com/KellerJordan/Muon) | <https://x.com/kellerjordan0/status/1842300916864844014> | [cite](https://github.com/KellerJordan/Muon) |
|
|
236
|
+
| LaProp | *Separating Momentum and Adaptivity in Adam* | [github](https://github.com/Z-T-WANG/LaProp-Optimizer) | <https://arxiv.org/abs/2002.04839> | [cite](https://github.com/Z-T-WANG/LaProp-Optimizer?tab=readme-ov-file#citation) |
|
|
237
|
+
| APOLLO | *SGD-like Memory, AdamW-level Performance* | [github](https://github.com/zhuhanqing/APOLLO) | <https://arxiv.org/abs/2412.05270> | [cite](https://github.com/zhuhanqing/APOLLO?tab=readme-ov-file#-citation) |
|
|
238
|
+
| MARS | *Unleashing the Power of Variance Reduction for Training Large Models* | [github](https://github.com/AGI-Arena/MARS) | <https://arxiv.org/abs/2411.10438> | [cite](https://github.com/AGI-Arena/MARS/tree/main?tab=readme-ov-file#citation) |
|
|
239
|
+
| SGDSaI | *No More Adam: Learning Rate Scaling at Initialization is All You Need* | [github](https://github.com/AnonymousAlethiometer/SGD_SaI) | <https://arxiv.org/abs/2411.10438> | [cite](https://github.com/AnonymousAlethiometer/SGD_SaI?tab=readme-ov-file#citation) |
|
|
240
|
+
| Grams | *Gradient Descent with Adaptive Momentum Scaling* | | <https://arxiv.org/abs/2412.17107> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv241217107C/exportcitation) |
|
|
241
|
+
| OrthoGrad | *Grokking at the Edge of Numerical Stability* | [github](https://github.com/LucasPrietoAl/grokking-at-the-edge-of-numerical-stability) | <https://arxiv.org/abs/2501.04697> | [cite](https://github.com/LucasPrietoAl/grokking-at-the-edge-of-numerical-stability?tab=readme-ov-file#citation) |
|
|
242
|
+
| Adam-ATAN2 | *Scaling Exponents Across Parameterizations and Optimizers* | | <https://arxiv.org/abs/2407.05872> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240705872E/exportcitation) |
|
|
243
|
+
| SPAM | *Spike-Aware Adam with Momentum Reset for Stable LLM Training* | [github](https://github.com/TianjinYellow/SPAM-Optimizer) | <https://arxiv.org/abs/2501.06842> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250106842H/exportcitation) |
|
|
244
|
+
| TAM | *Torque-Aware Momentum* | | <https://arxiv.org/abs/2412.18790> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv241218790M/exportcitation) |
|
|
245
|
+
| FOCUS | *First Order Concentrated Updating Scheme* | [github](https://github.com/liuyz0/FOCUS) | <https://arxiv.org/abs/2501.12243> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250112243M/exportcitation) |
|
|
246
|
+
| PSGD | *Preconditioned Stochastic Gradient Descent* | [github](https://github.com/lixilinx/psgd_torch) | <https://arxiv.org/abs/1512.04202> | [cite](https://github.com/lixilinx/psgd_torch?tab=readme-ov-file#resources) |
|
|
247
|
+
| EXAdam | *The Power of Adaptive Cross-Moments* | [github](https://github.com/AhmedMostafa16/EXAdam) | <https://arxiv.org/abs/2412.20302> | [cite](https://github.com/AhmedMostafa16/EXAdam?tab=readme-ov-file#citation) |
|
|
248
|
+
| GCSAM | *Gradient Centralized Sharpness Aware Minimization* | [github](https://github.com/mhassann22/GCSAM) | <https://arxiv.org/abs/2501.11584> | [cite](https://github.com/mhassann22/GCSAM?tab=readme-ov-file#citation) |
|
|
249
|
+
| LookSAM | *Towards Efficient and Scalable Sharpness-Aware Minimization* | [github](https://github.com/rollovd/LookSAM) | <https://arxiv.org/abs/2203.02714> | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220302714L/exportcitation) |
|
|
250
|
+
| SCION | *Training Deep Learning Models with Norm-Constrained LMOs* | [github](https://github.com/LIONS-EPFL/scion) | <https://arxiv.org/abs/2502.07529> | [cite](https://github.com/LIONS-EPFL/scion?tab=readme-ov-file#citation) |
|
|
251
|
+
| COSMOS | *SOAP with Muon* | [github](https://github.com/lliu606/COSMOS) | | |
|
|
252
|
+
| StableSPAM | *How to Train in 4-Bit More Stably than 16-Bit Adam* | [github](https://github.com/TianjinYellow/StableSPAM) | <https://arxiv.org/abs/2502.17055> | |
|
|
253
|
+
| AdaGC | *Improving Training Stability for Large Language Model Pretraining* | | <https://arxiv.org/abs/2502.11034> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250211034W/exportcitation) |
|
|
254
|
+
| Simplified-Ademamix | *Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD Variants* | [github](https://github.com/DepenM/Simplified-AdEMAMix/) | <https://arxiv.org/abs/2502.02431> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250202431M/exportcitation) |
|
|
255
|
+
| Fira | *Can We Achieve Full-rank Training of LLMs Under Low-rank Constraint?* | [github](https://github.com/xichen-fy/Fira) | <https://arxiv.org/abs/2410.01623> | [cite](https://github.com/xichen-fy/Fira/tree/main?tab=readme-ov-file#citation) |
|
|
256
|
+
| RACS & Alice | *Towards Efficient Optimizer Design for LLM via Structured Fisher Approximation with a Low-Rank Extension* | | <https://arxiv.org/pdf/2502.07752> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250207752G/exportcitation) |
|
|
257
|
+
| VSGD | *Variational Stochastic Gradient Descent for Deep Neural Networks* | [github](https://github.com/generativeai-tue/vsgd) | <https://openreview.net/forum?id=xu4ATNjcdy> | [cite](https://github.com/generativeai-tue/vsgd/tree/main?tab=readme-ov-file#cite) |
|
|
258
|
+
| SNSM | *Subset-Norm and Subspace-Momentum: Faster Memory-Efficient Adaptive Optimization with Convergence Guarantees* | [github](https://github.com/timmytonga/sn-sm) | <https://arxiv.org/abs/2411.07120> | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv241107120N/exportcitation) |
|
|
259
|
+
| AdamC | *Why Gradients Rapidly Increase Near the End of Training* | | <https://arxiv.org/abs/2506.02285> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250602285D/exportcitation) |
|
|
260
|
+
| AdaMuon | *Adaptive Muon Optimizer* | | <https://arxiv.org/abs/2507.11005v1> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250711005S/exportcitation) |
|
|
261
|
+
| SPlus | *A Stable Whitening Optimizer for Efficient Neural Network Training* | [github](https://github.com/kvfrans/splus) | <https://arxiv.org/abs/2506.07254> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250607254F/exportcitation) |
|
|
262
|
+
| EmoNavi | *An emotion-driven optimizer that feels loss and navigates accordingly* | [github](https://github.com/muooon/EmoNavi) | | |
|
|
263
|
+
| Refined Schedule-Free | *Through the River: Understanding the Benefit of Schedule-Free Methods for Language Model Training* | | <https://arxiv.org/abs/2507.09846> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250709846S/exportcitation) |
|
|
264
|
+
| FriendlySAM | *Friendly Sharpness-Aware Minimization* | [github](https://github.com/nblt/F-SAM) | <https://openaccess.thecvf.com/content/CVPR2024/papers/Li_Friendly_Sharpness-Aware_Minimization_CVPR_2024_paper.pdf> | [cite](https://github.com/nblt/F-SAM?tab=readme-ov-file#citation) |
|
|
265
|
+
| AdaGO | *AdaGrad Meets Muon: Adaptive Stepsizes for Orthogonal Updates* | | <https://arxiv.org/abs/2509.02981> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250902981Z/exportcitation) |
|
|
266
|
+
| Conda | *Column-Normalized Adam for Training Large Language Models Faster* | [github](https://github.com/jie040109/Conda) | <https://arxiv.org/abs/2509.24218> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250924218W/exportcitation) |
|
|
261
267
|
|
|
262
268
|
## Supported LR Scheduler
|
|
263
269
|
|