pytorch-optimizer 3.8.2__tar.gz → 3.9.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/PKG-INFO +13 -10
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/README.md +11 -8
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pyproject.toml +18 -25
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/__init__.py +2 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/base/optimizer.py +26 -8
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/bi_tempered.py +2 -2
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/dice.py +3 -7
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/jaccard.py +2 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/lovasz.py +3 -1
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/__init__.py +4 -4
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/__init__.py +58 -72
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/a2grad.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adabelief.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adabound.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adadelta.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adafactor.py +6 -6
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adagc.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adahessian.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adai.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adalite.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adam_mini.py +4 -6
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adamax.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adamc.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adamg.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adamod.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adamp.py +10 -10
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adams.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adamw.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adan.py +6 -6
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adanorm.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adapnm.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adashift.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adasmooth.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/ademamix.py +10 -11
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/adopt.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/aggmo.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/aida.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/alig.py +8 -8
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/amos.py +5 -5
- pytorch_optimizer-3.9.0/pytorch_optimizer/optimizer/ano.py +130 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/apollo.py +10 -10
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/avagrad.py +6 -5
- pytorch_optimizer-3.9.0/pytorch_optimizer/optimizer/bcos.py +156 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/came.py +6 -6
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/conda.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/dadapt.py +17 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/diffgrad.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/emonavi.py +175 -72
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/exadam.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/experimental/ranger25.py +9 -20
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/fadam.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/fira.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/focus.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/fp16.py +7 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/fromage.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/ftrl.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/galore.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/galore_utils.py +19 -16
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/grams.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/gravity.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/grokfast.py +11 -9
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/kate.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/lamb.py +83 -69
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/laprop.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/lars.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/lion.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/lomo.py +6 -2
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/lookahead.py +2 -1
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/madgrad.py +5 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/mars.py +8 -8
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/msvag.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/muon.py +25 -24
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/nero.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/novograd.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/orthograd.py +2 -1
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/padam.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/pcgrad.py +1 -1
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/pid.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/pnm.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/prodigy.py +3 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/psgd.py +12 -10
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/qhadam.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/qhm.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/racs.py +8 -12
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/radam.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/ranger.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/ranger21.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/sam.py +32 -29
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/schedulefree.py +20 -20
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/scion.py +9 -11
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/sgd.py +25 -23
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/shampoo.py +10 -10
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/shampoo_utils.py +10 -8
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/sm3.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/snsm.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/soap.py +13 -9
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/sophia.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/spam.py +9 -11
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/splus.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/srmm.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/swats.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/tam.py +10 -10
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/tiger.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/trac.py +5 -1
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/utils.py +6 -3
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/yogi.py +5 -5
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/LICENSE +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/base/__init__.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/base/exception.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/base/scheduler.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/base/type.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/__init__.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/cross_entropy.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/f1.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/focal.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/ldam.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/loss/tversky.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/chebyshev.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/cosine_anealing.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/experimental/__init__.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/experimental/deberta_v3_lr_scheduler.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/linear_warmup.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/proportion.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/rex.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/wsd.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/agc.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/demo.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/experimental/__init__.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/gradient_centralization.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/psgd_utils.py +0 -0
- {pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/optimizer/rotograd.py +0 -0
|
@@ -1,10 +1,10 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: pytorch_optimizer
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.9.0
|
|
4
4
|
Summary: optimizer & lr scheduler & objective function collections in PyTorch
|
|
5
5
|
License: Apache-2.0
|
|
6
6
|
License-File: LICENSE
|
|
7
|
-
Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,Alice,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaGC,AdaMax,AdaMuon,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,Simplified-AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,CAME,Conda,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,EmoFact,EmoLynx,EmoNavi,EmoNeco,EmoZeal,EXAdam,FAdam,Fira,FOCUS,FriendlySAM,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,PSGD,QHAdam,QHM,RACS,RAdam,Ranger,Ranger21,RotoGrad,SAM,GCSAM,LookSAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SCION,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,StableSPAM,SPlus,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,VSGD,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
|
|
7
|
+
Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,Alice,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaGC,AdaMax,AdaMuon,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,Simplified-AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,BCOS,CAME,Conda,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,EmoFact,EmoLynx,EmoNavi,EmoNeco,EmoZeal,EXAdam,FAdam,Fira,FOCUS,FriendlySAM,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,PSGD,QHAdam,QHM,RACS,RAdam,Ranger,Ranger21,RotoGrad,SAM,GCSAM,LookSAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SCION,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,StableSPAM,SPlus,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,VSGD,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
|
|
8
8
|
Author: kozistr
|
|
9
9
|
Author-email: kozistr@gmail.com
|
|
10
10
|
Maintainer: kozistr
|
|
@@ -41,17 +41,17 @@ Description-Content-Type: text/markdown
|
|
|
41
41
|
|
|
42
42
|
# pytorch-optimizer
|
|
43
43
|
|
|
44
|
-
| |
|
|
45
|
-
|
|
46
|
-
| Build |  [](https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest)
|
|
47
|
-
| Quality | [](https://codecov.io/gh/kozistr/pytorch_optimizer) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/pr_codeql.yml)  [](https://github.com/charliermarsh/ruff)
|
|
48
|
-
| Package | [](https://badge.fury.io/py/pytorch-optimizer) [](https://pypi.python.org/pypi/pytorch-optimizer/)
|
|
49
|
-
| Status | [](https://pepy.tech/project/pytorch-optimizer) [](https://pepy.tech/project/pytorch-optimizer)
|
|
50
|
-
| License | [](https://opensource.org/licenses/Apache-2.0)
|
|
44
|
+
| | |
|
|
45
|
+
|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
46
|
+
| Build |  [](https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest) |
|
|
47
|
+
| Quality | [](https://codecov.io/gh/kozistr/pytorch_optimizer) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/pr_codeql.yml)  [](https://github.com/charliermarsh/ruff) |
|
|
48
|
+
| Package | [](https://badge.fury.io/py/pytorch-optimizer) [](https://pypi.python.org/pypi/pytorch-optimizer/) |
|
|
49
|
+
| Status | [](https://pepy.tech/project/pytorch-optimizer) [](https://pepy.tech/project/pytorch-optimizer) |
|
|
50
|
+
| License | [](https://opensource.org/licenses/Apache-2.0) |
|
|
51
51
|
|
|
52
52
|
## The reasons why you use `pytorch-optimizer`.
|
|
53
53
|
|
|
54
|
-
* Wide range of supported optimizers. Currently, **
|
|
54
|
+
* Wide range of supported optimizers. Currently, **131 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
|
|
55
55
|
* Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
|
|
56
56
|
* Easy to use, clean, and tested codes
|
|
57
57
|
* Active maintenance
|
|
@@ -264,6 +264,9 @@ get_supported_optimizers(['adam*', 'ranger*'])
|
|
|
264
264
|
| FriendlySAM | *Friendly Sharpness-Aware Minimization* | [github](https://github.com/nblt/F-SAM) | <https://openaccess.thecvf.com/content/CVPR2024/papers/Li_Friendly_Sharpness-Aware_Minimization_CVPR_2024_paper.pdf> | [cite](https://github.com/nblt/F-SAM?tab=readme-ov-file#citation) |
|
|
265
265
|
| AdaGO | *AdaGrad Meets Muon: Adaptive Stepsizes for Orthogonal Updates* | | <https://arxiv.org/abs/2509.02981> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250902981Z/exportcitation) |
|
|
266
266
|
| Conda | *Column-Normalized Adam for Training Large Language Models Faster* | [github](https://github.com/jie040109/Conda) | <https://arxiv.org/abs/2509.24218> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250924218W/exportcitation) |
|
|
267
|
+
| BCOS | *Stochastic Approximation with Block Coordinate Optimal Stepsizes* | [github](https://github.com/facebookresearch/bcos) | <https://arxiv.org/abs/2507.08963> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250708963J/exportcitation) |
|
|
268
|
+
| Cautious WD | *Cautious Weight Decay* | | <https://arxiv.org/abs/2510.12402> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv251012402C/exportcitation) |
|
|
269
|
+
| Ano | *Faster is Better in Noisy Landscape* | [github](https://github.com/Adrienkgz/ano-optimizer) | <https://arxiv.org/abs/2508.18258> | [cite](https://github.com/Adrienkgz/ano-optimizer?tab=readme-ov-file#citation) |
|
|
267
270
|
|
|
268
271
|
## Supported LR Scheduler
|
|
269
272
|
|
|
@@ -1,16 +1,16 @@
|
|
|
1
1
|
# pytorch-optimizer
|
|
2
2
|
|
|
3
|
-
| |
|
|
4
|
-
|
|
5
|
-
| Build |  [](https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest)
|
|
6
|
-
| Quality | [](https://codecov.io/gh/kozistr/pytorch_optimizer) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/pr_codeql.yml)  [](https://github.com/charliermarsh/ruff)
|
|
7
|
-
| Package | [](https://badge.fury.io/py/pytorch-optimizer) [](https://pypi.python.org/pypi/pytorch-optimizer/)
|
|
8
|
-
| Status | [](https://pepy.tech/project/pytorch-optimizer) [](https://pepy.tech/project/pytorch-optimizer)
|
|
9
|
-
| License | [](https://opensource.org/licenses/Apache-2.0)
|
|
3
|
+
| | |
|
|
4
|
+
|---------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
|
|
5
|
+
| Build |  [](https://pytorch-optimizers.readthedocs.io/en/latest/?badge=latest) |
|
|
6
|
+
| Quality | [](https://codecov.io/gh/kozistr/pytorch_optimizer) [](https://github.com/kozistr/pytorch_optimizer/actions/workflows/pr_codeql.yml)  [](https://github.com/charliermarsh/ruff) |
|
|
7
|
+
| Package | [](https://badge.fury.io/py/pytorch-optimizer) [](https://pypi.python.org/pypi/pytorch-optimizer/) |
|
|
8
|
+
| Status | [](https://pepy.tech/project/pytorch-optimizer) [](https://pepy.tech/project/pytorch-optimizer) |
|
|
9
|
+
| License | [](https://opensource.org/licenses/Apache-2.0) |
|
|
10
10
|
|
|
11
11
|
## The reasons why you use `pytorch-optimizer`.
|
|
12
12
|
|
|
13
|
-
* Wide range of supported optimizers. Currently, **
|
|
13
|
+
* Wide range of supported optimizers. Currently, **131 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
|
|
14
14
|
* Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
|
|
15
15
|
* Easy to use, clean, and tested codes
|
|
16
16
|
* Active maintenance
|
|
@@ -223,6 +223,9 @@ get_supported_optimizers(['adam*', 'ranger*'])
|
|
|
223
223
|
| FriendlySAM | *Friendly Sharpness-Aware Minimization* | [github](https://github.com/nblt/F-SAM) | <https://openaccess.thecvf.com/content/CVPR2024/papers/Li_Friendly_Sharpness-Aware_Minimization_CVPR_2024_paper.pdf> | [cite](https://github.com/nblt/F-SAM?tab=readme-ov-file#citation) |
|
|
224
224
|
| AdaGO | *AdaGrad Meets Muon: Adaptive Stepsizes for Orthogonal Updates* | | <https://arxiv.org/abs/2509.02981> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250902981Z/exportcitation) |
|
|
225
225
|
| Conda | *Column-Normalized Adam for Training Large Language Models Faster* | [github](https://github.com/jie040109/Conda) | <https://arxiv.org/abs/2509.24218> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250924218W/exportcitation) |
|
|
226
|
+
| BCOS | *Stochastic Approximation with Block Coordinate Optimal Stepsizes* | [github](https://github.com/facebookresearch/bcos) | <https://arxiv.org/abs/2507.08963> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250708963J/exportcitation) |
|
|
227
|
+
| Cautious WD | *Cautious Weight Decay* | | <https://arxiv.org/abs/2510.12402> | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv251012402C/exportcitation) |
|
|
228
|
+
| Ano | *Faster is Better in Noisy Landscape* | [github](https://github.com/Adrienkgz/ano-optimizer) | <https://arxiv.org/abs/2508.18258> | [cite](https://github.com/Adrienkgz/ano-optimizer?tab=readme-ov-file#citation) |
|
|
226
229
|
|
|
227
230
|
## Supported LR Scheduler
|
|
228
231
|
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
[tool.poetry]
|
|
2
2
|
name = "pytorch_optimizer"
|
|
3
|
-
version = "3.
|
|
3
|
+
version = "3.9.0"
|
|
4
4
|
description = "optimizer & lr scheduler & objective function collections in PyTorch"
|
|
5
5
|
license = "Apache-2.0"
|
|
6
6
|
authors = ["kozistr <kozistr@gmail.com>"]
|
|
@@ -13,16 +13,16 @@ keywords = [
|
|
|
13
13
|
"pytorch", "deep-learning", "optimizer", "lr scheduler", "A2Grad", "Alice", "ASGD", "AccSGD", "AdaBelief",
|
|
14
14
|
"AdaBound", "AdaDelta", "AdaFactor", "AdaGC", "AdaMax", "AdaMuon", "AdamG", "AdaMod", "AdaNorm", "AdaPNM",
|
|
15
15
|
"AdaSmooth", "AdEMAMix", "Simplified-AdEMAMix", "ADOPT", "AdaHessian", "Adai", "Adalite", "AdaLomo", "AdamMini",
|
|
16
|
-
"AdamP", "AdamS", "Adan", "AggMo", "Aida", "AliG", "Amos", "Apollo", "APOLLO", "AvaGrad", "bSAM", "
|
|
17
|
-
"DAdaptAdaGrad", "DAdaptAdam", "DAdaptAdan", "DAdaptSGD", "DAdaptLion", "DeMo", "DiffGrad", "EmoFact",
|
|
18
|
-
"EmoNavi", "EmoNeco", "EmoZeal", "EXAdam", "FAdam", "Fira", "FOCUS", "FriendlySAM", "Fromage", "FTRL",
|
|
19
|
-
"Grams", "Gravity", "GrokFast", "GSAM", "Kate", "Lamb", "LaProp", "LARS", "Lion", "LOMO", "Lookahead",
|
|
20
|
-
"MARS", "MSVAG", "Muno", "Nero", "NovoGrad", "OrthoGrad", "PAdam", "PCGrad", "PID", "PNM", "Prodigy",
|
|
21
|
-
"QHAdam", "QHM", "RACS", "RAdam", "Ranger", "Ranger21", "RotoGrad", "SAM", "GCSAM", "LookSAM",
|
|
22
|
-
"ScheduleFreeAdamW", "ScheduleFreeRAdam", "SCION", "SGDP", "Shampoo", "ScalableShampoo", "SGDW",
|
|
23
|
-
"SOAP", "SopihaH", "SPAM", "StableSPAM", "SPlus", "SRMM", "StableAdamW", "SWATS", "TAM", "Tiger",
|
|
24
|
-
"WSAM", "Yogi", "BCE", "BCEFocal", "Focal", "FocalCosine", "SoftF1", "Dice", "LDAM", "Jaccard",
|
|
25
|
-
"Tversky", "FocalTversky", "LovaszHinge", "bitsandbytes", "WSD", "QGaLore",
|
|
16
|
+
"AdamP", "AdamS", "Adan", "AggMo", "Aida", "AliG", "Amos", "Apollo", "APOLLO", "AvaGrad", "bSAM", "BCOS", "CAME",
|
|
17
|
+
"Conda", "DAdaptAdaGrad", "DAdaptAdam", "DAdaptAdan", "DAdaptSGD", "DAdaptLion", "DeMo", "DiffGrad", "EmoFact",
|
|
18
|
+
"EmoLynx", "EmoNavi", "EmoNeco", "EmoZeal", "EXAdam", "FAdam", "Fira", "FOCUS", "FriendlySAM", "Fromage", "FTRL",
|
|
19
|
+
"GaLore", "Grams", "Gravity", "GrokFast", "GSAM", "Kate", "Lamb", "LaProp", "LARS", "Lion", "LOMO", "Lookahead",
|
|
20
|
+
"MADGRAD", "MARS", "MSVAG", "Muno", "Nero", "NovoGrad", "OrthoGrad", "PAdam", "PCGrad", "PID", "PNM", "Prodigy",
|
|
21
|
+
"PSGD", "QHAdam", "QHM", "RACS", "RAdam", "Ranger", "Ranger21", "RotoGrad", "SAM", "GCSAM", "LookSAM",
|
|
22
|
+
"ScheduleFreeSGD", "ScheduleFreeAdamW", "ScheduleFreeRAdam", "SCION", "SGDP", "Shampoo", "ScalableShampoo", "SGDW",
|
|
23
|
+
"SignSGD", "SM3", "SOAP", "SopihaH", "SPAM", "StableSPAM", "SPlus", "SRMM", "StableAdamW", "SWATS", "TAM", "Tiger",
|
|
24
|
+
"TRAC", "VSGD", "WSAM", "Yogi", "BCE", "BCEFocal", "Focal", "FocalCosine", "SoftF1", "Dice", "LDAM", "Jaccard",
|
|
25
|
+
"Bi-Tempered", "Tversky", "FocalTversky", "LovaszHinge", "bitsandbytes", "WSD", "QGaLore",
|
|
26
26
|
]
|
|
27
27
|
classifiers = [
|
|
28
28
|
"License :: OSI Approved :: Apache Software License",
|
|
@@ -65,7 +65,10 @@ black = [
|
|
|
65
65
|
]
|
|
66
66
|
ruff = "^0.14"
|
|
67
67
|
pyright = "^1.1"
|
|
68
|
-
pytest =
|
|
68
|
+
pytest = [
|
|
69
|
+
{ version = "<9", python = "<3.10" },
|
|
70
|
+
{ version = "^9", python = ">=3.10" },
|
|
71
|
+
]
|
|
69
72
|
pytest-cov = [
|
|
70
73
|
{ version = "<6", python = "<3.9" },
|
|
71
74
|
{ version = "^6", python = ">=3.9" },
|
|
@@ -103,7 +106,7 @@ ignore = [
|
|
|
103
106
|
"B905",
|
|
104
107
|
"D100", "D102", "D104", "D105", "D107", "D203", "D213", "D413",
|
|
105
108
|
"PLR0913", "PLR2004", "PLW2901",
|
|
106
|
-
"Q003", "ARG002",
|
|
109
|
+
"Q003", "ARG002", "RUF100",
|
|
107
110
|
]
|
|
108
111
|
fixable = ["ALL"]
|
|
109
112
|
unfixable = ["F401"]
|
|
@@ -113,7 +116,8 @@ flake8-quotes.inline-quotes = "single"
|
|
|
113
116
|
|
|
114
117
|
[tool.ruff.lint.extend-per-file-ignores]
|
|
115
118
|
"examples/visualize_optimizers.py" = ["D103", "D400", "D415"]
|
|
116
|
-
"{tests,examples}/*.py" = ["D", "S101"]
|
|
119
|
+
"{tests,examples,scripts}/*.py" = ["D", "S101", "T201"]
|
|
120
|
+
"{examples,scripts}/*.py" = ["INP001"]
|
|
117
121
|
"**/__init__.py" = ["F401"]
|
|
118
122
|
"hubconf.py" = ["D", "INP001"]
|
|
119
123
|
|
|
@@ -138,23 +142,12 @@ exclude = [
|
|
|
138
142
|
"./pytorch_optimizer/optimizer/rotograd.py",
|
|
139
143
|
]
|
|
140
144
|
reportArgumentType = false
|
|
141
|
-
reportAssignmentType = false
|
|
142
145
|
reportAttributeAccessIssue = false
|
|
143
146
|
reportCallIssue = false
|
|
144
147
|
reportIncompatibleMethodOverride = false
|
|
145
148
|
reportIncompatibleVariableOverride = false
|
|
146
|
-
reportIndexIssue = false
|
|
147
149
|
reportMissingImports = false
|
|
148
|
-
reportMissingTypeStubs = false
|
|
149
|
-
reportOperatorIssue = false
|
|
150
|
-
reportOptionalIterable = false
|
|
151
150
|
reportOptionalMemberAccess = false
|
|
152
|
-
reportOptionalOperand = false
|
|
153
|
-
reportOptionalSubscript = false
|
|
154
|
-
reportPossiblyUnboundVariable = false
|
|
155
|
-
reportPrivateImportUsage = false
|
|
156
|
-
reportRedeclaration = false
|
|
157
|
-
reportReturnType = false
|
|
158
151
|
pythonVersion = "3.12"
|
|
159
152
|
pythonPlatform = "Linux"
|
|
160
153
|
|
|
@@ -45,6 +45,7 @@ from pytorch_optimizer.optimizer import (
|
|
|
45
45
|
APOLLO,
|
|
46
46
|
ASGD,
|
|
47
47
|
BSAM,
|
|
48
|
+
BCOS,
|
|
48
49
|
CAME,
|
|
49
50
|
FOCUS,
|
|
50
51
|
FTRL,
|
|
@@ -109,6 +110,7 @@ from pytorch_optimizer.optimizer import (
|
|
|
109
110
|
Alice,
|
|
110
111
|
AliG,
|
|
111
112
|
Amos,
|
|
113
|
+
Ano,
|
|
112
114
|
ApolloDQN,
|
|
113
115
|
AvaGrad,
|
|
114
116
|
DAdaptAdaGrad,
|
|
@@ -58,7 +58,7 @@ class BaseOptimizer(ABC, Optimizer):
|
|
|
58
58
|
>>> optimizer.step(hessian=hessian_diag_est)
|
|
59
59
|
"""
|
|
60
60
|
i: int = 0
|
|
61
|
-
for group in param_groups:
|
|
61
|
+
for group in param_groups or []:
|
|
62
62
|
for p in group['params']:
|
|
63
63
|
if p.size() != hessian[i].size():
|
|
64
64
|
raise ValueError(
|
|
@@ -77,7 +77,7 @@ class BaseOptimizer(ABC, Optimizer):
|
|
|
77
77
|
state (State): Optimizer state dictionary.
|
|
78
78
|
pre_zero (bool): If True, zero-out the Hessian before computing/updating it.
|
|
79
79
|
"""
|
|
80
|
-
for group in param_groups:
|
|
80
|
+
for group in param_groups or []:
|
|
81
81
|
for p in group['params']:
|
|
82
82
|
if p.requires_grad and p.grad is not None and not p.grad.is_sparse:
|
|
83
83
|
if 'hessian' not in state[p]:
|
|
@@ -108,7 +108,7 @@ class BaseOptimizer(ABC, Optimizer):
|
|
|
108
108
|
|
|
109
109
|
params: List[torch.Tensor] = [
|
|
110
110
|
p
|
|
111
|
-
for group in param_groups
|
|
111
|
+
for group in param_groups or []
|
|
112
112
|
for p in group['params']
|
|
113
113
|
if p.requires_grad and p.grad is not None and not p.grad.is_sparse
|
|
114
114
|
]
|
|
@@ -137,7 +137,7 @@ class BaseOptimizer(ABC, Optimizer):
|
|
|
137
137
|
fixed_decay: bool,
|
|
138
138
|
ratio: Optional[float] = None,
|
|
139
139
|
) -> None:
|
|
140
|
-
"""Apply weight decay.
|
|
140
|
+
"""Apply weight decay in an in-place manner.
|
|
141
141
|
|
|
142
142
|
Args:
|
|
143
143
|
p (torch.Tensor): Parameter tensor to apply weight decay to.
|
|
@@ -153,6 +153,23 @@ class BaseOptimizer(ABC, Optimizer):
|
|
|
153
153
|
elif weight_decay > 0.0 and grad is not None:
|
|
154
154
|
grad.add_(p, alpha=weight_decay)
|
|
155
155
|
|
|
156
|
+
@staticmethod
|
|
157
|
+
def apply_cautious_weight_decay(
|
|
158
|
+
p: torch.Tensor,
|
|
159
|
+
update: torch.Tensor,
|
|
160
|
+
lr: float,
|
|
161
|
+
weight_decay: float,
|
|
162
|
+
) -> None:
|
|
163
|
+
"""Apply cautious weight decay (CWD) in an in-place manner.
|
|
164
|
+
|
|
165
|
+
Args:
|
|
166
|
+
p (torch.Tensor): Parameter tensor to apply weight decay to.
|
|
167
|
+
update (torch.Tensor): update tensor.
|
|
168
|
+
lr (float): Learning rate to scale the update.
|
|
169
|
+
weight_decay (float): Weight decay coefficient (L2 penalty).
|
|
170
|
+
"""
|
|
171
|
+
p.copy_(torch.where(update * p >= 0, p * (1.0 - weight_decay * lr), p))
|
|
172
|
+
|
|
156
173
|
@staticmethod
|
|
157
174
|
def apply_ams_bound(
|
|
158
175
|
ams_bound: bool,
|
|
@@ -385,11 +402,12 @@ class BaseOptimizer(ABC, Optimizer):
|
|
|
385
402
|
self.validate_range(betas[2], 'beta3', 0.0, 1.0, range_type=beta3_range_type)
|
|
386
403
|
|
|
387
404
|
def validate_nus(self, nus: Union[float, Tuple[float, float]]) -> None:
|
|
388
|
-
if isinstance(nus,
|
|
389
|
-
|
|
405
|
+
if isinstance(nus, tuple):
|
|
406
|
+
nu1, nu2 = nus
|
|
407
|
+
self.validate_range(nu1, 'nu1', 0.0, 1.0, range_type='[]')
|
|
408
|
+
self.validate_range(nu2, 'nu2', 0.0, 1.0, range_type='[]')
|
|
390
409
|
else:
|
|
391
|
-
self.validate_range(nus
|
|
392
|
-
self.validate_range(nus[1], 'nu2', 0.0, 1.0, range_type='[]')
|
|
410
|
+
self.validate_range(nus, 'nu', 0.0, 1.0, range_type='[]')
|
|
393
411
|
|
|
394
412
|
@abstractmethod
|
|
395
413
|
def init_group(self, group: ParamGroup, **kwargs) -> None: # pragma: no cover
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
from typing import Optional, Tuple
|
|
1
|
+
from typing import Optional, Tuple, cast
|
|
2
2
|
|
|
3
3
|
import torch
|
|
4
4
|
from torch import nn
|
|
@@ -107,7 +107,7 @@ def compute_normalization(activations: torch.Tensor, t: float, num_iters: int =
|
|
|
107
107
|
t (float): Temperature parameter (> 1.0 for tail heaviness).
|
|
108
108
|
num_iters (int): Number of iterations to run the method.
|
|
109
109
|
"""
|
|
110
|
-
return ComputeNormalization.apply(activations, t, num_iters)
|
|
110
|
+
return cast(torch.Tensor, ComputeNormalization.apply(activations, t, num_iters))
|
|
111
111
|
|
|
112
112
|
|
|
113
113
|
def tempered_softmax(activations: torch.Tensor, t: float, num_iters: int = 5) -> torch.Tensor:
|
|
@@ -41,8 +41,7 @@ class DiceLoss(_Loss):
|
|
|
41
41
|
|
|
42
42
|
Args:
|
|
43
43
|
mode (CLASS_MODE): Loss mode - 'binary', 'multiclass', or 'multilabel'.
|
|
44
|
-
classes (Optional[List[int]]): List of classes to include in loss computation.
|
|
45
|
-
Defaults to all classes.
|
|
44
|
+
classes (Optional[List[int]]): List of classes to include in loss computation. Defaults to all classes.
|
|
46
45
|
log_loss (bool): If True, loss is computed as `-log(dice_coeff)`; otherwise `1 - dice_coeff`.
|
|
47
46
|
from_logits (bool): If True, assumes input is raw logits.
|
|
48
47
|
label_smooth (float): Smoothness constant for dice coefficient numerator and denominator.
|
|
@@ -62,11 +61,8 @@ class DiceLoss(_Loss):
|
|
|
62
61
|
):
|
|
63
62
|
super().__init__()
|
|
64
63
|
|
|
65
|
-
if classes is not None:
|
|
66
|
-
|
|
67
|
-
raise ValueError('masking classes is not supported with mode=binary')
|
|
68
|
-
|
|
69
|
-
classes = torch.LongTensor(classes)
|
|
64
|
+
if classes is not None and mode == 'binary':
|
|
65
|
+
raise ValueError('masking classes is not supported with mode=binary')
|
|
70
66
|
|
|
71
67
|
self.mode = mode
|
|
72
68
|
self.classes = classes
|
|
@@ -60,11 +60,8 @@ class JaccardLoss(_Loss):
|
|
|
60
60
|
):
|
|
61
61
|
super().__init__()
|
|
62
62
|
|
|
63
|
-
if classes is not None:
|
|
64
|
-
|
|
65
|
-
raise ValueError('masking classes is not supported with mode=binary')
|
|
66
|
-
|
|
67
|
-
classes = torch.LongTensor(classes)
|
|
63
|
+
if classes is not None and mode == 'binary':
|
|
64
|
+
raise ValueError('masking classes is not supported with mode=binary')
|
|
68
65
|
|
|
69
66
|
self.mode = mode
|
|
70
67
|
self.classes = classes
|
|
@@ -49,4 +49,6 @@ class LovaszHingeLoss(nn.Module):
|
|
|
49
49
|
def forward(self, y_pred: torch.Tensor, y_true: torch.Tensor) -> torch.Tensor:
|
|
50
50
|
if not self.per_image:
|
|
51
51
|
return lovasz_hinge_flat(y_pred, y_true)
|
|
52
|
-
|
|
52
|
+
|
|
53
|
+
losses = torch.stack([lovasz_hinge_flat(y_p, y_t) for y_p, y_t in zip(y_pred, y_true)])
|
|
54
|
+
return losses.mean()
|
{pytorch_optimizer-3.8.2 → pytorch_optimizer-3.9.0}/pytorch_optimizer/lr_scheduler/__init__.py
RENAMED
|
@@ -74,12 +74,12 @@ def load_lr_scheduler(lr_scheduler_name: str) -> SCHEDULER:
|
|
|
74
74
|
|
|
75
75
|
:param lr_scheduler_name: learning rate scheduler name.
|
|
76
76
|
"""
|
|
77
|
-
|
|
77
|
+
lrs_name: str = lr_scheduler_name.lower()
|
|
78
78
|
|
|
79
|
-
if
|
|
80
|
-
raise NotImplementedError(f'not implemented lr_scheduler {
|
|
79
|
+
if lrs_name not in LR_SCHEDULERS:
|
|
80
|
+
raise NotImplementedError(f'not implemented lr_scheduler {lrs_name}')
|
|
81
81
|
|
|
82
|
-
return LR_SCHEDULERS[
|
|
82
|
+
return LR_SCHEDULERS[lrs_name]
|
|
83
83
|
|
|
84
84
|
|
|
85
85
|
def get_supported_lr_schedulers(filters: Optional[Union[str, List[str]]] = None) -> List[str]:
|
|
@@ -37,8 +37,10 @@ from pytorch_optimizer.optimizer.aggmo import AggMo
|
|
|
37
37
|
from pytorch_optimizer.optimizer.aida import Aida
|
|
38
38
|
from pytorch_optimizer.optimizer.alig import AliG
|
|
39
39
|
from pytorch_optimizer.optimizer.amos import Amos
|
|
40
|
+
from pytorch_optimizer.optimizer.ano import Ano
|
|
40
41
|
from pytorch_optimizer.optimizer.apollo import APOLLO, ApolloDQN
|
|
41
42
|
from pytorch_optimizer.optimizer.avagrad import AvaGrad
|
|
43
|
+
from pytorch_optimizer.optimizer.bcos import BCOS
|
|
42
44
|
from pytorch_optimizer.optimizer.came import CAME
|
|
43
45
|
from pytorch_optimizer.optimizer.conda import Conda
|
|
44
46
|
from pytorch_optimizer.optimizer.dadapt import DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptLion, DAdaptSGD
|
|
@@ -154,7 +156,9 @@ OPTIMIZER_LIST: List[OPTIMIZER] = [
|
|
|
154
156
|
Aida,
|
|
155
157
|
AliG,
|
|
156
158
|
Alice,
|
|
159
|
+
BCOS,
|
|
157
160
|
Amos,
|
|
161
|
+
Ano,
|
|
158
162
|
ApolloDQN,
|
|
159
163
|
AvaGrad,
|
|
160
164
|
BSAM,
|
|
@@ -235,75 +239,57 @@ OPTIMIZER_LIST: List[OPTIMIZER] = [
|
|
|
235
239
|
]
|
|
236
240
|
OPTIMIZERS: Dict[str, OPTIMIZER] = {str(optimizer.__name__).lower(): optimizer for optimizer in OPTIMIZER_LIST}
|
|
237
241
|
|
|
242
|
+
BNB_OPTIMIZERS = (
|
|
243
|
+
('paged_ademamix8bit', 'PagedAdEMAMix8bit'),
|
|
244
|
+
('paged_ademamix32bit', 'PagedAdEMAMix32bit'),
|
|
245
|
+
('paged_adam8bit', 'PagedAdam8bit'),
|
|
246
|
+
('paged_adam32bit', 'PagedAdam32bit'),
|
|
247
|
+
('paged_adamw8bit', 'PagedAdamW8bit'),
|
|
248
|
+
('paged_lion32bit', 'PagedLion32bit'),
|
|
249
|
+
('ademamix8bit', 'AdEMAMix8bit'),
|
|
250
|
+
('ademamix32bit', 'AdEMAMix32bit'),
|
|
251
|
+
('adam8bit', 'Adam8bit'),
|
|
252
|
+
('adam32bit', 'Adam32bit'),
|
|
253
|
+
('adamw8bit', 'AdamW8bit'),
|
|
254
|
+
('adamw32bit', 'AdamW32bit'),
|
|
255
|
+
('adagrad8bit', 'Adagrad8bit'),
|
|
256
|
+
('adagrad32bit', 'Adagrad32bit'),
|
|
257
|
+
('lamb8bit', 'LAMB8bit'),
|
|
258
|
+
('lamb32bit', 'LAMB32bit'),
|
|
259
|
+
('lars8bit', 'LARS8bit'),
|
|
260
|
+
('lars32bit', 'LARS32bit'),
|
|
261
|
+
('lion8bit', 'Lion8bit'),
|
|
262
|
+
('lion32bit', 'Lion32bit'),
|
|
263
|
+
('rmsprop8bit', 'RMSprop8bit'),
|
|
264
|
+
('rmsprop32bit', 'RMSprop32bit'),
|
|
265
|
+
('sgd8bit', 'SGD8bit'),
|
|
266
|
+
('sgd32bit', 'SGD32bit'),
|
|
267
|
+
)
|
|
268
|
+
|
|
238
269
|
|
|
239
|
-
def load_bnb_optimizer(optimizer: str) -> OPTIMIZER: # pragma: no cover
|
|
240
|
-
|
|
270
|
+
def load_bnb_optimizer(optimizer: str) -> OPTIMIZER: # pragma: no cover
|
|
271
|
+
"""Load bnb optimizer instance."""
|
|
241
272
|
from bitsandbytes import optim # noqa: PLC0415
|
|
242
273
|
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
return optim.PagedAdam8bit
|
|
249
|
-
if 'adamw8bit' in optimizer:
|
|
250
|
-
return optim.AdamW8bit
|
|
251
|
-
if 'paged_adamw8bit' in optimizer:
|
|
252
|
-
return optim.PagedAdamW8bit
|
|
253
|
-
if 'lamb8bit' in optimizer:
|
|
254
|
-
return optim.LAMB8bit
|
|
255
|
-
if 'lars8bit' in optimizer:
|
|
256
|
-
return optim.LARS8bit
|
|
257
|
-
if 'lion8bit' in optimizer:
|
|
258
|
-
return optim.Lion8bit
|
|
259
|
-
if 'adagrad8bit' in optimizer:
|
|
260
|
-
return optim.Adagrad8bit
|
|
261
|
-
if 'rmsprop8bit' in optimizer:
|
|
262
|
-
return optim.RMSprop8bit
|
|
263
|
-
if 'adagrad32bit' in optimizer:
|
|
264
|
-
return optim.Adagrad32bit
|
|
265
|
-
if 'adam32bit' in optimizer:
|
|
266
|
-
return optim.Adam32bit
|
|
267
|
-
if 'paged_adam32bit' in optimizer:
|
|
268
|
-
return optim.PagedAdam32bit
|
|
269
|
-
if 'adamw32bit' in optimizer:
|
|
270
|
-
return optim.AdamW32bit
|
|
271
|
-
if 'lamb32bit' in optimizer:
|
|
272
|
-
return optim.LAMB32bit
|
|
273
|
-
if 'lars32bit' in optimizer:
|
|
274
|
-
return optim.LARS32bit
|
|
275
|
-
if 'lion32bit' in optimizer:
|
|
276
|
-
return optim.Lion32bit
|
|
277
|
-
if 'paged_lion32bit' in optimizer:
|
|
278
|
-
return optim.PagedLion32bit
|
|
279
|
-
if 'rmsprop32bit' in optimizer:
|
|
280
|
-
return optim.RMSprop32bit
|
|
281
|
-
if 'sgd32bit' in optimizer:
|
|
282
|
-
return optim.SGD32bit
|
|
283
|
-
if 'ademamix8bit' in optimizer:
|
|
284
|
-
return optim.AdEMAMix8bit
|
|
285
|
-
if 'ademamix32bit' in optimizer:
|
|
286
|
-
return optim.AdEMAMix32bit
|
|
287
|
-
if 'paged_ademamix8bit' in optimizer:
|
|
288
|
-
return optim.PagedAdEMAMix8bit
|
|
289
|
-
if 'paged_ademamix32bit' in optimizer:
|
|
290
|
-
return optim.PagedAdEMAMix32bit
|
|
291
|
-
|
|
292
|
-
raise NotImplementedError(f'[-] not implemented optimizer : {optimizer}')
|
|
274
|
+
for name, cls_name in BNB_OPTIMIZERS:
|
|
275
|
+
if name in optimizer:
|
|
276
|
+
return getattr(optim, cls_name)
|
|
277
|
+
|
|
278
|
+
raise NotImplementedError(f'not implemented optimizer {optimizer}')
|
|
293
279
|
|
|
294
280
|
|
|
295
281
|
def load_q_galore_optimizer(optimizer: str) -> OPTIMIZER: # pragma: no cover
|
|
296
|
-
|
|
282
|
+
"""Load Q-GaLore optimizer instance."""
|
|
297
283
|
import q_galore_torch # noqa: PLC0415
|
|
298
284
|
|
|
299
285
|
if 'adamw8bit' in optimizer:
|
|
300
286
|
return q_galore_torch.QGaLoreAdamW8bit
|
|
301
287
|
|
|
302
|
-
raise NotImplementedError(f'
|
|
288
|
+
raise NotImplementedError(f'not implemented optimizer {optimizer}')
|
|
303
289
|
|
|
304
290
|
|
|
305
291
|
def load_ao_optimizer(optimizer: str) -> OPTIMIZER: # pragma: no cover
|
|
306
|
-
|
|
292
|
+
"""Load TorchAO optimizer instance."""
|
|
307
293
|
from torchao.prototype import low_bit_optim # noqa: PLC0415
|
|
308
294
|
|
|
309
295
|
if 'adamw8bit' in optimizer:
|
|
@@ -313,32 +299,32 @@ def load_ao_optimizer(optimizer: str) -> OPTIMIZER: # pragma: no cover
|
|
|
313
299
|
if 'adamwfp8' in optimizer:
|
|
314
300
|
return low_bit_optim.AdamWFp8
|
|
315
301
|
|
|
316
|
-
raise NotImplementedError(f'
|
|
302
|
+
raise NotImplementedError(f'not implemented optimizer {optimizer}')
|
|
317
303
|
|
|
318
304
|
|
|
319
305
|
def load_optimizer(optimizer: str) -> OPTIMIZER:
|
|
320
|
-
|
|
321
|
-
|
|
306
|
+
"""Load optimizers."""
|
|
307
|
+
optimizer_name: str = optimizer.lower()
|
|
322
308
|
|
|
323
|
-
if
|
|
309
|
+
if optimizer_name.startswith('bnb'):
|
|
324
310
|
if HAS_BNB and torch.cuda.is_available():
|
|
325
|
-
return load_bnb_optimizer(
|
|
326
|
-
raise ImportError(f'bitsandbytes and CUDA required for the optimizer {
|
|
327
|
-
if
|
|
311
|
+
return load_bnb_optimizer(optimizer_name) # pragma: no cover
|
|
312
|
+
raise ImportError(f'bitsandbytes and CUDA required for the optimizer {optimizer_name}')
|
|
313
|
+
if optimizer_name.startswith('q_galore'):
|
|
328
314
|
if HAS_Q_GALORE and torch.cuda.is_available():
|
|
329
|
-
return load_q_galore_optimizer(
|
|
330
|
-
raise ImportError(f'bitsandbytes, q-galore-torch, and CUDA required for the optimizer {
|
|
331
|
-
if
|
|
315
|
+
return load_q_galore_optimizer(optimizer_name) # pragma: no cover
|
|
316
|
+
raise ImportError(f'bitsandbytes, q-galore-torch, and CUDA required for the optimizer {optimizer_name}')
|
|
317
|
+
if optimizer_name.startswith('torchao'):
|
|
332
318
|
if HAS_TORCHAO and torch.cuda.is_available():
|
|
333
|
-
return load_ao_optimizer(
|
|
319
|
+
return load_ao_optimizer(optimizer_name) # pragma: no cover
|
|
334
320
|
raise ImportError(
|
|
335
|
-
f'torchao required for the optimizer {
|
|
321
|
+
f'torchao required for the optimizer {optimizer_name}. '
|
|
336
322
|
'usage: https://github.com/pytorch/ao/tree/main/torchao/prototype/low_bit_optim#usage'
|
|
337
323
|
)
|
|
338
|
-
if
|
|
339
|
-
raise NotImplementedError(f'not implemented optimizer : {
|
|
324
|
+
if optimizer_name not in OPTIMIZERS:
|
|
325
|
+
raise NotImplementedError(f'not implemented optimizer : {optimizer_name}')
|
|
340
326
|
|
|
341
|
-
return OPTIMIZERS[
|
|
327
|
+
return OPTIMIZERS[optimizer_name]
|
|
342
328
|
|
|
343
329
|
|
|
344
330
|
def create_optimizer(
|
|
@@ -374,7 +360,7 @@ def create_optimizer(
|
|
|
374
360
|
elif optimizer_name in ('lomo', 'adalomo', 'adammini'):
|
|
375
361
|
optimizer = optimizer_class(model, lr=lr, **kwargs)
|
|
376
362
|
elif optimizer_name in ('muon', 'adamuon', 'adago'):
|
|
377
|
-
warn(f'highly recommend you to manually create the {optimizer_name} manually.', UserWarning, 1)
|
|
363
|
+
warn(f'highly recommend you to manually create the {optimizer_name} manually.', UserWarning, stacklevel=1)
|
|
378
364
|
|
|
379
365
|
optimizer = prepare_muon_parameters(model, optimizer_name, lr=lr, weight_decay=weight_decay, **kwargs)
|
|
380
366
|
else:
|
|
@@ -385,7 +371,7 @@ def create_optimizer(
|
|
|
385
371
|
|
|
386
372
|
if use_lookahead:
|
|
387
373
|
if optimizer_name in ('ranger', 'ranger21', 'ranger25'):
|
|
388
|
-
warn(f'{optimizer} already has a Lookahead variant.', UserWarning, 1)
|
|
374
|
+
warn(f'{optimizer} already has a Lookahead variant.', UserWarning, stacklevel=1)
|
|
389
375
|
return optimizer
|
|
390
376
|
|
|
391
377
|
optimizer = Lookahead(
|
|
@@ -52,6 +52,9 @@ class A2Grad(BaseOptimizer):
|
|
|
52
52
|
return 'A2Grad'
|
|
53
53
|
|
|
54
54
|
def init_group(self, group: ParamGroup, **kwargs) -> None:
|
|
55
|
+
if 'step' not in group:
|
|
56
|
+
group['step'] = 0
|
|
57
|
+
|
|
55
58
|
for p in group['params']:
|
|
56
59
|
if p.grad is None:
|
|
57
60
|
continue
|
|
@@ -81,11 +84,8 @@ class A2Grad(BaseOptimizer):
|
|
|
81
84
|
loss = closure()
|
|
82
85
|
|
|
83
86
|
for group in self.param_groups:
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
group['step'] = 1
|
|
87
|
-
else:
|
|
88
|
-
group['step'] += 1
|
|
87
|
+
self.init_group(group)
|
|
88
|
+
group['step'] += 1
|
|
89
89
|
|
|
90
90
|
gamma_k: float = 2.0 * group['lips'] / (group['step'] + 1)
|
|
91
91
|
alpha_k_1: float = 2.0 / (group['step'] + 3)
|