pytorch-optimizer 3.4.0__tar.gz → 3.4.2__tar.gz

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: pytorch_optimizer
-Version: 3.4.0
+Version: 3.4.2
 Summary: optimizer & lr scheduler & objective function collections in PyTorch
 License: Apache-2.0
-Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaMax,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,CAME,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,EXAdam,FAdam,FOCUS,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,PSGD,QHAdam,QHM,RAdam,Ranger,Ranger21,RotoGrad,SAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
+Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaMax,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,CAME,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,EXAdam,FAdam,FOCUS,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,PSGD,QHAdam,QHM,RAdam,Ranger,Ranger21,RotoGrad,SAM,GCSAM,LookSAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SCION,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
 Author: kozistr
 Author-email: kozistr@gmail.com
 Maintainer: kozistr
@@ -22,8 +22,6 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development
@@ -31,7 +29,8 @@ Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
 Requires-Dist: numpy (<=1.24.4) ; python_version < "3.9"
 Requires-Dist: numpy (>1.24.4) ; python_version >= "3.9"
-Requires-Dist: torch (>=1.10) ; python_version >= "3.8"
+Requires-Dist: torch (>=1.10) ; python_version >= "3.9"
+Requires-Dist: torch (>=1.10,<2.5) ; python_version < "3.9"
 Project-URL: Documentation, https://pytorch-optimizers.readthedocs.io/en/latest
 Project-URL: Homepage, https://github.com/kozistr/pytorch_optimizer
 Project-URL: Repository, https://github.com/kozistr/pytorch_optimizer
@@ -49,7 +48,7 @@ Description-Content-Type: text/markdown
 
 ## The reasons why you use `pytorch-optimizer`.
 
-* Wide range of supported optimizers. Currently, **95 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
+* Wide range of supported optimizers. Currently, **99 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
 * Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
 * Easy to use, clean, and tested codes
 * Active maintenance
@@ -118,6 +117,7 @@ optimizer = create_optimizer(
     weight_decay=1e-3,
     use_gc=True,
     use_lookahead=True,
+    use_orthograd=False,
 )
 ```
 
@@ -242,6 +242,9 @@ get_supported_optimizers(['adam*', 'ranger*'])
 | FOCUS         | *First Order Concentrated Updating Scheme*                                                        | [github](https://github.com/liuyz0/FOCUS)                                                                      | <https://arxiv.org/abs/2501.12243>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250112243M/exportcitation)                                                        |
 | PSGD          | *Preconditioned Stochastic Gradient Descent*                                                      | [github](https://github.com/lixilinx/psgd_torch)                                                               | <https://arxiv.org/abs/1512.04202>                                                          | [cite](https://github.com/lixilinx/psgd_torch?tab=readme-ov-file#resources)                                                         |
 | EXAdam        | *The Power of Adaptive Cross-Moments*                                                             | [github](https://github.com/AhmedMostafa16/EXAdam)                                                             | <https://arxiv.org/abs/2412.20302>                                                          | [cite](https://github.com/AhmedMostafa16/EXAdam?tab=readme-ov-file#citation)                                                        |
+| GCSAM         | *Gradient Centralized Sharpness Aware Minimization*                                               | [github](https://github.com/mhassann22/GCSAM)                                                                  | <https://arxiv.org/abs/2501.11584>                                                          | [cite](https://github.com/mhassann22/GCSAM?tab=readme-ov-file#citation)                                                             |
+| LookSAM       | *Towards Efficient and Scalable Sharpness-Aware Minimization*                                     | [github](https://github.com/rollovd/LookSAM)                                                                   | <https://arxiv.org/abs/2203.02714>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220302714L/exportcitation)                                                        |
+| SCION         | *Training Deep Learning Models with Norm-Constrained LMOs*                                        |                                                                                                                | <https://arxiv.org/abs/2502.07529>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250207529P/exportcitation)                                                        |
 
 ## Supported LR Scheduler
 

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/README.md

@@ -10,7 +10,7 @@
 
 ## The reasons why you use `pytorch-optimizer`.
 
-* Wide range of supported optimizers. Currently, **95 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
+* Wide range of supported optimizers. Currently, **99 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
 * Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
 * Easy to use, clean, and tested codes
 * Active maintenance
@@ -79,6 +79,7 @@ optimizer = create_optimizer(
     weight_decay=1e-3,
     use_gc=True,
     use_lookahead=True,
+    use_orthograd=False,
 )
 ```
 
@@ -203,6 +204,9 @@ get_supported_optimizers(['adam*', 'ranger*'])
 | FOCUS         | *First Order Concentrated Updating Scheme*                                                        | [github](https://github.com/liuyz0/FOCUS)                                                                      | <https://arxiv.org/abs/2501.12243>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250112243M/exportcitation)                                                        |
 | PSGD          | *Preconditioned Stochastic Gradient Descent*                                                      | [github](https://github.com/lixilinx/psgd_torch)                                                               | <https://arxiv.org/abs/1512.04202>                                                          | [cite](https://github.com/lixilinx/psgd_torch?tab=readme-ov-file#resources)                                                         |
 | EXAdam        | *The Power of Adaptive Cross-Moments*                                                             | [github](https://github.com/AhmedMostafa16/EXAdam)                                                             | <https://arxiv.org/abs/2412.20302>                                                          | [cite](https://github.com/AhmedMostafa16/EXAdam?tab=readme-ov-file#citation)                                                        |
+| GCSAM         | *Gradient Centralized Sharpness Aware Minimization*                                               | [github](https://github.com/mhassann22/GCSAM)                                                                  | <https://arxiv.org/abs/2501.11584>                                                          | [cite](https://github.com/mhassann22/GCSAM?tab=readme-ov-file#citation)                                                             |
+| LookSAM       | *Towards Efficient and Scalable Sharpness-Aware Minimization*                                     | [github](https://github.com/rollovd/LookSAM)                                                                   | <https://arxiv.org/abs/2203.02714>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220302714L/exportcitation)                                                        |
+| SCION         | *Training Deep Learning Models with Norm-Constrained LMOs*                                        |                                                                                                                | <https://arxiv.org/abs/2502.07529>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250207529P/exportcitation)                                                        |
 
 ## Supported LR Scheduler
 

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytorch_optimizer"
-version = "3.4.0"
+version = "3.4.2"
 description = "optimizer & lr scheduler & objective function collections in PyTorch"
 license = "Apache-2.0"
 authors = ["kozistr <kozistr@gmail.com>"]
@@ -17,10 +17,11 @@ keywords = [
     "DAdaptLion", "DeMo", "DiffGrad", "EXAdam", "FAdam", "FOCUS", "Fromage", "FTRL", "GaLore", "Grams", "Gravity",
     "GrokFast", "GSAM", "Kate", "Lamb", "LaProp", "LARS", "Lion", "LOMO", "Lookahead", "MADGRAD", "MARS", "MSVAG",
     "Muno", "Nero", "NovoGrad", "OrthoGrad", "PAdam", "PCGrad", "PID", "PNM", "Prodigy", "PSGD", "QHAdam", "QHM",
-    "RAdam", "Ranger", "Ranger21", "RotoGrad", "SAM", "ScheduleFreeSGD", "ScheduleFreeAdamW", "ScheduleFreeRAdam",
-    "SGDP", "Shampoo", "ScalableShampoo", "SGDW", "SignSGD", "SM3", "SOAP", "SopihaH", "SPAM", "SRMM", "StableAdamW",
-    "SWATS", "TAM", "Tiger", "TRAC", "WSAM", "Yogi", "BCE", "BCEFocal", "Focal", "FocalCosine", "SoftF1", "Dice",
-    "LDAM", "Jaccard", "Bi-Tempered", "Tversky", "FocalTversky", "LovaszHinge", "bitsandbytes", "WSD", "QGaLore",
+    "RAdam", "Ranger", "Ranger21", "RotoGrad", "SAM", "GCSAM", "LookSAM", "ScheduleFreeSGD", "ScheduleFreeAdamW",
+    "ScheduleFreeRAdam", "SCION", "SGDP", "Shampoo", "ScalableShampoo", "SGDW", "SignSGD", "SM3", "SOAP", "SopihaH",
+    "SPAM", "SRMM", "StableAdamW", "SWATS", "TAM", "Tiger", "TRAC", "WSAM", "Yogi", "BCE", "BCEFocal", "Focal",
+    "FocalCosine", "SoftF1", "Dice", "LDAM", "Jaccard", "Bi-Tempered", "Tversky", "FocalTversky", "LovaszHinge",
+    "bitsandbytes", "WSD", "QGaLore",
 ]
 classifiers = [
     "License :: OSI Approved :: Apache Software License",
@@ -29,8 +30,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.6",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
@@ -51,7 +50,10 @@ numpy = [
     { version = ">1.24.4", python = ">=3.9" },
     { version = "<=1.24.4", python = "<3.9" },
 ]
-torch = { version = ">=1.10", python = ">=3.8", source = "torch" }
+torch = [
+    { version = ">=1.10,<2.5", python = "<3.9", source = "torch" },
+    { version = ">=1.10", python = ">=3.9", source = "torch" },
+]
 
 [tool.poetry.group.dev.dependencies]
 isort = [
@@ -95,7 +97,7 @@ select = [
     "TID", "ARG", "ERA", "RUF", "YTT", "PL", "Q"
 ]
 ignore = [
-    "A005", "B905",
+    "B905",
     "D100", "D102", "D104", "D105", "D107", "D203", "D213", "D413",
     "PLR0912", "PLR0913", "PLR0915", "PLR2004",
     "Q003", "ARG002",

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/__init__.py

@@ -58,6 +58,7 @@ from pytorch_optimizer.optimizer import (
     PNM,
     QHM,
     SAM,
+    SCION,
     SGDP,
     SGDW,
     SM3,
@@ -119,6 +120,7 @@ from pytorch_optimizer.optimizer import (
     LaProp,
     Lion,
     Lookahead,
+    LookSAM,
     Muon,
     Nero,
     NovoGrad,

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/base/exception.py

@@ -7,7 +7,7 @@ class NoSparseGradientError(Exception):
 
     def __init__(self, optimizer_name: str, note: str = ''):
         self.note: str = ' ' if not note else f' w/ {note} '
-        self.message: str = f'[-] {optimizer_name}{self.note}does not support sparse gradient.'
+        self.message: str = f'{optimizer_name}{self.note}does not support sparse gradient.'
         super().__init__(self.message)
 
 
@@ -15,7 +15,7 @@ class ZeroParameterSizeError(Exception):
     """Raised when the parameter size is 0."""
 
     def __init__(self):
-        self.message: str = '[-] parameter size is 0'
+        self.message: str = 'parameter size is 0'
         super().__init__(self.message)
 
 
@@ -23,7 +23,7 @@ class NoClosureError(Exception):
     """Raised when there's no closure function."""
 
     def __init__(self, optimizer_name: str, note: str = ''):
-        self.message: str = f'[-] {optimizer_name} requires closure.{note}'
+        self.message: str = f'{optimizer_name} requires closure.{note}'
         super().__init__(self.message)
 
 
@@ -32,7 +32,7 @@ class NegativeLRError(Exception):
 
     def __init__(self, lr: float, lr_type: str = ''):
         self.note: str = lr_type if lr_type else 'learning rate'
-        self.message: str = f'[-] {self.note} must be positive. ({lr} > 0)'
+        self.message: str = f'{self.note} must be positive. ({lr} > 0)'
         super().__init__(self.message)
 
 
@@ -41,5 +41,5 @@ class NegativeStepError(Exception):
 
     def __init__(self, num_steps: int, step_type: str = ''):
         self.note: str = step_type if step_type else 'step'
-        self.message: str = f'[-] {self.note} must be positive. ({num_steps} > 0)'
+        self.message: str = f'{self.note} must be positive. ({num_steps} > 0)'
         super().__init__(self.message)

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/base/optimizer.py

@@ -6,7 +6,7 @@ import torch
 from torch.optim import Optimizer
 
 from pytorch_optimizer.base.exception import NegativeLRError, NegativeStepError
-from pytorch_optimizer.base.types import (
+from pytorch_optimizer.base.type import (
     BETAS,
     CLOSURE,
     DEFAULTS,
@@ -103,7 +103,7 @@ class BaseOptimizer(ABC, Optimizer):
         :param distribution: HUTCHINSON_G. type of distribution.
         """
         if distribution not in ('gaussian', 'rademacher'):
-            raise NotImplementedError(f'[-] Hessian with distribution {distribution} is not implemented.')
+            raise NotImplementedError(f'hessian with distribution {distribution} is not implemented.')
 
         params: List[torch.Tensor] = [
             p

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/loss/dice.py

@@ -4,7 +4,7 @@ import torch
 from torch.nn.functional import logsigmoid, one_hot
 from torch.nn.modules.loss import _Loss
 
-from pytorch_optimizer.base.types import CLASS_MODE
+from pytorch_optimizer.base.type import CLASS_MODE
 
 
 def soft_dice_score(

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/loss/jaccard.py

@@ -4,7 +4,7 @@ import torch
 from torch.nn.functional import logsigmoid, one_hot
 from torch.nn.modules.loss import _Loss
 
-from pytorch_optimizer.base.types import CLASS_MODE
+from pytorch_optimizer.base.type import CLASS_MODE
 
 
 def soft_jaccard_score(

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/lr_scheduler/__init__.py

@@ -14,7 +14,7 @@ from torch.optim.lr_scheduler import (
     StepLR,
 )
 
-from pytorch_optimizer.base.types import SCHEDULER
+from pytorch_optimizer.base.type import SCHEDULER
 from pytorch_optimizer.lr_scheduler.chebyshev import get_chebyshev_perm_steps, get_chebyshev_schedule
 from pytorch_optimizer.lr_scheduler.cosine_anealing import CosineAnnealingWarmupRestarts
 from pytorch_optimizer.lr_scheduler.experimental.deberta_v3_lr_scheduler import deberta_v3_large_lr_scheduler

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/lr_scheduler/experimental/deberta_v3_lr_scheduler.py

@@ -1,6 +1,6 @@
 from torch import nn
 
-from pytorch_optimizer.base.types import PARAMETERS
+from pytorch_optimizer.base.type import PARAMETERS
 
 
 def deberta_v3_large_lr_scheduler(

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/__init__.py

@@ -1,12 +1,13 @@
 import fnmatch
 from importlib.util import find_spec
-from typing import Dict, List, Optional, Sequence, Set, Type, Union
+from typing import Dict, List, Optional, Sequence, Set, Union
+from warnings import warn
 
 import torch
 from torch import nn
-from torch.optim import AdamW, Optimizer
+from torch.optim import SGD, Adam, AdamW, Optimizer
 
-from pytorch_optimizer.base.types import OPTIMIZER, PARAMETERS
+from pytorch_optimizer.base.type import OPTIMIZER, PARAMETERS
 from pytorch_optimizer.optimizer.a2grad import A2Grad
 from pytorch_optimizer.optimizer.adabelief import AdaBelief
 from pytorch_optimizer.optimizer.adabound import AdaBound
@@ -48,7 +49,7 @@ from pytorch_optimizer.optimizer.fp16 import DynamicLossScaler, SafeFP16Optimize
 from pytorch_optimizer.optimizer.fromage import Fromage
 from pytorch_optimizer.optimizer.ftrl import FTRL
 from pytorch_optimizer.optimizer.galore import GaLore
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 from pytorch_optimizer.optimizer.grams import Grams
 from pytorch_optimizer.optimizer.gravity import Gravity
 from pytorch_optimizer.optimizer.grokfast import GrokFastAdamW
@@ -78,8 +79,9 @@ from pytorch_optimizer.optimizer.radam import RAdam
 from pytorch_optimizer.optimizer.ranger import Ranger
 from pytorch_optimizer.optimizer.ranger21 import Ranger21
 from pytorch_optimizer.optimizer.rotograd import RotoGrad
-from pytorch_optimizer.optimizer.sam import BSAM, GSAM, SAM, WSAM
+from pytorch_optimizer.optimizer.sam import BSAM, GSAM, SAM, WSAM, LookSAM
 from pytorch_optimizer.optimizer.schedulefree import ScheduleFreeAdamW, ScheduleFreeRAdam, ScheduleFreeSGD
+from pytorch_optimizer.optimizer.scion import SCION
 from pytorch_optimizer.optimizer.sgd import ASGD, SGDW, AccSGD, SGDSaI, SignSGD
 from pytorch_optimizer.optimizer.sgdp import SGDP
 from pytorch_optimizer.optimizer.shampoo import ScalableShampoo, Shampoo
@@ -206,6 +208,8 @@ def load_optimizer(optimizer: str) -> OPTIMIZER:
 
 OPTIMIZER_LIST: List[OPTIMIZER] = [
     AdamW,
+    Adam,
+    SGD,
     AdaBelief,
     AdaBound,
     PID,
@@ -297,6 +301,7 @@ OPTIMIZER_LIST: List[OPTIMIZER] = [
     SPAM,
     Kron,
     EXAdam,
+    SCION,
     Ranger25,
 ]
 OPTIMIZERS: Dict[str, OPTIMIZER] = {str(optimizer.__name__).lower(): optimizer for optimizer in OPTIMIZER_LIST}
@@ -341,6 +346,10 @@ def create_optimizer(
         optimizer = OrthoGrad(optimizer, **kwargs)
 
     if use_lookahead:
+        if optimizer_name in ('ranger', 'ranger21', 'ranger25'):
+            warn(f'{optimizer} already has a Lookahead variant.', UserWarning, 1)
+            return optimizer
+
         optimizer = Lookahead(
             optimizer,
             k=kwargs.get('k', 5),

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/a2grad.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 VARIANTS = Literal['uni', 'inc', 'exp']
 

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adabelief.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaBelief(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adabound.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaBound(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adadelta.py

@@ -2,7 +2,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaDelta(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adafactor.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaFactor(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adahessian.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, HUTCHINSON_G, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, HUTCHINSON_G, LOSS, PARAMETERS
 
 
 class AdaHessian(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adai.py

@@ -4,8 +4,8 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError, ZeroParameterSizeError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 
 
 class Adai(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adalite.py

@@ -3,7 +3,7 @@ from torch.nn.functional import softmax
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class Adalite(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adam_mini.py

@@ -7,7 +7,7 @@ from torch import nn
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS
 
 
 class AdamMini(BaseOptimizer):  # pragma: no cover

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adamax.py

@@ -2,7 +2,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaMax(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adamg.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdamG(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adamod.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaMod(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adamp.py

@@ -4,8 +4,8 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 from pytorch_optimizer.optimizer.utils import projection
 
 

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adams.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError, ZeroParameterSizeError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdamS(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adamw.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class StableAdamW(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adan.py

@@ -5,8 +5,8 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 from pytorch_optimizer.optimizer.utils import get_global_gradient_norm
 
 

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adanorm.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaNorm(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adapnm.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaPNM(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adashift.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaShift(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adasmooth.py

@@ -2,7 +2,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaSmooth(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/ademamix.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdEMAMix(BaseOptimizer):

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/adopt.py

@@ -1,11 +1,11 @@
 import math
-from typing import Callable, Optional
+from typing import Callable
 
 import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class ADOPT(BaseOptimizer):
@@ -14,6 +14,7 @@ class ADOPT(BaseOptimizer):
     :param params: PARAMETERS. iterable of parameters to optimize or dicts defining parameter groups.
     :param lr: float. learning rate.
     :param betas: BETAS. coefficients used for computing running averages of gradient and the squared hessian trace.
+    :param clip_lambda: Callable[[float], float]. function to clip gradient. default is `step ** 0.25`
     :param weight_decay: float. weight decay (L2 penalty).
     :param weight_decouple: bool. the optimizer uses decoupled weight decay as in AdamW.
     :param fixed_decay: bool. fix weight decay.
@@ -27,7 +28,7 @@ class ADOPT(BaseOptimizer):
         params: PARAMETERS,
         lr: float = 1e-3,
         betas: BETAS = (0.9, 0.9999),
-        clip_lambda: Optional[Callable[[float], float]] = lambda step: math.pow(step, 0.25),
+        clip_lambda: Callable[[float], float] = lambda step: math.pow(step, 0.25),
         weight_decay: float = 0.0,
         weight_decouple: bool = False,
         fixed_decay: bool = False,
@@ -114,8 +115,6 @@ class ADOPT(BaseOptimizer):
                     exp_avg_sq.addcmul_(grad, grad.conj())
                     continue
 
-                exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value=1.0 - beta2)
-
                 de_nom = exp_avg_sq.sqrt().clamp_(min=group['eps'])
 
                 normed_grad = grad.div(de_nom)
@@ -136,4 +135,6 @@ class ADOPT(BaseOptimizer):
 
                 p.add_(update, alpha=-lr)
 
+                exp_avg_sq.mul_(beta2).addcmul_(grad, grad.conj(), value=1.0 - beta2)
+
         return loss

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/agc.py

@@ -3,7 +3,9 @@ import torch
 from pytorch_optimizer.optimizer.utils import unit_norm
 
 
-def agc(p: torch.Tensor, grad: torch.Tensor, agc_eps: float, agc_clip_val: float, eps: float = 1e-6) -> torch.Tensor:
+def agc(
+    p: torch.Tensor, grad: torch.Tensor, agc_eps: float = 1e-3, agc_clip_val: float = 1e-2, eps: float = 1e-6
+) -> torch.Tensor:
     r"""Clip gradient values in excess of the unit wise norm.
 
     :param p: torch.Tensor. parameter.
@@ -12,11 +14,9 @@ def agc(p: torch.Tensor, grad: torch.Tensor, agc_eps: float, agc_clip_val: float
     :param agc_clip_val: float. norm clip.
     :param eps: float. simple stop from div by zero and no relation to standard optimizer eps.
     """
-    p_norm = unit_norm(p).clamp_(agc_eps)
-    g_norm = unit_norm(grad)
+    max_norm = unit_norm(p).clamp_min_(agc_eps).mul_(agc_clip_val)
+    g_norm = unit_norm(grad).clamp_min_(eps)
 
-    max_norm = p_norm * agc_clip_val
-
-    clipped_grad = grad * (max_norm / g_norm.clamp_min_(eps))
+    clipped_grad = grad * (max_norm / g_norm)
 
     return torch.where(g_norm > max_norm, clipped_grad, grad)

{pytorch_optimizer-3.4.0 → pytorch_optimizer-3.4.2}/pytorch_optimizer/optimizer/aggmo.py

@@ -2,7 +2,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AggMo(BaseOptimizer):