pytorch-optimizer 3.3.4__tar.gz → 3.4.1__tar.gz

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/PKG-INFO

@@ -1,9 +1,9 @@
 Metadata-Version: 2.3
 Name: pytorch_optimizer
-Version: 3.3.4
+Version: 3.4.1
 Summary: optimizer & lr scheduler & objective function collections in PyTorch
 License: Apache-2.0
-Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaMax,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,CAME,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,FAdam,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,QHAdam,QHM,RAdam,Ranger,Ranger21,RotoGrad,SAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
+Keywords: pytorch,deep-learning,optimizer,lr scheduler,A2Grad,ASGD,AccSGD,AdaBelief,AdaBound,AdaDelta,AdaFactor,AdaMax,AdamG,AdaMod,AdaNorm,AdaPNM,AdaSmooth,AdEMAMix,ADOPT,AdaHessian,Adai,Adalite,AdaLomo,AdamMini,AdamP,AdamS,Adan,AggMo,Aida,AliG,Amos,Apollo,APOLLO,AvaGrad,bSAM,CAME,DAdaptAdaGrad,DAdaptAdam,DAdaptAdan,DAdaptSGD,DAdaptLion,DeMo,DiffGrad,EXAdam,FAdam,FOCUS,Fromage,FTRL,GaLore,Grams,Gravity,GrokFast,GSAM,Kate,Lamb,LaProp,LARS,Lion,LOMO,Lookahead,MADGRAD,MARS,MSVAG,Muno,Nero,NovoGrad,OrthoGrad,PAdam,PCGrad,PID,PNM,Prodigy,PSGD,QHAdam,QHM,RAdam,Ranger,Ranger21,RotoGrad,SAM,GCSAM,LookSAM,ScheduleFreeSGD,ScheduleFreeAdamW,ScheduleFreeRAdam,SGDP,Shampoo,ScalableShampoo,SGDW,SignSGD,SM3,SOAP,SopihaH,SPAM,SRMM,StableAdamW,SWATS,TAM,Tiger,TRAC,WSAM,Yogi,BCE,BCEFocal,Focal,FocalCosine,SoftF1,Dice,LDAM,Jaccard,Bi-Tempered,Tversky,FocalTversky,LovaszHinge,bitsandbytes,WSD,QGaLore
 Author: kozistr
 Author-email: kozistr@gmail.com
 Maintainer: kozistr
@@ -22,18 +22,15 @@ Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
-Classifier: Programming Language :: Python :: 3.6
-Classifier: Programming Language :: Python :: 3.7
 Classifier: Topic :: Scientific/Engineering
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
 Classifier: Topic :: Software Development
 Classifier: Topic :: Software Development :: Libraries
 Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Provides-Extra: bitsandbytes
-Requires-Dist: bitsandbytes (>=0.44,<0.45) ; extra == "bitsandbytes"
 Requires-Dist: numpy (<=1.24.4) ; python_version < "3.9"
 Requires-Dist: numpy (>1.24.4) ; python_version >= "3.9"
-Requires-Dist: torch (>=1.10) ; python_version >= "3.8"
+Requires-Dist: torch (>=1.10) ; python_version >= "3.9"
+Requires-Dist: torch (>=1.10,<2.5) ; python_version < "3.9"
 Project-URL: Documentation, https://pytorch-optimizers.readthedocs.io/en/latest
 Project-URL: Homepage, https://github.com/kozistr/pytorch_optimizer
 Project-URL: Repository, https://github.com/kozistr/pytorch_optimizer
@@ -51,7 +48,7 @@ Description-Content-Type: text/markdown
 
 ## The reasons why you use `pytorch-optimizer`.
 
-* Wide range of supported optimizers. Currently, **92 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
+* Wide range of supported optimizers. Currently, **98 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
 * Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
 * Easy to use, clean, and tested codes
 * Active maintenance
@@ -120,6 +117,7 @@ optimizer = create_optimizer(
     weight_decay=1e-3,
     use_gc=True,
     use_lookahead=True,
+    use_orthograd=False,
 )
 ```
 
@@ -241,6 +239,11 @@ get_supported_optimizers(['adam*', 'ranger*'])
 | Adam-ATAN2    | *Scaling Exponents Across Parameterizations and Optimizers*                                       |                                                                                                                | <https://arxiv.org/abs/2407.05872>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240705872E/exportcitation)                                                        |
 | SPAM          | *Spike-Aware Adam with Momentum Reset for Stable LLM Training*                                    | [github](https://github.com/TianjinYellow/SPAM-Optimizer)                                                      | <https://arxiv.org/abs/2501.06842>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250106842H/exportcitation)                                                        |
 | TAM           | *Torque-Aware Momentum*                                                                           |                                                                                                                | <https://arxiv.org/abs/2412.18790>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv241218790M/exportcitation)                                                        |
+| FOCUS         | *First Order Concentrated Updating Scheme*                                                        | [github](https://github.com/liuyz0/FOCUS)                                                                      | <https://arxiv.org/abs/2501.12243>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250112243M/exportcitation)                                                        |
+| PSGD          | *Preconditioned Stochastic Gradient Descent*                                                      | [github](https://github.com/lixilinx/psgd_torch)                                                               | <https://arxiv.org/abs/1512.04202>                                                          | [cite](https://github.com/lixilinx/psgd_torch?tab=readme-ov-file#resources)                                                         |
+| EXAdam        | *The Power of Adaptive Cross-Moments*                                                             | [github](https://github.com/AhmedMostafa16/EXAdam)                                                             | <https://arxiv.org/abs/2412.20302>                                                          | [cite](https://github.com/AhmedMostafa16/EXAdam?tab=readme-ov-file#citation)                                                        |
+| GCSAM         | *Gradient Centralized Sharpness Aware Minimization*                                               | [github](https://github.com/mhassann22/GCSAM)                                                                  | <https://arxiv.org/abs/2501.11584>                                                          | [cite](https://github.com/mhassann22/GCSAM?tab=readme-ov-file#citation)                                                             |
+| LookSAM       | *Towards Efficient and Scalable Sharpness-Aware Minimization*                                     | [github](https://github.com/rollovd/LookSAM)                                                                   | <https://arxiv.org/abs/2203.02714>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220302714L/exportcitation)                                                        |
 
 ## Supported LR Scheduler
 

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/README.md

@@ -10,7 +10,7 @@
 
 ## The reasons why you use `pytorch-optimizer`.
 
-* Wide range of supported optimizers. Currently, **92 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
+* Wide range of supported optimizers. Currently, **98 optimizers (+ `bitsandbytes`, `qgalore`, `torchao`)**, **16 lr schedulers**, and **13 loss functions** are supported!
 * Including many variants such as `ADOPT`, `Cautious`, `AdamD`, `StableAdamW`, and `Gradient Centrailiaztion`
 * Easy to use, clean, and tested codes
 * Active maintenance
@@ -79,6 +79,7 @@ optimizer = create_optimizer(
     weight_decay=1e-3,
     use_gc=True,
     use_lookahead=True,
+    use_orthograd=False,
 )
 ```
 
@@ -200,6 +201,11 @@ get_supported_optimizers(['adam*', 'ranger*'])
 | Adam-ATAN2    | *Scaling Exponents Across Parameterizations and Optimizers*                                       |                                                                                                                | <https://arxiv.org/abs/2407.05872>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv240705872E/exportcitation)                                                        |
 | SPAM          | *Spike-Aware Adam with Momentum Reset for Stable LLM Training*                                    | [github](https://github.com/TianjinYellow/SPAM-Optimizer)                                                      | <https://arxiv.org/abs/2501.06842>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250106842H/exportcitation)                                                        |
 | TAM           | *Torque-Aware Momentum*                                                                           |                                                                                                                | <https://arxiv.org/abs/2412.18790>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2024arXiv241218790M/exportcitation)                                                        |
+| FOCUS         | *First Order Concentrated Updating Scheme*                                                        | [github](https://github.com/liuyz0/FOCUS)                                                                      | <https://arxiv.org/abs/2501.12243>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2025arXiv250112243M/exportcitation)                                                        |
+| PSGD          | *Preconditioned Stochastic Gradient Descent*                                                      | [github](https://github.com/lixilinx/psgd_torch)                                                               | <https://arxiv.org/abs/1512.04202>                                                          | [cite](https://github.com/lixilinx/psgd_torch?tab=readme-ov-file#resources)                                                         |
+| EXAdam        | *The Power of Adaptive Cross-Moments*                                                             | [github](https://github.com/AhmedMostafa16/EXAdam)                                                             | <https://arxiv.org/abs/2412.20302>                                                          | [cite](https://github.com/AhmedMostafa16/EXAdam?tab=readme-ov-file#citation)                                                        |
+| GCSAM         | *Gradient Centralized Sharpness Aware Minimization*                                               | [github](https://github.com/mhassann22/GCSAM)                                                                  | <https://arxiv.org/abs/2501.11584>                                                          | [cite](https://github.com/mhassann22/GCSAM?tab=readme-ov-file#citation)                                                             |
+| LookSAM       | *Towards Efficient and Scalable Sharpness-Aware Minimization*                                     | [github](https://github.com/rollovd/LookSAM)                                                                   | <https://arxiv.org/abs/2203.02714>                                                          | [cite](https://ui.adsabs.harvard.edu/abs/2022arXiv220302714L/exportcitation)                                                        |
 
 ## Supported LR Scheduler
 

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "pytorch_optimizer"
-version = "3.3.4"
+version = "3.4.1"
 description = "optimizer & lr scheduler & objective function collections in PyTorch"
 license = "Apache-2.0"
 authors = ["kozistr <kozistr@gmail.com>"]
@@ -14,13 +14,14 @@ keywords = [
     "AdaDelta", "AdaFactor", "AdaMax", "AdamG", "AdaMod", "AdaNorm", "AdaPNM", "AdaSmooth", "AdEMAMix", "ADOPT",
     "AdaHessian", "Adai", "Adalite", "AdaLomo", "AdamMini", "AdamP", "AdamS", "Adan", "AggMo", "Aida", "AliG", "Amos",
     "Apollo", "APOLLO", "AvaGrad", "bSAM", "CAME", "DAdaptAdaGrad", "DAdaptAdam", "DAdaptAdan", "DAdaptSGD",
-    "DAdaptLion", "DeMo", "DiffGrad", "FAdam", "Fromage", "FTRL", "GaLore", "Grams", "Gravity", "GrokFast", "GSAM",
-    "Kate", "Lamb", "LaProp", "LARS", "Lion", "LOMO", "Lookahead", "MADGRAD", "MARS", "MSVAG", "Muno", "Nero",
-    "NovoGrad", "OrthoGrad", "PAdam", "PCGrad", "PID", "PNM", "Prodigy", "QHAdam", "QHM", "RAdam", "Ranger",
-    "Ranger21", "RotoGrad", "SAM", "ScheduleFreeSGD", "ScheduleFreeAdamW", "ScheduleFreeRAdam", "SGDP", "Shampoo",
-    "ScalableShampoo", "SGDW", "SignSGD", "SM3", "SOAP", "SopihaH", "SPAM", "SRMM", "StableAdamW", "SWATS", "TAM",
-    "Tiger", "TRAC", "WSAM", "Yogi", "BCE", "BCEFocal", "Focal", "FocalCosine", "SoftF1", "Dice", "LDAM", "Jaccard",
-    "Bi-Tempered", "Tversky", "FocalTversky", "LovaszHinge", "bitsandbytes", "WSD", "QGaLore",
+    "DAdaptLion", "DeMo", "DiffGrad", "EXAdam", "FAdam", "FOCUS", "Fromage", "FTRL", "GaLore", "Grams", "Gravity",
+    "GrokFast", "GSAM", "Kate", "Lamb", "LaProp", "LARS", "Lion", "LOMO", "Lookahead", "MADGRAD", "MARS", "MSVAG",
+    "Muno", "Nero", "NovoGrad", "OrthoGrad", "PAdam", "PCGrad", "PID", "PNM", "Prodigy", "PSGD", "QHAdam", "QHM",
+    "RAdam", "Ranger", "Ranger21", "RotoGrad", "SAM", "GCSAM", "LookSAM", "ScheduleFreeSGD", "ScheduleFreeAdamW",
+    "ScheduleFreeRAdam", "SGDP", "Shampoo", "ScalableShampoo", "SGDW", "SignSGD", "SM3", "SOAP", "SopihaH", "SPAM",
+    "SRMM", "StableAdamW", "SWATS", "TAM", "Tiger", "TRAC", "WSAM", "Yogi", "BCE", "BCEFocal", "Focal", "FocalCosine",
+    "SoftF1", "Dice", "LDAM", "Jaccard", "Bi-Tempered", "Tversky", "FocalTversky", "LovaszHinge", "bitsandbytes", "WSD",
+    "QGaLore",
 ]
 classifiers = [
     "License :: OSI Approved :: Apache Software License",
@@ -29,8 +30,6 @@ classifiers = [
     "Intended Audience :: Developers",
     "Intended Audience :: Science/Research",
     "Programming Language :: Python :: 3",
-    "Programming Language :: Python :: 3.6",
-    "Programming Language :: Python :: 3.7",
     "Programming Language :: Python :: 3.8",
     "Programming Language :: Python :: 3.9",
     "Programming Language :: Python :: 3.10",
@@ -51,30 +50,31 @@ numpy = [
     { version = ">1.24.4", python = ">=3.9" },
     { version = "<=1.24.4", python = "<3.9" },
 ]
-torch = { version = ">=1.10", python = ">=3.8", source = "torch" }
-bitsandbytes = { version = "^0.44", optional = true }
+torch = [
+    { version = ">=1.10,<2.5", python = "<3.9", source = "torch" },
+    { version = ">=1.10", python = ">=3.9", source = "torch" },
+]
 
-[tool.poetry.dev-dependencies]
-isort = { version = "^5", python = ">=3.8" }
-black = { version = "^24", python = ">=3.8" }
+[tool.poetry.group.dev.dependencies]
+isort = [
+    { version = "<6", python = "<3.9" },
+    { version = "^6", python = ">=3.9" },
+]
+black = [
+    { version = "<25", python = "<3.9" },
+    { version = "^25", python = ">=3.9" },
+]
 ruff = "*"
 pytest = "*"
 pytest-cov = "*"
 
-[tool.poetry.extras]
-bitsandbytes = ["bitsandbytes"]
-
 [[tool.poetry.source]]
 name = "torch"
 url = "https://download.pytorch.org/whl/cpu"
 priority = "explicit"
 
 [tool.ruff]
-src = [
-    "pytorch_optimizer",
-    "tests",
-    "examples",
-]
+src = ["pytorch_optimizer", "tests", "examples"]
 target-version = "py312"
 line-length = 119
 exclude = [
@@ -97,8 +97,10 @@ select = [
     "TID", "ARG", "ERA", "RUF", "YTT", "PL", "Q"
 ]
 ignore = [
-    "A005", "B905", "D100", "D102", "D104", "D105", "D107", "D203", "D213", "D413", "PIE790", "PLR0912", "PLR0913",
-    "PLR0915", "PLR2004", "RUF013", "Q003", "ARG002",
+    "B905",
+    "D100", "D102", "D104", "D105", "D107", "D203", "D213", "D413",
+    "PLR0912", "PLR0913", "PLR0915", "PLR2004",
+    "Q003", "ARG002",
 ]
 fixable = ["ALL"]
 unfixable = ["F401"]
@@ -107,10 +109,10 @@ flake8-quotes.docstring-quotes = "double"
 flake8-quotes.inline-quotes = "single"
 
 [tool.ruff.lint.extend-per-file-ignores]
-"hubconf.py" = ["D", "INP001"]
 "examples/visualize_optimizers.py" = ["D103", "D400", "D415"]
-"**/__init__.py" = ["F401"]
 "{tests,examples}/*.py" = ["D", "S101"]
+"**/__init__.py" = ["F401"]
+"hubconf.py" = ["D", "INP001"]
 
 [tool.ruff.lint.isort]
 combine-as-imports = false
@@ -129,9 +131,9 @@ testpaths = "tests"
 
 [tool.coverage.run]
 omit = [
-    "./pytorch_optimizer/optimizer/rotograd.py",
     "./pytorch_optimizer/optimizer/adam_mini.py",
     "./pytorch_optimizer/optimizer/demo.py",
+    "./pytorch_optimizer/optimizer/rotograd.py",
 ]
 
 [build-system]

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/__init__.py

@@ -46,6 +46,7 @@ from pytorch_optimizer.optimizer import (
     ASGD,
     BSAM,
     CAME,
+    FOCUS,
     FTRL,
     GSAM,
     LARS,
@@ -105,6 +106,7 @@ from pytorch_optimizer.optimizer import (
     DeMo,
     DiffGrad,
     DynamicLossScaler,
+    EXAdam,
     FAdam,
     Fromage,
     GaLore,
@@ -112,10 +114,12 @@ from pytorch_optimizer.optimizer import (
     Gravity,
     GrokFastAdamW,
     Kate,
+    Kron,
     Lamb,
     LaProp,
     Lion,
     Lookahead,
+    LookSAM,
     Muon,
     Nero,
     NovoGrad,

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/base/exception.py

@@ -7,7 +7,7 @@ class NoSparseGradientError(Exception):
 
     def __init__(self, optimizer_name: str, note: str = ''):
         self.note: str = ' ' if not note else f' w/ {note} '
-        self.message: str = f'[-] {optimizer_name}{self.note}does not support sparse gradient.'
+        self.message: str = f'{optimizer_name}{self.note}does not support sparse gradient.'
         super().__init__(self.message)
 
 
@@ -15,7 +15,7 @@ class ZeroParameterSizeError(Exception):
     """Raised when the parameter size is 0."""
 
     def __init__(self):
-        self.message: str = '[-] parameter size is 0'
+        self.message: str = 'parameter size is 0'
         super().__init__(self.message)
 
 
@@ -23,7 +23,7 @@ class NoClosureError(Exception):
     """Raised when there's no closure function."""
 
     def __init__(self, optimizer_name: str, note: str = ''):
-        self.message: str = f'[-] {optimizer_name} requires closure.{note}'
+        self.message: str = f'{optimizer_name} requires closure.{note}'
         super().__init__(self.message)
 
 
@@ -32,7 +32,7 @@ class NegativeLRError(Exception):
 
     def __init__(self, lr: float, lr_type: str = ''):
         self.note: str = lr_type if lr_type else 'learning rate'
-        self.message: str = f'[-] {self.note} must be positive. ({lr} > 0)'
+        self.message: str = f'{self.note} must be positive. ({lr} > 0)'
         super().__init__(self.message)
 
 
@@ -41,5 +41,5 @@ class NegativeStepError(Exception):
 
     def __init__(self, num_steps: int, step_type: str = ''):
         self.note: str = step_type if step_type else 'step'
-        self.message: str = f'[-] {self.note} must be positive. ({num_steps} > 0)'
+        self.message: str = f'{self.note} must be positive. ({num_steps} > 0)'
         super().__init__(self.message)

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/base/optimizer.py

@@ -6,7 +6,16 @@ import torch
 from torch.optim import Optimizer
 
 from pytorch_optimizer.base.exception import NegativeLRError, NegativeStepError
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, HUTCHINSON_G, LOSS, PARAMETERS, STATE
+from pytorch_optimizer.base.type import (
+    BETAS,
+    CLOSURE,
+    DEFAULTS,
+    HUTCHINSON_G,
+    LOSS,
+    OPTIMIZER_INSTANCE_OR_CLASS,
+    PARAMETERS,
+    STATE,
+)
 
 
 class BaseOptimizer(ABC, Optimizer):
@@ -15,6 +24,18 @@ class BaseOptimizer(ABC, Optimizer):
     def __init__(self, params: PARAMETERS, defaults: DEFAULTS) -> None:
         super().__init__(params, defaults)
 
+    @staticmethod
+    def load_optimizer(optimizer: OPTIMIZER_INSTANCE_OR_CLASS, **kwargs) -> Optimizer:
+        r"""Build torch.optim.Optimizer class."""
+        if isinstance(optimizer, Optimizer):
+            return optimizer
+
+        if 'params' in kwargs:
+            params = kwargs.pop('params')
+            return optimizer(params, **kwargs)
+
+        raise ValueError('need to pass `params` when you pass the `torch.optim.Optimizer` instance.')
+
     @staticmethod
     @torch.no_grad()
     def set_hessian(param_groups: PARAMETERS, state: STATE, hessian: List[torch.Tensor]) -> None:
@@ -82,7 +103,7 @@ class BaseOptimizer(ABC, Optimizer):
         :param distribution: HUTCHINSON_G. type of distribution.
         """
         if distribution not in ('gaussian', 'rademacher'):
-            raise NotImplementedError(f'[-] Hessian with distribution {distribution} is not implemented.')
+            raise NotImplementedError(f'hessian with distribution {distribution} is not implemented.')
 
         params: List[torch.Tensor] = [
             p
@@ -317,7 +338,7 @@ class BaseOptimizer(ABC, Optimizer):
     @staticmethod
     def validate_options(x: str, name: str, options: List[str]) -> None:
         if x not in options:
-            opts: str = ' or '.join([f'\'{option}\'' for option in options]).strip()
+            opts: str = ' or '.join([f"'{option}'" for option in options]).strip()
             raise ValueError(f'[-] {name} {x} must be one of ({opts})')
 
     @staticmethod

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/loss/dice.py

@@ -4,7 +4,7 @@ import torch
 from torch.nn.functional import logsigmoid, one_hot
 from torch.nn.modules.loss import _Loss
 
-from pytorch_optimizer.base.types import CLASS_MODE
+from pytorch_optimizer.base.type import CLASS_MODE
 
 
 def soft_dice_score(

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/loss/jaccard.py

@@ -4,7 +4,7 @@ import torch
 from torch.nn.functional import logsigmoid, one_hot
 from torch.nn.modules.loss import _Loss
 
-from pytorch_optimizer.base.types import CLASS_MODE
+from pytorch_optimizer.base.type import CLASS_MODE
 
 
 def soft_jaccard_score(
@@ -49,7 +49,7 @@ class JaccardLoss(_Loss):
     def __init__(
         self,
         mode: CLASS_MODE,
-        classes: List[int] = None,
+        classes: Optional[List[int]] = None,
         log_loss: bool = False,
         from_logits: bool = True,
         label_smooth: float = 0.0,
@@ -59,7 +59,7 @@ class JaccardLoss(_Loss):
 
         if classes is not None:
             if mode == 'binary':
-                raise ValueError('[-] Masking classes is not supported with mode=binary')
+                raise ValueError('masking classes is not supported with mode=binary')
 
             classes = torch.LongTensor(classes)
 

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/lr_scheduler/__init__.py

@@ -14,7 +14,7 @@ from torch.optim.lr_scheduler import (
     StepLR,
 )
 
-from pytorch_optimizer.base.types import SCHEDULER
+from pytorch_optimizer.base.type import SCHEDULER
 from pytorch_optimizer.lr_scheduler.chebyshev import get_chebyshev_perm_steps, get_chebyshev_schedule
 from pytorch_optimizer.lr_scheduler.cosine_anealing import CosineAnnealingWarmupRestarts
 from pytorch_optimizer.lr_scheduler.experimental.deberta_v3_lr_scheduler import deberta_v3_large_lr_scheduler

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/lr_scheduler/experimental/deberta_v3_lr_scheduler.py

@@ -1,6 +1,6 @@
 from torch import nn
 
-from pytorch_optimizer.base.types import PARAMETERS
+from pytorch_optimizer.base.type import PARAMETERS
 
 
 def deberta_v3_large_lr_scheduler(

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/__init__.py

@@ -1,12 +1,13 @@
 import fnmatch
 from importlib.util import find_spec
-from typing import Dict, List, Optional, Sequence, Set, Type, Union
+from typing import Dict, List, Optional, Sequence, Set, Union
+from warnings import warn
 
 import torch
 from torch import nn
-from torch.optim import AdamW, Optimizer
+from torch.optim import SGD, Adam, AdamW, Optimizer
 
-from pytorch_optimizer.base.types import OPTIMIZER, PARAMETERS
+from pytorch_optimizer.base.type import OPTIMIZER, PARAMETERS
 from pytorch_optimizer.optimizer.a2grad import A2Grad
 from pytorch_optimizer.optimizer.adabelief import AdaBelief
 from pytorch_optimizer.optimizer.adabound import AdaBound
@@ -40,13 +41,15 @@ from pytorch_optimizer.optimizer.came import CAME
 from pytorch_optimizer.optimizer.dadapt import DAdaptAdaGrad, DAdaptAdam, DAdaptAdan, DAdaptLion, DAdaptSGD
 from pytorch_optimizer.optimizer.demo import DeMo
 from pytorch_optimizer.optimizer.diffgrad import DiffGrad
+from pytorch_optimizer.optimizer.exadam import EXAdam
 from pytorch_optimizer.optimizer.experimental.ranger25 import Ranger25
 from pytorch_optimizer.optimizer.fadam import FAdam
+from pytorch_optimizer.optimizer.focus import FOCUS
 from pytorch_optimizer.optimizer.fp16 import DynamicLossScaler, SafeFP16Optimizer
 from pytorch_optimizer.optimizer.fromage import Fromage
 from pytorch_optimizer.optimizer.ftrl import FTRL
 from pytorch_optimizer.optimizer.galore import GaLore
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 from pytorch_optimizer.optimizer.grams import Grams
 from pytorch_optimizer.optimizer.gravity import Gravity
 from pytorch_optimizer.optimizer.grokfast import GrokFastAdamW
@@ -69,13 +72,14 @@ from pytorch_optimizer.optimizer.pcgrad import PCGrad
 from pytorch_optimizer.optimizer.pid import PID
 from pytorch_optimizer.optimizer.pnm import PNM
 from pytorch_optimizer.optimizer.prodigy import Prodigy
+from pytorch_optimizer.optimizer.psgd import Kron
 from pytorch_optimizer.optimizer.qhadam import QHAdam
 from pytorch_optimizer.optimizer.qhm import QHM
 from pytorch_optimizer.optimizer.radam import RAdam
 from pytorch_optimizer.optimizer.ranger import Ranger
 from pytorch_optimizer.optimizer.ranger21 import Ranger21
 from pytorch_optimizer.optimizer.rotograd import RotoGrad
-from pytorch_optimizer.optimizer.sam import BSAM, GSAM, SAM, WSAM
+from pytorch_optimizer.optimizer.sam import BSAM, GSAM, SAM, WSAM, LookSAM
 from pytorch_optimizer.optimizer.schedulefree import ScheduleFreeAdamW, ScheduleFreeRAdam, ScheduleFreeSGD
 from pytorch_optimizer.optimizer.sgd import ASGD, SGDW, AccSGD, SGDSaI, SignSGD
 from pytorch_optimizer.optimizer.sgdp import SGDP
@@ -203,6 +207,8 @@ def load_optimizer(optimizer: str) -> OPTIMIZER:
 
 OPTIMIZER_LIST: List[OPTIMIZER] = [
     AdamW,
+    Adam,
+    SGD,
     AdaBelief,
     AdaBound,
     PID,
@@ -289,8 +295,11 @@ OPTIMIZER_LIST: List[OPTIMIZER] = [
     LaProp,
     MARS,
     SGDSaI,
+    FOCUS,
     Grams,
     SPAM,
+    Kron,
+    EXAdam,
     Ranger25,
 ]
 OPTIMIZERS: Dict[str, OPTIMIZER] = {str(optimizer.__name__).lower(): optimizer for optimizer in OPTIMIZER_LIST}
@@ -335,6 +344,10 @@ def create_optimizer(
         optimizer = OrthoGrad(optimizer, **kwargs)
 
     if use_lookahead:
+        if optimizer_name in ('ranger', 'ranger21', 'ranger25'):
+            warn(f'{optimizer} already has a Lookahead variant.', UserWarning, 1)
+            return optimizer
+
         optimizer = Lookahead(
             optimizer,
             k=kwargs.get('k', 5),

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/a2grad.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 VARIANTS = Literal['uni', 'inc', 'exp']
 

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adabelief.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaBelief(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adabound.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaBound(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adadelta.py

@@ -2,7 +2,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaDelta(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adafactor.py

@@ -5,7 +5,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaFactor(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adahessian.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, HUTCHINSON_G, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, HUTCHINSON_G, LOSS, PARAMETERS
 
 
 class AdaHessian(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adai.py

@@ -4,8 +4,8 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError, ZeroParameterSizeError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 
 
 class Adai(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adalite.py

@@ -3,7 +3,7 @@ from torch.nn.functional import softmax
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class Adalite(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adam_mini.py

@@ -7,7 +7,7 @@ from torch import nn
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS
 
 
 class AdamMini(BaseOptimizer):  # pragma: no cover

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adamax.py

@@ -2,7 +2,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaMax(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adamg.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdamG(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adamod.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdaMod(BaseOptimizer):

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adamp.py

@@ -4,8 +4,8 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
-from pytorch_optimizer.optimizer.gc import centralize_gradient
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.optimizer.gradient_centralization import centralize_gradient
 from pytorch_optimizer.optimizer.utils import projection
 
 

{pytorch_optimizer-3.3.4 → pytorch_optimizer-3.4.1}/pytorch_optimizer/optimizer/adams.py

@@ -4,7 +4,7 @@ import torch
 
 from pytorch_optimizer.base.exception import NoSparseGradientError, ZeroParameterSizeError
 from pytorch_optimizer.base.optimizer import BaseOptimizer
-from pytorch_optimizer.base.types import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
+from pytorch_optimizer.base.type import BETAS, CLOSURE, DEFAULTS, LOSS, PARAMETERS
 
 
 class AdamS(BaseOptimizer):