adv-optm 2.2.1.dev2__tar.gz → 2.2.3__tar.gz

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.2.1.dev2
+Version: 2.2.3
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu
@@ -37,6 +37,10 @@ A comprehensive, all-in-one collection of optimization algorithms for deep learn
 
 ## 🔥 What's New
 
+### in 2.2.2
+
+- `Simplified_AdEMAMix` now uses the same LR as AdamW for all `beta1` and `alpha_grad` values!
+
 ### in 2.1.x
 
 - Added Signum (SignSGD with momentum): A new optimizer in the family (SignSGD_adv)
@@ -195,14 +199,6 @@ This library integrates multiple state-of-the-art optimization techniques valida
 | `beta1` | 0.99 | Controls accumulator memory length:<br>• Small BS: **0.99–0.9999**<br>• Large BS: **0.9** |
 | `Grad α` | 100 | Most critical parameter:<br>• Inversely scales with batch size<br>• **100–10** for small BS (≤32)<br>• **1–0.1** for large BS (≥512) |
 
-> ⚠️ **Critical**: Requires **~100x smaller learning rate** than AdamW (e.g., 1e-6 vs 1e-4).
-> For `Prodigy_Adv`, set `initial_d` to:
-> - **LoRA**: `1e-8`
-> - **Full FT**: `1e-10`
-> - **Embedding**: `1e-7`
-
-> ⚠️ **Incompatible** with: **Cautious**, **Grams**, **atan2**, and standard update clipping.
-
 ---
 
 ### atan2

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/README.md

@@ -6,6 +6,10 @@ A comprehensive, all-in-one collection of optimization algorithms for deep learn
 
 ## 🔥 What's New
 
+### in 2.2.2
+
+- `Simplified_AdEMAMix` now uses the same LR as AdamW for all `beta1` and `alpha_grad` values!
+
 ### in 2.1.x
 
 - Added Signum (SignSGD with momentum): A new optimizer in the family (SignSGD_adv)
@@ -164,14 +168,6 @@ This library integrates multiple state-of-the-art optimization techniques valida
 | `beta1` | 0.99 | Controls accumulator memory length:<br>• Small BS: **0.99–0.9999**<br>• Large BS: **0.9** |
 | `Grad α` | 100 | Most critical parameter:<br>• Inversely scales with batch size<br>• **100–10** for small BS (≤32)<br>• **1–0.1** for large BS (≥512) |
 
-> ⚠️ **Critical**: Requires **~100x smaller learning rate** than AdamW (e.g., 1e-6 vs 1e-4).
-> For `Prodigy_Adv`, set `initial_d` to:
-> - **LoRA**: `1e-8`
-> - **Full FT**: `1e-10`
-> - **Embedding**: `1e-7`
-
-> ⚠️ **Incompatible** with: **Cautious**, **Grams**, **atan2**, and standard update clipping.
-
 ---
 
 ### atan2

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/adv_optm/__init__.py

@@ -22,4 +22,4 @@ __all__ = [
     "SignSGD_adv",
 ]
 
-__version__ = "2.2.1.dev2"
+__version__ = "2.2.3"

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/adv_optm/optim/Adopt_adv.py

@@ -7,7 +7,7 @@ from ..util import param_update
 from ..util.factorization_util import _get_effective_shape, _reconstruct_state, _factorize_state, _nnmf
 from ..util.OrthoGrad import _orthogonalize_gradient
 from ..util.Kourkoutas import KourkoutasHelper
-from ..util.update_util import _grams_update, _cautious_update
+from ..util.update_util import _grams_update, _cautious_update, _scale_sim_AdEMAMix_update
 
 A = 4 / math.pi
 
@@ -279,6 +279,8 @@ class Adopt_adv(torch.optim.Optimizer):
             lr = group['lr']
             step_param_fn = self._step_parameter
 
+        if self.Simplified_AdEMAMix:
+            lr = _scale_sim_AdEMAMix_update(beta1, state['step'] + 1, group["alpha_grad"], lr)
 
         step_param_fn(p, grad, state, group, lr, beta1, beta2, random_int_tensor)
 

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/adv_optm/optim/Prodigy_adv.py

@@ -9,7 +9,7 @@ from ..util import param_update
 from ..util.OrthoGrad import _orthogonalize_gradient
 from ..util.Kourkoutas import KourkoutasHelper
 from ..util.factorization_util import _get_effective_shape, _reconstruct_state, _factorize_state
-from ..util.update_util import _grams_update, _cautious_update
+from ..util.update_util import _grams_update, _cautious_update, _scale_sim_AdEMAMix_update
 
 A = 4 / math.pi
 
@@ -188,9 +188,6 @@ class Prodigy_adv(torch.optim.Optimizer):
             use_atan2 = False
         if kourkoutas_beta and not (betas[1] > beta2_min):
             raise ValueError(f"For Kourkoutas-β, betas[1] (as beta2_max) must be > beta2_min. Got {betas[1]} and {beta2_min}")
-        if Simplified_AdEMAMix and alpha_grad > 0 and not d_limiter:
-            # scales d_coef by alpha_grad, this force prodigy to behave well with Simplified_AdEMAMix.
-            d_coef = d_coef/alpha_grad
 
         defaults = {
             "lr": lr, "betas": betas, "eps": eps, "weight_decay": weight_decay, "cautious_wd": cautious_wd,
@@ -349,6 +346,9 @@ class Prodigy_adv(torch.optim.Optimizer):
             d = group['d']
             step_param_fn = self._step_parameter
 
+        if self.Simplified_AdEMAMix:
+            dlr = _scale_sim_AdEMAMix_update(self.beta1, state['step'] + 1, group["alpha_grad"], dlr)
+
         step_param_fn(p, grad, state, group, beta2, d, dlr, random_int_tensor)
 
         state['step'] += 1

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/adv_optm/optim/Simplified_AdEMAMix.py

@@ -7,6 +7,7 @@ from ..util import param_update
 from ..util.OrthoGrad import _orthogonalize_gradient
 from ..util.Kourkoutas import KourkoutasHelper
 from ..util.factorization_util import _get_effective_shape, _reconstruct_state, _factorize_state
+from ..util.update_util import _scale_sim_AdEMAMix_update
 
 # A little helper from the original simplified_AdEMAMix
 def linear_hl_warmup_scheduler(step, beta_end, beta_start=0, warmup=1):
@@ -237,6 +238,8 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
 
         lr = group["lr"]
 
+        lr = _scale_sim_AdEMAMix_update(beta1, state['step'] + 1, group["alpha_grad"], lr)
+
         random_int_tensor = None
 
         if group.get('compiled_optimizer', False):

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/adv_optm/util/update_util.py

@@ -22,3 +22,9 @@ def _cautious_update(mt: torch.Tensor, grad: torch.Tensor, inplace: bool=False):
         update_mt = mt.mul(mask)
     del mask
     return update_mt
+
+def _scale_sim_AdEMAMix_update(beta: float, current_step: int, alpha_grad: float, lr: float):
+    momentum_scale = (1 - beta ** current_step) / (1 - beta)
+    total_scale = 1 / (momentum_scale + alpha_grad)
+    lr = lr * total_scale
+    return lr

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/adv_optm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.2.1.dev2
+Version: 2.2.3
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu
@@ -37,6 +37,10 @@ A comprehensive, all-in-one collection of optimization algorithms for deep learn
 
 ## 🔥 What's New
 
+### in 2.2.2
+
+- `Simplified_AdEMAMix` now uses the same LR as AdamW for all `beta1` and `alpha_grad` values!
+
 ### in 2.1.x
 
 - Added Signum (SignSGD with momentum): A new optimizer in the family (SignSGD_adv)
@@ -195,14 +199,6 @@ This library integrates multiple state-of-the-art optimization techniques valida
 | `beta1` | 0.99 | Controls accumulator memory length:<br>• Small BS: **0.99–0.9999**<br>• Large BS: **0.9** |
 | `Grad α` | 100 | Most critical parameter:<br>• Inversely scales with batch size<br>• **100–10** for small BS (≤32)<br>• **1–0.1** for large BS (≥512) |
 
-> ⚠️ **Critical**: Requires **~100x smaller learning rate** than AdamW (e.g., 1e-6 vs 1e-4).
-> For `Prodigy_Adv`, set `initial_d` to:
-> - **LoRA**: `1e-8`
-> - **Full FT**: `1e-10`
-> - **Embedding**: `1e-7`
-
-> ⚠️ **Incompatible** with: **Cautious**, **Grams**, **atan2**, and standard update clipping.
-
 ---
 
 ### atan2

{adv_optm-2.2.1.dev2 → adv_optm-2.2.3}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="adv_optm",
-    version="2.2.1.dev2",
+    version="2.2.3",
     author="Koratahiu",
     author_email="hiuhonor@gmail.com",
     license='Apache 2.0',