adv-optm 2.4.dev5__tar.gz → 2.4.dev6__tar.gz

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.4.dev5
+Version: 2.4.dev6
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/adv_optm/__init__.py

@@ -22,4 +22,4 @@ __all__ = [
     "SignSGD_adv",
 ]
 
-__version__ = "2.4.dev5"
+__version__ = "2.4.dev6"

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/adv_optm/optim/AdamW_adv.py

@@ -9,7 +9,7 @@ from ..util.factorization_util import _get_effective_shape, _reconstruct_state,
 from ..util.update_util import _grams_update, _cautious_update, _init_fisher_wd_scaler, _get_fisher_wd_scaler
 from ..util.OrthoGrad import _orthogonalize_gradient
 from ..util.Kourkoutas import KourkoutasHelper
-from ..util.scaled_optm import scale_update, is_spectral, init_spectral_norm
+from ..util.scaled_optm import scale_update, is_spectral, init_spectral_norm, scale_eps
 from ..util.centered_decay import _init_anchor
 
 A = 4 / math.pi
@@ -153,6 +153,9 @@ class AdamW_adv(torch.optim.Optimizer):
             raise ValueError(f"Weight-decay should be >= 0.0. Got {weight_decay}")
         if kourkoutas_beta and not (betas[1] > beta2_min):
             raise ValueError(f"For Kourkoutas-β, betas[1] (as beta2_max) must be > beta2_min. Got {betas[1]} and {beta2_min}")
+        if scaled_optm and use_atan2:
+            print("Warning: use_atan2 is incompatible with scaled_optm, Disabling atan2.")
+            use_atan2 = False
 
         if cautious_mask and grams_moment:
             print("Warning: cautious is incompatible with grams, Disabling cautious.")
@@ -330,6 +333,8 @@ class AdamW_adv(torch.optim.Optimizer):
         # Determine if we are using dense first-moments alongside a factored second-order second-moment
         factored_2nd = group.get('factored_2nd', False)
 
+        adaptive_eps = scale_eps(group, p)
+
         if state['factored']:
             d1, d2 = state['effective_shape']
             grad_reshaped = grad.view(d1, d2)
@@ -394,7 +399,7 @@ class AdamW_adv(torch.optim.Optimizer):
                 update.atan2_(denom)
             else:
                 denom = vt.sqrt_()
-                denom.div_(sqrt_bias_correction2).add_(group['eps'])
+                denom.div_(sqrt_bias_correction2).add_(adaptive_eps)
                 update.div_(denom)
 
             wd_scaler = _get_fisher_wd_scaler(group, state.get("wd_scaler"), p, denom, group['use_atan2'])
@@ -438,7 +443,7 @@ class AdamW_adv(torch.optim.Optimizer):
                 update.atan2_(denom)
             else:
                 denom = exp_avg_sq.sqrt()
-                denom.div_(sqrt_bias_correction2).add_(group['eps'])
+                denom.div_(sqrt_bias_correction2).add_(adaptive_eps)
                 update.div_(denom)
 
             wd_scaler = _get_fisher_wd_scaler(group, state.get("wd_scaler"), p, denom, group['use_atan2'])

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/adv_optm/optim/Adopt_adv.py

@@ -8,7 +8,7 @@ from ..util.factorization_util import _get_effective_shape, _reconstruct_state,
 from ..util.OrthoGrad import _orthogonalize_gradient
 from ..util.Kourkoutas import KourkoutasHelper
 from ..util.update_util import _grams_update, _cautious_update, _scale_sim_AdEMAMix_update, _init_fisher_wd_scaler, _get_fisher_wd_scaler
-from ..util.scaled_optm import scale_update, is_spectral, init_spectral_norm
+from ..util.scaled_optm import scale_update, is_spectral, init_spectral_norm, scale_eps
 from ..util.centered_decay import _init_anchor
 
 A = 4 / math.pi
@@ -183,6 +183,9 @@ class Adopt_adv(torch.optim.Optimizer):
             print("Warning: grams is incompatible with Simplified_AdEMAMix, Disabling grams.")
         if cautious_mask and Simplified_AdEMAMix:
             print("Warning: cautious is incompatible with Simplified_AdEMAMix, Disabling cautious.")
+        if scaled_optm and use_atan2:
+            print("Warning: use_atan2 is incompatible with scaled_optm, Disabling atan2.")
+            use_atan2 = False
 
         defaults = {
             "lr": lr, "betas": betas, "eps": eps, "weight_decay": weight_decay,
@@ -364,6 +367,8 @@ class Adopt_adv(torch.optim.Optimizer):
         # Determine if we are using dense first-moments alongside a factored second-order second-moment
         factored_2nd = group.get('factored_2nd', False)
 
+        adaptive_eps = scale_eps(group, p)
+
         if state['factored']:
             d1, d2 = state['effective_shape']
             grad_reshaped = grad.view(d1, d2)
@@ -387,7 +392,7 @@ class Adopt_adv(torch.optim.Optimizer):
             if self.use_atan2:
                 normalized_grad = torch.atan2(grad_reshaped, denom, out=denom)
             else:
-                normalized_grad = torch.div(grad_reshaped, denom.add_(group['eps']), out=denom)
+                normalized_grad = torch.div(grad_reshaped, denom.add_(adaptive_eps), out=denom)
                 if self.clip_lambda is not None:
                     clip_val = self.clip_lambda(state['step'])
                     normalized_grad.clamp_(-clip_val, clip_val)
@@ -457,7 +462,7 @@ class Adopt_adv(torch.optim.Optimizer):
             if self.use_atan2:
                 normalized_grad = torch.atan2(grad, denom, out=denom)
             else:
-                normalized_grad = torch.div(grad, denom.add_(group['eps']), out=denom)
+                normalized_grad = torch.div(grad, denom.add_(adaptive_eps), out=denom)
                 if self.clip_lambda is not None:
                     clip_val = self.clip_lambda(state['step'])
                     normalized_grad.clamp_(-clip_val, clip_val)

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/adv_optm/util/scaled_optm.py

@@ -2,6 +2,8 @@ import torch
 
 from . import param_update
 
+import math
+
 def scale_update(
     p: torch.Tensor,
     update: torch.Tensor,
@@ -26,16 +28,16 @@ def scale_update(
 
     # DoRA Magnitude Scales (1D) or 1D Bias/Norm layers
     if is_dora_scale or p.ndim == 1:
-        return rms_normalization(update, dim=None, lr=lr)
+        return l2_normalization(update, dim=None, lr=lr)
 
     # Orthogonal Fine-Tuning (OFT)
     # This guarantees O(1) update complexity scaling, independent of block sizes.
     if is_oft:
         n = update.shape[1]
         # Calculate block size (b)
-        b = (1 + (1 + 8 * n) ** 0.5) / 2
-        target_norm = (b / 8) ** 0.5
-        scale = target_norm / (n ** 0.5)
+        b = (1 + math.sqrt(1 + 8 * n)) / 2
+        target_norm = math.sqrt(b / 8)
+        scale = target_norm / math.sqrt(n)
         return rms_normalization(update, dim=1, lr=lr * scale)
 
     # LoRA Factors or Full Finetuning weights
@@ -46,6 +48,16 @@ def scale_update(
     return update.mul_(lr)
 
 
+def scale_eps(group: dict, p) -> tuple[float, float]:
+    """
+    Scales Adam eps to be scale-invariant.
+    """
+    if group.get('scaled_optm', False):
+        adaptive_eps = (1.0 / group['n_layers']) * (1.0 / math.sqrt(p.numel()))
+    else:
+        adaptive_eps = group['eps']
+    return adaptive_eps
+
 def adjust_wds(wd: float, cwd: float, p: torch.Tensor) -> tuple[float, float]:
     """
     Adjusts standard weight decay and centered weight decay.
@@ -93,7 +105,7 @@ def rms_normalization(update: torch.Tensor, dim: int | None, lr: float) -> torch
     """Performs Root Mean Square normalization on the update tensor."""
     n = update.numel() if dim is None else update.shape[dim]
     norm = torch.linalg.vector_norm(update, ord=2, dim=dim, keepdim=True).clamp_min_(1e-12)
-    scale_n = n**0.5
+    scale_n = math.sqrt(n)
     return update.mul_(lr * scale_n / norm)
 
 
@@ -123,7 +135,7 @@ def spectral_normalization(update: torch.Tensor, vector_state: torch.Tensor, lr:
     update = update.to(vector_state.dtype)
     update_flat = update.view(d_out, d_in)
     # Target scale derived from the "Modular Norm" paper
-    target_scale = (d_out / d_in) ** 0.5
+    target_scale = math.sqrt(d_out / d_in)
     # Power Iteration step to estimate the largest singular value (sigma)
     # u = Wv
     u = torch.mv(update_flat, vector_state)

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/adv_optm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.4.dev5
+Version: 2.4.dev6
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.4.dev5 → adv_optm-2.4.dev6}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="adv_optm",
-    version="2.4.dev5",
+    version="2.4.dev6",
     author="Koratahiu",
     author_email="hiuhonor@gmail.com",
     license='Apache 2.0',