adv-optm 2.4.dev24__tar.gz → 2.4.dev25__tar.gz

{adv_optm-2.4.dev24 → adv_optm-2.4.dev25}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.4.dev24
+Version: 2.4.dev25
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.4.dev24 → adv_optm-2.4.dev25}/adv_optm/__init__.py

@@ -20,4 +20,4 @@ __all__ = [
     "SinkSGD_adv",
 ]
 
-__version__ = "2.4.dev24"
+__version__ = "2.4.dev25"

{adv_optm-2.4.dev24 → adv_optm-2.4.dev25}/adv_optm/optim/SignSGD_adv.py

@@ -280,16 +280,18 @@ class SignSGD_adv(torch.optim.Optimizer):
                 if snr_cond:
                     denom = (1.0 - exp_avg.square()).clamp_min_(1e-30).sqrt_().view_as(p)
 
+                if nesterov and normed_mt:
+                    # Scale the normalized gradient using empirical buffer magnitude (SNR recovery)
+                    normed_grad = grad_reshaped * exp_avg.abs()
+
                 exp_avg.lerp_(grad_reshaped, 1 - momentum)
 
                 if nesterov:
                     nv_coef = momentum if nesterov_coef is None else nesterov_coef
                     if normed_mt:
-                        # Scale the normalized gradient down to match the buffer's variance
-                        ema_std = math.sqrt((1 - momentum) / (1 + momentum))
-                        raw_update = (grad_reshaped * ema_std).lerp_(exp_avg, nv_coef)
+                        raw_update = normed_grad.lerp_(exp_avg, nv_coef)
                     else:
-                        raw_update = grad.lerp(exp_avg, nv_coef)
+                        raw_update = grad_reshaped.lerp(exp_avg, nv_coef)
                 else:
                     raw_update = exp_avg.clone()
 
@@ -309,14 +311,16 @@ class SignSGD_adv(torch.optim.Optimizer):
                 if snr_cond:
                     denom = (1.0 - exp_avg.square()).clamp_min_(1e-30).sqrt_()
 
+                if nesterov and normed_mt:
+                    # Scale the normalized gradient using empirical buffer magnitude (SNR recovery)
+                    normed_grad = grad * exp_avg.abs()
+
                 exp_avg.lerp_(grad, 1 - momentum)
 
                 if nesterov:
                     nv_coef = momentum if nesterov_coef is None else nesterov_coef
                     if normed_mt:
-                        # Scale the normalized gradient down to match the buffer's variance
-                        ema_std = math.sqrt((1 - momentum) / (1 + momentum))
-                        raw_update = (grad * ema_std).lerp_(exp_avg, nv_coef)
+                        raw_update = normed_grad.lerp_(exp_avg, nv_coef)
                     else:
                         raw_update = grad.lerp(exp_avg, nv_coef)
                 else:
@@ -351,7 +355,7 @@ class SignSGD_adv(torch.optim.Optimizer):
             update_scaling = lr * A if snr_cond else lr
             update.mul_(update_scaling)
 
-        param_update.apply_parameter_update(self, p, group, update, lr, random_int_tensor=random_int_tensor, wd_target=wd_target, cwd_target=cwd_target)
+        param_update.apply_parameter_update(self, p, group, update, lr, random_int_tensor=random_int_tensor, wd_target=wd_target, cwd_target=cwd_target, decoupled=True)
 
     def compile(self, *args, **kwargs):
         self._compiled_step_parameter = torch.compile(self._step_parameter, *args, **kwargs)

{adv_optm-2.4.dev24 → adv_optm-2.4.dev25}/adv_optm/optim/SinkSGD_adv.py

@@ -264,6 +264,10 @@ class SinkSGD_adv(torch.optim.Optimizer):
                     else:
                         denom = (1.0 - buf.square()).clamp_min_(1e-30).sqrt_().view_as(p)
 
+                if nesterov and normed_mt:
+                    # Scale the normalized gradient using empirical buffer magnitude (SNR recovery)
+                    normed_grad = grad_reshaped * buf.abs()
+
                 buf.lerp_(grad_reshaped, 1 - momentum)
 
                 # Factorize updated buffer
@@ -272,9 +276,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
                 if nesterov:
                     nv_coef = momentum if nesterov_coef is None else nesterov_coef
                     if normed_mt:
-                        # Scale the normalized gradient down to match the buffer's variance
-                        ema_std = math.sqrt((1 - momentum) / (1 + momentum))
-                        update = (grad_reshaped * ema_std).lerp_(buf, nv_coef)
+                        update = normed_grad.lerp_(buf, nv_coef)
                     else:
                         update = grad_reshaped.lerp(buf, nv_coef)
                 else:
@@ -299,6 +301,10 @@ class SinkSGD_adv(torch.optim.Optimizer):
                     else:
                         denom = (1.0 - buf.square()).clamp_min_(1e-30).sqrt_()
 
+                if nesterov and normed_mt:
+                    # Scale the normalized gradient using empirical buffer magnitude (SNR recovery)
+                    normed_grad = grad * buf.abs()
+
                 buf.lerp_(grad, 1 - momentum)
 
                 set_state(state, 'momentum_buffer', buf, actual_precision, random_int_state_tensor)
@@ -306,9 +312,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
                 if nesterov:
                     nv_coef = momentum if nesterov_coef is None else nesterov_coef
                     if normed_mt:
-                        # Scale the normalized gradient down to match the buffer's variance
-                        ema_std = math.sqrt((1 - momentum) / (1 + momentum))
-                        update = (grad * ema_std).lerp_(buf, nv_coef)
+                        update = normed_grad.lerp_(buf, nv_coef)
                     else:
                         update = grad.lerp(buf, nv_coef)
                 else:

{adv_optm-2.4.dev24 → adv_optm-2.4.dev25}/adv_optm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.4.dev24
+Version: 2.4.dev25
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.4.dev24 → adv_optm-2.4.dev25}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="adv_optm",
-    version="2.4.dev24",
+    version="2.4.dev25",
     author="Koratahiu",
     author_email="hiuhonor@gmail.com",
     license='Apache 2.0',