adv-optm 2.4.dev21__tar.gz → 2.4.dev23__tar.gz

{adv_optm-2.4.dev21 → adv_optm-2.4.dev23}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.4.dev21
+Version: 2.4.dev23
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.4.dev21 → adv_optm-2.4.dev23}/adv_optm/__init__.py

@@ -20,4 +20,4 @@ __all__ = [
     "SinkSGD_adv",
 ]
 
-__version__ = "2.4.dev21"
+__version__ = "2.4.dev23"

{adv_optm-2.4.dev21 → adv_optm-2.4.dev23}/adv_optm/optim/SignSGD_adv.py

@@ -70,8 +70,8 @@ class SignSGD_adv(torch.optim.Optimizer):
         nesterov_coef: float | None = None,
         # Normalization then Momentum
         normed_momentum: bool = False,
-        # Centered Variance Precondition
-        centered_vt: bool = False,
+        # SNR Precondition
+        snr_cond: bool = False,
         # Centered WD
         centered_wd: float = 0.0,
         centered_wd_mode: str = 'float8',
@@ -91,8 +91,8 @@ class SignSGD_adv(torch.optim.Optimizer):
             raise ValueError(f"momentum should be in [0.0, 1.0], but got {momentum}")
         if not weight_decay >= 0.0:
             raise ValueError(f"Weight decay must be >= 0.0, but got {weight_decay}")
-        if centered_vt and not normed_momentum and not momentum > 0:
-            raise NotImplementedError(f"centered_vt is intended to be used with normed_momentum")
+        if snr_cond and not normed_momentum and not momentum > 0:
+            raise NotImplementedError(f"snr_cond is intended to be used with normed_momentum")
 
         state_precision = state_precision.lower()
         valid_precisions = {"auto", "fp32", "factored", "bf16_sr", "fp16", "fp8_sr", "int8_sr"}
@@ -115,7 +115,7 @@ class SignSGD_adv(torch.optim.Optimizer):
             nesterov=nesterov,
             nesterov_coef=nesterov_coef,
             normed_momentum=normed_momentum,
-            centered_vt=centered_vt,
+            snr_cond=snr_cond,
             spectral_normalization=spectral_normalization,
             centered_wd= centered_wd,
             centered_wd_mode= centered_wd_mode,
@@ -254,7 +254,7 @@ class SignSGD_adv(torch.optim.Optimizer):
         nesterov = group.get('nesterov', False)
         nesterov_coef = group.get('nesterov_coef', None)
         sso = group.get('stochastic_sign', False)
-        centered_vt = group.get('centered_vt', False) and group.get('normed_momentum', False) and momentum > 0
+        snr_cond = group.get('snr_cond', False) and group.get('normed_momentum', False) and momentum > 0
 
         denom = None
         wd_target = None
@@ -278,7 +278,7 @@ class SignSGD_adv(torch.optim.Optimizer):
                 # Reconstruct momentum m_{t-1}
                 exp_avg = _reconstruct_state((state['mu_m_nmf'], state['mv_m_nmf'], state['sign'], d2), signed=True, shifter=state['shifter'])
 
-                if centered_vt:
+                if snr_cond:
                     denom = (1.0 - exp_avg.square()).clamp_min_(1e-30).sqrt_().view_as(p)
 
                 exp_avg.lerp_(grad_reshaped, 1 - momentum)
@@ -302,7 +302,7 @@ class SignSGD_adv(torch.optim.Optimizer):
                 actual_precision = group['actual_state_precision']
                 exp_avg = get_state(state, 'exp_avg', actual_precision)
 
-                if centered_vt:
+                if snr_cond:
                     denom = (1.0 - exp_avg.square()).clamp_min_(1e-30).sqrt_()
 
                 exp_avg.lerp_(grad, 1 - momentum)
@@ -325,7 +325,7 @@ class SignSGD_adv(torch.optim.Optimizer):
         else:
             update = raw_update
 
-        if centered_vt:
+        if snr_cond:
             update.atan2_(denom)
 
         if group.get('geometric_wd', False) and group["weight_decay"] > 0 :
@@ -339,7 +339,7 @@ class SignSGD_adv(torch.optim.Optimizer):
         if group.get('spectral_normalization', False):
             update = scale_update(p, update, lr, state=state)
         else:
-            update_scaling = lr * A if centered_vt else lr
+            update_scaling = lr * A if snr_cond else lr
             update.mul_(update_scaling)
 
         param_update.apply_parameter_update(self, p, group, update, lr, random_int_tensor=random_int_tensor, wd_target=wd_target, cwd_target=cwd_target)

{adv_optm-2.4.dev21 → adv_optm-2.4.dev23}/adv_optm/optim/SinkSGD_adv.py

@@ -58,8 +58,8 @@ class SinkSGD_adv(torch.optim.Optimizer):
         orthogonal_sinkhorn: bool = False,
         # Normalization then Momentum
         normed_momentum: bool = False,
-        # Centered Variance Precondition
-        centered_vt: bool = False,
+        # SNR Precondition
+        snr_cond: bool = False,
         # Nesterov Momentum
         nesterov: bool = False,
         nesterov_coef: float | None = None,
@@ -89,8 +89,8 @@ class SinkSGD_adv(torch.optim.Optimizer):
             raise ValueError(f"Momentum should be >= 0.0. Got {momentum}")
         if not (weight_decay >= 0.0):
             raise ValueError(f"Weight-decay should be >= 0.0. Got {weight_decay}")
-        if centered_vt and not normed_momentum:
-            raise NotImplementedError(f"centered_vt is intended to be used with normed_momentum")
+        if snr_cond and not normed_momentum:
+            raise NotImplementedError(f"snr_cond is intended to be used with normed_momentum")
 
         state_precision = state_precision.lower()
         valid_precisions = {"auto", "fp32", "factored", "bf16_sr", "fp16", "fp8_sr", "int8_sr"}
@@ -102,7 +102,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
 
         defaults = {
             "lr": lr, "momentum": momentum,
-            "weight_decay": weight_decay, "nesterov": nesterov, "nesterov_coef": nesterov_coef, "normed_momentum": normed_momentum, "centered_vt": centered_vt,
+            "weight_decay": weight_decay, "nesterov": nesterov, "nesterov_coef": nesterov_coef, "normed_momentum": normed_momentum, "snr_cond": snr_cond,
             "geometric_wd": geometric_wd, "cautious_wd": cautious_wd,
             "orthogonal_gradient": orthogonal_gradient, 
             "compiled_optimizer": compiled_optimizer,
@@ -228,7 +228,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
         momentum = group['momentum']
         nesterov = group['nesterov']
         nesterov_coef = group.get('nesterov_coef', None)
-        centered_vt = group.get('centered_vt', False)
+        snr_cond = group.get('snr_cond', False)
 
         vt_row = None
         vt_col = None
@@ -238,6 +238,9 @@ class SinkSGD_adv(torch.optim.Optimizer):
         wd_target = None
         cwd_target = None
 
+        if group["orthogonal_gradient"]:
+            grad = _orthogonalize_gradient(p, grad)
+
         if group.get('normed_momentum', False):
             if not is_vector:
                 # Sinkhorn iterative normalization
@@ -246,9 +249,6 @@ class SinkSGD_adv(torch.optim.Optimizer):
                 # For vectors, apply sign operation
                 grad = grad.sign_()
 
-        if group["orthogonal_gradient"]:
-            grad = _orthogonalize_gradient(p, grad)
-
         if state['factored']:
             d1, d2 = state['effective_shape']
             grad_reshaped = grad.view(d1, d2)
@@ -256,7 +256,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
             if momentum != 0:
                 buf = _reconstruct_state((state['mu_b_nmf'], state['mv_b_nmf'], state['sign'], d2), signed=True, shifter=state['shifter'])
 
-                if centered_vt:
+                if snr_cond:
                     if not is_vector:
                         buf_2d_sq = buf.view(grad.shape[0], -1).square()
                         vt_row = (1 - buf_2d_sq.mean(dim=-1)).clamp_min_(1e-30)
@@ -286,7 +286,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
             if momentum != 0:
                 buf = get_state(state, 'momentum_buffer', actual_precision)
 
-                if centered_vt:
+                if snr_cond:
                     if not is_vector:
                         buf_2d_sq = buf.view(grad.shape[0], -1).square()
                         vt_row = (1 - buf_2d_sq.mean(dim=-1)).clamp_min_(1e-30)
@@ -309,7 +309,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
 
             del random_int_state_tensor
 
-        if centered_vt:
+        if snr_cond:
             if not is_vector:
                 # Align with Sinkhorn: Alternate row/col preconditioning
                 update_2d = update.view(update.shape[0], -1)
@@ -342,7 +342,7 @@ class SinkSGD_adv(torch.optim.Optimizer):
         if group.get('spectral_normalization', False):
             update = scale_update(p, update, update_scaling, state=state)
         else:
-            if centered_vt:
+            if snr_cond:
                 update_scaling = update_scaling * (4/math.pi)
             update.mul_(update_scaling)
 

{adv_optm-2.4.dev21 → adv_optm-2.4.dev23}/adv_optm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.4.dev21
+Version: 2.4.dev23
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.4.dev21 → adv_optm-2.4.dev23}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="adv_optm",
-    version="2.4.dev21",
+    version="2.4.dev23",
     author="Koratahiu",
     author_email="hiuhonor@gmail.com",
     license='Apache 2.0',