adv-optm 2.6.1.dev1__tar.gz → 2.6.1.dev3__tar.gz

{adv_optm-2.6.1.dev1 → adv_optm-2.6.1.dev3}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.6.1.dev1
+Version: 2.6.1.dev3
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.6.1.dev1 → adv_optm-2.6.1.dev3}/adv_optm/__init__.py

@@ -20,4 +20,4 @@ __all__ = [
     "SinkSGD_adv",
 ]
 
-__version__ = "2.6.1.dev1"
+__version__ = "2.6.1.dev3"

{adv_optm-2.6.1.dev1 → adv_optm-2.6.1.dev3}/adv_optm/util/param_update.py

@@ -106,6 +106,7 @@ def apply_parameter_update(
     cwd = group.get("centered_wd", 0.0)
     wd, cwd = adjust_wds(wd, cwd, p)
     scaled_wd = group.get("scaled_wd", False)
+    decoupled = scaled_wd or decoupled
 
     # Calculate global decay factor for decoupled vs standard
     decay_factor = (lr / self._init_lr) if decoupled else lr
@@ -113,14 +114,14 @@ def apply_parameter_update(
     eff_wd = (wd * decay_factor) if wd != 0 else None
     eff_cwd = (cwd * decay_factor) if cwd != 0 else None
 
+    if scaled_wd:
+        eff_wd = scale_wd(eff_wd, p, skip_vectors=True)
+        eff_cwd = scale_wd(eff_cwd, p, skip_vectors=False)
+
     if wd_scaler is not None:
         if eff_wd is not None:
-            if scaled_wd:
-                eff_wd = scale_wd(eff_wd, p)
             eff_wd = eff_wd * wd_scaler
         if eff_cwd is not None:
-            if scaled_wd:
-                eff_cwd = scale_wd(eff_cwd, p)
             eff_cwd = eff_cwd * wd_scaler
 
     state = self.state[p]

{adv_optm-2.6.1.dev1 → adv_optm-2.6.1.dev3}/adv_optm/util/scaled_optm.py

@@ -97,21 +97,26 @@ def adjust_wds(wd: float, cwd: float, p: torch.Tensor) -> tuple[float, float]:
         # Centered WD safely regularizes the delta without collapsing base feature variance.
         return wd, cwd
 
-def scale_wd(wd: float, p: torch.Tensor) -> float:
+def scale_wd(wd: float | None, p: torch.Tensor, skip_vectors: bool = False) -> float:
     """
     Scale-invariant, dimension-scaled weight decay.
     """
+    if wd is None:
+        return wd
     if getattr(p, '_is_oft', False):
         n_el = p.shape[-1]
         b = (1.0 + math.sqrt(1.0 + 8.0 * n_el)) / 2.0
         wd = (2 * wd) / (b - 1)
         return wd
 
+    is_vector = p.ndim < 2 or getattr(p, '_is_dora_scale', False) or getattr(p, 'is_vector', False) 
+    if is_vector:
+        return 0.0 if skip_vectors else wd
+
     if p.ndim >= 2:
         width = p.numel() // p.shape[0]
         return wd / width
 
-
 def is_spectral(p: torch.Tensor) -> bool:
     """Determines if a parameter should undergo spectral normalization updates."""
     if getattr(p, '_is_oft', False):
@@ -227,14 +232,14 @@ def apply_spectral_riemannian_oft(
     # Estimate sigma (The spectral norm) for each block
     sigma = torch.sum(next_u * u_raw, dim=1, keepdim=True)
 
-    # We constrain the spectral norm of the entire block-diagonal update matrix
-    # which is the maximum of the spectral norms of its blocks.
-    max_sigma = sigma.max()
+    # Squeeze out the last dimension so shape becomes (batch_size, 1)
+    sigma = sigma.squeeze(-1) 
 
     target_scale = 0.5 * scale_factor
     spectral_eps = 1.0 / (2.0 * math.sqrt(block_size))
 
-    scale = lr * (target_scale / max_sigma.clamp_min(spectral_eps))
+    # Apply the clamp and scaling block-wise
+    scale = lr * (target_scale / sigma.clamp_min(spectral_eps))
 
     return update_flat.mul_(scale).view(orig_shape)
 

{adv_optm-2.6.1.dev1 → adv_optm-2.6.1.dev3}/adv_optm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.6.1.dev1
+Version: 2.6.1.dev3
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.6.1.dev1 → adv_optm-2.6.1.dev3}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="adv_optm",
-    version="2.6.1.dev1",
+    version="2.6.1.dev3",
     author="Koratahiu",
     author_email="hiuhonor@gmail.com",
     license='Apache 2.0',