adv-optm 2.6.2.dev1__tar.gz → 2.6.2.dev2__tar.gz

{adv_optm-2.6.2.dev1 → adv_optm-2.6.2.dev2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.6.2.dev1
+Version: 2.6.2.dev2
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.6.2.dev1 → adv_optm-2.6.2.dev2}/adv_optm/__init__.py

@@ -20,4 +20,4 @@ __all__ = [
     "SinkSGD_adv",
 ]
 
-__version__ = "2.6.2.dev1"
+__version__ = "2.6.2.dev2"

{adv_optm-2.6.2.dev1 → adv_optm-2.6.2.dev2}/adv_optm/optim/SinkSGD_adv.py

@@ -373,9 +373,13 @@ class SinkSGD_adv(torch.optim.Optimizer):
                 del anchor
 
         update_scaling = step_size
-        if group.get('spectral_normalization', False) and not (oft_sym and not normed_mt):
+
+        # If spectral normalization was already applied natively in the OFT sinkhorn, skip the scaling block
+        spectral_sinkhorn_applied = oft_sym and not normed_mt and group.get('spectral_normalization', False)
+
+        if group.get('spectral_normalization', False) and not spectral_sinkhorn_applied:
             update = scale_update(p, update, update_scaling, state=state)
-        else:
+        elif not spectral_sinkhorn_applied:
             if snr_cond:
                 update_scaling = update_scaling * (4/math.pi)
             update.mul_(update_scaling)

{adv_optm-2.6.2.dev1 → adv_optm-2.6.2.dev2}/adv_optm/util/sinkhorn.py

@@ -174,14 +174,13 @@ def apply_oft_sinkhorn(
             g_orth_norm = torch.linalg.vector_norm(G, ord=2, dim=(1, 2), keepdim=True).clamp_min_(1e-12)
             G.mul_(target_norm / g_orth_norm)
 
-    # Global normalization
-    norm = torch.linalg.vector_norm(G).clamp_min_(1e-12)
-    target_norm_global = math.sqrt(G.numel())
-    G.mul_(target_norm_global / norm)
-
-    # Extract upper triangular elements back into the flat format
     update.copy_(G[batch_idx, rows, cols])
 
+    # Global normalization
+    norm = torch.linalg.vector_norm(update).clamp_min_(1e-12)
+    target_norm_global = math.sqrt(update.numel())
+    update.mul_(target_norm_global / norm)
+
     return update
 
 @torch.no_grad()
@@ -201,7 +200,6 @@ def apply_spectral_oft_sinkhorn(
     device, dtype = p.device, p.dtype
     rows, cols = scaled_optm.get_cached_structural_tensors(block_size, device)
 
-    # Flatten any prepended batch dimensions for processing
     orig_shape = p.shape
 
     # Align the scale of p with the forward pass

{adv_optm-2.6.2.dev1 → adv_optm-2.6.2.dev2}/adv_optm.egg-info/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 2.6.2.dev1
+Version: 2.6.2.dev2
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-2.6.2.dev1 → adv_optm-2.6.2.dev2}/setup.py

@@ -5,7 +5,7 @@ with open("README.md", "r", encoding="utf-8") as fh:
 
 setup(
     name="adv_optm",
-    version="2.6.2.dev1",
+    version="2.6.2.dev2",
     author="Koratahiu",
     author_email="hiuhonor@gmail.com",
     license='Apache 2.0',