adv-optm 0.1.2__py3-none-any.whl → 0.1.3__py3-none-any.whl

adv_optm/__init__.py

@@ -14,4 +14,4 @@ __all__ = [
     "Lion_Prodigy_adv",
 ]
 
-__version__ = "0.1.2"
+__version__ = "0.1.3"

adv_optm/optim/Lion_Prodigy_adv.py

@@ -82,8 +82,6 @@ class Lion_Prodigy_adv(torch.optim.Optimizer):
             raise ValueError(f"Betas should be in [0.0, 1.0], but got {betas}")
         if not weight_decay >= 0.0:
             raise ValueError(f"Weight decay must be >= 0.0, but got {weight_decay}")
-        if variance_reduction and use_cautious:
-            print("Warning: Using both 'variance_reduction' and 'use_cautious' is not recommended and may lead to unintended effects.")
 
         defaults = dict(
             lr=lr,

adv_optm/optim/Lion_adv.py

@@ -57,8 +57,6 @@ class Lion_adv(torch.optim.Optimizer):
             raise ValueError(f"Betas should be in [0.0, 1.0], but got {betas}")
         if not weight_decay >= 0.0:
             raise ValueError(f"Weight decay must be >= 0.0, but got {weight_decay}")
-        if variance_reduction and use_cautious:
-            print("Warning: Using both 'variance_reduction' and 'use_cautious' is not recommended and may lead to unintended effects.")
 
         defaults = dict(
             lr=lr,

adv_optm/optim/Prodigy_adv.py

@@ -194,11 +194,12 @@ class Prodigy_adv(torch.optim.Optimizer):
                 d1, d2 = state['effective_shape']
 
                 # First moment (m)
-                state['mu_m_nmf'] = torch.zeros(d1, device=device, dtype=dtype) 
-                state['mv_m_nmf'] = torch.zeros(d2, device=device, dtype=dtype)
-                if not self.use_grams:
-                    packed_d2 = (d2 + 7) // 8
-                    state['sign'] = torch.zeros((d1, packed_d2), dtype=torch.uint8, device=device)
+                if self.beta1 > 0:
+                    state['mu_m_nmf'] = torch.zeros(d1, device=device, dtype=dtype) 
+                    state['mv_m_nmf'] = torch.zeros(d2, device=device, dtype=dtype)
+                    if not self.use_grams:
+                        packed_d2 = (d2 + 7) // 8
+                        state['sign'] = torch.zeros((d1, packed_d2), dtype=torch.uint8, device=device)
                 if self.use_AdEMAMix:
                     state['mu_m_slow_nmf'] = torch.zeros(d1, device=p.device, dtype=dtype) 
                     state['mv_m_slow_nmf'] = torch.zeros(d2, device=p.device, dtype=dtype)
@@ -208,7 +209,8 @@ class Prodigy_adv(torch.optim.Optimizer):
                 state['mu_v_nmf'] = torch.zeros(d1, device=device, dtype=dtype) 
                 state['mv_v_nmf'] = torch.zeros(d2, device=device, dtype=dtype)
             else:  # Fallback to standard AdamW for non-factored tensors
-                state['exp_avg'] = torch.zeros_like(p, device=device, dtype=dtype)
+                if self.beta1 > 0:
+                    state['exp_avg'] = torch.zeros_like(p, device=device, dtype=dtype)
                 if self.use_AdEMAMix:
                     state['exp_avg_slow'] = torch.zeros_like(p, dtype=dtype)
                 state['exp_avg_sq'] = torch.zeros_like(p, device=device, dtype=dtype)
@@ -231,22 +233,24 @@ class Prodigy_adv(torch.optim.Optimizer):
         if state['factored']:
             d1, d2 = state['effective_shape']
 
-            # Reconstruct momentum from previous step's factors
-            mt = _unnmf((state['mu_m_nmf'], state['mv_m_nmf']))
-            if not self.use_grams:
-                unpacked_sign = _unpack_bools(state['sign'], original_m=d2)
-                torch.where(unpacked_sign, mt, -mt, out=mt)
-                del unpacked_sign
-            # Update momentum in full-size
             grad_reshaped = grad.view(d1, d2)
-            mt.mul_(self.beta1).add_(grad_reshaped, alpha=self.d * (1.0 - self.beta1))
-            if self.use_grams:
-                mt.copy_(grad_reshaped.sign() * mt.abs())
-            elif self.use_cautious:
-                mask = (mt * grad_reshaped > 0).to(grad_reshaped.dtype)
-                mask.div_(mask.mean().clamp_(min=1e-3))
-                mt.mul_(mask)
-                del mask
+
+            # Reconstruct momentum from previous step's factors
+            if self.beta1 > 0:
+                mt = _unnmf((state['mu_m_nmf'], state['mv_m_nmf']))
+                if not self.use_grams:
+                    unpacked_sign = _unpack_bools(state['sign'], original_m=d2)
+                    torch.where(unpacked_sign, mt, -mt, out=mt)
+                    del unpacked_sign
+                # Update momentum in full-size
+                mt.mul_(self.beta1).add_(grad_reshaped, alpha=self.d * (1.0 - self.beta1))
+                if self.use_grams:
+                    mt.copy_(grad_reshaped.sign() * mt.abs())
+                elif self.use_cautious:
+                    mask = (mt * grad_reshaped > 0).to(grad_reshaped.dtype)
+                    mask.div_(mask.mean().clamp_(min=1e-3))
+                    mt.mul_(mask)
+                    del mask
 
             vt = _unnmf((state['mu_v_nmf'], state['mv_v_nmf']))
             vt.mul_(self.beta2).addcmul_(grad_reshaped, grad_reshaped, value=self.d * self.d * (1.0 - self.beta2))
@@ -258,30 +262,29 @@ class Prodigy_adv(torch.optim.Optimizer):
                 unpacked_sign_slow = _unpack_bools(state['sign_slow'], original_m=d2)
                 torch.where(unpacked_sign_slow, mt_slow, -mt_slow, out=mt_slow)
                 del unpacked_sign_slow
-
                 mt_slow.mul_(beta3_ema).add_(grad_reshaped, alpha=self.d * (1.0 - beta3_ema))
-                update_m = mt + (alpha_t * mt_slow)
+                update = mt + (alpha_t * mt_slow) if self.beta1 > 0 else grad_reshaped + (alpha_t * mt_slow)
             else:
-                update_m = mt
+                update = mt if self.beta1 > 0 else grad_reshaped
             del grad_reshaped
 
             if group['use_atan2']:
                 a = 1.2732395
                 denom = vt.sqrt()
-                update = torch.atan2(update_m, denom).mul_(a)
+                update.atan2_(denom).mul_(a)
             else:
-                denom = vt.sqrt().add_(self.d * group['eps'])
-                update = update_m / denom
-            del update_m, denom
+                denom = vt.sqrt()
+                update.div_(denom.add_(self.d * group['eps']))
+            del denom
 
-            update = update.view(p.shape)
-            update.mul_(self.dlr)
+            update.view(p.shape).mul_(self.dlr)
 
             # Compress updated moments and store new factors
-            if not self.use_grams:
-                state['sign'] = _pack_bools(mt > 0)
-            _nnmf(mt.abs(), out=(state['mu_m_nmf'], state['mv_m_nmf']))
-            del mt
+            if self.beta1 > 0:
+                if not self.use_grams:
+                    state['sign'] = _pack_bools(mt > 0)
+                _nnmf(mt.abs(), out=(state['mu_m_nmf'], state['mv_m_nmf']))
+                del mt
             if self.use_AdEMAMix:
                 state['sign_slow'] = _pack_bools(mt_slow > 0)
                 _nnmf(mt_slow.abs(), out=(state['mu_m_slow_nmf'], state['mv_m_slow_nmf']))
@@ -290,36 +293,38 @@ class Prodigy_adv(torch.optim.Optimizer):
             del vt
 
         else:  # Standard AdamW logic for non-factored tensors
-            exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']
-
-            exp_avg.mul_(self.beta1).add_(grad, alpha=self.d * (1.0 - self.beta1))
-            if self.use_grams:
-                exp_avg = grad.sign() * exp_avg.abs()
-            elif self.use_cautious:
-                mask = (exp_avg * grad > 0).to(grad.dtype)
-                mask.div_(mask.mean().clamp_(min=1e-3))
-                exp_avg.mul_(mask)
-                del mask
+            exp_avg_sq = state['exp_avg_sq']
+
+            if self.beta1 > 0:
+                exp_avg = state['exp_avg']
+                exp_avg.mul_(self.beta1).add_(grad, alpha=self.d * (1.0 - self.beta1))
+                if self.use_grams:
+                    exp_avg = grad.sign() * exp_avg.abs()
+                elif self.use_cautious:
+                    mask = (exp_avg * grad > 0).to(grad.dtype)
+                    mask.div_(mask.mean().clamp_(min=1e-3))
+                    exp_avg.mul_(mask)
+                    del mask
 
             if self.use_AdEMAMix:
                 exp_avg_slow = state['exp_avg_slow']
                 exp_avg_slow.mul_(beta3_ema).add_(grad, alpha=self.d * (1.0 - beta3_ema))
-                update_m = exp_avg + (alpha_t * exp_avg_slow)
+                update = exp_avg + (alpha_t * exp_avg_slow) if self.beta1 > 0 else grad + (alpha_t * exp_avg_slow)
             else:
-                update_m = exp_avg
+                update = exp_avg if self.beta1 > 0 else grad
 
             exp_avg_sq.mul_(self.beta2).addcmul_(grad, grad.conj(), value=self.d * self.d * (1.0 - self.beta2))
 
             if group['use_atan2']:
                 a = 1.2732395
                 denom = exp_avg_sq.sqrt()
-                update = torch.atan2(update_m, denom).mul_(a)
+                update.atan2_(denom).mul_(a)
             else:
-                denom = exp_avg_sq.sqrt().add_(self.d * group['eps'])
-                update = update_m / denom
-            del update_m, denom
+                denom = exp_avg_sq.sqrt()
+                update.div_(denom.add_(self.d * group['eps']))
+            del denom
 
-            update = update.mul_(self.dlr)
+            update.mul_(self.dlr)
 
         # --- Accumulate Prodigy stats ---
         d0, safeguard_warmup, slice_p = group['d0'], group['safeguard_warmup'], group['slice_p']

{adv_optm-0.1.2.dist-info → adv_optm-0.1.3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 0.1.2
+Version: 0.1.3
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu

{adv_optm-0.1.2.dist-info → adv_optm-0.1.3.dist-info}/RECORD

@@ -1,9 +1,9 @@
-adv_optm/__init__.py,sha256=BNYlxkuU8MFsWSY1_PLzp2XBSzpt-sxhnVuWVKRZGZ8,252
+adv_optm/__init__.py,sha256=5Mmq6ovFOuVKvEuEVVHD4UfO9ObsxbJ4KtsuoOtgvxc,252
 adv_optm/optim/AdamW_adv.py,sha256=_4Vt79EB18rnIkHttA0CdMpli8sZ5f03pesdrwT5K58,12887
 adv_optm/optim/Adopt_adv.py,sha256=rzBWfFOPrMuC6vwETsw7QPKmVXcv4IJRDCTj-6eU1Qk,14798
-adv_optm/optim/Lion_Prodigy_adv.py,sha256=ql6506h_IIZvTPdGYrQdd6iEhCXHTMntqmg739fc_dw,14102
-adv_optm/optim/Lion_adv.py,sha256=jOoRbJ6u9HCK7IBI9ILOCcwprKIGTUNvUzhRd99WJK0,9410
-adv_optm/optim/Prodigy_adv.py,sha256=InR50MoE32zG6qgEkg_JzXl7uXAVRy4EYG0JDl4eKok,17324
+adv_optm/optim/Lion_Prodigy_adv.py,sha256=Gc_o0HqZTau-cyP2x4ssKgdQnPYPjJsPVsqTFgz6vGw,13918
+adv_optm/optim/Lion_adv.py,sha256=ZxnV6fQUvOmvJVkeUbStNjeXBWMxDlfMcSLjNpL1uFU,9226
+adv_optm/optim/Prodigy_adv.py,sha256=H7MrZMjCkZdsHBXY17Jm7aTFNySoVkIXQSszdoHn6u4,17697
 adv_optm/optim/__init__.py,sha256=e5UighM92LDvDB2JJwj8gDsTpXEedpytScwqS6F2FR8,300
 adv_optm/util/BF16_Stochastic_Rounding.py,sha256=Q5H0BcogmE4atP65dLoI21HKSf50lRdsBDfeF6v9Tbg,1548
 adv_optm/util/Effective_Shape.py,sha256=TBvIk1V8IuTbbBsxuekJA4e_v8JlR5Nujtut8RTWAm4,318
@@ -11,8 +11,8 @@ adv_optm/util/NNMF.py,sha256=yRf5IP5Sjq0Uf0DxN0Q8NxEGSdD-f1ULziLVDOjY8K4,639
 adv_optm/util/One_Bit_Boolean.py,sha256=Wat49esdwohuN-OHOFMW8D0aOQgV9cP5Rl8z6yfmpos,1068
 adv_optm/util/OrthoGrad.py,sha256=NzInuBQGy_Ja__M1R9XbvqVaQ0fhGbtGgFE9YON7B3I,707
 adv_optm/util/__init__.py,sha256=qoyIF0jcLjs_vSEcsv36clw5LFNBEbifyXrrVxMH-G4,349
-adv_optm-0.1.2.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-adv_optm-0.1.2.dist-info/METADATA,sha256=iV5GBWtl4WphBeSIIsUoq1ay6-GJGnDD3XF6aSWWrqg,5846
-adv_optm-0.1.2.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-adv_optm-0.1.2.dist-info/top_level.txt,sha256=iNfBIIzu-lPrQ7jyC56WBCcbkRwitM2nJ15-MRQ_6fg,9
-adv_optm-0.1.2.dist-info/RECORD,,
+adv_optm-0.1.3.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+adv_optm-0.1.3.dist-info/METADATA,sha256=xv7wytTibFrp0MWf2htvY8N413qNPQs6P9cB-r2HOPY,5846
+adv_optm-0.1.3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+adv_optm-0.1.3.dist-info/top_level.txt,sha256=iNfBIIzu-lPrQ7jyC56WBCcbkRwitM2nJ15-MRQ_6fg,9
+adv_optm-0.1.3.dist-info/RECORD,,