adv-optm 1.2.dev19__py3-none-any.whl → 2.dev3__py3-none-any.whl

adv_optm/optim/Simplified_AdEMAMix.py

@@ -66,10 +66,10 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
         k_logging (int): if > 0 and kourkoutas_beta=True, enables periodic console
             logging of Kourkoutas-β statistics (min, max, mean of `β₂` across layers)
             every logging steps. Useful for debugging and tuning. Set to 0 to disable
-            logging (default: 0). 
+            logging (default: 0).
         layer_key_fn (Optional[Callable]): A function that takes a parameter `p`
             and returns a unique, hashable key representing its "layer" or "bucket".
-            If `None`, parameters are bucketed by their memory ID (tensor-wise).
+            If `None`, parameters are bucketed by their shape.
             (default: None)
         nnmf_factor (bool): whether to use the factorization or disable it to use
             the uncompressed optimizer. (default: False)
@@ -86,7 +86,7 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
         beta1_warmup: int | None = None,
         min_beta1: float | None = 0.9,
         use_bias_correction: bool = True,
-        vector_reshape: bool = True,
+        vector_reshape: bool = False,
         stochastic_rounding: bool = True,
         orthogonal_gradient: bool = False,
         kourkoutas_beta: bool = False,
@@ -97,6 +97,8 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
         k_logging: int = 0,
         layer_key_fn: Optional[Callable] = None,
         nnmf_factor: bool = False,
+        # Compiled
+        compiled_optimizer: bool = False,
     ):
         if not (lr >= 0.0):
             raise ValueError(f"Learning-rate should be >= 0.0. Got {lr}")
@@ -108,7 +110,8 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
             raise ValueError(f"Weight-decay should be >= 0.0. Got {weight_decay}")
         if not 0.0 <= alpha_grad:
             raise ValueError("Invalid alpha value: {}".format(alpha_grad))
-        if kourkoutas_beta and not (betas[1] > beta2_min): raise ValueError(f"For Kourkoutas-β, betas[1] (as beta2_max) must be > beta2_min. Got {betas[1]} and {beta2_min}")
+        if kourkoutas_beta and not (betas[1] > beta2_min):
+            raise ValueError(f"For Kourkoutas-β, betas[1] (as beta2_max) must be > beta2_min. Got {betas[1]} and {beta2_min}")
 
         defaults = {
             "lr": lr, "betas": betas, "eps": eps, "weight_decay": weight_decay,
@@ -117,16 +120,33 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
             "orthogonal_gradient": orthogonal_gradient, "use_bias_correction": use_bias_correction,
             "kourkoutas_beta": kourkoutas_beta, "beta2_min": beta2_min, "ema_alpha": ema_alpha,
             "tiny_spike": tiny_spike, "k_warmup_steps": k_warmup_steps, "k_logging": k_logging,
+            "compiled_optimizer": compiled_optimizer,
         }
         self.stochastic_rounding = stochastic_rounding
         self.factored = nnmf_factor
         self.kourkoutas_beta = kourkoutas_beta
         self.layer_key_fn = layer_key_fn
+        self.use_bias_correction = use_bias_correction
+        if use_bias_correction:
+            self.num_sum = betas[0] * 1.0
+            self.den_sum = betas[1] * (1.0 - betas[1])
+        else:
+            self.num_sum = 1.0
+            self.den_sum = 1.0
+
         super().__init__(params, defaults)
 
+        self.init_step()
+
         if self.kourkoutas_beta:
             self.kourkoutas_helper = KourkoutasHelper(self)
 
+        self.global_step = 0
+
+        if compiled_optimizer:
+            torch._dynamo.config.cache_size_limit = 8192
+            self.compile(fullgraph=True)
+
     @property
     def supports_fused_back_pass(self):
         return True
@@ -139,29 +159,22 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
     def supports_flat_params(self):
         return False
 
-    @torch.no_grad()
-    def step_parameter(self, p: torch.Tensor, group: dict, i: int | None = None):
-        if p.grad is None:
-            return
+    def init_step(self):
+        for group in self.param_groups:
+            for p in group['params']:
+                self.__init_state(p, group)
 
-        grad = p.grad
-        if grad.dtype != torch.float32 and self.factored:
-            grad = grad.float()
-        if group["orthogonal_gradient"]:
-            grad = _orthogonalize_gradient(p, grad)
+    @torch.no_grad()
+    def __init_state(self, p, group):
         state = self.state[p]
 
-        # State Initialization
-        if 'step' not in state:
-            state['step'] = 0
+        if len(state) == 0:
 
-            should_factor = (
+            state['factored'] = (
                 self.factored and
                 not (len(p.shape) == 1 and not group['vector_reshape'])
             )
 
-            state['factored'] = should_factor
-
             dtype = torch.float32 if self.factored else p.dtype
             device = p.device
 
@@ -170,50 +183,42 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
                 d1, d2 = state['effective_shape']
 
                 # First moment (m)
-                state['mu_m_nmf'] = torch.zeros(d1, device=device, dtype=dtype) 
+                state['mu_m_nmf'] = torch.zeros(d1, device=device, dtype=dtype)
                 state['mv_m_nmf'] = torch.zeros(d2, device=device, dtype=dtype)
                 packed_d2 = (d2 + 7) // 8
                 state['sign'] = torch.zeros((d1, packed_d2), dtype=torch.uint8, device=device)
                 # Second moment (v)
-                state['mu_v_nmf'] = torch.zeros(d1, device=device, dtype=dtype) 
+                state['mu_v_nmf'] = torch.zeros(d1, device=device, dtype=dtype)
                 state['mv_v_nmf'] = torch.zeros(d2, device=device, dtype=dtype)
             else:  # Fallback to standard optimizer for non-factored tensors
                 state['exp_avg'] = torch.zeros_like(p, device=device, dtype=dtype)
                 state['exp_avg_sq'] = torch.zeros_like(p, device=device, dtype=dtype)
-            
-            if group['use_bias_correction']:
-                state['num_sum'] = 0.0
-                state['den_sum'] = 0.0
-            else:
-                state['num_sum'] = 1.0
-                state['den_sum'] = 1.0
 
-        beta1_final, beta2 = group["betas"]
 
-        current_step = state['step']
+
+    @torch.no_grad()
+    def __step_parameter(self, p: torch.Tensor, group: dict, lr: torch.Tensor | float, beta1, num_sum, den_sum):
+        if p.grad is None:
+            return
+
+        grad = p.grad
+        if grad.dtype != torch.float32 and self.factored:
+            grad = grad.float()
+        if group["orthogonal_gradient"]:
+            grad = _orthogonalize_gradient(p, grad)
+        state = self.state[p]
+
+
+        ___, beta2 = group["betas"]
+
         if group.get('kourkoutas_beta', False):
-            # Call prepare_step() once at the beginning of the step for all params
-            self.kourkoutas_helper.maybe_prepare_step(current_step)
             # Accumulate current grad's norm for the *next* step
             self.kourkoutas_helper.accumulate_gradient_sq_norm(p, grad)
             # Get the dynamic beta2 calculated in prepare_step()
-            beta2 = self.kourkoutas_helper.get_beta2(p, group, current_step)
+            beta2 = self.kourkoutas_helper.get_beta2(p, group)
 
-        beta1_warmup = group["beta1_warmup"]
         alpha_grad = group["alpha_grad"]
 
-        if beta1_warmup is not None:
-            step = state['step'] + 1
-            beta1 = linear_hl_warmup_scheduler(step, beta_end=beta1_final, beta_start=group['min_beta1'], warmup=beta1_warmup)
-        else:
-            beta1 = beta1_final
-
-        if group['use_bias_correction']:
-            state['num_sum'] = beta1 * state['num_sum'] + 1.0
-            if group.get('kourkoutas_beta', False):
-                state['den_sum'] = group['betas'][1] * state['den_sum'] + (1.0 - group['betas'][1])
-            else:
-                state['den_sum'] = beta2 * state['den_sum'] + (1.0 - beta2)
 
         if state['factored']:
             d1, d2 = state['effective_shape']
@@ -233,12 +238,12 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
             update = torch.add(mt, grad_reshaped, alpha=alpha_grad)
             del grad_reshaped
 
-            denom = vt.sqrt().add_(group['eps'] * math.sqrt(state['den_sum']))
+            denom = vt.sqrt().add_(group['eps'] * math.sqrt(den_sum))
             update.div_(denom)
             del denom
 
             if group['use_bias_correction']:
-                update = (update / state['num_sum']) * math.sqrt(state['den_sum'])
+                update = (update / num_sum) * math.sqrt(den_sum)
 
             update = update.view(p.shape).mul_(group['lr'])
 
@@ -259,12 +264,12 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
 
             exp_avg_sq.mul_(beta2).addcmul_(grad, grad, value=1 - beta2)
 
-            denom = exp_avg_sq.sqrt().add_(group['eps'] * math.sqrt(state['den_sum']))
+            denom = exp_avg_sq.sqrt().add_(group['eps'] * math.sqrt(den_sum))
             update.div_(denom)
             del denom
 
             if group['use_bias_correction']:
-                update = (update / state['num_sum']) * math.sqrt(state['den_sum'])
+                update = (update / num_sum) * math.sqrt(den_sum)
 
             update.mul_(group['lr'])
 
@@ -281,7 +286,36 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
             p.data.add_(-update)
         del update
 
-        state['step'] += 1
+
+    @torch.no_grad()
+    def step_parameter(self, p: torch.Tensor, group: dict, i: int | None = None):
+        if self.global_step is None and 'step' in self.state[p]:
+            # For backward compatibility
+            g_state = self.state[p]
+            self.global_step = g_state['step']
+            self.num_sum = group["betas"][0] * g_state['num_sum'] + 1.0
+            self.den_sum = group['betas'][1] * g_state['den_sum'] + (1.0 - group['betas'][1])
+
+        if group["beta1_warmup"] is not None:
+            step = self.global_step + 1
+            beta1 = linear_hl_warmup_scheduler(step, beta_end=group["betas"][0], beta_start=group['min_beta1'], warmup=group["beta1_warmup"])
+        else:
+            beta1 = group["betas"][0]
+
+        if group.get('kourkoutas_beta', False):
+            # Prepare Kourkoutas-β once per step using the global step counter.
+            self.kourkoutas_helper.maybe_prepare_step(self.global_step)
+
+        if not group.get('compiled_optimizer', False):
+            self.__step_parameter(p, group, group['lr'], beta1, self.num_sum, self.den_sum)
+        else:
+            lr_tensor = torch.tensor(group['lr'], device=p.device)
+            num_sum_tesnor = torch.tensor(self.num_sum, device=p.device)
+            den_sum_tesnor = torch.tensor(self.den_sum, device=p.device)
+            self._compiled_step_parameter(p, group, lr_tensor, beta1, self.num_sum, self.den_sum)
+
+    def compile(self, *args, **kwargs):
+        self._compiled_step_parameter = torch.compile(self.__step_parameter, *args, **kwargs)
 
     @torch.no_grad()
     def step(self, closure=None):
@@ -294,5 +328,12 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
         for group in self.param_groups:
             for i, p in enumerate(group['params']):
                 self.step_parameter(p, group, i)
+        
+        g_group = self.param_groups[0]
+        if g_group['use_bias_correction']:
+            self.num_sum = g_group["betas"][0] * self.num_sum + 1.0
+            self.den_sum = g_group['betas'][1] * self.den_sum + (1.0 - g_group['betas'][1])
+
+        self.global_step += 1
 
-        return loss
+        return loss

adv_optm/optim/__init__.py

@@ -16,4 +16,4 @@ __all__ = [
     "Lion_Prodigy_adv",
     "Muon_adv",
     "AdaMuon_adv",
-]
+]

adv_optm/util/BF16_Stochastic_Rounding.py

@@ -44,4 +44,4 @@ def add_stochastic_(input: Tensor, other: Tensor, alpha: float = 1.0):
     result = other.clone() if other.dtype == torch.float32 else other.to(dtype=torch.float32)
 
     result.add_(input, alpha=alpha)
-    copy_stochastic_(input, result)
+    copy_stochastic_(input, result)

adv_optm/util/Effective_Shape.py

@@ -5,4 +5,4 @@ def _get_effective_shape(numel: int) -> tuple[int, int]:
     for i in reversed(range(1, int(numel ** 0.5) + 1)):
         if numel % i == 0:
             return (numel // i, i)
-    return (numel, 1)
+    return (numel, 1)

adv_optm/util/Kourkoutas.py

@@ -1,6 +1,5 @@
 import torch
 from torch.optim import Optimizer
-from typing import Callable
 
 class KourkoutasHelper:
     """
@@ -58,7 +57,7 @@ class KourkoutasHelper:
         Calculates dynamic beta2 for all layers using the completed scalar accumulators
         from the PREVIOUS step. Should be called once at the start of an optimizer step.
         """
-        
+
         beta2_log = []
         # These are just for the sample log, initialize them
         sun, pooled_grad_norm, r_ema_tensor = (torch.tensor(0.0),)*3
@@ -69,7 +68,7 @@ class KourkoutasHelper:
         master_defaults = self.optimizer.defaults
 
         for layer_key, info in self.layer_info.items():
-            params, group = info['params'], info['group_ref']
+            group = info['group_ref']
 
             if not group.get('kourkoutas_beta', False) and not group.get('adam_kourkoutas_beta', False):
                 continue
@@ -81,7 +80,7 @@ class KourkoutasHelper:
                 self.layer_state[layer_key] = {
                     'sum_sq_accumulator': torch.tensor(0.0, device=first_param_in_layer.device, dtype=torch.float32)
                 }
-            
+
             if 'kourkoutas_r_ema' not in param_state:
                 param_state['kourkoutas_r_ema'] = torch.tensor(0.0, device=first_param_in_layer.device, dtype=torch.float32)
 
@@ -96,14 +95,14 @@ class KourkoutasHelper:
 
             r_ema_tensor = param_state['kourkoutas_r_ema']
             accumulator = self.layer_state[layer_key]['sum_sq_accumulator']
-            
+
             pooled_grad_norm = torch.sqrt(accumulator)
-            
+
             # Update the persistent EMA tensor in-place.
             r_ema_tensor.mul_(ema_alpha).add_(pooled_grad_norm, alpha=1.0 - ema_alpha)
-            
+
             sun = torch.tensor(0.0, device=r_ema_tensor.device) # Default sun to 0 for warmup
-            
+
             if current_step < k_warmup_steps:
                 beta2 = beta2_max
             else:
@@ -113,7 +112,7 @@ class KourkoutasHelper:
 
             # Store the final calculated beta2 in the helper's transient state for this step.
             self.layer_state[layer_key]['dynamic_beta2'] = beta2.item() if isinstance(beta2, torch.Tensor) else beta2
-            
+
             # Reset the accumulator for the next optimizer step.
             accumulator.zero_()
 
@@ -149,11 +148,10 @@ class KourkoutasHelper:
             # Accumulate for the *next* step's prepare_step call
             self.layer_state[layer_key]['sum_sq_accumulator'] += torch.sum(grad.detach().pow(2)).float()
 
-    def get_beta2(self, p: torch.Tensor, group: dict, current_step: int) -> float:
+    def get_beta2(self, p: torch.Tensor, group: dict) -> float:
         """
         Gets the appropriate beta2 for the current parameter, handling warmup and dynamic value fetching.
         """
         layer_key = self.optimizer.layer_key_fn(p)
         # The default is the max value, which is correct for unmapped params or edge cases
-        beta2_default = group.get('betas', group.get('adam_betas'))[1] if group.get('betas', group.get('adam_betas')) else 0.999
-        return self.layer_state.get(layer_key, {}).get('dynamic_beta2', beta2_default)
+        return self.layer_state.get(layer_key, {}).get('dynamic_beta2', group['betas'][1])

adv_optm/util/NNMF.py

@@ -9,10 +9,15 @@ def _nnmf(matrix: torch.Tensor, out: tuple):
     shape = matrix.shape
     torch.sum(matrix, dim=1, out=out[0])
     torch.sum(matrix, dim=0, out=out[1])
+
+    # Add a small epsilon for numerical stability and to remove
+    # data-dependent branching, making it compatible with torch.dynamo.
+    epsilon = 1e-12
+
     # Normalize one of the factors for stability
     if shape[0] < shape[1]:
         scale = out[0].sum()
-        if scale != 0: out[0].div_(scale)
+        out[0].div_(scale + epsilon)
     else:
         scale = out[1].sum()
-        if scale != 0: out[1].div_(scale)
+        out[1].div_(scale + epsilon)

adv_optm/util/One_Bit_Boolean.py

@@ -19,4 +19,4 @@ def _unpack_bools(packed_tensor: torch.Tensor, original_m: int) -> torch.Tensor:
     shifter = (2**torch.arange(8, device=packed_tensor.device, dtype=torch.uint8)).view(1, 1, 8)
     unpacked_padded = (packed_tensor.unsqueeze(2) & shifter) != 0
     unpacked = unpacked_padded.view(packed_tensor.shape[0], -1)[:, :original_m]
-    return unpacked
+    return unpacked

adv_optm/util/OrthoGrad.py

@@ -2,15 +2,16 @@ import torch
 
 def _orthogonalize_gradient(p: torch.Tensor, grad: torch.Tensor) -> torch.Tensor:
     """Projects the gradient `grad` to be orthogonal to the parameter `p`."""
-    if grad.is_sparse: raise RuntimeError("OrthoGrad logic does not support sparse gradients.")
+    if grad.is_sparse:
+        raise RuntimeError("OrthoGrad logic does not support sparse gradients.")
     original_shape = grad.shape
     original_dtype = grad.dtype
     w = p.view(-1).float()
     g = grad.view(-1).float()
     w_norm_sq = torch.dot(w, w).add_(1e-30)
     proj = torch.dot(w, g) / w_norm_sq
-    g_orth = g.sub(w, alpha=proj)
+    g_orth = g.sub(w * proj)
     g_norm = g.norm(2)
     g_orth_norm = g_orth.norm(2).add_(1e-30)
     g_orth_scaled = g_orth * (g_norm / g_orth_norm)
-    return g_orth_scaled.view(original_shape).to(original_dtype)
+    return g_orth_scaled.view(original_shape).to(original_dtype)

adv_optm/util/__init__.py

@@ -10,4 +10,4 @@ __all__ = [
     "_get_effective_shape",
     "_orthogonalize_gradient",
     "_newton_schulz_iteration",
-]
+]

{adv_optm-1.2.dev19.dist-info → adv_optm-2.dev3.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: adv_optm
-Version: 1.2.dev19
+Version: 2.dev3
 Summary: A family of highly efficient, lightweight yet powerful optimizers.
 Home-page: https://github.com/Koratahiu/Advanced_Optimizers
 Author: Koratahiu
@@ -52,7 +52,7 @@ This library integrates multiple state-of-the-art optimization techniques valida
 ### **Memory-Efficient Optimization (SMMF-inspired)**
 - **Paper**: [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)
 - **Approach**: Uses rank-1 non-negative matrix factorization with reconstruction cycle (factor → reconstruct → update → factor)
-- **Innovation**: 
+- **Innovation**:
   - First moment split into **1-bit sign + absolute value**
   - Final storage: **four factored vectors + one 1-bit sign state**
   - Preserves Adam-like update quality with drastically reduced memory
@@ -110,7 +110,7 @@ This library integrates multiple state-of-the-art optimization techniques valida
 
 ## 🛠️ Comprehensive Feature Guide
 
-### A. Universal Safe Features  
+### A. Universal Safe Features
 *These features work with all optimizers and are generally safe to enable.*
 
 | Feature | Description | Recommended Usage | Performance Impact | Theoretical Basis | Compatibility |
@@ -165,7 +165,7 @@ This library integrates multiple state-of-the-art optimization techniques valida
 | `beta1` | 0.99 | Controls accumulator memory length:<br>• Small BS: **0.99–0.9999**<br>• Large BS: **0.9** |
 | `Grad α` | 100 | Most critical parameter:<br>• Inversely scales with batch size<br>• **100–10** for small BS (≤32)<br>• **1–0.1** for large BS (≥512) |
 
-> ⚠️ **Critical**: Requires **~100x smaller learning rate** than AdamW (e.g., 1e-6 vs 1e-4).  
+> ⚠️ **Critical**: Requires **~100x smaller learning rate** than AdamW (e.g., 1e-6 vs 1e-4).
 > For `Prodigy_Adv`, set `initial_d` to:
 > - **LoRA**: `1e-8`
 > - **Full FT**: `1e-10`
@@ -175,10 +175,10 @@ This library integrates multiple state-of-the-art optimization techniques valida
 
 #### Performance Validation
 
-**Small Batch Training (SDXL, BS=2, 1.8K steps)**  
+**Small Batch Training (SDXL, BS=2, 1.8K steps)**
 ![Training Comparison](https://github.com/user-attachments/assets/7eff0671-cc59-47fc-8b63-d5205456d649)
 
-- **🟢 Prodigy_Adv** (beta1=0.9, d0=1e-5): Final LR = 2.9e-4  
+- **🟢 Prodigy_Adv** (beta1=0.9, d0=1e-5): Final LR = 2.9e-4
 - **🔵 Prodigy_Adv + Simplified_AdEMAMix** (beta1=0.99, α=100, d0=1e-7): Final LR = 5.8e-6
 
 **Results**:
@@ -202,8 +202,8 @@ This library integrates multiple state-of-the-art optimization techniques valida
 
 Instead of using a fixed β₂ (e.g., 0.999 or 0.95), it **dynamically modulates β₂ per layer** based on a bounded *sunspike ratio*:
 
-- **During gradient bursts** → β₂ ↓ toward `Lower β₂` → faster reaction  
-- **During calm phases** → β₂ ↑ toward `The Selected β₂` → stronger smoothing  
+- **During gradient bursts** → β₂ ↓ toward `Lower β₂` → faster reaction
+- **During calm phases** → β₂ ↑ toward `The Selected β₂` → stronger smoothing
 
 This is especially effective for **noisy training, small batch sizes, and high learning rates**, where gradient norms shift abruptly due to noise or aggressive LR schedules.
 
@@ -220,17 +220,17 @@ This is especially effective for **noisy training, small batch sizes, and high l
 
 #### 📊 Performance Validation
 
-**ADAMW_ADV - full SDXL finetuning (aggressive LR: 3e-5) (BS=4, 2.5K steps)**  
+**ADAMW_ADV - full SDXL finetuning (aggressive LR: 3e-5) (BS=4, 2.5K steps)**
 <img width="1460" height="382" alt="image" src="https://github.com/user-attachments/assets/007f278a-fbac-4f3d-9cc7-274c3b959cdd" />
 
-- 🟣 Fixed `beta2=0.999`  
-- 🟠 Auto K-beta  
+- 🟣 Fixed `beta2=0.999`
+- 🟠 Auto K-beta
 
-**Observations:**  
+**Observations:**
 - K-beta is clearly better and more robust/stable for high LRs.
 
-> 📚 **Reference**:  
-> - Paper: [Kourkoutas-β: A Sunspike-Driven Adam Optimizer with Desert Flair](https://arxiv.org/abs/2508.12996)  
+> 📚 **Reference**:
+> - Paper: [Kourkoutas-β: A Sunspike-Driven Adam Optimizer with Desert Flair](https://arxiv.org/abs/2508.12996)
 > - Code: [kbeta](https://github.com/sck-at-ucy/kbeta)
 
 ---
@@ -258,7 +258,7 @@ settings:
   - factored: False  # Can be true or false, quality should not degrade due to Simplified_AdEMAMix’s high tolerance to 1-bit factorization.
 ```
 
-> ✅ **Why it works**:  
+> ✅ **Why it works**:
 > - `Kourkoutas-β` handles beta2 values
 > - `Simplified_AdEMAMix` ensures responsiveness in small-batch noise
 > - `OrthoGrad` prevents overfitting without weight decay
@@ -267,9 +267,9 @@ settings:
 
 ## 📚 References
 
-1. [Revisiting BFloat16 Training](https://arxiv.org/abs/2010.06192)  
-2. [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)  
-3. [The AdEMAMix Optimizer](https://arxiv.org/abs/2409.03137)  
-4. [Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD](https://arxiv.org/abs/2502.02431)  
-5. [AdaMeM: Memory Efficient Momentum for Adafactor](https://openreview.net/forum?id=fZqMVTz7K5)  
+1. [Revisiting BFloat16 Training](https://arxiv.org/abs/2010.06192)
+2. [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)
+3. [The AdEMAMix Optimizer](https://arxiv.org/abs/2409.03137)
+4. [Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD](https://arxiv.org/abs/2502.02431)
+5. [AdaMeM: Memory Efficient Momentum for Adafactor](https://openreview.net/forum?id=fZqMVTz7K5)
 6. [Kourkoutas-β: A Sunspike-Driven Adam Optimizer with Desert Flair](https://arxiv.org/abs/2508.12996)

adv_optm-2.dev3.dist-info/RECORD

@@ -0,0 +1,23 @@
+adv_optm/__init__.py,sha256=5x-lSBvBDU_bD2E4mS2a4b2ElrfIgg9kHmQdBUwghbk,379
+adv_optm/optim/AdaMuon_adv.py,sha256=zjZHFS7ng5KwemQzePjFiGtNZlcgbzmmnqF6A80h_Tg,34652
+adv_optm/optim/AdamW_adv.py,sha256=VC6NpR9lDaRS6CIDIWdEXE_-2Z1opa0lXCxYZy8FEEI,18242
+adv_optm/optim/Adopt_adv.py,sha256=FRYaqCyxzxUzt1geQj00WCWX0_71_8-cQyVNXaZeVBU,21898
+adv_optm/optim/Lion_Prodigy_adv.py,sha256=ZRld1lt2wQtMBideFA-FStStvfV_oEMCrswww5rYAso,14103
+adv_optm/optim/Lion_adv.py,sha256=GNkuFIwIjKwQElXjVbwjfwhe4lv4D_Qb0gbOjHl151g,8452
+adv_optm/optim/Muon_adv.py,sha256=d91wvmKKt_3IPqsqK1ZZ5cY71kuXyzy04IU3krn2NQ8,33316
+adv_optm/optim/Prodigy_adv.py,sha256=jY7zEWJ49ICqBERFf1fue126sZg0-o1Mu7M9pa_66Gs,26529
+adv_optm/optim/Simplified_AdEMAMix.py,sha256=4JGaX6DDm0zdY8NxXzRIGm4pqb33on8Xw-uImxO3WNE,14399
+adv_optm/optim/__init__.py,sha256=F4f-D8QGIByXHAZAu0keJf4foA22NpK-L9QgywVxAm8,491
+adv_optm/util/BF16_Stochastic_Rounding.py,sha256=b8bE7xGtJxZnQYCqdPKtYb8xYGrDftO6jCLLKLa9Ut8,1550
+adv_optm/util/Effective_Shape.py,sha256=h9pF4HaCkjDyo2dxlUpM66oD6FtclQnb7yPPfvReHyI,320
+adv_optm/util/Kourkoutas.py,sha256=8Lik30MACDwM77aNWmMecmPS9g31fT4jE6fuIG4QMTk,7366
+adv_optm/util/NNMF.py,sha256=hrvNGERj8evhPIWnWzsKdm5DwIZblTB4pkhc9xWytSY,794
+adv_optm/util/Newton_Schulz.py,sha256=bBboYw_jm5_FMf0Citl79uqNedkHOTjQnUI7rZgLBmY,3341
+adv_optm/util/One_Bit_Boolean.py,sha256=tE8lSnbKR3oO-EtM0Kzvf0E4hmuBvhmtFR_75su-DNI,1070
+adv_optm/util/OrthoGrad.py,sha256=doP667YpdiEdP3-cpyWiRNkAdkT-nzs45VSafOCRDHw,713
+adv_optm/util/__init__.py,sha256=cA5zt5dvznkOw2lqbaGvFjslznB1UEFYYZMMFsXrWBg,437
+adv_optm-2.dev3.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+adv_optm-2.dev3.dist-info/METADATA,sha256=ttkFBXVB97D9Fi3_AgO2bA9b-x-9sm0YSKujVtSLuBU,13983
+adv_optm-2.dev3.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+adv_optm-2.dev3.dist-info/top_level.txt,sha256=iNfBIIzu-lPrQ7jyC56WBCcbkRwitM2nJ15-MRQ_6fg,9
+adv_optm-2.dev3.dist-info/RECORD,,

adv_optm-1.2.dev19.dist-info/RECORD

@@ -1,23 +0,0 @@
-adv_optm/__init__.py,sha256=1AKxG--scx5Bl9G08tQcnfzAMaQVSgmW99uy3v2QWMw,380
-adv_optm/optim/AdaMuon_adv.py,sha256=7Had92OcsCiN1E9UJRyrpPV7VzHqmIvS-qM6OEcc24I,34671
-adv_optm/optim/AdamW_adv.py,sha256=jgMuRAfsnUh_2wUEZgYpJX5uwoT_kQjtMs2Xn2vJ3x0,17480
-adv_optm/optim/Adopt_adv.py,sha256=kbAeBG4bXWBvgj_qrE9W67J6c0swpEi4Erj2rfYrMXE,21252
-adv_optm/optim/Lion_Prodigy_adv.py,sha256=LEA3UYJpPeFnmxeniLNv1u2LKKj4ufx3Bq_MLw-nWXk,14617
-adv_optm/optim/Lion_adv.py,sha256=aGNAplZlyXYgVllYcV_s4bK8iC4fv6EizFoWIMNLdBc,8299
-adv_optm/optim/Muon_adv.py,sha256=tZY8K3pNBCGk1V09GbK05lJooFw92NfkF7_T548up3Q,33171
-adv_optm/optim/Prodigy_adv.py,sha256=k7f2J_RQpnrUXjwER_XOokISlQWpTSwGG-OL-bjMfBk,26061
-adv_optm/optim/Simplified_AdEMAMix.py,sha256=nEIA3yM11nBooKzHudB5l3x4UdFRBYRwiKVUkGmO0K8,12971
-adv_optm/optim/__init__.py,sha256=hpUWE6CKtt_rvMdgQVb3PtjhfZAvAxTq6hp8H8rIpBo,489
-adv_optm/util/BF16_Stochastic_Rounding.py,sha256=Q5H0BcogmE4atP65dLoI21HKSf50lRdsBDfeF6v9Tbg,1548
-adv_optm/util/Effective_Shape.py,sha256=TBvIk1V8IuTbbBsxuekJA4e_v8JlR5Nujtut8RTWAm4,318
-adv_optm/util/Kourkoutas.py,sha256=BnBj4WlTOJXOW0dv_vBBE27HxDTbI_1qDIWW2J7Bxdo,7644
-adv_optm/util/NNMF.py,sha256=yRf5IP5Sjq0Uf0DxN0Q8NxEGSdD-f1ULziLVDOjY8K4,639
-adv_optm/util/Newton_Schulz.py,sha256=bBboYw_jm5_FMf0Citl79uqNedkHOTjQnUI7rZgLBmY,3341
-adv_optm/util/One_Bit_Boolean.py,sha256=Wat49esdwohuN-OHOFMW8D0aOQgV9cP5Rl8z6yfmpos,1068
-adv_optm/util/OrthoGrad.py,sha256=NzInuBQGy_Ja__M1R9XbvqVaQ0fhGbtGgFE9YON7B3I,707
-adv_optm/util/__init__.py,sha256=CXzS703GB4gil85khZi7sgKOnbzXGBOltshIOSPqj18,435
-adv_optm-1.2.dev19.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-adv_optm-1.2.dev19.dist-info/METADATA,sha256=pQm5WuMKvf5Xse10viziVK9ry1UufcYRDwOd55jad8Y,14023
-adv_optm-1.2.dev19.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-adv_optm-1.2.dev19.dist-info/top_level.txt,sha256=iNfBIIzu-lPrQ7jyC56WBCcbkRwitM2nJ15-MRQ_6fg,9
-adv_optm-1.2.dev19.dist-info/RECORD,,