adv-optm 1.1.0.dev4__py3-none-any.whl → 1.1.1__py3-none-any.whl

adv_optm/__init__.py

@@ -16,4 +16,4 @@ __all__ = [
     "Lion_Prodigy_adv",
 ]
 
-__version__ = "1.1.0.dev4"
+__version__ = "1.1.1"

adv_optm/optim/AdamW_adv.py

@@ -10,7 +10,7 @@ from ..util.Kourkoutas import KourkoutasHelper
 
 class AdamW_adv(torch.optim.Optimizer):
     """
-    Implements a factored AdamW algorithm.
+    Implements an advanced AdamW algorithm.
     This is an advanced version of AdamW with optional features like
     low-rank factorization of optimizer states (SMMF), OrthoGrad, etc.
 
@@ -67,7 +67,7 @@ class AdamW_adv(torch.optim.Optimizer):
             "sunspike" ratio calculation to prevent division by zero. Corresponds
             to `ε_spike` in the paper. (default: 1e-9)
         k_warmup_steps (int): The number of initial steps during which β₂ is held
-            at a fixed average value (`(beta2_min + beta2_max) / 2`) before the
+            at a fixed beta2 value before the
             dynamic logic activates. (default: 0)
         k_logging (int): if > 0 and kourkoutas_beta=True, enables periodic console
             logging of Kourkoutas-β statistics (min, max, mean of `β₂` across layers)
@@ -100,8 +100,8 @@ class AdamW_adv(torch.optim.Optimizer):
         alpha: float = 5.0,
         t_alpha: int | None = None,
         kourkoutas_beta: bool = False,
-        beta2_min: float = 0.88,
-        ema_alpha: float = 0.93,
+        beta2_min: float = 0.9,
+        ema_alpha: float = 0.95,
         tiny_spike: float = 1e-9,
         k_warmup_steps: int = 0,
         k_logging: int = 0,
@@ -167,7 +167,7 @@ class AdamW_adv(torch.optim.Optimizer):
         state = self.state[p]
 
         # State Initialization
-        if len(state) == 0:
+        if 'step' not in state:
             state['step'] = 0
 
             should_factor = (

adv_optm/optim/Adopt_adv.py

@@ -10,7 +10,7 @@ from ..util.Kourkoutas import KourkoutasHelper
 
 class Adopt_adv(torch.optim.Optimizer):
     """
-    Implements a fusion of SMMF, and the ADOPT algorithm.
+    Implements an advanced ADOPT algorithm.
 
     The ADOPT update rule modifies Adam by:
     1.  **Initialization:** The second moment `v` is initialized as `v₀ = g₀²`.
@@ -85,7 +85,7 @@ class Adopt_adv(torch.optim.Optimizer):
             "sunspike" ratio calculation to prevent division by zero. Corresponds
             to `ε_spike` in the paper. (default: 1e-9)
         k_warmup_steps (int): The number of initial steps during which β₂ is held
-            at a fixed average value (`(beta2_min + beta2_max) / 2`) before the
+            at a fixed beta2 value before the
             dynamic logic activates. (default: 0)
         k_logging (int): if > 0 and kourkoutas_beta=True, enables periodic console
             logging of Kourkoutas-β statistics (min, max, mean of `β₂` across layers)
@@ -120,8 +120,8 @@ class Adopt_adv(torch.optim.Optimizer):
         Simplified_AdEMAMix: bool = False,
         alpha_grad: float = 100.0,
         kourkoutas_beta: bool = False,
-        beta2_min: float = 0.88,
-        ema_alpha: float = 0.93,
+        beta2_min: float = 0.9,
+        ema_alpha: float = 0.95,
         tiny_spike: float = 1e-9,
         k_warmup_steps: int = 0,
         k_logging: int = 0,
@@ -195,7 +195,7 @@ class Adopt_adv(torch.optim.Optimizer):
         state = self.state[p]
 
         # State Initialization
-        if len(state) == 0:
+        if 'step' not in state:
             state['step'] = 0
 
             should_factor = (

adv_optm/optim/Lion_Prodigy_adv.py

@@ -151,7 +151,7 @@ class Lion_Prodigy_adv(torch.optim.Optimizer):
         state = self.state[p]
 
         # State Initialization
-        if len(state) == 0:
+        if 'step' not in state:
             state['step'] = 0
 
             should_factor = (

adv_optm/optim/Lion_adv.py

@@ -99,7 +99,7 @@ class Lion_adv(torch.optim.Optimizer):
         state = self.state[p]
 
         # State Initialization
-        if len(state) == 0:
+        if 'step' not in state:
             state['step'] = 0
 
             should_factor = (

adv_optm/optim/Prodigy_adv.py

@@ -14,7 +14,7 @@ from ..util.Kourkoutas import KourkoutasHelper
 
 class Prodigy_adv(torch.optim.Optimizer):
     """
-    Implements a factored Prodigy/AdamW algorithm.
+    Implements an advanced Prodigy algorithm.
     This is an advanced version of Prodigy with optional features like
     low-rank factorization of optimizer states (SMMF), OrthoGrad, etc.
 
@@ -88,6 +88,9 @@ class Prodigy_adv(torch.optim.Optimizer):
         prodigy_steps (int): If greater than zero, disable Prodigy's stepsize adjustments
             after the specified optimiser step and release all state memory required by Prodigy
             (default: 0).
+        d_limiter (bool): whether to clamp the new step size estimate (`d_hat`)
+            to prevent sudden, volatile increases in the adaptive step size (`d`).
+            (default: False)
         kourkoutas_beta (bool): whether to enable the layer-wise dynamic β₂ logic.
             If `False`, the optimizer behaves as standard AdamW/Prodigy. (default: False)
         beta2_min (float): The minimum value for dynamic β₂, used during periods of
@@ -100,7 +103,7 @@ class Prodigy_adv(torch.optim.Optimizer):
             "sunspike" ratio calculation to prevent division by zero. Corresponds
             to `ε_spike` in the paper. (default: 1e-9)
         k_warmup_steps (int): The number of initial steps during which β₂ is held
-            at a fixed average value (`(beta2_min + beta2_max) / 2`) before the
+            at a fixed beta2 value before the
             dynamic logic activates. (default: 0)
         k_logging (int): if > 0 and kourkoutas_beta=True, enables periodic console
             logging of Kourkoutas-β statistics (min, max, mean of `β₂` across layers)
@@ -141,9 +144,11 @@ class Prodigy_adv(torch.optim.Optimizer):
         fsdp_in_use: bool = False,
         slice_p: int = 11,
         prodigy_steps: int = 0,
+        d_limiter: bool = False,
+        # K-b parameters
         kourkoutas_beta: bool = False,
-        beta2_min: float = 0.88,
-        ema_alpha: float = 0.93,
+        beta2_min: float = 0.9,
+        ema_alpha: float = 0.95,
         tiny_spike: float = 1e-9,
         k_warmup_steps: int = 0,
         k_logging: int = 0,
@@ -175,8 +180,8 @@ class Prodigy_adv(torch.optim.Optimizer):
             use_atan2 = False
         if kourkoutas_beta and not (betas[1] > beta2_min):
             raise ValueError(f"For Kourkoutas-β, betas[1] (as beta2_max) must be > beta2_min. Got {betas[1]} and {beta2_min}")
-        if Simplified_AdEMAMix and alpha_grad > 0:
-            # scales d_coef by alpha_grad, this force prodigy to behave well with Simplified_AdEMAMix
+        if Simplified_AdEMAMix and alpha_grad > 0 and not d_limiter:
+            # scales d_coef by alpha_grad, this force prodigy to behave well with Simplified_AdEMAMix.
             d_coef = d_coef/alpha_grad
 
         defaults = {
@@ -186,7 +191,7 @@ class Prodigy_adv(torch.optim.Optimizer):
             "beta3_ema": beta3_ema, "alpha": alpha, "t_alpha": t_alpha,
             "beta3": beta3, "d": d0, "d0": d0, "d_max": d0, "d_numerator": 0.0, "d_coef": d_coef,
             "growth_rate": growth_rate, "safeguard_warmup": safeguard_warmup, "k": 0, "slice_p": slice_p,
-            "fsdp_in_use": fsdp_in_use, "prodigy_steps": prodigy_steps,
+            "fsdp_in_use": fsdp_in_use, "prodigy_steps": prodigy_steps, "d_limiter": d_limiter,
             "alpha_grad": alpha_grad,
             "kourkoutas_beta": kourkoutas_beta, "beta2_min": beta2_min, "ema_alpha": ema_alpha,
             "tiny_spike": tiny_spike, "k_warmup_steps": k_warmup_steps, "k_logging": k_logging,
@@ -251,7 +256,7 @@ class Prodigy_adv(torch.optim.Optimizer):
         state = self.state[p]
 
         # State Initialization
-        if len(state) == 0:
+        if 'step' not in state:
             state['step'] = 0
 
             should_factor = (
@@ -306,10 +311,8 @@ class Prodigy_adv(torch.optim.Optimizer):
             self.kourkoutas_helper.accumulate_gradient_sq_norm(p, grad)
             # Get the dynamic beta2 calculated in prepare_step()
             beta2 = self.kourkoutas_helper.get_beta2(p, group, current_step)
-            beta3 = math.sqrt(beta2)
         else:
             beta2 = self.beta2_default
-            beta3 = self.beta3
 
         if self.use_AdEMAMix:
             beta3_ema = group['beta3_ema']
@@ -446,7 +449,7 @@ class Prodigy_adv(torch.optim.Optimizer):
             self.d_numerator += (self.d / d0) * self.dlr * torch.dot(grad_flat[::slice_p], p0.data - p_flat[::slice_p]).item()
 
             alpha = ((self.d / d0) * self.d) if safeguard_warmup else ((self.d / d0) * self.dlr)
-            s.mul_(beta3).add_(grad_flat[::slice_p], alpha=alpha)
+            s.mul_(self.beta3).add_(grad_flat[::slice_p], alpha=alpha)
             self.d_denom += s.abs().sum().item()
 
             del s, p0, grad_flat, p_flat, alpha
@@ -512,6 +515,8 @@ class Prodigy_adv(torch.optim.Optimizer):
             d_hat = self.d
             if global_d_denom > 0:
                 d_hat = d_coef * global_d_numerator / global_d_denom
+                if g_group['d_limiter']:
+                    d_hat = min(self.d * (2 ** 0.25), d_hat)
                 if self.d == g_group['d0']:
                     self.d = max(self.d, d_hat)
                 d_max = max(d_max, d_hat)

adv_optm/optim/Simplified_AdEMAMix.py

@@ -61,7 +61,7 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
             "sunspike" ratio calculation to prevent division by zero. Corresponds
             to `ε_spike` in the paper. (default: 1e-9)
         k_warmup_steps (int): The number of initial steps during which β₂ is held
-            at a fixed average value (`(beta2_min + beta2_max) / 2`) before the
+            at a fixed beta2 value before the
             dynamic logic activates. (default: 0)
         k_logging (int): if > 0 and kourkoutas_beta=True, enables periodic console
             logging of Kourkoutas-β statistics (min, max, mean of `β₂` across layers)
@@ -90,8 +90,8 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
         stochastic_rounding: bool = True,
         orthogonal_gradient: bool = False,
         kourkoutas_beta: bool = False,
-        beta2_min: float = 0.88,
-        ema_alpha: float = 0.93,
+        beta2_min: float = 0.9,
+        ema_alpha: float = 0.95,
         tiny_spike: float = 1e-9,
         k_warmup_steps: int = 0,
         k_logging: int = 0,
@@ -152,7 +152,7 @@ class Simplified_AdEMAMix(torch.optim.Optimizer):
         state = self.state[p]
 
         # State Initialization
-        if len(state) == 0:
+        if 'step' not in state:
             state['step'] = 0
 
             should_factor = (

adv_optm/util/Kourkoutas.py

@@ -11,9 +11,8 @@ class KourkoutasHelper:
         if not hasattr(optimizer, 'param_groups'):
             raise TypeError("optimizer must be a valid torch.optim.Optimizer instance.")
         self.optimizer = optimizer
-
-        # State managed by the helper
         self.layer_state = {}
+
         self.layer_info = {}
         self._layer_info_built = False
         self._current_step_prepared = -1
@@ -25,14 +24,24 @@ class KourkoutasHelper:
         # making it compatible with fused back pass mechanisms.
         self._build_layer_info_if_needed()
 
+        if self.optimizer.param_groups[0].get('k_logging', 0) > 0:
+            self.print_layer_info()
+
     def _build_layer_info_if_needed(self):
         """Builds a map of layers and the parameters they contain."""
         if self._layer_info_built:
             return
 
-        if not hasattr(self.optimizer, 'layer_key_fn') or self.optimizer.layer_key_fn is None:
-            print("Warning: KourkoutasHelper requires 'layer_key_fn' on the optimizer. Defaulting to tensor-wise (id).")
-            self.optimizer.layer_key_fn = lambda p: id(p)
+        if hasattr(self.optimizer, 'layer_key_fn') and self.optimizer.layer_key_fn is not None:
+            # A custom key function was provided by the user. We will use it.
+            pass
+        else:
+            # No key function was provided. Default to coarse, shape-based bucketing.
+            self.optimizer.layer_key_fn = lambda p: \
+                (id(p),) if p.dim() == 2 and 1 <= p.shape[0] <= 10 and p.shape[1] in {768, 1280, 4096} \
+                else tuple(p.shape)
+            # This ensures that we won't mix embeddings with tokens (1 to 10)
+            # TODO find a better way to safeguard the embeddings
 
         for group in self.optimizer.param_groups:
             for p in group['params']:
@@ -48,6 +57,24 @@ class KourkoutasHelper:
 
         self._layer_info_built = True
 
+    def print_layer_info(self):
+        """Prints the contents of self.layer_info for debugging."""
+        print("\n--- BEGIN self.layer_info DUMP ---")
+        if not self.layer_info:
+            print("Layer info is empty. Make sure the optimizer has parameters.")
+            return
+
+        for layer_key, info in self.layer_info.items():
+            param_count = len(info['params'])
+            first_param_details = ""
+            if param_count > 0:
+                p = info['params'][0]
+                first_param_details = f" (Example param shape: {list(p.shape)}, dtype: {p.dtype})"
+            
+            print(f"Key: {layer_key}, Params: {param_count}{first_param_details}")
+
+        print("--- END self.layer_info DUMP ---\n")
+
     def prepare_step(self, current_step: int):
         """
         Calculates dynamic beta2 for all layers using the completed scalar accumulators
@@ -55,44 +82,50 @@ class KourkoutasHelper:
         """
         
         beta2_log = []
+        first_layer_key = next(iter(self.layer_info), None)
         # These are just for the sample log, initialize them
-        sun, pooled_grad_norm, r_ema = (torch.tensor(0.0),)*3
-
+        sun, pooled_grad_norm, prev_r_ema_val, r_ema_tensor = (torch.tensor(0.0),)*4
 
         for layer_key, info in self.layer_info.items():
             params, group = info['params'], info['group_ref']
-            
+
+            first_param_in_layer = info['params'][0]
+            param_state = self.optimizer.state[first_param_in_layer]
+
             if layer_key not in self.layer_state:
                 self.layer_state[layer_key] = {
-                    'r_ema_grad_norm': torch.tensor(0.0, device=params[0].device, dtype=torch.float32),
-                    'sum_sq_accumulator': torch.tensor(0.0, device=params[0].device, dtype=torch.float32)
+                    'sum_sq_accumulator': torch.tensor(0.0, device=first_param_in_layer.device, dtype=torch.float32)
                 }
             
-            layer_state = self.layer_state[layer_key]
-
-            # Use the completed accumulator from the previous step
-            pooled_grad_norm = torch.sqrt(layer_state['sum_sq_accumulator'])
-
-            r_ema = layer_state['r_ema_grad_norm']
+            if 'kourkoutas_r_ema' not in param_state:
+                param_state['kourkoutas_r_ema'] = torch.tensor(0.0, device=first_param_in_layer.device, dtype=torch.float32)
 
-            # EMA is always updated, even during warmup
-            r_ema.mul_(group['ema_alpha']).add_(pooled_grad_norm, alpha=1.0 - group['ema_alpha'])
-
-            sun = torch.tensor(0.0, device=r_ema.device) # Default sun to 0 for warmup
+            r_ema_tensor = param_state['kourkoutas_r_ema']
+            accumulator = self.layer_state[layer_key]['sum_sq_accumulator']
+            
+            pooled_grad_norm = torch.sqrt(accumulator)
+            prev_r_ema_val = r_ema_tensor.item() # for logging
+            
+            # Update the persistent EMA tensor in-place.
+            r_ema_tensor.mul_(group['ema_alpha']).add_(pooled_grad_norm, alpha=1.0 - group['ema_alpha'])
+            
             beta2_max = group['betas'][1]
-
-            # --- CONSOLIDATED WARMUP LOGIC ---
+            sun = torch.tensor(0.0, device=r_ema_tensor.device) # Default sun to 0 for warmup
+            
             if current_step < group['k_warmup_steps']:
                 beta2 = beta2_max
             else:
-                raw = pooled_grad_norm / (r_ema + group['tiny_spike'])
+                raw = pooled_grad_norm / (r_ema_tensor + group['tiny_spike'])
                 sun = raw / (1.0 + raw)
                 beta2 = beta2_max - (beta2_max - group['beta2_min']) * sun
 
-            layer_state['dynamic_beta2'] = beta2.item() if isinstance(beta2, torch.Tensor) else beta2
-            layer_state['sum_sq_accumulator'].zero_()
+            # Store the final calculated beta2 in the helper's transient state for this step.
+            self.layer_state[layer_key]['dynamic_beta2'] = beta2.item() if isinstance(beta2, torch.Tensor) else beta2
+            
+            # Reset the accumulator for the next optimizer step.
+            accumulator.zero_()
 
-            beta2_log.append(layer_state['dynamic_beta2'])
+            beta2_log.append(self.layer_state[layer_key]['dynamic_beta2'])
 
         # Always compute stats for TensorBoard
         if beta2_log:
@@ -107,9 +140,12 @@ class KourkoutasHelper:
         k_logging_interval = self.optimizer.param_groups[0].get('k_logging', 0)
         is_logging_step = k_logging_interval > 0 and (current_step + 1) % k_logging_interval == 0
         if is_logging_step and self.last_beta2_stats:
+            if first_layer_key:
+                print(f"\n[Kourkoutas-β Debug] Step {current_step + 1} - Sample Layer '{first_layer_key}':")
+                print(f"  - Grad Norm: {pooled_grad_norm.item():.4e}, Prev EMA: {prev_r_ema_val:.4e}, New EMA: {r_ema_tensor.item():.4e}")
+                print(f"  - Sunspike: {sun.item():.4f}, Dynamic Beta2: {self.layer_state[first_layer_key]['dynamic_beta2']:.4f}")
             print(f"[Kourkoutas-β Debug] Step {current_step + 1} Overall Beta2 Stats: Min={self.last_beta2_stats['min']:.4f}, Max={self.last_beta2_stats['max']:.4f}, Mean={self.last_beta2_stats['mean']:.4f}")
 
-
     def maybe_prepare_step(self, current_step: int):
         """
         A universal guard that calls prepare_step() exactly once per training step.
@@ -125,9 +161,9 @@ class KourkoutasHelper:
         layer_key = self.optimizer.layer_key_fn(p)
 
         if layer_key in self.layer_info:
+            # Initialize the transient state for this layer if it's the first time in the step.
             if layer_key not in self.layer_state:
                     self.layer_state[layer_key] = {
-                    'r_ema_grad_norm': torch.tensor(0.0, device=p.device, dtype=torch.float32),
                     'sum_sq_accumulator': torch.tensor(0.0, device=p.device, dtype=torch.float32)
                 }
             # Accumulate for the *next* step's prepare_step call

adv_optm-1.1.1.dist-info/METADATA

@@ -0,0 +1,275 @@
+Metadata-Version: 2.4
+Name: adv_optm
+Version: 1.1.1
+Summary: A family of highly efficient, lightweight yet powerful optimizers.
+Home-page: https://github.com/Koratahiu/Advanced_Optimizers
+Author: Koratahiu
+Author-email: hiuhonor@gmail.com
+License: Apache 2.0
+Keywords: llm,fine-tuning,memory-efficient,low-rank,compression,pytorch,optimizer,adam
+Classifier: Programming Language :: Python :: 3
+Classifier: License :: OSI Approved :: Apache Software License
+Classifier: Operating System :: OS Independent
+Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
+Classifier: Topic :: Software Development :: Libraries :: Python Modules
+Requires-Python: >=3.8
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: torch>=2.0
+Dynamic: author
+Dynamic: author-email
+Dynamic: classifier
+Dynamic: description
+Dynamic: description-content-type
+Dynamic: home-page
+Dynamic: keywords
+Dynamic: license
+Dynamic: license-file
+Dynamic: requires-dist
+Dynamic: requires-python
+Dynamic: summary
+
+# Advanced Optimizers (AIO)
+
+A comprehensive, all-in-one collection of optimization algorithms for deep learning, designed for **maximum efficiency**, **minimal memory footprint**, and **superior performance** across diverse model architectures and training scenarios.
+
+[![PyPI](https://img.shields.io/pypi/v/adv_optm)](https://pypi.org/project/adv_optm/)
+
+---
+
+## 📦 Installation
+
+```bash
+pip install adv_optm
+```
+
+---
+
+## 🧠 Core Innovations
+
+This library integrates multiple state-of-the-art optimization techniques validated through extensive research and practical training, with **1-bit compression for optimizer states**:
+
+### **Memory-Efficient Optimization (SMMF-inspired)**
+- **Paper**: [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)
+- **Approach**: Uses rank-1 non-negative matrix factorization with reconstruction cycle (factor → reconstruct → update → factor)
+- **Innovation**: 
+  - First moment split into **1-bit sign + absolute value**
+  - Final storage: **four factored vectors + one 1-bit sign state**
+  - Preserves Adam-like update quality with drastically reduced memory
+
+---
+
+## ⚡ Performance Characteristics
+
+### Memory Efficiency (SDXL Model – 6.5GB)
+| Optimizer | Memory Usage | Description |
+|-----------|--------------|-------------|
+| `Adopt_Factored` | 328 MB | 4 small vectors + 1-bit state |
+| `Adopt_Factored + AdEMAMix` | 625 MB | 6 small vectors + two 1-bit states |
+| `Simplified_AdEMAMix` | 328 MB | Same as standard factored (no extra state) |
+
+### Speed Comparison (SDXL, Batch Size 4)
+| Optimizer | Speed | Notes |
+|-----------|-------|-------|
+| `Adafactor` | ~8.5s/it | Baseline |
+| `Adopt_Factored` | ~10s/it | +18% overhead from compression |
+| `Adopt_Factored + AdEMAMix` | ~12s/it | +41% overhead (3 factored states) |
+
+---
+
+## 🧪 Available Optimizers
+
+### Standard Optimizers (All support `factored=True/False`)
+| Optimizer | Description | Best For |
+|-----------|-------------|----------|
+| `Adam_Adv` | Advanced Adam implementation | General purpose |
+| `Adopt_Adv` | Adam-variant with independent beta2 | Stable training for small batch size regimes |
+| `Prodigy_Adv` | Prodigy with D-Adaptation | Adam with automatic LR tuning |
+| `Simplified_AdEMAMix` | Adam variant with accumulator momentum | Small/large batch training when tuned correctly |
+| `Lion_Adv` | Advanced Lion implementation | Memory-constrained environments |
+| `Prodigy_Lion_Adv` | Prodigy + Lion combination | Lion with automatic LR tuning |
+
+---
+
+## ⚙️ Feature Matrix
+
+| Feature | Adam_Adv | Adopt_Adv | Prodigy_Adv | Simplified_AdEMAMix | Lion_Adv |
+|---------|----------|-----------|-------------|---------------------|----------|
+| Factored | ✓ | ✓ | ✓ | ✓ | ✓ |
+| AdEMAMix | ✓ | ✓ | ✓ | ✗ | ✗ |
+| Simplified_AdEMAMix | ✗ | ✓ | ✓ | ✓ | ✗ |
+| OrthoGrad | ✓ | ✓ | ✓ | ✓ | ✓ |
+| Grams | ✓ | ✓ | ✓ | ✗ | ✗ |
+| Cautious | ✓ | ✓ | ✓ | ✗ | ✓ |
+| atan2 | ✓ | ✓ | ✓ | ✗ | ✗ |
+| Stochastic Rounding | ✓ | ✓ | ✓ | ✓ | ✓ |
+| Fused Backward Pass | ✓ | ✓ | ✓ | ✓ | ✓ |
+| **Kourkoutas-β** | ✓ | ✓ | ✓ | ✓ | ✗ |
+
+---
+
+## 🛠️ Comprehensive Feature Guide
+
+### A. Universal Safe Features  
+*These features work with all optimizers and are generally safe to enable.*
+
+| Feature | Description | Recommended Usage | Performance Impact | Theoretical Basis | Compatibility |
+|--------|-------------|-------------------|--------------------|-------------------|--------------|
+| **Fused Back Pass** | Fuses backward pass; gradients used immediately and memory freed on-the-fly | Memory-constrained environments | Reduces peak memory | Memory optimization | All optimizers |
+| **Stochastic Rounding** | Replaces nearest rounding with stochastic rounding to preserve small gradient updates in BF16 | BF16 training | Minimal overhead (<5%) | [Revisiting BFloat16 Training](https://arxiv.org/abs/2010.06192) | All optimizers |
+| **OrthoGrad** | Removes gradient component parallel to weights to reduce overfitting | Full fine-tuning without weight decay | +33% time overhead (BS=4); less at larger BS | [Grokking at Edge](https://github.com/LucasPrietoAl/grokking-at-the-edge-of-numerical-stability) | All optimizers |
+| **Factored** | Memory-efficient optimization via rank-1 1-bit factorization of optimizer states | Large models / memory-limited hardware | Adds compression overhead | [SMMF](https://arxiv.org/abs/2412.08894) | All optimizers |
+
+### B. Individual Features
+
+| Feature | Description | Recommended Usage | Performance Impact | Theoretical Basis | Compatibility |
+|--------|-------------|-------------------|--------------------|-------------------|--------------|
+| **Cautious** | Only applies update if gradient direction aligns with momentum direction | Accelerating convergence | No overhead | [C-Optim](https://github.com/kyleliang919/C-Optim) | Adam/Adopt/Prodigy/Lion |
+| **Grams** | Update direction derived purely from current gradient | When Cautious is insufficient | No overhead | [Grams](https://github.com/Gunale0926/Grams) | Adam/Adopt/Prodigy |
+| **AdEMAMix** | Dual EMA system that retains relevance of gradients over tens of thousands of steps | Long training runs, especially where model forgetting is a concern | +1 state memory | [AdEMAMix](https://arxiv.org/abs/2409.03137) | Adam/Adopt/Prodigy |
+| **Simplified_AdEMAMix** | Accumulator-based momentum, single EMA variant of AdEMAMix | All scenarios when tuned correctly | No overhead | [Connections](https://arxiv.org/abs/2502.02431) | Adam/Adopt/Prodigy |
+| **atan2** | Robust epsilon replacement with built-in gradient clipping | Use for stable bounded updates (or for Adopt as it needs that) | No overhead | [Adam-atan2](https://github.com/lucidrains/adam-atan2-pytorch) | Adam/Adopt/Prodigy |
+| **Kourkoutas-β** | Layer-wise adaptive β₂ based on gradient “sunspike” ratio | Noisy/small/large-batch/high-LR training | No overhead | [Kourkoutas-β]() | Adam/Adopt/Prodigy/Simplified_AdEMAMix |
+
+> **Note**: If both **Cautious** and **Grams** are enabled, **Grams takes precedence** and Cautious is disabled.
+
+---
+
+## 🔍 Feature Deep Dives
+
+### AdEMAMix
+
+- Adds a **slow-decaying second EMA** (`beta3`) that retains gradient memory over tens of thousands of steps.
+- Particularly effective for **small batch sizes**, where Adam’s standard first moment is nearly useless.
+- **Reference**: [AdaMeM: Memory Efficient Momentum for Adafactor](https://openreview.net/forum?id=fZqMVTz7K5)
+
+#### Tunable Hyperparameters
+| Parameter | Default | Tuning Guide |
+|-----------|---------|--------------|
+| `beta3` | 0.9999 | • Runs >120k steps: **0.9999**<br>• Runs ≤120k steps: **0.999** |
+| `alpha` | 5 | • Reduce to **2–3** if diverging<br>• Increase to strengthen long-term memory |
+
+> ✅ **Pro Tip**: Set `beta1=0` in Adam/Adopt/Prodigy to skip standard EMA entirely and rely solely on AdEMAMix’s slow EMA, ideal for small-batch regimes.
+
+---
+
+### Simplified_AdEMAMix
+
+- Introduced in [Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD Variants (arXiv:2502.02431)](https://arxiv.org/abs/2502.02431).
+- Replaces Adam’s first moment with a **gradient accumulator**, combining the stability of long memory with responsiveness to recent gradients.
+- **Key insight**: Classical momentum **does not accelerate** in noisy (small-batch) regimes; this accumulator do.
+
+#### Tunable Hyperparameters
+| Parameter | Default | Tuning Guide |
+|----------|---------|--------------|
+| `beta1` | 0.99 | Controls accumulator memory length:<br>• Small BS: **0.99–0.9999**<br>• Large BS: **0.9** |
+| `Grad α` | 100 | Most critical parameter:<br>• Inversely scales with batch size<br>• **100–10** for small BS (≤32)<br>• **1–0.1** for large BS (≥512) |
+
+> ⚠️ **Critical**: Requires **~100x smaller learning rate** than AdamW (e.g., 1e-6 vs 1e-4).  
+> For `Prodigy_Adv`, set `initial_d` to:
+> - **LoRA**: `1e-8`
+> - **Full FT**: `1e-10`
+> - **Embedding**: `1e-7`
+
+> ⚠️ **Incompatible** with: **Cautious**, **Grams**, **atan2**, and standard gradient clipping.
+
+#### Performance Validation
+
+**Small Batch Training (SDXL, BS=2, 1.8K steps)**  
+![Training Comparison](https://github.com/user-attachments/assets/7eff0671-cc59-47fc-8b63-d5205456d649)
+
+- **🟢 Prodigy_Adv** (beta1=0.9, d0=1e-5): Final LR = 2.9e-4  
+- **🔵 Prodigy_Adv + Simplified_AdEMAMix** (beta1=0.99, α=100, d0=1e-7): Final LR = 5.8e-6
+
+**Results**:
+- Faster convergence and higher final performance with Simplified_AdEMAMix
+- D-Adaptation automatically compensates for aggressive updates
+- Generated samples show **significantly better quality**
+
+---
+
+### atan2
+
+- Replaces `eps` in Adam-family optimizers with a **scale-invariant**, bounded update rule.
+- Automatically clips updates to **[-2, 2]**, preventing destabilizing jumps.
+- **Highly recommended** for `Adopt_Adv`, which is prone to instability without clipping.
+
+---
+
+### **Kourkoutas-β**
+
+**Kourkoutas-β** introduces a **sunspike-driven, layer-wise adaptive second-moment decay (β₂)** as an optional enhancement for `Adam_Adv`, `Adopt_Adv`, `Prodigy_Adv`, and `Simplified_AdEMAMix`.
+
+Instead of using a fixed β₂ (e.g., 0.999 or 0.95), it **dynamically modulates β₂ per layer** based on a bounded *sunspike ratio*:
+
+- **During gradient bursts** → β₂ ↓ toward `Lower β₂` → faster reaction  
+- **During calm phases** → β₂ ↑ toward `The Selected β₂` → stronger smoothing  
+
+This is especially effective for **noisy training, small batch sizes, and high learning rates**, where gradient norms shift abruptly due to noise or aggressive LR schedules.
+
+#### Pros/Cons
+
+| **Category** | **Details** |
+|--------------|-------------|
+| ✅ **Pros** | • **Layer-wise adaptation** blends benefits of high β₂ (strong smoothing) and low β₂ (fast reaction).<br>• **Robust to sudden loss landscape shifts**, reacts quickly during gradient bursts, smooths during calm phases.<br>• **High tolerance to aggressive learning rates**. |
+| ⚠️ **Cons** | • **Potentially unstable at the start of training** due to unreliable early gradient norms; mitigated by using `K-β Warmup Steps`. |
+
+> 💡 **Best Practice**: Set `K_warmup_steps` equal to your standard LR warmup steps. During warmup, the optimizer uses the static `beta2`; adaptation begins only after warmup ends.
+
+> 🔍 **Debugging Aid**: Enable `K_Logging` to monitor (min, max, mean) of dynamic β₂ values across layers every *N* steps.
+
+#### 📊 Performance Validation
+
+**ADAMW_ADV - full SDXL finetuning (aggressive LR: 3e-5) (BS=4, 2.5K steps)**  
+<img width="1460" height="382" alt="image" src="https://github.com/user-attachments/assets/007f278a-fbac-4f3d-9cc7-274c3b959cdd" />
+
+- 🟣 Fixed `beta2=0.999`  
+- 🟠 Auto K-beta  
+
+**Observations:**  
+- K-beta is clearly better and more robust/stable for high LRs.
+
+> 📚 **Reference**:  
+> - Paper: [Kourkoutas-β: A Sunspike-Driven Adam Optimizer with Desert Flair](https://arxiv.org/abs/2508.12996)  
+> - Code: [kbeta](https://github.com/sck-at-ucy/kbeta)
+
+---
+
+## Recommended Preset (Tested on LoRA/FT/Embedding)
+
+```yaml
+Learning Rate: 1
+optimizer: PRODIGY_Adv
+settings:
+  - beta1: 0.99 # Controls momentum decay, ~100-step effective memory. Adjust to 0.999 (1000 steps) or 0.9999 (10000 steps) based on training length and stability needs.
+  - beta2: 0.999
+  - kourkoutas_beta: True   # For Kourkoutas-β
+  - K-β Warmup Steps: 50    # Or 100, 200, depending on your run
+  - Simplified_AdEMAMix: True
+  - Grad α: 100
+  - OrthoGrad: True
+  - weight_decay: 0.0
+  - initial_d:
+      • LoRA: 1e-8
+      • Full fine-tune: 1e-10
+      • Embedding: 1e-7
+  - d_coef: 1
+  - d_limiter: True # To stablizie Prodigy with Simplified_AdEMAMix
+  - factored: False  # Can be true or false, quality should not degrade due to Simplified_AdEMAMix’s high tolerance to 1-bit factorization.
+```
+
+> ✅ **Why it works**:  
+> - `Kourkoutas-β` handles beta2 values
+> - `Simplified_AdEMAMix` ensures responsiveness in small-batch noise
+> - `OrthoGrad` prevents overfitting without weight decay
+
+---
+
+## 📚 References
+
+1. [Revisiting BFloat16 Training](https://arxiv.org/abs/2010.06192)  
+2. [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)  
+3. [The AdEMAMix Optimizer](https://arxiv.org/abs/2409.03137)  
+4. [Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD](https://arxiv.org/abs/2502.02431)  
+5. [AdaMeM: Memory Efficient Momentum for Adafactor](https://openreview.net/forum?id=fZqMVTz7K5)  
+6. [Kourkoutas-β: A Sunspike-Driven Adam Optimizer with Desert Flair](https://arxiv.org/abs/2508.12996)

adv_optm-1.1.1.dist-info/RECORD

@@ -0,0 +1,20 @@
+adv_optm/__init__.py,sha256=TL9XFW3kQQ2Xrxl6UULMftBzNvg7uTIcxMRD0vTttPk,306
+adv_optm/optim/AdamW_adv.py,sha256=ddEUVOif1gfZPgEJNrEGZ2wnha4MPMWw5ppPd8acQ3o,17457
+adv_optm/optim/Adopt_adv.py,sha256=fhH3hS9K6z5Blxc7NFfzpCrUGbl9EQnwLPmKDxBC1zg,21415
+adv_optm/optim/Lion_Prodigy_adv.py,sha256=aJ9orEEw0QYbrDzn1be0SHvOBlIkLwWG9RpWFuNMskM,13163
+adv_optm/optim/Lion_adv.py,sha256=aGNAplZlyXYgVllYcV_s4bK8iC4fv6EizFoWIMNLdBc,8299
+adv_optm/optim/Prodigy_adv.py,sha256=nD59cAWOJJCjZdIiuD5hD9MWO5sTjPQSvq-3dwGTcEM,25875
+adv_optm/optim/Simplified_AdEMAMix.py,sha256=gPjMhKulzmAeO42foe-d7xW0AcB50vKFYsvHgxbD3uc,12949
+adv_optm/optim/__init__.py,sha256=pcP865H2j1tut2VfTUhzQh7V8TF_tzPjqFnjMfFed2k,382
+adv_optm/util/BF16_Stochastic_Rounding.py,sha256=Q5H0BcogmE4atP65dLoI21HKSf50lRdsBDfeF6v9Tbg,1548
+adv_optm/util/Effective_Shape.py,sha256=TBvIk1V8IuTbbBsxuekJA4e_v8JlR5Nujtut8RTWAm4,318
+adv_optm/util/Kourkoutas.py,sha256=woyJfX7l4eieeg0pC5XrILBLvwECwbD3a6ou1K6qjKU,8706
+adv_optm/util/NNMF.py,sha256=yRf5IP5Sjq0Uf0DxN0Q8NxEGSdD-f1ULziLVDOjY8K4,639
+adv_optm/util/One_Bit_Boolean.py,sha256=Wat49esdwohuN-OHOFMW8D0aOQgV9cP5Rl8z6yfmpos,1068
+adv_optm/util/OrthoGrad.py,sha256=NzInuBQGy_Ja__M1R9XbvqVaQ0fhGbtGgFE9YON7B3I,707
+adv_optm/util/__init__.py,sha256=qoyIF0jcLjs_vSEcsv36clw5LFNBEbifyXrrVxMH-G4,349
+adv_optm-1.1.1.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
+adv_optm-1.1.1.dist-info/METADATA,sha256=F30-DuFinS-633wznIM27NBGU5asYpnKdiExchOFPcI,14019
+adv_optm-1.1.1.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
+adv_optm-1.1.1.dist-info/top_level.txt,sha256=iNfBIIzu-lPrQ7jyC56WBCcbkRwitM2nJ15-MRQ_6fg,9
+adv_optm-1.1.1.dist-info/RECORD,,

adv_optm-1.1.0.dev4.dist-info/METADATA

@@ -1,174 +0,0 @@
-Metadata-Version: 2.4
-Name: adv_optm
-Version: 1.1.0.dev4
-Summary: A family of highly efficient, lightweight yet powerful optimizers.
-Home-page: https://github.com/Koratahiu/Advanced_Optimizers
-Author: Koratahiu
-Author-email: hiuhonor@gmail.com
-License: Apache 2.0
-Keywords: llm,fine-tuning,memory-efficient,low-rank,compression,pytorch,optimizer,adam
-Classifier: Programming Language :: Python :: 3
-Classifier: License :: OSI Approved :: Apache Software License
-Classifier: Operating System :: OS Independent
-Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Classifier: Topic :: Software Development :: Libraries :: Python Modules
-Requires-Python: >=3.8
-Description-Content-Type: text/markdown
-License-File: LICENSE
-Requires-Dist: torch>=2.0
-Dynamic: author
-Dynamic: author-email
-Dynamic: classifier
-Dynamic: description
-Dynamic: description-content-type
-Dynamic: home-page
-Dynamic: keywords
-Dynamic: license
-Dynamic: license-file
-Dynamic: requires-dist
-Dynamic: requires-python
-Dynamic: summary
-
-# Advanced Optimizers (AIO)
-
-A comprehensive, all-in-one collection of optimization algorithms for deep learning, designed for maximum efficiency, minimal memory footprint, and superior performance across diverse model architectures and training scenarios.
-
-[![PyPI](https://img.shields.io/pypi/v/adv_optm)](https://pypi.org/project/adv_optm/)
-
----
-
-## 📦 Installation
-
-```bash
-pip install adv_optm
-```
-
----
-
-## 🧠 Core Innovations
-
-This library integrates multiple state-of-the-art optimization techniques validated through extensive research and practical training, with 1-bit compression for optimizer states:
-
-### **Memory-Efficient Optimization (SMMF-inspired)**
-- **Paper**: [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)
-- **Approach**: Uses rank-1 non-negative matrix factorization with reconstruction cycle (factor → reconstruct → update → factor)
-- **Innovation**: 
-  - First moment split into **1-bit sign + absolute value**
-  - Final storage: **four factored vectors + one 1-bit sign state**
-  - Preserves Adam-like update quality with drastically reduced memory
-
----
-
-## ⚡ Performance Characteristics
-
-### Memory Efficiency (SDXL Model - 6.5GB)
-| Optimizer | Memory Usage | Description |
-|-----------|--------------|-------------|
-| `Adopt_Factored` | 328 MB | 4 small vectors + 1-bit state |
-| `Adopt_Factored + AdEMAMix` | 625 MB | 6 small vectors + two 1-bit states |
-| `Simplified_AdEMAMix` | 328 MB | Same as standard factored (no extra state) |
-
-### Speed Comparison (SDXL, Batch Size 4)
-| Optimizer | Speed | Notes |
-|-----------|-------|-------|
-| `Adafactor` | ~8.5s/it | Baseline |
-| `Adopt_Factored` | ~10s/it | +18% overhead from compression |
-| `Adopt_Factored + AdEMAMix` | ~12s/it | +41% overhead (3 factored states) |
-
----
-
-## 🧪 Available Optimizers
-
-### Standard Optimizers (All support `factored=True/False`)
-| Optimizer | Description | Best For |
-|-----------|-------------|----------|
-| `Adam_Adv` | Advanced Adam implementation | General purpose |
-| `Adopt_Adv` | Adam-variant with independent beta2 | Stable training for small batch size regimes |
-| `Prodigy_Adv` | Prodigy with D-Adaptation | Adam with automatic LR tuning |
-| `Simplified_AdEMAMix` | Adam variant with accumulator momentum | Small/large batch training when tuned correctly |
-| `Lion_Adv` | Advanced Lion implementation | Memory-constrained environments |
-| `Prodigy_Lion_Adv` | Prodigy + Lion combination | Lion with automatic LR tuning |
-
-### Feature Matrix
-| Feature | Adam_Adv | Adopt_Adv | Prodigy_Adv | Simplified_AdEMAMix | Lion_Adv |
-|---------|----------|-----------|-------------|---------------------|----------|
-| Factored | ✓ | ✓ | ✓ | ✓ | ✓ |
-| AdEMAMix | ✓ | ✓ | ✓ | ✗ | ✗ |
-| Simplified_AdEMAMix | ✗ | ✗ | ✓ | ✓ | ✗ |
-| OrthoGrad | ✓ | ✓ | ✓ | ✓ | ✓ |
-| Grams | ✓ | ✓ | ✓ | ✗ | ✗ |
-| Cautious | ✓ | ✓ | ✓ | ✗ | ✓ |
-| atan2 | ✓ | ✓ | ✓ | ✗ | ✗ |
-| Stochastic Rounding | ✓ | ✓ | ✓ | ✓ | ✓ |
-| Fused Backward Pass | ✓ | ✓ | ✓ | ✓ | ✓ |
-
----
-
-## ⚙️ Key Features & Parameters
-
-### Comprehensive Feature Guide
-
-| Feature | Description | Recommended Usage | Performance Impact | Theoretical Basis | Compatibility |
-|---------|-------------|-------------------|--------------------|-------------------|--------------|
-| **Factored** | Memory-efficient optimization using rank-1 factorization | Enable for large models (>1B params) or limited VRAM | +12-41% time overhead, 1-bit memory usage | [SMMF](https://arxiv.org/abs/2412.08894) | All optimizers |
-| **AdEMAMix** | Dual EMA system for momentum | Use for long training runs (10k+ steps) | +1 state memory. | [AdEMAMix](https://arxiv.org/abs/2409.03137) | Adam/Adopt/Prodigy |
-| **Simplified_AdEMAMix** | Accumulator-based momentum | Small batch training (≤32) | Same memory as standard, no extra overhead | [Schedule-Free Connections](https://arxiv.org/abs/2502.02431) | Adam/Prodigy |
-| **OrthoGrad** | Removes gradient component parallel to weights | Full finetuning without weight decay | +33% time overhead, no memory impact | [Grokking at Edge](https://github.com/LucasPrietoAl/grokking-at-the-edge-of-numerical-stability) | All optimizers |
-| **Stochastic Rounding** | Improves precision for BF16 training | BF16 training | Minimal overhead (<5%) | [Revisiting BFloat16 Training](https://arxiv.org/abs/2010.06192) | All optimizers |
-| **atan2** | Robust eps replacement + built-in clipping | Use with Adopt or unstable training | No overhead | [Adam-atan2](https://github.com/lucidrains/adam-atan2-pytorch) | Adam/Adopt/prodigy |
-| **Cautious** | Update only when the direction align with the gradients | should faster the convergence | No overhead | [C-Optim](https://github.com/kyleliang919/C-Optim) | Adam/Adopt/prodigy |
-| **Grams** | Update direction from the gradients | should have a stronger effect than cautious | No overhead | [Grams](https://github.com/Gunale0926/Grams) | Adam/Adopt/prodigy |
-
----
-
-## Simplified_AdEMAMix Parameters
-Simplified_AdEMAMix replaces standard momentum with an accumulator for better small-large batch performance.
-
-| Parameter | Recommended Values | Description |
-|-----------|---------------------|-------------|
-| `beta1` | 0.9 (large BS), 0.99-0.9999 (small BS) | Determines memory length of accumulator |
-| `alpha` | 100-10 (small BS), 1-0 (large BS) | Gradient smoothing factor |
-
-**Alpha Tuning Guide**:
-| Batch Size | Recommended α | Rationale |
-|------------|---------------|-----------|
-| Small (≤32) | 100, 50, 20, 10 | Emphasizes recent gradients for quick adaptation |
-| Medium (32-512) | 10, 5, 2, 1 | Balanced approach |
-| Large (≥512) | 1, 0.5, 0 | Emphasizes historical gradients for stability |
-
-⚠️ **Important**: Use **~100x smaller learning rate** with Simplified_AdEMAMix compared to AdamW (e.g., 1e-6 instead of 1e-4)
-
-### 📊 Performance Validation
-Small Batch Training (SDXL, BS=2, 1.8K steps)
-![Training Comparison](https://github.com/user-attachments/assets/7eff0671-cc59-47fc-8b63-d5205456d649)
-
-- **🟢 Prodigy_adv** (beta1=0.9, d0=1e-5): Final LR=2.9e-4
-- **🔵 Prodigy_adv + Simplified_AdEMAMix** (beta1=0.99, α=100, d0=1e-7): Final LR=5.8e-6
-
-**Results**:
-- Simplified_AdEMAMix shows faster convergence and better final performance
-- D-Adaptation automatically handles aggressive updates (50x smaller LR)
-- Generated samples show significantly better quality with Simplified_AdEMAMix
-
----
-
-## ⚠️ Known Limitations
-
-### 1. Prodigy_Adv Sensitivity
-- Highly sensitive to gradient modifications (Adopt normalization, low-rank factorization)
-- May fail to increase learning rate in some LoRA scenarios
-- **Fix**: Disable factorization or set beta1=0
-
-### 2. Aggressive Learning Rates
-- Can destabilize factored first moment
-- **Recommendation**: Check Prodigy learning rate as reference for safe LR threshold
-
----
-
-## 📚 References
-
-1. [SMMF: Square-Matricized Momentum Factorization](https://arxiv.org/abs/2412.08894)
-2. [The AdEMAMix Optimizer: Better, Faster, Older](https://arxiv.org/abs/2409.03137)
-3. [Connections between Schedule-Free Optimizers, AdEMAMix, and Accelerated SGD Variants](https://arxiv.org/abs/2502.02431)
-
----

adv_optm-1.1.0.dev4.dist-info/RECORD

@@ -1,20 +0,0 @@
-adv_optm/__init__.py,sha256=H4E_1__pXxRu4PSgQCzGi7WuFqVjTfex2Yduz3B3peI,311
-adv_optm/optim/AdamW_adv.py,sha256=H4XlYZELwiFvXt0A9wMlRNiw9c8rmPMspHDCvR_SZIQ,17487
-adv_optm/optim/Adopt_adv.py,sha256=0uMROjCw3wGOyp0ZX_xjwMVaXHJ395ifntcgY0MZt3M,21460
-adv_optm/optim/Lion_Prodigy_adv.py,sha256=xIrwibQ2i919EHEACLCrKe5JBnS-s2Ai35yeJ1Bn1MA,13159
-adv_optm/optim/Lion_adv.py,sha256=6G1CukJB_pC7l9HwFEuY1ydsNHZFabVmOvcHDsHHVuQ,8295
-adv_optm/optim/Prodigy_adv.py,sha256=EeSfYu8IIeZX1Dk8MlD71vGOpMadtnW2iMhHxPDL2XQ,25574
-adv_optm/optim/Simplified_AdEMAMix.py,sha256=b4GaSI-TX6wFBqGxZeoJPbf2nVRCEtB3WVb1olDgY14,12980
-adv_optm/optim/__init__.py,sha256=pcP865H2j1tut2VfTUhzQh7V8TF_tzPjqFnjMfFed2k,382
-adv_optm/util/BF16_Stochastic_Rounding.py,sha256=Q5H0BcogmE4atP65dLoI21HKSf50lRdsBDfeF6v9Tbg,1548
-adv_optm/util/Effective_Shape.py,sha256=TBvIk1V8IuTbbBsxuekJA4e_v8JlR5Nujtut8RTWAm4,318
-adv_optm/util/Kourkoutas.py,sha256=st9hO2I0Xcby0LLq1MhxiEsPyNzEkNpJO_WfYvkioKg,6606
-adv_optm/util/NNMF.py,sha256=yRf5IP5Sjq0Uf0DxN0Q8NxEGSdD-f1ULziLVDOjY8K4,639
-adv_optm/util/One_Bit_Boolean.py,sha256=Wat49esdwohuN-OHOFMW8D0aOQgV9cP5Rl8z6yfmpos,1068
-adv_optm/util/OrthoGrad.py,sha256=NzInuBQGy_Ja__M1R9XbvqVaQ0fhGbtGgFE9YON7B3I,707
-adv_optm/util/__init__.py,sha256=qoyIF0jcLjs_vSEcsv36clw5LFNBEbifyXrrVxMH-G4,349
-adv_optm-1.1.0.dev4.dist-info/licenses/LICENSE,sha256=HrhfyXIkWY2tGFK11kg7vPCqhgh5DcxleloqdhrpyMY,11558
-adv_optm-1.1.0.dev4.dist-info/METADATA,sha256=Ue6x-vthnxradX5tH1ver4LVbWMEMmqPjMVO8KjTdhI,8427
-adv_optm-1.1.0.dev4.dist-info/WHEEL,sha256=_zCd3N1l69ArxyTb8rzEoP9TpbYXkqRFSNOD5OuxnTs,91
-adv_optm-1.1.0.dev4.dist-info/top_level.txt,sha256=iNfBIIzu-lPrQ7jyC56WBCcbkRwitM2nJ15-MRQ_6fg,9
-adv_optm-1.1.0.dev4.dist-info/RECORD,,