cloves 2.0.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Potentially problematic release.
This version of cloves might be problematic. Click here for more details.
- cloves-2.0.0/LICENSE +3 -0
- cloves-2.0.0/PKG-INFO +256 -0
- cloves-2.0.0/README.md +224 -0
- cloves-2.0.0/pyproject.toml +59 -0
- cloves-2.0.0/setup.cfg +4 -0
- cloves-2.0.0/setup.py +9 -0
- cloves-2.0.0/src/cloves/__init__.py +24 -0
- cloves-2.0.0/src/cloves/_version.py +1 -0
- cloves-2.0.0/src/cloves/adaptive.py +160 -0
- cloves-2.0.0/src/cloves/affine.py +129 -0
- cloves-2.0.0/src/cloves/apex.py +143 -0
- cloves-2.0.0/src/cloves/bisector.py +131 -0
- cloves-2.0.0/src/cloves/llm.py +135 -0
- cloves-2.0.0/src/cloves/matrix_polar.py +143 -0
- cloves-2.0.0/src/cloves/perpendicular.py +131 -0
- cloves-2.0.0/src/cloves.egg-info/PKG-INFO +256 -0
- cloves-2.0.0/src/cloves.egg-info/SOURCES.txt +21 -0
- cloves-2.0.0/src/cloves.egg-info/dependency_links.txt +1 -0
- cloves-2.0.0/src/cloves.egg-info/requires.txt +6 -0
- cloves-2.0.0/src/cloves.egg-info/top_level.txt +1 -0
- cloves-2.0.0/tests/test_adaptive_optimization.py +135 -0
- cloves-2.0.0/tests/test_apex_optimization.py +218 -0
- cloves-2.0.0/tests/test_optimizers.py +81 -0
cloves-2.0.0/LICENSE
ADDED
cloves-2.0.0/PKG-INFO
ADDED
|
@@ -0,0 +1,256 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: cloves
|
|
3
|
+
Version: 2.0.0
|
|
4
|
+
Summary: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
|
|
5
|
+
License-Expression: MIT
|
|
6
|
+
Project-URL: Homepage, https://github.com/huynhldvn/CLOVES
|
|
7
|
+
Project-URL: Documentation, https://github.com/huynhldvn/CLOVES#readme
|
|
8
|
+
Project-URL: Repository, https://github.com/huynhldvn/CLOVES.git
|
|
9
|
+
Project-URL: Issues, https://github.com/huynhldvn/CLOVES/issues
|
|
10
|
+
Keywords: deep-learning,optimizer,pytorch,optimization,llm,cleft-overstep,adamw,lion,muon
|
|
11
|
+
Classifier: Development Status :: 5 - Production/Stable
|
|
12
|
+
Classifier: Intended Audience :: Science/Research
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: Programming Language :: Python :: 3
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.8
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
21
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
22
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
23
|
+
Requires-Python: >=3.8
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
License-File: LICENSE
|
|
26
|
+
Requires-Dist: torch>=1.12.0
|
|
27
|
+
Provides-Extra: dev
|
|
28
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
29
|
+
Requires-Dist: build; extra == "dev"
|
|
30
|
+
Requires-Dist: twine; extra == "dev"
|
|
31
|
+
Dynamic: license-file
|
|
32
|
+
|
|
33
|
+
# CLOVES: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
|
|
34
|
+
|
|
35
|
+
[](https://pypi.org/project/cloves/)
|
|
36
|
+
[](https://pypi.org/project/cloves/)
|
|
37
|
+
[](LICENSE)
|
|
38
|
+
[](https://github.com/huynhldvn/CLOVES)
|
|
39
|
+
|
|
40
|
+
**CLOVES** (*Cleft-Overstep*) is an optimizer family designed for deep neural networks and large language models (LLMs). It suppresses transversal oscillations across steep canyon walls while accelerating descent along narrow valley floors.
|
|
41
|
+
|
|
42
|
+
<p align="center">
|
|
43
|
+
<img src="docs/images/cloves_optimization_dynamics_2d_3d.gif" alt="CLOVES 2D & 3D Optimization Dynamics Animation" width="100%">
|
|
44
|
+
</p>
|
|
45
|
+
<p align="center">
|
|
46
|
+
<em>Figure 1: Step-by-step optimization dynamics in 2D contour and 3D canyon space (<a href="scripts/generate_optimization_animation.py">reproducible via scripts/generate_optimization_animation.py</a>). While standard optimizers (AdamW, LION, SGD-M) oscillate across the steep canyon walls, CLOVES-Perpendicular and CLOVES-Bisector damp transversal bouncing and follow the valley floor toward the optimum.</em>
|
|
47
|
+
</p>
|
|
48
|
+
|
|
49
|
+
---
|
|
50
|
+
|
|
51
|
+
## 📦 Installation
|
|
52
|
+
|
|
53
|
+
Install the package directly via pip:
|
|
54
|
+
|
|
55
|
+
```bash
|
|
56
|
+
pip install cloves
|
|
57
|
+
```
|
|
58
|
+
|
|
59
|
+
For development and running benchmarks:
|
|
60
|
+
|
|
61
|
+
```bash
|
|
62
|
+
git clone https://github.com/huynhldvn/CLOVES.git
|
|
63
|
+
cd CLOVES
|
|
64
|
+
pip install -e ".[dev]"
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## 📓 Interactive Notebooks & 1-Click Demos
|
|
70
|
+
|
|
71
|
+
Try CLOVES immediately in Google Colab or Jupyter without local setup:
|
|
72
|
+
|
|
73
|
+
<p align="center">
|
|
74
|
+
<a href="https://colab.research.google.com/github/huynhldvn/CLOVES/blob/master/notebooks/quickstart_interactive_demo.ipynb">
|
|
75
|
+
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
|
|
76
|
+
</a>
|
|
77
|
+
</p>
|
|
78
|
+
|
|
79
|
+
- **Quickstart Demo**: [`notebooks/quickstart_interactive_demo.ipynb`](notebooks/quickstart_interactive_demo.ipynb) — 1-click Colab interactive simulation & PyTorch training.
|
|
80
|
+
- **Dedicated Benchmarks**: Check out [`notebooks/`](notebooks/) for focused notebooks on [CIFAR-10](notebooks/01_cifar10_vision_benchmark.ipynb), [LLM QLoRA](notebooks/02_llm_qlora_tuning_benchmark.ipynb), and [2D Ravine Simulation](notebooks/03_2d_ravine_simulation.ipynb).
|
|
81
|
+
- **Standalone Scripts**: Run minimal scripts in [`examples/`](examples/): `examples/01_quickstart_vision.py` and `examples/02_quickstart_llm.py`.
|
|
82
|
+
|
|
83
|
+
---
|
|
84
|
+
|
|
85
|
+
## 🎯 Which Optimizer Should You Use? (Practical Guide)
|
|
86
|
+
|
|
87
|
+
Choose the right variant based on your workload:
|
|
88
|
+
|
|
89
|
+
| Workload / Architecture | Recommended Optimizer | Key Advantage | State Memory | Recommended Settings |
|
|
90
|
+
| :--- | :--- | :--- | :---: | :--- |
|
|
91
|
+
| 🤖 **LLM LoRA / QLoRA Tuning** | `CLOVES_Apex_Asymmetric` ⭐ | **Recommended for LoRA**. Automatically accelerates Matrix A ($q_A=1.30$) and stabilizes Matrix B ($q_B=1.05$). Matches or slightly improves on AdamW loss while saving **> 3.3 GB VRAM**. | 2 States (8 B) | `lr=1.6e-4`, `total_steps=500` |
|
|
92
|
+
| 🧠 **LLM Full Tuning / Transformers** | `CLOVES_Adaptive` | **Drop-in replacement for AdamW**. Damps coordinate oscillations, maintains stability on deep networks, reduces synthetic ravine loss by 92%. | 2 States (8 B) | `lr=2e-4`, `q=1.25`, `gamma=0.6` |
|
|
93
|
+
| ⚡ **Extreme Memory Constraints (LLM)** | `CLOVES_LLM` | **50% state memory savings** (4 B/param), eliminates LION's coordinate-wise loss drift. | 1 State (4 B) | `lr=1e-4`, `q=1.20`, `gamma=0.6` |
|
|
94
|
+
| 👁️ **Standard Vision (CNN, ResNet, ViT)** | `CLOVES_Perpendicular` ⭐ | **Recommended for vision tasks**. Rejects orthogonal transverse wall force ($g_\perp$), reaches **93.07%** on CIFAR-10, runs with light 1-state memory. | 1 State (4 B) | `lr=0.05 - 0.1`, `beta=0.9` |
|
|
95
|
+
| 📈 **Tabular / Smooth Convex Tasks** | `CLOVES` (Bisector) | Fastest computational throughput, follows the ravine floor along the angular bisector. | 1 State (4 B) | `lr=0.1`, `q=1.5`, `gamma=0.1` |
|
|
96
|
+
|
|
97
|
+
> ⚠️ **Scope & Evaluation Note:**
|
|
98
|
+
> The LLM-optimized variants (`CLOVES_Apex_Asymmetric`, `CLOVES_Adaptive`, `CLOVES_LLM`, `CLOVES_Matrix_Polar`) were designed and validated specifically on Transformer and LoRA / QLoRA architectures (LLaMA-3, Qwen-1.5B, Qwen-7B). **These LLM variants have not yet been evaluated on other tasks (such as computer vision, audio, or tabular data)**.
|
|
99
|
+
> 👉 For computer vision and general classification workloads, use `CLOVES_Perpendicular` or `CLOVES` (Bisector), which have been thoroughly evaluated on vision benchmarks.
|
|
100
|
+
|
|
101
|
+
---
|
|
102
|
+
|
|
103
|
+
## ⚡ Quickstart
|
|
104
|
+
|
|
105
|
+
### 1. Minimal Runnable Snippet (Copy & Run)
|
|
106
|
+
Test CLOVES immediately in any Python environment:
|
|
107
|
+
|
|
108
|
+
```python
|
|
109
|
+
import torch
|
|
110
|
+
import torch.nn as nn
|
|
111
|
+
from cloves import CLOVES_Perpendicular
|
|
112
|
+
|
|
113
|
+
# Define a minimal model & optimizer
|
|
114
|
+
model = nn.Linear(10, 2)
|
|
115
|
+
optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05)
|
|
116
|
+
|
|
117
|
+
# Standard training step
|
|
118
|
+
x, y = torch.randn(16, 10), torch.randn(16, 2)
|
|
119
|
+
loss = nn.MSELoss()(model(x), y)
|
|
120
|
+
loss.backward()
|
|
121
|
+
optimizer.step()
|
|
122
|
+
optimizer.zero_grad()
|
|
123
|
+
|
|
124
|
+
print("Step completed! Loss:", float(loss.detach()))
|
|
125
|
+
```
|
|
126
|
+
|
|
127
|
+
### 2. Drop-in Replacement for Real Models
|
|
128
|
+
Simply swap your existing optimizer in your training loop:
|
|
129
|
+
|
|
130
|
+
```python
|
|
131
|
+
# A. Vision (ResNet, CNN, ViT) — lightweight 1-state memory:
|
|
132
|
+
from cloves import CLOVES_Perpendicular
|
|
133
|
+
optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05, beta=0.9)
|
|
134
|
+
|
|
135
|
+
# B. LLM & LoRA Fine-Tuning — saves >3.3 GB VRAM:
|
|
136
|
+
from cloves import CLOVES_Apex_Asymmetric
|
|
137
|
+
optimizer = CLOVES_Apex_Asymmetric(model.parameters(), lr=1.6e-4, total_steps=500)
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
---
|
|
141
|
+
|
|
142
|
+
## 🚀 Milestone Versions & Experimental Reports
|
|
143
|
+
|
|
144
|
+
CLOVES bridges classical first-order valley descent theory into large-scale stochastic deep learning and LLM fine-tuning:
|
|
145
|
+
|
|
146
|
+
```
|
|
147
|
+
THE CLOVES OPTIMIZATION PHILOSOPHY
|
|
148
|
+
(Transversal Wall Damping γ < 1 & Longitudinal Floor Overstep q > 1)
|
|
149
|
+
│
|
|
150
|
+
┌────────────────────────────────┴────────────────────────────────┐
|
|
151
|
+
▼ ▼
|
|
152
|
+
[MILESTONE 1: DEEP VISION & 1-STATE PARITY] [MILESTONE 2: LLM & ADAPTIVE SCALING]
|
|
153
|
+
• Problem: Wall chattering in narrow ravines • Problem: Small-norm stagnation (||g|| ~ 1e-4)
|
|
154
|
+
• Memory: Strict 4 bytes/param (1-state) • Memory: 50% state memory savings or RMS scaling
|
|
155
|
+
• Mechanism: Angular bisector & orthogonal • Mechanism: Coordinate overstep, LoRA A/B
|
|
156
|
+
transversal force rejection (g_perp) asymmetric velocity, Newton-Schulz polar
|
|
157
|
+
│ │
|
|
158
|
+
▼ ▼
|
|
159
|
+
CLOVES-Bisector / CLOVES-Perpendicular CLOVES-Apex / CLOVES-Adaptive
|
|
160
|
+
(Strong vision accuracy, 50% state memory) (Matches AdamW, beats Muon on QLoRA)
|
|
161
|
+
```
|
|
162
|
+
|
|
163
|
+
The project is organized into two developmental milestones:
|
|
164
|
+
- **[Version 1.0 — Geometric Vision & Ravine Damping](benchmarks/v1_geometric_vision/README.md)**: First-order directional cleft damping (Bisector, Perpendicular, Affine) maintaining strict 1-state memory parity (4 bytes/param) for vision architectures (ResNet-18, DenseNet-121).
|
|
165
|
+
- **[Version 2.0 — LLM Fine-Tuning & Adaptive Cleft Scaling](benchmarks/v2_llm_adaptive/README.md)**: Addresses small-gradient stagnation ($\|g\| \sim 10^{-4}$) via coordinate cleft alignment, adaptive second-moment curvature scaling, and asymmetric LoRA velocity for Transformer architectures (LLaMA-3, Qwen-1.5B, Qwen-7B).
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## 📈 Comprehensive Master Benchmark Results
|
|
170
|
+
|
|
171
|
+
The table below summarizes empirical validations across synthetic optimization ravines, deep vision architectures, and large language model instruction tuning suites, including convergence accuracy/loss, runtime per step/epoch, total training duration, and GPU memory consumption (measured on NVIDIA Tesla T4 GPUs):
|
|
172
|
+
|
|
173
|
+
> [!NOTE]
|
|
174
|
+
> **Evaluation Protocol & Objectivity**: To guarantee rigorous, unbiased comparison and avoid overfitting artifacts, all deep learning benchmarks (CIFAR-10, CIFAR-100, LLaMA-3, Qwen) are strictly evaluated on held-out **Validation Sets** (`val_loss` and `val_acc`), never on training batches. Synthetic ravine benchmarks evaluate the exact mathematical objective loss function $f(x)$ under identical random seeds and initialization.
|
|
175
|
+
>
|
|
176
|
+
> **Metric Interpretation**: **↑** = Higher is better | **↓** = Lower is better
|
|
177
|
+
|
|
178
|
+
| Benchmark Suite & Dataset | Architecture / Setup | Optimizer | Convergence Metric<br>(Val Loss ↓ / Val Acc ↑) | Step / Epoch Time<br>(↓) | Total Execution Time<br>(↓) | Peak GPU VRAM<br>(↓) | Optimizer State<br>(↓) |
|
|
179
|
+
| :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
|
|
180
|
+
| **2D Ill-Conditioned Ravine** ($\kappa=1000$) | Quadratic canyon (500 steps) | **CLOVES-Bisector**<br>AdamW<br>LION | Objective Loss: **$7.11 \times 10^{-5}$**<br>Objective Loss: 20.20<br>Objective Loss: 24.52 | **0.82 ms / step**<br>1.14 ms / step<br>0.89 ms / step | **0.41 s**<br>0.57 s<br>0.45 s | < 10 MB<br>< 10 MB<br>< 10 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
|
|
181
|
+
| **CIFAR-10 Vision** (Standard Vision) | ResNet-18 (20 Epochs, bs=128) | **CLOVES-Perpendicular**<br>CLOVES-Affine<br>AdamW<br>Official LION | Val Acc: **93.07%** (Val Loss: 0.2237)<br>Val Acc: 92.51% (Val Loss: 0.2310)<br>Val Acc: 92.35% (Val Loss: 0.2512)<br>Val Acc: 89.20% (Val Loss: 0.3840) | **51.1 s / epoch**<br>51.6 s / epoch<br>61.4 s / epoch<br>51.0 s / epoch | **1,022.7 s** (~17.0 min)<br>1,032.0 s (~17.2 min)<br>1,228.0 s (~20.5 min)<br>1,020.0 s (~17.0 min) | **696.0 MB**<br>696.0 MB<br>827.8 MB<br>696.0 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
|
|
182
|
+
| **CIFAR-100 Deep Vision** (100 Classes) | DenseNet-121 (30 Epochs, Cosine) | **CLOVES (Proposed)**<br>AdamW<br>SGD-M | Top-1 Val Acc: **73.98%** (Val Loss: 1.3993)<br>Top-1 Val Acc: 72.77% (Val Loss: 1.4435)<br>Top-1 Val Acc: 72.56% (Val Loss: 1.4304) | **132.5 s / epoch**<br>133.1 s / epoch<br>131.8 s / epoch | **3,975.0 s** (~66.2 min)<br>3,993.0 s (~66.5 min)<br>3,954.0 s (~65.9 min) | **5,001.5 MB**<br>5,028.4 MB<br>5,001.5 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
|
|
183
|
+
| **1000D Non-Convex Ravine** ($\kappa=10,000$) | Anisotropic manifold (500 steps) | **CLOVES-Adaptive**<br>AdamW<br>**CLOVES-LLM**<br>Official LION | Objective Loss: **17.12** (-92.0%)<br>Objective Loss: 213.23 (Baseline)<br>Objective Loss: **12,297.97** (-60.2%)<br>Objective Loss: 30,861.02 (Baseline) | **1.21 ms / step**<br>1.18 ms / step<br>0.94 ms / step<br>0.92 ms / step | **0.61 s**<br>0.59 s<br>0.47 s<br>0.46 s | < 15 MB<br>< 15 MB<br>< 15 MB<br>< 15 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
|
|
184
|
+
| **LLaMA-3-8B Instruction Tuning** | 4-bit QLoRA, Alpaca (500 steps) | **CLOVES-Perpendicular**<br>CLOVES-Matrix-Polar<br>AdamW (Baseline)<br>Official Muon | Val Loss: **5.7338**<br>Val Loss: **5.7822**<br>Val Loss: 5.7269<br>Val Loss: 5.7941 | **12.01 s / step**<br>12.48 s / step<br>11.71 s / step<br>12.52 s / step | **16,010.2 s** (~4.44 h)<br>16,485.3 s (~4.58 h)<br>16,376.8 s (~4.55 h)<br>16,428.7 s (~4.56 h) | **10,594.5 MB**<br>10,594.5 MB<br>14,349.5 MB<br>14,189.5 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
|
|
185
|
+
| **Qwen-1.5B Instruction Tuning** | QLoRA, Alpaca (1000 steps) | **CLOVES-Apex (Tuned)**<br>AdamW (Baseline) | Val Loss: **0.4295**<br>Val Loss: 0.4298 | 562.8 ms / step<br>470.3 ms / step | **2,681.9 s** (~44.7 min)<br>2,648.4 s (~44.1 min) | **8,202.3 MB**<br>8,202.3 MB | **2-State (8 B)**<br>2-State (8 B) |
|
|
186
|
+
| **Qwen-7B Instruction Tuning** | QLoRA, Alpaca (500 steps) | **CLOVES-Apex (Annealed)**<br>AdamW (Baseline)<br>**CLOVES-Sign-Apex**<br>Official LION (Baseline) | Val Loss: **0.5573**<br>Val Loss: 0.5583<br>Val Loss: **0.6249**<br>Val Loss: 0.6365 | 2,956.0 ms / step<br>2,746.3 ms / step<br>2,732.9 ms / step<br>2,714.6 ms / step | **3,911.1 s** (~65.1 min)<br>3,871.5 s (~64.5 min)<br>3,624.2 s (~60.4 min)<br>3,613.7 s (~60.2 min) | **10,793.6 MB** (-3.36 GB)<br>14,159.3 MB<br>**10,639.6 MB** (-3.37 GB)<br>14,005.3 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
|
|
187
|
+
|
|
188
|
+
### Pareto Frontier: Loss vs. Memory vs. Runtime Trade-off
|
|
189
|
+
|
|
190
|
+
The empirical evaluations demonstrate favorable trade-offs across memory footprint, runtime efficiency, and convergence loss:
|
|
191
|
+
|
|
192
|
+
<p align="center">
|
|
193
|
+
<img src="docs/images/cloves_pareto_frontier_master_clean.png" alt="CLOVES Pareto Frontier" width="85%">
|
|
194
|
+
</p>
|
|
195
|
+
|
|
196
|
+
---
|
|
197
|
+
|
|
198
|
+
## 📊 Optimizer Selection Matrix
|
|
199
|
+
|
|
200
|
+
| Optimizer Class | Target Workload | State Buffers | Memory Footprint | Special Characteristics |
|
|
201
|
+
| :--- | :--- | :---: | :---: | :--- |
|
|
202
|
+
| `CLOVES` | Vision (CNNs, ResNet) | 1 ($m$) | 4 bytes / param | Angular bisector damping; low memory |
|
|
203
|
+
| `CLOVES_Perpendicular` | Highly ill-conditioned loss | 1 ($m$) | 4 bytes / param | Orthogonal wall force rejection |
|
|
204
|
+
| `CLOVES_Affine` | Deep classification networks | 1 ($m$) | 4 bytes / param | Continuous affine subspace projection |
|
|
205
|
+
| `CLOVES_Adaptive` | LLMs, Transformers | 2 ($m, v$) | 8 bytes / param | Coordinate cleft overstep + RMS scaling |
|
|
206
|
+
| `CLOVES_LLM` | Memory-constrained LLMs | 1 ($m$) | 4 bytes / param | Sign overstep; 50% state memory savings |
|
|
207
|
+
| `CLOVES_Apex_Asymmetric` | LoRA / QLoRA Instruction Tuning | 2 ($m, v$) | 8 bytes / param | Matrix A/B ratio, soft tanh damping |
|
|
208
|
+
| `CLOVES_Matrix_Polar` | Linear layer weights (2D) | 1 ($m$) | 4 bytes / param | Newton-Schulz quintic polar decomposition |
|
|
209
|
+
|
|
210
|
+
---
|
|
211
|
+
|
|
212
|
+
## 📚 References & Baseline Implementations
|
|
213
|
+
|
|
214
|
+
CLOVES benchmarks are evaluated against official implementations of standard and contemporary optimizers:
|
|
215
|
+
|
|
216
|
+
1. **AdamW**: Loshchilov & Hutter, *Decoupled Weight Decay Regularization*, ICLR 2019.
|
|
217
|
+
2. **Lion** (*EvoLved Sign Momentum*): Chen et al., *Symbolic Discovery of Optimization Algorithms*, NeurIPS 2023. arXiv: [2302.06675](https://arxiv.org/abs/2302.06675).
|
|
218
|
+
- Code: [google-research/lion](https://github.com/google-research/google-research/tree/master/lion) | [lucidrains/lion-pytorch](https://github.com/lucidrains/lion-pytorch)
|
|
219
|
+
3. **Muon** (*Momentum Orthogonalized by Newton-schulz*): Keller Jordan et al., *Muon: An Optimizer for Hidden Layers in Neural Networks*, 2024.
|
|
220
|
+
- Code: [KellerJordan/Muon](https://github.com/KellerJordan/Muon)
|
|
221
|
+
|
|
222
|
+
---
|
|
223
|
+
|
|
224
|
+
## 🏛️ Research Inspiration & Lineage
|
|
225
|
+
|
|
226
|
+
The algorithmic concepts in CLOVES are **inspired by** foundational ravine traversal theories in mathematical programming (Nguyễn & Bùi, 1990) and subsequent exploratory studies evaluating cleft-overstep heuristics in machine learning tasks (Phan & Lê, 2020).
|
|
227
|
+
|
|
228
|
+
### Pragmatic Adaptation from Theory to Practice
|
|
229
|
+
The original cleft-overstep formulations were conceived within classical, low-dimensional, deterministic optimization settings. Translating these geometric concepts to modern deep learning and large-scale language models ($d \gg 10^7$) inherently involves **empirical, heuristic (ad-hoc) adaptations**:
|
|
230
|
+
- The authors make **no claim of 100% formal mathematical equivalence** or complete theoretical convergence proofs matching the original deterministic algorithms.
|
|
231
|
+
- Instead, CLOVES adapts the core geometric intuition—*damping transversal wall-to-wall bouncing while accelerating along the longitudinal ravine bed*—into efficient, zero-allocation, fused PyTorch tensor operations tailored for stochastic mini-batch gradients.
|
|
232
|
+
- Extensions such as coordinate-wise variance scaling (`CLOVES_Adaptive`), 1-state sign momentum (`CLOVES_LLM`), asymmetric LoRA velocity ratios (`CLOVES_Apex_Asymmetric`), and polar matrix decomposition (`CLOVES_Matrix_Polar`) are practical designs driven by empirical deep learning dynamics.
|
|
233
|
+
|
|
234
|
+
**Predecessor & Historical References**:
|
|
235
|
+
1. Nguyễn Văn Mạnh & Bùi Minh Trí, *"Methods for finding ravine directions in non-differentiable optimization,"* *Acta Mathematica Vietnamica*, 1990.
|
|
236
|
+
2. Duy-Hung Phan & Le Dinh Huynh, *"Evaluation of the Cleft-Overstep Algorithm for Linear Regression Analysis,"* in *Computational Collective Intelligence (ICCCI 2020)*, Lecture Notes in Computer Science (LNCS, vol. 12496), pp. 400–411, Springer, 2020. DOI: [10.1007/978-3-030-63007-2_31](https://doi.org/10.1007/978-3-030-63007-2_31).
|
|
237
|
+
|
|
238
|
+
---
|
|
239
|
+
|
|
240
|
+
## 🤝 Feedback & Contributions
|
|
241
|
+
|
|
242
|
+
CLOVES is an empirical project developed with an open mindset. If you experiment with CLOVES on your models and find it helpful—or discover interesting edge cases—we would love to hear from you!
|
|
243
|
+
|
|
244
|
+
Bug reports, critiques, and pull requests are warmly welcomed via [GitHub Issues](https://github.com/huynhldvn/CLOVES/issues) or [Discussions](https://github.com/huynhldvn/CLOVES/discussions).
|
|
245
|
+
|
|
246
|
+
---
|
|
247
|
+
|
|
248
|
+
## 🙏 Acknowledgements
|
|
249
|
+
|
|
250
|
+
We thank **Kaggle** for providing the free GPU compute (Tesla T4) for our benchmarks, and **Antigravity** for assisting with code refactoring and documentation.
|
|
251
|
+
|
|
252
|
+
---
|
|
253
|
+
|
|
254
|
+
## 📄 License
|
|
255
|
+
|
|
256
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
cloves-2.0.0/README.md
ADDED
|
@@ -0,0 +1,224 @@
|
|
|
1
|
+
# CLOVES: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
|
|
2
|
+
|
|
3
|
+
[](https://pypi.org/project/cloves/)
|
|
4
|
+
[](https://pypi.org/project/cloves/)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
[](https://github.com/huynhldvn/CLOVES)
|
|
7
|
+
|
|
8
|
+
**CLOVES** (*Cleft-Overstep*) is an optimizer family designed for deep neural networks and large language models (LLMs). It suppresses transversal oscillations across steep canyon walls while accelerating descent along narrow valley floors.
|
|
9
|
+
|
|
10
|
+
<p align="center">
|
|
11
|
+
<img src="docs/images/cloves_optimization_dynamics_2d_3d.gif" alt="CLOVES 2D & 3D Optimization Dynamics Animation" width="100%">
|
|
12
|
+
</p>
|
|
13
|
+
<p align="center">
|
|
14
|
+
<em>Figure 1: Step-by-step optimization dynamics in 2D contour and 3D canyon space (<a href="scripts/generate_optimization_animation.py">reproducible via scripts/generate_optimization_animation.py</a>). While standard optimizers (AdamW, LION, SGD-M) oscillate across the steep canyon walls, CLOVES-Perpendicular and CLOVES-Bisector damp transversal bouncing and follow the valley floor toward the optimum.</em>
|
|
15
|
+
</p>
|
|
16
|
+
|
|
17
|
+
---
|
|
18
|
+
|
|
19
|
+
## 📦 Installation
|
|
20
|
+
|
|
21
|
+
Install the package directly via pip:
|
|
22
|
+
|
|
23
|
+
```bash
|
|
24
|
+
pip install cloves
|
|
25
|
+
```
|
|
26
|
+
|
|
27
|
+
For development and running benchmarks:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
git clone https://github.com/huynhldvn/CLOVES.git
|
|
31
|
+
cd CLOVES
|
|
32
|
+
pip install -e ".[dev]"
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
---
|
|
36
|
+
|
|
37
|
+
## 📓 Interactive Notebooks & 1-Click Demos
|
|
38
|
+
|
|
39
|
+
Try CLOVES immediately in Google Colab or Jupyter without local setup:
|
|
40
|
+
|
|
41
|
+
<p align="center">
|
|
42
|
+
<a href="https://colab.research.google.com/github/huynhldvn/CLOVES/blob/master/notebooks/quickstart_interactive_demo.ipynb">
|
|
43
|
+
<img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
|
|
44
|
+
</a>
|
|
45
|
+
</p>
|
|
46
|
+
|
|
47
|
+
- **Quickstart Demo**: [`notebooks/quickstart_interactive_demo.ipynb`](notebooks/quickstart_interactive_demo.ipynb) — 1-click Colab interactive simulation & PyTorch training.
|
|
48
|
+
- **Dedicated Benchmarks**: Check out [`notebooks/`](notebooks/) for focused notebooks on [CIFAR-10](notebooks/01_cifar10_vision_benchmark.ipynb), [LLM QLoRA](notebooks/02_llm_qlora_tuning_benchmark.ipynb), and [2D Ravine Simulation](notebooks/03_2d_ravine_simulation.ipynb).
|
|
49
|
+
- **Standalone Scripts**: Run minimal scripts in [`examples/`](examples/): `examples/01_quickstart_vision.py` and `examples/02_quickstart_llm.py`.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## 🎯 Which Optimizer Should You Use? (Practical Guide)
|
|
54
|
+
|
|
55
|
+
Choose the right variant based on your workload:
|
|
56
|
+
|
|
57
|
+
| Workload / Architecture | Recommended Optimizer | Key Advantage | State Memory | Recommended Settings |
|
|
58
|
+
| :--- | :--- | :--- | :---: | :--- |
|
|
59
|
+
| 🤖 **LLM LoRA / QLoRA Tuning** | `CLOVES_Apex_Asymmetric` ⭐ | **Recommended for LoRA**. Automatically accelerates Matrix A ($q_A=1.30$) and stabilizes Matrix B ($q_B=1.05$). Matches or slightly improves on AdamW loss while saving **> 3.3 GB VRAM**. | 2 States (8 B) | `lr=1.6e-4`, `total_steps=500` |
|
|
60
|
+
| 🧠 **LLM Full Tuning / Transformers** | `CLOVES_Adaptive` | **Drop-in replacement for AdamW**. Damps coordinate oscillations, maintains stability on deep networks, reduces synthetic ravine loss by 92%. | 2 States (8 B) | `lr=2e-4`, `q=1.25`, `gamma=0.6` |
|
|
61
|
+
| ⚡ **Extreme Memory Constraints (LLM)** | `CLOVES_LLM` | **50% state memory savings** (4 B/param), eliminates LION's coordinate-wise loss drift. | 1 State (4 B) | `lr=1e-4`, `q=1.20`, `gamma=0.6` |
|
|
62
|
+
| 👁️ **Standard Vision (CNN, ResNet, ViT)** | `CLOVES_Perpendicular` ⭐ | **Recommended for vision tasks**. Rejects orthogonal transverse wall force ($g_\perp$), reaches **93.07%** on CIFAR-10, runs with light 1-state memory. | 1 State (4 B) | `lr=0.05 - 0.1`, `beta=0.9` |
|
|
63
|
+
| 📈 **Tabular / Smooth Convex Tasks** | `CLOVES` (Bisector) | Fastest computational throughput, follows the ravine floor along the angular bisector. | 1 State (4 B) | `lr=0.1`, `q=1.5`, `gamma=0.1` |
|
|
64
|
+
|
|
65
|
+
> ⚠️ **Scope & Evaluation Note:**
|
|
66
|
+
> The LLM-optimized variants (`CLOVES_Apex_Asymmetric`, `CLOVES_Adaptive`, `CLOVES_LLM`, `CLOVES_Matrix_Polar`) were designed and validated specifically on Transformer and LoRA / QLoRA architectures (LLaMA-3, Qwen-1.5B, Qwen-7B). **These LLM variants have not yet been evaluated on other tasks (such as computer vision, audio, or tabular data)**.
|
|
67
|
+
> 👉 For computer vision and general classification workloads, use `CLOVES_Perpendicular` or `CLOVES` (Bisector), which have been thoroughly evaluated on vision benchmarks.
|
|
68
|
+
|
|
69
|
+
---
|
|
70
|
+
|
|
71
|
+
## ⚡ Quickstart
|
|
72
|
+
|
|
73
|
+
### 1. Minimal Runnable Snippet (Copy & Run)
|
|
74
|
+
Test CLOVES immediately in any Python environment:
|
|
75
|
+
|
|
76
|
+
```python
|
|
77
|
+
import torch
|
|
78
|
+
import torch.nn as nn
|
|
79
|
+
from cloves import CLOVES_Perpendicular
|
|
80
|
+
|
|
81
|
+
# Define a minimal model & optimizer
|
|
82
|
+
model = nn.Linear(10, 2)
|
|
83
|
+
optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05)
|
|
84
|
+
|
|
85
|
+
# Standard training step
|
|
86
|
+
x, y = torch.randn(16, 10), torch.randn(16, 2)
|
|
87
|
+
loss = nn.MSELoss()(model(x), y)
|
|
88
|
+
loss.backward()
|
|
89
|
+
optimizer.step()
|
|
90
|
+
optimizer.zero_grad()
|
|
91
|
+
|
|
92
|
+
print("Step completed! Loss:", float(loss.detach()))
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
### 2. Drop-in Replacement for Real Models
|
|
96
|
+
Simply swap your existing optimizer in your training loop:
|
|
97
|
+
|
|
98
|
+
```python
|
|
99
|
+
# A. Vision (ResNet, CNN, ViT) — lightweight 1-state memory:
|
|
100
|
+
from cloves import CLOVES_Perpendicular
|
|
101
|
+
optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05, beta=0.9)
|
|
102
|
+
|
|
103
|
+
# B. LLM & LoRA Fine-Tuning — saves >3.3 GB VRAM:
|
|
104
|
+
from cloves import CLOVES_Apex_Asymmetric
|
|
105
|
+
optimizer = CLOVES_Apex_Asymmetric(model.parameters(), lr=1.6e-4, total_steps=500)
|
|
106
|
+
```
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## 🚀 Milestone Versions & Experimental Reports
|
|
111
|
+
|
|
112
|
+
CLOVES bridges classical first-order valley descent theory into large-scale stochastic deep learning and LLM fine-tuning:
|
|
113
|
+
|
|
114
|
+
```
|
|
115
|
+
THE CLOVES OPTIMIZATION PHILOSOPHY
|
|
116
|
+
(Transversal Wall Damping γ < 1 & Longitudinal Floor Overstep q > 1)
|
|
117
|
+
│
|
|
118
|
+
┌────────────────────────────────┴────────────────────────────────┐
|
|
119
|
+
▼ ▼
|
|
120
|
+
[MILESTONE 1: DEEP VISION & 1-STATE PARITY] [MILESTONE 2: LLM & ADAPTIVE SCALING]
|
|
121
|
+
• Problem: Wall chattering in narrow ravines • Problem: Small-norm stagnation (||g|| ~ 1e-4)
|
|
122
|
+
• Memory: Strict 4 bytes/param (1-state) • Memory: 50% state memory savings or RMS scaling
|
|
123
|
+
• Mechanism: Angular bisector & orthogonal • Mechanism: Coordinate overstep, LoRA A/B
|
|
124
|
+
transversal force rejection (g_perp) asymmetric velocity, Newton-Schulz polar
|
|
125
|
+
│ │
|
|
126
|
+
▼ ▼
|
|
127
|
+
CLOVES-Bisector / CLOVES-Perpendicular CLOVES-Apex / CLOVES-Adaptive
|
|
128
|
+
(Strong vision accuracy, 50% state memory) (Matches AdamW, beats Muon on QLoRA)
|
|
129
|
+
```
|
|
130
|
+
|
|
131
|
+
The project is organized into two developmental milestones:
|
|
132
|
+
- **[Version 1.0 — Geometric Vision & Ravine Damping](benchmarks/v1_geometric_vision/README.md)**: First-order directional cleft damping (Bisector, Perpendicular, Affine) maintaining strict 1-state memory parity (4 bytes/param) for vision architectures (ResNet-18, DenseNet-121).
|
|
133
|
+
- **[Version 2.0 — LLM Fine-Tuning & Adaptive Cleft Scaling](benchmarks/v2_llm_adaptive/README.md)**: Addresses small-gradient stagnation ($\|g\| \sim 10^{-4}$) via coordinate cleft alignment, adaptive second-moment curvature scaling, and asymmetric LoRA velocity for Transformer architectures (LLaMA-3, Qwen-1.5B, Qwen-7B).
|
|
134
|
+
|
|
135
|
+
---
|
|
136
|
+
|
|
137
|
+
## 📈 Comprehensive Master Benchmark Results
|
|
138
|
+
|
|
139
|
+
The table below summarizes empirical validations across synthetic optimization ravines, deep vision architectures, and large language model instruction tuning suites, including convergence accuracy/loss, runtime per step/epoch, total training duration, and GPU memory consumption (measured on NVIDIA Tesla T4 GPUs):
|
|
140
|
+
|
|
141
|
+
> [!NOTE]
|
|
142
|
+
> **Evaluation Protocol & Objectivity**: To guarantee rigorous, unbiased comparison and avoid overfitting artifacts, all deep learning benchmarks (CIFAR-10, CIFAR-100, LLaMA-3, Qwen) are strictly evaluated on held-out **Validation Sets** (`val_loss` and `val_acc`), never on training batches. Synthetic ravine benchmarks evaluate the exact mathematical objective loss function $f(x)$ under identical random seeds and initialization.
|
|
143
|
+
>
|
|
144
|
+
> **Metric Interpretation**: **↑** = Higher is better | **↓** = Lower is better
|
|
145
|
+
|
|
146
|
+
| Benchmark Suite & Dataset | Architecture / Setup | Optimizer | Convergence Metric<br>(Val Loss ↓ / Val Acc ↑) | Step / Epoch Time<br>(↓) | Total Execution Time<br>(↓) | Peak GPU VRAM<br>(↓) | Optimizer State<br>(↓) |
|
|
147
|
+
| :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
|
|
148
|
+
| **2D Ill-Conditioned Ravine** ($\kappa=1000$) | Quadratic canyon (500 steps) | **CLOVES-Bisector**<br>AdamW<br>LION | Objective Loss: **$7.11 \times 10^{-5}$**<br>Objective Loss: 20.20<br>Objective Loss: 24.52 | **0.82 ms / step**<br>1.14 ms / step<br>0.89 ms / step | **0.41 s**<br>0.57 s<br>0.45 s | < 10 MB<br>< 10 MB<br>< 10 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
|
|
149
|
+
| **CIFAR-10 Vision** (Standard Vision) | ResNet-18 (20 Epochs, bs=128) | **CLOVES-Perpendicular**<br>CLOVES-Affine<br>AdamW<br>Official LION | Val Acc: **93.07%** (Val Loss: 0.2237)<br>Val Acc: 92.51% (Val Loss: 0.2310)<br>Val Acc: 92.35% (Val Loss: 0.2512)<br>Val Acc: 89.20% (Val Loss: 0.3840) | **51.1 s / epoch**<br>51.6 s / epoch<br>61.4 s / epoch<br>51.0 s / epoch | **1,022.7 s** (~17.0 min)<br>1,032.0 s (~17.2 min)<br>1,228.0 s (~20.5 min)<br>1,020.0 s (~17.0 min) | **696.0 MB**<br>696.0 MB<br>827.8 MB<br>696.0 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
|
|
150
|
+
| **CIFAR-100 Deep Vision** (100 Classes) | DenseNet-121 (30 Epochs, Cosine) | **CLOVES (Proposed)**<br>AdamW<br>SGD-M | Top-1 Val Acc: **73.98%** (Val Loss: 1.3993)<br>Top-1 Val Acc: 72.77% (Val Loss: 1.4435)<br>Top-1 Val Acc: 72.56% (Val Loss: 1.4304) | **132.5 s / epoch**<br>133.1 s / epoch<br>131.8 s / epoch | **3,975.0 s** (~66.2 min)<br>3,993.0 s (~66.5 min)<br>3,954.0 s (~65.9 min) | **5,001.5 MB**<br>5,028.4 MB<br>5,001.5 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
|
|
151
|
+
| **1000D Non-Convex Ravine** ($\kappa=10,000$) | Anisotropic manifold (500 steps) | **CLOVES-Adaptive**<br>AdamW<br>**CLOVES-LLM**<br>Official LION | Objective Loss: **17.12** (-92.0%)<br>Objective Loss: 213.23 (Baseline)<br>Objective Loss: **12,297.97** (-60.2%)<br>Objective Loss: 30,861.02 (Baseline) | **1.21 ms / step**<br>1.18 ms / step<br>0.94 ms / step<br>0.92 ms / step | **0.61 s**<br>0.59 s<br>0.47 s<br>0.46 s | < 15 MB<br>< 15 MB<br>< 15 MB<br>< 15 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
|
|
152
|
+
| **LLaMA-3-8B Instruction Tuning** | 4-bit QLoRA, Alpaca (500 steps) | **CLOVES-Perpendicular**<br>CLOVES-Matrix-Polar<br>AdamW (Baseline)<br>Official Muon | Val Loss: **5.7338**<br>Val Loss: **5.7822**<br>Val Loss: 5.7269<br>Val Loss: 5.7941 | **12.01 s / step**<br>12.48 s / step<br>11.71 s / step<br>12.52 s / step | **16,010.2 s** (~4.44 h)<br>16,485.3 s (~4.58 h)<br>16,376.8 s (~4.55 h)<br>16,428.7 s (~4.56 h) | **10,594.5 MB**<br>10,594.5 MB<br>14,349.5 MB<br>14,189.5 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
|
|
153
|
+
| **Qwen-1.5B Instruction Tuning** | QLoRA, Alpaca (1000 steps) | **CLOVES-Apex (Tuned)**<br>AdamW (Baseline) | Val Loss: **0.4295**<br>Val Loss: 0.4298 | 562.8 ms / step<br>470.3 ms / step | **2,681.9 s** (~44.7 min)<br>2,648.4 s (~44.1 min) | **8,202.3 MB**<br>8,202.3 MB | **2-State (8 B)**<br>2-State (8 B) |
|
|
154
|
+
| **Qwen-7B Instruction Tuning** | QLoRA, Alpaca (500 steps) | **CLOVES-Apex (Annealed)**<br>AdamW (Baseline)<br>**CLOVES-Sign-Apex**<br>Official LION (Baseline) | Val Loss: **0.5573**<br>Val Loss: 0.5583<br>Val Loss: **0.6249**<br>Val Loss: 0.6365 | 2,956.0 ms / step<br>2,746.3 ms / step<br>2,732.9 ms / step<br>2,714.6 ms / step | **3,911.1 s** (~65.1 min)<br>3,871.5 s (~64.5 min)<br>3,624.2 s (~60.4 min)<br>3,613.7 s (~60.2 min) | **10,793.6 MB** (-3.36 GB)<br>14,159.3 MB<br>**10,639.6 MB** (-3.37 GB)<br>14,005.3 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
|
|
155
|
+
|
|
156
|
+
### Pareto Frontier: Loss vs. Memory vs. Runtime Trade-off
|
|
157
|
+
|
|
158
|
+
The empirical evaluations demonstrate favorable trade-offs across memory footprint, runtime efficiency, and convergence loss:
|
|
159
|
+
|
|
160
|
+
<p align="center">
|
|
161
|
+
<img src="docs/images/cloves_pareto_frontier_master_clean.png" alt="CLOVES Pareto Frontier" width="85%">
|
|
162
|
+
</p>
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## 📊 Optimizer Selection Matrix
|
|
167
|
+
|
|
168
|
+
| Optimizer Class | Target Workload | State Buffers | Memory Footprint | Special Characteristics |
|
|
169
|
+
| :--- | :--- | :---: | :---: | :--- |
|
|
170
|
+
| `CLOVES` | Vision (CNNs, ResNet) | 1 ($m$) | 4 bytes / param | Angular bisector damping; low memory |
|
|
171
|
+
| `CLOVES_Perpendicular` | Highly ill-conditioned loss | 1 ($m$) | 4 bytes / param | Orthogonal wall force rejection |
|
|
172
|
+
| `CLOVES_Affine` | Deep classification networks | 1 ($m$) | 4 bytes / param | Continuous affine subspace projection |
|
|
173
|
+
| `CLOVES_Adaptive` | LLMs, Transformers | 2 ($m, v$) | 8 bytes / param | Coordinate cleft overstep + RMS scaling |
|
|
174
|
+
| `CLOVES_LLM` | Memory-constrained LLMs | 1 ($m$) | 4 bytes / param | Sign overstep; 50% state memory savings |
|
|
175
|
+
| `CLOVES_Apex_Asymmetric` | LoRA / QLoRA Instruction Tuning | 2 ($m, v$) | 8 bytes / param | Matrix A/B ratio, soft tanh damping |
|
|
176
|
+
| `CLOVES_Matrix_Polar` | Linear layer weights (2D) | 1 ($m$) | 4 bytes / param | Newton-Schulz quintic polar decomposition |
|
|
177
|
+
|
|
178
|
+
---
|
|
179
|
+
|
|
180
|
+
## 📚 References & Baseline Implementations
|
|
181
|
+
|
|
182
|
+
CLOVES benchmarks are evaluated against official implementations of standard and contemporary optimizers:
|
|
183
|
+
|
|
184
|
+
1. **AdamW**: Loshchilov & Hutter, *Decoupled Weight Decay Regularization*, ICLR 2019.
|
|
185
|
+
2. **Lion** (*EvoLved Sign Momentum*): Chen et al., *Symbolic Discovery of Optimization Algorithms*, NeurIPS 2023. arXiv: [2302.06675](https://arxiv.org/abs/2302.06675).
|
|
186
|
+
- Code: [google-research/lion](https://github.com/google-research/google-research/tree/master/lion) | [lucidrains/lion-pytorch](https://github.com/lucidrains/lion-pytorch)
|
|
187
|
+
3. **Muon** (*Momentum Orthogonalized by Newton-schulz*): Keller Jordan et al., *Muon: An Optimizer for Hidden Layers in Neural Networks*, 2024.
|
|
188
|
+
- Code: [KellerJordan/Muon](https://github.com/KellerJordan/Muon)
|
|
189
|
+
|
|
190
|
+
---
|
|
191
|
+
|
|
192
|
+
## 🏛️ Research Inspiration & Lineage
|
|
193
|
+
|
|
194
|
+
The algorithmic concepts in CLOVES are **inspired by** foundational ravine traversal theories in mathematical programming (Nguyễn & Bùi, 1990) and subsequent exploratory studies evaluating cleft-overstep heuristics in machine learning tasks (Phan & Lê, 2020).
|
|
195
|
+
|
|
196
|
+
### Pragmatic Adaptation from Theory to Practice
|
|
197
|
+
The original cleft-overstep formulations were conceived within classical, low-dimensional, deterministic optimization settings. Translating these geometric concepts to modern deep learning and large-scale language models ($d \gg 10^7$) inherently involves **empirical, heuristic (ad-hoc) adaptations**:
|
|
198
|
+
- The authors make **no claim of 100% formal mathematical equivalence** or complete theoretical convergence proofs matching the original deterministic algorithms.
|
|
199
|
+
- Instead, CLOVES adapts the core geometric intuition—*damping transversal wall-to-wall bouncing while accelerating along the longitudinal ravine bed*—into efficient, zero-allocation, fused PyTorch tensor operations tailored for stochastic mini-batch gradients.
|
|
200
|
+
- Extensions such as coordinate-wise variance scaling (`CLOVES_Adaptive`), 1-state sign momentum (`CLOVES_LLM`), asymmetric LoRA velocity ratios (`CLOVES_Apex_Asymmetric`), and polar matrix decomposition (`CLOVES_Matrix_Polar`) are practical designs driven by empirical deep learning dynamics.
|
|
201
|
+
|
|
202
|
+
**Predecessor & Historical References**:
|
|
203
|
+
1. Nguyễn Văn Mạnh & Bùi Minh Trí, *"Methods for finding ravine directions in non-differentiable optimization,"* *Acta Mathematica Vietnamica*, 1990.
|
|
204
|
+
2. Duy-Hung Phan & Le Dinh Huynh, *"Evaluation of the Cleft-Overstep Algorithm for Linear Regression Analysis,"* in *Computational Collective Intelligence (ICCCI 2020)*, Lecture Notes in Computer Science (LNCS, vol. 12496), pp. 400–411, Springer, 2020. DOI: [10.1007/978-3-030-63007-2_31](https://doi.org/10.1007/978-3-030-63007-2_31).
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## 🤝 Feedback & Contributions
|
|
209
|
+
|
|
210
|
+
CLOVES is an empirical project developed with an open mindset. If you experiment with CLOVES on your models and find it helpful—or discover interesting edge cases—we would love to hear from you!
|
|
211
|
+
|
|
212
|
+
Bug reports, critiques, and pull requests are warmly welcomed via [GitHub Issues](https://github.com/huynhldvn/CLOVES/issues) or [Discussions](https://github.com/huynhldvn/CLOVES/discussions).
|
|
213
|
+
|
|
214
|
+
---
|
|
215
|
+
|
|
216
|
+
## 🙏 Acknowledgements
|
|
217
|
+
|
|
218
|
+
We thank **Kaggle** for providing the free GPU compute (Tesla T4) for our benchmarks, and **Antigravity** for assisting with code refactoring and documentation.
|
|
219
|
+
|
|
220
|
+
---
|
|
221
|
+
|
|
222
|
+
## 📄 License
|
|
223
|
+
|
|
224
|
+
MIT License. See [LICENSE](LICENSE) for details.
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=61.0.0", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "cloves"
|
|
7
|
+
version = "2.0.0"
|
|
8
|
+
description = "Cleft-Overstep Optimization for Deep Neural Networks & LLMs"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.8"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
keywords = [
|
|
13
|
+
"deep-learning",
|
|
14
|
+
"optimizer",
|
|
15
|
+
"pytorch",
|
|
16
|
+
"optimization",
|
|
17
|
+
"llm",
|
|
18
|
+
"cleft-overstep",
|
|
19
|
+
"adamw",
|
|
20
|
+
"lion",
|
|
21
|
+
"muon"
|
|
22
|
+
]
|
|
23
|
+
classifiers = [
|
|
24
|
+
"Development Status :: 5 - Production/Stable",
|
|
25
|
+
"Intended Audience :: Science/Research",
|
|
26
|
+
"Intended Audience :: Developers",
|
|
27
|
+
"Programming Language :: Python :: 3",
|
|
28
|
+
"Programming Language :: Python :: 3.8",
|
|
29
|
+
"Programming Language :: Python :: 3.9",
|
|
30
|
+
"Programming Language :: Python :: 3.10",
|
|
31
|
+
"Programming Language :: Python :: 3.11",
|
|
32
|
+
"Programming Language :: Python :: 3.12",
|
|
33
|
+
"Programming Language :: Python :: 3.13",
|
|
34
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
35
|
+
"Topic :: Software Development :: Libraries :: Python Modules"
|
|
36
|
+
]
|
|
37
|
+
dependencies = [
|
|
38
|
+
"torch>=1.12.0"
|
|
39
|
+
]
|
|
40
|
+
|
|
41
|
+
[project.optional-dependencies]
|
|
42
|
+
dev = [
|
|
43
|
+
"pytest>=7.0.0",
|
|
44
|
+
"build",
|
|
45
|
+
"twine"
|
|
46
|
+
]
|
|
47
|
+
|
|
48
|
+
[project.urls]
|
|
49
|
+
Homepage = "https://github.com/huynhldvn/CLOVES"
|
|
50
|
+
Documentation = "https://github.com/huynhldvn/CLOVES#readme"
|
|
51
|
+
Repository = "https://github.com/huynhldvn/CLOVES.git"
|
|
52
|
+
Issues = "https://github.com/huynhldvn/CLOVES/issues"
|
|
53
|
+
|
|
54
|
+
[tool.setuptools.packages.find]
|
|
55
|
+
where = ["src"]
|
|
56
|
+
|
|
57
|
+
[tool.pytest.ini_options]
|
|
58
|
+
pythonpath = ["src"]
|
|
59
|
+
testpaths = ["tests"]
|
cloves-2.0.0/setup.cfg
ADDED
cloves-2.0.0/setup.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
"""
|
|
2
|
+
CLOVES: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
|
|
3
|
+
"""
|
|
4
|
+
|
|
5
|
+
from ._version import __version__
|
|
6
|
+
from .bisector import CLOVES, CLOVES_Bisector
|
|
7
|
+
from .perpendicular import CLOVES_Perpendicular
|
|
8
|
+
from .affine import CLOVES_Affine
|
|
9
|
+
from .adaptive import CLOVES_Adaptive
|
|
10
|
+
from .llm import CLOVES_LLM
|
|
11
|
+
from .apex import CLOVES_Apex_Asymmetric
|
|
12
|
+
from .matrix_polar import CLOVES_Matrix_Polar
|
|
13
|
+
|
|
14
|
+
__all__ = [
|
|
15
|
+
"__version__",
|
|
16
|
+
"CLOVES",
|
|
17
|
+
"CLOVES_Bisector",
|
|
18
|
+
"CLOVES_Perpendicular",
|
|
19
|
+
"CLOVES_Affine",
|
|
20
|
+
"CLOVES_Adaptive",
|
|
21
|
+
"CLOVES_LLM",
|
|
22
|
+
"CLOVES_Apex_Asymmetric",
|
|
23
|
+
"CLOVES_Matrix_Polar",
|
|
24
|
+
]
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "2.0.0"
|