cloves 2.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
cloves-2.0.1/LICENSE ADDED
@@ -0,0 +1 @@
1
+ MIT License
cloves-2.0.1/PKG-INFO ADDED
@@ -0,0 +1,256 @@
1
+ Metadata-Version: 2.4
2
+ Name: cloves
3
+ Version: 2.0.1
4
+ Summary: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
5
+ License-Expression: MIT
6
+ Project-URL: Homepage, https://github.com/huynhldvn/CLOVES
7
+ Project-URL: Documentation, https://github.com/huynhldvn/CLOVES#readme
8
+ Project-URL: Repository, https://github.com/huynhldvn/CLOVES.git
9
+ Project-URL: Issues, https://github.com/huynhldvn/CLOVES/issues
10
+ Keywords: deep-learning,optimizer,pytorch,optimization,llm,cleft-overstep,adamw,lion,muon
11
+ Classifier: Development Status :: 5 - Production/Stable
12
+ Classifier: Intended Audience :: Science/Research
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: Programming Language :: Python :: 3
15
+ Classifier: Programming Language :: Python :: 3.8
16
+ Classifier: Programming Language :: Python :: 3.9
17
+ Classifier: Programming Language :: Python :: 3.10
18
+ Classifier: Programming Language :: Python :: 3.11
19
+ Classifier: Programming Language :: Python :: 3.12
20
+ Classifier: Programming Language :: Python :: 3.13
21
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
22
+ Classifier: Topic :: Software Development :: Libraries :: Python Modules
23
+ Requires-Python: >=3.8
24
+ Description-Content-Type: text/markdown
25
+ License-File: LICENSE
26
+ Requires-Dist: torch>=1.12.0
27
+ Provides-Extra: dev
28
+ Requires-Dist: pytest>=7.0.0; extra == "dev"
29
+ Requires-Dist: build; extra == "dev"
30
+ Requires-Dist: twine; extra == "dev"
31
+ Dynamic: license-file
32
+
33
+ # CLOVES: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
34
+
35
+ [![PyPI version](https://img.shields.io/pypi/v/cloves.svg)](https://pypi.org/project/cloves/)
36
+ [![Python Versions](https://img.shields.io/pypi/pyversions/cloves.svg)](https://pypi.org/project/cloves/)
37
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/huynhldvn/CLOVES/blob/main/LICENSE)
38
+ [![GitHub stars](https://img.shields.io/github/stars/huynhldvn/CLOVES.svg)](https://github.com/huynhldvn/CLOVES)
39
+
40
+ **CLOVES** (*Cleft-Overstep*) is an optimizer family designed for deep neural networks and large language models (LLMs). It suppresses transversal oscillations across steep canyon walls while accelerating descent along narrow valley floors.
41
+
42
+ <p align="center">
43
+ <img src="https://raw.githubusercontent.com/huynhldvn/CLOVES/main/docs/images/cloves_optimization_dynamics_2d_3d.gif" alt="CLOVES 2D & 3D Optimization Dynamics Animation" width="100%">
44
+ </p>
45
+ <p align="center">
46
+ <em>Figure 1: Step-by-step optimization dynamics in 2D contour and 3D canyon space (<a href="https://github.com/huynhldvn/CLOVES/blob/main/scripts/generate_optimization_animation.py">reproducible via scripts/generate_optimization_animation.py</a>). While standard optimizers (AdamW, LION, SGD-M) oscillate across the steep canyon walls, CLOVES-Perpendicular and CLOVES-Bisector damp transversal bouncing and follow the valley floor toward the optimum.</em>
47
+ </p>
48
+
49
+ ---
50
+
51
+ ## 📦 Installation
52
+
53
+ Install the package directly via pip:
54
+
55
+ ```bash
56
+ pip install cloves
57
+ ```
58
+
59
+ For development and running benchmarks:
60
+
61
+ ```bash
62
+ git clone https://github.com/huynhldvn/CLOVES.git
63
+ cd CLOVES
64
+ pip install -e ".[dev]"
65
+ ```
66
+
67
+ ---
68
+
69
+ ## 📓 Interactive Notebooks & 1-Click Demos
70
+
71
+ Try CLOVES immediately in Google Colab or Jupyter without local setup:
72
+
73
+ <p align="center">
74
+ <a href="https://colab.research.google.com/github/huynhldvn/CLOVES/blob/main/notebooks/quickstart_interactive_demo.ipynb">
75
+ <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
76
+ </a>
77
+ </p>
78
+
79
+ - **Quickstart Demo**: [`notebooks/quickstart_interactive_demo.ipynb`](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/quickstart_interactive_demo.ipynb) — 1-click Colab interactive simulation & PyTorch training.
80
+ - **Dedicated Benchmarks**: Check out [notebooks/](https://github.com/huynhldvn/CLOVES/tree/main/notebooks) for focused notebooks on [CIFAR-10](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/01_cifar10_vision_benchmark.ipynb), [LLM QLoRA](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/02_llm_qlora_tuning_benchmark.ipynb), and [2D Ravine Simulation](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/03_2d_ravine_simulation.ipynb).
81
+ - **Standalone Scripts**: Run minimal scripts in [examples/](https://github.com/huynhldvn/CLOVES/tree/main/examples): `examples/01_quickstart_vision.py` and `examples/02_quickstart_llm.py`.
82
+
83
+ ---
84
+
85
+ ## 🎯 Which Optimizer Should You Use? (Practical Guide)
86
+
87
+ Choose the right variant based on your workload:
88
+
89
+ | Workload / Architecture | Recommended Optimizer | Key Advantage | State Memory | Recommended Settings |
90
+ | :--- | :--- | :--- | :---: | :--- |
91
+ | 🤖 **LLM LoRA / QLoRA Tuning** | `CLOVES_Apex_Asymmetric` ⭐ | **Recommended for LoRA**. Automatically accelerates Matrix A ($q_A=1.30$) and stabilizes Matrix B ($q_B=1.05$). Matches or slightly improves on AdamW loss while saving **> 3.3 GB VRAM**. | 2 States (8 B) | `lr=1.6e-4`, `total_steps=500` |
92
+ | 🧠 **LLM Full Tuning / Transformers** | `CLOVES_Adaptive` | **Drop-in replacement for AdamW**. Damps coordinate oscillations, maintains stability on deep networks, reduces synthetic ravine loss by 92%. | 2 States (8 B) | `lr=2e-4`, `q=1.25`, `gamma=0.6` |
93
+ | ⚡ **Extreme Memory Constraints (LLM)** | `CLOVES_LLM` | **50% state memory savings** (4 B/param), eliminates LION's coordinate-wise loss drift. | 1 State (4 B) | `lr=1e-4`, `q=1.20`, `gamma=0.6` |
94
+ | 👁️ **Standard Vision (CNN, ResNet, ViT)** | `CLOVES_Perpendicular` ⭐ | **Recommended for vision tasks**. Rejects orthogonal transverse wall force ($g_\perp$), reaches **93.07%** on CIFAR-10, runs with light 1-state memory. | 1 State (4 B) | `lr=0.05 - 0.1`, `beta=0.9` |
95
+ | 📈 **Tabular / Smooth Convex Tasks** | `CLOVES` (Bisector) | Fastest computational throughput, follows the ravine floor along the angular bisector. | 1 State (4 B) | `lr=0.1`, `q=1.5`, `gamma=0.1` |
96
+
97
+ > ⚠️ **Scope & Evaluation Note:**
98
+ > The LLM-optimized variants (`CLOVES_Apex_Asymmetric`, `CLOVES_Adaptive`, `CLOVES_LLM`, `CLOVES_Matrix_Polar`) were designed and validated specifically on Transformer and LoRA / QLoRA architectures (LLaMA-3, Qwen-1.5B, Qwen-7B). **These LLM variants have not yet been evaluated on other tasks (such as computer vision, audio, or tabular data)**.
99
+ > 👉 For computer vision and general classification workloads, use `CLOVES_Perpendicular` or `CLOVES` (Bisector), which have been thoroughly evaluated on vision benchmarks.
100
+
101
+ ---
102
+
103
+ ## ⚡ Quickstart
104
+
105
+ ### 1. Minimal Runnable Snippet (Copy & Run)
106
+ Test CLOVES immediately in any Python environment:
107
+
108
+ ```python
109
+ import torch
110
+ import torch.nn as nn
111
+ from cloves import CLOVES_Perpendicular
112
+
113
+ # Define a minimal model & optimizer
114
+ model = nn.Linear(10, 2)
115
+ optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05)
116
+
117
+ # Standard training step
118
+ x, y = torch.randn(16, 10), torch.randn(16, 2)
119
+ loss = nn.MSELoss()(model(x), y)
120
+ loss.backward()
121
+ optimizer.step()
122
+ optimizer.zero_grad()
123
+
124
+ print("Step completed! Loss:", float(loss.detach()))
125
+ ```
126
+
127
+ ### 2. Drop-in Replacement for Real Models
128
+ Simply swap your existing optimizer in your training loop:
129
+
130
+ ```python
131
+ # A. Vision (ResNet, CNN, ViT) — lightweight 1-state memory:
132
+ from cloves import CLOVES_Perpendicular
133
+ optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05, beta=0.9)
134
+
135
+ # B. LLM & LoRA Fine-Tuning — saves >3.3 GB VRAM:
136
+ from cloves import CLOVES_Apex_Asymmetric
137
+ optimizer = CLOVES_Apex_Asymmetric(model.parameters(), lr=1.6e-4, total_steps=500)
138
+ ```
139
+
140
+ ---
141
+
142
+ ## 🚀 Milestone Versions & Experimental Reports
143
+
144
+ CLOVES bridges classical first-order valley descent theory into large-scale stochastic deep learning and LLM fine-tuning:
145
+
146
+ ```
147
+ THE CLOVES OPTIMIZATION PHILOSOPHY
148
+ (Transversal Wall Damping γ < 1 & Longitudinal Floor Overstep q > 1)
149
+ │
150
+ ┌────────────────────────────────┴────────────────────────────────┐
151
+ ▼ ▼
152
+ [MILESTONE 1: DEEP VISION & 1-STATE PARITY] [MILESTONE 2: LLM & ADAPTIVE SCALING]
153
+ • Problem: Wall chattering in narrow ravines • Problem: Small-norm stagnation (||g|| ~ 1e-4)
154
+ • Memory: Strict 4 bytes/param (1-state) • Memory: 50% state memory savings or RMS scaling
155
+ • Mechanism: Angular bisector & orthogonal • Mechanism: Coordinate overstep, LoRA A/B
156
+ transversal force rejection (g_perp) asymmetric velocity, Newton-Schulz polar
157
+ │ │
158
+ ▼ ▼
159
+ CLOVES-Bisector / CLOVES-Perpendicular CLOVES-Apex / CLOVES-Adaptive
160
+ (Strong vision accuracy, 50% state memory) (Matches AdamW, beats Muon on QLoRA)
161
+ ```
162
+
163
+ The project is organized into two developmental milestones:
164
+ - **[Version 1.0 — Geometric Vision & Ravine Damping](https://github.com/huynhldvn/CLOVES/tree/main/benchmarks/v1_geometric_vision)**: First-order directional cleft damping (Bisector, Perpendicular, Affine) maintaining strict 1-state memory parity (4 bytes/param) for vision architectures (ResNet-18, DenseNet-121).
165
+ - **[Version 2.0 — LLM Fine-Tuning & Adaptive Cleft Scaling](https://github.com/huynhldvn/CLOVES/tree/main/benchmarks/v2_llm_adaptive)**: Addresses small-gradient stagnation ($\|g\| \sim 10^{-4}$) via coordinate cleft alignment, adaptive second-moment curvature scaling, and asymmetric LoRA velocity for Transformer architectures (LLaMA-3, Qwen-1.5B, Qwen-7B).
166
+
167
+ ---
168
+
169
+ ## 📈 Comprehensive Master Benchmark Results
170
+
171
+ The table below summarizes empirical validations across synthetic optimization ravines, deep vision architectures, and large language model instruction tuning suites, including convergence accuracy/loss, runtime per step/epoch, total training duration, and GPU memory consumption (measured on NVIDIA Tesla T4 GPUs):
172
+
173
+ > [!NOTE]
174
+ > **Evaluation Protocol & Objectivity**: To guarantee rigorous, unbiased comparison and avoid overfitting artifacts, all deep learning benchmarks (CIFAR-10, CIFAR-100, LLaMA-3, Qwen) are strictly evaluated on held-out **Validation Sets** (`val_loss` and `val_acc`), never on training batches. Synthetic ravine benchmarks evaluate the exact mathematical objective loss function $f(x)$ under identical random seeds and initialization.
175
+ >
176
+ > **Metric Interpretation**: **↑** = Higher is better &nbsp;|&nbsp; **↓** = Lower is better
177
+
178
+ | Benchmark Suite & Dataset | Architecture / Setup | Optimizer | Convergence Metric<br>(Val Loss ↓ / Val Acc ↑) | Step / Epoch Time<br>(↓) | Total Execution Time<br>(↓) | Peak GPU VRAM<br>(↓) | Optimizer State<br>(↓) |
179
+ | :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
180
+ | **2D Ill-Conditioned Ravine** ($\kappa=100$) | Quadratic canyon (250 steps) | **CLOVES-Bisector**<br>AdamW<br>LION | Objective Loss: **$7.11 \times 10^{-5}$**<br>Objective Loss: 20.20<br>Objective Loss: 24.52 | **0.82 ms / step**<br>1.14 ms / step<br>0.89 ms / step | **0.41 s**<br>0.57 s<br>0.45 s | < 10 MB<br>< 10 MB<br>< 10 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
181
+ | **CIFAR-10 Vision** (Standard Vision) | ResNet-18 (20 Epochs, bs=128) | **CLOVES-Perpendicular**<br>CLOVES-Affine<br>AdamW<br>Official LION | Val Acc: **93.07%** (Val Loss: 0.2237)<br>Val Acc: 92.51% (Val Loss: 0.2310)<br>Val Acc: 92.35% (Val Loss: 0.2512)<br>Val Acc: 89.20% (Val Loss: 0.3840) | **51.1 s / epoch**<br>51.6 s / epoch<br>61.4 s / epoch<br>51.0 s / epoch | **1,022.7 s** (~17.0 min)<br>1,032.0 s (~17.2 min)<br>1,228.0 s (~20.5 min)<br>1,020.0 s (~17.0 min) | **696.0 MB**<br>696.0 MB<br>827.8 MB<br>696.0 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
182
+ | **CIFAR-100 Deep Vision** (100 Classes) | DenseNet-121 (30 Epochs, Cosine) | **CLOVES (Proposed)**<br>AdamW<br>SGD-M | Top-1 Val Acc: **73.98%** (Val Loss: 1.3993)<br>Top-1 Val Acc: 72.77% (Val Loss: 1.4435)<br>Top-1 Val Acc: 72.56% (Val Loss: 1.4304) | **132.5 s / epoch**<br>133.1 s / epoch<br>131.8 s / epoch | **3,975.0 s** (~66.2 min)<br>3,993.0 s (~66.5 min)<br>3,954.0 s (~65.9 min) | **5,001.5 MB**<br>5,028.4 MB<br>5,001.5 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
183
+ | **1000D Non-Convex Ravine** ($\kappa=10,000$) | Anisotropic manifold (500 steps) | **CLOVES-Adaptive**<br>AdamW<br>**CLOVES-LLM**<br>Official LION | Objective Loss: **17.12** (-92.0%)<br>Objective Loss: 213.23 (Baseline)<br>Objective Loss: **12,297.97** (-60.2%)<br>Objective Loss: 30,861.02 (Baseline) | **1.21 ms / step**<br>1.18 ms / step<br>0.94 ms / step<br>0.92 ms / step | **0.61 s**<br>0.59 s<br>0.47 s<br>0.46 s | < 15 MB<br>< 15 MB<br>< 15 MB<br>< 15 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
184
+ | **LLaMA-3-8B Instruction Tuning** | 4-bit QLoRA, Alpaca (500 steps) | **CLOVES-Perpendicular**<br>CLOVES-Matrix-Polar<br>AdamW (Baseline)<br>Official Muon | Val Loss: **5.7338**<br>Val Loss: **5.7822**<br>Val Loss: 5.7269<br>Val Loss: 5.7941 | **12.01 s / step**<br>12.48 s / step<br>11.71 s / step<br>12.52 s / step | **16,010.2 s** (~4.44 h)<br>16,485.3 s (~4.58 h)<br>16,376.8 s (~4.55 h)<br>16,428.7 s (~4.56 h) | **10,594.5 MB**<br>10,594.5 MB<br>14,349.5 MB<br>14,189.5 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
185
+ | **Qwen-1.5B Instruction Tuning** | QLoRA, Alpaca (1000 steps) | **CLOVES-Apex (Tuned)**<br>AdamW (Baseline) | Val Loss: **0.4295**<br>Val Loss: 0.4298 | 562.8 ms / step<br>470.3 ms / step | **2,681.9 s** (~44.7 min)<br>2,648.4 s (~44.1 min) | **8,202.3 MB**<br>8,202.3 MB | **2-State (8 B)**<br>2-State (8 B) |
186
+ | **Qwen-7B Instruction Tuning** | QLoRA, Alpaca (500 steps) | **CLOVES-Apex (Annealed)**<br>AdamW (Baseline)<br>**CLOVES-Sign-Apex**<br>Official LION (Baseline) | Val Loss: **0.5573**<br>Val Loss: 0.5583<br>Val Loss: **0.6249**<br>Val Loss: 0.6365 | 2,956.0 ms / step<br>2,746.3 ms / step<br>2,732.9 ms / step<br>2,714.6 ms / step | **3,911.1 s** (~65.1 min)<br>3,871.5 s (~64.5 min)<br>3,624.2 s (~60.4 min)<br>3,613.7 s (~60.2 min) | **10,793.6 MB** (-3.36 GB)<br>14,159.3 MB<br>**10,639.6 MB** (-3.37 GB)<br>14,005.3 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
187
+
188
+ ### Pareto Frontier: Loss vs. Memory vs. Runtime Trade-off
189
+
190
+ The empirical evaluations demonstrate favorable trade-offs across memory footprint, runtime efficiency, and convergence loss:
191
+
192
+ <p align="center">
193
+ <img src="https://raw.githubusercontent.com/huynhldvn/CLOVES/main/docs/images/cloves_pareto_frontier_master_clean.png" alt="CLOVES Pareto Frontier" width="85%">
194
+ </p>
195
+
196
+ ---
197
+
198
+ ## 📊 Optimizer Selection Matrix
199
+
200
+ | Optimizer Class | Target Workload | State Buffers | Memory Footprint | Special Characteristics |
201
+ | :--- | :--- | :---: | :---: | :--- |
202
+ | `CLOVES` | Vision (CNNs, ResNet) | 1 ($m$) | 4 bytes / param | Angular bisector damping; low memory |
203
+ | `CLOVES_Perpendicular` | Highly ill-conditioned loss | 1 ($m$) | 4 bytes / param | Orthogonal wall force rejection |
204
+ | `CLOVES_Affine` | Deep classification networks | 1 ($m$) | 4 bytes / param | Continuous affine subspace projection |
205
+ | `CLOVES_Adaptive` | LLMs, Transformers | 2 ($m, v$) | 8 bytes / param | Coordinate cleft overstep + RMS scaling |
206
+ | `CLOVES_LLM` | Memory-constrained LLMs | 1 ($m$) | 4 bytes / param | Sign overstep; 50% state memory savings |
207
+ | `CLOVES_Apex_Asymmetric` | LoRA / QLoRA Instruction Tuning | 2 ($m, v$) | 8 bytes / param | Matrix A/B ratio, soft tanh damping |
208
+ | `CLOVES_Matrix_Polar` | Linear layer weights (2D) | 1 ($m$) | 4 bytes / param | Newton-Schulz quintic polar decomposition |
209
+
210
+ ---
211
+
212
+ ## 📚 References & Baseline Implementations
213
+
214
+ CLOVES benchmarks are evaluated against official implementations of standard and contemporary optimizers:
215
+
216
+ 1. **AdamW**: Loshchilov & Hutter, *Decoupled Weight Decay Regularization*, ICLR 2019.
217
+ 2. **Lion** (*EvoLved Sign Momentum*): Chen et al., *Symbolic Discovery of Optimization Algorithms*, NeurIPS 2023. arXiv: [2302.06675](https://arxiv.org/abs/2302.06675).
218
+ - Code: [google-research/lion](https://github.com/google-research/google-research/tree/master/lion) | [lucidrains/lion-pytorch](https://github.com/lucidrains/lion-pytorch)
219
+ 3. **Muon** (*Momentum Orthogonalized by Newton-schulz*): Keller Jordan et al., *Muon: An Optimizer for Hidden Layers in Neural Networks*, 2024.
220
+ - Code: [KellerJordan/Muon](https://github.com/KellerJordan/Muon)
221
+
222
+ ---
223
+
224
+ ## 🏛️ Research Inspiration & Lineage
225
+
226
+ The algorithmic concepts in CLOVES are **inspired by** foundational ravine traversal theories in mathematical programming (Nguyễn & Bùi, 1990) and subsequent exploratory studies evaluating cleft-overstep heuristics in machine learning tasks (Phan & Lê, 2020).
227
+
228
+ ### Pragmatic Adaptation from Theory to Practice
229
+ The original cleft-overstep formulations were conceived within classical, low-dimensional, deterministic optimization settings. Translating these geometric concepts to modern deep learning and large-scale language models ($d \gg 10^7$) inherently involves **empirical, heuristic (ad-hoc) adaptations**:
230
+ - The authors make **no claim of 100% formal mathematical equivalence** or complete theoretical convergence proofs matching the original deterministic algorithms.
231
+ - Instead, CLOVES adapts the core geometric intuition—*damping transversal wall-to-wall bouncing while accelerating along the longitudinal ravine bed*—into efficient, zero-allocation, fused PyTorch tensor operations tailored for stochastic mini-batch gradients.
232
+ - Extensions such as coordinate-wise variance scaling (`CLOVES_Adaptive`), 1-state sign momentum (`CLOVES_LLM`), asymmetric LoRA velocity ratios (`CLOVES_Apex_Asymmetric`), and polar matrix decomposition (`CLOVES_Matrix_Polar`) are practical designs driven by empirical deep learning dynamics.
233
+
234
+ **Predecessor & Historical References**:
235
+ 1. Nguyễn Văn Mạnh & Bùi Minh Trí, *"Methods for finding ravine directions in non-differentiable optimization,"* *Acta Mathematica Vietnamica*, 1990.
236
+ 2. Duy-Hung Phan & Le Dinh Huynh, *"Evaluation of the Cleft-Overstep Algorithm for Linear Regression Analysis,"* in *Computational Collective Intelligence (ICCCI 2020)*, Lecture Notes in Computer Science (LNCS, vol. 12496), pp. 400–411, Springer, 2020. DOI: [10.1007/978-3-030-63007-2_31](https://doi.org/10.1007/978-3-030-63007-2_31).
237
+
238
+ ---
239
+
240
+ ## 🤝 Feedback & Contributions
241
+
242
+ CLOVES is an empirical project developed with an open mindset. If you experiment with CLOVES on your models and find it helpful—or discover interesting edge cases—we would love to hear from you!
243
+
244
+ Bug reports, critiques, and pull requests are warmly welcomed via [GitHub Issues](https://github.com/huynhldvn/CLOVES/issues) or [Discussions](https://github.com/huynhldvn/CLOVES/discussions).
245
+
246
+ ---
247
+
248
+ ## 🙏 Acknowledgements
249
+
250
+ We thank **Kaggle** for providing the free GPU compute (Tesla T4) for our benchmarks, and **Antigravity** for assisting with code refactoring and documentation.
251
+
252
+ ---
253
+
254
+ ## 📄 License
255
+
256
+ MIT License. See [LICENSE](https://github.com/huynhldvn/CLOVES/blob/main/LICENSE) for details.
cloves-2.0.1/README.md ADDED
@@ -0,0 +1,224 @@
1
+ # CLOVES: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
2
+
3
+ [![PyPI version](https://img.shields.io/pypi/v/cloves.svg)](https://pypi.org/project/cloves/)
4
+ [![Python Versions](https://img.shields.io/pypi/pyversions/cloves.svg)](https://pypi.org/project/cloves/)
5
+ [![License: MIT](https://img.shields.io/badge/License-MIT-blue.svg)](https://github.com/huynhldvn/CLOVES/blob/main/LICENSE)
6
+ [![GitHub stars](https://img.shields.io/github/stars/huynhldvn/CLOVES.svg)](https://github.com/huynhldvn/CLOVES)
7
+
8
+ **CLOVES** (*Cleft-Overstep*) is an optimizer family designed for deep neural networks and large language models (LLMs). It suppresses transversal oscillations across steep canyon walls while accelerating descent along narrow valley floors.
9
+
10
+ <p align="center">
11
+ <img src="https://raw.githubusercontent.com/huynhldvn/CLOVES/main/docs/images/cloves_optimization_dynamics_2d_3d.gif" alt="CLOVES 2D & 3D Optimization Dynamics Animation" width="100%">
12
+ </p>
13
+ <p align="center">
14
+ <em>Figure 1: Step-by-step optimization dynamics in 2D contour and 3D canyon space (<a href="https://github.com/huynhldvn/CLOVES/blob/main/scripts/generate_optimization_animation.py">reproducible via scripts/generate_optimization_animation.py</a>). While standard optimizers (AdamW, LION, SGD-M) oscillate across the steep canyon walls, CLOVES-Perpendicular and CLOVES-Bisector damp transversal bouncing and follow the valley floor toward the optimum.</em>
15
+ </p>
16
+
17
+ ---
18
+
19
+ ## 📦 Installation
20
+
21
+ Install the package directly via pip:
22
+
23
+ ```bash
24
+ pip install cloves
25
+ ```
26
+
27
+ For development and running benchmarks:
28
+
29
+ ```bash
30
+ git clone https://github.com/huynhldvn/CLOVES.git
31
+ cd CLOVES
32
+ pip install -e ".[dev]"
33
+ ```
34
+
35
+ ---
36
+
37
+ ## 📓 Interactive Notebooks & 1-Click Demos
38
+
39
+ Try CLOVES immediately in Google Colab or Jupyter without local setup:
40
+
41
+ <p align="center">
42
+ <a href="https://colab.research.google.com/github/huynhldvn/CLOVES/blob/main/notebooks/quickstart_interactive_demo.ipynb">
43
+ <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab">
44
+ </a>
45
+ </p>
46
+
47
+ - **Quickstart Demo**: [`notebooks/quickstart_interactive_demo.ipynb`](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/quickstart_interactive_demo.ipynb) — 1-click Colab interactive simulation & PyTorch training.
48
+ - **Dedicated Benchmarks**: Check out [notebooks/](https://github.com/huynhldvn/CLOVES/tree/main/notebooks) for focused notebooks on [CIFAR-10](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/01_cifar10_vision_benchmark.ipynb), [LLM QLoRA](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/02_llm_qlora_tuning_benchmark.ipynb), and [2D Ravine Simulation](https://github.com/huynhldvn/CLOVES/blob/main/notebooks/03_2d_ravine_simulation.ipynb).
49
+ - **Standalone Scripts**: Run minimal scripts in [examples/](https://github.com/huynhldvn/CLOVES/tree/main/examples): `examples/01_quickstart_vision.py` and `examples/02_quickstart_llm.py`.
50
+
51
+ ---
52
+
53
+ ## 🎯 Which Optimizer Should You Use? (Practical Guide)
54
+
55
+ Choose the right variant based on your workload:
56
+
57
+ | Workload / Architecture | Recommended Optimizer | Key Advantage | State Memory | Recommended Settings |
58
+ | :--- | :--- | :--- | :---: | :--- |
59
+ | 🤖 **LLM LoRA / QLoRA Tuning** | `CLOVES_Apex_Asymmetric` ⭐ | **Recommended for LoRA**. Automatically accelerates Matrix A ($q_A=1.30$) and stabilizes Matrix B ($q_B=1.05$). Matches or slightly improves on AdamW loss while saving **> 3.3 GB VRAM**. | 2 States (8 B) | `lr=1.6e-4`, `total_steps=500` |
60
+ | 🧠 **LLM Full Tuning / Transformers** | `CLOVES_Adaptive` | **Drop-in replacement for AdamW**. Damps coordinate oscillations, maintains stability on deep networks, reduces synthetic ravine loss by 92%. | 2 States (8 B) | `lr=2e-4`, `q=1.25`, `gamma=0.6` |
61
+ | ⚡ **Extreme Memory Constraints (LLM)** | `CLOVES_LLM` | **50% state memory savings** (4 B/param), eliminates LION's coordinate-wise loss drift. | 1 State (4 B) | `lr=1e-4`, `q=1.20`, `gamma=0.6` |
62
+ | 👁️ **Standard Vision (CNN, ResNet, ViT)** | `CLOVES_Perpendicular` ⭐ | **Recommended for vision tasks**. Rejects orthogonal transverse wall force ($g_\perp$), reaches **93.07%** on CIFAR-10, runs with light 1-state memory. | 1 State (4 B) | `lr=0.05 - 0.1`, `beta=0.9` |
63
+ | 📈 **Tabular / Smooth Convex Tasks** | `CLOVES` (Bisector) | Fastest computational throughput, follows the ravine floor along the angular bisector. | 1 State (4 B) | `lr=0.1`, `q=1.5`, `gamma=0.1` |
64
+
65
+ > ⚠️ **Scope & Evaluation Note:**
66
+ > The LLM-optimized variants (`CLOVES_Apex_Asymmetric`, `CLOVES_Adaptive`, `CLOVES_LLM`, `CLOVES_Matrix_Polar`) were designed and validated specifically on Transformer and LoRA / QLoRA architectures (LLaMA-3, Qwen-1.5B, Qwen-7B). **These LLM variants have not yet been evaluated on other tasks (such as computer vision, audio, or tabular data)**.
67
+ > 👉 For computer vision and general classification workloads, use `CLOVES_Perpendicular` or `CLOVES` (Bisector), which have been thoroughly evaluated on vision benchmarks.
68
+
69
+ ---
70
+
71
+ ## ⚡ Quickstart
72
+
73
+ ### 1. Minimal Runnable Snippet (Copy & Run)
74
+ Test CLOVES immediately in any Python environment:
75
+
76
+ ```python
77
+ import torch
78
+ import torch.nn as nn
79
+ from cloves import CLOVES_Perpendicular
80
+
81
+ # Define a minimal model & optimizer
82
+ model = nn.Linear(10, 2)
83
+ optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05)
84
+
85
+ # Standard training step
86
+ x, y = torch.randn(16, 10), torch.randn(16, 2)
87
+ loss = nn.MSELoss()(model(x), y)
88
+ loss.backward()
89
+ optimizer.step()
90
+ optimizer.zero_grad()
91
+
92
+ print("Step completed! Loss:", float(loss.detach()))
93
+ ```
94
+
95
+ ### 2. Drop-in Replacement for Real Models
96
+ Simply swap your existing optimizer in your training loop:
97
+
98
+ ```python
99
+ # A. Vision (ResNet, CNN, ViT) — lightweight 1-state memory:
100
+ from cloves import CLOVES_Perpendicular
101
+ optimizer = CLOVES_Perpendicular(model.parameters(), lr=0.05, beta=0.9)
102
+
103
+ # B. LLM & LoRA Fine-Tuning — saves >3.3 GB VRAM:
104
+ from cloves import CLOVES_Apex_Asymmetric
105
+ optimizer = CLOVES_Apex_Asymmetric(model.parameters(), lr=1.6e-4, total_steps=500)
106
+ ```
107
+
108
+ ---
109
+
110
+ ## 🚀 Milestone Versions & Experimental Reports
111
+
112
+ CLOVES bridges classical first-order valley descent theory into large-scale stochastic deep learning and LLM fine-tuning:
113
+
114
+ ```
115
+ THE CLOVES OPTIMIZATION PHILOSOPHY
116
+ (Transversal Wall Damping γ < 1 & Longitudinal Floor Overstep q > 1)
117
+ │
118
+ ┌────────────────────────────────┴────────────────────────────────┐
119
+ ▼ ▼
120
+ [MILESTONE 1: DEEP VISION & 1-STATE PARITY] [MILESTONE 2: LLM & ADAPTIVE SCALING]
121
+ • Problem: Wall chattering in narrow ravines • Problem: Small-norm stagnation (||g|| ~ 1e-4)
122
+ • Memory: Strict 4 bytes/param (1-state) • Memory: 50% state memory savings or RMS scaling
123
+ • Mechanism: Angular bisector & orthogonal • Mechanism: Coordinate overstep, LoRA A/B
124
+ transversal force rejection (g_perp) asymmetric velocity, Newton-Schulz polar
125
+ │ │
126
+ ▼ ▼
127
+ CLOVES-Bisector / CLOVES-Perpendicular CLOVES-Apex / CLOVES-Adaptive
128
+ (Strong vision accuracy, 50% state memory) (Matches AdamW, beats Muon on QLoRA)
129
+ ```
130
+
131
+ The project is organized into two developmental milestones:
132
+ - **[Version 1.0 — Geometric Vision & Ravine Damping](https://github.com/huynhldvn/CLOVES/tree/main/benchmarks/v1_geometric_vision)**: First-order directional cleft damping (Bisector, Perpendicular, Affine) maintaining strict 1-state memory parity (4 bytes/param) for vision architectures (ResNet-18, DenseNet-121).
133
+ - **[Version 2.0 — LLM Fine-Tuning & Adaptive Cleft Scaling](https://github.com/huynhldvn/CLOVES/tree/main/benchmarks/v2_llm_adaptive)**: Addresses small-gradient stagnation ($\|g\| \sim 10^{-4}$) via coordinate cleft alignment, adaptive second-moment curvature scaling, and asymmetric LoRA velocity for Transformer architectures (LLaMA-3, Qwen-1.5B, Qwen-7B).
134
+
135
+ ---
136
+
137
+ ## 📈 Comprehensive Master Benchmark Results
138
+
139
+ The table below summarizes empirical validations across synthetic optimization ravines, deep vision architectures, and large language model instruction tuning suites, including convergence accuracy/loss, runtime per step/epoch, total training duration, and GPU memory consumption (measured on NVIDIA Tesla T4 GPUs):
140
+
141
+ > [!NOTE]
142
+ > **Evaluation Protocol & Objectivity**: To guarantee rigorous, unbiased comparison and avoid overfitting artifacts, all deep learning benchmarks (CIFAR-10, CIFAR-100, LLaMA-3, Qwen) are strictly evaluated on held-out **Validation Sets** (`val_loss` and `val_acc`), never on training batches. Synthetic ravine benchmarks evaluate the exact mathematical objective loss function $f(x)$ under identical random seeds and initialization.
143
+ >
144
+ > **Metric Interpretation**: **↑** = Higher is better &nbsp;|&nbsp; **↓** = Lower is better
145
+
146
+ | Benchmark Suite & Dataset | Architecture / Setup | Optimizer | Convergence Metric<br>(Val Loss ↓ / Val Acc ↑) | Step / Epoch Time<br>(↓) | Total Execution Time<br>(↓) | Peak GPU VRAM<br>(↓) | Optimizer State<br>(↓) |
147
+ | :--- | :--- | :--- | :---: | :---: | :---: | :---: | :---: |
148
+ | **2D Ill-Conditioned Ravine** ($\kappa=100$) | Quadratic canyon (250 steps) | **CLOVES-Bisector**<br>AdamW<br>LION | Objective Loss: **$7.11 \times 10^{-5}$**<br>Objective Loss: 20.20<br>Objective Loss: 24.52 | **0.82 ms / step**<br>1.14 ms / step<br>0.89 ms / step | **0.41 s**<br>0.57 s<br>0.45 s | < 10 MB<br>< 10 MB<br>< 10 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
149
+ | **CIFAR-10 Vision** (Standard Vision) | ResNet-18 (20 Epochs, bs=128) | **CLOVES-Perpendicular**<br>CLOVES-Affine<br>AdamW<br>Official LION | Val Acc: **93.07%** (Val Loss: 0.2237)<br>Val Acc: 92.51% (Val Loss: 0.2310)<br>Val Acc: 92.35% (Val Loss: 0.2512)<br>Val Acc: 89.20% (Val Loss: 0.3840) | **51.1 s / epoch**<br>51.6 s / epoch<br>61.4 s / epoch<br>51.0 s / epoch | **1,022.7 s** (~17.0 min)<br>1,032.0 s (~17.2 min)<br>1,228.0 s (~20.5 min)<br>1,020.0 s (~17.0 min) | **696.0 MB**<br>696.0 MB<br>827.8 MB<br>696.0 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
150
+ | **CIFAR-100 Deep Vision** (100 Classes) | DenseNet-121 (30 Epochs, Cosine) | **CLOVES (Proposed)**<br>AdamW<br>SGD-M | Top-1 Val Acc: **73.98%** (Val Loss: 1.3993)<br>Top-1 Val Acc: 72.77% (Val Loss: 1.4435)<br>Top-1 Val Acc: 72.56% (Val Loss: 1.4304) | **132.5 s / epoch**<br>133.1 s / epoch<br>131.8 s / epoch | **3,975.0 s** (~66.2 min)<br>3,993.0 s (~66.5 min)<br>3,954.0 s (~65.9 min) | **5,001.5 MB**<br>5,028.4 MB<br>5,001.5 MB | **1-State (4 B)**<br>2-State (8 B)<br>1-State (4 B) |
151
+ | **1000D Non-Convex Ravine** ($\kappa=10,000$) | Anisotropic manifold (500 steps) | **CLOVES-Adaptive**<br>AdamW<br>**CLOVES-LLM**<br>Official LION | Objective Loss: **17.12** (-92.0%)<br>Objective Loss: 213.23 (Baseline)<br>Objective Loss: **12,297.97** (-60.2%)<br>Objective Loss: 30,861.02 (Baseline) | **1.21 ms / step**<br>1.18 ms / step<br>0.94 ms / step<br>0.92 ms / step | **0.61 s**<br>0.59 s<br>0.47 s<br>0.46 s | < 15 MB<br>< 15 MB<br>< 15 MB<br>< 15 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
152
+ | **LLaMA-3-8B Instruction Tuning** | 4-bit QLoRA, Alpaca (500 steps) | **CLOVES-Perpendicular**<br>CLOVES-Matrix-Polar<br>AdamW (Baseline)<br>Official Muon | Val Loss: **5.7338**<br>Val Loss: **5.7822**<br>Val Loss: 5.7269<br>Val Loss: 5.7941 | **12.01 s / step**<br>12.48 s / step<br>11.71 s / step<br>12.52 s / step | **16,010.2 s** (~4.44 h)<br>16,485.3 s (~4.58 h)<br>16,376.8 s (~4.55 h)<br>16,428.7 s (~4.56 h) | **10,594.5 MB**<br>10,594.5 MB<br>14,349.5 MB<br>14,189.5 MB | **1-State (4 B)**<br>1-State (4 B)<br>2-State (8 B)<br>1-State (4 B) |
153
+ | **Qwen-1.5B Instruction Tuning** | QLoRA, Alpaca (1000 steps) | **CLOVES-Apex (Tuned)**<br>AdamW (Baseline) | Val Loss: **0.4295**<br>Val Loss: 0.4298 | 562.8 ms / step<br>470.3 ms / step | **2,681.9 s** (~44.7 min)<br>2,648.4 s (~44.1 min) | **8,202.3 MB**<br>8,202.3 MB | **2-State (8 B)**<br>2-State (8 B) |
154
+ | **Qwen-7B Instruction Tuning** | QLoRA, Alpaca (500 steps) | **CLOVES-Apex (Annealed)**<br>AdamW (Baseline)<br>**CLOVES-Sign-Apex**<br>Official LION (Baseline) | Val Loss: **0.5573**<br>Val Loss: 0.5583<br>Val Loss: **0.6249**<br>Val Loss: 0.6365 | 2,956.0 ms / step<br>2,746.3 ms / step<br>2,732.9 ms / step<br>2,714.6 ms / step | **3,911.1 s** (~65.1 min)<br>3,871.5 s (~64.5 min)<br>3,624.2 s (~60.4 min)<br>3,613.7 s (~60.2 min) | **10,793.6 MB** (-3.36 GB)<br>14,159.3 MB<br>**10,639.6 MB** (-3.37 GB)<br>14,005.3 MB | **2-State (8 B)**<br>2-State (8 B)<br>**1-State (4 B)**<br>1-State (4 B) |
155
+
156
+ ### Pareto Frontier: Loss vs. Memory vs. Runtime Trade-off
157
+
158
+ The empirical evaluations demonstrate favorable trade-offs across memory footprint, runtime efficiency, and convergence loss:
159
+
160
+ <p align="center">
161
+ <img src="https://raw.githubusercontent.com/huynhldvn/CLOVES/main/docs/images/cloves_pareto_frontier_master_clean.png" alt="CLOVES Pareto Frontier" width="85%">
162
+ </p>
163
+
164
+ ---
165
+
166
+ ## 📊 Optimizer Selection Matrix
167
+
168
+ | Optimizer Class | Target Workload | State Buffers | Memory Footprint | Special Characteristics |
169
+ | :--- | :--- | :---: | :---: | :--- |
170
+ | `CLOVES` | Vision (CNNs, ResNet) | 1 ($m$) | 4 bytes / param | Angular bisector damping; low memory |
171
+ | `CLOVES_Perpendicular` | Highly ill-conditioned loss | 1 ($m$) | 4 bytes / param | Orthogonal wall force rejection |
172
+ | `CLOVES_Affine` | Deep classification networks | 1 ($m$) | 4 bytes / param | Continuous affine subspace projection |
173
+ | `CLOVES_Adaptive` | LLMs, Transformers | 2 ($m, v$) | 8 bytes / param | Coordinate cleft overstep + RMS scaling |
174
+ | `CLOVES_LLM` | Memory-constrained LLMs | 1 ($m$) | 4 bytes / param | Sign overstep; 50% state memory savings |
175
+ | `CLOVES_Apex_Asymmetric` | LoRA / QLoRA Instruction Tuning | 2 ($m, v$) | 8 bytes / param | Matrix A/B ratio, soft tanh damping |
176
+ | `CLOVES_Matrix_Polar` | Linear layer weights (2D) | 1 ($m$) | 4 bytes / param | Newton-Schulz quintic polar decomposition |
177
+
178
+ ---
179
+
180
+ ## 📚 References & Baseline Implementations
181
+
182
+ CLOVES benchmarks are evaluated against official implementations of standard and contemporary optimizers:
183
+
184
+ 1. **AdamW**: Loshchilov & Hutter, *Decoupled Weight Decay Regularization*, ICLR 2019.
185
+ 2. **Lion** (*EvoLved Sign Momentum*): Chen et al., *Symbolic Discovery of Optimization Algorithms*, NeurIPS 2023. arXiv: [2302.06675](https://arxiv.org/abs/2302.06675).
186
+ - Code: [google-research/lion](https://github.com/google-research/google-research/tree/master/lion) | [lucidrains/lion-pytorch](https://github.com/lucidrains/lion-pytorch)
187
+ 3. **Muon** (*Momentum Orthogonalized by Newton-schulz*): Keller Jordan et al., *Muon: An Optimizer for Hidden Layers in Neural Networks*, 2024.
188
+ - Code: [KellerJordan/Muon](https://github.com/KellerJordan/Muon)
189
+
190
+ ---
191
+
192
+ ## 🏛️ Research Inspiration & Lineage
193
+
194
+ The algorithmic concepts in CLOVES are **inspired by** foundational ravine traversal theories in mathematical programming (Nguyễn & Bùi, 1990) and subsequent exploratory studies evaluating cleft-overstep heuristics in machine learning tasks (Phan & Lê, 2020).
195
+
196
+ ### Pragmatic Adaptation from Theory to Practice
197
+ The original cleft-overstep formulations were conceived within classical, low-dimensional, deterministic optimization settings. Translating these geometric concepts to modern deep learning and large-scale language models ($d \gg 10^7$) inherently involves **empirical, heuristic (ad-hoc) adaptations**:
198
+ - The authors make **no claim of 100% formal mathematical equivalence** or complete theoretical convergence proofs matching the original deterministic algorithms.
199
+ - Instead, CLOVES adapts the core geometric intuition—*damping transversal wall-to-wall bouncing while accelerating along the longitudinal ravine bed*—into efficient, zero-allocation, fused PyTorch tensor operations tailored for stochastic mini-batch gradients.
200
+ - Extensions such as coordinate-wise variance scaling (`CLOVES_Adaptive`), 1-state sign momentum (`CLOVES_LLM`), asymmetric LoRA velocity ratios (`CLOVES_Apex_Asymmetric`), and polar matrix decomposition (`CLOVES_Matrix_Polar`) are practical designs driven by empirical deep learning dynamics.
201
+
202
+ **Predecessor & Historical References**:
203
+ 1. Nguyễn Văn Mạnh & Bùi Minh Trí, *"Methods for finding ravine directions in non-differentiable optimization,"* *Acta Mathematica Vietnamica*, 1990.
204
+ 2. Duy-Hung Phan & Le Dinh Huynh, *"Evaluation of the Cleft-Overstep Algorithm for Linear Regression Analysis,"* in *Computational Collective Intelligence (ICCCI 2020)*, Lecture Notes in Computer Science (LNCS, vol. 12496), pp. 400–411, Springer, 2020. DOI: [10.1007/978-3-030-63007-2_31](https://doi.org/10.1007/978-3-030-63007-2_31).
205
+
206
+ ---
207
+
208
+ ## 🤝 Feedback & Contributions
209
+
210
+ CLOVES is an empirical project developed with an open mindset. If you experiment with CLOVES on your models and find it helpful—or discover interesting edge cases—we would love to hear from you!
211
+
212
+ Bug reports, critiques, and pull requests are warmly welcomed via [GitHub Issues](https://github.com/huynhldvn/CLOVES/issues) or [Discussions](https://github.com/huynhldvn/CLOVES/discussions).
213
+
214
+ ---
215
+
216
+ ## 🙏 Acknowledgements
217
+
218
+ We thank **Kaggle** for providing the free GPU compute (Tesla T4) for our benchmarks, and **Antigravity** for assisting with code refactoring and documentation.
219
+
220
+ ---
221
+
222
+ ## 📄 License
223
+
224
+ MIT License. See [LICENSE](https://github.com/huynhldvn/CLOVES/blob/main/LICENSE) for details.
@@ -0,0 +1,59 @@
1
+ [build-system]
2
+ requires = ["setuptools>=61.0.0", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "cloves"
7
+ version = "2.0.1"
8
+ description = "Cleft-Overstep Optimization for Deep Neural Networks & LLMs"
9
+ readme = "README.md"
10
+ requires-python = ">=3.8"
11
+ license = "MIT"
12
+ keywords = [
13
+ "deep-learning",
14
+ "optimizer",
15
+ "pytorch",
16
+ "optimization",
17
+ "llm",
18
+ "cleft-overstep",
19
+ "adamw",
20
+ "lion",
21
+ "muon"
22
+ ]
23
+ classifiers = [
24
+ "Development Status :: 5 - Production/Stable",
25
+ "Intended Audience :: Science/Research",
26
+ "Intended Audience :: Developers",
27
+ "Programming Language :: Python :: 3",
28
+ "Programming Language :: Python :: 3.8",
29
+ "Programming Language :: Python :: 3.9",
30
+ "Programming Language :: Python :: 3.10",
31
+ "Programming Language :: Python :: 3.11",
32
+ "Programming Language :: Python :: 3.12",
33
+ "Programming Language :: Python :: 3.13",
34
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
35
+ "Topic :: Software Development :: Libraries :: Python Modules"
36
+ ]
37
+ dependencies = [
38
+ "torch>=1.12.0"
39
+ ]
40
+
41
+ [project.optional-dependencies]
42
+ dev = [
43
+ "pytest>=7.0.0",
44
+ "build",
45
+ "twine"
46
+ ]
47
+
48
+ [project.urls]
49
+ Homepage = "https://github.com/huynhldvn/CLOVES"
50
+ Documentation = "https://github.com/huynhldvn/CLOVES#readme"
51
+ Repository = "https://github.com/huynhldvn/CLOVES.git"
52
+ Issues = "https://github.com/huynhldvn/CLOVES/issues"
53
+
54
+ [tool.setuptools.packages.find]
55
+ where = ["src"]
56
+
57
+ [tool.pytest.ini_options]
58
+ pythonpath = ["src"]
59
+ testpaths = ["tests"]
cloves-2.0.1/setup.cfg ADDED
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
cloves-2.0.1/setup.py ADDED
@@ -0,0 +1,9 @@
1
+ """
2
+ Thin setup.py for backward compatibility with legacy tooling.
3
+ Primary configuration is defined in pyproject.toml (PEP 621).
4
+ """
5
+
6
+ from setuptools import setup
7
+
8
+ if __name__ == "__main__":
9
+ setup()
@@ -0,0 +1,24 @@
1
+ """
2
+ CLOVES: Cleft-Overstep Optimization for Deep Neural Networks & LLMs
3
+ """
4
+
5
+ from ._version import __version__
6
+ from .bisector import CLOVES, CLOVES_Bisector
7
+ from .perpendicular import CLOVES_Perpendicular
8
+ from .affine import CLOVES_Affine
9
+ from .adaptive import CLOVES_Adaptive
10
+ from .llm import CLOVES_LLM
11
+ from .apex import CLOVES_Apex_Asymmetric
12
+ from .matrix_polar import CLOVES_Matrix_Polar
13
+
14
+ __all__ = [
15
+ "__version__",
16
+ "CLOVES",
17
+ "CLOVES_Bisector",
18
+ "CLOVES_Perpendicular",
19
+ "CLOVES_Affine",
20
+ "CLOVES_Adaptive",
21
+ "CLOVES_LLM",
22
+ "CLOVES_Apex_Asymmetric",
23
+ "CLOVES_Matrix_Polar",
24
+ ]
@@ -0,0 +1 @@
1
+ __version__ = "2.0.1"