luminav 1.1.5__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- luminav-1.1.5/LICENSE +17 -0
- luminav-1.1.5/PKG-INFO +202 -0
- luminav-1.1.5/README.md +168 -0
- luminav-1.1.5/pyproject.toml +56 -0
- luminav-1.1.5/setup.cfg +4 -0
- luminav-1.1.5/src/luminav/__init__.py +3 -0
- luminav-1.1.5/src/luminav/luminav.py +651 -0
- luminav-1.1.5/src/luminav.egg-info/PKG-INFO +202 -0
- luminav-1.1.5/src/luminav.egg-info/SOURCES.txt +11 -0
- luminav-1.1.5/src/luminav.egg-info/dependency_links.txt +1 -0
- luminav-1.1.5/src/luminav.egg-info/requires.txt +4 -0
- luminav-1.1.5/src/luminav.egg-info/top_level.txt +1 -0
- luminav-1.1.5/tests/test_smollm_t4.py +88 -0
luminav-1.1.5/LICENSE
ADDED
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
Apache License
|
|
2
|
+
Version 2.0, January 2004
|
|
3
|
+
http://www.apache.org/licenses/
|
|
4
|
+
|
|
5
|
+
Copyright (c) 2026 Lumina Moon and Contributors.
|
|
6
|
+
|
|
7
|
+
Licensed under the Apache License, Version 2.0 (the "License");
|
|
8
|
+
you may not use this file except in compliance with the License.
|
|
9
|
+
You may obtain a copy of the License at
|
|
10
|
+
|
|
11
|
+
http://www.apache.org/licenses/LICENSE-2.0
|
|
12
|
+
|
|
13
|
+
Unless required by applicable law or agreed to in writing, software
|
|
14
|
+
distributed under the License is distributed on an "AS IS" BASIS,
|
|
15
|
+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
16
|
+
See the License for the specific language governing permissions and
|
|
17
|
+
limitations under the License.
|
luminav-1.1.5/PKG-INFO
ADDED
|
@@ -0,0 +1,202 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: luminav
|
|
3
|
+
Version: 1.1.5
|
|
4
|
+
Summary: A master-free, memory-efficient adaptive optimizer for low-precision deep learning
|
|
5
|
+
Author: Silver Moon (cloverxion)
|
|
6
|
+
Maintainer: Lumina Moon
|
|
7
|
+
License: Apache-2.0
|
|
8
|
+
Project-URL: Homepage, https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper
|
|
9
|
+
Project-URL: Author Profile, https://huggingface.co/cloverxion
|
|
10
|
+
Project-URL: Organization, https://huggingface.co/Lumina-Moon
|
|
11
|
+
Project-URL: Software DOI, https://doi.org/10.57967/hf/10365
|
|
12
|
+
Project-URL: Paper DOI, https://doi.org/10.57967/hf/10270
|
|
13
|
+
Project-URL: Bug Tracker, https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper/discussions
|
|
14
|
+
Project-URL: Changelog, https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper/blob/main/CHANGELOG.md
|
|
15
|
+
Keywords: pytorch,optimizer,low-vram,fp16,bf16,triton,deep-learning,llm,transformer
|
|
16
|
+
Classifier: Development Status :: 4 - Beta
|
|
17
|
+
Classifier: Intended Audience :: Science/Research
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: License :: OSI Approved :: Apache Software License
|
|
20
|
+
Classifier: Programming Language :: Python :: 3
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
22
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
23
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
24
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
25
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
26
|
+
Classifier: Operating System :: OS Independent
|
|
27
|
+
Requires-Python: >=3.10
|
|
28
|
+
Description-Content-Type: text/markdown
|
|
29
|
+
License-File: LICENSE
|
|
30
|
+
Requires-Dist: torch>=2.0.0
|
|
31
|
+
Provides-Extra: triton
|
|
32
|
+
Requires-Dist: triton; extra == "triton"
|
|
33
|
+
Dynamic: license-file
|
|
34
|
+
|
|
35
|
+
# LuminaV Optimizer
|
|
36
|
+
|
|
37
|
+
> **We Were Too Broke for AdamW So We Trapped Gradients in a Hyperbolic Straitjacket and Hired a Traffic Cop to Slap Them**
|
|
38
|
+
|
|
39
|
+
[](https://pypi.org/project/luminav/)
|
|
40
|
+
[](https://doi.org/10.57967/hf/10365)
|
|
41
|
+
[](https://doi.org/10.57967/hf/10270)
|
|
42
|
+
[](https://pypi.org/project/luminav/)
|
|
43
|
+
[](LICENSE)
|
|
44
|
+
|
|
45
|
+
---
|
|
46
|
+
|
|
47
|
+
## Official Repository & Research Paper
|
|
48
|
+
|
|
49
|
+
- **Official Repository & Issue Tracker:** [https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper](https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper)
|
|
50
|
+
- **Primary Research Paper Archive:** [https://huggingface.co/cloverx-id/XoneLM-1.0-Paper](https://huggingface.co/cloverx-id/XoneLM-1.0-Paper)
|
|
51
|
+
- **Software DOI:** [10.57967/hf/10365](https://doi.org/10.57967/hf/10365)
|
|
52
|
+
- **Paper DOI:** [10.57967/hf/10270](https://doi.org/10.57967/hf/10270)
|
|
53
|
+
|
|
54
|
+
---
|
|
55
|
+
|
|
56
|
+
## What's New in v1.1.5 (Latest Release)
|
|
57
|
+
|
|
58
|
+
The **v1.1.5** release introduces the **Zero-VRAM FP16 Numerical Safety Shield**, resolving early pretraining instability and eliminating `NaN` collapses in pure half-precision training without allocating 4-byte master weights:
|
|
59
|
+
|
|
60
|
+
- **Dynamic Epsilon Floor for FP16:** Automatically floors effective epsilon to `max(eps, 1e-4)` for `torch.float16` parameters, preventing the bias-corrected scale factor `c2` from underflowing into IEEE-754 subnormal/zero limits (~5.96e-8) and eliminating step-1 `0.0 / 0.0 = NaN` errors on zero-gradient parameters (e.g., unselected vocabulary tokens, padding, or dropout paths).
|
|
61
|
+
- **On-Chip Second-Moment Clamping:** Added an upper-bound register ceiling (`65,000.0`) to the centered innovation variance in Triton kernels and PyTorch loops before storing into FP16 pointers. This prevents unclipped initial gradient spikes (`|g| > 256`) from exceeding the FP16 maximum dynamic range (65,504) and permanently saturating state buffers to `+inf`.
|
|
62
|
+
- **Zero-Sigma Division Guard:** Embedded an explicit `sigma + 1e-6` protection across all Triton kernel passes and PyTorch fallbacks to ensure division safety during cold-start iterations.
|
|
63
|
+
- **Native Hardware Capability Detection:** Integrated `_supports_native_bf16()` to automatically detect Nvidia Ampere SM80+ architectures for optimal native execution.
|
|
64
|
+
- **Empirically Proven on Tesla T4:** Empirically verified by pretraining `SmolLM-135M` from scratch on Wikimedia Wikipedia in pure FP16 on an Nvidia Tesla T4 GPU for 100 consecutive steps with zero `NaN` occurrences—even with external gradient clipping completely disabled.
|
|
65
|
+
|
|
66
|
+
---
|
|
67
|
+
|
|
68
|
+
## Overview
|
|
69
|
+
|
|
70
|
+
**LuminaV** is a master-free, memory-efficient adaptive optimizer engineered specifically for deep learning workloads running directly in low precision (`FP16` / `BF16`) without maintaining redundant 4-byte FP32 master weights.
|
|
71
|
+
|
|
72
|
+
By combining **Centered Innovation Variance**, **Hyperbolic Tangent (tanh) Coordinate Bounding**, a **Directional Traffic-Cop Mask**, and **On-Chip Bitwise Stochastic Rounding**, LuminaV eliminates the standard 16-byte-per-parameter memory tax imposed by AdamW while preventing weight stagnation and numerical explosions.
|
|
73
|
+
|
|
74
|
+
---
|
|
75
|
+
|
|
76
|
+
## Key Features
|
|
77
|
+
|
|
78
|
+
1. **Zero Master-Weight Copies:** Directly mutates parameter weights in native `FP16` or `BF16`, eliminating the redundant 4-byte FP32 master weight allocation.
|
|
79
|
+
2. **On-Chip Bitwise Stochastic Rounding (SR):** Implements in-register bitcast hashing in Triton to provide unbiased stochastic rounding, preventing weight stagnation during fine-grained updates or learning rate decay.
|
|
80
|
+
3. **Hyperbolic tanh Bounding Envelope:** Maps normalized momentum through a `(-1.0, 1.0)` transfer function, guaranteeing coordinate updates cannot explode beyond the step learning rate.
|
|
81
|
+
4. **The Traffic-Cop Directional Gate:** Dynamically eliminates coordinate updates whenever historical momentum conflicts with the incoming mini-batch gradient direction (`u_t · g_t <= 0`).
|
|
82
|
+
5. **Centered Innovation Variance:** Tracks centered innovation dispersion `(g_t - m_t)^2` rather than uncentered raw second moments, suppressing variance inflation during confident descent.
|
|
83
|
+
6. **Dual Execution Engine:** Fully accelerated custom OpenAI Triton kernels for CUDA devices, paired with vectorized C++ `torch._foreach` multi-tensor fallbacks.
|
|
84
|
+
|
|
85
|
+
---
|
|
86
|
+
|
|
87
|
+
## Installation
|
|
88
|
+
|
|
89
|
+
Install directly via PyPI:
|
|
90
|
+
|
|
91
|
+
```bash
|
|
92
|
+
pip install luminav
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
For GPU acceleration via OpenAI Triton:
|
|
96
|
+
|
|
97
|
+
```bash
|
|
98
|
+
pip install luminav[triton]
|
|
99
|
+
```
|
|
100
|
+
|
|
101
|
+
Or install in editable mode from source:
|
|
102
|
+
|
|
103
|
+
```bash
|
|
104
|
+
git clone https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper
|
|
105
|
+
cd LuminaV-Optimizer-Paper
|
|
106
|
+
pip install -e .
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Quickstart
|
|
112
|
+
|
|
113
|
+
```python
|
|
114
|
+
import torch
|
|
115
|
+
import torch.nn as nn
|
|
116
|
+
from luminav import LuminaV
|
|
117
|
+
|
|
118
|
+
# 1. Instantiate model directly in native half precision (e.g. BF16 or FP16)
|
|
119
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
120
|
+
model = nn.Linear(1024, 1024).to(device=device, dtype=torch.float16)
|
|
121
|
+
|
|
122
|
+
# 2. Initialize LuminaV
|
|
123
|
+
optimizer = LuminaV(
|
|
124
|
+
model.parameters(),
|
|
125
|
+
lr=8e-4,
|
|
126
|
+
betas=(0.9, 0.999),
|
|
127
|
+
eps=1e-8,
|
|
128
|
+
weight_decay=0.08,
|
|
129
|
+
tau=0.8,
|
|
130
|
+
buffer=2, # 2 = Dual-Buffer (Standard), 1 = Single-Buffer (Low VRAM)
|
|
131
|
+
stochastic_rounding=True,
|
|
132
|
+
execution="auto"
|
|
133
|
+
)
|
|
134
|
+
|
|
135
|
+
# 3. Standard training loop
|
|
136
|
+
data = torch.randn(32, 1024, device=device, dtype=torch.float16)
|
|
137
|
+
target = torch.randn(32, 1024, device=device, dtype=torch.float16)
|
|
138
|
+
criterion = nn.MSELoss()
|
|
139
|
+
|
|
140
|
+
optimizer.zero_grad(set_to_none=True)
|
|
141
|
+
output = model(data)
|
|
142
|
+
loss = criterion(output, target)
|
|
143
|
+
loss.backward()
|
|
144
|
+
|
|
145
|
+
optimizer.step()
|
|
146
|
+
```
|
|
147
|
+
|
|
148
|
+
---
|
|
149
|
+
|
|
150
|
+
## Parameter Reference
|
|
151
|
+
|
|
152
|
+
| Parameter | Type | Default | Description |
|
|
153
|
+
| :--- | :--- | :--- | :--- |
|
|
154
|
+
| `params` | `iterable` | *Required* | Iterable of parameters to optimize or dicts defining parameter groups. |
|
|
155
|
+
| `lr` | `float` | `8e-4` | Learning rate (eta). |
|
|
156
|
+
| `betas` | `Tuple[float, float]` | `(0.9, 0.999)` | Coefficients (beta1, beta2) for running momentum and centered innovation variance. |
|
|
157
|
+
| `eps` | `float` | `1e-8` | Numerical stability term (epsilon). Automatically floored to `1e-4` in FP16 to prevent subnormal underflow. |
|
|
158
|
+
| `weight_decay` | `float` | `8e-2` | Decoupled weight decay coefficient (lambda). |
|
|
159
|
+
| `tau` | `float` | `0.8` | Analytical bias correction temperature parameter (tau). |
|
|
160
|
+
| `alpha_ss` | `float` | `0.5` | Softsign dampening factor (alpha_ss) used in single-buffer mode (`buffer=1`). |
|
|
161
|
+
| `cautious` | `bool` | `True` | If `True`, enables Traffic-Cop directional verification masking. |
|
|
162
|
+
| `cautious_clamp_min` | `float` | `0.2` | Safety floor density clamp (gamma_min) preventing division by zero in masked normalization. |
|
|
163
|
+
| `buffer` | `int` | `2` | Buffer mode: `2` (Dual-buffer tracking m_t and v_t) or `1` (Single-buffer scalar RMS tracking). |
|
|
164
|
+
| `stochastic_rounding` | `bool` | `True` | Enables bitwise stochastic rounding on native FP16/BF16 weights. |
|
|
165
|
+
| `execution` | `str` | `"auto"` | Execution engine: `"auto"`, `"triton"`, `"foreach"`, or `"single"`. |
|
|
166
|
+
|
|
167
|
+
---
|
|
168
|
+
|
|
169
|
+
## Citation
|
|
170
|
+
|
|
171
|
+
If you utilize LuminaV in your research or applications, please cite both the foundational paper and this software implementation:
|
|
172
|
+
|
|
173
|
+
```bibtex
|
|
174
|
+
# 1. To cite the official research paper & theoretical mechanics
|
|
175
|
+
@misc{luminamoon2026luminav_paper,
|
|
176
|
+
author = {{Silver Moon (cloverxion)}},
|
|
177
|
+
organization = {Lumina Moon},
|
|
178
|
+
title = {{LuminaV: We Were Too Broke for AdamW So We Trapped Gradients in a Hyperbolic Straitjacket and Hired a Traffic Cop to Slap Them}},
|
|
179
|
+
year = {2026},
|
|
180
|
+
publisher = {Hugging Face},
|
|
181
|
+
doi = {10.57967/hf/10270},
|
|
182
|
+
url = {https://huggingface.co/cloverx-id/XoneLM-1.0-Paper}
|
|
183
|
+
}
|
|
184
|
+
|
|
185
|
+
# 2. To cite this software implementation & standalone codebase
|
|
186
|
+
@software{luminamoon2026luminav_code,
|
|
187
|
+
author = {{Silver Moon (cloverxion)}},
|
|
188
|
+
organization = {Lumina Moon},
|
|
189
|
+
title = {{LuminaV Optimizer: Official PyTorch Implementation}},
|
|
190
|
+
year = {2026},
|
|
191
|
+
publisher = {Hugging Face},
|
|
192
|
+
version = {1.1.5},
|
|
193
|
+
doi = {10.57967/hf/10365},
|
|
194
|
+
url = {https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper}
|
|
195
|
+
}
|
|
196
|
+
```
|
|
197
|
+
|
|
198
|
+
---
|
|
199
|
+
|
|
200
|
+
## License
|
|
201
|
+
|
|
202
|
+
Apache License 2.0. See [LICENSE](LICENSE) for full terms.
|
luminav-1.1.5/README.md
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
# LuminaV Optimizer
|
|
2
|
+
|
|
3
|
+
> **We Were Too Broke for AdamW So We Trapped Gradients in a Hyperbolic Straitjacket and Hired a Traffic Cop to Slap Them**
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/luminav/)
|
|
6
|
+
[](https://doi.org/10.57967/hf/10365)
|
|
7
|
+
[](https://doi.org/10.57967/hf/10270)
|
|
8
|
+
[](https://pypi.org/project/luminav/)
|
|
9
|
+
[](LICENSE)
|
|
10
|
+
|
|
11
|
+
---
|
|
12
|
+
|
|
13
|
+
## Official Repository & Research Paper
|
|
14
|
+
|
|
15
|
+
- **Official Repository & Issue Tracker:** [https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper](https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper)
|
|
16
|
+
- **Primary Research Paper Archive:** [https://huggingface.co/cloverx-id/XoneLM-1.0-Paper](https://huggingface.co/cloverx-id/XoneLM-1.0-Paper)
|
|
17
|
+
- **Software DOI:** [10.57967/hf/10365](https://doi.org/10.57967/hf/10365)
|
|
18
|
+
- **Paper DOI:** [10.57967/hf/10270](https://doi.org/10.57967/hf/10270)
|
|
19
|
+
|
|
20
|
+
---
|
|
21
|
+
|
|
22
|
+
## What's New in v1.1.5 (Latest Release)
|
|
23
|
+
|
|
24
|
+
The **v1.1.5** release introduces the **Zero-VRAM FP16 Numerical Safety Shield**, resolving early pretraining instability and eliminating `NaN` collapses in pure half-precision training without allocating 4-byte master weights:
|
|
25
|
+
|
|
26
|
+
- **Dynamic Epsilon Floor for FP16:** Automatically floors effective epsilon to `max(eps, 1e-4)` for `torch.float16` parameters, preventing the bias-corrected scale factor `c2` from underflowing into IEEE-754 subnormal/zero limits (~5.96e-8) and eliminating step-1 `0.0 / 0.0 = NaN` errors on zero-gradient parameters (e.g., unselected vocabulary tokens, padding, or dropout paths).
|
|
27
|
+
- **On-Chip Second-Moment Clamping:** Added an upper-bound register ceiling (`65,000.0`) to the centered innovation variance in Triton kernels and PyTorch loops before storing into FP16 pointers. This prevents unclipped initial gradient spikes (`|g| > 256`) from exceeding the FP16 maximum dynamic range (65,504) and permanently saturating state buffers to `+inf`.
|
|
28
|
+
- **Zero-Sigma Division Guard:** Embedded an explicit `sigma + 1e-6` protection across all Triton kernel passes and PyTorch fallbacks to ensure division safety during cold-start iterations.
|
|
29
|
+
- **Native Hardware Capability Detection:** Integrated `_supports_native_bf16()` to automatically detect Nvidia Ampere SM80+ architectures for optimal native execution.
|
|
30
|
+
- **Empirically Proven on Tesla T4:** Empirically verified by pretraining `SmolLM-135M` from scratch on Wikimedia Wikipedia in pure FP16 on an Nvidia Tesla T4 GPU for 100 consecutive steps with zero `NaN` occurrences—even with external gradient clipping completely disabled.
|
|
31
|
+
|
|
32
|
+
---
|
|
33
|
+
|
|
34
|
+
## Overview
|
|
35
|
+
|
|
36
|
+
**LuminaV** is a master-free, memory-efficient adaptive optimizer engineered specifically for deep learning workloads running directly in low precision (`FP16` / `BF16`) without maintaining redundant 4-byte FP32 master weights.
|
|
37
|
+
|
|
38
|
+
By combining **Centered Innovation Variance**, **Hyperbolic Tangent (tanh) Coordinate Bounding**, a **Directional Traffic-Cop Mask**, and **On-Chip Bitwise Stochastic Rounding**, LuminaV eliminates the standard 16-byte-per-parameter memory tax imposed by AdamW while preventing weight stagnation and numerical explosions.
|
|
39
|
+
|
|
40
|
+
---
|
|
41
|
+
|
|
42
|
+
## Key Features
|
|
43
|
+
|
|
44
|
+
1. **Zero Master-Weight Copies:** Directly mutates parameter weights in native `FP16` or `BF16`, eliminating the redundant 4-byte FP32 master weight allocation.
|
|
45
|
+
2. **On-Chip Bitwise Stochastic Rounding (SR):** Implements in-register bitcast hashing in Triton to provide unbiased stochastic rounding, preventing weight stagnation during fine-grained updates or learning rate decay.
|
|
46
|
+
3. **Hyperbolic tanh Bounding Envelope:** Maps normalized momentum through a `(-1.0, 1.0)` transfer function, guaranteeing coordinate updates cannot explode beyond the step learning rate.
|
|
47
|
+
4. **The Traffic-Cop Directional Gate:** Dynamically eliminates coordinate updates whenever historical momentum conflicts with the incoming mini-batch gradient direction (`u_t · g_t <= 0`).
|
|
48
|
+
5. **Centered Innovation Variance:** Tracks centered innovation dispersion `(g_t - m_t)^2` rather than uncentered raw second moments, suppressing variance inflation during confident descent.
|
|
49
|
+
6. **Dual Execution Engine:** Fully accelerated custom OpenAI Triton kernels for CUDA devices, paired with vectorized C++ `torch._foreach` multi-tensor fallbacks.
|
|
50
|
+
|
|
51
|
+
---
|
|
52
|
+
|
|
53
|
+
## Installation
|
|
54
|
+
|
|
55
|
+
Install directly via PyPI:
|
|
56
|
+
|
|
57
|
+
```bash
|
|
58
|
+
pip install luminav
|
|
59
|
+
```
|
|
60
|
+
|
|
61
|
+
For GPU acceleration via OpenAI Triton:
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
pip install luminav[triton]
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
Or install in editable mode from source:
|
|
68
|
+
|
|
69
|
+
```bash
|
|
70
|
+
git clone https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper
|
|
71
|
+
cd LuminaV-Optimizer-Paper
|
|
72
|
+
pip install -e .
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
---
|
|
76
|
+
|
|
77
|
+
## Quickstart
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
import torch
|
|
81
|
+
import torch.nn as nn
|
|
82
|
+
from luminav import LuminaV
|
|
83
|
+
|
|
84
|
+
# 1. Instantiate model directly in native half precision (e.g. BF16 or FP16)
|
|
85
|
+
device = "cuda" if torch.cuda.is_available() else "cpu"
|
|
86
|
+
model = nn.Linear(1024, 1024).to(device=device, dtype=torch.float16)
|
|
87
|
+
|
|
88
|
+
# 2. Initialize LuminaV
|
|
89
|
+
optimizer = LuminaV(
|
|
90
|
+
model.parameters(),
|
|
91
|
+
lr=8e-4,
|
|
92
|
+
betas=(0.9, 0.999),
|
|
93
|
+
eps=1e-8,
|
|
94
|
+
weight_decay=0.08,
|
|
95
|
+
tau=0.8,
|
|
96
|
+
buffer=2, # 2 = Dual-Buffer (Standard), 1 = Single-Buffer (Low VRAM)
|
|
97
|
+
stochastic_rounding=True,
|
|
98
|
+
execution="auto"
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
# 3. Standard training loop
|
|
102
|
+
data = torch.randn(32, 1024, device=device, dtype=torch.float16)
|
|
103
|
+
target = torch.randn(32, 1024, device=device, dtype=torch.float16)
|
|
104
|
+
criterion = nn.MSELoss()
|
|
105
|
+
|
|
106
|
+
optimizer.zero_grad(set_to_none=True)
|
|
107
|
+
output = model(data)
|
|
108
|
+
loss = criterion(output, target)
|
|
109
|
+
loss.backward()
|
|
110
|
+
|
|
111
|
+
optimizer.step()
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
---
|
|
115
|
+
|
|
116
|
+
## Parameter Reference
|
|
117
|
+
|
|
118
|
+
| Parameter | Type | Default | Description |
|
|
119
|
+
| :--- | :--- | :--- | :--- |
|
|
120
|
+
| `params` | `iterable` | *Required* | Iterable of parameters to optimize or dicts defining parameter groups. |
|
|
121
|
+
| `lr` | `float` | `8e-4` | Learning rate (eta). |
|
|
122
|
+
| `betas` | `Tuple[float, float]` | `(0.9, 0.999)` | Coefficients (beta1, beta2) for running momentum and centered innovation variance. |
|
|
123
|
+
| `eps` | `float` | `1e-8` | Numerical stability term (epsilon). Automatically floored to `1e-4` in FP16 to prevent subnormal underflow. |
|
|
124
|
+
| `weight_decay` | `float` | `8e-2` | Decoupled weight decay coefficient (lambda). |
|
|
125
|
+
| `tau` | `float` | `0.8` | Analytical bias correction temperature parameter (tau). |
|
|
126
|
+
| `alpha_ss` | `float` | `0.5` | Softsign dampening factor (alpha_ss) used in single-buffer mode (`buffer=1`). |
|
|
127
|
+
| `cautious` | `bool` | `True` | If `True`, enables Traffic-Cop directional verification masking. |
|
|
128
|
+
| `cautious_clamp_min` | `float` | `0.2` | Safety floor density clamp (gamma_min) preventing division by zero in masked normalization. |
|
|
129
|
+
| `buffer` | `int` | `2` | Buffer mode: `2` (Dual-buffer tracking m_t and v_t) or `1` (Single-buffer scalar RMS tracking). |
|
|
130
|
+
| `stochastic_rounding` | `bool` | `True` | Enables bitwise stochastic rounding on native FP16/BF16 weights. |
|
|
131
|
+
| `execution` | `str` | `"auto"` | Execution engine: `"auto"`, `"triton"`, `"foreach"`, or `"single"`. |
|
|
132
|
+
|
|
133
|
+
---
|
|
134
|
+
|
|
135
|
+
## Citation
|
|
136
|
+
|
|
137
|
+
If you utilize LuminaV in your research or applications, please cite both the foundational paper and this software implementation:
|
|
138
|
+
|
|
139
|
+
```bibtex
|
|
140
|
+
# 1. To cite the official research paper & theoretical mechanics
|
|
141
|
+
@misc{luminamoon2026luminav_paper,
|
|
142
|
+
author = {{Silver Moon (cloverxion)}},
|
|
143
|
+
organization = {Lumina Moon},
|
|
144
|
+
title = {{LuminaV: We Were Too Broke for AdamW So We Trapped Gradients in a Hyperbolic Straitjacket and Hired a Traffic Cop to Slap Them}},
|
|
145
|
+
year = {2026},
|
|
146
|
+
publisher = {Hugging Face},
|
|
147
|
+
doi = {10.57967/hf/10270},
|
|
148
|
+
url = {https://huggingface.co/cloverx-id/XoneLM-1.0-Paper}
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
# 2. To cite this software implementation & standalone codebase
|
|
152
|
+
@software{luminamoon2026luminav_code,
|
|
153
|
+
author = {{Silver Moon (cloverxion)}},
|
|
154
|
+
organization = {Lumina Moon},
|
|
155
|
+
title = {{LuminaV Optimizer: Official PyTorch Implementation}},
|
|
156
|
+
year = {2026},
|
|
157
|
+
publisher = {Hugging Face},
|
|
158
|
+
version = {1.1.5},
|
|
159
|
+
doi = {10.57967/hf/10365},
|
|
160
|
+
url = {https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper}
|
|
161
|
+
}
|
|
162
|
+
```
|
|
163
|
+
|
|
164
|
+
---
|
|
165
|
+
|
|
166
|
+
## License
|
|
167
|
+
|
|
168
|
+
Apache License 2.0. See [LICENSE](LICENSE) for full terms.
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=75.0.0"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "luminav"
|
|
7
|
+
version = "1.1.5"
|
|
8
|
+
description = "A master-free, memory-efficient adaptive optimizer for low-precision deep learning"
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.10"
|
|
11
|
+
license = { text = "Apache-2.0" }
|
|
12
|
+
authors = [
|
|
13
|
+
{ name = "Silver Moon (cloverxion)" }
|
|
14
|
+
]
|
|
15
|
+
maintainers = [
|
|
16
|
+
{ name = "Lumina Moon" }
|
|
17
|
+
]
|
|
18
|
+
keywords = [
|
|
19
|
+
"pytorch",
|
|
20
|
+
"optimizer",
|
|
21
|
+
"low-vram",
|
|
22
|
+
"fp16",
|
|
23
|
+
"bf16",
|
|
24
|
+
"triton",
|
|
25
|
+
"deep-learning",
|
|
26
|
+
"llm",
|
|
27
|
+
"transformer"
|
|
28
|
+
]
|
|
29
|
+
classifiers = [
|
|
30
|
+
"Development Status :: 4 - Beta",
|
|
31
|
+
"Intended Audience :: Science/Research",
|
|
32
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
33
|
+
"License :: OSI Approved :: Apache Software License",
|
|
34
|
+
"Programming Language :: Python :: 3",
|
|
35
|
+
"Programming Language :: Python :: 3.10",
|
|
36
|
+
"Programming Language :: Python :: 3.11",
|
|
37
|
+
"Programming Language :: Python :: 3.12",
|
|
38
|
+
"Programming Language :: Python :: 3.13",
|
|
39
|
+
"Programming Language :: Python :: 3.14",
|
|
40
|
+
"Operating System :: OS Independent",
|
|
41
|
+
]
|
|
42
|
+
dependencies = [
|
|
43
|
+
"torch>=2.0.0",
|
|
44
|
+
]
|
|
45
|
+
|
|
46
|
+
[project.optional-dependencies]
|
|
47
|
+
triton = ["triton"]
|
|
48
|
+
|
|
49
|
+
[project.urls]
|
|
50
|
+
"Homepage" = "https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper"
|
|
51
|
+
"Author Profile" = "https://huggingface.co/cloverxion"
|
|
52
|
+
"Organization" = "https://huggingface.co/Lumina-Moon"
|
|
53
|
+
"Software DOI" = "https://doi.org/10.57967/hf/10365"
|
|
54
|
+
"Paper DOI" = "https://doi.org/10.57967/hf/10270"
|
|
55
|
+
"Bug Tracker" = "https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper/discussions"
|
|
56
|
+
"Changelog" = "https://huggingface.co/cloverx-id/LuminaV-Optimizer-Paper/blob/main/CHANGELOG.md"
|
luminav-1.1.5/setup.cfg
ADDED