dirichlet-regularization 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirichlet_regularization-1.0.0.dist-info/METADATA +292 -0
- dirichlet_regularization-1.0.0.dist-info/RECORD +11 -0
- dirichlet_regularization-1.0.0.dist-info/WHEEL +5 -0
- dirichlet_regularization-1.0.0.dist-info/licenses/LICENSE +21 -0
- dirichlet_regularization-1.0.0.dist-info/top_level.txt +1 -0
- dreg/__init__.py +45 -0
- dreg/baselines.py +93 -0
- dreg/model.py +189 -0
- dreg/quantization.py +419 -0
- dreg/spectral.py +164 -0
- dreg/topology.py +338 -0
|
@@ -0,0 +1,292 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: dirichlet-regularization
|
|
3
|
+
Version: 1.0.0
|
|
4
|
+
Summary: 2D Dirichlet spatial regularization for neural network weights: inducing topographic smoothness for 2D-DCT spectral compression and Base-3 trit quantization
|
|
5
|
+
Author-email: Mario Raúl Carbonell Martínez <marioraulcarbonell@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/mcarbonell/dirichlet-regularization
|
|
8
|
+
Project-URL: Repository, https://github.com/mcarbonell/dirichlet-regularization
|
|
9
|
+
Keywords: deep-learning,pytorch,regularization,dirichlet-energy,dct,quantization,model-compression
|
|
10
|
+
Classifier: Development Status :: 4 - Beta
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Programming Language :: Python :: 3
|
|
13
|
+
Classifier: Programming Language :: Python :: 3.9
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: C
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Operating System :: OS Independent
|
|
20
|
+
Requires-Python: >=3.9
|
|
21
|
+
Description-Content-Type: text/markdown
|
|
22
|
+
License-File: LICENSE
|
|
23
|
+
Requires-Dist: torch>=2.0.0
|
|
24
|
+
Requires-Dist: numpy>=1.20.0
|
|
25
|
+
Provides-Extra: examples
|
|
26
|
+
Requires-Dist: torchvision>=0.15.0; extra == "examples"
|
|
27
|
+
Requires-Dist: matplotlib>=3.5.0; extra == "examples"
|
|
28
|
+
Provides-Extra: benchmarks
|
|
29
|
+
Requires-Dist: modal>=0.50.0; extra == "benchmarks"
|
|
30
|
+
Requires-Dist: datasets>=2.0.0; extra == "benchmarks"
|
|
31
|
+
Requires-Dist: tokenizers>=0.13.0; extra == "benchmarks"
|
|
32
|
+
Requires-Dist: transformers>=4.28.0; extra == "benchmarks"
|
|
33
|
+
Requires-Dist: matplotlib>=3.5.0; extra == "benchmarks"
|
|
34
|
+
Requires-Dist: tqdm>=4.65.0; extra == "benchmarks"
|
|
35
|
+
Provides-Extra: dev
|
|
36
|
+
Requires-Dist: pytest>=7.0.0; extra == "dev"
|
|
37
|
+
Requires-Dist: pytest-cov>=4.0.0; extra == "dev"
|
|
38
|
+
Requires-Dist: ruff>=0.1.0; extra == "dev"
|
|
39
|
+
Dynamic: license-file
|
|
40
|
+
|
|
41
|
+
<div align="center">
|
|
42
|
+
|
|
43
|
+
# Dirichlet Regularization (`dreg`)
|
|
44
|
+
### Spatial Smoothness in Neural Weight Manifolds for 2D-DCT Spectral Compression
|
|
45
|
+
|
|
46
|
+
[](https://www.python.org/downloads/)
|
|
47
|
+
[](https://pytorch.org/)
|
|
48
|
+
[](https://github.com/mcarbonell/dirichlet-regularization/actions/workflows/ci.yml)
|
|
49
|
+
[](https://github.com/mcarbonell/dirichlet-regularization/tree/main/tests)
|
|
50
|
+
[](https://github.com/mcarbonell/dirichlet-regularization/tree/main/tests)
|
|
51
|
+
[](https://github.com/astral-sh/ruff)
|
|
52
|
+
[](https://github.com/mcarbonell/dirichlet-regularization/blob/main/paper/paper-draft.pdf)
|
|
53
|
+
[](https://github.com/mcarbonell/dirichlet-regularization/blob/main/LICENSE)
|
|
54
|
+
|
|
55
|
+
</div>
|
|
56
|
+
|
|
57
|
+
---
|
|
58
|
+
|
|
59
|
+
## The Core Idea
|
|
60
|
+
|
|
61
|
+
Standard neural networks treat weight matrices as unstructured, permutation-invariant point clouds. After training with isotropic $L_2$ weight decay, their 2D spatial frequency spectrum resembles **flat white noise** — variance is spread uniformly across all frequencies, making frequency-domain compression ineffective.
|
|
62
|
+
|
|
63
|
+
**Dirichlet Regularization (`dreg`)** introduces a simple inductive bias during training: *neighboring entries on a 2D weight lattice should vary smoothly*, analogous to topographic organization in biological cortex.
|
|
64
|
+
|
|
65
|
+
Penalizing discrete 2D spatial differences breaks permutation symmetry and **concentrates spectral energy in the low-frequency harmonics of the orthonormal 2D-DCT** — up to **~72%** of Frobenius energy below $\rho \le 0.40$ on square Attention projections (`256×256`) and **52.8%** on MNIST MLPs — enabling hierarchical Base-3 Trit spectral quantization (`.tritq`) across a configurable `0.43–3.77 bpp` rate–distortion frontier.
|
|
66
|
+
|
|
67
|
+
<p align="center">
|
|
68
|
+
<img src="https://raw.githubusercontent.com/mcarbonell/dirichlet-regularization/main/docs/figures/fig1_weight_heatmaps.png" alt="Spatial Weight Structures: Standard AdamW vs Dirichlet Regularization" width="90%">
|
|
69
|
+
</p>
|
|
70
|
+
|
|
71
|
+
> **Figure 1: Illustrative Spatial Weight Structures (MNIST MLP, 64×64 top-left patches).** 3-epoch MNIST MLP (`fc1` top, `fc2` bottom, unnormalized $\epsilon = 2 \times 10^{-3}$). Left: Standard AdamW yields high-frequency unstructured weights (edge-normalized $E_D = 0.0016$ on `fc1`). Right: 2D Dirichlet regularization induces smooth spatial manifolds ($E_D = 0.0001$, $-94\%$ roughness). On full TinyStories-10M Transformers (Table 2 of the preprint), mean $E_D$ drops by **$-72.1\%$** ($0.0123 \to 0.0034$ `numel`; $0.0062 \to 0.0017$ `edges`).
|
|
72
|
+
|
|
73
|
+
---
|
|
74
|
+
|
|
75
|
+
## The Mathematics
|
|
76
|
+
|
|
77
|
+
Standard $L_2$ weight decay penalizes **magnitude** and is strictly invariant to row/column permutations:
|
|
78
|
+
|
|
79
|
+
$$\mathcal{L}_{L_2}(W) = \frac{\gamma}{2} \|W\|_F^2$$
|
|
80
|
+
|
|
81
|
+
**Dirichlet Regularization** penalizes **2D discrete spatial differences** across the $M \times N$ weight lattice via the separable Neumann graph Laplacians $L_M \in \mathbb{R}^{M \times M}$ and $L_N \in \mathbb{R}^{N \times N}$:
|
|
82
|
+
|
|
83
|
+
$$\mathcal{E}_D^{\text{var}}(W) = \frac{1}{2MN}\!\left(\sum_{i=1}^{M-1}\sum_{j=1}^N (W_{i+1,j}-W_{i,j})^2 + \sum_{i=1}^M\sum_{j=1}^{N-1} (W_{i,j+1}-W_{i,j})^2\right) = \frac{1}{2MN}\Big(\operatorname{Tr}(W^\top L_M W) + \operatorname{Tr}(W L_N W^\top)\Big)$$
|
|
84
|
+
|
|
85
|
+
Because the eigenvectors of the 1D Neumann path-graph Laplacian are exactly the orthonormal **Type-II DCT** basis vectors, $\mathcal{E}_D$ diagonalizes in the 2D-DCT domain $S = D_M W D_N^\top$ (`numel` convention $= 2\,\mathcal{E}_D^{\text{var}}$):
|
|
86
|
+
|
|
87
|
+
$$\mathcal{E}_D^{\text{numel}}(W) = \frac{1}{MN} \sum_{k=0}^{M-1}\sum_{l=0}^{N-1} 4\!\left[\sin^2\!\left(\frac{\pi k}{2M}\right) + \sin^2\!\left(\frac{\pi l}{2N}\right)\right] S_{k,l}^2$$
|
|
88
|
+
|
|
89
|
+
High-frequency modes $(k, l)$ are penalized quadratically, concentrating weight energy into the lowest 2D-DCT harmonics.
|
|
90
|
+
|
|
91
|
+
> **Normalization note:** `dreg` provides `normalization="numel"` ($\frac{1}{MN}\sum \Delta^2 = 2\,\mathcal{E}_D^{\text{var}}$, default), `normalization="edges"` ($\approx \mathcal{E}_D^{\text{var}}$), and scale-invariant `normalization="rayleigh"` ($\frac{\sum \Delta^2}{2\|W\|_F^2} \in [0, 4]$, immune to $L_2$ magnitude shrinkage), plus `layer_weighting="param_weighted"` to balance gradient strength equally across square Attention and wide FFN layers. See [dreg/topology.py](https://github.com/mcarbonell/dirichlet-regularization/blob/main/dreg/topology.py) and §3.1 of the preprint.
|
|
92
|
+
|
|
93
|
+
<p align="center">
|
|
94
|
+
<img src="https://raw.githubusercontent.com/mcarbonell/dirichlet-regularization/main/docs/figures/fig2_dct_energy_spectra.png" alt="2D-DCT Spectral Energy Compaction" width="90%">
|
|
95
|
+
</p>
|
|
96
|
+
|
|
97
|
+
> **Figure 2: 2D-DCT Cumulative Spectral Energy (TinyStories-10M, Layer 3).** Cumulative Frobenius energy vs. normalized radius $\rho \in [0, \sqrt{2}]$. Standard AdamW stays flat ($\sim 13\text{--}14\%$ below $\rho = 0.40$). Dirichlet regularization concentrates **~72%** of energy below $\rho = 0.40$ in Attention $W_q$ (`256×256`), and **~25%** in the $4\times$ larger FFN gate projection (`256×1024`) due to $1/(MN)$ per-weight gradient scaling.
|
|
98
|
+
|
|
99
|
+
---
|
|
100
|
+
|
|
101
|
+
## Quick Start
|
|
102
|
+
|
|
103
|
+
### Installation
|
|
104
|
+
|
|
105
|
+
```bash
|
|
106
|
+
pip install dirichlet-regularization
|
|
107
|
+
```
|
|
108
|
+
|
|
109
|
+
Or from source:
|
|
110
|
+
```bash
|
|
111
|
+
git clone https://github.com/mcarbonell/dirichlet-regularization.git
|
|
112
|
+
cd dirichlet-regularization
|
|
113
|
+
pip install -e .
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
Optional extras:
|
|
117
|
+
```bash
|
|
118
|
+
pip install -e ".[examples]" # torchvision + matplotlib for examples
|
|
119
|
+
pip install -e ".[dev]" # pytest, pytest-cov, ruff
|
|
120
|
+
```
|
|
121
|
+
|
|
122
|
+
### Apply to Any PyTorch Model (2 lines of code)
|
|
123
|
+
|
|
124
|
+
#### Option A: Universal loss wrapper (zero model modifications)
|
|
125
|
+
```python
|
|
126
|
+
from dreg import DirichletLoss
|
|
127
|
+
|
|
128
|
+
# Automatically skips nn.Embedding, lm_head, frozen weights, and deduplicates tied weights.
|
|
129
|
+
# Use normalization="rayleigh" + layer_weighting="param_weighted" for scale-invariant,
|
|
130
|
+
# FFN-balanced smoothing (recommended: weight_decay=0.25–0.50), or classic normalization="numel"
|
|
131
|
+
# with weight_decay=15.0–30.0.
|
|
132
|
+
topo_reg = DirichletLoss(
|
|
133
|
+
weight_decay=0.50,
|
|
134
|
+
normalization="rayleigh",
|
|
135
|
+
layer_weighting="param_weighted",
|
|
136
|
+
)
|
|
137
|
+
|
|
138
|
+
# Inside your training loop:
|
|
139
|
+
task_loss = criterion(model(inputs), targets)
|
|
140
|
+
total_loss = task_loss + topo_reg(model)
|
|
141
|
+
total_loss.backward()
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
#### Option B: Drop-in replacement for `nn.Linear`
|
|
145
|
+
```python
|
|
146
|
+
import torch.nn as nn
|
|
147
|
+
from dreg import TopographicLinear
|
|
148
|
+
|
|
149
|
+
class MyModel(nn.Module):
|
|
150
|
+
def __init__(self):
|
|
151
|
+
super().__init__()
|
|
152
|
+
self.fc1 = TopographicLinear(512, 2048)
|
|
153
|
+
self.fc2 = TopographicLinear(2048, 10)
|
|
154
|
+
self.act = nn.GELU()
|
|
155
|
+
|
|
156
|
+
def forward(self, x):
|
|
157
|
+
return self.fc2(self.act(self.fc1(x)))
|
|
158
|
+
|
|
159
|
+
def dirichlet_loss(self):
|
|
160
|
+
return 0.5 * (self.fc1.dirichlet_energy() + self.fc2.dirichlet_energy())
|
|
161
|
+
|
|
162
|
+
model = MyModel()
|
|
163
|
+
loss = criterion(model(x), y) + 15.0 * model.dirichlet_loss()
|
|
164
|
+
loss.backward()
|
|
165
|
+
```
|
|
166
|
+
|
|
167
|
+
> **Calibration tip:** Because `dirichlet_energy_2d` is normalized by $MN$, raw values are $\sim 10^{-3}\text{--}10^{-2}$. Use $\lambda_{\text{topo}} \in [5.0, 30.0]$ (rather than `0.01`, which is washed out by AdamW).
|
|
168
|
+
|
|
169
|
+
---
|
|
170
|
+
|
|
171
|
+
## Empirical Highlights
|
|
172
|
+
|
|
173
|
+
### 1. TinyStories 10.56M Transformer (81.92M tokens, A10G GPU)
|
|
174
|
+
|
|
175
|
+
Evaluated on 25 validation batches (204,800 tokens; full results in [results/pareto_curve_tinystories_results.json](https://github.com/mcarbonell/dirichlet-regularization/blob/main/results/pareto_curve_tinystories_results.json) and Preprint Table 5):
|
|
176
|
+
|
|
177
|
+
| Quantization Regime | Radii $(r_0, r_1, r_2)$ | Linear Rate | Compression | Std PPL | Topo PPL ($\lambda=30$) | Dirichlet Gain |
|
|
178
|
+
| :--- | :---: | :---: | :---: | :---: | :---: | :---: |
|
|
179
|
+
| Dense FP32 (Reference) | — | 32.000 bpp | 1.0× | **6.11** | 6.46 | $-0.35$ PPL |
|
|
180
|
+
| Conservative | `(0.45, 0.90, 1.41)` | 3.774 bpp | 8.5× | 8.87 | **8.22** | **+0.65 PPL** |
|
|
181
|
+
| High-Fidelity | `(0.30, 0.70, 1.30)` | 2.794 bpp | 11.5× | 14.97 | **12.43** | **+2.54 PPL** |
|
|
182
|
+
| Mid-Band | `(0.20, 0.55, 1.15)` | 2.181 bpp | 14.7× | 30.10 | **23.08** | **+7.02 PPL** |
|
|
183
|
+
| **Calibrated TritQ** | **`(0.15, 0.40, 1.00)`** | **1.638 bpp** | **19.5×** | 73.75 | **66.50** | **+7.25 PPL** |
|
|
184
|
+
| Aggressive | `(0.10, 0.30, 0.75)` | 0.914 bpp | 35.0× | **835.92** | 1117.65 | $-281.73$ PPL |
|
|
185
|
+
| Ultra-Aggressive | `(0.08, 0.22, 0.50)` | 0.430 bpp | 74.4× | 5274.18 | **2890.48** | **+2383.70 PPL** |
|
|
186
|
+
|
|
187
|
+
* Roughness reduction: **$-72.1\%$** ($E_D^{\text{edges}}: 0.006178 \to 0.001724$; $E_D^{\text{numel}}: 0.012316 \to 0.003438$).
|
|
188
|
+
* At `2.181 bpp`, Topographic TritQ (`23.08 PPL`) outperforms both Standard TritQ (`30.10 PPL`) and per-tensor Spatial INT2 RTN (`41.67 PPL` at `2.00 bpp`).
|
|
189
|
+
|
|
190
|
+
<p align="center">
|
|
191
|
+
<img src="https://raw.githubusercontent.com/mcarbonell/dirichlet-regularization/main/docs/figures/fig3_pareto_quantization.png" alt="Calibration Sweep and Empirical Rate-Distortion Frontier" width="95%">
|
|
192
|
+
</p>
|
|
193
|
+
|
|
194
|
+
> **Figure 3: Calibration Sweep and Rate–Distortion Frontier (TinyStories-10M).** (a) 5-point calibration sweep ($\lambda_{\text{topo}} \in \{0, 0.01, 5, 15, 30\}$, 2k steps) at `1.638 bpp`. (b) Converged 10k-step rate–distortion frontier across `1.638–3.774 bpp` (Preprint Table 5).
|
|
195
|
+
|
|
196
|
+
### 2. Vision MLP on MNIST (`234.75K` parameters, 5 epochs)
|
|
197
|
+
|
|
198
|
+
| Metric | Standard MLP | Topographic MLP (`dreg`, $\lambda=15$) | Gain |
|
|
199
|
+
| :--- | :---: | :---: | :---: |
|
|
200
|
+
| FP32 Test Accuracy | **97.87%** | 97.58% | $-0.29\%$ (preserved) |
|
|
201
|
+
| Dirichlet Energy ($E_D$, `numel`) | 0.0219 | **0.0034** | **$-84.5\%$ roughness** |
|
|
202
|
+
| Low-Frequency Energy ($\rho \le 0.40$) | 27.87% | **52.84%** | **$1.90\times$ compaction** |
|
|
203
|
+
| Low-Pass Truncation Acc ($\rho \le 0.40$) | 30.69% | **41.41%** | **+10.72% abs.** (+11.01% rel. retention) |
|
|
204
|
+
| Base-3 TritQ Acc (`1.64 bpp`) | 87.40% | **88.59%** | **+1.19% accuracy** |
|
|
205
|
+
|
|
206
|
+
---
|
|
207
|
+
|
|
208
|
+
## Quantization Format (`.tritq`) & C11 Direct Spectral Micro-Kernel
|
|
209
|
+
|
|
210
|
+
* **Exact Base-3 Trit Packing + Lloyd-Max Band-2:** 5 balanced trits $\{-1, 0, +1\}$ pack bijectively into 1 byte ($3^5 = 243 \le 256 \Rightarrow 1.60\text{ bits/trit}$), quantized via 1D Lloyd-Max MSE-optimal scaling and decoded in $\mathcal{O}(1)$ time via a 1.25 KB L1-cache lookup table. Non-topographic parameters (`token_emb`, `lm_head`, LayerNorms) are preserved uncompressed in FP16.
|
|
211
|
+
* **Portable C11 Micro-Kernel (`kernel/spectral_dma_kernel.c`):**
|
|
212
|
+
* **Zero-copy async DMA double-buffering** (Win32 Events / POSIX pthreads) verified to `< 1e-5` float32 parity against PyTorch.
|
|
213
|
+
* **Direct Spectral-Domain Streaming MatVec (`c_spectral_matvec_direct`):** Computes $y = \operatorname{IDCT}_{1D}(S \cdot \operatorname{DCT}_{1D}(x))$ via radix-2 $\mathcal{O}(N \log N)$ 1D-DCT directly from `.tritq` bitstreams **without materializing $M \times N$ weights or storing DCT basis tables** — cutting `1024×256` FFN working SRAM from `8.4 MB` down to **`13.0 KB`** (`650×` reduction) and running **`~45×` faster** than full-matrix 2D-IDCT reconstruction (`1.9e-6` float32 parity).
|
|
214
|
+
|
|
215
|
+
---
|
|
216
|
+
|
|
217
|
+
## Reproducing Locally (CPU)
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
# 1. Run the master end-to-end verification suite (99 tests + C parity + trained MLP + controls)
|
|
221
|
+
python examples/reproduce_all.py
|
|
222
|
+
|
|
223
|
+
# 2. Build and benchmark the native C11 DMA micro-kernel
|
|
224
|
+
python kernel/build_kernel.py
|
|
225
|
+
python examples/benchmark_c_dma.py
|
|
226
|
+
|
|
227
|
+
# 3. Train a small Topographic Transformer or MNIST MLP locally
|
|
228
|
+
python examples/train_topographic.py
|
|
229
|
+
python examples/train_mlp_dirichlet.py --dataset mnist --epochs 5
|
|
230
|
+
python examples/evaluate_falsification.py
|
|
231
|
+
```
|
|
232
|
+
|
|
233
|
+
---
|
|
234
|
+
|
|
235
|
+
## Repository Structure
|
|
236
|
+
|
|
237
|
+
```text
|
|
238
|
+
dirichlet-regularization/
|
|
239
|
+
├── dreg/ # Core Python package
|
|
240
|
+
│ ├── topology.py # 2D Dirichlet energy, DirichletLoss & annealing schedulers
|
|
241
|
+
│ ├── spectral.py # Orthonormal 2D-DCT/IDCT & BlockDCTTiler
|
|
242
|
+
│ ├── quantization.py # Base3TritQuantizer & .tritq binary checkpoint format
|
|
243
|
+
│ ├── model.py # Reference TopographicTransformer & TopographicLinear
|
|
244
|
+
│ └── baselines.py # Haar random orthogonal rotation & truncated SVD controls
|
|
245
|
+
├── kernel/ # Native C11 zero-copy DMA micro-kernel
|
|
246
|
+
│ ├── spectral_dma_kernel.c # O(1) trit LUT + Win32/POSIX async double-buffered DMA
|
|
247
|
+
│ ├── build_kernel.py # Cross-platform compiler wrapper (gcc/clang/cl)
|
|
248
|
+
│ └── Makefile
|
|
249
|
+
├── examples/ # Local CPU/GPU reproducible scripts
|
|
250
|
+
│ ├── reproduce_all.py # End-to-end verification (tests, C parity, trained MLP, controls)
|
|
251
|
+
│ ├── train_topographic.py # Minimal TopographicTransformer training example
|
|
252
|
+
│ ├── train_mlp_dirichlet.py # Vision MLP benchmark (synthetic / MNIST)
|
|
253
|
+
│ ├── evaluate_falsification.py # Controlled Standard vs Topographic comparison
|
|
254
|
+
│ ├── evaluate_pareto_curve.py # Local rate-distortion sweep & plotting
|
|
255
|
+
│ ├── train_multiseed_sweep.py # Multi-seed robustness & annealing sweep
|
|
256
|
+
│ ├── train_gpt2_dirichlet.py # GPT-2 fine-tuning & TritQ audit
|
|
257
|
+
│ └── benchmark_c_dma.py # Native C kernel parity & throughput benchmark
|
|
258
|
+
├── benchmarks/ # Modal cloud A10G GPU training & evaluation scripts
|
|
259
|
+
├── results/ # Raw JSON logs backing paper Tables 4–5 and diagnostics
|
|
260
|
+
├── paper/ # Academic preprint (LaTeX source, BibTeX, figures, PDF)
|
|
261
|
+
│ ├── paper-draft.pdf
|
|
262
|
+
│ ├── paper-draft.tex
|
|
263
|
+
│ ├── references.bib
|
|
264
|
+
│ └── figures/
|
|
265
|
+
├── tests/ # 99 automated unit, integration, and C-kernel tests (96% cov)
|
|
266
|
+
├── CITATION.cff
|
|
267
|
+
├── pyproject.toml
|
|
268
|
+
└── README.md
|
|
269
|
+
```
|
|
270
|
+
|
|
271
|
+
---
|
|
272
|
+
|
|
273
|
+
## Preprint & Citation
|
|
274
|
+
|
|
275
|
+
Read the full manuscript in [`paper/paper-draft.pdf`](https://github.com/mcarbonell/dirichlet-regularization/blob/main/paper/paper-draft.pdf):
|
|
276
|
+
> **"Inducing Spectral Smoothness in Neural Weight Manifolds via 2D Dirichlet Regularization"**
|
|
277
|
+
> *Mario Raúl Carbonell Martínez (Independent Researcher)*
|
|
278
|
+
|
|
279
|
+
```bibtex
|
|
280
|
+
@misc{carbonell2026dirichlet,
|
|
281
|
+
title={Inducing Spectral Smoothness in Neural Weight Manifolds via 2D Dirichlet Regularization},
|
|
282
|
+
author={Carbonell Mart{\'i}nez, Mario Ra{\'u}l},
|
|
283
|
+
year={2026},
|
|
284
|
+
howpublished={\url{https://github.com/mcarbonell/dirichlet-regularization}}
|
|
285
|
+
}
|
|
286
|
+
```
|
|
287
|
+
|
|
288
|
+
---
|
|
289
|
+
|
|
290
|
+
## License
|
|
291
|
+
|
|
292
|
+
This project is licensed under the MIT License — see the [LICENSE](https://github.com/mcarbonell/dirichlet-regularization/blob/main/LICENSE) file for details.
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
dirichlet_regularization-1.0.0.dist-info/licenses/LICENSE,sha256=NJET5xT0oUWaizC_RiqJChZcsQ3NRP0w2SfOYrdZZZw,1105
|
|
2
|
+
dreg/__init__.py,sha256=6mnhU2H-3IBGkGKx1rqK0quN1n3zj1A3lBWnV1jMIhc,1371
|
|
3
|
+
dreg/baselines.py,sha256=nwVFIuhqFTK20mx1Q3ruuN7RxqoTiPIX8oAmmfhkx00,3300
|
|
4
|
+
dreg/model.py,sha256=MBTVOGJJqrr5xt9BzOIjmJKFy2TdihsFlEPAz1md5uQ,8064
|
|
5
|
+
dreg/quantization.py,sha256=5CWIYoR9v4wAzaVEF8X_3c_yiTFigi96SzYojcKk4KU,19735
|
|
6
|
+
dreg/spectral.py,sha256=mUhgSGJOzANQC7Pyrox8duY94wRfVT5HTjA9up0SHfI,6325
|
|
7
|
+
dreg/topology.py,sha256=-PHOEuQCpISZ98L921pe8_VQahytamGIYnBOmXKVb1U,13362
|
|
8
|
+
dirichlet_regularization-1.0.0.dist-info/METADATA,sha256=OBC5kuaP12_jMd59xFwxwMVgDKtSztV4_LvEQSvFgT4,17191
|
|
9
|
+
dirichlet_regularization-1.0.0.dist-info/WHEEL,sha256=mOdXR9C3wMVov7cSY8PY-XCzo3SmmWCHvGQr5csJoao,91
|
|
10
|
+
dirichlet_regularization-1.0.0.dist-info/top_level.txt,sha256=jzsvYtyMktEPIp4ZFTXT67veW7jAlQ2f6OvveeGN_7M,5
|
|
11
|
+
dirichlet_regularization-1.0.0.dist-info/RECORD,,
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mcarbonell / Topographic Spectral Research Group
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
dreg
|
dreg/__init__.py
ADDED
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Dirichlet Regularization (dreg)
|
|
3
|
+
===============================
|
|
4
|
+
A mathematically principled framework for 2D Dirichlet spatial regularization of neural
|
|
5
|
+
network weights: cortical-inspired topographic smoothness that concentrates 2D-DCT
|
|
6
|
+
spectral energy and improves low-bit Base-3 trit spectral quantization.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from .baselines import random_orthogonal_transform_2d, svd_low_rank_approximation
|
|
10
|
+
from .model import TopographicConfig, TopographicLinear, TopographicTransformer
|
|
11
|
+
from .quantization import Base3TritQuantizer, TritQFormat
|
|
12
|
+
from .spectral import BlockDCTTiler, dct1d, dct2d, dct_matrix_1d, idct1d, idct2d, spectral_matvec
|
|
13
|
+
from .topology import (
|
|
14
|
+
CosineDirichletScheduler,
|
|
15
|
+
DirichletLoss,
|
|
16
|
+
DirichletScheduler,
|
|
17
|
+
StepCooldownScheduler,
|
|
18
|
+
dirichlet_energy_2d,
|
|
19
|
+
get_grid_dimensions,
|
|
20
|
+
)
|
|
21
|
+
|
|
22
|
+
__version__ = "1.0.0"
|
|
23
|
+
__all__ = [
|
|
24
|
+
"DirichletLoss",
|
|
25
|
+
"dirichlet_energy_2d",
|
|
26
|
+
"get_grid_dimensions",
|
|
27
|
+
"DirichletScheduler",
|
|
28
|
+
"StepCooldownScheduler",
|
|
29
|
+
"CosineDirichletScheduler",
|
|
30
|
+
"dct_matrix_1d",
|
|
31
|
+
"dct1d",
|
|
32
|
+
"idct1d",
|
|
33
|
+
"dct2d",
|
|
34
|
+
"idct2d",
|
|
35
|
+
"spectral_matvec",
|
|
36
|
+
"BlockDCTTiler",
|
|
37
|
+
"Base3TritQuantizer",
|
|
38
|
+
"TritQFormat",
|
|
39
|
+
"TopographicTransformer",
|
|
40
|
+
"TopographicConfig",
|
|
41
|
+
"TopographicLinear",
|
|
42
|
+
"random_orthogonal_transform_2d",
|
|
43
|
+
"svd_low_rank_approximation",
|
|
44
|
+
]
|
|
45
|
+
|
dreg/baselines.py
ADDED
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Comparative Baselines Module
|
|
3
|
+
============================
|
|
4
|
+
Implements standard quantization and compression controls demanded in peer review:
|
|
5
|
+
1. Random Orthogonal Rotation / Incoherence Transform (QuIP/QuaRot-style control)
|
|
6
|
+
2. SVD Low-Rank Truncation (budget-matched low-rank baseline)
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
import math
|
|
10
|
+
from typing import Optional, Tuple
|
|
11
|
+
|
|
12
|
+
import torch
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def random_orthogonal_transform_2d(
|
|
16
|
+
weight: torch.Tensor,
|
|
17
|
+
seed: Optional[int] = None,
|
|
18
|
+
) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
19
|
+
"""
|
|
20
|
+
Applies random orthogonal rotation matrices Q_L and Q_R to a weight matrix:
|
|
21
|
+
W_rot = Q_L @ W @ Q_R^T
|
|
22
|
+
where Q_L in O(M) and Q_R in O(N).
|
|
23
|
+
|
|
24
|
+
This serves as the critical control against Hadamard/QuIP-style incoherence:
|
|
25
|
+
it rotates coordinates without imposing spatial continuity.
|
|
26
|
+
"""
|
|
27
|
+
if seed is not None:
|
|
28
|
+
gen = torch.Generator(device=weight.device).manual_seed(seed)
|
|
29
|
+
else:
|
|
30
|
+
gen = None
|
|
31
|
+
|
|
32
|
+
m, n = weight.shape
|
|
33
|
+
# Generate Haar-uniform random orthogonal matrices via QR + Mezzadri (2007) sign correction
|
|
34
|
+
g_l = torch.randn((m, m), device=weight.device, dtype=weight.dtype, generator=gen)
|
|
35
|
+
q_l, r_l = torch.linalg.qr(g_l)
|
|
36
|
+
diag_l = torch.sign(torch.diag(r_l))
|
|
37
|
+
diag_l = torch.where(diag_l == 0, torch.ones_like(diag_l), diag_l)
|
|
38
|
+
q_l = q_l * diag_l.unsqueeze(0)
|
|
39
|
+
|
|
40
|
+
g_r = torch.randn((n, n), device=weight.device, dtype=weight.dtype, generator=gen)
|
|
41
|
+
q_r, r_r = torch.linalg.qr(g_r)
|
|
42
|
+
diag_r = torch.sign(torch.diag(r_r))
|
|
43
|
+
diag_r = torch.where(diag_r == 0, torch.ones_like(diag_r), diag_r)
|
|
44
|
+
q_r = q_r * diag_r.unsqueeze(0)
|
|
45
|
+
|
|
46
|
+
w_rot = q_l @ weight @ q_r.t()
|
|
47
|
+
return w_rot, q_l, q_r
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def svd_low_rank_approximation(
|
|
51
|
+
weight: torch.Tensor,
|
|
52
|
+
rank: Optional[int] = None,
|
|
53
|
+
compression_ratio: Optional[float] = None,
|
|
54
|
+
) -> Tuple[torch.Tensor, int]:
|
|
55
|
+
"""
|
|
56
|
+
Computes optimal truncated SVD rank-k approximation of a weight matrix:
|
|
57
|
+
W_approx = U[:, :k] @ diag(S[:k]) @ Vh[:k, :]
|
|
58
|
+
|
|
59
|
+
Args:
|
|
60
|
+
weight: [M, N] tensor
|
|
61
|
+
rank: Explicit rank cutoff k.
|
|
62
|
+
compression_ratio: If rank is None, determines k such that k*(M+N) achieves
|
|
63
|
+
the target compression relative to M*N FP32 elements.
|
|
64
|
+
|
|
65
|
+
Returns:
|
|
66
|
+
(W_approx, rank_k)
|
|
67
|
+
"""
|
|
68
|
+
m, n = weight.shape
|
|
69
|
+
max_rank = min(m, n)
|
|
70
|
+
|
|
71
|
+
if rank is None:
|
|
72
|
+
if compression_ratio is not None and compression_ratio > 1.0:
|
|
73
|
+
# target_elements = (M * N) / compression_ratio
|
|
74
|
+
# k * (M + N) <= target_elements => k = target_elements / (M + N)
|
|
75
|
+
target_params = (m * n) / compression_ratio
|
|
76
|
+
rank = max(1, min(max_rank, int(math.floor(target_params / (m + n)))))
|
|
77
|
+
else:
|
|
78
|
+
rank = max(1, max_rank // 4)
|
|
79
|
+
else:
|
|
80
|
+
rank = max(1, min(max_rank, rank))
|
|
81
|
+
|
|
82
|
+
u, s, vh = torch.linalg.svd(weight, full_matrices=False)
|
|
83
|
+
w_approx = u[:, :rank] @ torch.diag(s[:rank]) @ vh[:rank, :]
|
|
84
|
+
return w_approx, rank
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def spectral_hadamard_benchmark_control(weight: torch.Tensor, seed: Optional[int] = None) -> torch.Tensor:
|
|
88
|
+
"""
|
|
89
|
+
Applies a Haar-uniform random orthogonal 2D rotation control (incoherence
|
|
90
|
+
baseline without imposing spatial manifold smoothing).
|
|
91
|
+
"""
|
|
92
|
+
w_rot, _, _ = random_orthogonal_transform_2d(weight, seed=seed)
|
|
93
|
+
return w_rot
|
dreg/model.py
ADDED
|
@@ -0,0 +1,189 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Topographic Transformer Architecture Module
|
|
3
|
+
============================================
|
|
4
|
+
Implements the causal autoregressive Transformer with native 2D cortical lattice
|
|
5
|
+
topology, SwiGLU FFNs, and integrated Dirichlet loss accumulation.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import math
|
|
9
|
+
from dataclasses import dataclass
|
|
10
|
+
from typing import List, Optional, Tuple
|
|
11
|
+
|
|
12
|
+
import torch
|
|
13
|
+
import torch.nn as nn
|
|
14
|
+
import torch.nn.functional as F
|
|
15
|
+
|
|
16
|
+
from .topology import dirichlet_energy_2d
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
@dataclass
|
|
20
|
+
class TopographicConfig:
|
|
21
|
+
vocab_size: int = 4096
|
|
22
|
+
d_model: int = 256
|
|
23
|
+
n_heads: int = 4
|
|
24
|
+
n_layers: int = 6
|
|
25
|
+
ffn_dim: int = 512
|
|
26
|
+
max_seq_len: int = 256
|
|
27
|
+
dropout: float = 0.0
|
|
28
|
+
topo_lambda: float = 0.01
|
|
29
|
+
topo_normalization: str = "numel"
|
|
30
|
+
topo_layer_weighting: str = "uniform"
|
|
31
|
+
use_block_dct: bool = True
|
|
32
|
+
block_size: int = 64
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class TopographicLinear(nn.Linear):
|
|
36
|
+
"""
|
|
37
|
+
nn.Linear subclass that tracks its 2D cortical grid mapping
|
|
38
|
+
and computes its local Dirichlet harmonic energy.
|
|
39
|
+
"""
|
|
40
|
+
def __init__(self, in_features: int, out_features: int, bias: bool = False):
|
|
41
|
+
super().__init__(in_features, out_features, bias=bias)
|
|
42
|
+
self.grid_shape = (out_features, in_features)
|
|
43
|
+
|
|
44
|
+
def dirichlet_energy(self, normalization: str = "numel") -> torch.Tensor:
|
|
45
|
+
return dirichlet_energy_2d(self.weight, normalization=normalization)
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
class TopographicAttention(nn.Module):
|
|
49
|
+
def __init__(self, cfg: TopographicConfig):
|
|
50
|
+
super().__init__()
|
|
51
|
+
self.cfg = cfg
|
|
52
|
+
self.d_model = cfg.d_model
|
|
53
|
+
self.n_heads = cfg.n_heads
|
|
54
|
+
self.d_head = cfg.d_model // cfg.n_heads
|
|
55
|
+
assert self.d_head * cfg.n_heads == cfg.d_model, "d_model must be divisible by n_heads"
|
|
56
|
+
|
|
57
|
+
self.q_proj = TopographicLinear(cfg.d_model, cfg.d_model, bias=False)
|
|
58
|
+
self.k_proj = TopographicLinear(cfg.d_model, cfg.d_model, bias=False)
|
|
59
|
+
self.v_proj = TopographicLinear(cfg.d_model, cfg.d_model, bias=False)
|
|
60
|
+
self.out_proj = TopographicLinear(cfg.d_model, cfg.d_model, bias=False)
|
|
61
|
+
self.attn_dropout = nn.Dropout(cfg.dropout) if cfg.dropout > 0.0 else nn.Identity()
|
|
62
|
+
|
|
63
|
+
def forward(self, x: torch.Tensor, mask: Optional[torch.Tensor] = None) -> torch.Tensor:
|
|
64
|
+
b, s, d = x.shape
|
|
65
|
+
q = self.q_proj(x).view(b, s, self.n_heads, self.d_head).transpose(1, 2)
|
|
66
|
+
k = self.k_proj(x).view(b, s, self.n_heads, self.d_head).transpose(1, 2)
|
|
67
|
+
v = self.v_proj(x).view(b, s, self.n_heads, self.d_head).transpose(1, 2)
|
|
68
|
+
|
|
69
|
+
# Scaled dot-product attention
|
|
70
|
+
scores = torch.matmul(q, k.transpose(-2, -1)) / math.sqrt(self.d_head)
|
|
71
|
+
if mask is not None:
|
|
72
|
+
scores = scores.masked_fill(mask[:, :, :s, :s] == 0, float("-inf"))
|
|
73
|
+
else:
|
|
74
|
+
causal_mask = torch.triu(torch.full((s, s), float("-inf"), device=x.device), diagonal=1)
|
|
75
|
+
scores = scores + causal_mask
|
|
76
|
+
|
|
77
|
+
attn = self.attn_dropout(F.softmax(scores, dim=-1))
|
|
78
|
+
out = torch.matmul(attn, v).transpose(1, 2).contiguous().view(b, s, d)
|
|
79
|
+
return self.out_proj(out)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
class TopographicMLP(nn.Module):
|
|
83
|
+
"""
|
|
84
|
+
SwiGLU feed-forward network with topographic linear projections.
|
|
85
|
+
"""
|
|
86
|
+
def __init__(self, cfg: TopographicConfig):
|
|
87
|
+
super().__init__()
|
|
88
|
+
self.gate_proj = TopographicLinear(cfg.d_model, cfg.ffn_dim, bias=False)
|
|
89
|
+
self.up_proj = TopographicLinear(cfg.d_model, cfg.ffn_dim, bias=False)
|
|
90
|
+
self.down_proj = TopographicLinear(cfg.ffn_dim, cfg.d_model, bias=False)
|
|
91
|
+
|
|
92
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
93
|
+
return self.down_proj(F.silu(self.gate_proj(x)) * self.up_proj(x))
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class TopographicBlock(nn.Module):
|
|
97
|
+
def __init__(self, cfg: TopographicConfig):
|
|
98
|
+
super().__init__()
|
|
99
|
+
self.ln1 = nn.LayerNorm(cfg.d_model)
|
|
100
|
+
self.attn = TopographicAttention(cfg)
|
|
101
|
+
self.ln2 = nn.LayerNorm(cfg.d_model)
|
|
102
|
+
self.mlp = TopographicMLP(cfg)
|
|
103
|
+
self.resid_dropout = nn.Dropout(cfg.dropout) if cfg.dropout > 0.0 else nn.Identity()
|
|
104
|
+
|
|
105
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
106
|
+
x = x + self.resid_dropout(self.attn(self.ln1(x)))
|
|
107
|
+
x = x + self.resid_dropout(self.mlp(self.ln2(x)))
|
|
108
|
+
return x
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class TopographicTransformer(nn.Module):
|
|
112
|
+
def __init__(self, cfg: TopographicConfig):
|
|
113
|
+
super().__init__()
|
|
114
|
+
self.cfg = cfg
|
|
115
|
+
self.token_emb = nn.Embedding(cfg.vocab_size, cfg.d_model)
|
|
116
|
+
self.pos_emb = nn.Parameter(torch.zeros(1, cfg.max_seq_len, cfg.d_model))
|
|
117
|
+
self.blocks = nn.ModuleList([TopographicBlock(cfg) for _ in range(cfg.n_layers)])
|
|
118
|
+
self.ln_f = nn.LayerNorm(cfg.d_model)
|
|
119
|
+
# NOTE: lm_head intentionally uses standard nn.Linear, not TopographicLinear.
|
|
120
|
+
# Applying Dirichlet smoothness to the vocabulary projection would constrain
|
|
121
|
+
# the logit geometry, forcing semantically unrelated tokens with adjacent indices
|
|
122
|
+
# to share similar output weights and degrading generation quality.
|
|
123
|
+
self.lm_head = nn.Linear(cfg.d_model, cfg.vocab_size, bias=False)
|
|
124
|
+
|
|
125
|
+
nn.init.normal_(self.pos_emb, std=0.02)
|
|
126
|
+
self.apply(self._init_weights)
|
|
127
|
+
|
|
128
|
+
def _init_weights(self, m: nn.Module):
|
|
129
|
+
if isinstance(m, nn.Linear):
|
|
130
|
+
nn.init.normal_(m.weight, mean=0.0, std=0.02)
|
|
131
|
+
if m.bias is not None:
|
|
132
|
+
nn.init.zeros_(m.bias)
|
|
133
|
+
elif isinstance(m, nn.Embedding):
|
|
134
|
+
nn.init.normal_(m.weight, mean=0.0, std=0.02)
|
|
135
|
+
|
|
136
|
+
def forward(self, idx: torch.Tensor, targets: Optional[torch.Tensor] = None) -> Tuple[torch.Tensor, Optional[torch.Tensor]]:
|
|
137
|
+
b, s = idx.shape
|
|
138
|
+
if s > self.cfg.max_seq_len:
|
|
139
|
+
raise ValueError(f"Sequence length {s} exceeds max_seq_len {self.cfg.max_seq_len}")
|
|
140
|
+
x = self.token_emb(idx) + self.pos_emb[:, :s, :]
|
|
141
|
+
for blk in self.blocks:
|
|
142
|
+
x = blk(x)
|
|
143
|
+
x = self.ln_f(x)
|
|
144
|
+
logits = self.lm_head(x)
|
|
145
|
+
|
|
146
|
+
loss = None
|
|
147
|
+
if targets is not None:
|
|
148
|
+
loss = F.cross_entropy(logits.view(-1, self.cfg.vocab_size), targets.reshape(-1))
|
|
149
|
+
return logits, loss
|
|
150
|
+
|
|
151
|
+
def topographic_loss(self) -> torch.Tensor:
|
|
152
|
+
"""
|
|
153
|
+
Computes the 2D Dirichlet harmonic energy across all TopographicLinear
|
|
154
|
+
projections using the configured normalization and layer weighting.
|
|
155
|
+
"""
|
|
156
|
+
energies = []
|
|
157
|
+
numels = []
|
|
158
|
+
for m in self.modules():
|
|
159
|
+
if isinstance(m, TopographicLinear):
|
|
160
|
+
energies.append(m.dirichlet_energy(normalization=self.cfg.topo_normalization))
|
|
161
|
+
numels.append(m.weight.numel())
|
|
162
|
+
if not energies:
|
|
163
|
+
return torch.tensor(0.0, device=self.token_emb.weight.device)
|
|
164
|
+
if self.cfg.topo_layer_weighting == "param_weighted":
|
|
165
|
+
total_numel = float(sum(numels))
|
|
166
|
+
agg = sum(e * (n / total_numel) for e, n in zip(energies, numels))
|
|
167
|
+
else:
|
|
168
|
+
agg = sum(energies) / len(energies)
|
|
169
|
+
return agg * self.cfg.topo_lambda
|
|
170
|
+
|
|
171
|
+
def get_linear_projections(self) -> List[TopographicLinear]:
|
|
172
|
+
return [m for m in self.modules() if isinstance(m, TopographicLinear)]
|
|
173
|
+
|
|
174
|
+
@torch.no_grad()
|
|
175
|
+
def generate(self, prompt_tokens: torch.Tensor, max_new_tokens: int = 64, temperature: float = 0.8) -> torch.Tensor:
|
|
176
|
+
if temperature < 0.0:
|
|
177
|
+
raise ValueError(f"temperature must be non-negative, got {temperature}")
|
|
178
|
+
out = prompt_tokens.clone()
|
|
179
|
+
for _ in range(max_new_tokens):
|
|
180
|
+
idx_cond = out[:, -self.cfg.max_seq_len:]
|
|
181
|
+
logits, _ = self(idx_cond)
|
|
182
|
+
last_logits = logits[:, -1, :]
|
|
183
|
+
if temperature == 0.0:
|
|
184
|
+
next_token = torch.argmax(last_logits, dim=-1, keepdim=True)
|
|
185
|
+
else:
|
|
186
|
+
probs = F.softmax(last_logits / temperature, dim=-1)
|
|
187
|
+
next_token = torch.multinomial(probs, num_samples=1)
|
|
188
|
+
out = torch.cat([out, next_token], dim=1)
|
|
189
|
+
return out
|