dirichlet-regularization 1.0.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- dirichlet_regularization-1.0.0.dist-info/METADATA +292 -0
- dirichlet_regularization-1.0.0.dist-info/RECORD +11 -0
- dirichlet_regularization-1.0.0.dist-info/WHEEL +5 -0
- dirichlet_regularization-1.0.0.dist-info/licenses/LICENSE +21 -0
- dirichlet_regularization-1.0.0.dist-info/top_level.txt +1 -0
- dreg/__init__.py +45 -0
- dreg/baselines.py +93 -0
- dreg/model.py +189 -0
- dreg/quantization.py +419 -0
- dreg/spectral.py +164 -0
- dreg/topology.py +338 -0
dreg/topology.py
ADDED
|
@@ -0,0 +1,338 @@
|
|
|
1
|
+
"""
|
|
2
|
+
Topological Dirichlet Regularization Module
|
|
3
|
+
===========================================
|
|
4
|
+
Implements 2D cortical manifold embedding and Dirichlet harmonic pinning
|
|
5
|
+
to eliminate permutation invariance and concentrate energy into low spatial frequencies.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
import math
|
|
9
|
+
from typing import Iterable, Optional, Tuple, Union
|
|
10
|
+
|
|
11
|
+
import torch
|
|
12
|
+
import torch.nn as nn
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def get_grid_dimensions(n: int) -> Tuple[int, int]:
|
|
16
|
+
"""
|
|
17
|
+
Finds the integer factor pair (H, W) such that H * W = n
|
|
18
|
+
with the aspect ratio closest to 1.0 (square-like 2D cortical lattice).
|
|
19
|
+
"""
|
|
20
|
+
if n <= 0:
|
|
21
|
+
raise ValueError(f"n must be a positive integer, got {n}")
|
|
22
|
+
sqrt_n = int(math.isqrt(n))
|
|
23
|
+
for h in range(sqrt_n, 0, -1):
|
|
24
|
+
if n % h == 0:
|
|
25
|
+
return h, n // h
|
|
26
|
+
return 1, n
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def dirichlet_energy_2d(
|
|
30
|
+
weight: torch.Tensor,
|
|
31
|
+
grid_shape: Optional[Tuple[int, int]] = None,
|
|
32
|
+
normalization: str = "numel",
|
|
33
|
+
) -> torch.Tensor:
|
|
34
|
+
"""
|
|
35
|
+
Computes the 2D Dirichlet Harmonic Energy of a weight matrix.
|
|
36
|
+
|
|
37
|
+
Variational form (discrete unit lattice normalized by 2*M*N):
|
|
38
|
+
E_var(W) = 1/(2*M*N) * sum_{(u,v) in E} ||w_u - w_v||^2
|
|
39
|
+
|
|
40
|
+
Implementation note — factor 2:
|
|
41
|
+
For backward compatibility the ``'numel'`` mode returns
|
|
42
|
+
``sum_sq / (M*N)`` which equals ``2 * E_var``. The ``'edges'``
|
|
43
|
+
mode (``sum_sq / num_edges``) is asymptotically equal to ``E_var``
|
|
44
|
+
for large matrices since ``num_edges ≈ 2*M*N``. Both modes are
|
|
45
|
+
monotonic in roughness; only the absolute scale differs by ~2×.
|
|
46
|
+
|
|
47
|
+
Args:
|
|
48
|
+
weight: Tensor of shape (out_features, in_features)
|
|
49
|
+
grid_shape: Optional (H, W) tuple. If it equals ``weight.shape``, the
|
|
50
|
+
natural 2D grid is used. If ``H * W`` matches ``weight.shape[0]``
|
|
51
|
+
or ``weight.shape[1]``, that axis is reshaped onto a 2D sheet.
|
|
52
|
+
normalization: Normalization mode for the energy scalar:
|
|
53
|
+
- 'numel' (default): Normalizes by total elements (M * N). Backward-compatible.
|
|
54
|
+
Equals 2× the variational form.
|
|
55
|
+
- 'edges': Normalizes by the exact number of grid edges ((M-1)*N + M*(N-1)),
|
|
56
|
+
providing scale-invariant roughness estimation across varying matrix shapes.
|
|
57
|
+
Asymptotically equal to the variational form.
|
|
58
|
+
- 'rayleigh': Variance-normalized Rayleigh quotient of the 2D Neumann Laplacian:
|
|
59
|
+
``sum_sq / (2 * ||W||_F^2 + 1e-8)`` in ``[0, 4]``. Strictly scale-invariant
|
|
60
|
+
(immune to L2 magnitude shrinkage) and dimensionless across matrix scales.
|
|
61
|
+
- 'none': Raw sum of squared differences without division.
|
|
62
|
+
|
|
63
|
+
Returns:
|
|
64
|
+
Scalar torch.Tensor representing the normalized Dirichlet energy.
|
|
65
|
+
"""
|
|
66
|
+
if grid_shape is not None:
|
|
67
|
+
h, w = grid_shape
|
|
68
|
+
if weight.dim() == 2 and (h, w) == (weight.shape[0], weight.shape[1]):
|
|
69
|
+
sheet = weight
|
|
70
|
+
elif weight.shape[0] == h * w:
|
|
71
|
+
sheet = weight.reshape(h, w, -1)
|
|
72
|
+
elif weight.shape[1] == h * w:
|
|
73
|
+
sheet = weight.reshape(-1, h, w).permute(1, 2, 0)
|
|
74
|
+
else:
|
|
75
|
+
raise ValueError(
|
|
76
|
+
f"grid_shape {grid_shape} is incompatible with weight shape {tuple(weight.shape)}"
|
|
77
|
+
)
|
|
78
|
+
else:
|
|
79
|
+
sheet = weight
|
|
80
|
+
|
|
81
|
+
if sheet.dim() == 2:
|
|
82
|
+
diff_h = sheet[1:, :] - sheet[:-1, :]
|
|
83
|
+
diff_w = sheet[:, 1:] - sheet[:, :-1]
|
|
84
|
+
sum_sq = diff_h.pow(2).sum() + diff_w.pow(2).sum()
|
|
85
|
+
h, w = sheet.shape
|
|
86
|
+
num_edges = (h - 1) * w + h * (w - 1)
|
|
87
|
+
elif sheet.dim() == 3:
|
|
88
|
+
diff_h = sheet[1:, :, :] - sheet[:-1, :, :]
|
|
89
|
+
diff_w = sheet[:, 1:, :] - sheet[:, :-1, :]
|
|
90
|
+
sum_sq = diff_h.pow(2).sum() + diff_w.pow(2).sum()
|
|
91
|
+
h, w, c = sheet.shape
|
|
92
|
+
num_edges = ((h - 1) * w + h * (w - 1)) * c
|
|
93
|
+
else:
|
|
94
|
+
raise ValueError(f"Unsupported weight tensor shape for Dirichlet energy: {sheet.shape}")
|
|
95
|
+
|
|
96
|
+
if normalization == "numel":
|
|
97
|
+
return sum_sq / max(sheet.numel(), 1)
|
|
98
|
+
elif normalization == "edges":
|
|
99
|
+
return sum_sq / max(num_edges, 1)
|
|
100
|
+
elif normalization == "rayleigh":
|
|
101
|
+
frob_sq = sheet.pow(2).sum()
|
|
102
|
+
return sum_sq / (2.0 * frob_sq + 1e-8)
|
|
103
|
+
elif normalization == "none":
|
|
104
|
+
return sum_sq
|
|
105
|
+
else:
|
|
106
|
+
raise ValueError(
|
|
107
|
+
f"Unknown normalization mode: {normalization}. Expected 'numel', 'edges', 'rayleigh', or 'none'."
|
|
108
|
+
)
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
class DirichletLoss(nn.Module):
|
|
112
|
+
"""
|
|
113
|
+
PyTorch loss wrapper for Dirichlet harmonic pinning.
|
|
114
|
+
Traverses 2D linear projections and accumulates topological gradient penalty.
|
|
115
|
+
|
|
116
|
+
By default, ``nn.Embedding`` layers and modules matching ``exclude_names``
|
|
117
|
+
(e.g. ``lm_head``) are skipped because token indices have no spatial topology.
|
|
118
|
+
Frozen weights (``requires_grad=False``) are skipped and tied weights are
|
|
119
|
+
deduplicated by storage pointer.
|
|
120
|
+
|
|
121
|
+
Args:
|
|
122
|
+
weight_decay: Coupling strength lambda_topo.
|
|
123
|
+
normalization: Per-matrix normalization ('numel', 'edges', 'rayleigh', 'none').
|
|
124
|
+
layer_weighting: Aggregation across layers:
|
|
125
|
+
- 'uniform' (default): Equal average across layers (1/L).
|
|
126
|
+
- 'param_weighted': Weighted by parameter share (M_l*N_l / sum M_j*N_j),
|
|
127
|
+
eliminating per-weight gradient attenuation on wider layers (e.g. FFNs).
|
|
128
|
+
exclude_embeddings: Skip nn.Embedding layers (default True).
|
|
129
|
+
exclude_names: Module name substrings to skip (default ("lm_head",)).
|
|
130
|
+
"""
|
|
131
|
+
def __init__(
|
|
132
|
+
self,
|
|
133
|
+
weight_decay: float = 0.01,
|
|
134
|
+
normalization: str = "numel",
|
|
135
|
+
layer_weighting: str = "uniform",
|
|
136
|
+
exclude_embeddings: bool = True,
|
|
137
|
+
exclude_names: Tuple[str, ...] = ("lm_head",),
|
|
138
|
+
):
|
|
139
|
+
super().__init__()
|
|
140
|
+
if layer_weighting not in {"uniform", "param_weighted"}:
|
|
141
|
+
raise ValueError(
|
|
142
|
+
f"Unknown layer_weighting: {layer_weighting}. Expected 'uniform' or 'param_weighted'."
|
|
143
|
+
)
|
|
144
|
+
self.weight_decay = weight_decay
|
|
145
|
+
self.normalization = normalization
|
|
146
|
+
self.layer_weighting = layer_weighting
|
|
147
|
+
self.exclude_embeddings = exclude_embeddings
|
|
148
|
+
self.exclude_names = tuple(n.lower() for n in exclude_names)
|
|
149
|
+
|
|
150
|
+
def _is_excluded(self, name: str, mod: nn.Module) -> bool:
|
|
151
|
+
if self.exclude_embeddings and isinstance(mod, nn.Embedding):
|
|
152
|
+
return True
|
|
153
|
+
if name:
|
|
154
|
+
leaf = name.rsplit(".", 1)[-1].lower()
|
|
155
|
+
if any(leaf == ex or ex in name.lower() for ex in self.exclude_names):
|
|
156
|
+
return True
|
|
157
|
+
return False
|
|
158
|
+
|
|
159
|
+
def forward(self, target: Union[nn.Module, Iterable[nn.Module]]) -> torch.Tensor:
|
|
160
|
+
if isinstance(target, nn.Module):
|
|
161
|
+
named_iter = target.named_modules()
|
|
162
|
+
else:
|
|
163
|
+
named_iter = (("", mod) for mod in target)
|
|
164
|
+
|
|
165
|
+
energies = []
|
|
166
|
+
numels = []
|
|
167
|
+
seen_ptrs = set()
|
|
168
|
+
|
|
169
|
+
for name, mod in named_iter:
|
|
170
|
+
if self._is_excluded(name, mod):
|
|
171
|
+
continue
|
|
172
|
+
weight = getattr(mod, "weight", None)
|
|
173
|
+
if weight is None or not isinstance(weight, torch.Tensor) or weight.dim() != 2:
|
|
174
|
+
continue
|
|
175
|
+
if not weight.requires_grad:
|
|
176
|
+
continue
|
|
177
|
+
ptr = weight.data_ptr()
|
|
178
|
+
if ptr in seen_ptrs:
|
|
179
|
+
continue
|
|
180
|
+
seen_ptrs.add(ptr)
|
|
181
|
+
|
|
182
|
+
energies.append(dirichlet_energy_2d(weight, normalization=self.normalization))
|
|
183
|
+
numels.append(weight.numel())
|
|
184
|
+
|
|
185
|
+
if not energies:
|
|
186
|
+
return torch.tensor(0.0)
|
|
187
|
+
|
|
188
|
+
if self.layer_weighting == "param_weighted":
|
|
189
|
+
total_numel = float(sum(numels))
|
|
190
|
+
agg = sum(e * (n / total_numel) for e, n in zip(energies, numels))
|
|
191
|
+
else:
|
|
192
|
+
agg = sum(energies) / len(energies)
|
|
193
|
+
|
|
194
|
+
return self.weight_decay * agg
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
class DirichletScheduler:
|
|
198
|
+
"""
|
|
199
|
+
Base dynamic schedule manager for Dirichlet loss weight_decay (surface tension).
|
|
200
|
+
Allows scheduling lambda_topo over training steps (e.g. cosine annealing, step cooldown).
|
|
201
|
+
"""
|
|
202
|
+
|
|
203
|
+
def __init__(
|
|
204
|
+
self,
|
|
205
|
+
loss_fn: DirichletLoss,
|
|
206
|
+
total_steps: int,
|
|
207
|
+
schedule: str = "cosine",
|
|
208
|
+
initial_weight_decay: Optional[float] = None,
|
|
209
|
+
min_weight_decay: float = 0.0,
|
|
210
|
+
cooldown_ratio: float = 0.20,
|
|
211
|
+
):
|
|
212
|
+
if total_steps <= 0:
|
|
213
|
+
raise ValueError(f"total_steps must be positive, got {total_steps}")
|
|
214
|
+
if not (0.0 <= cooldown_ratio <= 1.0):
|
|
215
|
+
raise ValueError(f"cooldown_ratio must be in [0.0, 1.0], got {cooldown_ratio}")
|
|
216
|
+
|
|
217
|
+
self.loss_fn = loss_fn
|
|
218
|
+
self.total_steps = total_steps
|
|
219
|
+
self.schedule = schedule.lower()
|
|
220
|
+
self.initial_weight_decay = (
|
|
221
|
+
float(initial_weight_decay) if initial_weight_decay is not None else float(loss_fn.weight_decay)
|
|
222
|
+
)
|
|
223
|
+
self.min_weight_decay = float(min_weight_decay)
|
|
224
|
+
self.cooldown_ratio = float(cooldown_ratio)
|
|
225
|
+
self.current_step = 0
|
|
226
|
+
|
|
227
|
+
valid_schedules = {"constant", "step_cooldown", "cosine", "linear"}
|
|
228
|
+
if self.schedule not in valid_schedules:
|
|
229
|
+
raise ValueError(f"Unknown schedule: {self.schedule}. Supported: {sorted(valid_schedules)}")
|
|
230
|
+
|
|
231
|
+
def get_weight_decay(self, step: Optional[int] = None) -> float:
|
|
232
|
+
"""Computes the scheduled weight_decay for the specified step."""
|
|
233
|
+
t = self.current_step if step is None else step
|
|
234
|
+
if t < 0:
|
|
235
|
+
t = 0
|
|
236
|
+
|
|
237
|
+
l0 = self.initial_weight_decay
|
|
238
|
+
l_min = self.min_weight_decay
|
|
239
|
+
t_total = self.total_steps
|
|
240
|
+
|
|
241
|
+
if self.schedule == "constant":
|
|
242
|
+
return l0
|
|
243
|
+
elif self.schedule == "step_cooldown":
|
|
244
|
+
cooldown_start = int((1.0 - self.cooldown_ratio) * t_total)
|
|
245
|
+
return l0 if t < cooldown_start else l_min
|
|
246
|
+
elif self.schedule == "cosine":
|
|
247
|
+
if t >= t_total:
|
|
248
|
+
return l_min
|
|
249
|
+
progress = min(max(t / t_total, 0.0), 1.0)
|
|
250
|
+
cosine_factor = 0.5 * (1.0 + math.cos(math.pi * progress))
|
|
251
|
+
return l_min + (l0 - l_min) * cosine_factor
|
|
252
|
+
elif self.schedule == "linear":
|
|
253
|
+
if t >= t_total:
|
|
254
|
+
return l_min
|
|
255
|
+
progress = min(max(t / t_total, 0.0), 1.0)
|
|
256
|
+
return l0 + (l_min - l0) * progress
|
|
257
|
+
return l0
|
|
258
|
+
|
|
259
|
+
def step(self, step: Optional[int] = None) -> float:
|
|
260
|
+
"""
|
|
261
|
+
Advances the scheduler by one step (or jumps to specified step),
|
|
262
|
+
updates loss_fn.weight_decay, and returns the new value.
|
|
263
|
+
"""
|
|
264
|
+
if step is not None:
|
|
265
|
+
self.current_step = step
|
|
266
|
+
wd = self.get_weight_decay(self.current_step)
|
|
267
|
+
self.loss_fn.weight_decay = wd
|
|
268
|
+
if step is None:
|
|
269
|
+
self.current_step += 1
|
|
270
|
+
else:
|
|
271
|
+
self.current_step = step + 1
|
|
272
|
+
return wd
|
|
273
|
+
|
|
274
|
+
def state_dict(self) -> dict:
|
|
275
|
+
return {
|
|
276
|
+
"current_step": self.current_step,
|
|
277
|
+
"total_steps": self.total_steps,
|
|
278
|
+
"initial_weight_decay": self.initial_weight_decay,
|
|
279
|
+
"min_weight_decay": self.min_weight_decay,
|
|
280
|
+
"cooldown_ratio": self.cooldown_ratio,
|
|
281
|
+
"schedule": self.schedule,
|
|
282
|
+
}
|
|
283
|
+
|
|
284
|
+
def load_state_dict(self, state_dict: dict) -> None:
|
|
285
|
+
self.current_step = state_dict["current_step"]
|
|
286
|
+
self.total_steps = state_dict["total_steps"]
|
|
287
|
+
self.initial_weight_decay = state_dict["initial_weight_decay"]
|
|
288
|
+
self.min_weight_decay = state_dict["min_weight_decay"]
|
|
289
|
+
self.cooldown_ratio = state_dict.get("cooldown_ratio", 0.20)
|
|
290
|
+
self.schedule = state_dict["schedule"]
|
|
291
|
+
self.loss_fn.weight_decay = self.get_weight_decay(self.current_step)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
class StepCooldownScheduler(DirichletScheduler):
|
|
295
|
+
"""
|
|
296
|
+
Convenience wrapper for two-stage step cooldown annealing:
|
|
297
|
+
lambda(t) = lambda_0 for t < (1 - cooldown_ratio) * total_steps, else min_weight_decay.
|
|
298
|
+
"""
|
|
299
|
+
|
|
300
|
+
def __init__(
|
|
301
|
+
self,
|
|
302
|
+
loss_fn: DirichletLoss,
|
|
303
|
+
total_steps: int,
|
|
304
|
+
cooldown_ratio: float = 0.20,
|
|
305
|
+
initial_weight_decay: Optional[float] = None,
|
|
306
|
+
min_weight_decay: float = 0.0,
|
|
307
|
+
):
|
|
308
|
+
super().__init__(
|
|
309
|
+
loss_fn=loss_fn,
|
|
310
|
+
total_steps=total_steps,
|
|
311
|
+
schedule="step_cooldown",
|
|
312
|
+
initial_weight_decay=initial_weight_decay,
|
|
313
|
+
min_weight_decay=min_weight_decay,
|
|
314
|
+
cooldown_ratio=cooldown_ratio,
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
class CosineDirichletScheduler(DirichletScheduler):
|
|
319
|
+
"""
|
|
320
|
+
Convenience wrapper for smooth half-cosine decay annealing:
|
|
321
|
+
lambda(t) = lambda_min + 0.5 * (lambda_0 - lambda_min) * (1 + cos(pi * t / T))
|
|
322
|
+
"""
|
|
323
|
+
|
|
324
|
+
def __init__(
|
|
325
|
+
self,
|
|
326
|
+
loss_fn: DirichletLoss,
|
|
327
|
+
total_steps: int,
|
|
328
|
+
initial_weight_decay: Optional[float] = None,
|
|
329
|
+
min_weight_decay: float = 0.0,
|
|
330
|
+
):
|
|
331
|
+
super().__init__(
|
|
332
|
+
loss_fn=loss_fn,
|
|
333
|
+
total_steps=total_steps,
|
|
334
|
+
schedule="cosine",
|
|
335
|
+
initial_weight_decay=initial_weight_decay,
|
|
336
|
+
min_weight_decay=min_weight_decay,
|
|
337
|
+
)
|
|
338
|
+
|