dirichlet-regularization 1.0.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
dreg/topology.py ADDED
@@ -0,0 +1,338 @@
1
+ """
2
+ Topological Dirichlet Regularization Module
3
+ ===========================================
4
+ Implements 2D cortical manifold embedding and Dirichlet harmonic pinning
5
+ to eliminate permutation invariance and concentrate energy into low spatial frequencies.
6
+ """
7
+
8
+ import math
9
+ from typing import Iterable, Optional, Tuple, Union
10
+
11
+ import torch
12
+ import torch.nn as nn
13
+
14
+
15
+ def get_grid_dimensions(n: int) -> Tuple[int, int]:
16
+ """
17
+ Finds the integer factor pair (H, W) such that H * W = n
18
+ with the aspect ratio closest to 1.0 (square-like 2D cortical lattice).
19
+ """
20
+ if n <= 0:
21
+ raise ValueError(f"n must be a positive integer, got {n}")
22
+ sqrt_n = int(math.isqrt(n))
23
+ for h in range(sqrt_n, 0, -1):
24
+ if n % h == 0:
25
+ return h, n // h
26
+ return 1, n
27
+
28
+
29
+ def dirichlet_energy_2d(
30
+ weight: torch.Tensor,
31
+ grid_shape: Optional[Tuple[int, int]] = None,
32
+ normalization: str = "numel",
33
+ ) -> torch.Tensor:
34
+ """
35
+ Computes the 2D Dirichlet Harmonic Energy of a weight matrix.
36
+
37
+ Variational form (discrete unit lattice normalized by 2*M*N):
38
+ E_var(W) = 1/(2*M*N) * sum_{(u,v) in E} ||w_u - w_v||^2
39
+
40
+ Implementation note — factor 2:
41
+ For backward compatibility the ``'numel'`` mode returns
42
+ ``sum_sq / (M*N)`` which equals ``2 * E_var``. The ``'edges'``
43
+ mode (``sum_sq / num_edges``) is asymptotically equal to ``E_var``
44
+ for large matrices since ``num_edges ≈ 2*M*N``. Both modes are
45
+ monotonic in roughness; only the absolute scale differs by ~2×.
46
+
47
+ Args:
48
+ weight: Tensor of shape (out_features, in_features)
49
+ grid_shape: Optional (H, W) tuple. If it equals ``weight.shape``, the
50
+ natural 2D grid is used. If ``H * W`` matches ``weight.shape[0]``
51
+ or ``weight.shape[1]``, that axis is reshaped onto a 2D sheet.
52
+ normalization: Normalization mode for the energy scalar:
53
+ - 'numel' (default): Normalizes by total elements (M * N). Backward-compatible.
54
+ Equals 2× the variational form.
55
+ - 'edges': Normalizes by the exact number of grid edges ((M-1)*N + M*(N-1)),
56
+ providing scale-invariant roughness estimation across varying matrix shapes.
57
+ Asymptotically equal to the variational form.
58
+ - 'rayleigh': Variance-normalized Rayleigh quotient of the 2D Neumann Laplacian:
59
+ ``sum_sq / (2 * ||W||_F^2 + 1e-8)`` in ``[0, 4]``. Strictly scale-invariant
60
+ (immune to L2 magnitude shrinkage) and dimensionless across matrix scales.
61
+ - 'none': Raw sum of squared differences without division.
62
+
63
+ Returns:
64
+ Scalar torch.Tensor representing the normalized Dirichlet energy.
65
+ """
66
+ if grid_shape is not None:
67
+ h, w = grid_shape
68
+ if weight.dim() == 2 and (h, w) == (weight.shape[0], weight.shape[1]):
69
+ sheet = weight
70
+ elif weight.shape[0] == h * w:
71
+ sheet = weight.reshape(h, w, -1)
72
+ elif weight.shape[1] == h * w:
73
+ sheet = weight.reshape(-1, h, w).permute(1, 2, 0)
74
+ else:
75
+ raise ValueError(
76
+ f"grid_shape {grid_shape} is incompatible with weight shape {tuple(weight.shape)}"
77
+ )
78
+ else:
79
+ sheet = weight
80
+
81
+ if sheet.dim() == 2:
82
+ diff_h = sheet[1:, :] - sheet[:-1, :]
83
+ diff_w = sheet[:, 1:] - sheet[:, :-1]
84
+ sum_sq = diff_h.pow(2).sum() + diff_w.pow(2).sum()
85
+ h, w = sheet.shape
86
+ num_edges = (h - 1) * w + h * (w - 1)
87
+ elif sheet.dim() == 3:
88
+ diff_h = sheet[1:, :, :] - sheet[:-1, :, :]
89
+ diff_w = sheet[:, 1:, :] - sheet[:, :-1, :]
90
+ sum_sq = diff_h.pow(2).sum() + diff_w.pow(2).sum()
91
+ h, w, c = sheet.shape
92
+ num_edges = ((h - 1) * w + h * (w - 1)) * c
93
+ else:
94
+ raise ValueError(f"Unsupported weight tensor shape for Dirichlet energy: {sheet.shape}")
95
+
96
+ if normalization == "numel":
97
+ return sum_sq / max(sheet.numel(), 1)
98
+ elif normalization == "edges":
99
+ return sum_sq / max(num_edges, 1)
100
+ elif normalization == "rayleigh":
101
+ frob_sq = sheet.pow(2).sum()
102
+ return sum_sq / (2.0 * frob_sq + 1e-8)
103
+ elif normalization == "none":
104
+ return sum_sq
105
+ else:
106
+ raise ValueError(
107
+ f"Unknown normalization mode: {normalization}. Expected 'numel', 'edges', 'rayleigh', or 'none'."
108
+ )
109
+
110
+
111
+ class DirichletLoss(nn.Module):
112
+ """
113
+ PyTorch loss wrapper for Dirichlet harmonic pinning.
114
+ Traverses 2D linear projections and accumulates topological gradient penalty.
115
+
116
+ By default, ``nn.Embedding`` layers and modules matching ``exclude_names``
117
+ (e.g. ``lm_head``) are skipped because token indices have no spatial topology.
118
+ Frozen weights (``requires_grad=False``) are skipped and tied weights are
119
+ deduplicated by storage pointer.
120
+
121
+ Args:
122
+ weight_decay: Coupling strength lambda_topo.
123
+ normalization: Per-matrix normalization ('numel', 'edges', 'rayleigh', 'none').
124
+ layer_weighting: Aggregation across layers:
125
+ - 'uniform' (default): Equal average across layers (1/L).
126
+ - 'param_weighted': Weighted by parameter share (M_l*N_l / sum M_j*N_j),
127
+ eliminating per-weight gradient attenuation on wider layers (e.g. FFNs).
128
+ exclude_embeddings: Skip nn.Embedding layers (default True).
129
+ exclude_names: Module name substrings to skip (default ("lm_head",)).
130
+ """
131
+ def __init__(
132
+ self,
133
+ weight_decay: float = 0.01,
134
+ normalization: str = "numel",
135
+ layer_weighting: str = "uniform",
136
+ exclude_embeddings: bool = True,
137
+ exclude_names: Tuple[str, ...] = ("lm_head",),
138
+ ):
139
+ super().__init__()
140
+ if layer_weighting not in {"uniform", "param_weighted"}:
141
+ raise ValueError(
142
+ f"Unknown layer_weighting: {layer_weighting}. Expected 'uniform' or 'param_weighted'."
143
+ )
144
+ self.weight_decay = weight_decay
145
+ self.normalization = normalization
146
+ self.layer_weighting = layer_weighting
147
+ self.exclude_embeddings = exclude_embeddings
148
+ self.exclude_names = tuple(n.lower() for n in exclude_names)
149
+
150
+ def _is_excluded(self, name: str, mod: nn.Module) -> bool:
151
+ if self.exclude_embeddings and isinstance(mod, nn.Embedding):
152
+ return True
153
+ if name:
154
+ leaf = name.rsplit(".", 1)[-1].lower()
155
+ if any(leaf == ex or ex in name.lower() for ex in self.exclude_names):
156
+ return True
157
+ return False
158
+
159
+ def forward(self, target: Union[nn.Module, Iterable[nn.Module]]) -> torch.Tensor:
160
+ if isinstance(target, nn.Module):
161
+ named_iter = target.named_modules()
162
+ else:
163
+ named_iter = (("", mod) for mod in target)
164
+
165
+ energies = []
166
+ numels = []
167
+ seen_ptrs = set()
168
+
169
+ for name, mod in named_iter:
170
+ if self._is_excluded(name, mod):
171
+ continue
172
+ weight = getattr(mod, "weight", None)
173
+ if weight is None or not isinstance(weight, torch.Tensor) or weight.dim() != 2:
174
+ continue
175
+ if not weight.requires_grad:
176
+ continue
177
+ ptr = weight.data_ptr()
178
+ if ptr in seen_ptrs:
179
+ continue
180
+ seen_ptrs.add(ptr)
181
+
182
+ energies.append(dirichlet_energy_2d(weight, normalization=self.normalization))
183
+ numels.append(weight.numel())
184
+
185
+ if not energies:
186
+ return torch.tensor(0.0)
187
+
188
+ if self.layer_weighting == "param_weighted":
189
+ total_numel = float(sum(numels))
190
+ agg = sum(e * (n / total_numel) for e, n in zip(energies, numels))
191
+ else:
192
+ agg = sum(energies) / len(energies)
193
+
194
+ return self.weight_decay * agg
195
+
196
+
197
+ class DirichletScheduler:
198
+ """
199
+ Base dynamic schedule manager for Dirichlet loss weight_decay (surface tension).
200
+ Allows scheduling lambda_topo over training steps (e.g. cosine annealing, step cooldown).
201
+ """
202
+
203
+ def __init__(
204
+ self,
205
+ loss_fn: DirichletLoss,
206
+ total_steps: int,
207
+ schedule: str = "cosine",
208
+ initial_weight_decay: Optional[float] = None,
209
+ min_weight_decay: float = 0.0,
210
+ cooldown_ratio: float = 0.20,
211
+ ):
212
+ if total_steps <= 0:
213
+ raise ValueError(f"total_steps must be positive, got {total_steps}")
214
+ if not (0.0 <= cooldown_ratio <= 1.0):
215
+ raise ValueError(f"cooldown_ratio must be in [0.0, 1.0], got {cooldown_ratio}")
216
+
217
+ self.loss_fn = loss_fn
218
+ self.total_steps = total_steps
219
+ self.schedule = schedule.lower()
220
+ self.initial_weight_decay = (
221
+ float(initial_weight_decay) if initial_weight_decay is not None else float(loss_fn.weight_decay)
222
+ )
223
+ self.min_weight_decay = float(min_weight_decay)
224
+ self.cooldown_ratio = float(cooldown_ratio)
225
+ self.current_step = 0
226
+
227
+ valid_schedules = {"constant", "step_cooldown", "cosine", "linear"}
228
+ if self.schedule not in valid_schedules:
229
+ raise ValueError(f"Unknown schedule: {self.schedule}. Supported: {sorted(valid_schedules)}")
230
+
231
+ def get_weight_decay(self, step: Optional[int] = None) -> float:
232
+ """Computes the scheduled weight_decay for the specified step."""
233
+ t = self.current_step if step is None else step
234
+ if t < 0:
235
+ t = 0
236
+
237
+ l0 = self.initial_weight_decay
238
+ l_min = self.min_weight_decay
239
+ t_total = self.total_steps
240
+
241
+ if self.schedule == "constant":
242
+ return l0
243
+ elif self.schedule == "step_cooldown":
244
+ cooldown_start = int((1.0 - self.cooldown_ratio) * t_total)
245
+ return l0 if t < cooldown_start else l_min
246
+ elif self.schedule == "cosine":
247
+ if t >= t_total:
248
+ return l_min
249
+ progress = min(max(t / t_total, 0.0), 1.0)
250
+ cosine_factor = 0.5 * (1.0 + math.cos(math.pi * progress))
251
+ return l_min + (l0 - l_min) * cosine_factor
252
+ elif self.schedule == "linear":
253
+ if t >= t_total:
254
+ return l_min
255
+ progress = min(max(t / t_total, 0.0), 1.0)
256
+ return l0 + (l_min - l0) * progress
257
+ return l0
258
+
259
+ def step(self, step: Optional[int] = None) -> float:
260
+ """
261
+ Advances the scheduler by one step (or jumps to specified step),
262
+ updates loss_fn.weight_decay, and returns the new value.
263
+ """
264
+ if step is not None:
265
+ self.current_step = step
266
+ wd = self.get_weight_decay(self.current_step)
267
+ self.loss_fn.weight_decay = wd
268
+ if step is None:
269
+ self.current_step += 1
270
+ else:
271
+ self.current_step = step + 1
272
+ return wd
273
+
274
+ def state_dict(self) -> dict:
275
+ return {
276
+ "current_step": self.current_step,
277
+ "total_steps": self.total_steps,
278
+ "initial_weight_decay": self.initial_weight_decay,
279
+ "min_weight_decay": self.min_weight_decay,
280
+ "cooldown_ratio": self.cooldown_ratio,
281
+ "schedule": self.schedule,
282
+ }
283
+
284
+ def load_state_dict(self, state_dict: dict) -> None:
285
+ self.current_step = state_dict["current_step"]
286
+ self.total_steps = state_dict["total_steps"]
287
+ self.initial_weight_decay = state_dict["initial_weight_decay"]
288
+ self.min_weight_decay = state_dict["min_weight_decay"]
289
+ self.cooldown_ratio = state_dict.get("cooldown_ratio", 0.20)
290
+ self.schedule = state_dict["schedule"]
291
+ self.loss_fn.weight_decay = self.get_weight_decay(self.current_step)
292
+
293
+
294
+ class StepCooldownScheduler(DirichletScheduler):
295
+ """
296
+ Convenience wrapper for two-stage step cooldown annealing:
297
+ lambda(t) = lambda_0 for t < (1 - cooldown_ratio) * total_steps, else min_weight_decay.
298
+ """
299
+
300
+ def __init__(
301
+ self,
302
+ loss_fn: DirichletLoss,
303
+ total_steps: int,
304
+ cooldown_ratio: float = 0.20,
305
+ initial_weight_decay: Optional[float] = None,
306
+ min_weight_decay: float = 0.0,
307
+ ):
308
+ super().__init__(
309
+ loss_fn=loss_fn,
310
+ total_steps=total_steps,
311
+ schedule="step_cooldown",
312
+ initial_weight_decay=initial_weight_decay,
313
+ min_weight_decay=min_weight_decay,
314
+ cooldown_ratio=cooldown_ratio,
315
+ )
316
+
317
+
318
+ class CosineDirichletScheduler(DirichletScheduler):
319
+ """
320
+ Convenience wrapper for smooth half-cosine decay annealing:
321
+ lambda(t) = lambda_min + 0.5 * (lambda_0 - lambda_min) * (1 + cos(pi * t / T))
322
+ """
323
+
324
+ def __init__(
325
+ self,
326
+ loss_fn: DirichletLoss,
327
+ total_steps: int,
328
+ initial_weight_decay: Optional[float] = None,
329
+ min_weight_decay: float = 0.0,
330
+ ):
331
+ super().__init__(
332
+ loss_fn=loss_fn,
333
+ total_steps=total_steps,
334
+ schedule="cosine",
335
+ initial_weight_decay=initial_weight_decay,
336
+ min_weight_decay=min_weight_decay,
337
+ )
338
+