torch-type-nn 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,126 @@
1
+ """Functional core of type-nn.
2
+
3
+ For a batch x of shape (B, n) and a layer with m units of up to R Ors each:
4
+
5
+ O[b,k,r] = w[k,r] . x[b] + b[k,r] Or, a sum type
6
+ A[b,k] = prod_r sign(O) |O| ^ a[k,r] And, a product type
7
+ z[b,k] = sign(A) ln(1 + |A|) partition function
8
+
9
+ The forward runs in log space so a deep product never overflows:
10
+
11
+ l = sum_r a ln|O|, s = prod_r sign(O), z = s ln(1 + e^l)
12
+
13
+ and the backward is written by hand (``AndOrFunction``) so it can take the
14
+ exact one-sided rule at a factor that is exactly zero, which autograd
15
+ through ``log|O|`` would turn into NaN:
16
+
17
+ dz/dO_r = s sigma(l) a_r / O_r (O_r != 0)
18
+ dz/da_r = s sigma(l) ln|O_r| (O_r != 0, else 0)
19
+ at O_r = 0: dz/dO_r = cofactor if a_r = 1 and it is the only zero, else 0
20
+
21
+ Slots with ``mask == False`` are padding. They hold the identity Or
22
+ (w = 0, b = 1, a = 1), whose value is exactly 1, so they do not change A;
23
+ their parameter gradients are zeroed so no optimizer can move them.
24
+ """
25
+
26
+ from __future__ import annotations
27
+
28
+ import torch
29
+ from torch.autograd.function import once_differentiable
30
+
31
+ __all__ = ["readout", "readout_grad", "and_or", "AndOrFunction"]
32
+
33
+
34
+ def readout(u: torch.Tensor) -> torch.Tensor:
35
+ """F(u) = sign(u) ln(1 + |u|): odd, monotone, F(0) = 0."""
36
+ return torch.sign(u) * torch.log1p(u.abs())
37
+
38
+
39
+ def readout_grad(u: torch.Tensor) -> torch.Tensor:
40
+ """F'(u) = 1 / (1 + |u|)."""
41
+ return 1.0 / (1.0 + u.abs())
42
+
43
+
44
+ def _ors(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
45
+ m, R, n = weight.shape
46
+ return (x @ weight.reshape(m * R, n).T).reshape(x.shape[0], m, R) + bias
47
+
48
+
49
+ class AndOrFunction(torch.autograd.Function):
50
+ """z = sign(A) ln(1 + |A|), A = prod_r (w_r . x + b_r)^{a_r}. See module docs."""
51
+
52
+ @staticmethod
53
+ def forward(ctx, x, weight, bias, assembly, mask):
54
+ O = _ors(x, weight, bias) # (B, m, R)
55
+ logabs = torch.log(O.abs()) # -inf at an exact zero
56
+ ell = (assembly * logabs).sum(-1) # (B, m); -inf if a factor is 0
57
+ sgn = torch.sign(O).prod(-1) # 0 if a factor is 0
58
+ z = sgn * torch.logaddexp(ell, torch.zeros_like(ell))
59
+ ctx.save_for_backward(x, weight, assembly, mask, O, logabs, ell, sgn)
60
+ return z
61
+
62
+ @staticmethod
63
+ @once_differentiable
64
+ def backward(ctx, g):
65
+ x, weight, assembly, mask, O, logabs, ell, sgn = ctx.saved_tensors
66
+ m, R, n = weight.shape
67
+ B = x.shape[0]
68
+ a = assembly.unsqueeze(0) # (1, m, R)
69
+ ss = (sgn * torch.sigmoid(ell)).unsqueeze(-1) # s sigma(l), (B, m, 1)
70
+ nz = O != 0
71
+ safe_O = torch.where(nz, O, torch.ones_like(O))
72
+ dz_dO = torch.where(nz, ss * a / safe_O, torch.zeros_like(O))
73
+ dz_da = torch.where(nz, ss * torch.where(nz, logabs, torch.zeros_like(O)),
74
+ torch.zeros_like(O))
75
+ zero = ~nz
76
+ if bool(zero.any()):
77
+ # The only zero factor with a = 1 has the cofactor as its slope
78
+ # (dz/dA = 1 at A = 0); every other case at a zero is 0.
79
+ n_zero = zero.sum(-1, keepdim=True) # (B, m, 1)
80
+ ell_nz = (a * torch.where(nz, logabs, torch.zeros_like(O))).sum(-1, keepdim=True)
81
+ sgn_nz = torch.where(nz, torch.sign(O), torch.ones_like(O)).prod(-1, keepdim=True)
82
+ cof = sgn_nz * torch.exp(ell_nz)
83
+ use = zero & (n_zero == 1) & (a <= 1)
84
+ dz_dO = torch.where(use, cof.expand_as(O), dz_dO)
85
+ gk = g.unsqueeze(-1) # (B, m, 1)
86
+ dO = gk * dz_dO # dL/dO, (B, m, R)
87
+ grad_x = grad_w = grad_b = grad_a = None
88
+ if ctx.needs_input_grad[0]:
89
+ grad_x = dO.reshape(B, m * R) @ weight.reshape(m * R, n)
90
+ maskf = mask.to(O.dtype)
91
+ dOp = dO * maskf
92
+ if ctx.needs_input_grad[1]:
93
+ grad_w = (dOp.reshape(B, m * R).T @ x).reshape(m, R, n)
94
+ if ctx.needs_input_grad[2]:
95
+ grad_b = dOp.sum(0)
96
+ if ctx.needs_input_grad[3]:
97
+ grad_a = (gk * dz_da).sum(0) * maskf
98
+ return grad_x, grad_w, grad_b, grad_a, None
99
+
100
+
101
+ def and_or(
102
+ x: torch.Tensor,
103
+ weight: torch.Tensor,
104
+ bias: torch.Tensor,
105
+ assembly: torch.Tensor,
106
+ mask: torch.Tensor | None = None,
107
+ ) -> torch.Tensor:
108
+ """Apply one type-nn layer.
109
+
110
+ Args:
111
+ x: input of shape ``(*, n)``.
112
+ weight: ``(m, R, n)``, the dense weights of every Or.
113
+ bias: ``(m, R)``.
114
+ assembly: ``(m, R)``, assembly indices, expected ``>= 1``.
115
+ mask: optional ``(m, R)`` bool; ``False`` marks a padding slot, which
116
+ must hold the identity Or (w = 0, b = 1).
117
+
118
+ Returns:
119
+ ``(*, m)``.
120
+ """
121
+ m, R, n = weight.shape
122
+ if mask is None:
123
+ mask = torch.ones(m, R, dtype=torch.bool, device=weight.device)
124
+ lead = x.shape[:-1]
125
+ z = AndOrFunction.apply(x.reshape(-1, n), weight, bias, assembly, mask)
126
+ return z.reshape(*lead, m)
torch_type_nn/layer.py ADDED
@@ -0,0 +1,354 @@
1
+ """The type-nn layer as an ``nn.Module`` whose structure can change.
2
+
3
+ Storage is dense and padded so a whole layer is a few tensor ops:
4
+
5
+ weight (m, R, n) w of every Or slot
6
+ bias (m, R) b
7
+ assembly (m, R) a >= 1
8
+ mask (m, R) True where the slot holds a live Or
9
+
10
+ ``R`` is the largest degree (Ors per And) in the layer. A free slot holds
11
+ the identity Or (w = 0, b = 1, a = 1), which is exactly 1, so padding never
12
+ changes the output (see :mod:`torch_type_nn.functional`).
13
+
14
+ Structural edits (add/drop an Or, a unit, an input column) replace the
15
+ parameter tensors. The layer therefore carries its own per-Or Adam state
16
+ (``adam_*`` buffers): every edit moves the optimizer state with the
17
+ parameters, which is what :class:`torch_type_nn.optim.TypeAdam` uses. A
18
+ static layer also works with any stock ``torch.optim`` optimizer; the
19
+ ``adam_*`` buffers then simply stay zero.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import math
25
+
26
+ import torch
27
+ from torch import nn
28
+
29
+ from .edits import Edit, grow_index, keep_index
30
+ from .functional import and_or, readout
31
+
32
+ __all__ = ["AndOr", "uniform"]
33
+
34
+ # name -> fill value of a free slot
35
+ _OR_PARAMS = {"bias": 1.0, "assembly": 1.0}
36
+ _OR_BUFFERS = {
37
+ "mask": False,
38
+ "or_probe": False,
39
+ "or_born": 0,
40
+ "adam_step": 0,
41
+ "adam_m_bias": 0.0,
42
+ "adam_v_bias": 0.0,
43
+ "adam_m_assembly": 0.0,
44
+ "adam_v_assembly": 0.0,
45
+ }
46
+ _ORN_BUFFERS = ("adam_m_weight", "adam_v_weight") # (m, R, n), fill 0
47
+ _UNIT_BUFFERS = {"unit_probe": False, "unit_born": 0} # (m,)
48
+ _STAT_BUFFERS = ("stat_sx", "stat_sxx", "stat_su", "stat_suu", "stat_sxu") # (n,)
49
+
50
+
51
+ class AndOr(nn.Module):
52
+ r"""One type-nn layer: ``out_features`` Ands, each a product of Ors over the input.
53
+
54
+ .. math::
55
+ z_k = \mathrm{sign}(A_k)\ln(1+|A_k|),\quad
56
+ A_k = \prod_r (w_{kr}\cdot x + b_{kr})^{a_{kr}}
57
+
58
+ Args:
59
+ in_features: input width ``n``.
60
+ out_features: number of units (Ands) ``m``.
61
+ ors_per_unit: initial degree; every initial Or is drawn with the
62
+ ``nn.Linear`` law ``U(+-1/sqrt(n))`` and ``a = 1``.
63
+ generator: optional ``torch.Generator`` for the initial draw.
64
+ """
65
+
66
+ def __init__(
67
+ self,
68
+ in_features: int,
69
+ out_features: int,
70
+ ors_per_unit: int = 1,
71
+ *,
72
+ generator: torch.Generator | None = None,
73
+ device=None,
74
+ dtype=None,
75
+ ) -> None:
76
+ super().__init__()
77
+ dtype = dtype or torch.get_default_dtype()
78
+ n, m, R = int(in_features), int(out_features), int(ors_per_unit)
79
+ self.in_features, self.out_features = n, m
80
+ self.probe = False # True while this layer is the depth probe
81
+ self.born = 0 # training step at birth
82
+ self.track_stats = False
83
+ # When a list, every parameter replacement is appended to it as an
84
+ # :class:`~torch_type_nn.edits.Edit`, so a stock optimizer can follow.
85
+ self.edit_sink: list | None = None
86
+ f = {"device": device, "dtype": dtype}
87
+ self.weight = nn.Parameter(torch.zeros(m, R, n, **f))
88
+ self.bias = nn.Parameter(torch.ones(m, R, **f))
89
+ self.assembly = nn.Parameter(torch.ones(m, R, **f))
90
+ for name, fill in _OR_BUFFERS.items():
91
+ self.register_buffer(name, _full((m, R), fill, dtype, device))
92
+ for name in _ORN_BUFFERS:
93
+ self.register_buffer(name, torch.zeros(m, R, n, **f))
94
+ for name, fill in _UNIT_BUFFERS.items():
95
+ self.register_buffer(name, _full((m,), fill, dtype, device))
96
+ for name in _STAT_BUFFERS:
97
+ self.register_buffer(name, torch.zeros(n, **f), persistent=False)
98
+ self.register_buffer("stat_gin", torch.zeros((), **f), persistent=False)
99
+ self.register_buffer("stat_ns", torch.zeros((), **f), persistent=False)
100
+ self.mask.fill_(True)
101
+ self.reset_parameters(generator)
102
+
103
+ # ------------------------------------------------------------------ basics
104
+
105
+ @property
106
+ def degree(self) -> int:
107
+ """Slots per unit (R)."""
108
+ return self.weight.shape[1]
109
+
110
+ def reset_parameters(self, generator: torch.Generator | None = None) -> None:
111
+ """Every live Or gets the ``nn.Linear`` law ``U(+-1/sqrt(n))``, a = 1."""
112
+ with torch.no_grad():
113
+ k = 1.0 / math.sqrt(max(self.in_features, 1))
114
+ w = uniform(self.weight.shape, k, generator, self.weight)
115
+ b = uniform(self.bias.shape, k, generator, self.bias)
116
+ live = self.mask
117
+ self.weight.copy_(torch.where(live.unsqueeze(-1), w, torch.zeros_like(w)))
118
+ self.bias.copy_(torch.where(live, b, torch.ones_like(b)))
119
+ self.assembly.fill_(1.0)
120
+ self.reset_adam_state()
121
+
122
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
123
+ if self.track_stats and self.training and torch.is_grad_enabled():
124
+ x = self._observe(x)
125
+ return and_or(x, self.weight, self.bias, self.assembly, self.mask)
126
+
127
+ def num_ors(self) -> torch.Tensor:
128
+ """Live Ors per unit, shape (m,)."""
129
+ return self.mask.sum(1)
130
+
131
+ def num_params(self) -> int:
132
+ """Every learnable scalar of every live Or: w, b and a."""
133
+ return int(self.mask.sum()) * (self.in_features + 2)
134
+
135
+ def extra_repr(self) -> str:
136
+ return (f"in_features={self.in_features}, out_features={self.out_features}, "
137
+ f"ors={int(self.mask.sum())}, degree<={self.degree}"
138
+ + (", probe" if self.probe else ""))
139
+
140
+ @torch.no_grad()
141
+ def project_(self) -> None:
142
+ """Enforce the invariants: a >= 1, and every free slot is the identity Or."""
143
+ self.assembly.clamp_(min=1.0)
144
+ free = ~self.mask
145
+ self.weight.masked_fill_(free.unsqueeze(-1), 0.0)
146
+ self.bias.masked_fill_(free, 1.0)
147
+ self.assembly.masked_fill_(free, 1.0)
148
+
149
+ @torch.no_grad()
150
+ def reset_adam_state(self, unit: int | None = None, slot: int | None = None) -> None:
151
+ """Zero the per-Or Adam state (everywhere, or of one Or)."""
152
+ idx = (slice(None), slice(None)) if unit is None else (unit, slot)
153
+ for name in ("adam_step", "adam_m_bias", "adam_v_bias",
154
+ "adam_m_assembly", "adam_v_assembly", *_ORN_BUFFERS):
155
+ getattr(self, name)[idx] = 0
156
+
157
+ # ------------------------------------------------------ input statistics
158
+
159
+ @torch.no_grad()
160
+ def reset_stats(self) -> None:
161
+ """Clear the epoch statistics of the layer input."""
162
+ for name in (*_STAT_BUFFERS, "stat_gin", "stat_ns"):
163
+ getattr(self, name).zero_()
164
+
165
+ def _observe(self, x: torch.Tensor) -> torch.Tensor:
166
+ # Statistics of x and of u = F(x) (what a type-identity layer would
167
+ # emit), used to fold a depth edit; and the mean |dL/dx|, used to
168
+ # pick the gap where depth is wanted.
169
+ n = self.in_features
170
+ with torch.no_grad():
171
+ xf = x.detach().reshape(-1, n)
172
+ u = readout(xf)
173
+ self.stat_sx.add_(xf.sum(0))
174
+ self.stat_sxx.add_((xf * xf).sum(0))
175
+ self.stat_su.add_(u.sum(0))
176
+ self.stat_suu.add_((u * u).sum(0))
177
+ self.stat_sxu.add_((xf * u).sum(0))
178
+ self.stat_ns.add_(xf.shape[0])
179
+ if not x.requires_grad:
180
+ x = x.detach().requires_grad_(True)
181
+ if n:
182
+ gin = self.stat_gin
183
+
184
+ def hook(g: torch.Tensor) -> None:
185
+ with torch.no_grad():
186
+ gin.add_(g.reshape(-1, n).abs().mean(1).sum())
187
+
188
+ x.register_hook(hook)
189
+ return x
190
+
191
+ # -------------------------------------------------------- structure edits
192
+
193
+ def _set(self, name: str, value: torch.Tensor, dim: int | None = None,
194
+ index: torch.Tensor | None = None) -> None:
195
+ if name in self._parameters:
196
+ old = self._parameters[name]
197
+ new = nn.Parameter(value.contiguous())
198
+ self._parameters[name] = new
199
+ if self.edit_sink is not None and index is not None:
200
+ d = dim if dim >= 0 else old.dim() + dim
201
+ self.edit_sink.append(Edit(old, new, d, index))
202
+ else:
203
+ self._buffers[name] = value.contiguous()
204
+
205
+ def _or_tensors(self):
206
+ """(name, fill) of every tensor indexed by (unit, slot[, input])."""
207
+ yield "weight", 0.0
208
+ for name, fill in _OR_PARAMS.items():
209
+ yield name, fill
210
+ for name, fill in _OR_BUFFERS.items():
211
+ yield name, fill
212
+ for name in _ORN_BUFFERS:
213
+ yield name, 0.0
214
+
215
+ def _unit_tensors(self):
216
+ yield from self._or_tensors()
217
+ yield from _UNIT_BUFFERS.items()
218
+
219
+ @torch.no_grad()
220
+ def _grow_slots(self) -> None:
221
+ for name, fill in self._or_tensors():
222
+ t = getattr(self, name)
223
+ pad = _full((t.shape[0], 1, *t.shape[2:]), fill, t.dtype, t.device)
224
+ self._set(name, torch.cat([t.detach(), pad], 1), 1,
225
+ grow_index(t.shape[1], t.device))
226
+
227
+ @torch.no_grad()
228
+ def add_or(self, unit: int, weight: torch.Tensor | None = None, bias: float = 1.0,
229
+ assembly: float = 1.0, *, probe: bool = False, born: int = 0) -> int:
230
+ """Put a live Or in a free slot of ``unit`` (growing R if needed); returns the slot."""
231
+ free = (~self.mask[unit]).nonzero()
232
+ if len(free) == 0:
233
+ self._grow_slots()
234
+ r = self.degree - 1
235
+ else:
236
+ r = int(free[0])
237
+ self.weight[unit, r] = 0.0 if weight is None else weight
238
+ self.bias[unit, r] = bias
239
+ self.assembly[unit, r] = assembly
240
+ self.mask[unit, r] = True
241
+ self.or_probe[unit, r] = probe
242
+ self.or_born[unit, r] = born
243
+ self.reset_adam_state(unit, r)
244
+ return r
245
+
246
+ @torch.no_grad()
247
+ def drop_or(self, unit: int, slot: int) -> None:
248
+ """Free a slot: it becomes the identity Or again."""
249
+ self.weight[unit, slot] = 0.0
250
+ self.bias[unit, slot] = 1.0
251
+ self.assembly[unit, slot] = 1.0
252
+ self.mask[unit, slot] = False
253
+ self.or_probe[unit, slot] = False
254
+ self.or_born[unit, slot] = 0
255
+ self.reset_adam_state(unit, slot)
256
+
257
+ @torch.no_grad()
258
+ def compact(self) -> None:
259
+ """Drop slot columns that are free in every unit."""
260
+ keep = self.mask.any(0)
261
+ if bool(keep.all()):
262
+ return
263
+ idx = keep.nonzero().flatten()
264
+ for name, _ in self._or_tensors():
265
+ self._set(name, getattr(self, name).detach().index_select(1, idx), 1,
266
+ keep_index(idx))
267
+
268
+ @torch.no_grad()
269
+ def add_unit(self) -> int:
270
+ """Append a unit with no Ors (it outputs ln 2 until it gets one); returns its index."""
271
+ for name, fill in self._unit_tensors():
272
+ t = getattr(self, name)
273
+ pad = _full((1, *t.shape[1:]), fill, t.dtype, t.device)
274
+ self._set(name, torch.cat([t.detach(), pad], 0), 0,
275
+ grow_index(t.shape[0], t.device))
276
+ self.out_features += 1
277
+ return self.out_features - 1
278
+
279
+ @torch.no_grad()
280
+ def drop_unit(self, unit: int) -> None:
281
+ keep = _without(self.out_features, unit, self.weight.device)
282
+ for name, _ in self._unit_tensors():
283
+ self._set(name, getattr(self, name).detach().index_select(0, keep), 0,
284
+ keep_index(keep))
285
+ self.out_features -= 1
286
+
287
+ @torch.no_grad()
288
+ def add_input(self) -> None:
289
+ """New incoming coordinate. Every Or meets it with weight exactly 0,
290
+ so the layer computes the same function (Coq: widen_preserves)."""
291
+ for name in ("weight", *_ORN_BUFFERS):
292
+ t = getattr(self, name)
293
+ self._set(name, torch.cat([t.detach(), torch.zeros_like(t[..., :1])], -1), -1,
294
+ grow_index(t.shape[-1], t.device))
295
+ self.in_features += 1
296
+ self._resize_stats()
297
+
298
+ @torch.no_grad()
299
+ def drop_input(self, j: int) -> None:
300
+ keep = _without(self.in_features, j, self.weight.device)
301
+ for name in ("weight", *_ORN_BUFFERS):
302
+ self._set(name, getattr(self, name).detach().index_select(-1, keep), -1,
303
+ keep_index(keep))
304
+ self.in_features -= 1
305
+ self._resize_stats()
306
+
307
+ def _resize_stats(self) -> None:
308
+ for name in _STAT_BUFFERS:
309
+ self._buffers[name] = self.weight.new_zeros(self.in_features)
310
+ self.reset_stats()
311
+
312
+ # ------------------------------------------------------------ persistence
313
+
314
+ def get_extra_state(self):
315
+ return {"probe": self.probe, "born": self.born}
316
+
317
+ def set_extra_state(self, state) -> None:
318
+ self.probe = bool(state.get("probe", False))
319
+ self.born = int(state.get("born", 0))
320
+
321
+ def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
322
+ # A checkpoint may hold a different structure: take its shapes first.
323
+ w = state_dict.get(prefix + "weight")
324
+ if w is not None and w.shape != self.weight.shape:
325
+ m, R, n = w.shape
326
+ with torch.no_grad():
327
+ for name, t in [*self._parameters.items(), *self._buffers.items()]:
328
+ src = state_dict.get(prefix + name)
329
+ if src is not None:
330
+ self._set(name, torch.empty(src.shape, dtype=t.dtype, device=t.device))
331
+ self.in_features, self.out_features = n, m
332
+ self._resize_stats()
333
+ super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
334
+
335
+
336
+ def uniform(shape, bound: float, generator: torch.Generator | None,
337
+ like: torch.Tensor) -> torch.Tensor:
338
+ """U(-bound, bound) drawn from ``generator`` (on its own device), then moved to ``like``."""
339
+ dev = generator.device if generator is not None else like.device
340
+ t = torch.empty(shape, dtype=like.dtype, device=dev)
341
+ t.uniform_(-bound, bound, generator=generator)
342
+ return t.to(like.device)
343
+
344
+
345
+ def _full(shape, fill, dtype, device) -> torch.Tensor:
346
+ if isinstance(fill, bool):
347
+ return torch.full(shape, fill, dtype=torch.bool, device=device)
348
+ if isinstance(fill, int):
349
+ return torch.full(shape, fill, dtype=torch.long, device=device)
350
+ return torch.full(shape, fill, dtype=dtype, device=device)
351
+
352
+
353
+ def _without(n: int, i: int, device) -> torch.Tensor:
354
+ return torch.cat([torch.arange(i, device=device), torch.arange(i + 1, n, device=device)])