torch-type-nn 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- torch_type_nn/__init__.py +38 -0
- torch_type_nn/adapters/__init__.py +6 -0
- torch_type_nn/adapters/mlp.py +322 -0
- torch_type_nn/adapters/typenn.py +546 -0
- torch_type_nn/edits.py +99 -0
- torch_type_nn/functional.py +126 -0
- torch_type_nn/layer.py +354 -0
- torch_type_nn/mlp.py +251 -0
- torch_type_nn/network.py +115 -0
- torch_type_nn/optim.py +122 -0
- torch_type_nn/protocol.py +148 -0
- torch_type_nn/py.typed +0 -0
- torch_type_nn/scaling.py +523 -0
- torch_type_nn/train.py +97 -0
- torch_type_nn-0.1.0.dev0.dist-info/METADATA +302 -0
- torch_type_nn-0.1.0.dev0.dist-info/RECORD +18 -0
- torch_type_nn-0.1.0.dev0.dist-info/WHEEL +4 -0
- torch_type_nn-0.1.0.dev0.dist-info/licenses/LICENSE +23 -0
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""Functional core of type-nn.
|
|
2
|
+
|
|
3
|
+
For a batch x of shape (B, n) and a layer with m units of up to R Ors each:
|
|
4
|
+
|
|
5
|
+
O[b,k,r] = w[k,r] . x[b] + b[k,r] Or, a sum type
|
|
6
|
+
A[b,k] = prod_r sign(O) |O| ^ a[k,r] And, a product type
|
|
7
|
+
z[b,k] = sign(A) ln(1 + |A|) partition function
|
|
8
|
+
|
|
9
|
+
The forward runs in log space so a deep product never overflows:
|
|
10
|
+
|
|
11
|
+
l = sum_r a ln|O|, s = prod_r sign(O), z = s ln(1 + e^l)
|
|
12
|
+
|
|
13
|
+
and the backward is written by hand (``AndOrFunction``) so it can take the
|
|
14
|
+
exact one-sided rule at a factor that is exactly zero, which autograd
|
|
15
|
+
through ``log|O|`` would turn into NaN:
|
|
16
|
+
|
|
17
|
+
dz/dO_r = s sigma(l) a_r / O_r (O_r != 0)
|
|
18
|
+
dz/da_r = s sigma(l) ln|O_r| (O_r != 0, else 0)
|
|
19
|
+
at O_r = 0: dz/dO_r = cofactor if a_r = 1 and it is the only zero, else 0
|
|
20
|
+
|
|
21
|
+
Slots with ``mask == False`` are padding. They hold the identity Or
|
|
22
|
+
(w = 0, b = 1, a = 1), whose value is exactly 1, so they do not change A;
|
|
23
|
+
their parameter gradients are zeroed so no optimizer can move them.
|
|
24
|
+
"""
|
|
25
|
+
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import torch
|
|
29
|
+
from torch.autograd.function import once_differentiable
|
|
30
|
+
|
|
31
|
+
__all__ = ["readout", "readout_grad", "and_or", "AndOrFunction"]
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def readout(u: torch.Tensor) -> torch.Tensor:
|
|
35
|
+
"""F(u) = sign(u) ln(1 + |u|): odd, monotone, F(0) = 0."""
|
|
36
|
+
return torch.sign(u) * torch.log1p(u.abs())
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def readout_grad(u: torch.Tensor) -> torch.Tensor:
|
|
40
|
+
"""F'(u) = 1 / (1 + |u|)."""
|
|
41
|
+
return 1.0 / (1.0 + u.abs())
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _ors(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor) -> torch.Tensor:
|
|
45
|
+
m, R, n = weight.shape
|
|
46
|
+
return (x @ weight.reshape(m * R, n).T).reshape(x.shape[0], m, R) + bias
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
class AndOrFunction(torch.autograd.Function):
|
|
50
|
+
"""z = sign(A) ln(1 + |A|), A = prod_r (w_r . x + b_r)^{a_r}. See module docs."""
|
|
51
|
+
|
|
52
|
+
@staticmethod
|
|
53
|
+
def forward(ctx, x, weight, bias, assembly, mask):
|
|
54
|
+
O = _ors(x, weight, bias) # (B, m, R)
|
|
55
|
+
logabs = torch.log(O.abs()) # -inf at an exact zero
|
|
56
|
+
ell = (assembly * logabs).sum(-1) # (B, m); -inf if a factor is 0
|
|
57
|
+
sgn = torch.sign(O).prod(-1) # 0 if a factor is 0
|
|
58
|
+
z = sgn * torch.logaddexp(ell, torch.zeros_like(ell))
|
|
59
|
+
ctx.save_for_backward(x, weight, assembly, mask, O, logabs, ell, sgn)
|
|
60
|
+
return z
|
|
61
|
+
|
|
62
|
+
@staticmethod
|
|
63
|
+
@once_differentiable
|
|
64
|
+
def backward(ctx, g):
|
|
65
|
+
x, weight, assembly, mask, O, logabs, ell, sgn = ctx.saved_tensors
|
|
66
|
+
m, R, n = weight.shape
|
|
67
|
+
B = x.shape[0]
|
|
68
|
+
a = assembly.unsqueeze(0) # (1, m, R)
|
|
69
|
+
ss = (sgn * torch.sigmoid(ell)).unsqueeze(-1) # s sigma(l), (B, m, 1)
|
|
70
|
+
nz = O != 0
|
|
71
|
+
safe_O = torch.where(nz, O, torch.ones_like(O))
|
|
72
|
+
dz_dO = torch.where(nz, ss * a / safe_O, torch.zeros_like(O))
|
|
73
|
+
dz_da = torch.where(nz, ss * torch.where(nz, logabs, torch.zeros_like(O)),
|
|
74
|
+
torch.zeros_like(O))
|
|
75
|
+
zero = ~nz
|
|
76
|
+
if bool(zero.any()):
|
|
77
|
+
# The only zero factor with a = 1 has the cofactor as its slope
|
|
78
|
+
# (dz/dA = 1 at A = 0); every other case at a zero is 0.
|
|
79
|
+
n_zero = zero.sum(-1, keepdim=True) # (B, m, 1)
|
|
80
|
+
ell_nz = (a * torch.where(nz, logabs, torch.zeros_like(O))).sum(-1, keepdim=True)
|
|
81
|
+
sgn_nz = torch.where(nz, torch.sign(O), torch.ones_like(O)).prod(-1, keepdim=True)
|
|
82
|
+
cof = sgn_nz * torch.exp(ell_nz)
|
|
83
|
+
use = zero & (n_zero == 1) & (a <= 1)
|
|
84
|
+
dz_dO = torch.where(use, cof.expand_as(O), dz_dO)
|
|
85
|
+
gk = g.unsqueeze(-1) # (B, m, 1)
|
|
86
|
+
dO = gk * dz_dO # dL/dO, (B, m, R)
|
|
87
|
+
grad_x = grad_w = grad_b = grad_a = None
|
|
88
|
+
if ctx.needs_input_grad[0]:
|
|
89
|
+
grad_x = dO.reshape(B, m * R) @ weight.reshape(m * R, n)
|
|
90
|
+
maskf = mask.to(O.dtype)
|
|
91
|
+
dOp = dO * maskf
|
|
92
|
+
if ctx.needs_input_grad[1]:
|
|
93
|
+
grad_w = (dOp.reshape(B, m * R).T @ x).reshape(m, R, n)
|
|
94
|
+
if ctx.needs_input_grad[2]:
|
|
95
|
+
grad_b = dOp.sum(0)
|
|
96
|
+
if ctx.needs_input_grad[3]:
|
|
97
|
+
grad_a = (gk * dz_da).sum(0) * maskf
|
|
98
|
+
return grad_x, grad_w, grad_b, grad_a, None
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
def and_or(
|
|
102
|
+
x: torch.Tensor,
|
|
103
|
+
weight: torch.Tensor,
|
|
104
|
+
bias: torch.Tensor,
|
|
105
|
+
assembly: torch.Tensor,
|
|
106
|
+
mask: torch.Tensor | None = None,
|
|
107
|
+
) -> torch.Tensor:
|
|
108
|
+
"""Apply one type-nn layer.
|
|
109
|
+
|
|
110
|
+
Args:
|
|
111
|
+
x: input of shape ``(*, n)``.
|
|
112
|
+
weight: ``(m, R, n)``, the dense weights of every Or.
|
|
113
|
+
bias: ``(m, R)``.
|
|
114
|
+
assembly: ``(m, R)``, assembly indices, expected ``>= 1``.
|
|
115
|
+
mask: optional ``(m, R)`` bool; ``False`` marks a padding slot, which
|
|
116
|
+
must hold the identity Or (w = 0, b = 1).
|
|
117
|
+
|
|
118
|
+
Returns:
|
|
119
|
+
``(*, m)``.
|
|
120
|
+
"""
|
|
121
|
+
m, R, n = weight.shape
|
|
122
|
+
if mask is None:
|
|
123
|
+
mask = torch.ones(m, R, dtype=torch.bool, device=weight.device)
|
|
124
|
+
lead = x.shape[:-1]
|
|
125
|
+
z = AndOrFunction.apply(x.reshape(-1, n), weight, bias, assembly, mask)
|
|
126
|
+
return z.reshape(*lead, m)
|
torch_type_nn/layer.py
ADDED
|
@@ -0,0 +1,354 @@
|
|
|
1
|
+
"""The type-nn layer as an ``nn.Module`` whose structure can change.
|
|
2
|
+
|
|
3
|
+
Storage is dense and padded so a whole layer is a few tensor ops:
|
|
4
|
+
|
|
5
|
+
weight (m, R, n) w of every Or slot
|
|
6
|
+
bias (m, R) b
|
|
7
|
+
assembly (m, R) a >= 1
|
|
8
|
+
mask (m, R) True where the slot holds a live Or
|
|
9
|
+
|
|
10
|
+
``R`` is the largest degree (Ors per And) in the layer. A free slot holds
|
|
11
|
+
the identity Or (w = 0, b = 1, a = 1), which is exactly 1, so padding never
|
|
12
|
+
changes the output (see :mod:`torch_type_nn.functional`).
|
|
13
|
+
|
|
14
|
+
Structural edits (add/drop an Or, a unit, an input column) replace the
|
|
15
|
+
parameter tensors. The layer therefore carries its own per-Or Adam state
|
|
16
|
+
(``adam_*`` buffers): every edit moves the optimizer state with the
|
|
17
|
+
parameters, which is what :class:`torch_type_nn.optim.TypeAdam` uses. A
|
|
18
|
+
static layer also works with any stock ``torch.optim`` optimizer; the
|
|
19
|
+
``adam_*`` buffers then simply stay zero.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
import math
|
|
25
|
+
|
|
26
|
+
import torch
|
|
27
|
+
from torch import nn
|
|
28
|
+
|
|
29
|
+
from .edits import Edit, grow_index, keep_index
|
|
30
|
+
from .functional import and_or, readout
|
|
31
|
+
|
|
32
|
+
__all__ = ["AndOr", "uniform"]
|
|
33
|
+
|
|
34
|
+
# name -> fill value of a free slot
|
|
35
|
+
_OR_PARAMS = {"bias": 1.0, "assembly": 1.0}
|
|
36
|
+
_OR_BUFFERS = {
|
|
37
|
+
"mask": False,
|
|
38
|
+
"or_probe": False,
|
|
39
|
+
"or_born": 0,
|
|
40
|
+
"adam_step": 0,
|
|
41
|
+
"adam_m_bias": 0.0,
|
|
42
|
+
"adam_v_bias": 0.0,
|
|
43
|
+
"adam_m_assembly": 0.0,
|
|
44
|
+
"adam_v_assembly": 0.0,
|
|
45
|
+
}
|
|
46
|
+
_ORN_BUFFERS = ("adam_m_weight", "adam_v_weight") # (m, R, n), fill 0
|
|
47
|
+
_UNIT_BUFFERS = {"unit_probe": False, "unit_born": 0} # (m,)
|
|
48
|
+
_STAT_BUFFERS = ("stat_sx", "stat_sxx", "stat_su", "stat_suu", "stat_sxu") # (n,)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
class AndOr(nn.Module):
|
|
52
|
+
r"""One type-nn layer: ``out_features`` Ands, each a product of Ors over the input.
|
|
53
|
+
|
|
54
|
+
.. math::
|
|
55
|
+
z_k = \mathrm{sign}(A_k)\ln(1+|A_k|),\quad
|
|
56
|
+
A_k = \prod_r (w_{kr}\cdot x + b_{kr})^{a_{kr}}
|
|
57
|
+
|
|
58
|
+
Args:
|
|
59
|
+
in_features: input width ``n``.
|
|
60
|
+
out_features: number of units (Ands) ``m``.
|
|
61
|
+
ors_per_unit: initial degree; every initial Or is drawn with the
|
|
62
|
+
``nn.Linear`` law ``U(+-1/sqrt(n))`` and ``a = 1``.
|
|
63
|
+
generator: optional ``torch.Generator`` for the initial draw.
|
|
64
|
+
"""
|
|
65
|
+
|
|
66
|
+
def __init__(
|
|
67
|
+
self,
|
|
68
|
+
in_features: int,
|
|
69
|
+
out_features: int,
|
|
70
|
+
ors_per_unit: int = 1,
|
|
71
|
+
*,
|
|
72
|
+
generator: torch.Generator | None = None,
|
|
73
|
+
device=None,
|
|
74
|
+
dtype=None,
|
|
75
|
+
) -> None:
|
|
76
|
+
super().__init__()
|
|
77
|
+
dtype = dtype or torch.get_default_dtype()
|
|
78
|
+
n, m, R = int(in_features), int(out_features), int(ors_per_unit)
|
|
79
|
+
self.in_features, self.out_features = n, m
|
|
80
|
+
self.probe = False # True while this layer is the depth probe
|
|
81
|
+
self.born = 0 # training step at birth
|
|
82
|
+
self.track_stats = False
|
|
83
|
+
# When a list, every parameter replacement is appended to it as an
|
|
84
|
+
# :class:`~torch_type_nn.edits.Edit`, so a stock optimizer can follow.
|
|
85
|
+
self.edit_sink: list | None = None
|
|
86
|
+
f = {"device": device, "dtype": dtype}
|
|
87
|
+
self.weight = nn.Parameter(torch.zeros(m, R, n, **f))
|
|
88
|
+
self.bias = nn.Parameter(torch.ones(m, R, **f))
|
|
89
|
+
self.assembly = nn.Parameter(torch.ones(m, R, **f))
|
|
90
|
+
for name, fill in _OR_BUFFERS.items():
|
|
91
|
+
self.register_buffer(name, _full((m, R), fill, dtype, device))
|
|
92
|
+
for name in _ORN_BUFFERS:
|
|
93
|
+
self.register_buffer(name, torch.zeros(m, R, n, **f))
|
|
94
|
+
for name, fill in _UNIT_BUFFERS.items():
|
|
95
|
+
self.register_buffer(name, _full((m,), fill, dtype, device))
|
|
96
|
+
for name in _STAT_BUFFERS:
|
|
97
|
+
self.register_buffer(name, torch.zeros(n, **f), persistent=False)
|
|
98
|
+
self.register_buffer("stat_gin", torch.zeros((), **f), persistent=False)
|
|
99
|
+
self.register_buffer("stat_ns", torch.zeros((), **f), persistent=False)
|
|
100
|
+
self.mask.fill_(True)
|
|
101
|
+
self.reset_parameters(generator)
|
|
102
|
+
|
|
103
|
+
# ------------------------------------------------------------------ basics
|
|
104
|
+
|
|
105
|
+
@property
|
|
106
|
+
def degree(self) -> int:
|
|
107
|
+
"""Slots per unit (R)."""
|
|
108
|
+
return self.weight.shape[1]
|
|
109
|
+
|
|
110
|
+
def reset_parameters(self, generator: torch.Generator | None = None) -> None:
|
|
111
|
+
"""Every live Or gets the ``nn.Linear`` law ``U(+-1/sqrt(n))``, a = 1."""
|
|
112
|
+
with torch.no_grad():
|
|
113
|
+
k = 1.0 / math.sqrt(max(self.in_features, 1))
|
|
114
|
+
w = uniform(self.weight.shape, k, generator, self.weight)
|
|
115
|
+
b = uniform(self.bias.shape, k, generator, self.bias)
|
|
116
|
+
live = self.mask
|
|
117
|
+
self.weight.copy_(torch.where(live.unsqueeze(-1), w, torch.zeros_like(w)))
|
|
118
|
+
self.bias.copy_(torch.where(live, b, torch.ones_like(b)))
|
|
119
|
+
self.assembly.fill_(1.0)
|
|
120
|
+
self.reset_adam_state()
|
|
121
|
+
|
|
122
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
123
|
+
if self.track_stats and self.training and torch.is_grad_enabled():
|
|
124
|
+
x = self._observe(x)
|
|
125
|
+
return and_or(x, self.weight, self.bias, self.assembly, self.mask)
|
|
126
|
+
|
|
127
|
+
def num_ors(self) -> torch.Tensor:
|
|
128
|
+
"""Live Ors per unit, shape (m,)."""
|
|
129
|
+
return self.mask.sum(1)
|
|
130
|
+
|
|
131
|
+
def num_params(self) -> int:
|
|
132
|
+
"""Every learnable scalar of every live Or: w, b and a."""
|
|
133
|
+
return int(self.mask.sum()) * (self.in_features + 2)
|
|
134
|
+
|
|
135
|
+
def extra_repr(self) -> str:
|
|
136
|
+
return (f"in_features={self.in_features}, out_features={self.out_features}, "
|
|
137
|
+
f"ors={int(self.mask.sum())}, degree<={self.degree}"
|
|
138
|
+
+ (", probe" if self.probe else ""))
|
|
139
|
+
|
|
140
|
+
@torch.no_grad()
|
|
141
|
+
def project_(self) -> None:
|
|
142
|
+
"""Enforce the invariants: a >= 1, and every free slot is the identity Or."""
|
|
143
|
+
self.assembly.clamp_(min=1.0)
|
|
144
|
+
free = ~self.mask
|
|
145
|
+
self.weight.masked_fill_(free.unsqueeze(-1), 0.0)
|
|
146
|
+
self.bias.masked_fill_(free, 1.0)
|
|
147
|
+
self.assembly.masked_fill_(free, 1.0)
|
|
148
|
+
|
|
149
|
+
@torch.no_grad()
|
|
150
|
+
def reset_adam_state(self, unit: int | None = None, slot: int | None = None) -> None:
|
|
151
|
+
"""Zero the per-Or Adam state (everywhere, or of one Or)."""
|
|
152
|
+
idx = (slice(None), slice(None)) if unit is None else (unit, slot)
|
|
153
|
+
for name in ("adam_step", "adam_m_bias", "adam_v_bias",
|
|
154
|
+
"adam_m_assembly", "adam_v_assembly", *_ORN_BUFFERS):
|
|
155
|
+
getattr(self, name)[idx] = 0
|
|
156
|
+
|
|
157
|
+
# ------------------------------------------------------ input statistics
|
|
158
|
+
|
|
159
|
+
@torch.no_grad()
|
|
160
|
+
def reset_stats(self) -> None:
|
|
161
|
+
"""Clear the epoch statistics of the layer input."""
|
|
162
|
+
for name in (*_STAT_BUFFERS, "stat_gin", "stat_ns"):
|
|
163
|
+
getattr(self, name).zero_()
|
|
164
|
+
|
|
165
|
+
def _observe(self, x: torch.Tensor) -> torch.Tensor:
|
|
166
|
+
# Statistics of x and of u = F(x) (what a type-identity layer would
|
|
167
|
+
# emit), used to fold a depth edit; and the mean |dL/dx|, used to
|
|
168
|
+
# pick the gap where depth is wanted.
|
|
169
|
+
n = self.in_features
|
|
170
|
+
with torch.no_grad():
|
|
171
|
+
xf = x.detach().reshape(-1, n)
|
|
172
|
+
u = readout(xf)
|
|
173
|
+
self.stat_sx.add_(xf.sum(0))
|
|
174
|
+
self.stat_sxx.add_((xf * xf).sum(0))
|
|
175
|
+
self.stat_su.add_(u.sum(0))
|
|
176
|
+
self.stat_suu.add_((u * u).sum(0))
|
|
177
|
+
self.stat_sxu.add_((xf * u).sum(0))
|
|
178
|
+
self.stat_ns.add_(xf.shape[0])
|
|
179
|
+
if not x.requires_grad:
|
|
180
|
+
x = x.detach().requires_grad_(True)
|
|
181
|
+
if n:
|
|
182
|
+
gin = self.stat_gin
|
|
183
|
+
|
|
184
|
+
def hook(g: torch.Tensor) -> None:
|
|
185
|
+
with torch.no_grad():
|
|
186
|
+
gin.add_(g.reshape(-1, n).abs().mean(1).sum())
|
|
187
|
+
|
|
188
|
+
x.register_hook(hook)
|
|
189
|
+
return x
|
|
190
|
+
|
|
191
|
+
# -------------------------------------------------------- structure edits
|
|
192
|
+
|
|
193
|
+
def _set(self, name: str, value: torch.Tensor, dim: int | None = None,
|
|
194
|
+
index: torch.Tensor | None = None) -> None:
|
|
195
|
+
if name in self._parameters:
|
|
196
|
+
old = self._parameters[name]
|
|
197
|
+
new = nn.Parameter(value.contiguous())
|
|
198
|
+
self._parameters[name] = new
|
|
199
|
+
if self.edit_sink is not None and index is not None:
|
|
200
|
+
d = dim if dim >= 0 else old.dim() + dim
|
|
201
|
+
self.edit_sink.append(Edit(old, new, d, index))
|
|
202
|
+
else:
|
|
203
|
+
self._buffers[name] = value.contiguous()
|
|
204
|
+
|
|
205
|
+
def _or_tensors(self):
|
|
206
|
+
"""(name, fill) of every tensor indexed by (unit, slot[, input])."""
|
|
207
|
+
yield "weight", 0.0
|
|
208
|
+
for name, fill in _OR_PARAMS.items():
|
|
209
|
+
yield name, fill
|
|
210
|
+
for name, fill in _OR_BUFFERS.items():
|
|
211
|
+
yield name, fill
|
|
212
|
+
for name in _ORN_BUFFERS:
|
|
213
|
+
yield name, 0.0
|
|
214
|
+
|
|
215
|
+
def _unit_tensors(self):
|
|
216
|
+
yield from self._or_tensors()
|
|
217
|
+
yield from _UNIT_BUFFERS.items()
|
|
218
|
+
|
|
219
|
+
@torch.no_grad()
|
|
220
|
+
def _grow_slots(self) -> None:
|
|
221
|
+
for name, fill in self._or_tensors():
|
|
222
|
+
t = getattr(self, name)
|
|
223
|
+
pad = _full((t.shape[0], 1, *t.shape[2:]), fill, t.dtype, t.device)
|
|
224
|
+
self._set(name, torch.cat([t.detach(), pad], 1), 1,
|
|
225
|
+
grow_index(t.shape[1], t.device))
|
|
226
|
+
|
|
227
|
+
@torch.no_grad()
|
|
228
|
+
def add_or(self, unit: int, weight: torch.Tensor | None = None, bias: float = 1.0,
|
|
229
|
+
assembly: float = 1.0, *, probe: bool = False, born: int = 0) -> int:
|
|
230
|
+
"""Put a live Or in a free slot of ``unit`` (growing R if needed); returns the slot."""
|
|
231
|
+
free = (~self.mask[unit]).nonzero()
|
|
232
|
+
if len(free) == 0:
|
|
233
|
+
self._grow_slots()
|
|
234
|
+
r = self.degree - 1
|
|
235
|
+
else:
|
|
236
|
+
r = int(free[0])
|
|
237
|
+
self.weight[unit, r] = 0.0 if weight is None else weight
|
|
238
|
+
self.bias[unit, r] = bias
|
|
239
|
+
self.assembly[unit, r] = assembly
|
|
240
|
+
self.mask[unit, r] = True
|
|
241
|
+
self.or_probe[unit, r] = probe
|
|
242
|
+
self.or_born[unit, r] = born
|
|
243
|
+
self.reset_adam_state(unit, r)
|
|
244
|
+
return r
|
|
245
|
+
|
|
246
|
+
@torch.no_grad()
|
|
247
|
+
def drop_or(self, unit: int, slot: int) -> None:
|
|
248
|
+
"""Free a slot: it becomes the identity Or again."""
|
|
249
|
+
self.weight[unit, slot] = 0.0
|
|
250
|
+
self.bias[unit, slot] = 1.0
|
|
251
|
+
self.assembly[unit, slot] = 1.0
|
|
252
|
+
self.mask[unit, slot] = False
|
|
253
|
+
self.or_probe[unit, slot] = False
|
|
254
|
+
self.or_born[unit, slot] = 0
|
|
255
|
+
self.reset_adam_state(unit, slot)
|
|
256
|
+
|
|
257
|
+
@torch.no_grad()
|
|
258
|
+
def compact(self) -> None:
|
|
259
|
+
"""Drop slot columns that are free in every unit."""
|
|
260
|
+
keep = self.mask.any(0)
|
|
261
|
+
if bool(keep.all()):
|
|
262
|
+
return
|
|
263
|
+
idx = keep.nonzero().flatten()
|
|
264
|
+
for name, _ in self._or_tensors():
|
|
265
|
+
self._set(name, getattr(self, name).detach().index_select(1, idx), 1,
|
|
266
|
+
keep_index(idx))
|
|
267
|
+
|
|
268
|
+
@torch.no_grad()
|
|
269
|
+
def add_unit(self) -> int:
|
|
270
|
+
"""Append a unit with no Ors (it outputs ln 2 until it gets one); returns its index."""
|
|
271
|
+
for name, fill in self._unit_tensors():
|
|
272
|
+
t = getattr(self, name)
|
|
273
|
+
pad = _full((1, *t.shape[1:]), fill, t.dtype, t.device)
|
|
274
|
+
self._set(name, torch.cat([t.detach(), pad], 0), 0,
|
|
275
|
+
grow_index(t.shape[0], t.device))
|
|
276
|
+
self.out_features += 1
|
|
277
|
+
return self.out_features - 1
|
|
278
|
+
|
|
279
|
+
@torch.no_grad()
|
|
280
|
+
def drop_unit(self, unit: int) -> None:
|
|
281
|
+
keep = _without(self.out_features, unit, self.weight.device)
|
|
282
|
+
for name, _ in self._unit_tensors():
|
|
283
|
+
self._set(name, getattr(self, name).detach().index_select(0, keep), 0,
|
|
284
|
+
keep_index(keep))
|
|
285
|
+
self.out_features -= 1
|
|
286
|
+
|
|
287
|
+
@torch.no_grad()
|
|
288
|
+
def add_input(self) -> None:
|
|
289
|
+
"""New incoming coordinate. Every Or meets it with weight exactly 0,
|
|
290
|
+
so the layer computes the same function (Coq: widen_preserves)."""
|
|
291
|
+
for name in ("weight", *_ORN_BUFFERS):
|
|
292
|
+
t = getattr(self, name)
|
|
293
|
+
self._set(name, torch.cat([t.detach(), torch.zeros_like(t[..., :1])], -1), -1,
|
|
294
|
+
grow_index(t.shape[-1], t.device))
|
|
295
|
+
self.in_features += 1
|
|
296
|
+
self._resize_stats()
|
|
297
|
+
|
|
298
|
+
@torch.no_grad()
|
|
299
|
+
def drop_input(self, j: int) -> None:
|
|
300
|
+
keep = _without(self.in_features, j, self.weight.device)
|
|
301
|
+
for name in ("weight", *_ORN_BUFFERS):
|
|
302
|
+
self._set(name, getattr(self, name).detach().index_select(-1, keep), -1,
|
|
303
|
+
keep_index(keep))
|
|
304
|
+
self.in_features -= 1
|
|
305
|
+
self._resize_stats()
|
|
306
|
+
|
|
307
|
+
def _resize_stats(self) -> None:
|
|
308
|
+
for name in _STAT_BUFFERS:
|
|
309
|
+
self._buffers[name] = self.weight.new_zeros(self.in_features)
|
|
310
|
+
self.reset_stats()
|
|
311
|
+
|
|
312
|
+
# ------------------------------------------------------------ persistence
|
|
313
|
+
|
|
314
|
+
def get_extra_state(self):
|
|
315
|
+
return {"probe": self.probe, "born": self.born}
|
|
316
|
+
|
|
317
|
+
def set_extra_state(self, state) -> None:
|
|
318
|
+
self.probe = bool(state.get("probe", False))
|
|
319
|
+
self.born = int(state.get("born", 0))
|
|
320
|
+
|
|
321
|
+
def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
|
|
322
|
+
# A checkpoint may hold a different structure: take its shapes first.
|
|
323
|
+
w = state_dict.get(prefix + "weight")
|
|
324
|
+
if w is not None and w.shape != self.weight.shape:
|
|
325
|
+
m, R, n = w.shape
|
|
326
|
+
with torch.no_grad():
|
|
327
|
+
for name, t in [*self._parameters.items(), *self._buffers.items()]:
|
|
328
|
+
src = state_dict.get(prefix + name)
|
|
329
|
+
if src is not None:
|
|
330
|
+
self._set(name, torch.empty(src.shape, dtype=t.dtype, device=t.device))
|
|
331
|
+
self.in_features, self.out_features = n, m
|
|
332
|
+
self._resize_stats()
|
|
333
|
+
super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
|
|
334
|
+
|
|
335
|
+
|
|
336
|
+
def uniform(shape, bound: float, generator: torch.Generator | None,
|
|
337
|
+
like: torch.Tensor) -> torch.Tensor:
|
|
338
|
+
"""U(-bound, bound) drawn from ``generator`` (on its own device), then moved to ``like``."""
|
|
339
|
+
dev = generator.device if generator is not None else like.device
|
|
340
|
+
t = torch.empty(shape, dtype=like.dtype, device=dev)
|
|
341
|
+
t.uniform_(-bound, bound, generator=generator)
|
|
342
|
+
return t.to(like.device)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _full(shape, fill, dtype, device) -> torch.Tensor:
|
|
346
|
+
if isinstance(fill, bool):
|
|
347
|
+
return torch.full(shape, fill, dtype=torch.bool, device=device)
|
|
348
|
+
if isinstance(fill, int):
|
|
349
|
+
return torch.full(shape, fill, dtype=torch.long, device=device)
|
|
350
|
+
return torch.full(shape, fill, dtype=dtype, device=device)
|
|
351
|
+
|
|
352
|
+
|
|
353
|
+
def _without(n: int, i: int, device) -> torch.Tensor:
|
|
354
|
+
return torch.cat([torch.arange(i, device=device), torch.arange(i + 1, n, device=device)])
|