torch-type-nn 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- torch_type_nn/__init__.py +38 -0
- torch_type_nn/adapters/__init__.py +6 -0
- torch_type_nn/adapters/mlp.py +322 -0
- torch_type_nn/adapters/typenn.py +546 -0
- torch_type_nn/edits.py +99 -0
- torch_type_nn/functional.py +126 -0
- torch_type_nn/layer.py +354 -0
- torch_type_nn/mlp.py +251 -0
- torch_type_nn/network.py +115 -0
- torch_type_nn/optim.py +122 -0
- torch_type_nn/protocol.py +148 -0
- torch_type_nn/py.typed +0 -0
- torch_type_nn/scaling.py +523 -0
- torch_type_nn/train.py +97 -0
- torch_type_nn-0.1.0.dev0.dist-info/METADATA +302 -0
- torch_type_nn-0.1.0.dev0.dist-info/RECORD +18 -0
- torch_type_nn-0.1.0.dev0.dist-info/WHEEL +4 -0
- torch_type_nn-0.1.0.dev0.dist-info/licenses/LICENSE +23 -0
torch_type_nn/mlp.py
ADDED
|
@@ -0,0 +1,251 @@
|
|
|
1
|
+
"""An MLP whose width and depth can change while it trains.
|
|
2
|
+
|
|
3
|
+
``ScalableMLP`` is ``Linear -> ReLU -> ... -> Linear`` with an optional
|
|
4
|
+
type-nn readout ``F(u) = sign(u) ln(1 + |u|)`` on the output (so it can be
|
|
5
|
+
compared with the type-nn board's MLP baseline). Its layers are
|
|
6
|
+
:class:`TrackedLinear`: an ``nn.Linear`` that can add or drop output units and
|
|
7
|
+
input columns, reports each parameter replacement as an
|
|
8
|
+
:class:`~torch_type_nn.edits.Edit`, and collects the input statistics the
|
|
9
|
+
scaler needs. Grow and prune it with
|
|
10
|
+
:class:`~torch_type_nn.scaling.StructureScaler`, which wraps it in
|
|
11
|
+
:class:`~torch_type_nn.adapters.mlp.MLPAdapter`.
|
|
12
|
+
|
|
13
|
+
The identities it offers (Net2Net, Chen et al. 2016):
|
|
14
|
+
|
|
15
|
+
* width: a new hidden unit read with an all-zero column is exact;
|
|
16
|
+
* depth: after a ReLU the activations are >= 0, so ``ReLU(I a + 0) = a``;
|
|
17
|
+
an identity layer is exact in any gap between hidden layers (but not in
|
|
18
|
+
front of the first layer, whose input may be negative).
|
|
19
|
+
"""
|
|
20
|
+
|
|
21
|
+
from __future__ import annotations
|
|
22
|
+
|
|
23
|
+
import math
|
|
24
|
+
import re
|
|
25
|
+
|
|
26
|
+
import torch
|
|
27
|
+
from torch import nn
|
|
28
|
+
|
|
29
|
+
from .edits import Edit, grow_index, keep_index
|
|
30
|
+
from .functional import readout
|
|
31
|
+
from .layer import uniform
|
|
32
|
+
|
|
33
|
+
__all__ = ["TrackedLinear", "ScalableMLP"]
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
class TrackedLinear(nn.Linear):
|
|
37
|
+
"""``nn.Linear`` with structural edits, edit reporting and input statistics."""
|
|
38
|
+
|
|
39
|
+
def __init__(self, in_features: int, out_features: int, *,
|
|
40
|
+
generator: torch.Generator | None = None, device=None, dtype=None) -> None:
|
|
41
|
+
super().__init__(in_features, out_features, device=device, dtype=dtype)
|
|
42
|
+
self.probe = False
|
|
43
|
+
self.born = 0
|
|
44
|
+
self.track_stats = False
|
|
45
|
+
self.edit_sink: list | None = None
|
|
46
|
+
f = {"device": device, "dtype": self.weight.dtype}
|
|
47
|
+
self.register_buffer("unit_probe", torch.zeros(out_features, dtype=torch.bool,
|
|
48
|
+
device=device))
|
|
49
|
+
self.register_buffer("unit_born", torch.zeros(out_features, dtype=torch.long,
|
|
50
|
+
device=device))
|
|
51
|
+
self.register_buffer("stat_sx", torch.zeros(in_features, **f), persistent=False)
|
|
52
|
+
self.register_buffer("stat_gin", torch.zeros((), **f), persistent=False)
|
|
53
|
+
self.register_buffer("stat_ns", torch.zeros((), **f), persistent=False)
|
|
54
|
+
if generator is not None:
|
|
55
|
+
self.reset_from(generator)
|
|
56
|
+
|
|
57
|
+
@torch.no_grad()
|
|
58
|
+
def reset_from(self, generator: torch.Generator) -> None:
|
|
59
|
+
"""The ``nn.Linear`` law U(+-1/sqrt(n)), drawn from ``generator``."""
|
|
60
|
+
k = 1.0 / math.sqrt(max(self.in_features, 1))
|
|
61
|
+
self.weight.copy_(uniform(self.weight.shape, k, generator, self.weight))
|
|
62
|
+
self.bias.copy_(uniform(self.bias.shape, k, generator, self.bias))
|
|
63
|
+
|
|
64
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
65
|
+
if self.track_stats and self.training and torch.is_grad_enabled():
|
|
66
|
+
x = self._observe(x)
|
|
67
|
+
return super().forward(x)
|
|
68
|
+
|
|
69
|
+
def _observe(self, x: torch.Tensor) -> torch.Tensor:
|
|
70
|
+
n = self.in_features
|
|
71
|
+
with torch.no_grad():
|
|
72
|
+
xf = x.detach().reshape(-1, n)
|
|
73
|
+
self.stat_sx.add_(xf.sum(0))
|
|
74
|
+
self.stat_ns.add_(xf.shape[0])
|
|
75
|
+
if not x.requires_grad:
|
|
76
|
+
x = x.detach().requires_grad_(True)
|
|
77
|
+
gin = self.stat_gin
|
|
78
|
+
|
|
79
|
+
def hook(g: torch.Tensor) -> None:
|
|
80
|
+
with torch.no_grad():
|
|
81
|
+
gin.add_(g.reshape(-1, n).abs().mean(1).sum())
|
|
82
|
+
|
|
83
|
+
x.register_hook(hook)
|
|
84
|
+
return x
|
|
85
|
+
|
|
86
|
+
@torch.no_grad()
|
|
87
|
+
def reset_stats(self) -> None:
|
|
88
|
+
for t in (self.stat_sx, self.stat_gin, self.stat_ns):
|
|
89
|
+
t.zero_()
|
|
90
|
+
|
|
91
|
+
def num_params(self) -> int:
|
|
92
|
+
return self.out_features * (self.in_features + 1)
|
|
93
|
+
|
|
94
|
+
def extra_repr(self) -> str:
|
|
95
|
+
return super().extra_repr() + (", probe" if self.probe else "")
|
|
96
|
+
|
|
97
|
+
# -------------------------------------------------------- structure edits
|
|
98
|
+
|
|
99
|
+
def _set(self, name: str, value: torch.Tensor, dim: int, index: torch.Tensor) -> None:
|
|
100
|
+
if name in self._parameters:
|
|
101
|
+
old = self._parameters[name]
|
|
102
|
+
new = nn.Parameter(value.contiguous())
|
|
103
|
+
self._parameters[name] = new
|
|
104
|
+
if self.edit_sink is not None:
|
|
105
|
+
self.edit_sink.append(Edit(old, new, dim, index))
|
|
106
|
+
else:
|
|
107
|
+
self._buffers[name] = value.contiguous()
|
|
108
|
+
|
|
109
|
+
@torch.no_grad()
|
|
110
|
+
def add_out(self, weight: torch.Tensor, bias: float, *, probe: bool = False,
|
|
111
|
+
born: int = 0) -> int:
|
|
112
|
+
"""Append an output unit; returns its index."""
|
|
113
|
+
m = self.out_features
|
|
114
|
+
idx = grow_index(m, self.weight.device)
|
|
115
|
+
self._set("weight", torch.cat([self.weight.detach(), weight.reshape(1, -1)], 0), 0, idx)
|
|
116
|
+
self._set("bias", torch.cat([self.bias.detach(), self.bias.new_full((1,), bias)]), 0, idx)
|
|
117
|
+
self._set("unit_probe", torch.cat([self.unit_probe, self.unit_probe.new_full((1,), probe)]),
|
|
118
|
+
0, idx)
|
|
119
|
+
self._set("unit_born", torch.cat([self.unit_born, self.unit_born.new_full((1,), born)]),
|
|
120
|
+
0, idx)
|
|
121
|
+
self.out_features = m + 1
|
|
122
|
+
return m
|
|
123
|
+
|
|
124
|
+
@torch.no_grad()
|
|
125
|
+
def drop_out(self, k: int) -> None:
|
|
126
|
+
keep = _without(self.out_features, k, self.weight.device)
|
|
127
|
+
for name in ("weight", "bias", "unit_probe", "unit_born"):
|
|
128
|
+
self._set(name, getattr(self, name).detach().index_select(0, keep), 0,
|
|
129
|
+
keep_index(keep))
|
|
130
|
+
self.out_features -= 1
|
|
131
|
+
|
|
132
|
+
@torch.no_grad()
|
|
133
|
+
def add_in(self) -> None:
|
|
134
|
+
"""A new input column of zeros: the function is unchanged."""
|
|
135
|
+
idx = grow_index(self.in_features, self.weight.device)
|
|
136
|
+
w = self.weight.detach()
|
|
137
|
+
self._set("weight", torch.cat([w, torch.zeros_like(w[:, :1])], 1), 1, idx)
|
|
138
|
+
self.in_features += 1
|
|
139
|
+
self._resize_stats()
|
|
140
|
+
|
|
141
|
+
@torch.no_grad()
|
|
142
|
+
def drop_in(self, j: int) -> None:
|
|
143
|
+
keep = _without(self.in_features, j, self.weight.device)
|
|
144
|
+
self._set("weight", self.weight.detach().index_select(1, keep), 1, keep_index(keep))
|
|
145
|
+
self.in_features -= 1
|
|
146
|
+
self._resize_stats()
|
|
147
|
+
|
|
148
|
+
def _resize_stats(self) -> None:
|
|
149
|
+
self._buffers["stat_sx"] = self.weight.new_zeros(self.in_features)
|
|
150
|
+
self.reset_stats()
|
|
151
|
+
|
|
152
|
+
# ------------------------------------------------------------ persistence
|
|
153
|
+
|
|
154
|
+
def get_extra_state(self):
|
|
155
|
+
return {"probe": self.probe, "born": self.born}
|
|
156
|
+
|
|
157
|
+
def set_extra_state(self, state) -> None:
|
|
158
|
+
self.probe = bool(state.get("probe", False))
|
|
159
|
+
self.born = int(state.get("born", 0))
|
|
160
|
+
|
|
161
|
+
def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
|
|
162
|
+
w = state_dict.get(prefix + "weight")
|
|
163
|
+
if w is not None and w.shape != self.weight.shape:
|
|
164
|
+
m, n = w.shape
|
|
165
|
+
for name, t in [*self._parameters.items(), *self._buffers.items()]:
|
|
166
|
+
src = state_dict.get(prefix + name)
|
|
167
|
+
if src is not None:
|
|
168
|
+
new = torch.empty(src.shape, dtype=t.dtype, device=t.device)
|
|
169
|
+
if name in self._parameters:
|
|
170
|
+
self._parameters[name] = nn.Parameter(new)
|
|
171
|
+
else:
|
|
172
|
+
self._buffers[name] = new
|
|
173
|
+
self.in_features, self.out_features = n, m
|
|
174
|
+
self._resize_stats()
|
|
175
|
+
super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
class ScalableMLP(nn.Module):
|
|
179
|
+
"""``Linear -> ReLU -> ... -> Linear [-> F]`` whose width and depth can change.
|
|
180
|
+
|
|
181
|
+
Args:
|
|
182
|
+
in_features, out_features: task widths (fixed for life).
|
|
183
|
+
hidden: widths of the hidden layers at birth; default one hidden layer
|
|
184
|
+
of ``max(2, out_features)`` units: small, since the scaler grows it.
|
|
185
|
+
readout: apply ``F(u) = sign(u) ln(1 + |u|)`` to the output, as the
|
|
186
|
+
type-nn board's MLP baseline does.
|
|
187
|
+
seed: seeds ``self.generator`` (initialisation and structural draws).
|
|
188
|
+
"""
|
|
189
|
+
|
|
190
|
+
def __init__(self, in_features: int, out_features: int, hidden: list[int] | None = None,
|
|
191
|
+
*, readout: bool = True, seed: int = 1, device=None, dtype=None) -> None:
|
|
192
|
+
super().__init__()
|
|
193
|
+
self.in_features, self.out_features = int(in_features), int(out_features)
|
|
194
|
+
self.readout = readout
|
|
195
|
+
self.generator = torch.Generator().manual_seed(int(seed))
|
|
196
|
+
hidden = [max(2, self.out_features)] if hidden is None else list(hidden)
|
|
197
|
+
widths = [self.in_features, *hidden, self.out_features]
|
|
198
|
+
self.linears = nn.ModuleList(
|
|
199
|
+
TrackedLinear(a, b, generator=self.generator, device=device, dtype=dtype)
|
|
200
|
+
for a, b in zip(widths[:-1], widths[1:], strict=True))
|
|
201
|
+
|
|
202
|
+
@property
|
|
203
|
+
def depth(self) -> int:
|
|
204
|
+
return len(self.linears)
|
|
205
|
+
|
|
206
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
207
|
+
last = len(self.linears) - 1
|
|
208
|
+
for i, lin in enumerate(self.linears):
|
|
209
|
+
x = lin(x)
|
|
210
|
+
if i < last:
|
|
211
|
+
x = torch.relu(x)
|
|
212
|
+
return readout(x) if self.readout else x
|
|
213
|
+
|
|
214
|
+
def num_params(self) -> int:
|
|
215
|
+
return sum(l.num_params() for l in self.linears)
|
|
216
|
+
|
|
217
|
+
def structure(self) -> list[int]:
|
|
218
|
+
"""Hidden widths."""
|
|
219
|
+
return [l.out_features for l in self.linears[:-1]]
|
|
220
|
+
|
|
221
|
+
def extra_repr(self) -> str:
|
|
222
|
+
return (f"in_features={self.in_features}, out_features={self.out_features}, "
|
|
223
|
+
f"hidden={self.structure()}, params={self.num_params()}")
|
|
224
|
+
|
|
225
|
+
def __getstate__(self):
|
|
226
|
+
state = self.__dict__.copy()
|
|
227
|
+
state["_generator_state"] = state.pop("generator").get_state()
|
|
228
|
+
return state
|
|
229
|
+
|
|
230
|
+
def __setstate__(self, state):
|
|
231
|
+
gen_state = state.pop("_generator_state", None)
|
|
232
|
+
self.__dict__.update(state)
|
|
233
|
+
self.generator = torch.Generator()
|
|
234
|
+
if gen_state is not None:
|
|
235
|
+
self.generator.set_state(gen_state)
|
|
236
|
+
|
|
237
|
+
def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
|
|
238
|
+
"""Accepts a checkpoint of any structure."""
|
|
239
|
+
idx = {int(m.group(1)) for k in state_dict
|
|
240
|
+
if (m := re.match(r"linears\.(\d+)\.weight$", k))}
|
|
241
|
+
D = max(idx) + 1 if idx else 0
|
|
242
|
+
ref = self.linears[0].weight
|
|
243
|
+
while len(self.linears) < D:
|
|
244
|
+
self.linears.append(TrackedLinear(1, 1, device=ref.device, dtype=ref.dtype))
|
|
245
|
+
while len(self.linears) > D:
|
|
246
|
+
del self.linears[len(self.linears) - 1]
|
|
247
|
+
return super().load_state_dict(state_dict, strict=strict, assign=assign)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _without(n: int, i: int, device) -> torch.Tensor:
|
|
251
|
+
return torch.cat([torch.arange(i, device=device), torch.arange(i + 1, n, device=device)])
|
torch_type_nn/network.py
ADDED
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""A stack of type-nn layers whose depth, widths and degrees can change."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
import re
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
from torch import nn
|
|
10
|
+
|
|
11
|
+
from .layer import AndOr
|
|
12
|
+
|
|
13
|
+
__all__ = ["TypeNN", "birth_depth"]
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def birth_depth(in_features: int, out_features: int) -> int:
|
|
17
|
+
"""round(ln(1 + n m)), at least 1: xor 1, iris 3, wine 4, wdbc 3, diabetes 2, ionosphere 4."""
|
|
18
|
+
# C lround: half away from zero (Python round() is half-to-even)
|
|
19
|
+
return max(1, int(math.floor(math.log(1.0 + in_features * out_features) + 0.5)))
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
class TypeNN(nn.Module):
|
|
23
|
+
"""A dense stack of :class:`AndOr` layers, ``n -> m -> ... -> m``.
|
|
24
|
+
|
|
25
|
+
At birth every unit is a single random Or (a degree-1 And) and there are
|
|
26
|
+
:func:`birth_depth` layers. Nothing about the structure is a
|
|
27
|
+
hyper-parameter: train it with :class:`~torch_type_nn.scaling.StructureScaler`
|
|
28
|
+
(or :func:`~torch_type_nn.train.fit`) and width, degree and depth are
|
|
29
|
+
grown early and pruned late. Without a scaler it is an ordinary static
|
|
30
|
+
network.
|
|
31
|
+
|
|
32
|
+
Args:
|
|
33
|
+
in_features, out_features: task widths (fixed for life).
|
|
34
|
+
depth: birth depth; default :func:`birth_depth`.
|
|
35
|
+
seed: seeds ``self.generator``, used for the birth draw and for
|
|
36
|
+
every random draw of structural edits.
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
def __init__(self, in_features: int, out_features: int, *, depth: int | None = None,
|
|
40
|
+
seed: int = 1, device=None, dtype=None) -> None:
|
|
41
|
+
super().__init__()
|
|
42
|
+
self.in_features, self.out_features = int(in_features), int(out_features)
|
|
43
|
+
self.generator = torch.Generator().manual_seed(int(seed))
|
|
44
|
+
D = birth_depth(in_features, out_features) if depth is None else int(depth)
|
|
45
|
+
self.birth_depth = D
|
|
46
|
+
layers, n = [], self.in_features
|
|
47
|
+
for _ in range(D):
|
|
48
|
+
layers.append(AndOr(n, self.out_features, 1, generator=self.generator,
|
|
49
|
+
device=device, dtype=dtype))
|
|
50
|
+
n = self.out_features
|
|
51
|
+
self.layers = nn.ModuleList(layers)
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def depth(self) -> int:
|
|
55
|
+
return len(self.layers)
|
|
56
|
+
|
|
57
|
+
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
|
58
|
+
for layer in self.layers:
|
|
59
|
+
x = layer(x)
|
|
60
|
+
return x
|
|
61
|
+
|
|
62
|
+
def num_params(self) -> int:
|
|
63
|
+
"""Every learnable scalar present: (n_in + 2) per live Or, probes included."""
|
|
64
|
+
return sum(layer.num_params() for layer in self.layers)
|
|
65
|
+
|
|
66
|
+
def insert_layer(self, index: int, layer: AndOr) -> None:
|
|
67
|
+
self.layers.insert(index, layer)
|
|
68
|
+
|
|
69
|
+
def remove_layer(self, index: int) -> AndOr:
|
|
70
|
+
layer = self.layers[index]
|
|
71
|
+
del self.layers[index]
|
|
72
|
+
return layer
|
|
73
|
+
|
|
74
|
+
def structure(self) -> list[list[int]]:
|
|
75
|
+
"""Live Ors per unit, per layer."""
|
|
76
|
+
return [layer.num_ors().tolist() for layer in self.layers]
|
|
77
|
+
|
|
78
|
+
def extra_repr(self) -> str:
|
|
79
|
+
return (f"in_features={self.in_features}, out_features={self.out_features}, "
|
|
80
|
+
f"depth={self.depth}, params={self.num_params()}")
|
|
81
|
+
|
|
82
|
+
def get_extra_state(self):
|
|
83
|
+
return {"birth_depth": self.birth_depth, "generator": self.generator.get_state()}
|
|
84
|
+
|
|
85
|
+
def set_extra_state(self, state) -> None:
|
|
86
|
+
self.birth_depth = int(state.get("birth_depth", self.birth_depth))
|
|
87
|
+
if "generator" in state:
|
|
88
|
+
self.generator.set_state(state["generator"])
|
|
89
|
+
|
|
90
|
+
# torch.Generator does not pickle: carry its state instead
|
|
91
|
+
def __getstate__(self):
|
|
92
|
+
state = self.__dict__.copy()
|
|
93
|
+
gen = state.pop("generator")
|
|
94
|
+
state["_generator_state"] = gen.get_state()
|
|
95
|
+
return state
|
|
96
|
+
|
|
97
|
+
def __setstate__(self, state):
|
|
98
|
+
gen_state = state.pop("_generator_state", None)
|
|
99
|
+
self.__dict__.update(state)
|
|
100
|
+
self.generator = torch.Generator()
|
|
101
|
+
if gen_state is not None:
|
|
102
|
+
self.generator.set_state(gen_state)
|
|
103
|
+
|
|
104
|
+
def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
|
|
105
|
+
"""Accepts a checkpoint of any structure: depth is taken from the keys,
|
|
106
|
+
widths and degrees from the tensors."""
|
|
107
|
+
idx = {int(m.group(1)) for k in state_dict
|
|
108
|
+
if (m := re.match(r"layers\.(\d+)\.weight$", k))}
|
|
109
|
+
D = max(idx) + 1 if idx else 0
|
|
110
|
+
ref = self.layers[0].weight if len(self.layers) else torch.empty(0)
|
|
111
|
+
while len(self.layers) < D:
|
|
112
|
+
self.layers.append(AndOr(1, 1, 1, device=ref.device, dtype=ref.dtype))
|
|
113
|
+
while len(self.layers) > D:
|
|
114
|
+
del self.layers[len(self.layers) - 1]
|
|
115
|
+
return super().load_state_dict(state_dict, strict=strict, assign=assign)
|
torch_type_nn/optim.py
ADDED
|
@@ -0,0 +1,122 @@
|
|
|
1
|
+
"""Adam with one step counter per Or.
|
|
2
|
+
|
|
3
|
+
type-nn gives each Or its own Adam bias correction: an Or born late starts
|
|
4
|
+
at t = 1, so its first steps are corrected exactly like those of an Or that
|
|
5
|
+
existed from the start. Its moments live in the layer (``AndOr.adam_*``) so
|
|
6
|
+
structural edits carry them along.
|
|
7
|
+
|
|
8
|
+
After every step the assembly index is projected back to ``a >= 1`` and
|
|
9
|
+
free slots stay the identity Or.
|
|
10
|
+
|
|
11
|
+
Parameters that do not belong to an :class:`~torch_type_nn.layer.AndOr`
|
|
12
|
+
(say, a ``nn.Linear`` head in the same model) get ordinary Adam, so one
|
|
13
|
+
``TypeAdam`` can drive a mixed model.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import torch
|
|
19
|
+
from torch import nn
|
|
20
|
+
|
|
21
|
+
from .layer import AndOr
|
|
22
|
+
|
|
23
|
+
__all__ = ["TypeAdam", "keep_invariants"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def keep_invariants(optimizer: torch.optim.Optimizer, module: nn.Module):
|
|
27
|
+
"""Make a stock optimizer respect the architecture: after every
|
|
28
|
+
``optimizer.step()`` each :class:`AndOr` in ``module`` is projected back to
|
|
29
|
+
``a >= 1`` with its free slots the identity Or (``AndOr.project_``).
|
|
30
|
+
:class:`TypeAdam` does this itself. Returns the hook handle; calling it
|
|
31
|
+
twice on one optimizer attaches one hook.
|
|
32
|
+
"""
|
|
33
|
+
if isinstance(optimizer, TypeAdam) or getattr(optimizer, "_tnn_projects", None) is module:
|
|
34
|
+
return None
|
|
35
|
+
|
|
36
|
+
def project(opt, args, kwargs):
|
|
37
|
+
for layer in module.modules():
|
|
38
|
+
if isinstance(layer, AndOr):
|
|
39
|
+
layer.project_()
|
|
40
|
+
|
|
41
|
+
handle = optimizer.register_step_post_hook(project)
|
|
42
|
+
optimizer._tnn_projects = module
|
|
43
|
+
return handle
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
class TypeAdam(torch.optim.Optimizer):
|
|
47
|
+
"""Adam whose step count is per Or, for models that contain :class:`AndOr` layers.
|
|
48
|
+
|
|
49
|
+
Unlike stock optimizers, ``TypeAdam`` takes the *module*, not a parameter
|
|
50
|
+
list: structural edits replace parameter tensors, and the optimizer
|
|
51
|
+
re-reads them from the module on every step.
|
|
52
|
+
|
|
53
|
+
Args:
|
|
54
|
+
module: the model.
|
|
55
|
+
lr: learning rate.
|
|
56
|
+
betas, eps: as in :class:`torch.optim.Adam`.
|
|
57
|
+
"""
|
|
58
|
+
|
|
59
|
+
def __init__(self, module: nn.Module, lr: float = 1e-3,
|
|
60
|
+
betas: tuple[float, float] = (0.9, 0.999), eps: float = 1e-8) -> None:
|
|
61
|
+
if lr < 0:
|
|
62
|
+
raise ValueError(f"invalid lr {lr}")
|
|
63
|
+
self.module = module
|
|
64
|
+
super().__init__(list(module.parameters()), {"lr": lr, "betas": betas, "eps": eps})
|
|
65
|
+
|
|
66
|
+
def _sync(self) -> None:
|
|
67
|
+
self.param_groups[0]["params"] = list(self.module.parameters())
|
|
68
|
+
|
|
69
|
+
def zero_grad(self, set_to_none: bool = True) -> None:
|
|
70
|
+
self._sync()
|
|
71
|
+
super().zero_grad(set_to_none)
|
|
72
|
+
|
|
73
|
+
@torch.no_grad()
|
|
74
|
+
def step(self, closure=None):
|
|
75
|
+
loss = None
|
|
76
|
+
if closure is not None:
|
|
77
|
+
with torch.enable_grad():
|
|
78
|
+
loss = closure()
|
|
79
|
+
self._sync()
|
|
80
|
+
group = self.param_groups[0]
|
|
81
|
+
lr, (b1, b2), eps = group["lr"], group["betas"], group["eps"]
|
|
82
|
+
owned = set()
|
|
83
|
+
for layer in self.module.modules():
|
|
84
|
+
if isinstance(layer, AndOr):
|
|
85
|
+
_step_layer(layer, lr, b1, b2, eps)
|
|
86
|
+
owned.update(id(p) for p in layer.parameters(recurse=False))
|
|
87
|
+
for p in group["params"]:
|
|
88
|
+
if id(p) in owned or p.grad is None:
|
|
89
|
+
continue
|
|
90
|
+
st = self.state[p]
|
|
91
|
+
if not st:
|
|
92
|
+
st["step"] = 0
|
|
93
|
+
st["exp_avg"] = torch.zeros_like(p)
|
|
94
|
+
st["exp_avg_sq"] = torch.zeros_like(p)
|
|
95
|
+
st["step"] += 1
|
|
96
|
+
t = st["step"]
|
|
97
|
+
st["exp_avg"].mul_(b1).add_(p.grad, alpha=1 - b1)
|
|
98
|
+
st["exp_avg_sq"].mul_(b2).addcmul_(p.grad, p.grad, value=1 - b2)
|
|
99
|
+
c1, c2 = 1.0 / (1.0 - b1 ** t), 1.0 / (1.0 - b2 ** t)
|
|
100
|
+
p.sub_(lr * (st["exp_avg"] * c1) / ((st["exp_avg_sq"] * c2).sqrt() + eps))
|
|
101
|
+
return loss
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def _step_layer(layer: AndOr, lr: float, b1: float, b2: float, eps: float) -> None:
|
|
105
|
+
live = layer.mask
|
|
106
|
+
layer.adam_step.add_(live.long())
|
|
107
|
+
t = layer.adam_step.clamp(min=1).to(layer.weight.dtype)
|
|
108
|
+
c1 = 1.0 / (1.0 - b1 ** t) # (m, R)
|
|
109
|
+
c2 = 1.0 / (1.0 - b2 ** t)
|
|
110
|
+
livef = live.to(layer.weight.dtype)
|
|
111
|
+
for p, m, v, cc1, cc2, lf in (
|
|
112
|
+
(layer.weight, layer.adam_m_weight, layer.adam_v_weight,
|
|
113
|
+
c1.unsqueeze(-1), c2.unsqueeze(-1), livef.unsqueeze(-1)),
|
|
114
|
+
(layer.bias, layer.adam_m_bias, layer.adam_v_bias, c1, c2, livef),
|
|
115
|
+
(layer.assembly, layer.adam_m_assembly, layer.adam_v_assembly, c1, c2, livef),
|
|
116
|
+
):
|
|
117
|
+
g = p.grad if p.grad is not None else torch.zeros_like(p)
|
|
118
|
+
g = g * lf
|
|
119
|
+
m.mul_(b1).add_(g, alpha=1 - b1)
|
|
120
|
+
v.mul_(b2).addcmul_(g, g, value=1 - b2)
|
|
121
|
+
p.sub_(lf * lr * (m * cc1) / ((v * cc2).sqrt() + eps))
|
|
122
|
+
layer.assembly.clamp_(min=1.0) # at least one copy
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""The contract between the structure scaler and a model family.
|
|
2
|
+
|
|
3
|
+
type-nn's scaling rule is architecture-agnostic: on each axis a *probe* is
|
|
4
|
+
an exact identity that back-prop may move; a probe that moved past
|
|
5
|
+
``theta(T)`` is promoted (under ``rule="bic"``: and pays its BIC price); late
|
|
6
|
+
in training items back inside the identity band are removed (under
|
|
7
|
+
``rule="bic"``: items are ablated and removed while the model criterion
|
|
8
|
+
allows).
|
|
9
|
+
What is specific to an architecture is only *what* an item is: how to
|
|
10
|
+
insert an identity, how far an item is from it, how to reset it and how to
|
|
11
|
+
remove it. That is this protocol.
|
|
12
|
+
|
|
13
|
+
Items live on three axes:
|
|
14
|
+
|
|
15
|
+
``WIDTH``
|
|
16
|
+
a coordinate between two layers (a unit and the weights that read it)
|
|
17
|
+
``DEGREE``
|
|
18
|
+
a factor inside a unit (for type-nn: an Or inside an And)
|
|
19
|
+
``DEPTH``
|
|
20
|
+
a layer (there is at most one depth probe at a time)
|
|
21
|
+
|
|
22
|
+
An adapter may leave an axis empty (return no sites or candidates).
|
|
23
|
+
:class:`~torch_type_nn.adapters.typenn.TypeNNAdapter` is the reference
|
|
24
|
+
implementation.
|
|
25
|
+
"""
|
|
26
|
+
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
from collections.abc import Callable, Hashable, Sequence
|
|
30
|
+
from dataclasses import dataclass
|
|
31
|
+
from typing import Protocol, runtime_checkable
|
|
32
|
+
|
|
33
|
+
import torch
|
|
34
|
+
from torch import nn
|
|
35
|
+
|
|
36
|
+
__all__ = ["WIDTH", "DEGREE", "DEPTH", "AXES", "Item", "Trial", "EditContext", "Scalable"]
|
|
37
|
+
|
|
38
|
+
WIDTH, DEGREE, DEPTH = "width", "degree", "depth"
|
|
39
|
+
AXES = (WIDTH, DEGREE, DEPTH)
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
@dataclass(frozen=True)
|
|
43
|
+
class Item:
|
|
44
|
+
"""One structural item: an axis and an adapter-defined address.
|
|
45
|
+
|
|
46
|
+
Addresses are only valid until the next structural edit.
|
|
47
|
+
"""
|
|
48
|
+
|
|
49
|
+
axis: str
|
|
50
|
+
address: tuple
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
@dataclass
|
|
54
|
+
class Trial:
|
|
55
|
+
"""A structural edit made tentatively: keep it with ``commit`` or revert it with ``undo``."""
|
|
56
|
+
|
|
57
|
+
undo: Callable[[], None]
|
|
58
|
+
commit: Callable[[], None]
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@dataclass
|
|
62
|
+
class EditContext:
|
|
63
|
+
"""What a new probe needs: its birth step, noise amplitude, and the RNG."""
|
|
64
|
+
|
|
65
|
+
step: int
|
|
66
|
+
noise: float
|
|
67
|
+
generator: torch.Generator
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@runtime_checkable
|
|
71
|
+
class Scalable(Protocol):
|
|
72
|
+
"""A model family the :class:`~torch_type_nn.scaling.StructureScaler` can grow and prune."""
|
|
73
|
+
|
|
74
|
+
model: nn.Module
|
|
75
|
+
|
|
76
|
+
# ------------------------------------------------------------ bookkeeping
|
|
77
|
+
def num_params(self) -> int:
|
|
78
|
+
"""Every learnable scalar currently present (probes included)."""
|
|
79
|
+
|
|
80
|
+
def set_tracking(self, on: bool) -> None:
|
|
81
|
+
"""Collect (or stop collecting) training statistics in forward passes."""
|
|
82
|
+
|
|
83
|
+
def reset_stats(self) -> None:
|
|
84
|
+
"""Forget the statistics of the epoch that ended."""
|
|
85
|
+
|
|
86
|
+
def set_edit_sink(self, sink: list | None) -> None:
|
|
87
|
+
"""Report parameter replacements (``Edit``) to ``sink`` while it is a list."""
|
|
88
|
+
|
|
89
|
+
def finalize(self) -> None:
|
|
90
|
+
"""Training is over; tidy storage (e.g. drop free padding)."""
|
|
91
|
+
|
|
92
|
+
# ----------------------------------------------------------------- probes
|
|
93
|
+
def probe_sites(self, axis: str) -> Sequence[Hashable]:
|
|
94
|
+
"""Where probes of ``axis`` live now (WIDTH and DEGREE), in visiting order."""
|
|
95
|
+
|
|
96
|
+
def probe_at(self, axis: str, site: Hashable | None = None) -> Item | None:
|
|
97
|
+
"""The probe at ``site`` (for DEPTH: the depth probe, ``site`` ignored)."""
|
|
98
|
+
|
|
99
|
+
def add_probe(self, axis: str, site: Hashable, ctx: EditContext) -> None:
|
|
100
|
+
"""Insert an exact identity (up to ``ctx.noise``) at ``site``."""
|
|
101
|
+
|
|
102
|
+
def promote(self, item: Item) -> None:
|
|
103
|
+
"""The probe becomes a live item."""
|
|
104
|
+
|
|
105
|
+
def born(self, item: Item) -> int:
|
|
106
|
+
"""Training step at which ``item`` was inserted."""
|
|
107
|
+
|
|
108
|
+
def displacement(self, item: Item) -> float:
|
|
109
|
+
"""RMS distance of ``item`` from its identity (``inf`` if it has none)."""
|
|
110
|
+
|
|
111
|
+
def best_site(self, axis: str, moving: Item | None = None) -> Hashable:
|
|
112
|
+
"""Where a new probe of ``axis`` is most wanted (DEPTH: the loudest gap,
|
|
113
|
+
counted as if ``moving`` were absent)."""
|
|
114
|
+
|
|
115
|
+
def site_of(self, item: Item) -> Hashable:
|
|
116
|
+
"""The site a probe occupies, in :meth:`best_site` coordinates."""
|
|
117
|
+
|
|
118
|
+
def remove_probe(self, item: Item) -> None:
|
|
119
|
+
"""Take a probe out (near-exactly) so it can move elsewhere."""
|
|
120
|
+
|
|
121
|
+
# -------------------------------------------------- measurement / pruning
|
|
122
|
+
def ablate(self, item: Item) -> Callable[[], None] | None:
|
|
123
|
+
"""Reset ``item`` to its identity in place; return the exact undo
|
|
124
|
+
(``None`` if the item has no identity)."""
|
|
125
|
+
|
|
126
|
+
def cost(self, item: Item) -> int:
|
|
127
|
+
"""Parameters owned by ``item``."""
|
|
128
|
+
|
|
129
|
+
def trial_remove(self, item: Item) -> Trial | None:
|
|
130
|
+
"""Remove ``item`` (DEPTH) tentatively; ``None`` if it cannot be removed."""
|
|
131
|
+
|
|
132
|
+
def removal_order(self, axis: str) -> Sequence[Item]:
|
|
133
|
+
"""Live items of ``axis`` to try removing one at a time (DEPTH)."""
|
|
134
|
+
|
|
135
|
+
def prune_candidates(self) -> Sequence[Item]:
|
|
136
|
+
"""Items (WIDTH and DEGREE) for the joint ablation pool, in a fixed order."""
|
|
137
|
+
|
|
138
|
+
def can_remove(self, item: Item, removed: Sequence[Item]) -> bool:
|
|
139
|
+
"""Whether ``item`` may join the ablated set ``removed`` (keep-one rules)."""
|
|
140
|
+
|
|
141
|
+
def params_without(self, removed: Sequence[Item]) -> int:
|
|
142
|
+
"""Exact parameter count of the model with ``removed`` taken out."""
|
|
143
|
+
|
|
144
|
+
def commit_removals(self, removed: Sequence[Item],
|
|
145
|
+
axes: Sequence[str] = (WIDTH, DEGREE)) -> dict[str, int]:
|
|
146
|
+
"""Remove ``removed`` for real and clear the surviving probe flags, on the
|
|
147
|
+
given ``axes`` only (the threshold rule commits width, then degree).
|
|
148
|
+
Returns counter increments (``or_drop``, ``and_add``, ...)."""
|
torch_type_nn/py.typed
ADDED
|
File without changes
|