torch-type-nn 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
torch_type_nn/mlp.py ADDED
@@ -0,0 +1,251 @@
1
+ """An MLP whose width and depth can change while it trains.
2
+
3
+ ``ScalableMLP`` is ``Linear -> ReLU -> ... -> Linear`` with an optional
4
+ type-nn readout ``F(u) = sign(u) ln(1 + |u|)`` on the output (so it can be
5
+ compared with the type-nn board's MLP baseline). Its layers are
6
+ :class:`TrackedLinear`: an ``nn.Linear`` that can add or drop output units and
7
+ input columns, reports each parameter replacement as an
8
+ :class:`~torch_type_nn.edits.Edit`, and collects the input statistics the
9
+ scaler needs. Grow and prune it with
10
+ :class:`~torch_type_nn.scaling.StructureScaler`, which wraps it in
11
+ :class:`~torch_type_nn.adapters.mlp.MLPAdapter`.
12
+
13
+ The identities it offers (Net2Net, Chen et al. 2016):
14
+
15
+ * width: a new hidden unit read with an all-zero column is exact;
16
+ * depth: after a ReLU the activations are >= 0, so ``ReLU(I a + 0) = a``;
17
+ an identity layer is exact in any gap between hidden layers (but not in
18
+ front of the first layer, whose input may be negative).
19
+ """
20
+
21
+ from __future__ import annotations
22
+
23
+ import math
24
+ import re
25
+
26
+ import torch
27
+ from torch import nn
28
+
29
+ from .edits import Edit, grow_index, keep_index
30
+ from .functional import readout
31
+ from .layer import uniform
32
+
33
+ __all__ = ["TrackedLinear", "ScalableMLP"]
34
+
35
+
36
+ class TrackedLinear(nn.Linear):
37
+ """``nn.Linear`` with structural edits, edit reporting and input statistics."""
38
+
39
+ def __init__(self, in_features: int, out_features: int, *,
40
+ generator: torch.Generator | None = None, device=None, dtype=None) -> None:
41
+ super().__init__(in_features, out_features, device=device, dtype=dtype)
42
+ self.probe = False
43
+ self.born = 0
44
+ self.track_stats = False
45
+ self.edit_sink: list | None = None
46
+ f = {"device": device, "dtype": self.weight.dtype}
47
+ self.register_buffer("unit_probe", torch.zeros(out_features, dtype=torch.bool,
48
+ device=device))
49
+ self.register_buffer("unit_born", torch.zeros(out_features, dtype=torch.long,
50
+ device=device))
51
+ self.register_buffer("stat_sx", torch.zeros(in_features, **f), persistent=False)
52
+ self.register_buffer("stat_gin", torch.zeros((), **f), persistent=False)
53
+ self.register_buffer("stat_ns", torch.zeros((), **f), persistent=False)
54
+ if generator is not None:
55
+ self.reset_from(generator)
56
+
57
+ @torch.no_grad()
58
+ def reset_from(self, generator: torch.Generator) -> None:
59
+ """The ``nn.Linear`` law U(+-1/sqrt(n)), drawn from ``generator``."""
60
+ k = 1.0 / math.sqrt(max(self.in_features, 1))
61
+ self.weight.copy_(uniform(self.weight.shape, k, generator, self.weight))
62
+ self.bias.copy_(uniform(self.bias.shape, k, generator, self.bias))
63
+
64
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
65
+ if self.track_stats and self.training and torch.is_grad_enabled():
66
+ x = self._observe(x)
67
+ return super().forward(x)
68
+
69
+ def _observe(self, x: torch.Tensor) -> torch.Tensor:
70
+ n = self.in_features
71
+ with torch.no_grad():
72
+ xf = x.detach().reshape(-1, n)
73
+ self.stat_sx.add_(xf.sum(0))
74
+ self.stat_ns.add_(xf.shape[0])
75
+ if not x.requires_grad:
76
+ x = x.detach().requires_grad_(True)
77
+ gin = self.stat_gin
78
+
79
+ def hook(g: torch.Tensor) -> None:
80
+ with torch.no_grad():
81
+ gin.add_(g.reshape(-1, n).abs().mean(1).sum())
82
+
83
+ x.register_hook(hook)
84
+ return x
85
+
86
+ @torch.no_grad()
87
+ def reset_stats(self) -> None:
88
+ for t in (self.stat_sx, self.stat_gin, self.stat_ns):
89
+ t.zero_()
90
+
91
+ def num_params(self) -> int:
92
+ return self.out_features * (self.in_features + 1)
93
+
94
+ def extra_repr(self) -> str:
95
+ return super().extra_repr() + (", probe" if self.probe else "")
96
+
97
+ # -------------------------------------------------------- structure edits
98
+
99
+ def _set(self, name: str, value: torch.Tensor, dim: int, index: torch.Tensor) -> None:
100
+ if name in self._parameters:
101
+ old = self._parameters[name]
102
+ new = nn.Parameter(value.contiguous())
103
+ self._parameters[name] = new
104
+ if self.edit_sink is not None:
105
+ self.edit_sink.append(Edit(old, new, dim, index))
106
+ else:
107
+ self._buffers[name] = value.contiguous()
108
+
109
+ @torch.no_grad()
110
+ def add_out(self, weight: torch.Tensor, bias: float, *, probe: bool = False,
111
+ born: int = 0) -> int:
112
+ """Append an output unit; returns its index."""
113
+ m = self.out_features
114
+ idx = grow_index(m, self.weight.device)
115
+ self._set("weight", torch.cat([self.weight.detach(), weight.reshape(1, -1)], 0), 0, idx)
116
+ self._set("bias", torch.cat([self.bias.detach(), self.bias.new_full((1,), bias)]), 0, idx)
117
+ self._set("unit_probe", torch.cat([self.unit_probe, self.unit_probe.new_full((1,), probe)]),
118
+ 0, idx)
119
+ self._set("unit_born", torch.cat([self.unit_born, self.unit_born.new_full((1,), born)]),
120
+ 0, idx)
121
+ self.out_features = m + 1
122
+ return m
123
+
124
+ @torch.no_grad()
125
+ def drop_out(self, k: int) -> None:
126
+ keep = _without(self.out_features, k, self.weight.device)
127
+ for name in ("weight", "bias", "unit_probe", "unit_born"):
128
+ self._set(name, getattr(self, name).detach().index_select(0, keep), 0,
129
+ keep_index(keep))
130
+ self.out_features -= 1
131
+
132
+ @torch.no_grad()
133
+ def add_in(self) -> None:
134
+ """A new input column of zeros: the function is unchanged."""
135
+ idx = grow_index(self.in_features, self.weight.device)
136
+ w = self.weight.detach()
137
+ self._set("weight", torch.cat([w, torch.zeros_like(w[:, :1])], 1), 1, idx)
138
+ self.in_features += 1
139
+ self._resize_stats()
140
+
141
+ @torch.no_grad()
142
+ def drop_in(self, j: int) -> None:
143
+ keep = _without(self.in_features, j, self.weight.device)
144
+ self._set("weight", self.weight.detach().index_select(1, keep), 1, keep_index(keep))
145
+ self.in_features -= 1
146
+ self._resize_stats()
147
+
148
+ def _resize_stats(self) -> None:
149
+ self._buffers["stat_sx"] = self.weight.new_zeros(self.in_features)
150
+ self.reset_stats()
151
+
152
+ # ------------------------------------------------------------ persistence
153
+
154
+ def get_extra_state(self):
155
+ return {"probe": self.probe, "born": self.born}
156
+
157
+ def set_extra_state(self, state) -> None:
158
+ self.probe = bool(state.get("probe", False))
159
+ self.born = int(state.get("born", 0))
160
+
161
+ def _load_from_state_dict(self, state_dict, prefix, *args, **kwargs):
162
+ w = state_dict.get(prefix + "weight")
163
+ if w is not None and w.shape != self.weight.shape:
164
+ m, n = w.shape
165
+ for name, t in [*self._parameters.items(), *self._buffers.items()]:
166
+ src = state_dict.get(prefix + name)
167
+ if src is not None:
168
+ new = torch.empty(src.shape, dtype=t.dtype, device=t.device)
169
+ if name in self._parameters:
170
+ self._parameters[name] = nn.Parameter(new)
171
+ else:
172
+ self._buffers[name] = new
173
+ self.in_features, self.out_features = n, m
174
+ self._resize_stats()
175
+ super()._load_from_state_dict(state_dict, prefix, *args, **kwargs)
176
+
177
+
178
+ class ScalableMLP(nn.Module):
179
+ """``Linear -> ReLU -> ... -> Linear [-> F]`` whose width and depth can change.
180
+
181
+ Args:
182
+ in_features, out_features: task widths (fixed for life).
183
+ hidden: widths of the hidden layers at birth; default one hidden layer
184
+ of ``max(2, out_features)`` units: small, since the scaler grows it.
185
+ readout: apply ``F(u) = sign(u) ln(1 + |u|)`` to the output, as the
186
+ type-nn board's MLP baseline does.
187
+ seed: seeds ``self.generator`` (initialisation and structural draws).
188
+ """
189
+
190
+ def __init__(self, in_features: int, out_features: int, hidden: list[int] | None = None,
191
+ *, readout: bool = True, seed: int = 1, device=None, dtype=None) -> None:
192
+ super().__init__()
193
+ self.in_features, self.out_features = int(in_features), int(out_features)
194
+ self.readout = readout
195
+ self.generator = torch.Generator().manual_seed(int(seed))
196
+ hidden = [max(2, self.out_features)] if hidden is None else list(hidden)
197
+ widths = [self.in_features, *hidden, self.out_features]
198
+ self.linears = nn.ModuleList(
199
+ TrackedLinear(a, b, generator=self.generator, device=device, dtype=dtype)
200
+ for a, b in zip(widths[:-1], widths[1:], strict=True))
201
+
202
+ @property
203
+ def depth(self) -> int:
204
+ return len(self.linears)
205
+
206
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
207
+ last = len(self.linears) - 1
208
+ for i, lin in enumerate(self.linears):
209
+ x = lin(x)
210
+ if i < last:
211
+ x = torch.relu(x)
212
+ return readout(x) if self.readout else x
213
+
214
+ def num_params(self) -> int:
215
+ return sum(l.num_params() for l in self.linears)
216
+
217
+ def structure(self) -> list[int]:
218
+ """Hidden widths."""
219
+ return [l.out_features for l in self.linears[:-1]]
220
+
221
+ def extra_repr(self) -> str:
222
+ return (f"in_features={self.in_features}, out_features={self.out_features}, "
223
+ f"hidden={self.structure()}, params={self.num_params()}")
224
+
225
+ def __getstate__(self):
226
+ state = self.__dict__.copy()
227
+ state["_generator_state"] = state.pop("generator").get_state()
228
+ return state
229
+
230
+ def __setstate__(self, state):
231
+ gen_state = state.pop("_generator_state", None)
232
+ self.__dict__.update(state)
233
+ self.generator = torch.Generator()
234
+ if gen_state is not None:
235
+ self.generator.set_state(gen_state)
236
+
237
+ def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
238
+ """Accepts a checkpoint of any structure."""
239
+ idx = {int(m.group(1)) for k in state_dict
240
+ if (m := re.match(r"linears\.(\d+)\.weight$", k))}
241
+ D = max(idx) + 1 if idx else 0
242
+ ref = self.linears[0].weight
243
+ while len(self.linears) < D:
244
+ self.linears.append(TrackedLinear(1, 1, device=ref.device, dtype=ref.dtype))
245
+ while len(self.linears) > D:
246
+ del self.linears[len(self.linears) - 1]
247
+ return super().load_state_dict(state_dict, strict=strict, assign=assign)
248
+
249
+
250
+ def _without(n: int, i: int, device) -> torch.Tensor:
251
+ return torch.cat([torch.arange(i, device=device), torch.arange(i + 1, n, device=device)])
@@ -0,0 +1,115 @@
1
+ """A stack of type-nn layers whose depth, widths and degrees can change."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ import re
7
+
8
+ import torch
9
+ from torch import nn
10
+
11
+ from .layer import AndOr
12
+
13
+ __all__ = ["TypeNN", "birth_depth"]
14
+
15
+
16
+ def birth_depth(in_features: int, out_features: int) -> int:
17
+ """round(ln(1 + n m)), at least 1: xor 1, iris 3, wine 4, wdbc 3, diabetes 2, ionosphere 4."""
18
+ # C lround: half away from zero (Python round() is half-to-even)
19
+ return max(1, int(math.floor(math.log(1.0 + in_features * out_features) + 0.5)))
20
+
21
+
22
+ class TypeNN(nn.Module):
23
+ """A dense stack of :class:`AndOr` layers, ``n -> m -> ... -> m``.
24
+
25
+ At birth every unit is a single random Or (a degree-1 And) and there are
26
+ :func:`birth_depth` layers. Nothing about the structure is a
27
+ hyper-parameter: train it with :class:`~torch_type_nn.scaling.StructureScaler`
28
+ (or :func:`~torch_type_nn.train.fit`) and width, degree and depth are
29
+ grown early and pruned late. Without a scaler it is an ordinary static
30
+ network.
31
+
32
+ Args:
33
+ in_features, out_features: task widths (fixed for life).
34
+ depth: birth depth; default :func:`birth_depth`.
35
+ seed: seeds ``self.generator``, used for the birth draw and for
36
+ every random draw of structural edits.
37
+ """
38
+
39
+ def __init__(self, in_features: int, out_features: int, *, depth: int | None = None,
40
+ seed: int = 1, device=None, dtype=None) -> None:
41
+ super().__init__()
42
+ self.in_features, self.out_features = int(in_features), int(out_features)
43
+ self.generator = torch.Generator().manual_seed(int(seed))
44
+ D = birth_depth(in_features, out_features) if depth is None else int(depth)
45
+ self.birth_depth = D
46
+ layers, n = [], self.in_features
47
+ for _ in range(D):
48
+ layers.append(AndOr(n, self.out_features, 1, generator=self.generator,
49
+ device=device, dtype=dtype))
50
+ n = self.out_features
51
+ self.layers = nn.ModuleList(layers)
52
+
53
+ @property
54
+ def depth(self) -> int:
55
+ return len(self.layers)
56
+
57
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
58
+ for layer in self.layers:
59
+ x = layer(x)
60
+ return x
61
+
62
+ def num_params(self) -> int:
63
+ """Every learnable scalar present: (n_in + 2) per live Or, probes included."""
64
+ return sum(layer.num_params() for layer in self.layers)
65
+
66
+ def insert_layer(self, index: int, layer: AndOr) -> None:
67
+ self.layers.insert(index, layer)
68
+
69
+ def remove_layer(self, index: int) -> AndOr:
70
+ layer = self.layers[index]
71
+ del self.layers[index]
72
+ return layer
73
+
74
+ def structure(self) -> list[list[int]]:
75
+ """Live Ors per unit, per layer."""
76
+ return [layer.num_ors().tolist() for layer in self.layers]
77
+
78
+ def extra_repr(self) -> str:
79
+ return (f"in_features={self.in_features}, out_features={self.out_features}, "
80
+ f"depth={self.depth}, params={self.num_params()}")
81
+
82
+ def get_extra_state(self):
83
+ return {"birth_depth": self.birth_depth, "generator": self.generator.get_state()}
84
+
85
+ def set_extra_state(self, state) -> None:
86
+ self.birth_depth = int(state.get("birth_depth", self.birth_depth))
87
+ if "generator" in state:
88
+ self.generator.set_state(state["generator"])
89
+
90
+ # torch.Generator does not pickle: carry its state instead
91
+ def __getstate__(self):
92
+ state = self.__dict__.copy()
93
+ gen = state.pop("generator")
94
+ state["_generator_state"] = gen.get_state()
95
+ return state
96
+
97
+ def __setstate__(self, state):
98
+ gen_state = state.pop("_generator_state", None)
99
+ self.__dict__.update(state)
100
+ self.generator = torch.Generator()
101
+ if gen_state is not None:
102
+ self.generator.set_state(gen_state)
103
+
104
+ def load_state_dict(self, state_dict, strict: bool = True, assign: bool = False):
105
+ """Accepts a checkpoint of any structure: depth is taken from the keys,
106
+ widths and degrees from the tensors."""
107
+ idx = {int(m.group(1)) for k in state_dict
108
+ if (m := re.match(r"layers\.(\d+)\.weight$", k))}
109
+ D = max(idx) + 1 if idx else 0
110
+ ref = self.layers[0].weight if len(self.layers) else torch.empty(0)
111
+ while len(self.layers) < D:
112
+ self.layers.append(AndOr(1, 1, 1, device=ref.device, dtype=ref.dtype))
113
+ while len(self.layers) > D:
114
+ del self.layers[len(self.layers) - 1]
115
+ return super().load_state_dict(state_dict, strict=strict, assign=assign)
torch_type_nn/optim.py ADDED
@@ -0,0 +1,122 @@
1
+ """Adam with one step counter per Or.
2
+
3
+ type-nn gives each Or its own Adam bias correction: an Or born late starts
4
+ at t = 1, so its first steps are corrected exactly like those of an Or that
5
+ existed from the start. Its moments live in the layer (``AndOr.adam_*``) so
6
+ structural edits carry them along.
7
+
8
+ After every step the assembly index is projected back to ``a >= 1`` and
9
+ free slots stay the identity Or.
10
+
11
+ Parameters that do not belong to an :class:`~torch_type_nn.layer.AndOr`
12
+ (say, a ``nn.Linear`` head in the same model) get ordinary Adam, so one
13
+ ``TypeAdam`` can drive a mixed model.
14
+ """
15
+
16
+ from __future__ import annotations
17
+
18
+ import torch
19
+ from torch import nn
20
+
21
+ from .layer import AndOr
22
+
23
+ __all__ = ["TypeAdam", "keep_invariants"]
24
+
25
+
26
+ def keep_invariants(optimizer: torch.optim.Optimizer, module: nn.Module):
27
+ """Make a stock optimizer respect the architecture: after every
28
+ ``optimizer.step()`` each :class:`AndOr` in ``module`` is projected back to
29
+ ``a >= 1`` with its free slots the identity Or (``AndOr.project_``).
30
+ :class:`TypeAdam` does this itself. Returns the hook handle; calling it
31
+ twice on one optimizer attaches one hook.
32
+ """
33
+ if isinstance(optimizer, TypeAdam) or getattr(optimizer, "_tnn_projects", None) is module:
34
+ return None
35
+
36
+ def project(opt, args, kwargs):
37
+ for layer in module.modules():
38
+ if isinstance(layer, AndOr):
39
+ layer.project_()
40
+
41
+ handle = optimizer.register_step_post_hook(project)
42
+ optimizer._tnn_projects = module
43
+ return handle
44
+
45
+
46
+ class TypeAdam(torch.optim.Optimizer):
47
+ """Adam whose step count is per Or, for models that contain :class:`AndOr` layers.
48
+
49
+ Unlike stock optimizers, ``TypeAdam`` takes the *module*, not a parameter
50
+ list: structural edits replace parameter tensors, and the optimizer
51
+ re-reads them from the module on every step.
52
+
53
+ Args:
54
+ module: the model.
55
+ lr: learning rate.
56
+ betas, eps: as in :class:`torch.optim.Adam`.
57
+ """
58
+
59
+ def __init__(self, module: nn.Module, lr: float = 1e-3,
60
+ betas: tuple[float, float] = (0.9, 0.999), eps: float = 1e-8) -> None:
61
+ if lr < 0:
62
+ raise ValueError(f"invalid lr {lr}")
63
+ self.module = module
64
+ super().__init__(list(module.parameters()), {"lr": lr, "betas": betas, "eps": eps})
65
+
66
+ def _sync(self) -> None:
67
+ self.param_groups[0]["params"] = list(self.module.parameters())
68
+
69
+ def zero_grad(self, set_to_none: bool = True) -> None:
70
+ self._sync()
71
+ super().zero_grad(set_to_none)
72
+
73
+ @torch.no_grad()
74
+ def step(self, closure=None):
75
+ loss = None
76
+ if closure is not None:
77
+ with torch.enable_grad():
78
+ loss = closure()
79
+ self._sync()
80
+ group = self.param_groups[0]
81
+ lr, (b1, b2), eps = group["lr"], group["betas"], group["eps"]
82
+ owned = set()
83
+ for layer in self.module.modules():
84
+ if isinstance(layer, AndOr):
85
+ _step_layer(layer, lr, b1, b2, eps)
86
+ owned.update(id(p) for p in layer.parameters(recurse=False))
87
+ for p in group["params"]:
88
+ if id(p) in owned or p.grad is None:
89
+ continue
90
+ st = self.state[p]
91
+ if not st:
92
+ st["step"] = 0
93
+ st["exp_avg"] = torch.zeros_like(p)
94
+ st["exp_avg_sq"] = torch.zeros_like(p)
95
+ st["step"] += 1
96
+ t = st["step"]
97
+ st["exp_avg"].mul_(b1).add_(p.grad, alpha=1 - b1)
98
+ st["exp_avg_sq"].mul_(b2).addcmul_(p.grad, p.grad, value=1 - b2)
99
+ c1, c2 = 1.0 / (1.0 - b1 ** t), 1.0 / (1.0 - b2 ** t)
100
+ p.sub_(lr * (st["exp_avg"] * c1) / ((st["exp_avg_sq"] * c2).sqrt() + eps))
101
+ return loss
102
+
103
+
104
+ def _step_layer(layer: AndOr, lr: float, b1: float, b2: float, eps: float) -> None:
105
+ live = layer.mask
106
+ layer.adam_step.add_(live.long())
107
+ t = layer.adam_step.clamp(min=1).to(layer.weight.dtype)
108
+ c1 = 1.0 / (1.0 - b1 ** t) # (m, R)
109
+ c2 = 1.0 / (1.0 - b2 ** t)
110
+ livef = live.to(layer.weight.dtype)
111
+ for p, m, v, cc1, cc2, lf in (
112
+ (layer.weight, layer.adam_m_weight, layer.adam_v_weight,
113
+ c1.unsqueeze(-1), c2.unsqueeze(-1), livef.unsqueeze(-1)),
114
+ (layer.bias, layer.adam_m_bias, layer.adam_v_bias, c1, c2, livef),
115
+ (layer.assembly, layer.adam_m_assembly, layer.adam_v_assembly, c1, c2, livef),
116
+ ):
117
+ g = p.grad if p.grad is not None else torch.zeros_like(p)
118
+ g = g * lf
119
+ m.mul_(b1).add_(g, alpha=1 - b1)
120
+ v.mul_(b2).addcmul_(g, g, value=1 - b2)
121
+ p.sub_(lf * lr * (m * cc1) / ((v * cc2).sqrt() + eps))
122
+ layer.assembly.clamp_(min=1.0) # at least one copy
@@ -0,0 +1,148 @@
1
+ """The contract between the structure scaler and a model family.
2
+
3
+ type-nn's scaling rule is architecture-agnostic: on each axis a *probe* is
4
+ an exact identity that back-prop may move; a probe that moved past
5
+ ``theta(T)`` is promoted (under ``rule="bic"``: and pays its BIC price); late
6
+ in training items back inside the identity band are removed (under
7
+ ``rule="bic"``: items are ablated and removed while the model criterion
8
+ allows).
9
+ What is specific to an architecture is only *what* an item is: how to
10
+ insert an identity, how far an item is from it, how to reset it and how to
11
+ remove it. That is this protocol.
12
+
13
+ Items live on three axes:
14
+
15
+ ``WIDTH``
16
+ a coordinate between two layers (a unit and the weights that read it)
17
+ ``DEGREE``
18
+ a factor inside a unit (for type-nn: an Or inside an And)
19
+ ``DEPTH``
20
+ a layer (there is at most one depth probe at a time)
21
+
22
+ An adapter may leave an axis empty (return no sites or candidates).
23
+ :class:`~torch_type_nn.adapters.typenn.TypeNNAdapter` is the reference
24
+ implementation.
25
+ """
26
+
27
+ from __future__ import annotations
28
+
29
+ from collections.abc import Callable, Hashable, Sequence
30
+ from dataclasses import dataclass
31
+ from typing import Protocol, runtime_checkable
32
+
33
+ import torch
34
+ from torch import nn
35
+
36
+ __all__ = ["WIDTH", "DEGREE", "DEPTH", "AXES", "Item", "Trial", "EditContext", "Scalable"]
37
+
38
+ WIDTH, DEGREE, DEPTH = "width", "degree", "depth"
39
+ AXES = (WIDTH, DEGREE, DEPTH)
40
+
41
+
42
+ @dataclass(frozen=True)
43
+ class Item:
44
+ """One structural item: an axis and an adapter-defined address.
45
+
46
+ Addresses are only valid until the next structural edit.
47
+ """
48
+
49
+ axis: str
50
+ address: tuple
51
+
52
+
53
+ @dataclass
54
+ class Trial:
55
+ """A structural edit made tentatively: keep it with ``commit`` or revert it with ``undo``."""
56
+
57
+ undo: Callable[[], None]
58
+ commit: Callable[[], None]
59
+
60
+
61
+ @dataclass
62
+ class EditContext:
63
+ """What a new probe needs: its birth step, noise amplitude, and the RNG."""
64
+
65
+ step: int
66
+ noise: float
67
+ generator: torch.Generator
68
+
69
+
70
+ @runtime_checkable
71
+ class Scalable(Protocol):
72
+ """A model family the :class:`~torch_type_nn.scaling.StructureScaler` can grow and prune."""
73
+
74
+ model: nn.Module
75
+
76
+ # ------------------------------------------------------------ bookkeeping
77
+ def num_params(self) -> int:
78
+ """Every learnable scalar currently present (probes included)."""
79
+
80
+ def set_tracking(self, on: bool) -> None:
81
+ """Collect (or stop collecting) training statistics in forward passes."""
82
+
83
+ def reset_stats(self) -> None:
84
+ """Forget the statistics of the epoch that ended."""
85
+
86
+ def set_edit_sink(self, sink: list | None) -> None:
87
+ """Report parameter replacements (``Edit``) to ``sink`` while it is a list."""
88
+
89
+ def finalize(self) -> None:
90
+ """Training is over; tidy storage (e.g. drop free padding)."""
91
+
92
+ # ----------------------------------------------------------------- probes
93
+ def probe_sites(self, axis: str) -> Sequence[Hashable]:
94
+ """Where probes of ``axis`` live now (WIDTH and DEGREE), in visiting order."""
95
+
96
+ def probe_at(self, axis: str, site: Hashable | None = None) -> Item | None:
97
+ """The probe at ``site`` (for DEPTH: the depth probe, ``site`` ignored)."""
98
+
99
+ def add_probe(self, axis: str, site: Hashable, ctx: EditContext) -> None:
100
+ """Insert an exact identity (up to ``ctx.noise``) at ``site``."""
101
+
102
+ def promote(self, item: Item) -> None:
103
+ """The probe becomes a live item."""
104
+
105
+ def born(self, item: Item) -> int:
106
+ """Training step at which ``item`` was inserted."""
107
+
108
+ def displacement(self, item: Item) -> float:
109
+ """RMS distance of ``item`` from its identity (``inf`` if it has none)."""
110
+
111
+ def best_site(self, axis: str, moving: Item | None = None) -> Hashable:
112
+ """Where a new probe of ``axis`` is most wanted (DEPTH: the loudest gap,
113
+ counted as if ``moving`` were absent)."""
114
+
115
+ def site_of(self, item: Item) -> Hashable:
116
+ """The site a probe occupies, in :meth:`best_site` coordinates."""
117
+
118
+ def remove_probe(self, item: Item) -> None:
119
+ """Take a probe out (near-exactly) so it can move elsewhere."""
120
+
121
+ # -------------------------------------------------- measurement / pruning
122
+ def ablate(self, item: Item) -> Callable[[], None] | None:
123
+ """Reset ``item`` to its identity in place; return the exact undo
124
+ (``None`` if the item has no identity)."""
125
+
126
+ def cost(self, item: Item) -> int:
127
+ """Parameters owned by ``item``."""
128
+
129
+ def trial_remove(self, item: Item) -> Trial | None:
130
+ """Remove ``item`` (DEPTH) tentatively; ``None`` if it cannot be removed."""
131
+
132
+ def removal_order(self, axis: str) -> Sequence[Item]:
133
+ """Live items of ``axis`` to try removing one at a time (DEPTH)."""
134
+
135
+ def prune_candidates(self) -> Sequence[Item]:
136
+ """Items (WIDTH and DEGREE) for the joint ablation pool, in a fixed order."""
137
+
138
+ def can_remove(self, item: Item, removed: Sequence[Item]) -> bool:
139
+ """Whether ``item`` may join the ablated set ``removed`` (keep-one rules)."""
140
+
141
+ def params_without(self, removed: Sequence[Item]) -> int:
142
+ """Exact parameter count of the model with ``removed`` taken out."""
143
+
144
+ def commit_removals(self, removed: Sequence[Item],
145
+ axes: Sequence[str] = (WIDTH, DEGREE)) -> dict[str, int]:
146
+ """Remove ``removed`` for real and clear the surviving probe flags, on the
147
+ given ``axes`` only (the threshold rule commits width, then degree).
148
+ Returns counter increments (``or_drop``, ``and_add``, ...)."""
torch_type_nn/py.typed ADDED
File without changes