wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/ir/semantics.py
ADDED
|
@@ -0,0 +1,1255 @@
|
|
|
1
|
+
"""Semantics derived from the op graph (§5.1.2). Nothing here reads parameter names.
|
|
2
|
+
|
|
3
|
+
The pass works on the prefill graph:
|
|
4
|
+
|
|
5
|
+
1. **Residual stream.** The first ``x + f(x)`` add fixes where the stream starts;
|
|
6
|
+
from there each step takes the next add with the current stream value as one
|
|
7
|
+
operand. Blocks are groups of consecutive residual adds executed by the same
|
|
8
|
+
module, numbered by position along the stream.
|
|
9
|
+
2. **Writers and readers.** A branch's writers are the parameter projections
|
|
10
|
+
reached walking back from its add operand (following full-shape data only, so
|
|
11
|
+
routing weights and gates do not count). Its readers are the projections
|
|
12
|
+
reached walking forward from the stream value.
|
|
13
|
+
3. **Mixer type.** Each writer's sub-branch is attention if it contains a
|
|
14
|
+
fused attention op or a softmax over an activation-activation product, an SSM
|
|
15
|
+
or linear attention if it mixes along the sequence without softmax, MoE if it
|
|
16
|
+
contains a router ``topk``, and an MLP otherwise.
|
|
17
|
+
4. **Roles.** q/k/v come from which projection features reach which attention
|
|
18
|
+
argument; gate/up from which features pass through the activation function;
|
|
19
|
+
fused weights get sub-ranges straight from the tape's feature provenance.
|
|
20
|
+
"""
|
|
21
|
+
|
|
22
|
+
from __future__ import annotations
|
|
23
|
+
|
|
24
|
+
from collections import defaultdict
|
|
25
|
+
from collections.abc import Callable, Iterable
|
|
26
|
+
from dataclasses import asdict, dataclass, field
|
|
27
|
+
from typing import Any
|
|
28
|
+
|
|
29
|
+
from wspx.lab.ir.graph import Node, OpGraph
|
|
30
|
+
from wspx.lab.ir.tape import ACTIVATION_OPS, CONV_OPS, MATMUL_OPS, SOFTMAX_OPS
|
|
31
|
+
|
|
32
|
+
ALIAS_OPS = frozenset(
|
|
33
|
+
{
|
|
34
|
+
"view",
|
|
35
|
+
"_unsafe_view",
|
|
36
|
+
"reshape",
|
|
37
|
+
"clone",
|
|
38
|
+
"contiguous",
|
|
39
|
+
"_to_copy",
|
|
40
|
+
"detach",
|
|
41
|
+
"alias",
|
|
42
|
+
"unsqueeze",
|
|
43
|
+
"squeeze",
|
|
44
|
+
"lift_fresh",
|
|
45
|
+
"flatten",
|
|
46
|
+
"unflatten",
|
|
47
|
+
"view_as",
|
|
48
|
+
"native_dropout",
|
|
49
|
+
"dropout",
|
|
50
|
+
}
|
|
51
|
+
)
|
|
52
|
+
RNN_OPS = frozenset({"lstm", "gru", "rnn_tanh", "rnn_relu", "_cudnn_rnn", "mkldnn_rnn"})
|
|
53
|
+
SEQ_MIX_OPS = frozenset({"cumsum", "cumprod", "_cummax_helper", "logcumsumexp"})
|
|
54
|
+
RNN_GATES = {
|
|
55
|
+
"lstm": ("input", "forget", "cell", "output"),
|
|
56
|
+
"gru": ("reset", "update", "new"),
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
@dataclass
|
|
61
|
+
class Role:
|
|
62
|
+
address: str
|
|
63
|
+
"""Short address without slice, e.g. ``L3.attn.q`` or ``embed``."""
|
|
64
|
+
param: str
|
|
65
|
+
kind: str
|
|
66
|
+
"""reader, writer, norm, embed, pos_embed, stem, unembed, final_norm, router, expert, bias, aux, layer."""
|
|
67
|
+
block: int | None = None
|
|
68
|
+
branch: str | None = None
|
|
69
|
+
module: str = ""
|
|
70
|
+
in_dim: int | None = None
|
|
71
|
+
out_dim: int | None = None
|
|
72
|
+
split: list[list[int]] | None = None
|
|
73
|
+
"""Runs ``[start, stop, step]`` along ``out_dim`` for a sub-role of a fused weight."""
|
|
74
|
+
expert: int | None = None
|
|
75
|
+
"""Index along dim 0 when this role is one expert of a 3-D expert weight."""
|
|
76
|
+
experts: int | None = None
|
|
77
|
+
gates: list[str] | None = None
|
|
78
|
+
"""RNN gate order along the stacked dim (PyTorch's documented layout)."""
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
@dataclass
|
|
82
|
+
class Block:
|
|
83
|
+
index: int
|
|
84
|
+
module: str
|
|
85
|
+
branches: list[str]
|
|
86
|
+
residual_adds: list[int]
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass
|
|
90
|
+
class Semantics:
|
|
91
|
+
roles: list[Role] = field(default_factory=list)
|
|
92
|
+
blocks: list[Block] = field(default_factory=list)
|
|
93
|
+
stream: list[int] = field(default_factory=list)
|
|
94
|
+
embed: str | None = None
|
|
95
|
+
unembed: str | None = None
|
|
96
|
+
tied: bool = False
|
|
97
|
+
final_norm: list[str] = field(default_factory=list)
|
|
98
|
+
readers: list[str] = field(default_factory=list)
|
|
99
|
+
writers: list[str] = field(default_factory=list)
|
|
100
|
+
unassigned: list[str] = field(default_factory=list)
|
|
101
|
+
head_dims: dict[str, int] = field(default_factory=dict)
|
|
102
|
+
"""Per-head width of attention inputs, by role address (``L3.attn.q`` → 128)."""
|
|
103
|
+
notes: list[str] = field(default_factory=list)
|
|
104
|
+
|
|
105
|
+
def to_dict(self) -> dict[str, Any]:
|
|
106
|
+
return asdict(self)
|
|
107
|
+
|
|
108
|
+
@classmethod
|
|
109
|
+
def from_dict(cls, data: dict[str, Any]) -> Semantics:
|
|
110
|
+
data = dict(data)
|
|
111
|
+
data["roles"] = [Role(**r) for r in data.get("roles", [])]
|
|
112
|
+
data["blocks"] = [Block(**b) for b in data.get("blocks", [])]
|
|
113
|
+
return cls(**data)
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def base_op(node: Node) -> str:
|
|
117
|
+
name = node.short_op
|
|
118
|
+
return name[:-1] if name.endswith("_") and not name.startswith("_") else name
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
class _Graph:
|
|
122
|
+
"""Read-only helpers over an :class:`OpGraph`."""
|
|
123
|
+
|
|
124
|
+
def __init__(self, graph: OpGraph, opaque: tuple[str, ...] = ()) -> None:
|
|
125
|
+
self.g = graph
|
|
126
|
+
self.cons = graph.consumers
|
|
127
|
+
self._param_anc: dict[int, bool] = {}
|
|
128
|
+
self.opaque = opaque
|
|
129
|
+
"""Module paths whose internals are handled as a unit (recurrent layers)."""
|
|
130
|
+
|
|
131
|
+
def in_opaque(self, node: Node) -> str | None:
|
|
132
|
+
for prefix in self.opaque:
|
|
133
|
+
if node.module == prefix or node.module.startswith(prefix + "."):
|
|
134
|
+
return prefix
|
|
135
|
+
return None
|
|
136
|
+
|
|
137
|
+
def node(self, idx: int) -> Node:
|
|
138
|
+
return self.g.nodes[idx]
|
|
139
|
+
|
|
140
|
+
def producer_idx(self, vid: int) -> int:
|
|
141
|
+
return self.g.values[vid].producer
|
|
142
|
+
|
|
143
|
+
def shape(self, vid: int) -> tuple[int, ...]:
|
|
144
|
+
return self.g.values[vid].shape
|
|
145
|
+
|
|
146
|
+
def floating(self, vid: int) -> bool:
|
|
147
|
+
dtype = self.g.values[vid].dtype
|
|
148
|
+
return "float" in dtype or "bfloat" in dtype or "complex" in dtype
|
|
149
|
+
|
|
150
|
+
def float_args(self, node: Node) -> list[tuple[int, int]]:
|
|
151
|
+
return [(p, v) for p, v in node.activation_args if self.floating(v)]
|
|
152
|
+
|
|
153
|
+
def is_proj(self, node: Node) -> bool:
|
|
154
|
+
op = base_op(node)
|
|
155
|
+
return (
|
|
156
|
+
bool(node.params)
|
|
157
|
+
and (op in MATMUL_OPS or op in CONV_OPS)
|
|
158
|
+
and bool(node.activation_args)
|
|
159
|
+
and self.in_opaque(node) is None
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
def is_actact(self, node: Node) -> bool:
|
|
163
|
+
return base_op(node) in MATMUL_OPS and not node.params and len(self.float_args(node)) >= 2
|
|
164
|
+
|
|
165
|
+
def is_attention(self, node: Node) -> bool:
|
|
166
|
+
return "scaled_dot_product" in base_op(node) or base_op(node).startswith("_flash_attention")
|
|
167
|
+
|
|
168
|
+
def aliases(self, vid: int) -> set[int]:
|
|
169
|
+
out = {vid}
|
|
170
|
+
stack = [vid]
|
|
171
|
+
while stack:
|
|
172
|
+
v = stack.pop()
|
|
173
|
+
numel = _numel(self.shape(v))
|
|
174
|
+
for nidx, _ in self.cons.get(v, []):
|
|
175
|
+
node = self.node(nidx)
|
|
176
|
+
if base_op(node) in ALIAS_OPS and len(node.activation_args) == 1 and node.outs:
|
|
177
|
+
o = node.outs[0]
|
|
178
|
+
if o >= 0 and o not in out and _numel(self.shape(o)) == numel:
|
|
179
|
+
out.add(o)
|
|
180
|
+
stack.append(o)
|
|
181
|
+
return out
|
|
182
|
+
|
|
183
|
+
def reach(self, vids: Iterable[int], *, until: int | None = None) -> set[int]:
|
|
184
|
+
"""Values forward-reachable from ``vids`` through nodes with index < ``until``."""
|
|
185
|
+
seen = set(vids)
|
|
186
|
+
stack = list(seen)
|
|
187
|
+
while stack:
|
|
188
|
+
v = stack.pop()
|
|
189
|
+
for nidx, _ in self.cons.get(v, []):
|
|
190
|
+
if until is not None and nidx >= until:
|
|
191
|
+
continue
|
|
192
|
+
for o in self.node(nidx).outs:
|
|
193
|
+
if o >= 0 and o not in seen:
|
|
194
|
+
seen.add(o)
|
|
195
|
+
stack.append(o)
|
|
196
|
+
return seen
|
|
197
|
+
|
|
198
|
+
def ancestors(self, vid: int, *, stop: set[int] | None = None) -> set[int]:
|
|
199
|
+
"""Node indices in the backward closure of ``vid`` (not crossing ``stop`` values)."""
|
|
200
|
+
stop = stop or set()
|
|
201
|
+
seen_nodes: set[int] = set()
|
|
202
|
+
stack = [vid]
|
|
203
|
+
seen_vals = {vid}
|
|
204
|
+
while stack:
|
|
205
|
+
v = stack.pop()
|
|
206
|
+
if v in stop:
|
|
207
|
+
continue
|
|
208
|
+
p = self.producer_idx(v)
|
|
209
|
+
if p < 0 or p in seen_nodes:
|
|
210
|
+
continue
|
|
211
|
+
seen_nodes.add(p)
|
|
212
|
+
for _, a in self.node(p).activation_args:
|
|
213
|
+
if a not in seen_vals:
|
|
214
|
+
seen_vals.add(a)
|
|
215
|
+
stack.append(a)
|
|
216
|
+
return seen_nodes
|
|
217
|
+
|
|
218
|
+
def has_param_ancestry(self, vid: int) -> bool:
|
|
219
|
+
if vid in self._param_anc:
|
|
220
|
+
return self._param_anc[vid]
|
|
221
|
+
result = any(self.node(n).params for n in self.ancestors(vid))
|
|
222
|
+
self._param_anc[vid] = result
|
|
223
|
+
return result
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def _numel(shape: tuple[int, ...]) -> int:
|
|
227
|
+
n = 1
|
|
228
|
+
for s in shape:
|
|
229
|
+
n *= s
|
|
230
|
+
return n
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _broadcastable(small: tuple[int, ...], big: tuple[int, ...]) -> bool:
|
|
234
|
+
if len(small) > len(big):
|
|
235
|
+
return False
|
|
236
|
+
return all(s in (1, b) for s, b in zip(reversed(small), reversed(big), strict=False))
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
# --------------------------------------------------------------------------- stream
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
@dataclass
|
|
243
|
+
class _ResAdd:
|
|
244
|
+
node: int
|
|
245
|
+
skip: int
|
|
246
|
+
branch: int
|
|
247
|
+
injection: bool = False
|
|
248
|
+
shortcut: int | None = None
|
|
249
|
+
"""For projection shortcuts (ResNet downsample) the shortcut operand."""
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
def _find_stream(G: _Graph) -> tuple[int | None, list[_ResAdd]]:
|
|
253
|
+
start = _stream_start(G)
|
|
254
|
+
if start is None:
|
|
255
|
+
return None, []
|
|
256
|
+
adds: list[_ResAdd] = []
|
|
257
|
+
s = start
|
|
258
|
+
for _ in range(len(G.g.nodes)):
|
|
259
|
+
found = _next_residual(G, s)
|
|
260
|
+
if found is None:
|
|
261
|
+
advanced = _advance(G, s)
|
|
262
|
+
if advanced is not None and advanced != s:
|
|
263
|
+
s = advanced
|
|
264
|
+
continue
|
|
265
|
+
break
|
|
266
|
+
adds.append(found)
|
|
267
|
+
s = G.node(found.node).outs[0]
|
|
268
|
+
s = _advance_unary(G, s)
|
|
269
|
+
return start, adds
|
|
270
|
+
|
|
271
|
+
|
|
272
|
+
def _stream_start(G: _Graph) -> int | None:
|
|
273
|
+
"""Where the residual stream begins: the skip operand of the first residual add.
|
|
274
|
+
|
|
275
|
+
An add is residual if one operand derives from the other through a projection
|
|
276
|
+
(``x + f(x)``), or if both derive from one value through projections of their
|
|
277
|
+
own (a ResNet projection shortcut) and the sum is not itself a branch of a
|
|
278
|
+
later ``x + f(x)`` (Falcon's parallel attention + MLP).
|
|
279
|
+
"""
|
|
280
|
+
candidates: list[tuple[Node, int, int]] = []
|
|
281
|
+
for node in G.g.nodes:
|
|
282
|
+
if base_op(node) != "add" or not node.outs or node.outs[0] < 0 or G.in_opaque(node):
|
|
283
|
+
continue
|
|
284
|
+
fa = G.float_args(node)
|
|
285
|
+
if len(fa) != 2:
|
|
286
|
+
continue
|
|
287
|
+
(_, a), (_, b) = fa
|
|
288
|
+
out_shape = G.shape(node.outs[0])
|
|
289
|
+
if G.shape(a) == out_shape and G.shape(b) == out_shape:
|
|
290
|
+
candidates.append((node, a, b))
|
|
291
|
+
for node, a, b in candidates:
|
|
292
|
+
if b in G.reach({a}, until=node.idx) and _path_has_proj(G, b, stop={a}):
|
|
293
|
+
return a
|
|
294
|
+
if a in G.reach({b}, until=node.idx) and _path_has_proj(G, a, stop={b}):
|
|
295
|
+
return b
|
|
296
|
+
common = _common_source(G, a, b)
|
|
297
|
+
if common is not None and not _is_sub_branch(G, node, common, candidates):
|
|
298
|
+
return common
|
|
299
|
+
return None
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _is_sub_branch(
|
|
303
|
+
G: _Graph, node: Node, source: int, candidates: list[tuple[Node, int, int]]
|
|
304
|
+
) -> bool:
|
|
305
|
+
"""True if this add's output feeds the branch side of a later ``x + f(x)`` add."""
|
|
306
|
+
downstream = G.reach(set(node.outs))
|
|
307
|
+
for later, a, b in candidates:
|
|
308
|
+
if later.idx <= node.idx:
|
|
309
|
+
continue
|
|
310
|
+
for skip, other in ((a, b), (b, a)):
|
|
311
|
+
if other in downstream and skip not in downstream and source in G.reach({skip}):
|
|
312
|
+
return True
|
|
313
|
+
return False
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def _path_has_proj(G: _Graph, vid: int, stop: set[int]) -> bool:
|
|
317
|
+
return any(G.is_proj(G.node(n)) for n in G.ancestors(vid, stop=stop))
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _common_source(G: _Graph, a: int, b: int) -> int | None:
|
|
321
|
+
"""Latest value both operands derive from, each through a projection of its own."""
|
|
322
|
+
anc_a, anc_b = G.ancestors(a), G.ancestors(b)
|
|
323
|
+
shared = anc_a & anc_b
|
|
324
|
+
own_a = any(G.is_proj(G.node(n)) for n in anc_a - shared)
|
|
325
|
+
own_b = any(G.is_proj(G.node(n)) for n in anc_b - shared)
|
|
326
|
+
if not (own_a and own_b) or not shared:
|
|
327
|
+
return None
|
|
328
|
+
for nidx in sorted(shared, reverse=True):
|
|
329
|
+
for out in G.node(nidx).outs:
|
|
330
|
+
if out >= 0 and G.floating(out):
|
|
331
|
+
return out
|
|
332
|
+
return None
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def _next_residual(G: _Graph, s: int) -> _ResAdd | None:
|
|
336
|
+
aliases = G.aliases(s)
|
|
337
|
+
first = max(G.producer_idx(v) for v in aliases) + 1
|
|
338
|
+
reach = None
|
|
339
|
+
for node in G.g.nodes[first:]:
|
|
340
|
+
if base_op(node) != "add" or not node.outs or node.outs[0] < 0:
|
|
341
|
+
continue
|
|
342
|
+
fa = G.float_args(node)
|
|
343
|
+
if len(fa) != 2:
|
|
344
|
+
continue
|
|
345
|
+
(_, a), (_, b) = fa
|
|
346
|
+
out_shape = G.shape(node.outs[0])
|
|
347
|
+
for skip, other in ((a, b), (b, a)):
|
|
348
|
+
if skip in aliases and other not in aliases and G.shape(skip) == out_shape:
|
|
349
|
+
if G.shape(other) != out_shape and not _broadcastable(G.shape(other), out_shape):
|
|
350
|
+
continue
|
|
351
|
+
if reach is None:
|
|
352
|
+
reach = G.reach({s})
|
|
353
|
+
injection = other not in reach
|
|
354
|
+
return _ResAdd(node.idx, skip, other, injection=injection)
|
|
355
|
+
# Projection shortcut: both operands derive from s, neither is s itself.
|
|
356
|
+
reach = reach or G.reach({s})
|
|
357
|
+
for node in G.g.nodes[first:]:
|
|
358
|
+
if base_op(node) != "add" or not node.outs or node.outs[0] < 0:
|
|
359
|
+
continue
|
|
360
|
+
fa = G.float_args(node)
|
|
361
|
+
if len(fa) != 2:
|
|
362
|
+
continue
|
|
363
|
+
(_, a), (_, b) = fa
|
|
364
|
+
out_shape = G.shape(node.outs[0])
|
|
365
|
+
if G.shape(a) != out_shape or G.shape(b) != out_shape or a not in reach or b not in reach:
|
|
366
|
+
continue
|
|
367
|
+
anc_a, anc_b = G.ancestors(a, stop=aliases), G.ancestors(b, stop=aliases)
|
|
368
|
+
shared = anc_a & anc_b
|
|
369
|
+
pa = [n for n in anc_a - shared if G.is_proj(G.node(n))]
|
|
370
|
+
pb = [n for n in anc_b - shared if G.is_proj(G.node(n))]
|
|
371
|
+
if pa and pb:
|
|
372
|
+
main, short = (a, b) if len(pa) >= len(pb) else (b, a)
|
|
373
|
+
return _ResAdd(node.idx, short, main, shortcut=short)
|
|
374
|
+
return None
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def _advance(G: _Graph, s: int) -> int | None:
|
|
378
|
+
"""Follow a single-consumer pointwise op whose other inputs are constants (embed scaling)."""
|
|
379
|
+
cons = G.cons.get(s, [])
|
|
380
|
+
if len(cons) != 1:
|
|
381
|
+
return None
|
|
382
|
+
node = G.node(cons[0][0])
|
|
383
|
+
if not node.outs or node.outs[0] < 0 or G.shape(node.outs[0]) != G.shape(s):
|
|
384
|
+
return None
|
|
385
|
+
if base_op(node) in ALIAS_OPS:
|
|
386
|
+
return node.outs[0]
|
|
387
|
+
if not _is_pointwise_node(node):
|
|
388
|
+
return None
|
|
389
|
+
for _, v in node.activation_args:
|
|
390
|
+
if v != s and G.has_param_ancestry(v):
|
|
391
|
+
return None
|
|
392
|
+
return node.outs[0]
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _advance_unary(G: _Graph, s: int) -> int:
|
|
396
|
+
"""After a residual add, step through a sole following activation (ResNet's ReLU)."""
|
|
397
|
+
while True:
|
|
398
|
+
cons = G.cons.get(s, [])
|
|
399
|
+
if len(cons) != 1:
|
|
400
|
+
return s
|
|
401
|
+
node = G.node(cons[0][0])
|
|
402
|
+
if base_op(node) not in ACTIVATION_OPS or len(node.activation_args) != 1 or not node.outs:
|
|
403
|
+
return s
|
|
404
|
+
s = node.outs[0]
|
|
405
|
+
|
|
406
|
+
|
|
407
|
+
_POINTWISE_EXTRA = frozenset(
|
|
408
|
+
{"mul", "add", "sub", "div", "neg", "pow", "rsqrt", "where", "masked_fill"}
|
|
409
|
+
)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
def _is_pointwise_node(node: Node) -> bool:
|
|
413
|
+
from wspx.lab.ir.tape import _pointwise_name
|
|
414
|
+
|
|
415
|
+
return base_op(node) in _POINTWISE_EXTRA or _pointwise_name(node.op)
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
# --------------------------------------------------------------------------- branches
|
|
419
|
+
|
|
420
|
+
|
|
421
|
+
def _writers(G: _Graph, vid: int, stop: set[int]) -> tuple[list[int], list[str]]:
|
|
422
|
+
"""Projections whose output features make up ``vid``, plus params on the way (post-norms)."""
|
|
423
|
+
writers: list[int] = []
|
|
424
|
+
norms: list[str] = []
|
|
425
|
+
seen: set[int] = set()
|
|
426
|
+
stack = [vid]
|
|
427
|
+
while stack:
|
|
428
|
+
v = stack.pop()
|
|
429
|
+
if v in stop or v in seen:
|
|
430
|
+
continue
|
|
431
|
+
seen.add(v)
|
|
432
|
+
p = G.producer_idx(v)
|
|
433
|
+
if p < 0:
|
|
434
|
+
continue
|
|
435
|
+
node = G.node(p)
|
|
436
|
+
if G.is_proj(node):
|
|
437
|
+
writers.append(node.idx)
|
|
438
|
+
continue
|
|
439
|
+
if G.is_attention(node) or G.is_actact(node):
|
|
440
|
+
continue
|
|
441
|
+
out_shape = G.shape(v)
|
|
442
|
+
args = G.float_args(node)
|
|
443
|
+
if _is_pointwise_node(node):
|
|
444
|
+
full = [a for _, a in args if G.shape(a) == out_shape]
|
|
445
|
+
follow = full or [a for _, a in args]
|
|
446
|
+
else:
|
|
447
|
+
follow = [a for _, a in args]
|
|
448
|
+
norms.extend(u.name for u in node.params if u.name not in norms)
|
|
449
|
+
stack.extend(follow)
|
|
450
|
+
return sorted(set(writers)), norms
|
|
451
|
+
|
|
452
|
+
|
|
453
|
+
def _readers(G: _Graph, stream_vals: set[int], until: int) -> tuple[list[int], dict[str, int]]:
|
|
454
|
+
"""First projections reached forward from the stream, plus params on the way.
|
|
455
|
+
|
|
456
|
+
Returns the reader node indices and ``{param: node idx}`` for the norm params.
|
|
457
|
+
"""
|
|
458
|
+
readers: list[int] = []
|
|
459
|
+
norms: dict[str, int] = {}
|
|
460
|
+
seen: set[int] = set()
|
|
461
|
+
stack = list(stream_vals)
|
|
462
|
+
while stack:
|
|
463
|
+
v = stack.pop()
|
|
464
|
+
if v in seen:
|
|
465
|
+
continue
|
|
466
|
+
seen.add(v)
|
|
467
|
+
for nidx, _ in G.cons.get(v, []):
|
|
468
|
+
if nidx >= until:
|
|
469
|
+
continue
|
|
470
|
+
node = G.node(nidx)
|
|
471
|
+
if G.is_proj(node):
|
|
472
|
+
if nidx not in readers:
|
|
473
|
+
readers.append(nidx)
|
|
474
|
+
continue
|
|
475
|
+
if G.is_attention(node) or G.is_actact(node) or base_op(node) == "embedding":
|
|
476
|
+
continue
|
|
477
|
+
for use in node.params:
|
|
478
|
+
norms.setdefault(use.name, nidx)
|
|
479
|
+
stack.extend(o for o in node.outs if o >= 0)
|
|
480
|
+
return sorted(readers), norms
|
|
481
|
+
|
|
482
|
+
|
|
483
|
+
def _classify(G: _Graph, nodes: set[int]) -> str:
|
|
484
|
+
ops = {base_op(G.node(n)) for n in nodes}
|
|
485
|
+
if any(G.is_attention(G.node(n)) for n in nodes):
|
|
486
|
+
return "attn"
|
|
487
|
+
actact = [n for n in nodes if G.is_actact(G.node(n))]
|
|
488
|
+
if actact and ops & SOFTMAX_OPS:
|
|
489
|
+
return "attn"
|
|
490
|
+
if any(G.node(n).meta.get("forced_routing") for n in nodes):
|
|
491
|
+
return "moe"
|
|
492
|
+
if ops & RNN_OPS:
|
|
493
|
+
return "rnn"
|
|
494
|
+
if actact or ops & SEQ_MIX_OPS:
|
|
495
|
+
return "ssm"
|
|
496
|
+
if any(base_op(G.node(n)) in CONV_OPS and G.node(n).params for n in nodes):
|
|
497
|
+
return "conv"
|
|
498
|
+
return "mlp"
|
|
499
|
+
|
|
500
|
+
|
|
501
|
+
# --------------------------------------------------------------------------- main
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
RoleFn = Callable[..., None]
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
def derive(
|
|
508
|
+
graph: OpGraph,
|
|
509
|
+
params: dict[str, tuple[int, ...]],
|
|
510
|
+
*,
|
|
511
|
+
module_of: dict[str, str] | None = None,
|
|
512
|
+
recurrent: dict[str, RecurrentLayout] | None = None,
|
|
513
|
+
) -> Semantics:
|
|
514
|
+
"""Derive roles for every parameter from the prefill graph.
|
|
515
|
+
|
|
516
|
+
``params`` maps each canonical parameter name to its shape; ``module_of`` maps
|
|
517
|
+
a name to its owning module (defaults to the name minus its last segment);
|
|
518
|
+
``recurrent`` describes ``nn.LSTM``/``nn.GRU``/``nn.RNN`` modules, whose kernels
|
|
519
|
+
are opaque (cuDNN) or unrolled per step (CPU) and so are handled as a unit.
|
|
520
|
+
"""
|
|
521
|
+
recurrent = recurrent or {}
|
|
522
|
+
G = _Graph(graph, opaque=tuple(recurrent))
|
|
523
|
+
sem = Semantics()
|
|
524
|
+
module_of = module_of or {name: name.rpartition(".")[0] for name in params}
|
|
525
|
+
roles: list[Role] = []
|
|
526
|
+
taken: set[str] = set()
|
|
527
|
+
|
|
528
|
+
def role(**kw: Any) -> None:
|
|
529
|
+
force = kw.pop("force", False)
|
|
530
|
+
whole = kw.get("split") is None and kw.get("expert") is None
|
|
531
|
+
if whole and kw["param"] in taken and not force:
|
|
532
|
+
return
|
|
533
|
+
if whole:
|
|
534
|
+
taken.add(kw["param"])
|
|
535
|
+
kw.setdefault("module", module_of.get(kw["param"], ""))
|
|
536
|
+
roles.append(Role(**kw))
|
|
537
|
+
|
|
538
|
+
start, adds = _find_stream(G)
|
|
539
|
+
if start is None or not any(not a.injection for a in adds):
|
|
540
|
+
_derive_sequential(G, sem, role, module_of, recurrent)
|
|
541
|
+
else:
|
|
542
|
+
sem.stream = [start] + [G.node(a.node).outs[0] for a in adds]
|
|
543
|
+
_derive_residual(G, sem, start, adds, role, module_of, params)
|
|
544
|
+
|
|
545
|
+
sem.tied = sem.embed is not None and sem.embed == sem.unembed
|
|
546
|
+
sem.roles = roles
|
|
547
|
+
sem.unassigned = sorted(set(params) - {r.param for r in roles})
|
|
548
|
+
return sem
|
|
549
|
+
|
|
550
|
+
|
|
551
|
+
def _weight_uses(node: Node) -> list[Any]:
|
|
552
|
+
uses = [u for u in node.params if u.in_dim is not None or u.out_dim is not None]
|
|
553
|
+
return uses or list(node.params[:1])
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _emitter(G: _Graph, role: RoleFn, prefix: str, block: int | None, branch: str | None) -> Any:
|
|
557
|
+
"""Return ``emit(node idx, name, kind, split=None, expert=None, experts=None)``.
|
|
558
|
+
|
|
559
|
+
Emits a role for the projection's weight and ``<name>.bias`` for any other
|
|
560
|
+
parameter of the same op, carrying the same split and expert.
|
|
561
|
+
"""
|
|
562
|
+
|
|
563
|
+
def emit(
|
|
564
|
+
nidx: int,
|
|
565
|
+
name: str,
|
|
566
|
+
kind: str,
|
|
567
|
+
split: list[list[int]] | None = None,
|
|
568
|
+
expert: int | None = None,
|
|
569
|
+
experts: int | None = None,
|
|
570
|
+
) -> None:
|
|
571
|
+
node = G.node(nidx)
|
|
572
|
+
weights = _weight_uses(node)
|
|
573
|
+
for use in weights:
|
|
574
|
+
role(
|
|
575
|
+
address=f"{prefix}.{name}",
|
|
576
|
+
param=use.name,
|
|
577
|
+
kind=kind,
|
|
578
|
+
block=block,
|
|
579
|
+
branch=branch,
|
|
580
|
+
in_dim=use.in_dim,
|
|
581
|
+
out_dim=use.out_dim,
|
|
582
|
+
split=split,
|
|
583
|
+
expert=expert,
|
|
584
|
+
experts=experts,
|
|
585
|
+
)
|
|
586
|
+
names = {u.name for u in weights}
|
|
587
|
+
for use in [*node.params, *_bias_adds(G, nidx)]:
|
|
588
|
+
if use.name not in names:
|
|
589
|
+
names.add(use.name)
|
|
590
|
+
role(
|
|
591
|
+
address=f"{prefix}.{name}.bias",
|
|
592
|
+
param=use.name,
|
|
593
|
+
kind="bias",
|
|
594
|
+
block=block,
|
|
595
|
+
branch=branch,
|
|
596
|
+
out_dim=None,
|
|
597
|
+
split=split,
|
|
598
|
+
expert=expert,
|
|
599
|
+
experts=experts,
|
|
600
|
+
)
|
|
601
|
+
|
|
602
|
+
return emit
|
|
603
|
+
|
|
604
|
+
|
|
605
|
+
def _bias_adds(G: _Graph, nidx: int) -> list[Any]:
|
|
606
|
+
"""Parameters added straight onto a projection's output (``x @ W + b`` written as two ops)."""
|
|
607
|
+
out = []
|
|
608
|
+
frontier = [o for o in G.node(nidx).outs if o >= 0]
|
|
609
|
+
for _ in range(4):
|
|
610
|
+
nxt: list[int] = []
|
|
611
|
+
for v in frontier:
|
|
612
|
+
for cidx, _ in G.cons.get(v, []):
|
|
613
|
+
cnode = G.node(cidx)
|
|
614
|
+
if base_op(cnode) in ALIAS_OPS:
|
|
615
|
+
nxt.extend(o for o in cnode.outs if o >= 0)
|
|
616
|
+
elif base_op(cnode) == "add" and cnode.params and len(cnode.activation_args) == 1:
|
|
617
|
+
out.extend(cnode.params)
|
|
618
|
+
frontier = nxt
|
|
619
|
+
return out
|
|
620
|
+
|
|
621
|
+
|
|
622
|
+
def _norm_roles(role: RoleFn, names: list[str], address: str, **kw: Any) -> None:
|
|
623
|
+
"""First param of a norm is its scale (``address``); a second is ``address.bias``."""
|
|
624
|
+
kind = kw.pop("kind", "norm")
|
|
625
|
+
for i, name in enumerate(names):
|
|
626
|
+
suffix = "" if i == 0 else (".bias" if i == 1 else f".{i}")
|
|
627
|
+
role(address=f"{address}{suffix}", param=name, kind=kind, **kw)
|
|
628
|
+
|
|
629
|
+
|
|
630
|
+
def _token_embedding(G: _Graph, node_ids: Iterable[int]) -> int | None:
|
|
631
|
+
"""The embedding op whose indices come straight from a graph input (input ids)."""
|
|
632
|
+
for nidx in sorted(node_ids):
|
|
633
|
+
node = G.node(nidx)
|
|
634
|
+
if base_op(node) != "embedding" or not node.params:
|
|
635
|
+
continue
|
|
636
|
+
for _, v in node.activation_args:
|
|
637
|
+
if G.floating(v):
|
|
638
|
+
continue
|
|
639
|
+
root = v
|
|
640
|
+
while True:
|
|
641
|
+
p = G.producer_idx(root)
|
|
642
|
+
if p < 0:
|
|
643
|
+
return nidx
|
|
644
|
+
pnode = G.node(p)
|
|
645
|
+
if base_op(pnode) not in ALIAS_OPS or len(pnode.activation_args) != 1:
|
|
646
|
+
break
|
|
647
|
+
root = pnode.activation_args[0][1]
|
|
648
|
+
return None
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def _derive_residual(
|
|
652
|
+
G: _Graph,
|
|
653
|
+
sem: Semantics,
|
|
654
|
+
start: int,
|
|
655
|
+
adds: list[_ResAdd],
|
|
656
|
+
role: RoleFn,
|
|
657
|
+
module_of: dict[str, str],
|
|
658
|
+
shapes: dict[str, tuple[int, ...]],
|
|
659
|
+
) -> None:
|
|
660
|
+
# ---- stem: everything computed before the stream starts
|
|
661
|
+
stem = G.ancestors(start)
|
|
662
|
+
tok = _token_embedding(G, stem)
|
|
663
|
+
if tok is not None:
|
|
664
|
+
sem.embed = _weight_uses(G.node(tok))[0].name
|
|
665
|
+
role(address="embed", param=sem.embed, kind="embed", out_dim=1)
|
|
666
|
+
n_pos = 0
|
|
667
|
+
for nidx in sorted(stem):
|
|
668
|
+
node = G.node(nidx)
|
|
669
|
+
if nidx == tok:
|
|
670
|
+
continue
|
|
671
|
+
for use in node.params:
|
|
672
|
+
if base_op(node) == "embedding":
|
|
673
|
+
role(
|
|
674
|
+
address="pos_embed" if n_pos == 0 else f"pos_embed{n_pos}",
|
|
675
|
+
param=use.name,
|
|
676
|
+
kind="pos_embed",
|
|
677
|
+
out_dim=1,
|
|
678
|
+
)
|
|
679
|
+
n_pos += 1
|
|
680
|
+
else:
|
|
681
|
+
role(
|
|
682
|
+
address=f"stem.{_label(use.name, module_of)}",
|
|
683
|
+
param=use.name,
|
|
684
|
+
kind="stem",
|
|
685
|
+
in_dim=use.in_dim,
|
|
686
|
+
out_dim=use.out_dim,
|
|
687
|
+
)
|
|
688
|
+
if sem.embed:
|
|
689
|
+
sem.writers.append(sem.embed)
|
|
690
|
+
|
|
691
|
+
# ---- blocks along the stream
|
|
692
|
+
stream_val = start
|
|
693
|
+
block_idx = -1
|
|
694
|
+
block_module: str | None = None
|
|
695
|
+
for add in adds:
|
|
696
|
+
add_node = G.node(add.node)
|
|
697
|
+
aliases = G.aliases(stream_val)
|
|
698
|
+
if add.injection:
|
|
699
|
+
for nidx in sorted(G.ancestors(add.branch, stop=aliases)):
|
|
700
|
+
node = G.node(nidx)
|
|
701
|
+
for use in node.params:
|
|
702
|
+
role(
|
|
703
|
+
address="pos_embed" if n_pos == 0 else f"pos_embed{n_pos}",
|
|
704
|
+
param=use.name,
|
|
705
|
+
kind="pos_embed",
|
|
706
|
+
out_dim=1 if base_op(node) == "embedding" else use.out_dim,
|
|
707
|
+
)
|
|
708
|
+
n_pos += 1
|
|
709
|
+
stream_val = _advance_unary(G, add_node.outs[0])
|
|
710
|
+
continue
|
|
711
|
+
|
|
712
|
+
if block_idx < 0 or add_node.module != block_module:
|
|
713
|
+
block_idx += 1
|
|
714
|
+
block_module = add_node.module
|
|
715
|
+
sem.blocks.append(Block(block_idx, add_node.module, [], []))
|
|
716
|
+
block = sem.blocks[-1]
|
|
717
|
+
block.residual_adds.append(add.node)
|
|
718
|
+
|
|
719
|
+
writers, post_norm = _writers(G, add.branch, stop=aliases)
|
|
720
|
+
readers, pre_norm = _readers(G, aliases, add.node)
|
|
721
|
+
short_nodes = G.ancestors(add.shortcut, stop=aliases) if add.shortcut is not None else set()
|
|
722
|
+
readers = [r for r in readers if r not in short_nodes]
|
|
723
|
+
branch_nodes = G.ancestors(add.branch, stop=aliases)
|
|
724
|
+
subs: list[tuple[int, set[int], str]] = []
|
|
725
|
+
for w in writers:
|
|
726
|
+
nodes = G.ancestors(G.node(w).activation_args[0][1], stop=aliases) | {w}
|
|
727
|
+
after = {
|
|
728
|
+
G.producer_idx(v)
|
|
729
|
+
for v in G.reach(set(G.node(w).outs), until=add.node)
|
|
730
|
+
if G.producer_idx(v) >= 0
|
|
731
|
+
}
|
|
732
|
+
nodes |= after & branch_nodes
|
|
733
|
+
subs.append((w, nodes, _classify(G, nodes)))
|
|
734
|
+
kinds = {k for _, _, k in subs}
|
|
735
|
+
labels: list[str] = []
|
|
736
|
+
for w, nodes, kind in subs:
|
|
737
|
+
label = "moe.shared" if kind == "mlp" and "moe" in kinds else kind
|
|
738
|
+
labels.append(label)
|
|
739
|
+
if label not in block.branches:
|
|
740
|
+
block.branches.append(label)
|
|
741
|
+
sub_readers = [r for r in readers if r in nodes]
|
|
742
|
+
_assign_branch(
|
|
743
|
+
G,
|
|
744
|
+
role,
|
|
745
|
+
block_idx,
|
|
746
|
+
label,
|
|
747
|
+
kind,
|
|
748
|
+
w,
|
|
749
|
+
nodes,
|
|
750
|
+
sub_readers,
|
|
751
|
+
module_of,
|
|
752
|
+
shapes,
|
|
753
|
+
sem.head_dims,
|
|
754
|
+
)
|
|
755
|
+
sem.readers.extend(u.name for r in sub_readers for u in _weight_uses(G.node(r)))
|
|
756
|
+
sem.writers.extend(u.name for u in _weight_uses(G.node(w)))
|
|
757
|
+
first = labels[0] if labels else "mix"
|
|
758
|
+
|
|
759
|
+
# readers that feed no writer's features (a shared-expert gate, say)
|
|
760
|
+
covered = set().union(*(n for _, n, _ in subs)) if subs else set()
|
|
761
|
+
emit_first = _emitter(G, role, f"L{block_idx}.{first}", block_idx, first)
|
|
762
|
+
for r in readers:
|
|
763
|
+
if r not in covered:
|
|
764
|
+
emit_first(r, _label(_weight_uses(G.node(r))[0].name, module_of), "reader")
|
|
765
|
+
|
|
766
|
+
# pre-norms go to the sub-branch whose readers they feed; post-norms to the writer's
|
|
767
|
+
by_label: dict[str, list[str]] = defaultdict(list)
|
|
768
|
+
for name, nidx in pre_norm.items():
|
|
769
|
+
label = next(
|
|
770
|
+
(lab for (_, nodes, _), lab in zip(subs, labels, strict=True) if nidx in nodes),
|
|
771
|
+
first,
|
|
772
|
+
)
|
|
773
|
+
by_label[label].append(name)
|
|
774
|
+
for label, names in by_label.items():
|
|
775
|
+
_norm_roles(role, names, f"L{block_idx}.{label}.norm", block=block_idx, branch=label)
|
|
776
|
+
if post_norm:
|
|
777
|
+
last = labels[-1] if labels else first
|
|
778
|
+
_norm_roles(
|
|
779
|
+
role, post_norm, f"L{block_idx}.{last}.post_norm", block=block_idx, branch=last
|
|
780
|
+
)
|
|
781
|
+
|
|
782
|
+
if add.shortcut is not None:
|
|
783
|
+
short = short_nodes
|
|
784
|
+
for i, nidx in enumerate(sorted(n for n in short if G.is_proj(G.node(n)))):
|
|
785
|
+
emit = _emitter(G, role, f"L{block_idx}.shortcut", block_idx, "shortcut")
|
|
786
|
+
emit(nidx, "proj" if i == 0 else f"proj{i}", "writer")
|
|
787
|
+
for nidx in sorted(short):
|
|
788
|
+
for use in G.node(nidx).params:
|
|
789
|
+
role(
|
|
790
|
+
address=f"L{block_idx}.shortcut.{_label(use.name, module_of)}",
|
|
791
|
+
param=use.name,
|
|
792
|
+
kind="aux",
|
|
793
|
+
block=block_idx,
|
|
794
|
+
branch="shortcut",
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
# anything else read inside the sub-branches (SSM constants, attention sinks…)
|
|
798
|
+
for (_, nodes, _), label in zip(subs, labels, strict=True):
|
|
799
|
+
for nidx in sorted(nodes):
|
|
800
|
+
for use in G.node(nidx).params:
|
|
801
|
+
role(
|
|
802
|
+
address=f"L{block_idx}.{label}.{_label(use.name, module_of)}",
|
|
803
|
+
param=use.name,
|
|
804
|
+
kind="aux",
|
|
805
|
+
block=block_idx,
|
|
806
|
+
branch=label,
|
|
807
|
+
)
|
|
808
|
+
stream_val = _advance_unary(G, add_node.outs[0])
|
|
809
|
+
|
|
810
|
+
# ---- tail: final norm and readout
|
|
811
|
+
readout = _readout(G)
|
|
812
|
+
if readout is not None:
|
|
813
|
+
rnode = G.node(readout)
|
|
814
|
+
tail = G.ancestors(rnode.activation_args[0][1], stop=G.aliases(stream_val))
|
|
815
|
+
names = list(dict.fromkeys(u.name for n in sorted(tail) for u in G.node(n).params))
|
|
816
|
+
sem.final_norm = names
|
|
817
|
+
_norm_roles(role, names, "final_norm", kind="final_norm")
|
|
818
|
+
weights = _weight_uses(rnode)
|
|
819
|
+
if weights:
|
|
820
|
+
sem.unembed = weights[0].name
|
|
821
|
+
sem.readers.append(sem.unembed)
|
|
822
|
+
# the unembedding may be tied to the embedding, so it always gets its own role
|
|
823
|
+
role(
|
|
824
|
+
address="unembed",
|
|
825
|
+
param=sem.unembed,
|
|
826
|
+
kind="unembed",
|
|
827
|
+
in_dim=weights[0].in_dim,
|
|
828
|
+
out_dim=weights[0].out_dim,
|
|
829
|
+
force=True,
|
|
830
|
+
)
|
|
831
|
+
for use in rnode.params:
|
|
832
|
+
if use.name not in {w.name for w in weights}:
|
|
833
|
+
role(address="unembed.bias", param=use.name, kind="bias", out_dim=0)
|
|
834
|
+
sem.readers = sorted(set(sem.readers))
|
|
835
|
+
sem.writers = sorted(set(sem.writers))
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
def _readout(G: _Graph) -> int | None:
|
|
839
|
+
for out in G.g.outputs:
|
|
840
|
+
v = out
|
|
841
|
+
for _ in range(16):
|
|
842
|
+
p = G.producer_idx(v)
|
|
843
|
+
if p < 0:
|
|
844
|
+
break
|
|
845
|
+
node = G.node(p)
|
|
846
|
+
if G.is_proj(node):
|
|
847
|
+
return node.idx
|
|
848
|
+
if base_op(node) in ALIAS_OPS or _is_pointwise_node(node):
|
|
849
|
+
args = G.float_args(node)
|
|
850
|
+
if not args:
|
|
851
|
+
break
|
|
852
|
+
v = args[0][1]
|
|
853
|
+
continue
|
|
854
|
+
break
|
|
855
|
+
return None
|
|
856
|
+
|
|
857
|
+
|
|
858
|
+
def _label(name: str, module_of: dict[str, str]) -> str:
|
|
859
|
+
"""A short, stable label for a parameter with no structural role."""
|
|
860
|
+
module = module_of.get(name, "")
|
|
861
|
+
leaf = name[len(module) + 1 :] if module and name.startswith(module + ".") else name
|
|
862
|
+
owner = module.rpartition(".")[2] if module else ""
|
|
863
|
+
if owner and not owner.isdigit() and leaf == "weight":
|
|
864
|
+
return owner
|
|
865
|
+
if owner and not owner.isdigit() and leaf == "bias":
|
|
866
|
+
return f"{owner}.bias"
|
|
867
|
+
return leaf.replace(".", "_")
|
|
868
|
+
|
|
869
|
+
|
|
870
|
+
Sources = list[tuple[int, list[list[int]]]]
|
|
871
|
+
"""``[(projection node, feature runs), …]`` arriving at one op argument."""
|
|
872
|
+
|
|
873
|
+
|
|
874
|
+
def _features(node: Node, pos: int) -> Sources:
|
|
875
|
+
feats = node.meta.get("features", {}).get(str(pos)) or []
|
|
876
|
+
return [(int(reader), [list(r) for r in runs]) for reader, runs in feats]
|
|
877
|
+
|
|
878
|
+
|
|
879
|
+
def _assign_branch(
|
|
880
|
+
G: _Graph,
|
|
881
|
+
role: RoleFn,
|
|
882
|
+
block: int,
|
|
883
|
+
label: str,
|
|
884
|
+
kind: str,
|
|
885
|
+
writer: int,
|
|
886
|
+
nodes: set[int],
|
|
887
|
+
readers: list[int],
|
|
888
|
+
module_of: dict[str, str],
|
|
889
|
+
shapes: dict[str, tuple[int, ...]],
|
|
890
|
+
head_dims: dict[str, int],
|
|
891
|
+
) -> None:
|
|
892
|
+
prefix = f"L{block}.{label}"
|
|
893
|
+
emit = _emitter(G, role, prefix, block, label)
|
|
894
|
+
projs = sorted(n for n in nodes if G.is_proj(G.node(n)))
|
|
895
|
+
|
|
896
|
+
if kind == "attn":
|
|
897
|
+
qkv = _attention_inputs(G, nodes)
|
|
898
|
+
by_reader: dict[int, dict[str, list[list[int]]]] = defaultdict(dict)
|
|
899
|
+
for name, sources in qkv.items():
|
|
900
|
+
for reader, runs in sources:
|
|
901
|
+
by_reader[reader][name] = runs
|
|
902
|
+
for reader, parts in sorted(by_reader.items()):
|
|
903
|
+
covered = sum(len(range(*r)) for runs in parts.values() for r in runs)
|
|
904
|
+
whole = covered == _out_size(G, reader, shapes)
|
|
905
|
+
if whole and len(parts) == 1:
|
|
906
|
+
emit(reader, next(iter(parts)), "reader")
|
|
907
|
+
continue
|
|
908
|
+
fused = "".join(n for n in ("q", "k", "v") if n in parts)
|
|
909
|
+
emit(
|
|
910
|
+
reader,
|
|
911
|
+
fused if whole else _label(_weight_uses(G.node(reader))[0].name, module_of),
|
|
912
|
+
"reader" if reader in readers else "aux",
|
|
913
|
+
)
|
|
914
|
+
for name, runs in parts.items():
|
|
915
|
+
emit(reader, name, "reader", split=runs)
|
|
916
|
+
# an output gate: projection features through an activation that scale the result
|
|
917
|
+
gated: set[int] = set()
|
|
918
|
+
for nidx in sorted(nodes):
|
|
919
|
+
if base_op(G.node(nidx)) in ACTIVATION_OPS:
|
|
920
|
+
for reader, runs in _features(G.node(nidx), 0):
|
|
921
|
+
if reader != writer and reader not in gated:
|
|
922
|
+
gated.add(reader)
|
|
923
|
+
emit(
|
|
924
|
+
reader,
|
|
925
|
+
"gate",
|
|
926
|
+
"reader",
|
|
927
|
+
split=None if reader not in by_reader else runs,
|
|
928
|
+
)
|
|
929
|
+
for n in projs:
|
|
930
|
+
if n not in by_reader and n not in gated and n != writer:
|
|
931
|
+
emit(
|
|
932
|
+
n,
|
|
933
|
+
_label(_weight_uses(G.node(n))[0].name, module_of),
|
|
934
|
+
"reader" if n in readers else "aux",
|
|
935
|
+
)
|
|
936
|
+
emit(writer, "o", "writer")
|
|
937
|
+
_qk_norms(G, role, nodes, prefix, block, label)
|
|
938
|
+
widths = _attention_widths(G, nodes)
|
|
939
|
+
for name, sources in qkv.items():
|
|
940
|
+
if len(sources) == 1 and name in widths:
|
|
941
|
+
head_dims[f"{prefix}.{name}"] = widths[name]
|
|
942
|
+
return
|
|
943
|
+
|
|
944
|
+
if kind == "moe":
|
|
945
|
+
routers = list(
|
|
946
|
+
dict.fromkeys(
|
|
947
|
+
G.node(n).meta["forced_routing"]["router"]
|
|
948
|
+
for n in sorted(nodes)
|
|
949
|
+
if G.node(n).meta.get("forced_routing")
|
|
950
|
+
)
|
|
951
|
+
)
|
|
952
|
+
for r in routers:
|
|
953
|
+
emit(r, "router", "router")
|
|
954
|
+
experts = [n for n in projs if n not in routers]
|
|
955
|
+
grouped = all(any(u.rows is not None for u in _weight_uses(G.node(n))) for n in experts)
|
|
956
|
+
if grouped:
|
|
957
|
+
count = max(
|
|
958
|
+
(_expert_count(G, u.name) or 0 for n in experts for u in _weight_uses(G.node(n))),
|
|
959
|
+
default=0,
|
|
960
|
+
)
|
|
961
|
+
sub_emit = _emitter(G, role, f"{prefix}.experts", block, label)
|
|
962
|
+
|
|
963
|
+
def ex_emit(
|
|
964
|
+
nidx: int, name: str, _kind: str, split: list[list[int]] | None = None
|
|
965
|
+
) -> None:
|
|
966
|
+
sub_emit(nidx, name, "expert", split=split, experts=count)
|
|
967
|
+
|
|
968
|
+
_gated(G, ex_emit, nodes, experts, writer)
|
|
969
|
+
else:
|
|
970
|
+
index = _expert_index(writer, G, module_of)
|
|
971
|
+
sub_emit = _emitter(G, role, f"{prefix}.e{index}", block, label)
|
|
972
|
+
|
|
973
|
+
def ex_emit(
|
|
974
|
+
nidx: int, name: str, _kind: str, split: list[list[int]] | None = None
|
|
975
|
+
) -> None:
|
|
976
|
+
sub_emit(nidx, name, "expert", split=split, expert=index)
|
|
977
|
+
|
|
978
|
+
_gated(G, ex_emit, nodes, experts, writer)
|
|
979
|
+
return
|
|
980
|
+
|
|
981
|
+
if kind == "mlp":
|
|
982
|
+
_gated(G, emit, nodes, projs, writer)
|
|
983
|
+
return
|
|
984
|
+
|
|
985
|
+
if kind == "conv":
|
|
986
|
+
conv_i = 0
|
|
987
|
+
for nidx in sorted(nodes):
|
|
988
|
+
node = G.node(nidx)
|
|
989
|
+
if G.is_proj(node):
|
|
990
|
+
conv_i += 1
|
|
991
|
+
emit(nidx, f"conv{conv_i}", "writer" if nidx == writer else "reader")
|
|
992
|
+
elif node.params:
|
|
993
|
+
_norm_roles(
|
|
994
|
+
role,
|
|
995
|
+
[u.name for u in node.params],
|
|
996
|
+
f"{prefix}.norm{max(conv_i, 1)}",
|
|
997
|
+
block=block,
|
|
998
|
+
branch=label,
|
|
999
|
+
)
|
|
1000
|
+
return
|
|
1001
|
+
|
|
1002
|
+
# ssm, rnn and anything else that mixes along the sequence
|
|
1003
|
+
named = set()
|
|
1004
|
+
if len(readers) == 1:
|
|
1005
|
+
emit(readers[0], "in", "reader")
|
|
1006
|
+
named.add(readers[0])
|
|
1007
|
+
for n in projs:
|
|
1008
|
+
if n not in named and n != writer:
|
|
1009
|
+
emit(
|
|
1010
|
+
n,
|
|
1011
|
+
_label(_weight_uses(G.node(n))[0].name, module_of),
|
|
1012
|
+
"reader" if n in readers else "aux",
|
|
1013
|
+
)
|
|
1014
|
+
emit(writer, "out", "writer")
|
|
1015
|
+
|
|
1016
|
+
|
|
1017
|
+
def _out_size(G: _Graph, nidx: int, shapes: dict[str, tuple[int, ...]]) -> int:
|
|
1018
|
+
use = _weight_uses(G.node(nidx))[0]
|
|
1019
|
+
shape = shapes.get(use.name, ())
|
|
1020
|
+
return shape[use.out_dim] if use.out_dim is not None and use.out_dim < len(shape) else -1
|
|
1021
|
+
|
|
1022
|
+
|
|
1023
|
+
def _attention_inputs(G: _Graph, nodes: set[int]) -> dict[str, Sources]:
|
|
1024
|
+
"""Which projection features arrive at q, k and v."""
|
|
1025
|
+
out: dict[str, Sources] = {}
|
|
1026
|
+
for nidx in sorted(nodes):
|
|
1027
|
+
node = G.node(nidx)
|
|
1028
|
+
if G.is_attention(node):
|
|
1029
|
+
for pos, name in ((0, "q"), (1, "k"), (2, "v")):
|
|
1030
|
+
if (f := _features(node, pos)) and name not in out:
|
|
1031
|
+
out[name] = f
|
|
1032
|
+
if out:
|
|
1033
|
+
return out
|
|
1034
|
+
softmax = [n for n in sorted(nodes) if base_op(G.node(n)) in SOFTMAX_OPS]
|
|
1035
|
+
if not softmax:
|
|
1036
|
+
return out
|
|
1037
|
+
sm = G.node(softmax[0])
|
|
1038
|
+
for n in sorted(G.ancestors(sm.activation_args[0][1]) & nodes, reverse=True):
|
|
1039
|
+
node = G.node(n)
|
|
1040
|
+
if G.is_actact(node):
|
|
1041
|
+
for (pos, _), name in zip(G.float_args(node)[:2], ("q", "k"), strict=False):
|
|
1042
|
+
if f := _features(node, pos):
|
|
1043
|
+
out[name] = f
|
|
1044
|
+
break
|
|
1045
|
+
after = G.reach(set(sm.outs))
|
|
1046
|
+
for n in sorted(nodes):
|
|
1047
|
+
node = G.node(n)
|
|
1048
|
+
if n > sm.idx and G.is_actact(node) and any(v in after for _, v in node.activation_args):
|
|
1049
|
+
for pos, v in G.float_args(node):
|
|
1050
|
+
if v not in after and (f := _features(node, pos)):
|
|
1051
|
+
out["v"] = f
|
|
1052
|
+
break
|
|
1053
|
+
return out
|
|
1054
|
+
|
|
1055
|
+
|
|
1056
|
+
def _attention_widths(G: _Graph, nodes: set[int]) -> dict[str, int]:
|
|
1057
|
+
"""Head width of q, k and v: the last dim of each attention input."""
|
|
1058
|
+
for nidx in sorted(nodes):
|
|
1059
|
+
node = G.node(nidx)
|
|
1060
|
+
if G.is_attention(node):
|
|
1061
|
+
return {
|
|
1062
|
+
name: G.shape(node.args[pos])[-1]
|
|
1063
|
+
for pos, name in ((0, "q"), (1, "k"), (2, "v"))
|
|
1064
|
+
if pos < len(node.args) and node.args[pos] >= 0
|
|
1065
|
+
}
|
|
1066
|
+
return {}
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def _qk_norms(
|
|
1070
|
+
G: _Graph, role: RoleFn, nodes: set[int], prefix: str, block: int, label: str
|
|
1071
|
+
) -> None:
|
|
1072
|
+
"""Per-head norms between a q/k projection and the attention op."""
|
|
1073
|
+
proj_outs = {o for n in nodes if G.is_proj(G.node(n)) for o in G.node(n).outs if o >= 0}
|
|
1074
|
+
for nidx in sorted(nodes):
|
|
1075
|
+
node = G.node(nidx)
|
|
1076
|
+
if not G.is_attention(node):
|
|
1077
|
+
continue
|
|
1078
|
+
for pos, name in ((0, "q"), (1, "k")):
|
|
1079
|
+
if pos >= len(node.args) or node.args[pos] < 0:
|
|
1080
|
+
continue
|
|
1081
|
+
between = G.ancestors(node.args[pos], stop=proj_outs)
|
|
1082
|
+
names = list(
|
|
1083
|
+
dict.fromkeys(
|
|
1084
|
+
u.name for a in sorted(between) if a in nodes for u in G.node(a).params
|
|
1085
|
+
)
|
|
1086
|
+
)
|
|
1087
|
+
if names:
|
|
1088
|
+
_norm_roles(role, names, f"{prefix}.{name}_norm", block=block, branch=label)
|
|
1089
|
+
return
|
|
1090
|
+
|
|
1091
|
+
|
|
1092
|
+
def _expert_count(G: _Graph, name: str) -> int | None:
|
|
1093
|
+
rows: set[int] = set()
|
|
1094
|
+
for node in G.g.nodes:
|
|
1095
|
+
for use in node.params:
|
|
1096
|
+
if use.name == name and use.rows is not None:
|
|
1097
|
+
rows.update(use.rows)
|
|
1098
|
+
return (max(rows) + 1) if rows else None
|
|
1099
|
+
|
|
1100
|
+
|
|
1101
|
+
def _expert_index(nidx: int, G: _Graph, module_of: dict[str, str]) -> int:
|
|
1102
|
+
"""Position of a per-expert module in its ModuleList (structural, not a name convention)."""
|
|
1103
|
+
name = _weight_uses(G.node(nidx))[0].name
|
|
1104
|
+
parts = module_of.get(name, "").split(".")
|
|
1105
|
+
for part in reversed(parts):
|
|
1106
|
+
if part.isdigit():
|
|
1107
|
+
return int(part)
|
|
1108
|
+
return 0
|
|
1109
|
+
|
|
1110
|
+
|
|
1111
|
+
def _gated(G: _Graph, emit: Any, nodes: set[int], projs: list[int], writer: int) -> None:
|
|
1112
|
+
"""Name MLP projections from data flow: gate, up and down.
|
|
1113
|
+
|
|
1114
|
+
The gate is whatever passes through the activation function before meeting
|
|
1115
|
+
other features in a product; up is the other side of that product. With no
|
|
1116
|
+
gating product, the single inner projection is ``up``.
|
|
1117
|
+
"""
|
|
1118
|
+
through_act: dict[int, list[list[int]]] = {}
|
|
1119
|
+
for nidx in sorted(nodes):
|
|
1120
|
+
node = G.node(nidx)
|
|
1121
|
+
if base_op(node) in ACTIVATION_OPS:
|
|
1122
|
+
for reader, runs in _features(node, 0):
|
|
1123
|
+
through_act.setdefault(reader, runs)
|
|
1124
|
+
up: dict[int, list[list[int]]] = {}
|
|
1125
|
+
for nidx in sorted(nodes):
|
|
1126
|
+
feats = G.node(nidx).meta.get("features", {})
|
|
1127
|
+
if base_op(G.node(nidx)) != "mul" or len(feats) < 2:
|
|
1128
|
+
continue
|
|
1129
|
+
entries = [entry for pos in feats for entry in _features(G.node(nidx), int(pos))]
|
|
1130
|
+
if len({(r, str(runs)) for r, runs in entries}) < 2:
|
|
1131
|
+
continue # x * f(x) on identical features is an activation, not a gate
|
|
1132
|
+
for reader, runs in entries:
|
|
1133
|
+
if through_act.get(reader) == runs:
|
|
1134
|
+
continue
|
|
1135
|
+
up.setdefault(reader, runs)
|
|
1136
|
+
gate = {r: runs for r, runs in through_act.items() if up}
|
|
1137
|
+
|
|
1138
|
+
named: set[int] = set()
|
|
1139
|
+
for reader in sorted(set(gate) | set(up)):
|
|
1140
|
+
if reader == writer:
|
|
1141
|
+
continue
|
|
1142
|
+
if reader in gate and reader in up:
|
|
1143
|
+
emit(reader, "gate_up", "reader")
|
|
1144
|
+
emit(reader, "gate", "reader", split=gate[reader])
|
|
1145
|
+
emit(reader, "up", "reader", split=up[reader])
|
|
1146
|
+
elif reader in gate:
|
|
1147
|
+
emit(reader, "gate", "reader")
|
|
1148
|
+
else:
|
|
1149
|
+
emit(reader, "up", "reader")
|
|
1150
|
+
named.add(reader)
|
|
1151
|
+
inner = [n for n in projs if n not in named and n != writer]
|
|
1152
|
+
for i, n in enumerate(inner):
|
|
1153
|
+
emit(n, "up" if not named and i == 0 else f"in{i}", "reader")
|
|
1154
|
+
emit(writer, "down", "writer")
|
|
1155
|
+
|
|
1156
|
+
|
|
1157
|
+
# --------------------------------------------------------------------------- sequential
|
|
1158
|
+
|
|
1159
|
+
|
|
1160
|
+
@dataclass
|
|
1161
|
+
class RecurrentLayout:
|
|
1162
|
+
"""PyTorch's documented layout for an ``nn.RNNBase`` module."""
|
|
1163
|
+
|
|
1164
|
+
mode: str
|
|
1165
|
+
"""``LSTM``, ``GRU``, ``RNN_TANH`` or ``RNN_RELU``."""
|
|
1166
|
+
num_layers: int
|
|
1167
|
+
bidirectional: bool
|
|
1168
|
+
flat: list[str]
|
|
1169
|
+
"""Canonical parameter names in ``_flat_weights_names`` order."""
|
|
1170
|
+
|
|
1171
|
+
|
|
1172
|
+
def _derive_sequential(
|
|
1173
|
+
G: _Graph,
|
|
1174
|
+
sem: Semantics,
|
|
1175
|
+
role: RoleFn,
|
|
1176
|
+
module_of: dict[str, str],
|
|
1177
|
+
recurrent: dict[str, RecurrentLayout],
|
|
1178
|
+
) -> None:
|
|
1179
|
+
"""Models without a residual stream: one block per layer along the data path."""
|
|
1180
|
+
index = -1
|
|
1181
|
+
last_kind = "linear"
|
|
1182
|
+
norms_in_block: dict[int, int] = defaultdict(int)
|
|
1183
|
+
readout = _readout(G)
|
|
1184
|
+
done_rnn: set[str] = set()
|
|
1185
|
+
for node in G.g.nodes:
|
|
1186
|
+
op = base_op(node)
|
|
1187
|
+
rnn = G.in_opaque(node)
|
|
1188
|
+
if rnn is not None:
|
|
1189
|
+
if rnn not in done_rnn:
|
|
1190
|
+
done_rnn.add(rnn)
|
|
1191
|
+
index = _rnn_roles(recurrent[rnn], role, index + 1) - 1
|
|
1192
|
+
last_kind = recurrent[rnn].mode.lower()
|
|
1193
|
+
continue
|
|
1194
|
+
if op == "embedding" and node.params and sem.embed is None:
|
|
1195
|
+
sem.embed = node.params[0].name
|
|
1196
|
+
role(address="embed", param=sem.embed, kind="embed", out_dim=1)
|
|
1197
|
+
continue
|
|
1198
|
+
if G.is_proj(node):
|
|
1199
|
+
index += 1
|
|
1200
|
+
last_kind = "conv" if op in CONV_OPS else "linear"
|
|
1201
|
+
emit = _emitter(G, role, f"L{index}", index, last_kind)
|
|
1202
|
+
emit(node.idx, last_kind, "layer")
|
|
1203
|
+
if node.idx == readout:
|
|
1204
|
+
weight = _weight_uses(node)[0]
|
|
1205
|
+
sem.unembed = weight.name
|
|
1206
|
+
role(
|
|
1207
|
+
address="unembed",
|
|
1208
|
+
param=weight.name,
|
|
1209
|
+
kind="unembed",
|
|
1210
|
+
in_dim=weight.in_dim,
|
|
1211
|
+
out_dim=weight.out_dim,
|
|
1212
|
+
force=True,
|
|
1213
|
+
)
|
|
1214
|
+
continue
|
|
1215
|
+
if node.params:
|
|
1216
|
+
target = max(index, 0)
|
|
1217
|
+
n = norms_in_block[target]
|
|
1218
|
+
norms_in_block[target] += 1
|
|
1219
|
+
_norm_roles(
|
|
1220
|
+
role,
|
|
1221
|
+
[u.name for u in node.params],
|
|
1222
|
+
f"L{target}.norm" + (str(n) if n else ""),
|
|
1223
|
+
block=target,
|
|
1224
|
+
branch=last_kind,
|
|
1225
|
+
)
|
|
1226
|
+
|
|
1227
|
+
|
|
1228
|
+
def _rnn_roles(layout: RecurrentLayout, role: RoleFn, first: int) -> int:
|
|
1229
|
+
"""Roles for a recurrent module; returns the next free block index.
|
|
1230
|
+
|
|
1231
|
+
Per layer and direction PyTorch stores ``w_ih, w_hh[, b_ih, b_hh]``, each with
|
|
1232
|
+
the gates stacked along dim 0 in a documented order.
|
|
1233
|
+
"""
|
|
1234
|
+
mode = layout.mode.lower().replace("rnn_tanh", "rnn").replace("rnn_relu", "rnn")
|
|
1235
|
+
gates = list(RNN_GATES.get(layout.mode.lower(), ())) or None
|
|
1236
|
+
directions = 2 if layout.bidirectional else 1
|
|
1237
|
+
per = len(layout.flat) // (layout.num_layers * directions)
|
|
1238
|
+
slots = ["ih", "hh", "ih.bias", "hh.bias"][:per]
|
|
1239
|
+
for i, name in enumerate(layout.flat):
|
|
1240
|
+
chunk, slot = divmod(i, per)
|
|
1241
|
+
layer, direction = divmod(chunk, directions)
|
|
1242
|
+
reverse = "_reverse" if direction else ""
|
|
1243
|
+
head, _, tail = slots[slot].partition(".")
|
|
1244
|
+
address = f"L{first + layer}.{mode}.{head}{reverse}" + (f".{tail}" if tail else "")
|
|
1245
|
+
role(
|
|
1246
|
+
address=address,
|
|
1247
|
+
param=name,
|
|
1248
|
+
kind="bias" if tail else "reader",
|
|
1249
|
+
block=first + layer,
|
|
1250
|
+
branch=mode,
|
|
1251
|
+
in_dim=None if tail else 1,
|
|
1252
|
+
out_dim=0,
|
|
1253
|
+
gates=gates,
|
|
1254
|
+
)
|
|
1255
|
+
return first + layout.num_layers
|