wspx 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. wspx/__init__.py +24 -0
  2. wspx/_version.py +1 -0
  3. wspx/addressing.py +415 -0
  4. wspx/auth.py +77 -0
  5. wspx/cli/__init__.py +1 -0
  6. wspx/cli/client.py +124 -0
  7. wspx/cli/config.py +89 -0
  8. wspx/cli/connect.py +88 -0
  9. wspx/cli/main.py +848 -0
  10. wspx/cloud/__init__.py +20 -0
  11. wspx/cloud/app.py +191 -0
  12. wspx/cloud/backend.py +141 -0
  13. wspx/cloud/compose.py +85 -0
  14. wspx/cloud/deploy.py +102 -0
  15. wspx/config.py +189 -0
  16. wspx/control/__init__.py +5 -0
  17. wspx/control/app.py +488 -0
  18. wspx/control/backend.py +244 -0
  19. wspx/control/gateway_client.py +79 -0
  20. wspx/control/mcp_server.py +71 -0
  21. wspx/control/sessions.py +424 -0
  22. wspx/control/tracedb.py +183 -0
  23. wspx/control/ui/assets/index-BJFweZAn.css +1 -0
  24. wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
  25. wspx/control/ui/index.html +13 -0
  26. wspx/control/views.py +180 -0
  27. wspx/diagnostics.py +146 -0
  28. wspx/driver/__init__.py +7 -0
  29. wspx/driver/loop.py +255 -0
  30. wspx/driver/messages.py +62 -0
  31. wspx/driver/providers/__init__.py +84 -0
  32. wspx/driver/providers/anthropic.py +149 -0
  33. wspx/driver/providers/compat.py +115 -0
  34. wspx/driver/providers/google.py +138 -0
  35. wspx/driver/providers/openai.py +125 -0
  36. wspx/driver/run.py +134 -0
  37. wspx/errors.py +69 -0
  38. wspx/gateway/__init__.py +8 -0
  39. wspx/gateway/app.py +626 -0
  40. wspx/gateway/ledger.py +536 -0
  41. wspx/gateway/prices.py +75 -0
  42. wspx/gateway/usage.py +284 -0
  43. wspx/handles.py +117 -0
  44. wspx/integrations/AGENTS.snippet.md +8 -0
  45. wspx/integrations/SKILL.md +52 -0
  46. wspx/integrations/__init__.py +17 -0
  47. wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
  48. wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
  49. wspx/integrations/codex-hooks.toml +9 -0
  50. wspx/lab/__init__.py +5 -0
  51. wspx/lab/analysis.py +485 -0
  52. wspx/lab/check.py +489 -0
  53. wspx/lab/evaluate.py +108 -0
  54. wspx/lab/export.py +275 -0
  55. wspx/lab/grep.py +754 -0
  56. wspx/lab/index.py +239 -0
  57. wspx/lab/ir/__init__.py +1 -0
  58. wspx/lab/ir/capture.py +299 -0
  59. wspx/lab/ir/graph.py +148 -0
  60. wspx/lab/ir/model_ir.py +522 -0
  61. wspx/lab/ir/semantics.py +1255 -0
  62. wspx/lab/ir/tape.py +853 -0
  63. wspx/lab/loader.py +204 -0
  64. wspx/lab/merge.py +200 -0
  65. wspx/lab/ops/__init__.py +76 -0
  66. wspx/lab/ops/builtin.py +379 -0
  67. wspx/lab/patch.py +320 -0
  68. wspx/lab/probe.py +206 -0
  69. wspx/lab/results.py +114 -0
  70. wspx/lab/sdk.py +120 -0
  71. wspx/lab/server.py +236 -0
  72. wspx/lab/session.py +1574 -0
  73. wspx/lab/tensors.py +119 -0
  74. wspx/lab/text.py +69 -0
  75. wspx/lab/tree.py +302 -0
  76. wspx/loops.py +127 -0
  77. wspx/project.py +143 -0
  78. wspx/py.typed +0 -0
  79. wspx/safety/__init__.py +15 -0
  80. wspx/safety/kill.py +47 -0
  81. wspx/safety/sweeper.py +68 -0
  82. wspx/store/__init__.py +64 -0
  83. wspx/store/objects.py +130 -0
  84. wspx/store/overlay.py +209 -0
  85. wspx/store/refs.py +150 -0
  86. wspx/testing/__init__.py +21 -0
  87. wspx/testing/api.py +155 -0
  88. wspx/testing/builtin.py +139 -0
  89. wspx/testing/runner.py +257 -0
  90. wspx/tools/__init__.py +17 -0
  91. wspx/tools/dispatch.py +129 -0
  92. wspx/tools/schemas.py +313 -0
  93. wspx/vcs/__init__.py +17 -0
  94. wspx/vcs/repo.py +431 -0
  95. wspx-0.1.0.dev0.dist-info/METADATA +278 -0
  96. wspx-0.1.0.dev0.dist-info/RECORD +99 -0
  97. wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
  98. wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
  99. wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,1255 @@
1
+ """Semantics derived from the op graph (§5.1.2). Nothing here reads parameter names.
2
+
3
+ The pass works on the prefill graph:
4
+
5
+ 1. **Residual stream.** The first ``x + f(x)`` add fixes where the stream starts;
6
+ from there each step takes the next add with the current stream value as one
7
+ operand. Blocks are groups of consecutive residual adds executed by the same
8
+ module, numbered by position along the stream.
9
+ 2. **Writers and readers.** A branch's writers are the parameter projections
10
+ reached walking back from its add operand (following full-shape data only, so
11
+ routing weights and gates do not count). Its readers are the projections
12
+ reached walking forward from the stream value.
13
+ 3. **Mixer type.** Each writer's sub-branch is attention if it contains a
14
+ fused attention op or a softmax over an activation-activation product, an SSM
15
+ or linear attention if it mixes along the sequence without softmax, MoE if it
16
+ contains a router ``topk``, and an MLP otherwise.
17
+ 4. **Roles.** q/k/v come from which projection features reach which attention
18
+ argument; gate/up from which features pass through the activation function;
19
+ fused weights get sub-ranges straight from the tape's feature provenance.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ from collections import defaultdict
25
+ from collections.abc import Callable, Iterable
26
+ from dataclasses import asdict, dataclass, field
27
+ from typing import Any
28
+
29
+ from wspx.lab.ir.graph import Node, OpGraph
30
+ from wspx.lab.ir.tape import ACTIVATION_OPS, CONV_OPS, MATMUL_OPS, SOFTMAX_OPS
31
+
32
+ ALIAS_OPS = frozenset(
33
+ {
34
+ "view",
35
+ "_unsafe_view",
36
+ "reshape",
37
+ "clone",
38
+ "contiguous",
39
+ "_to_copy",
40
+ "detach",
41
+ "alias",
42
+ "unsqueeze",
43
+ "squeeze",
44
+ "lift_fresh",
45
+ "flatten",
46
+ "unflatten",
47
+ "view_as",
48
+ "native_dropout",
49
+ "dropout",
50
+ }
51
+ )
52
+ RNN_OPS = frozenset({"lstm", "gru", "rnn_tanh", "rnn_relu", "_cudnn_rnn", "mkldnn_rnn"})
53
+ SEQ_MIX_OPS = frozenset({"cumsum", "cumprod", "_cummax_helper", "logcumsumexp"})
54
+ RNN_GATES = {
55
+ "lstm": ("input", "forget", "cell", "output"),
56
+ "gru": ("reset", "update", "new"),
57
+ }
58
+
59
+
60
+ @dataclass
61
+ class Role:
62
+ address: str
63
+ """Short address without slice, e.g. ``L3.attn.q`` or ``embed``."""
64
+ param: str
65
+ kind: str
66
+ """reader, writer, norm, embed, pos_embed, stem, unembed, final_norm, router, expert, bias, aux, layer."""
67
+ block: int | None = None
68
+ branch: str | None = None
69
+ module: str = ""
70
+ in_dim: int | None = None
71
+ out_dim: int | None = None
72
+ split: list[list[int]] | None = None
73
+ """Runs ``[start, stop, step]`` along ``out_dim`` for a sub-role of a fused weight."""
74
+ expert: int | None = None
75
+ """Index along dim 0 when this role is one expert of a 3-D expert weight."""
76
+ experts: int | None = None
77
+ gates: list[str] | None = None
78
+ """RNN gate order along the stacked dim (PyTorch's documented layout)."""
79
+
80
+
81
+ @dataclass
82
+ class Block:
83
+ index: int
84
+ module: str
85
+ branches: list[str]
86
+ residual_adds: list[int]
87
+
88
+
89
+ @dataclass
90
+ class Semantics:
91
+ roles: list[Role] = field(default_factory=list)
92
+ blocks: list[Block] = field(default_factory=list)
93
+ stream: list[int] = field(default_factory=list)
94
+ embed: str | None = None
95
+ unembed: str | None = None
96
+ tied: bool = False
97
+ final_norm: list[str] = field(default_factory=list)
98
+ readers: list[str] = field(default_factory=list)
99
+ writers: list[str] = field(default_factory=list)
100
+ unassigned: list[str] = field(default_factory=list)
101
+ head_dims: dict[str, int] = field(default_factory=dict)
102
+ """Per-head width of attention inputs, by role address (``L3.attn.q`` → 128)."""
103
+ notes: list[str] = field(default_factory=list)
104
+
105
+ def to_dict(self) -> dict[str, Any]:
106
+ return asdict(self)
107
+
108
+ @classmethod
109
+ def from_dict(cls, data: dict[str, Any]) -> Semantics:
110
+ data = dict(data)
111
+ data["roles"] = [Role(**r) for r in data.get("roles", [])]
112
+ data["blocks"] = [Block(**b) for b in data.get("blocks", [])]
113
+ return cls(**data)
114
+
115
+
116
+ def base_op(node: Node) -> str:
117
+ name = node.short_op
118
+ return name[:-1] if name.endswith("_") and not name.startswith("_") else name
119
+
120
+
121
+ class _Graph:
122
+ """Read-only helpers over an :class:`OpGraph`."""
123
+
124
+ def __init__(self, graph: OpGraph, opaque: tuple[str, ...] = ()) -> None:
125
+ self.g = graph
126
+ self.cons = graph.consumers
127
+ self._param_anc: dict[int, bool] = {}
128
+ self.opaque = opaque
129
+ """Module paths whose internals are handled as a unit (recurrent layers)."""
130
+
131
+ def in_opaque(self, node: Node) -> str | None:
132
+ for prefix in self.opaque:
133
+ if node.module == prefix or node.module.startswith(prefix + "."):
134
+ return prefix
135
+ return None
136
+
137
+ def node(self, idx: int) -> Node:
138
+ return self.g.nodes[idx]
139
+
140
+ def producer_idx(self, vid: int) -> int:
141
+ return self.g.values[vid].producer
142
+
143
+ def shape(self, vid: int) -> tuple[int, ...]:
144
+ return self.g.values[vid].shape
145
+
146
+ def floating(self, vid: int) -> bool:
147
+ dtype = self.g.values[vid].dtype
148
+ return "float" in dtype or "bfloat" in dtype or "complex" in dtype
149
+
150
+ def float_args(self, node: Node) -> list[tuple[int, int]]:
151
+ return [(p, v) for p, v in node.activation_args if self.floating(v)]
152
+
153
+ def is_proj(self, node: Node) -> bool:
154
+ op = base_op(node)
155
+ return (
156
+ bool(node.params)
157
+ and (op in MATMUL_OPS or op in CONV_OPS)
158
+ and bool(node.activation_args)
159
+ and self.in_opaque(node) is None
160
+ )
161
+
162
+ def is_actact(self, node: Node) -> bool:
163
+ return base_op(node) in MATMUL_OPS and not node.params and len(self.float_args(node)) >= 2
164
+
165
+ def is_attention(self, node: Node) -> bool:
166
+ return "scaled_dot_product" in base_op(node) or base_op(node).startswith("_flash_attention")
167
+
168
+ def aliases(self, vid: int) -> set[int]:
169
+ out = {vid}
170
+ stack = [vid]
171
+ while stack:
172
+ v = stack.pop()
173
+ numel = _numel(self.shape(v))
174
+ for nidx, _ in self.cons.get(v, []):
175
+ node = self.node(nidx)
176
+ if base_op(node) in ALIAS_OPS and len(node.activation_args) == 1 and node.outs:
177
+ o = node.outs[0]
178
+ if o >= 0 and o not in out and _numel(self.shape(o)) == numel:
179
+ out.add(o)
180
+ stack.append(o)
181
+ return out
182
+
183
+ def reach(self, vids: Iterable[int], *, until: int | None = None) -> set[int]:
184
+ """Values forward-reachable from ``vids`` through nodes with index < ``until``."""
185
+ seen = set(vids)
186
+ stack = list(seen)
187
+ while stack:
188
+ v = stack.pop()
189
+ for nidx, _ in self.cons.get(v, []):
190
+ if until is not None and nidx >= until:
191
+ continue
192
+ for o in self.node(nidx).outs:
193
+ if o >= 0 and o not in seen:
194
+ seen.add(o)
195
+ stack.append(o)
196
+ return seen
197
+
198
+ def ancestors(self, vid: int, *, stop: set[int] | None = None) -> set[int]:
199
+ """Node indices in the backward closure of ``vid`` (not crossing ``stop`` values)."""
200
+ stop = stop or set()
201
+ seen_nodes: set[int] = set()
202
+ stack = [vid]
203
+ seen_vals = {vid}
204
+ while stack:
205
+ v = stack.pop()
206
+ if v in stop:
207
+ continue
208
+ p = self.producer_idx(v)
209
+ if p < 0 or p in seen_nodes:
210
+ continue
211
+ seen_nodes.add(p)
212
+ for _, a in self.node(p).activation_args:
213
+ if a not in seen_vals:
214
+ seen_vals.add(a)
215
+ stack.append(a)
216
+ return seen_nodes
217
+
218
+ def has_param_ancestry(self, vid: int) -> bool:
219
+ if vid in self._param_anc:
220
+ return self._param_anc[vid]
221
+ result = any(self.node(n).params for n in self.ancestors(vid))
222
+ self._param_anc[vid] = result
223
+ return result
224
+
225
+
226
+ def _numel(shape: tuple[int, ...]) -> int:
227
+ n = 1
228
+ for s in shape:
229
+ n *= s
230
+ return n
231
+
232
+
233
+ def _broadcastable(small: tuple[int, ...], big: tuple[int, ...]) -> bool:
234
+ if len(small) > len(big):
235
+ return False
236
+ return all(s in (1, b) for s, b in zip(reversed(small), reversed(big), strict=False))
237
+
238
+
239
+ # --------------------------------------------------------------------------- stream
240
+
241
+
242
+ @dataclass
243
+ class _ResAdd:
244
+ node: int
245
+ skip: int
246
+ branch: int
247
+ injection: bool = False
248
+ shortcut: int | None = None
249
+ """For projection shortcuts (ResNet downsample) the shortcut operand."""
250
+
251
+
252
+ def _find_stream(G: _Graph) -> tuple[int | None, list[_ResAdd]]:
253
+ start = _stream_start(G)
254
+ if start is None:
255
+ return None, []
256
+ adds: list[_ResAdd] = []
257
+ s = start
258
+ for _ in range(len(G.g.nodes)):
259
+ found = _next_residual(G, s)
260
+ if found is None:
261
+ advanced = _advance(G, s)
262
+ if advanced is not None and advanced != s:
263
+ s = advanced
264
+ continue
265
+ break
266
+ adds.append(found)
267
+ s = G.node(found.node).outs[0]
268
+ s = _advance_unary(G, s)
269
+ return start, adds
270
+
271
+
272
+ def _stream_start(G: _Graph) -> int | None:
273
+ """Where the residual stream begins: the skip operand of the first residual add.
274
+
275
+ An add is residual if one operand derives from the other through a projection
276
+ (``x + f(x)``), or if both derive from one value through projections of their
277
+ own (a ResNet projection shortcut) and the sum is not itself a branch of a
278
+ later ``x + f(x)`` (Falcon's parallel attention + MLP).
279
+ """
280
+ candidates: list[tuple[Node, int, int]] = []
281
+ for node in G.g.nodes:
282
+ if base_op(node) != "add" or not node.outs or node.outs[0] < 0 or G.in_opaque(node):
283
+ continue
284
+ fa = G.float_args(node)
285
+ if len(fa) != 2:
286
+ continue
287
+ (_, a), (_, b) = fa
288
+ out_shape = G.shape(node.outs[0])
289
+ if G.shape(a) == out_shape and G.shape(b) == out_shape:
290
+ candidates.append((node, a, b))
291
+ for node, a, b in candidates:
292
+ if b in G.reach({a}, until=node.idx) and _path_has_proj(G, b, stop={a}):
293
+ return a
294
+ if a in G.reach({b}, until=node.idx) and _path_has_proj(G, a, stop={b}):
295
+ return b
296
+ common = _common_source(G, a, b)
297
+ if common is not None and not _is_sub_branch(G, node, common, candidates):
298
+ return common
299
+ return None
300
+
301
+
302
+ def _is_sub_branch(
303
+ G: _Graph, node: Node, source: int, candidates: list[tuple[Node, int, int]]
304
+ ) -> bool:
305
+ """True if this add's output feeds the branch side of a later ``x + f(x)`` add."""
306
+ downstream = G.reach(set(node.outs))
307
+ for later, a, b in candidates:
308
+ if later.idx <= node.idx:
309
+ continue
310
+ for skip, other in ((a, b), (b, a)):
311
+ if other in downstream and skip not in downstream and source in G.reach({skip}):
312
+ return True
313
+ return False
314
+
315
+
316
+ def _path_has_proj(G: _Graph, vid: int, stop: set[int]) -> bool:
317
+ return any(G.is_proj(G.node(n)) for n in G.ancestors(vid, stop=stop))
318
+
319
+
320
+ def _common_source(G: _Graph, a: int, b: int) -> int | None:
321
+ """Latest value both operands derive from, each through a projection of its own."""
322
+ anc_a, anc_b = G.ancestors(a), G.ancestors(b)
323
+ shared = anc_a & anc_b
324
+ own_a = any(G.is_proj(G.node(n)) for n in anc_a - shared)
325
+ own_b = any(G.is_proj(G.node(n)) for n in anc_b - shared)
326
+ if not (own_a and own_b) or not shared:
327
+ return None
328
+ for nidx in sorted(shared, reverse=True):
329
+ for out in G.node(nidx).outs:
330
+ if out >= 0 and G.floating(out):
331
+ return out
332
+ return None
333
+
334
+
335
+ def _next_residual(G: _Graph, s: int) -> _ResAdd | None:
336
+ aliases = G.aliases(s)
337
+ first = max(G.producer_idx(v) for v in aliases) + 1
338
+ reach = None
339
+ for node in G.g.nodes[first:]:
340
+ if base_op(node) != "add" or not node.outs or node.outs[0] < 0:
341
+ continue
342
+ fa = G.float_args(node)
343
+ if len(fa) != 2:
344
+ continue
345
+ (_, a), (_, b) = fa
346
+ out_shape = G.shape(node.outs[0])
347
+ for skip, other in ((a, b), (b, a)):
348
+ if skip in aliases and other not in aliases and G.shape(skip) == out_shape:
349
+ if G.shape(other) != out_shape and not _broadcastable(G.shape(other), out_shape):
350
+ continue
351
+ if reach is None:
352
+ reach = G.reach({s})
353
+ injection = other not in reach
354
+ return _ResAdd(node.idx, skip, other, injection=injection)
355
+ # Projection shortcut: both operands derive from s, neither is s itself.
356
+ reach = reach or G.reach({s})
357
+ for node in G.g.nodes[first:]:
358
+ if base_op(node) != "add" or not node.outs or node.outs[0] < 0:
359
+ continue
360
+ fa = G.float_args(node)
361
+ if len(fa) != 2:
362
+ continue
363
+ (_, a), (_, b) = fa
364
+ out_shape = G.shape(node.outs[0])
365
+ if G.shape(a) != out_shape or G.shape(b) != out_shape or a not in reach or b not in reach:
366
+ continue
367
+ anc_a, anc_b = G.ancestors(a, stop=aliases), G.ancestors(b, stop=aliases)
368
+ shared = anc_a & anc_b
369
+ pa = [n for n in anc_a - shared if G.is_proj(G.node(n))]
370
+ pb = [n for n in anc_b - shared if G.is_proj(G.node(n))]
371
+ if pa and pb:
372
+ main, short = (a, b) if len(pa) >= len(pb) else (b, a)
373
+ return _ResAdd(node.idx, short, main, shortcut=short)
374
+ return None
375
+
376
+
377
+ def _advance(G: _Graph, s: int) -> int | None:
378
+ """Follow a single-consumer pointwise op whose other inputs are constants (embed scaling)."""
379
+ cons = G.cons.get(s, [])
380
+ if len(cons) != 1:
381
+ return None
382
+ node = G.node(cons[0][0])
383
+ if not node.outs or node.outs[0] < 0 or G.shape(node.outs[0]) != G.shape(s):
384
+ return None
385
+ if base_op(node) in ALIAS_OPS:
386
+ return node.outs[0]
387
+ if not _is_pointwise_node(node):
388
+ return None
389
+ for _, v in node.activation_args:
390
+ if v != s and G.has_param_ancestry(v):
391
+ return None
392
+ return node.outs[0]
393
+
394
+
395
+ def _advance_unary(G: _Graph, s: int) -> int:
396
+ """After a residual add, step through a sole following activation (ResNet's ReLU)."""
397
+ while True:
398
+ cons = G.cons.get(s, [])
399
+ if len(cons) != 1:
400
+ return s
401
+ node = G.node(cons[0][0])
402
+ if base_op(node) not in ACTIVATION_OPS or len(node.activation_args) != 1 or not node.outs:
403
+ return s
404
+ s = node.outs[0]
405
+
406
+
407
+ _POINTWISE_EXTRA = frozenset(
408
+ {"mul", "add", "sub", "div", "neg", "pow", "rsqrt", "where", "masked_fill"}
409
+ )
410
+
411
+
412
+ def _is_pointwise_node(node: Node) -> bool:
413
+ from wspx.lab.ir.tape import _pointwise_name
414
+
415
+ return base_op(node) in _POINTWISE_EXTRA or _pointwise_name(node.op)
416
+
417
+
418
+ # --------------------------------------------------------------------------- branches
419
+
420
+
421
+ def _writers(G: _Graph, vid: int, stop: set[int]) -> tuple[list[int], list[str]]:
422
+ """Projections whose output features make up ``vid``, plus params on the way (post-norms)."""
423
+ writers: list[int] = []
424
+ norms: list[str] = []
425
+ seen: set[int] = set()
426
+ stack = [vid]
427
+ while stack:
428
+ v = stack.pop()
429
+ if v in stop or v in seen:
430
+ continue
431
+ seen.add(v)
432
+ p = G.producer_idx(v)
433
+ if p < 0:
434
+ continue
435
+ node = G.node(p)
436
+ if G.is_proj(node):
437
+ writers.append(node.idx)
438
+ continue
439
+ if G.is_attention(node) or G.is_actact(node):
440
+ continue
441
+ out_shape = G.shape(v)
442
+ args = G.float_args(node)
443
+ if _is_pointwise_node(node):
444
+ full = [a for _, a in args if G.shape(a) == out_shape]
445
+ follow = full or [a for _, a in args]
446
+ else:
447
+ follow = [a for _, a in args]
448
+ norms.extend(u.name for u in node.params if u.name not in norms)
449
+ stack.extend(follow)
450
+ return sorted(set(writers)), norms
451
+
452
+
453
+ def _readers(G: _Graph, stream_vals: set[int], until: int) -> tuple[list[int], dict[str, int]]:
454
+ """First projections reached forward from the stream, plus params on the way.
455
+
456
+ Returns the reader node indices and ``{param: node idx}`` for the norm params.
457
+ """
458
+ readers: list[int] = []
459
+ norms: dict[str, int] = {}
460
+ seen: set[int] = set()
461
+ stack = list(stream_vals)
462
+ while stack:
463
+ v = stack.pop()
464
+ if v in seen:
465
+ continue
466
+ seen.add(v)
467
+ for nidx, _ in G.cons.get(v, []):
468
+ if nidx >= until:
469
+ continue
470
+ node = G.node(nidx)
471
+ if G.is_proj(node):
472
+ if nidx not in readers:
473
+ readers.append(nidx)
474
+ continue
475
+ if G.is_attention(node) or G.is_actact(node) or base_op(node) == "embedding":
476
+ continue
477
+ for use in node.params:
478
+ norms.setdefault(use.name, nidx)
479
+ stack.extend(o for o in node.outs if o >= 0)
480
+ return sorted(readers), norms
481
+
482
+
483
+ def _classify(G: _Graph, nodes: set[int]) -> str:
484
+ ops = {base_op(G.node(n)) for n in nodes}
485
+ if any(G.is_attention(G.node(n)) for n in nodes):
486
+ return "attn"
487
+ actact = [n for n in nodes if G.is_actact(G.node(n))]
488
+ if actact and ops & SOFTMAX_OPS:
489
+ return "attn"
490
+ if any(G.node(n).meta.get("forced_routing") for n in nodes):
491
+ return "moe"
492
+ if ops & RNN_OPS:
493
+ return "rnn"
494
+ if actact or ops & SEQ_MIX_OPS:
495
+ return "ssm"
496
+ if any(base_op(G.node(n)) in CONV_OPS and G.node(n).params for n in nodes):
497
+ return "conv"
498
+ return "mlp"
499
+
500
+
501
+ # --------------------------------------------------------------------------- main
502
+
503
+
504
+ RoleFn = Callable[..., None]
505
+
506
+
507
+ def derive(
508
+ graph: OpGraph,
509
+ params: dict[str, tuple[int, ...]],
510
+ *,
511
+ module_of: dict[str, str] | None = None,
512
+ recurrent: dict[str, RecurrentLayout] | None = None,
513
+ ) -> Semantics:
514
+ """Derive roles for every parameter from the prefill graph.
515
+
516
+ ``params`` maps each canonical parameter name to its shape; ``module_of`` maps
517
+ a name to its owning module (defaults to the name minus its last segment);
518
+ ``recurrent`` describes ``nn.LSTM``/``nn.GRU``/``nn.RNN`` modules, whose kernels
519
+ are opaque (cuDNN) or unrolled per step (CPU) and so are handled as a unit.
520
+ """
521
+ recurrent = recurrent or {}
522
+ G = _Graph(graph, opaque=tuple(recurrent))
523
+ sem = Semantics()
524
+ module_of = module_of or {name: name.rpartition(".")[0] for name in params}
525
+ roles: list[Role] = []
526
+ taken: set[str] = set()
527
+
528
+ def role(**kw: Any) -> None:
529
+ force = kw.pop("force", False)
530
+ whole = kw.get("split") is None and kw.get("expert") is None
531
+ if whole and kw["param"] in taken and not force:
532
+ return
533
+ if whole:
534
+ taken.add(kw["param"])
535
+ kw.setdefault("module", module_of.get(kw["param"], ""))
536
+ roles.append(Role(**kw))
537
+
538
+ start, adds = _find_stream(G)
539
+ if start is None or not any(not a.injection for a in adds):
540
+ _derive_sequential(G, sem, role, module_of, recurrent)
541
+ else:
542
+ sem.stream = [start] + [G.node(a.node).outs[0] for a in adds]
543
+ _derive_residual(G, sem, start, adds, role, module_of, params)
544
+
545
+ sem.tied = sem.embed is not None and sem.embed == sem.unembed
546
+ sem.roles = roles
547
+ sem.unassigned = sorted(set(params) - {r.param for r in roles})
548
+ return sem
549
+
550
+
551
+ def _weight_uses(node: Node) -> list[Any]:
552
+ uses = [u for u in node.params if u.in_dim is not None or u.out_dim is not None]
553
+ return uses or list(node.params[:1])
554
+
555
+
556
+ def _emitter(G: _Graph, role: RoleFn, prefix: str, block: int | None, branch: str | None) -> Any:
557
+ """Return ``emit(node idx, name, kind, split=None, expert=None, experts=None)``.
558
+
559
+ Emits a role for the projection's weight and ``<name>.bias`` for any other
560
+ parameter of the same op, carrying the same split and expert.
561
+ """
562
+
563
+ def emit(
564
+ nidx: int,
565
+ name: str,
566
+ kind: str,
567
+ split: list[list[int]] | None = None,
568
+ expert: int | None = None,
569
+ experts: int | None = None,
570
+ ) -> None:
571
+ node = G.node(nidx)
572
+ weights = _weight_uses(node)
573
+ for use in weights:
574
+ role(
575
+ address=f"{prefix}.{name}",
576
+ param=use.name,
577
+ kind=kind,
578
+ block=block,
579
+ branch=branch,
580
+ in_dim=use.in_dim,
581
+ out_dim=use.out_dim,
582
+ split=split,
583
+ expert=expert,
584
+ experts=experts,
585
+ )
586
+ names = {u.name for u in weights}
587
+ for use in [*node.params, *_bias_adds(G, nidx)]:
588
+ if use.name not in names:
589
+ names.add(use.name)
590
+ role(
591
+ address=f"{prefix}.{name}.bias",
592
+ param=use.name,
593
+ kind="bias",
594
+ block=block,
595
+ branch=branch,
596
+ out_dim=None,
597
+ split=split,
598
+ expert=expert,
599
+ experts=experts,
600
+ )
601
+
602
+ return emit
603
+
604
+
605
+ def _bias_adds(G: _Graph, nidx: int) -> list[Any]:
606
+ """Parameters added straight onto a projection's output (``x @ W + b`` written as two ops)."""
607
+ out = []
608
+ frontier = [o for o in G.node(nidx).outs if o >= 0]
609
+ for _ in range(4):
610
+ nxt: list[int] = []
611
+ for v in frontier:
612
+ for cidx, _ in G.cons.get(v, []):
613
+ cnode = G.node(cidx)
614
+ if base_op(cnode) in ALIAS_OPS:
615
+ nxt.extend(o for o in cnode.outs if o >= 0)
616
+ elif base_op(cnode) == "add" and cnode.params and len(cnode.activation_args) == 1:
617
+ out.extend(cnode.params)
618
+ frontier = nxt
619
+ return out
620
+
621
+
622
+ def _norm_roles(role: RoleFn, names: list[str], address: str, **kw: Any) -> None:
623
+ """First param of a norm is its scale (``address``); a second is ``address.bias``."""
624
+ kind = kw.pop("kind", "norm")
625
+ for i, name in enumerate(names):
626
+ suffix = "" if i == 0 else (".bias" if i == 1 else f".{i}")
627
+ role(address=f"{address}{suffix}", param=name, kind=kind, **kw)
628
+
629
+
630
+ def _token_embedding(G: _Graph, node_ids: Iterable[int]) -> int | None:
631
+ """The embedding op whose indices come straight from a graph input (input ids)."""
632
+ for nidx in sorted(node_ids):
633
+ node = G.node(nidx)
634
+ if base_op(node) != "embedding" or not node.params:
635
+ continue
636
+ for _, v in node.activation_args:
637
+ if G.floating(v):
638
+ continue
639
+ root = v
640
+ while True:
641
+ p = G.producer_idx(root)
642
+ if p < 0:
643
+ return nidx
644
+ pnode = G.node(p)
645
+ if base_op(pnode) not in ALIAS_OPS or len(pnode.activation_args) != 1:
646
+ break
647
+ root = pnode.activation_args[0][1]
648
+ return None
649
+
650
+
651
+ def _derive_residual(
652
+ G: _Graph,
653
+ sem: Semantics,
654
+ start: int,
655
+ adds: list[_ResAdd],
656
+ role: RoleFn,
657
+ module_of: dict[str, str],
658
+ shapes: dict[str, tuple[int, ...]],
659
+ ) -> None:
660
+ # ---- stem: everything computed before the stream starts
661
+ stem = G.ancestors(start)
662
+ tok = _token_embedding(G, stem)
663
+ if tok is not None:
664
+ sem.embed = _weight_uses(G.node(tok))[0].name
665
+ role(address="embed", param=sem.embed, kind="embed", out_dim=1)
666
+ n_pos = 0
667
+ for nidx in sorted(stem):
668
+ node = G.node(nidx)
669
+ if nidx == tok:
670
+ continue
671
+ for use in node.params:
672
+ if base_op(node) == "embedding":
673
+ role(
674
+ address="pos_embed" if n_pos == 0 else f"pos_embed{n_pos}",
675
+ param=use.name,
676
+ kind="pos_embed",
677
+ out_dim=1,
678
+ )
679
+ n_pos += 1
680
+ else:
681
+ role(
682
+ address=f"stem.{_label(use.name, module_of)}",
683
+ param=use.name,
684
+ kind="stem",
685
+ in_dim=use.in_dim,
686
+ out_dim=use.out_dim,
687
+ )
688
+ if sem.embed:
689
+ sem.writers.append(sem.embed)
690
+
691
+ # ---- blocks along the stream
692
+ stream_val = start
693
+ block_idx = -1
694
+ block_module: str | None = None
695
+ for add in adds:
696
+ add_node = G.node(add.node)
697
+ aliases = G.aliases(stream_val)
698
+ if add.injection:
699
+ for nidx in sorted(G.ancestors(add.branch, stop=aliases)):
700
+ node = G.node(nidx)
701
+ for use in node.params:
702
+ role(
703
+ address="pos_embed" if n_pos == 0 else f"pos_embed{n_pos}",
704
+ param=use.name,
705
+ kind="pos_embed",
706
+ out_dim=1 if base_op(node) == "embedding" else use.out_dim,
707
+ )
708
+ n_pos += 1
709
+ stream_val = _advance_unary(G, add_node.outs[0])
710
+ continue
711
+
712
+ if block_idx < 0 or add_node.module != block_module:
713
+ block_idx += 1
714
+ block_module = add_node.module
715
+ sem.blocks.append(Block(block_idx, add_node.module, [], []))
716
+ block = sem.blocks[-1]
717
+ block.residual_adds.append(add.node)
718
+
719
+ writers, post_norm = _writers(G, add.branch, stop=aliases)
720
+ readers, pre_norm = _readers(G, aliases, add.node)
721
+ short_nodes = G.ancestors(add.shortcut, stop=aliases) if add.shortcut is not None else set()
722
+ readers = [r for r in readers if r not in short_nodes]
723
+ branch_nodes = G.ancestors(add.branch, stop=aliases)
724
+ subs: list[tuple[int, set[int], str]] = []
725
+ for w in writers:
726
+ nodes = G.ancestors(G.node(w).activation_args[0][1], stop=aliases) | {w}
727
+ after = {
728
+ G.producer_idx(v)
729
+ for v in G.reach(set(G.node(w).outs), until=add.node)
730
+ if G.producer_idx(v) >= 0
731
+ }
732
+ nodes |= after & branch_nodes
733
+ subs.append((w, nodes, _classify(G, nodes)))
734
+ kinds = {k for _, _, k in subs}
735
+ labels: list[str] = []
736
+ for w, nodes, kind in subs:
737
+ label = "moe.shared" if kind == "mlp" and "moe" in kinds else kind
738
+ labels.append(label)
739
+ if label not in block.branches:
740
+ block.branches.append(label)
741
+ sub_readers = [r for r in readers if r in nodes]
742
+ _assign_branch(
743
+ G,
744
+ role,
745
+ block_idx,
746
+ label,
747
+ kind,
748
+ w,
749
+ nodes,
750
+ sub_readers,
751
+ module_of,
752
+ shapes,
753
+ sem.head_dims,
754
+ )
755
+ sem.readers.extend(u.name for r in sub_readers for u in _weight_uses(G.node(r)))
756
+ sem.writers.extend(u.name for u in _weight_uses(G.node(w)))
757
+ first = labels[0] if labels else "mix"
758
+
759
+ # readers that feed no writer's features (a shared-expert gate, say)
760
+ covered = set().union(*(n for _, n, _ in subs)) if subs else set()
761
+ emit_first = _emitter(G, role, f"L{block_idx}.{first}", block_idx, first)
762
+ for r in readers:
763
+ if r not in covered:
764
+ emit_first(r, _label(_weight_uses(G.node(r))[0].name, module_of), "reader")
765
+
766
+ # pre-norms go to the sub-branch whose readers they feed; post-norms to the writer's
767
+ by_label: dict[str, list[str]] = defaultdict(list)
768
+ for name, nidx in pre_norm.items():
769
+ label = next(
770
+ (lab for (_, nodes, _), lab in zip(subs, labels, strict=True) if nidx in nodes),
771
+ first,
772
+ )
773
+ by_label[label].append(name)
774
+ for label, names in by_label.items():
775
+ _norm_roles(role, names, f"L{block_idx}.{label}.norm", block=block_idx, branch=label)
776
+ if post_norm:
777
+ last = labels[-1] if labels else first
778
+ _norm_roles(
779
+ role, post_norm, f"L{block_idx}.{last}.post_norm", block=block_idx, branch=last
780
+ )
781
+
782
+ if add.shortcut is not None:
783
+ short = short_nodes
784
+ for i, nidx in enumerate(sorted(n for n in short if G.is_proj(G.node(n)))):
785
+ emit = _emitter(G, role, f"L{block_idx}.shortcut", block_idx, "shortcut")
786
+ emit(nidx, "proj" if i == 0 else f"proj{i}", "writer")
787
+ for nidx in sorted(short):
788
+ for use in G.node(nidx).params:
789
+ role(
790
+ address=f"L{block_idx}.shortcut.{_label(use.name, module_of)}",
791
+ param=use.name,
792
+ kind="aux",
793
+ block=block_idx,
794
+ branch="shortcut",
795
+ )
796
+
797
+ # anything else read inside the sub-branches (SSM constants, attention sinks…)
798
+ for (_, nodes, _), label in zip(subs, labels, strict=True):
799
+ for nidx in sorted(nodes):
800
+ for use in G.node(nidx).params:
801
+ role(
802
+ address=f"L{block_idx}.{label}.{_label(use.name, module_of)}",
803
+ param=use.name,
804
+ kind="aux",
805
+ block=block_idx,
806
+ branch=label,
807
+ )
808
+ stream_val = _advance_unary(G, add_node.outs[0])
809
+
810
+ # ---- tail: final norm and readout
811
+ readout = _readout(G)
812
+ if readout is not None:
813
+ rnode = G.node(readout)
814
+ tail = G.ancestors(rnode.activation_args[0][1], stop=G.aliases(stream_val))
815
+ names = list(dict.fromkeys(u.name for n in sorted(tail) for u in G.node(n).params))
816
+ sem.final_norm = names
817
+ _norm_roles(role, names, "final_norm", kind="final_norm")
818
+ weights = _weight_uses(rnode)
819
+ if weights:
820
+ sem.unembed = weights[0].name
821
+ sem.readers.append(sem.unembed)
822
+ # the unembedding may be tied to the embedding, so it always gets its own role
823
+ role(
824
+ address="unembed",
825
+ param=sem.unembed,
826
+ kind="unembed",
827
+ in_dim=weights[0].in_dim,
828
+ out_dim=weights[0].out_dim,
829
+ force=True,
830
+ )
831
+ for use in rnode.params:
832
+ if use.name not in {w.name for w in weights}:
833
+ role(address="unembed.bias", param=use.name, kind="bias", out_dim=0)
834
+ sem.readers = sorted(set(sem.readers))
835
+ sem.writers = sorted(set(sem.writers))
836
+
837
+
838
+ def _readout(G: _Graph) -> int | None:
839
+ for out in G.g.outputs:
840
+ v = out
841
+ for _ in range(16):
842
+ p = G.producer_idx(v)
843
+ if p < 0:
844
+ break
845
+ node = G.node(p)
846
+ if G.is_proj(node):
847
+ return node.idx
848
+ if base_op(node) in ALIAS_OPS or _is_pointwise_node(node):
849
+ args = G.float_args(node)
850
+ if not args:
851
+ break
852
+ v = args[0][1]
853
+ continue
854
+ break
855
+ return None
856
+
857
+
858
+ def _label(name: str, module_of: dict[str, str]) -> str:
859
+ """A short, stable label for a parameter with no structural role."""
860
+ module = module_of.get(name, "")
861
+ leaf = name[len(module) + 1 :] if module and name.startswith(module + ".") else name
862
+ owner = module.rpartition(".")[2] if module else ""
863
+ if owner and not owner.isdigit() and leaf == "weight":
864
+ return owner
865
+ if owner and not owner.isdigit() and leaf == "bias":
866
+ return f"{owner}.bias"
867
+ return leaf.replace(".", "_")
868
+
869
+
870
+ Sources = list[tuple[int, list[list[int]]]]
871
+ """``[(projection node, feature runs), …]`` arriving at one op argument."""
872
+
873
+
874
+ def _features(node: Node, pos: int) -> Sources:
875
+ feats = node.meta.get("features", {}).get(str(pos)) or []
876
+ return [(int(reader), [list(r) for r in runs]) for reader, runs in feats]
877
+
878
+
879
+ def _assign_branch(
880
+ G: _Graph,
881
+ role: RoleFn,
882
+ block: int,
883
+ label: str,
884
+ kind: str,
885
+ writer: int,
886
+ nodes: set[int],
887
+ readers: list[int],
888
+ module_of: dict[str, str],
889
+ shapes: dict[str, tuple[int, ...]],
890
+ head_dims: dict[str, int],
891
+ ) -> None:
892
+ prefix = f"L{block}.{label}"
893
+ emit = _emitter(G, role, prefix, block, label)
894
+ projs = sorted(n for n in nodes if G.is_proj(G.node(n)))
895
+
896
+ if kind == "attn":
897
+ qkv = _attention_inputs(G, nodes)
898
+ by_reader: dict[int, dict[str, list[list[int]]]] = defaultdict(dict)
899
+ for name, sources in qkv.items():
900
+ for reader, runs in sources:
901
+ by_reader[reader][name] = runs
902
+ for reader, parts in sorted(by_reader.items()):
903
+ covered = sum(len(range(*r)) for runs in parts.values() for r in runs)
904
+ whole = covered == _out_size(G, reader, shapes)
905
+ if whole and len(parts) == 1:
906
+ emit(reader, next(iter(parts)), "reader")
907
+ continue
908
+ fused = "".join(n for n in ("q", "k", "v") if n in parts)
909
+ emit(
910
+ reader,
911
+ fused if whole else _label(_weight_uses(G.node(reader))[0].name, module_of),
912
+ "reader" if reader in readers else "aux",
913
+ )
914
+ for name, runs in parts.items():
915
+ emit(reader, name, "reader", split=runs)
916
+ # an output gate: projection features through an activation that scale the result
917
+ gated: set[int] = set()
918
+ for nidx in sorted(nodes):
919
+ if base_op(G.node(nidx)) in ACTIVATION_OPS:
920
+ for reader, runs in _features(G.node(nidx), 0):
921
+ if reader != writer and reader not in gated:
922
+ gated.add(reader)
923
+ emit(
924
+ reader,
925
+ "gate",
926
+ "reader",
927
+ split=None if reader not in by_reader else runs,
928
+ )
929
+ for n in projs:
930
+ if n not in by_reader and n not in gated and n != writer:
931
+ emit(
932
+ n,
933
+ _label(_weight_uses(G.node(n))[0].name, module_of),
934
+ "reader" if n in readers else "aux",
935
+ )
936
+ emit(writer, "o", "writer")
937
+ _qk_norms(G, role, nodes, prefix, block, label)
938
+ widths = _attention_widths(G, nodes)
939
+ for name, sources in qkv.items():
940
+ if len(sources) == 1 and name in widths:
941
+ head_dims[f"{prefix}.{name}"] = widths[name]
942
+ return
943
+
944
+ if kind == "moe":
945
+ routers = list(
946
+ dict.fromkeys(
947
+ G.node(n).meta["forced_routing"]["router"]
948
+ for n in sorted(nodes)
949
+ if G.node(n).meta.get("forced_routing")
950
+ )
951
+ )
952
+ for r in routers:
953
+ emit(r, "router", "router")
954
+ experts = [n for n in projs if n not in routers]
955
+ grouped = all(any(u.rows is not None for u in _weight_uses(G.node(n))) for n in experts)
956
+ if grouped:
957
+ count = max(
958
+ (_expert_count(G, u.name) or 0 for n in experts for u in _weight_uses(G.node(n))),
959
+ default=0,
960
+ )
961
+ sub_emit = _emitter(G, role, f"{prefix}.experts", block, label)
962
+
963
+ def ex_emit(
964
+ nidx: int, name: str, _kind: str, split: list[list[int]] | None = None
965
+ ) -> None:
966
+ sub_emit(nidx, name, "expert", split=split, experts=count)
967
+
968
+ _gated(G, ex_emit, nodes, experts, writer)
969
+ else:
970
+ index = _expert_index(writer, G, module_of)
971
+ sub_emit = _emitter(G, role, f"{prefix}.e{index}", block, label)
972
+
973
+ def ex_emit(
974
+ nidx: int, name: str, _kind: str, split: list[list[int]] | None = None
975
+ ) -> None:
976
+ sub_emit(nidx, name, "expert", split=split, expert=index)
977
+
978
+ _gated(G, ex_emit, nodes, experts, writer)
979
+ return
980
+
981
+ if kind == "mlp":
982
+ _gated(G, emit, nodes, projs, writer)
983
+ return
984
+
985
+ if kind == "conv":
986
+ conv_i = 0
987
+ for nidx in sorted(nodes):
988
+ node = G.node(nidx)
989
+ if G.is_proj(node):
990
+ conv_i += 1
991
+ emit(nidx, f"conv{conv_i}", "writer" if nidx == writer else "reader")
992
+ elif node.params:
993
+ _norm_roles(
994
+ role,
995
+ [u.name for u in node.params],
996
+ f"{prefix}.norm{max(conv_i, 1)}",
997
+ block=block,
998
+ branch=label,
999
+ )
1000
+ return
1001
+
1002
+ # ssm, rnn and anything else that mixes along the sequence
1003
+ named = set()
1004
+ if len(readers) == 1:
1005
+ emit(readers[0], "in", "reader")
1006
+ named.add(readers[0])
1007
+ for n in projs:
1008
+ if n not in named and n != writer:
1009
+ emit(
1010
+ n,
1011
+ _label(_weight_uses(G.node(n))[0].name, module_of),
1012
+ "reader" if n in readers else "aux",
1013
+ )
1014
+ emit(writer, "out", "writer")
1015
+
1016
+
1017
+ def _out_size(G: _Graph, nidx: int, shapes: dict[str, tuple[int, ...]]) -> int:
1018
+ use = _weight_uses(G.node(nidx))[0]
1019
+ shape = shapes.get(use.name, ())
1020
+ return shape[use.out_dim] if use.out_dim is not None and use.out_dim < len(shape) else -1
1021
+
1022
+
1023
+ def _attention_inputs(G: _Graph, nodes: set[int]) -> dict[str, Sources]:
1024
+ """Which projection features arrive at q, k and v."""
1025
+ out: dict[str, Sources] = {}
1026
+ for nidx in sorted(nodes):
1027
+ node = G.node(nidx)
1028
+ if G.is_attention(node):
1029
+ for pos, name in ((0, "q"), (1, "k"), (2, "v")):
1030
+ if (f := _features(node, pos)) and name not in out:
1031
+ out[name] = f
1032
+ if out:
1033
+ return out
1034
+ softmax = [n for n in sorted(nodes) if base_op(G.node(n)) in SOFTMAX_OPS]
1035
+ if not softmax:
1036
+ return out
1037
+ sm = G.node(softmax[0])
1038
+ for n in sorted(G.ancestors(sm.activation_args[0][1]) & nodes, reverse=True):
1039
+ node = G.node(n)
1040
+ if G.is_actact(node):
1041
+ for (pos, _), name in zip(G.float_args(node)[:2], ("q", "k"), strict=False):
1042
+ if f := _features(node, pos):
1043
+ out[name] = f
1044
+ break
1045
+ after = G.reach(set(sm.outs))
1046
+ for n in sorted(nodes):
1047
+ node = G.node(n)
1048
+ if n > sm.idx and G.is_actact(node) and any(v in after for _, v in node.activation_args):
1049
+ for pos, v in G.float_args(node):
1050
+ if v not in after and (f := _features(node, pos)):
1051
+ out["v"] = f
1052
+ break
1053
+ return out
1054
+
1055
+
1056
+ def _attention_widths(G: _Graph, nodes: set[int]) -> dict[str, int]:
1057
+ """Head width of q, k and v: the last dim of each attention input."""
1058
+ for nidx in sorted(nodes):
1059
+ node = G.node(nidx)
1060
+ if G.is_attention(node):
1061
+ return {
1062
+ name: G.shape(node.args[pos])[-1]
1063
+ for pos, name in ((0, "q"), (1, "k"), (2, "v"))
1064
+ if pos < len(node.args) and node.args[pos] >= 0
1065
+ }
1066
+ return {}
1067
+
1068
+
1069
+ def _qk_norms(
1070
+ G: _Graph, role: RoleFn, nodes: set[int], prefix: str, block: int, label: str
1071
+ ) -> None:
1072
+ """Per-head norms between a q/k projection and the attention op."""
1073
+ proj_outs = {o for n in nodes if G.is_proj(G.node(n)) for o in G.node(n).outs if o >= 0}
1074
+ for nidx in sorted(nodes):
1075
+ node = G.node(nidx)
1076
+ if not G.is_attention(node):
1077
+ continue
1078
+ for pos, name in ((0, "q"), (1, "k")):
1079
+ if pos >= len(node.args) or node.args[pos] < 0:
1080
+ continue
1081
+ between = G.ancestors(node.args[pos], stop=proj_outs)
1082
+ names = list(
1083
+ dict.fromkeys(
1084
+ u.name for a in sorted(between) if a in nodes for u in G.node(a).params
1085
+ )
1086
+ )
1087
+ if names:
1088
+ _norm_roles(role, names, f"{prefix}.{name}_norm", block=block, branch=label)
1089
+ return
1090
+
1091
+
1092
+ def _expert_count(G: _Graph, name: str) -> int | None:
1093
+ rows: set[int] = set()
1094
+ for node in G.g.nodes:
1095
+ for use in node.params:
1096
+ if use.name == name and use.rows is not None:
1097
+ rows.update(use.rows)
1098
+ return (max(rows) + 1) if rows else None
1099
+
1100
+
1101
+ def _expert_index(nidx: int, G: _Graph, module_of: dict[str, str]) -> int:
1102
+ """Position of a per-expert module in its ModuleList (structural, not a name convention)."""
1103
+ name = _weight_uses(G.node(nidx))[0].name
1104
+ parts = module_of.get(name, "").split(".")
1105
+ for part in reversed(parts):
1106
+ if part.isdigit():
1107
+ return int(part)
1108
+ return 0
1109
+
1110
+
1111
+ def _gated(G: _Graph, emit: Any, nodes: set[int], projs: list[int], writer: int) -> None:
1112
+ """Name MLP projections from data flow: gate, up and down.
1113
+
1114
+ The gate is whatever passes through the activation function before meeting
1115
+ other features in a product; up is the other side of that product. With no
1116
+ gating product, the single inner projection is ``up``.
1117
+ """
1118
+ through_act: dict[int, list[list[int]]] = {}
1119
+ for nidx in sorted(nodes):
1120
+ node = G.node(nidx)
1121
+ if base_op(node) in ACTIVATION_OPS:
1122
+ for reader, runs in _features(node, 0):
1123
+ through_act.setdefault(reader, runs)
1124
+ up: dict[int, list[list[int]]] = {}
1125
+ for nidx in sorted(nodes):
1126
+ feats = G.node(nidx).meta.get("features", {})
1127
+ if base_op(G.node(nidx)) != "mul" or len(feats) < 2:
1128
+ continue
1129
+ entries = [entry for pos in feats for entry in _features(G.node(nidx), int(pos))]
1130
+ if len({(r, str(runs)) for r, runs in entries}) < 2:
1131
+ continue # x * f(x) on identical features is an activation, not a gate
1132
+ for reader, runs in entries:
1133
+ if through_act.get(reader) == runs:
1134
+ continue
1135
+ up.setdefault(reader, runs)
1136
+ gate = {r: runs for r, runs in through_act.items() if up}
1137
+
1138
+ named: set[int] = set()
1139
+ for reader in sorted(set(gate) | set(up)):
1140
+ if reader == writer:
1141
+ continue
1142
+ if reader in gate and reader in up:
1143
+ emit(reader, "gate_up", "reader")
1144
+ emit(reader, "gate", "reader", split=gate[reader])
1145
+ emit(reader, "up", "reader", split=up[reader])
1146
+ elif reader in gate:
1147
+ emit(reader, "gate", "reader")
1148
+ else:
1149
+ emit(reader, "up", "reader")
1150
+ named.add(reader)
1151
+ inner = [n for n in projs if n not in named and n != writer]
1152
+ for i, n in enumerate(inner):
1153
+ emit(n, "up" if not named and i == 0 else f"in{i}", "reader")
1154
+ emit(writer, "down", "writer")
1155
+
1156
+
1157
+ # --------------------------------------------------------------------------- sequential
1158
+
1159
+
1160
+ @dataclass
1161
+ class RecurrentLayout:
1162
+ """PyTorch's documented layout for an ``nn.RNNBase`` module."""
1163
+
1164
+ mode: str
1165
+ """``LSTM``, ``GRU``, ``RNN_TANH`` or ``RNN_RELU``."""
1166
+ num_layers: int
1167
+ bidirectional: bool
1168
+ flat: list[str]
1169
+ """Canonical parameter names in ``_flat_weights_names`` order."""
1170
+
1171
+
1172
+ def _derive_sequential(
1173
+ G: _Graph,
1174
+ sem: Semantics,
1175
+ role: RoleFn,
1176
+ module_of: dict[str, str],
1177
+ recurrent: dict[str, RecurrentLayout],
1178
+ ) -> None:
1179
+ """Models without a residual stream: one block per layer along the data path."""
1180
+ index = -1
1181
+ last_kind = "linear"
1182
+ norms_in_block: dict[int, int] = defaultdict(int)
1183
+ readout = _readout(G)
1184
+ done_rnn: set[str] = set()
1185
+ for node in G.g.nodes:
1186
+ op = base_op(node)
1187
+ rnn = G.in_opaque(node)
1188
+ if rnn is not None:
1189
+ if rnn not in done_rnn:
1190
+ done_rnn.add(rnn)
1191
+ index = _rnn_roles(recurrent[rnn], role, index + 1) - 1
1192
+ last_kind = recurrent[rnn].mode.lower()
1193
+ continue
1194
+ if op == "embedding" and node.params and sem.embed is None:
1195
+ sem.embed = node.params[0].name
1196
+ role(address="embed", param=sem.embed, kind="embed", out_dim=1)
1197
+ continue
1198
+ if G.is_proj(node):
1199
+ index += 1
1200
+ last_kind = "conv" if op in CONV_OPS else "linear"
1201
+ emit = _emitter(G, role, f"L{index}", index, last_kind)
1202
+ emit(node.idx, last_kind, "layer")
1203
+ if node.idx == readout:
1204
+ weight = _weight_uses(node)[0]
1205
+ sem.unembed = weight.name
1206
+ role(
1207
+ address="unembed",
1208
+ param=weight.name,
1209
+ kind="unembed",
1210
+ in_dim=weight.in_dim,
1211
+ out_dim=weight.out_dim,
1212
+ force=True,
1213
+ )
1214
+ continue
1215
+ if node.params:
1216
+ target = max(index, 0)
1217
+ n = norms_in_block[target]
1218
+ norms_in_block[target] += 1
1219
+ _norm_roles(
1220
+ role,
1221
+ [u.name for u in node.params],
1222
+ f"L{target}.norm" + (str(n) if n else ""),
1223
+ block=target,
1224
+ branch=last_kind,
1225
+ )
1226
+
1227
+
1228
+ def _rnn_roles(layout: RecurrentLayout, role: RoleFn, first: int) -> int:
1229
+ """Roles for a recurrent module; returns the next free block index.
1230
+
1231
+ Per layer and direction PyTorch stores ``w_ih, w_hh[, b_ih, b_hh]``, each with
1232
+ the gates stacked along dim 0 in a documented order.
1233
+ """
1234
+ mode = layout.mode.lower().replace("rnn_tanh", "rnn").replace("rnn_relu", "rnn")
1235
+ gates = list(RNN_GATES.get(layout.mode.lower(), ())) or None
1236
+ directions = 2 if layout.bidirectional else 1
1237
+ per = len(layout.flat) // (layout.num_layers * directions)
1238
+ slots = ["ih", "hh", "ih.bias", "hh.bias"][:per]
1239
+ for i, name in enumerate(layout.flat):
1240
+ chunk, slot = divmod(i, per)
1241
+ layer, direction = divmod(chunk, directions)
1242
+ reverse = "_reverse" if direction else ""
1243
+ head, _, tail = slots[slot].partition(".")
1244
+ address = f"L{first + layer}.{mode}.{head}{reverse}" + (f".{tail}" if tail else "")
1245
+ role(
1246
+ address=address,
1247
+ param=name,
1248
+ kind="bias" if tail else "reader",
1249
+ block=first + layer,
1250
+ branch=mode,
1251
+ in_dim=None if tail else 1,
1252
+ out_dim=0,
1253
+ gates=gates,
1254
+ )
1255
+ return first + layout.num_layers