wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/ir/model_ir.py
ADDED
|
@@ -0,0 +1,522 @@
|
|
|
1
|
+
"""The Model IR: one inspectable object holding architecture, wiring and roles (§5.1).
|
|
2
|
+
|
|
3
|
+
The parameter table (from instantiating the real module) defines what exists;
|
|
4
|
+
the captured graphs define how each weight is wired; the semantics give each
|
|
5
|
+
weight its role. Addresses resolve against this object into exact
|
|
6
|
+
per-dimension selections of real parameters.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import difflib
|
|
12
|
+
import fnmatch
|
|
13
|
+
import re
|
|
14
|
+
from collections import defaultdict
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from dataclasses import asdict, dataclass, field
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
from wspx.addressing import (
|
|
20
|
+
Address,
|
|
21
|
+
Handle,
|
|
22
|
+
HandleRef,
|
|
23
|
+
Index,
|
|
24
|
+
Named,
|
|
25
|
+
Range,
|
|
26
|
+
ScopeKind,
|
|
27
|
+
SliceItem,
|
|
28
|
+
Token,
|
|
29
|
+
parse_address,
|
|
30
|
+
)
|
|
31
|
+
from wspx.errors import AddressError
|
|
32
|
+
from wspx.lab.ir.graph import OpGraph, expand_runs
|
|
33
|
+
from wspx.lab.ir.semantics import Role, Semantics
|
|
34
|
+
|
|
35
|
+
Selector = int | slice | list[int]
|
|
36
|
+
|
|
37
|
+
_EXPERT_RE = re.compile(r"^e(\d+)$")
|
|
38
|
+
_HEAD_RE = re.compile(r"^h(\d+)$")
|
|
39
|
+
_NAMED_AXES = {
|
|
40
|
+
"oc": "out",
|
|
41
|
+
"out": "out",
|
|
42
|
+
"ic": "in",
|
|
43
|
+
"in": "in",
|
|
44
|
+
"row": 0,
|
|
45
|
+
"col": 1,
|
|
46
|
+
"e": 0,
|
|
47
|
+
"expert": 0,
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
@dataclass
|
|
52
|
+
class ParamInfo:
|
|
53
|
+
name: str
|
|
54
|
+
shape: tuple[int, ...]
|
|
55
|
+
dtype: str
|
|
56
|
+
module: str
|
|
57
|
+
module_class: str
|
|
58
|
+
aliases: list[str] = field(default_factory=list)
|
|
59
|
+
"""Other names for the same tensor (tied weights)."""
|
|
60
|
+
hash: str | None = None
|
|
61
|
+
|
|
62
|
+
@property
|
|
63
|
+
def numel(self) -> int:
|
|
64
|
+
n = 1
|
|
65
|
+
for s in self.shape:
|
|
66
|
+
n *= s
|
|
67
|
+
return n
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
@dataclass
|
|
71
|
+
class Coverage:
|
|
72
|
+
exercised: bool
|
|
73
|
+
phases: list[str] = field(default_factory=list)
|
|
74
|
+
rows_seen: int | None = None
|
|
75
|
+
rows_total: int | None = None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class Target:
|
|
80
|
+
"""A resolved address: one parameter and a per-dim selection of it."""
|
|
81
|
+
|
|
82
|
+
param: str
|
|
83
|
+
index: tuple[Selector, ...]
|
|
84
|
+
address: str
|
|
85
|
+
role: Role | None = None
|
|
86
|
+
|
|
87
|
+
@property
|
|
88
|
+
def is_whole(self) -> bool:
|
|
89
|
+
return all(isinstance(s, slice) and s == slice(None) for s in self.index)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
TokenLookup = Callable[[str], int]
|
|
93
|
+
HandleLookup = Callable[[Handle, str], list[int]]
|
|
94
|
+
"""(handle, parameter) → unit indices along the tensor dim being narrowed."""
|
|
95
|
+
|
|
96
|
+
|
|
97
|
+
class ModelIR:
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
*,
|
|
101
|
+
fingerprint: str,
|
|
102
|
+
arch: str,
|
|
103
|
+
params: dict[str, ParamInfo],
|
|
104
|
+
semantics: Semantics,
|
|
105
|
+
graphs: dict[str, OpGraph] | None = None,
|
|
106
|
+
coverage: dict[str, Coverage] | None = None,
|
|
107
|
+
modules: dict[str, str] | None = None,
|
|
108
|
+
is_lm: bool = False,
|
|
109
|
+
vocab_size: int | None = None,
|
|
110
|
+
labels: dict[int, str] | None = None,
|
|
111
|
+
capture: dict[str, Any] | None = None,
|
|
112
|
+
) -> None:
|
|
113
|
+
self.fingerprint = fingerprint
|
|
114
|
+
self.arch = arch
|
|
115
|
+
self.params = params
|
|
116
|
+
self.semantics = semantics
|
|
117
|
+
self.graphs = graphs or {}
|
|
118
|
+
self.coverage = coverage or {}
|
|
119
|
+
self.modules = modules or {}
|
|
120
|
+
self.is_lm = is_lm
|
|
121
|
+
self.vocab_size = vocab_size
|
|
122
|
+
self.labels = labels or {}
|
|
123
|
+
self.capture = capture or {}
|
|
124
|
+
self._alias = {a: p.name for p in params.values() for a in p.aliases}
|
|
125
|
+
self._by_param: dict[str, list[Role]] = defaultdict(list)
|
|
126
|
+
for role in semantics.roles:
|
|
127
|
+
self._by_param[role.param].append(role)
|
|
128
|
+
|
|
129
|
+
# ------------------------------------------------------------------ lookup
|
|
130
|
+
|
|
131
|
+
def canonical(self, name: str) -> str:
|
|
132
|
+
return self._alias.get(name, name)
|
|
133
|
+
|
|
134
|
+
def roles_of(self, param: str) -> list[Role]:
|
|
135
|
+
return list(self._by_param.get(self.canonical(param), []))
|
|
136
|
+
|
|
137
|
+
def primary_role(self, param: str) -> Role | None:
|
|
138
|
+
roles = [r for r in self.roles_of(param) if r.split is None and r.expert is None]
|
|
139
|
+
return roles[0] if roles else None
|
|
140
|
+
|
|
141
|
+
@property
|
|
142
|
+
def roles(self) -> list[Role]:
|
|
143
|
+
return self.semantics.roles
|
|
144
|
+
|
|
145
|
+
@property
|
|
146
|
+
def num_blocks(self) -> int:
|
|
147
|
+
return len(self.semantics.blocks) or len(
|
|
148
|
+
{r.block for r in self.roles if r.block is not None}
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
@property
|
|
152
|
+
def unexercised(self) -> list[str]:
|
|
153
|
+
return sorted(name for name, cov in self.coverage.items() if not cov.exercised)
|
|
154
|
+
|
|
155
|
+
def is_tied(self, param: str) -> bool:
|
|
156
|
+
kinds = {r.kind for r in self.roles_of(param)}
|
|
157
|
+
return bool(
|
|
158
|
+
self.params.get(self.canonical(param), ParamInfo("", (), "", "", "")).aliases
|
|
159
|
+
) or ({"embed", "unembed"} <= kinds)
|
|
160
|
+
|
|
161
|
+
# ------------------------------------------------------------------ resolve
|
|
162
|
+
|
|
163
|
+
def resolve(
|
|
164
|
+
self,
|
|
165
|
+
address: str | Address,
|
|
166
|
+
*,
|
|
167
|
+
token_id: TokenLookup | None = None,
|
|
168
|
+
handle_indices: HandleLookup | None = None,
|
|
169
|
+
) -> list[Target]:
|
|
170
|
+
"""Resolve an address into targets. Raises :class:`AddressError` (E101) if nothing matches."""
|
|
171
|
+
addr = parse_address(address) if isinstance(address, str) else address
|
|
172
|
+
if addr.kind is ScopeKind.RAW:
|
|
173
|
+
base = self._resolve_raw(addr)
|
|
174
|
+
elif addr.kind is ScopeKind.SPECIAL:
|
|
175
|
+
base = self._from_roles([r for r in self.roles if r.address == addr.path[0]])
|
|
176
|
+
else:
|
|
177
|
+
base = self._resolve_short(addr)
|
|
178
|
+
if not base:
|
|
179
|
+
raise AddressError(self._no_match(addr))
|
|
180
|
+
if addr.slice is None:
|
|
181
|
+
return base
|
|
182
|
+
return [self._apply_slice(t, addr, token_id, handle_indices) for t in base]
|
|
183
|
+
|
|
184
|
+
def _resolve_raw(self, addr: Address) -> list[Target]:
|
|
185
|
+
assert addr.raw is not None
|
|
186
|
+
names = list(self.params) + list(self._alias)
|
|
187
|
+
if addr.has_glob:
|
|
188
|
+
matched = sorted({self.canonical(n) for n in names if fnmatch.fnmatchcase(n, addr.raw)})
|
|
189
|
+
else:
|
|
190
|
+
matched = (
|
|
191
|
+
[self.canonical(addr.raw)]
|
|
192
|
+
if addr.raw in self.params or addr.raw in self._alias
|
|
193
|
+
else []
|
|
194
|
+
)
|
|
195
|
+
if matched:
|
|
196
|
+
return [Target(p, self._full(p), p, self.primary_role(p)) for p in matched]
|
|
197
|
+
# Raw-looking scopes may also be role addresses with no special keyword (pos_embed, stem.*).
|
|
198
|
+
roles = [r for r in self.roles if fnmatch.fnmatchcase(r.address, addr.raw)]
|
|
199
|
+
return self._from_roles(roles)
|
|
200
|
+
|
|
201
|
+
def _resolve_short(self, addr: Address) -> list[Target]:
|
|
202
|
+
assert addr.layers is not None
|
|
203
|
+
pattern = ".".join(addr.path)
|
|
204
|
+
in_range = [r for r in self.roles if r.block is not None and r.block in addr.layers]
|
|
205
|
+
|
|
206
|
+
def rest(role: Role) -> str:
|
|
207
|
+
return role.address.split(".", 1)[1] if "." in role.address else ""
|
|
208
|
+
|
|
209
|
+
direct = [r for r in in_range if fnmatch.fnmatchcase(rest(r), pattern)]
|
|
210
|
+
if direct:
|
|
211
|
+
return self._from_roles(_dedupe_globbed(direct) if addr.has_glob else direct)
|
|
212
|
+
|
|
213
|
+
# Sub-selectors: expert (e5), head (h3) and recurrent gate (forget).
|
|
214
|
+
parts = list(addr.path)
|
|
215
|
+
expert: int | None = None
|
|
216
|
+
head: int | None = None
|
|
217
|
+
gate: str | None = None
|
|
218
|
+
for i, part in enumerate(parts):
|
|
219
|
+
if m := _EXPERT_RE.match(part):
|
|
220
|
+
expert = int(m.group(1))
|
|
221
|
+
parts[i] = "experts"
|
|
222
|
+
break
|
|
223
|
+
if parts and (m := _HEAD_RE.match(parts[-1])):
|
|
224
|
+
head = int(m.group(1))
|
|
225
|
+
parts = parts[:-1]
|
|
226
|
+
elif parts:
|
|
227
|
+
gates = {g for r in in_range if r.gates for g in r.gates}
|
|
228
|
+
if parts[-1] in gates:
|
|
229
|
+
gate = parts.pop()
|
|
230
|
+
sub_pattern = ".".join(parts)
|
|
231
|
+
if expert is not None:
|
|
232
|
+
# per-expert modules carry their own e<N> roles; 3-D expert weights get a row
|
|
233
|
+
literal = ".".join(addr.path[: len(parts)])
|
|
234
|
+
per_module = [r for r in in_range if fnmatch.fnmatchcase(rest(r), literal)]
|
|
235
|
+
if per_module and head is None and gate is None:
|
|
236
|
+
return self._from_roles(per_module)
|
|
237
|
+
matched = [r for r in in_range if fnmatch.fnmatchcase(rest(r), sub_pattern)]
|
|
238
|
+
if not matched:
|
|
239
|
+
return []
|
|
240
|
+
targets = []
|
|
241
|
+
for role in matched:
|
|
242
|
+
index = list(self._role_index(role))
|
|
243
|
+
info = self.params[role.param]
|
|
244
|
+
if expert is not None:
|
|
245
|
+
if role.experts is None:
|
|
246
|
+
raise AddressError(f"{addr}: {role.address} is not an expert weight")
|
|
247
|
+
if not 0 <= expert < role.experts:
|
|
248
|
+
raise AddressError(
|
|
249
|
+
f"{addr}: expert {expert} out of range (0-{role.experts - 1})"
|
|
250
|
+
)
|
|
251
|
+
index[0] = expert
|
|
252
|
+
if head is not None:
|
|
253
|
+
index = self._select_head(addr, role, info, index, head)
|
|
254
|
+
if gate is not None:
|
|
255
|
+
assert role.gates is not None
|
|
256
|
+
k = role.gates.index(gate)
|
|
257
|
+
size = info.shape[0] // len(role.gates)
|
|
258
|
+
index[0] = list(range(k * size, (k + 1) * size))
|
|
259
|
+
targets.append(Target(role.param, tuple(index), str(addr.with_slice(None)), role))
|
|
260
|
+
return targets
|
|
261
|
+
|
|
262
|
+
def _select_head(
|
|
263
|
+
self, addr: Address, role: Role, info: ParamInfo, index: list[Selector], head: int
|
|
264
|
+
) -> list[Selector]:
|
|
265
|
+
width = self.semantics_head_dim(role.address)
|
|
266
|
+
if width is None or role.out_dim is None:
|
|
267
|
+
raise AddressError(f"{addr}: {role.address} has no per-head layout")
|
|
268
|
+
dim = role.out_dim
|
|
269
|
+
current = index[dim]
|
|
270
|
+
features = list(range(info.shape[dim])) if isinstance(current, slice) else list(current) # type: ignore[arg-type]
|
|
271
|
+
n_heads = len(features) // width
|
|
272
|
+
if not 0 <= head < n_heads:
|
|
273
|
+
raise AddressError(f"{addr}: head {head} out of range (0-{n_heads - 1})")
|
|
274
|
+
index[dim] = features[head * width : (head + 1) * width]
|
|
275
|
+
return index
|
|
276
|
+
|
|
277
|
+
def semantics_head_dim(self, address: str) -> int | None:
|
|
278
|
+
return self.semantics.head_dims.get(address)
|
|
279
|
+
|
|
280
|
+
def _from_roles(self, roles: list[Role]) -> list[Target]:
|
|
281
|
+
return [Target(r.param, tuple(self._role_index(r)), r.address, r) for r in roles]
|
|
282
|
+
|
|
283
|
+
def _full(self, param: str) -> tuple[Selector, ...]:
|
|
284
|
+
return tuple(slice(None) for _ in self.params[param].shape)
|
|
285
|
+
|
|
286
|
+
def _role_index(self, role: Role) -> list[Selector]:
|
|
287
|
+
info = self.params[role.param]
|
|
288
|
+
index: list[Selector] = [slice(None)] * len(info.shape)
|
|
289
|
+
if role.split is not None:
|
|
290
|
+
dim = role.out_dim if role.out_dim is not None else len(info.shape) - 1
|
|
291
|
+
index[dim] = expand_runs(role.split)
|
|
292
|
+
return index
|
|
293
|
+
|
|
294
|
+
def _apply_slice(
|
|
295
|
+
self,
|
|
296
|
+
target: Target,
|
|
297
|
+
addr: Address,
|
|
298
|
+
token_id: TokenLookup | None,
|
|
299
|
+
handle_indices: HandleLookup | None,
|
|
300
|
+
) -> Target:
|
|
301
|
+
assert addr.slice is not None
|
|
302
|
+
info = self.params[target.param]
|
|
303
|
+
index = list(target.index)
|
|
304
|
+
free = [d for d, s in enumerate(index) if not isinstance(s, int)]
|
|
305
|
+
positional = 0
|
|
306
|
+
for item in addr.slice:
|
|
307
|
+
if isinstance(item, Named):
|
|
308
|
+
dim = self._named_dim(item.axis, target, addr)
|
|
309
|
+
index[dim] = _narrow(index[dim], item.item, info.shape[dim], addr)
|
|
310
|
+
continue
|
|
311
|
+
if positional >= len(free):
|
|
312
|
+
raise AddressError(f"{addr}: too many indices for {target.param} {info.shape}")
|
|
313
|
+
dim = free[positional]
|
|
314
|
+
positional += 1
|
|
315
|
+
index[dim] = self._narrow_item(
|
|
316
|
+
index[dim], item, info.shape[dim], addr, token_id, handle_indices, target.param
|
|
317
|
+
)
|
|
318
|
+
return Target(target.param, tuple(index), str(addr), target.role)
|
|
319
|
+
|
|
320
|
+
def _narrow_item(
|
|
321
|
+
self,
|
|
322
|
+
current: Selector,
|
|
323
|
+
item: SliceItem,
|
|
324
|
+
size: int,
|
|
325
|
+
addr: Address,
|
|
326
|
+
token_id: TokenLookup | None,
|
|
327
|
+
handle_indices: HandleLookup | None,
|
|
328
|
+
param: str = "",
|
|
329
|
+
) -> Selector:
|
|
330
|
+
if isinstance(item, Token):
|
|
331
|
+
if token_id is None:
|
|
332
|
+
raise AddressError(
|
|
333
|
+
f"{addr}: token selectors need the tokenizer (resolve in the lab)"
|
|
334
|
+
)
|
|
335
|
+
return _narrow(current, _as_index(token_id(item.text)), size, addr)
|
|
336
|
+
if isinstance(item, HandleRef):
|
|
337
|
+
if handle_indices is None:
|
|
338
|
+
raise AddressError(f"{addr}: handle selectors need a session")
|
|
339
|
+
allowed = set(_as_list(current, size))
|
|
340
|
+
picks = [i for i in handle_indices(item.handle, param) if i in allowed]
|
|
341
|
+
if not picks:
|
|
342
|
+
raise AddressError(
|
|
343
|
+
f"{addr}: {item.handle} has no units in this selection of {param}"
|
|
344
|
+
)
|
|
345
|
+
return picks
|
|
346
|
+
assert isinstance(item, Index | Range)
|
|
347
|
+
return _narrow(current, item, size, addr)
|
|
348
|
+
|
|
349
|
+
def _named_dim(self, axis: str, target: Target, addr: Address) -> int:
|
|
350
|
+
spec = _NAMED_AXES.get(axis)
|
|
351
|
+
role = target.role
|
|
352
|
+
if spec == "out":
|
|
353
|
+
return role.out_dim if role and role.out_dim is not None else 0
|
|
354
|
+
if spec == "in":
|
|
355
|
+
return role.in_dim if role and role.in_dim is not None else 1
|
|
356
|
+
if isinstance(spec, int):
|
|
357
|
+
return spec
|
|
358
|
+
raise AddressError(f"{addr}: unknown axis {axis!r}; use oc, ic, row, col or e")
|
|
359
|
+
|
|
360
|
+
def _no_match(self, addr: Address) -> str:
|
|
361
|
+
known = sorted({r.address for r in self.roles} | set(self.params))
|
|
362
|
+
hint = difflib.get_close_matches(addr.scope, known, n=3)
|
|
363
|
+
more = f"; did you mean {', '.join(hint)}?" if hint else ""
|
|
364
|
+
return f"E101 {addr}: no weights match{more}"
|
|
365
|
+
|
|
366
|
+
# ------------------------------------------------------------------ summary
|
|
367
|
+
|
|
368
|
+
def layer_types(self) -> list[tuple[str, str]]:
|
|
369
|
+
"""Compressed block layout, e.g. ``[("0-23", "attn+mlp"), ("24-31", "ssm+mlp")]``."""
|
|
370
|
+
kinds = [("+".join(b.branches) or "?") for b in self.semantics.blocks]
|
|
371
|
+
out: list[tuple[str, str]] = []
|
|
372
|
+
start = 0
|
|
373
|
+
for i in range(1, len(kinds) + 1):
|
|
374
|
+
if i == len(kinds) or kinds[i] != kinds[start]:
|
|
375
|
+
span = str(start) if i - 1 == start else f"{start}-{i - 1}"
|
|
376
|
+
out.append((span, kinds[start]))
|
|
377
|
+
start = i
|
|
378
|
+
return out
|
|
379
|
+
|
|
380
|
+
def summary(self) -> dict[str, Any]:
|
|
381
|
+
total = sum(p.numel for p in self.params.values())
|
|
382
|
+
exercised = sum(1 for c in self.coverage.values() if c.exercised)
|
|
383
|
+
return {
|
|
384
|
+
"arch": self.arch,
|
|
385
|
+
"fingerprint": self.fingerprint,
|
|
386
|
+
"params": len(self.params),
|
|
387
|
+
"numel": total,
|
|
388
|
+
"blocks": self.num_blocks,
|
|
389
|
+
"layers": self.layer_types(),
|
|
390
|
+
"embed": self.semantics.embed,
|
|
391
|
+
"unembed": self.semantics.unembed,
|
|
392
|
+
"tied": self.semantics.tied,
|
|
393
|
+
"final_norm": self.semantics.final_norm,
|
|
394
|
+
"coverage": f"{exercised}/{len(self.coverage)}" if self.coverage else "n/a",
|
|
395
|
+
"unexercised": self.unexercised,
|
|
396
|
+
"unassigned": self.semantics.unassigned,
|
|
397
|
+
"is_lm": self.is_lm,
|
|
398
|
+
"vocab_size": self.vocab_size,
|
|
399
|
+
"phases": list(self.graphs),
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
def render_info(self) -> str:
|
|
403
|
+
s = self.summary()
|
|
404
|
+
dtypes = sorted({p.dtype for p in self.params.values()})
|
|
405
|
+
lines = [
|
|
406
|
+
f"{self.arch} · {_human(s['numel'])} params in {s['params']} tensors · {', '.join(dtypes)}",
|
|
407
|
+
f"fingerprint {self.fingerprint[:23]}…",
|
|
408
|
+
]
|
|
409
|
+
if s["layers"]:
|
|
410
|
+
layout = ", ".join(f"L{span}: {kind}" for span, kind in s["layers"])
|
|
411
|
+
lines.append(f"{s['blocks']} blocks on a residual stream: {layout}")
|
|
412
|
+
else:
|
|
413
|
+
lines.append(f"{s['blocks']} layers, no residual stream")
|
|
414
|
+
if s["embed"] or s["unembed"]:
|
|
415
|
+
tie = " (tied)" if s["tied"] else ""
|
|
416
|
+
lines.append(f"embed {s['embed']} · unembed {s['unembed']}{tie}")
|
|
417
|
+
lines.append(f"coverage {s['coverage']} params exercised · phases {', '.join(s['phases'])}")
|
|
418
|
+
if s["unexercised"]:
|
|
419
|
+
lines.append(
|
|
420
|
+
f"UNEXERCISED ({len(s['unexercised'])}): {', '.join(s['unexercised'][:8])}"
|
|
421
|
+
)
|
|
422
|
+
if s["unassigned"]:
|
|
423
|
+
lines.append(f"no role ({len(s['unassigned'])}): {', '.join(s['unassigned'][:8])}")
|
|
424
|
+
example = next((r.address for r in self.roles if r.kind == "writer"), None)
|
|
425
|
+
if example:
|
|
426
|
+
lines.append(f"addresses look like {example} · raw parameter names always work")
|
|
427
|
+
return "\n".join(lines)
|
|
428
|
+
|
|
429
|
+
# ------------------------------------------------------------------ io
|
|
430
|
+
|
|
431
|
+
def to_dict(self, *, with_graphs: bool = True) -> dict[str, Any]:
|
|
432
|
+
return {
|
|
433
|
+
"version": 1,
|
|
434
|
+
"fingerprint": self.fingerprint,
|
|
435
|
+
"arch": self.arch,
|
|
436
|
+
"params": {k: asdict(v) for k, v in self.params.items()},
|
|
437
|
+
"semantics": self.semantics.to_dict(),
|
|
438
|
+
"coverage": {k: asdict(v) for k, v in self.coverage.items()},
|
|
439
|
+
"modules": self.modules,
|
|
440
|
+
"is_lm": self.is_lm,
|
|
441
|
+
"vocab_size": self.vocab_size,
|
|
442
|
+
"labels": {str(k): v for k, v in self.labels.items()},
|
|
443
|
+
"capture": self.capture,
|
|
444
|
+
"graphs": {k: g.to_dict() for k, g in self.graphs.items()} if with_graphs else {},
|
|
445
|
+
}
|
|
446
|
+
|
|
447
|
+
@classmethod
|
|
448
|
+
def from_dict(cls, data: dict[str, Any]) -> ModelIR:
|
|
449
|
+
params = {
|
|
450
|
+
k: ParamInfo(
|
|
451
|
+
name=v["name"],
|
|
452
|
+
shape=tuple(v["shape"]),
|
|
453
|
+
dtype=v["dtype"],
|
|
454
|
+
module=v["module"],
|
|
455
|
+
module_class=v["module_class"],
|
|
456
|
+
aliases=list(v.get("aliases", [])),
|
|
457
|
+
hash=v.get("hash"),
|
|
458
|
+
)
|
|
459
|
+
for k, v in data["params"].items()
|
|
460
|
+
}
|
|
461
|
+
return cls(
|
|
462
|
+
fingerprint=data["fingerprint"],
|
|
463
|
+
arch=data["arch"],
|
|
464
|
+
params=params,
|
|
465
|
+
semantics=Semantics.from_dict(data["semantics"]),
|
|
466
|
+
graphs={k: OpGraph.from_dict(g) for k, g in data.get("graphs", {}).items()},
|
|
467
|
+
coverage={k: Coverage(**v) for k, v in data.get("coverage", {}).items()},
|
|
468
|
+
modules=data.get("modules", {}),
|
|
469
|
+
is_lm=data.get("is_lm", False),
|
|
470
|
+
vocab_size=data.get("vocab_size"),
|
|
471
|
+
labels={int(k): v for k, v in data.get("labels", {}).items()},
|
|
472
|
+
capture=data.get("capture", {}),
|
|
473
|
+
)
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
# --------------------------------------------------------------------------- helpers
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _dedupe_globbed(roles: list[Role]) -> list[Role]:
|
|
480
|
+
"""A glob matching a fused weight and its sub-roles selects the whole weight once."""
|
|
481
|
+
whole = {r.param for r in roles if r.split is None and r.expert is None and r.kind != "bias"}
|
|
482
|
+
return [r for r in roles if r.split is None or r.param not in whole]
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _as_index(value: int) -> Index:
|
|
486
|
+
return Index(value)
|
|
487
|
+
|
|
488
|
+
|
|
489
|
+
def _as_list(current: Selector, size: int) -> list[int]:
|
|
490
|
+
if isinstance(current, slice):
|
|
491
|
+
return list(range(*current.indices(size)))
|
|
492
|
+
if isinstance(current, int):
|
|
493
|
+
return [current]
|
|
494
|
+
return list(current)
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def _narrow(current: Selector, item: Index | Range, size: int, addr: Address) -> Selector:
|
|
498
|
+
"""Apply an index or range to a dim that may already be narrowed."""
|
|
499
|
+
if isinstance(item, Range) and item == Range():
|
|
500
|
+
return current
|
|
501
|
+
base = _as_list(current, size)
|
|
502
|
+
try:
|
|
503
|
+
if isinstance(item, Range):
|
|
504
|
+
picked = base[item.to_slice()]
|
|
505
|
+
return _compact(picked)
|
|
506
|
+
return base[item.value]
|
|
507
|
+
except IndexError:
|
|
508
|
+
raise AddressError(f"{addr}: index {item} out of range for size {len(base)}") from None
|
|
509
|
+
|
|
510
|
+
|
|
511
|
+
def _compact(values: list[int]) -> Selector:
|
|
512
|
+
"""Contiguous ascending runs become slices (cheaper and view-friendly)."""
|
|
513
|
+
if values and values == list(range(values[0], values[-1] + 1)):
|
|
514
|
+
return slice(values[0], values[-1] + 1)
|
|
515
|
+
return values
|
|
516
|
+
|
|
517
|
+
|
|
518
|
+
def _human(n: int) -> str:
|
|
519
|
+
for unit, size in (("T", 10**12), ("B", 10**9), ("M", 10**6), ("K", 10**3)):
|
|
520
|
+
if n >= size:
|
|
521
|
+
return f"{n / size:.1f}{unit}"
|
|
522
|
+
return str(n)
|