wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/grep.py
ADDED
|
@@ -0,0 +1,754 @@
|
|
|
1
|
+
"""Grep over weights (§5.2): expressions evaluated where the weights are.
|
|
2
|
+
|
|
3
|
+
An expression scores every *unit* in scope: a column, a row, or a whole tensor.
|
|
4
|
+
For a writer the default unit is what one neuron writes into the residual
|
|
5
|
+
stream; for a reader, what one neuron reads from it.
|
|
6
|
+
|
|
7
|
+
Fields: ``norm colnorm rownorm absmax mean std kurtosis max min rank nan inf``.
|
|
8
|
+
Functions: ``writes("tok")`` (cosine with the token's unembedding, final-norm
|
|
9
|
+
scale folded in), ``reads("tok")`` (cosine with the token's embedding, the
|
|
10
|
+
block's pre-norm scale folded in), ``cos(dir:name)``, ``abs(x)``. Tokens may
|
|
11
|
+
be written as ids (``writes(1234)``). Comparisons, ``and``/``or``/``not`` and
|
|
12
|
+
arithmetic combine them; ``4σ`` on the right of a comparison means four
|
|
13
|
+
standard deviations above the mean of the left side across the scope.
|
|
14
|
+
|
|
15
|
+
The ranking key is the left side of the first comparison (or the whole
|
|
16
|
+
expression when it is a plain value).
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
import math
|
|
22
|
+
import re
|
|
23
|
+
from collections.abc import Callable, Iterator
|
|
24
|
+
from dataclasses import dataclass, field
|
|
25
|
+
from typing import Any
|
|
26
|
+
|
|
27
|
+
import torch
|
|
28
|
+
|
|
29
|
+
from wspx.errors import GrepError
|
|
30
|
+
from wspx.lab.ir.model_ir import ModelIR, Target
|
|
31
|
+
from wspx.lab.tensors import normalize_index, select
|
|
32
|
+
|
|
33
|
+
FIELDS = frozenset(
|
|
34
|
+
{
|
|
35
|
+
"norm",
|
|
36
|
+
"colnorm",
|
|
37
|
+
"rownorm",
|
|
38
|
+
"absmax",
|
|
39
|
+
"mean",
|
|
40
|
+
"std",
|
|
41
|
+
"kurtosis",
|
|
42
|
+
"max",
|
|
43
|
+
"min",
|
|
44
|
+
"rank",
|
|
45
|
+
"nan",
|
|
46
|
+
"inf",
|
|
47
|
+
}
|
|
48
|
+
)
|
|
49
|
+
FUNCS = frozenset({"writes", "reads", "cos", "abs"})
|
|
50
|
+
CHUNK_UNITS = 8192
|
|
51
|
+
|
|
52
|
+
# --------------------------------------------------------------------------- parsing
|
|
53
|
+
|
|
54
|
+
_TOKEN_RE = re.compile(
|
|
55
|
+
r"""\s*(?:
|
|
56
|
+
(?P<num>\d+(?:\.\d*)?(?:[eE][-+]?\d+)?|\.\d+(?:[eE][-+]?\d+)?)(?P<sigma>σ|sigma)?
|
|
57
|
+
| (?P<str>"(?:[^"\\]|\\.)*"|'(?:[^'\\]|\\.)*')
|
|
58
|
+
| (?P<handle>dir:[A-Za-z_][\w\-]*(?:\([^)]*\))?)
|
|
59
|
+
| (?P<op>>=|<=|==|!=|>|<|\(|\)|\+|-|\*|/|,)
|
|
60
|
+
| (?P<name>[A-Za-z_][A-Za-z0-9_]*)
|
|
61
|
+
)""",
|
|
62
|
+
re.VERBOSE,
|
|
63
|
+
)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass(frozen=True)
|
|
67
|
+
class Num:
|
|
68
|
+
value: float
|
|
69
|
+
sigma: bool = False
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
@dataclass(frozen=True)
|
|
73
|
+
class Field:
|
|
74
|
+
name: str
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
@dataclass(frozen=True)
|
|
78
|
+
class Call:
|
|
79
|
+
func: str
|
|
80
|
+
arg: Any
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
@dataclass(frozen=True)
|
|
84
|
+
class Unary:
|
|
85
|
+
op: str
|
|
86
|
+
operand: Any
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
@dataclass(frozen=True)
|
|
90
|
+
class Binary:
|
|
91
|
+
op: str
|
|
92
|
+
left: Any
|
|
93
|
+
right: Any
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
Node = Num | Field | Call | Unary | Binary
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _tokens(text: str) -> list[tuple[str, str]]:
|
|
100
|
+
out: list[tuple[str, str]] = []
|
|
101
|
+
pos = 0
|
|
102
|
+
text = text.strip()
|
|
103
|
+
while pos < len(text):
|
|
104
|
+
m = _TOKEN_RE.match(text, pos)
|
|
105
|
+
if not m or m.end() == pos:
|
|
106
|
+
raise GrepError(f"unexpected {text[pos : pos + 10]!r} at column {pos + 1}")
|
|
107
|
+
pos = m.end()
|
|
108
|
+
kind = m.lastgroup
|
|
109
|
+
if m.group("num") is not None:
|
|
110
|
+
out.append(("sigma" if m.group("sigma") else "num", m.group("num")))
|
|
111
|
+
elif kind is not None:
|
|
112
|
+
out.append((kind, m.group(kind)))
|
|
113
|
+
return out
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
class _Parser:
|
|
117
|
+
def __init__(self, text: str) -> None:
|
|
118
|
+
self.text = text
|
|
119
|
+
self.toks = _tokens(text)
|
|
120
|
+
self.i = 0
|
|
121
|
+
|
|
122
|
+
def peek(self) -> tuple[str, str] | None:
|
|
123
|
+
return self.toks[self.i] if self.i < len(self.toks) else None
|
|
124
|
+
|
|
125
|
+
def take(self, value: str | None = None) -> tuple[str, str]:
|
|
126
|
+
tok = self.peek()
|
|
127
|
+
if tok is None or (value is not None and tok[1] != value):
|
|
128
|
+
raise GrepError(f"{self.text!r}: expected {value or 'more input'}")
|
|
129
|
+
self.i += 1
|
|
130
|
+
return tok
|
|
131
|
+
|
|
132
|
+
def parse(self) -> Node:
|
|
133
|
+
node = self.or_()
|
|
134
|
+
if self.peek() is not None:
|
|
135
|
+
raise GrepError(f"{self.text!r}: unexpected {self.peek()[1]!r}") # type: ignore[index]
|
|
136
|
+
return node
|
|
137
|
+
|
|
138
|
+
def or_(self) -> Node:
|
|
139
|
+
node = self.and_()
|
|
140
|
+
while self.peek() == ("name", "or"):
|
|
141
|
+
self.take()
|
|
142
|
+
node = Binary("or", node, self.and_())
|
|
143
|
+
return node
|
|
144
|
+
|
|
145
|
+
def and_(self) -> Node:
|
|
146
|
+
node = self.not_()
|
|
147
|
+
while self.peek() == ("name", "and"):
|
|
148
|
+
self.take()
|
|
149
|
+
node = Binary("and", node, self.not_())
|
|
150
|
+
return node
|
|
151
|
+
|
|
152
|
+
def not_(self) -> Node:
|
|
153
|
+
if self.peek() == ("name", "not"):
|
|
154
|
+
self.take()
|
|
155
|
+
return Unary("not", self.not_())
|
|
156
|
+
return self.cmp()
|
|
157
|
+
|
|
158
|
+
def cmp(self) -> Node:
|
|
159
|
+
node = self.sum()
|
|
160
|
+
tok = self.peek()
|
|
161
|
+
if tok and tok[0] == "op" and tok[1] in (">", "<", ">=", "<=", "==", "!="):
|
|
162
|
+
self.take()
|
|
163
|
+
node = Binary(tok[1], node, self.sum())
|
|
164
|
+
return node
|
|
165
|
+
|
|
166
|
+
def sum(self) -> Node:
|
|
167
|
+
node = self.term()
|
|
168
|
+
while (tok := self.peek()) and tok[1] in ("+", "-") and tok[0] == "op":
|
|
169
|
+
self.take()
|
|
170
|
+
node = Binary(tok[1], node, self.term())
|
|
171
|
+
return node
|
|
172
|
+
|
|
173
|
+
def term(self) -> Node:
|
|
174
|
+
node = self.unary()
|
|
175
|
+
while (tok := self.peek()) and tok[1] in ("*", "/") and tok[0] == "op":
|
|
176
|
+
self.take()
|
|
177
|
+
node = Binary(tok[1], node, self.unary())
|
|
178
|
+
return node
|
|
179
|
+
|
|
180
|
+
def unary(self) -> Node:
|
|
181
|
+
if self.peek() == ("op", "-"):
|
|
182
|
+
self.take()
|
|
183
|
+
return Unary("-", self.unary())
|
|
184
|
+
return self.atom()
|
|
185
|
+
|
|
186
|
+
def atom(self) -> Node:
|
|
187
|
+
tok = self.peek()
|
|
188
|
+
if tok is None:
|
|
189
|
+
raise GrepError(f"{self.text!r}: expression ends early")
|
|
190
|
+
kind, value = tok
|
|
191
|
+
if kind == "num":
|
|
192
|
+
self.take()
|
|
193
|
+
return Num(float(value))
|
|
194
|
+
if kind == "sigma":
|
|
195
|
+
self.take()
|
|
196
|
+
return Num(float(value), sigma=True)
|
|
197
|
+
if value == "(":
|
|
198
|
+
self.take()
|
|
199
|
+
node = self.or_()
|
|
200
|
+
self.take(")")
|
|
201
|
+
return node
|
|
202
|
+
if kind == "name" and value in FUNCS:
|
|
203
|
+
self.take()
|
|
204
|
+
self.take("(")
|
|
205
|
+
arg = self._arg(value)
|
|
206
|
+
self.take(")")
|
|
207
|
+
return Call(value, arg)
|
|
208
|
+
if kind == "name" and value in FIELDS:
|
|
209
|
+
self.take()
|
|
210
|
+
return Field(value)
|
|
211
|
+
raise GrepError(
|
|
212
|
+
f"{self.text!r}: unknown name {value!r}; fields are {', '.join(sorted(FIELDS))}"
|
|
213
|
+
)
|
|
214
|
+
|
|
215
|
+
def _arg(self, func: str) -> Any:
|
|
216
|
+
tok = self.peek()
|
|
217
|
+
if tok is None:
|
|
218
|
+
raise GrepError(f"{func}() needs an argument")
|
|
219
|
+
if func == "abs":
|
|
220
|
+
inner: Node = self.or_()
|
|
221
|
+
return inner
|
|
222
|
+
self.take()
|
|
223
|
+
kind, value = tok
|
|
224
|
+
if kind == "str":
|
|
225
|
+
return _unquote(value)
|
|
226
|
+
if kind == "num":
|
|
227
|
+
return int(float(value))
|
|
228
|
+
if kind == "handle":
|
|
229
|
+
return value
|
|
230
|
+
raise GrepError(f"{func}() takes a token string, a token id or a dir: handle")
|
|
231
|
+
|
|
232
|
+
|
|
233
|
+
def _unquote(text: str) -> str:
|
|
234
|
+
body = text[1:-1]
|
|
235
|
+
return re.sub(r"\\(.)", r"\1", body)
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def parse(text: str) -> Node:
|
|
239
|
+
return _Parser(text).parse()
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def ranking_key(node: Node) -> Node:
|
|
243
|
+
"""The value hits are ranked by."""
|
|
244
|
+
if isinstance(node, Binary) and node.op in ("and", "or"):
|
|
245
|
+
return ranking_key(node.left)
|
|
246
|
+
if isinstance(node, Unary) and node.op == "not":
|
|
247
|
+
return ranking_key(node.operand)
|
|
248
|
+
if isinstance(node, Binary) and node.op in (">", "<", ">=", "<=", "==", "!="):
|
|
249
|
+
left: Node = node.left
|
|
250
|
+
return left
|
|
251
|
+
return node
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def describe(node: Node) -> str:
|
|
255
|
+
"""Compact text for slugs: ``writes(" Wait")`` → ``writes-wait``."""
|
|
256
|
+
if isinstance(node, Call):
|
|
257
|
+
return f"{node.func}-{node.arg}"
|
|
258
|
+
if isinstance(node, Field):
|
|
259
|
+
return node.name
|
|
260
|
+
if isinstance(node, Binary):
|
|
261
|
+
return describe(node.left)
|
|
262
|
+
if isinstance(node, Unary):
|
|
263
|
+
return describe(node.operand)
|
|
264
|
+
return "value"
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def uses_fields(node: Node) -> set[str]:
|
|
268
|
+
if isinstance(node, Field):
|
|
269
|
+
return {node.name}
|
|
270
|
+
if isinstance(node, Call):
|
|
271
|
+
return {node.func} | (
|
|
272
|
+
uses_fields(node.arg)
|
|
273
|
+
if isinstance(node.arg, Num | Field | Call | Unary | Binary)
|
|
274
|
+
else set()
|
|
275
|
+
)
|
|
276
|
+
if isinstance(node, Unary):
|
|
277
|
+
return uses_fields(node.operand)
|
|
278
|
+
if isinstance(node, Binary):
|
|
279
|
+
return uses_fields(node.left) | uses_fields(node.right)
|
|
280
|
+
return set()
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
# --------------------------------------------------------------------------- units
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
@dataclass
|
|
287
|
+
class UnitBatch:
|
|
288
|
+
"""Units of one tensor (or one expert), as rows of ``vectors``."""
|
|
289
|
+
|
|
290
|
+
target: Target
|
|
291
|
+
expert: int | None
|
|
292
|
+
vectors: torch.Tensor
|
|
293
|
+
"""``[n_units, length]`` fp32."""
|
|
294
|
+
locals: list[int]
|
|
295
|
+
"""Unit index within the role's view (what an address means)."""
|
|
296
|
+
axis: str
|
|
297
|
+
whole: torch.Tensor | None = None
|
|
298
|
+
addresses: list[str] = field(default_factory=list)
|
|
299
|
+
globals: list[int] = field(default_factory=list)
|
|
300
|
+
"""Unit index along the tensor dim, for result-set handles."""
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
@dataclass
|
|
304
|
+
class Environment:
|
|
305
|
+
"""What functions need: vocabulary vectors, directions and folded norm scales."""
|
|
306
|
+
|
|
307
|
+
ir: ModelIR
|
|
308
|
+
params: dict[str, torch.Tensor]
|
|
309
|
+
token_id: Callable[[str | int], int] | None = None
|
|
310
|
+
direction: Callable[[str], torch.Tensor] | None = None
|
|
311
|
+
_cache: dict[str, torch.Tensor] = field(default_factory=dict)
|
|
312
|
+
|
|
313
|
+
def vocab_row(self, kind: str, token: str | int) -> torch.Tensor:
|
|
314
|
+
if self.token_id is None and isinstance(token, str):
|
|
315
|
+
raise GrepError(
|
|
316
|
+
f'{kind}("{token}") needs a tokenizer; use a token id instead, e.g. {kind}(1234)'
|
|
317
|
+
)
|
|
318
|
+
tid = token if isinstance(token, int) else self.token_id(token) # type: ignore[misc]
|
|
319
|
+
sem = self.ir.semantics
|
|
320
|
+
name = sem.unembed if kind == "writes" else sem.embed
|
|
321
|
+
if name is None:
|
|
322
|
+
raise GrepError(
|
|
323
|
+
f"{kind}() needs a model with {'an unembedding' if kind == 'writes' else 'an embedding'}"
|
|
324
|
+
)
|
|
325
|
+
weight = self.params[name]
|
|
326
|
+
role = next(
|
|
327
|
+
(
|
|
328
|
+
r
|
|
329
|
+
for r in self.ir.roles_of(name)
|
|
330
|
+
if r.kind == ("unembed" if kind == "writes" else "embed")
|
|
331
|
+
),
|
|
332
|
+
None,
|
|
333
|
+
)
|
|
334
|
+
vocab_dim = (role.out_dim if kind == "writes" else 0) if role else 0
|
|
335
|
+
vocab_dim = 0 if vocab_dim is None else vocab_dim
|
|
336
|
+
if not 0 <= tid < weight.shape[vocab_dim]:
|
|
337
|
+
raise GrepError(
|
|
338
|
+
f"token id {tid} outside the vocabulary (0-{weight.shape[vocab_dim] - 1})"
|
|
339
|
+
)
|
|
340
|
+
return weight.detach().select(vocab_dim, tid).float()
|
|
341
|
+
|
|
342
|
+
def final_scale(self) -> torch.Tensor | None:
|
|
343
|
+
if "final" not in self._cache:
|
|
344
|
+
names = self.ir.semantics.final_norm
|
|
345
|
+
scale = (
|
|
346
|
+
self.params[names[0]].detach().float()
|
|
347
|
+
if names and self.params[names[0]].dim() == 1
|
|
348
|
+
else None
|
|
349
|
+
)
|
|
350
|
+
if scale is not None:
|
|
351
|
+
self._cache["final"] = scale
|
|
352
|
+
return self._cache.get("final")
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def unit_batches(
|
|
356
|
+
ir: ModelIR, params: dict[str, torch.Tensor], targets: list[Target], axis: str | None
|
|
357
|
+
) -> Iterator[UnitBatch]:
|
|
358
|
+
"""Split each target into units along ``axis`` (default: the role's unit axis)."""
|
|
359
|
+
for target in targets:
|
|
360
|
+
weight = params[target.param].detach()
|
|
361
|
+
index = normalize_index(target.index, weight.dim())
|
|
362
|
+
role = target.role or ir.primary_role(target.param)
|
|
363
|
+
experts = role is not None and role.experts is not None and isinstance(index[0], slice)
|
|
364
|
+
if experts:
|
|
365
|
+
count = weight.shape[0]
|
|
366
|
+
rows = range(*index[0].indices(count)) # type: ignore[union-attr]
|
|
367
|
+
for e in rows:
|
|
368
|
+
sub = Target(target.param, (e, *index[1:]), target.address, target.role)
|
|
369
|
+
yield from _units(ir, weight, sub, axis, expert=e)
|
|
370
|
+
else:
|
|
371
|
+
expert = (
|
|
372
|
+
index[0]
|
|
373
|
+
if isinstance(index[0], int) and role is not None and role.experts
|
|
374
|
+
else None
|
|
375
|
+
)
|
|
376
|
+
yield from _units(ir, weight, target, axis, expert=expert)
|
|
377
|
+
|
|
378
|
+
|
|
379
|
+
def _units(
|
|
380
|
+
ir: ModelIR, weight: torch.Tensor, target: Target, axis: str | None, expert: int | None
|
|
381
|
+
) -> Iterator[UnitBatch]:
|
|
382
|
+
index = normalize_index(target.index, weight.dim())
|
|
383
|
+
block = select(weight, index).float()
|
|
384
|
+
free = [d for d, s in enumerate(index) if not isinstance(s, int)]
|
|
385
|
+
if block.dim() == 0:
|
|
386
|
+
block = block.reshape(1)
|
|
387
|
+
free = [0]
|
|
388
|
+
if axis is None:
|
|
389
|
+
axis = _default_axis(ir, target, free)
|
|
390
|
+
role = target.role or ir.primary_role(target.param)
|
|
391
|
+
base = role.address if role is not None and target.role is not None else target.param
|
|
392
|
+
if expert is not None and ".experts." in base:
|
|
393
|
+
base = base.replace(".experts.", f".e{expert}.")
|
|
394
|
+
if axis == "tensor":
|
|
395
|
+
yield UnitBatch(
|
|
396
|
+
target,
|
|
397
|
+
expert,
|
|
398
|
+
block.reshape(1, -1),
|
|
399
|
+
[0],
|
|
400
|
+
"tensor",
|
|
401
|
+
whole=block,
|
|
402
|
+
addresses=[base],
|
|
403
|
+
globals=[0],
|
|
404
|
+
)
|
|
405
|
+
return
|
|
406
|
+
if block.dim() == 1:
|
|
407
|
+
if axis not in ("elem", "row", "col"):
|
|
408
|
+
raise GrepError(f"{target.address} is 1-D; use --axis elem or tensor")
|
|
409
|
+
unit_dim, axis = free[0], "elem"
|
|
410
|
+
vectors = block.reshape(-1, 1)
|
|
411
|
+
elif block.dim() == 2:
|
|
412
|
+
unit_dim = free[0] if axis == "row" else free[1]
|
|
413
|
+
vectors = block if axis == "row" else block.T
|
|
414
|
+
else:
|
|
415
|
+
raise GrepError(f"{target.address}: {block.dim()}-D selection; narrow it to a matrix")
|
|
416
|
+
size = weight.shape[unit_dim]
|
|
417
|
+
sel = index[unit_dim]
|
|
418
|
+
globals_ = list(range(*sel.indices(size))) if isinstance(sel, slice) else list(sel) # type: ignore[arg-type]
|
|
419
|
+
view = list(range(size))
|
|
420
|
+
if target.role is not None:
|
|
421
|
+
rsel = ir._role_index(target.role)[unit_dim]
|
|
422
|
+
view = list(range(*rsel.indices(size))) if isinstance(rsel, slice) else list(rsel) # type: ignore[arg-type]
|
|
423
|
+
position = {g: i for i, g in enumerate(view)}
|
|
424
|
+
locals_ = [position.get(g, g) for g in globals_]
|
|
425
|
+
in_view = target.role is not None and all(g in position for g in globals_)
|
|
426
|
+
name = base if in_view else target.param
|
|
427
|
+
pick = locals_ if in_view else globals_
|
|
428
|
+
addresses = (
|
|
429
|
+
[f"{name}[{u}]" for u in pick]
|
|
430
|
+
if axis in ("row", "elem")
|
|
431
|
+
else [f"{name}[:, {u}]" for u in pick]
|
|
432
|
+
)
|
|
433
|
+
yield UnitBatch(
|
|
434
|
+
target, expert, vectors, pick, axis, whole=block, addresses=addresses, globals=globals_
|
|
435
|
+
)
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def _default_axis(ir: ModelIR, target: Target, free: list[int]) -> str:
|
|
439
|
+
role = target.role or ir.primary_role(target.param)
|
|
440
|
+
if len(free) != 2 or role is None or role.in_dim is None or role.out_dim is None:
|
|
441
|
+
return "row" if len(free) == 2 else ("elem" if len(free) == 1 else "tensor")
|
|
442
|
+
writes_residual = role.kind in ("writer", "embed", "pos_embed", "stem") or (
|
|
443
|
+
role.kind == "expert" and role.address.endswith(".down")
|
|
444
|
+
)
|
|
445
|
+
unit_dim = role.in_dim if writes_residual else role.out_dim
|
|
446
|
+
if role.kind in ("embed", "unembed", "pos_embed"):
|
|
447
|
+
unit_dim = 0 if role.kind != "unembed" else role.out_dim
|
|
448
|
+
return "col" if free.index(unit_dim) == 1 else "row"
|
|
449
|
+
|
|
450
|
+
|
|
451
|
+
# --------------------------------------------------------------------------- evaluation
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
class _Eval:
|
|
455
|
+
def __init__(self, env: Environment, batch: UnitBatch) -> None:
|
|
456
|
+
self.env = env
|
|
457
|
+
self.b = batch
|
|
458
|
+
self._memo: dict[str, torch.Tensor] = {}
|
|
459
|
+
|
|
460
|
+
def field(self, name: str) -> torch.Tensor:
|
|
461
|
+
if name in self._memo:
|
|
462
|
+
return self._memo[name]
|
|
463
|
+
v = self.b.vectors
|
|
464
|
+
out: torch.Tensor
|
|
465
|
+
if name in ("norm", "colnorm", "rownorm"):
|
|
466
|
+
if name != "norm" and self.b.axis not in (name[:3], "tensor", "elem"):
|
|
467
|
+
raise GrepError(f"{name} with --axis {self.b.axis}; drop --axis or use norm")
|
|
468
|
+
out = v.norm(dim=1)
|
|
469
|
+
elif name == "absmax":
|
|
470
|
+
out = v.abs().amax(dim=1)
|
|
471
|
+
elif name == "max":
|
|
472
|
+
out = v.amax(dim=1)
|
|
473
|
+
elif name == "min":
|
|
474
|
+
out = v.amin(dim=1)
|
|
475
|
+
elif name == "mean":
|
|
476
|
+
out = v.mean(dim=1)
|
|
477
|
+
elif name == "std":
|
|
478
|
+
out = (
|
|
479
|
+
v.std(dim=1, correction=0)
|
|
480
|
+
if v.shape[1] > 1
|
|
481
|
+
else torch.zeros(v.shape[0], device=v.device)
|
|
482
|
+
)
|
|
483
|
+
elif name == "kurtosis":
|
|
484
|
+
mu = v.mean(dim=1, keepdim=True)
|
|
485
|
+
var = ((v - mu) ** 2).mean(dim=1)
|
|
486
|
+
out = ((v - mu) ** 4).mean(dim=1) / var.clamp_min(1e-30) ** 2 - 3.0
|
|
487
|
+
elif name == "rank":
|
|
488
|
+
whole = self.b.whole if self.b.whole is not None else v
|
|
489
|
+
mat = whole if whole.dim() == 2 else whole.reshape(1, -1)
|
|
490
|
+
out = torch.full((v.shape[0],), float(torch.linalg.matrix_rank(mat)), device=v.device)
|
|
491
|
+
elif name == "nan":
|
|
492
|
+
out = torch.isnan(v).any(dim=1).float()
|
|
493
|
+
elif name == "inf":
|
|
494
|
+
out = torch.isinf(v).any(dim=1).float()
|
|
495
|
+
else: # pragma: no cover - parser restricts names
|
|
496
|
+
raise GrepError(f"unknown field {name}")
|
|
497
|
+
self._memo[name] = out
|
|
498
|
+
return out
|
|
499
|
+
|
|
500
|
+
def cosine(self, ref: torch.Tensor, scale: torch.Tensor | None = None) -> torch.Tensor:
|
|
501
|
+
v = self.b.vectors
|
|
502
|
+
if v.shape[1] != ref.numel():
|
|
503
|
+
raise GrepError(
|
|
504
|
+
f"{self.b.target.address}: units have length {v.shape[1]}, the reference vector "
|
|
505
|
+
f"{ref.numel()}; pick units that live in the residual stream (writers' columns, readers' rows)"
|
|
506
|
+
)
|
|
507
|
+
ref = ref.to(v.device)
|
|
508
|
+
if scale is not None:
|
|
509
|
+
v = v * scale.to(v.device)
|
|
510
|
+
result: torch.Tensor = (v @ ref) / (v.norm(dim=1) * ref.norm()).clamp_min(1e-30)
|
|
511
|
+
return result
|
|
512
|
+
|
|
513
|
+
def eval(self, node: Node) -> torch.Tensor | float:
|
|
514
|
+
if isinstance(node, Num):
|
|
515
|
+
if node.sigma:
|
|
516
|
+
raise GrepError(
|
|
517
|
+
"σ thresholds belong on the right of a comparison, e.g. colnorm > 4σ"
|
|
518
|
+
)
|
|
519
|
+
return node.value
|
|
520
|
+
if isinstance(node, Field):
|
|
521
|
+
return self.field(node.name)
|
|
522
|
+
if isinstance(node, Call):
|
|
523
|
+
if node.func == "abs":
|
|
524
|
+
inner = self.eval(node.arg)
|
|
525
|
+
return inner.abs() if isinstance(inner, torch.Tensor) else abs(inner)
|
|
526
|
+
if node.func == "writes":
|
|
527
|
+
return self.cosine(self.env.vocab_row("writes", node.arg), self.env.final_scale())
|
|
528
|
+
if node.func == "reads":
|
|
529
|
+
return self.cosine(self.env.vocab_row("reads", node.arg), self._pre_norm_scale())
|
|
530
|
+
if node.func == "cos":
|
|
531
|
+
if self.env.direction is None:
|
|
532
|
+
raise GrepError("cos() needs a session with directions")
|
|
533
|
+
return self.cosine(self.env.direction(str(node.arg)))
|
|
534
|
+
if isinstance(node, Unary):
|
|
535
|
+
val = self.eval(node.operand)
|
|
536
|
+
if node.op == "-":
|
|
537
|
+
return -val
|
|
538
|
+
return (
|
|
539
|
+
_as_bool(val).logical_not().float()
|
|
540
|
+
if isinstance(val, torch.Tensor)
|
|
541
|
+
else float(not val)
|
|
542
|
+
)
|
|
543
|
+
if isinstance(node, Binary):
|
|
544
|
+
if node.op in ("and", "or"):
|
|
545
|
+
a, b = _as_bool(self.eval(node.left)), _as_bool(self.eval(node.right))
|
|
546
|
+
return (a & b if node.op == "and" else a | b).float()
|
|
547
|
+
left = self.eval(node.left)
|
|
548
|
+
right = node.right
|
|
549
|
+
if isinstance(right, Num) and right.sigma:
|
|
550
|
+
if not isinstance(left, torch.Tensor):
|
|
551
|
+
raise GrepError("σ needs a per-unit quantity on the left")
|
|
552
|
+
rv: Any = self.env._cache.get(f"sigma:{id(node)}")
|
|
553
|
+
if rv is None:
|
|
554
|
+
raise GrepError("internal: σ threshold not prepared")
|
|
555
|
+
else:
|
|
556
|
+
rv = self.eval(right)
|
|
557
|
+
value: torch.Tensor | float = _apply(node.op, left, rv)
|
|
558
|
+
return value
|
|
559
|
+
raise GrepError(f"cannot evaluate {node!r}")
|
|
560
|
+
|
|
561
|
+
def _pre_norm_scale(self) -> torch.Tensor | None:
|
|
562
|
+
role = self.b.target.role
|
|
563
|
+
if role is None or role.block is None or role.branch is None:
|
|
564
|
+
return None
|
|
565
|
+
norm = next(
|
|
566
|
+
(
|
|
567
|
+
r
|
|
568
|
+
for r in self.env.ir.roles
|
|
569
|
+
if r.block == role.block
|
|
570
|
+
and r.branch == role.branch
|
|
571
|
+
and r.address.endswith(f"{role.branch}.norm")
|
|
572
|
+
),
|
|
573
|
+
None,
|
|
574
|
+
)
|
|
575
|
+
if norm is None:
|
|
576
|
+
return None
|
|
577
|
+
weight = self.env.params[norm.param]
|
|
578
|
+
return weight.detach().float() if weight.dim() == 1 else None
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def _as_bool(v: torch.Tensor | float) -> torch.Tensor:
|
|
582
|
+
if isinstance(v, torch.Tensor):
|
|
583
|
+
return v != 0
|
|
584
|
+
return torch.tensor(bool(v))
|
|
585
|
+
|
|
586
|
+
|
|
587
|
+
def _apply(op: str, a: Any, b: Any) -> Any:
|
|
588
|
+
if op == ">":
|
|
589
|
+
return (
|
|
590
|
+
(a > b).float()
|
|
591
|
+
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)
|
|
592
|
+
else float(a > b)
|
|
593
|
+
)
|
|
594
|
+
if op == "<":
|
|
595
|
+
return (
|
|
596
|
+
(a < b).float()
|
|
597
|
+
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)
|
|
598
|
+
else float(a < b)
|
|
599
|
+
)
|
|
600
|
+
if op == ">=":
|
|
601
|
+
return (
|
|
602
|
+
(a >= b).float()
|
|
603
|
+
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)
|
|
604
|
+
else float(a >= b)
|
|
605
|
+
)
|
|
606
|
+
if op == "<=":
|
|
607
|
+
return (
|
|
608
|
+
(a <= b).float()
|
|
609
|
+
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)
|
|
610
|
+
else float(a <= b)
|
|
611
|
+
)
|
|
612
|
+
if op == "==":
|
|
613
|
+
return (
|
|
614
|
+
(a == b).float()
|
|
615
|
+
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)
|
|
616
|
+
else float(a == b)
|
|
617
|
+
)
|
|
618
|
+
if op == "!=":
|
|
619
|
+
return (
|
|
620
|
+
(a != b).float()
|
|
621
|
+
if isinstance(a, torch.Tensor) or isinstance(b, torch.Tensor)
|
|
622
|
+
else float(a != b)
|
|
623
|
+
)
|
|
624
|
+
if op == "+":
|
|
625
|
+
return a + b
|
|
626
|
+
if op == "-":
|
|
627
|
+
return a - b
|
|
628
|
+
if op == "*":
|
|
629
|
+
return a * b
|
|
630
|
+
if op == "/":
|
|
631
|
+
return a / b
|
|
632
|
+
raise GrepError(f"unknown operator {op}")
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _is_boolean(node: Node) -> bool:
|
|
636
|
+
if isinstance(node, Binary):
|
|
637
|
+
return node.op in (">", "<", ">=", "<=", "==", "!=", "and", "or")
|
|
638
|
+
if isinstance(node, Unary):
|
|
639
|
+
return node.op == "not"
|
|
640
|
+
return isinstance(node, Field) and node.name in ("nan", "inf")
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _sigma_nodes(node: Node) -> list[Binary]:
|
|
644
|
+
out: list[Binary] = []
|
|
645
|
+
if isinstance(node, Binary):
|
|
646
|
+
if isinstance(node.right, Num) and node.right.sigma:
|
|
647
|
+
out.append(node)
|
|
648
|
+
out += _sigma_nodes(node.left) + _sigma_nodes(node.right)
|
|
649
|
+
elif isinstance(node, Unary):
|
|
650
|
+
out += _sigma_nodes(node.operand)
|
|
651
|
+
return out
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
# --------------------------------------------------------------------------- run
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
@dataclass
|
|
658
|
+
class Hit:
|
|
659
|
+
address: str
|
|
660
|
+
param: str
|
|
661
|
+
expert: int | None
|
|
662
|
+
unit: int
|
|
663
|
+
score: float
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
@dataclass
|
|
667
|
+
class GrepResult:
|
|
668
|
+
expression: str
|
|
669
|
+
scope: str
|
|
670
|
+
axis: str
|
|
671
|
+
candidates: int
|
|
672
|
+
hits: list[Hit]
|
|
673
|
+
"""Every unit that matched, best first (the full ranked list)."""
|
|
674
|
+
scores: list[Hit]
|
|
675
|
+
"""Every candidate unit with its ranking score, best first."""
|
|
676
|
+
seconds: float = 0.0
|
|
677
|
+
|
|
678
|
+
|
|
679
|
+
def grep(
|
|
680
|
+
env: Environment,
|
|
681
|
+
expression: str,
|
|
682
|
+
targets: list[Target],
|
|
683
|
+
*,
|
|
684
|
+
axis: str | None = None,
|
|
685
|
+
) -> GrepResult:
|
|
686
|
+
"""Evaluate ``expression`` over every unit of ``targets``."""
|
|
687
|
+
import time
|
|
688
|
+
|
|
689
|
+
tree = parse(expression)
|
|
690
|
+
key = ranking_key(tree)
|
|
691
|
+
boolean = _is_boolean(tree)
|
|
692
|
+
started = time.perf_counter()
|
|
693
|
+
batches = list(unit_batches(env.ir, env.params, targets, axis))
|
|
694
|
+
if not batches:
|
|
695
|
+
raise GrepError("scope contains no weights")
|
|
696
|
+
|
|
697
|
+
# σ thresholds need the population of the left side across the whole scope.
|
|
698
|
+
for node in _sigma_nodes(tree):
|
|
699
|
+
values = torch.cat([_flat(_Eval(env, b).eval(node.left), b) for b in batches])
|
|
700
|
+
mu, sd = (
|
|
701
|
+
float(values.mean()),
|
|
702
|
+
float(values.std(correction=0)) if values.numel() > 1 else 0.0,
|
|
703
|
+
)
|
|
704
|
+
right = node.right
|
|
705
|
+
assert isinstance(right, Num)
|
|
706
|
+
env._cache[f"sigma:{id(node)}"] = torch.tensor(mu + right.value * sd)
|
|
707
|
+
|
|
708
|
+
scores: list[Hit] = []
|
|
709
|
+
hits: list[Hit] = []
|
|
710
|
+
used_axis = batches[0].axis
|
|
711
|
+
for batch in batches:
|
|
712
|
+
ev = _Eval(env, batch)
|
|
713
|
+
score = _flat(ev.eval(key), batch)
|
|
714
|
+
keep = (
|
|
715
|
+
_flat(ev.eval(tree), batch) != 0
|
|
716
|
+
if boolean
|
|
717
|
+
else torch.ones_like(score, dtype=torch.bool)
|
|
718
|
+
)
|
|
719
|
+
score_l = score.detach().cpu().tolist()
|
|
720
|
+
keep_l = keep.detach().cpu().tolist()
|
|
721
|
+
for address, unit, s, k in zip(
|
|
722
|
+
batch.addresses, batch.globals, score_l, keep_l, strict=True
|
|
723
|
+
):
|
|
724
|
+
hit = Hit(
|
|
725
|
+
address,
|
|
726
|
+
batch.target.param,
|
|
727
|
+
batch.expert,
|
|
728
|
+
unit,
|
|
729
|
+
float("nan") if math.isnan(s) else s,
|
|
730
|
+
)
|
|
731
|
+
scores.append(hit)
|
|
732
|
+
if k:
|
|
733
|
+
hits.append(hit)
|
|
734
|
+
order = lambda h: (math.isnan(h.score), -h.score if not math.isnan(h.score) else 0.0) # noqa: E731
|
|
735
|
+
scores.sort(key=order)
|
|
736
|
+
hits.sort(key=order)
|
|
737
|
+
return GrepResult(
|
|
738
|
+
expression,
|
|
739
|
+
", ".join(sorted({t.address for t in targets})),
|
|
740
|
+
used_axis,
|
|
741
|
+
len(scores),
|
|
742
|
+
hits,
|
|
743
|
+
scores,
|
|
744
|
+
time.perf_counter() - started,
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
|
|
748
|
+
def _flat(value: torch.Tensor | float, batch: UnitBatch) -> torch.Tensor:
|
|
749
|
+
n = batch.vectors.shape[0]
|
|
750
|
+
if isinstance(value, torch.Tensor):
|
|
751
|
+
if value.dim() == 0:
|
|
752
|
+
return value.expand(n).float()
|
|
753
|
+
return value.float()
|
|
754
|
+
return torch.full((n,), float(value), device=batch.vectors.device)
|