wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/check.py
ADDED
|
@@ -0,0 +1,489 @@
|
|
|
1
|
+
"""Checks: "does this edit compile?" (§5.4).
|
|
2
|
+
|
|
3
|
+
* **Static** (IR only, milliseconds): addresses, shapes, base hashes, op/role
|
|
4
|
+
validity, tied weights, fused sub-role boundaries, declared scope, quantized
|
|
5
|
+
storage.
|
|
6
|
+
* **Numeric** (weights, under a second): non-finite values, overflow, sub-ULP
|
|
7
|
+
loss, relative norm change, spectrum shift, no-ops, dense-edit size.
|
|
8
|
+
* **Smoke** (a short forward pass): KL against the base on a fixed canary,
|
|
9
|
+
perplexity change and degenerate output.
|
|
10
|
+
|
|
11
|
+
Every diagnostic carries fixes. Severity scales with damage: sub-ULP loss
|
|
12
|
+
warns below ``subulp_error_at`` and blocks above it.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import itertools
|
|
18
|
+
import math
|
|
19
|
+
from collections import defaultdict
|
|
20
|
+
from dataclasses import dataclass, field
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import torch
|
|
24
|
+
|
|
25
|
+
from wspx.config import CheckSection
|
|
26
|
+
from wspx.diagnostics import Code, Fix, Report, Severity, diag
|
|
27
|
+
from wspx.lab.ir.graph import expand_runs
|
|
28
|
+
from wspx.lab.ir.model_ir import ModelIR
|
|
29
|
+
from wspx.lab.patch import TensorDelta
|
|
30
|
+
from wspx.lab.tree import WorkingTree
|
|
31
|
+
|
|
32
|
+
WRITER_OPS = frozenset({"project_out", "bake_steer", "rank_one_write"})
|
|
33
|
+
SUBULP_WARN_AT = 0.05
|
|
34
|
+
SPECTRUM_SHIFT = 0.10
|
|
35
|
+
SPECTRUM_MAX_NUMEL = 64 * 1024 * 1024
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass
|
|
39
|
+
class EditPlan:
|
|
40
|
+
deltas: list[TensorDelta]
|
|
41
|
+
op: str = "edit"
|
|
42
|
+
precision: str = "merged"
|
|
43
|
+
scope: list[str] | None = None
|
|
44
|
+
"""Declared parameter names the edit may touch (from the addresses given)."""
|
|
45
|
+
args: dict[str, Any] = field(default_factory=dict)
|
|
46
|
+
|
|
47
|
+
def by_tensor(self) -> dict[str, list[TensorDelta]]:
|
|
48
|
+
out: dict[str, list[TensorDelta]] = defaultdict(list)
|
|
49
|
+
for d in self.deltas:
|
|
50
|
+
out[d.tensor].append(d)
|
|
51
|
+
return dict(out)
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
# --------------------------------------------------------------------------- static
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def static_check(ir: ModelIR, plan: EditPlan) -> Report:
|
|
58
|
+
report = Report(levels=["static"])
|
|
59
|
+
add = report.diagnostics.append
|
|
60
|
+
for name, deltas in plan.by_tensor().items():
|
|
61
|
+
info = ir.params.get(name)
|
|
62
|
+
if info is None:
|
|
63
|
+
add(diag(Code.UNRESOLVED_ADDRESS, "no such parameter", target=name))
|
|
64
|
+
continue
|
|
65
|
+
target = _display(ir, name)
|
|
66
|
+
for d in deltas:
|
|
67
|
+
if tuple(d.shape) != tuple(info.shape):
|
|
68
|
+
add(
|
|
69
|
+
diag(
|
|
70
|
+
Code.SHAPE_MISMATCH,
|
|
71
|
+
f"delta shape {tuple(d.shape)} vs tensor {info.shape}",
|
|
72
|
+
target=target,
|
|
73
|
+
)
|
|
74
|
+
)
|
|
75
|
+
if info.hash and d.base != info.hash:
|
|
76
|
+
add(
|
|
77
|
+
diag(
|
|
78
|
+
Code.BASE_MISMATCH,
|
|
79
|
+
"delta was computed against a different base tensor",
|
|
80
|
+
target=target,
|
|
81
|
+
fixes=[Fix(title="recompute the edit on this model")],
|
|
82
|
+
)
|
|
83
|
+
)
|
|
84
|
+
roles = ir.roles_of(name)
|
|
85
|
+
kinds = {r.kind for r in roles}
|
|
86
|
+
if plan.op in WRITER_OPS and (len(info.shape) < 2 and plan.op != "bake_steer"):
|
|
87
|
+
add(
|
|
88
|
+
diag(
|
|
89
|
+
Code.OP_INVALID_FOR_ROLE,
|
|
90
|
+
f"{plan.op} needs a matrix, {name} is {len(info.shape)}-D",
|
|
91
|
+
target=target,
|
|
92
|
+
fixes=[
|
|
93
|
+
Fix(
|
|
94
|
+
title="target a projection weight such as L*.mlp.down",
|
|
95
|
+
action={"address": "L*.mlp.down"},
|
|
96
|
+
)
|
|
97
|
+
],
|
|
98
|
+
)
|
|
99
|
+
)
|
|
100
|
+
if plan.op == "project_out" and kinds & {"norm", "final_norm", "bias"}:
|
|
101
|
+
add(
|
|
102
|
+
diag(
|
|
103
|
+
Code.OP_INVALID_FOR_ROLE,
|
|
104
|
+
f"project_out on a {'/'.join(sorted(kinds))} weight",
|
|
105
|
+
target=target,
|
|
106
|
+
)
|
|
107
|
+
)
|
|
108
|
+
if plan.op in WRITER_OPS and "reader" in kinds and "writer" not in kinds:
|
|
109
|
+
add(
|
|
110
|
+
diag(
|
|
111
|
+
Code.READER_NOT_WRITER,
|
|
112
|
+
"this edit touches a reader: it changes what the layer reads, not what it writes",
|
|
113
|
+
target=target,
|
|
114
|
+
severity=Severity.INFO,
|
|
115
|
+
fixes=[
|
|
116
|
+
Fix(
|
|
117
|
+
title="target the block's writer (o, down, out) to change what it writes"
|
|
118
|
+
)
|
|
119
|
+
],
|
|
120
|
+
)
|
|
121
|
+
)
|
|
122
|
+
if info.aliases or {"embed", "unembed"} <= kinds:
|
|
123
|
+
names = ", ".join([name, *info.aliases])
|
|
124
|
+
add(
|
|
125
|
+
diag(
|
|
126
|
+
Code.TIED_WEIGHT,
|
|
127
|
+
"tensor is tied: the edit changes both embedding and unembedding",
|
|
128
|
+
target=target,
|
|
129
|
+
detail=[f"shared storage: {names}"],
|
|
130
|
+
fixes=[
|
|
131
|
+
Fix(title="accept: both roles change together"),
|
|
132
|
+
Fix(
|
|
133
|
+
title='precision="adapter" keeps the change on one side',
|
|
134
|
+
action={"precision": "adapter"},
|
|
135
|
+
),
|
|
136
|
+
],
|
|
137
|
+
)
|
|
138
|
+
)
|
|
139
|
+
_fused_boundaries(ir, name, deltas, report)
|
|
140
|
+
if plan.scope is not None and name not in plan.scope:
|
|
141
|
+
add(
|
|
142
|
+
diag(
|
|
143
|
+
Code.OUTSIDE_SCOPE,
|
|
144
|
+
"edit touches a tensor outside the declared scope",
|
|
145
|
+
target=target,
|
|
146
|
+
)
|
|
147
|
+
)
|
|
148
|
+
dtype = info.dtype
|
|
149
|
+
if dtype in ("int8", "uint8", "int32") and plan.precision != "adapter":
|
|
150
|
+
add(
|
|
151
|
+
diag(
|
|
152
|
+
Code.QUANTIZED_NEEDS_ADAPTER,
|
|
153
|
+
f"{dtype} storage is quantized; small deltas would be lost",
|
|
154
|
+
target=target,
|
|
155
|
+
fixes=[Fix(title='precision="adapter"', action={"precision": "adapter"})],
|
|
156
|
+
)
|
|
157
|
+
)
|
|
158
|
+
elif dtype.startswith("float8") and plan.precision == "merged":
|
|
159
|
+
add(
|
|
160
|
+
diag(
|
|
161
|
+
Code.REQUANTIZED,
|
|
162
|
+
f"{dtype} target: the edit is requantized on materialisation",
|
|
163
|
+
target=target,
|
|
164
|
+
severity=Severity.INFO,
|
|
165
|
+
fixes=[
|
|
166
|
+
Fix(
|
|
167
|
+
title='precision="adapter" for small edits',
|
|
168
|
+
action={"precision": "adapter"},
|
|
169
|
+
)
|
|
170
|
+
],
|
|
171
|
+
)
|
|
172
|
+
)
|
|
173
|
+
return report
|
|
174
|
+
|
|
175
|
+
|
|
176
|
+
def _fused_boundaries(ir: ModelIR, name: str, deltas: list[TensorDelta], report: Report) -> None:
|
|
177
|
+
subs = [r for r in ir.roles_of(name) if r.split is not None and r.kind != "bias"]
|
|
178
|
+
if not subs:
|
|
179
|
+
return
|
|
180
|
+
touched_parts: set[str] = set()
|
|
181
|
+
for d in deltas:
|
|
182
|
+
if d.index is None or d.kind == "sparse":
|
|
183
|
+
touched_parts.update(r.address.rsplit(".", 1)[-1] for r in subs)
|
|
184
|
+
continue
|
|
185
|
+
role = subs[0]
|
|
186
|
+
dim = role.out_dim if role.out_dim is not None else 0
|
|
187
|
+
sel = d.index[dim] if dim < len(d.index) else slice(None)
|
|
188
|
+
size = ir.params[name].shape[dim]
|
|
189
|
+
rows = (
|
|
190
|
+
set(range(*sel.indices(size)))
|
|
191
|
+
if isinstance(sel, slice)
|
|
192
|
+
else ({sel} if isinstance(sel, int) else set(sel))
|
|
193
|
+
)
|
|
194
|
+
for r in subs:
|
|
195
|
+
if rows & set(expand_runs(r.split or [])):
|
|
196
|
+
touched_parts.add(r.address.rsplit(".", 1)[-1])
|
|
197
|
+
if len(touched_parts) > 1:
|
|
198
|
+
report.diagnostics.append(
|
|
199
|
+
diag(
|
|
200
|
+
Code.FUSED_SPLIT,
|
|
201
|
+
f"edit spans fused parts {', '.join(sorted(touched_parts))} of one weight",
|
|
202
|
+
target=_display(ir, name),
|
|
203
|
+
severity=Severity.INFO,
|
|
204
|
+
fixes=[
|
|
205
|
+
Fix(
|
|
206
|
+
title=f"address one part, e.g. {subs[0].address}",
|
|
207
|
+
action={"address": subs[0].address},
|
|
208
|
+
)
|
|
209
|
+
],
|
|
210
|
+
)
|
|
211
|
+
)
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
# --------------------------------------------------------------------------- numeric
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def numeric_check(ir: ModelIR, tree: WorkingTree, plan: EditPlan, cfg: CheckSection) -> Report:
|
|
218
|
+
report = Report(levels=["numeric"])
|
|
219
|
+
add = report.diagnostics.append
|
|
220
|
+
total_numel = sum(p.numel for p in ir.params.values()) or 1
|
|
221
|
+
touched = 0
|
|
222
|
+
unchanged = 0
|
|
223
|
+
tensors = plan.by_tensor()
|
|
224
|
+
ranks = [d.rank for d in plan.deltas if d.rank is not None]
|
|
225
|
+
for name, deltas in tensors.items():
|
|
226
|
+
if name not in tree.params:
|
|
227
|
+
continue
|
|
228
|
+
target = _display(ir, name)
|
|
229
|
+
old = tree.params[name].detach()
|
|
230
|
+
dtype = torch.float32 if plan.precision == "promote" else old.dtype
|
|
231
|
+
if plan.precision == "adapter":
|
|
232
|
+
continue
|
|
233
|
+
new = tree.preview(name, deltas, dtype=dtype)
|
|
234
|
+
intended = torch.zeros(old.shape, dtype=torch.float32, device=old.device)
|
|
235
|
+
for d in deltas:
|
|
236
|
+
d.add_into(intended)
|
|
237
|
+
if not torch.isfinite(intended).all() or not torch.isfinite(new.float()).all():
|
|
238
|
+
add(
|
|
239
|
+
diag(
|
|
240
|
+
Code.NON_FINITE,
|
|
241
|
+
"edit produces NaN or Inf",
|
|
242
|
+
target=target,
|
|
243
|
+
fixes=[Fix(title="check the direction and scale for zeros or overflow")],
|
|
244
|
+
)
|
|
245
|
+
)
|
|
246
|
+
continue
|
|
247
|
+
if old.is_floating_point() and dtype != torch.float32:
|
|
248
|
+
limit = torch.finfo(dtype).max
|
|
249
|
+
peak = float((old.float() + intended).abs().max())
|
|
250
|
+
if peak > limit:
|
|
251
|
+
add(
|
|
252
|
+
diag(
|
|
253
|
+
Code.DTYPE_OVERFLOW,
|
|
254
|
+
f"values reach {peak:.3g}, above {dtype} max {limit:.3g}",
|
|
255
|
+
target=target,
|
|
256
|
+
fixes=[Fix(title="reduce the edit's scale", action={"scale": 0.5})],
|
|
257
|
+
)
|
|
258
|
+
)
|
|
259
|
+
continue
|
|
260
|
+
effective = new.float() - old.float()
|
|
261
|
+
changed = (
|
|
262
|
+
int((new != old.to(new.dtype)).sum())
|
|
263
|
+
if new.dtype == old.dtype
|
|
264
|
+
else int((effective != 0).sum())
|
|
265
|
+
)
|
|
266
|
+
touched += changed
|
|
267
|
+
if changed == 0:
|
|
268
|
+
unchanged += 1
|
|
269
|
+
mass = float(intended.abs().sum())
|
|
270
|
+
if mass > 0 and plan.precision == "merged":
|
|
271
|
+
lost = float((intended - effective).abs().sum()) / mass
|
|
272
|
+
if lost >= SUBULP_WARN_AT:
|
|
273
|
+
report.diagnostics.append(_subulp(target, old, intended, lost, cfg, deltas))
|
|
274
|
+
w_norm = float(old.float().norm())
|
|
275
|
+
rel = float(effective.norm()) / w_norm if w_norm > 0 else float("inf")
|
|
276
|
+
report.stats.setdefault("rel_norm", {})[name] = rel
|
|
277
|
+
if rel > cfg.max_rel_norm:
|
|
278
|
+
add(
|
|
279
|
+
diag(
|
|
280
|
+
Code.REL_NORM,
|
|
281
|
+
f"relative norm change {rel:.3f} > {cfg.max_rel_norm}",
|
|
282
|
+
target=target,
|
|
283
|
+
fixes=[
|
|
284
|
+
Fix(
|
|
285
|
+
title="lower the strength", action={"strength": cfg.max_rel_norm / rel}
|
|
286
|
+
),
|
|
287
|
+
Fix(title="narrow the selection (fewer units or a lower rank)"),
|
|
288
|
+
],
|
|
289
|
+
)
|
|
290
|
+
)
|
|
291
|
+
if old.dim() == 2 and old.numel() <= SPECTRUM_MAX_NUMEL and rel > 1e-3:
|
|
292
|
+
s_old, s_new = _top_singular(old.float()), _top_singular(new.float())
|
|
293
|
+
shift = abs(s_new - s_old) / s_old if s_old > 0 else 0.0
|
|
294
|
+
report.stats.setdefault("sigma_shift", {})[name] = shift
|
|
295
|
+
if shift > SPECTRUM_SHIFT:
|
|
296
|
+
add(
|
|
297
|
+
diag(
|
|
298
|
+
Code.SPECTRUM_SHIFT,
|
|
299
|
+
f"top singular value moves {shift:.1%} ({s_old:.3g} → {s_new:.3g})",
|
|
300
|
+
target=target,
|
|
301
|
+
fixes=[Fix(title="lower the strength")],
|
|
302
|
+
)
|
|
303
|
+
)
|
|
304
|
+
if tensors and unchanged == len(tensors) and plan.precision == "merged":
|
|
305
|
+
add(
|
|
306
|
+
diag(
|
|
307
|
+
Code.NO_OP,
|
|
308
|
+
"edit leaves every touched tensor bit-identical",
|
|
309
|
+
detail=["the change is below the stored dtype's resolution everywhere, or zero"],
|
|
310
|
+
fixes=[
|
|
311
|
+
Fix(title="scale the edit up", action={"scale_at_least": 2}),
|
|
312
|
+
Fix(title='precision="adapter"', action={"precision": "adapter"}),
|
|
313
|
+
Fix(title='precision="promote"', action={"precision": "promote"}),
|
|
314
|
+
],
|
|
315
|
+
)
|
|
316
|
+
)
|
|
317
|
+
frac = touched / total_numel
|
|
318
|
+
report.stats.update(
|
|
319
|
+
{"params_touched": touched, "tensors_touched": len(tensors), "delta_ranks": ranks}
|
|
320
|
+
)
|
|
321
|
+
if frac > cfg.warn_dense_fraction:
|
|
322
|
+
add(
|
|
323
|
+
diag(
|
|
324
|
+
Code.DENSE_EDIT,
|
|
325
|
+
f"edit changes {frac:.1%} of all parameters (store size)",
|
|
326
|
+
severity=Severity.WARNING,
|
|
327
|
+
fixes=[Fix(title="prefer low-rank or sparse edits")],
|
|
328
|
+
)
|
|
329
|
+
)
|
|
330
|
+
return report
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _subulp(
|
|
334
|
+
target: str,
|
|
335
|
+
old: torch.Tensor,
|
|
336
|
+
intended: torch.Tensor,
|
|
337
|
+
lost: float,
|
|
338
|
+
cfg: CheckSection,
|
|
339
|
+
deltas: list[TensorDelta],
|
|
340
|
+
) -> Any:
|
|
341
|
+
mask = intended != 0
|
|
342
|
+
w = old.float().abs()[mask]
|
|
343
|
+
w = w[w > 0]
|
|
344
|
+
med_w = float(w.median()) if w.numel() else 0.0
|
|
345
|
+
mantissa = {torch.bfloat16: 7, torch.float16: 10, torch.float32: 23}.get(old.dtype, 7)
|
|
346
|
+
ulp = 2.0 ** (math.floor(math.log2(med_w)) - mantissa) if med_w > 0 else 0.0
|
|
347
|
+
med_d = float(intended.abs()[mask].median()) if mask.any() else 0.0
|
|
348
|
+
severity = Severity.ERROR if lost >= cfg.subulp_error_at else Severity.WARNING
|
|
349
|
+
fixes = [
|
|
350
|
+
Fix(title="scale the edit ≥ 2× (if its magnitude is free)", action={"scale_at_least": 2}),
|
|
351
|
+
Fix(title="concentrate it: same norm on fewer columns or a lower rank"),
|
|
352
|
+
]
|
|
353
|
+
if all(d.kind == "lowrank" and d.index is None for d in deltas) and old.dim() == 2:
|
|
354
|
+
fixes.append(
|
|
355
|
+
Fix(
|
|
356
|
+
title='precision="adapter" (unmerged side path, exported as PEFT)',
|
|
357
|
+
action={"precision": "adapter"},
|
|
358
|
+
)
|
|
359
|
+
)
|
|
360
|
+
fixes.append(
|
|
361
|
+
Fix(
|
|
362
|
+
title='precision="promote" (keep this tensor in fp32)', action={"precision": "promote"}
|
|
363
|
+
)
|
|
364
|
+
)
|
|
365
|
+
name = str(old.dtype).removeprefix("torch.")
|
|
366
|
+
return diag(
|
|
367
|
+
Code.SUB_ULP,
|
|
368
|
+
f"{lost:.0%} of delta magnitude below {name} resolution",
|
|
369
|
+
target=target,
|
|
370
|
+
severity=severity,
|
|
371
|
+
detail=[
|
|
372
|
+
f"median |w| {med_w:.3g} → ULP ≈ {ulp:.2g}, so changes below ≈ {ulp / 2:.2g} vanish; "
|
|
373
|
+
f"median |Δ| {med_d:.2g}"
|
|
374
|
+
],
|
|
375
|
+
fixes=fixes,
|
|
376
|
+
lost=lost,
|
|
377
|
+
)
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
def _top_singular(mat: torch.Tensor, iters: int = 30) -> float:
|
|
381
|
+
"""Largest singular value by power iteration (deterministic start)."""
|
|
382
|
+
gen = torch.Generator(device="cpu").manual_seed(0)
|
|
383
|
+
v = torch.randn(mat.shape[1], generator=gen).to(mat.device, mat.dtype)
|
|
384
|
+
v = v / v.norm()
|
|
385
|
+
sigma = 0.0
|
|
386
|
+
for _ in range(iters):
|
|
387
|
+
u = mat @ v
|
|
388
|
+
sigma = float(u.norm())
|
|
389
|
+
if sigma == 0:
|
|
390
|
+
return 0.0
|
|
391
|
+
v = mat.T @ (u / sigma)
|
|
392
|
+
v = v / v.norm()
|
|
393
|
+
return sigma
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _display(ir: ModelIR, name: str) -> str:
|
|
397
|
+
role = ir.primary_role(name)
|
|
398
|
+
return role.address if role else name
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
# --------------------------------------------------------------------------- smoke
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
@dataclass
|
|
405
|
+
class SmokeMetrics:
|
|
406
|
+
kl: float
|
|
407
|
+
"""Mean per-token KL(base ‖ edited) on the canary."""
|
|
408
|
+
ppl_base: float | None = None
|
|
409
|
+
ppl_new: float | None = None
|
|
410
|
+
degenerate: list[str] = field(default_factory=list)
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def token_kl(
|
|
414
|
+
base_logits: torch.Tensor, new_logits: torch.Tensor, mask: torch.Tensor | None = None
|
|
415
|
+
) -> float:
|
|
416
|
+
"""Mean KL(base ‖ new) over positions, computed in fp32."""
|
|
417
|
+
p = torch.log_softmax(base_logits.float(), dim=-1)
|
|
418
|
+
q = torch.log_softmax(new_logits.float(), dim=-1)
|
|
419
|
+
kl = (p.exp() * (p - q)).sum(-1)
|
|
420
|
+
if mask is not None:
|
|
421
|
+
m = mask.to(kl.dtype)
|
|
422
|
+
return float((kl * m).sum() / m.sum().clamp_min(1))
|
|
423
|
+
return float(kl.mean())
|
|
424
|
+
|
|
425
|
+
|
|
426
|
+
def perplexity(logits: torch.Tensor, ids: torch.Tensor, mask: torch.Tensor | None = None) -> float:
|
|
427
|
+
logp = torch.log_softmax(logits[:, :-1].float(), dim=-1)
|
|
428
|
+
nll = -logp.gather(-1, ids[:, 1:, None]).squeeze(-1)
|
|
429
|
+
if mask is not None:
|
|
430
|
+
m = mask[:, 1:].to(nll.dtype)
|
|
431
|
+
return math.exp(float((nll * m).sum() / m.sum().clamp_min(1)))
|
|
432
|
+
return math.exp(float(nll.mean()))
|
|
433
|
+
|
|
434
|
+
|
|
435
|
+
def degenerate_reasons(tokens: list[int], *, eos: int | None = None) -> list[str]:
|
|
436
|
+
"""Why a greedy continuation looks broken: empty, repetitive or a single token."""
|
|
437
|
+
if not tokens or (eos is not None and tokens[0] == eos):
|
|
438
|
+
return ["empty output"]
|
|
439
|
+
reasons = []
|
|
440
|
+
run = longest = 1
|
|
441
|
+
for a, b in itertools.pairwise(tokens):
|
|
442
|
+
run = run + 1 if a == b else 1
|
|
443
|
+
longest = max(longest, run)
|
|
444
|
+
if longest >= 8:
|
|
445
|
+
reasons.append(f"one token repeated {longest}×")
|
|
446
|
+
for n in (2, 3, 4):
|
|
447
|
+
grams = [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)]
|
|
448
|
+
if len(grams) >= 6 and len(set(grams)) <= len(grams) // 3:
|
|
449
|
+
reasons.append(f"repeating {n}-grams")
|
|
450
|
+
break
|
|
451
|
+
return reasons
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def smoke_report(metrics: SmokeMetrics, cfg: CheckSection) -> Report:
|
|
455
|
+
report = Report(levels=["smoke"], stats={"canary_kl": metrics.kl})
|
|
456
|
+
add = report.diagnostics.append
|
|
457
|
+
if metrics.kl > cfg.max_canary_kl:
|
|
458
|
+
add(
|
|
459
|
+
diag(
|
|
460
|
+
Code.CANARY_KL,
|
|
461
|
+
f"canary KL vs base {metrics.kl:.3f} > {cfg.max_canary_kl}",
|
|
462
|
+
fixes=[
|
|
463
|
+
Fix(title="lower the strength"),
|
|
464
|
+
Fix(title="narrow the edit to fewer units or layers"),
|
|
465
|
+
Fix(title="trace --vs base on a canary prompt to see where it diverges"),
|
|
466
|
+
],
|
|
467
|
+
)
|
|
468
|
+
)
|
|
469
|
+
if metrics.ppl_base and metrics.ppl_new:
|
|
470
|
+
rise = metrics.ppl_new / metrics.ppl_base - 1
|
|
471
|
+
report.stats.update({"ppl_base": metrics.ppl_base, "ppl_new": metrics.ppl_new})
|
|
472
|
+
if rise > cfg.max_ppl_increase:
|
|
473
|
+
add(
|
|
474
|
+
diag(
|
|
475
|
+
Code.PERPLEXITY,
|
|
476
|
+
f"canary perplexity {metrics.ppl_base:.2f} → {metrics.ppl_new:.2f} "
|
|
477
|
+
f"(+{rise:.0%})",
|
|
478
|
+
fixes=[Fix(title="lower the strength")],
|
|
479
|
+
)
|
|
480
|
+
)
|
|
481
|
+
for reason in metrics.degenerate:
|
|
482
|
+
add(
|
|
483
|
+
diag(
|
|
484
|
+
Code.DEGENERATE_OUTPUT,
|
|
485
|
+
f"degenerate output: {reason}",
|
|
486
|
+
fixes=[Fix(title="revert and retry with a smaller edit")],
|
|
487
|
+
)
|
|
488
|
+
)
|
|
489
|
+
return report
|
wspx/lab/evaluate.py
ADDED
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
"""The model object suites receive: generation and scoring on the current weights."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import math
|
|
6
|
+
from collections.abc import Callable, Sequence
|
|
7
|
+
from contextlib import AbstractContextManager
|
|
8
|
+
from typing import Any
|
|
9
|
+
|
|
10
|
+
import torch
|
|
11
|
+
|
|
12
|
+
from wspx.lab import analysis
|
|
13
|
+
from wspx.lab.check import perplexity, token_kl
|
|
14
|
+
from wspx.lab.ir.model_ir import ModelIR
|
|
15
|
+
from wspx.lab.text import Prompt, Text
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
class SuiteModel:
|
|
19
|
+
"""Passed to ``@suite`` functions. Everything runs on the live working tree."""
|
|
20
|
+
|
|
21
|
+
def __init__(
|
|
22
|
+
self,
|
|
23
|
+
model: torch.nn.Module,
|
|
24
|
+
ir: ModelIR,
|
|
25
|
+
text: Text,
|
|
26
|
+
at_base: Callable[[], AbstractContextManager[Any]],
|
|
27
|
+
) -> None:
|
|
28
|
+
self._model = model
|
|
29
|
+
self._ir = ir
|
|
30
|
+
self._text = text
|
|
31
|
+
self._at_base = at_base
|
|
32
|
+
self._base_cache: dict[str, Any] = {}
|
|
33
|
+
|
|
34
|
+
def canary_prompts(self) -> list[Prompt]:
|
|
35
|
+
"""The fixed canary: text prompts with a tokenizer, seeded token ids without one."""
|
|
36
|
+
from wspx.testing.builtin import CANARY_PROMPTS
|
|
37
|
+
|
|
38
|
+
if self._text.tokenizer is not None:
|
|
39
|
+
return list(CANARY_PROMPTS)
|
|
40
|
+
gen = torch.Generator().manual_seed(1234)
|
|
41
|
+
vocab = self._ir.vocab_size or 2
|
|
42
|
+
return [
|
|
43
|
+
torch.randint(0, vocab, (16,), generator=gen).tolist()
|
|
44
|
+
for _ in range(len(CANARY_PROMPTS))
|
|
45
|
+
]
|
|
46
|
+
|
|
47
|
+
def generate(
|
|
48
|
+
self, prompt: Prompt, *, max_tokens: int = 64, temperature: float = 0.0, seed: int = 0
|
|
49
|
+
) -> str:
|
|
50
|
+
ids = analysis.generate(
|
|
51
|
+
self._model,
|
|
52
|
+
self._text,
|
|
53
|
+
[prompt],
|
|
54
|
+
max_tokens=max_tokens,
|
|
55
|
+
temperature=temperature,
|
|
56
|
+
seed=seed,
|
|
57
|
+
)[0]
|
|
58
|
+
eos = self._text.eos_id
|
|
59
|
+
if eos is not None and eos in ids:
|
|
60
|
+
ids = ids[: ids.index(eos)]
|
|
61
|
+
return self._text.decode(ids)
|
|
62
|
+
|
|
63
|
+
def _logits(self, prompts: Sequence[Prompt]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
|
|
64
|
+
device = next(self._model.parameters()).device
|
|
65
|
+
ids, mask = self._text.batch(prompts, device)
|
|
66
|
+
with torch.no_grad():
|
|
67
|
+
return analysis.forward(self._model, self._ir, ids, mask), ids, mask
|
|
68
|
+
|
|
69
|
+
def logprob(self, prompt: Prompt, continuation: Prompt) -> float:
|
|
70
|
+
"""Total log-probability of ``continuation`` after ``prompt``."""
|
|
71
|
+
p = self._text.ids(prompt)
|
|
72
|
+
if isinstance(continuation, str):
|
|
73
|
+
if self._text.tokenizer is None:
|
|
74
|
+
raise ValueError("text continuations need a tokenizer; pass token ids")
|
|
75
|
+
c = list(self._text.tokenizer(continuation, add_special_tokens=False)["input_ids"])
|
|
76
|
+
else:
|
|
77
|
+
c = [int(t) for t in continuation]
|
|
78
|
+
logits, _, _ = self._logits([p + c])
|
|
79
|
+
logp = torch.log_softmax(logits[0].float(), dim=-1)
|
|
80
|
+
start = len(p)
|
|
81
|
+
return float(sum(logp[start - 1 + i, tok] for i, tok in enumerate(c)))
|
|
82
|
+
|
|
83
|
+
def logit_diff(self, prompt: Prompt, good: str | int, bad: str | int) -> float:
|
|
84
|
+
logits, _, _ = self._logits([prompt])
|
|
85
|
+
last = logits[0, -1].float()
|
|
86
|
+
return float(last[self._text.token_id(good)] - last[self._text.token_id(bad)])
|
|
87
|
+
|
|
88
|
+
def kl_vs_base(self, prompt: Prompt) -> float:
|
|
89
|
+
logits, _, mask = self._logits([prompt])
|
|
90
|
+
key = repr(prompt)
|
|
91
|
+
if key not in self._base_cache:
|
|
92
|
+
with self._at_base():
|
|
93
|
+
self._base_cache[key] = self._logits([prompt])[0]
|
|
94
|
+
return token_kl(self._base_cache[key], logits, mask)
|
|
95
|
+
|
|
96
|
+
def perplexity(self, texts: Sequence[Prompt]) -> float:
|
|
97
|
+
values = []
|
|
98
|
+
for t in texts:
|
|
99
|
+
logits, ids, mask = self._logits([t])
|
|
100
|
+
values.append(math.log(perplexity(logits, ids, mask)))
|
|
101
|
+
return math.exp(sum(values) / len(values))
|
|
102
|
+
|
|
103
|
+
def base_perplexity(self, texts: Sequence[Prompt]) -> float:
|
|
104
|
+
key = "ppl:" + repr(list(texts))
|
|
105
|
+
if key not in self._base_cache:
|
|
106
|
+
with self._at_base():
|
|
107
|
+
self._base_cache[key] = self.perplexity(texts)
|
|
108
|
+
return float(self._base_cache[key])
|