wspx 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. wspx/__init__.py +24 -0
  2. wspx/_version.py +1 -0
  3. wspx/addressing.py +415 -0
  4. wspx/auth.py +77 -0
  5. wspx/cli/__init__.py +1 -0
  6. wspx/cli/client.py +124 -0
  7. wspx/cli/config.py +89 -0
  8. wspx/cli/connect.py +88 -0
  9. wspx/cli/main.py +848 -0
  10. wspx/cloud/__init__.py +20 -0
  11. wspx/cloud/app.py +191 -0
  12. wspx/cloud/backend.py +141 -0
  13. wspx/cloud/compose.py +85 -0
  14. wspx/cloud/deploy.py +102 -0
  15. wspx/config.py +189 -0
  16. wspx/control/__init__.py +5 -0
  17. wspx/control/app.py +488 -0
  18. wspx/control/backend.py +244 -0
  19. wspx/control/gateway_client.py +79 -0
  20. wspx/control/mcp_server.py +71 -0
  21. wspx/control/sessions.py +424 -0
  22. wspx/control/tracedb.py +183 -0
  23. wspx/control/ui/assets/index-BJFweZAn.css +1 -0
  24. wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
  25. wspx/control/ui/index.html +13 -0
  26. wspx/control/views.py +180 -0
  27. wspx/diagnostics.py +146 -0
  28. wspx/driver/__init__.py +7 -0
  29. wspx/driver/loop.py +255 -0
  30. wspx/driver/messages.py +62 -0
  31. wspx/driver/providers/__init__.py +84 -0
  32. wspx/driver/providers/anthropic.py +149 -0
  33. wspx/driver/providers/compat.py +115 -0
  34. wspx/driver/providers/google.py +138 -0
  35. wspx/driver/providers/openai.py +125 -0
  36. wspx/driver/run.py +134 -0
  37. wspx/errors.py +69 -0
  38. wspx/gateway/__init__.py +8 -0
  39. wspx/gateway/app.py +626 -0
  40. wspx/gateway/ledger.py +536 -0
  41. wspx/gateway/prices.py +75 -0
  42. wspx/gateway/usage.py +284 -0
  43. wspx/handles.py +117 -0
  44. wspx/integrations/AGENTS.snippet.md +8 -0
  45. wspx/integrations/SKILL.md +52 -0
  46. wspx/integrations/__init__.py +17 -0
  47. wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
  48. wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
  49. wspx/integrations/codex-hooks.toml +9 -0
  50. wspx/lab/__init__.py +5 -0
  51. wspx/lab/analysis.py +485 -0
  52. wspx/lab/check.py +489 -0
  53. wspx/lab/evaluate.py +108 -0
  54. wspx/lab/export.py +275 -0
  55. wspx/lab/grep.py +754 -0
  56. wspx/lab/index.py +239 -0
  57. wspx/lab/ir/__init__.py +1 -0
  58. wspx/lab/ir/capture.py +299 -0
  59. wspx/lab/ir/graph.py +148 -0
  60. wspx/lab/ir/model_ir.py +522 -0
  61. wspx/lab/ir/semantics.py +1255 -0
  62. wspx/lab/ir/tape.py +853 -0
  63. wspx/lab/loader.py +204 -0
  64. wspx/lab/merge.py +200 -0
  65. wspx/lab/ops/__init__.py +76 -0
  66. wspx/lab/ops/builtin.py +379 -0
  67. wspx/lab/patch.py +320 -0
  68. wspx/lab/probe.py +206 -0
  69. wspx/lab/results.py +114 -0
  70. wspx/lab/sdk.py +120 -0
  71. wspx/lab/server.py +236 -0
  72. wspx/lab/session.py +1574 -0
  73. wspx/lab/tensors.py +119 -0
  74. wspx/lab/text.py +69 -0
  75. wspx/lab/tree.py +302 -0
  76. wspx/loops.py +127 -0
  77. wspx/project.py +143 -0
  78. wspx/py.typed +0 -0
  79. wspx/safety/__init__.py +15 -0
  80. wspx/safety/kill.py +47 -0
  81. wspx/safety/sweeper.py +68 -0
  82. wspx/store/__init__.py +64 -0
  83. wspx/store/objects.py +130 -0
  84. wspx/store/overlay.py +209 -0
  85. wspx/store/refs.py +150 -0
  86. wspx/testing/__init__.py +21 -0
  87. wspx/testing/api.py +155 -0
  88. wspx/testing/builtin.py +139 -0
  89. wspx/testing/runner.py +257 -0
  90. wspx/tools/__init__.py +17 -0
  91. wspx/tools/dispatch.py +129 -0
  92. wspx/tools/schemas.py +313 -0
  93. wspx/vcs/__init__.py +17 -0
  94. wspx/vcs/repo.py +431 -0
  95. wspx-0.1.0.dev0.dist-info/METADATA +278 -0
  96. wspx-0.1.0.dev0.dist-info/RECORD +99 -0
  97. wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
  98. wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
  99. wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/check.py ADDED
@@ -0,0 +1,489 @@
1
+ """Checks: "does this edit compile?" (§5.4).
2
+
3
+ * **Static** (IR only, milliseconds): addresses, shapes, base hashes, op/role
4
+ validity, tied weights, fused sub-role boundaries, declared scope, quantized
5
+ storage.
6
+ * **Numeric** (weights, under a second): non-finite values, overflow, sub-ULP
7
+ loss, relative norm change, spectrum shift, no-ops, dense-edit size.
8
+ * **Smoke** (a short forward pass): KL against the base on a fixed canary,
9
+ perplexity change and degenerate output.
10
+
11
+ Every diagnostic carries fixes. Severity scales with damage: sub-ULP loss
12
+ warns below ``subulp_error_at`` and blocks above it.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import itertools
18
+ import math
19
+ from collections import defaultdict
20
+ from dataclasses import dataclass, field
21
+ from typing import Any
22
+
23
+ import torch
24
+
25
+ from wspx.config import CheckSection
26
+ from wspx.diagnostics import Code, Fix, Report, Severity, diag
27
+ from wspx.lab.ir.graph import expand_runs
28
+ from wspx.lab.ir.model_ir import ModelIR
29
+ from wspx.lab.patch import TensorDelta
30
+ from wspx.lab.tree import WorkingTree
31
+
32
+ WRITER_OPS = frozenset({"project_out", "bake_steer", "rank_one_write"})
33
+ SUBULP_WARN_AT = 0.05
34
+ SPECTRUM_SHIFT = 0.10
35
+ SPECTRUM_MAX_NUMEL = 64 * 1024 * 1024
36
+
37
+
38
+ @dataclass
39
+ class EditPlan:
40
+ deltas: list[TensorDelta]
41
+ op: str = "edit"
42
+ precision: str = "merged"
43
+ scope: list[str] | None = None
44
+ """Declared parameter names the edit may touch (from the addresses given)."""
45
+ args: dict[str, Any] = field(default_factory=dict)
46
+
47
+ def by_tensor(self) -> dict[str, list[TensorDelta]]:
48
+ out: dict[str, list[TensorDelta]] = defaultdict(list)
49
+ for d in self.deltas:
50
+ out[d.tensor].append(d)
51
+ return dict(out)
52
+
53
+
54
+ # --------------------------------------------------------------------------- static
55
+
56
+
57
+ def static_check(ir: ModelIR, plan: EditPlan) -> Report:
58
+ report = Report(levels=["static"])
59
+ add = report.diagnostics.append
60
+ for name, deltas in plan.by_tensor().items():
61
+ info = ir.params.get(name)
62
+ if info is None:
63
+ add(diag(Code.UNRESOLVED_ADDRESS, "no such parameter", target=name))
64
+ continue
65
+ target = _display(ir, name)
66
+ for d in deltas:
67
+ if tuple(d.shape) != tuple(info.shape):
68
+ add(
69
+ diag(
70
+ Code.SHAPE_MISMATCH,
71
+ f"delta shape {tuple(d.shape)} vs tensor {info.shape}",
72
+ target=target,
73
+ )
74
+ )
75
+ if info.hash and d.base != info.hash:
76
+ add(
77
+ diag(
78
+ Code.BASE_MISMATCH,
79
+ "delta was computed against a different base tensor",
80
+ target=target,
81
+ fixes=[Fix(title="recompute the edit on this model")],
82
+ )
83
+ )
84
+ roles = ir.roles_of(name)
85
+ kinds = {r.kind for r in roles}
86
+ if plan.op in WRITER_OPS and (len(info.shape) < 2 and plan.op != "bake_steer"):
87
+ add(
88
+ diag(
89
+ Code.OP_INVALID_FOR_ROLE,
90
+ f"{plan.op} needs a matrix, {name} is {len(info.shape)}-D",
91
+ target=target,
92
+ fixes=[
93
+ Fix(
94
+ title="target a projection weight such as L*.mlp.down",
95
+ action={"address": "L*.mlp.down"},
96
+ )
97
+ ],
98
+ )
99
+ )
100
+ if plan.op == "project_out" and kinds & {"norm", "final_norm", "bias"}:
101
+ add(
102
+ diag(
103
+ Code.OP_INVALID_FOR_ROLE,
104
+ f"project_out on a {'/'.join(sorted(kinds))} weight",
105
+ target=target,
106
+ )
107
+ )
108
+ if plan.op in WRITER_OPS and "reader" in kinds and "writer" not in kinds:
109
+ add(
110
+ diag(
111
+ Code.READER_NOT_WRITER,
112
+ "this edit touches a reader: it changes what the layer reads, not what it writes",
113
+ target=target,
114
+ severity=Severity.INFO,
115
+ fixes=[
116
+ Fix(
117
+ title="target the block's writer (o, down, out) to change what it writes"
118
+ )
119
+ ],
120
+ )
121
+ )
122
+ if info.aliases or {"embed", "unembed"} <= kinds:
123
+ names = ", ".join([name, *info.aliases])
124
+ add(
125
+ diag(
126
+ Code.TIED_WEIGHT,
127
+ "tensor is tied: the edit changes both embedding and unembedding",
128
+ target=target,
129
+ detail=[f"shared storage: {names}"],
130
+ fixes=[
131
+ Fix(title="accept: both roles change together"),
132
+ Fix(
133
+ title='precision="adapter" keeps the change on one side',
134
+ action={"precision": "adapter"},
135
+ ),
136
+ ],
137
+ )
138
+ )
139
+ _fused_boundaries(ir, name, deltas, report)
140
+ if plan.scope is not None and name not in plan.scope:
141
+ add(
142
+ diag(
143
+ Code.OUTSIDE_SCOPE,
144
+ "edit touches a tensor outside the declared scope",
145
+ target=target,
146
+ )
147
+ )
148
+ dtype = info.dtype
149
+ if dtype in ("int8", "uint8", "int32") and plan.precision != "adapter":
150
+ add(
151
+ diag(
152
+ Code.QUANTIZED_NEEDS_ADAPTER,
153
+ f"{dtype} storage is quantized; small deltas would be lost",
154
+ target=target,
155
+ fixes=[Fix(title='precision="adapter"', action={"precision": "adapter"})],
156
+ )
157
+ )
158
+ elif dtype.startswith("float8") and plan.precision == "merged":
159
+ add(
160
+ diag(
161
+ Code.REQUANTIZED,
162
+ f"{dtype} target: the edit is requantized on materialisation",
163
+ target=target,
164
+ severity=Severity.INFO,
165
+ fixes=[
166
+ Fix(
167
+ title='precision="adapter" for small edits',
168
+ action={"precision": "adapter"},
169
+ )
170
+ ],
171
+ )
172
+ )
173
+ return report
174
+
175
+
176
+ def _fused_boundaries(ir: ModelIR, name: str, deltas: list[TensorDelta], report: Report) -> None:
177
+ subs = [r for r in ir.roles_of(name) if r.split is not None and r.kind != "bias"]
178
+ if not subs:
179
+ return
180
+ touched_parts: set[str] = set()
181
+ for d in deltas:
182
+ if d.index is None or d.kind == "sparse":
183
+ touched_parts.update(r.address.rsplit(".", 1)[-1] for r in subs)
184
+ continue
185
+ role = subs[0]
186
+ dim = role.out_dim if role.out_dim is not None else 0
187
+ sel = d.index[dim] if dim < len(d.index) else slice(None)
188
+ size = ir.params[name].shape[dim]
189
+ rows = (
190
+ set(range(*sel.indices(size)))
191
+ if isinstance(sel, slice)
192
+ else ({sel} if isinstance(sel, int) else set(sel))
193
+ )
194
+ for r in subs:
195
+ if rows & set(expand_runs(r.split or [])):
196
+ touched_parts.add(r.address.rsplit(".", 1)[-1])
197
+ if len(touched_parts) > 1:
198
+ report.diagnostics.append(
199
+ diag(
200
+ Code.FUSED_SPLIT,
201
+ f"edit spans fused parts {', '.join(sorted(touched_parts))} of one weight",
202
+ target=_display(ir, name),
203
+ severity=Severity.INFO,
204
+ fixes=[
205
+ Fix(
206
+ title=f"address one part, e.g. {subs[0].address}",
207
+ action={"address": subs[0].address},
208
+ )
209
+ ],
210
+ )
211
+ )
212
+
213
+
214
+ # --------------------------------------------------------------------------- numeric
215
+
216
+
217
+ def numeric_check(ir: ModelIR, tree: WorkingTree, plan: EditPlan, cfg: CheckSection) -> Report:
218
+ report = Report(levels=["numeric"])
219
+ add = report.diagnostics.append
220
+ total_numel = sum(p.numel for p in ir.params.values()) or 1
221
+ touched = 0
222
+ unchanged = 0
223
+ tensors = plan.by_tensor()
224
+ ranks = [d.rank for d in plan.deltas if d.rank is not None]
225
+ for name, deltas in tensors.items():
226
+ if name not in tree.params:
227
+ continue
228
+ target = _display(ir, name)
229
+ old = tree.params[name].detach()
230
+ dtype = torch.float32 if plan.precision == "promote" else old.dtype
231
+ if plan.precision == "adapter":
232
+ continue
233
+ new = tree.preview(name, deltas, dtype=dtype)
234
+ intended = torch.zeros(old.shape, dtype=torch.float32, device=old.device)
235
+ for d in deltas:
236
+ d.add_into(intended)
237
+ if not torch.isfinite(intended).all() or not torch.isfinite(new.float()).all():
238
+ add(
239
+ diag(
240
+ Code.NON_FINITE,
241
+ "edit produces NaN or Inf",
242
+ target=target,
243
+ fixes=[Fix(title="check the direction and scale for zeros or overflow")],
244
+ )
245
+ )
246
+ continue
247
+ if old.is_floating_point() and dtype != torch.float32:
248
+ limit = torch.finfo(dtype).max
249
+ peak = float((old.float() + intended).abs().max())
250
+ if peak > limit:
251
+ add(
252
+ diag(
253
+ Code.DTYPE_OVERFLOW,
254
+ f"values reach {peak:.3g}, above {dtype} max {limit:.3g}",
255
+ target=target,
256
+ fixes=[Fix(title="reduce the edit's scale", action={"scale": 0.5})],
257
+ )
258
+ )
259
+ continue
260
+ effective = new.float() - old.float()
261
+ changed = (
262
+ int((new != old.to(new.dtype)).sum())
263
+ if new.dtype == old.dtype
264
+ else int((effective != 0).sum())
265
+ )
266
+ touched += changed
267
+ if changed == 0:
268
+ unchanged += 1
269
+ mass = float(intended.abs().sum())
270
+ if mass > 0 and plan.precision == "merged":
271
+ lost = float((intended - effective).abs().sum()) / mass
272
+ if lost >= SUBULP_WARN_AT:
273
+ report.diagnostics.append(_subulp(target, old, intended, lost, cfg, deltas))
274
+ w_norm = float(old.float().norm())
275
+ rel = float(effective.norm()) / w_norm if w_norm > 0 else float("inf")
276
+ report.stats.setdefault("rel_norm", {})[name] = rel
277
+ if rel > cfg.max_rel_norm:
278
+ add(
279
+ diag(
280
+ Code.REL_NORM,
281
+ f"relative norm change {rel:.3f} > {cfg.max_rel_norm}",
282
+ target=target,
283
+ fixes=[
284
+ Fix(
285
+ title="lower the strength", action={"strength": cfg.max_rel_norm / rel}
286
+ ),
287
+ Fix(title="narrow the selection (fewer units or a lower rank)"),
288
+ ],
289
+ )
290
+ )
291
+ if old.dim() == 2 and old.numel() <= SPECTRUM_MAX_NUMEL and rel > 1e-3:
292
+ s_old, s_new = _top_singular(old.float()), _top_singular(new.float())
293
+ shift = abs(s_new - s_old) / s_old if s_old > 0 else 0.0
294
+ report.stats.setdefault("sigma_shift", {})[name] = shift
295
+ if shift > SPECTRUM_SHIFT:
296
+ add(
297
+ diag(
298
+ Code.SPECTRUM_SHIFT,
299
+ f"top singular value moves {shift:.1%} ({s_old:.3g} → {s_new:.3g})",
300
+ target=target,
301
+ fixes=[Fix(title="lower the strength")],
302
+ )
303
+ )
304
+ if tensors and unchanged == len(tensors) and plan.precision == "merged":
305
+ add(
306
+ diag(
307
+ Code.NO_OP,
308
+ "edit leaves every touched tensor bit-identical",
309
+ detail=["the change is below the stored dtype's resolution everywhere, or zero"],
310
+ fixes=[
311
+ Fix(title="scale the edit up", action={"scale_at_least": 2}),
312
+ Fix(title='precision="adapter"', action={"precision": "adapter"}),
313
+ Fix(title='precision="promote"', action={"precision": "promote"}),
314
+ ],
315
+ )
316
+ )
317
+ frac = touched / total_numel
318
+ report.stats.update(
319
+ {"params_touched": touched, "tensors_touched": len(tensors), "delta_ranks": ranks}
320
+ )
321
+ if frac > cfg.warn_dense_fraction:
322
+ add(
323
+ diag(
324
+ Code.DENSE_EDIT,
325
+ f"edit changes {frac:.1%} of all parameters (store size)",
326
+ severity=Severity.WARNING,
327
+ fixes=[Fix(title="prefer low-rank or sparse edits")],
328
+ )
329
+ )
330
+ return report
331
+
332
+
333
+ def _subulp(
334
+ target: str,
335
+ old: torch.Tensor,
336
+ intended: torch.Tensor,
337
+ lost: float,
338
+ cfg: CheckSection,
339
+ deltas: list[TensorDelta],
340
+ ) -> Any:
341
+ mask = intended != 0
342
+ w = old.float().abs()[mask]
343
+ w = w[w > 0]
344
+ med_w = float(w.median()) if w.numel() else 0.0
345
+ mantissa = {torch.bfloat16: 7, torch.float16: 10, torch.float32: 23}.get(old.dtype, 7)
346
+ ulp = 2.0 ** (math.floor(math.log2(med_w)) - mantissa) if med_w > 0 else 0.0
347
+ med_d = float(intended.abs()[mask].median()) if mask.any() else 0.0
348
+ severity = Severity.ERROR if lost >= cfg.subulp_error_at else Severity.WARNING
349
+ fixes = [
350
+ Fix(title="scale the edit ≥ 2× (if its magnitude is free)", action={"scale_at_least": 2}),
351
+ Fix(title="concentrate it: same norm on fewer columns or a lower rank"),
352
+ ]
353
+ if all(d.kind == "lowrank" and d.index is None for d in deltas) and old.dim() == 2:
354
+ fixes.append(
355
+ Fix(
356
+ title='precision="adapter" (unmerged side path, exported as PEFT)',
357
+ action={"precision": "adapter"},
358
+ )
359
+ )
360
+ fixes.append(
361
+ Fix(
362
+ title='precision="promote" (keep this tensor in fp32)', action={"precision": "promote"}
363
+ )
364
+ )
365
+ name = str(old.dtype).removeprefix("torch.")
366
+ return diag(
367
+ Code.SUB_ULP,
368
+ f"{lost:.0%} of delta magnitude below {name} resolution",
369
+ target=target,
370
+ severity=severity,
371
+ detail=[
372
+ f"median |w| {med_w:.3g} → ULP ≈ {ulp:.2g}, so changes below ≈ {ulp / 2:.2g} vanish; "
373
+ f"median |Δ| {med_d:.2g}"
374
+ ],
375
+ fixes=fixes,
376
+ lost=lost,
377
+ )
378
+
379
+
380
+ def _top_singular(mat: torch.Tensor, iters: int = 30) -> float:
381
+ """Largest singular value by power iteration (deterministic start)."""
382
+ gen = torch.Generator(device="cpu").manual_seed(0)
383
+ v = torch.randn(mat.shape[1], generator=gen).to(mat.device, mat.dtype)
384
+ v = v / v.norm()
385
+ sigma = 0.0
386
+ for _ in range(iters):
387
+ u = mat @ v
388
+ sigma = float(u.norm())
389
+ if sigma == 0:
390
+ return 0.0
391
+ v = mat.T @ (u / sigma)
392
+ v = v / v.norm()
393
+ return sigma
394
+
395
+
396
+ def _display(ir: ModelIR, name: str) -> str:
397
+ role = ir.primary_role(name)
398
+ return role.address if role else name
399
+
400
+
401
+ # --------------------------------------------------------------------------- smoke
402
+
403
+
404
+ @dataclass
405
+ class SmokeMetrics:
406
+ kl: float
407
+ """Mean per-token KL(base ‖ edited) on the canary."""
408
+ ppl_base: float | None = None
409
+ ppl_new: float | None = None
410
+ degenerate: list[str] = field(default_factory=list)
411
+
412
+
413
+ def token_kl(
414
+ base_logits: torch.Tensor, new_logits: torch.Tensor, mask: torch.Tensor | None = None
415
+ ) -> float:
416
+ """Mean KL(base ‖ new) over positions, computed in fp32."""
417
+ p = torch.log_softmax(base_logits.float(), dim=-1)
418
+ q = torch.log_softmax(new_logits.float(), dim=-1)
419
+ kl = (p.exp() * (p - q)).sum(-1)
420
+ if mask is not None:
421
+ m = mask.to(kl.dtype)
422
+ return float((kl * m).sum() / m.sum().clamp_min(1))
423
+ return float(kl.mean())
424
+
425
+
426
+ def perplexity(logits: torch.Tensor, ids: torch.Tensor, mask: torch.Tensor | None = None) -> float:
427
+ logp = torch.log_softmax(logits[:, :-1].float(), dim=-1)
428
+ nll = -logp.gather(-1, ids[:, 1:, None]).squeeze(-1)
429
+ if mask is not None:
430
+ m = mask[:, 1:].to(nll.dtype)
431
+ return math.exp(float((nll * m).sum() / m.sum().clamp_min(1)))
432
+ return math.exp(float(nll.mean()))
433
+
434
+
435
+ def degenerate_reasons(tokens: list[int], *, eos: int | None = None) -> list[str]:
436
+ """Why a greedy continuation looks broken: empty, repetitive or a single token."""
437
+ if not tokens or (eos is not None and tokens[0] == eos):
438
+ return ["empty output"]
439
+ reasons = []
440
+ run = longest = 1
441
+ for a, b in itertools.pairwise(tokens):
442
+ run = run + 1 if a == b else 1
443
+ longest = max(longest, run)
444
+ if longest >= 8:
445
+ reasons.append(f"one token repeated {longest}×")
446
+ for n in (2, 3, 4):
447
+ grams = [tuple(tokens[i : i + n]) for i in range(len(tokens) - n + 1)]
448
+ if len(grams) >= 6 and len(set(grams)) <= len(grams) // 3:
449
+ reasons.append(f"repeating {n}-grams")
450
+ break
451
+ return reasons
452
+
453
+
454
+ def smoke_report(metrics: SmokeMetrics, cfg: CheckSection) -> Report:
455
+ report = Report(levels=["smoke"], stats={"canary_kl": metrics.kl})
456
+ add = report.diagnostics.append
457
+ if metrics.kl > cfg.max_canary_kl:
458
+ add(
459
+ diag(
460
+ Code.CANARY_KL,
461
+ f"canary KL vs base {metrics.kl:.3f} > {cfg.max_canary_kl}",
462
+ fixes=[
463
+ Fix(title="lower the strength"),
464
+ Fix(title="narrow the edit to fewer units or layers"),
465
+ Fix(title="trace --vs base on a canary prompt to see where it diverges"),
466
+ ],
467
+ )
468
+ )
469
+ if metrics.ppl_base and metrics.ppl_new:
470
+ rise = metrics.ppl_new / metrics.ppl_base - 1
471
+ report.stats.update({"ppl_base": metrics.ppl_base, "ppl_new": metrics.ppl_new})
472
+ if rise > cfg.max_ppl_increase:
473
+ add(
474
+ diag(
475
+ Code.PERPLEXITY,
476
+ f"canary perplexity {metrics.ppl_base:.2f} → {metrics.ppl_new:.2f} "
477
+ f"(+{rise:.0%})",
478
+ fixes=[Fix(title="lower the strength")],
479
+ )
480
+ )
481
+ for reason in metrics.degenerate:
482
+ add(
483
+ diag(
484
+ Code.DEGENERATE_OUTPUT,
485
+ f"degenerate output: {reason}",
486
+ fixes=[Fix(title="revert and retry with a smaller edit")],
487
+ )
488
+ )
489
+ return report
wspx/lab/evaluate.py ADDED
@@ -0,0 +1,108 @@
1
+ """The model object suites receive: generation and scoring on the current weights."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import math
6
+ from collections.abc import Callable, Sequence
7
+ from contextlib import AbstractContextManager
8
+ from typing import Any
9
+
10
+ import torch
11
+
12
+ from wspx.lab import analysis
13
+ from wspx.lab.check import perplexity, token_kl
14
+ from wspx.lab.ir.model_ir import ModelIR
15
+ from wspx.lab.text import Prompt, Text
16
+
17
+
18
+ class SuiteModel:
19
+ """Passed to ``@suite`` functions. Everything runs on the live working tree."""
20
+
21
+ def __init__(
22
+ self,
23
+ model: torch.nn.Module,
24
+ ir: ModelIR,
25
+ text: Text,
26
+ at_base: Callable[[], AbstractContextManager[Any]],
27
+ ) -> None:
28
+ self._model = model
29
+ self._ir = ir
30
+ self._text = text
31
+ self._at_base = at_base
32
+ self._base_cache: dict[str, Any] = {}
33
+
34
+ def canary_prompts(self) -> list[Prompt]:
35
+ """The fixed canary: text prompts with a tokenizer, seeded token ids without one."""
36
+ from wspx.testing.builtin import CANARY_PROMPTS
37
+
38
+ if self._text.tokenizer is not None:
39
+ return list(CANARY_PROMPTS)
40
+ gen = torch.Generator().manual_seed(1234)
41
+ vocab = self._ir.vocab_size or 2
42
+ return [
43
+ torch.randint(0, vocab, (16,), generator=gen).tolist()
44
+ for _ in range(len(CANARY_PROMPTS))
45
+ ]
46
+
47
+ def generate(
48
+ self, prompt: Prompt, *, max_tokens: int = 64, temperature: float = 0.0, seed: int = 0
49
+ ) -> str:
50
+ ids = analysis.generate(
51
+ self._model,
52
+ self._text,
53
+ [prompt],
54
+ max_tokens=max_tokens,
55
+ temperature=temperature,
56
+ seed=seed,
57
+ )[0]
58
+ eos = self._text.eos_id
59
+ if eos is not None and eos in ids:
60
+ ids = ids[: ids.index(eos)]
61
+ return self._text.decode(ids)
62
+
63
+ def _logits(self, prompts: Sequence[Prompt]) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
64
+ device = next(self._model.parameters()).device
65
+ ids, mask = self._text.batch(prompts, device)
66
+ with torch.no_grad():
67
+ return analysis.forward(self._model, self._ir, ids, mask), ids, mask
68
+
69
+ def logprob(self, prompt: Prompt, continuation: Prompt) -> float:
70
+ """Total log-probability of ``continuation`` after ``prompt``."""
71
+ p = self._text.ids(prompt)
72
+ if isinstance(continuation, str):
73
+ if self._text.tokenizer is None:
74
+ raise ValueError("text continuations need a tokenizer; pass token ids")
75
+ c = list(self._text.tokenizer(continuation, add_special_tokens=False)["input_ids"])
76
+ else:
77
+ c = [int(t) for t in continuation]
78
+ logits, _, _ = self._logits([p + c])
79
+ logp = torch.log_softmax(logits[0].float(), dim=-1)
80
+ start = len(p)
81
+ return float(sum(logp[start - 1 + i, tok] for i, tok in enumerate(c)))
82
+
83
+ def logit_diff(self, prompt: Prompt, good: str | int, bad: str | int) -> float:
84
+ logits, _, _ = self._logits([prompt])
85
+ last = logits[0, -1].float()
86
+ return float(last[self._text.token_id(good)] - last[self._text.token_id(bad)])
87
+
88
+ def kl_vs_base(self, prompt: Prompt) -> float:
89
+ logits, _, mask = self._logits([prompt])
90
+ key = repr(prompt)
91
+ if key not in self._base_cache:
92
+ with self._at_base():
93
+ self._base_cache[key] = self._logits([prompt])[0]
94
+ return token_kl(self._base_cache[key], logits, mask)
95
+
96
+ def perplexity(self, texts: Sequence[Prompt]) -> float:
97
+ values = []
98
+ for t in texts:
99
+ logits, ids, mask = self._logits([t])
100
+ values.append(math.log(perplexity(logits, ids, mask)))
101
+ return math.exp(sum(values) / len(values))
102
+
103
+ def base_perplexity(self, texts: Sequence[Prompt]) -> float:
104
+ key = "ppl:" + repr(list(texts))
105
+ if key not in self._base_cache:
106
+ with self._at_base():
107
+ self._base_cache[key] = self.perplexity(texts)
108
+ return float(self._base_cache[key])