wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/ops/builtin.py
ADDED
|
@@ -0,0 +1,379 @@
|
|
|
1
|
+
"""Built-in ops. Each compiles against the current weights into concrete deltas."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
from collections.abc import Callable
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
import torch
|
|
9
|
+
|
|
10
|
+
from wspx.errors import PatchError
|
|
11
|
+
from wspx.lab import patch
|
|
12
|
+
from wspx.lab.ir.model_ir import Target
|
|
13
|
+
from wspx.lab.ops import OpContext
|
|
14
|
+
from wspx.lab.patch import TensorDelta
|
|
15
|
+
from wspx.lab.tensors import Index, normalize_index, select
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def _free_dims(index: Index) -> list[int]:
|
|
19
|
+
return [d for d, s in enumerate(index) if not isinstance(s, int)]
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def _index(ctx: OpContext, target: Target) -> Index:
|
|
23
|
+
return normalize_index(target.index, len(ctx.ir.params[target.param].shape))
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
def _narrow(index: Index, dim: int, picks: list[int] | None, size: int) -> Index:
|
|
27
|
+
if picks is None:
|
|
28
|
+
return index
|
|
29
|
+
current = index[dim]
|
|
30
|
+
base = list(range(*current.indices(size))) if isinstance(current, slice) else list(current) # type: ignore[arg-type]
|
|
31
|
+
try:
|
|
32
|
+
chosen = [base[i] for i in picks]
|
|
33
|
+
except IndexError:
|
|
34
|
+
raise PatchError(f"unit index out of range (0-{len(base) - 1})") from None
|
|
35
|
+
return (*index[:dim], chosen, *index[dim + 1 :])
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _lowrank_on(
|
|
39
|
+
ctx: OpContext, target: Target, index: Index, along: dict[int, torch.Tensor], **meta: Any
|
|
40
|
+
) -> TensorDelta:
|
|
41
|
+
"""Build ``Δ[index] = a ⊗ b`` where ``along`` maps two param dims to their vectors."""
|
|
42
|
+
free = _free_dims(index)
|
|
43
|
+
if len(free) != 2:
|
|
44
|
+
raise PatchError(f"{target.address}: needs a 2-D block, selection leaves {len(free)} dims")
|
|
45
|
+
first, second = free
|
|
46
|
+
if set(along) != {first, second}:
|
|
47
|
+
raise PatchError(
|
|
48
|
+
f"{target.address}: vectors given for dims {sorted(along)}, block has {free}"
|
|
49
|
+
)
|
|
50
|
+
shape = ctx.ir.params[target.param].shape
|
|
51
|
+
return patch.lowrank(
|
|
52
|
+
target.param,
|
|
53
|
+
tuple(shape),
|
|
54
|
+
ctx.base_hash(target.param),
|
|
55
|
+
along[first],
|
|
56
|
+
along[second],
|
|
57
|
+
index=index,
|
|
58
|
+
**meta,
|
|
59
|
+
)
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def _layout(ctx: OpContext, target: Target) -> tuple[int, int]:
|
|
63
|
+
role = target.role or ctx.ir.primary_role(target.param)
|
|
64
|
+
shape = ctx.ir.params[target.param].shape
|
|
65
|
+
if role is not None and role.in_dim is not None and role.out_dim is not None:
|
|
66
|
+
return role.in_dim, role.out_dim
|
|
67
|
+
if len(shape) == 2:
|
|
68
|
+
return 1, 0 # nn.Linear convention when the graph said nothing
|
|
69
|
+
raise PatchError(f"{target.address}: unknown matrix layout")
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def _residual_side(ctx: OpContext, target: Target) -> str:
|
|
73
|
+
"""Whether the residual stream is this weight's output ("out") or its input ("in")."""
|
|
74
|
+
role = target.role or ctx.ir.primary_role(target.param)
|
|
75
|
+
if role is None:
|
|
76
|
+
return "out"
|
|
77
|
+
if role.kind in ("reader", "router", "unembed"):
|
|
78
|
+
return "in"
|
|
79
|
+
if role.kind == "expert":
|
|
80
|
+
return "out" if role.address.rsplit(".", 1)[-1] == "down" else "in"
|
|
81
|
+
return "out"
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
# --------------------------------------------------------------------------- primitives
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def ablate(ctx: OpContext, targets: Any, *, units: list[int] | None = None) -> list[TensorDelta]:
|
|
88
|
+
"""Zero the selected weights (``units`` narrows the unit axis: columns of writers, rows of readers)."""
|
|
89
|
+
return set_(ctx, targets, value=0.0, units=units)
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def set_(
|
|
93
|
+
ctx: OpContext,
|
|
94
|
+
targets: Any,
|
|
95
|
+
*,
|
|
96
|
+
value: float | torch.Tensor = 0.0,
|
|
97
|
+
units: list[int] | None = None,
|
|
98
|
+
) -> list[TensorDelta]:
|
|
99
|
+
"""Set the selected weights to ``value``."""
|
|
100
|
+
out = []
|
|
101
|
+
for target in ctx.targets(targets):
|
|
102
|
+
index = _unit_index(ctx, target, units)
|
|
103
|
+
current = select(ctx.weight(target.param), index)
|
|
104
|
+
new = torch.as_tensor(value, dtype=torch.float32, device=current.device).expand_as(current)
|
|
105
|
+
out.append(
|
|
106
|
+
patch.dense(
|
|
107
|
+
target.param,
|
|
108
|
+
tuple(ctx.ir.params[target.param].shape),
|
|
109
|
+
ctx.base_hash(target.param),
|
|
110
|
+
new - current,
|
|
111
|
+
index=index,
|
|
112
|
+
kind="set",
|
|
113
|
+
)
|
|
114
|
+
)
|
|
115
|
+
return out
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def scale(
|
|
119
|
+
ctx: OpContext, targets: Any, *, factor: float, units: list[int] | None = None
|
|
120
|
+
) -> list[TensorDelta]:
|
|
121
|
+
"""Multiply the selected weights by ``factor``."""
|
|
122
|
+
out = []
|
|
123
|
+
for target in ctx.targets(targets):
|
|
124
|
+
index = _unit_index(ctx, target, units)
|
|
125
|
+
current = select(ctx.weight(target.param), index)
|
|
126
|
+
out.append(
|
|
127
|
+
patch.dense(
|
|
128
|
+
target.param,
|
|
129
|
+
tuple(ctx.ir.params[target.param].shape),
|
|
130
|
+
ctx.base_hash(target.param),
|
|
131
|
+
(factor - 1.0) * current,
|
|
132
|
+
index=index,
|
|
133
|
+
kind="scale",
|
|
134
|
+
factor=factor,
|
|
135
|
+
)
|
|
136
|
+
)
|
|
137
|
+
return out
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def _unit_index(ctx: OpContext, target: Target, units: list[int] | None) -> Index:
|
|
141
|
+
index = _index(ctx, target)
|
|
142
|
+
if units is None:
|
|
143
|
+
return index
|
|
144
|
+
shape = ctx.ir.params[target.param].shape
|
|
145
|
+
if len(shape) == 1:
|
|
146
|
+
return _narrow(index, 0, units, shape[0])
|
|
147
|
+
in_dim, out_dim = _layout(ctx, target)
|
|
148
|
+
dim = in_dim if _residual_side(ctx, target) == "out" else out_dim
|
|
149
|
+
return _narrow(index, dim, units, shape[dim])
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
# --------------------------------------------------------------------------- directional
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def project_out(
|
|
156
|
+
ctx: OpContext,
|
|
157
|
+
direction: Any,
|
|
158
|
+
targets: Any,
|
|
159
|
+
*,
|
|
160
|
+
cols: list[int] | None = None,
|
|
161
|
+
strength: float = 1.0,
|
|
162
|
+
) -> list[TensorDelta]:
|
|
163
|
+
"""Remove ``direction`` from what each target writes to (or reads from) the residual stream.
|
|
164
|
+
|
|
165
|
+
Writers: ``W ← W − s·d dᵀ W`` on the selected units; readers: ``W ← W − s·(W d) dᵀ``.
|
|
166
|
+
Each is a rank-1 delta.
|
|
167
|
+
"""
|
|
168
|
+
d = ctx.direction(direction)
|
|
169
|
+
d = d / d.norm()
|
|
170
|
+
out = []
|
|
171
|
+
for target in ctx.targets(targets):
|
|
172
|
+
shape = ctx.ir.params[target.param].shape
|
|
173
|
+
in_dim, out_dim = _layout(ctx, target)
|
|
174
|
+
side = _residual_side(ctx, target)
|
|
175
|
+
index = _index(ctx, target)
|
|
176
|
+
if cols is not None:
|
|
177
|
+
unit_dim = in_dim if side == "out" else out_dim
|
|
178
|
+
index = _narrow(index, unit_dim, cols, shape[unit_dim])
|
|
179
|
+
block = select(ctx.weight(target.param), index)
|
|
180
|
+
free = _free_dims(index)
|
|
181
|
+
res_dim = out_dim if side == "out" else in_dim
|
|
182
|
+
other = in_dim if side == "out" else out_dim
|
|
183
|
+
if res_dim not in free or other not in free:
|
|
184
|
+
raise PatchError(f"{target.address}: the selection must keep both matrix dims")
|
|
185
|
+
dd = d.to(block.device)
|
|
186
|
+
# Express the block as (residual, other) to compute the projection.
|
|
187
|
+
mat = block if free.index(res_dim) == 0 else block.T
|
|
188
|
+
if mat.shape[0] != dd.numel():
|
|
189
|
+
raise PatchError(
|
|
190
|
+
f"{target.address}: direction has {dd.numel()} dims, residual side has {mat.shape[0]}"
|
|
191
|
+
)
|
|
192
|
+
coeff = mat.T @ dd # what each unit writes/reads along d
|
|
193
|
+
along = {res_dim: -strength * dd, other: coeff}
|
|
194
|
+
out.append(_lowrank_on(ctx, target, index, along, op="project_out", strength=strength))
|
|
195
|
+
return out
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def bake_steer(ctx: OpContext, direction: Any, *, scale: float, at: int) -> list[TensorDelta]:
|
|
199
|
+
"""Add ``scale · d`` to the residual stream after block ``at``, permanently.
|
|
200
|
+
|
|
201
|
+
Uses a writer bias in that block when one exists; at block 0 the token
|
|
202
|
+
embedding works too (every row shifts by the same vector).
|
|
203
|
+
"""
|
|
204
|
+
d = ctx.direction(direction)
|
|
205
|
+
d = d / d.norm()
|
|
206
|
+
biases = [
|
|
207
|
+
r
|
|
208
|
+
for r in ctx.ir.roles
|
|
209
|
+
if r.block == at
|
|
210
|
+
and r.kind == "bias"
|
|
211
|
+
and r.split is None
|
|
212
|
+
and r.address.endswith((".o.bias", ".down.bias", ".out.bias"))
|
|
213
|
+
]
|
|
214
|
+
if biases:
|
|
215
|
+
role = biases[-1]
|
|
216
|
+
shape = ctx.ir.params[role.param].shape
|
|
217
|
+
return [
|
|
218
|
+
patch.dense(
|
|
219
|
+
role.param,
|
|
220
|
+
tuple(shape),
|
|
221
|
+
ctx.base_hash(role.param),
|
|
222
|
+
(scale * d).to(torch.float32),
|
|
223
|
+
op="bake_steer",
|
|
224
|
+
at=at,
|
|
225
|
+
)
|
|
226
|
+
]
|
|
227
|
+
embed = ctx.ir.semantics.embed
|
|
228
|
+
if at in (0, -1) and embed is not None:
|
|
229
|
+
shape = ctx.ir.params[embed].shape
|
|
230
|
+
ones = torch.ones(shape[0], dtype=torch.float32, device=d.device)
|
|
231
|
+
return [
|
|
232
|
+
patch.lowrank(
|
|
233
|
+
embed, tuple(shape), ctx.base_hash(embed), ones, scale * d, op="bake_steer", at=at
|
|
234
|
+
)
|
|
235
|
+
]
|
|
236
|
+
raise PatchError(
|
|
237
|
+
f"block {at} has no writer bias to carry a steering vector; use at=0 (via the embedding) "
|
|
238
|
+
"or a block whose attention/MLP output has a bias"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def rank_one_write(
|
|
243
|
+
ctx: OpContext,
|
|
244
|
+
target: Any,
|
|
245
|
+
*,
|
|
246
|
+
key: Any,
|
|
247
|
+
value: Any,
|
|
248
|
+
cov: torch.Tensor | None = None,
|
|
249
|
+
) -> list[TensorDelta]:
|
|
250
|
+
"""ROME-style: make ``W k = v*`` with the smallest change, ``Δ = (v* − W k)(C⁻¹k)ᵀ / (kᵀC⁻¹k)``."""
|
|
251
|
+
(t,) = ctx.targets(target)
|
|
252
|
+
in_dim, out_dim = _layout(ctx, t)
|
|
253
|
+
index = _index(ctx, t)
|
|
254
|
+
block = select(ctx.weight(t.param), index)
|
|
255
|
+
free = _free_dims(index)
|
|
256
|
+
mat = block if free.index(out_dim) == 0 else block.T # (out, in)
|
|
257
|
+
k = ctx.direction(key).to(mat.device)
|
|
258
|
+
v_star = ctx.direction(value).to(mat.device)
|
|
259
|
+
ck = torch.linalg.solve(cov.to(mat.device).float(), k) if cov is not None else k
|
|
260
|
+
residual = v_star - mat @ k
|
|
261
|
+
along = {out_dim: residual / float(k @ ck), in_dim: ck}
|
|
262
|
+
return [_lowrank_on(ctx, t, index, along, op="rank_one_write")]
|
|
263
|
+
|
|
264
|
+
|
|
265
|
+
def transplant(ctx: OpContext, targets: Any, *, source: str) -> list[TensorDelta]:
|
|
266
|
+
"""Copy the selected weights from another loaded model with matching IR roles."""
|
|
267
|
+
if source not in ctx.sources:
|
|
268
|
+
raise PatchError(f"no source model {source!r} is loaded")
|
|
269
|
+
src_ir, src_model = ctx.sources[source]
|
|
270
|
+
src_params = dict(src_model.named_parameters())
|
|
271
|
+
out = []
|
|
272
|
+
for target in ctx.targets(targets):
|
|
273
|
+
matches = src_ir.resolve(target.address)
|
|
274
|
+
match = next(
|
|
275
|
+
(m for m in matches if m.index == target.index), matches[0] if matches else None
|
|
276
|
+
)
|
|
277
|
+
if match is None:
|
|
278
|
+
raise PatchError(f"{target.address}: no matching weight in {source}")
|
|
279
|
+
index = _index(ctx, target)
|
|
280
|
+
src = select(
|
|
281
|
+
src_params[match.param].detach().float(), normalize_index(match.index, len(index))
|
|
282
|
+
)
|
|
283
|
+
cur = select(ctx.weight(target.param), index)
|
|
284
|
+
if src.shape != cur.shape:
|
|
285
|
+
raise PatchError(
|
|
286
|
+
f"{target.address}: shape {tuple(src.shape)} in {source}, {tuple(cur.shape)} here"
|
|
287
|
+
)
|
|
288
|
+
out.append(
|
|
289
|
+
patch.dense(
|
|
290
|
+
target.param,
|
|
291
|
+
tuple(ctx.ir.params[target.param].shape),
|
|
292
|
+
ctx.base_hash(target.param),
|
|
293
|
+
src.to(cur.device) - cur,
|
|
294
|
+
index=index,
|
|
295
|
+
kind="set",
|
|
296
|
+
op="transplant",
|
|
297
|
+
source=source,
|
|
298
|
+
)
|
|
299
|
+
)
|
|
300
|
+
return out
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def orthogonalize(
|
|
304
|
+
ctx: OpContext, deltas: list[TensorDelta], *, against: list[TensorDelta], side: str = "out"
|
|
305
|
+
) -> list[TensorDelta]:
|
|
306
|
+
"""Project each delta off the output (or input) subspace of ``against`` on the same tensor.
|
|
307
|
+
|
|
308
|
+
``Δ_B′ = (I − U_A U_Aᵀ) Δ_B``: B can no longer write what A writes.
|
|
309
|
+
"""
|
|
310
|
+
out = []
|
|
311
|
+
for delta in deltas:
|
|
312
|
+
basis = _subspace([a for a in against if a.tensor == delta.tensor], side)
|
|
313
|
+
if basis is None:
|
|
314
|
+
out.append(delta)
|
|
315
|
+
continue
|
|
316
|
+
if delta.kind == "lowrank" and delta.index is None and side == "out":
|
|
317
|
+
assert delta.u is not None
|
|
318
|
+
u = delta.u - basis @ (basis.T @ delta.u)
|
|
319
|
+
out.append(
|
|
320
|
+
patch.lowrank(
|
|
321
|
+
delta.tensor,
|
|
322
|
+
delta.shape,
|
|
323
|
+
delta.base,
|
|
324
|
+
u,
|
|
325
|
+
delta.v, # type: ignore[arg-type]
|
|
326
|
+
scale=delta.scale,
|
|
327
|
+
op="orthogonalize",
|
|
328
|
+
)
|
|
329
|
+
)
|
|
330
|
+
continue
|
|
331
|
+
full = delta.dense()
|
|
332
|
+
mat = full if side == "out" else full.T
|
|
333
|
+
proj = mat - basis @ (basis.T @ mat)
|
|
334
|
+
out.append(
|
|
335
|
+
patch.compact(
|
|
336
|
+
patch.dense(
|
|
337
|
+
delta.tensor,
|
|
338
|
+
delta.shape,
|
|
339
|
+
delta.base,
|
|
340
|
+
proj if side == "out" else proj.T,
|
|
341
|
+
op="orthogonalize",
|
|
342
|
+
)
|
|
343
|
+
)
|
|
344
|
+
)
|
|
345
|
+
del ctx
|
|
346
|
+
return out
|
|
347
|
+
|
|
348
|
+
|
|
349
|
+
def _subspace(deltas: list[TensorDelta], side: str, tol: float = 1e-6) -> torch.Tensor | None:
|
|
350
|
+
if not deltas:
|
|
351
|
+
return None
|
|
352
|
+
cols = []
|
|
353
|
+
for d in deltas:
|
|
354
|
+
if d.kind == "lowrank" and d.index is None:
|
|
355
|
+
assert d.u is not None
|
|
356
|
+
assert d.v is not None
|
|
357
|
+
cols.append(d.u if side == "out" else d.v)
|
|
358
|
+
else:
|
|
359
|
+
full = d.dense()
|
|
360
|
+
mat = full if side == "out" else full.T
|
|
361
|
+
u, s, _ = torch.linalg.svd(mat, full_matrices=False)
|
|
362
|
+
cols.append(u[:, s > tol * max(float(s.max()), 1e-30)])
|
|
363
|
+
stacked = torch.cat(cols, dim=1)
|
|
364
|
+
q, r = torch.linalg.qr(stacked)
|
|
365
|
+
keep = r.diagonal().abs() > tol * max(float(r.diagonal().abs().max()), 1e-30)
|
|
366
|
+
basis: torch.Tensor = q[:, keep]
|
|
367
|
+
return basis
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
BUILTIN: dict[str, Callable[..., list[TensorDelta]]] = {
|
|
371
|
+
"ablate": ablate,
|
|
372
|
+
"set": set_,
|
|
373
|
+
"scale": scale,
|
|
374
|
+
"project_out": project_out,
|
|
375
|
+
"bake_steer": bake_steer,
|
|
376
|
+
"rank_one_write": rank_one_write,
|
|
377
|
+
"transplant": transplant,
|
|
378
|
+
"orthogonalize": orthogonalize,
|
|
379
|
+
}
|
wspx/lab/patch.py
ADDED
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
"""The patch IR (§5.3): the only edit forms the store and the lab understand.
|
|
2
|
+
|
|
3
|
+
Every primitive is an fp32 delta from the base tensor, kept compact:
|
|
4
|
+
|
|
5
|
+
======== =================================================================
|
|
6
|
+
lowrank ``Δ[index] = scale · Σ_k u[:, k] ⊗ v[:, k]`` on a 2-D block
|
|
7
|
+
sparse ``Δ.flat[indices] = values`` (indices unique)
|
|
8
|
+
dense ``Δ[index] = delta`` (a block, or the whole tensor)
|
|
9
|
+
scale a dense block delta, recorded as a scaling (``(f − 1) · W``)
|
|
10
|
+
set a dense block delta, recorded as an assignment (``values − W``)
|
|
11
|
+
======== =================================================================
|
|
12
|
+
|
|
13
|
+
Materialisation sums every delta of a tensor in fp32, in canonical order
|
|
14
|
+
(sorted by blob id), onto the fp32 base and rounds once (§5.5.2). Low-rank
|
|
15
|
+
products are built as sequential outer products, never a GEMM, so every
|
|
16
|
+
element is a fixed sequence of IEEE operations: the same history gives the
|
|
17
|
+
same bits on any device and in any order.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import json
|
|
23
|
+
from dataclasses import dataclass, field
|
|
24
|
+
from typing import Any
|
|
25
|
+
|
|
26
|
+
import torch
|
|
27
|
+
from safetensors.torch import load as st_load
|
|
28
|
+
from safetensors.torch import save as st_save
|
|
29
|
+
|
|
30
|
+
from wspx.errors import PatchError
|
|
31
|
+
from wspx.lab.tensors import Index, Selector, assign, select
|
|
32
|
+
from wspx.store.objects import canonical_json, oid_of
|
|
33
|
+
|
|
34
|
+
KINDS = ("lowrank", "sparse", "dense", "scale", "set")
|
|
35
|
+
FORMAT = "wspx.delta/1"
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def encode_index(index: Index | None) -> list[Any] | None:
|
|
39
|
+
if index is None:
|
|
40
|
+
return None
|
|
41
|
+
out: list[Any] = []
|
|
42
|
+
for sel in index:
|
|
43
|
+
if isinstance(sel, int):
|
|
44
|
+
out.append(sel)
|
|
45
|
+
elif isinstance(sel, slice):
|
|
46
|
+
out.append({"s": [sel.start, sel.stop, sel.step]})
|
|
47
|
+
else:
|
|
48
|
+
out.append({"l": list(sel)})
|
|
49
|
+
return out
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def decode_index(data: list[Any] | None) -> Index | None:
|
|
53
|
+
if data is None:
|
|
54
|
+
return None
|
|
55
|
+
out: list[Selector] = []
|
|
56
|
+
for item in data:
|
|
57
|
+
if isinstance(item, int):
|
|
58
|
+
out.append(item)
|
|
59
|
+
elif "s" in item:
|
|
60
|
+
out.append(slice(*item["s"]))
|
|
61
|
+
else:
|
|
62
|
+
out.append(list(item["l"]))
|
|
63
|
+
return tuple(out)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
@dataclass
|
|
67
|
+
class TensorDelta:
|
|
68
|
+
"""One tensor's delta. Immutable once encoded; identified by the hash of its bytes."""
|
|
69
|
+
|
|
70
|
+
tensor: str
|
|
71
|
+
kind: str
|
|
72
|
+
shape: tuple[int, ...]
|
|
73
|
+
base: str
|
|
74
|
+
"""Hash of the base tensor this delta applies to."""
|
|
75
|
+
index: Index | None = None
|
|
76
|
+
u: torch.Tensor | None = None
|
|
77
|
+
v: torch.Tensor | None = None
|
|
78
|
+
scale: float = 1.0
|
|
79
|
+
indices: torch.Tensor | None = None
|
|
80
|
+
values: torch.Tensor | None = None
|
|
81
|
+
delta: torch.Tensor | None = None
|
|
82
|
+
meta: dict[str, Any] = field(default_factory=dict)
|
|
83
|
+
_bytes: bytes | None = field(default=None, repr=False, compare=False)
|
|
84
|
+
|
|
85
|
+
def __post_init__(self) -> None:
|
|
86
|
+
if self.kind not in KINDS:
|
|
87
|
+
raise PatchError(f"unknown delta kind {self.kind!r}")
|
|
88
|
+
if self.index is not None and all(
|
|
89
|
+
isinstance(s, slice) and s == slice(None) for s in self.index
|
|
90
|
+
):
|
|
91
|
+
self.index = None # a whole-tensor selection is stored as no selection
|
|
92
|
+
if self.kind == "lowrank":
|
|
93
|
+
if self.u is None or self.v is None:
|
|
94
|
+
raise PatchError("lowrank delta needs u and v")
|
|
95
|
+
if self.u.dim() == 1:
|
|
96
|
+
self.u = self.u[:, None]
|
|
97
|
+
if self.v.dim() == 1:
|
|
98
|
+
self.v = self.v[:, None]
|
|
99
|
+
if self.u.shape[1] != self.v.shape[1]:
|
|
100
|
+
raise PatchError(
|
|
101
|
+
f"lowrank rank mismatch: u {tuple(self.u.shape)} v {tuple(self.v.shape)}"
|
|
102
|
+
)
|
|
103
|
+
self.u = self.u.detach().to(torch.float32).contiguous()
|
|
104
|
+
self.v = self.v.detach().to(torch.float32).contiguous()
|
|
105
|
+
elif self.kind == "sparse":
|
|
106
|
+
if self.indices is None or self.values is None:
|
|
107
|
+
raise PatchError("sparse delta needs indices and values")
|
|
108
|
+
order = torch.argsort(self.indices.detach().to(torch.long).cpu())
|
|
109
|
+
self.indices = self.indices.detach().to(torch.long).cpu()[order].contiguous()
|
|
110
|
+
self.values = self.values.detach().to(torch.float32).cpu()[order].contiguous()
|
|
111
|
+
if self.indices.numel() and bool((self.indices[1:] == self.indices[:-1]).any()):
|
|
112
|
+
raise PatchError("sparse delta indices must be unique")
|
|
113
|
+
else:
|
|
114
|
+
if self.delta is None:
|
|
115
|
+
raise PatchError(f"{self.kind} delta needs a delta block")
|
|
116
|
+
self.delta = self.delta.detach().to(torch.float32).contiguous()
|
|
117
|
+
|
|
118
|
+
# ------------------------------------------------------------------ geometry
|
|
119
|
+
|
|
120
|
+
@property
|
|
121
|
+
def rank(self) -> int | None:
|
|
122
|
+
return int(self.u.shape[1]) if self.u is not None else None
|
|
123
|
+
|
|
124
|
+
@property
|
|
125
|
+
def nnz(self) -> int:
|
|
126
|
+
"""Elements the delta can touch (an upper bound for low-rank blocks)."""
|
|
127
|
+
if self.kind == "sparse":
|
|
128
|
+
assert self.indices is not None
|
|
129
|
+
return int(self.indices.numel())
|
|
130
|
+
if self.kind == "lowrank":
|
|
131
|
+
assert self.u is not None
|
|
132
|
+
assert self.v is not None
|
|
133
|
+
return int(self.u.shape[0] * self.v.shape[0])
|
|
134
|
+
assert self.delta is not None
|
|
135
|
+
return int(self.delta.numel())
|
|
136
|
+
|
|
137
|
+
def nbytes(self) -> int:
|
|
138
|
+
return sum(
|
|
139
|
+
t.numel() * t.element_size()
|
|
140
|
+
for t in (self.u, self.v, self.indices, self.values, self.delta)
|
|
141
|
+
if t is not None
|
|
142
|
+
)
|
|
143
|
+
|
|
144
|
+
# ------------------------------------------------------------------ apply
|
|
145
|
+
|
|
146
|
+
def block(self, device: torch.device | str = "cpu") -> torch.Tensor:
|
|
147
|
+
"""The delta over its block (lowrank/dense kinds), fp32."""
|
|
148
|
+
if self.kind == "lowrank":
|
|
149
|
+
assert self.u is not None
|
|
150
|
+
assert self.v is not None
|
|
151
|
+
u = self.u.to(device)
|
|
152
|
+
v = self.v.to(device)
|
|
153
|
+
out = torch.zeros(u.shape[0], v.shape[0], dtype=torch.float32, device=device)
|
|
154
|
+
for k in range(u.shape[1]):
|
|
155
|
+
out.add_(torch.outer(u[:, k], v[:, k]))
|
|
156
|
+
if self.scale != 1.0:
|
|
157
|
+
out.mul_(self.scale)
|
|
158
|
+
return out
|
|
159
|
+
if self.kind == "sparse":
|
|
160
|
+
raise PatchError("sparse deltas have no block form")
|
|
161
|
+
assert self.delta is not None
|
|
162
|
+
return self.delta.to(device)
|
|
163
|
+
|
|
164
|
+
def add_into(self, acc: torch.Tensor) -> None:
|
|
165
|
+
"""``acc += Δ`` for an fp32 accumulator of the full tensor shape."""
|
|
166
|
+
if tuple(acc.shape) != tuple(self.shape):
|
|
167
|
+
raise PatchError(
|
|
168
|
+
f"{self.tensor}: delta for shape {self.shape} applied to {tuple(acc.shape)}"
|
|
169
|
+
)
|
|
170
|
+
if self.kind == "sparse":
|
|
171
|
+
assert self.indices is not None
|
|
172
|
+
assert self.values is not None
|
|
173
|
+
flat = acc.view(-1)
|
|
174
|
+
flat.index_add_(0, self.indices.to(acc.device), self.values.to(acc.device))
|
|
175
|
+
return
|
|
176
|
+
block = self.block(acc.device)
|
|
177
|
+
target = select(acc, self.index)
|
|
178
|
+
if tuple(target.shape) != tuple(block.shape):
|
|
179
|
+
raise PatchError(
|
|
180
|
+
f"{self.tensor}: block {tuple(block.shape)} does not match selection {tuple(target.shape)}"
|
|
181
|
+
)
|
|
182
|
+
if self.index is None or not any(isinstance(s, list) for s in self.index):
|
|
183
|
+
target.add_(block) # a view: writes through
|
|
184
|
+
else:
|
|
185
|
+
assign(acc, self.index, target + block)
|
|
186
|
+
|
|
187
|
+
def dense(self, device: torch.device | str = "cpu") -> torch.Tensor:
|
|
188
|
+
out = torch.zeros(self.shape, dtype=torch.float32, device=device)
|
|
189
|
+
self.add_into(out)
|
|
190
|
+
return out
|
|
191
|
+
|
|
192
|
+
# ------------------------------------------------------------------ codec
|
|
193
|
+
|
|
194
|
+
def to_bytes(self) -> bytes:
|
|
195
|
+
if self._bytes is None:
|
|
196
|
+
tensors: dict[str, torch.Tensor] = {}
|
|
197
|
+
for name in ("u", "v", "indices", "values", "delta"):
|
|
198
|
+
value = getattr(self, name)
|
|
199
|
+
if value is not None:
|
|
200
|
+
tensors[name] = value.detach().cpu().contiguous()
|
|
201
|
+
header = {
|
|
202
|
+
"format": FORMAT,
|
|
203
|
+
"tensor": self.tensor,
|
|
204
|
+
"kind": self.kind,
|
|
205
|
+
"shape": list(self.shape),
|
|
206
|
+
"base": self.base,
|
|
207
|
+
"index": encode_index(self.index),
|
|
208
|
+
"scale": self.scale,
|
|
209
|
+
"meta": self.meta,
|
|
210
|
+
}
|
|
211
|
+
self._bytes = st_save(tensors, metadata={"wspx": canonical_json(header).decode()})
|
|
212
|
+
return self._bytes
|
|
213
|
+
|
|
214
|
+
@property
|
|
215
|
+
def oid(self) -> str:
|
|
216
|
+
return oid_of(self.to_bytes())
|
|
217
|
+
|
|
218
|
+
@classmethod
|
|
219
|
+
def from_bytes(cls, data: bytes) -> TensorDelta:
|
|
220
|
+
try:
|
|
221
|
+
header_len = int.from_bytes(data[:8], "little")
|
|
222
|
+
meta = json.loads(data[8 : 8 + header_len])["__metadata__"]
|
|
223
|
+
header = json.loads(meta["wspx"])
|
|
224
|
+
except (ValueError, KeyError, json.JSONDecodeError) as exc:
|
|
225
|
+
raise PatchError(f"not a wspx delta blob: {exc}") from None
|
|
226
|
+
if header.get("format") != FORMAT:
|
|
227
|
+
raise PatchError(f"unsupported delta format {header.get('format')!r}")
|
|
228
|
+
tensors = st_load(data)
|
|
229
|
+
delta = cls(
|
|
230
|
+
tensor=header["tensor"],
|
|
231
|
+
kind=header["kind"],
|
|
232
|
+
shape=tuple(header["shape"]),
|
|
233
|
+
base=header["base"],
|
|
234
|
+
index=decode_index(header["index"]),
|
|
235
|
+
scale=float(header["scale"]),
|
|
236
|
+
meta=header.get("meta", {}),
|
|
237
|
+
u=tensors.get("u"),
|
|
238
|
+
v=tensors.get("v"),
|
|
239
|
+
indices=tensors.get("indices"),
|
|
240
|
+
values=tensors.get("values"),
|
|
241
|
+
delta=tensors.get("delta"),
|
|
242
|
+
)
|
|
243
|
+
delta._bytes = data
|
|
244
|
+
return delta
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def materialize(
|
|
248
|
+
base: torch.Tensor, deltas: list[TensorDelta], *, dtype: torch.dtype | None = None
|
|
249
|
+
) -> torch.Tensor:
|
|
250
|
+
"""``cast(fp32(base) + Σ fp32(Δ))`` with deltas in canonical order, rounded once."""
|
|
251
|
+
target = dtype or base.dtype
|
|
252
|
+
if not deltas:
|
|
253
|
+
return base.to(target, copy=True)
|
|
254
|
+
acc = base.to(torch.float32, copy=True)
|
|
255
|
+
for delta in sorted(deltas, key=lambda d: d.oid):
|
|
256
|
+
delta.add_into(acc)
|
|
257
|
+
return acc.to(target)
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
# --------------------------------------------------------------------------- builders
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def lowrank(
|
|
264
|
+
tensor: str,
|
|
265
|
+
shape: tuple[int, ...],
|
|
266
|
+
base: str,
|
|
267
|
+
u: torch.Tensor,
|
|
268
|
+
v: torch.Tensor,
|
|
269
|
+
*,
|
|
270
|
+
index: Index | None = None,
|
|
271
|
+
scale: float = 1.0,
|
|
272
|
+
**meta: Any,
|
|
273
|
+
) -> TensorDelta:
|
|
274
|
+
return TensorDelta(
|
|
275
|
+
tensor, "lowrank", shape, base, index=index, u=u, v=v, scale=scale, meta=meta
|
|
276
|
+
)
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def dense(
|
|
280
|
+
tensor: str,
|
|
281
|
+
shape: tuple[int, ...],
|
|
282
|
+
base: str,
|
|
283
|
+
delta: torch.Tensor,
|
|
284
|
+
*,
|
|
285
|
+
index: Index | None = None,
|
|
286
|
+
kind: str = "dense",
|
|
287
|
+
**meta: Any,
|
|
288
|
+
) -> TensorDelta:
|
|
289
|
+
return TensorDelta(tensor, kind, shape, base, index=index, delta=delta, meta=meta)
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
def sparse(
|
|
293
|
+
tensor: str,
|
|
294
|
+
shape: tuple[int, ...],
|
|
295
|
+
base: str,
|
|
296
|
+
indices: torch.Tensor,
|
|
297
|
+
values: torch.Tensor,
|
|
298
|
+
**meta: Any,
|
|
299
|
+
) -> TensorDelta:
|
|
300
|
+
return TensorDelta(tensor, "sparse", shape, base, indices=indices, values=values, meta=meta)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def compact(delta: TensorDelta) -> TensorDelta:
|
|
304
|
+
"""Re-encode a dense delta as sparse when that is smaller. Values are unchanged."""
|
|
305
|
+
if delta.kind not in ("dense", "scale", "set") or delta.index is not None:
|
|
306
|
+
return delta
|
|
307
|
+
assert delta.delta is not None
|
|
308
|
+
nz = torch.nonzero(delta.delta.reshape(-1)).reshape(-1)
|
|
309
|
+
if nz.numel() * 3 >= delta.delta.numel():
|
|
310
|
+
return delta
|
|
311
|
+
values = delta.delta.reshape(-1)[nz]
|
|
312
|
+
return TensorDelta(
|
|
313
|
+
delta.tensor,
|
|
314
|
+
"sparse",
|
|
315
|
+
delta.shape,
|
|
316
|
+
delta.base,
|
|
317
|
+
indices=nz,
|
|
318
|
+
values=values,
|
|
319
|
+
meta={**delta.meta, "from": delta.kind},
|
|
320
|
+
)
|