wspx 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. wspx/__init__.py +24 -0
  2. wspx/_version.py +1 -0
  3. wspx/addressing.py +415 -0
  4. wspx/auth.py +77 -0
  5. wspx/cli/__init__.py +1 -0
  6. wspx/cli/client.py +124 -0
  7. wspx/cli/config.py +89 -0
  8. wspx/cli/connect.py +88 -0
  9. wspx/cli/main.py +848 -0
  10. wspx/cloud/__init__.py +20 -0
  11. wspx/cloud/app.py +191 -0
  12. wspx/cloud/backend.py +141 -0
  13. wspx/cloud/compose.py +85 -0
  14. wspx/cloud/deploy.py +102 -0
  15. wspx/config.py +189 -0
  16. wspx/control/__init__.py +5 -0
  17. wspx/control/app.py +488 -0
  18. wspx/control/backend.py +244 -0
  19. wspx/control/gateway_client.py +79 -0
  20. wspx/control/mcp_server.py +71 -0
  21. wspx/control/sessions.py +424 -0
  22. wspx/control/tracedb.py +183 -0
  23. wspx/control/ui/assets/index-BJFweZAn.css +1 -0
  24. wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
  25. wspx/control/ui/index.html +13 -0
  26. wspx/control/views.py +180 -0
  27. wspx/diagnostics.py +146 -0
  28. wspx/driver/__init__.py +7 -0
  29. wspx/driver/loop.py +255 -0
  30. wspx/driver/messages.py +62 -0
  31. wspx/driver/providers/__init__.py +84 -0
  32. wspx/driver/providers/anthropic.py +149 -0
  33. wspx/driver/providers/compat.py +115 -0
  34. wspx/driver/providers/google.py +138 -0
  35. wspx/driver/providers/openai.py +125 -0
  36. wspx/driver/run.py +134 -0
  37. wspx/errors.py +69 -0
  38. wspx/gateway/__init__.py +8 -0
  39. wspx/gateway/app.py +626 -0
  40. wspx/gateway/ledger.py +536 -0
  41. wspx/gateway/prices.py +75 -0
  42. wspx/gateway/usage.py +284 -0
  43. wspx/handles.py +117 -0
  44. wspx/integrations/AGENTS.snippet.md +8 -0
  45. wspx/integrations/SKILL.md +52 -0
  46. wspx/integrations/__init__.py +17 -0
  47. wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
  48. wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
  49. wspx/integrations/codex-hooks.toml +9 -0
  50. wspx/lab/__init__.py +5 -0
  51. wspx/lab/analysis.py +485 -0
  52. wspx/lab/check.py +489 -0
  53. wspx/lab/evaluate.py +108 -0
  54. wspx/lab/export.py +275 -0
  55. wspx/lab/grep.py +754 -0
  56. wspx/lab/index.py +239 -0
  57. wspx/lab/ir/__init__.py +1 -0
  58. wspx/lab/ir/capture.py +299 -0
  59. wspx/lab/ir/graph.py +148 -0
  60. wspx/lab/ir/model_ir.py +522 -0
  61. wspx/lab/ir/semantics.py +1255 -0
  62. wspx/lab/ir/tape.py +853 -0
  63. wspx/lab/loader.py +204 -0
  64. wspx/lab/merge.py +200 -0
  65. wspx/lab/ops/__init__.py +76 -0
  66. wspx/lab/ops/builtin.py +379 -0
  67. wspx/lab/patch.py +320 -0
  68. wspx/lab/probe.py +206 -0
  69. wspx/lab/results.py +114 -0
  70. wspx/lab/sdk.py +120 -0
  71. wspx/lab/server.py +236 -0
  72. wspx/lab/session.py +1574 -0
  73. wspx/lab/tensors.py +119 -0
  74. wspx/lab/text.py +69 -0
  75. wspx/lab/tree.py +302 -0
  76. wspx/loops.py +127 -0
  77. wspx/project.py +143 -0
  78. wspx/py.typed +0 -0
  79. wspx/safety/__init__.py +15 -0
  80. wspx/safety/kill.py +47 -0
  81. wspx/safety/sweeper.py +68 -0
  82. wspx/store/__init__.py +64 -0
  83. wspx/store/objects.py +130 -0
  84. wspx/store/overlay.py +209 -0
  85. wspx/store/refs.py +150 -0
  86. wspx/testing/__init__.py +21 -0
  87. wspx/testing/api.py +155 -0
  88. wspx/testing/builtin.py +139 -0
  89. wspx/testing/runner.py +257 -0
  90. wspx/tools/__init__.py +17 -0
  91. wspx/tools/dispatch.py +129 -0
  92. wspx/tools/schemas.py +313 -0
  93. wspx/vcs/__init__.py +17 -0
  94. wspx/vcs/repo.py +431 -0
  95. wspx-0.1.0.dev0.dist-info/METADATA +278 -0
  96. wspx-0.1.0.dev0.dist-info/RECORD +99 -0
  97. wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
  98. wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
  99. wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
@@ -0,0 +1,379 @@
1
+ """Built-in ops. Each compiles against the current weights into concrete deltas."""
2
+
3
+ from __future__ import annotations
4
+
5
+ from collections.abc import Callable
6
+ from typing import Any
7
+
8
+ import torch
9
+
10
+ from wspx.errors import PatchError
11
+ from wspx.lab import patch
12
+ from wspx.lab.ir.model_ir import Target
13
+ from wspx.lab.ops import OpContext
14
+ from wspx.lab.patch import TensorDelta
15
+ from wspx.lab.tensors import Index, normalize_index, select
16
+
17
+
18
+ def _free_dims(index: Index) -> list[int]:
19
+ return [d for d, s in enumerate(index) if not isinstance(s, int)]
20
+
21
+
22
+ def _index(ctx: OpContext, target: Target) -> Index:
23
+ return normalize_index(target.index, len(ctx.ir.params[target.param].shape))
24
+
25
+
26
+ def _narrow(index: Index, dim: int, picks: list[int] | None, size: int) -> Index:
27
+ if picks is None:
28
+ return index
29
+ current = index[dim]
30
+ base = list(range(*current.indices(size))) if isinstance(current, slice) else list(current) # type: ignore[arg-type]
31
+ try:
32
+ chosen = [base[i] for i in picks]
33
+ except IndexError:
34
+ raise PatchError(f"unit index out of range (0-{len(base) - 1})") from None
35
+ return (*index[:dim], chosen, *index[dim + 1 :])
36
+
37
+
38
+ def _lowrank_on(
39
+ ctx: OpContext, target: Target, index: Index, along: dict[int, torch.Tensor], **meta: Any
40
+ ) -> TensorDelta:
41
+ """Build ``Δ[index] = a ⊗ b`` where ``along`` maps two param dims to their vectors."""
42
+ free = _free_dims(index)
43
+ if len(free) != 2:
44
+ raise PatchError(f"{target.address}: needs a 2-D block, selection leaves {len(free)} dims")
45
+ first, second = free
46
+ if set(along) != {first, second}:
47
+ raise PatchError(
48
+ f"{target.address}: vectors given for dims {sorted(along)}, block has {free}"
49
+ )
50
+ shape = ctx.ir.params[target.param].shape
51
+ return patch.lowrank(
52
+ target.param,
53
+ tuple(shape),
54
+ ctx.base_hash(target.param),
55
+ along[first],
56
+ along[second],
57
+ index=index,
58
+ **meta,
59
+ )
60
+
61
+
62
+ def _layout(ctx: OpContext, target: Target) -> tuple[int, int]:
63
+ role = target.role or ctx.ir.primary_role(target.param)
64
+ shape = ctx.ir.params[target.param].shape
65
+ if role is not None and role.in_dim is not None and role.out_dim is not None:
66
+ return role.in_dim, role.out_dim
67
+ if len(shape) == 2:
68
+ return 1, 0 # nn.Linear convention when the graph said nothing
69
+ raise PatchError(f"{target.address}: unknown matrix layout")
70
+
71
+
72
+ def _residual_side(ctx: OpContext, target: Target) -> str:
73
+ """Whether the residual stream is this weight's output ("out") or its input ("in")."""
74
+ role = target.role or ctx.ir.primary_role(target.param)
75
+ if role is None:
76
+ return "out"
77
+ if role.kind in ("reader", "router", "unembed"):
78
+ return "in"
79
+ if role.kind == "expert":
80
+ return "out" if role.address.rsplit(".", 1)[-1] == "down" else "in"
81
+ return "out"
82
+
83
+
84
+ # --------------------------------------------------------------------------- primitives
85
+
86
+
87
+ def ablate(ctx: OpContext, targets: Any, *, units: list[int] | None = None) -> list[TensorDelta]:
88
+ """Zero the selected weights (``units`` narrows the unit axis: columns of writers, rows of readers)."""
89
+ return set_(ctx, targets, value=0.0, units=units)
90
+
91
+
92
+ def set_(
93
+ ctx: OpContext,
94
+ targets: Any,
95
+ *,
96
+ value: float | torch.Tensor = 0.0,
97
+ units: list[int] | None = None,
98
+ ) -> list[TensorDelta]:
99
+ """Set the selected weights to ``value``."""
100
+ out = []
101
+ for target in ctx.targets(targets):
102
+ index = _unit_index(ctx, target, units)
103
+ current = select(ctx.weight(target.param), index)
104
+ new = torch.as_tensor(value, dtype=torch.float32, device=current.device).expand_as(current)
105
+ out.append(
106
+ patch.dense(
107
+ target.param,
108
+ tuple(ctx.ir.params[target.param].shape),
109
+ ctx.base_hash(target.param),
110
+ new - current,
111
+ index=index,
112
+ kind="set",
113
+ )
114
+ )
115
+ return out
116
+
117
+
118
+ def scale(
119
+ ctx: OpContext, targets: Any, *, factor: float, units: list[int] | None = None
120
+ ) -> list[TensorDelta]:
121
+ """Multiply the selected weights by ``factor``."""
122
+ out = []
123
+ for target in ctx.targets(targets):
124
+ index = _unit_index(ctx, target, units)
125
+ current = select(ctx.weight(target.param), index)
126
+ out.append(
127
+ patch.dense(
128
+ target.param,
129
+ tuple(ctx.ir.params[target.param].shape),
130
+ ctx.base_hash(target.param),
131
+ (factor - 1.0) * current,
132
+ index=index,
133
+ kind="scale",
134
+ factor=factor,
135
+ )
136
+ )
137
+ return out
138
+
139
+
140
+ def _unit_index(ctx: OpContext, target: Target, units: list[int] | None) -> Index:
141
+ index = _index(ctx, target)
142
+ if units is None:
143
+ return index
144
+ shape = ctx.ir.params[target.param].shape
145
+ if len(shape) == 1:
146
+ return _narrow(index, 0, units, shape[0])
147
+ in_dim, out_dim = _layout(ctx, target)
148
+ dim = in_dim if _residual_side(ctx, target) == "out" else out_dim
149
+ return _narrow(index, dim, units, shape[dim])
150
+
151
+
152
+ # --------------------------------------------------------------------------- directional
153
+
154
+
155
+ def project_out(
156
+ ctx: OpContext,
157
+ direction: Any,
158
+ targets: Any,
159
+ *,
160
+ cols: list[int] | None = None,
161
+ strength: float = 1.0,
162
+ ) -> list[TensorDelta]:
163
+ """Remove ``direction`` from what each target writes to (or reads from) the residual stream.
164
+
165
+ Writers: ``W ← W − s·d dᵀ W`` on the selected units; readers: ``W ← W − s·(W d) dᵀ``.
166
+ Each is a rank-1 delta.
167
+ """
168
+ d = ctx.direction(direction)
169
+ d = d / d.norm()
170
+ out = []
171
+ for target in ctx.targets(targets):
172
+ shape = ctx.ir.params[target.param].shape
173
+ in_dim, out_dim = _layout(ctx, target)
174
+ side = _residual_side(ctx, target)
175
+ index = _index(ctx, target)
176
+ if cols is not None:
177
+ unit_dim = in_dim if side == "out" else out_dim
178
+ index = _narrow(index, unit_dim, cols, shape[unit_dim])
179
+ block = select(ctx.weight(target.param), index)
180
+ free = _free_dims(index)
181
+ res_dim = out_dim if side == "out" else in_dim
182
+ other = in_dim if side == "out" else out_dim
183
+ if res_dim not in free or other not in free:
184
+ raise PatchError(f"{target.address}: the selection must keep both matrix dims")
185
+ dd = d.to(block.device)
186
+ # Express the block as (residual, other) to compute the projection.
187
+ mat = block if free.index(res_dim) == 0 else block.T
188
+ if mat.shape[0] != dd.numel():
189
+ raise PatchError(
190
+ f"{target.address}: direction has {dd.numel()} dims, residual side has {mat.shape[0]}"
191
+ )
192
+ coeff = mat.T @ dd # what each unit writes/reads along d
193
+ along = {res_dim: -strength * dd, other: coeff}
194
+ out.append(_lowrank_on(ctx, target, index, along, op="project_out", strength=strength))
195
+ return out
196
+
197
+
198
+ def bake_steer(ctx: OpContext, direction: Any, *, scale: float, at: int) -> list[TensorDelta]:
199
+ """Add ``scale · d`` to the residual stream after block ``at``, permanently.
200
+
201
+ Uses a writer bias in that block when one exists; at block 0 the token
202
+ embedding works too (every row shifts by the same vector).
203
+ """
204
+ d = ctx.direction(direction)
205
+ d = d / d.norm()
206
+ biases = [
207
+ r
208
+ for r in ctx.ir.roles
209
+ if r.block == at
210
+ and r.kind == "bias"
211
+ and r.split is None
212
+ and r.address.endswith((".o.bias", ".down.bias", ".out.bias"))
213
+ ]
214
+ if biases:
215
+ role = biases[-1]
216
+ shape = ctx.ir.params[role.param].shape
217
+ return [
218
+ patch.dense(
219
+ role.param,
220
+ tuple(shape),
221
+ ctx.base_hash(role.param),
222
+ (scale * d).to(torch.float32),
223
+ op="bake_steer",
224
+ at=at,
225
+ )
226
+ ]
227
+ embed = ctx.ir.semantics.embed
228
+ if at in (0, -1) and embed is not None:
229
+ shape = ctx.ir.params[embed].shape
230
+ ones = torch.ones(shape[0], dtype=torch.float32, device=d.device)
231
+ return [
232
+ patch.lowrank(
233
+ embed, tuple(shape), ctx.base_hash(embed), ones, scale * d, op="bake_steer", at=at
234
+ )
235
+ ]
236
+ raise PatchError(
237
+ f"block {at} has no writer bias to carry a steering vector; use at=0 (via the embedding) "
238
+ "or a block whose attention/MLP output has a bias"
239
+ )
240
+
241
+
242
+ def rank_one_write(
243
+ ctx: OpContext,
244
+ target: Any,
245
+ *,
246
+ key: Any,
247
+ value: Any,
248
+ cov: torch.Tensor | None = None,
249
+ ) -> list[TensorDelta]:
250
+ """ROME-style: make ``W k = v*`` with the smallest change, ``Δ = (v* − W k)(C⁻¹k)ᵀ / (kᵀC⁻¹k)``."""
251
+ (t,) = ctx.targets(target)
252
+ in_dim, out_dim = _layout(ctx, t)
253
+ index = _index(ctx, t)
254
+ block = select(ctx.weight(t.param), index)
255
+ free = _free_dims(index)
256
+ mat = block if free.index(out_dim) == 0 else block.T # (out, in)
257
+ k = ctx.direction(key).to(mat.device)
258
+ v_star = ctx.direction(value).to(mat.device)
259
+ ck = torch.linalg.solve(cov.to(mat.device).float(), k) if cov is not None else k
260
+ residual = v_star - mat @ k
261
+ along = {out_dim: residual / float(k @ ck), in_dim: ck}
262
+ return [_lowrank_on(ctx, t, index, along, op="rank_one_write")]
263
+
264
+
265
+ def transplant(ctx: OpContext, targets: Any, *, source: str) -> list[TensorDelta]:
266
+ """Copy the selected weights from another loaded model with matching IR roles."""
267
+ if source not in ctx.sources:
268
+ raise PatchError(f"no source model {source!r} is loaded")
269
+ src_ir, src_model = ctx.sources[source]
270
+ src_params = dict(src_model.named_parameters())
271
+ out = []
272
+ for target in ctx.targets(targets):
273
+ matches = src_ir.resolve(target.address)
274
+ match = next(
275
+ (m for m in matches if m.index == target.index), matches[0] if matches else None
276
+ )
277
+ if match is None:
278
+ raise PatchError(f"{target.address}: no matching weight in {source}")
279
+ index = _index(ctx, target)
280
+ src = select(
281
+ src_params[match.param].detach().float(), normalize_index(match.index, len(index))
282
+ )
283
+ cur = select(ctx.weight(target.param), index)
284
+ if src.shape != cur.shape:
285
+ raise PatchError(
286
+ f"{target.address}: shape {tuple(src.shape)} in {source}, {tuple(cur.shape)} here"
287
+ )
288
+ out.append(
289
+ patch.dense(
290
+ target.param,
291
+ tuple(ctx.ir.params[target.param].shape),
292
+ ctx.base_hash(target.param),
293
+ src.to(cur.device) - cur,
294
+ index=index,
295
+ kind="set",
296
+ op="transplant",
297
+ source=source,
298
+ )
299
+ )
300
+ return out
301
+
302
+
303
+ def orthogonalize(
304
+ ctx: OpContext, deltas: list[TensorDelta], *, against: list[TensorDelta], side: str = "out"
305
+ ) -> list[TensorDelta]:
306
+ """Project each delta off the output (or input) subspace of ``against`` on the same tensor.
307
+
308
+ ``Δ_B′ = (I − U_A U_Aᵀ) Δ_B``: B can no longer write what A writes.
309
+ """
310
+ out = []
311
+ for delta in deltas:
312
+ basis = _subspace([a for a in against if a.tensor == delta.tensor], side)
313
+ if basis is None:
314
+ out.append(delta)
315
+ continue
316
+ if delta.kind == "lowrank" and delta.index is None and side == "out":
317
+ assert delta.u is not None
318
+ u = delta.u - basis @ (basis.T @ delta.u)
319
+ out.append(
320
+ patch.lowrank(
321
+ delta.tensor,
322
+ delta.shape,
323
+ delta.base,
324
+ u,
325
+ delta.v, # type: ignore[arg-type]
326
+ scale=delta.scale,
327
+ op="orthogonalize",
328
+ )
329
+ )
330
+ continue
331
+ full = delta.dense()
332
+ mat = full if side == "out" else full.T
333
+ proj = mat - basis @ (basis.T @ mat)
334
+ out.append(
335
+ patch.compact(
336
+ patch.dense(
337
+ delta.tensor,
338
+ delta.shape,
339
+ delta.base,
340
+ proj if side == "out" else proj.T,
341
+ op="orthogonalize",
342
+ )
343
+ )
344
+ )
345
+ del ctx
346
+ return out
347
+
348
+
349
+ def _subspace(deltas: list[TensorDelta], side: str, tol: float = 1e-6) -> torch.Tensor | None:
350
+ if not deltas:
351
+ return None
352
+ cols = []
353
+ for d in deltas:
354
+ if d.kind == "lowrank" and d.index is None:
355
+ assert d.u is not None
356
+ assert d.v is not None
357
+ cols.append(d.u if side == "out" else d.v)
358
+ else:
359
+ full = d.dense()
360
+ mat = full if side == "out" else full.T
361
+ u, s, _ = torch.linalg.svd(mat, full_matrices=False)
362
+ cols.append(u[:, s > tol * max(float(s.max()), 1e-30)])
363
+ stacked = torch.cat(cols, dim=1)
364
+ q, r = torch.linalg.qr(stacked)
365
+ keep = r.diagonal().abs() > tol * max(float(r.diagonal().abs().max()), 1e-30)
366
+ basis: torch.Tensor = q[:, keep]
367
+ return basis
368
+
369
+
370
+ BUILTIN: dict[str, Callable[..., list[TensorDelta]]] = {
371
+ "ablate": ablate,
372
+ "set": set_,
373
+ "scale": scale,
374
+ "project_out": project_out,
375
+ "bake_steer": bake_steer,
376
+ "rank_one_write": rank_one_write,
377
+ "transplant": transplant,
378
+ "orthogonalize": orthogonalize,
379
+ }
wspx/lab/patch.py ADDED
@@ -0,0 +1,320 @@
1
+ """The patch IR (§5.3): the only edit forms the store and the lab understand.
2
+
3
+ Every primitive is an fp32 delta from the base tensor, kept compact:
4
+
5
+ ======== =================================================================
6
+ lowrank ``Δ[index] = scale · Σ_k u[:, k] ⊗ v[:, k]`` on a 2-D block
7
+ sparse ``Δ.flat[indices] = values`` (indices unique)
8
+ dense ``Δ[index] = delta`` (a block, or the whole tensor)
9
+ scale a dense block delta, recorded as a scaling (``(f − 1) · W``)
10
+ set a dense block delta, recorded as an assignment (``values − W``)
11
+ ======== =================================================================
12
+
13
+ Materialisation sums every delta of a tensor in fp32, in canonical order
14
+ (sorted by blob id), onto the fp32 base and rounds once (§5.5.2). Low-rank
15
+ products are built as sequential outer products, never a GEMM, so every
16
+ element is a fixed sequence of IEEE operations: the same history gives the
17
+ same bits on any device and in any order.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import json
23
+ from dataclasses import dataclass, field
24
+ from typing import Any
25
+
26
+ import torch
27
+ from safetensors.torch import load as st_load
28
+ from safetensors.torch import save as st_save
29
+
30
+ from wspx.errors import PatchError
31
+ from wspx.lab.tensors import Index, Selector, assign, select
32
+ from wspx.store.objects import canonical_json, oid_of
33
+
34
+ KINDS = ("lowrank", "sparse", "dense", "scale", "set")
35
+ FORMAT = "wspx.delta/1"
36
+
37
+
38
+ def encode_index(index: Index | None) -> list[Any] | None:
39
+ if index is None:
40
+ return None
41
+ out: list[Any] = []
42
+ for sel in index:
43
+ if isinstance(sel, int):
44
+ out.append(sel)
45
+ elif isinstance(sel, slice):
46
+ out.append({"s": [sel.start, sel.stop, sel.step]})
47
+ else:
48
+ out.append({"l": list(sel)})
49
+ return out
50
+
51
+
52
+ def decode_index(data: list[Any] | None) -> Index | None:
53
+ if data is None:
54
+ return None
55
+ out: list[Selector] = []
56
+ for item in data:
57
+ if isinstance(item, int):
58
+ out.append(item)
59
+ elif "s" in item:
60
+ out.append(slice(*item["s"]))
61
+ else:
62
+ out.append(list(item["l"]))
63
+ return tuple(out)
64
+
65
+
66
+ @dataclass
67
+ class TensorDelta:
68
+ """One tensor's delta. Immutable once encoded; identified by the hash of its bytes."""
69
+
70
+ tensor: str
71
+ kind: str
72
+ shape: tuple[int, ...]
73
+ base: str
74
+ """Hash of the base tensor this delta applies to."""
75
+ index: Index | None = None
76
+ u: torch.Tensor | None = None
77
+ v: torch.Tensor | None = None
78
+ scale: float = 1.0
79
+ indices: torch.Tensor | None = None
80
+ values: torch.Tensor | None = None
81
+ delta: torch.Tensor | None = None
82
+ meta: dict[str, Any] = field(default_factory=dict)
83
+ _bytes: bytes | None = field(default=None, repr=False, compare=False)
84
+
85
+ def __post_init__(self) -> None:
86
+ if self.kind not in KINDS:
87
+ raise PatchError(f"unknown delta kind {self.kind!r}")
88
+ if self.index is not None and all(
89
+ isinstance(s, slice) and s == slice(None) for s in self.index
90
+ ):
91
+ self.index = None # a whole-tensor selection is stored as no selection
92
+ if self.kind == "lowrank":
93
+ if self.u is None or self.v is None:
94
+ raise PatchError("lowrank delta needs u and v")
95
+ if self.u.dim() == 1:
96
+ self.u = self.u[:, None]
97
+ if self.v.dim() == 1:
98
+ self.v = self.v[:, None]
99
+ if self.u.shape[1] != self.v.shape[1]:
100
+ raise PatchError(
101
+ f"lowrank rank mismatch: u {tuple(self.u.shape)} v {tuple(self.v.shape)}"
102
+ )
103
+ self.u = self.u.detach().to(torch.float32).contiguous()
104
+ self.v = self.v.detach().to(torch.float32).contiguous()
105
+ elif self.kind == "sparse":
106
+ if self.indices is None or self.values is None:
107
+ raise PatchError("sparse delta needs indices and values")
108
+ order = torch.argsort(self.indices.detach().to(torch.long).cpu())
109
+ self.indices = self.indices.detach().to(torch.long).cpu()[order].contiguous()
110
+ self.values = self.values.detach().to(torch.float32).cpu()[order].contiguous()
111
+ if self.indices.numel() and bool((self.indices[1:] == self.indices[:-1]).any()):
112
+ raise PatchError("sparse delta indices must be unique")
113
+ else:
114
+ if self.delta is None:
115
+ raise PatchError(f"{self.kind} delta needs a delta block")
116
+ self.delta = self.delta.detach().to(torch.float32).contiguous()
117
+
118
+ # ------------------------------------------------------------------ geometry
119
+
120
+ @property
121
+ def rank(self) -> int | None:
122
+ return int(self.u.shape[1]) if self.u is not None else None
123
+
124
+ @property
125
+ def nnz(self) -> int:
126
+ """Elements the delta can touch (an upper bound for low-rank blocks)."""
127
+ if self.kind == "sparse":
128
+ assert self.indices is not None
129
+ return int(self.indices.numel())
130
+ if self.kind == "lowrank":
131
+ assert self.u is not None
132
+ assert self.v is not None
133
+ return int(self.u.shape[0] * self.v.shape[0])
134
+ assert self.delta is not None
135
+ return int(self.delta.numel())
136
+
137
+ def nbytes(self) -> int:
138
+ return sum(
139
+ t.numel() * t.element_size()
140
+ for t in (self.u, self.v, self.indices, self.values, self.delta)
141
+ if t is not None
142
+ )
143
+
144
+ # ------------------------------------------------------------------ apply
145
+
146
+ def block(self, device: torch.device | str = "cpu") -> torch.Tensor:
147
+ """The delta over its block (lowrank/dense kinds), fp32."""
148
+ if self.kind == "lowrank":
149
+ assert self.u is not None
150
+ assert self.v is not None
151
+ u = self.u.to(device)
152
+ v = self.v.to(device)
153
+ out = torch.zeros(u.shape[0], v.shape[0], dtype=torch.float32, device=device)
154
+ for k in range(u.shape[1]):
155
+ out.add_(torch.outer(u[:, k], v[:, k]))
156
+ if self.scale != 1.0:
157
+ out.mul_(self.scale)
158
+ return out
159
+ if self.kind == "sparse":
160
+ raise PatchError("sparse deltas have no block form")
161
+ assert self.delta is not None
162
+ return self.delta.to(device)
163
+
164
+ def add_into(self, acc: torch.Tensor) -> None:
165
+ """``acc += Δ`` for an fp32 accumulator of the full tensor shape."""
166
+ if tuple(acc.shape) != tuple(self.shape):
167
+ raise PatchError(
168
+ f"{self.tensor}: delta for shape {self.shape} applied to {tuple(acc.shape)}"
169
+ )
170
+ if self.kind == "sparse":
171
+ assert self.indices is not None
172
+ assert self.values is not None
173
+ flat = acc.view(-1)
174
+ flat.index_add_(0, self.indices.to(acc.device), self.values.to(acc.device))
175
+ return
176
+ block = self.block(acc.device)
177
+ target = select(acc, self.index)
178
+ if tuple(target.shape) != tuple(block.shape):
179
+ raise PatchError(
180
+ f"{self.tensor}: block {tuple(block.shape)} does not match selection {tuple(target.shape)}"
181
+ )
182
+ if self.index is None or not any(isinstance(s, list) for s in self.index):
183
+ target.add_(block) # a view: writes through
184
+ else:
185
+ assign(acc, self.index, target + block)
186
+
187
+ def dense(self, device: torch.device | str = "cpu") -> torch.Tensor:
188
+ out = torch.zeros(self.shape, dtype=torch.float32, device=device)
189
+ self.add_into(out)
190
+ return out
191
+
192
+ # ------------------------------------------------------------------ codec
193
+
194
+ def to_bytes(self) -> bytes:
195
+ if self._bytes is None:
196
+ tensors: dict[str, torch.Tensor] = {}
197
+ for name in ("u", "v", "indices", "values", "delta"):
198
+ value = getattr(self, name)
199
+ if value is not None:
200
+ tensors[name] = value.detach().cpu().contiguous()
201
+ header = {
202
+ "format": FORMAT,
203
+ "tensor": self.tensor,
204
+ "kind": self.kind,
205
+ "shape": list(self.shape),
206
+ "base": self.base,
207
+ "index": encode_index(self.index),
208
+ "scale": self.scale,
209
+ "meta": self.meta,
210
+ }
211
+ self._bytes = st_save(tensors, metadata={"wspx": canonical_json(header).decode()})
212
+ return self._bytes
213
+
214
+ @property
215
+ def oid(self) -> str:
216
+ return oid_of(self.to_bytes())
217
+
218
+ @classmethod
219
+ def from_bytes(cls, data: bytes) -> TensorDelta:
220
+ try:
221
+ header_len = int.from_bytes(data[:8], "little")
222
+ meta = json.loads(data[8 : 8 + header_len])["__metadata__"]
223
+ header = json.loads(meta["wspx"])
224
+ except (ValueError, KeyError, json.JSONDecodeError) as exc:
225
+ raise PatchError(f"not a wspx delta blob: {exc}") from None
226
+ if header.get("format") != FORMAT:
227
+ raise PatchError(f"unsupported delta format {header.get('format')!r}")
228
+ tensors = st_load(data)
229
+ delta = cls(
230
+ tensor=header["tensor"],
231
+ kind=header["kind"],
232
+ shape=tuple(header["shape"]),
233
+ base=header["base"],
234
+ index=decode_index(header["index"]),
235
+ scale=float(header["scale"]),
236
+ meta=header.get("meta", {}),
237
+ u=tensors.get("u"),
238
+ v=tensors.get("v"),
239
+ indices=tensors.get("indices"),
240
+ values=tensors.get("values"),
241
+ delta=tensors.get("delta"),
242
+ )
243
+ delta._bytes = data
244
+ return delta
245
+
246
+
247
+ def materialize(
248
+ base: torch.Tensor, deltas: list[TensorDelta], *, dtype: torch.dtype | None = None
249
+ ) -> torch.Tensor:
250
+ """``cast(fp32(base) + Σ fp32(Δ))`` with deltas in canonical order, rounded once."""
251
+ target = dtype or base.dtype
252
+ if not deltas:
253
+ return base.to(target, copy=True)
254
+ acc = base.to(torch.float32, copy=True)
255
+ for delta in sorted(deltas, key=lambda d: d.oid):
256
+ delta.add_into(acc)
257
+ return acc.to(target)
258
+
259
+
260
+ # --------------------------------------------------------------------------- builders
261
+
262
+
263
+ def lowrank(
264
+ tensor: str,
265
+ shape: tuple[int, ...],
266
+ base: str,
267
+ u: torch.Tensor,
268
+ v: torch.Tensor,
269
+ *,
270
+ index: Index | None = None,
271
+ scale: float = 1.0,
272
+ **meta: Any,
273
+ ) -> TensorDelta:
274
+ return TensorDelta(
275
+ tensor, "lowrank", shape, base, index=index, u=u, v=v, scale=scale, meta=meta
276
+ )
277
+
278
+
279
+ def dense(
280
+ tensor: str,
281
+ shape: tuple[int, ...],
282
+ base: str,
283
+ delta: torch.Tensor,
284
+ *,
285
+ index: Index | None = None,
286
+ kind: str = "dense",
287
+ **meta: Any,
288
+ ) -> TensorDelta:
289
+ return TensorDelta(tensor, kind, shape, base, index=index, delta=delta, meta=meta)
290
+
291
+
292
+ def sparse(
293
+ tensor: str,
294
+ shape: tuple[int, ...],
295
+ base: str,
296
+ indices: torch.Tensor,
297
+ values: torch.Tensor,
298
+ **meta: Any,
299
+ ) -> TensorDelta:
300
+ return TensorDelta(tensor, "sparse", shape, base, indices=indices, values=values, meta=meta)
301
+
302
+
303
+ def compact(delta: TensorDelta) -> TensorDelta:
304
+ """Re-encode a dense delta as sparse when that is smaller. Values are unchanged."""
305
+ if delta.kind not in ("dense", "scale", "set") or delta.index is not None:
306
+ return delta
307
+ assert delta.delta is not None
308
+ nz = torch.nonzero(delta.delta.reshape(-1)).reshape(-1)
309
+ if nz.numel() * 3 >= delta.delta.numel():
310
+ return delta
311
+ values = delta.delta.reshape(-1)[nz]
312
+ return TensorDelta(
313
+ delta.tensor,
314
+ "sparse",
315
+ delta.shape,
316
+ delta.base,
317
+ indices=nz,
318
+ values=values,
319
+ meta={**delta.meta, "from": delta.kind},
320
+ )