wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/session.py
ADDED
|
@@ -0,0 +1,1574 @@
|
|
|
1
|
+
"""The lab session: one live model, its IR, working tree and every tool (§4.2).
|
|
2
|
+
|
|
3
|
+
Each tool returns a :class:`ToolResult`: a short, budgeted summary for the
|
|
4
|
+
driver plus structured data for the UI and SDK. All state that must survive a
|
|
5
|
+
cold start (handles, directions, result sets, staged patches) lives in the
|
|
6
|
+
store, so resuming costs one model load plus a replay of small deltas.
|
|
7
|
+
"""
|
|
8
|
+
|
|
9
|
+
from __future__ import annotations
|
|
10
|
+
|
|
11
|
+
import contextlib
|
|
12
|
+
import hashlib
|
|
13
|
+
import io
|
|
14
|
+
import json
|
|
15
|
+
import math
|
|
16
|
+
import re
|
|
17
|
+
import runpy
|
|
18
|
+
import time
|
|
19
|
+
import traceback
|
|
20
|
+
from collections.abc import Callable, Iterator, Sequence
|
|
21
|
+
from dataclasses import dataclass, field
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
import torch
|
|
26
|
+
from safetensors.torch import load as st_load
|
|
27
|
+
from safetensors.torch import save as st_save
|
|
28
|
+
|
|
29
|
+
from wspx.addressing import HandleKind, make_slug, parse_handle
|
|
30
|
+
from wspx.config import WspxConfig
|
|
31
|
+
from wspx.diagnostics import Code, Report, diag
|
|
32
|
+
from wspx.errors import (
|
|
33
|
+
AddressError,
|
|
34
|
+
CheckFailed,
|
|
35
|
+
GrepError,
|
|
36
|
+
HandleError,
|
|
37
|
+
IntegrityError,
|
|
38
|
+
PatchError,
|
|
39
|
+
VCSError,
|
|
40
|
+
WspxError,
|
|
41
|
+
)
|
|
42
|
+
from wspx.handles import HandleTable
|
|
43
|
+
from wspx.lab import analysis, export, merge, results
|
|
44
|
+
from wspx.lab import ops as op_registry
|
|
45
|
+
from wspx.lab.check import (
|
|
46
|
+
EditPlan,
|
|
47
|
+
SmokeMetrics,
|
|
48
|
+
degenerate_reasons,
|
|
49
|
+
numeric_check,
|
|
50
|
+
perplexity,
|
|
51
|
+
smoke_report,
|
|
52
|
+
static_check,
|
|
53
|
+
token_kl,
|
|
54
|
+
)
|
|
55
|
+
from wspx.lab.evaluate import SuiteModel
|
|
56
|
+
from wspx.lab.grep import Environment, GrepResult, Hit, grep
|
|
57
|
+
from wspx.lab.index import ModelIndex, build_index
|
|
58
|
+
from wspx.lab.ir.model_ir import ModelIR, Target
|
|
59
|
+
from wspx.lab.ops import OpContext
|
|
60
|
+
from wspx.lab.patch import TensorDelta
|
|
61
|
+
from wspx.lab.probe import Probe
|
|
62
|
+
from wspx.lab.tensors import normalize_index, select, tensor_hash
|
|
63
|
+
from wspx.lab.text import Prompt, Text
|
|
64
|
+
from wspx.lab.tree import TreeState, WorkingTree
|
|
65
|
+
from wspx.store import Store, canonical_json, short
|
|
66
|
+
from wspx.testing import runner
|
|
67
|
+
from wspx.testing.api import Suite
|
|
68
|
+
from wspx.testing.builtin import BUILTIN, CANARY_PROMPTS
|
|
69
|
+
from wspx.vcs import Commit, PatchRecord, Repo, tree_key
|
|
70
|
+
|
|
71
|
+
DEFAULT_TOP = 20
|
|
72
|
+
MAX_TOP = 200
|
|
73
|
+
BUDGET_CHARS = 6000
|
|
74
|
+
RUN_TAIL_LINES = 200
|
|
75
|
+
RUN_TAIL_BYTES = 16 * 1024
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass
|
|
79
|
+
class ToolResult:
|
|
80
|
+
summary: str
|
|
81
|
+
data: dict[str, Any] = field(default_factory=dict)
|
|
82
|
+
handles: list[str] = field(default_factory=list)
|
|
83
|
+
ok: bool = True
|
|
84
|
+
|
|
85
|
+
def budgeted(self, limit: int = BUDGET_CHARS) -> str:
|
|
86
|
+
if len(self.summary) <= limit:
|
|
87
|
+
return self.summary
|
|
88
|
+
cut = self.summary[:limit].rsplit("\n", 1)[0]
|
|
89
|
+
hidden = self.summary.count("\n") - cut.count("\n")
|
|
90
|
+
return f"{cut}\n… {hidden} more lines (narrow the query or use a handle)"
|
|
91
|
+
|
|
92
|
+
def __str__(self) -> str:
|
|
93
|
+
return self.budgeted()
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
class Lab:
|
|
97
|
+
def __init__(
|
|
98
|
+
self,
|
|
99
|
+
model: torch.nn.Module,
|
|
100
|
+
ir: ModelIR,
|
|
101
|
+
store: Store,
|
|
102
|
+
*,
|
|
103
|
+
session: str = "local",
|
|
104
|
+
branch: str | None = None,
|
|
105
|
+
tokenizer: Any = None,
|
|
106
|
+
config: WspxConfig | None = None,
|
|
107
|
+
base_reader: Callable[[str], torch.Tensor] | None = None,
|
|
108
|
+
workdir: Path | None = None,
|
|
109
|
+
cache_bytes: int | None = None,
|
|
110
|
+
) -> None:
|
|
111
|
+
self.model = model
|
|
112
|
+
self.ir = ir
|
|
113
|
+
self.store = store
|
|
114
|
+
self.session = session
|
|
115
|
+
self.config = config or WspxConfig()
|
|
116
|
+
self.workdir = Path(workdir or Path.cwd())
|
|
117
|
+
self.repo = Repo(store, ir.fingerprint)
|
|
118
|
+
self.text = Text(tokenizer, vocab_size=ir.vocab_size)
|
|
119
|
+
self.tree = WorkingTree(
|
|
120
|
+
model, ir, load_delta=self._load_delta, base_reader=base_reader, cache_bytes=cache_bytes
|
|
121
|
+
)
|
|
122
|
+
self.probe = Probe(model, ir)
|
|
123
|
+
self.handles = HandleTable()
|
|
124
|
+
self.directions: dict[str, torch.Tensor] = {}
|
|
125
|
+
self.results: dict[str, list[Hit]] = {}
|
|
126
|
+
self.staged: list[PatchRecord] = []
|
|
127
|
+
self.sources: dict[str, tuple[ModelIR, torch.nn.Module]] = {}
|
|
128
|
+
self.context: dict[str, Any] = {}
|
|
129
|
+
"""Per-call provenance set by the caller: driver, call id."""
|
|
130
|
+
self.index: ModelIndex | None = None
|
|
131
|
+
self.heldout_queries = 0
|
|
132
|
+
self._script: str | None = None
|
|
133
|
+
self._base_canary: dict[str, Any] | None = None
|
|
134
|
+
self._resume(branch)
|
|
135
|
+
|
|
136
|
+
# ------------------------------------------------------------------ persistence
|
|
137
|
+
|
|
138
|
+
@property
|
|
139
|
+
def device(self) -> torch.device:
|
|
140
|
+
return next(self.model.parameters()).device
|
|
141
|
+
|
|
142
|
+
@property
|
|
143
|
+
def params(self) -> dict[str, torch.nn.Parameter]:
|
|
144
|
+
return self.tree.params
|
|
145
|
+
|
|
146
|
+
def _state_path(self) -> str:
|
|
147
|
+
return f"sessions/{self.session}/state.json"
|
|
148
|
+
|
|
149
|
+
def _resume(self, branch: str | None) -> None:
|
|
150
|
+
raw = self.store.read_file(self._state_path())
|
|
151
|
+
if not self.store.refs.exists(self.repo.head_ref(self.session)):
|
|
152
|
+
self.repo.init_session(self.session, branch or "main")
|
|
153
|
+
if raw is not None:
|
|
154
|
+
state = json.loads(raw)
|
|
155
|
+
self.handles = HandleTable.load(state.get("handles", []))
|
|
156
|
+
for name, oid in state.get("directions", {}).items():
|
|
157
|
+
self.directions[name] = st_load(self.store.objects.get(oid))["v"]
|
|
158
|
+
for key, rel in state.get("results", {}).items():
|
|
159
|
+
data = self.store.read_file(rel)
|
|
160
|
+
if data is not None:
|
|
161
|
+
self.results[key] = results.load_hits(data)
|
|
162
|
+
self.staged = [self.repo.get_patch(p) for p in state.get("staged", [])]
|
|
163
|
+
self.heldout_queries = int(state.get("heldout_queries", 0))
|
|
164
|
+
head = self.repo.head(self.session)
|
|
165
|
+
tree = TreeState.from_dict(self.repo.tree_of_oid(head))
|
|
166
|
+
for record in self.staged:
|
|
167
|
+
deltas = [self._load_delta(d["blob"]) for d in record.deltas]
|
|
168
|
+
tree = tree.with_deltas(deltas, record.precision)
|
|
169
|
+
if tree.tensors():
|
|
170
|
+
self.tree.checkout(tree)
|
|
171
|
+
if head:
|
|
172
|
+
self._verify(self.repo.get_commit(head), staged=bool(self.staged))
|
|
173
|
+
|
|
174
|
+
def _save(self) -> None:
|
|
175
|
+
directions = {}
|
|
176
|
+
for name, vec in self.directions.items():
|
|
177
|
+
directions[name] = self.store.objects.put(
|
|
178
|
+
st_save({"v": vec.detach().float().cpu().contiguous()})
|
|
179
|
+
)
|
|
180
|
+
state = {
|
|
181
|
+
"handles": self.handles.dump(),
|
|
182
|
+
"directions": directions,
|
|
183
|
+
"results": {k: f"results/{self.session}/{k[1:]}.parquet" for k in self.results},
|
|
184
|
+
"staged": [p.oid for p in self.staged],
|
|
185
|
+
"heldout_queries": self.heldout_queries,
|
|
186
|
+
}
|
|
187
|
+
self.store.write_file(self._state_path(), canonical_json(state))
|
|
188
|
+
|
|
189
|
+
def _load_delta(self, oid: str) -> TensorDelta:
|
|
190
|
+
return TensorDelta.from_bytes(self.store.objects.get(oid))
|
|
191
|
+
|
|
192
|
+
def _verify(self, commit: Commit, *, staged: bool = False) -> None:
|
|
193
|
+
if staged:
|
|
194
|
+
return
|
|
195
|
+
bad = self.tree.verify(commit.result_hashes)
|
|
196
|
+
if bad:
|
|
197
|
+
raise IntegrityError(
|
|
198
|
+
f"checkout of {commit.short}: hashes differ for {', '.join(bad[:3])}"
|
|
199
|
+
)
|
|
200
|
+
|
|
201
|
+
# ------------------------------------------------------------------ resolution
|
|
202
|
+
|
|
203
|
+
def resolve(self, address: str) -> list[Target]:
|
|
204
|
+
"""Resolve an address, or a comma-separated list of them (matching nothing is an error)."""
|
|
205
|
+
parts = _split_scopes(address)
|
|
206
|
+
if len(parts) == 1:
|
|
207
|
+
return self.ir.resolve(
|
|
208
|
+
parts[0], token_id=self._token_id, handle_indices=self._handle_indices
|
|
209
|
+
)
|
|
210
|
+
out: list[Target] = []
|
|
211
|
+
for part in parts:
|
|
212
|
+
try:
|
|
213
|
+
out.extend(
|
|
214
|
+
self.ir.resolve(
|
|
215
|
+
part, token_id=self._token_id, handle_indices=self._handle_indices
|
|
216
|
+
)
|
|
217
|
+
)
|
|
218
|
+
except AddressError:
|
|
219
|
+
continue
|
|
220
|
+
if not out:
|
|
221
|
+
raise AddressError(f"E101 {address}: no weights match")
|
|
222
|
+
return out
|
|
223
|
+
|
|
224
|
+
def _token_id(self, token: str) -> int:
|
|
225
|
+
if token.startswith("#") and token[1:].isdigit():
|
|
226
|
+
return int(token[1:])
|
|
227
|
+
return self.text.token_id(token)
|
|
228
|
+
|
|
229
|
+
def _handle_indices(self, handle: Any, param: str) -> list[int]:
|
|
230
|
+
hits = self._hits(str(handle))
|
|
231
|
+
return results.indices(hits, param)
|
|
232
|
+
|
|
233
|
+
def _hits(self, ref: str | Sequence[Hit]) -> list[Hit]:
|
|
234
|
+
if not isinstance(ref, str):
|
|
235
|
+
return list(ref)
|
|
236
|
+
entry, _ = self.handles.resolve(ref)
|
|
237
|
+
if entry.key not in self.results:
|
|
238
|
+
raise HandleError(f"{entry.key} holds no units")
|
|
239
|
+
return self.results[entry.key]
|
|
240
|
+
|
|
241
|
+
def direction(self, ref: Any) -> torch.Tensor:
|
|
242
|
+
"""``dir:name``, a list of numbers or a tensor, as an fp32 vector on the model's device."""
|
|
243
|
+
if isinstance(ref, DirectionRef):
|
|
244
|
+
ref = ref.handle
|
|
245
|
+
if isinstance(ref, str):
|
|
246
|
+
handle = parse_handle(ref, allow_revision=False)
|
|
247
|
+
if handle.kind is not HandleKind.DIRECTION or handle.id not in self.directions:
|
|
248
|
+
raise HandleError(
|
|
249
|
+
f"{ref} is not a known direction ({', '.join(sorted(self.directions)) or 'none'})"
|
|
250
|
+
)
|
|
251
|
+
return self.directions[handle.id].to(self.device).float()
|
|
252
|
+
return torch.as_tensor(ref, dtype=torch.float32, device=self.device)
|
|
253
|
+
|
|
254
|
+
def base_hash(self, name: str) -> str:
|
|
255
|
+
info = self.ir.params[name]
|
|
256
|
+
if info.hash is None:
|
|
257
|
+
info.hash = tensor_hash(self.tree.base(name))
|
|
258
|
+
return info.hash
|
|
259
|
+
|
|
260
|
+
def _ctx(self) -> OpContext:
|
|
261
|
+
ctx = OpContext(
|
|
262
|
+
ir=self.ir,
|
|
263
|
+
tree=self.tree,
|
|
264
|
+
resolve=self.resolve,
|
|
265
|
+
direction=self.direction,
|
|
266
|
+
sources=self.sources,
|
|
267
|
+
)
|
|
268
|
+
ctx.base_hash = self.base_hash # type: ignore[method-assign]
|
|
269
|
+
return ctx
|
|
270
|
+
|
|
271
|
+
@contextlib.contextmanager
|
|
272
|
+
def at_state(self, state: TreeState) -> Iterator[None]:
|
|
273
|
+
"""Temporarily switch the live weights to ``state`` (exact both ways)."""
|
|
274
|
+
current = self.tree.state
|
|
275
|
+
self.tree.checkout(state)
|
|
276
|
+
try:
|
|
277
|
+
yield
|
|
278
|
+
finally:
|
|
279
|
+
self.tree.checkout(current)
|
|
280
|
+
|
|
281
|
+
def at_base(self) -> contextlib.AbstractContextManager[None]:
|
|
282
|
+
return self.at_state(TreeState())
|
|
283
|
+
|
|
284
|
+
def tree_key(self, state: TreeState | None = None) -> str:
|
|
285
|
+
return tree_key((state or self.tree.state).to_dict())
|
|
286
|
+
|
|
287
|
+
# ------------------------------------------------------------------ info / ls
|
|
288
|
+
|
|
289
|
+
def info(self, *, handles: bool = False) -> ToolResult:
|
|
290
|
+
head = self.repo.head(self.session)
|
|
291
|
+
branch = self.repo.current_branch(self.session) or "(detached)"
|
|
292
|
+
lines = [
|
|
293
|
+
self.ir.render_info(),
|
|
294
|
+
f"branch {branch} · HEAD {short(head) if head else '(base)'} · {len(self.staged)} staged patch(es)",
|
|
295
|
+
]
|
|
296
|
+
if self.heldout_queries:
|
|
297
|
+
lines.append(f"held-out queries so far: {self.heldout_queries}")
|
|
298
|
+
if handles:
|
|
299
|
+
lines.append("handles:\n" + self.handles.table(limit=200))
|
|
300
|
+
return ToolResult(
|
|
301
|
+
"\n".join(lines), {"summary": self.ir.summary(), "branch": branch, "head": head}
|
|
302
|
+
)
|
|
303
|
+
|
|
304
|
+
def ls(
|
|
305
|
+
self,
|
|
306
|
+
scope: str = "L*.*",
|
|
307
|
+
*,
|
|
308
|
+
role: str | None = None,
|
|
309
|
+
layers: str | None = None,
|
|
310
|
+
limit: int = 100,
|
|
311
|
+
) -> ToolResult:
|
|
312
|
+
if role is not None:
|
|
313
|
+
scope = f"L{layers or '*'}.{role}"
|
|
314
|
+
targets = self.resolve(scope)
|
|
315
|
+
rows = []
|
|
316
|
+
for t in targets:
|
|
317
|
+
info = self.ir.params[t.param]
|
|
318
|
+
shape = tuple(select(torch.empty(info.shape, device="meta"), t.index).shape)
|
|
319
|
+
kind = t.role.kind if t.role else "param"
|
|
320
|
+
rows.append((t.address, kind, t.param, shape, info.dtype))
|
|
321
|
+
lines = [f"{a:<28} {k:<9} {tuple(s)!s:<16} {d:<9} {p}" for a, k, p, s, d in rows[:limit]]
|
|
322
|
+
if len(rows) > limit:
|
|
323
|
+
lines.append(f"… {len(rows) - limit} more; narrow the scope")
|
|
324
|
+
return ToolResult(
|
|
325
|
+
"\n".join(lines) or "(nothing matches)", {"rows": [list(r[:3]) for r in rows]}
|
|
326
|
+
)
|
|
327
|
+
|
|
328
|
+
# ------------------------------------------------------------------ grep
|
|
329
|
+
|
|
330
|
+
def grep(
|
|
331
|
+
self,
|
|
332
|
+
expression: str | None = None,
|
|
333
|
+
*,
|
|
334
|
+
scope: str | None = None,
|
|
335
|
+
axis: str | None = None,
|
|
336
|
+
top: int = DEFAULT_TOP,
|
|
337
|
+
name: str | None = None,
|
|
338
|
+
fires_on: Sequence[Prompt] | str | None = None,
|
|
339
|
+
) -> GrepHits:
|
|
340
|
+
top = max(1, min(int(top), MAX_TOP))
|
|
341
|
+
if fires_on is not None:
|
|
342
|
+
result = self._activation_grep(self._prompts(fires_on), scope)
|
|
343
|
+
else:
|
|
344
|
+
if not expression:
|
|
345
|
+
raise GrepError("grep needs an expression or --fires-on prompts")
|
|
346
|
+
if scope is None:
|
|
347
|
+
scope = _default_scope(expression)
|
|
348
|
+
env = Environment(
|
|
349
|
+
self.ir, dict(self.params), token_id=self._token_or_id, direction=self.direction
|
|
350
|
+
)
|
|
351
|
+
result = grep(env, expression, self.resolve(scope), axis=axis)
|
|
352
|
+
entry = self.handles.register_result(
|
|
353
|
+
(
|
|
354
|
+
result.expression.split(" ")[0] if fires_on is None else "fires-on",
|
|
355
|
+
result.scope.split(",")[0],
|
|
356
|
+
f"top{top}",
|
|
357
|
+
),
|
|
358
|
+
name=name,
|
|
359
|
+
description=f"{len(result.hits)} of {result.candidates} {result.axis}s matching {result.expression}",
|
|
360
|
+
size=f"{min(len(result.hits), top)} hits",
|
|
361
|
+
call_id=self.context.get("call_id"),
|
|
362
|
+
)
|
|
363
|
+
rel = f"results/{self.session}/{entry.key[1:]}.parquet"
|
|
364
|
+
self.store.write_file(rel, results.to_bytes(result, top=top))
|
|
365
|
+
self.results[entry.key] = result.hits[:top]
|
|
366
|
+
self._save()
|
|
367
|
+
return GrepHits(self, entry.display, result, top)
|
|
368
|
+
|
|
369
|
+
def _token_or_id(self, token: str | int) -> int:
|
|
370
|
+
return token if isinstance(token, int) else self._token_id(token)
|
|
371
|
+
|
|
372
|
+
def _activation_grep(self, prompts: list[Prompt], scope: str | None) -> GrepResult:
|
|
373
|
+
"""Writer units ranked by mean |activation| over the prompts' tokens."""
|
|
374
|
+
from wspx.lab.probe import writer_units
|
|
375
|
+
|
|
376
|
+
params = [t.param for t in self.resolve(scope)] if scope else writer_units(self.ir)
|
|
377
|
+
units = [p for p in params if p in writer_units(self.ir)]
|
|
378
|
+
if not units:
|
|
379
|
+
raise GrepError("--fires-on ranks writer units; the scope has none")
|
|
380
|
+
ids, mask = self.text.batch(prompts, self.device)
|
|
381
|
+
with torch.no_grad(), self.probe.capture([], units=units) as cap:
|
|
382
|
+
analysis.forward(self.model, self.ir, ids, mask)
|
|
383
|
+
scores: list[Hit] = []
|
|
384
|
+
m = mask.unsqueeze(-1).float()
|
|
385
|
+
for param in units:
|
|
386
|
+
role = self.ir.primary_role(param)
|
|
387
|
+
acts = cap.units[param].float()
|
|
388
|
+
mean = ((acts.abs() * m).sum((0, 1)) / m.sum()).tolist()
|
|
389
|
+
addr = role.address if role else param
|
|
390
|
+
scores.extend(Hit(f"{addr}[:, {j}]", param, None, j, s) for j, s in enumerate(mean))
|
|
391
|
+
scores.sort(key=lambda h: -h.score)
|
|
392
|
+
return GrepResult(
|
|
393
|
+
f"fires-on n={len(prompts)}", scope or "writers", "col", len(scores), scores, scores
|
|
394
|
+
)
|
|
395
|
+
|
|
396
|
+
def _prompts(self, value: Sequence[Prompt] | str) -> list[Prompt]:
|
|
397
|
+
if isinstance(value, str):
|
|
398
|
+
path = self.workdir / value
|
|
399
|
+
if not path.exists():
|
|
400
|
+
raise WspxError(f"{value}: no such prompt file")
|
|
401
|
+
rows: list[Any] = [
|
|
402
|
+
json.loads(line) for line in path.read_text().splitlines() if line.strip()
|
|
403
|
+
]
|
|
404
|
+
prompts: list[Any] = [
|
|
405
|
+
r.get("prompt", r.get("text")) if isinstance(r, dict) else r for r in rows
|
|
406
|
+
]
|
|
407
|
+
if any(p is None for p in prompts):
|
|
408
|
+
raise WspxError(f"{value}: each line needs a 'prompt' or 'text' field")
|
|
409
|
+
return prompts
|
|
410
|
+
return list(value)
|
|
411
|
+
|
|
412
|
+
# ------------------------------------------------------------------ show / generate
|
|
413
|
+
|
|
414
|
+
def show(self, address: str, *, values: int | None = None) -> ToolResult:
|
|
415
|
+
targets = self.resolve(address)
|
|
416
|
+
scale = None
|
|
417
|
+
if self.ir.semantics.final_norm:
|
|
418
|
+
w = self.params[self.ir.semantics.final_norm[0]]
|
|
419
|
+
scale = w.detach().float() if w.dim() == 1 else None
|
|
420
|
+
infos = [
|
|
421
|
+
analysis.show(
|
|
422
|
+
self.ir, dict(self.params), t, text=self.text, values=values, final_scale=scale
|
|
423
|
+
)
|
|
424
|
+
for t in targets[:4]
|
|
425
|
+
]
|
|
426
|
+
text = "\n".join(analysis.render_show(i) for i in infos)
|
|
427
|
+
if len(targets) > 4:
|
|
428
|
+
text += f"\n… {len(targets) - 4} more tensors match; narrow the address"
|
|
429
|
+
return ToolResult(text, {"show": [{**i, "stats": i["stats"].__dict__} for i in infos]})
|
|
430
|
+
|
|
431
|
+
def generate(
|
|
432
|
+
self,
|
|
433
|
+
prompts: Prompt | Sequence[Prompt],
|
|
434
|
+
*,
|
|
435
|
+
max_tokens: int = 64,
|
|
436
|
+
temperature: float = 0.0,
|
|
437
|
+
seed: int = 0,
|
|
438
|
+
) -> ToolResult:
|
|
439
|
+
single = isinstance(prompts, str) or (bool(prompts) and isinstance(prompts[0], int))
|
|
440
|
+
batch: list[Any] = [prompts] if single else list(prompts)
|
|
441
|
+
outs = analysis.generate(
|
|
442
|
+
self.model, self.text, batch, max_tokens=max_tokens, temperature=temperature, seed=seed
|
|
443
|
+
)
|
|
444
|
+
texts = [self.text.decode(o) for o in outs]
|
|
445
|
+
body = "\n".join(f"[{i}] {t!r}" for i, t in enumerate(texts))
|
|
446
|
+
return ToolResult(body, {"outputs": texts, "ids": outs})
|
|
447
|
+
|
|
448
|
+
# ------------------------------------------------------------------ contrast
|
|
449
|
+
|
|
450
|
+
def contrast(
|
|
451
|
+
self,
|
|
452
|
+
positive: Sequence[Prompt] | str,
|
|
453
|
+
negative: Sequence[Prompt] | str,
|
|
454
|
+
*,
|
|
455
|
+
name: str,
|
|
456
|
+
layer: int | None = None,
|
|
457
|
+
position: str = "last",
|
|
458
|
+
) -> DirectionRef:
|
|
459
|
+
if not re.fullmatch(r"[A-Za-z_][\w\-]*", name):
|
|
460
|
+
raise WspxError(f"direction name {name!r}: letters, digits, _ and -")
|
|
461
|
+
d = analysis.contrast(
|
|
462
|
+
self.model,
|
|
463
|
+
self.ir,
|
|
464
|
+
self.probe,
|
|
465
|
+
self.text,
|
|
466
|
+
self._prompts(positive),
|
|
467
|
+
self._prompts(negative),
|
|
468
|
+
name=name,
|
|
469
|
+
layer=layer,
|
|
470
|
+
position=position,
|
|
471
|
+
)
|
|
472
|
+
self.directions[name] = d.vector.detach().float().cpu()
|
|
473
|
+
entry = self.handles.register(
|
|
474
|
+
HandleKind.DIRECTION,
|
|
475
|
+
f"dir:{name}",
|
|
476
|
+
d.provenance,
|
|
477
|
+
description=f"difference of means at L{d.layer}, separation {d.separation:.2f}",
|
|
478
|
+
size=f"{d.vector.numel()}-d",
|
|
479
|
+
call_id=self.context.get("call_id"),
|
|
480
|
+
)
|
|
481
|
+
self._save()
|
|
482
|
+
writers = self.grep(f"abs(cos(dir:{name}))", scope=_writer_scope(self.ir), top=5)
|
|
483
|
+
return DirectionRef(self, entry.display, d, writers)
|
|
484
|
+
|
|
485
|
+
# ------------------------------------------------------------------ trace
|
|
486
|
+
|
|
487
|
+
def trace(
|
|
488
|
+
self,
|
|
489
|
+
prompt: Prompt,
|
|
490
|
+
*,
|
|
491
|
+
vs: str | None = None,
|
|
492
|
+
reference: Prompt | None = None,
|
|
493
|
+
target: str | int | None = None,
|
|
494
|
+
top: int = 5,
|
|
495
|
+
) -> ToolResult:
|
|
496
|
+
if reference is not None:
|
|
497
|
+
t = analysis.attribution_trace(
|
|
498
|
+
self.model,
|
|
499
|
+
self.ir,
|
|
500
|
+
self.probe,
|
|
501
|
+
self.text,
|
|
502
|
+
prompt,
|
|
503
|
+
reference,
|
|
504
|
+
target=target,
|
|
505
|
+
top=max(top, 10),
|
|
506
|
+
)
|
|
507
|
+
elif vs is not None:
|
|
508
|
+
other = (
|
|
509
|
+
TreeState()
|
|
510
|
+
if vs == "base"
|
|
511
|
+
else TreeState.from_dict(self.repo.tree_of_oid(self.repo.resolve(vs, self.session)))
|
|
512
|
+
)
|
|
513
|
+
t = analysis.diff_trace(
|
|
514
|
+
self.model,
|
|
515
|
+
self.ir,
|
|
516
|
+
self.probe,
|
|
517
|
+
self.text,
|
|
518
|
+
prompt,
|
|
519
|
+
switch=self.at_state(other),
|
|
520
|
+
top=top,
|
|
521
|
+
)
|
|
522
|
+
else:
|
|
523
|
+
t = analysis.forward_trace(self.model, self.ir, self.probe, self.text, prompt, top=top)
|
|
524
|
+
blob = canonical_json(
|
|
525
|
+
{"kind": t.kind, "arrays": t.arrays, "data": _jsonable(t.data), "summary": t.summary}
|
|
526
|
+
)
|
|
527
|
+
oid = self.store.objects.put(blob)
|
|
528
|
+
key = "t_" + short(oid, 8)
|
|
529
|
+
self.handles.register(
|
|
530
|
+
HandleKind.TRACE,
|
|
531
|
+
key,
|
|
532
|
+
make_slug(t.kind, "trace"),
|
|
533
|
+
description=t.summary.split("\n")[0],
|
|
534
|
+
ref=oid,
|
|
535
|
+
call_id=self.context.get("call_id"),
|
|
536
|
+
)
|
|
537
|
+
self._save()
|
|
538
|
+
return ToolResult(
|
|
539
|
+
f"{t.summary}\nsaved as {key}:{make_slug(t.kind, 'trace')}",
|
|
540
|
+
{"trace": key, "kind": t.kind, **_jsonable(t.data)},
|
|
541
|
+
[key],
|
|
542
|
+
)
|
|
543
|
+
|
|
544
|
+
# ------------------------------------------------------------------ edit
|
|
545
|
+
|
|
546
|
+
@property
|
|
547
|
+
def edit(self) -> EditNamespace:
|
|
548
|
+
return EditNamespace(self)
|
|
549
|
+
|
|
550
|
+
def edit_op(
|
|
551
|
+
self, op: str, *, precision: str = "merged", scope: str | None = None, **args: Any
|
|
552
|
+
) -> PatchRef:
|
|
553
|
+
if precision not in ("merged", "adapter", "promote"):
|
|
554
|
+
raise PatchError(f"precision must be merged, adapter or promote, not {precision!r}")
|
|
555
|
+
fn = op_registry.get(op)
|
|
556
|
+
args = self._expand_unit_handles(args)
|
|
557
|
+
deltas = fn(self._ctx(), **args)
|
|
558
|
+
if not deltas:
|
|
559
|
+
raise PatchError(f"{op} produced no change")
|
|
560
|
+
plan = EditPlan(
|
|
561
|
+
deltas,
|
|
562
|
+
op=op,
|
|
563
|
+
precision=precision,
|
|
564
|
+
scope=[t.param for t in self.resolve(scope)] if scope else None,
|
|
565
|
+
args=args,
|
|
566
|
+
)
|
|
567
|
+
report = static_check(self.ir, plan)
|
|
568
|
+
report.extend(numeric_check(self.ir, self.tree, plan, self.config.check))
|
|
569
|
+
if not report.ok:
|
|
570
|
+
raise CheckFailed(report.errors)
|
|
571
|
+
blobs = []
|
|
572
|
+
for d in deltas:
|
|
573
|
+
oid = self.store.objects.put(d.to_bytes())
|
|
574
|
+
blobs.append(
|
|
575
|
+
{
|
|
576
|
+
"tensor": d.tensor,
|
|
577
|
+
"kind": d.kind,
|
|
578
|
+
"blob": oid,
|
|
579
|
+
**({"rank": d.rank} if d.rank else {}),
|
|
580
|
+
}
|
|
581
|
+
)
|
|
582
|
+
record = self.repo.put_patch(
|
|
583
|
+
PatchRecord(
|
|
584
|
+
op,
|
|
585
|
+
self._recordable(args),
|
|
586
|
+
precision,
|
|
587
|
+
blobs,
|
|
588
|
+
call=self.context.get("call_id"),
|
|
589
|
+
script=self._script,
|
|
590
|
+
)
|
|
591
|
+
)
|
|
592
|
+
self.tree.stage(deltas, precision)
|
|
593
|
+
self.staged.append(record)
|
|
594
|
+
targets = sorted({_display(self.ir, d.tensor) for d in deltas})
|
|
595
|
+
slug = make_slug(op, targets[0] if len(targets) == 1 else f"{len(targets)}-tensors")
|
|
596
|
+
self.handles.register(
|
|
597
|
+
HandleKind.PATCH,
|
|
598
|
+
record.handle,
|
|
599
|
+
slug,
|
|
600
|
+
description=f"{op} on {', '.join(targets[:4])}",
|
|
601
|
+
size=f"{sum(d.nbytes() for d in deltas) / 1024:.0f} KB",
|
|
602
|
+
ref=record.oid,
|
|
603
|
+
call_id=self.context.get("call_id"),
|
|
604
|
+
)
|
|
605
|
+
self._save()
|
|
606
|
+
return PatchRef(self, f"{record.handle}:{slug}", record, report)
|
|
607
|
+
|
|
608
|
+
def _expand_unit_handles(self, args: dict[str, Any]) -> dict[str, Any]:
|
|
609
|
+
"""``cols=@r1`` (or a GrepHits) narrows each target to the result set's units for that weight."""
|
|
610
|
+
hits_ref = args.pop("cols", None) if not isinstance(args.get("cols"), list) else None
|
|
611
|
+
if hits_ref is None:
|
|
612
|
+
return args
|
|
613
|
+
hits = self._hits(hits_ref.handle if isinstance(hits_ref, GrepHits) else str(hits_ref))
|
|
614
|
+
key = "targets" if "targets" in args else ("target" if "target" in args else None)
|
|
615
|
+
if key is None:
|
|
616
|
+
raise PatchError("cols= needs targets")
|
|
617
|
+
specs = args[key] if isinstance(args[key], list) else [args[key]]
|
|
618
|
+
narrowed: list[Target] = []
|
|
619
|
+
for spec in specs:
|
|
620
|
+
for t in self.resolve(spec) if isinstance(spec, str) else [spec]:
|
|
621
|
+
units = results.indices(hits, t.param)
|
|
622
|
+
if not units:
|
|
623
|
+
continue
|
|
624
|
+
index = list(normalize_index(t.index, len(self.ir.params[t.param].shape)))
|
|
625
|
+
axis_dim = _unit_dim(self.ir, t)
|
|
626
|
+
index[axis_dim] = units
|
|
627
|
+
narrowed.append(Target(t.param, tuple(index), t.address, t.role))
|
|
628
|
+
if not narrowed:
|
|
629
|
+
raise PatchError("the result set has no units in these targets")
|
|
630
|
+
args[key] = narrowed
|
|
631
|
+
return args
|
|
632
|
+
|
|
633
|
+
def _recordable(self, args: dict[str, Any]) -> dict[str, Any]:
|
|
634
|
+
out: dict[str, Any] = {}
|
|
635
|
+
for k, v in args.items():
|
|
636
|
+
if isinstance(v, torch.Tensor):
|
|
637
|
+
out[k] = {
|
|
638
|
+
"tensor": self.store.objects.put(
|
|
639
|
+
st_save({"v": v.detach().float().cpu().contiguous()})
|
|
640
|
+
)
|
|
641
|
+
}
|
|
642
|
+
elif isinstance(v, Target):
|
|
643
|
+
out[k] = {"address": v.address, "param": v.param}
|
|
644
|
+
elif isinstance(v, list) and v and isinstance(v[0], Target):
|
|
645
|
+
out[k] = [{"address": t.address, "param": t.param} for t in v]
|
|
646
|
+
elif isinstance(v, DirectionRef):
|
|
647
|
+
out[k] = v.handle
|
|
648
|
+
else:
|
|
649
|
+
try:
|
|
650
|
+
canonical_json(v)
|
|
651
|
+
out[k] = v
|
|
652
|
+
except (TypeError, ValueError):
|
|
653
|
+
out[k] = repr(v)
|
|
654
|
+
return out
|
|
655
|
+
|
|
656
|
+
# ------------------------------------------------------------------ check
|
|
657
|
+
|
|
658
|
+
def check(self, patch: Any = None, *, level: str = "smoke") -> Report:
|
|
659
|
+
records = self.staged if patch is None else [self._record(patch)]
|
|
660
|
+
if not records:
|
|
661
|
+
return Report(levels=[], diagnostics=[diag(Code.NO_OP, "nothing staged to check")])
|
|
662
|
+
deltas = [self._load_delta(d["blob"]) for r in records for d in r.deltas]
|
|
663
|
+
precision = records[0].precision if len({r.precision for r in records}) == 1 else "merged"
|
|
664
|
+
plan = EditPlan(
|
|
665
|
+
deltas, op=records[0].op if len(records) == 1 else "edit", precision=precision
|
|
666
|
+
)
|
|
667
|
+
report = static_check(self.ir, plan)
|
|
668
|
+
head_state = self._committed_state()
|
|
669
|
+
without = [r for r in self.staged if r not in records]
|
|
670
|
+
reference = head_state
|
|
671
|
+
for r in without:
|
|
672
|
+
reference = reference.with_deltas(
|
|
673
|
+
[self._load_delta(d["blob"]) for d in r.deltas], r.precision
|
|
674
|
+
)
|
|
675
|
+
with self.at_state(reference):
|
|
676
|
+
report.extend(numeric_check(self.ir, self.tree, plan, self.config.check))
|
|
677
|
+
if level == "smoke" and self.ir.is_lm:
|
|
678
|
+
report.extend(smoke_report(self._smoke(), self.config.check))
|
|
679
|
+
return report
|
|
680
|
+
|
|
681
|
+
def _record(self, ref: Any) -> PatchRecord:
|
|
682
|
+
handle = ref.handle if isinstance(ref, PatchRef) else str(ref)
|
|
683
|
+
key = handle.split(":")[0]
|
|
684
|
+
for r in self.staged:
|
|
685
|
+
if r.handle == key:
|
|
686
|
+
return r
|
|
687
|
+
return self.repo.get_patch(key)
|
|
688
|
+
|
|
689
|
+
def _committed_state(self) -> TreeState:
|
|
690
|
+
return TreeState.from_dict(self.repo.tree_of_oid(self.repo.head(self.session)))
|
|
691
|
+
|
|
692
|
+
def _canary(self) -> tuple[torch.Tensor, torch.Tensor]:
|
|
693
|
+
if self.text.tokenizer is not None:
|
|
694
|
+
return self.text.batch(CANARY_PROMPTS, self.device)
|
|
695
|
+
gen = torch.Generator().manual_seed(1234)
|
|
696
|
+
vocab = self.ir.vocab_size or 2
|
|
697
|
+
ids = torch.randint(0, vocab, (8, 16), generator=gen).to(self.device)
|
|
698
|
+
return ids, torch.ones_like(ids)
|
|
699
|
+
|
|
700
|
+
def _smoke(self) -> SmokeMetrics:
|
|
701
|
+
ids, mask = self._canary()
|
|
702
|
+
if self._base_canary is None:
|
|
703
|
+
with torch.no_grad(), self.at_base():
|
|
704
|
+
logits = analysis.forward(self.model, self.ir, ids, mask)
|
|
705
|
+
self._base_canary = {"logits": logits.cpu(), "ppl": perplexity(logits, ids, mask)}
|
|
706
|
+
with torch.no_grad():
|
|
707
|
+
logits = analysis.forward(self.model, self.ir, ids, mask)
|
|
708
|
+
kl = token_kl(self._base_canary["logits"].to(logits.device), logits, mask)
|
|
709
|
+
reasons: list[str] = []
|
|
710
|
+
if self.text.tokenizer is not None:
|
|
711
|
+
outs = analysis.generate(self.model, self.text, CANARY_PROMPTS[:4], max_tokens=16)
|
|
712
|
+
for out in outs:
|
|
713
|
+
reasons.extend(
|
|
714
|
+
r for r in degenerate_reasons(out, eos=self.text.eos_id) if r not in reasons
|
|
715
|
+
)
|
|
716
|
+
return SmokeMetrics(kl, self._base_canary["ppl"], perplexity(logits, ids, mask), reasons)
|
|
717
|
+
|
|
718
|
+
# ------------------------------------------------------------------ test
|
|
719
|
+
|
|
720
|
+
def _suites(self) -> dict[str, Suite]:
|
|
721
|
+
found = {name: factory() for name, factory in BUILTIN.items()}
|
|
722
|
+
paths = [p for p in self.config.tests.paths if (self.workdir / p).exists()]
|
|
723
|
+
for s in runner.discover(paths, root=self.workdir) if paths else []:
|
|
724
|
+
found[s.name] = s
|
|
725
|
+
return found
|
|
726
|
+
|
|
727
|
+
def test(
|
|
728
|
+
self, suite: str | None = None, *, vs: str | None = None, caller: str = "driver"
|
|
729
|
+
) -> ToolResult:
|
|
730
|
+
suites = self._suites()
|
|
731
|
+
names = [suite] if suite else [n for n in suites if n not in BUILTIN]
|
|
732
|
+
if not names:
|
|
733
|
+
names = ["canary"]
|
|
734
|
+
missing = [n for n in names if n not in suites]
|
|
735
|
+
if missing:
|
|
736
|
+
raise WspxError(f"no suite {missing[0]!r}; available: {', '.join(sorted(suites))}")
|
|
737
|
+
lines, data = [], []
|
|
738
|
+
for name in names:
|
|
739
|
+
s = suites[name]
|
|
740
|
+
current = self._run(s, self.tree.state)
|
|
741
|
+
reference = None
|
|
742
|
+
if vs is not None:
|
|
743
|
+
state = (
|
|
744
|
+
TreeState()
|
|
745
|
+
if vs == "base"
|
|
746
|
+
else TreeState.from_dict(
|
|
747
|
+
self.repo.tree_of_oid(self.repo.resolve(vs, self.session))
|
|
748
|
+
)
|
|
749
|
+
)
|
|
750
|
+
reference = self._run(s, state)
|
|
751
|
+
if s.heldout:
|
|
752
|
+
self.heldout_queries += 1
|
|
753
|
+
lines.append(runner.summary(current, vs=reference))
|
|
754
|
+
data.append(runner.driver_view(current) if caller != "ui" else current.__dict__)
|
|
755
|
+
self._save()
|
|
756
|
+
return ToolResult("\n".join(lines), {"runs": data})
|
|
757
|
+
|
|
758
|
+
def _run(self, suite: Suite, state: TreeState) -> runner.RunResult:
|
|
759
|
+
key = self.tree_key(state)
|
|
760
|
+
rel = f"runs/{key}/{suite.name}/{suite.hash}-{suite.seed}.json"
|
|
761
|
+
raw = self.store.read_file(rel)
|
|
762
|
+
if raw is not None:
|
|
763
|
+
cached = runner.RunResult.from_json(raw)
|
|
764
|
+
if self._data_unchanged(cached):
|
|
765
|
+
cached.cached = True
|
|
766
|
+
return cached
|
|
767
|
+
with self.at_state(state):
|
|
768
|
+
model = SuiteModel(self.model, self.ir, self.text, self.at_base)
|
|
769
|
+
result = runner.run_suite(suite, model, key)
|
|
770
|
+
self.store.write_file(rel, result.to_json())
|
|
771
|
+
return result
|
|
772
|
+
|
|
773
|
+
def _data_unchanged(self, run: runner.RunResult) -> bool:
|
|
774
|
+
for file, digest in run.data_hashes.items():
|
|
775
|
+
p = Path(file)
|
|
776
|
+
if not p.exists() or hashlib.sha256(p.read_bytes()).hexdigest()[:16] != digest:
|
|
777
|
+
return False
|
|
778
|
+
return True
|
|
779
|
+
|
|
780
|
+
# ------------------------------------------------------------------ vcs
|
|
781
|
+
|
|
782
|
+
def status(self) -> ToolResult:
|
|
783
|
+
head = self.repo.head(self.session)
|
|
784
|
+
branch = self.repo.current_branch(self.session) or "(detached)"
|
|
785
|
+
lines = [f"on {branch} at {short(head) if head else '(base)'}"]
|
|
786
|
+
if not self.staged:
|
|
787
|
+
lines.append("nothing staged")
|
|
788
|
+
for r in self.staged:
|
|
789
|
+
lines.append(
|
|
790
|
+
f" staged {r.handle} {r.op} → {', '.join(_display(self.ir, t) for t in r.tensors)}"
|
|
791
|
+
f" [{r.precision}]"
|
|
792
|
+
)
|
|
793
|
+
return ToolResult(
|
|
794
|
+
"\n".join(lines), {"staged": [r.handle for r in self.staged], "head": head}
|
|
795
|
+
)
|
|
796
|
+
|
|
797
|
+
def commit(self, message: str, *, force: bool = False) -> ToolResult:
|
|
798
|
+
if not self.staged:
|
|
799
|
+
raise VCSError("nothing to commit; stage an edit first")
|
|
800
|
+
report = self.check(level="smoke")
|
|
801
|
+
if not report.ok and not force:
|
|
802
|
+
raise CheckFailed(report.errors)
|
|
803
|
+
branch = self.repo.current_branch(self.session)
|
|
804
|
+
if (
|
|
805
|
+
branch in self.config.policy.protected_branches
|
|
806
|
+
and self.context.get("caller") == "driver"
|
|
807
|
+
):
|
|
808
|
+
raise VCSError(
|
|
809
|
+
f"{branch} is protected; commit on a session branch and merge with approval"
|
|
810
|
+
)
|
|
811
|
+
checks = self.store.objects.put_json(report.model_dump(mode="json"))
|
|
812
|
+
state = self.tree.state
|
|
813
|
+
hashes = {name: self.tree.hash(name) for name in sorted(state.tensors())}
|
|
814
|
+
evals = [
|
|
815
|
+
f"run:{rel.split('/')[-2]}@{self.tree_key()}"
|
|
816
|
+
for rel in self.store.glob(f"runs/{self.tree_key()}", "*/*.json")
|
|
817
|
+
]
|
|
818
|
+
author = {
|
|
819
|
+
"session": self.session,
|
|
820
|
+
**{k: v for k, v in self.context.items() if k in ("driver", "calls")},
|
|
821
|
+
}
|
|
822
|
+
commit = self.repo.commit(
|
|
823
|
+
self.session,
|
|
824
|
+
tree=state.to_dict(),
|
|
825
|
+
patches=[r.oid for r in self.staged],
|
|
826
|
+
result_hashes=hashes,
|
|
827
|
+
message=message,
|
|
828
|
+
author=author,
|
|
829
|
+
checks=checks,
|
|
830
|
+
evals=sorted(set(evals)),
|
|
831
|
+
forced=force and not report.ok,
|
|
832
|
+
)
|
|
833
|
+
touched = sorted({t for r in self.staged for t in r.tensors})
|
|
834
|
+
self.staged = []
|
|
835
|
+
if self.index is not None:
|
|
836
|
+
self.index = build_index(self.ir, dict(self.params), names=touched, existing=self.index)
|
|
837
|
+
self._save()
|
|
838
|
+
warn = f" ({len(report.warnings)} warning(s))" if report.warnings else ""
|
|
839
|
+
return ToolResult(
|
|
840
|
+
f"committed {commit.short} on {branch or '(detached)'}: {commit.message}{warn}",
|
|
841
|
+
{"commit": commit.oid, "touched": touched},
|
|
842
|
+
)
|
|
843
|
+
|
|
844
|
+
def log(self, rev: str = "HEAD", n: int = 20) -> ToolResult:
|
|
845
|
+
start = self.repo.resolve(rev, self.session)
|
|
846
|
+
rows = self.repo.log(start, n)
|
|
847
|
+
lines = [
|
|
848
|
+
f"{c.short} {time.strftime('%Y-%m-%d %H:%M', time.localtime(c.time))} "
|
|
849
|
+
f"{'[needs-review] ' if c.status != 'ok' else ''}{c.message.splitlines()[0]}"
|
|
850
|
+
for c in rows
|
|
851
|
+
]
|
|
852
|
+
return ToolResult("\n".join(lines) or "(no commits)", {"commits": [c.oid for c in rows]})
|
|
853
|
+
|
|
854
|
+
def checkout(self, rev: str, *, force: bool = False) -> ToolResult:
|
|
855
|
+
if self.staged and not force:
|
|
856
|
+
raise VCSError(
|
|
857
|
+
f"{len(self.staged)} staged patch(es) would be lost; commit them or use force"
|
|
858
|
+
)
|
|
859
|
+
branch = (
|
|
860
|
+
rev if rev in self.repo.branches() or self.store.refs.exists(f"heads/{rev}") else None
|
|
861
|
+
)
|
|
862
|
+
oid = self.repo.resolve(rev, self.session)
|
|
863
|
+
tree = TreeState.from_dict(self.repo.tree_of_oid(oid))
|
|
864
|
+
started = time.perf_counter()
|
|
865
|
+
self.tree.checkout(tree)
|
|
866
|
+
if oid:
|
|
867
|
+
self._verify(self.repo.get_commit(oid))
|
|
868
|
+
if branch:
|
|
869
|
+
self.repo.switch(self.session, branch)
|
|
870
|
+
elif oid:
|
|
871
|
+
self.repo.detach(self.session, oid)
|
|
872
|
+
self.staged = []
|
|
873
|
+
self._save()
|
|
874
|
+
ms = (time.perf_counter() - started) * 1000
|
|
875
|
+
where = branch or (short(oid) if oid else "base")
|
|
876
|
+
return ToolResult(
|
|
877
|
+
f"checked out {where} · {len(tree.tensors())} tensor(s) edited vs base · "
|
|
878
|
+
f"hashes verified · {ms:.0f} ms",
|
|
879
|
+
{"head": oid},
|
|
880
|
+
)
|
|
881
|
+
|
|
882
|
+
def branch(
|
|
883
|
+
self, name: str | None = None, *, at: str = "HEAD", switch: bool = True
|
|
884
|
+
) -> ToolResult:
|
|
885
|
+
if name is None:
|
|
886
|
+
current = self.repo.current_branch(self.session)
|
|
887
|
+
lines = [
|
|
888
|
+
f"{'*' if b == current else ' '} {b} {short(o)}"
|
|
889
|
+
for b, o in sorted(self.repo.branches().items())
|
|
890
|
+
]
|
|
891
|
+
return ToolResult(
|
|
892
|
+
"\n".join(lines) or "(no branches)", {"branches": self.repo.branches()}
|
|
893
|
+
)
|
|
894
|
+
self.repo.create_branch(name, self.repo.resolve(at, self.session))
|
|
895
|
+
if switch:
|
|
896
|
+
self.repo.switch(self.session, name)
|
|
897
|
+
return ToolResult(f"branch {name} at {at}" + (" (switched)" if switch else ""))
|
|
898
|
+
|
|
899
|
+
def revert(self, rev: str = "HEAD") -> ToolResult:
|
|
900
|
+
if self.staged:
|
|
901
|
+
raise VCSError("commit or discard staged patches before reverting")
|
|
902
|
+
oid = self.repo.resolve(rev, self.session)
|
|
903
|
+
tree = self.repo.revert_tree(self.session, rev)
|
|
904
|
+
state = TreeState.from_dict(tree)
|
|
905
|
+
self.tree.checkout(state)
|
|
906
|
+
hashes = {name: self.tree.hash(name) for name in sorted(state.tensors())}
|
|
907
|
+
target = self.repo.get_commit(oid) if oid else None
|
|
908
|
+
commit = self.repo.commit(
|
|
909
|
+
self.session,
|
|
910
|
+
tree=tree,
|
|
911
|
+
patches=[],
|
|
912
|
+
result_hashes=hashes,
|
|
913
|
+
message=f'Revert "{target.message.splitlines()[0] if target else rev}"',
|
|
914
|
+
author={"session": self.session},
|
|
915
|
+
)
|
|
916
|
+
restored = [
|
|
917
|
+
n
|
|
918
|
+
for n in self.ir.params
|
|
919
|
+
if n not in state.tensors() and (target is not None and n in target.result_hashes)
|
|
920
|
+
]
|
|
921
|
+
exact = all(self.tree.hash(n) == self.base_hash(n) for n in restored)
|
|
922
|
+
return ToolResult(
|
|
923
|
+
f"reverted {short(oid) if oid else rev} as {commit.short}"
|
|
924
|
+
+ (
|
|
925
|
+
f" · {len(restored)} tensor(s) back to base, hashes match"
|
|
926
|
+
if restored and exact
|
|
927
|
+
else ""
|
|
928
|
+
),
|
|
929
|
+
{"commit": commit.oid},
|
|
930
|
+
)
|
|
931
|
+
|
|
932
|
+
def diff(self, a: str = "HEAD", b: str | None = None) -> ToolResult:
|
|
933
|
+
sa = self._state_of(a)
|
|
934
|
+
sb = self.tree.state if b is None else self._state_of(b)
|
|
935
|
+
names = sorted(
|
|
936
|
+
t
|
|
937
|
+
for t in sa.tensors() | sb.tensors()
|
|
938
|
+
if sa.merged.get(t) != sb.merged.get(t) or (t in sa.promote) != (t in sb.promote)
|
|
939
|
+
)
|
|
940
|
+
rows = []
|
|
941
|
+
for name in names:
|
|
942
|
+
wa = self.tree.materialize_state(name, sa, dtype=torch.float32)
|
|
943
|
+
wb = self.tree.materialize_state(name, sb, dtype=torch.float32)
|
|
944
|
+
delta = wb - wa
|
|
945
|
+
nz = delta != 0
|
|
946
|
+
rel = float(delta.norm() / wa.norm().clamp_min(1e-30))
|
|
947
|
+
rank = (
|
|
948
|
+
int(torch.linalg.matrix_rank(delta))
|
|
949
|
+
if delta.dim() == 2 and delta.numel() <= 1 << 24
|
|
950
|
+
else None
|
|
951
|
+
)
|
|
952
|
+
rows.append(
|
|
953
|
+
{
|
|
954
|
+
"tensor": name,
|
|
955
|
+
"address": _display(self.ir, name),
|
|
956
|
+
"norm": float(delta.norm()),
|
|
957
|
+
"rel": rel,
|
|
958
|
+
"rank": rank,
|
|
959
|
+
"touched": int(nz.sum()),
|
|
960
|
+
"slices": _bbox(nz),
|
|
961
|
+
}
|
|
962
|
+
)
|
|
963
|
+
roles = sorted(
|
|
964
|
+
{(r.block, r.branch) for n in names for r in self.ir.roles_of(n) if r.block is not None}
|
|
965
|
+
)
|
|
966
|
+
lines = [
|
|
967
|
+
f"weight diff {a}..{b or 'working tree'}: {len(rows)} tensor(s)"
|
|
968
|
+
+ (f" across {len({blk for blk, _ in roles})} block(s)" if roles else "")
|
|
969
|
+
]
|
|
970
|
+
lines.extend(
|
|
971
|
+
f" {r['address']:<24} ‖Δ‖ {r['norm']:.3g} ({r['rel']:.2%}) rank {r['rank']} · "
|
|
972
|
+
f"{r['touched']} elements in {r['slices']}"
|
|
973
|
+
for r in rows
|
|
974
|
+
)
|
|
975
|
+
behaviour = self._behaviour_diff(sa, sb)
|
|
976
|
+
lines.extend(behaviour)
|
|
977
|
+
return ToolResult("\n".join(lines), {"tensors": rows, "behaviour": behaviour})
|
|
978
|
+
|
|
979
|
+
def _state_of(self, rev: str) -> TreeState:
|
|
980
|
+
if rev == "base":
|
|
981
|
+
return TreeState()
|
|
982
|
+
return TreeState.from_dict(self.repo.tree_of_oid(self.repo.resolve(rev, self.session)))
|
|
983
|
+
|
|
984
|
+
def _behaviour_diff(self, sa: TreeState, sb: TreeState) -> list[str]:
|
|
985
|
+
ka, kb = self.tree_key(sa), self.tree_key(sb)
|
|
986
|
+
out = []
|
|
987
|
+
for rel_b in self.store.glob(f"runs/{kb}", "*/*.json"):
|
|
988
|
+
suite_file = rel_b.split("/", 2)[2]
|
|
989
|
+
raw_a = self.store.read_file(f"runs/{ka}/{suite_file}")
|
|
990
|
+
raw_b = self.store.read_file(rel_b)
|
|
991
|
+
if raw_a is not None and raw_b is not None:
|
|
992
|
+
ra, rb = runner.RunResult.from_json(raw_a), runner.RunResult.from_json(raw_b)
|
|
993
|
+
p = runner.paired(rb, ra)
|
|
994
|
+
out.append(
|
|
995
|
+
f" behaviour {rb.suite}: {ra.pass_rate:.1%} → {rb.pass_rate:.1%} "
|
|
996
|
+
f"(Δ {p.mean_delta:+.3f}, {p.flips_up} fixed, {p.flips_down} broken)"
|
|
997
|
+
)
|
|
998
|
+
return out
|
|
999
|
+
|
|
1000
|
+
def blame(self, address: str) -> ToolResult:
|
|
1001
|
+
lines = []
|
|
1002
|
+
for t in self.resolve(address):
|
|
1003
|
+
boxes = [
|
|
1004
|
+
set(range(*s.indices(n)))
|
|
1005
|
+
if isinstance(s, slice)
|
|
1006
|
+
else ({s} if isinstance(s, int) else set(s))
|
|
1007
|
+
for s, n in zip(
|
|
1008
|
+
normalize_index(t.index, len(self.ir.params[t.param].shape)),
|
|
1009
|
+
self.ir.params[t.param].shape,
|
|
1010
|
+
strict=True,
|
|
1011
|
+
)
|
|
1012
|
+
]
|
|
1013
|
+
|
|
1014
|
+
def touches(entry: dict[str, Any], boxes: list[set[int]] = boxes) -> bool:
|
|
1015
|
+
dims, _flat = merge._selection(self._load_delta(entry["blob"]))
|
|
1016
|
+
if dims is None:
|
|
1017
|
+
return True
|
|
1018
|
+
return all(a & b for a, b in zip(dims, boxes, strict=True))
|
|
1019
|
+
|
|
1020
|
+
for c in self.repo.blame(self.repo.head(self.session), t.param, touches):
|
|
1021
|
+
lines.append(f"{c.short} {t.address}: {c.message.splitlines()[0]}")
|
|
1022
|
+
return ToolResult("\n".join(lines) or f"no commit touched {address}", {"lines": lines})
|
|
1023
|
+
|
|
1024
|
+
def bisect(self, suite: str, *, good: str = "base", bad: str = "HEAD") -> ToolResult:
|
|
1025
|
+
suites = self._suites()
|
|
1026
|
+
if suite not in suites:
|
|
1027
|
+
raise WspxError(f"no suite {suite!r}")
|
|
1028
|
+
s = suites[suite]
|
|
1029
|
+
bad_oid = self.repo.resolve(bad, self.session)
|
|
1030
|
+
good_oid = None if good == "base" else self.repo.resolve(good, self.session)
|
|
1031
|
+
chain = []
|
|
1032
|
+
for c in self.repo.walk(bad_oid, first_parent=True):
|
|
1033
|
+
if c.oid == good_oid:
|
|
1034
|
+
break
|
|
1035
|
+
chain.append(c)
|
|
1036
|
+
chain.reverse()
|
|
1037
|
+
if not chain:
|
|
1038
|
+
raise VCSError("no commits between good and bad")
|
|
1039
|
+
threshold = self._run(s, self._state_of(good)).pass_rate
|
|
1040
|
+
lo, hi = 0, len(chain) - 1
|
|
1041
|
+
steps = 0
|
|
1042
|
+
if self._run(s, self._state_of(chain[hi].oid)).pass_rate >= threshold:
|
|
1043
|
+
return ToolResult(
|
|
1044
|
+
f"{suite} does not regress between {good} and {bad} (pass rate ≥ {threshold:.1%})"
|
|
1045
|
+
)
|
|
1046
|
+
while lo < hi:
|
|
1047
|
+
mid = (lo + hi) // 2
|
|
1048
|
+
steps += 1
|
|
1049
|
+
if self._run(s, self._state_of(chain[mid].oid)).pass_rate < threshold:
|
|
1050
|
+
hi = mid
|
|
1051
|
+
else:
|
|
1052
|
+
lo = mid + 1
|
|
1053
|
+
first = chain[lo]
|
|
1054
|
+
return ToolResult(
|
|
1055
|
+
f"first bad commit: {first.short} {first.message.splitlines()[0]} "
|
|
1056
|
+
f"({steps} step(s); {suite} below {threshold:.1%})",
|
|
1057
|
+
{"commit": first.oid},
|
|
1058
|
+
)
|
|
1059
|
+
|
|
1060
|
+
def merge(
|
|
1061
|
+
self, branch: str, *, strategies: dict[str, str] | None = None, message: str | None = None
|
|
1062
|
+
) -> ToolResult:
|
|
1063
|
+
if self.staged:
|
|
1064
|
+
raise VCSError("commit staged patches before merging")
|
|
1065
|
+
ours = self.repo.head(self.session)
|
|
1066
|
+
theirs = self.repo.resolve(branch, self.session)
|
|
1067
|
+
if theirs is None:
|
|
1068
|
+
raise VCSError(f"{branch} has no commits")
|
|
1069
|
+
if self.repo.is_ancestor(theirs, ours):
|
|
1070
|
+
return ToolResult(f"already up to date with {branch}")
|
|
1071
|
+
base = self.repo.merge_base(ours, theirs)
|
|
1072
|
+
mine = self.repo.introduced(base, ours)
|
|
1073
|
+
other = self.repo.introduced(base, theirs)
|
|
1074
|
+
tree = self.repo.tree_of_oid(ours)
|
|
1075
|
+
merged = {k: list(v) for k, v in tree.get("merged", {}).items()}
|
|
1076
|
+
requested = self._strategy_map(strategies or {})
|
|
1077
|
+
report: list[dict[str, Any]] = []
|
|
1078
|
+
unresolved = []
|
|
1079
|
+
acts = self._canary_inputs(list(set(mine) & set(other)))
|
|
1080
|
+
for tensor, blobs in other.items():
|
|
1081
|
+
if tensor not in mine:
|
|
1082
|
+
merged[tensor] = sorted({*merged.get(tensor, []), *blobs})
|
|
1083
|
+
continue
|
|
1084
|
+
a = [self._load_delta(o) for o in mine[tensor]]
|
|
1085
|
+
b = [self._load_delta(o) for o in blobs]
|
|
1086
|
+
role = self.ir.primary_role(tensor)
|
|
1087
|
+
ov = merge.overlap(
|
|
1088
|
+
tensor,
|
|
1089
|
+
a,
|
|
1090
|
+
b,
|
|
1091
|
+
out_dim=(role.out_dim if role and role.out_dim is not None else 0),
|
|
1092
|
+
in_dim=(role.in_dim if role and role.in_dim is not None else 1),
|
|
1093
|
+
acts=acts.get(tensor),
|
|
1094
|
+
)
|
|
1095
|
+
choice = merge.choose(ov, requested.get(tensor))
|
|
1096
|
+
ov.strategy = choice
|
|
1097
|
+
report.append(ov.to_dict())
|
|
1098
|
+
if choice == "needs-strategy":
|
|
1099
|
+
unresolved.append(tensor)
|
|
1100
|
+
continue
|
|
1101
|
+
merged[tensor] = self._apply_strategy(
|
|
1102
|
+
choice, tensor, merged.get(tensor, []), a, b, theirs
|
|
1103
|
+
)
|
|
1104
|
+
if unresolved:
|
|
1105
|
+
table = "\n".join(
|
|
1106
|
+
f" {_display(self.ir, r['tensor'])}: slice {r['slice']:.2f}, out-space "
|
|
1107
|
+
f"{r['out_space']:.2f}, in-space {r['in_space']:.2f}"
|
|
1108
|
+
for r in report
|
|
1109
|
+
if r["tensor"] in unresolved
|
|
1110
|
+
)
|
|
1111
|
+
return ToolResult(
|
|
1112
|
+
f"merge needs a strategy for {len(unresolved)} tensor(s):\n{table}\n"
|
|
1113
|
+
f'fix: pass strategies, e.g. {{"{_display(self.ir, unresolved[0])}": "orthogonalize"}}'
|
|
1114
|
+
f" (sum, rebase, orthogonalize, ties, pick:ours, pick:theirs)",
|
|
1115
|
+
{"conflicts": report},
|
|
1116
|
+
ok=False,
|
|
1117
|
+
)
|
|
1118
|
+
new_tree = {
|
|
1119
|
+
"merged": {k: v for k, v in merged.items() if v},
|
|
1120
|
+
"adapter": tree.get("adapter", {}),
|
|
1121
|
+
"promote": tree.get("promote", []),
|
|
1122
|
+
}
|
|
1123
|
+
state = TreeState.from_dict(new_tree)
|
|
1124
|
+
kl_ours = kl_theirs = None
|
|
1125
|
+
if self.ir.is_lm:
|
|
1126
|
+
ids, mask = self._canary()
|
|
1127
|
+
with torch.no_grad():
|
|
1128
|
+
with self.at_state(state):
|
|
1129
|
+
m = analysis.forward(self.model, self.ir, ids, mask)
|
|
1130
|
+
with self.at_state(self._state_of(theirs)):
|
|
1131
|
+
t = analysis.forward(self.model, self.ir, ids, mask)
|
|
1132
|
+
o = analysis.forward(self.model, self.ir, ids, mask)
|
|
1133
|
+
kl_ours, kl_theirs = token_kl(o, m, mask), token_kl(t, m, mask)
|
|
1134
|
+
self.tree.checkout(state)
|
|
1135
|
+
suites_failed = self._rerun_recorded(ours, theirs)
|
|
1136
|
+
limit = self.config.check.max_canary_kl
|
|
1137
|
+
failed = bool(suites_failed) or any(
|
|
1138
|
+
k is not None and k > limit for k in (kl_ours, kl_theirs)
|
|
1139
|
+
)
|
|
1140
|
+
hashes = {n: self.tree.hash(n) for n in sorted(state.tensors())}
|
|
1141
|
+
branch_now = self.repo.current_branch(self.session)
|
|
1142
|
+
status = "needs-review" if failed else "ok"
|
|
1143
|
+
info = {
|
|
1144
|
+
"from": theirs,
|
|
1145
|
+
"base": base,
|
|
1146
|
+
"conflicts": report,
|
|
1147
|
+
"kl_vs_ours": kl_ours,
|
|
1148
|
+
"kl_vs_theirs": kl_theirs,
|
|
1149
|
+
"suites_failed": suites_failed,
|
|
1150
|
+
}
|
|
1151
|
+
if failed and branch_now in self.config.policy.protected_branches:
|
|
1152
|
+
self.tree.checkout(self._state_of(ours) if ours else TreeState())
|
|
1153
|
+
return ToolResult(
|
|
1154
|
+
f"merge into protected {branch_now} needs review: interference check failed "
|
|
1155
|
+
f"(KL {kl_ours}, {kl_theirs}; suites {suites_failed}); nothing changed",
|
|
1156
|
+
{"merge": info},
|
|
1157
|
+
ok=False,
|
|
1158
|
+
)
|
|
1159
|
+
commit = self.repo.commit(
|
|
1160
|
+
self.session,
|
|
1161
|
+
tree=new_tree,
|
|
1162
|
+
patches=[],
|
|
1163
|
+
result_hashes=hashes,
|
|
1164
|
+
message=message or f"Merge {branch} into {branch_now or 'HEAD'}",
|
|
1165
|
+
author={"session": self.session},
|
|
1166
|
+
extra_parents=[theirs],
|
|
1167
|
+
merge=info,
|
|
1168
|
+
status=status,
|
|
1169
|
+
)
|
|
1170
|
+
self._save()
|
|
1171
|
+
kl_text = (
|
|
1172
|
+
f"KL vs ours {kl_ours:.3f}, vs theirs {kl_theirs:.3f}"
|
|
1173
|
+
if kl_ours is not None
|
|
1174
|
+
else "no canary"
|
|
1175
|
+
)
|
|
1176
|
+
return ToolResult(
|
|
1177
|
+
f"merged {branch} as {commit.short} [{status}] · {len(report)} overlapping tensor(s) · "
|
|
1178
|
+
f"{kl_text}",
|
|
1179
|
+
{"commit": commit.oid, "merge": info},
|
|
1180
|
+
)
|
|
1181
|
+
|
|
1182
|
+
def _strategy_map(self, strategies: dict[str, str]) -> dict[str, str]:
|
|
1183
|
+
out = {}
|
|
1184
|
+
for key, value in strategies.items():
|
|
1185
|
+
params = [key] if key in self.ir.params else [t.param for t in self.resolve(key)]
|
|
1186
|
+
out.update(dict.fromkeys(params, value))
|
|
1187
|
+
return out
|
|
1188
|
+
|
|
1189
|
+
def _apply_strategy(
|
|
1190
|
+
self,
|
|
1191
|
+
choice: str,
|
|
1192
|
+
tensor: str,
|
|
1193
|
+
current: list[str],
|
|
1194
|
+
ours: list[TensorDelta],
|
|
1195
|
+
theirs: list[TensorDelta],
|
|
1196
|
+
theirs_head: str,
|
|
1197
|
+
) -> list[str]:
|
|
1198
|
+
mine = {d.oid for d in ours}
|
|
1199
|
+
keep = [o for o in current if o not in mine]
|
|
1200
|
+
if choice == "sum":
|
|
1201
|
+
return sorted({*current, *(d.oid for d in theirs)})
|
|
1202
|
+
if choice == "pick:ours":
|
|
1203
|
+
return current
|
|
1204
|
+
if choice == "pick:theirs":
|
|
1205
|
+
return sorted({*keep, *(d.oid for d in theirs)})
|
|
1206
|
+
if choice == "orthogonalize":
|
|
1207
|
+
new = op_registry.get("orthogonalize")(self._ctx(), theirs, against=ours)
|
|
1208
|
+
elif choice == "ties":
|
|
1209
|
+
new = [merge.ties(ours, theirs)]
|
|
1210
|
+
current = keep
|
|
1211
|
+
elif choice == "rebase":
|
|
1212
|
+
new = self._replay(theirs_head, tensor)
|
|
1213
|
+
else: # pragma: no cover - choose() validates
|
|
1214
|
+
raise VCSError(f"unknown strategy {choice}")
|
|
1215
|
+
oids = [self.store.objects.put(d.to_bytes()) for d in new]
|
|
1216
|
+
return sorted({*current, *oids})
|
|
1217
|
+
|
|
1218
|
+
def _replay(self, head: str, tensor: str) -> list[TensorDelta]:
|
|
1219
|
+
"""Re-run the other branch's ops for ``tensor`` against the current weights."""
|
|
1220
|
+
out: list[TensorDelta] = []
|
|
1221
|
+
base = self.repo.merge_base(self.repo.head(self.session), head)
|
|
1222
|
+
for commit in reversed(list(self.repo.walk(head))):
|
|
1223
|
+
if base and self.repo.is_ancestor(commit.oid, base):
|
|
1224
|
+
continue
|
|
1225
|
+
for pid in commit.patches:
|
|
1226
|
+
record = self.repo.get_patch(pid)
|
|
1227
|
+
if tensor not in record.tensors:
|
|
1228
|
+
continue
|
|
1229
|
+
args = self._unrecord(record.args)
|
|
1230
|
+
deltas = op_registry.get(record.op)(self._ctx(), **args)
|
|
1231
|
+
out.extend(d for d in deltas if d.tensor == tensor)
|
|
1232
|
+
if not out:
|
|
1233
|
+
raise VCSError(f"rebase: no replayable ops touch {tensor}")
|
|
1234
|
+
return out
|
|
1235
|
+
|
|
1236
|
+
def _unrecord(self, args: dict[str, Any]) -> dict[str, Any]:
|
|
1237
|
+
out: dict[str, Any] = {}
|
|
1238
|
+
for k, v in args.items():
|
|
1239
|
+
if isinstance(v, dict) and "tensor" in v:
|
|
1240
|
+
out[k] = st_load(self.store.objects.get(v["tensor"]))["v"]
|
|
1241
|
+
elif isinstance(v, dict) and "address" in v:
|
|
1242
|
+
out[k] = v["address"]
|
|
1243
|
+
elif isinstance(v, list) and v and isinstance(v[0], dict) and "address" in v[0]:
|
|
1244
|
+
out[k] = [x["address"] for x in v]
|
|
1245
|
+
else:
|
|
1246
|
+
out[k] = v
|
|
1247
|
+
return out
|
|
1248
|
+
|
|
1249
|
+
def _canary_inputs(self, tensors: list[str]) -> dict[str, torch.Tensor]:
|
|
1250
|
+
"""Recorded inputs of each 2-D weight's module on the canary (for activation overlap)."""
|
|
1251
|
+
wanted = [
|
|
1252
|
+
t for t in tensors if len(self.ir.params[t].shape) == 2 and self.ir.params[t].module
|
|
1253
|
+
]
|
|
1254
|
+
if not wanted or not self.ir.is_lm:
|
|
1255
|
+
return {}
|
|
1256
|
+
ids, mask = self._canary()
|
|
1257
|
+
with torch.no_grad(), self.probe.capture([], units=wanted) as cap:
|
|
1258
|
+
analysis.forward(self.model, self.ir, ids, mask)
|
|
1259
|
+
return {k: v.reshape(-1, v.shape[-1]).float().cpu() for k, v in cap.units.items()}
|
|
1260
|
+
|
|
1261
|
+
def _rerun_recorded(self, ours: str | None, theirs: str) -> list[str]:
|
|
1262
|
+
names: set[str] = set()
|
|
1263
|
+
for oid in (ours, theirs):
|
|
1264
|
+
if oid:
|
|
1265
|
+
names.update(
|
|
1266
|
+
e.split(":", 1)[1].split("@")[0] for e in self.repo.get_commit(oid).evals
|
|
1267
|
+
)
|
|
1268
|
+
suites = self._suites()
|
|
1269
|
+
failed = []
|
|
1270
|
+
for name in sorted(names & set(suites)):
|
|
1271
|
+
now = self._run(suites[name], self.tree.state)
|
|
1272
|
+
before = [self._run(suites[name], self._state_of(o)) for o in (ours, theirs) if o]
|
|
1273
|
+
if before and now.pass_rate < min(r.pass_rate for r in before):
|
|
1274
|
+
failed.append(name)
|
|
1275
|
+
return failed
|
|
1276
|
+
|
|
1277
|
+
# ------------------------------------------------------------------ export / run
|
|
1278
|
+
|
|
1279
|
+
def export(
|
|
1280
|
+
self, kind: str, path: str, *, base_model: str | None = None, max_rank: int = 64
|
|
1281
|
+
) -> ToolResult:
|
|
1282
|
+
out = (self.workdir / path).resolve()
|
|
1283
|
+
if kind == "merged":
|
|
1284
|
+
info = export.export_merged(self.model, self.ir, out)
|
|
1285
|
+
elif kind == "peft":
|
|
1286
|
+
info = export.export_peft(
|
|
1287
|
+
self.model, self.ir, self.tree, out, base_model=base_model, max_rank=max_rank
|
|
1288
|
+
)
|
|
1289
|
+
elif kind == "gguf":
|
|
1290
|
+
tmp = out.with_suffix(".merged")
|
|
1291
|
+
export.export_merged(self.model, self.ir, tmp)
|
|
1292
|
+
info = export.export_gguf(tmp, out)
|
|
1293
|
+
elif kind == "bundle":
|
|
1294
|
+
info = self._bundle(out)
|
|
1295
|
+
else:
|
|
1296
|
+
raise WspxError(f"unknown export kind {kind!r}; use merged, peft, gguf or bundle")
|
|
1297
|
+
notes = "".join(f"\n ! {w}" for w in info.get("warnings", []))
|
|
1298
|
+
errors = info.get("svd_error") or {}
|
|
1299
|
+
if errors:
|
|
1300
|
+
notes += "".join(
|
|
1301
|
+
f"\n ~ {_display(self.ir, k)}: SVD relative error {v:.2e}"
|
|
1302
|
+
for k, v in errors.items()
|
|
1303
|
+
)
|
|
1304
|
+
return ToolResult(
|
|
1305
|
+
f"exported {kind} to {info['path']}{notes}",
|
|
1306
|
+
{k: v for k, v in info.items() if k != "hashes"},
|
|
1307
|
+
)
|
|
1308
|
+
|
|
1309
|
+
def _bundle(self, out: Path) -> dict[str, Any]:
|
|
1310
|
+
head = self.repo.head(self.session)
|
|
1311
|
+
objects: dict[str, bytes] = {}
|
|
1312
|
+
for commit in self.repo.walk(head):
|
|
1313
|
+
objects[commit.oid] = self.store.objects.get(commit.oid)
|
|
1314
|
+
objects[commit.tree] = self.store.objects.get(commit.tree)
|
|
1315
|
+
for pid in commit.patches:
|
|
1316
|
+
objects[pid] = self.store.objects.get(pid)
|
|
1317
|
+
for section in ("merged", "adapter"):
|
|
1318
|
+
for blobs in self.repo.get_tree(commit.tree).get(section, {}).values():
|
|
1319
|
+
for b in blobs:
|
|
1320
|
+
objects[b] = self.store.objects.get(b)
|
|
1321
|
+
manifest = {
|
|
1322
|
+
"format": "wspx.bundle/1",
|
|
1323
|
+
"base": self.ir.fingerprint,
|
|
1324
|
+
"arch": self.ir.arch,
|
|
1325
|
+
"head": head,
|
|
1326
|
+
}
|
|
1327
|
+
return export.export_bundle(objects, manifest, out)
|
|
1328
|
+
|
|
1329
|
+
def run(self, script: str, *, args: Sequence[str] = ()) -> ToolResult:
|
|
1330
|
+
from wspx.lab import sdk
|
|
1331
|
+
|
|
1332
|
+
path = (self.workdir / script).resolve()
|
|
1333
|
+
if not path.is_file():
|
|
1334
|
+
raise WspxError(f"{script}: no such script")
|
|
1335
|
+
source = path.read_bytes()
|
|
1336
|
+
digest = "sha256:" + hashlib.sha256(source).hexdigest()
|
|
1337
|
+
buf = io.StringIO()
|
|
1338
|
+
previous = sdk.activate(self)
|
|
1339
|
+
self._script = digest
|
|
1340
|
+
ok = True
|
|
1341
|
+
try:
|
|
1342
|
+
with contextlib.redirect_stdout(buf), contextlib.redirect_stderr(buf):
|
|
1343
|
+
import sys
|
|
1344
|
+
|
|
1345
|
+
argv = sys.argv
|
|
1346
|
+
sys.argv = [str(path), *args]
|
|
1347
|
+
try:
|
|
1348
|
+
runpy.run_path(str(path), run_name="__main__")
|
|
1349
|
+
except SystemExit as exc:
|
|
1350
|
+
ok = exc.code in (None, 0)
|
|
1351
|
+
except Exception:
|
|
1352
|
+
ok = False
|
|
1353
|
+
traceback.print_exc()
|
|
1354
|
+
finally:
|
|
1355
|
+
sys.argv = argv
|
|
1356
|
+
finally:
|
|
1357
|
+
self._script = None
|
|
1358
|
+
sdk.activate(previous)
|
|
1359
|
+
output = buf.getvalue()
|
|
1360
|
+
oid = self.store.objects.put(output.encode())
|
|
1361
|
+
tail = output.splitlines()[-RUN_TAIL_LINES:]
|
|
1362
|
+
text = "\n".join(tail)[-RUN_TAIL_BYTES:]
|
|
1363
|
+
return ToolResult(
|
|
1364
|
+
f"{text}\n[{'ok' if ok else 'failed'} · script {digest[7:19]} · full output {short(oid, 12)}]",
|
|
1365
|
+
{"output": oid, "script": digest, "ok": ok},
|
|
1366
|
+
ok=ok,
|
|
1367
|
+
)
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
# --------------------------------------------------------------------------- SDK objects
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
class GrepHits:
|
|
1374
|
+
"""A grep result: iterable hits plus the handle that names them."""
|
|
1375
|
+
|
|
1376
|
+
def __init__(self, lab: Lab, handle: str, result: GrepResult, top: int) -> None:
|
|
1377
|
+
self.lab = lab
|
|
1378
|
+
self.handle = handle
|
|
1379
|
+
self.result = result
|
|
1380
|
+
self.hits = result.hits[:top]
|
|
1381
|
+
|
|
1382
|
+
def __iter__(self) -> Iterator[Hit]:
|
|
1383
|
+
return iter(self.hits)
|
|
1384
|
+
|
|
1385
|
+
def __len__(self) -> int:
|
|
1386
|
+
return len(self.hits)
|
|
1387
|
+
|
|
1388
|
+
def __getitem__(self, i: int) -> Hit:
|
|
1389
|
+
return self.hits[i]
|
|
1390
|
+
|
|
1391
|
+
@property
|
|
1392
|
+
def summary(self) -> str:
|
|
1393
|
+
lines = []
|
|
1394
|
+
index = self.lab.index
|
|
1395
|
+
for h in self.hits:
|
|
1396
|
+
tokens = ""
|
|
1397
|
+
if index is not None:
|
|
1398
|
+
tops = index.unit_tokens(h.param, h.unit, h.expert)[:4]
|
|
1399
|
+
if tops:
|
|
1400
|
+
tokens = " writes: " + ", ".join(self.lab.text.show(t) for t, _ in tops)
|
|
1401
|
+
lines.append(f"{h.address:<32} {h.score:+.4f}{tokens}")
|
|
1402
|
+
more = len(self.result.hits) - len(self.hits)
|
|
1403
|
+
tail = (
|
|
1404
|
+
f"{len(self.hits)} of {self.result.candidates:,} {self.result.axis}s"
|
|
1405
|
+
+ (f" ({more} more matched; raise --top or narrow)" if more > 0 else "")
|
|
1406
|
+
+ f" · {self.result.seconds * 1000:.0f} ms · saved as {self.handle}"
|
|
1407
|
+
)
|
|
1408
|
+
return "\n".join([*lines, tail])
|
|
1409
|
+
|
|
1410
|
+
def __str__(self) -> str:
|
|
1411
|
+
return self.summary
|
|
1412
|
+
|
|
1413
|
+
|
|
1414
|
+
class DirectionRef:
|
|
1415
|
+
def __init__(
|
|
1416
|
+
self, lab: Lab, handle: str, direction: analysis.Direction, writers: GrepHits
|
|
1417
|
+
) -> None:
|
|
1418
|
+
self.lab = lab
|
|
1419
|
+
self.handle = handle
|
|
1420
|
+
self.direction = direction
|
|
1421
|
+
self.writers = writers
|
|
1422
|
+
|
|
1423
|
+
@property
|
|
1424
|
+
def vector(self) -> torch.Tensor:
|
|
1425
|
+
return self.direction.vector
|
|
1426
|
+
|
|
1427
|
+
@property
|
|
1428
|
+
def summary(self) -> str:
|
|
1429
|
+
d = self.direction
|
|
1430
|
+
return (
|
|
1431
|
+
f"{self.handle} · ‖v‖ {float(d.vector.norm()):.3g} · separation {d.separation:.2f} "
|
|
1432
|
+
f"({d.n_pos} vs {d.n_neg} prompts)\ntop writers:\n{self.writers.summary}"
|
|
1433
|
+
)
|
|
1434
|
+
|
|
1435
|
+
def __str__(self) -> str:
|
|
1436
|
+
return self.summary
|
|
1437
|
+
|
|
1438
|
+
|
|
1439
|
+
class PatchRef:
|
|
1440
|
+
def __init__(self, lab: Lab, handle: str, record: PatchRecord, report: Report) -> None:
|
|
1441
|
+
self.lab = lab
|
|
1442
|
+
self.handle = handle
|
|
1443
|
+
self.record = record
|
|
1444
|
+
self.report = report
|
|
1445
|
+
|
|
1446
|
+
@property
|
|
1447
|
+
def summary(self) -> str:
|
|
1448
|
+
tensors = ", ".join(_display(self.lab.ir, t) for t in self.record.tensors[:6])
|
|
1449
|
+
body = f"staged {self.handle}: {self.record.op} on {tensors} [{self.record.precision}]"
|
|
1450
|
+
notes = [d.render() for d in self.report.diagnostics]
|
|
1451
|
+
return body + ("\n" + "\n".join(notes) if notes else "")
|
|
1452
|
+
|
|
1453
|
+
def __str__(self) -> str:
|
|
1454
|
+
return self.summary
|
|
1455
|
+
|
|
1456
|
+
|
|
1457
|
+
class EditNamespace:
|
|
1458
|
+
"""``lab.edit.project_out(...)`` or ``lab.edit("project_out", ...)``."""
|
|
1459
|
+
|
|
1460
|
+
def __init__(self, lab: Lab) -> None:
|
|
1461
|
+
self._lab = lab
|
|
1462
|
+
|
|
1463
|
+
def __call__(self, op: str, **args: Any) -> PatchRef:
|
|
1464
|
+
return self._lab.edit_op(op, **args)
|
|
1465
|
+
|
|
1466
|
+
def __getattr__(self, op: str) -> Callable[..., PatchRef]:
|
|
1467
|
+
if op.startswith("_"):
|
|
1468
|
+
raise AttributeError(op)
|
|
1469
|
+
op_registry.get(op)
|
|
1470
|
+
|
|
1471
|
+
def call(*positional: Any, **args: Any) -> PatchRef:
|
|
1472
|
+
return self._lab.edit_op(op, **_bind_positional(op, positional, args))
|
|
1473
|
+
|
|
1474
|
+
return call
|
|
1475
|
+
|
|
1476
|
+
|
|
1477
|
+
def _bind_positional(op: str, positional: tuple[Any, ...], args: dict[str, Any]) -> dict[str, Any]:
|
|
1478
|
+
import inspect
|
|
1479
|
+
|
|
1480
|
+
fn = op_registry.get(op)
|
|
1481
|
+
names = [
|
|
1482
|
+
p.name
|
|
1483
|
+
for p in list(inspect.signature(fn).parameters.values())[1:]
|
|
1484
|
+
if p.kind in (p.POSITIONAL_ONLY, p.POSITIONAL_OR_KEYWORD)
|
|
1485
|
+
]
|
|
1486
|
+
if len(positional) > len(names):
|
|
1487
|
+
raise PatchError(f"{op} takes at most {len(names)} positional argument(s)")
|
|
1488
|
+
return {**dict(zip(names, positional, strict=False)), **args}
|
|
1489
|
+
|
|
1490
|
+
|
|
1491
|
+
# --------------------------------------------------------------------------- helpers
|
|
1492
|
+
|
|
1493
|
+
|
|
1494
|
+
def _display(ir: ModelIR, name: str) -> str:
|
|
1495
|
+
role = ir.primary_role(name)
|
|
1496
|
+
return role.address if role else name
|
|
1497
|
+
|
|
1498
|
+
|
|
1499
|
+
def _default_scope(expression: str) -> str:
|
|
1500
|
+
if "writes(" in expression or "cos(" in expression:
|
|
1501
|
+
return "L*.*.down,L*.*.o,L*.*.out"
|
|
1502
|
+
if "reads(" in expression:
|
|
1503
|
+
return "L*.*.q,L*.*.k,L*.*.v,L*.*.up,L*.*.gate"
|
|
1504
|
+
return "L*.*"
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
def _writer_scope(ir: ModelIR) -> str:
|
|
1508
|
+
return ",".join(
|
|
1509
|
+
sorted(
|
|
1510
|
+
{
|
|
1511
|
+
".".join(["L*", *r.address.split(".")[1:]])
|
|
1512
|
+
for r in ir.roles
|
|
1513
|
+
if r.kind == "writer" and r.split is None
|
|
1514
|
+
}
|
|
1515
|
+
)
|
|
1516
|
+
)
|
|
1517
|
+
|
|
1518
|
+
|
|
1519
|
+
def _unit_dim(ir: ModelIR, target: Target) -> int:
|
|
1520
|
+
role = target.role or ir.primary_role(target.param)
|
|
1521
|
+
if role is None or role.in_dim is None or role.out_dim is None:
|
|
1522
|
+
return len(ir.params[target.param].shape) - 1
|
|
1523
|
+
writes = role.kind in ("writer", "embed", "pos_embed", "stem") or (
|
|
1524
|
+
role.kind == "expert" and role.address.endswith(".down")
|
|
1525
|
+
)
|
|
1526
|
+
return role.in_dim if writes else role.out_dim
|
|
1527
|
+
|
|
1528
|
+
|
|
1529
|
+
def _bbox(mask: torch.Tensor) -> str:
|
|
1530
|
+
if mask.dim() == 0 or not mask.any():
|
|
1531
|
+
return "nothing"
|
|
1532
|
+
parts = []
|
|
1533
|
+
for d in range(mask.dim()):
|
|
1534
|
+
other = tuple(i for i in range(mask.dim()) if i != d)
|
|
1535
|
+
hit = mask.any(dim=other) if other else mask
|
|
1536
|
+
idx = torch.nonzero(hit).reshape(-1)
|
|
1537
|
+
lo, hi = int(idx.min()), int(idx.max())
|
|
1538
|
+
parts.append(":" if lo == 0 and hi == mask.shape[d] - 1 else f"{lo}:{hi + 1}")
|
|
1539
|
+
return "[" + ", ".join(parts) + "]"
|
|
1540
|
+
|
|
1541
|
+
|
|
1542
|
+
def _jsonable(value: Any) -> Any:
|
|
1543
|
+
if isinstance(value, dict):
|
|
1544
|
+
return {str(k): _jsonable(v) for k, v in value.items()}
|
|
1545
|
+
if isinstance(value, list | tuple):
|
|
1546
|
+
return [_jsonable(v) for v in value]
|
|
1547
|
+
if isinstance(value, float) and math.isnan(value):
|
|
1548
|
+
return None
|
|
1549
|
+
if isinstance(value, torch.Tensor):
|
|
1550
|
+
return value.tolist()
|
|
1551
|
+
return value
|
|
1552
|
+
|
|
1553
|
+
|
|
1554
|
+
def _split_scopes(text: str) -> list[str]:
|
|
1555
|
+
"""Split ``a, b[:, 3], c`` on top-level commas (not inside brackets or quotes)."""
|
|
1556
|
+
parts: list[str] = []
|
|
1557
|
+
buf: list[str] = []
|
|
1558
|
+
depth, quote = 0, ""
|
|
1559
|
+
for ch in text:
|
|
1560
|
+
if quote:
|
|
1561
|
+
quote = "" if ch == quote else quote
|
|
1562
|
+
elif ch in "\"'":
|
|
1563
|
+
quote = ch
|
|
1564
|
+
elif ch == "[":
|
|
1565
|
+
depth += 1
|
|
1566
|
+
elif ch == "]":
|
|
1567
|
+
depth -= 1
|
|
1568
|
+
elif ch == "," and depth == 0:
|
|
1569
|
+
parts.append("".join(buf).strip())
|
|
1570
|
+
buf = []
|
|
1571
|
+
continue
|
|
1572
|
+
buf.append(ch)
|
|
1573
|
+
parts.append("".join(buf).strip())
|
|
1574
|
+
return [p for p in parts if p]
|