wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/loader.py
ADDED
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
"""Loading models into the lab (§5.1.1).
|
|
2
|
+
|
|
3
|
+
1. **Resolve the source**: a local directory or a Hugging Face repo (already
|
|
4
|
+
fetched into the models volume by the downloader; fetched here when run
|
|
5
|
+
locally), or a custom ``nn.Module`` given as ``pkg.module:Class``.
|
|
6
|
+
2. **Instantiate on the meta device** to get the true parameter table at no cost.
|
|
7
|
+
3. **Bind the weights**; missing or unexpected keys are an error.
|
|
8
|
+
4. **Fingerprint**: repo, revision, modeling-code hash and checkpoint file hashes.
|
|
9
|
+
|
|
10
|
+
Base tensors that map one-to-one onto a checkpoint tensor can be re-read from
|
|
11
|
+
the read-only files, so the working tree's base cache may evict them.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
import importlib
|
|
18
|
+
import json
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from pathlib import Path
|
|
21
|
+
from typing import Any
|
|
22
|
+
|
|
23
|
+
import torch
|
|
24
|
+
|
|
25
|
+
from wspx.errors import ModelLoadError
|
|
26
|
+
|
|
27
|
+
DTYPES = {"bfloat16": torch.bfloat16, "float16": torch.float16, "float32": torch.float32}
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
@dataclass
|
|
31
|
+
class Loaded:
|
|
32
|
+
model: torch.nn.Module
|
|
33
|
+
tokenizer: Any
|
|
34
|
+
fingerprint: str
|
|
35
|
+
source: str
|
|
36
|
+
path: Path | None
|
|
37
|
+
readable: dict[str, tuple[Path, str]] = field(default_factory=dict)
|
|
38
|
+
"""Parameter → (safetensors file, key) for tensors stored verbatim."""
|
|
39
|
+
|
|
40
|
+
def base_reader(self, name: str) -> torch.Tensor:
|
|
41
|
+
from safetensors import safe_open
|
|
42
|
+
|
|
43
|
+
if name not in self.readable:
|
|
44
|
+
raise ModelLoadError(f"{name} is not stored verbatim in the checkpoint")
|
|
45
|
+
path, key = self.readable[name]
|
|
46
|
+
with safe_open(str(path), framework="pt") as fh:
|
|
47
|
+
tensor: torch.Tensor = fh.get_tensor(key)
|
|
48
|
+
dtype = dict(self.model.named_parameters())[name].dtype
|
|
49
|
+
return tensor.to(dtype)
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def resolve_source(source: str, *, revision: str | None = None, local_only: bool = False) -> Path:
|
|
53
|
+
path = Path(source).expanduser()
|
|
54
|
+
if path.is_dir():
|
|
55
|
+
return path
|
|
56
|
+
try:
|
|
57
|
+
from huggingface_hub import snapshot_download
|
|
58
|
+
except ImportError as exc: # pragma: no cover - lab dependency
|
|
59
|
+
raise ModelLoadError("huggingface_hub is needed to fetch models") from exc
|
|
60
|
+
try:
|
|
61
|
+
return Path(
|
|
62
|
+
snapshot_download(
|
|
63
|
+
source,
|
|
64
|
+
revision=revision,
|
|
65
|
+
local_files_only=local_only,
|
|
66
|
+
allow_patterns=[
|
|
67
|
+
"*.json",
|
|
68
|
+
"*.safetensors",
|
|
69
|
+
"*.py",
|
|
70
|
+
"*.model",
|
|
71
|
+
"*.txt",
|
|
72
|
+
"*.jinja",
|
|
73
|
+
"*.tiktoken",
|
|
74
|
+
],
|
|
75
|
+
)
|
|
76
|
+
)
|
|
77
|
+
except Exception as exc:
|
|
78
|
+
raise ModelLoadError(f"cannot fetch {source}: {exc}") from exc
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def load_hf(
|
|
82
|
+
source: str,
|
|
83
|
+
*,
|
|
84
|
+
revision: str | None = None,
|
|
85
|
+
dtype: str = "bfloat16",
|
|
86
|
+
trust_remote_code: bool = False,
|
|
87
|
+
device: str | None = None,
|
|
88
|
+
local_only: bool = False,
|
|
89
|
+
) -> Loaded:
|
|
90
|
+
"""Load a causal LM (or any AutoModel) and fingerprint it."""
|
|
91
|
+
from transformers import AutoConfig, AutoModel, AutoModelForCausalLM, AutoTokenizer
|
|
92
|
+
|
|
93
|
+
path = resolve_source(source, revision=revision, local_only=local_only)
|
|
94
|
+
config = AutoConfig.from_pretrained(str(path), trust_remote_code=trust_remote_code)
|
|
95
|
+
torch_dtype = DTYPES.get(dtype, "auto")
|
|
96
|
+
is_lm = bool(getattr(config, "architectures", None)) and any(
|
|
97
|
+
a.endswith(("ForCausalLM", "LMHeadModel", "ForConditionalGeneration"))
|
|
98
|
+
for a in config.architectures or []
|
|
99
|
+
)
|
|
100
|
+
auto = AutoModelForCausalLM if is_lm else AutoModel
|
|
101
|
+
with torch.device("meta"):
|
|
102
|
+
skeleton = auto.from_config(config, trust_remote_code=trust_remote_code) # type: ignore[no-untyped-call]
|
|
103
|
+
expected = {n for n, _ in skeleton.named_parameters()}
|
|
104
|
+
try:
|
|
105
|
+
loaded: Any = auto.from_pretrained(
|
|
106
|
+
str(path),
|
|
107
|
+
dtype=torch_dtype,
|
|
108
|
+
trust_remote_code=trust_remote_code,
|
|
109
|
+
device_map=device or ("auto" if torch.cuda.is_available() else None),
|
|
110
|
+
output_loading_info=True,
|
|
111
|
+
)
|
|
112
|
+
model, info = loaded
|
|
113
|
+
except Exception as exc:
|
|
114
|
+
raise ModelLoadError(f"cannot load {source}: {exc}") from exc
|
|
115
|
+
missing = [k for k in info.get("missing_keys", []) if k in expected]
|
|
116
|
+
unexpected = list(info.get("unexpected_keys", []))
|
|
117
|
+
if missing or unexpected:
|
|
118
|
+
raise ModelLoadError(
|
|
119
|
+
f"{source}: checkpoint does not match the architecture "
|
|
120
|
+
f"(missing {missing[:5]}, unexpected {unexpected[:5]})"
|
|
121
|
+
)
|
|
122
|
+
model.eval()
|
|
123
|
+
tokenizer = None
|
|
124
|
+
try:
|
|
125
|
+
tokenizer = AutoTokenizer.from_pretrained(str(path), trust_remote_code=trust_remote_code)
|
|
126
|
+
except Exception:
|
|
127
|
+
tokenizer = None
|
|
128
|
+
readable = _verbatim_tensors(path, model)
|
|
129
|
+
return Loaded(model, tokenizer, fingerprint(path, source, revision), source, path, readable)
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def load_module(spec: str, weights: str | None = None, **kwargs: Any) -> Loaded:
|
|
133
|
+
"""``pkg.module:Class`` plus optional safetensors weights."""
|
|
134
|
+
module_name, _, attr = spec.partition(":")
|
|
135
|
+
if not attr:
|
|
136
|
+
raise ModelLoadError(f"{spec!r}: expected 'package.module:Class'")
|
|
137
|
+
try:
|
|
138
|
+
cls = getattr(importlib.import_module(module_name), attr)
|
|
139
|
+
except (ImportError, AttributeError) as exc:
|
|
140
|
+
raise ModelLoadError(f"cannot import {spec}: {exc}") from exc
|
|
141
|
+
model = cls(**kwargs)
|
|
142
|
+
path: Path | None = None
|
|
143
|
+
if weights:
|
|
144
|
+
from safetensors.torch import load_file
|
|
145
|
+
|
|
146
|
+
path = Path(weights)
|
|
147
|
+
state = load_file(str(path))
|
|
148
|
+
result = model.load_state_dict(state, strict=True)
|
|
149
|
+
del result
|
|
150
|
+
digest = hashlib.sha256(spec.encode())
|
|
151
|
+
if path is not None:
|
|
152
|
+
digest.update(_file_hash(path).encode())
|
|
153
|
+
model.eval()
|
|
154
|
+
readable = {n: (path, n) for n, _ in model.named_parameters()} if path is not None else {}
|
|
155
|
+
return Loaded(model, None, "fp:" + digest.hexdigest(), spec, path, readable)
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def fingerprint(path: Path, source: str, revision: str | None) -> str:
|
|
159
|
+
"""repo + revision + code hash + checkpoint file hashes."""
|
|
160
|
+
digest = hashlib.sha256()
|
|
161
|
+
digest.update(f"{source}@{revision or ''}\n".encode())
|
|
162
|
+
for file in sorted(path.glob("*.py")):
|
|
163
|
+
digest.update(f"code {file.name} {_file_hash(file)}\n".encode())
|
|
164
|
+
for file in sorted(path.glob("*.safetensors")):
|
|
165
|
+
digest.update(f"weights {file.name} {_file_hash(file)}\n".encode())
|
|
166
|
+
config = path / "config.json"
|
|
167
|
+
if config.exists():
|
|
168
|
+
digest.update(f"config {_file_hash(config)}\n".encode())
|
|
169
|
+
return "fp:" + digest.hexdigest()
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _file_hash(path: Path) -> str:
|
|
173
|
+
h = hashlib.sha256()
|
|
174
|
+
with path.open("rb") as fh:
|
|
175
|
+
for chunk in iter(lambda: fh.read(1 << 24), b""):
|
|
176
|
+
h.update(chunk)
|
|
177
|
+
return h.hexdigest()
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def _verbatim_tensors(path: Path, model: torch.nn.Module) -> dict[str, tuple[Path, str]]:
|
|
181
|
+
"""Parameters whose checkpoint tensor has the same name and shape (re-readable bases)."""
|
|
182
|
+
from safetensors import safe_open
|
|
183
|
+
|
|
184
|
+
index = path / "model.safetensors.index.json"
|
|
185
|
+
files: dict[str, Path] = {}
|
|
186
|
+
if index.exists():
|
|
187
|
+
files = {k: path / v for k, v in json.loads(index.read_text())["weight_map"].items()}
|
|
188
|
+
else:
|
|
189
|
+
for file in path.glob("*.safetensors"):
|
|
190
|
+
with safe_open(str(file), framework="pt") as fh:
|
|
191
|
+
files.update(dict.fromkeys(fh.keys(), file))
|
|
192
|
+
out: dict[str, tuple[Path, str]] = {}
|
|
193
|
+
shapes: dict[Path, Any] = {}
|
|
194
|
+
for name, param in model.named_parameters():
|
|
195
|
+
found = files.get(name)
|
|
196
|
+
if found is None:
|
|
197
|
+
continue
|
|
198
|
+
file = found
|
|
199
|
+
if file not in shapes:
|
|
200
|
+
with safe_open(str(file), framework="pt") as fh:
|
|
201
|
+
shapes[file] = {k: fh.get_slice(k).get_shape() for k in fh.keys()} # noqa: SIM118
|
|
202
|
+
if list(shapes[file].get(name, [])) == list(param.shape):
|
|
203
|
+
out[name] = (file, name)
|
|
204
|
+
return out
|
wspx/lab/merge.py
ADDED
|
@@ -0,0 +1,200 @@
|
|
|
1
|
+
"""Merging in weight space (§5.5.4): detect overlap by action, resolve it with math.
|
|
2
|
+
|
|
3
|
+
Text-git semantics don't carry over: nobody resolves a weight conflict by
|
|
4
|
+
reading numbers, and edits on disjoint slices of one layer can still interact.
|
|
5
|
+
So every tensor both sides touched gets a conflict score from three signals:
|
|
6
|
+
|
|
7
|
+
* **slice**: the share of touched elements both sides touched;
|
|
8
|
+
* **subspace**: the largest cosine between the two deltas' output (column)
|
|
9
|
+
spaces and between their input (row) spaces;
|
|
10
|
+
* **activation**: on recorded canary inputs, the correlation between the
|
|
11
|
+
sizes of ``Δ_A x`` and ``Δ_B x``.
|
|
12
|
+
|
|
13
|
+
Strategies per tensor: ``sum`` (default when overlap is low), ``rebase``
|
|
14
|
+
(replay the other side's ops on top), ``orthogonalize`` (project B's delta off
|
|
15
|
+
A's output subspace), ``ties`` (trim, elect sign, disjoint mean) and
|
|
16
|
+
``pick:ours`` / ``pick:theirs``.
|
|
17
|
+
"""
|
|
18
|
+
|
|
19
|
+
from __future__ import annotations
|
|
20
|
+
|
|
21
|
+
from dataclasses import asdict, dataclass
|
|
22
|
+
from typing import Any
|
|
23
|
+
|
|
24
|
+
import torch
|
|
25
|
+
|
|
26
|
+
from wspx.lab import patch
|
|
27
|
+
from wspx.lab.patch import TensorDelta
|
|
28
|
+
from wspx.lab.tensors import normalize_index
|
|
29
|
+
|
|
30
|
+
STRATEGIES = ("sum", "rebase", "orthogonalize", "ties", "pick:ours", "pick:theirs")
|
|
31
|
+
AUTO_SUM_BELOW = 0.3
|
|
32
|
+
MAX_SUBSPACE_RANK = 32
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Overlap:
|
|
37
|
+
tensor: str
|
|
38
|
+
slice: float
|
|
39
|
+
out_space: float
|
|
40
|
+
in_space: float
|
|
41
|
+
activation: float | None
|
|
42
|
+
score: float
|
|
43
|
+
strategy: str = "sum"
|
|
44
|
+
|
|
45
|
+
def to_dict(self) -> dict[str, Any]:
|
|
46
|
+
return asdict(self)
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def _selection(delta: TensorDelta) -> tuple[list[set[int]] | None, set[int] | None]:
|
|
50
|
+
"""Outer-product per-dim index sets, or a set of flat indices for sparse deltas."""
|
|
51
|
+
if delta.kind == "sparse":
|
|
52
|
+
assert delta.indices is not None
|
|
53
|
+
return None, set(delta.indices.tolist())
|
|
54
|
+
index = normalize_index(delta.index, len(delta.shape))
|
|
55
|
+
dims = []
|
|
56
|
+
for sel, size in zip(index, delta.shape, strict=True):
|
|
57
|
+
if isinstance(sel, int):
|
|
58
|
+
dims.append({sel})
|
|
59
|
+
elif isinstance(sel, slice):
|
|
60
|
+
dims.append(set(range(*sel.indices(size))))
|
|
61
|
+
else:
|
|
62
|
+
dims.append(set(sel))
|
|
63
|
+
return dims, None
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _flat(dims: list[set[int]], shape: tuple[int, ...]) -> set[int]:
|
|
67
|
+
flat = {0}
|
|
68
|
+
for d, (values, size) in enumerate(zip(dims, shape, strict=True)):
|
|
69
|
+
del d
|
|
70
|
+
flat = {f * size + v for f in flat for v in values}
|
|
71
|
+
return flat
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def slice_overlap(a: list[TensorDelta], b: list[TensorDelta]) -> float:
|
|
75
|
+
"""|touched_A ∩ touched_B| / min(|touched_A|, |touched_B|)."""
|
|
76
|
+
if not a or not b:
|
|
77
|
+
return 0.0
|
|
78
|
+
shape = a[0].shape
|
|
79
|
+
numel = 1
|
|
80
|
+
for s in shape:
|
|
81
|
+
numel *= s
|
|
82
|
+
if numel <= 1 << 22:
|
|
83
|
+
mask_a = torch.zeros(numel, dtype=torch.bool)
|
|
84
|
+
mask_b = torch.zeros(numel, dtype=torch.bool)
|
|
85
|
+
for deltas, mask in ((a, mask_a), (b, mask_b)):
|
|
86
|
+
for d in deltas:
|
|
87
|
+
dims, flat = _selection(d)
|
|
88
|
+
idx = torch.tensor(
|
|
89
|
+
sorted(flat if flat is not None else _flat(dims or [], shape)), dtype=torch.long
|
|
90
|
+
)
|
|
91
|
+
mask[idx] = True
|
|
92
|
+
both = int((mask_a & mask_b).sum())
|
|
93
|
+
return both / max(1, min(int(mask_a.sum()), int(mask_b.sum())))
|
|
94
|
+
# Large tensors: outer-product boxes intersect dim by dim.
|
|
95
|
+
best = 0.0
|
|
96
|
+
for da in a:
|
|
97
|
+
for db in b:
|
|
98
|
+
sa, fa = _selection(da)
|
|
99
|
+
sb, fb = _selection(db)
|
|
100
|
+
if sa is None or sb is None:
|
|
101
|
+
best = max(best, 1.0 if (fa or set()) & (fb or set()) else 0.0)
|
|
102
|
+
continue
|
|
103
|
+
inter = 1
|
|
104
|
+
size_a = size_b = 1
|
|
105
|
+
for xa, xb in zip(sa, sb, strict=True):
|
|
106
|
+
inter *= len(xa & xb)
|
|
107
|
+
size_a *= len(xa)
|
|
108
|
+
size_b *= len(xb)
|
|
109
|
+
best = max(best, inter / max(1, min(size_a, size_b)))
|
|
110
|
+
return best
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
def _as_matrix(delta: torch.Tensor, out_dim: int, in_dim: int) -> torch.Tensor:
|
|
114
|
+
if delta.dim() == 2:
|
|
115
|
+
return delta if (out_dim, in_dim) == (0, 1) else delta.T
|
|
116
|
+
return delta.reshape(-1, delta.shape[-1])
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _basis(mat: torch.Tensor, side: str) -> torch.Tensor:
|
|
120
|
+
m = mat if side == "out" else mat.T
|
|
121
|
+
if not torch.any(m):
|
|
122
|
+
return torch.zeros(m.shape[0], 0)
|
|
123
|
+
u, s, _ = torch.linalg.svd(m.float(), full_matrices=False)
|
|
124
|
+
keep = s > 1e-6 * float(s.max())
|
|
125
|
+
basis: torch.Tensor = u[:, keep][:, :MAX_SUBSPACE_RANK]
|
|
126
|
+
return basis
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def subspace_overlap(da: torch.Tensor, db: torch.Tensor, side: str) -> float:
|
|
130
|
+
"""Largest cosine of the principal angles between the two deltas' spaces."""
|
|
131
|
+
ua, ub = _basis(da, side), _basis(db, side)
|
|
132
|
+
if ua.shape[1] == 0 or ub.shape[1] == 0:
|
|
133
|
+
return 0.0
|
|
134
|
+
return float(torch.linalg.svdvals(ua.T @ ub).max().clamp(0, 1))
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def activation_overlap(da: torch.Tensor, db: torch.Tensor, acts: torch.Tensor) -> float | None:
|
|
138
|
+
"""Pearson correlation of ‖Δ_A x‖ and ‖Δ_B x‖ over canary inputs ``acts`` [n, in]."""
|
|
139
|
+
if acts.shape[-1] != da.shape[1]:
|
|
140
|
+
return None
|
|
141
|
+
x = acts.reshape(-1, acts.shape[-1]).float()
|
|
142
|
+
a = (x @ da.T.float()).norm(dim=1)
|
|
143
|
+
b = (x @ db.T.float()).norm(dim=1)
|
|
144
|
+
if a.std() == 0 or b.std() == 0:
|
|
145
|
+
return 0.0
|
|
146
|
+
return float(torch.corrcoef(torch.stack([a, b]))[0, 1].clamp(-1, 1))
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def overlap(
|
|
150
|
+
tensor: str,
|
|
151
|
+
ours: list[TensorDelta],
|
|
152
|
+
theirs: list[TensorDelta],
|
|
153
|
+
*,
|
|
154
|
+
out_dim: int = 0,
|
|
155
|
+
in_dim: int = 1,
|
|
156
|
+
acts: torch.Tensor | None = None,
|
|
157
|
+
) -> Overlap:
|
|
158
|
+
dense_a = sum((d.dense() for d in ours), torch.zeros(ours[0].shape))
|
|
159
|
+
dense_b = sum((d.dense() for d in theirs), torch.zeros(theirs[0].shape))
|
|
160
|
+
ma, mb = _as_matrix(dense_a, out_dim, in_dim), _as_matrix(dense_b, out_dim, in_dim)
|
|
161
|
+
s = slice_overlap(ours, theirs)
|
|
162
|
+
o = subspace_overlap(ma, mb, "out")
|
|
163
|
+
i = subspace_overlap(ma, mb, "in")
|
|
164
|
+
act = activation_overlap(ma, mb, acts) if acts is not None and ma.dim() == 2 else None
|
|
165
|
+
signals = [s, o, i] + ([abs(act)] if act is not None else [])
|
|
166
|
+
# Disjoint slices that write the same directions still conflict, so take the max.
|
|
167
|
+
score = max(signals)
|
|
168
|
+
return Overlap(tensor, s, o, i, act, score)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def choose(ov: Overlap, requested: str | None) -> str:
|
|
172
|
+
if requested:
|
|
173
|
+
if requested not in STRATEGIES:
|
|
174
|
+
raise ValueError(
|
|
175
|
+
f"unknown merge strategy {requested!r}; use one of {', '.join(STRATEGIES)}"
|
|
176
|
+
)
|
|
177
|
+
return requested
|
|
178
|
+
return "sum" if ov.score < AUTO_SUM_BELOW else "needs-strategy"
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def ties(ours: list[TensorDelta], theirs: list[TensorDelta], *, keep: float = 0.2) -> TensorDelta:
|
|
182
|
+
"""TIES-merging: trim each side to its largest ``keep`` share, elect a sign, average agreeing values."""
|
|
183
|
+
first = ours[0] if ours else theirs[0]
|
|
184
|
+
sides = [
|
|
185
|
+
sum((d.dense() for d in side), torch.zeros(first.shape)) for side in (ours, theirs) if side
|
|
186
|
+
]
|
|
187
|
+
trimmed = []
|
|
188
|
+
for delta in sides:
|
|
189
|
+
flat = delta.abs().reshape(-1)
|
|
190
|
+
k = max(1, int(flat.numel() * keep))
|
|
191
|
+
threshold = torch.topk(flat, k).values[-1]
|
|
192
|
+
trimmed.append(torch.where(delta.abs() >= threshold, delta, torch.zeros_like(delta)))
|
|
193
|
+
stacked = torch.stack(trimmed)
|
|
194
|
+
sign = torch.sign(stacked.sum(0))
|
|
195
|
+
agree = (torch.sign(stacked) == sign) & (stacked != 0)
|
|
196
|
+
count = agree.sum(0).clamp_min(1)
|
|
197
|
+
merged = (stacked * agree).sum(0) / count
|
|
198
|
+
return patch.compact(
|
|
199
|
+
patch.dense(first.tensor, first.shape, first.base, merged, op="ties", keep=keep)
|
|
200
|
+
)
|
wspx/lab/ops/__init__.py
ADDED
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
"""Edit ops: high-level edits that compile to the patch IR at staging time (§5.3).
|
|
2
|
+
|
|
3
|
+
An op runs once against the current working tree and returns concrete fp32
|
|
4
|
+
deltas. Its name and arguments are kept for readability (and so a merge can
|
|
5
|
+
re-apply the intent), but replay never re-runs it.
|
|
6
|
+
|
|
7
|
+
Third-party ops register under the ``wspx.ops`` entry-point group with the
|
|
8
|
+
signature ``op(ctx: OpContext, **args) -> list[TensorDelta]``.
|
|
9
|
+
"""
|
|
10
|
+
|
|
11
|
+
from __future__ import annotations
|
|
12
|
+
|
|
13
|
+
from collections.abc import Callable
|
|
14
|
+
from dataclasses import dataclass, field
|
|
15
|
+
from functools import cache
|
|
16
|
+
from importlib.metadata import entry_points
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
import torch
|
|
20
|
+
|
|
21
|
+
from wspx.errors import PatchError
|
|
22
|
+
from wspx.lab.ir.model_ir import ModelIR, Target
|
|
23
|
+
from wspx.lab.patch import TensorDelta
|
|
24
|
+
from wspx.lab.tree import WorkingTree
|
|
25
|
+
|
|
26
|
+
Op = Callable[..., list[TensorDelta]]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass
|
|
30
|
+
class OpContext:
|
|
31
|
+
"""What an op may see: the IR, the live weights and ways to resolve names."""
|
|
32
|
+
|
|
33
|
+
ir: ModelIR
|
|
34
|
+
tree: WorkingTree
|
|
35
|
+
resolve: Callable[[str], list[Target]]
|
|
36
|
+
direction: Callable[[Any], torch.Tensor]
|
|
37
|
+
"""``dir:name`` handle, list or tensor → fp32 vector on the model's device."""
|
|
38
|
+
sources: dict[str, tuple[ModelIR, torch.nn.Module]] = field(default_factory=dict)
|
|
39
|
+
"""Other loaded models, for ``transplant``."""
|
|
40
|
+
|
|
41
|
+
def weight(self, name: str) -> torch.Tensor:
|
|
42
|
+
"""The live tensor (current working tree), as fp32."""
|
|
43
|
+
return self.tree.params[name].detach().float()
|
|
44
|
+
|
|
45
|
+
def base_hash(self, name: str) -> str:
|
|
46
|
+
info = self.ir.params[name]
|
|
47
|
+
if info.hash is None:
|
|
48
|
+
raise PatchError(f"{name}: base hash unknown (IR built without hashes)")
|
|
49
|
+
return info.hash
|
|
50
|
+
|
|
51
|
+
def targets(self, spec: str | list[str] | Target | list[Target]) -> list[Target]:
|
|
52
|
+
items = spec if isinstance(spec, list) else [spec]
|
|
53
|
+
out: list[Target] = []
|
|
54
|
+
for item in items:
|
|
55
|
+
out.extend(self.resolve(item) if isinstance(item, str) else [item])
|
|
56
|
+
if not out:
|
|
57
|
+
raise PatchError(f"{spec!r} resolves to no weights")
|
|
58
|
+
return out
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
@cache
|
|
62
|
+
def registry() -> dict[str, Op]:
|
|
63
|
+
from wspx.lab.ops import builtin
|
|
64
|
+
|
|
65
|
+
ops: dict[str, Op] = dict(builtin.BUILTIN)
|
|
66
|
+
for ep in entry_points(group="wspx.ops"):
|
|
67
|
+
if ep.name not in ops:
|
|
68
|
+
ops[ep.name] = ep.load()
|
|
69
|
+
return ops
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def get(name: str) -> Op:
|
|
73
|
+
ops = registry()
|
|
74
|
+
if name not in ops:
|
|
75
|
+
raise PatchError(f"unknown op {name!r}; available: {', '.join(sorted(ops))}")
|
|
76
|
+
return ops[name]
|