wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/index.py
ADDED
|
@@ -0,0 +1,239 @@
|
|
|
1
|
+
"""The tier-0 index (§5.2.3): computed from weights alone, once per fingerprint.
|
|
2
|
+
|
|
3
|
+
* per-tensor statistics and a 10-bin histogram;
|
|
4
|
+
* per-unit norms and absmax along each weight's unit axis;
|
|
5
|
+
* top singular values of every matrix (and of every expert matrix);
|
|
6
|
+
* vocabulary projections: the tokens each writer unit writes most and each
|
|
7
|
+
reader unit reads most (final-norm and pre-norm scales folded in).
|
|
8
|
+
|
|
9
|
+
It answers ``info``, ``ls``, ``show --stats/--vocab`` and stat greps while the
|
|
10
|
+
lab is cold, and annotates grep hits. At commit time the lab refreshes the
|
|
11
|
+
entries of the tensors it touched.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import json
|
|
17
|
+
from dataclasses import asdict, dataclass, field
|
|
18
|
+
from pathlib import Path
|
|
19
|
+
from typing import Any
|
|
20
|
+
|
|
21
|
+
import pyarrow as pa
|
|
22
|
+
import pyarrow.parquet as pq
|
|
23
|
+
import torch
|
|
24
|
+
|
|
25
|
+
from wspx.lab.grep import Environment, unit_batches
|
|
26
|
+
from wspx.lab.ir.model_ir import ModelIR
|
|
27
|
+
|
|
28
|
+
SPECTRUM_K = 16
|
|
29
|
+
VOCAB_K = 10
|
|
30
|
+
MAX_SVD_NUMEL = 64 * 1024 * 1024
|
|
31
|
+
LOGIT_BUDGET = 1 << 26 # elements per vocab-projection chunk
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass
|
|
35
|
+
class TensorStats:
|
|
36
|
+
shape: list[int]
|
|
37
|
+
dtype: str
|
|
38
|
+
norm: float
|
|
39
|
+
mean: float
|
|
40
|
+
std: float
|
|
41
|
+
absmax: float
|
|
42
|
+
kurtosis: float
|
|
43
|
+
zeros: float
|
|
44
|
+
nan: int
|
|
45
|
+
inf: int
|
|
46
|
+
hist_edges: list[float]
|
|
47
|
+
hist_counts: list[int]
|
|
48
|
+
spectrum: list[float] = field(default_factory=list)
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def tensor_stats(t: torch.Tensor, *, spectrum: bool = True) -> TensorStats:
|
|
52
|
+
x = t.detach().float()
|
|
53
|
+
flat = x.reshape(-1)
|
|
54
|
+
finite = flat[torch.isfinite(flat)]
|
|
55
|
+
mean = float(finite.mean()) if finite.numel() else float("nan")
|
|
56
|
+
std = float(finite.std(correction=0)) if finite.numel() > 1 else 0.0
|
|
57
|
+
kurt = (
|
|
58
|
+
float(((finite - mean) ** 4).mean() / max(std, 1e-30) ** 4 - 3)
|
|
59
|
+
if finite.numel() > 1
|
|
60
|
+
else 0.0
|
|
61
|
+
)
|
|
62
|
+
lo, hi = (float(finite.min()), float(finite.max())) if finite.numel() else (0.0, 0.0)
|
|
63
|
+
if hi == lo:
|
|
64
|
+
hi = lo + 1e-12
|
|
65
|
+
counts = torch.histc(finite, bins=10, min=lo, max=hi) if finite.numel() else torch.zeros(10)
|
|
66
|
+
edges = [lo + (hi - lo) * i / 10 for i in range(11)]
|
|
67
|
+
spec: list[float] = []
|
|
68
|
+
if spectrum and x.dim() >= 2:
|
|
69
|
+
mat = x[0] if x.dim() >= 3 else x # expert weights: the first expert's spectrum
|
|
70
|
+
mat = mat.reshape(mat.shape[0], -1)
|
|
71
|
+
if mat.numel() <= MAX_SVD_NUMEL and torch.isfinite(mat).all():
|
|
72
|
+
spec = [float(v) for v in torch.linalg.svdvals(mat)[:SPECTRUM_K]]
|
|
73
|
+
return TensorStats(
|
|
74
|
+
shape=list(t.shape),
|
|
75
|
+
dtype=str(t.dtype).removeprefix("torch."),
|
|
76
|
+
norm=float(finite.norm()),
|
|
77
|
+
mean=mean,
|
|
78
|
+
std=std,
|
|
79
|
+
absmax=float(finite.abs().max()) if finite.numel() else 0.0,
|
|
80
|
+
kurtosis=kurt,
|
|
81
|
+
zeros=float((flat == 0).float().mean()),
|
|
82
|
+
nan=int(torch.isnan(flat).sum()),
|
|
83
|
+
inf=int(torch.isinf(flat).sum()),
|
|
84
|
+
hist_edges=edges,
|
|
85
|
+
hist_counts=[int(c) for c in counts.tolist()],
|
|
86
|
+
spectrum=spec,
|
|
87
|
+
)
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
@dataclass
|
|
91
|
+
class ModelIndex:
|
|
92
|
+
fingerprint: str
|
|
93
|
+
tensors: dict[str, TensorStats] = field(default_factory=dict)
|
|
94
|
+
units: dict[str, dict[str, Any]] = field(default_factory=dict)
|
|
95
|
+
"""param → {"axis", "address", "norm", "absmax", "top_ids", "top_scores"} (lists per unit)."""
|
|
96
|
+
|
|
97
|
+
def save(self, directory: Path) -> None:
|
|
98
|
+
directory.mkdir(parents=True, exist_ok=True)
|
|
99
|
+
(directory / "tensors.json").write_text(
|
|
100
|
+
json.dumps(
|
|
101
|
+
{
|
|
102
|
+
"fingerprint": self.fingerprint,
|
|
103
|
+
"tensors": {k: asdict(v) for k, v in self.tensors.items()},
|
|
104
|
+
}
|
|
105
|
+
)
|
|
106
|
+
)
|
|
107
|
+
rows: dict[str, list[Any]] = {
|
|
108
|
+
k: []
|
|
109
|
+
for k in (
|
|
110
|
+
"param",
|
|
111
|
+
"expert",
|
|
112
|
+
"unit",
|
|
113
|
+
"address",
|
|
114
|
+
"norm",
|
|
115
|
+
"absmax",
|
|
116
|
+
"top_ids",
|
|
117
|
+
"top_scores",
|
|
118
|
+
)
|
|
119
|
+
}
|
|
120
|
+
for param, data in self.units.items():
|
|
121
|
+
n = len(data["norm"])
|
|
122
|
+
rows["param"].extend([param] * n)
|
|
123
|
+
rows["expert"].extend(data["expert"])
|
|
124
|
+
rows["unit"].extend(data["unit"])
|
|
125
|
+
rows["address"].extend(data["address"])
|
|
126
|
+
rows["norm"].extend(data["norm"])
|
|
127
|
+
rows["absmax"].extend(data["absmax"])
|
|
128
|
+
rows["top_ids"].extend(data.get("top_ids") or [[] for _ in range(n)])
|
|
129
|
+
rows["top_scores"].extend(data.get("top_scores") or [[] for _ in range(n)])
|
|
130
|
+
pq.write_table(pa.table(rows), directory / "units.parquet", compression="zstd")
|
|
131
|
+
|
|
132
|
+
@classmethod
|
|
133
|
+
def load(cls, directory: Path) -> ModelIndex:
|
|
134
|
+
meta = json.loads((directory / "tensors.json").read_text())
|
|
135
|
+
index = cls(meta["fingerprint"], {k: TensorStats(**v) for k, v in meta["tensors"].items()})
|
|
136
|
+
path = directory / "units.parquet"
|
|
137
|
+
if path.exists():
|
|
138
|
+
for row in pq.read_table(path).to_pylist():
|
|
139
|
+
entry = index.units.setdefault(
|
|
140
|
+
row["param"],
|
|
141
|
+
{
|
|
142
|
+
k: []
|
|
143
|
+
for k in (
|
|
144
|
+
"expert",
|
|
145
|
+
"unit",
|
|
146
|
+
"address",
|
|
147
|
+
"norm",
|
|
148
|
+
"absmax",
|
|
149
|
+
"top_ids",
|
|
150
|
+
"top_scores",
|
|
151
|
+
)
|
|
152
|
+
},
|
|
153
|
+
)
|
|
154
|
+
for key in entry:
|
|
155
|
+
entry[key].append(row[key])
|
|
156
|
+
return index
|
|
157
|
+
|
|
158
|
+
def unit_tokens(
|
|
159
|
+
self, param: str, unit: int, expert: int | None = None
|
|
160
|
+
) -> list[tuple[int, float]]:
|
|
161
|
+
data = self.units.get(param)
|
|
162
|
+
if not data:
|
|
163
|
+
return []
|
|
164
|
+
for i, (e, u) in enumerate(zip(data["expert"], data["unit"], strict=True)):
|
|
165
|
+
if u == unit and (expert is None or e == expert):
|
|
166
|
+
return list(
|
|
167
|
+
zip(data["top_ids"][i] or [], data["top_scores"][i] or [], strict=False)
|
|
168
|
+
)
|
|
169
|
+
return []
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def build_index(
|
|
173
|
+
ir: ModelIR,
|
|
174
|
+
params: dict[str, torch.Tensor],
|
|
175
|
+
*,
|
|
176
|
+
names: list[str] | None = None,
|
|
177
|
+
vocab: bool = True,
|
|
178
|
+
existing: ModelIndex | None = None,
|
|
179
|
+
) -> ModelIndex:
|
|
180
|
+
"""Compute (or refresh, for ``names``) the tier-0 index."""
|
|
181
|
+
index = existing or ModelIndex(ir.fingerprint)
|
|
182
|
+
names = names if names is not None else list(ir.params)
|
|
183
|
+
env = Environment(ir, params)
|
|
184
|
+
with torch.no_grad():
|
|
185
|
+
for name in names:
|
|
186
|
+
index.tensors[name] = tensor_stats(params[name])
|
|
187
|
+
role = ir.primary_role(name)
|
|
188
|
+
if role is None or len(ir.params[name].shape) < 2:
|
|
189
|
+
index.units.pop(name, None)
|
|
190
|
+
continue
|
|
191
|
+
targets = list(ir.resolve(name))
|
|
192
|
+
entry: dict[str, list[Any]] = {
|
|
193
|
+
k: []
|
|
194
|
+
for k in ("expert", "unit", "address", "norm", "absmax", "top_ids", "top_scores")
|
|
195
|
+
}
|
|
196
|
+
for batch in unit_batches(ir, params, targets, None):
|
|
197
|
+
v = batch.vectors
|
|
198
|
+
entry["expert"].extend([-1 if batch.expert is None else batch.expert] * v.shape[0])
|
|
199
|
+
entry["unit"].extend(batch.globals)
|
|
200
|
+
entry["address"].extend(batch.addresses)
|
|
201
|
+
entry["norm"].extend(v.norm(dim=1).tolist())
|
|
202
|
+
entry["absmax"].extend(v.abs().amax(dim=1).tolist())
|
|
203
|
+
ids, scores = _vocab_top(env, role.kind, v) if vocab else (None, None)
|
|
204
|
+
entry["top_ids"].extend(ids or [[] for _ in range(v.shape[0])])
|
|
205
|
+
entry["top_scores"].extend(scores or [[] for _ in range(v.shape[0])])
|
|
206
|
+
index.units[name] = entry
|
|
207
|
+
return index
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _vocab_top(
|
|
211
|
+
env: Environment, kind: str, vectors: torch.Tensor
|
|
212
|
+
) -> tuple[list[list[int]] | None, list[list[float]] | None]:
|
|
213
|
+
sem = env.ir.semantics
|
|
214
|
+
writes = kind in ("writer", "embed", "pos_embed", "stem", "expert")
|
|
215
|
+
name = sem.unembed if writes else sem.embed
|
|
216
|
+
if name is None:
|
|
217
|
+
return None, None
|
|
218
|
+
table = env.params[name].detach().float()
|
|
219
|
+
role = next(
|
|
220
|
+
(r for r in env.ir.roles_of(name) if r.kind == ("unembed" if writes else "embed")), None
|
|
221
|
+
)
|
|
222
|
+
vocab_dim = 0 if role is None or role.kind == "embed" else (role.out_dim or 0)
|
|
223
|
+
table = table if vocab_dim == 0 else table.T
|
|
224
|
+
if vectors.shape[1] != table.shape[1]:
|
|
225
|
+
return None, None
|
|
226
|
+
if writes:
|
|
227
|
+
scale = env.final_scale()
|
|
228
|
+
if scale is not None and scale.numel() == vectors.shape[1]:
|
|
229
|
+
vectors = vectors * scale.to(vectors.device)
|
|
230
|
+
chunk = max(1, LOGIT_BUDGET // max(table.shape[0], 1))
|
|
231
|
+
ids: list[list[int]] = []
|
|
232
|
+
scores: list[list[float]] = []
|
|
233
|
+
table = table.to(vectors.device)
|
|
234
|
+
for start in range(0, vectors.shape[0], chunk):
|
|
235
|
+
logits = vectors[start : start + chunk] @ table.T
|
|
236
|
+
top = torch.topk(logits, k=min(VOCAB_K, logits.shape[1]), dim=1)
|
|
237
|
+
ids.extend(top.indices.tolist())
|
|
238
|
+
scores.extend(top.values.tolist())
|
|
239
|
+
return ids, scores
|
wspx/lab/ir/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
"""The Model IR: architecture plus weights compiled into one inspectable object (§5.1)."""
|
wspx/lab/ir/capture.py
ADDED
|
@@ -0,0 +1,299 @@
|
|
|
1
|
+
"""Build a Model IR from a live module: profiling passes, coverage and semantics (§5.1.1).
|
|
2
|
+
|
|
3
|
+
Language models get a prefill pass (batch 2, 16 tokens, one row left-padded)
|
|
4
|
+
and a decode step against the KV cache, so ops that differ between the two
|
|
5
|
+
phases are both recorded. MoE routers are forced to round-robin assignments,
|
|
6
|
+
with extra passes until every expert row has been read. Any parameter that
|
|
7
|
+
never appears in a graph is reported as unexercised and gets no guessed role.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
import hashlib
|
|
13
|
+
from collections.abc import Mapping
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import torch
|
|
17
|
+
|
|
18
|
+
from wspx.errors import ModelLoadError
|
|
19
|
+
from wspx.lab.ir.graph import OpGraph
|
|
20
|
+
from wspx.lab.ir.model_ir import Coverage, ModelIR, ParamInfo
|
|
21
|
+
from wspx.lab.ir.semantics import RecurrentLayout, derive
|
|
22
|
+
from wspx.lab.ir.tape import Tape, _iter_tensors
|
|
23
|
+
from wspx.lab.tensors import dtype_name, tensor_hash
|
|
24
|
+
|
|
25
|
+
PREFILL_TOKENS = 16
|
|
26
|
+
BATCH = 2
|
|
27
|
+
LEFT_PAD = 3
|
|
28
|
+
MAX_ROUTING_PASSES = 64
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def build_ir(
|
|
32
|
+
model: torch.nn.Module,
|
|
33
|
+
*,
|
|
34
|
+
example_inputs: Any = None,
|
|
35
|
+
fingerprint: str | None = None,
|
|
36
|
+
param_hashes: Mapping[str, str] | None = None,
|
|
37
|
+
force_routing: bool = True,
|
|
38
|
+
seed: int = 0,
|
|
39
|
+
hash_params: bool = True,
|
|
40
|
+
) -> ModelIR:
|
|
41
|
+
"""Capture ``model`` and derive its IR.
|
|
42
|
+
|
|
43
|
+
``example_inputs`` (a tuple of positional args or a dict of keyword args) is
|
|
44
|
+
required for models that are neither causal LMs nor Hugging Face vision models.
|
|
45
|
+
"""
|
|
46
|
+
model.eval()
|
|
47
|
+
params, canonical = _param_table(model)
|
|
48
|
+
recurrent = _recurrent_layouts(model, canonical)
|
|
49
|
+
is_lm = _is_causal_lm(model)
|
|
50
|
+
graphs: dict[str, OpGraph] = {}
|
|
51
|
+
rows: dict[str, set[int]] = {}
|
|
52
|
+
full: set[str] = set()
|
|
53
|
+
passes = 0
|
|
54
|
+
gen = torch.Generator().manual_seed(seed)
|
|
55
|
+
device = _device_of(model)
|
|
56
|
+
|
|
57
|
+
def absorb(tape: Tape) -> None:
|
|
58
|
+
for name, seen in tape.rows_read.items():
|
|
59
|
+
rows.setdefault(name, set()).update(seen)
|
|
60
|
+
full.update(tape.full_read)
|
|
61
|
+
|
|
62
|
+
with torch.no_grad():
|
|
63
|
+
if is_lm:
|
|
64
|
+
vocab = int(getattr(model, "config").vocab_size) # noqa: B009
|
|
65
|
+
ids = torch.randint(0, vocab, (BATCH, PREFILL_TOKENS), generator=gen).to(device)
|
|
66
|
+
mask = torch.ones(BATCH, PREFILL_TOKENS, dtype=torch.long, device=device)
|
|
67
|
+
mask[1, :LEFT_PAD] = 0
|
|
68
|
+
tape = Tape(model, "prefill", force_routing=force_routing)
|
|
69
|
+
_generate_two(model, tape, ids, mask)
|
|
70
|
+
graphs.update(tape.graphs)
|
|
71
|
+
absorb(tape)
|
|
72
|
+
routers = dict(tape.routers)
|
|
73
|
+
tape.release()
|
|
74
|
+
passes += len(tape.graphs)
|
|
75
|
+
|
|
76
|
+
# Extra prefill passes rotate the forced assignment until every expert is hit.
|
|
77
|
+
if force_routing and routers:
|
|
78
|
+
step = min(r["slots"] for r in routers.values())
|
|
79
|
+
for i in range(1, MAX_ROUTING_PASSES):
|
|
80
|
+
if not _row_deficit(rows, full, params):
|
|
81
|
+
break
|
|
82
|
+
tape = Tape(
|
|
83
|
+
model,
|
|
84
|
+
"routing",
|
|
85
|
+
force_routing=True,
|
|
86
|
+
route_offset=i * step,
|
|
87
|
+
track_features=False,
|
|
88
|
+
)
|
|
89
|
+
with tape.recording():
|
|
90
|
+
model(input_ids=ids, attention_mask=mask, use_cache=False)
|
|
91
|
+
absorb(tape)
|
|
92
|
+
tape.release()
|
|
93
|
+
passes += 1
|
|
94
|
+
else:
|
|
95
|
+
inputs = (
|
|
96
|
+
example_inputs
|
|
97
|
+
if example_inputs is not None
|
|
98
|
+
else _default_inputs(model, gen, device)
|
|
99
|
+
)
|
|
100
|
+
tape = Tape(model, "forward", force_routing=force_routing)
|
|
101
|
+
with tape.recording():
|
|
102
|
+
out = _call(model, inputs)
|
|
103
|
+
tape.mark_outputs(_primary_output(out))
|
|
104
|
+
graphs["forward"] = tape.graph
|
|
105
|
+
absorb(tape)
|
|
106
|
+
tape.release()
|
|
107
|
+
passes += 1
|
|
108
|
+
|
|
109
|
+
main = graphs.get("prefill") or graphs["forward"]
|
|
110
|
+
shapes = {name: info.shape for name, info in params.items()}
|
|
111
|
+
module_of = {name: info.module for name, info in params.items()}
|
|
112
|
+
semantics = derive(main, shapes, module_of=module_of, recurrent=recurrent)
|
|
113
|
+
|
|
114
|
+
coverage: dict[str, Coverage] = {}
|
|
115
|
+
for name, info in params.items():
|
|
116
|
+
phases = [phase for phase, g in graphs.items() if name in g.params_read()]
|
|
117
|
+
read = bool(phases) or name in full or name in rows
|
|
118
|
+
cov = Coverage(exercised=read, phases=phases)
|
|
119
|
+
if name in rows and name not in full and info.shape:
|
|
120
|
+
cov.rows_seen = len(rows[name])
|
|
121
|
+
cov.rows_total = info.shape[0]
|
|
122
|
+
cov.exercised = read and cov.rows_seen == cov.rows_total
|
|
123
|
+
coverage[name] = cov
|
|
124
|
+
|
|
125
|
+
if param_hashes is not None:
|
|
126
|
+
hashes = dict(param_hashes)
|
|
127
|
+
elif hash_params:
|
|
128
|
+
hashes = {name: tensor_hash(p) for name, p in model.named_parameters()}
|
|
129
|
+
else:
|
|
130
|
+
hashes = {} # computed lazily, before a tensor's first edit
|
|
131
|
+
if fingerprint is None:
|
|
132
|
+
raise ModelLoadError("a fingerprint is required when parameters are not hashed")
|
|
133
|
+
for name, info in params.items():
|
|
134
|
+
info.hash = hashes.get(name)
|
|
135
|
+
if fingerprint is None:
|
|
136
|
+
digest = hashlib.sha256()
|
|
137
|
+
digest.update(type(model).__qualname__.encode())
|
|
138
|
+
for name in sorted(hashes):
|
|
139
|
+
digest.update(f"{name}={hashes[name]}\n".encode())
|
|
140
|
+
fingerprint = "fp:" + digest.hexdigest()
|
|
141
|
+
|
|
142
|
+
config = getattr(model, "config", None)
|
|
143
|
+
labels: dict[int, str] = {}
|
|
144
|
+
id2label = getattr(config, "id2label", None) if config is not None and not is_lm else None
|
|
145
|
+
if isinstance(id2label, dict) and not all(
|
|
146
|
+
str(v).startswith("LABEL_") for v in id2label.values()
|
|
147
|
+
):
|
|
148
|
+
labels = {int(k): str(v) for k, v in id2label.items()}
|
|
149
|
+
|
|
150
|
+
return ModelIR(
|
|
151
|
+
fingerprint=fingerprint,
|
|
152
|
+
arch=type(model).__name__,
|
|
153
|
+
params=params,
|
|
154
|
+
semantics=semantics,
|
|
155
|
+
graphs=graphs,
|
|
156
|
+
coverage=coverage,
|
|
157
|
+
modules={path: type(m).__name__ for path, m in model.named_modules()},
|
|
158
|
+
is_lm=is_lm,
|
|
159
|
+
vocab_size=int(config.vocab_size) if is_lm and config is not None else None,
|
|
160
|
+
labels=labels,
|
|
161
|
+
capture={"method": "tape", "passes": passes, "batch": BATCH, "tokens": PREFILL_TOKENS},
|
|
162
|
+
)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _generate_two(
|
|
166
|
+
model: torch.nn.Module, tape: Tape, ids: torch.Tensor, mask: torch.Tensor
|
|
167
|
+
) -> None:
|
|
168
|
+
"""Prefill plus one decode step, through the model's own generation code.
|
|
169
|
+
|
|
170
|
+
``generate`` knows every cache type (KV, sliding window, SSM state), so this
|
|
171
|
+
exercises the real decode path. Glue between forward calls is not recorded.
|
|
172
|
+
"""
|
|
173
|
+
calls = 0
|
|
174
|
+
|
|
175
|
+
def pre(_module: torch.nn.Module, _args: Any, _kwargs: Any) -> None:
|
|
176
|
+
nonlocal calls
|
|
177
|
+
if calls == 1:
|
|
178
|
+
tape.start_phase("decode")
|
|
179
|
+
tape.active = calls < 2
|
|
180
|
+
calls += 1
|
|
181
|
+
|
|
182
|
+
def post(_module: torch.nn.Module, _args: Any, _kwargs: Any, out: Any) -> None:
|
|
183
|
+
if tape.active:
|
|
184
|
+
tape.mark_outputs(_logits(out))
|
|
185
|
+
tape.active = False
|
|
186
|
+
|
|
187
|
+
h_pre = model.register_forward_pre_hook(pre, with_kwargs=True)
|
|
188
|
+
h_post = model.register_forward_hook(post, with_kwargs=True)
|
|
189
|
+
config = getattr(model, "generation_config", None)
|
|
190
|
+
pad = getattr(config, "pad_token_id", None)
|
|
191
|
+
if pad is None:
|
|
192
|
+
pad = 0
|
|
193
|
+
tape.active = False # only forward calls are recorded
|
|
194
|
+
try:
|
|
195
|
+
with tape.recording():
|
|
196
|
+
model.generate( # type: ignore[operator]
|
|
197
|
+
input_ids=ids,
|
|
198
|
+
attention_mask=mask,
|
|
199
|
+
max_new_tokens=2,
|
|
200
|
+
min_new_tokens=2,
|
|
201
|
+
do_sample=False,
|
|
202
|
+
pad_token_id=pad,
|
|
203
|
+
use_cache=True,
|
|
204
|
+
)
|
|
205
|
+
finally:
|
|
206
|
+
h_pre.remove()
|
|
207
|
+
h_post.remove()
|
|
208
|
+
tape.active = True
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _param_table(model: torch.nn.Module) -> tuple[dict[str, ParamInfo], dict[int, str]]:
|
|
212
|
+
owners = {}
|
|
213
|
+
for path, module in model.named_modules(remove_duplicate=False):
|
|
214
|
+
for leaf, _ in module.named_parameters(recurse=False, remove_duplicate=False):
|
|
215
|
+
owners[f"{path}.{leaf}" if path else leaf] = (path, type(module).__name__)
|
|
216
|
+
params: dict[str, ParamInfo] = {}
|
|
217
|
+
canonical: dict[int, str] = {}
|
|
218
|
+
for name, param in model.named_parameters(remove_duplicate=False):
|
|
219
|
+
key = id(param)
|
|
220
|
+
if key in canonical:
|
|
221
|
+
params[canonical[key]].aliases.append(name)
|
|
222
|
+
continue
|
|
223
|
+
canonical[key] = name
|
|
224
|
+
module, cls = owners.get(name, (name.rpartition(".")[0], ""))
|
|
225
|
+
params[name] = ParamInfo(name, tuple(param.shape), dtype_name(param.dtype), module, cls)
|
|
226
|
+
return params, canonical
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def _recurrent_layouts(
|
|
230
|
+
model: torch.nn.Module, canonical: dict[int, str]
|
|
231
|
+
) -> dict[str, RecurrentLayout]:
|
|
232
|
+
out: dict[str, RecurrentLayout] = {}
|
|
233
|
+
for path, module in model.named_modules():
|
|
234
|
+
if isinstance(module, torch.nn.RNNBase):
|
|
235
|
+
flat = [canonical[id(getattr(module, n))] for n in module._flat_weights_names]
|
|
236
|
+
out[path] = RecurrentLayout(module.mode, module.num_layers, module.bidirectional, flat)
|
|
237
|
+
return out
|
|
238
|
+
|
|
239
|
+
|
|
240
|
+
def _row_deficit(rows: dict[str, set[int]], full: set[str], params: dict[str, ParamInfo]) -> bool:
|
|
241
|
+
return any(
|
|
242
|
+
name not in full and len(seen) < params[name].shape[0]
|
|
243
|
+
for name, seen in rows.items()
|
|
244
|
+
if name in params and params[name].shape
|
|
245
|
+
)
|
|
246
|
+
|
|
247
|
+
|
|
248
|
+
def _is_causal_lm(model: torch.nn.Module) -> bool:
|
|
249
|
+
try:
|
|
250
|
+
from transformers import GenerationMixin
|
|
251
|
+
except ImportError: # pragma: no cover - transformers is a lab dependency
|
|
252
|
+
return False
|
|
253
|
+
config = getattr(model, "config", None)
|
|
254
|
+
return (
|
|
255
|
+
isinstance(model, GenerationMixin)
|
|
256
|
+
and config is not None
|
|
257
|
+
and not getattr(config, "is_encoder_decoder", False)
|
|
258
|
+
and getattr(config, "vocab_size", None) is not None
|
|
259
|
+
)
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def _default_inputs(model: torch.nn.Module, gen: torch.Generator, device: torch.device) -> Any:
|
|
263
|
+
config = getattr(model, "config", None)
|
|
264
|
+
channels = getattr(config, "num_channels", None)
|
|
265
|
+
if channels:
|
|
266
|
+
size = getattr(config, "image_size", None) or 32
|
|
267
|
+
size = size if isinstance(size, int) else size[0]
|
|
268
|
+
return {"pixel_values": torch.randn(BATCH, channels, size, size, generator=gen).to(device)}
|
|
269
|
+
raise ModelLoadError(
|
|
270
|
+
f"{type(model).__name__}: pass example inputs for profiling (a tuple of args or a dict)"
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _call(model: torch.nn.Module, inputs: Any) -> Any:
|
|
275
|
+
if isinstance(inputs, dict):
|
|
276
|
+
return model(**inputs)
|
|
277
|
+
if isinstance(inputs, tuple | list):
|
|
278
|
+
return model(*inputs)
|
|
279
|
+
return model(inputs)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _logits(out: Any) -> Any:
|
|
283
|
+
return getattr(out, "logits", out)
|
|
284
|
+
|
|
285
|
+
|
|
286
|
+
def _primary_output(out: Any) -> Any:
|
|
287
|
+
"""The model's main output: ``logits`` if present, else the first tensor."""
|
|
288
|
+
logits = getattr(out, "logits", None)
|
|
289
|
+
if isinstance(logits, torch.Tensor):
|
|
290
|
+
return logits
|
|
291
|
+
for tensor in _iter_tensors(out):
|
|
292
|
+
return tensor
|
|
293
|
+
return out
|
|
294
|
+
|
|
295
|
+
|
|
296
|
+
def _device_of(model: torch.nn.Module) -> torch.device:
|
|
297
|
+
for param in model.parameters():
|
|
298
|
+
return param.device
|
|
299
|
+
return torch.device("cpu")
|
wspx/lab/ir/graph.py
ADDED
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
"""The op graph: a framework-neutral record of one profiling pass.
|
|
2
|
+
|
|
3
|
+
Each node is one executed ATen op, tagged with the innermost module that ran
|
|
4
|
+
it. Activation tensors become numbered values; parameters never do. A node
|
|
5
|
+
lists the parameters it read (directly, through views, or through tensors
|
|
6
|
+
computed only from parameters) together with which parameter dims it
|
|
7
|
+
contracted, so every weight's wiring into every op is explicit.
|
|
8
|
+
"""
|
|
9
|
+
|
|
10
|
+
from __future__ import annotations
|
|
11
|
+
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from dataclasses import asdict, dataclass, field
|
|
14
|
+
from functools import cached_property
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
INPUT = -1
|
|
18
|
+
"""Producer id of a value that entered the graph from outside (input ids, masks)."""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
@dataclass(slots=True)
|
|
22
|
+
class ParamUse:
|
|
23
|
+
name: str
|
|
24
|
+
arg: int
|
|
25
|
+
"""Position among the node's flattened tensor arguments."""
|
|
26
|
+
rows: list[int] | None = None
|
|
27
|
+
"""Leading-dim indices read (expert ids for 3-D expert weights); ``None`` = whole tensor."""
|
|
28
|
+
in_dim: int | None = None
|
|
29
|
+
"""Parameter dim contracted against the activation (matmul-like ops only)."""
|
|
30
|
+
out_dim: int | None = None
|
|
31
|
+
"""Parameter dim that becomes the op's output features."""
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
@dataclass(slots=True)
|
|
35
|
+
class Node:
|
|
36
|
+
idx: int
|
|
37
|
+
op: str
|
|
38
|
+
module: str
|
|
39
|
+
args: list[int]
|
|
40
|
+
"""Value id per flattened tensor argument; ``-1`` for parameters, buffers and constants."""
|
|
41
|
+
params: list[ParamUse]
|
|
42
|
+
outs: list[int]
|
|
43
|
+
meta: dict[str, Any] = field(default_factory=dict)
|
|
44
|
+
|
|
45
|
+
@property
|
|
46
|
+
def short_op(self) -> str:
|
|
47
|
+
"""``aten.mm.default`` → ``mm``."""
|
|
48
|
+
parts = self.op.split(".")
|
|
49
|
+
return parts[1] if len(parts) > 1 else parts[0]
|
|
50
|
+
|
|
51
|
+
@property
|
|
52
|
+
def activation_args(self) -> list[tuple[int, int]]:
|
|
53
|
+
"""``(position, value id)`` for each activation argument."""
|
|
54
|
+
return [(pos, vid) for pos, vid in enumerate(self.args) if vid >= 0]
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
@dataclass(slots=True)
|
|
58
|
+
class Value:
|
|
59
|
+
id: int
|
|
60
|
+
shape: tuple[int, ...]
|
|
61
|
+
dtype: str
|
|
62
|
+
producer: int
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
@dataclass
|
|
66
|
+
class OpGraph:
|
|
67
|
+
phase: str
|
|
68
|
+
nodes: list[Node] = field(default_factory=list)
|
|
69
|
+
values: dict[int, Value] = field(default_factory=dict)
|
|
70
|
+
inputs: list[int] = field(default_factory=list)
|
|
71
|
+
outputs: list[int] = field(default_factory=list)
|
|
72
|
+
|
|
73
|
+
@cached_property
|
|
74
|
+
def consumers(self) -> dict[int, list[tuple[int, int]]]:
|
|
75
|
+
"""value id → ``[(node idx, arg position), …]`` in execution order."""
|
|
76
|
+
out: dict[int, list[tuple[int, int]]] = defaultdict(list)
|
|
77
|
+
for node in self.nodes:
|
|
78
|
+
for pos, vid in node.activation_args:
|
|
79
|
+
out[vid].append((node.idx, pos))
|
|
80
|
+
return dict(out)
|
|
81
|
+
|
|
82
|
+
def producer(self, vid: int) -> Node | None:
|
|
83
|
+
value = self.values.get(vid)
|
|
84
|
+
if value is None or value.producer < 0:
|
|
85
|
+
return None
|
|
86
|
+
return self.nodes[value.producer]
|
|
87
|
+
|
|
88
|
+
def params_read(self) -> set[str]:
|
|
89
|
+
return {use.name for node in self.nodes for use in node.params}
|
|
90
|
+
|
|
91
|
+
def to_dict(self) -> dict[str, Any]:
|
|
92
|
+
return {
|
|
93
|
+
"phase": self.phase,
|
|
94
|
+
"nodes": [asdict(n) for n in self.nodes],
|
|
95
|
+
"values": [asdict(v) for v in self.values.values()],
|
|
96
|
+
"inputs": self.inputs,
|
|
97
|
+
"outputs": self.outputs,
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
@classmethod
|
|
101
|
+
def from_dict(cls, data: dict[str, Any]) -> OpGraph:
|
|
102
|
+
nodes = [
|
|
103
|
+
Node(
|
|
104
|
+
idx=n["idx"],
|
|
105
|
+
op=n["op"],
|
|
106
|
+
module=n["module"],
|
|
107
|
+
args=list(n["args"]),
|
|
108
|
+
params=[ParamUse(**p) for p in n["params"]],
|
|
109
|
+
outs=list(n["outs"]),
|
|
110
|
+
meta=n.get("meta", {}),
|
|
111
|
+
)
|
|
112
|
+
for n in data["nodes"]
|
|
113
|
+
]
|
|
114
|
+
values = {
|
|
115
|
+
v["id"]: Value(v["id"], tuple(v["shape"]), v["dtype"], v["producer"])
|
|
116
|
+
for v in data["values"]
|
|
117
|
+
}
|
|
118
|
+
return cls(data["phase"], nodes, values, list(data["inputs"]), list(data["outputs"]))
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
def compress_indices(indices: list[int]) -> list[list[int]]:
|
|
122
|
+
"""Encode sorted unique ints as arithmetic runs ``[start, stop, step]``.
|
|
123
|
+
|
|
124
|
+
``[0..63]`` → ``[[0, 64, 1]]``; ``[0, 2, 4, …]`` → ``[[0, 2n, 2]]``.
|
|
125
|
+
"""
|
|
126
|
+
runs: list[list[int]] = []
|
|
127
|
+
i, n = 0, len(indices)
|
|
128
|
+
while i < n:
|
|
129
|
+
start = indices[i]
|
|
130
|
+
if i + 1 == n:
|
|
131
|
+
runs.append([start, start + 1, 1])
|
|
132
|
+
break
|
|
133
|
+
step = indices[i + 1] - start
|
|
134
|
+
j = i + 1
|
|
135
|
+
while j + 1 < n and indices[j + 1] - indices[j] == step:
|
|
136
|
+
j += 1
|
|
137
|
+
if j == i + 1 and i + 2 < n and step != 1:
|
|
138
|
+
# A lone pair with an odd gap is better as two singletons than a run.
|
|
139
|
+
runs.append([start, start + 1, 1])
|
|
140
|
+
i += 1
|
|
141
|
+
continue
|
|
142
|
+
runs.append([start, indices[j] + 1, step])
|
|
143
|
+
i = j + 1
|
|
144
|
+
return runs
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def expand_runs(runs: list[list[int]]) -> list[int]:
|
|
148
|
+
return [i for start, stop, step in runs for i in range(start, stop, step)]
|