wspx 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wspx/__init__.py +24 -0
- wspx/_version.py +1 -0
- wspx/addressing.py +415 -0
- wspx/auth.py +77 -0
- wspx/cli/__init__.py +1 -0
- wspx/cli/client.py +124 -0
- wspx/cli/config.py +89 -0
- wspx/cli/connect.py +88 -0
- wspx/cli/main.py +848 -0
- wspx/cloud/__init__.py +20 -0
- wspx/cloud/app.py +191 -0
- wspx/cloud/backend.py +141 -0
- wspx/cloud/compose.py +85 -0
- wspx/cloud/deploy.py +102 -0
- wspx/config.py +189 -0
- wspx/control/__init__.py +5 -0
- wspx/control/app.py +488 -0
- wspx/control/backend.py +244 -0
- wspx/control/gateway_client.py +79 -0
- wspx/control/mcp_server.py +71 -0
- wspx/control/sessions.py +424 -0
- wspx/control/tracedb.py +183 -0
- wspx/control/ui/assets/index-BJFweZAn.css +1 -0
- wspx/control/ui/assets/index-DhUVhWIK.js +10 -0
- wspx/control/ui/index.html +13 -0
- wspx/control/views.py +180 -0
- wspx/diagnostics.py +146 -0
- wspx/driver/__init__.py +7 -0
- wspx/driver/loop.py +255 -0
- wspx/driver/messages.py +62 -0
- wspx/driver/providers/__init__.py +84 -0
- wspx/driver/providers/anthropic.py +149 -0
- wspx/driver/providers/compat.py +115 -0
- wspx/driver/providers/google.py +138 -0
- wspx/driver/providers/openai.py +125 -0
- wspx/driver/run.py +134 -0
- wspx/errors.py +69 -0
- wspx/gateway/__init__.py +8 -0
- wspx/gateway/app.py +626 -0
- wspx/gateway/ledger.py +536 -0
- wspx/gateway/prices.py +75 -0
- wspx/gateway/usage.py +284 -0
- wspx/handles.py +117 -0
- wspx/integrations/AGENTS.snippet.md +8 -0
- wspx/integrations/SKILL.md +52 -0
- wspx/integrations/__init__.py +17 -0
- wspx/integrations/claude-plugin/.claude-plugin/plugin.json +30 -0
- wspx/integrations/claude-plugin/skills/wspx/SKILL.md +52 -0
- wspx/integrations/codex-hooks.toml +9 -0
- wspx/lab/__init__.py +5 -0
- wspx/lab/analysis.py +485 -0
- wspx/lab/check.py +489 -0
- wspx/lab/evaluate.py +108 -0
- wspx/lab/export.py +275 -0
- wspx/lab/grep.py +754 -0
- wspx/lab/index.py +239 -0
- wspx/lab/ir/__init__.py +1 -0
- wspx/lab/ir/capture.py +299 -0
- wspx/lab/ir/graph.py +148 -0
- wspx/lab/ir/model_ir.py +522 -0
- wspx/lab/ir/semantics.py +1255 -0
- wspx/lab/ir/tape.py +853 -0
- wspx/lab/loader.py +204 -0
- wspx/lab/merge.py +200 -0
- wspx/lab/ops/__init__.py +76 -0
- wspx/lab/ops/builtin.py +379 -0
- wspx/lab/patch.py +320 -0
- wspx/lab/probe.py +206 -0
- wspx/lab/results.py +114 -0
- wspx/lab/sdk.py +120 -0
- wspx/lab/server.py +236 -0
- wspx/lab/session.py +1574 -0
- wspx/lab/tensors.py +119 -0
- wspx/lab/text.py +69 -0
- wspx/lab/tree.py +302 -0
- wspx/loops.py +127 -0
- wspx/project.py +143 -0
- wspx/py.typed +0 -0
- wspx/safety/__init__.py +15 -0
- wspx/safety/kill.py +47 -0
- wspx/safety/sweeper.py +68 -0
- wspx/store/__init__.py +64 -0
- wspx/store/objects.py +130 -0
- wspx/store/overlay.py +209 -0
- wspx/store/refs.py +150 -0
- wspx/testing/__init__.py +21 -0
- wspx/testing/api.py +155 -0
- wspx/testing/builtin.py +139 -0
- wspx/testing/runner.py +257 -0
- wspx/tools/__init__.py +17 -0
- wspx/tools/dispatch.py +129 -0
- wspx/tools/schemas.py +313 -0
- wspx/vcs/__init__.py +17 -0
- wspx/vcs/repo.py +431 -0
- wspx-0.1.0.dev0.dist-info/METADATA +278 -0
- wspx-0.1.0.dev0.dist-info/RECORD +99 -0
- wspx-0.1.0.dev0.dist-info/WHEEL +4 -0
- wspx-0.1.0.dev0.dist-info/entry_points.txt +10 -0
- wspx-0.1.0.dev0.dist-info/licenses/LICENSE +201 -0
wspx/lab/probe.py
ADDED
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
"""Activation capture at exact points from the IR (§5.8.1).
|
|
2
|
+
|
|
3
|
+
A *point* is the k-th occurrence of an op inside a module during a forward
|
|
4
|
+
pass, read off the captured graph. The residual stream after block L is the
|
|
5
|
+
output of that block's last residual add; attention inputs are the arguments
|
|
6
|
+
of the attention op; routes are the outputs of the router's ``topk``. Writer
|
|
7
|
+
unit activations are the input of the module that owns the writer weight.
|
|
8
|
+
|
|
9
|
+
Points are recorded by a light dispatch mode; nothing is installed on the
|
|
10
|
+
model outside a ``with probe.capture()`` block, so the default forward stays
|
|
11
|
+
hook-free.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
from collections import defaultdict
|
|
17
|
+
from collections.abc import Iterator
|
|
18
|
+
from contextlib import contextmanager
|
|
19
|
+
from dataclasses import dataclass, field
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
import torch
|
|
23
|
+
from torch.utils._python_dispatch import TorchDispatchMode
|
|
24
|
+
|
|
25
|
+
from wspx.errors import WspxError
|
|
26
|
+
from wspx.lab.ir.model_ir import ModelIR
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
@dataclass(frozen=True)
|
|
30
|
+
class Point:
|
|
31
|
+
module: str
|
|
32
|
+
op: str
|
|
33
|
+
occurrence: int
|
|
34
|
+
which: str = "out"
|
|
35
|
+
"""``out`` (first output), ``args`` (all tensor args) or ``outs`` (all outputs)."""
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def points_from_ir(ir: ModelIR) -> dict[str, Point]:
|
|
39
|
+
"""Named capture points: ``resid.L{n}``, ``attn.L{n}`` and ``route.L{n}``."""
|
|
40
|
+
graph = ir.graphs.get("prefill") or ir.graphs.get("forward")
|
|
41
|
+
if graph is None:
|
|
42
|
+
return {}
|
|
43
|
+
seen: dict[tuple[str, str], int] = defaultdict(int)
|
|
44
|
+
occurrence: dict[int, int] = {}
|
|
45
|
+
for node in graph.nodes:
|
|
46
|
+
key = (node.module, node.op)
|
|
47
|
+
occurrence[node.idx] = seen[key]
|
|
48
|
+
seen[key] += 1
|
|
49
|
+
out: dict[str, Point] = {}
|
|
50
|
+
for block in ir.semantics.blocks:
|
|
51
|
+
last = graph.nodes[block.residual_adds[-1]]
|
|
52
|
+
out[f"resid.L{block.index}"] = Point(last.module, last.op, occurrence[last.idx])
|
|
53
|
+
stream = ir.semantics.stream
|
|
54
|
+
if stream:
|
|
55
|
+
producer = graph.values[stream[0]].producer
|
|
56
|
+
if producer >= 0:
|
|
57
|
+
first = graph.nodes[producer]
|
|
58
|
+
out["resid.embed"] = Point(first.module, first.op, occurrence[first.idx])
|
|
59
|
+
for block in ir.semantics.blocks:
|
|
60
|
+
lo = block.residual_adds[0]
|
|
61
|
+
hi = block.residual_adds[-1]
|
|
62
|
+
prev_hi = max(
|
|
63
|
+
(b.residual_adds[-1] for b in ir.semantics.blocks if b.index == block.index - 1),
|
|
64
|
+
default=-1,
|
|
65
|
+
)
|
|
66
|
+
for node in graph.nodes[prev_hi + 1 : hi + 1]:
|
|
67
|
+
op = node.short_op
|
|
68
|
+
if "scaled_dot_product" in op and f"attn.L{block.index}" not in out:
|
|
69
|
+
out[f"attn.L{block.index}"] = Point(
|
|
70
|
+
node.module, node.op, occurrence[node.idx], "args"
|
|
71
|
+
)
|
|
72
|
+
if node.meta.get("forced_routing") and f"route.L{block.index}" not in out:
|
|
73
|
+
out[f"route.L{block.index}"] = Point(
|
|
74
|
+
node.module, node.op, occurrence[node.idx], "outs"
|
|
75
|
+
)
|
|
76
|
+
del lo
|
|
77
|
+
return out
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
class _Recorder(TorchDispatchMode): # type: ignore[no-untyped-call]
|
|
81
|
+
def __init__(
|
|
82
|
+
self, wanted: dict[tuple[str, str], dict[int, list[str]]], stack: list[str]
|
|
83
|
+
) -> None:
|
|
84
|
+
super().__init__() # type: ignore[no-untyped-call]
|
|
85
|
+
self.wanted = wanted
|
|
86
|
+
self.stack = stack
|
|
87
|
+
self.counts: dict[tuple[str, str], int] = defaultdict(int)
|
|
88
|
+
self.found: dict[str, Any] = {}
|
|
89
|
+
self.which: dict[str, str] = {}
|
|
90
|
+
|
|
91
|
+
def __torch_dispatch__(
|
|
92
|
+
self,
|
|
93
|
+
func: Any,
|
|
94
|
+
types: Any,
|
|
95
|
+
args: tuple[Any, ...] = (),
|
|
96
|
+
kwargs: dict[str, Any] | None = None,
|
|
97
|
+
) -> Any:
|
|
98
|
+
out = func(*args, **(kwargs or {}))
|
|
99
|
+
module = self.stack[-1] if self.stack else ""
|
|
100
|
+
key = (module, str(func))
|
|
101
|
+
slots = self.wanted.get(key)
|
|
102
|
+
if slots is not None:
|
|
103
|
+
k = self.counts[key]
|
|
104
|
+
self.counts[key] += 1
|
|
105
|
+
for name in slots.get(k, []):
|
|
106
|
+
which = self.which[name]
|
|
107
|
+
if which == "args":
|
|
108
|
+
self.found.setdefault(name, []).append(
|
|
109
|
+
[a.detach() for a in args if isinstance(a, torch.Tensor)]
|
|
110
|
+
)
|
|
111
|
+
elif which == "outs":
|
|
112
|
+
outs = out if isinstance(out, tuple | list) else (out,)
|
|
113
|
+
self.found.setdefault(name, []).append(
|
|
114
|
+
[o.detach() for o in outs if isinstance(o, torch.Tensor)]
|
|
115
|
+
)
|
|
116
|
+
else:
|
|
117
|
+
first = out[0] if isinstance(out, tuple | list) else out
|
|
118
|
+
self.found.setdefault(name, []).append(first.detach())
|
|
119
|
+
return out
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
@dataclass
|
|
123
|
+
class Capture:
|
|
124
|
+
points: dict[str, Any] = field(default_factory=dict)
|
|
125
|
+
"""Point name → captured tensor(s), first call only unless ``all_calls``."""
|
|
126
|
+
units: dict[str, torch.Tensor] = field(default_factory=dict)
|
|
127
|
+
"""Writer param → input activations of its module (the units)."""
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
class Probe:
|
|
131
|
+
def __init__(self, model: torch.nn.Module, ir: ModelIR) -> None:
|
|
132
|
+
self.model = model
|
|
133
|
+
self.ir = ir
|
|
134
|
+
self.points = points_from_ir(ir)
|
|
135
|
+
|
|
136
|
+
@contextmanager
|
|
137
|
+
def capture(
|
|
138
|
+
self,
|
|
139
|
+
names: list[str] | None = None,
|
|
140
|
+
*,
|
|
141
|
+
units: list[str] | None = None,
|
|
142
|
+
grad_units: bool = False,
|
|
143
|
+
) -> Iterator[Capture]:
|
|
144
|
+
"""Record the named points (default: all residual points) and writer units."""
|
|
145
|
+
names = names if names is not None else [n for n in self.points if n.startswith("resid.")]
|
|
146
|
+
missing = [n for n in names if n not in self.points]
|
|
147
|
+
if missing:
|
|
148
|
+
raise WspxError(
|
|
149
|
+
f"no capture point {missing[0]!r}; known: {', '.join(sorted(self.points)[:6])}…"
|
|
150
|
+
)
|
|
151
|
+
result = Capture()
|
|
152
|
+
stack: list[str] = []
|
|
153
|
+
wanted: dict[tuple[str, str], dict[int, list[str]]] = defaultdict(lambda: defaultdict(list))
|
|
154
|
+
recorder = _Recorder({}, stack)
|
|
155
|
+
for name in names:
|
|
156
|
+
point = self.points[name]
|
|
157
|
+
wanted[(point.module, point.op)][point.occurrence].append(name)
|
|
158
|
+
recorder.which[name] = point.which
|
|
159
|
+
recorder.wanted = {k: dict(v) for k, v in wanted.items()}
|
|
160
|
+
|
|
161
|
+
handles = []
|
|
162
|
+
module_names = {id(m): n for n, m in self.model.named_modules(remove_duplicate=False)}
|
|
163
|
+
|
|
164
|
+
def pre(module: torch.nn.Module, _args: Any) -> None:
|
|
165
|
+
stack.append(module_names.get(id(module), ""))
|
|
166
|
+
|
|
167
|
+
def post(_module: torch.nn.Module, _args: Any, _out: Any) -> None:
|
|
168
|
+
if stack:
|
|
169
|
+
stack.pop()
|
|
170
|
+
|
|
171
|
+
if names:
|
|
172
|
+
for module in self.model.modules():
|
|
173
|
+
handles.append(module.register_forward_pre_hook(pre))
|
|
174
|
+
handles.append(module.register_forward_hook(post, always_call=True))
|
|
175
|
+
for param in units or []:
|
|
176
|
+
info = self.ir.params[param]
|
|
177
|
+
module = self.model.get_submodule(info.module)
|
|
178
|
+
|
|
179
|
+
def grab(_m: torch.nn.Module, args: tuple[Any, ...], key: str = param) -> None:
|
|
180
|
+
x = args[0]
|
|
181
|
+
if grad_units and x.requires_grad:
|
|
182
|
+
x.retain_grad()
|
|
183
|
+
result.units.setdefault(key, x if grad_units else x.detach())
|
|
184
|
+
|
|
185
|
+
handles.append(module.register_forward_pre_hook(grab))
|
|
186
|
+
try:
|
|
187
|
+
if names:
|
|
188
|
+
with recorder:
|
|
189
|
+
yield result
|
|
190
|
+
else:
|
|
191
|
+
yield result
|
|
192
|
+
finally:
|
|
193
|
+
for handle in handles:
|
|
194
|
+
handle.remove()
|
|
195
|
+
for name, values in recorder.found.items():
|
|
196
|
+
result.points[name] = values[0]
|
|
197
|
+
|
|
198
|
+
|
|
199
|
+
def writer_units(ir: ModelIR) -> list[str]:
|
|
200
|
+
"""Writer weights whose module input is the vector of units (2-D projections)."""
|
|
201
|
+
out = [
|
|
202
|
+
role.param
|
|
203
|
+
for role in ir.roles
|
|
204
|
+
if role.kind == "writer" and role.split is None and len(ir.params[role.param].shape) == 2
|
|
205
|
+
]
|
|
206
|
+
return list(dict.fromkeys(out))
|
wspx/lab/results.py
ADDED
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
"""Result sets: the full ranked list of a grep, persisted as Parquet, plus small views.
|
|
2
|
+
|
|
3
|
+
The driver sees the top hits; the store keeps every score. The UI gets
|
|
4
|
+
server-side aggregates (a histogram, per-block maxima, a downsampled heatmap)
|
|
5
|
+
as a few KB of JSON. Parquet never goes to the browser.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import io
|
|
11
|
+
import math
|
|
12
|
+
from collections import defaultdict
|
|
13
|
+
from pathlib import Path
|
|
14
|
+
from typing import Any
|
|
15
|
+
|
|
16
|
+
import pyarrow as pa
|
|
17
|
+
import pyarrow.parquet as pq
|
|
18
|
+
|
|
19
|
+
from wspx.lab.grep import GrepResult, Hit
|
|
20
|
+
|
|
21
|
+
SCHEMA = pa.schema(
|
|
22
|
+
[
|
|
23
|
+
("rank", pa.int64()),
|
|
24
|
+
("address", pa.string()),
|
|
25
|
+
("param", pa.string()),
|
|
26
|
+
("expert", pa.int64()),
|
|
27
|
+
("unit", pa.int64()),
|
|
28
|
+
("score", pa.float64()),
|
|
29
|
+
("hit", pa.bool_()),
|
|
30
|
+
]
|
|
31
|
+
)
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def to_bytes(result: GrepResult, *, top: int | None = None) -> bytes:
|
|
35
|
+
"""Every scored unit as Parquet; ``hit`` marks the matches that were returned (top-k)."""
|
|
36
|
+
returned = {
|
|
37
|
+
(h.param, h.expert, h.unit)
|
|
38
|
+
for h in result.hits[: top if top is not None else len(result.hits)]
|
|
39
|
+
}
|
|
40
|
+
rows = result.scores
|
|
41
|
+
table = pa.table(
|
|
42
|
+
{
|
|
43
|
+
"rank": list(range(len(rows))),
|
|
44
|
+
"address": [h.address for h in rows],
|
|
45
|
+
"param": [h.param for h in rows],
|
|
46
|
+
"expert": [-1 if h.expert is None else h.expert for h in rows],
|
|
47
|
+
"unit": [h.unit for h in rows],
|
|
48
|
+
"score": [h.score for h in rows],
|
|
49
|
+
"hit": [(h.param, h.expert, h.unit) in returned for h in rows],
|
|
50
|
+
},
|
|
51
|
+
schema=SCHEMA,
|
|
52
|
+
)
|
|
53
|
+
buf = io.BytesIO()
|
|
54
|
+
pq.write_table(table, buf, compression="zstd")
|
|
55
|
+
return buf.getvalue()
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def save(result: GrepResult, path: Path, *, top: int | None = None) -> None:
|
|
59
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
60
|
+
path.write_bytes(to_bytes(result, top=top))
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def load_hits(source: Path | bytes, *, only_hits: bool = True) -> list[Hit]:
|
|
64
|
+
data = source if isinstance(source, bytes) else source.read_bytes()
|
|
65
|
+
table = pq.read_table(io.BytesIO(data)).to_pylist()
|
|
66
|
+
return [
|
|
67
|
+
Hit(
|
|
68
|
+
r["address"],
|
|
69
|
+
r["param"],
|
|
70
|
+
None if r["expert"] < 0 else r["expert"],
|
|
71
|
+
r["unit"],
|
|
72
|
+
r["score"],
|
|
73
|
+
)
|
|
74
|
+
for r in table
|
|
75
|
+
if r["hit"] or not only_hits
|
|
76
|
+
]
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def indices(hits: list[Hit], param: str, expert: int | None = None) -> list[int]:
|
|
80
|
+
"""Unit indices of ``param`` (and expert) among ``hits``, in rank order."""
|
|
81
|
+
return [h.unit for h in hits if h.param == param and (expert is None or h.expert == expert)]
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def views(
|
|
85
|
+
result: GrepResult, block_of: dict[str, int | None], *, bins: int = 50, width: int = 64
|
|
86
|
+
) -> dict[str, Any]:
|
|
87
|
+
scores = [h.score for h in result.scores if not math.isnan(h.score)]
|
|
88
|
+
if not scores:
|
|
89
|
+
return {"hist": [], "per_block_max": {}, "heatmap": []}
|
|
90
|
+
lo, hi = min(scores), max(scores)
|
|
91
|
+
span = (hi - lo) or 1.0
|
|
92
|
+
counts = [0] * bins
|
|
93
|
+
for s in scores:
|
|
94
|
+
counts[min(bins - 1, int((s - lo) / span * bins))] += 1
|
|
95
|
+
per_block: dict[str, float] = {}
|
|
96
|
+
rows: dict[str, list[float]] = defaultdict(list)
|
|
97
|
+
for h in result.scores:
|
|
98
|
+
block = block_of.get(h.param)
|
|
99
|
+
key = str(block) if block is not None else h.param
|
|
100
|
+
if not math.isnan(h.score):
|
|
101
|
+
per_block[key] = max(per_block.get(key, -math.inf), h.score)
|
|
102
|
+
rows[key].append(h.score)
|
|
103
|
+
heatmap = []
|
|
104
|
+
for key in sorted(rows, key=lambda k: (not k.isdigit(), int(k) if k.isdigit() else 0, k)):
|
|
105
|
+
values = rows[key]
|
|
106
|
+
step = max(1, math.ceil(len(values) / width))
|
|
107
|
+
heatmap.append(
|
|
108
|
+
{"row": key, "cells": [max(values[i : i + step]) for i in range(0, len(values), step)]}
|
|
109
|
+
)
|
|
110
|
+
return {
|
|
111
|
+
"hist": {"lo": lo, "hi": hi, "counts": counts},
|
|
112
|
+
"per_block_max": per_block,
|
|
113
|
+
"heatmap": heatmap,
|
|
114
|
+
}
|
wspx/lab/sdk.py
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""The Python SDK: ``wspx.session()`` (§4.5).
|
|
2
|
+
|
|
3
|
+
Inside the lab (and inside scripts that ``run`` executes) ``wspx.session()``
|
|
4
|
+
returns the live session. On a machine with the ``lab`` extra installed it can
|
|
5
|
+
also open a local session directly::
|
|
6
|
+
|
|
7
|
+
import wspx
|
|
8
|
+
m = wspx.session("Qwen/Qwen3.5-0.8B", store="./wspx-store")
|
|
9
|
+
hits = m.grep('writes(" Wait") > 0.3', scope="L*.mlp.down", top=50)
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
from typing import Any
|
|
16
|
+
|
|
17
|
+
import torch
|
|
18
|
+
|
|
19
|
+
from wspx.config import WspxConfig
|
|
20
|
+
from wspx.errors import WspxError
|
|
21
|
+
from wspx.lab.ir.capture import build_ir
|
|
22
|
+
from wspx.lab.ir.model_ir import ModelIR
|
|
23
|
+
from wspx.lab.session import Lab
|
|
24
|
+
from wspx.store import Store
|
|
25
|
+
|
|
26
|
+
__all__ = ["Lab", "activate", "open_lab", "session"]
|
|
27
|
+
|
|
28
|
+
_ACTIVE: Lab | None = None
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def activate(lab: Lab | None) -> Lab | None:
|
|
32
|
+
"""Make ``lab`` the session ``wspx.session()`` returns; returns the previous one."""
|
|
33
|
+
global _ACTIVE
|
|
34
|
+
previous, _ACTIVE = _ACTIVE, lab
|
|
35
|
+
return previous
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def session(model: Any = None, **kwargs: Any) -> Lab:
|
|
39
|
+
"""The live session, or a new local one when a model is given."""
|
|
40
|
+
if model is None and not kwargs:
|
|
41
|
+
if _ACTIVE is None:
|
|
42
|
+
raise WspxError("no active wspx session; pass a model to open one locally")
|
|
43
|
+
return _ACTIVE
|
|
44
|
+
lab = open_lab(model, **kwargs)
|
|
45
|
+
activate(lab)
|
|
46
|
+
return lab
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def open_lab(
|
|
50
|
+
model: str | torch.nn.Module,
|
|
51
|
+
*,
|
|
52
|
+
store: str | Path = "wspx-store",
|
|
53
|
+
session: str = "local",
|
|
54
|
+
branch: str | None = None,
|
|
55
|
+
tokenizer: Any = None,
|
|
56
|
+
example_inputs: Any = None,
|
|
57
|
+
config: WspxConfig | None = None,
|
|
58
|
+
revision: str | None = None,
|
|
59
|
+
dtype: str = "bfloat16",
|
|
60
|
+
trust_remote_code: bool = False,
|
|
61
|
+
workdir: str | Path | None = None,
|
|
62
|
+
) -> Lab:
|
|
63
|
+
"""Load (or take) a model, build or reuse its IR from the store, and open a session."""
|
|
64
|
+
st = Store(store)
|
|
65
|
+
base_reader = None
|
|
66
|
+
hashes = None
|
|
67
|
+
fingerprint = None
|
|
68
|
+
if isinstance(model, str):
|
|
69
|
+
from wspx.lab.loader import load_hf
|
|
70
|
+
|
|
71
|
+
loaded = load_hf(model, revision=revision, dtype=dtype, trust_remote_code=trust_remote_code)
|
|
72
|
+
module, tokenizer = loaded.model, tokenizer or loaded.tokenizer
|
|
73
|
+
fingerprint = loaded.fingerprint
|
|
74
|
+
readable = loaded.readable
|
|
75
|
+
if readable:
|
|
76
|
+
base_reader = loaded.base_reader
|
|
77
|
+
hashes = False
|
|
78
|
+
else:
|
|
79
|
+
module = model
|
|
80
|
+
ir = _cached_ir(st, fingerprint)
|
|
81
|
+
if ir is None:
|
|
82
|
+
ir = build_ir(
|
|
83
|
+
module,
|
|
84
|
+
example_inputs=example_inputs,
|
|
85
|
+
fingerprint=fingerprint,
|
|
86
|
+
hash_params=hashes is None,
|
|
87
|
+
)
|
|
88
|
+
_save_ir(st, ir)
|
|
89
|
+
return Lab(
|
|
90
|
+
module,
|
|
91
|
+
ir,
|
|
92
|
+
st,
|
|
93
|
+
session=session,
|
|
94
|
+
branch=branch,
|
|
95
|
+
tokenizer=tokenizer,
|
|
96
|
+
config=config or WspxConfig.discover(workdir),
|
|
97
|
+
base_reader=base_reader,
|
|
98
|
+
workdir=Path(workdir) if workdir else None,
|
|
99
|
+
)
|
|
100
|
+
|
|
101
|
+
|
|
102
|
+
def _ir_path(fingerprint: str) -> str:
|
|
103
|
+
return f"models/{fingerprint.replace(':', '_')}/ir.json"
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _cached_ir(store: Store, fingerprint: str | None) -> ModelIR | None:
|
|
107
|
+
if fingerprint is None:
|
|
108
|
+
return None
|
|
109
|
+
raw = store.read_file(_ir_path(fingerprint))
|
|
110
|
+
if raw is None:
|
|
111
|
+
return None
|
|
112
|
+
import json
|
|
113
|
+
|
|
114
|
+
return ModelIR.from_dict(json.loads(raw))
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _save_ir(store: Store, ir: ModelIR) -> None:
|
|
118
|
+
import json
|
|
119
|
+
|
|
120
|
+
store.write_file(_ir_path(ir.fingerprint), json.dumps(ir.to_dict()).encode())
|
wspx/lab/server.py
ADDED
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
"""The lab server: one session's lab, reachable only through the control plane.
|
|
2
|
+
|
|
3
|
+
It runs inside the GPU sandbox with no secrets and no outbound network. The
|
|
4
|
+
control plane reaches it over an encrypted port with a connect token and
|
|
5
|
+
renews its lease; if renewals stop (the control plane crashed, hung or was
|
|
6
|
+
deleted) the process exits on its own when the lease lapses. Every response
|
|
7
|
+
carries the store journal, which the control plane persists.
|
|
8
|
+
|
|
9
|
+
``python -m wspx.lab.server --model /models/<id> --session s_… --store /store``
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import dataclasses
|
|
16
|
+
import json
|
|
17
|
+
import math
|
|
18
|
+
import os
|
|
19
|
+
import sys
|
|
20
|
+
import threading
|
|
21
|
+
import time
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
from typing import Any
|
|
24
|
+
|
|
25
|
+
from starlette.applications import Starlette
|
|
26
|
+
from starlette.concurrency import run_in_threadpool
|
|
27
|
+
from starlette.requests import Request
|
|
28
|
+
from starlette.responses import JSONResponse, Response
|
|
29
|
+
from starlette.routing import Route
|
|
30
|
+
|
|
31
|
+
from wspx.lab.session import Lab
|
|
32
|
+
from wspx.store import Store
|
|
33
|
+
from wspx.store.overlay import OverlayStore
|
|
34
|
+
from wspx.tools.dispatch import call
|
|
35
|
+
|
|
36
|
+
DEFAULT_LEASE_S = 300.0
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def _clean(value: Any) -> Any:
|
|
40
|
+
"""JSON-safe: NaN and infinities become null; anything else unknown becomes its string."""
|
|
41
|
+
if isinstance(value, float):
|
|
42
|
+
return value if math.isfinite(value) else None
|
|
43
|
+
if isinstance(value, dict):
|
|
44
|
+
return {str(k): _clean(v) for k, v in value.items()}
|
|
45
|
+
if isinstance(value, list | tuple):
|
|
46
|
+
return [_clean(v) for v in value]
|
|
47
|
+
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
|
48
|
+
return _clean(dataclasses.asdict(value))
|
|
49
|
+
return value
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
class _JSON(JSONResponse):
|
|
53
|
+
def render(self, content: Any) -> bytes:
|
|
54
|
+
return json.dumps(
|
|
55
|
+
_clean(content), default=str, allow_nan=False, separators=(",", ":")
|
|
56
|
+
).encode()
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class LabServer:
|
|
60
|
+
def __init__(
|
|
61
|
+
self,
|
|
62
|
+
lab: Lab,
|
|
63
|
+
*,
|
|
64
|
+
lease_s: float = DEFAULT_LEASE_S,
|
|
65
|
+
token: str | None = None,
|
|
66
|
+
exit_on_lapse: bool = True,
|
|
67
|
+
) -> None:
|
|
68
|
+
self.lab = lab
|
|
69
|
+
self.lease_expires = time.time() + lease_s
|
|
70
|
+
self.token = token
|
|
71
|
+
self.exit_on_lapse = exit_on_lapse
|
|
72
|
+
self._lock = threading.Lock()
|
|
73
|
+
self.started = time.time()
|
|
74
|
+
self.calls = 0
|
|
75
|
+
|
|
76
|
+
# ------------------------------------------------------------------ lease
|
|
77
|
+
|
|
78
|
+
def renew(self, lease_s: float) -> None:
|
|
79
|
+
self.lease_expires = max(self.lease_expires, time.time() + lease_s)
|
|
80
|
+
|
|
81
|
+
def watchdog(self, interval: float = 10.0) -> threading.Thread:
|
|
82
|
+
def loop() -> None:
|
|
83
|
+
while True:
|
|
84
|
+
time.sleep(interval)
|
|
85
|
+
if time.time() > self.lease_expires:
|
|
86
|
+
print("wspx lab: lease lapsed; shutting down", file=sys.stderr, flush=True)
|
|
87
|
+
if self.exit_on_lapse:
|
|
88
|
+
os._exit(0)
|
|
89
|
+
return
|
|
90
|
+
|
|
91
|
+
thread = threading.Thread(target=loop, name="wspx-lease", daemon=True)
|
|
92
|
+
thread.start()
|
|
93
|
+
return thread
|
|
94
|
+
|
|
95
|
+
# ------------------------------------------------------------------ handlers
|
|
96
|
+
|
|
97
|
+
def journal(self) -> dict[str, Any]:
|
|
98
|
+
store = self.lab.store
|
|
99
|
+
return store.journal.drain() if isinstance(store, OverlayStore) else {}
|
|
100
|
+
|
|
101
|
+
def workspace(self) -> str:
|
|
102
|
+
lab = self.lab
|
|
103
|
+
head = lab.repo.head(lab.session)
|
|
104
|
+
branch = lab.repo.current_branch(lab.session) or "(detached)"
|
|
105
|
+
lines = [
|
|
106
|
+
f"branch {branch} · HEAD {head[7:14] if head else '(base)'} · {len(lab.staged)} staged"
|
|
107
|
+
]
|
|
108
|
+
lines.append(lab.handles.table(limit=30))
|
|
109
|
+
return "\n".join(lines)
|
|
110
|
+
|
|
111
|
+
def _authorized(self, request: Request) -> bool:
|
|
112
|
+
"""The lab's own token, in its own header (``Authorization`` carries Modal's connect token)."""
|
|
113
|
+
if self.token is None:
|
|
114
|
+
return True
|
|
115
|
+
import hmac
|
|
116
|
+
|
|
117
|
+
return hmac.compare_digest(request.headers.get("x-wspx-lab-token", ""), self.token)
|
|
118
|
+
|
|
119
|
+
def app(self) -> Starlette:
|
|
120
|
+
async def health(_request: Request) -> Response:
|
|
121
|
+
return _JSON(
|
|
122
|
+
{
|
|
123
|
+
"ok": True,
|
|
124
|
+
"session": self.lab.session,
|
|
125
|
+
"arch": self.lab.ir.arch,
|
|
126
|
+
"lease_expires": self.lease_expires,
|
|
127
|
+
"uptime": time.time() - self.started,
|
|
128
|
+
"calls": self.calls,
|
|
129
|
+
}
|
|
130
|
+
)
|
|
131
|
+
|
|
132
|
+
async def heartbeat(request: Request) -> Response:
|
|
133
|
+
if not self._authorized(request):
|
|
134
|
+
return _JSON({"error": "unauthorized"}, status_code=401)
|
|
135
|
+
body = await request.json() if await request.body() else {}
|
|
136
|
+
self.renew(float(body.get("lease_s", DEFAULT_LEASE_S)))
|
|
137
|
+
with self._lock:
|
|
138
|
+
journal = self.journal()
|
|
139
|
+
return _JSON(
|
|
140
|
+
{
|
|
141
|
+
"lease_expires": self.lease_expires,
|
|
142
|
+
"journal": journal,
|
|
143
|
+
"workspace": self.workspace(),
|
|
144
|
+
"fingerprint": self.lab.ir.fingerprint,
|
|
145
|
+
}
|
|
146
|
+
)
|
|
147
|
+
|
|
148
|
+
async def tool(request: Request) -> Response:
|
|
149
|
+
if not self._authorized(request):
|
|
150
|
+
return _JSON({"error": "unauthorized"}, status_code=401)
|
|
151
|
+
name = request.path_params["name"]
|
|
152
|
+
body = await request.json()
|
|
153
|
+
|
|
154
|
+
def run() -> dict[str, Any]:
|
|
155
|
+
with self._lock:
|
|
156
|
+
self.lab.context = dict(body.get("context") or {})
|
|
157
|
+
self.lab.context["call_id"] = body.get("call_id")
|
|
158
|
+
result = call(self.lab, name, body.get("args") or {})
|
|
159
|
+
self.calls += 1
|
|
160
|
+
return {
|
|
161
|
+
"summary": result.budgeted(),
|
|
162
|
+
"data": result.data,
|
|
163
|
+
"handles": result.handles,
|
|
164
|
+
"ok": result.ok,
|
|
165
|
+
"journal": self.journal(),
|
|
166
|
+
"workspace": self.workspace(),
|
|
167
|
+
"fingerprint": self.lab.ir.fingerprint,
|
|
168
|
+
}
|
|
169
|
+
|
|
170
|
+
return _JSON(await run_in_threadpool(run))
|
|
171
|
+
|
|
172
|
+
return Starlette(
|
|
173
|
+
routes=[
|
|
174
|
+
Route("/health", health),
|
|
175
|
+
Route("/heartbeat", heartbeat, methods=["POST"]),
|
|
176
|
+
Route("/tool/{name:str}", tool, methods=["POST"]),
|
|
177
|
+
]
|
|
178
|
+
)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def main(argv: list[str] | None = None) -> None:
|
|
182
|
+
parser = argparse.ArgumentParser(prog="wspx-lab")
|
|
183
|
+
parser.add_argument(
|
|
184
|
+
"--model", required=True, help="model directory (or Hugging Face id when run locally)"
|
|
185
|
+
)
|
|
186
|
+
parser.add_argument("--session", required=True)
|
|
187
|
+
parser.add_argument("--branch", default=None)
|
|
188
|
+
parser.add_argument("--store", required=True, help="the store volume (read-only in the cloud)")
|
|
189
|
+
parser.add_argument("--overlay", default="/tmp/wspx-overlay")
|
|
190
|
+
parser.add_argument("--workdir", default="/workspace")
|
|
191
|
+
parser.add_argument(
|
|
192
|
+
"--snapshot", default=None, help="object id of the project to unpack into --workdir"
|
|
193
|
+
)
|
|
194
|
+
parser.add_argument("--dtype", default="bfloat16")
|
|
195
|
+
parser.add_argument("--trust-remote-code", action="store_true")
|
|
196
|
+
parser.add_argument("--port", type=int, default=8080)
|
|
197
|
+
parser.add_argument("--lease", type=float, default=DEFAULT_LEASE_S)
|
|
198
|
+
args = parser.parse_args(argv)
|
|
199
|
+
|
|
200
|
+
import uvicorn
|
|
201
|
+
|
|
202
|
+
from wspx.config import WspxConfig
|
|
203
|
+
from wspx.lab.ir.capture import build_ir
|
|
204
|
+
from wspx.lab.loader import load_hf
|
|
205
|
+
from wspx.lab.sdk import _cached_ir, _save_ir
|
|
206
|
+
from wspx.project import extract
|
|
207
|
+
|
|
208
|
+
store: Store = OverlayStore(args.store, args.overlay)
|
|
209
|
+
workdir = Path(args.workdir)
|
|
210
|
+
if args.snapshot:
|
|
211
|
+
extract(store.objects.get(args.snapshot), workdir)
|
|
212
|
+
loaded = load_hf(
|
|
213
|
+
args.model, dtype=args.dtype, trust_remote_code=args.trust_remote_code, local_only=True
|
|
214
|
+
)
|
|
215
|
+
ir = _cached_ir(store, loaded.fingerprint)
|
|
216
|
+
if ir is None:
|
|
217
|
+
ir = build_ir(loaded.model, fingerprint=loaded.fingerprint, hash_params=False)
|
|
218
|
+
_save_ir(store, ir)
|
|
219
|
+
lab = Lab(
|
|
220
|
+
loaded.model,
|
|
221
|
+
ir,
|
|
222
|
+
store,
|
|
223
|
+
session=args.session,
|
|
224
|
+
branch=args.branch,
|
|
225
|
+
tokenizer=loaded.tokenizer,
|
|
226
|
+
base_reader=loaded.base_reader if loaded.readable else None,
|
|
227
|
+
workdir=workdir,
|
|
228
|
+
config=WspxConfig.discover(workdir),
|
|
229
|
+
)
|
|
230
|
+
server = LabServer(lab, lease_s=args.lease, token=os.environ.get("WSPX_LAB_TOKEN"))
|
|
231
|
+
server.watchdog()
|
|
232
|
+
uvicorn.run(server.app(), host="0.0.0.0", port=args.port, log_level="warning")
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
if __name__ == "__main__":
|
|
236
|
+
main()
|