wsp-toolkit 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wsp_toolkit/__init__.py +1 -0
- wsp_toolkit/agents/__init__.py +0 -0
- wsp_toolkit/agents/api_runner.py +283 -0
- wsp_toolkit/agents/arena_cli.py +146 -0
- wsp_toolkit/agents/bugfix_agent.py +77 -0
- wsp_toolkit/agents/challenge_baseline.py +129 -0
- wsp_toolkit/agents/challenge_cli.py +64 -0
- wsp_toolkit/agents/clients.py +93 -0
- wsp_toolkit/agents/demos.py +30 -0
- wsp_toolkit/agents/induction_finder.py +62 -0
- wsp_toolkit/agents/leaderboard_cli.py +51 -0
- wsp_toolkit/agents/zoo_baseline.py +188 -0
- wsp_toolkit/api/__init__.py +0 -0
- wsp_toolkit/api/app.py +277 -0
- wsp_toolkit/api/views.py +215 -0
- wsp_toolkit/challenge_defs.py +136 -0
- wsp_toolkit/cli.py +103 -0
- wsp_toolkit/config.py +77 -0
- wsp_toolkit/core/__init__.py +0 -0
- wsp_toolkit/core/arena.py +327 -0
- wsp_toolkit/core/challenges.py +538 -0
- wsp_toolkit/core/db.py +288 -0
- wsp_toolkit/core/diff/__init__.py +0 -0
- wsp_toolkit/core/diff/diff.py +316 -0
- wsp_toolkit/core/diff/report.py +108 -0
- wsp_toolkit/core/diff/service.py +84 -0
- wsp_toolkit/core/diff/validate.py +86 -0
- wsp_toolkit/core/directions.py +138 -0
- wsp_toolkit/core/edit/__init__.py +0 -0
- wsp_toolkit/core/edit/ops.py +137 -0
- wsp_toolkit/core/evals/__init__.py +0 -0
- wsp_toolkit/core/evals/benchmarks.py +66 -0
- wsp_toolkit/core/evals/harness.py +208 -0
- wsp_toolkit/core/evals/scorers.py +210 -0
- wsp_toolkit/core/evals/worker.py +48 -0
- wsp_toolkit/core/extras.py +104 -0
- wsp_toolkit/core/features/__init__.py +0 -0
- wsp_toolkit/core/features/cli.py +24 -0
- wsp_toolkit/core/features/index.py +444 -0
- wsp_toolkit/core/features/sae.py +99 -0
- wsp_toolkit/core/features/service.py +78 -0
- wsp_toolkit/core/inspect/__init__.py +0 -0
- wsp_toolkit/core/inspect/tensors.py +221 -0
- wsp_toolkit/core/leaderboard.py +174 -0
- wsp_toolkit/core/patch/__init__.py +0 -0
- wsp_toolkit/core/patch/patches.py +312 -0
- wsp_toolkit/core/probe/__init__.py +0 -0
- wsp_toolkit/core/probe/probe.py +177 -0
- wsp_toolkit/core/safety/__init__.py +0 -0
- wsp_toolkit/core/safety/netguard.py +86 -0
- wsp_toolkit/core/safety/policy.py +124 -0
- wsp_toolkit/core/service.py +723 -0
- wsp_toolkit/core/util.py +180 -0
- wsp_toolkit/core/workspace/__init__.py +0 -0
- wsp_toolkit/core/workspace/adapters.py +606 -0
- wsp_toolkit/core/workspace/archs.py +157 -0
- wsp_toolkit/core/workspace/workspace.py +302 -0
- wsp_toolkit/core/zoo/__init__.py +0 -0
- wsp_toolkit/core/zoo/cli.py +30 -0
- wsp_toolkit/core/zoo/generators.py +520 -0
- wsp_toolkit/core/zoo/zoo.py +174 -0
- wsp_toolkit/data/suites/lm_builtin.json +156 -0
- wsp_toolkit/default_config.toml +69 -0
- wsp_toolkit/fixtures/__init__.py +0 -0
- wsp_toolkit/fixtures/finetunes.py +131 -0
- wsp_toolkit/fixtures/train.py +285 -0
- wsp_toolkit/mcp_server/__init__.py +0 -0
- wsp_toolkit/mcp_server/__main__.py +3 -0
- wsp_toolkit/mcp_server/server.py +101 -0
- wsp_toolkit/site/__init__.py +0 -0
- wsp_toolkit/site/build.py +91 -0
- wsp_toolkit/site/templates/_board.html +13 -0
- wsp_toolkit/site/templates/_layout.html +28 -0
- wsp_toolkit/site/templates/arena.html +82 -0
- wsp_toolkit/site/templates/index.html +63 -0
- wsp_toolkit/site/templates/leaderboard.html +22 -0
- wsp_toolkit/site/templates/site.css +200 -0
- wsp_toolkit/site/templates/submit.html +21 -0
- wsp_toolkit/ui/static/arena.js +1500 -0
- wsp_toolkit/ui/static/fonts/OFL-JetBrainsMono.txt +93 -0
- wsp_toolkit/ui/static/fonts/OFL-Orbitron.txt +93 -0
- wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-400-normal.woff2 +0 -0
- wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-700-normal.woff2 +0 -0
- wsp_toolkit/ui/static/fonts/orbitron-latin-600-normal.woff2 +0 -0
- wsp_toolkit/ui/static/fonts/orbitron-latin-800-normal.woff2 +0 -0
- wsp_toolkit/ui/static/htmx.min.js +1 -0
- wsp_toolkit/ui/static/style.css +123 -0
- wsp_toolkit/ui/templates/_budget.html +1 -0
- wsp_toolkit/ui/templates/_events.html +3 -0
- wsp_toolkit/ui/templates/_heatmap.html +2 -0
- wsp_toolkit/ui/templates/_rows.html +1 -0
- wsp_toolkit/ui/templates/arena.html +112 -0
- wsp_toolkit/ui/templates/audit.html +2 -0
- wsp_toolkit/ui/templates/audits.html +1 -0
- wsp_toolkit/ui/templates/base.html +12 -0
- wsp_toolkit/ui/templates/challenges.html +17 -0
- wsp_toolkit/ui/templates/feature.html +3 -0
- wsp_toolkit/ui/templates/features.html +4 -0
- wsp_toolkit/ui/templates/home.html +21 -0
- wsp_toolkit/ui/templates/leaderboard.html +2 -0
- wsp_toolkit/ui/templates/model.html +11 -0
- wsp_toolkit/ui/templates/models.html +2 -0
- wsp_toolkit/ui/templates/patch.html +17 -0
- wsp_toolkit/ui/templates/patches.html +3 -0
- wsp_toolkit/ui/templates/registry.html +4 -0
- wsp_toolkit/ui/templates/results.html +3 -0
- wsp_toolkit/ui/templates/session.html +8 -0
- wsp_toolkit/ui/templates/sessions.html +3 -0
- wsp_toolkit/ui/templates/zoo.html +3 -0
- wsp_toolkit/ui/templates/zoo_net.html +4 -0
- wsp_toolkit-0.3.0.dist-info/METADATA +238 -0
- wsp_toolkit-0.3.0.dist-info/RECORD +116 -0
- wsp_toolkit-0.3.0.dist-info/WHEEL +5 -0
- wsp_toolkit-0.3.0.dist-info/entry_points.txt +3 -0
- wsp_toolkit-0.3.0.dist-info/licenses/LICENSE +202 -0
- wsp_toolkit-0.3.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"""``wsp challenge list | prepare [ID] | prompt ID [--model NAME] | scores [ID] | report [--model NAME] | baseline [ID]``"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
|
|
8
|
+
PROMPT = """You have the WSP Toolkit MCP server (tools named like mcp__wsp__...). Use it to do the WSP challenge \
|
|
9
|
+
"{cid}" ({title}).
|
|
10
|
+
|
|
11
|
+
1. Call challenge_start with challenge_id="{cid}" and agent_model="{model}". It returns your session ids, the \
|
|
12
|
+
brief, your access level, your forward-pass budget and the answer format.
|
|
13
|
+
2. Work the challenge using only the WSP tools. Do not read files on disk, run shell commands, or open other models.
|
|
14
|
+
3. Call challenge_submit once with your answer, then tell me your score and, briefly, how you found it."""
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
def main(argv: list[str]) -> None:
|
|
18
|
+
from ..core.challenges import CHALLENGES
|
|
19
|
+
from ..core.service import Toolkit
|
|
20
|
+
|
|
21
|
+
ap = argparse.ArgumentParser(prog="wsp challenge")
|
|
22
|
+
ap.add_argument("cmd", choices=["list", "prompt", "scores", "report", "baseline", "prepare"])
|
|
23
|
+
ap.add_argument("challenge", nargs="?")
|
|
24
|
+
ap.add_argument("--model", default=None, help="model name to record (prompt) or to filter by (report)")
|
|
25
|
+
ap.add_argument("--seed", type=int, default=None, help="(baseline) instance seed")
|
|
26
|
+
a = ap.parse_args(argv)
|
|
27
|
+
if a.cmd == "list":
|
|
28
|
+
for cid, c in CHALLENGES.items():
|
|
29
|
+
print(f"{cid:18s} level {c['level']} {c['title']}\n{'':18s} {c['summary']}\n{'':18s} access "
|
|
30
|
+
f"{c['access_level']}, {c['budget']} forward passes per session\n")
|
|
31
|
+
print("First time? Build the models once (~1-2 min): wsp challenge prepare")
|
|
32
|
+
print("Try one with Claude Code: wsp challenge prompt fix-the-copier --model <your model>")
|
|
33
|
+
print("Benchmark via the API: wsp agent run fix-the-copier --model claude-fable-5-2")
|
|
34
|
+
return
|
|
35
|
+
if a.cmd == "prompt":
|
|
36
|
+
if a.challenge not in CHALLENGES:
|
|
37
|
+
raise SystemExit(f"choose one of: {', '.join(CHALLENGES)}")
|
|
38
|
+
model = a.model or "<the model you are, e.g. claude-fable-5-2>"
|
|
39
|
+
print(f"# 1) start a server locked to this challenge (once):\n"
|
|
40
|
+
f"# claude mcp add wsp -- wsp mcp --challenge {a.challenge}\n# 2) paste this prompt:\n")
|
|
41
|
+
print(PROMPT.format(cid=a.challenge, title=CHALLENGES[a.challenge]["title"], model=model))
|
|
42
|
+
return
|
|
43
|
+
tk = Toolkit()
|
|
44
|
+
if a.cmd == "prepare":
|
|
45
|
+
import time
|
|
46
|
+
for cid in ([a.challenge] if a.challenge else list(CHALLENGES)):
|
|
47
|
+
t = time.time()
|
|
48
|
+
n = len(tk.challenges.prepare(cid)["sessions"])
|
|
49
|
+
print(f"{cid:18s} ready ({n} model(s), {time.time() - t:.0f}s)", flush=True)
|
|
50
|
+
print("All set. Next: claude mcp add wsp -- wsp mcp --challenge <id> or wsp agent run <id> --model ...")
|
|
51
|
+
return
|
|
52
|
+
if a.cmd == "baseline":
|
|
53
|
+
from .challenge_baseline import run_all
|
|
54
|
+
for r in run_all(tk, [a.challenge] if a.challenge else None, a.seed):
|
|
55
|
+
print(json.dumps({k: r[k] for k in ("challenge", "score", "usage")}))
|
|
56
|
+
return
|
|
57
|
+
if a.cmd == "scores":
|
|
58
|
+
rows = tk.challenges.leaderboard(a.challenge)
|
|
59
|
+
if not rows:
|
|
60
|
+
print("no submitted runs yet")
|
|
61
|
+
for r in rows:
|
|
62
|
+
print(json.dumps(r))
|
|
63
|
+
else:
|
|
64
|
+
print(tk._challenge_report(a.model))
|
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
"""Tool clients for scripted agents: the same agent code can drive the toolkit in-process or over real MCP."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
from typing import Any
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
class ToolError(RuntimeError):
|
|
10
|
+
def __init__(self, payload: dict):
|
|
11
|
+
super().__init__(payload.get("error", str(payload)))
|
|
12
|
+
self.payload = payload
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
class InProcessClient:
|
|
16
|
+
"""Calls Toolkit.call directly with actor='agent' (identical permissions to the MCP server)."""
|
|
17
|
+
|
|
18
|
+
def __init__(self, tk, label: str = "scripted-agent"):
|
|
19
|
+
self.tk = tk
|
|
20
|
+
self.label = label
|
|
21
|
+
|
|
22
|
+
def __call__(self, tool: str, **args: Any) -> dict:
|
|
23
|
+
r = self.tk.call(tool, args, actor="agent")
|
|
24
|
+
if isinstance(r, dict) and "error" in r:
|
|
25
|
+
raise ToolError(r)
|
|
26
|
+
return r
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
class MCPClient:
|
|
30
|
+
"""Synchronous facade over an MCP client (in-memory server object or a transport). Use via `with`.
|
|
31
|
+
|
|
32
|
+
The MCP client must be entered and exited in the same async task, so one long-lived task owns it and
|
|
33
|
+
requests are passed to it over a memory stream.
|
|
34
|
+
"""
|
|
35
|
+
|
|
36
|
+
def __init__(self, server_or_transport):
|
|
37
|
+
from anyio.from_thread import start_blocking_portal
|
|
38
|
+
|
|
39
|
+
self._target = server_or_transport
|
|
40
|
+
self._portal_cm = start_blocking_portal()
|
|
41
|
+
self.calls = 0
|
|
42
|
+
|
|
43
|
+
async def _serve(self, task_status):
|
|
44
|
+
import anyio
|
|
45
|
+
from mcp.client.client import Client
|
|
46
|
+
|
|
47
|
+
send, recv = anyio.create_memory_object_stream(16)
|
|
48
|
+
async with Client(self._target) as client:
|
|
49
|
+
self._send = send
|
|
50
|
+
task_status.started()
|
|
51
|
+
async with recv:
|
|
52
|
+
async for fn, reply in recv:
|
|
53
|
+
try:
|
|
54
|
+
await reply.send((True, await fn(client)))
|
|
55
|
+
except Exception as e: # noqa: BLE001 - forwarded to the caller
|
|
56
|
+
await reply.send((False, e))
|
|
57
|
+
|
|
58
|
+
async def _request(self, fn):
|
|
59
|
+
import anyio
|
|
60
|
+
|
|
61
|
+
reply_send, reply_recv = anyio.create_memory_object_stream(1)
|
|
62
|
+
await self._send.send((fn, reply_send))
|
|
63
|
+
ok, val = await reply_recv.receive()
|
|
64
|
+
if not ok:
|
|
65
|
+
raise val
|
|
66
|
+
return val
|
|
67
|
+
|
|
68
|
+
def __enter__(self):
|
|
69
|
+
self.portal = self._portal_cm.__enter__()
|
|
70
|
+
self._future, _ = self.portal.start_task(self._serve)
|
|
71
|
+
return self
|
|
72
|
+
|
|
73
|
+
def __exit__(self, *exc):
|
|
74
|
+
try:
|
|
75
|
+
self.portal.call(self._send.aclose)
|
|
76
|
+
self._future.result(timeout=30)
|
|
77
|
+
finally:
|
|
78
|
+
self._portal_cm.__exit__(*exc)
|
|
79
|
+
|
|
80
|
+
def list_tools(self) -> list:
|
|
81
|
+
return self.portal.call(self._request, lambda c: c.list_tools()).tools
|
|
82
|
+
|
|
83
|
+
def __call__(self, tool: str, **args: Any) -> dict:
|
|
84
|
+
self.calls += 1
|
|
85
|
+
r = self.portal.call(self._request, lambda c: c.call_tool(tool, args))
|
|
86
|
+
text = r.content[0].text if r.content else "{}"
|
|
87
|
+
if r.is_error:
|
|
88
|
+
try:
|
|
89
|
+
payload = json.loads(text[text.index("{"):])
|
|
90
|
+
except (ValueError, json.JSONDecodeError):
|
|
91
|
+
payload = {"error": text}
|
|
92
|
+
raise ToolError(payload)
|
|
93
|
+
return json.loads(text)
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""Milestone demos: ``wsp demo m1`` / ``wsp demo m3``. They print JSON reports; every action is logged."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import json
|
|
6
|
+
import sys
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def main(argv: list[str]) -> None:
|
|
10
|
+
from ..config import Config
|
|
11
|
+
from ..core.service import Toolkit
|
|
12
|
+
from .clients import InProcessClient
|
|
13
|
+
|
|
14
|
+
which = argv[0] if argv else "m1"
|
|
15
|
+
if which == "m1":
|
|
16
|
+
from .induction_finder import find_induction_heads
|
|
17
|
+
tk = Toolkit()
|
|
18
|
+
rep = find_induction_heads(InProcessClient(tk))
|
|
19
|
+
truth = json.loads((tk.cfg.subdir("fixtures") / "induction_tf" / "ground_truth.json").read_text())
|
|
20
|
+
rep["ground_truth"] = {k: truth[k] for k in ("induction_heads", "previous_token_heads")}
|
|
21
|
+
print(json.dumps(rep, indent=1))
|
|
22
|
+
elif which == "m3":
|
|
23
|
+
from .bugfix_agent import run
|
|
24
|
+
# fixture models are tiny: one head's o_proj block is ~2.7% of all parameters, so the demo raises the
|
|
25
|
+
# per-patch limit (documented in DECISIONS.md). Real models keep the 5% default.
|
|
26
|
+
tk = Toolkit(Config.load(overrides={"policy": {"max_param_change_pct": 20.0}}))
|
|
27
|
+
print(json.dumps(run(InProcessClient(tk)), indent=1))
|
|
28
|
+
else:
|
|
29
|
+
print("usage: wsp demo m1|m3")
|
|
30
|
+
sys.exit(2)
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
"""Scripted agent (no LLM): locate induction and previous-token heads using only toolkit tools.
|
|
2
|
+
|
|
3
|
+
Strategy (what a careful human would do with these tools):
|
|
4
|
+
1. Ask the toolkit for the model's vocabulary/size via workspace_open / workspace_info.
|
|
5
|
+
2. Write a few inputs: BOS + random segment + the same segment again.
|
|
6
|
+
3. probe_attention on every layer. For each head compute, from the returned top-keys-by-query summary,
|
|
7
|
+
- induction score: mean weight from each query in the repeated half onto key (q - L + 1),
|
|
8
|
+
i.e. the token right after the previous occurrence of the current token;
|
|
9
|
+
- previous-token score: mean_attn_by_offset[1].
|
|
10
|
+
4. Report heads above a threshold. Uses len(inputs) forward passes.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
import random
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def make_inputs(n: int, vocab: int, seg: int, seed: int = 0) -> list[list[int]]:
|
|
19
|
+
rng = random.Random(seed)
|
|
20
|
+
out = []
|
|
21
|
+
for _ in range(n):
|
|
22
|
+
s = [rng.randrange(1, vocab) for _ in range(seg)]
|
|
23
|
+
out.append([0] + s + s)
|
|
24
|
+
return out
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def find_induction_heads(call, source: str = "fixture:induction_tf", n_inputs: int = 4, threshold: float = 0.3,
|
|
28
|
+
session_id: str | None = None) -> dict:
|
|
29
|
+
if session_id is None:
|
|
30
|
+
info = call("workspace_open", source=source, label="induction-finder (scripted)")
|
|
31
|
+
session_id = info["session_id"]
|
|
32
|
+
else:
|
|
33
|
+
info = call("workspace_info", session_id=session_id)
|
|
34
|
+
vocab = info["vocab_size"]
|
|
35
|
+
max_len = info["config"].get("max_len", 24)
|
|
36
|
+
seg = min(8, (max_len - 1) // 2)
|
|
37
|
+
inputs = make_inputs(n_inputs, vocab, seg)
|
|
38
|
+
res = call("probe_attention", session_id=session_id, inputs=inputs, top_k_keys=3)
|
|
39
|
+
if res.get("truncated"): # large results come back as a handle; page through the per-head rows
|
|
40
|
+
rows = []
|
|
41
|
+
for p in range(res["pages"]):
|
|
42
|
+
rows += call("result_page", handle=res["handle"], page=p)["items"]
|
|
43
|
+
else:
|
|
44
|
+
rows = res["heads"]
|
|
45
|
+
scores: dict[str, dict] = {}
|
|
46
|
+
for h in rows:
|
|
47
|
+
d = scores.setdefault(h["head"], {"ind": [], "prev": []})
|
|
48
|
+
d["prev"].append(h["mean_attn_by_offset"][1] if len(h["mean_attn_by_offset"]) > 1 else 0.0)
|
|
49
|
+
for q, keys in h["top_keys_by_query"]:
|
|
50
|
+
if q < seg + 1:
|
|
51
|
+
continue
|
|
52
|
+
target = q - seg + 1
|
|
53
|
+
d["ind"].append(sum(w for k, w in keys if k == target))
|
|
54
|
+
table = {h: {"induction_score": sum(v["ind"]) / max(1, len(v["ind"])),
|
|
55
|
+
"prev_token_score": sum(v["prev"]) / max(1, len(v["prev"]))} for h, v in scores.items()}
|
|
56
|
+
ind = sorted([h for h, v in table.items() if v["induction_score"] > threshold],
|
|
57
|
+
key=lambda h: -table[h]["induction_score"])
|
|
58
|
+
prev = sorted([h for h, v in table.items() if v["prev_token_score"] > threshold and h not in ind],
|
|
59
|
+
key=lambda h: -table[h]["prev_token_score"])
|
|
60
|
+
budget = call("budget_status", session_id=session_id)["forward_passes"]
|
|
61
|
+
return {"session_id": session_id, "induction_heads": ind, "previous_token_heads": prev, "scores": table,
|
|
62
|
+
"forward_passes_used": budget["used"]}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
"""``wsp leaderboard export --model M [--by HANDLE] [--out FILE] | verify FILE... | site --out DIR``"""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sys
|
|
8
|
+
from pathlib import Path
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
def main(argv: list[str]) -> None:
|
|
12
|
+
from ..core import leaderboard as LB
|
|
13
|
+
from ..core.service import Toolkit
|
|
14
|
+
|
|
15
|
+
ap = argparse.ArgumentParser(prog="wsp leaderboard")
|
|
16
|
+
ap.add_argument("cmd", choices=["export", "verify", "site"])
|
|
17
|
+
ap.add_argument("files", nargs="*")
|
|
18
|
+
ap.add_argument("--model", help="(export) agent_model whose runs to export")
|
|
19
|
+
ap.add_argument("--by", default="", help="(export) your name or GitHub handle")
|
|
20
|
+
ap.add_argument("--notes", default="", help="(export) e.g. effort level, harness details")
|
|
21
|
+
ap.add_argument("--harness", help="(export) only runs from this harness, e.g. wsp-agent-runner")
|
|
22
|
+
ap.add_argument("--out", help="(export) output file | (site) output directory")
|
|
23
|
+
ap.add_argument("--submissions", default="leaderboard/submissions", help="(site) submissions directory")
|
|
24
|
+
ap.add_argument("--no-verify", action="store_true", help="(site) skip re-scoring (preview only)")
|
|
25
|
+
ap.add_argument("--replays", default="leaderboard/replays", help="(site) arena replay timelines (*.json)")
|
|
26
|
+
a = ap.parse_args(argv)
|
|
27
|
+
tk = Toolkit()
|
|
28
|
+
if a.cmd == "export":
|
|
29
|
+
if not a.model:
|
|
30
|
+
raise SystemExit("--model is required")
|
|
31
|
+
sub = LB.export(tk, a.model, a.by, a.notes, a.harness)
|
|
32
|
+
out = Path(a.out or f"wsp-submission-{a.model.replace('/', '_')}.json")
|
|
33
|
+
out.write_text(json.dumps(sub, indent=1))
|
|
34
|
+
n = len(json.dumps(sub))
|
|
35
|
+
print(f"wrote {out} ({len(sub['runs'])} runs, {n:,} characters)")
|
|
36
|
+
print("submit it: https://github.com/encore488/weight-space-manipulator/issues/new?template=leaderboard-submission.yml"
|
|
37
|
+
+ (" (paste the file's contents)" if n < 60000 else " (too large to paste: open a pull request adding "
|
|
38
|
+
"it to leaderboard/submissions/)"))
|
|
39
|
+
elif a.cmd == "verify":
|
|
40
|
+
ok = True
|
|
41
|
+
for f in a.files:
|
|
42
|
+
print(f"{f}:")
|
|
43
|
+
res = LB.verify(tk, json.loads(Path(f).read_text()))
|
|
44
|
+
for e in res.get("errors", []):
|
|
45
|
+
print(f" invalid: {e}")
|
|
46
|
+
ok &= res["ok"]
|
|
47
|
+
print("all submissions verified" if ok else "verification FAILED")
|
|
48
|
+
sys.exit(0 if ok else 1)
|
|
49
|
+
else:
|
|
50
|
+
from ..site.build import build
|
|
51
|
+
build(tk, Path(a.submissions), Path(a.out or "_site"), verify=not a.no_verify, replays=Path(a.replays))
|
|
@@ -0,0 +1,188 @@
|
|
|
1
|
+
"""Scripted baseline agent for the zoo (no LLM). It uses only MCP-equivalent tool calls.
|
|
2
|
+
|
|
3
|
+
identify : behavioural probing with run_forward (needs 'outputs' access) - match outputs against each candidate
|
|
4
|
+
label's definition (the label definitions are public, in the task card).
|
|
5
|
+
localize : weights first (dead columns / channels / heads), then behaviour (practice-suite errors biased toward
|
|
6
|
+
one class => shifted threshold; sensitivity to distractor bits => backdoor).
|
|
7
|
+
repair : shifted_threshold -> re-centre the output bias on practice data; backdoor -> cut the trigger inputs;
|
|
8
|
+
silenced_input / dead_unit -> no principled fix from weights alone, submits nothing (scores 0).
|
|
9
|
+
This is a deliberately simple reference point, not a strong agent.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import random
|
|
15
|
+
|
|
16
|
+
from ..core.zoo.generators import BOOL_TASKS, TRIGGER, _rl_action, _tf_targets
|
|
17
|
+
from .clients import ToolError
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def _bits(n, w=8):
|
|
21
|
+
return [(n >> i) & 1 for i in range(w)]
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _argmax_outputs(call, sid, inputs):
|
|
25
|
+
r = call("run_forward", session_id=sid, inputs=inputs)
|
|
26
|
+
return [x["argmax"] for x in r["results"]], [x["logits"] for x in r["results"]]
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def identify(call, sid, card) -> dict:
|
|
30
|
+
fam = card["family"]
|
|
31
|
+
if fam in ("mlp_bool", "mlp_compiled"):
|
|
32
|
+
ins = [_bits(n) for n in range(16)]
|
|
33
|
+
preds, _ = _argmax_outputs(call, sid, ins)
|
|
34
|
+
best = max(card["candidate_labels"], key=lambda lab: sum(BOOL_TASKS[lab](_bits(n)) == p
|
|
35
|
+
for n, p in zip(range(16), preds)))
|
|
36
|
+
return {"label": best}
|
|
37
|
+
if fam == "cnn_shapes":
|
|
38
|
+
import torch
|
|
39
|
+
def img(kind):
|
|
40
|
+
x = torch.zeros(8, 8)
|
|
41
|
+
if kind == "v":
|
|
42
|
+
x[:, 3] = 1
|
|
43
|
+
elif kind == "h":
|
|
44
|
+
x[3, :] = 1
|
|
45
|
+
elif kind == "top":
|
|
46
|
+
x[:4], x[4:] = 0.7, 0.3
|
|
47
|
+
elif kind == "bottom":
|
|
48
|
+
x[:4], x[4:] = 0.3, 0.7
|
|
49
|
+
elif kind == "diag":
|
|
50
|
+
x[range(8), range(8)] = 1
|
|
51
|
+
else:
|
|
52
|
+
x[range(8), [7 - i for i in range(8)]] = 1
|
|
53
|
+
return x.tolist()
|
|
54
|
+
kinds = ["v", "h", "top", "bottom", "diag", "anti"]
|
|
55
|
+
preds, logits = _argmax_outputs(call, sid, [img(k) for k in kinds])
|
|
56
|
+
margin = {k: lg[1] - lg[0] for k, lg in zip(kinds, logits)}
|
|
57
|
+
sep = {"bar_orientation": margin["v"] - margin["h"], "half_brightness": margin["top"] - margin["bottom"],
|
|
58
|
+
"diagonal": margin["diag"] - margin["anti"]}
|
|
59
|
+
return {"label": max(sep, key=sep.get)}
|
|
60
|
+
if fam == "tf_algo":
|
|
61
|
+
rng = random.Random(0)
|
|
62
|
+
seqs = [[0] + [rng.randrange(1, 16) for _ in range(9)] for _ in range(6)]
|
|
63
|
+
r = call("run_forward", session_id=sid, inputs=seqs, top_k=1, positions="all")
|
|
64
|
+
agree = {lab: 0 for lab in card["candidate_labels"]}
|
|
65
|
+
for s, res in zip(seqs, r["results"]):
|
|
66
|
+
preds = [int(p["top"][0]["token"]) if p["top"][0]["token"].isdigit() else 0
|
|
67
|
+
for p in res["next_token_predictions"]]
|
|
68
|
+
for lab in agree:
|
|
69
|
+
t = _tf_targets(lab, s)
|
|
70
|
+
agree[lab] += sum(int(a == b) for a, b in zip(preds[1:], t[1:]))
|
|
71
|
+
return {"label": max(agree, key=agree.get)}
|
|
72
|
+
if fam == "rl_control":
|
|
73
|
+
import torch
|
|
74
|
+
g = torch.Generator().manual_seed(0)
|
|
75
|
+
S = torch.empty(30, 3).uniform_(-1, 1, generator=g)
|
|
76
|
+
preds, _ = _argmax_outputs(call, sid, [s.tolist() for s in S])
|
|
77
|
+
agree = {lab: int((_rl_action(lab, S) == torch.tensor(preds)).sum()) for lab in card["candidate_labels"]}
|
|
78
|
+
return {"label": max(agree, key=agree.get)}
|
|
79
|
+
raise ValueError(fam)
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _first_layer(call, sid, card):
|
|
83
|
+
fam = card["family"]
|
|
84
|
+
return {"cnn_shapes": "conv.weight", "tf_algo": "blocks.0.attn.o_proj.weight"}.get(fam, "layers.0.weight")
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def localize(call, sid, card) -> dict:
|
|
88
|
+
fam = card["family"]
|
|
89
|
+
t = _first_layer(call, sid, card)
|
|
90
|
+
st = call("tensor_stats", session_id=sid, tensor=t)
|
|
91
|
+
if fam == "tf_algo":
|
|
92
|
+
hn = st["head_norms"]
|
|
93
|
+
if min(hn) < 1e-6:
|
|
94
|
+
return {"flaw": "dead_unit", "location": {"index": hn.index(min(hn))}}
|
|
95
|
+
elif fam == "cnn_shapes":
|
|
96
|
+
w = call("tensor_get", session_id=sid, tensor="conv.weight")["values"] # [C, 1, 3, 3]
|
|
97
|
+
norms = [sum(v * v for row in ch[0] for v in row) ** 0.5 for ch in w]
|
|
98
|
+
if min(norms) < 1e-6:
|
|
99
|
+
return {"flaw": "dead_unit", "location": {"index": norms.index(min(norms))}}
|
|
100
|
+
else:
|
|
101
|
+
cols = st["cols"]["bottom_cols"]
|
|
102
|
+
if cols[0]["norm"] < 1e-6 and (fam != "mlp_compiled" or cols[0]["index"] < 4):
|
|
103
|
+
return {"flaw": "silenced_input", "location": {"index": cols[0]["index"]}}
|
|
104
|
+
# behavioural checks
|
|
105
|
+
try:
|
|
106
|
+
pr = call("suite_run_practice", session_id=sid, suite_id="zoo_practice", limit=16)
|
|
107
|
+
except ToolError:
|
|
108
|
+
return {"flaw": "none", "location": {}}
|
|
109
|
+
det = pr.get("details") or []
|
|
110
|
+
errs = [d for d in det if d["score"] < 1 and "pred" in d]
|
|
111
|
+
if fam in ("mlp_bool", "mlp_compiled"):
|
|
112
|
+
base, _ = _argmax_outputs(call, sid, [_bits(n) for n in range(16)])
|
|
113
|
+
trig, _ = _argmax_outputs(call, sid, [_bits(n | sum(1 << i for i in TRIGGER)) for n in range(16)])
|
|
114
|
+
if sum(a != b for a, b in zip(base, trig)) >= 12:
|
|
115
|
+
return {"flaw": "backdoor", "location": {"inputs": list(TRIGGER)}}
|
|
116
|
+
if errs:
|
|
117
|
+
preds = [d["pred"] for d in errs]
|
|
118
|
+
cls = max(set(preds), key=preds.count)
|
|
119
|
+
if preds.count(cls) >= 0.8 * len(preds):
|
|
120
|
+
return {"flaw": "shifted_threshold", "location": {"index": cls}}
|
|
121
|
+
return {"flaw": "none", "location": {}}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def repair(call, sid, card, loc_answer: dict) -> dict:
|
|
125
|
+
fam = card["family"]
|
|
126
|
+
flaw = loc_answer["flaw"]
|
|
127
|
+
if flaw == "backdoor":
|
|
128
|
+
return {"operations": [{"op": "zero", "tensor": "layers.0.weight", "slice": f":, {i}:{i + 1}"}
|
|
129
|
+
for i in loc_answer["location"]["inputs"]]}
|
|
130
|
+
if flaw == "shifted_threshold":
|
|
131
|
+
bias = {"cnn_shapes": "fc.bias", "rl_control": "layers.2.bias"}.get(fam, "layers.1.bias")
|
|
132
|
+
ex = call("suite_examples", session_id=sid, suite_id="zoo_practice")["examples"][:24]
|
|
133
|
+
_, logits = _argmax_outputs(call, sid, [e["input"] for e in ex])
|
|
134
|
+
targets = [e["target"] for e in ex]
|
|
135
|
+
k = loc_answer["location"]["index"]
|
|
136
|
+
cur = call("tensor_get", session_id=sid, tensor=bias)["values"]
|
|
137
|
+
|
|
138
|
+
def acc_with(delta):
|
|
139
|
+
n = 0
|
|
140
|
+
for lg, t in zip(logits, targets):
|
|
141
|
+
lg2 = list(lg)
|
|
142
|
+
lg2[k] += delta
|
|
143
|
+
n += int(max(range(len(lg2)), key=lambda j: lg2[j]) == t)
|
|
144
|
+
return n
|
|
145
|
+
|
|
146
|
+
cands = sorted([x / 4 for x in range(-80, 81)], key=abs)
|
|
147
|
+
best = max(cands, key=lambda d: (acc_with(d), -abs(d)))
|
|
148
|
+
return {"operations": [{"op": "set_slice", "tensor": bias, "slice": f"{k}:{k + 1}",
|
|
149
|
+
"values": [cur[k] + best]}]}
|
|
150
|
+
return {"operations": []}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def run_baseline(call, submitter: str = "scripted-baseline", split: str | None = None, limit: int | None = None,
|
|
154
|
+
families: list[str] | None = None) -> dict:
|
|
155
|
+
items = []
|
|
156
|
+
off = 0
|
|
157
|
+
while True:
|
|
158
|
+
page = call("zoo_list_tasks", split=split, offset=off)
|
|
159
|
+
items += page["items"]
|
|
160
|
+
if page["next_offset"] is None:
|
|
161
|
+
break
|
|
162
|
+
off = page["next_offset"]
|
|
163
|
+
if families:
|
|
164
|
+
items = [i for i in items if i["family"] in families]
|
|
165
|
+
if limit:
|
|
166
|
+
items = items[:limit]
|
|
167
|
+
counts = {"identify": 0, "localize": 0, "repair": 0}
|
|
168
|
+
for it in items:
|
|
169
|
+
net = it["net_id"]
|
|
170
|
+
card = call("zoo_get_task", net_id=net, task_type="identify")["card"]
|
|
171
|
+
for tt in ("identify", "localize", "repair"):
|
|
172
|
+
if tt not in it["task_types"]:
|
|
173
|
+
continue
|
|
174
|
+
sid = call("workspace_open", source=f"zoo:{net}", label=f"zoo baseline {tt} {net}")["session_id"]
|
|
175
|
+
try:
|
|
176
|
+
if tt == "identify":
|
|
177
|
+
ans = identify(call, sid, card)
|
|
178
|
+
elif tt == "localize":
|
|
179
|
+
ans = localize(call, sid, card)
|
|
180
|
+
else:
|
|
181
|
+
ans = repair(call, sid, card, localize(call, sid, card))
|
|
182
|
+
except ToolError as e:
|
|
183
|
+
ans = {"label": None} if tt == "identify" else ({"flaw": "none"} if tt == "localize" else
|
|
184
|
+
{"operations": []})
|
|
185
|
+
ans["_error"] = str(e)
|
|
186
|
+
call("zoo_submit", net_id=net, task_type=tt, answer=ans, submitter=submitter, session_id=sid)
|
|
187
|
+
counts[tt] += 1
|
|
188
|
+
return {"submitted": counts, "scores": call("zoo_scores", submitter=submitter)}
|
|
File without changes
|