wsp-toolkit 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. wsp_toolkit/__init__.py +1 -0
  2. wsp_toolkit/agents/__init__.py +0 -0
  3. wsp_toolkit/agents/api_runner.py +283 -0
  4. wsp_toolkit/agents/arena_cli.py +146 -0
  5. wsp_toolkit/agents/bugfix_agent.py +77 -0
  6. wsp_toolkit/agents/challenge_baseline.py +129 -0
  7. wsp_toolkit/agents/challenge_cli.py +64 -0
  8. wsp_toolkit/agents/clients.py +93 -0
  9. wsp_toolkit/agents/demos.py +30 -0
  10. wsp_toolkit/agents/induction_finder.py +62 -0
  11. wsp_toolkit/agents/leaderboard_cli.py +51 -0
  12. wsp_toolkit/agents/zoo_baseline.py +188 -0
  13. wsp_toolkit/api/__init__.py +0 -0
  14. wsp_toolkit/api/app.py +277 -0
  15. wsp_toolkit/api/views.py +215 -0
  16. wsp_toolkit/challenge_defs.py +136 -0
  17. wsp_toolkit/cli.py +103 -0
  18. wsp_toolkit/config.py +77 -0
  19. wsp_toolkit/core/__init__.py +0 -0
  20. wsp_toolkit/core/arena.py +327 -0
  21. wsp_toolkit/core/challenges.py +538 -0
  22. wsp_toolkit/core/db.py +288 -0
  23. wsp_toolkit/core/diff/__init__.py +0 -0
  24. wsp_toolkit/core/diff/diff.py +316 -0
  25. wsp_toolkit/core/diff/report.py +108 -0
  26. wsp_toolkit/core/diff/service.py +84 -0
  27. wsp_toolkit/core/diff/validate.py +86 -0
  28. wsp_toolkit/core/directions.py +138 -0
  29. wsp_toolkit/core/edit/__init__.py +0 -0
  30. wsp_toolkit/core/edit/ops.py +137 -0
  31. wsp_toolkit/core/evals/__init__.py +0 -0
  32. wsp_toolkit/core/evals/benchmarks.py +66 -0
  33. wsp_toolkit/core/evals/harness.py +208 -0
  34. wsp_toolkit/core/evals/scorers.py +210 -0
  35. wsp_toolkit/core/evals/worker.py +48 -0
  36. wsp_toolkit/core/extras.py +104 -0
  37. wsp_toolkit/core/features/__init__.py +0 -0
  38. wsp_toolkit/core/features/cli.py +24 -0
  39. wsp_toolkit/core/features/index.py +444 -0
  40. wsp_toolkit/core/features/sae.py +99 -0
  41. wsp_toolkit/core/features/service.py +78 -0
  42. wsp_toolkit/core/inspect/__init__.py +0 -0
  43. wsp_toolkit/core/inspect/tensors.py +221 -0
  44. wsp_toolkit/core/leaderboard.py +174 -0
  45. wsp_toolkit/core/patch/__init__.py +0 -0
  46. wsp_toolkit/core/patch/patches.py +312 -0
  47. wsp_toolkit/core/probe/__init__.py +0 -0
  48. wsp_toolkit/core/probe/probe.py +177 -0
  49. wsp_toolkit/core/safety/__init__.py +0 -0
  50. wsp_toolkit/core/safety/netguard.py +86 -0
  51. wsp_toolkit/core/safety/policy.py +124 -0
  52. wsp_toolkit/core/service.py +723 -0
  53. wsp_toolkit/core/util.py +180 -0
  54. wsp_toolkit/core/workspace/__init__.py +0 -0
  55. wsp_toolkit/core/workspace/adapters.py +606 -0
  56. wsp_toolkit/core/workspace/archs.py +157 -0
  57. wsp_toolkit/core/workspace/workspace.py +302 -0
  58. wsp_toolkit/core/zoo/__init__.py +0 -0
  59. wsp_toolkit/core/zoo/cli.py +30 -0
  60. wsp_toolkit/core/zoo/generators.py +520 -0
  61. wsp_toolkit/core/zoo/zoo.py +174 -0
  62. wsp_toolkit/data/suites/lm_builtin.json +156 -0
  63. wsp_toolkit/default_config.toml +69 -0
  64. wsp_toolkit/fixtures/__init__.py +0 -0
  65. wsp_toolkit/fixtures/finetunes.py +131 -0
  66. wsp_toolkit/fixtures/train.py +285 -0
  67. wsp_toolkit/mcp_server/__init__.py +0 -0
  68. wsp_toolkit/mcp_server/__main__.py +3 -0
  69. wsp_toolkit/mcp_server/server.py +101 -0
  70. wsp_toolkit/site/__init__.py +0 -0
  71. wsp_toolkit/site/build.py +91 -0
  72. wsp_toolkit/site/templates/_board.html +13 -0
  73. wsp_toolkit/site/templates/_layout.html +28 -0
  74. wsp_toolkit/site/templates/arena.html +82 -0
  75. wsp_toolkit/site/templates/index.html +63 -0
  76. wsp_toolkit/site/templates/leaderboard.html +22 -0
  77. wsp_toolkit/site/templates/site.css +200 -0
  78. wsp_toolkit/site/templates/submit.html +21 -0
  79. wsp_toolkit/ui/static/arena.js +1500 -0
  80. wsp_toolkit/ui/static/fonts/OFL-JetBrainsMono.txt +93 -0
  81. wsp_toolkit/ui/static/fonts/OFL-Orbitron.txt +93 -0
  82. wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-400-normal.woff2 +0 -0
  83. wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-700-normal.woff2 +0 -0
  84. wsp_toolkit/ui/static/fonts/orbitron-latin-600-normal.woff2 +0 -0
  85. wsp_toolkit/ui/static/fonts/orbitron-latin-800-normal.woff2 +0 -0
  86. wsp_toolkit/ui/static/htmx.min.js +1 -0
  87. wsp_toolkit/ui/static/style.css +123 -0
  88. wsp_toolkit/ui/templates/_budget.html +1 -0
  89. wsp_toolkit/ui/templates/_events.html +3 -0
  90. wsp_toolkit/ui/templates/_heatmap.html +2 -0
  91. wsp_toolkit/ui/templates/_rows.html +1 -0
  92. wsp_toolkit/ui/templates/arena.html +112 -0
  93. wsp_toolkit/ui/templates/audit.html +2 -0
  94. wsp_toolkit/ui/templates/audits.html +1 -0
  95. wsp_toolkit/ui/templates/base.html +12 -0
  96. wsp_toolkit/ui/templates/challenges.html +17 -0
  97. wsp_toolkit/ui/templates/feature.html +3 -0
  98. wsp_toolkit/ui/templates/features.html +4 -0
  99. wsp_toolkit/ui/templates/home.html +21 -0
  100. wsp_toolkit/ui/templates/leaderboard.html +2 -0
  101. wsp_toolkit/ui/templates/model.html +11 -0
  102. wsp_toolkit/ui/templates/models.html +2 -0
  103. wsp_toolkit/ui/templates/patch.html +17 -0
  104. wsp_toolkit/ui/templates/patches.html +3 -0
  105. wsp_toolkit/ui/templates/registry.html +4 -0
  106. wsp_toolkit/ui/templates/results.html +3 -0
  107. wsp_toolkit/ui/templates/session.html +8 -0
  108. wsp_toolkit/ui/templates/sessions.html +3 -0
  109. wsp_toolkit/ui/templates/zoo.html +3 -0
  110. wsp_toolkit/ui/templates/zoo_net.html +4 -0
  111. wsp_toolkit-0.3.0.dist-info/METADATA +238 -0
  112. wsp_toolkit-0.3.0.dist-info/RECORD +116 -0
  113. wsp_toolkit-0.3.0.dist-info/WHEEL +5 -0
  114. wsp_toolkit-0.3.0.dist-info/entry_points.txt +3 -0
  115. wsp_toolkit-0.3.0.dist-info/licenses/LICENSE +202 -0
  116. wsp_toolkit-0.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1 @@
1
+ __version__ = "0.3.0"
File without changes
@@ -0,0 +1,283 @@
1
+ """Benchmark a Claude model on a WSP challenge through the Anthropic API.
2
+
3
+ pip install anthropic
4
+ export ANTHROPIC_API_KEY=... # or `ant auth login`
5
+ wsp agent run fix-the-copier --model claude-fable-5-2
6
+
7
+ The model gets ONLY the toolkit's tools (no shell, no files), so these runs are the comparable ones on the
8
+ challenge leaderboard (harness "wsp-agent-runner"). The server is locked to the challenge's models.
9
+
10
+ Loop: manual tool-use loop over streaming Messages requests (append-only history, so preserved thinking works),
11
+ eager input streaming with client-side schema validation, automatic prompt caching, server-side refusal fallback
12
+ on models that support it, and hard limits on turns and estimated cost.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import json
18
+ import time
19
+ from pathlib import Path
20
+ from typing import Any
21
+
22
+ # Tools the model does not get: the runner opens the challenge itself, and the model must not open other models
23
+ # or write to the zoo leaderboard.
24
+ EXCLUDED_TOOLS = {"workspace_open", "session_start", "challenge_start", "challenge_list", "challenge_scores",
25
+ "zoo_list_tasks", "zoo_submit", "zoo_scores",
26
+ "narrate"} # the runner already shows the model's text and thinking as live captions
27
+
28
+ # Models that accept server-side refusal fallbacks (`fallbacks: "default"`).
29
+ FALLBACK_MODELS = {"claude-fable-5-2", "claude-fable-5-1", "claude-mythos-5-2", "claude-mythos-5-1",
30
+ "claude-opus-5-5", "claude-opus-5", "claude-sonnet-5-5"}
31
+ FALLBACK_BETA = "server-side-fallback-2026-07-01"
32
+
33
+ # $ per million tokens: (input, output, cache read). Cache writes are estimated at 1.25x input. Estimates only.
34
+ PRICES = {"claude-fable-5-2": (8.0, 40.0, 0.20), "claude-mythos-5-2": (8.0, 40.0, 0.20),
35
+ "claude-fable-5-1": (10.0, 50.0, 1.0), "claude-fable-5": (10.0, 50.0, 1.0),
36
+ "claude-opus-5-5": (4.0, 20.0, 0.20), "claude-opus-5": (5.0, 25.0, 0.5),
37
+ "claude-opus-4-8": (5.0, 25.0, 0.5), "claude-sonnet-5-5": (2.0, 10.0, 0.10),
38
+ "claude-sonnet-5": (2.0, 10.0, 0.2), "claude-haiku-5-5": (0.10, 0.50, 0.01)}
39
+
40
+ SYSTEM = """You are an AI agent doing weight-space programming: you read, probe and edit the weights of small \
41
+ neural networks directly, through the WSP Toolkit's tools. Weight tools are free; forward passes are budgeted per \
42
+ session; some tools are unavailable at lower access levels. Tool results are JSON; very large results come back as \
43
+ a handle you can page with result_page. Edits must go through edit_apply with a rationale; you can revert them.
44
+
45
+ You are being evaluated on a challenge. Work it out from the evidence the tools give you, then call \
46
+ challenge_submit exactly once with your answer. Your final message should briefly say what you found."""
47
+
48
+
49
+ def tool_definitions(tk) -> list[dict]:
50
+ """The same JSON schemas the MCP server advertises, minus excluded tools, for the Messages API."""
51
+ import anyio
52
+
53
+ from ..mcp_server.server import build_server
54
+
55
+ server = build_server(tk)
56
+ tools = anyio.run(server.list_tools)
57
+ out = []
58
+ for t in tools:
59
+ if t.name in EXCLUDED_TOOLS:
60
+ continue
61
+ out.append({"name": t.name, "description": t.description or t.name, "input_schema": t.input_schema,
62
+ "eager_input_streaming": True})
63
+ return out
64
+
65
+
66
+ def _valid(args: Any, schema: dict) -> str | None:
67
+ import jsonschema
68
+
69
+ if not isinstance(args, dict):
70
+ return "tool input is not an object"
71
+ try:
72
+ jsonschema.validate(args, schema)
73
+ except jsonschema.ValidationError as e:
74
+ return e.message
75
+ return None
76
+
77
+
78
+ def _cost(model: str, u: dict) -> float | None:
79
+ p = PRICES.get(model)
80
+ if not p:
81
+ return None
82
+ pin, pout, pread = p
83
+ return (u["input_tokens"] * pin + u["cache_creation_input_tokens"] * pin * 1.25 +
84
+ u["cache_read_input_tokens"] * pread + u["output_tokens"] * pout) / 1e6
85
+
86
+
87
+ def _block_dict(b) -> dict:
88
+ t = getattr(b, "type", None)
89
+ if t == "text":
90
+ return {"type": "text", "text": b.text}
91
+ if t == "thinking":
92
+ return {"type": "thinking", "summary": getattr(b, "thinking", "")}
93
+ if t == "tool_use":
94
+ return {"type": "tool_use", "name": b.name, "input": b.input}
95
+ return {"type": str(t)}
96
+
97
+
98
+ def run(tk, challenge_id: str, model: str, client=None, effort: str | None = "high", thinking: str = "summarized",
99
+ max_turns: int = 80, max_cost: float = 20.0, max_tokens: int = 64000, label: str | None = None,
100
+ verbose: bool = True, seed: int | None = None) -> dict:
101
+ from ..mcp_server.server import lock_to_challenge
102
+
103
+ if client is None:
104
+ import anthropic
105
+ client = anthropic.Anthropic()
106
+ lock_to_challenge(tk, challenge_id)
107
+ start = tk.challenges.start(challenge_id, model, "wsp-agent-runner", label or f"api runner: {model}", "agent",
108
+ seed)
109
+ run_id = start["run_id"]
110
+ note_sid = (start.get("sessions") or [{}])[0].get("session_id")
111
+ tools = tool_definitions(tk)
112
+ schemas = {t["name"]: t["input_schema"] for t in tools}
113
+ log_path = tk.cfg.subdir("agent_runs") / f"{run_id}.jsonl"
114
+ log = log_path.open("a")
115
+
116
+ def note(entry: dict) -> None:
117
+ log.write(json.dumps({"t": round(time.time(), 2), **entry}, default=str) + "\n")
118
+ log.flush()
119
+
120
+ brief = {k: start[k] for k in ("run_id", "challenge", "seed", "title", "brief", "parameters", "access_level",
121
+ "budget_per_session", "sessions", "answer_format", "rules")}
122
+ messages: list[dict] = [{"role": "user", "content": "Challenge:\n" + json.dumps(brief, indent=1)}]
123
+ note({"type": "start", "challenge": challenge_id, "model": model, "run_id": run_id})
124
+ kwargs: dict = {"model": model, "max_tokens": max_tokens, "system": SYSTEM, "tools": tools,
125
+ "cache_control": {"type": "ephemeral"}}
126
+ if effort:
127
+ kwargs["output_config"] = {"effort": effort}
128
+ if thinking == "summarized":
129
+ kwargs["thinking"] = {"type": "adaptive", "display": "summarized"}
130
+ if model in FALLBACK_MODELS:
131
+ kwargs["betas"] = [FALLBACK_BETA]
132
+ kwargs["fallbacks"] = "default"
133
+ usage = {"input_tokens": 0, "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}
134
+ submitted = None
135
+ stop = "max_turns"
136
+ json_retries = 0
137
+ warned = nudged = False
138
+ turn = 0
139
+ while turn < max_turns:
140
+ turn += 1
141
+ try:
142
+ with client.beta.messages.stream(messages=messages, **kwargs) as stream:
143
+ for _event in stream:
144
+ pass
145
+ response = stream.get_final_message()
146
+ json_retries = 0
147
+ except ValueError:
148
+ # JSON the SDK could not parse at all (eager input streaming): no tool_use id to answer, re-issue
149
+ json_retries += 1
150
+ if json_retries > 2:
151
+ stop = "unparseable_tool_input"
152
+ break
153
+ turn -= 1
154
+ continue
155
+ for k in usage:
156
+ usage[k] += int(getattr(response.usage, k, 0) or 0)
157
+ for b in response.content:
158
+ d = _block_dict(b)
159
+ note({"turn": turn, **d})
160
+ said = d.get("text") if d["type"] == "text" else d.get("summary") if d["type"] == "thinking" else None
161
+ if said and str(said).strip() and note_sid:
162
+ tk.store.log_event(session_id=note_sid, actor="agent", kind="note", tool=d["type"],
163
+ result_summary=str(said).strip())
164
+ if verbose and d["type"] == "text" and d["text"].strip():
165
+ print(f"\n[{model}] {d['text'].strip()}\n", flush=True)
166
+ if verbose and d["type"] == "tool_use":
167
+ print(f" -> {d['name']}({json.dumps(d['input'])[:160]})", flush=True)
168
+
169
+ if response.stop_reason == "pause_turn":
170
+ messages.append({"role": "assistant", "content": response.content})
171
+ continue
172
+ tool_uses = [b for b in response.content if b.type == "tool_use"]
173
+ if response.stop_reason == "refusal":
174
+ stop = "refusal"
175
+ break
176
+ if response.stop_reason == "max_tokens" and tool_uses:
177
+ stop = "max_tokens"
178
+ break
179
+ messages.append({"role": "assistant", "content": response.content})
180
+ if not tool_uses:
181
+ if submitted is None and not nudged and turn < max_turns:
182
+ nudged = True
183
+ messages.append({"role": "user", "content": "You have not called challenge_submit yet. Submit your "
184
+ "answer now (once)."})
185
+ continue
186
+ stop = "end_turn"
187
+ break
188
+ results = []
189
+ for b in tool_uses:
190
+ err = _valid(b.input, schemas.get(b.name, {"type": "object"})) if b.name in schemas else \
191
+ f"unknown tool {b.name!r}"
192
+ if err:
193
+ content, is_error = json.dumps({"INVALID_JSON": json.dumps(b.input, default=str), "why": err}), True
194
+ else:
195
+ r = tk.call(b.name, dict(b.input), actor="agent")
196
+ is_error = isinstance(r, dict) and "error" in r
197
+ content = json.dumps(r, default=str)
198
+ if b.name == "challenge_submit" and not is_error:
199
+ submitted = r
200
+ results.append({"type": "tool_result", "tool_use_id": b.id, "content": content,
201
+ **({"is_error": True} if is_error else {})})
202
+ note({"turn": turn, "type": "tool_result", "name": b.name, "is_error": is_error, "chars": len(content)})
203
+ cost = _cost(model, usage)
204
+ if submitted is not None:
205
+ messages.append({"role": "user", "content": results})
206
+ stop = "submitted"
207
+ break
208
+ if not warned and ((cost is not None and cost > 0.8 * max_cost) or turn >= max_turns - 3):
209
+ warned = True
210
+ results.append({"type": "text", "text": "Runner note: you are close to the turn/cost limit for this "
211
+ "run. Submit your answer with challenge_submit within the next turn or two."})
212
+ messages.append({"role": "user", "content": results})
213
+ if cost is not None and cost > max_cost:
214
+ stop = "max_cost"
215
+ break
216
+
217
+ if submitted is None: # limits hit: score whatever state the run is in (empty answer)
218
+ submitted = tk.challenges.submit(run_id, {})
219
+ submitted["auto_submitted"] = True
220
+ cost = _cost(model, usage)
221
+ tk.store.execute("UPDATE challenge_runs SET usage_json=? WHERE id=?",
222
+ (json.dumps({**(submitted.get("usage") or {}), "api_usage": usage,
223
+ "estimated_cost_usd": round(cost, 3) if cost is not None else None,
224
+ "turns": turn, "stop": stop}), run_id))
225
+ result = {"run_id": run_id, "challenge": challenge_id, "seed": start["seed"], "model": model,
226
+ "score": submitted.get("score"),
227
+ "detail": submitted.get("detail"), "auto_submitted": submitted.get("auto_submitted", False),
228
+ "turns": turn, "stop": stop, "api_usage": usage,
229
+ "estimated_cost_usd": round(cost, 3) if cost is not None else None, "transcript": str(log_path)}
230
+ note({"type": "end", **result})
231
+ log.close()
232
+ return result
233
+
234
+
235
+ def main(argv: list[str]) -> None:
236
+ import argparse
237
+
238
+ from ..core.challenges import CHALLENGES
239
+ from ..core.service import Toolkit
240
+
241
+ ap = argparse.ArgumentParser(prog="wsp agent run", description=__doc__,
242
+ formatter_class=argparse.RawDescriptionHelpFormatter)
243
+ ap.add_argument("cmd", choices=["run", "bench"])
244
+ ap.add_argument("challenge", nargs="?", choices=sorted(CHALLENGES), help="(run only)")
245
+ ap.add_argument("--seed", type=int, help="instance seed (default: random)")
246
+ ap.add_argument("--model", default="claude-fable-5-2")
247
+ ap.add_argument("--effort", default="high", help="low|medium|high|xhigh|max, or 'none' to omit")
248
+ ap.add_argument("--thinking", default="summarized", choices=["summarized", "omit"])
249
+ ap.add_argument("--max-turns", type=int, default=80)
250
+ ap.add_argument("--max-cost", type=float, default=20.0, help="stop when the estimated cost exceeds this (USD)")
251
+ ap.add_argument("--runs", type=int, default=1, help="repeat the challenge N times")
252
+ a = ap.parse_args(argv)
253
+ try:
254
+ import anthropic
255
+ except ImportError as e:
256
+ raise SystemExit("the API runner needs the Anthropic SDK: pip install anthropic") from e
257
+ tk = Toolkit()
258
+ if a.cmd == "run" and not a.challenge:
259
+ raise SystemExit("usage: wsp agent run <challenge> --model ... (or: wsp bench run --model ...)")
260
+ plan = [(a.challenge, i) for i in range(a.runs)] if a.cmd == "run" else \
261
+ [(c, i) for c in CHALLENGES for i in range(a.runs)]
262
+ results = []
263
+ for cid, i in plan:
264
+ try:
265
+ r = run(tk, cid, a.model, effort=None if a.effort == "none" else a.effort, thinking=a.thinking,
266
+ max_turns=a.max_turns, max_cost=a.max_cost, label=f"api runner: {a.model} #{i + 1}",
267
+ seed=a.seed)
268
+ except anthropic.AuthenticationError as e:
269
+ raise SystemExit("Anthropic API authentication failed: set ANTHROPIC_API_KEY or run `ant auth login`") from e
270
+ except anthropic.NotFoundError as e:
271
+ raise SystemExit(f"model {a.model!r} was not found for your API key: {e.message}") from e
272
+ except anthropic.APIStatusError as e:
273
+ raise SystemExit(f"Anthropic API error {e.status_code}: {e.message}") from e
274
+ results.append(r)
275
+ print(json.dumps({k: r[k] for k in ("challenge", "seed", "run_id", "score", "turns", "stop",
276
+ "estimated_cost_usd")}), flush=True)
277
+ print(f"\nmean score over {len(results)} run(s): {sum(r['score'] or 0 for r in results) / len(results):.3f}")
278
+ total = sum(r["estimated_cost_usd"] or 0 for r in results)
279
+ print(f"estimated API cost: ${total:.2f}")
280
+ print(f"next: wsp leaderboard export --model {a.model} --by <your GitHub handle> (then submit it on the site)")
281
+
282
+
283
+ _ = Path
@@ -0,0 +1,146 @@
1
+ """``wsp arena``: animated replays of challenge runs.
2
+
3
+ wsp arena list recent challenge runs (ids to replay)
4
+ wsp arena export [RUN|latest] [--out F.html] one self-contained HTML file that plays the run
5
+ wsp arena timeline [RUN|latest] [--out F.json] the animation timeline as JSON
6
+ wsp arena record [RUN|latest] [--out F.mp4] frame-exact MP4 (needs the [video] extra + a Chromium:
7
+ pip install "wsp-toolkit[video]" && playwright install chromium)
8
+
9
+ The live view is in the web UI: wsp ui, then open http://127.0.0.1:8765/arena (it has a real-time Rec button too).
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import base64
16
+ import json
17
+ import os
18
+ import shutil
19
+ import subprocess
20
+ import sys
21
+ import tempfile
22
+ import time
23
+ from pathlib import Path
24
+
25
+ SIZES = {"landscape": "1920x1080", "square": "1080x1080", "portrait": "1080x1350", "vertical": "1080x1920"}
26
+
27
+
28
+ def _ffmpeg() -> str:
29
+ exe = shutil.which("ffmpeg")
30
+ if exe:
31
+ return exe
32
+ try:
33
+ import imageio_ffmpeg
34
+ return imageio_ffmpeg.get_ffmpeg_exe()
35
+ except ImportError:
36
+ sys.exit("ffmpeg not found: install it (brew install ffmpeg / apt install ffmpeg) or "
37
+ 'pip install "wsp-toolkit[video]"')
38
+
39
+
40
+ def record(html: Path, out: Path, size: str, fps: int = 30, crf: int = 18, start: float = 0.0,
41
+ end: float | None = None, verbose: bool = True) -> dict:
42
+ """Render every frame with headless Chromium (deterministic: render(t) per frame) and encode H.264 MP4."""
43
+ try:
44
+ from playwright.sync_api import sync_playwright
45
+ except ImportError:
46
+ sys.exit('video export needs Playwright: pip install "wsp-toolkit[video]" && playwright install chromium\n'
47
+ "(or open the run in `wsp ui` → Arena and press ● Rec to save a WebM in real time)")
48
+ w, h = (int(x) for x in size.lower().split("x"))
49
+ ff = _ffmpeg()
50
+ out.parent.mkdir(parents=True, exist_ok=True)
51
+ cmd = [ff, "-y", "-loglevel", "error", "-f", "image2pipe", "-framerate", str(fps), "-c:v", "mjpeg", "-i", "-",
52
+ "-c:v", "libx264", "-preset", "slow", "-crf", str(crf), "-pix_fmt", "yuv420p", "-movflags", "+faststart",
53
+ str(out)]
54
+ t0 = time.time()
55
+ with sync_playwright() as pw:
56
+ exe = os.environ.get("WSP_CHROMIUM")
57
+ browser = pw.chromium.launch(**({"executable_path": exe} if exe else {}))
58
+ page = browser.new_page(viewport={"width": min(w, 1920), "height": min(h, 1920)})
59
+ errors: list[str] = []
60
+ page.on("pageerror", lambda e: errors.append(str(e)))
61
+ page.goto(html.resolve().as_uri() + "?record")
62
+ page.wait_for_function("window.arenaReady === true", timeout=30000)
63
+ duration = float(page.evaluate("window.arena.duration"))
64
+ stop = min(duration, end) if end else duration
65
+ frames = int((stop - start) * fps)
66
+ enc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
67
+ batch = 8
68
+ for i in range(0, frames, batch):
69
+ times = [start + (i + k) / fps for k in range(min(batch, frames - i))]
70
+ urls = page.evaluate("""(ts) => ts.map(t => { window.arena.render(t);
71
+ return document.getElementById('arena').toDataURL('image/jpeg', 0.95); })""", times)
72
+ for u in urls:
73
+ enc.stdin.write(base64.b64decode(u.split(",", 1)[1]))
74
+ if verbose and (i // batch) % 30 == 0:
75
+ print(f"\r frame {i}/{frames}", end="", flush=True)
76
+ enc.stdin.close()
77
+ enc.wait()
78
+ browser.close()
79
+ if errors:
80
+ raise RuntimeError("renderer errors: " + "; ".join(errors[:3]))
81
+ if enc.returncode:
82
+ raise RuntimeError(f"ffmpeg failed with exit code {enc.returncode}")
83
+ if verbose:
84
+ print(f"\r {frames} frames in {time.time() - t0:.0f}s")
85
+ return {"out": str(out), "frames": frames, "seconds": round(frames / fps, 2), "size": f"{w}x{h}"}
86
+
87
+
88
+ def main(argv: list[str]) -> None:
89
+ ap = argparse.ArgumentParser(prog="wsp arena", description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
90
+ ap.add_argument("action", choices=["list", "export", "timeline", "record"])
91
+ ap.add_argument("run", nargs="?", default="latest", help="challenge run id (cr_...) or 'latest'")
92
+ ap.add_argument("--out")
93
+ ap.add_argument("--size", default="1920x1080",
94
+ help="WxH or one of: " + ", ".join(f"{k} ({v})" for k, v in SIZES.items()))
95
+ ap.add_argument("--fps", type=int, default=30)
96
+ ap.add_argument("--crf", type=int, default=18, help="H.264 quality (lower = better, bigger)")
97
+ ap.add_argument("--timeline", help="record/export from a timeline JSON file instead of the local database")
98
+ ap.add_argument("--note", help="caption stored with the timeline (shown on the website's arena page)")
99
+ a = ap.parse_args(argv)
100
+ size = SIZES.get(a.size, a.size)
101
+
102
+ from ..core.arena import build_timeline, standalone_html
103
+ from ..core.service import Toolkit
104
+
105
+ if a.action == "list":
106
+ tk = Toolkit()
107
+ rows = tk.store.all("SELECT id, challenge_id, seed, agent_model, score, started_at FROM challenge_runs "
108
+ "ORDER BY started_at DESC LIMIT 30")
109
+ for r in rows:
110
+ sc = f"{r['score']:.3f}" if r["score"] is not None else "live"
111
+ print(f"{r['id']} {r['challenge_id']:<20} seed {r['seed']:<8} {r['agent_model']:<24} {sc}")
112
+ if not rows:
113
+ print("no challenge runs yet")
114
+ return
115
+
116
+ if a.timeline:
117
+ tl = json.loads(Path(a.timeline).read_text())
118
+ else:
119
+ tk = Toolkit()
120
+ rid = a.run
121
+ if rid == "latest":
122
+ row = tk.store.one("SELECT id FROM challenge_runs ORDER BY started_at DESC LIMIT 1")
123
+ if not row:
124
+ sys.exit("no challenge runs yet")
125
+ rid = row["id"]
126
+ tl = build_timeline(tk, rid)
127
+ if a.note:
128
+ tl["note"] = a.note
129
+ stem = f"wsp-arena-{tl['run']['challenge']}-s{tl['run']['seed']}"
130
+ if a.action == "timeline":
131
+ out = Path(a.out or stem + ".json")
132
+ out.write_text(json.dumps(tl, indent=1))
133
+ print(out)
134
+ elif a.action == "export":
135
+ out = Path(a.out or stem + ".html")
136
+ out.write_text(standalone_html(tl, size))
137
+ print(f"{out} (open it in a browser: space = play/pause, ←/→ = seek)")
138
+ else:
139
+ out = Path(a.out or f"{stem}-{size}.mp4")
140
+ with tempfile.TemporaryDirectory() as d:
141
+ html = Path(d) / "arena.html"
142
+ html.write_text(standalone_html(tl, size))
143
+ print(f"recording {tl['run']['challenge']} (seed {tl['run']['seed']}, {tl['run']['model']}): "
144
+ f"{tl['duration']:.0f}s at {a.fps} fps, {size}")
145
+ res = record(html, out, size, a.fps, a.crf)
146
+ print(json.dumps(res))
@@ -0,0 +1,77 @@
1
+ """Scripted repair agent (no LLM) for the M3 fixture demo: find and fix the planted bug in induction_tf_broken
2
+ with a forward-pass budget, using practice data only, then check once on held-out (aggregate only).
3
+
4
+ Steps:
5
+ 1. practice suite (limited) -> confirm the copy behaviour is broken
6
+ 2. probe_attention -> layer-1 heads still attend like induction heads, so the *routing* is intact
7
+ 3. tensor_stats on each o_proj -> compare per-head output norms across layers; flag heads whose o_proj block is
8
+ far smaller than the median head norm of the model
9
+ 4. hypothesis: those heads' outputs were scaled down. Try a few restore factors (each costs a practice run),
10
+ keep the best, revert the others
11
+ 5. one held-out evaluation of the final working copy
12
+ Every step is a logged tool call, visible in the UI.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import statistics
18
+
19
+
20
+ def run(call, source: str = "fixture:induction_tf_broken", budget_hint: int = 50, practice_limit: int = 8,
21
+ factors=(0.75, 1.0, 1.3), session_id: str | None = None, final_heldout: bool = True) -> dict:
22
+ log = []
23
+ if session_id:
24
+ o = call("workspace_info", session_id=session_id)
25
+ sid = session_id
26
+ else:
27
+ o = call("workspace_open", source=source, label="bugfix-agent (scripted)")
28
+ sid = o["session_id"]
29
+ n_layers = o["n_layers"]
30
+ suites = call("suite_list", session_id=sid)["suites"]
31
+ practice = next(s["id"] for s in suites if s["split"] == "practice")
32
+ heldout = next((s["id"] for s in suites if s["split"] == "heldout"), None)
33
+ base = call("suite_run_practice", session_id=sid, suite_id=practice, limit=practice_limit)
34
+ log.append(f"practice score before: {base['metrics']['score']}")
35
+
36
+ ex = next(s for s in suites if s["id"] == practice)["example_preview"][0]["input"]
37
+ att = call("probe_attention", session_id=sid, inputs=[ex], layers=[n_layers - 1], top_k_keys=1)
38
+ att_heads = att.get("heads", [])
39
+ log.append(f"probed attention on layer {n_layers - 1}: {len(att_heads)} heads")
40
+
41
+ norms = {}
42
+ for li in range(n_layers):
43
+ st = call("tensor_stats", session_id=sid, tensor=f"blocks.{li}.attn.o_proj.weight")
44
+ for h, n in enumerate(st["head_norms"]):
45
+ norms[(li, h)] = n
46
+ top = max(norms.values())
47
+ weak = [(li, h) for (li, h), n in norms.items() if n < 0.35 * top]
48
+ healthy = [n for k, n in norms.items() if k not in weak] or [top]
49
+ med = statistics.median(healthy)
50
+ log.append(f"healthy-head median output norm {med:.3g}; suspiciously small heads: {weak}")
51
+ if not weak:
52
+ return {"session_id": sid, "fixed": False, "log": log}
53
+
54
+ best = {"score": base["metrics"]["score"], "patch": None, "factor": None}
55
+ for f in factors: # multipliers of "restore each weak head to the healthy median norm"
56
+ ops = [{"op": "scale", "tensor": f"blocks.{li}.attn.o_proj.weight", "head": h,
57
+ "factor": round(f * med / norms[(li, h)], 4)} for li, h in weak]
58
+ p = call("edit_apply", session_id=sid, operations=ops,
59
+ rationale=f"Heads {weak} have o_proj output blocks under 35% of the largest head's norm; restoring "
60
+ f"them to {f}x the healthy median norm ({med:.3g}).")
61
+ r = call("suite_run_practice", session_id=sid, suite_id=practice, limit=practice_limit)
62
+ sc = r["metrics"]["score"]
63
+ log.append(f"factor {f}: patch {p['patch_id']} practice score {sc}; regression {p.get('regression')}")
64
+ if sc > best["score"]:
65
+ best = {"score": sc, "patch": p["patch_id"], "factor": f}
66
+ call("patch_revert", session_id=sid, patch_id=p["patch_id"])
67
+ if best["patch"] is None:
68
+ return {"session_id": sid, "fixed": False, "log": log}
69
+ call("patch_reapply", session_id=sid, patch_id=best["patch"])
70
+ if not final_heldout:
71
+ return {"session_id": sid, "fixed": True, "patch_id": best["patch"], "factor": best["factor"], "log": log}
72
+ ho = call("evaluate_heldout", session_id=sid, suite_id=heldout)
73
+ log.append(f"kept factor {best['factor']} ({best['patch']}); held-out score {ho['metrics']['score']}")
74
+ b = call("budget_status", session_id=sid)
75
+ return {"session_id": sid, "fixed": True, "patch_id": best["patch"], "factor": best["factor"],
76
+ "practice_before": base["metrics"]["score"], "practice_after": best["score"],
77
+ "heldout_after": ho["metrics"]["score"], "forward_passes_used": b["forward_passes"]["used"], "log": log}
@@ -0,0 +1,129 @@
1
+ """Scripted (no-LLM) reference agents for each challenge, so the leaderboard has a baseline row.
2
+
3
+ wsp challenge baseline # runs every challenge once as agent_model "scripted-baseline"
4
+
5
+ They use only the toolkit's tools, inside the same sessions, access levels and budgets a model gets.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import statistics
11
+
12
+ from . import bugfix_agent, zoo_baseline
13
+ from .clients import InProcessClient, ToolError
14
+ from .induction_finder import find_induction_heads
15
+
16
+ NAME = "scripted-baseline"
17
+
18
+
19
+ def _blind(call, sid: str) -> None:
20
+ """Weights only: scale up attention heads whose output (o_proj) norm is far below the median."""
21
+ info = call("workspace_info", session_id=sid)
22
+ norms = {}
23
+ for li in range(info["n_layers"]):
24
+ for h, n in enumerate(call("tensor_stats", session_id=sid, tensor=f"blocks.{li}.attn.o_proj.weight")
25
+ ["head_norms"]):
26
+ norms[(li, h)] = n
27
+ top = max(norms.values())
28
+ weak = {k for k, n in norms.items() if 0 < n < 0.35 * top}
29
+ med = statistics.median([n for k, n in norms.items() if k not in weak] or [top])
30
+ ops = [{"op": "scale", "tensor": f"blocks.{li}.attn.o_proj.weight", "head": h, "factor": round(med / norms[(li, h)], 3)}
31
+ for (li, h) in sorted(weak)]
32
+ if ops:
33
+ call("edit_apply", session_id=sid, operations=ops,
34
+ rationale="heads with o_proj norm < 35% of the largest look scaled down; restore them to the healthy median")
35
+
36
+
37
+ def _mystery(call, ref: str, sus: str) -> dict:
38
+ """Free weight diff: the trigger's embedding row and the forced token's unembedding row changed most."""
39
+ wd = call("diff_models", session_id=ref, other_session_id=sus)
40
+ rows = {t["tensor"]: t for t in wd["most_changed_tensors"]}
41
+ trig = rows.get("embed.weight", {}).get("top_rows", [[None]])[0][0]
42
+ forced = rows.get("unembed.weight", {}).get("top_rows", [[None]])[0][0]
43
+ return {"trigger_token": f"t{trig}" if trig is not None else None,
44
+ "forced_token": f"t{forced}" if forced is not None else None}
45
+
46
+
47
+ def _zoo(call, sessions: list[dict]) -> dict:
48
+ answers = {}
49
+ for s in sessions:
50
+ card = call("zoo_get_task", net_id=s["net_id"], task_type="localize")["card"]
51
+ try:
52
+ answers[s["net_id"]] = zoo_baseline.localize(call, s["session_id"], card)
53
+ except ToolError:
54
+ answers[s["net_id"]] = {"flaw": "none", "location": {}}
55
+ return {"answers": answers}
56
+
57
+
58
+ def _token_ban(call, sid: str, target: str) -> None:
59
+ """Zero the banned token's unembedding row (its logit becomes 0 everywhere)."""
60
+ tid = int(target.lstrip("t"))
61
+ call("edit_apply", session_id=sid, rationale=f"zero the unembedding row of {target}",
62
+ operations=[{"op": "zero", "tensor": "unembed.weight", "slice": f"{tid}, :"}])
63
+
64
+
65
+ def _install_rule(call, sid: str, trigger: str, output: str) -> None:
66
+ """Naive rank-1 write: add the trigger's embedding direction to the output token's unembedding row."""
67
+ a, b = int(trigger.lstrip("t")), int(output.lstrip("t"))
68
+ call("edit_apply", session_id=sid, rationale=f"rank-1 rule {trigger} -> {output} via the unembedding",
69
+ operations=[{"op": "add_low_rank", "tensor": "unembed.weight", "coeffs": [10.0],
70
+ "u": [{"vector": [1.0 if i == b else 0.0 for i in range(32)]}],
71
+ "v": [{"tensor": "embed.weight", "row": a}]}])
72
+
73
+
74
+ def _name_functions(call, sessions: list[dict]) -> dict:
75
+ """Weights-only heuristic: which inputs have non-negligible first-layer weight, then the first candidate label
76
+ whose relevant inputs match. (An agent that simulates the network exactly would score 1.0.)"""
77
+ from ..core.zoo.generators import BOOL_TASKS
78
+
79
+ def relevant(f):
80
+ return {i for i in range(4) if any(f([(n >> b) & 1 for b in range(4)]) !=
81
+ f([((n ^ (1 << i)) >> b) & 1 for b in range(4)]) for n in range(16))}
82
+ answers = {}
83
+ for s in sessions:
84
+ st = call("tensor_stats", session_id=s["session_id"], tensor="layers.0.weight")
85
+ cols = {c["index"]: c["norm"] for c in st["cols"]["top_cols"] + st["cols"]["bottom_cols"]}
86
+ mx = max(cols.values()) or 1.0
87
+ used = {i for i, n in cols.items() if n > 0.15 * mx and i < 4}
88
+ cands = [lab for lab, f in sorted(BOOL_TASKS.items()) if relevant(f) == used]
89
+ answers[s["net_id"]] = cands[0] if cands else "and_01"
90
+ return {"answers": answers}
91
+
92
+
93
+ def run_all(tk, challenges: list[str] | None = None, seed: int | None = None) -> list[dict]:
94
+ from ..core.challenges import CHALLENGES
95
+ from ..mcp_server.server import lock_to_challenge
96
+
97
+ call = InProcessClient(tk)
98
+ out = []
99
+ for cid in challenges or list(CHALLENGES):
100
+ lock_to_challenge(tk, cid)
101
+ r = tk.challenges.start(cid, NAME, "scripted (tools only)", "scripted baseline", "agent", seed)
102
+ ss = r["sessions"]
103
+ if cid == "induction-hunt":
104
+ rep = find_induction_heads(call, session_id=ss[0]["session_id"])
105
+ ans = {"induction_heads": rep["induction_heads"], "previous_token_heads": rep["previous_token_heads"]}
106
+ elif cid == "fix-the-copier":
107
+ bugfix_agent.run(call, session_id=ss[0]["session_id"], final_heldout=False)
108
+ ans = {}
109
+ elif cid == "blind-surgeon":
110
+ _blind(call, ss[0]["session_id"])
111
+ ans = {}
112
+ elif cid == "mystery-finetune":
113
+ ref = next(s["session_id"] for s in ss if s["role"] == "reference")
114
+ sus = next(s["session_id"] for s in ss if s["role"] == "suspect")
115
+ ans = _mystery(call, ref, sus)
116
+ elif cid == "token-ban":
117
+ _token_ban(call, ss[0]["session_id"], r["parameters"]["target"])
118
+ ans = {}
119
+ elif cid == "install-a-rule":
120
+ _install_rule(call, ss[0]["session_id"], r["parameters"]["trigger"], r["parameters"]["output"])
121
+ ans = {}
122
+ elif cid == "name-that-function":
123
+ ans = _name_functions(call, ss)
124
+ elif cid == "backdoor-removal":
125
+ ans = {} # no weights-only heuristic for finding an unknown trigger: the baseline makes no edit
126
+ else:
127
+ ans = _zoo(call, ss)
128
+ out.append(call("challenge_submit", run_id=r["run_id"], answer=ans))
129
+ return out