wsp-toolkit 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- wsp_toolkit/__init__.py +1 -0
- wsp_toolkit/agents/__init__.py +0 -0
- wsp_toolkit/agents/api_runner.py +283 -0
- wsp_toolkit/agents/arena_cli.py +146 -0
- wsp_toolkit/agents/bugfix_agent.py +77 -0
- wsp_toolkit/agents/challenge_baseline.py +129 -0
- wsp_toolkit/agents/challenge_cli.py +64 -0
- wsp_toolkit/agents/clients.py +93 -0
- wsp_toolkit/agents/demos.py +30 -0
- wsp_toolkit/agents/induction_finder.py +62 -0
- wsp_toolkit/agents/leaderboard_cli.py +51 -0
- wsp_toolkit/agents/zoo_baseline.py +188 -0
- wsp_toolkit/api/__init__.py +0 -0
- wsp_toolkit/api/app.py +277 -0
- wsp_toolkit/api/views.py +215 -0
- wsp_toolkit/challenge_defs.py +136 -0
- wsp_toolkit/cli.py +103 -0
- wsp_toolkit/config.py +77 -0
- wsp_toolkit/core/__init__.py +0 -0
- wsp_toolkit/core/arena.py +327 -0
- wsp_toolkit/core/challenges.py +538 -0
- wsp_toolkit/core/db.py +288 -0
- wsp_toolkit/core/diff/__init__.py +0 -0
- wsp_toolkit/core/diff/diff.py +316 -0
- wsp_toolkit/core/diff/report.py +108 -0
- wsp_toolkit/core/diff/service.py +84 -0
- wsp_toolkit/core/diff/validate.py +86 -0
- wsp_toolkit/core/directions.py +138 -0
- wsp_toolkit/core/edit/__init__.py +0 -0
- wsp_toolkit/core/edit/ops.py +137 -0
- wsp_toolkit/core/evals/__init__.py +0 -0
- wsp_toolkit/core/evals/benchmarks.py +66 -0
- wsp_toolkit/core/evals/harness.py +208 -0
- wsp_toolkit/core/evals/scorers.py +210 -0
- wsp_toolkit/core/evals/worker.py +48 -0
- wsp_toolkit/core/extras.py +104 -0
- wsp_toolkit/core/features/__init__.py +0 -0
- wsp_toolkit/core/features/cli.py +24 -0
- wsp_toolkit/core/features/index.py +444 -0
- wsp_toolkit/core/features/sae.py +99 -0
- wsp_toolkit/core/features/service.py +78 -0
- wsp_toolkit/core/inspect/__init__.py +0 -0
- wsp_toolkit/core/inspect/tensors.py +221 -0
- wsp_toolkit/core/leaderboard.py +174 -0
- wsp_toolkit/core/patch/__init__.py +0 -0
- wsp_toolkit/core/patch/patches.py +312 -0
- wsp_toolkit/core/probe/__init__.py +0 -0
- wsp_toolkit/core/probe/probe.py +177 -0
- wsp_toolkit/core/safety/__init__.py +0 -0
- wsp_toolkit/core/safety/netguard.py +86 -0
- wsp_toolkit/core/safety/policy.py +124 -0
- wsp_toolkit/core/service.py +723 -0
- wsp_toolkit/core/util.py +180 -0
- wsp_toolkit/core/workspace/__init__.py +0 -0
- wsp_toolkit/core/workspace/adapters.py +606 -0
- wsp_toolkit/core/workspace/archs.py +157 -0
- wsp_toolkit/core/workspace/workspace.py +302 -0
- wsp_toolkit/core/zoo/__init__.py +0 -0
- wsp_toolkit/core/zoo/cli.py +30 -0
- wsp_toolkit/core/zoo/generators.py +520 -0
- wsp_toolkit/core/zoo/zoo.py +174 -0
- wsp_toolkit/data/suites/lm_builtin.json +156 -0
- wsp_toolkit/default_config.toml +69 -0
- wsp_toolkit/fixtures/__init__.py +0 -0
- wsp_toolkit/fixtures/finetunes.py +131 -0
- wsp_toolkit/fixtures/train.py +285 -0
- wsp_toolkit/mcp_server/__init__.py +0 -0
- wsp_toolkit/mcp_server/__main__.py +3 -0
- wsp_toolkit/mcp_server/server.py +101 -0
- wsp_toolkit/site/__init__.py +0 -0
- wsp_toolkit/site/build.py +91 -0
- wsp_toolkit/site/templates/_board.html +13 -0
- wsp_toolkit/site/templates/_layout.html +28 -0
- wsp_toolkit/site/templates/arena.html +82 -0
- wsp_toolkit/site/templates/index.html +63 -0
- wsp_toolkit/site/templates/leaderboard.html +22 -0
- wsp_toolkit/site/templates/site.css +200 -0
- wsp_toolkit/site/templates/submit.html +21 -0
- wsp_toolkit/ui/static/arena.js +1500 -0
- wsp_toolkit/ui/static/fonts/OFL-JetBrainsMono.txt +93 -0
- wsp_toolkit/ui/static/fonts/OFL-Orbitron.txt +93 -0
- wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-400-normal.woff2 +0 -0
- wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-700-normal.woff2 +0 -0
- wsp_toolkit/ui/static/fonts/orbitron-latin-600-normal.woff2 +0 -0
- wsp_toolkit/ui/static/fonts/orbitron-latin-800-normal.woff2 +0 -0
- wsp_toolkit/ui/static/htmx.min.js +1 -0
- wsp_toolkit/ui/static/style.css +123 -0
- wsp_toolkit/ui/templates/_budget.html +1 -0
- wsp_toolkit/ui/templates/_events.html +3 -0
- wsp_toolkit/ui/templates/_heatmap.html +2 -0
- wsp_toolkit/ui/templates/_rows.html +1 -0
- wsp_toolkit/ui/templates/arena.html +112 -0
- wsp_toolkit/ui/templates/audit.html +2 -0
- wsp_toolkit/ui/templates/audits.html +1 -0
- wsp_toolkit/ui/templates/base.html +12 -0
- wsp_toolkit/ui/templates/challenges.html +17 -0
- wsp_toolkit/ui/templates/feature.html +3 -0
- wsp_toolkit/ui/templates/features.html +4 -0
- wsp_toolkit/ui/templates/home.html +21 -0
- wsp_toolkit/ui/templates/leaderboard.html +2 -0
- wsp_toolkit/ui/templates/model.html +11 -0
- wsp_toolkit/ui/templates/models.html +2 -0
- wsp_toolkit/ui/templates/patch.html +17 -0
- wsp_toolkit/ui/templates/patches.html +3 -0
- wsp_toolkit/ui/templates/registry.html +4 -0
- wsp_toolkit/ui/templates/results.html +3 -0
- wsp_toolkit/ui/templates/session.html +8 -0
- wsp_toolkit/ui/templates/sessions.html +3 -0
- wsp_toolkit/ui/templates/zoo.html +3 -0
- wsp_toolkit/ui/templates/zoo_net.html +4 -0
- wsp_toolkit-0.3.0.dist-info/METADATA +238 -0
- wsp_toolkit-0.3.0.dist-info/RECORD +116 -0
- wsp_toolkit-0.3.0.dist-info/WHEEL +5 -0
- wsp_toolkit-0.3.0.dist-info/entry_points.txt +3 -0
- wsp_toolkit-0.3.0.dist-info/licenses/LICENSE +202 -0
- wsp_toolkit-0.3.0.dist-info/top_level.txt +1 -0
wsp_toolkit/__init__.py
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
__version__ = "0.3.0"
|
|
File without changes
|
|
@@ -0,0 +1,283 @@
|
|
|
1
|
+
"""Benchmark a Claude model on a WSP challenge through the Anthropic API.
|
|
2
|
+
|
|
3
|
+
pip install anthropic
|
|
4
|
+
export ANTHROPIC_API_KEY=... # or `ant auth login`
|
|
5
|
+
wsp agent run fix-the-copier --model claude-fable-5-2
|
|
6
|
+
|
|
7
|
+
The model gets ONLY the toolkit's tools (no shell, no files), so these runs are the comparable ones on the
|
|
8
|
+
challenge leaderboard (harness "wsp-agent-runner"). The server is locked to the challenge's models.
|
|
9
|
+
|
|
10
|
+
Loop: manual tool-use loop over streaming Messages requests (append-only history, so preserved thinking works),
|
|
11
|
+
eager input streaming with client-side schema validation, automatic prompt caching, server-side refusal fallback
|
|
12
|
+
on models that support it, and hard limits on turns and estimated cost.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import json
|
|
18
|
+
import time
|
|
19
|
+
from pathlib import Path
|
|
20
|
+
from typing import Any
|
|
21
|
+
|
|
22
|
+
# Tools the model does not get: the runner opens the challenge itself, and the model must not open other models
|
|
23
|
+
# or write to the zoo leaderboard.
|
|
24
|
+
EXCLUDED_TOOLS = {"workspace_open", "session_start", "challenge_start", "challenge_list", "challenge_scores",
|
|
25
|
+
"zoo_list_tasks", "zoo_submit", "zoo_scores",
|
|
26
|
+
"narrate"} # the runner already shows the model's text and thinking as live captions
|
|
27
|
+
|
|
28
|
+
# Models that accept server-side refusal fallbacks (`fallbacks: "default"`).
|
|
29
|
+
FALLBACK_MODELS = {"claude-fable-5-2", "claude-fable-5-1", "claude-mythos-5-2", "claude-mythos-5-1",
|
|
30
|
+
"claude-opus-5-5", "claude-opus-5", "claude-sonnet-5-5"}
|
|
31
|
+
FALLBACK_BETA = "server-side-fallback-2026-07-01"
|
|
32
|
+
|
|
33
|
+
# $ per million tokens: (input, output, cache read). Cache writes are estimated at 1.25x input. Estimates only.
|
|
34
|
+
PRICES = {"claude-fable-5-2": (8.0, 40.0, 0.20), "claude-mythos-5-2": (8.0, 40.0, 0.20),
|
|
35
|
+
"claude-fable-5-1": (10.0, 50.0, 1.0), "claude-fable-5": (10.0, 50.0, 1.0),
|
|
36
|
+
"claude-opus-5-5": (4.0, 20.0, 0.20), "claude-opus-5": (5.0, 25.0, 0.5),
|
|
37
|
+
"claude-opus-4-8": (5.0, 25.0, 0.5), "claude-sonnet-5-5": (2.0, 10.0, 0.10),
|
|
38
|
+
"claude-sonnet-5": (2.0, 10.0, 0.2), "claude-haiku-5-5": (0.10, 0.50, 0.01)}
|
|
39
|
+
|
|
40
|
+
SYSTEM = """You are an AI agent doing weight-space programming: you read, probe and edit the weights of small \
|
|
41
|
+
neural networks directly, through the WSP Toolkit's tools. Weight tools are free; forward passes are budgeted per \
|
|
42
|
+
session; some tools are unavailable at lower access levels. Tool results are JSON; very large results come back as \
|
|
43
|
+
a handle you can page with result_page. Edits must go through edit_apply with a rationale; you can revert them.
|
|
44
|
+
|
|
45
|
+
You are being evaluated on a challenge. Work it out from the evidence the tools give you, then call \
|
|
46
|
+
challenge_submit exactly once with your answer. Your final message should briefly say what you found."""
|
|
47
|
+
|
|
48
|
+
|
|
49
|
+
def tool_definitions(tk) -> list[dict]:
|
|
50
|
+
"""The same JSON schemas the MCP server advertises, minus excluded tools, for the Messages API."""
|
|
51
|
+
import anyio
|
|
52
|
+
|
|
53
|
+
from ..mcp_server.server import build_server
|
|
54
|
+
|
|
55
|
+
server = build_server(tk)
|
|
56
|
+
tools = anyio.run(server.list_tools)
|
|
57
|
+
out = []
|
|
58
|
+
for t in tools:
|
|
59
|
+
if t.name in EXCLUDED_TOOLS:
|
|
60
|
+
continue
|
|
61
|
+
out.append({"name": t.name, "description": t.description or t.name, "input_schema": t.input_schema,
|
|
62
|
+
"eager_input_streaming": True})
|
|
63
|
+
return out
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _valid(args: Any, schema: dict) -> str | None:
|
|
67
|
+
import jsonschema
|
|
68
|
+
|
|
69
|
+
if not isinstance(args, dict):
|
|
70
|
+
return "tool input is not an object"
|
|
71
|
+
try:
|
|
72
|
+
jsonschema.validate(args, schema)
|
|
73
|
+
except jsonschema.ValidationError as e:
|
|
74
|
+
return e.message
|
|
75
|
+
return None
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _cost(model: str, u: dict) -> float | None:
|
|
79
|
+
p = PRICES.get(model)
|
|
80
|
+
if not p:
|
|
81
|
+
return None
|
|
82
|
+
pin, pout, pread = p
|
|
83
|
+
return (u["input_tokens"] * pin + u["cache_creation_input_tokens"] * pin * 1.25 +
|
|
84
|
+
u["cache_read_input_tokens"] * pread + u["output_tokens"] * pout) / 1e6
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def _block_dict(b) -> dict:
|
|
88
|
+
t = getattr(b, "type", None)
|
|
89
|
+
if t == "text":
|
|
90
|
+
return {"type": "text", "text": b.text}
|
|
91
|
+
if t == "thinking":
|
|
92
|
+
return {"type": "thinking", "summary": getattr(b, "thinking", "")}
|
|
93
|
+
if t == "tool_use":
|
|
94
|
+
return {"type": "tool_use", "name": b.name, "input": b.input}
|
|
95
|
+
return {"type": str(t)}
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def run(tk, challenge_id: str, model: str, client=None, effort: str | None = "high", thinking: str = "summarized",
|
|
99
|
+
max_turns: int = 80, max_cost: float = 20.0, max_tokens: int = 64000, label: str | None = None,
|
|
100
|
+
verbose: bool = True, seed: int | None = None) -> dict:
|
|
101
|
+
from ..mcp_server.server import lock_to_challenge
|
|
102
|
+
|
|
103
|
+
if client is None:
|
|
104
|
+
import anthropic
|
|
105
|
+
client = anthropic.Anthropic()
|
|
106
|
+
lock_to_challenge(tk, challenge_id)
|
|
107
|
+
start = tk.challenges.start(challenge_id, model, "wsp-agent-runner", label or f"api runner: {model}", "agent",
|
|
108
|
+
seed)
|
|
109
|
+
run_id = start["run_id"]
|
|
110
|
+
note_sid = (start.get("sessions") or [{}])[0].get("session_id")
|
|
111
|
+
tools = tool_definitions(tk)
|
|
112
|
+
schemas = {t["name"]: t["input_schema"] for t in tools}
|
|
113
|
+
log_path = tk.cfg.subdir("agent_runs") / f"{run_id}.jsonl"
|
|
114
|
+
log = log_path.open("a")
|
|
115
|
+
|
|
116
|
+
def note(entry: dict) -> None:
|
|
117
|
+
log.write(json.dumps({"t": round(time.time(), 2), **entry}, default=str) + "\n")
|
|
118
|
+
log.flush()
|
|
119
|
+
|
|
120
|
+
brief = {k: start[k] for k in ("run_id", "challenge", "seed", "title", "brief", "parameters", "access_level",
|
|
121
|
+
"budget_per_session", "sessions", "answer_format", "rules")}
|
|
122
|
+
messages: list[dict] = [{"role": "user", "content": "Challenge:\n" + json.dumps(brief, indent=1)}]
|
|
123
|
+
note({"type": "start", "challenge": challenge_id, "model": model, "run_id": run_id})
|
|
124
|
+
kwargs: dict = {"model": model, "max_tokens": max_tokens, "system": SYSTEM, "tools": tools,
|
|
125
|
+
"cache_control": {"type": "ephemeral"}}
|
|
126
|
+
if effort:
|
|
127
|
+
kwargs["output_config"] = {"effort": effort}
|
|
128
|
+
if thinking == "summarized":
|
|
129
|
+
kwargs["thinking"] = {"type": "adaptive", "display": "summarized"}
|
|
130
|
+
if model in FALLBACK_MODELS:
|
|
131
|
+
kwargs["betas"] = [FALLBACK_BETA]
|
|
132
|
+
kwargs["fallbacks"] = "default"
|
|
133
|
+
usage = {"input_tokens": 0, "output_tokens": 0, "cache_creation_input_tokens": 0, "cache_read_input_tokens": 0}
|
|
134
|
+
submitted = None
|
|
135
|
+
stop = "max_turns"
|
|
136
|
+
json_retries = 0
|
|
137
|
+
warned = nudged = False
|
|
138
|
+
turn = 0
|
|
139
|
+
while turn < max_turns:
|
|
140
|
+
turn += 1
|
|
141
|
+
try:
|
|
142
|
+
with client.beta.messages.stream(messages=messages, **kwargs) as stream:
|
|
143
|
+
for _event in stream:
|
|
144
|
+
pass
|
|
145
|
+
response = stream.get_final_message()
|
|
146
|
+
json_retries = 0
|
|
147
|
+
except ValueError:
|
|
148
|
+
# JSON the SDK could not parse at all (eager input streaming): no tool_use id to answer, re-issue
|
|
149
|
+
json_retries += 1
|
|
150
|
+
if json_retries > 2:
|
|
151
|
+
stop = "unparseable_tool_input"
|
|
152
|
+
break
|
|
153
|
+
turn -= 1
|
|
154
|
+
continue
|
|
155
|
+
for k in usage:
|
|
156
|
+
usage[k] += int(getattr(response.usage, k, 0) or 0)
|
|
157
|
+
for b in response.content:
|
|
158
|
+
d = _block_dict(b)
|
|
159
|
+
note({"turn": turn, **d})
|
|
160
|
+
said = d.get("text") if d["type"] == "text" else d.get("summary") if d["type"] == "thinking" else None
|
|
161
|
+
if said and str(said).strip() and note_sid:
|
|
162
|
+
tk.store.log_event(session_id=note_sid, actor="agent", kind="note", tool=d["type"],
|
|
163
|
+
result_summary=str(said).strip())
|
|
164
|
+
if verbose and d["type"] == "text" and d["text"].strip():
|
|
165
|
+
print(f"\n[{model}] {d['text'].strip()}\n", flush=True)
|
|
166
|
+
if verbose and d["type"] == "tool_use":
|
|
167
|
+
print(f" -> {d['name']}({json.dumps(d['input'])[:160]})", flush=True)
|
|
168
|
+
|
|
169
|
+
if response.stop_reason == "pause_turn":
|
|
170
|
+
messages.append({"role": "assistant", "content": response.content})
|
|
171
|
+
continue
|
|
172
|
+
tool_uses = [b for b in response.content if b.type == "tool_use"]
|
|
173
|
+
if response.stop_reason == "refusal":
|
|
174
|
+
stop = "refusal"
|
|
175
|
+
break
|
|
176
|
+
if response.stop_reason == "max_tokens" and tool_uses:
|
|
177
|
+
stop = "max_tokens"
|
|
178
|
+
break
|
|
179
|
+
messages.append({"role": "assistant", "content": response.content})
|
|
180
|
+
if not tool_uses:
|
|
181
|
+
if submitted is None and not nudged and turn < max_turns:
|
|
182
|
+
nudged = True
|
|
183
|
+
messages.append({"role": "user", "content": "You have not called challenge_submit yet. Submit your "
|
|
184
|
+
"answer now (once)."})
|
|
185
|
+
continue
|
|
186
|
+
stop = "end_turn"
|
|
187
|
+
break
|
|
188
|
+
results = []
|
|
189
|
+
for b in tool_uses:
|
|
190
|
+
err = _valid(b.input, schemas.get(b.name, {"type": "object"})) if b.name in schemas else \
|
|
191
|
+
f"unknown tool {b.name!r}"
|
|
192
|
+
if err:
|
|
193
|
+
content, is_error = json.dumps({"INVALID_JSON": json.dumps(b.input, default=str), "why": err}), True
|
|
194
|
+
else:
|
|
195
|
+
r = tk.call(b.name, dict(b.input), actor="agent")
|
|
196
|
+
is_error = isinstance(r, dict) and "error" in r
|
|
197
|
+
content = json.dumps(r, default=str)
|
|
198
|
+
if b.name == "challenge_submit" and not is_error:
|
|
199
|
+
submitted = r
|
|
200
|
+
results.append({"type": "tool_result", "tool_use_id": b.id, "content": content,
|
|
201
|
+
**({"is_error": True} if is_error else {})})
|
|
202
|
+
note({"turn": turn, "type": "tool_result", "name": b.name, "is_error": is_error, "chars": len(content)})
|
|
203
|
+
cost = _cost(model, usage)
|
|
204
|
+
if submitted is not None:
|
|
205
|
+
messages.append({"role": "user", "content": results})
|
|
206
|
+
stop = "submitted"
|
|
207
|
+
break
|
|
208
|
+
if not warned and ((cost is not None and cost > 0.8 * max_cost) or turn >= max_turns - 3):
|
|
209
|
+
warned = True
|
|
210
|
+
results.append({"type": "text", "text": "Runner note: you are close to the turn/cost limit for this "
|
|
211
|
+
"run. Submit your answer with challenge_submit within the next turn or two."})
|
|
212
|
+
messages.append({"role": "user", "content": results})
|
|
213
|
+
if cost is not None and cost > max_cost:
|
|
214
|
+
stop = "max_cost"
|
|
215
|
+
break
|
|
216
|
+
|
|
217
|
+
if submitted is None: # limits hit: score whatever state the run is in (empty answer)
|
|
218
|
+
submitted = tk.challenges.submit(run_id, {})
|
|
219
|
+
submitted["auto_submitted"] = True
|
|
220
|
+
cost = _cost(model, usage)
|
|
221
|
+
tk.store.execute("UPDATE challenge_runs SET usage_json=? WHERE id=?",
|
|
222
|
+
(json.dumps({**(submitted.get("usage") or {}), "api_usage": usage,
|
|
223
|
+
"estimated_cost_usd": round(cost, 3) if cost is not None else None,
|
|
224
|
+
"turns": turn, "stop": stop}), run_id))
|
|
225
|
+
result = {"run_id": run_id, "challenge": challenge_id, "seed": start["seed"], "model": model,
|
|
226
|
+
"score": submitted.get("score"),
|
|
227
|
+
"detail": submitted.get("detail"), "auto_submitted": submitted.get("auto_submitted", False),
|
|
228
|
+
"turns": turn, "stop": stop, "api_usage": usage,
|
|
229
|
+
"estimated_cost_usd": round(cost, 3) if cost is not None else None, "transcript": str(log_path)}
|
|
230
|
+
note({"type": "end", **result})
|
|
231
|
+
log.close()
|
|
232
|
+
return result
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def main(argv: list[str]) -> None:
|
|
236
|
+
import argparse
|
|
237
|
+
|
|
238
|
+
from ..core.challenges import CHALLENGES
|
|
239
|
+
from ..core.service import Toolkit
|
|
240
|
+
|
|
241
|
+
ap = argparse.ArgumentParser(prog="wsp agent run", description=__doc__,
|
|
242
|
+
formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
243
|
+
ap.add_argument("cmd", choices=["run", "bench"])
|
|
244
|
+
ap.add_argument("challenge", nargs="?", choices=sorted(CHALLENGES), help="(run only)")
|
|
245
|
+
ap.add_argument("--seed", type=int, help="instance seed (default: random)")
|
|
246
|
+
ap.add_argument("--model", default="claude-fable-5-2")
|
|
247
|
+
ap.add_argument("--effort", default="high", help="low|medium|high|xhigh|max, or 'none' to omit")
|
|
248
|
+
ap.add_argument("--thinking", default="summarized", choices=["summarized", "omit"])
|
|
249
|
+
ap.add_argument("--max-turns", type=int, default=80)
|
|
250
|
+
ap.add_argument("--max-cost", type=float, default=20.0, help="stop when the estimated cost exceeds this (USD)")
|
|
251
|
+
ap.add_argument("--runs", type=int, default=1, help="repeat the challenge N times")
|
|
252
|
+
a = ap.parse_args(argv)
|
|
253
|
+
try:
|
|
254
|
+
import anthropic
|
|
255
|
+
except ImportError as e:
|
|
256
|
+
raise SystemExit("the API runner needs the Anthropic SDK: pip install anthropic") from e
|
|
257
|
+
tk = Toolkit()
|
|
258
|
+
if a.cmd == "run" and not a.challenge:
|
|
259
|
+
raise SystemExit("usage: wsp agent run <challenge> --model ... (or: wsp bench run --model ...)")
|
|
260
|
+
plan = [(a.challenge, i) for i in range(a.runs)] if a.cmd == "run" else \
|
|
261
|
+
[(c, i) for c in CHALLENGES for i in range(a.runs)]
|
|
262
|
+
results = []
|
|
263
|
+
for cid, i in plan:
|
|
264
|
+
try:
|
|
265
|
+
r = run(tk, cid, a.model, effort=None if a.effort == "none" else a.effort, thinking=a.thinking,
|
|
266
|
+
max_turns=a.max_turns, max_cost=a.max_cost, label=f"api runner: {a.model} #{i + 1}",
|
|
267
|
+
seed=a.seed)
|
|
268
|
+
except anthropic.AuthenticationError as e:
|
|
269
|
+
raise SystemExit("Anthropic API authentication failed: set ANTHROPIC_API_KEY or run `ant auth login`") from e
|
|
270
|
+
except anthropic.NotFoundError as e:
|
|
271
|
+
raise SystemExit(f"model {a.model!r} was not found for your API key: {e.message}") from e
|
|
272
|
+
except anthropic.APIStatusError as e:
|
|
273
|
+
raise SystemExit(f"Anthropic API error {e.status_code}: {e.message}") from e
|
|
274
|
+
results.append(r)
|
|
275
|
+
print(json.dumps({k: r[k] for k in ("challenge", "seed", "run_id", "score", "turns", "stop",
|
|
276
|
+
"estimated_cost_usd")}), flush=True)
|
|
277
|
+
print(f"\nmean score over {len(results)} run(s): {sum(r['score'] or 0 for r in results) / len(results):.3f}")
|
|
278
|
+
total = sum(r["estimated_cost_usd"] or 0 for r in results)
|
|
279
|
+
print(f"estimated API cost: ${total:.2f}")
|
|
280
|
+
print(f"next: wsp leaderboard export --model {a.model} --by <your GitHub handle> (then submit it on the site)")
|
|
281
|
+
|
|
282
|
+
|
|
283
|
+
_ = Path
|
|
@@ -0,0 +1,146 @@
|
|
|
1
|
+
"""``wsp arena``: animated replays of challenge runs.
|
|
2
|
+
|
|
3
|
+
wsp arena list recent challenge runs (ids to replay)
|
|
4
|
+
wsp arena export [RUN|latest] [--out F.html] one self-contained HTML file that plays the run
|
|
5
|
+
wsp arena timeline [RUN|latest] [--out F.json] the animation timeline as JSON
|
|
6
|
+
wsp arena record [RUN|latest] [--out F.mp4] frame-exact MP4 (needs the [video] extra + a Chromium:
|
|
7
|
+
pip install "wsp-toolkit[video]" && playwright install chromium)
|
|
8
|
+
|
|
9
|
+
The live view is in the web UI: wsp ui, then open http://127.0.0.1:8765/arena (it has a real-time Rec button too).
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import base64
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import shutil
|
|
19
|
+
import subprocess
|
|
20
|
+
import sys
|
|
21
|
+
import tempfile
|
|
22
|
+
import time
|
|
23
|
+
from pathlib import Path
|
|
24
|
+
|
|
25
|
+
SIZES = {"landscape": "1920x1080", "square": "1080x1080", "portrait": "1080x1350", "vertical": "1080x1920"}
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _ffmpeg() -> str:
|
|
29
|
+
exe = shutil.which("ffmpeg")
|
|
30
|
+
if exe:
|
|
31
|
+
return exe
|
|
32
|
+
try:
|
|
33
|
+
import imageio_ffmpeg
|
|
34
|
+
return imageio_ffmpeg.get_ffmpeg_exe()
|
|
35
|
+
except ImportError:
|
|
36
|
+
sys.exit("ffmpeg not found: install it (brew install ffmpeg / apt install ffmpeg) or "
|
|
37
|
+
'pip install "wsp-toolkit[video]"')
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def record(html: Path, out: Path, size: str, fps: int = 30, crf: int = 18, start: float = 0.0,
|
|
41
|
+
end: float | None = None, verbose: bool = True) -> dict:
|
|
42
|
+
"""Render every frame with headless Chromium (deterministic: render(t) per frame) and encode H.264 MP4."""
|
|
43
|
+
try:
|
|
44
|
+
from playwright.sync_api import sync_playwright
|
|
45
|
+
except ImportError:
|
|
46
|
+
sys.exit('video export needs Playwright: pip install "wsp-toolkit[video]" && playwright install chromium\n'
|
|
47
|
+
"(or open the run in `wsp ui` → Arena and press ● Rec to save a WebM in real time)")
|
|
48
|
+
w, h = (int(x) for x in size.lower().split("x"))
|
|
49
|
+
ff = _ffmpeg()
|
|
50
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
51
|
+
cmd = [ff, "-y", "-loglevel", "error", "-f", "image2pipe", "-framerate", str(fps), "-c:v", "mjpeg", "-i", "-",
|
|
52
|
+
"-c:v", "libx264", "-preset", "slow", "-crf", str(crf), "-pix_fmt", "yuv420p", "-movflags", "+faststart",
|
|
53
|
+
str(out)]
|
|
54
|
+
t0 = time.time()
|
|
55
|
+
with sync_playwright() as pw:
|
|
56
|
+
exe = os.environ.get("WSP_CHROMIUM")
|
|
57
|
+
browser = pw.chromium.launch(**({"executable_path": exe} if exe else {}))
|
|
58
|
+
page = browser.new_page(viewport={"width": min(w, 1920), "height": min(h, 1920)})
|
|
59
|
+
errors: list[str] = []
|
|
60
|
+
page.on("pageerror", lambda e: errors.append(str(e)))
|
|
61
|
+
page.goto(html.resolve().as_uri() + "?record")
|
|
62
|
+
page.wait_for_function("window.arenaReady === true", timeout=30000)
|
|
63
|
+
duration = float(page.evaluate("window.arena.duration"))
|
|
64
|
+
stop = min(duration, end) if end else duration
|
|
65
|
+
frames = int((stop - start) * fps)
|
|
66
|
+
enc = subprocess.Popen(cmd, stdin=subprocess.PIPE)
|
|
67
|
+
batch = 8
|
|
68
|
+
for i in range(0, frames, batch):
|
|
69
|
+
times = [start + (i + k) / fps for k in range(min(batch, frames - i))]
|
|
70
|
+
urls = page.evaluate("""(ts) => ts.map(t => { window.arena.render(t);
|
|
71
|
+
return document.getElementById('arena').toDataURL('image/jpeg', 0.95); })""", times)
|
|
72
|
+
for u in urls:
|
|
73
|
+
enc.stdin.write(base64.b64decode(u.split(",", 1)[1]))
|
|
74
|
+
if verbose and (i // batch) % 30 == 0:
|
|
75
|
+
print(f"\r frame {i}/{frames}", end="", flush=True)
|
|
76
|
+
enc.stdin.close()
|
|
77
|
+
enc.wait()
|
|
78
|
+
browser.close()
|
|
79
|
+
if errors:
|
|
80
|
+
raise RuntimeError("renderer errors: " + "; ".join(errors[:3]))
|
|
81
|
+
if enc.returncode:
|
|
82
|
+
raise RuntimeError(f"ffmpeg failed with exit code {enc.returncode}")
|
|
83
|
+
if verbose:
|
|
84
|
+
print(f"\r {frames} frames in {time.time() - t0:.0f}s")
|
|
85
|
+
return {"out": str(out), "frames": frames, "seconds": round(frames / fps, 2), "size": f"{w}x{h}"}
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def main(argv: list[str]) -> None:
|
|
89
|
+
ap = argparse.ArgumentParser(prog="wsp arena", description=__doc__, formatter_class=argparse.RawTextHelpFormatter)
|
|
90
|
+
ap.add_argument("action", choices=["list", "export", "timeline", "record"])
|
|
91
|
+
ap.add_argument("run", nargs="?", default="latest", help="challenge run id (cr_...) or 'latest'")
|
|
92
|
+
ap.add_argument("--out")
|
|
93
|
+
ap.add_argument("--size", default="1920x1080",
|
|
94
|
+
help="WxH or one of: " + ", ".join(f"{k} ({v})" for k, v in SIZES.items()))
|
|
95
|
+
ap.add_argument("--fps", type=int, default=30)
|
|
96
|
+
ap.add_argument("--crf", type=int, default=18, help="H.264 quality (lower = better, bigger)")
|
|
97
|
+
ap.add_argument("--timeline", help="record/export from a timeline JSON file instead of the local database")
|
|
98
|
+
ap.add_argument("--note", help="caption stored with the timeline (shown on the website's arena page)")
|
|
99
|
+
a = ap.parse_args(argv)
|
|
100
|
+
size = SIZES.get(a.size, a.size)
|
|
101
|
+
|
|
102
|
+
from ..core.arena import build_timeline, standalone_html
|
|
103
|
+
from ..core.service import Toolkit
|
|
104
|
+
|
|
105
|
+
if a.action == "list":
|
|
106
|
+
tk = Toolkit()
|
|
107
|
+
rows = tk.store.all("SELECT id, challenge_id, seed, agent_model, score, started_at FROM challenge_runs "
|
|
108
|
+
"ORDER BY started_at DESC LIMIT 30")
|
|
109
|
+
for r in rows:
|
|
110
|
+
sc = f"{r['score']:.3f}" if r["score"] is not None else "live"
|
|
111
|
+
print(f"{r['id']} {r['challenge_id']:<20} seed {r['seed']:<8} {r['agent_model']:<24} {sc}")
|
|
112
|
+
if not rows:
|
|
113
|
+
print("no challenge runs yet")
|
|
114
|
+
return
|
|
115
|
+
|
|
116
|
+
if a.timeline:
|
|
117
|
+
tl = json.loads(Path(a.timeline).read_text())
|
|
118
|
+
else:
|
|
119
|
+
tk = Toolkit()
|
|
120
|
+
rid = a.run
|
|
121
|
+
if rid == "latest":
|
|
122
|
+
row = tk.store.one("SELECT id FROM challenge_runs ORDER BY started_at DESC LIMIT 1")
|
|
123
|
+
if not row:
|
|
124
|
+
sys.exit("no challenge runs yet")
|
|
125
|
+
rid = row["id"]
|
|
126
|
+
tl = build_timeline(tk, rid)
|
|
127
|
+
if a.note:
|
|
128
|
+
tl["note"] = a.note
|
|
129
|
+
stem = f"wsp-arena-{tl['run']['challenge']}-s{tl['run']['seed']}"
|
|
130
|
+
if a.action == "timeline":
|
|
131
|
+
out = Path(a.out or stem + ".json")
|
|
132
|
+
out.write_text(json.dumps(tl, indent=1))
|
|
133
|
+
print(out)
|
|
134
|
+
elif a.action == "export":
|
|
135
|
+
out = Path(a.out or stem + ".html")
|
|
136
|
+
out.write_text(standalone_html(tl, size))
|
|
137
|
+
print(f"{out} (open it in a browser: space = play/pause, ←/→ = seek)")
|
|
138
|
+
else:
|
|
139
|
+
out = Path(a.out or f"{stem}-{size}.mp4")
|
|
140
|
+
with tempfile.TemporaryDirectory() as d:
|
|
141
|
+
html = Path(d) / "arena.html"
|
|
142
|
+
html.write_text(standalone_html(tl, size))
|
|
143
|
+
print(f"recording {tl['run']['challenge']} (seed {tl['run']['seed']}, {tl['run']['model']}): "
|
|
144
|
+
f"{tl['duration']:.0f}s at {a.fps} fps, {size}")
|
|
145
|
+
res = record(html, out, size, a.fps, a.crf)
|
|
146
|
+
print(json.dumps(res))
|
|
@@ -0,0 +1,77 @@
|
|
|
1
|
+
"""Scripted repair agent (no LLM) for the M3 fixture demo: find and fix the planted bug in induction_tf_broken
|
|
2
|
+
with a forward-pass budget, using practice data only, then check once on held-out (aggregate only).
|
|
3
|
+
|
|
4
|
+
Steps:
|
|
5
|
+
1. practice suite (limited) -> confirm the copy behaviour is broken
|
|
6
|
+
2. probe_attention -> layer-1 heads still attend like induction heads, so the *routing* is intact
|
|
7
|
+
3. tensor_stats on each o_proj -> compare per-head output norms across layers; flag heads whose o_proj block is
|
|
8
|
+
far smaller than the median head norm of the model
|
|
9
|
+
4. hypothesis: those heads' outputs were scaled down. Try a few restore factors (each costs a practice run),
|
|
10
|
+
keep the best, revert the others
|
|
11
|
+
5. one held-out evaluation of the final working copy
|
|
12
|
+
Every step is a logged tool call, visible in the UI.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import statistics
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def run(call, source: str = "fixture:induction_tf_broken", budget_hint: int = 50, practice_limit: int = 8,
|
|
21
|
+
factors=(0.75, 1.0, 1.3), session_id: str | None = None, final_heldout: bool = True) -> dict:
|
|
22
|
+
log = []
|
|
23
|
+
if session_id:
|
|
24
|
+
o = call("workspace_info", session_id=session_id)
|
|
25
|
+
sid = session_id
|
|
26
|
+
else:
|
|
27
|
+
o = call("workspace_open", source=source, label="bugfix-agent (scripted)")
|
|
28
|
+
sid = o["session_id"]
|
|
29
|
+
n_layers = o["n_layers"]
|
|
30
|
+
suites = call("suite_list", session_id=sid)["suites"]
|
|
31
|
+
practice = next(s["id"] for s in suites if s["split"] == "practice")
|
|
32
|
+
heldout = next((s["id"] for s in suites if s["split"] == "heldout"), None)
|
|
33
|
+
base = call("suite_run_practice", session_id=sid, suite_id=practice, limit=practice_limit)
|
|
34
|
+
log.append(f"practice score before: {base['metrics']['score']}")
|
|
35
|
+
|
|
36
|
+
ex = next(s for s in suites if s["id"] == practice)["example_preview"][0]["input"]
|
|
37
|
+
att = call("probe_attention", session_id=sid, inputs=[ex], layers=[n_layers - 1], top_k_keys=1)
|
|
38
|
+
att_heads = att.get("heads", [])
|
|
39
|
+
log.append(f"probed attention on layer {n_layers - 1}: {len(att_heads)} heads")
|
|
40
|
+
|
|
41
|
+
norms = {}
|
|
42
|
+
for li in range(n_layers):
|
|
43
|
+
st = call("tensor_stats", session_id=sid, tensor=f"blocks.{li}.attn.o_proj.weight")
|
|
44
|
+
for h, n in enumerate(st["head_norms"]):
|
|
45
|
+
norms[(li, h)] = n
|
|
46
|
+
top = max(norms.values())
|
|
47
|
+
weak = [(li, h) for (li, h), n in norms.items() if n < 0.35 * top]
|
|
48
|
+
healthy = [n for k, n in norms.items() if k not in weak] or [top]
|
|
49
|
+
med = statistics.median(healthy)
|
|
50
|
+
log.append(f"healthy-head median output norm {med:.3g}; suspiciously small heads: {weak}")
|
|
51
|
+
if not weak:
|
|
52
|
+
return {"session_id": sid, "fixed": False, "log": log}
|
|
53
|
+
|
|
54
|
+
best = {"score": base["metrics"]["score"], "patch": None, "factor": None}
|
|
55
|
+
for f in factors: # multipliers of "restore each weak head to the healthy median norm"
|
|
56
|
+
ops = [{"op": "scale", "tensor": f"blocks.{li}.attn.o_proj.weight", "head": h,
|
|
57
|
+
"factor": round(f * med / norms[(li, h)], 4)} for li, h in weak]
|
|
58
|
+
p = call("edit_apply", session_id=sid, operations=ops,
|
|
59
|
+
rationale=f"Heads {weak} have o_proj output blocks under 35% of the largest head's norm; restoring "
|
|
60
|
+
f"them to {f}x the healthy median norm ({med:.3g}).")
|
|
61
|
+
r = call("suite_run_practice", session_id=sid, suite_id=practice, limit=practice_limit)
|
|
62
|
+
sc = r["metrics"]["score"]
|
|
63
|
+
log.append(f"factor {f}: patch {p['patch_id']} practice score {sc}; regression {p.get('regression')}")
|
|
64
|
+
if sc > best["score"]:
|
|
65
|
+
best = {"score": sc, "patch": p["patch_id"], "factor": f}
|
|
66
|
+
call("patch_revert", session_id=sid, patch_id=p["patch_id"])
|
|
67
|
+
if best["patch"] is None:
|
|
68
|
+
return {"session_id": sid, "fixed": False, "log": log}
|
|
69
|
+
call("patch_reapply", session_id=sid, patch_id=best["patch"])
|
|
70
|
+
if not final_heldout:
|
|
71
|
+
return {"session_id": sid, "fixed": True, "patch_id": best["patch"], "factor": best["factor"], "log": log}
|
|
72
|
+
ho = call("evaluate_heldout", session_id=sid, suite_id=heldout)
|
|
73
|
+
log.append(f"kept factor {best['factor']} ({best['patch']}); held-out score {ho['metrics']['score']}")
|
|
74
|
+
b = call("budget_status", session_id=sid)
|
|
75
|
+
return {"session_id": sid, "fixed": True, "patch_id": best["patch"], "factor": best["factor"],
|
|
76
|
+
"practice_before": base["metrics"]["score"], "practice_after": best["score"],
|
|
77
|
+
"heldout_after": ho["metrics"]["score"], "forward_passes_used": b["forward_passes"]["used"], "log": log}
|
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
"""Scripted (no-LLM) reference agents for each challenge, so the leaderboard has a baseline row.
|
|
2
|
+
|
|
3
|
+
wsp challenge baseline # runs every challenge once as agent_model "scripted-baseline"
|
|
4
|
+
|
|
5
|
+
They use only the toolkit's tools, inside the same sessions, access levels and budgets a model gets.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import statistics
|
|
11
|
+
|
|
12
|
+
from . import bugfix_agent, zoo_baseline
|
|
13
|
+
from .clients import InProcessClient, ToolError
|
|
14
|
+
from .induction_finder import find_induction_heads
|
|
15
|
+
|
|
16
|
+
NAME = "scripted-baseline"
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _blind(call, sid: str) -> None:
|
|
20
|
+
"""Weights only: scale up attention heads whose output (o_proj) norm is far below the median."""
|
|
21
|
+
info = call("workspace_info", session_id=sid)
|
|
22
|
+
norms = {}
|
|
23
|
+
for li in range(info["n_layers"]):
|
|
24
|
+
for h, n in enumerate(call("tensor_stats", session_id=sid, tensor=f"blocks.{li}.attn.o_proj.weight")
|
|
25
|
+
["head_norms"]):
|
|
26
|
+
norms[(li, h)] = n
|
|
27
|
+
top = max(norms.values())
|
|
28
|
+
weak = {k for k, n in norms.items() if 0 < n < 0.35 * top}
|
|
29
|
+
med = statistics.median([n for k, n in norms.items() if k not in weak] or [top])
|
|
30
|
+
ops = [{"op": "scale", "tensor": f"blocks.{li}.attn.o_proj.weight", "head": h, "factor": round(med / norms[(li, h)], 3)}
|
|
31
|
+
for (li, h) in sorted(weak)]
|
|
32
|
+
if ops:
|
|
33
|
+
call("edit_apply", session_id=sid, operations=ops,
|
|
34
|
+
rationale="heads with o_proj norm < 35% of the largest look scaled down; restore them to the healthy median")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _mystery(call, ref: str, sus: str) -> dict:
|
|
38
|
+
"""Free weight diff: the trigger's embedding row and the forced token's unembedding row changed most."""
|
|
39
|
+
wd = call("diff_models", session_id=ref, other_session_id=sus)
|
|
40
|
+
rows = {t["tensor"]: t for t in wd["most_changed_tensors"]}
|
|
41
|
+
trig = rows.get("embed.weight", {}).get("top_rows", [[None]])[0][0]
|
|
42
|
+
forced = rows.get("unembed.weight", {}).get("top_rows", [[None]])[0][0]
|
|
43
|
+
return {"trigger_token": f"t{trig}" if trig is not None else None,
|
|
44
|
+
"forced_token": f"t{forced}" if forced is not None else None}
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def _zoo(call, sessions: list[dict]) -> dict:
|
|
48
|
+
answers = {}
|
|
49
|
+
for s in sessions:
|
|
50
|
+
card = call("zoo_get_task", net_id=s["net_id"], task_type="localize")["card"]
|
|
51
|
+
try:
|
|
52
|
+
answers[s["net_id"]] = zoo_baseline.localize(call, s["session_id"], card)
|
|
53
|
+
except ToolError:
|
|
54
|
+
answers[s["net_id"]] = {"flaw": "none", "location": {}}
|
|
55
|
+
return {"answers": answers}
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
def _token_ban(call, sid: str, target: str) -> None:
|
|
59
|
+
"""Zero the banned token's unembedding row (its logit becomes 0 everywhere)."""
|
|
60
|
+
tid = int(target.lstrip("t"))
|
|
61
|
+
call("edit_apply", session_id=sid, rationale=f"zero the unembedding row of {target}",
|
|
62
|
+
operations=[{"op": "zero", "tensor": "unembed.weight", "slice": f"{tid}, :"}])
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
def _install_rule(call, sid: str, trigger: str, output: str) -> None:
|
|
66
|
+
"""Naive rank-1 write: add the trigger's embedding direction to the output token's unembedding row."""
|
|
67
|
+
a, b = int(trigger.lstrip("t")), int(output.lstrip("t"))
|
|
68
|
+
call("edit_apply", session_id=sid, rationale=f"rank-1 rule {trigger} -> {output} via the unembedding",
|
|
69
|
+
operations=[{"op": "add_low_rank", "tensor": "unembed.weight", "coeffs": [10.0],
|
|
70
|
+
"u": [{"vector": [1.0 if i == b else 0.0 for i in range(32)]}],
|
|
71
|
+
"v": [{"tensor": "embed.weight", "row": a}]}])
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def _name_functions(call, sessions: list[dict]) -> dict:
|
|
75
|
+
"""Weights-only heuristic: which inputs have non-negligible first-layer weight, then the first candidate label
|
|
76
|
+
whose relevant inputs match. (An agent that simulates the network exactly would score 1.0.)"""
|
|
77
|
+
from ..core.zoo.generators import BOOL_TASKS
|
|
78
|
+
|
|
79
|
+
def relevant(f):
|
|
80
|
+
return {i for i in range(4) if any(f([(n >> b) & 1 for b in range(4)]) !=
|
|
81
|
+
f([((n ^ (1 << i)) >> b) & 1 for b in range(4)]) for n in range(16))}
|
|
82
|
+
answers = {}
|
|
83
|
+
for s in sessions:
|
|
84
|
+
st = call("tensor_stats", session_id=s["session_id"], tensor="layers.0.weight")
|
|
85
|
+
cols = {c["index"]: c["norm"] for c in st["cols"]["top_cols"] + st["cols"]["bottom_cols"]}
|
|
86
|
+
mx = max(cols.values()) or 1.0
|
|
87
|
+
used = {i for i, n in cols.items() if n > 0.15 * mx and i < 4}
|
|
88
|
+
cands = [lab for lab, f in sorted(BOOL_TASKS.items()) if relevant(f) == used]
|
|
89
|
+
answers[s["net_id"]] = cands[0] if cands else "and_01"
|
|
90
|
+
return {"answers": answers}
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def run_all(tk, challenges: list[str] | None = None, seed: int | None = None) -> list[dict]:
|
|
94
|
+
from ..core.challenges import CHALLENGES
|
|
95
|
+
from ..mcp_server.server import lock_to_challenge
|
|
96
|
+
|
|
97
|
+
call = InProcessClient(tk)
|
|
98
|
+
out = []
|
|
99
|
+
for cid in challenges or list(CHALLENGES):
|
|
100
|
+
lock_to_challenge(tk, cid)
|
|
101
|
+
r = tk.challenges.start(cid, NAME, "scripted (tools only)", "scripted baseline", "agent", seed)
|
|
102
|
+
ss = r["sessions"]
|
|
103
|
+
if cid == "induction-hunt":
|
|
104
|
+
rep = find_induction_heads(call, session_id=ss[0]["session_id"])
|
|
105
|
+
ans = {"induction_heads": rep["induction_heads"], "previous_token_heads": rep["previous_token_heads"]}
|
|
106
|
+
elif cid == "fix-the-copier":
|
|
107
|
+
bugfix_agent.run(call, session_id=ss[0]["session_id"], final_heldout=False)
|
|
108
|
+
ans = {}
|
|
109
|
+
elif cid == "blind-surgeon":
|
|
110
|
+
_blind(call, ss[0]["session_id"])
|
|
111
|
+
ans = {}
|
|
112
|
+
elif cid == "mystery-finetune":
|
|
113
|
+
ref = next(s["session_id"] for s in ss if s["role"] == "reference")
|
|
114
|
+
sus = next(s["session_id"] for s in ss if s["role"] == "suspect")
|
|
115
|
+
ans = _mystery(call, ref, sus)
|
|
116
|
+
elif cid == "token-ban":
|
|
117
|
+
_token_ban(call, ss[0]["session_id"], r["parameters"]["target"])
|
|
118
|
+
ans = {}
|
|
119
|
+
elif cid == "install-a-rule":
|
|
120
|
+
_install_rule(call, ss[0]["session_id"], r["parameters"]["trigger"], r["parameters"]["output"])
|
|
121
|
+
ans = {}
|
|
122
|
+
elif cid == "name-that-function":
|
|
123
|
+
ans = _name_functions(call, ss)
|
|
124
|
+
elif cid == "backdoor-removal":
|
|
125
|
+
ans = {} # no weights-only heuristic for finding an unknown trigger: the baseline makes no edit
|
|
126
|
+
else:
|
|
127
|
+
ans = _zoo(call, ss)
|
|
128
|
+
out.append(call("challenge_submit", run_id=r["run_id"], answer=ans))
|
|
129
|
+
return out
|