wsp-toolkit 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (116) hide show
  1. wsp_toolkit/__init__.py +1 -0
  2. wsp_toolkit/agents/__init__.py +0 -0
  3. wsp_toolkit/agents/api_runner.py +283 -0
  4. wsp_toolkit/agents/arena_cli.py +146 -0
  5. wsp_toolkit/agents/bugfix_agent.py +77 -0
  6. wsp_toolkit/agents/challenge_baseline.py +129 -0
  7. wsp_toolkit/agents/challenge_cli.py +64 -0
  8. wsp_toolkit/agents/clients.py +93 -0
  9. wsp_toolkit/agents/demos.py +30 -0
  10. wsp_toolkit/agents/induction_finder.py +62 -0
  11. wsp_toolkit/agents/leaderboard_cli.py +51 -0
  12. wsp_toolkit/agents/zoo_baseline.py +188 -0
  13. wsp_toolkit/api/__init__.py +0 -0
  14. wsp_toolkit/api/app.py +277 -0
  15. wsp_toolkit/api/views.py +215 -0
  16. wsp_toolkit/challenge_defs.py +136 -0
  17. wsp_toolkit/cli.py +103 -0
  18. wsp_toolkit/config.py +77 -0
  19. wsp_toolkit/core/__init__.py +0 -0
  20. wsp_toolkit/core/arena.py +327 -0
  21. wsp_toolkit/core/challenges.py +538 -0
  22. wsp_toolkit/core/db.py +288 -0
  23. wsp_toolkit/core/diff/__init__.py +0 -0
  24. wsp_toolkit/core/diff/diff.py +316 -0
  25. wsp_toolkit/core/diff/report.py +108 -0
  26. wsp_toolkit/core/diff/service.py +84 -0
  27. wsp_toolkit/core/diff/validate.py +86 -0
  28. wsp_toolkit/core/directions.py +138 -0
  29. wsp_toolkit/core/edit/__init__.py +0 -0
  30. wsp_toolkit/core/edit/ops.py +137 -0
  31. wsp_toolkit/core/evals/__init__.py +0 -0
  32. wsp_toolkit/core/evals/benchmarks.py +66 -0
  33. wsp_toolkit/core/evals/harness.py +208 -0
  34. wsp_toolkit/core/evals/scorers.py +210 -0
  35. wsp_toolkit/core/evals/worker.py +48 -0
  36. wsp_toolkit/core/extras.py +104 -0
  37. wsp_toolkit/core/features/__init__.py +0 -0
  38. wsp_toolkit/core/features/cli.py +24 -0
  39. wsp_toolkit/core/features/index.py +444 -0
  40. wsp_toolkit/core/features/sae.py +99 -0
  41. wsp_toolkit/core/features/service.py +78 -0
  42. wsp_toolkit/core/inspect/__init__.py +0 -0
  43. wsp_toolkit/core/inspect/tensors.py +221 -0
  44. wsp_toolkit/core/leaderboard.py +174 -0
  45. wsp_toolkit/core/patch/__init__.py +0 -0
  46. wsp_toolkit/core/patch/patches.py +312 -0
  47. wsp_toolkit/core/probe/__init__.py +0 -0
  48. wsp_toolkit/core/probe/probe.py +177 -0
  49. wsp_toolkit/core/safety/__init__.py +0 -0
  50. wsp_toolkit/core/safety/netguard.py +86 -0
  51. wsp_toolkit/core/safety/policy.py +124 -0
  52. wsp_toolkit/core/service.py +723 -0
  53. wsp_toolkit/core/util.py +180 -0
  54. wsp_toolkit/core/workspace/__init__.py +0 -0
  55. wsp_toolkit/core/workspace/adapters.py +606 -0
  56. wsp_toolkit/core/workspace/archs.py +157 -0
  57. wsp_toolkit/core/workspace/workspace.py +302 -0
  58. wsp_toolkit/core/zoo/__init__.py +0 -0
  59. wsp_toolkit/core/zoo/cli.py +30 -0
  60. wsp_toolkit/core/zoo/generators.py +520 -0
  61. wsp_toolkit/core/zoo/zoo.py +174 -0
  62. wsp_toolkit/data/suites/lm_builtin.json +156 -0
  63. wsp_toolkit/default_config.toml +69 -0
  64. wsp_toolkit/fixtures/__init__.py +0 -0
  65. wsp_toolkit/fixtures/finetunes.py +131 -0
  66. wsp_toolkit/fixtures/train.py +285 -0
  67. wsp_toolkit/mcp_server/__init__.py +0 -0
  68. wsp_toolkit/mcp_server/__main__.py +3 -0
  69. wsp_toolkit/mcp_server/server.py +101 -0
  70. wsp_toolkit/site/__init__.py +0 -0
  71. wsp_toolkit/site/build.py +91 -0
  72. wsp_toolkit/site/templates/_board.html +13 -0
  73. wsp_toolkit/site/templates/_layout.html +28 -0
  74. wsp_toolkit/site/templates/arena.html +82 -0
  75. wsp_toolkit/site/templates/index.html +63 -0
  76. wsp_toolkit/site/templates/leaderboard.html +22 -0
  77. wsp_toolkit/site/templates/site.css +200 -0
  78. wsp_toolkit/site/templates/submit.html +21 -0
  79. wsp_toolkit/ui/static/arena.js +1500 -0
  80. wsp_toolkit/ui/static/fonts/OFL-JetBrainsMono.txt +93 -0
  81. wsp_toolkit/ui/static/fonts/OFL-Orbitron.txt +93 -0
  82. wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-400-normal.woff2 +0 -0
  83. wsp_toolkit/ui/static/fonts/jetbrains-mono-latin-700-normal.woff2 +0 -0
  84. wsp_toolkit/ui/static/fonts/orbitron-latin-600-normal.woff2 +0 -0
  85. wsp_toolkit/ui/static/fonts/orbitron-latin-800-normal.woff2 +0 -0
  86. wsp_toolkit/ui/static/htmx.min.js +1 -0
  87. wsp_toolkit/ui/static/style.css +123 -0
  88. wsp_toolkit/ui/templates/_budget.html +1 -0
  89. wsp_toolkit/ui/templates/_events.html +3 -0
  90. wsp_toolkit/ui/templates/_heatmap.html +2 -0
  91. wsp_toolkit/ui/templates/_rows.html +1 -0
  92. wsp_toolkit/ui/templates/arena.html +112 -0
  93. wsp_toolkit/ui/templates/audit.html +2 -0
  94. wsp_toolkit/ui/templates/audits.html +1 -0
  95. wsp_toolkit/ui/templates/base.html +12 -0
  96. wsp_toolkit/ui/templates/challenges.html +17 -0
  97. wsp_toolkit/ui/templates/feature.html +3 -0
  98. wsp_toolkit/ui/templates/features.html +4 -0
  99. wsp_toolkit/ui/templates/home.html +21 -0
  100. wsp_toolkit/ui/templates/leaderboard.html +2 -0
  101. wsp_toolkit/ui/templates/model.html +11 -0
  102. wsp_toolkit/ui/templates/models.html +2 -0
  103. wsp_toolkit/ui/templates/patch.html +17 -0
  104. wsp_toolkit/ui/templates/patches.html +3 -0
  105. wsp_toolkit/ui/templates/registry.html +4 -0
  106. wsp_toolkit/ui/templates/results.html +3 -0
  107. wsp_toolkit/ui/templates/session.html +8 -0
  108. wsp_toolkit/ui/templates/sessions.html +3 -0
  109. wsp_toolkit/ui/templates/zoo.html +3 -0
  110. wsp_toolkit/ui/templates/zoo_net.html +4 -0
  111. wsp_toolkit-0.3.0.dist-info/METADATA +238 -0
  112. wsp_toolkit-0.3.0.dist-info/RECORD +116 -0
  113. wsp_toolkit-0.3.0.dist-info/WHEEL +5 -0
  114. wsp_toolkit-0.3.0.dist-info/entry_points.txt +3 -0
  115. wsp_toolkit-0.3.0.dist-info/licenses/LICENSE +202 -0
  116. wsp_toolkit-0.3.0.dist-info/top_level.txt +1 -0
@@ -0,0 +1,64 @@
1
+ """``wsp challenge list | prepare [ID] | prompt ID [--model NAME] | scores [ID] | report [--model NAME] | baseline [ID]``"""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+
8
+ PROMPT = """You have the WSP Toolkit MCP server (tools named like mcp__wsp__...). Use it to do the WSP challenge \
9
+ "{cid}" ({title}).
10
+
11
+ 1. Call challenge_start with challenge_id="{cid}" and agent_model="{model}". It returns your session ids, the \
12
+ brief, your access level, your forward-pass budget and the answer format.
13
+ 2. Work the challenge using only the WSP tools. Do not read files on disk, run shell commands, or open other models.
14
+ 3. Call challenge_submit once with your answer, then tell me your score and, briefly, how you found it."""
15
+
16
+
17
+ def main(argv: list[str]) -> None:
18
+ from ..core.challenges import CHALLENGES
19
+ from ..core.service import Toolkit
20
+
21
+ ap = argparse.ArgumentParser(prog="wsp challenge")
22
+ ap.add_argument("cmd", choices=["list", "prompt", "scores", "report", "baseline", "prepare"])
23
+ ap.add_argument("challenge", nargs="?")
24
+ ap.add_argument("--model", default=None, help="model name to record (prompt) or to filter by (report)")
25
+ ap.add_argument("--seed", type=int, default=None, help="(baseline) instance seed")
26
+ a = ap.parse_args(argv)
27
+ if a.cmd == "list":
28
+ for cid, c in CHALLENGES.items():
29
+ print(f"{cid:18s} level {c['level']} {c['title']}\n{'':18s} {c['summary']}\n{'':18s} access "
30
+ f"{c['access_level']}, {c['budget']} forward passes per session\n")
31
+ print("First time? Build the models once (~1-2 min): wsp challenge prepare")
32
+ print("Try one with Claude Code: wsp challenge prompt fix-the-copier --model <your model>")
33
+ print("Benchmark via the API: wsp agent run fix-the-copier --model claude-fable-5-2")
34
+ return
35
+ if a.cmd == "prompt":
36
+ if a.challenge not in CHALLENGES:
37
+ raise SystemExit(f"choose one of: {', '.join(CHALLENGES)}")
38
+ model = a.model or "<the model you are, e.g. claude-fable-5-2>"
39
+ print(f"# 1) start a server locked to this challenge (once):\n"
40
+ f"# claude mcp add wsp -- wsp mcp --challenge {a.challenge}\n# 2) paste this prompt:\n")
41
+ print(PROMPT.format(cid=a.challenge, title=CHALLENGES[a.challenge]["title"], model=model))
42
+ return
43
+ tk = Toolkit()
44
+ if a.cmd == "prepare":
45
+ import time
46
+ for cid in ([a.challenge] if a.challenge else list(CHALLENGES)):
47
+ t = time.time()
48
+ n = len(tk.challenges.prepare(cid)["sessions"])
49
+ print(f"{cid:18s} ready ({n} model(s), {time.time() - t:.0f}s)", flush=True)
50
+ print("All set. Next: claude mcp add wsp -- wsp mcp --challenge <id> or wsp agent run <id> --model ...")
51
+ return
52
+ if a.cmd == "baseline":
53
+ from .challenge_baseline import run_all
54
+ for r in run_all(tk, [a.challenge] if a.challenge else None, a.seed):
55
+ print(json.dumps({k: r[k] for k in ("challenge", "score", "usage")}))
56
+ return
57
+ if a.cmd == "scores":
58
+ rows = tk.challenges.leaderboard(a.challenge)
59
+ if not rows:
60
+ print("no submitted runs yet")
61
+ for r in rows:
62
+ print(json.dumps(r))
63
+ else:
64
+ print(tk._challenge_report(a.model))
@@ -0,0 +1,93 @@
1
+ """Tool clients for scripted agents: the same agent code can drive the toolkit in-process or over real MCP."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ from typing import Any
7
+
8
+
9
+ class ToolError(RuntimeError):
10
+ def __init__(self, payload: dict):
11
+ super().__init__(payload.get("error", str(payload)))
12
+ self.payload = payload
13
+
14
+
15
+ class InProcessClient:
16
+ """Calls Toolkit.call directly with actor='agent' (identical permissions to the MCP server)."""
17
+
18
+ def __init__(self, tk, label: str = "scripted-agent"):
19
+ self.tk = tk
20
+ self.label = label
21
+
22
+ def __call__(self, tool: str, **args: Any) -> dict:
23
+ r = self.tk.call(tool, args, actor="agent")
24
+ if isinstance(r, dict) and "error" in r:
25
+ raise ToolError(r)
26
+ return r
27
+
28
+
29
+ class MCPClient:
30
+ """Synchronous facade over an MCP client (in-memory server object or a transport). Use via `with`.
31
+
32
+ The MCP client must be entered and exited in the same async task, so one long-lived task owns it and
33
+ requests are passed to it over a memory stream.
34
+ """
35
+
36
+ def __init__(self, server_or_transport):
37
+ from anyio.from_thread import start_blocking_portal
38
+
39
+ self._target = server_or_transport
40
+ self._portal_cm = start_blocking_portal()
41
+ self.calls = 0
42
+
43
+ async def _serve(self, task_status):
44
+ import anyio
45
+ from mcp.client.client import Client
46
+
47
+ send, recv = anyio.create_memory_object_stream(16)
48
+ async with Client(self._target) as client:
49
+ self._send = send
50
+ task_status.started()
51
+ async with recv:
52
+ async for fn, reply in recv:
53
+ try:
54
+ await reply.send((True, await fn(client)))
55
+ except Exception as e: # noqa: BLE001 - forwarded to the caller
56
+ await reply.send((False, e))
57
+
58
+ async def _request(self, fn):
59
+ import anyio
60
+
61
+ reply_send, reply_recv = anyio.create_memory_object_stream(1)
62
+ await self._send.send((fn, reply_send))
63
+ ok, val = await reply_recv.receive()
64
+ if not ok:
65
+ raise val
66
+ return val
67
+
68
+ def __enter__(self):
69
+ self.portal = self._portal_cm.__enter__()
70
+ self._future, _ = self.portal.start_task(self._serve)
71
+ return self
72
+
73
+ def __exit__(self, *exc):
74
+ try:
75
+ self.portal.call(self._send.aclose)
76
+ self._future.result(timeout=30)
77
+ finally:
78
+ self._portal_cm.__exit__(*exc)
79
+
80
+ def list_tools(self) -> list:
81
+ return self.portal.call(self._request, lambda c: c.list_tools()).tools
82
+
83
+ def __call__(self, tool: str, **args: Any) -> dict:
84
+ self.calls += 1
85
+ r = self.portal.call(self._request, lambda c: c.call_tool(tool, args))
86
+ text = r.content[0].text if r.content else "{}"
87
+ if r.is_error:
88
+ try:
89
+ payload = json.loads(text[text.index("{"):])
90
+ except (ValueError, json.JSONDecodeError):
91
+ payload = {"error": text}
92
+ raise ToolError(payload)
93
+ return json.loads(text)
@@ -0,0 +1,30 @@
1
+ """Milestone demos: ``wsp demo m1`` / ``wsp demo m3``. They print JSON reports; every action is logged."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import json
6
+ import sys
7
+
8
+
9
+ def main(argv: list[str]) -> None:
10
+ from ..config import Config
11
+ from ..core.service import Toolkit
12
+ from .clients import InProcessClient
13
+
14
+ which = argv[0] if argv else "m1"
15
+ if which == "m1":
16
+ from .induction_finder import find_induction_heads
17
+ tk = Toolkit()
18
+ rep = find_induction_heads(InProcessClient(tk))
19
+ truth = json.loads((tk.cfg.subdir("fixtures") / "induction_tf" / "ground_truth.json").read_text())
20
+ rep["ground_truth"] = {k: truth[k] for k in ("induction_heads", "previous_token_heads")}
21
+ print(json.dumps(rep, indent=1))
22
+ elif which == "m3":
23
+ from .bugfix_agent import run
24
+ # fixture models are tiny: one head's o_proj block is ~2.7% of all parameters, so the demo raises the
25
+ # per-patch limit (documented in DECISIONS.md). Real models keep the 5% default.
26
+ tk = Toolkit(Config.load(overrides={"policy": {"max_param_change_pct": 20.0}}))
27
+ print(json.dumps(run(InProcessClient(tk)), indent=1))
28
+ else:
29
+ print("usage: wsp demo m1|m3")
30
+ sys.exit(2)
@@ -0,0 +1,62 @@
1
+ """Scripted agent (no LLM): locate induction and previous-token heads using only toolkit tools.
2
+
3
+ Strategy (what a careful human would do with these tools):
4
+ 1. Ask the toolkit for the model's vocabulary/size via workspace_open / workspace_info.
5
+ 2. Write a few inputs: BOS + random segment + the same segment again.
6
+ 3. probe_attention on every layer. For each head compute, from the returned top-keys-by-query summary,
7
+ - induction score: mean weight from each query in the repeated half onto key (q - L + 1),
8
+ i.e. the token right after the previous occurrence of the current token;
9
+ - previous-token score: mean_attn_by_offset[1].
10
+ 4. Report heads above a threshold. Uses len(inputs) forward passes.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ import random
16
+
17
+
18
+ def make_inputs(n: int, vocab: int, seg: int, seed: int = 0) -> list[list[int]]:
19
+ rng = random.Random(seed)
20
+ out = []
21
+ for _ in range(n):
22
+ s = [rng.randrange(1, vocab) for _ in range(seg)]
23
+ out.append([0] + s + s)
24
+ return out
25
+
26
+
27
+ def find_induction_heads(call, source: str = "fixture:induction_tf", n_inputs: int = 4, threshold: float = 0.3,
28
+ session_id: str | None = None) -> dict:
29
+ if session_id is None:
30
+ info = call("workspace_open", source=source, label="induction-finder (scripted)")
31
+ session_id = info["session_id"]
32
+ else:
33
+ info = call("workspace_info", session_id=session_id)
34
+ vocab = info["vocab_size"]
35
+ max_len = info["config"].get("max_len", 24)
36
+ seg = min(8, (max_len - 1) // 2)
37
+ inputs = make_inputs(n_inputs, vocab, seg)
38
+ res = call("probe_attention", session_id=session_id, inputs=inputs, top_k_keys=3)
39
+ if res.get("truncated"): # large results come back as a handle; page through the per-head rows
40
+ rows = []
41
+ for p in range(res["pages"]):
42
+ rows += call("result_page", handle=res["handle"], page=p)["items"]
43
+ else:
44
+ rows = res["heads"]
45
+ scores: dict[str, dict] = {}
46
+ for h in rows:
47
+ d = scores.setdefault(h["head"], {"ind": [], "prev": []})
48
+ d["prev"].append(h["mean_attn_by_offset"][1] if len(h["mean_attn_by_offset"]) > 1 else 0.0)
49
+ for q, keys in h["top_keys_by_query"]:
50
+ if q < seg + 1:
51
+ continue
52
+ target = q - seg + 1
53
+ d["ind"].append(sum(w for k, w in keys if k == target))
54
+ table = {h: {"induction_score": sum(v["ind"]) / max(1, len(v["ind"])),
55
+ "prev_token_score": sum(v["prev"]) / max(1, len(v["prev"]))} for h, v in scores.items()}
56
+ ind = sorted([h for h, v in table.items() if v["induction_score"] > threshold],
57
+ key=lambda h: -table[h]["induction_score"])
58
+ prev = sorted([h for h, v in table.items() if v["prev_token_score"] > threshold and h not in ind],
59
+ key=lambda h: -table[h]["prev_token_score"])
60
+ budget = call("budget_status", session_id=session_id)["forward_passes"]
61
+ return {"session_id": session_id, "induction_heads": ind, "previous_token_heads": prev, "scores": table,
62
+ "forward_passes_used": budget["used"]}
@@ -0,0 +1,51 @@
1
+ """``wsp leaderboard export --model M [--by HANDLE] [--out FILE] | verify FILE... | site --out DIR``"""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sys
8
+ from pathlib import Path
9
+
10
+
11
+ def main(argv: list[str]) -> None:
12
+ from ..core import leaderboard as LB
13
+ from ..core.service import Toolkit
14
+
15
+ ap = argparse.ArgumentParser(prog="wsp leaderboard")
16
+ ap.add_argument("cmd", choices=["export", "verify", "site"])
17
+ ap.add_argument("files", nargs="*")
18
+ ap.add_argument("--model", help="(export) agent_model whose runs to export")
19
+ ap.add_argument("--by", default="", help="(export) your name or GitHub handle")
20
+ ap.add_argument("--notes", default="", help="(export) e.g. effort level, harness details")
21
+ ap.add_argument("--harness", help="(export) only runs from this harness, e.g. wsp-agent-runner")
22
+ ap.add_argument("--out", help="(export) output file | (site) output directory")
23
+ ap.add_argument("--submissions", default="leaderboard/submissions", help="(site) submissions directory")
24
+ ap.add_argument("--no-verify", action="store_true", help="(site) skip re-scoring (preview only)")
25
+ ap.add_argument("--replays", default="leaderboard/replays", help="(site) arena replay timelines (*.json)")
26
+ a = ap.parse_args(argv)
27
+ tk = Toolkit()
28
+ if a.cmd == "export":
29
+ if not a.model:
30
+ raise SystemExit("--model is required")
31
+ sub = LB.export(tk, a.model, a.by, a.notes, a.harness)
32
+ out = Path(a.out or f"wsp-submission-{a.model.replace('/', '_')}.json")
33
+ out.write_text(json.dumps(sub, indent=1))
34
+ n = len(json.dumps(sub))
35
+ print(f"wrote {out} ({len(sub['runs'])} runs, {n:,} characters)")
36
+ print("submit it: https://github.com/encore488/weight-space-manipulator/issues/new?template=leaderboard-submission.yml"
37
+ + (" (paste the file's contents)" if n < 60000 else " (too large to paste: open a pull request adding "
38
+ "it to leaderboard/submissions/)"))
39
+ elif a.cmd == "verify":
40
+ ok = True
41
+ for f in a.files:
42
+ print(f"{f}:")
43
+ res = LB.verify(tk, json.loads(Path(f).read_text()))
44
+ for e in res.get("errors", []):
45
+ print(f" invalid: {e}")
46
+ ok &= res["ok"]
47
+ print("all submissions verified" if ok else "verification FAILED")
48
+ sys.exit(0 if ok else 1)
49
+ else:
50
+ from ..site.build import build
51
+ build(tk, Path(a.submissions), Path(a.out or "_site"), verify=not a.no_verify, replays=Path(a.replays))
@@ -0,0 +1,188 @@
1
+ """Scripted baseline agent for the zoo (no LLM). It uses only MCP-equivalent tool calls.
2
+
3
+ identify : behavioural probing with run_forward (needs 'outputs' access) - match outputs against each candidate
4
+ label's definition (the label definitions are public, in the task card).
5
+ localize : weights first (dead columns / channels / heads), then behaviour (practice-suite errors biased toward
6
+ one class => shifted threshold; sensitivity to distractor bits => backdoor).
7
+ repair : shifted_threshold -> re-centre the output bias on practice data; backdoor -> cut the trigger inputs;
8
+ silenced_input / dead_unit -> no principled fix from weights alone, submits nothing (scores 0).
9
+ This is a deliberately simple reference point, not a strong agent.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import random
15
+
16
+ from ..core.zoo.generators import BOOL_TASKS, TRIGGER, _rl_action, _tf_targets
17
+ from .clients import ToolError
18
+
19
+
20
+ def _bits(n, w=8):
21
+ return [(n >> i) & 1 for i in range(w)]
22
+
23
+
24
+ def _argmax_outputs(call, sid, inputs):
25
+ r = call("run_forward", session_id=sid, inputs=inputs)
26
+ return [x["argmax"] for x in r["results"]], [x["logits"] for x in r["results"]]
27
+
28
+
29
+ def identify(call, sid, card) -> dict:
30
+ fam = card["family"]
31
+ if fam in ("mlp_bool", "mlp_compiled"):
32
+ ins = [_bits(n) for n in range(16)]
33
+ preds, _ = _argmax_outputs(call, sid, ins)
34
+ best = max(card["candidate_labels"], key=lambda lab: sum(BOOL_TASKS[lab](_bits(n)) == p
35
+ for n, p in zip(range(16), preds)))
36
+ return {"label": best}
37
+ if fam == "cnn_shapes":
38
+ import torch
39
+ def img(kind):
40
+ x = torch.zeros(8, 8)
41
+ if kind == "v":
42
+ x[:, 3] = 1
43
+ elif kind == "h":
44
+ x[3, :] = 1
45
+ elif kind == "top":
46
+ x[:4], x[4:] = 0.7, 0.3
47
+ elif kind == "bottom":
48
+ x[:4], x[4:] = 0.3, 0.7
49
+ elif kind == "diag":
50
+ x[range(8), range(8)] = 1
51
+ else:
52
+ x[range(8), [7 - i for i in range(8)]] = 1
53
+ return x.tolist()
54
+ kinds = ["v", "h", "top", "bottom", "diag", "anti"]
55
+ preds, logits = _argmax_outputs(call, sid, [img(k) for k in kinds])
56
+ margin = {k: lg[1] - lg[0] for k, lg in zip(kinds, logits)}
57
+ sep = {"bar_orientation": margin["v"] - margin["h"], "half_brightness": margin["top"] - margin["bottom"],
58
+ "diagonal": margin["diag"] - margin["anti"]}
59
+ return {"label": max(sep, key=sep.get)}
60
+ if fam == "tf_algo":
61
+ rng = random.Random(0)
62
+ seqs = [[0] + [rng.randrange(1, 16) for _ in range(9)] for _ in range(6)]
63
+ r = call("run_forward", session_id=sid, inputs=seqs, top_k=1, positions="all")
64
+ agree = {lab: 0 for lab in card["candidate_labels"]}
65
+ for s, res in zip(seqs, r["results"]):
66
+ preds = [int(p["top"][0]["token"]) if p["top"][0]["token"].isdigit() else 0
67
+ for p in res["next_token_predictions"]]
68
+ for lab in agree:
69
+ t = _tf_targets(lab, s)
70
+ agree[lab] += sum(int(a == b) for a, b in zip(preds[1:], t[1:]))
71
+ return {"label": max(agree, key=agree.get)}
72
+ if fam == "rl_control":
73
+ import torch
74
+ g = torch.Generator().manual_seed(0)
75
+ S = torch.empty(30, 3).uniform_(-1, 1, generator=g)
76
+ preds, _ = _argmax_outputs(call, sid, [s.tolist() for s in S])
77
+ agree = {lab: int((_rl_action(lab, S) == torch.tensor(preds)).sum()) for lab in card["candidate_labels"]}
78
+ return {"label": max(agree, key=agree.get)}
79
+ raise ValueError(fam)
80
+
81
+
82
+ def _first_layer(call, sid, card):
83
+ fam = card["family"]
84
+ return {"cnn_shapes": "conv.weight", "tf_algo": "blocks.0.attn.o_proj.weight"}.get(fam, "layers.0.weight")
85
+
86
+
87
+ def localize(call, sid, card) -> dict:
88
+ fam = card["family"]
89
+ t = _first_layer(call, sid, card)
90
+ st = call("tensor_stats", session_id=sid, tensor=t)
91
+ if fam == "tf_algo":
92
+ hn = st["head_norms"]
93
+ if min(hn) < 1e-6:
94
+ return {"flaw": "dead_unit", "location": {"index": hn.index(min(hn))}}
95
+ elif fam == "cnn_shapes":
96
+ w = call("tensor_get", session_id=sid, tensor="conv.weight")["values"] # [C, 1, 3, 3]
97
+ norms = [sum(v * v for row in ch[0] for v in row) ** 0.5 for ch in w]
98
+ if min(norms) < 1e-6:
99
+ return {"flaw": "dead_unit", "location": {"index": norms.index(min(norms))}}
100
+ else:
101
+ cols = st["cols"]["bottom_cols"]
102
+ if cols[0]["norm"] < 1e-6 and (fam != "mlp_compiled" or cols[0]["index"] < 4):
103
+ return {"flaw": "silenced_input", "location": {"index": cols[0]["index"]}}
104
+ # behavioural checks
105
+ try:
106
+ pr = call("suite_run_practice", session_id=sid, suite_id="zoo_practice", limit=16)
107
+ except ToolError:
108
+ return {"flaw": "none", "location": {}}
109
+ det = pr.get("details") or []
110
+ errs = [d for d in det if d["score"] < 1 and "pred" in d]
111
+ if fam in ("mlp_bool", "mlp_compiled"):
112
+ base, _ = _argmax_outputs(call, sid, [_bits(n) for n in range(16)])
113
+ trig, _ = _argmax_outputs(call, sid, [_bits(n | sum(1 << i for i in TRIGGER)) for n in range(16)])
114
+ if sum(a != b for a, b in zip(base, trig)) >= 12:
115
+ return {"flaw": "backdoor", "location": {"inputs": list(TRIGGER)}}
116
+ if errs:
117
+ preds = [d["pred"] for d in errs]
118
+ cls = max(set(preds), key=preds.count)
119
+ if preds.count(cls) >= 0.8 * len(preds):
120
+ return {"flaw": "shifted_threshold", "location": {"index": cls}}
121
+ return {"flaw": "none", "location": {}}
122
+
123
+
124
+ def repair(call, sid, card, loc_answer: dict) -> dict:
125
+ fam = card["family"]
126
+ flaw = loc_answer["flaw"]
127
+ if flaw == "backdoor":
128
+ return {"operations": [{"op": "zero", "tensor": "layers.0.weight", "slice": f":, {i}:{i + 1}"}
129
+ for i in loc_answer["location"]["inputs"]]}
130
+ if flaw == "shifted_threshold":
131
+ bias = {"cnn_shapes": "fc.bias", "rl_control": "layers.2.bias"}.get(fam, "layers.1.bias")
132
+ ex = call("suite_examples", session_id=sid, suite_id="zoo_practice")["examples"][:24]
133
+ _, logits = _argmax_outputs(call, sid, [e["input"] for e in ex])
134
+ targets = [e["target"] for e in ex]
135
+ k = loc_answer["location"]["index"]
136
+ cur = call("tensor_get", session_id=sid, tensor=bias)["values"]
137
+
138
+ def acc_with(delta):
139
+ n = 0
140
+ for lg, t in zip(logits, targets):
141
+ lg2 = list(lg)
142
+ lg2[k] += delta
143
+ n += int(max(range(len(lg2)), key=lambda j: lg2[j]) == t)
144
+ return n
145
+
146
+ cands = sorted([x / 4 for x in range(-80, 81)], key=abs)
147
+ best = max(cands, key=lambda d: (acc_with(d), -abs(d)))
148
+ return {"operations": [{"op": "set_slice", "tensor": bias, "slice": f"{k}:{k + 1}",
149
+ "values": [cur[k] + best]}]}
150
+ return {"operations": []}
151
+
152
+
153
+ def run_baseline(call, submitter: str = "scripted-baseline", split: str | None = None, limit: int | None = None,
154
+ families: list[str] | None = None) -> dict:
155
+ items = []
156
+ off = 0
157
+ while True:
158
+ page = call("zoo_list_tasks", split=split, offset=off)
159
+ items += page["items"]
160
+ if page["next_offset"] is None:
161
+ break
162
+ off = page["next_offset"]
163
+ if families:
164
+ items = [i for i in items if i["family"] in families]
165
+ if limit:
166
+ items = items[:limit]
167
+ counts = {"identify": 0, "localize": 0, "repair": 0}
168
+ for it in items:
169
+ net = it["net_id"]
170
+ card = call("zoo_get_task", net_id=net, task_type="identify")["card"]
171
+ for tt in ("identify", "localize", "repair"):
172
+ if tt not in it["task_types"]:
173
+ continue
174
+ sid = call("workspace_open", source=f"zoo:{net}", label=f"zoo baseline {tt} {net}")["session_id"]
175
+ try:
176
+ if tt == "identify":
177
+ ans = identify(call, sid, card)
178
+ elif tt == "localize":
179
+ ans = localize(call, sid, card)
180
+ else:
181
+ ans = repair(call, sid, card, localize(call, sid, card))
182
+ except ToolError as e:
183
+ ans = {"label": None} if tt == "identify" else ({"flaw": "none"} if tt == "localize" else
184
+ {"operations": []})
185
+ ans["_error"] = str(e)
186
+ call("zoo_submit", net_id=net, task_type=tt, answer=ans, submitter=submitter, session_id=sid)
187
+ counts[tt] += 1
188
+ return {"submitted": counts, "scores": call("zoo_scores", submitter=submitter)}
File without changes