nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,206 @@
1
+ """Kit for agent.tools/v1, version 2: five checks, each a different use of the
2
+ tools of one MCP server (``mcp_kit.py``) given to the session.
3
+
4
+ - ``one-tool``: the server offers a single tool without arguments;
5
+ - ``arguments``: a tool taking a string and an integer, its answer theirs;
6
+ - ``decoys``: the right tool among twenty of alike names and answers;
7
+ - ``chain``: three tools, each one's output the next one's input;
8
+ - ``retry``: the first call is refused as a tool error saying what was wrong,
9
+ and the agent calls again, corrected.
10
+
11
+ Each unit draws a seed the prompt never carries; every answer is derived from
12
+ it, so none can be guessed. A unit passes when the final message carries the
13
+ answer *and* the server's own log shows the calls that produce it - written
14
+ beside the trial's verdict (``verifier/mcp-calls.jsonl``), outside the
15
+ workspace the agent works in. The prompts say what to find, never which tool.
16
+ """
17
+ import hashlib
18
+ import importlib.util
19
+ import json
20
+ import os
21
+ import re
22
+ import secrets
23
+ import sys
24
+
25
+ from nullius.standard.container import CHECK_IMAGE
26
+ from nullius.standard.jsonl import serve
27
+
28
+ HERE = os.path.dirname(os.path.abspath(__file__))
29
+ _spec = importlib.util.spec_from_file_location("nullius_kit_mcp", os.path.join(HERE, "mcp_kit.py"))
30
+ mcp = importlib.util.module_from_spec(_spec)
31
+ _spec.loader.exec_module(mcp)
32
+
33
+ UNITS = ("one-tool", "arguments", "decoys", "chain", "retry")
34
+ WAREHOUSES = ("Riverside", "Northgate", "Old Quay", "Millbrook")
35
+ SITES = ("Harbour Road", "Cedar Yard", "Mill Lane", "Station Hill")
36
+ FINISH = " Reply with it and nothing else."
37
+ SEEDS = {}
38
+ # The agent starts the server: in its container on docker, by the image's own python3 (this machine's
39
+ # interpreter, mounted over it, would not run there); here, by this one.
40
+ PYTHON = "python3" if os.environ.get(CHECK_IMAGE) else sys.executable
41
+
42
+
43
+ def asked(seed):
44
+ """What the prompts name, from the seed: never an answer, only the question."""
45
+ n = int(hashlib.sha256(seed.encode()).hexdigest(), 16)
46
+ return {"warehouse": WAREHOUSES[n % 4], "item": 100 + n // 4 % 900, "parcel": f"P-{1000 + n // 3600 % 9000}",
47
+ "order": 1000 + n // 7 ** 9 % 9000, "site": SITES[n // 11 ** 9 % 4], "year": 2019 + n // 13 ** 9 % 8}
48
+
49
+
50
+ def prompt(unit, seed):
51
+ a = asked(seed)
52
+ use = "Using the tools of the MCP server named kit, given to this session, find "
53
+ return {
54
+ "one-tool": "The MCP server named kit, given to this session, holds the code word of this check. Find it." + FINISH,
55
+ "arguments": use + f"the stock code of item {a['item']} in the {a['warehouse']} warehouse." + FINISH,
56
+ "decoys": use + f"the carrier's tracking number of parcel {a['parcel']}." + FINISH,
57
+ "chain": use + f"the settlement code of the account that pays for order {a['order']}." + FINISH,
58
+ "retry": use + f"the permit code of the site called {a['site']} for the year {a['year']}." + FINISH,
59
+ }[unit]
60
+
61
+
62
+ def expected(unit, seed):
63
+ a = asked(seed)
64
+ if unit == "one-tool":
65
+ return mcp.call(seed, "lookup_code", {})
66
+ if unit == "arguments":
67
+ return mcp.code(seed, "stock", mcp.norm(a["warehouse"]), a["item"])
68
+ if unit == "decoys":
69
+ return mcp.code(seed, "get_tracking_number", mcp.norm(a["parcel"]))
70
+ if unit == "chain":
71
+ return mcp.chain(seed, a["order"])[2]
72
+ return mcp.code(seed, "permit", mcp.site_id(seed, a["site"]), a["year"])
73
+
74
+
75
+ def log_path(dir):
76
+ """The server's call log: beside the trial's verdict, not in the workspace."""
77
+ return os.path.join(os.path.dirname(os.path.abspath(dir)), "verifier", "mcp-calls.jsonl")
78
+
79
+
80
+ def read_log(path):
81
+ out = []
82
+ try:
83
+ with open(path) as f:
84
+ for line in f:
85
+ try:
86
+ out.append(json.loads(line))
87
+ except ValueError:
88
+ pass
89
+ except OSError:
90
+ pass
91
+ return out
92
+
93
+
94
+ def _flat(s):
95
+ return re.sub(r"[^A-Za-z0-9]", "", str(s or "")).upper()
96
+
97
+
98
+ def _ok(c, tool, **want):
99
+ """A call of ``tool`` that succeeded with these arguments (compared as the server compares them)."""
100
+ if c.get("tool") != tool or c.get("isError"):
101
+ return False
102
+ args = c.get("arguments") or {}
103
+ for k, v in want.items():
104
+ got = args.get(k)
105
+ if isinstance(v, int):
106
+ if isinstance(got, bool) or not isinstance(got, (int, float)) or got != v:
107
+ return False
108
+ elif mcp.norm(got) != mcp.norm(v):
109
+ return False
110
+ return True
111
+
112
+
113
+ def missing(unit, seed, calls):
114
+ """What the log lacks for ``unit``: None when the calls are there."""
115
+ a = asked(seed)
116
+ if unit == "one-tool":
117
+ return None if any(_ok(c, "lookup_code") for c in calls) else "lookup_code was never called"
118
+ if unit == "arguments":
119
+ return None if any(_ok(c, "get_stock_code", warehouse=a["warehouse"], item=a["item"]) for c in calls) \
120
+ else f"no successful get_stock_code call for item {a['item']} in {a['warehouse']}"
121
+ if unit == "decoys":
122
+ return None if any(_ok(c, "get_tracking_number", parcel=a["parcel"]) for c in calls) \
123
+ else f"get_tracking_number was never called for {a['parcel']}"
124
+ if unit == "chain":
125
+ customer, account, _ = mcp.chain(seed, a["order"])
126
+ hops = [("get_order_customer", {"order": a["order"]}), ("get_customer_account", {"customer": customer}),
127
+ ("get_account_settlement", {"account": account})]
128
+ i = 0
129
+ for c in calls: # the three hops, in order
130
+ if i < len(hops) and _ok(c, hops[i][0], **hops[i][1]):
131
+ i += 1
132
+ return None if i == len(hops) else f"the chain stopped before {hops[i][0]}"
133
+ sid = mcp.site_id(seed, a["site"])
134
+ refused = next((n for n, c in enumerate(calls) if c.get("tool") == "get_permit_code" and c.get("isError")), None)
135
+ if refused is None:
136
+ return "get_permit_code never answered with an error"
137
+ if not any(_ok(c, "get_permit_code", site=sid, year=a["year"]) for c in calls[refused + 1:]):
138
+ return "no corrected get_permit_code call after the error"
139
+ return None
140
+
141
+
142
+ def verdict(unit, seed, calls, answer):
143
+ """(passed, detail) for one unit, from the server's log and the final message."""
144
+ want = expected(unit, seed)
145
+ said = answer or ""
146
+ why = None if _flat(want) in _flat(said) else "the final message does not carry the answer"
147
+ why = why or missing(unit, seed, calls)
148
+ codes = set(re.findall(r"[A-Za-z0-9]{4}-?[A-Za-z0-9]{4}", said))
149
+ came = [c["tool"] for c in calls if not c.get("isError") and any(_flat(x) in _flat(c.get("text")) for x in codes)]
150
+ detail = {"said": said[:200], "expected": want, "calls": [c.get("tool") for c in calls]}
151
+ if why:
152
+ detail["why"] = why
153
+ if came:
154
+ detail["answer_from"] = sorted(set(came))
155
+ return why is None, detail
156
+
157
+
158
+ def units(dataset):
159
+ return {"units": list(UNITS)}
160
+
161
+
162
+ def setup(unit, dir):
163
+ if unit not in UNITS:
164
+ raise ValueError(f"no unit {unit!r}")
165
+ SEEDS[unit] = seed = secrets.token_hex(16)
166
+ log = log_path(dir)
167
+ os.makedirs(os.path.dirname(log), exist_ok=True)
168
+ if os.path.exists(log):
169
+ os.remove(log)
170
+ env = {"KIT_SEED": seed, "KIT_LOG": log}
171
+ if unit == "one-tool":
172
+ env["KIT_TOOLS"] = "lookup_code"
173
+ return {"prompt": prompt(unit, seed),
174
+ "mcp": [{"name": "kit", "command": PYTHON, "args": [os.path.join(HERE, "mcp_kit.py")], "env": env}]}
175
+
176
+
177
+ def score(unit, dir, transcript, answer):
178
+ if unit not in SEEDS:
179
+ raise ValueError(f"{unit} was not set up")
180
+ ok, detail = verdict(unit, SEEDS[unit], read_log(log_path(dir)), answer)
181
+ return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": detail}
182
+
183
+
184
+ def oracle(unit, dir):
185
+ """The reference: the calls a correct agent makes, sent to the kit's own
186
+ server code (``mcp.handle``, logged as the server logs them), and the
187
+ answer they lead to - so the reference passes through the same scorer."""
188
+ seed, a = SEEDS[unit], asked(SEEDS[unit])
189
+ if unit == "chain":
190
+ customer, account, _ = mcp.chain(seed, a["order"])
191
+ calls = [("get_order_customer", {"order": a["order"]}), ("get_customer_account", {"customer": customer}),
192
+ ("get_account_settlement", {"account": account})]
193
+ else:
194
+ calls = {"one-tool": [("lookup_code", {})],
195
+ "arguments": [("get_stock_code", {"warehouse": a["warehouse"], "item": a["item"]})],
196
+ "decoys": [("get_tracking_number", {"parcel": a["parcel"]})],
197
+ "retry": [("get_permit_code", {"site": a["site"], "year": a["year"]}), # refused: a name, not an id
198
+ ("get_permit_code", {"site": mcp.site_id(seed, a["site"]), "year": a["year"]})]}[unit]
199
+ for n, (tool, args) in enumerate(calls):
200
+ mcp.handle({"jsonrpc": "2.0", "id": n, "method": "tools/call", "params": {"name": tool, "arguments": args}},
201
+ seed, log=log_path(dir))
202
+ return {"answer": expected(unit, seed)}
203
+
204
+
205
+ if __name__ == "__main__":
206
+ serve({"units": units, "setup": setup, "score": score, "oracle": oracle})
@@ -0,0 +1,209 @@
1
+ """The tools kit's MCP server: JSON-RPC 2.0 over stdio, one message per line.
2
+
3
+ Twenty tools of a small shipping office - stock, parcels, orders, accounts,
4
+ site permits - most of them decoys for any one check: names alike, answers
5
+ of the same shape. Every value is derived from ``$KIT_SEED`` (one seed per
6
+ unit, drawn by the driver and never in a prompt), so an answer cannot be
7
+ guessed, only fetched with the right tool and the right arguments.
8
+
9
+ Every ``tools/call`` is appended to ``$KIT_LOG`` as one JSON line - the tool,
10
+ its arguments, whether it was an error, what it said - and the driver scores
11
+ from that log and the final answer. ``$KIT_TOOLS``, a comma-separated list,
12
+ narrows what ``tools/list`` offers (the one-tool check); every tool by default.
13
+
14
+ The driver imports this module for the same derivations (:func:`code`,
15
+ :func:`site_id`, :func:`chain`), so the server and the scoring cannot drift.
16
+ """
17
+ import base64
18
+ import hashlib
19
+ import hmac
20
+ import json
21
+ import os
22
+ import sys
23
+ import time
24
+
25
+ PROTOCOL = "2025-06-18"
26
+
27
+
28
+ def code(seed, *parts):
29
+ """A code such as ``Q7KD-29XF``: eight characters of an HMAC of ``parts``
30
+ under the unit's seed. Every tool answers in this shape."""
31
+ d = hmac.new(str(seed).encode(), "|".join(str(p) for p in parts).encode(), hashlib.sha256).digest()
32
+ s = base64.b32encode(d).decode()[:8]
33
+ return f"{s[:4]}-{s[4:]}"
34
+
35
+
36
+ def norm(s):
37
+ """An identifier as the tools compare it: case and outer spaces ignored."""
38
+ return " ".join(str(s).split()).upper()
39
+
40
+
41
+ def site_id(seed, name):
42
+ return "SITE-" + code(seed, "site", norm(name)).replace("-", "")[:5]
43
+
44
+
45
+ def chain(seed, order):
46
+ """Order -> customer -> account -> settlement code: the three hops."""
47
+ customer = "CUS-" + code(seed, "customer", int(order))
48
+ account = "ACC-" + code(seed, "account", customer)
49
+ return customer, account, code(seed, "settlement", account)
50
+
51
+
52
+ def _s(desc):
53
+ return {"type": "string", "description": desc}
54
+
55
+
56
+ def _i(desc):
57
+ return {"type": "integer", "description": desc}
58
+
59
+
60
+ PARCEL = {"parcel": _s("The parcel's id, such as P-1234.")}
61
+ NUMBERS = { # one family of near-twins: the decoy check asks for one of them
62
+ "get_tracking_number": "The carrier's tracking number of a parcel, as the carrier knows it.",
63
+ "get_tracing_number": "The internal trace number of a parcel in the warehouse scanner log.",
64
+ "get_order_number": "The number of the customer order a parcel belongs to.",
65
+ "get_invoice_number": "The number of the invoice billed for a parcel.",
66
+ "get_return_number": "The return authorisation number of a parcel sent back.",
67
+ "get_customs_number": "The customs declaration number of a parcel.",
68
+ "get_batch_number": "The packing batch number a parcel was packed in.",
69
+ "get_route_number": "The delivery route number a parcel is on.",
70
+ "get_pallet_number": "The number of the pallet a parcel travels on.",
71
+ "get_seal_number": "The number of the tamper seal on a parcel.",
72
+ }
73
+ # "The site", not "the site's id": a schema asking for an id the agent has no
74
+ # way to learn sends a careful agent looking for one instead of calling - the
75
+ # error is where it learns the id, as from many a real API.
76
+ SITE_YEAR = {"site": _s("The site."),"year": _i("The year, such as 2025.")}
77
+ STOCK = {"warehouse": _s("The warehouse's name."), "item": _i("The item number.")}
78
+
79
+ TOOLS = [
80
+ ("lookup_code", "Returns the code word of this check.", {}),
81
+ ("get_stock_code", "The stock code of an item in a warehouse.", STOCK),
82
+ ("get_stock_count", "How many of an item a warehouse holds.", STOCK),
83
+ *[(n, d, PARCEL) for n, d in NUMBERS.items()],
84
+ ("get_order_status", "Where an order stands: received, packed, shipped or delivered.", {"order": _i("The order number.")}),
85
+ ("get_order_customer", "The id of the customer who placed an order.", {"order": _i("The order number.")}),
86
+ ("get_customer_account", "The id of the account a customer pays from.", {"customer": _s("The customer's id.")}),
87
+ ("get_account_settlement", "The settlement code of an account.", {"account": _s("The account's id.")}),
88
+ ("get_account_balance", "The balance of an account, in euros.", {"account": _s("The account's id.")}),
89
+ ("get_permit_code", "The permit code of a site for a year.", SITE_YEAR),
90
+ ("get_permit_fee", "The permit fee of a site for a year, in euros.", SITE_YEAR),
91
+ ]
92
+
93
+
94
+ class ToolError(Exception):
95
+ """What was wrong with a call, said back to the agent as ``isError``."""
96
+
97
+
98
+ def _int(args, name):
99
+ v = args.get(name)
100
+ if isinstance(v, bool) or not (isinstance(v, int) or (isinstance(v, float) and v.is_integer())):
101
+ raise ToolError(f"{name} must be an integer, not {json.dumps(v)}")
102
+ return int(v)
103
+
104
+
105
+ def _str(args, name):
106
+ v = args.get(name)
107
+ if not isinstance(v, str) or not v.strip():
108
+ raise ToolError(f"{name} must be a non-empty string, not {json.dumps(v)}")
109
+ return v
110
+
111
+
112
+ def _site(seed, args):
113
+ """A site is named by its id; a name is refused with the id it has, the
114
+ one way to learn it - so the first call errs and the second can succeed."""
115
+ s = norm(_str(args, "site"))
116
+ if not s.startswith("SITE-"):
117
+ raise ToolError(f"{_str(args, 'site')!r} is not a site id: sites are named by id here, "
118
+ f"and the site called {_str(args, 'site').strip()!r} has the id {site_id(seed, s)}")
119
+ return s
120
+
121
+
122
+ def call(seed, name, args):
123
+ """The text a tool returns; :class:`ToolError` for a call it refuses."""
124
+ args = args if isinstance(args, dict) else {}
125
+ if name == "lookup_code":
126
+ return "kit-" + code(seed, "lookup").replace("-", "").lower()
127
+ if name in ("get_stock_code", "get_stock_count"):
128
+ w, i = norm(_str(args, "warehouse")), _int(args, "item")
129
+ if name == "get_stock_count":
130
+ return f"{int(code(seed, 'count', w, i).encode().hex(), 16) % 500} in stock"
131
+ return f"stock code of item {i} in {w.title()}: {code(seed, 'stock', w, i)}"
132
+ if name in NUMBERS:
133
+ p = norm(_str(args, "parcel"))
134
+ return f"{name[4:].replace('_', ' ')} of parcel {p}: {code(seed, name, p)}"
135
+ if name == "get_order_status":
136
+ o = _int(args, "order")
137
+ return f"order {o}: " + ("received", "packed", "shipped", "delivered")[ord(code(seed, "status", o)[0]) % 4]
138
+ if name == "get_order_customer":
139
+ return f"order {_int(args, 'order')} was placed by customer {chain(seed, _int(args, 'order'))[0]}"
140
+ if name == "get_customer_account":
141
+ c = norm(_str(args, "customer"))
142
+ return f"customer {c} pays from account ACC-{code(seed, 'account', c)}"
143
+ if name == "get_account_settlement":
144
+ a = norm(_str(args, "account"))
145
+ return f"settlement code of account {a}: {code(seed, 'settlement', a)}"
146
+ if name == "get_account_balance":
147
+ a = norm(_str(args, "account"))
148
+ return f"balance of account {a}: {int(code(seed, 'balance', a).encode().hex(), 16) % 90000 / 100:.2f} EUR"
149
+ if name in ("get_permit_code", "get_permit_fee"):
150
+ s, y = _site(seed, args), _int(args, "year")
151
+ if name == "get_permit_fee":
152
+ return f"permit fee of {s} for {y}: {100 + int(code(seed, 'fee', s, y).encode().hex(), 16) % 900} EUR"
153
+ return f"permit code of {s} for {y}: {code(seed, 'permit', s, y)}"
154
+ raise KeyError(name)
155
+
156
+
157
+ def listed(only=None):
158
+ keep = [t for t in TOOLS if not only or t[0] in only]
159
+ return [{"name": n, "description": d,
160
+ "inputSchema": dict({"type": "object", "properties": props}, **({"required": list(props)} if props else {}))}
161
+ for n, d, props in keep]
162
+
163
+
164
+ def handle(m, seed, only=None, log=None):
165
+ """One JSON-RPC message in, its reply out (None for a notification)."""
166
+ if not isinstance(m, dict) or "id" not in m:
167
+ return None
168
+ method, rid, params = m.get("method"), m["id"], m.get("params") or {}
169
+ if method == "initialize":
170
+ result = {"protocolVersion": params.get("protocolVersion") or PROTOCOL,
171
+ "capabilities": {"tools": {}}, "serverInfo": {"name": "kit", "version": "2"}}
172
+ elif method == "tools/list":
173
+ result = {"tools": listed(only)}
174
+ elif method == "tools/call":
175
+ name, args = params.get("name"), params.get("arguments") or {}
176
+ if name not in [t["name"] for t in listed(only)]:
177
+ return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32602, "message": f"unknown tool {name!r}"}}
178
+ try:
179
+ text, err = call(seed, name, args), False
180
+ except ToolError as e:
181
+ text, err = str(e), True
182
+ if log:
183
+ with open(log, "a") as f:
184
+ f.write(json.dumps({"t": time.time(), "tool": name, "arguments": args, "isError": err, "text": text}) + "\n")
185
+ result = {"content": [{"type": "text", "text": text}], "isError": err}
186
+ elif method == "ping":
187
+ result = {}
188
+ elif method in ("resources/list", "prompts/list"):
189
+ result = {method.split("/")[0]: []}
190
+ else:
191
+ return {"jsonrpc": "2.0", "id": rid, "error": {"code": -32601, "message": "no " + str(method)}}
192
+ return {"jsonrpc": "2.0", "id": rid, "result": result}
193
+
194
+
195
+ def main():
196
+ seed, log = os.environ.get("KIT_SEED", ""), os.environ.get("KIT_LOG") or None
197
+ only = [t for t in os.environ.get("KIT_TOOLS", "").split(",") if t] or None
198
+ for line in sys.stdin:
199
+ try:
200
+ m = json.loads(line)
201
+ except ValueError:
202
+ continue
203
+ reply = handle(m, seed, only, log)
204
+ if reply is not None:
205
+ print(json.dumps(reply), flush=True)
206
+
207
+
208
+ if __name__ == "__main__":
209
+ main()
@@ -0,0 +1,119 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "agent-tools-v1",
4
+ "kit_version": 2,
5
+ "label": "Kit: agent.tools/v1",
6
+ "summary": "Five checks against one MCP server of twenty tools: two basic - a tool without arguments, one with typed arguments - and three skills - the right tool among decoys, a chain of three, a call refused and corrected.",
7
+ "family": "agent",
8
+ "needs": [
9
+ "agent.tools/v1"
10
+ ],
11
+ "basic": [
12
+ "one-tool",
13
+ "arguments"
14
+ ],
15
+ "skills": {
16
+ "decoys": "chooses the right tool among similar ones",
17
+ "chain": "feeds one tool's answer into the next, three times over",
18
+ "retry": "reads a tool's error and calls it again, corrected"
19
+ },
20
+ "driver": {
21
+ "command": [
22
+ "python3",
23
+ "{here}/driver.py"
24
+ ]
25
+ },
26
+ "datasets": [
27
+ {
28
+ "id": "kit",
29
+ "units": 5
30
+ }
31
+ ],
32
+ "units": {
33
+ "noun": "check",
34
+ "nouns": "checks"
35
+ },
36
+ "controls": {
37
+ "nop": {
38
+ "source": "nullius",
39
+ "from": "Nullius: no agent runs; the answer is empty and the workspace is left as setup made it."
40
+ },
41
+ "oracle": {
42
+ "source": "nullius",
43
+ "from": "The code word the kit's setup gave the unit's MCP server."
44
+ }
45
+ },
46
+ "score": {
47
+ "kind": "pass",
48
+ "says": "the final message carries the answer, and the server's call log shows the calls that produce it"
49
+ },
50
+ "limits": {
51
+ "turn_seconds": 300
52
+ },
53
+ "proven": {
54
+ "kit": {
55
+ "units": {
56
+ "units": 5,
57
+ "passed": "5 ids, the same twice, unique",
58
+ "on": "2026-10-12"
59
+ },
60
+ "leak": {
61
+ "units": 5,
62
+ "looked_for": 1,
63
+ "short": 4,
64
+ "passed": "5/5",
65
+ "note": "4 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
66
+ "on": "2026-10-12"
67
+ },
68
+ "nop": {
69
+ "passed": "0/5",
70
+ "value": 0.0,
71
+ "expect": {
72
+ "at_most": 0.0
73
+ },
74
+ "units": 5,
75
+ "job": "2026-10-12__16-28-38__agent-tools-v1__kit__prove-nop",
76
+ "source": "nullius",
77
+ "on": "2026-10-12"
78
+ },
79
+ "oracle": {
80
+ "passed": "5/5",
81
+ "value": 1.0,
82
+ "expect": {
83
+ "at_least": 1.0
84
+ },
85
+ "units": 5,
86
+ "job": "2026-10-12__16-28-38__agent-tools-v1__kit__prove-oracle",
87
+ "source": "nullius",
88
+ "on": "2026-10-12"
89
+ }
90
+ }
91
+ },
92
+ "check_details": {
93
+ "one-tool": {
94
+ "label": "Call one tool",
95
+ "task": "Find the code word using the single available MCP tool.",
96
+ "passes": "The final answer contains the generated code and the server records a successful lookup_code call."
97
+ },
98
+ "arguments": {
99
+ "label": "Pass typed arguments",
100
+ "task": "Find an item’s stock code in a named warehouse.",
101
+ "passes": "The answer contains the code and the server records get_stock_code with the correct warehouse and numeric item."
102
+ },
103
+ "decoys": {
104
+ "label": "Choose among decoys",
105
+ "task": "Find a parcel’s tracking number among twenty similar tools.",
106
+ "passes": "The answer contains the tracking number and the server records get_tracking_number for that parcel."
107
+ },
108
+ "chain": {
109
+ "label": "Chain three calls",
110
+ "task": "Find the settlement code for the account paying for an order.",
111
+ "passes": "The answer contains the code; successful order → customer → account → settlement calls appear in that order."
112
+ },
113
+ "retry": {
114
+ "label": "Recover from a tool error",
115
+ "task": "Find a site’s permit code for a given year.",
116
+ "passes": "The answer contains the code; a failed get_permit_code call is followed by a successful call with the correct site ID and year."
117
+ }
118
+ }
119
+ }
@@ -0,0 +1,65 @@
1
+ """Kit for agent.workspace/v1: the agent reads and writes in its working directory."""
2
+ import os
3
+ import stat
4
+ import subprocess
5
+ import sys
6
+
7
+ from nullius.standard.container import check_argv
8
+ from nullius.standard.jsonl import serve
9
+
10
+ CALC = "def add(a, b):\n return a - b\n"
11
+ TEST = "from calc import add\nassert add(2, 3) == 5, add(2, 3)\nprint('OK')\n"
12
+
13
+
14
+ def units(dataset):
15
+ return {"units": ["write", "fix"]}
16
+
17
+
18
+ def setup(unit, dir):
19
+ if unit == "write":
20
+ return {"prompt": "In the current working directory, create a file named result.txt whose whole content is: nullius-ok"}
21
+ with open(os.path.join(dir, "calc.py"), "w") as f:
22
+ f.write(CALC)
23
+ with open(os.path.join(dir, "test_calc.py"), "w") as f:
24
+ f.write(TEST)
25
+ return {"prompt": "In the current working directory, calc.py has a bug: `python3 test_calc.py` fails. Fix calc.py so the test prints OK. Do not change test_calc.py."}
26
+
27
+
28
+ def score(unit, dir, transcript, answer):
29
+ if unit == "write":
30
+ got = _read_plain(os.path.join(dir, "result.txt"))
31
+ ok = got == "nullius-ok"
32
+ return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"result.txt": got}}
33
+ test = os.path.join(dir, "test_calc.py")
34
+ if os.path.lexists(test): # the test as given, whatever happened to it - a link included
35
+ os.unlink(test)
36
+ with open(test, "w") as f:
37
+ f.write(TEST)
38
+ argv = check_argv(dir, [sys.executable, "test_calc.py"], ["timeout", "60", "python3", "test_calc.py"])
39
+ r = subprocess.run(argv, cwd=dir, capture_output=True, text=True, timeout=120)
40
+ ok = r.returncode == 0 and "OK" in r.stdout
41
+ return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"stdout": r.stdout[-500:], "stderr": r.stderr[-500:]}}
42
+
43
+
44
+ def _read_plain(path):
45
+ """A regular file's text, or None: a link the agent planted is not followed
46
+ out of its workspace (on docker it could point at the user's settings)."""
47
+ try:
48
+ fd = os.open(path, os.O_RDONLY | getattr(os, "O_NOFOLLOW", 0))
49
+ except OSError:
50
+ return None
51
+ with os.fdopen(fd) as f:
52
+ return f.read().strip() if stat.S_ISREG(os.fstat(f.fileno()).st_mode) else None
53
+
54
+
55
+ def oracle(unit, dir):
56
+ if unit == "write":
57
+ with open(os.path.join(dir, "result.txt"), "w") as f:
58
+ f.write("nullius-ok\n")
59
+ else:
60
+ with open(os.path.join(dir, "calc.py"), "w") as f:
61
+ f.write(CALC.replace("a - b", "a + b"))
62
+ return {"answer": ""}
63
+
64
+
65
+ serve({"units": units, "setup": setup, "score": score, "oracle": oracle})