nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,129 @@
1
+ """A program spoken to in JSON lines: one request, one reply.
2
+
3
+ The benchmark's driver and a capability are both this: a process started from
4
+ its definition's command, a request written as one JSON line on its stdin
5
+ (``{"op": ..., ...}``), one JSON line back on its stdout. A reply carrying
6
+ ``error`` is raised as :class:`Failed`; anything that is not one JSON object
7
+ line, or no reply within the timeout, is raised too, naming what it got.
8
+ Its stderr goes to a log file, never into the reply.
9
+ """
10
+ from __future__ import annotations
11
+
12
+ import json
13
+ import os
14
+ import queue
15
+ from concurrent.futures import CancelledError
16
+ from nullius.standard.cancellation import reply as wait_reply
17
+ import signal
18
+ import subprocess
19
+ import threading
20
+ from typing import List, Optional
21
+
22
+
23
+ class Failed(Exception):
24
+ """The program answered with an error, or did not answer as it should."""
25
+
26
+
27
+ class Proc:
28
+ def __init__(self, cmd: List[str], cwd: str, env: dict, log: str, stopped=None):
29
+ self.cmd, self.cwd, self.env, self.log = cmd, cwd, env, log
30
+ self.stopped = stopped
31
+ self.p: Optional[subprocess.Popen] = None
32
+ self.lines: "queue.Queue[Optional[str]]" = queue.Queue()
33
+
34
+ def start(self) -> "Proc":
35
+ if self.stopped is not None and self.stopped.is_set():
36
+ raise CancelledError("the run was stopped")
37
+ self.err = open(self.log, "a")
38
+ self.p = subprocess.Popen(self.cmd, cwd=self.cwd, env=self.env, stdin=subprocess.PIPE,
39
+ stdout=subprocess.PIPE, stderr=self.err, text=True, bufsize=1,
40
+ start_new_session=True)
41
+ threading.Thread(target=self._read, daemon=True).start()
42
+ return self
43
+
44
+ def _read(self):
45
+ for line in self.p.stdout:
46
+ self.lines.put(line)
47
+ self.lines.put(None)
48
+
49
+ def request(self, op: str, timeout: float = 600, **kw) -> dict:
50
+ if self.stopped is not None and self.stopped.is_set():
51
+ raise CancelledError("the run was stopped")
52
+ if self.p is None:
53
+ self.start()
54
+ try:
55
+ self.p.stdin.write(json.dumps(dict(kw, op=op)) + "\n")
56
+ self.p.stdin.flush()
57
+ except (BrokenPipeError, OSError):
58
+ raise Failed(f"{op}: the program had already ended ({self._tail()})")
59
+ try:
60
+ line = wait_reply(self.lines, timeout, self.stopped)
61
+ except queue.Empty:
62
+ self.close()
63
+ raise Failed(f"{op}: no reply in {int(timeout)} s")
64
+ if line is None:
65
+ raise Failed(f"{op}: the program ended without replying ({self._tail()})")
66
+ try:
67
+ reply = json.loads(line)
68
+ except ValueError:
69
+ raise Failed(f"{op}: the reply is not JSON: {line[:200]!r}")
70
+ if not isinstance(reply, dict):
71
+ raise Failed(f"{op}: the reply is not an object")
72
+ if reply.get("error"):
73
+ raise Failed(f"{op}: {reply['error']}")
74
+ return reply
75
+
76
+ def _tail(self) -> str:
77
+ try:
78
+ self.err.flush()
79
+ with open(self.log, errors="replace") as f:
80
+ lines = [l.strip() for l in f.read().splitlines() if l.strip()]
81
+ return lines[-1][:240] if lines else "nothing on stderr"
82
+ except OSError:
83
+ return "no log"
84
+
85
+ def close(self):
86
+ if self.p and self.p.poll() is None:
87
+ try:
88
+ self.p.stdin.close()
89
+ self.p.wait(timeout=3)
90
+ except Exception:
91
+ try:
92
+ os.killpg(self.p.pid, signal.SIGKILL)
93
+ except OSError:
94
+ pass
95
+ if self.p:
96
+ for f in (self.p.stdin, self.p.stdout, self.err):
97
+ try:
98
+ f.close()
99
+ except Exception:
100
+ pass
101
+ try:
102
+ self.p.wait(timeout=3)
103
+ except Exception:
104
+ pass
105
+
106
+
107
+ def serve(handlers: dict, out=None) -> None:
108
+ """The other side, for a driver or a capability written in Python: read
109
+ requests from stdin, answer each with ``handlers[op](**request)``.
110
+ An exception becomes ``{"error": ...}`` and the loop goes on.
111
+
112
+ ``out``: where the replies go, ``sys.stdout`` when not given - a program
113
+ that sends ``sys.stdout`` elsewhere for good (threads printing while a
114
+ request is answered) passes the protocol's own stream here."""
115
+ import sys
116
+ out = out or sys.stdout
117
+ for line in sys.stdin:
118
+ if not line.strip():
119
+ continue
120
+ try:
121
+ req = json.loads(line)
122
+ op = req.pop("op")
123
+ if op not in handlers:
124
+ raise ValueError(f"unknown op {op!r}")
125
+ reply = handlers[op](**req)
126
+ except Exception as e: # the error is the reply
127
+ reply = {"error": f"{type(e).__name__}: {e}"}
128
+ out.write(json.dumps(reply) + "\n")
129
+ out.flush()
@@ -0,0 +1,27 @@
1
+ """Kit for agent.conversation/v1: the second turn needs the first."""
2
+ from nullius.standard.jsonl import serve
3
+
4
+ CODES = {"c1": "heron-42", "c2": "maple-17"}
5
+
6
+
7
+ def units(dataset):
8
+ return {"units": list(CODES)}
9
+
10
+
11
+ def setup(unit, dir):
12
+ return {"prompt": f"Remember this code word: {CODES[unit]}. Reply only with: noted. Do not use any tools."}
13
+
14
+
15
+ def turn(unit, dir, transcript):
16
+ if sum(1 for t in transcript if t["role"] == "user") >= 2:
17
+ return {"done": True}
18
+ return {"message": "What was the code word I gave you? Reply with the code word only."}
19
+
20
+
21
+ def score(unit, dir, transcript, answer):
22
+ asked = sum(1 for t in transcript if t["role"] == "user")
23
+ ok = asked == 2 and CODES[unit] in (answer or "")
24
+ return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"turns": asked, "said": (answer or "")[:200]}}
25
+
26
+
27
+ serve({"units": units, "setup": setup, "turn": turn, "score": score})
@@ -0,0 +1,26 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "agent-conversation-v1",
4
+ "kit_version": 1,
5
+ "label": "Kit: agent.conversation/v1",
6
+ "summary": "A code word said in the first turn, asked for in the second.",
7
+ "family": "agent",
8
+ "needs": ["agent.conversation/v1"],
9
+ "driver": {"command": ["python3", "{here}/driver.py"]},
10
+ "datasets": [{"id": "kit", "units": 2}],
11
+ "units": {"noun": "check", "nouns": "checks"},
12
+ "score": {"kind": "pass", "says": "the second turn's answer is the first turn's code word"},
13
+ "limits": {"turn_seconds": 300},
14
+ "check_details": {
15
+ "c1": {
16
+ "label": "Remember heron-42",
17
+ "task": "Remember heron-42 in the first turn; recall it in the second.",
18
+ "passes": "Exactly two user turns; the final answer contains heron-42."
19
+ },
20
+ "c2": {
21
+ "label": "Remember maple-17",
22
+ "task": "Remember maple-17 in the first turn; recall it in the second.",
23
+ "passes": "Exactly two user turns; the final answer contains maple-17."
24
+ }
25
+ }
26
+ }
@@ -0,0 +1,171 @@
1
+ """Search a generated HTTP collection with the agent's own tools.
2
+
3
+ No MCP server or search implementation is injected. The fixture supplies a
4
+ searchable website, random facts, source documents and trusted request logs.
5
+ Passing proves bounded web search/retrieval, not a public search provider or
6
+ access to arbitrary sites. Reference answers alone cannot pass: the HTTP log
7
+ must show a relevant query followed by the supporting document retrieval.
8
+ """
9
+ import atexit
10
+ import html
11
+ import json
12
+ import os
13
+ import secrets
14
+ import socketserver
15
+ import tempfile
16
+ import threading
17
+ from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
18
+ from urllib.parse import parse_qs, urlsplit, quote
19
+ from urllib.request import urlopen
20
+
21
+ from nullius.standard.container import CHECK_IMAGE
22
+ from nullius.standard.jsonl import serve
23
+
24
+ UNITS = ('find-source', 'disambiguate', 'follow-reference')
25
+ FIXTURES = {}
26
+
27
+
28
+ class UnixServer(socketserver.ThreadingUnixStreamServer):
29
+ daemon_threads = True
30
+
31
+
32
+ class Fixture:
33
+ def __init__(self, unit, directory):
34
+ self.unit = unit
35
+ self.target = 'Archive-' + secrets.token_hex(4)
36
+ self.answer = secrets.token_hex(12)
37
+ self.page = '/documents/' + secrets.token_hex(12)
38
+ self.index = '/documents/' + secrets.token_hex(12)
39
+ self.evidence = f'The current catalogue number of {self.target} is {self.answer}.'
40
+ self.calls = []
41
+ self.lock = threading.Lock()
42
+ self.log = os.path.join(os.path.dirname(directory), 'verifier', 'search-requests.jsonl')
43
+ os.makedirs(os.path.dirname(self.log), exist_ok=True)
44
+ self.temp = tempfile.TemporaryDirectory(prefix='ns-', dir='/tmp')
45
+ self.socket = os.path.join(self.temp.name, 'web.sock')
46
+ fixture = self
47
+
48
+ class Handler(BaseHTTPRequestHandler):
49
+ def log_message(self, *args):
50
+ pass
51
+
52
+ def do_GET(self):
53
+ status, body, event = fixture.response(self.path)
54
+ if event:
55
+ with fixture.lock:
56
+ fixture.calls.append(event)
57
+ with open(fixture.log, 'a') as f:
58
+ f.write(json.dumps(event) + '\n')
59
+ data = ('<!doctype html><html><body>' + body + '</body></html>').encode()
60
+ self.send_response(status)
61
+ self.send_header('Content-Type', 'text/html; charset=utf-8')
62
+ self.send_header('Content-Length', str(len(data)))
63
+ self.end_headers()
64
+ self.wfile.write(data)
65
+
66
+ self.tcp = ThreadingHTTPServer(('127.0.0.1', 0), Handler)
67
+ self.unix = UnixServer(self.socket, Handler)
68
+ self.local = f'http://127.0.0.1:{self.tcp.server_port}'
69
+ self.base = 'http://web-search:9001' if os.environ.get(CHECK_IMAGE) else self.local
70
+ for server in (self.tcp, self.unix):
71
+ threading.Thread(target=server.serve_forever, daemon=True).start()
72
+ self.closed = False
73
+
74
+ def response(self, path):
75
+ parsed = urlsplit(path)
76
+ if parsed.path == '/':
77
+ return 200, '<h1>Research archive</h1><form action="/search"><input name="q"><button>Search</button></form>', None
78
+ if parsed.path == '/search':
79
+ q = parse_qs(parsed.query).get('q', [''])[0]
80
+ if self.target.casefold() not in q.casefold():
81
+ return 200, '<p>No matching documents.</p>', {'kind': 'search', 'relevant': False}
82
+ link = self.index if self.unit == 'follow-reference' else self.page
83
+ rows = [f'<li><a href="{link}">{self.target}: current official record</a></li>']
84
+ if self.unit == 'disambiguate':
85
+ rows.insert(0, '<li><a href="/obsolete">Archived record (superseded)</a></li>')
86
+ rows.append('<li><a href="/unrelated">A similarly named archive</a></li>')
87
+ return 200, '<ul>' + ''.join(rows) + '</ul>', {'kind': 'search', 'relevant': True}
88
+ if parsed.path == self.index and self.unit == 'follow-reference':
89
+ return 200, f'<p>The current record is maintained in the <a href="{self.page}">source register</a>.</p>', {'kind': 'index'}
90
+ if parsed.path == self.page:
91
+ return 200, '<h1>Current source register</h1><p>' + html.escape(self.evidence) + '</p>', {'kind': 'source'}
92
+ if parsed.path in ('/obsolete', '/unrelated'):
93
+ return 200, '<p>Obsolete or unrelated catalogue number: NOT-CURRENT. Do not use as the current record.</p>', {'kind': 'decoy'}
94
+ return 404, '<p>Not found</p>', None
95
+
96
+ def close(self):
97
+ if not self.closed:
98
+ self.closed = True
99
+ for server in (self.tcp, self.unix):
100
+ server.shutdown()
101
+ server.server_close()
102
+ self.temp.cleanup()
103
+
104
+
105
+ def fixture(unit, directory):
106
+ return FIXTURES[(unit, os.path.abspath(directory))]
107
+
108
+
109
+ def units(dataset):
110
+ return {'units': list(UNITS)}
111
+
112
+
113
+ def setup(unit, dir):
114
+ if unit not in UNITS:
115
+ raise ValueError('unknown search check')
116
+ key = (unit, os.path.abspath(dir))
117
+ if key in FIXTURES:
118
+ FIXTURES[key].close()
119
+ f = FIXTURES[key] = Fixture(unit, os.path.abspath(dir))
120
+ return {'prompt': f'Search the research archive at {f.base}/ for the current catalogue number of {f.target}. '
121
+ 'Find and open the supporting source. Return only a JSON object with answer (the catalogue number), '
122
+ 'source (the full supporting document URL), and evidence (an exact sentence from that document). '
123
+ 'Use your available web tools; the website has a search form.',
124
+ 'web_services': [{'name': 'web-search', 'port': 9001, 'socket': f.socket}]}
125
+
126
+
127
+ def verdict(f, answer):
128
+ try:
129
+ result = json.loads(answer or '')
130
+ except (ValueError, TypeError):
131
+ result = {}
132
+ if not isinstance(result, dict):
133
+ result = {}
134
+ wanted = ['search', 'index', 'source'] if f.unit == 'follow-reference' else ['search', 'source']
135
+ position = 0
136
+ for event in f.calls:
137
+ if position < len(wanted) and event['kind'] == wanted[position] and event.get('relevant', True):
138
+ position += 1
139
+ checks = {'answer': result.get('answer') == f.answer,
140
+ 'source': result.get('source') == f.base + f.page,
141
+ 'evidence': result.get('evidence') == f.evidence,
142
+ 'retrieval': position == len(wanted)}
143
+ return {'passed': all(checks.values()), 'reward': float(all(checks.values())), 'detail': checks}
144
+
145
+
146
+ def score(unit, dir, transcript, answer):
147
+ f = fixture(unit, dir)
148
+ result = verdict(f, answer)
149
+ f.close()
150
+ return result
151
+
152
+
153
+ def oracle(unit, dir):
154
+ # The leak checker invokes the oracle on a copy of the workspace.
155
+ f = FIXTURES.get((unit, os.path.abspath(dir)))
156
+ if f is None:
157
+ f = next(value for (name, _), value in reversed(list(FIXTURES.items())) if name == unit)
158
+ for path in ['/search?q=' + quote(f.target)] + ([f.index] if unit == 'follow-reference' else []) + [f.page]:
159
+ with urlopen(f.local + path, timeout=5) as response:
160
+ response.read()
161
+ return {'answer': json.dumps({'answer': f.answer, 'source': f.base + f.page, 'evidence': f.evidence})}
162
+
163
+
164
+ @atexit.register
165
+ def close():
166
+ for f in FIXTURES.values():
167
+ f.close()
168
+
169
+
170
+ if __name__ == '__main__':
171
+ serve({'units': units, 'setup': setup, 'score': score, 'oracle': oracle})
@@ -0,0 +1,99 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "agent-search-v1",
4
+ "kit_version": 1,
5
+ "label": "Kit: agent.search/v1",
6
+ "summary": "Search a generated HTTP collection, distinguish the current source and follow a reference; retrieve the supporting document and cite its exact evidence. Uses the agent\u2019s available tools, with no injected MCP tools.",
7
+ "family": "agent",
8
+ "needs": [
9
+ "agent.search/v1"
10
+ ],
11
+ "driver": {
12
+ "command": [
13
+ "python3",
14
+ "{here}/driver.py"
15
+ ]
16
+ },
17
+ "datasets": [
18
+ {
19
+ "id": "kit",
20
+ "units": 3
21
+ }
22
+ ],
23
+ "units": {
24
+ "noun": "check",
25
+ "nouns": "checks"
26
+ },
27
+ "controls": {
28
+ "nop": {
29
+ "source": "nullius",
30
+ "from": "No agent runs; no HTTP retrieval occurs and the answer is empty."
31
+ },
32
+ "oracle": {
33
+ "source": "nullius",
34
+ "from": "The fixture client searches and retrieves the generated sources through HTTP, then returns their evidence."
35
+ }
36
+ },
37
+ "score": {
38
+ "kind": "pass",
39
+ "says": "Correct JSON answer, exact source URL and evidence sentence, plus ordered search and source retrieval in the fixture request log"
40
+ },
41
+ "limits": {
42
+ "turn_seconds": 300
43
+ },
44
+ "check_details": {
45
+ "find-source": {
46
+ "label": "Find a source",
47
+ "task": "Search for a named record and retrieve its current catalogue number.",
48
+ "passes": "The generated catalogue number, source URL and exact evidence sentence match; the trusted HTTP log shows a relevant search before source retrieval."
49
+ },
50
+ "disambiguate": {
51
+ "label": "Choose the current source",
52
+ "task": "Choose the current record among obsolete and unrelated results.",
53
+ "passes": "The generated catalogue number, source URL and exact evidence sentence match; the trusted HTTP log shows a relevant search before source retrieval."
54
+ },
55
+ "follow-reference": {
56
+ "label": "Follow a source reference",
57
+ "task": "Follow a search result\u2019s reference to the document containing the answer.",
58
+ "passes": "The generated catalogue number, source URL and exact evidence sentence match; the trusted HTTP log shows a relevant search before source retrieval with the intermediate record opened first."
59
+ }
60
+ },
61
+ "proven": {
62
+ "kit": {
63
+ "units": {
64
+ "units": 3,
65
+ "passed": "3 ids, the same twice, unique",
66
+ "on": "2026-11-11"
67
+ },
68
+ "leak": {
69
+ "units": 3,
70
+ "looked_for": 3,
71
+ "short": 0,
72
+ "passed": "3/3",
73
+ "on": "2026-11-11"
74
+ },
75
+ "nop": {
76
+ "passed": "0/3",
77
+ "value": 0.0,
78
+ "expect": {
79
+ "at_most": 0.0
80
+ },
81
+ "units": 3,
82
+ "job": "2026-11-11__23-26-20__agent-search-v1__kit__prove-nop",
83
+ "source": "nullius",
84
+ "on": "2026-11-11"
85
+ },
86
+ "oracle": {
87
+ "passed": "3/3",
88
+ "value": 1.0,
89
+ "expect": {
90
+ "at_least": 1.0
91
+ },
92
+ "units": 3,
93
+ "job": "2026-11-11__23-26-20__agent-search-v1__kit__prove-oracle",
94
+ "source": "nullius",
95
+ "on": "2026-11-11"
96
+ }
97
+ }
98
+ }
99
+ }
@@ -0,0 +1,25 @@
1
+ """Kit for agent.task/v1: the answer is the agent's final message."""
2
+ from nullius.standard.jsonl import serve
3
+
4
+ WORDS = {"t1": "lantern", "t2": "quartz", "t3": "meadow"}
5
+
6
+
7
+ def units(dataset):
8
+ return {"units": list(WORDS)}
9
+
10
+
11
+ def setup(unit, dir):
12
+ return {"prompt": f"Reply with exactly one word, the word {WORDS[unit]}, and nothing else. Do not use any tools."}
13
+
14
+
15
+ def score(unit, dir, transcript, answer):
16
+ said = (answer or "").strip().strip(".!*`\"'").lower()
17
+ ok = said == WORDS[unit]
18
+ return {"reward": 1.0 if ok else 0.0, "passed": ok, "detail": {"expected": WORDS[unit], "said": (answer or "")[:200]}}
19
+
20
+
21
+ def oracle(unit, dir):
22
+ return {"answer": WORDS[unit]}
23
+
24
+
25
+ serve({"units": units, "setup": setup, "score": score, "oracle": oracle})
@@ -0,0 +1,100 @@
1
+ {
2
+ "schema": "nullius.benchmark/v1",
3
+ "key": "agent-task-v1",
4
+ "kit_version": 1,
5
+ "label": "Kit: agent.task/v1",
6
+ "summary": "Three prompts asking for one exact word: the agent answers in its final message.",
7
+ "family": "agent",
8
+ "needs": [
9
+ "agent.task/v1"
10
+ ],
11
+ "driver": {
12
+ "command": [
13
+ "python3",
14
+ "{here}/driver.py"
15
+ ]
16
+ },
17
+ "datasets": [
18
+ {
19
+ "id": "kit",
20
+ "units": 3
21
+ }
22
+ ],
23
+ "units": {
24
+ "noun": "check",
25
+ "nouns": "checks"
26
+ },
27
+ "controls": {
28
+ "nop": {
29
+ "source": "nullius",
30
+ "from": "Nullius: no agent runs; the answer is empty and the workspace is left as setup made it."
31
+ },
32
+ "oracle": {
33
+ "source": "nullius",
34
+ "from": "The kit's own word for each check, written with the kit."
35
+ }
36
+ },
37
+ "score": {
38
+ "kind": "pass",
39
+ "says": "the final message is the word asked for"
40
+ },
41
+ "limits": {
42
+ "turn_seconds": 300
43
+ },
44
+ "proven": {
45
+ "kit": {
46
+ "units": {
47
+ "units": 3,
48
+ "passed": "3 ids, the same twice, unique",
49
+ "on": "2026-10-12"
50
+ },
51
+ "leak": {
52
+ "units": 3,
53
+ "looked_for": 0,
54
+ "short": 3,
55
+ "passed": "3/3",
56
+ "note": "3 reference(s) under 12 characters not looked for: too short to tell a leak from chance",
57
+ "on": "2026-10-12"
58
+ },
59
+ "nop": {
60
+ "passed": "0/3",
61
+ "value": 0.0,
62
+ "expect": {
63
+ "at_most": 0.0
64
+ },
65
+ "units": 3,
66
+ "job": "2026-10-12__16-13-24__agent-task-v1__kit__prove-nop",
67
+ "source": "nullius",
68
+ "on": "2026-10-12"
69
+ },
70
+ "oracle": {
71
+ "passed": "3/3",
72
+ "value": 1.0,
73
+ "expect": {
74
+ "at_least": 1.0
75
+ },
76
+ "units": 3,
77
+ "job": "2026-10-12__16-13-24__agent-task-v1__kit__prove-oracle",
78
+ "source": "nullius",
79
+ "on": "2026-10-12"
80
+ }
81
+ }
82
+ },
83
+ "check_details": {
84
+ "t1": {
85
+ "label": "Reply with lantern",
86
+ "task": "Reply with exactly one word: lantern. Do not use tools.",
87
+ "passes": "The final answer equals lantern, ignoring case, surrounding whitespace and . ! * ` quotes."
88
+ },
89
+ "t2": {
90
+ "label": "Reply with quartz",
91
+ "task": "Reply with exactly one word: quartz. Do not use tools.",
92
+ "passes": "The final answer equals quartz, ignoring case, surrounding whitespace and . ! * ` quotes."
93
+ },
94
+ "t3": {
95
+ "label": "Reply with meadow",
96
+ "task": "Reply with exactly one word: meadow. Do not use tools.",
97
+ "passes": "The final answer equals meadow, ignoring case, surrounding whitespace and . ! * ` quotes."
98
+ }
99
+ }
100
+ }