nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,373 @@
1
+ """A benchmark's proofs: its controls behave, its units are stable, nothing leaks.
2
+
3
+ An agent or a capability proves its ports by kits (``conform``); a benchmark
4
+ proves itself, per dataset, by four checks - what ``cli prove`` runs and
5
+ writes into the benchmark's ``proven``:
6
+
7
+ - ``nop``, the null control: no agent, an empty answer, the workspace as
8
+ setup left it. Nothing should pass - every score at the floor - up to the
9
+ bound the control declares (``controls.nop.expect.at_most``, 0 by default).
10
+ - ``oracle``, the reference control: the driver's ``oracle`` answer, and
11
+ what it writes into the workspace. Everything should pass - every score at
12
+ the ceiling - down to ``controls.oracle.expect.at_least`` (1 by default; a
13
+ judged benchmark may declare less, the judge being a model).
14
+ - ``units``: two ``units`` calls, each from a driver of its own, give the
15
+ same ids; the ids are unique, plain enough to name a trial, as many as the
16
+ dataset says it holds.
17
+ - ``leak``: after ``setup``, the reference is not already there to read. For
18
+ each unit the driver sets a workspace up, the oracle is run on a copy of
19
+ it, and what the oracle gives - its answer, and the lines it writes into
20
+ files - is looked for in the prompt (a capability benchmark's: its recall
21
+ questions, since the sessions it adds are meant to hold the evidence) and
22
+ in every file setup wrote. A text shorter than ``SHORT`` characters (a
23
+ number, a name) is not looked for - it would be found by chance - and the
24
+ record says how many were skipped. A benchmark whose questions may name
25
+ their answer among choices ("A or B?") says so (``controls.oracle.in_prompt``):
26
+ those units are counted and named in the record, not failed.
27
+
28
+ The controls are real runs, jobs like any other (named
29
+ ``<stamp>__<benchmark>__<dataset>__prove-nop`` and ``…-oracle``, so the runs
30
+ pages pair them as a launch's controls); the units and leak checks talk to
31
+ the driver directly and leave nothing behind. A control calls no model, so
32
+ it needs no key - except a judged benchmark's, whose judge still grades.
33
+
34
+ A check that passes is recorded with its date, its counts and its job; one
35
+ that fails is taken back, as ``conform`` takes back a port - what it reached kept
36
+ under ``failed``, for a page to say why. A check run on
37
+ fewer units (``--limit``) does not replace one that covered more. A
38
+ benchmark whose units may not be redistributed (``units.redistribute``) is
39
+ recorded without what names them: no ``in_prompt.first``, no problems. Parity
40
+ with the authors' own scorer is not checked yet (``docs/standard.md``).
41
+ """
42
+ from __future__ import annotations
43
+
44
+ import os
45
+ import re
46
+ import shutil
47
+ import tempfile
48
+ import time
49
+ from typing import Callable, Dict, List, Optional
50
+
51
+ from nullius.runs import parse
52
+ from nullius.standard import definitions, install, jsonl, run, shard as shards
53
+
54
+ STAMP = "%Y-%m-%d__%H-%M-%S"
55
+ SHORT = 12 # characters: a reference shorter than this is not looked for
56
+ CHECKS = ("units", "leak", "nop", "oracle")
57
+ _PLAIN_ID = re.compile(r"^[^\s\x00-\x1f]{1,1000}$") # whole in result.json; its trial folder takes the first 60 and the unit number
58
+ _MAX_FILE = 1 << 20 # a file setup wrote is read up to this size
59
+
60
+
61
+ def job_name(stamp: str, bench: str, dataset: str, control: str, shard: Optional[tuple] = None) -> str:
62
+ return f"{stamp}__{bench}__{dataset}__prove-{control}" + (f"__shard-{shard[0]}of{shard[1]}" if shard else "")
63
+
64
+
65
+ class _Driver:
66
+ """The benchmark's driver, started as a run starts it, for the checks
67
+ that talk to it directly (units, leak)."""
68
+
69
+ def __init__(self, bench: dict, ds: dict, work: str, licences=None):
70
+ self.up = install.upstream(bench)
71
+ self.data = install.dataset(bench, ds, self.up)
72
+ install.driver_packages(bench) # its driver.install, once: start() then finds it
73
+ self.bench, self.ds, self.work = bench, ds, work
74
+ self.licences = licences if licences is not None else run.Licences(bench)
75
+
76
+ def start(self, n: int) -> jsonl.Proc:
77
+ home = os.path.join(self.work, f"driver-home-{n}")
78
+ os.makedirs(home, exist_ok=True)
79
+ cmd, extra = install.driver_argv(self.bench)
80
+ credentials = run.driver_secrets(self.bench, self.licences)
81
+ env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
82
+ "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1",
83
+ "NULLIUS_VERSION": str(self.bench.get("_version") or ""),
84
+ **run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults
85
+ return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start()
86
+
87
+
88
+ def units_check(driver: _Driver, ds: dict) -> dict:
89
+ """Two drivers, one ``units`` call each: the same ids, unique, plain."""
90
+ got = []
91
+ for n in (1, 2):
92
+ p = driver.start(n)
93
+ try:
94
+ got.append(p.request("units", dataset=ds["id"], timeout=300).get("units"))
95
+ except jsonl.Failed as e:
96
+ return {"ok": False, "units": 0, "problems": [str(e)], "ids": []}
97
+ finally:
98
+ p.close()
99
+ a, b = got
100
+ problems = []
101
+ if not isinstance(a, list) or not a:
102
+ return {"ok": False, "units": 0, "problems": ["units: no ids"], "ids": []}
103
+ if a != b:
104
+ problems.append("two calls gave different ids" + ("" if len(a) == len(b) else f" ({len(a)}, then {len(b)})"))
105
+ bad = [u for u in a if not isinstance(u, str) or not _PLAIN_ID.match(u)]
106
+ if bad:
107
+ problems.append(f"{len(bad)} id(s) cannot name a trial (a string, no spaces, at most 1000 characters): {bad[:3]!r}")
108
+ names = [str(u).replace("/", "-") for u in a] # what the pages read as a task's name (job.write_trial)
109
+ if len(set(names)) != len(names):
110
+ dup = sorted({x for x in names if names.count(x) > 1})
111
+ problems.append(f"{len(names) - len(set(names))} id(s) repeat: {dup[:3]!r}")
112
+ if isinstance(ds.get("units"), int) and ds["units"] != len(a):
113
+ problems.append(f"the dataset says {ds['units']} units, the driver gives {len(a)}")
114
+ return {"ok": not problems, "units": len(a), "problems": problems, "ids": [str(u) for u in a],
115
+ "passed": f"{len(a)} ids, the same twice, unique" if not problems else ""}
116
+
117
+
118
+ def _files(root: str) -> Dict[str, bytes]:
119
+ out = {}
120
+ for d, dirs, files in os.walk(root):
121
+ dirs[:] = [x for x in dirs if x != ".git"]
122
+ for f in files:
123
+ p = os.path.join(d, f)
124
+ try:
125
+ if os.path.islink(p) or os.path.getsize(p) > _MAX_FILE:
126
+ continue
127
+ with open(p, "rb") as fh:
128
+ out[os.path.relpath(p, root)] = fh.read()
129
+ except OSError:
130
+ continue
131
+ return out
132
+
133
+
134
+ def _flat(text: str) -> str:
135
+ return " ".join(str(text).split())
136
+
137
+
138
+ def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = "") -> dict:
139
+ """For each unit: set up, run the oracle on a copy, and look for what it
140
+ gave in what setup had already shown. ``in_prompt``: the benchmark's
141
+ reason a prompt may name its own answer (``controls.oracle.in_prompt``,
142
+ a choice question offering it among others); such a unit is counted and
143
+ named, not failed. Found in a file setup wrote, it is always a leak."""
144
+ p = driver.start(0)
145
+ looked, short, nothing, leaks, failed, bad, named = 0, 0, 0, [], [], set(), []
146
+ try:
147
+ for i, unit in enumerate(units):
148
+ a, b = os.path.join(driver.work, "leak", f"{i}", "a"), os.path.join(driver.work, "leak", f"{i}", "b")
149
+ os.makedirs(a)
150
+ try:
151
+ s = p.request("setup", unit=unit, dir=a, timeout=600)
152
+ before = _files(a)
153
+ shutil.copytree(a, b)
154
+ o = p.request("oracle", unit=unit, dir=b, timeout=600)
155
+ except jsonl.Failed as e:
156
+ failed.append(f"{unit}: {e}")
157
+ bad.add(unit)
158
+ continue
159
+ after = _files(b)
160
+ if family == "capability":
161
+ shown = " ".join(q.get("question", "") for q in (s.get("memory") or {}).get("recall") or [])
162
+ else:
163
+ shown = s.get("prompt") or ""
164
+ shown = _flat(shown)
165
+ files = _flat("\n".join(x.decode("utf-8", "replace") for x in before.values()))
166
+ needles = []
167
+ if str(o.get("answer") or "").strip():
168
+ needles.append(("the answer", str(o["answer"])))
169
+ for path, new in after.items():
170
+ old = before.get(path)
171
+ if old == new:
172
+ continue
173
+ kept = set(old.decode("utf-8", "replace").splitlines()) if old is not None else set()
174
+ added = "\n".join(l for l in new.decode("utf-8", "replace").splitlines() if l not in kept)
175
+ if added.strip():
176
+ needles.append((path, added))
177
+ if not needles:
178
+ nothing += 1
179
+ for what, text in needles:
180
+ flat = _flat(text)
181
+ if len(flat) < SHORT:
182
+ short += 1
183
+ continue
184
+ looked += 1
185
+ if flat in files or (flat in shown and not in_prompt):
186
+ leaks.append(f"{unit}: {what} is already there after setup, in {'a file' if flat in files else 'the prompt'}")
187
+ bad.add(unit)
188
+ elif flat in shown:
189
+ named.append(unit)
190
+ shutil.rmtree(os.path.dirname(a), ignore_errors=True)
191
+ finally:
192
+ p.close()
193
+ problems = leaks + [f"could not check {f}" for f in failed]
194
+ rec = {"ok": not problems, "units": len(units), "looked_for": looked, "short": short, "problems": problems,
195
+ "passed": f"{len(units) - len(bad)}/{len(units)}"}
196
+ notes = []
197
+ if short:
198
+ notes.append(f"{short} reference(s) under {SHORT} characters not looked for: too short to tell a leak from chance")
199
+ if nothing:
200
+ notes.append(f"{nothing} unit(s) whose oracle gave nothing to look for")
201
+ if named:
202
+ rec["in_prompt"] = {"units": len(named), "first": named[:5], "why": in_prompt}
203
+ notes.append(f"{len(named)} prompt(s) name their answer, as the benchmark says they may: {in_prompt}")
204
+ if notes:
205
+ rec["note"] = "; ".join(notes)
206
+ return rec
207
+
208
+
209
+ def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str,
210
+ provider: str, model: str, judge_model: Optional[str], env, concurrency: int,
211
+ say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None,
212
+ licences=None, remove_images: bool = False) -> dict:
213
+ """Run one control over the dataset (or its first ``limit`` units, of
214
+ ``shard`` when given) and hold it to its bound: the pass rate, or the mean score."""
215
+ other = definitions.control(bench, which)
216
+ job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency,
217
+ jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s),
218
+ env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences,
219
+ remove_images=remove_images)
220
+ j = parse.parse_job(job_dir)
221
+ n = len(j.trials)
222
+ errored = sum(1 for t in j.trials if t.exception_type)
223
+ if bench["score"]["kind"] == "pass":
224
+ top = sum(1 for t in j.trials if t.reward == 1.0)
225
+ value = top / n if n else 0.0
226
+ else:
227
+ scores = [t.score for t in j.trials if t.score is not None]
228
+ top = sum(1 for s in scores if s >= 1.0)
229
+ value = sum(scores) / len(scores) if scores else 0.0
230
+ (bound, expected), = other["expect"].items()
231
+ within = value <= expected if bound == "at_most" else value >= expected
232
+ problems = []
233
+ if not n:
234
+ problems.append("no unit ran")
235
+ if errored:
236
+ problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials")
237
+ if n and not within:
238
+ problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}")
239
+ rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected},
240
+ "units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems}
241
+ if judge_model or (bench["score"].get("judge") or {}).get("model"):
242
+ rec["judge"] = judge_model or bench["score"]["judge"]["model"]
243
+ return rec
244
+
245
+
246
+ def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs",
247
+ provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None,
248
+ env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print,
249
+ shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None,
250
+ remove_images: bool = False) -> Dict[str, dict]:
251
+ """Every check on every dataset (or ``dataset``); with ``write``, what
252
+ passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the
253
+ leak check and the controls cover that shard's units only (``shard``),
254
+ the units check all of them, and each record says which shard.
255
+ ``remove_images``: each unit's own image is removed after it, so the
256
+ oracle pulls it again (by default they stay). Returns
257
+ {dataset: {check: record}}, each record with ``ok`` and its ``problems``."""
258
+ env = env if env is not None else os.environ
259
+ stamp = stamp or time.strftime(STAMP)
260
+ sets = [d for d in bench["datasets"] if dataset in (None, d["id"])]
261
+ if not sets:
262
+ raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}")
263
+ declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})]
264
+ if not declared:
265
+ say(f"{bench['label']} declares no controls: only its units and nothing else can be checked")
266
+ got: Dict[str, dict] = {}
267
+ for ds in sets:
268
+ say(f"{bench['key']}/{ds['id']}:")
269
+ work = tempfile.mkdtemp(prefix="prove-")
270
+ checks: Dict[str, dict] = {}
271
+ try:
272
+ try:
273
+ driver = _Driver(bench, ds, work, licences=licences)
274
+ except install.InstallFailed as e:
275
+ raise run.Refused(f"could not install what the proof needs: {e}")
276
+ checks["units"] = units_check(driver, ds)
277
+ ids = checks["units"].pop("ids")
278
+ if shard and shard[1] > 1:
279
+ ids = [u for _, u in shards.select(ids, *shard)]
280
+ ids = ids[:limit] if limit else ids
281
+ _say(say, "units", checks["units"])
282
+ if "oracle" in declared and ids:
283
+ checks["leak"] = leak_check(driver, ids, bench["family"], bench["controls"]["oracle"].get("in_prompt") or "")
284
+ _say(say, "leak", checks["leak"])
285
+ finally:
286
+ shutil.rmtree(work, ignore_errors=True)
287
+ for c in declared:
288
+ checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider,
289
+ model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say,
290
+ shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences,
291
+ remove_images=remove_images)
292
+ _say(say, c, checks[c])
293
+ total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"]
294
+ for name, r in checks.items():
295
+ r["on"] = time.strftime("%Y-%m-%d")
296
+ if limit and name != "units" and limit < total:
297
+ r["limit"] = limit
298
+ if shard and shard[1] > 1 and name != "units":
299
+ r["shard"] = f"{shard[0]}/{shard[1]}"
300
+ got[ds["id"]] = checks
301
+ if write:
302
+ record(bench, got)
303
+ return got
304
+
305
+
306
+ def _say(say, name, r):
307
+ say(f" {name}: {'passed' if r['ok'] else 'FAILED'} {r.get('passed', '')}".rstrip()
308
+ + "".join(f"\n {p}" for p in r.get("problems") or []) + (f"\n ({r['note']})" if r.get("note") else ""))
309
+
310
+
311
+ def record(bench: dict, got: Dict[str, dict]) -> None:
312
+ """Write what passed into ``bench``'s ``proven`` and save it: a check that
313
+ passed replaces its record unless that one covered more units; one that
314
+ failed is taken back. The failure itself is kept under ``failed`` - what
315
+ it reached, against what bound, its job - so a page can say "0.9955 < 1"
316
+ rather than only "not proven"; the next pass of that check clears it.
317
+ Units that may not be redistributed are never named (:func:`_unnamed`)."""
318
+ if (bench.get("units") or {}).get("redistribute") is False:
319
+ got = {ds: {name: _unnamed(r) for name, r in checks.items()} for ds, checks in got.items()}
320
+ definitions.save(bench, lambda cur: cur.update(proven=_merged(cur.get("proven") or {}, got)))
321
+
322
+
323
+ def _unnamed(r: dict) -> dict:
324
+ """A check's record without the units it names: the leak check's first
325
+ prompts that name their answer, and a failed check's problems."""
326
+ r = dict(r, problems=[])
327
+ if "in_prompt" in r:
328
+ r["in_prompt"] = {k: v for k, v in r["in_prompt"].items() if k != "first"}
329
+ return r
330
+
331
+
332
+ def _merged(proven: dict, got: Dict[str, dict]) -> dict:
333
+ proven = dict(proven)
334
+ for ds, checks in got.items():
335
+ cur = dict(proven.get(ds) or {})
336
+ failed = dict(cur.pop("failed", None) or {})
337
+ for name, r in checks.items():
338
+ if not r["ok"]:
339
+ cur.pop(name, None)
340
+ failed[name] = {k: r[k] for k in ("passed", "value", "expect", "units", "job", "on", "limit", "shard") if k in r}
341
+ failed[name]["problems"] = list(r.get("problems") or [])[:3]
342
+ continue
343
+ failed.pop(name, None)
344
+ old = cur.get(name) or {}
345
+ if old and old.get("units", 0) > r.get("units", 0):
346
+ continue
347
+ cur[name] = {k: v for k, v in r.items() if k not in ("ok", "problems")}
348
+ if cur or failed:
349
+ proven[ds] = {**{k: cur[k] for k in CHECKS if k in cur}, **({"failed": failed} if failed else {})}
350
+ else:
351
+ proven.pop(ds, None)
352
+ return proven
353
+
354
+
355
+ def status(bench: dict) -> Dict[str, Dict[str, dict]]:
356
+ """Per dataset, each control the benchmark declares - its source, where it
357
+ comes from, its bound - and whether it is proven on that dataset, with the
358
+ record; then the units and leak checks the same way. ``failed``: the last
359
+ attempt that did not pass, while none has since. What a page shows."""
360
+ out = {}
361
+ controls = bench.get("controls") or {}
362
+ for ds in bench["datasets"]:
363
+ rec = (bench.get("proven") or {}).get(ds["id"]) or {}
364
+ failed = rec.get("failed") if isinstance(rec.get("failed"), dict) else {}
365
+ row = {}
366
+ for c in definitions.CONTROLS:
367
+ if c in controls:
368
+ d = definitions.control(bench, c)
369
+ row[c] = {"source": d["source"], "from": d["from"], "expect": d["expect"], "proven": rec.get(c), "failed": failed.get(c)}
370
+ for c in ("units", "leak"):
371
+ row[c] = {"proven": rec.get(c), "failed": failed.get(c)}
372
+ out[ds["id"]] = row
373
+ return out