nullius-eval 0.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. _nullius_app/.env.example +54 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +16 -0
  5. _nullius_app/static/brand/favicon.svg +1 -0
  6. _nullius_app/static/brand/mark-dark.svg +1 -0
  7. _nullius_app/static/brand/marks.svg +34 -0
  8. _nullius_app/static/brand/tokens.json +51 -0
  9. _nullius_app/static/brand/typesafe.png +0 -0
  10. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  11. _nullius_app/static/builder/command-builder-core.js +744 -0
  12. _nullius_app/static/builder/command-builder.css +248 -0
  13. _nullius_app/static/builder/command-builder.js +994 -0
  14. _nullius_app/static/builder/model-picker.js +113 -0
  15. _nullius_app/static/catalogue/add.js +46 -0
  16. _nullius_app/static/catalogue/agents/README.md +19 -0
  17. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  18. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  19. _nullius_app/static/catalogue/agents/marks.json +8 -0
  20. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  21. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  22. _nullius_app/static/catalogue/catalogue.css +453 -0
  23. _nullius_app/static/catalogue/catalogue.js +87 -0
  24. _nullius_app/static/catalogue/marks.svg +25 -0
  25. _nullius_app/static/catalogue/new.js +78 -0
  26. _nullius_app/static/catalogue/versions.js +27 -0
  27. _nullius_app/static/compare/comparison.css +117 -0
  28. _nullius_app/static/controls.js +117 -0
  29. _nullius_app/static/disclosures.js +70 -0
  30. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  31. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  32. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  33. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  36. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/README.md +11 -0
  38. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  39. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  40. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  41. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  42. _nullius_app/static/fonts.css +49 -0
  43. _nullius_app/static/notifications.js +34 -0
  44. _nullius_app/static/onboarding.css +78 -0
  45. _nullius_app/static/onboarding.js +158 -0
  46. _nullius_app/static/runs/board.css +354 -0
  47. _nullius_app/static/runs/board.js +127 -0
  48. _nullius_app/static/runs/launches.css +87 -0
  49. _nullius_app/static/runs/live-board.css +186 -0
  50. _nullius_app/static/runs/live-board.js +387 -0
  51. _nullius_app/static/runs/path-suggestions.js +119 -0
  52. _nullius_app/static/runs/start.css +38 -0
  53. _nullius_app/static/settings/autosave.js +145 -0
  54. _nullius_app/static/settings/judges.js +61 -0
  55. _nullius_app/static/settings/modal.css +32 -0
  56. _nullius_app/static/settings/modal.js +91 -0
  57. _nullius_app/static/settings/navigation.js +55 -0
  58. _nullius_app/static/settings/providers.js +78 -0
  59. _nullius_app/static/settings/sandbox.js +83 -0
  60. _nullius_app/static/settings/settings.css +229 -0
  61. _nullius_app/static/style.css +1751 -0
  62. _nullius_app/static/traces/conversation.css +49 -0
  63. _nullius_app/static/traces/conversation.js +215 -0
  64. _nullius_app/templates/agents/entry.html +145 -0
  65. _nullius_app/templates/agents/kits.html +55 -0
  66. _nullius_app/templates/agents/standard.html +58 -0
  67. _nullius_app/templates/base.html +477 -0
  68. _nullius_app/templates/builder/new_run.html +55 -0
  69. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  70. _nullius_app/templates/catalogue/benchmark.html +301 -0
  71. _nullius_app/templates/catalogue/benchmarks.html +92 -0
  72. _nullius_app/templates/catalogue/new.html +141 -0
  73. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  74. _nullius_app/templates/compare/_detail.html +37 -0
  75. _nullius_app/templates/compare/compare.html +298 -0
  76. _nullius_app/templates/compare/compare_picker.html +51 -0
  77. _nullius_app/templates/compare/compare_task.html +329 -0
  78. _nullius_app/templates/runs/_execution.html +131 -0
  79. _nullius_app/templates/runs/_runtime.html +5 -0
  80. _nullius_app/templates/runs/ascii-mark.html +485 -0
  81. _nullius_app/templates/runs/feasibility.html +127 -0
  82. _nullius_app/templates/runs/index.html +609 -0
  83. _nullius_app/templates/runs/job.html +446 -0
  84. _nullius_app/templates/runs/preparing.html +14 -0
  85. _nullius_app/templates/runs/reconcile.html +124 -0
  86. _nullius_app/templates/runs/setup.html +272 -0
  87. _nullius_app/templates/runs/start.html +51 -0
  88. _nullius_app/templates/settings/settings.html +508 -0
  89. _nullius_app/templates/traces/_charts.html +165 -0
  90. _nullius_app/templates/traces/_conversation.html +37 -0
  91. _nullius_app/templates/traces/_judgement.html +72 -0
  92. _nullius_app/templates/traces/artifact.html +15 -0
  93. _nullius_app/templates/traces/trial.html +838 -0
  94. _nullius_app/templates/workspaces/workspaces.html +54 -0
  95. _nullius_app/web/__init__.py +1 -0
  96. _nullius_app/web/agents.py +230 -0
  97. _nullius_app/web/builder.py +392 -0
  98. _nullius_app/web/catalogue.py +374 -0
  99. _nullius_app/web/compare.py +489 -0
  100. _nullius_app/web/core.py +17 -0
  101. _nullius_app/web/execute.py +205 -0
  102. _nullius_app/web/feasibility.py +60 -0
  103. _nullius_app/web/filters.py +224 -0
  104. _nullius_app/web/judge.py +623 -0
  105. _nullius_app/web/judge_tasks.py +33 -0
  106. _nullius_app/web/onboarding.py +20 -0
  107. _nullius_app/web/reconcile.py +47 -0
  108. _nullius_app/web/runs.py +461 -0
  109. _nullius_app/web/settings.py +210 -0
  110. _nullius_app/web/shared.py +468 -0
  111. _nullius_app/web/traces.py +306 -0
  112. _nullius_app/web/workspaces.py +84 -0
  113. nullius/__init__.py +32 -0
  114. nullius/__main__.py +5 -0
  115. nullius/catalogue/__init__.py +0 -0
  116. nullius/catalogue/agents/codex/agent.json +202 -0
  117. nullius/catalogue/agents/codex/config.toml +25 -0
  118. nullius/catalogue/agents/goose/agent.json +120 -0
  119. nullius/catalogue/agents/goose/config.yaml +20 -0
  120. nullius/catalogue/agents/jiuwenswarm/agent.json +115 -0
  121. nullius/catalogue/agents/jiuwenswarm/agent.py +316 -0
  122. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  123. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  124. nullius/catalogue/agents/opencode/agent.json +124 -0
  125. nullius/catalogue/agents/opencode/opencode.json +32 -0
  126. nullius/catalogue/agents/reference/agent.json +89 -0
  127. nullius/catalogue/agents/reference/agent.py +209 -0
  128. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  129. nullius/catalogue/agents/terminus-2/run.py +139 -0
  130. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  131. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  132. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  133. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  134. nullius/catalogue/benchmarks/humaneval/benchmark.json +37 -0
  135. nullius/catalogue/benchmarks/humaneval/driver.py +208 -0
  136. nullius/catalogue/benchmarks/humaneval/standard.json +114 -0
  137. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  138. nullius/catalogue/benchmarks/locomo/benchmark.json +42 -0
  139. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  140. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  141. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  142. nullius/catalogue/benchmarks/longmemeval/benchmark.json +64 -0
  143. nullius/catalogue/benchmarks/longmemeval/driver.py +323 -0
  144. nullius/catalogue/benchmarks/longmemeval/standard.json +551 -0
  145. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +74 -0
  146. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  147. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +105 -0
  148. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  149. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  150. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  151. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  152. nullius/catalogue/benchmarks/tau3-bench/driver.py +605 -0
  153. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +111 -0
  154. nullius/catalogue/benchmarks/tau3-bench/standard.json +495 -0
  155. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  156. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +70 -0
  157. nullius/catalogue/benchmarks/terminal-bench/driver.py +189 -0
  158. nullius/catalogue/benchmarks/terminal-bench/standard.json +160 -0
  159. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  160. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  161. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  162. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  163. nullius/catalogue/capabilities/mem0/capability.py +170 -0
  164. nullius/catalogue/catalog.py +413 -0
  165. nullius/catalogue/citation.py +116 -0
  166. nullius/catalogue/harbor_entry.py +367 -0
  167. nullius/catalogue/sources.py +85 -0
  168. nullius/catalogue/storage.py +125 -0
  169. nullius/cli.py +83 -0
  170. nullius/compare/__init__.py +0 -0
  171. nullius/compare/cli.py +265 -0
  172. nullius/compare/comparison.py +184 -0
  173. nullius/execution.py +1624 -0
  174. nullius/gitrefs.py +128 -0
  175. nullius/harbor/__init__.py +0 -0
  176. nullius/harbor/feed.py +228 -0
  177. nullius/harbor/source.py +582 -0
  178. nullius/hosts.py +339 -0
  179. nullius/importers/__init__.py +140 -0
  180. nullius/importers/atif.py +387 -0
  181. nullius/importers/cli.py +135 -0
  182. nullius/importers/otel.py +274 -0
  183. nullius/judge/__init__.py +14 -0
  184. nullius/judge/batch.py +122 -0
  185. nullius/judge/client.py +416 -0
  186. nullius/judge/floors.json +86 -0
  187. nullius/judge/lanes.py +89 -0
  188. nullius/judge/questions.py +309 -0
  189. nullius/judge/reconcile.py +172 -0
  190. nullius/judge/state.py +224 -0
  191. nullius/judge/store.py +167 -0
  192. nullius/migrate.py +268 -0
  193. nullius/onboarding.py +112 -0
  194. nullius/paths.py +84 -0
  195. nullius/prerequisites.py +111 -0
  196. nullius/progress.py +294 -0
  197. nullius/providers.py +115 -0
  198. nullius/recipes/__init__.py +0 -0
  199. nullius/recipes/launch.py +78 -0
  200. nullius/redact.py +72 -0
  201. nullius/runs/__init__.py +0 -0
  202. nullius/runs/artifacts.py +67 -0
  203. nullius/runs/classify.py +189 -0
  204. nullius/runs/cli.py +316 -0
  205. nullius/runs/feasibility.py +442 -0
  206. nullius/runs/launches.py +139 -0
  207. nullius/runs/live.py +705 -0
  208. nullius/runs/parse.py +991 -0
  209. nullius/runs/verifier.py +241 -0
  210. nullius/sandbox.py +146 -0
  211. nullius/secrets.py +192 -0
  212. nullius/settings.py +456 -0
  213. nullius/standard/__init__.py +15 -0
  214. nullius/standard/acp.py +301 -0
  215. nullius/standard/cancellation.py +19 -0
  216. nullius/standard/cli.py +650 -0
  217. nullius/standard/conform.py +149 -0
  218. nullius/standard/container.py +1381 -0
  219. nullius/standard/definitions.py +1577 -0
  220. nullius/standard/egress.py +192 -0
  221. nullius/standard/fidelity.py +255 -0
  222. nullius/standard/forward.py +92 -0
  223. nullius/standard/hostreq.py +65 -0
  224. nullius/standard/howto.py +220 -0
  225. nullius/standard/install.py +397 -0
  226. nullius/standard/job.py +541 -0
  227. nullius/standard/jsonl.py +129 -0
  228. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  229. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  230. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  231. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  232. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  233. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  234. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  235. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  236. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  237. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  238. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  239. nullius/standard/match.py +66 -0
  240. nullius/standard/mcp_bridge.py +62 -0
  241. nullius/standard/mcp_machine.py +193 -0
  242. nullius/standard/merge.py +209 -0
  243. nullius/standard/names.py +30 -0
  244. nullius/standard/payload.py +255 -0
  245. nullius/standard/ports.py +64 -0
  246. nullius/standard/prove.py +357 -0
  247. nullius/standard/recipe.py +663 -0
  248. nullius/standard/relay.py +629 -0
  249. nullius/standard/run.py +1367 -0
  250. nullius/standard/shard.py +71 -0
  251. nullius/traces/__init__.py +0 -0
  252. nullius/traces/behavior.py +131 -0
  253. nullius/traces/charts.py +898 -0
  254. nullius/traces/conversation.py +159 -0
  255. nullius/traces/otel.py +608 -0
  256. nullius/traces/trace.py +1477 -0
  257. nullius/traces/trajectory.py +586 -0
  258. nullius/workspace.py +409 -0
  259. nullius/workspace_cli.py +119 -0
  260. nullius_eval-0.1.0rc1.dist-info/METADATA +93 -0
  261. nullius_eval-0.1.0rc1.dist-info/RECORD +268 -0
  262. nullius_eval-0.1.0rc1.dist-info/WHEEL +5 -0
  263. nullius_eval-0.1.0rc1.dist-info/entry_points.txt +2 -0
  264. nullius_eval-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
  265. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  266. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  267. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  268. nullius_eval-0.1.0rc1.dist-info/top_level.txt +2 -0
@@ -0,0 +1,663 @@
1
+ """The standard in the run builder: its choices, its recipe, its compile.
2
+
3
+ The builder offers the standard as a runner beside Harbor and the runner
4
+ definitions. A recipe for it carries *choices* - a benchmark, one of its
5
+ datasets, an agent or a capability and any of its options (``options``, name ->
6
+ value; the rest keep the vendor's defaults), a provider and a model, a limit,
7
+ how many units at a time, and for an agent the explicit yes to running it
8
+ unsandboxed - and :func:`compile` turns them into one command, here, on the server:
9
+ the preview route and ``/run`` compile again from the definitions, never from a
10
+ script the page sent.
11
+
12
+ Every pair is shown, and the ones that do not fit are refused with
13
+ ``match.fit``'s reasons, the same sentences the page greys a choice out with.
14
+
15
+ The command is ``nullius.standard.cli run``, and
16
+ ``--from-run`` makes it read the key and endpoint the run is handed
17
+ (``execution``'s launcher exports them as ``OPENAI_API_KEY`` and
18
+ ``OPENAI_BASE_URL``) instead of saved settings. It runs from
19
+ the installed ``nullius standard`` command, or the source/payload tree
20
+ named by ``--app`` or ``$NULLIUS_STANDARD``. Relative job paths stay relative
21
+ to the directory the script was started in.
22
+
23
+ *Where* (``where``) is this machine (``local``), a host from Settings, or a
24
+ list of them: one machine is one run, several split it - one shard per
25
+ machine (``shard.py``), each its own run with its own job
26
+ (``<name>-shard-<I>of<N>``), all under one launch id, the recipe's name, so
27
+ the runs page groups them and ``cli merge`` puts them together. A shard's
28
+ ``limit`` is its own. ``backend``: ``local`` processes, or ``docker`` - each
29
+ unit's agent or capability in a container - which a host must have docker for
30
+ (``docker`` in the result: the run checks it before anything is sent).
31
+ """
32
+ from __future__ import annotations
33
+
34
+ import re
35
+ import shlex
36
+ from typing import Callable, Dict, List, Optional
37
+
38
+ from nullius import hosts, providers
39
+ from nullius.providers import PROVIDER_LABEL, PROVIDERS, STEP
40
+ from nullius.standard import hostreq, definitions, match, ports, prove
41
+
42
+ HARNESS = "standard"
43
+ # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's.
44
+ RUNNER = {"key": HARNESS, "label": "Nullius", "builtin": True, "standard": True,
45
+ "summary": "Any agent or capability that proved the ports a benchmark needs: ACP, MCP and the relay, on this machine."}
46
+ MAX_CONCURRENT = 32
47
+ BACKENDS = ("local", "docker")
48
+ _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
49
+ _MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$")
50
+
51
+
52
+ def _script(command: str) -> str:
53
+ """Run an installed standard or an explicit source/payload, without
54
+ changing where relative job paths resolve or reading saved credentials."""
55
+ args = command.removeprefix("nullius standard ")
56
+ # A local snapshot can be launched by the interpreter that already has
57
+ # Nullius installed. Keep exactly one backend provider: ignore cwd and
58
+ # ambient Python paths (-I), omit site-packages (-S), then add this tree.
59
+ # -I ignores PYTHONDONTWRITEBYTECODE, so -B keeps the snapshot immutable.
60
+ # PYTHONPATH remains available to the standard's subprocess helpers.
61
+ bootstrap = ('import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); '
62
+ 'runpy.run_module("nullius.standard.cli", run_name="__main__", alter_sys=True)')
63
+ return ("set +x\n"
64
+ "set -euo pipefail\n"
65
+ 'nullius_app=${NULLIUS_STANDARD:-}\n'
66
+ 'nullius_die() { printf \'Nullius: %s\\n\' "$*" >&2; exit 1; }\n'
67
+ 'while (($#)); do\n'
68
+ ' case "$1" in\n'
69
+ ' --app) (($# >= 2)) && [[ -n "$2" ]] || nullius_die "--app requires a path"; nullius_app=$2; shift 2 ;;\n'
70
+ ' --help|-h) printf \'%s\\n\' "Usage: bash recipe-run.sh [--app PATH]" '
71
+ '"Uses installed nullius, or --app PATH for a source checkout." '
72
+ '"Relative job paths resolve from the current directory."; exit 0 ;;\n'
73
+ ' *) nullius_die "Unknown option; use --help" ;;\n'
74
+ ' esac\n'
75
+ 'done\n'
76
+ 'if [[ -z "$nullius_app" && -f app/nullius/standard/cli.py ]]; then nullius_app=$PWD; fi\n'
77
+ 'if [[ -n "$nullius_app" ]]; then\n'
78
+ ' nullius_app_asked=$nullius_app\n'
79
+ ' nullius_app=$(cd -- "$nullius_app_asked" 2>/dev/null && pwd -P) '
80
+ '|| nullius_die "Cannot open the application directory: $nullius_app_asked"\n'
81
+ ' [[ -f "$nullius_app/app/nullius/standard/cli.py" ]] '
82
+ '|| nullius_die "Required file is missing: $nullius_app/app/nullius/standard/cli.py"\n'
83
+ ' export PYTHONPATH="$nullius_app/app"\n'
84
+ f' exec "${{NULLIUS_PYTHON:-python3}}" -I -S -B -c {shlex.quote(bootstrap)} "$nullius_app/app" {args}\n'
85
+ 'fi\n'
86
+ 'command -v nullius >/dev/null 2>&1 || nullius_die "Install nullius or pass --app PATH"\n'
87
+ f'exec nullius standard {args}\n')
88
+
89
+
90
+ def _all() -> Dict[str, List[dict]]:
91
+ return {k: definitions.load(k, every=True)[0] for k in ("benchmark", "agent", "capability")}
92
+
93
+
94
+ def _bench_page(b: dict) -> dict:
95
+ """A benchmark as a page shows it. ``controls``: the ones it declares,
96
+ each with who wrote it and where it comes from - what the builder offers
97
+ beside the agents or capabilities. Each dataset carries the same controls,
98
+ each with its ``proven`` record on that dataset (None: not proven), and
99
+ ``proofs``, the units and leak checks the same way (``prove.status``)."""
100
+ kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b["family"] == "capability" else []
101
+ status = prove.status(b)
102
+ controls = [{k: c[k] for k in ("key", "label", "source", "from", "expect")}
103
+ for c in (definitions.control(b, w) for w in definitions.CONTROLS) if c]
104
+ versioned = definitions.version_of(b) is not None
105
+ return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": kinds[0] if kinds else "",
106
+ "ref": definitions.ref_of(b) if "_version" in b else b["key"],
107
+ "version": b["_version"] if versioned else "",
108
+ # its versions, each by the ref a recipe names it with (the bare key: the default)
109
+ "versions": [{"ref": b["key"] if v == b.get("_default") else f"{b['key']}@{v}", "version": v, "default": v == b.get("_default")}
110
+ for v in b.get("_versions") or []] if versioned else [],
111
+ "environments": {n: {"summary": e.get("summary", ""), "local_ok": bool(e.get("local_ok")),
112
+ "needs_secrets": [{"name": s["name"], "to": s["to"], "kind": s.get("kind") or "value",
113
+ "summary": s.get("summary", ""), **{k: s[k] for k in ("path", "env") if k in s}}
114
+ for s in e.get("needs_secrets") or []]}
115
+ for n, e in (b.get("environments") or {}).items()},
116
+ "needs": list(b.get("needs") or []), "units": b.get("units") or {"noun": "unit", "nouns": "units"},
117
+ "score": (b.get("score") or {}).get("says", ""), "judged": bool((b.get("score") or {}).get("judge")),
118
+ # the upstream it runs: its repository, the release tag when it has one, the commit that is the authority
119
+ "upstream": {k: str((b.get("upstream") or {}).get(k) or "") for k in ("repository", "tag", "tested")},
120
+ "controls": controls,
121
+ # One Original procedure, with separate reference-participant and paper-model metadata.
122
+ "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"],
123
+ "reference_agent": x.get("reference_agent", ""),
124
+ "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}),
125
+ **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]),
126
+ "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}}
127
+ if "models" in x else {"any": True, "providers": list(x["providers"])})}
128
+ for k, x in (b.get("setups") or {}).items() if k == "original"],
129
+ # its own options (a search, a mode): each value with its line and what it needs; and the host requirements
130
+ "simulator": (b.get("simulator") or {}).get("model", ""), "judge": ((b.get("score") or {}).get("judge") or {}).get("model", ""),
131
+ # what the upstream's leaderboard fixes besides the ranked model (Like the leaderboard)
132
+ "leaderboard": b.get("leaderboard") or None,
133
+ "attempts": (b.get("limits") or {}).get("attempts", ""),
134
+ "options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [],
135
+ "setups": o.get("setups") or ["yours"],
136
+ **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}}
137
+ for n, o in (b.get("options") or {}).items()},
138
+ "host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]}
139
+ for n, r in hostreq.all_of(b).items()},
140
+ "datasets": [{"id": d["id"], "label": d.get("label") or d["id"], "summary": d.get("summary", ""), "units": d.get("units"),
141
+ "environment": d.get("environment") or "",
142
+ "controls": [dict(c, proven=status[d["id"]][c["key"]]["proven"], failed=status[d["id"]][c["key"]]["failed"])
143
+ for c in controls],
144
+ "proofs": {k: status[d["id"]][k]["proven"] for k in ("units", "leak")},
145
+ "failed": {k: status[d["id"]][k]["failed"] for k in ("units", "leak") if status[d["id"]][k]["failed"]}}
146
+ for d in b["datasets"]]}
147
+
148
+
149
+ def protocol(o: dict) -> str:
150
+ """How Nullius talks to it, in a word or two for a page."""
151
+ if o["_kind"] == "control":
152
+ return "no agent"
153
+ if o["_kind"] == "capability":
154
+ return "JSON lines"
155
+ return "ACP" if "acp" in (o.get("start") or {}) else "a command"
156
+
157
+
158
+ def skills_of(port: str, record: Optional[dict]) -> List[dict]:
159
+ """The skills of ``port``'s kit as a page shows them under a proof:
160
+ {id, says, shown, passed} each, ``passed`` the count when the proof
161
+ recorded one. Empty without a proof, for a kit without skills, and for a
162
+ proof on an older kit from before skills were recorded - it says nothing
163
+ of them."""
164
+ have = definitions.port_skills(port)
165
+ if not record or not have or not (isinstance(record.get("skills"), dict) or definitions.on_current_kit(port, record)):
166
+ return []
167
+ shown, counts = definitions.skills_shown(port, record), record.get("skills") or {}
168
+ return [{"id": s, "says": says, "shown": s in shown, "passed": (counts.get(s) or {}).get("passed", "")}
169
+ for s, says in have.items()]
170
+
171
+
172
+ def _other_page(o: dict) -> dict:
173
+ return {"key": o["_ref"], "base": o["key"], "label": definitions.title(o), "summary": o["summary"], "kind": o["_kind"],
174
+ "featureKind": o.get("kind", ""), "protocol": protocol(o), "version": str((o.get("upstream") or {}).get("version") or ""),
175
+ "default": o.get("_version") == o.get("_default"),
176
+ "offers": list(o.get("offers") or []),
177
+ # each proof with the kit version it passed and, on an older kit than the current one, a note saying so
178
+ "proven": {p: {**{k: r.get(k) for k in ("passed", "on", "model")}, "kit": definitions.proof_kit(r),
179
+ "older": definitions.older_kit(p, r), "label": ports.PORTS[p]["label"], "skills": skills_of(p, r)}
180
+ for p, r in (o.get("proven") or {}).items() if ports.known(p)},
181
+ "options": {n: {"summary": x.get("summary", ""), "default": x.get("default"),
182
+ **({"values": list(x["values"])} if "values" in x else {"pattern": x.get("pattern")})}
183
+ for n, x in (o.get("options") or {}).items()},
184
+ **({"fidelity": {k: o["proven"]["fidelity"].get(k) for k in ("verdict", "on", "model")}}
185
+ if isinstance((o.get("proven") or {}).get("fidelity"), dict) else {})}
186
+
187
+
188
+ def choices() -> dict:
189
+ """What the page offers - choices and the reasons a pair does not fit,
190
+ never a command."""
191
+ d = _all()
192
+ # one entry per benchmark, its default version, which lists the others (``versions``); the fits cover every version
193
+ defaults = [b for b in d["benchmark"] if b.get("_version") == b.get("_default")]
194
+ return {"benchmarks": [_bench_page(b) for b in defaults],
195
+ # an example (the reference agent, the example memory) is there to read or copy, never to choose
196
+ "agents": [_other_page(o) for o in d["agent"] if not o.get("example")],
197
+ "capabilities": [_other_page(o) for o in d["capability"] if not o.get("example")],
198
+ "fits": match.table(d["benchmark"], d["agent"], d["capability"]),
199
+ "ports": {p: s["summary"] for p, s in ports.PORTS.items()},
200
+ "providers": list(PROVIDERS)}
201
+
202
+
203
+ def licences_of(key: str) -> List[dict]:
204
+ """The licences a benchmark's environments need, every version's, as its
205
+ page lists them: [{environment, summary, needs: [{name, kind, to,
206
+ summary, path?, env?, status}]}], ``status`` the store's
207
+ (``secrets.status``: dates, never a value) or {"set": False}."""
208
+ from nullius import secrets
209
+ have = secrets.status(key)
210
+ out: Dict[str, dict] = {}
211
+ for b in _all()["benchmark"]:
212
+ if b["key"] != key:
213
+ continue
214
+ for n, e in (b.get("environments") or {}).items():
215
+ env = out.setdefault(n, {"environment": n, "summary": e.get("summary", ""), "needs": []})
216
+ for s in definitions.secrets_of({"environments": {n: e}}):
217
+ if all(x["name"] != s["name"] for x in env["needs"]):
218
+ env["needs"].append(dict(s, status=have.get(s["name"]) or {"set": False}))
219
+ return [e for e in out.values() if e["needs"]]
220
+
221
+
222
+ def page_of(benchmark: str) -> Optional[dict]:
223
+ """One benchmark of the standard as its page shows it (``_bench_page``), by
224
+ key or key@version; None when the standard has no such benchmark."""
225
+ b = definitions.find(_all()["benchmark"], benchmark or "")
226
+ return _bench_page(dict(b, label=definitions.title(b))) if b else None
227
+
228
+
229
+ def chosen_of(recipe: dict) -> List[str]:
230
+ """The agents, capabilities and controls a recipe runs, each once, in its order:
231
+ ``others``, or the one ``other`` of a recipe from before several could be chosen."""
232
+ picked = recipe.get("others")
233
+ if not isinstance(picked, list):
234
+ picked = [recipe.get("other")] if recipe.get("other") else []
235
+ return list(dict.fromkeys(str(x) for x in picked if isinstance(x, str) and x))
236
+
237
+
238
+ def options_for(recipe: dict, ref: str, several: bool):
239
+ """One choice's options: ``options`` is {ref: {name: word}}; a recipe from before, or one with a
240
+ single choice, may give that choice's {name: word} directly."""
241
+ given = recipe.get("options") or {}
242
+ if isinstance(given, dict) and ref in given and isinstance(given[ref], dict):
243
+ return given[ref]
244
+ if not several and isinstance(given, dict) and all(isinstance(v, str) for v in given.values()):
245
+ return given
246
+ return {}
247
+
248
+
249
+ def _int(v, low: int, high: Optional[int] = None) -> Optional[int]:
250
+ if isinstance(v, bool) or not isinstance(v, int) or v < low or (high is not None and v > high):
251
+ return None
252
+ return v
253
+
254
+
255
+ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *,
256
+ docker_unusable: Optional[Callable[[], Optional[str]]] = None) -> dict:
257
+ """{errors, notes, review, jobs, confirm, script, parts, model}
258
+ for one recipe; errors name the step
259
+ that edits what is wrong. And ``shards``: one per machine of *Where*, each
260
+ {where, index, of, name, script, parts, jobs} - what one run on that
261
+ machine is given (``web/execute.py`` starts one per shard) - and
262
+ ``docker``, whether the runs need docker where they run.
263
+
264
+ ``docker_unusable``: asked, when the docker backend is chosen for this
265
+ machine, why docker cannot run here (``container.unusable``) - a reason
266
+ is the Where step's error. Not given, it is not asked: compiling stays a
267
+ function of the definitions alone, and the routes that preview or start
268
+ a run pass it."""
269
+ d = defs or _all()
270
+ errors, notes = [], []
271
+ add = lambda step, message: errors.append({"step": STEP[step], "message": message})
272
+ out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""}
273
+ name = str(recipe.get("name") or "")
274
+ if not _NAME.match(name):
275
+ add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.")
276
+ # the benchmark by key (its default version) or key@version
277
+ b = definitions.find(d["benchmark"], str(recipe.get("benchmark") or "")) if recipe.get("benchmark") else None
278
+ if b is None:
279
+ add("benchmark", "Choose a benchmark of the standard.")
280
+ return out
281
+ b = dict(b, label=definitions.title(b))
282
+ # one dataset or several (``datasets``; a recipe from before names one, ``dataset``): each its own jobs
283
+ wanted = recipe.get("datasets") if isinstance(recipe.get("datasets"), list) else ([recipe["dataset"]] if recipe.get("dataset") else [])
284
+ wanted = list(dict.fromkeys(str(x) for x in wanted if isinstance(x, str)))
285
+ dss = [x for x in b["datasets"] if x["id"] in wanted]
286
+ dss.sort(key=lambda x: wanted.index(x["id"]))
287
+ unknown = [x for x in wanted if x not in {y["id"] for y in b["datasets"]}]
288
+ if not dss or unknown:
289
+ add("dataset", f"Choose one or more of {b['label']}'s datasets" + (f" ({', '.join(unknown)} is not one)" if unknown else "") + ".")
290
+ side = "agent" if b["family"] == "agent" else "capability"
291
+ # A control the benchmark declares (nop, oracle) is a choice beside the agents or capabilities: it runs
292
+ # nothing as you and calls no model - a judged benchmark's judge only - so it asks for neither.
293
+ # Several may be chosen: each is its own job of the one run, one after the other.
294
+ backend = recipe.get("backend") or "local"
295
+ if backend not in BACKENDS:
296
+ add("where", "The backend is local processes or docker.")
297
+ backend = "local"
298
+ what = "an agent" if side == "agent" else f"a {_bench_page(b)['kind'] or 'capability'}"
299
+ refs = chosen_of(recipe)
300
+ if not refs:
301
+ add("agent", f"Choose how {b['label']} runs: " + ("as published, " if b.get("setups") else "") + f"with {what}, or a check.")
302
+ several = len(refs) > 1
303
+ picks = [] # [{o, control, given, opts}]
304
+ provider_ = recipe.get("provider") if recipe.get("provider") in PROVIDERS else ""
305
+ for ref in refs:
306
+ control = ref in definitions.CONTROLS
307
+ given, opts = {}, {}
308
+ if ref == "model":
309
+ add("agent", "Choose Original, then choose the model on the Model step.")
310
+ continue
311
+ if ref == "original":
312
+ orig = (b.get("setups") or {}).get("original") or {}
313
+ picked = str(recipe.get("model") or "").strip()
314
+ if not picked:
315
+ add("model", "Choose a model.")
316
+ continue
317
+ method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original"
318
+ o, why_not = definitions.setup(b, method, picked, provider_)
319
+ if o is None:
320
+ add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".")
321
+ continue
322
+ picks.append({"o": o, "control": False, "setup": True, "given": {}, "opts": {}, "model": picked or o["asked"]})
323
+ continue
324
+ if control:
325
+ o = definitions.control(b, ref)
326
+ if o is None:
327
+ add("agent", f"{b['label']} declares no {ref} control.")
328
+ continue
329
+ else:
330
+ o = definitions.find(d[side], ref)
331
+ if o is None:
332
+ add("agent", f"Choose {what} for {b['label']}: {ref} is not one.")
333
+ continue
334
+ o = dict(o, label=definitions.title(o))
335
+ why = match.fit(b, o)
336
+ if why:
337
+ add("agent", f"{o['label']} cannot run {b['label']}: " + "; ".join(why) + ".")
338
+ given = options_for(recipe, ref, several)
339
+ if not isinstance(given, dict) or not all(isinstance(v, str) for v in given.values()):
340
+ add("agent", "Options are names with a word each.")
341
+ given = {}
342
+ opts, bad = definitions.options_of(o, given)
343
+ for why_not in bad:
344
+ add("agent", f"{o['label']}: " + why_not)
345
+ picks.append({"o": o, "control": control, "given": given, "opts": opts})
346
+ # the benchmark's own options (``bench_options``), each on the datasets it applies to; what the chosen
347
+ # values need: an embedding model (its calls through the run's relay, on the OpenAI provider), the machine's programs
348
+ chosen_opts = recipe.get("bench_options") or {}
349
+ if not isinstance(chosen_opts, dict) or not all(isinstance(v, str) for v in chosen_opts.values()):
350
+ add("dataset", "The benchmark's options are names with a word each.")
351
+ chosen_opts = {}
352
+ lb = (b.get("leaderboard") or {}) if recipe.get("model_choice") == "leaderboard" else {}
353
+ for k, v in (lb.get("bench_options") or {}).items():
354
+ if chosen_opts.get(k, v) != v:
355
+ add("dataset", f"Like the leaderboard sets {(b.get('options') or {})[k].get('label') or k.replace('_', ' ')} to {v}: choose Any of your choice for {chosen_opts[k]}.")
356
+ chosen_opts = {**(lb.get("bench_options") or {}), **chosen_opts}
357
+ per_ds, needs = {}, {"embeddings": [], "host": []}
358
+ for x in dss:
359
+ vals, bad = definitions.bench_options_of(b, x["id"], chosen_opts)
360
+ for why_not in bad:
361
+ add("dataset", why_not[0].upper() + why_not[1:] + ".")
362
+ per_ds[x["id"]] = vals
363
+ n = definitions.option_needs(b, vals)
364
+ needs["embeddings"] += [m for m in n["embeddings"] if m not in needs["embeddings"]]
365
+ needs["host"] += [h for h in n["host"] if h not in needs["host"]]
366
+ if needs["embeddings"] and recipe.get("provider") not in (None, "openai"):
367
+ add("model", f"{', '.join(needs['embeddings'])} is OpenAI's embedding model: choose the OpenAI provider for this search.")
368
+ agents_local = [p["o"]["label"] for p in picks if side == "agent" and not p["control"] and not p.get("setup") and backend == "local"]
369
+ if backend != "local" and any(p.get("setup") for p in picks):
370
+ add("where", "A setup runs the benchmark's own code on the machine, as published: choose local processes.")
371
+ if agents_local and recipe.get("unsandboxed") is not True:
372
+ add("where", f"{', '.join(agents_local)} would run as you, not sandboxed: tick Run it as me, or choose Docker.")
373
+ only_controls = bool(picks) and all(p["control"] for p in picks)
374
+ needs_model = any(not p["control"] for p in picks)
375
+ model_choice = recipe.get("model_choice", "any")
376
+ if model_choice not in ("paper", "leaderboard", "any"):
377
+ add("model", "Choose Like the paper, Like the leaderboard or Any of your choice.")
378
+ if model_choice == "leaderboard" and not b.get("leaderboard"):
379
+ add("model", f"{b['label']} declares no leaderboard; choose Like the paper or Any of your choice.")
380
+ orig = (b.get("setups") or {}).get("original") or {}
381
+ official = {"judge": ((b.get("score") or {}).get("judge") or {}).get("model", ""), "simulator": (b.get("simulator") or {}).get("model", "")}
382
+ fixed = {**official, **{k: lb[k] for k in ("judge", "simulator") if lb.get(k)}}
383
+ if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()):
384
+ add("model", "No paper models are listed for this benchmark; choose Any of your choice.")
385
+ elif model_choice == "paper" and needs_model and orig.get("models"):
386
+ if recipe.get("model") not in orig["models"]:
387
+ add("model", "Choose one of the paper's listed models.")
388
+ elif provider_ not in orig.get("providers", {}):
389
+ add("model", "The paper's models are not listed for this provider.")
390
+ provider = recipe.get("provider")
391
+ if provider not in PROVIDERS:
392
+ add("model", "Choose a provider: OpenRouter, OpenAI or a local or self-hosted OpenAI-compatible endpoint.")
393
+ model = str(recipe.get("model") or "").strip()
394
+ if not _MODEL.match(model) and (needs_model or model):
395
+ add("model", "Choose a model or enter its ID.")
396
+ if model_choice == "paper" and model in orig.get("models", []) and provider_ in orig.get("providers", {}):
397
+ model = orig["providers"][provider_] + (orig.get("served_as") or {}).get(model, model)
398
+ # each unit tried K times (the upstream's --num-trials: pass^k); the user simulator's and the judge's models
399
+ repeat = _int(recipe.get("repeat") or lb.get("repeat") or 1, 1, 32)
400
+ if repeat is None:
401
+ add("limits", "Attempts per unit: a whole number from 1 to 32.")
402
+ repeat = 1
403
+ if lb and repeat < lb.get("repeat", 1):
404
+ add("limits", f"Like the leaderboard runs each {(b.get('units') or {}).get('noun', 'unit')} at least {lb['repeat']} times.")
405
+ sim_model = str(recipe.get("simulator_model") or "").strip()
406
+ judge_model = str(recipe.get("judge_model") or "").strip()
407
+ like = {"paper": "Like the paper", "leaderboard": "Like the leaderboard"}.get(model_choice)
408
+ for what, role, m in (("the user simulator's", "simulator", sim_model), ("the judge's", "judge", judge_model)):
409
+ if m and not _MODEL.match(m):
410
+ add("model", f"Enter {what} model as the provider lists it, or leave it empty for its official one.")
411
+ elif m and like and m not in (fixed[role], providers.served_as(fixed[role], provider_)):
412
+ add("model", f"{like} uses {what} model {fixed[role]}: choose Any of your choice for {m}.")
413
+ # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default
414
+ sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model
415
+ judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model
416
+ limit = recipe.get("limit")
417
+ if limit not in ("", None) and _int(limit, 1) is None:
418
+ add("limits", "The number of units is a whole number of at least 1, or empty for all of them.")
419
+ conc = _int(recipe.get("concurrency") or 1, 1, MAX_CONCURRENT) # empty: one at a time
420
+ if conc is None:
421
+ add("limits", f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.")
422
+ targets, why_not = where_of(recipe.get("where", "local"))
423
+ if why_not:
424
+ add("where", why_not)
425
+ elif backend == "docker" and "local" in targets and docker_unusable is not None:
426
+ why = docker_unusable()
427
+ if why:
428
+ add("where", f"The docker backend cannot run on this machine: {why}. Choose local processes, or a host with docker.")
429
+ if errors:
430
+ return out
431
+ nouns = (b.get("units") or {}).get("nouns", "units")
432
+ if picks and all(p.get("setup") for p in picks): # only setups: a unit is theirs (a conversation)
433
+ nouns = picks[0]["o"]["units"]["nouns"]
434
+ each = [None if picks and all(p.get("setup") for p in picks) else x.get("units") for x in dss] # a setup's count is not the dataset's
435
+ total = sum(each) if all(each) else None
436
+ n = sum(min(limit, t) for t in each) if isinstance(limit, int) and total else (limit or total)
437
+ of = len(targets)
438
+ many = len(dss) > 1
439
+ # what a run is made of: each dataset with each choice, a job each, in that order
440
+ pairs = [(x, p) for x in dss for p in picks]
441
+
442
+ def word_of(d: dict, p: dict) -> str:
443
+ return "-".join(([d["id"]] if many else []) + ([definitions.job_word(p["o"])] if several else []))
444
+
445
+ def job_of(d: dict, p: dict, index: int) -> str:
446
+ base = "-".join(filter(None, [name, word_of(d, p)]))
447
+ return f"{base}-shard-{index}of{of}" if of > 1 else base
448
+
449
+ def options_for_pick(d: dict, p: dict) -> dict:
450
+ """The benchmark's options for this dataset and this pick: a setup's own, or those of the port."""
451
+ return definitions.bench_options_of(b, d["id"], chosen_opts, p["o"]["key"] if p.get("setup") else "yours")[0]
452
+
453
+ def command_for(d: dict, p: dict, index: int) -> str:
454
+ o, control, given = p["o"], p["control"], p["given"]
455
+ launch = "-".join(filter(None, [name, word_of(d, p)]))
456
+ m = p["model"] if p.get("setup") else model
457
+ words = ["nullius", "standard", "run", definitions.ref_of(b), o["_ref"], *(["--model", m] if m else []), "--provider", provider,
458
+ "--dataset", d["id"], *(["--limit", str(limit)] if isinstance(limit, int) else []), "-j", str(conc),
459
+ *[w for k, v in options_for_pick(d, p).items() for w in ("--bench-option", f"{k}={v}")],
460
+ *(["--repeat", str(repeat)] if repeat > 1 else []),
461
+ *(["--simulator-model", sim_model] if sim_model and not p["control"] else []),
462
+ *(["--judge-model", judge_model] if judge_model else []),
463
+ "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")],
464
+ *(["--backend", "docker"] if backend == "docker" else []),
465
+ *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []),
466
+ *(["--shard", f"{index}/{of}", "--launch", launch] if of > 1 else []), "--from-run"]
467
+ return " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
468
+
469
+ def script_for(d: dict, p: dict, command: str, index: int) -> str:
470
+ return ("#!/usr/bin/env bash\n"
471
+ f"# {b['label']} ({d.get('label') or d['id']}) with {p['o']['label']}, through the Nullius standard (docs/standard.md)"
472
+ + (f": shard {index} of {of}.\n" if of > 1 else ".\n") +
473
+ "# Uses installed nullius or --app PATH ($NULLIUS_STANDARD for an execution payload). The key and\n"
474
+ "# endpoint come from the run's environment: OPENAI_API_KEY, and OPENAI_BASE_URL for an endpoint\n"
475
+ "# other than the provider's own.\n"
476
+ + _script(command))
477
+
478
+ out["shards"] = []
479
+ commands = []
480
+ for i, where in enumerate(targets, 1):
481
+ parts, jobs = [], []
482
+ for d, p in pairs:
483
+ command = command_for(d, p, i)
484
+ commands.append(command)
485
+ parts.append({"id": word_of(d, p) or definitions.job_word(p["o"]),
486
+ "label": " · ".join(([d.get("label") or d["id"]] if many else []) + [p["o"]["label"]]) + (f" · shard {i}/{of}" if of > 1 else ""),
487
+ "script": script_for(d, p, command, i)})
488
+ jobs.append({"name": job_of(d, p, i), "agent": p["o"]["_ref"], "dataset": definitions.ref_of(b)})
489
+ out["shards"].append({"where": where, "index": i, "of": of, "name": f"{name}-shard-{i}of{of}" if of > 1 else name,
490
+ "command": "\n".join(commands[-len(pairs):]), "script": parts[0]["script"], "parts": parts, "jobs": jobs})
491
+ out["script"] = "\n".join(pt["script"] for pt in out["shards"][0]["parts"]) if len(pairs) > 1 else out["shards"][0]["script"]
492
+ out["parts"] = [pt for sh in out["shards"] for pt in sh["parts"]]
493
+ out["jobs"] = [j for sh in out["shards"] for j in sh["jobs"]]
494
+ # docker where it runs: the docker backend, or a control run in each unit's own image, whatever the backend
495
+ own = any(definitions.own_image(definitions.environment_of(b, None, x)[1]) for x in dss)
496
+ out["docker"] = backend == "docker" or (own and any(p["control"] for p in picks))
497
+ out["model"] = model
498
+ # Nothing in the run asks a model: no key is handed to it, and none need be set.
499
+ out["keyless"] = only_controls and not (b.get("score") or {}).get("judge") and not needs["embeddings"]
500
+ # what the machines must have for the chosen options (``host``): the Where step and its test show it, Run checks it
501
+ reqs = hostreq.requirements(b, needs["host"])
502
+ out["host"] = {"benchmark": b["key"], "needs": [{"name": n, "summary": r["summary"], "check": r["check"]} for n, r in reqs.items()],
503
+ "check": [p for r in reqs.values() for p in r["check"]], "install": hostreq.install_lines(b, list(reqs))}
504
+ # The licences its environments may need, by name: what a run on a host is sent of the benchmark's store.
505
+ out["licences"] = {"benchmark": b["key"], "names": sorted({s["name"] for s in definitions.secrets_of(b)})}
506
+
507
+
508
+ def row(p: dict) -> list:
509
+ o = p["o"]
510
+ if p["control"]:
511
+ return [STEP["agent"], "Check", f"{o['label']} ({o['key']}) · {o['source']} · {o['from']}"]
512
+ if p.get("setup"):
513
+ original = (b.get("setups") or {}).get("original") or {}
514
+ label = original.get("reference_agent") or ("No agent · Original" if side == "agent" else "Original " + (_bench_page(b)["kind"] or "capability"))
515
+ return [STEP["agent"], "Participant", label]
516
+ proven = ", ".join(f"{q} {r.get('passed', '')}".strip() for q, r in (o.get("proven") or {}).items() if q in b["needs"])
517
+ return [STEP["agent"], "Agent" if side == "agent" else "Capability", f"{o['label']} · {protocol(o)} · proven {proven}"]
518
+ out["review"] = [[STEP["benchmark"], "Benchmark", f"{b['label']} · needs {', '.join(b['needs'])}"],
519
+ [STEP["runner"], "Runner", "Nullius"],
520
+ [STEP["dataset"], "Datasets" if many else "Dataset",
521
+ ", ".join(f"{x.get('label') or x['id']}" + (f" · {x['units']} {nouns}" if x.get("units") else "") for x in dss)],
522
+ *[[STEP["dataset"], (b.get("options") or {})[k].get("label") or k.replace("_", " ").capitalize(),
523
+ f"{v}" + ("" if k in chosen_opts else " (its default)") + f" · {', '.join(x['id'] for x in dss if per_ds[x['id']].get(k) == v)}"]
524
+ for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items()],
525
+ *[row(p) for p in picks],
526
+ *[[STEP["agent"], "Options" + (f" of {p['o']['label']}" if several else ""),
527
+ ", ".join(f"{k} {v}" + ("" if k in p["given"] else " (its default)") for k, v in p["opts"].items())]
528
+ for p in picks if p["opts"]],
529
+ [STEP["model"], "Model", "none: a control calls no model" if out["keyless"]
530
+ else f"{PROVIDER_LABEL[provider]} · " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only")
531
+ + (f" · user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "")
532
+ + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")],
533
+ [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}"
534
+ + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ")
535
+ + (" of each dataset" if many and isinstance(limit, int) else "")
536
+ + (f" · {len(pairs)} jobs, one after the other" if len(pairs) > 1 else "")
537
+ + (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "")
538
+ + (" · a setup counts " + next(p["o"]["units"]["nouns"] for p in picks if p.get("setup"))
539
+ if any(p.get("setup") for p in picks) and not all(p.get("setup") for p in picks) else "")],
540
+ [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "")
541
+ + ", ".join(where_label(t) for t in targets) + f" · {backend} backend"]]
542
+ if agents_local:
543
+ notes.append(f"{', '.join(agents_local)} {'run' if len(agents_local) > 1 else 'runs'} as you on {'each machine' if of > 1 else where_label(targets[0])}, with its permission requests granted: "
544
+ "the local backend is no sandbox.")
545
+ if of > 1:
546
+ notes.append(f"Each machine runs the units whose shard is its own, by their ids' hash: {of} jobs under one launch, "
547
+ f"{name}, put together by cli merge.")
548
+ docker_only = [n for n, e in (b.get("environments") or {}).items() if not e.get("local_ok")]
549
+ if docker_only and not only_controls and backend == "local":
550
+ notes.append(f"Units in {b['label']}'s environment{'s' if len(docker_only) > 1 else ''} {', '.join(docker_only)} "
551
+ "need the docker backend: on local processes they are recorded as not run.")
552
+ if any(p["control"] for p in picks) and any(definitions.own_image(e) for e in (b.get("environments") or {}).values()):
553
+ notes.append(f"Each {(b.get('units') or {}).get('noun', 'unit')} of {b['label']} is its own image: the control "
554
+ "runs in it, so docker is needed where it runs, whatever the backend.")
555
+ if not isinstance(limit, int):
556
+ out["confirm"] = f"Run all {total or ''} {nouns} of {b['label']}{' (' + ', '.join(x.get('label') or x['id'] for x in dss) + ')' if many else ''} with {', '.join(p['o']['label'] for p in picks)}?".replace(" ", " ")
557
+ return out
558
+
559
+
560
+ def where_of(where) -> tuple:
561
+ """*Where* as a list of machines - ``local`` or a host's entry - and why
562
+ it cannot be, or "": a word or a list, each machine once. Whether a host
563
+ is one of Settings' is the route's to check."""
564
+ targets = where if isinstance(where, list) else [where if where not in (None, "") else "local"]
565
+ if not targets:
566
+ return ["local"], "Choose where it runs: this machine, a host, or several."
567
+ out = []
568
+ for t in targets:
569
+ t = str(t).strip() if isinstance(t, str) else ""
570
+ if t != "local":
571
+ try:
572
+ t = hosts.parse(t).entry
573
+ except ValueError as e:
574
+ return ["local"], f"Where: {e}."
575
+ if t in out:
576
+ return ["local"], f"Where names {where_label(t)} twice: each machine runs one shard."
577
+ out.append(t)
578
+ return out, ""
579
+
580
+
581
+ def where_label(target: str) -> str:
582
+ return "this machine" if target == "local" else hosts.parse(target).label
583
+
584
+
585
+ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: bool, stamp: str, backend: str = "local") -> dict:
586
+ """The check an Agents or Capabilities page starts: every kit the agent or
587
+ capability can take, as one command (``cli conform``), the jobs it will
588
+ write named ahead (``conform.job_name``). {errors, script, jobs, label}."""
589
+ from nullius.standard import conform
590
+ o = definitions.get(kind, key) if kind in ("agent", "capability") else None
591
+ errors = []
592
+ if o is None:
593
+ return {"errors": [f"no {kind} {key!r}"], "script": "", "jobs": [], "label": key}
594
+ o = dict(o, label=definitions.title(o))
595
+ if provider not in PROVIDERS:
596
+ errors.append("Choose a provider: OpenRouter, OpenAI or a local or self-hosted OpenAI-compatible endpoint.")
597
+ if not _MODEL.match(model or ""):
598
+ errors.append("Choose a model or enter its ID.")
599
+ if backend not in ("local", "docker"):
600
+ errors.append("Choose local processes or Docker for the checks.")
601
+ if kind == "agent" and backend == "local" and unsandboxed is not True:
602
+ errors.append(f"{o['label']} runs on this machine as you while its kits run: the local backend is not a sandbox. Say yes to that to check it.")
603
+ if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""):
604
+ errors.append("a time stamp: YYYY-MM-DD__HH-MM-SS")
605
+ kits = [definitions.kit(p) for p in conform.candidates(o)]
606
+ kits = [k for k in kits if k]
607
+ if not kits:
608
+ errors.append(f"{o['label']} has no port with a kit to check.")
609
+ if errors:
610
+ return {"errors": errors, "script": "", "jobs": [], "label": o["label"]}
611
+ words = ["nullius", "standard", "conform", o["_ref"], "--model", model, "--provider", provider,
612
+ "--stamp", stamp, *(["--backend", "docker"] if backend == "docker" else []),
613
+ *(["--unsandboxed"] if kind == "agent" and backend == "local" else []), "--from-run"]
614
+ command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
615
+ script = ("#!/usr/bin/env bash\n"
616
+ f"# {o['label']}: the conformance kits of the ports it can offer (docs/standard.md, \"Ports\").\n"
617
+ + _script(command))
618
+ return {"errors": [], "script": script, "label": o["label"],
619
+ "jobs": [{"name": conform.job_name(stamp, k["key"], definitions.job_word(o)), "agent": o["_ref"], "dataset": k["key"]}
620
+ for k in kits]}
621
+
622
+
623
+ def proving(benchmark: str, *, dataset: str = "", limit=None, provider: str = "openai", judge_model: str = "",
624
+ stamp: str, concurrency: int = 4) -> dict:
625
+ """The proof a benchmark's page starts, as an Agents page starts a check
626
+ of ports (``conformance``): ``cli prove`` over one dataset or each - its
627
+ units, the leak check and every control it declares - as one command,
628
+ the control jobs it will write named ahead (``prove.job_name``).
629
+ ``keyless``: the controls call no model, so the run is handed no key at
630
+ all - unless the benchmark is judged, whose judge still grades.
631
+ {errors, script, jobs, label, keyless}."""
632
+ errors: List[str] = []
633
+ b = definitions.find(_all()["benchmark"], benchmark or "")
634
+ if b is None:
635
+ return {"errors": [f"no benchmark {benchmark!r}"], "script": "", "jobs": [], "label": benchmark, "keyless": False}
636
+ b = dict(b, label=definitions.title(b))
637
+ judged = bool((b.get("score") or {}).get("judge"))
638
+ sets = [d for d in b["datasets"] if dataset in ("", d["id"])]
639
+ if not sets:
640
+ errors.append(f"Choose one of {b['label']}'s datasets, or each of them.")
641
+ if limit not in ("", None) and _int(limit, 1) is None:
642
+ errors.append("The number of units is a whole number of at least 1, or empty for all of them.")
643
+ if _int(concurrency, 1, MAX_CONCURRENT) is None:
644
+ errors.append(f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.")
645
+ if judged and provider not in PROVIDERS:
646
+ errors.append("Choose the judge's provider: OpenRouter, OpenAI or a local or self-hosted OpenAI-compatible endpoint.")
647
+ if judge_model and not _MODEL.match(judge_model):
648
+ errors.append("The judge model, as the provider lists it - or empty for the benchmark's own.")
649
+ if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""):
650
+ errors.append("a time stamp: YYYY-MM-DD__HH-MM-SS")
651
+ if errors:
652
+ return {"errors": errors, "script": "", "jobs": [], "label": b["label"], "keyless": False}
653
+ words = ["nullius", "standard", "prove", definitions.ref_of(b), *(["--dataset", dataset] if dataset else []),
654
+ *(["--limit", str(limit)] if isinstance(limit, int) else []), *(["--provider", provider] if judged else []),
655
+ *(["--judge-model", judge_model] if judged and judge_model else []), "-j", str(concurrency), "--stamp", stamp, "--from-run"]
656
+ command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
657
+ script = ("#!/usr/bin/env bash\n"
658
+ f"# {b['label']}: its proofs - units, no leak, its controls (docs/standard.md, \"Controls and proofs\").\n"
659
+ + _script(command))
660
+ declared = [c for c in definitions.CONTROLS if c in (b.get("controls") or {})]
661
+ return {"errors": [], "script": script, "label": b["label"], "keyless": not judged,
662
+ "jobs": [{"name": prove.job_name(stamp, b["key"], d["id"], c), "agent": c, "dataset": definitions.ref_of(b)}
663
+ for d in sets for c in declared]}