nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,746 @@
1
+ """The standard in the run builder: its choices, its recipe, its compile.
2
+
3
+ The builder offers the standard as a runner beside Harbor and the runner
4
+ definitions. A recipe for it carries *choices* - a benchmark, one of its
5
+ datasets, an agent or a capability and any of its options (``options``, name ->
6
+ value; the rest keep the vendor's defaults), a provider and a model, a limit,
7
+ how many units at a time, and for an agent the explicit yes to running it
8
+ unsandboxed - and :func:`compile` turns them into one command, here, on the server:
9
+ the preview route and ``/run`` compile again from the definitions, never from a
10
+ script the page sent.
11
+
12
+ Every pair is shown, and the ones that do not fit are refused with
13
+ ``match.fit``'s reasons, the same sentences the page greys a choice out with.
14
+
15
+ The command is ``nullius.standard.cli run``, and
16
+ ``--from-run`` makes it read the key and endpoint the run is handed
17
+ (``execution``'s launcher exports them as ``OPENAI_API_KEY`` and
18
+ ``OPENAI_BASE_URL``) instead of saved settings. It runs from
19
+ the installed ``nullius standard`` command, or the source/payload tree
20
+ named by ``--app`` or ``$NULLIUS_STANDARD``. Relative job paths stay relative
21
+ to the directory the script was started in.
22
+
23
+ *Where* (``where``) is this machine (``local``), a host from Settings, or a
24
+ list of them: one machine is one run, several split it - one shard per
25
+ machine (``shard.py``), each its own run with its own job
26
+ (``<name>-shard-<I>of<N>``), all under one launch id, the recipe's name, so
27
+ the runs page groups them and ``cli merge`` puts them together. A shard's
28
+ ``limit`` is its own. ``backend``: ``local`` processes, or ``container`` - each
29
+ unit's agent or capability in a container, ``docker`` its earlier name - which a
30
+ host must have Docker or Podman for (``docker`` in the result: the run checks it
31
+ before anything is sent).
32
+ """
33
+ from __future__ import annotations
34
+
35
+ import math
36
+ import re
37
+ import shlex
38
+ from typing import Callable, Dict, List, Optional
39
+
40
+ from nullius import hosts, providers
41
+ from nullius.providers import PROVIDERS, STEP
42
+ from nullius.standard import hostreq, definitions, ledger, match, ports, prove
43
+
44
+ HARNESS = "standard"
45
+ # The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's.
46
+ RUNNER = {"key": HARNESS, "label": "Nullius", "builtin": True, "standard": True,
47
+ "summary": "Any agent or capability that proved the ports a benchmark needs: ACP, MCP and the relay, on this machine."}
48
+ MAX_CONCURRENT = 32
49
+ BACKENDS = definitions.names.BACKENDS
50
+ _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
51
+ _MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$")
52
+
53
+
54
+ def _script(command: str) -> str:
55
+ """Run an installed standard or an explicit source/payload, without
56
+ changing where relative job paths resolve or reading saved credentials."""
57
+ args = command.removeprefix("nullius standard ")
58
+ # A local snapshot can be launched by the interpreter that already has
59
+ # Nullius installed. Keep exactly one backend provider: ignore cwd and
60
+ # ambient Python paths (-I), omit site-packages (-S), then add this tree.
61
+ # -I ignores PYTHONDONTWRITEBYTECODE, so -B keeps the snapshot immutable.
62
+ # PYTHONPATH remains available to the standard's subprocess helpers.
63
+ bootstrap = ('import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); '
64
+ 'runpy.run_module("nullius.standard.cli", run_name="__main__", alter_sys=True)')
65
+ return ("set +x\n"
66
+ "set -euo pipefail\n"
67
+ 'nullius_app=${NULLIUS_STANDARD:-}\n'
68
+ 'nullius_die() { printf \'Nullius: %s\\n\' "$*" >&2; exit 1; }\n'
69
+ 'while (($#)); do\n'
70
+ ' case "$1" in\n'
71
+ ' --app) (($# >= 2)) && [[ -n "$2" ]] || nullius_die "--app requires a path"; nullius_app=$2; shift 2 ;;\n'
72
+ ' --help|-h) printf \'%s\\n\' "Usage: bash recipe-run.sh [--app PATH]" '
73
+ '"Uses installed nullius, or --app PATH for a source checkout." '
74
+ '"Relative job paths resolve from the current directory."; exit 0 ;;\n'
75
+ ' *) nullius_die "Unknown option; use --help" ;;\n'
76
+ ' esac\n'
77
+ 'done\n'
78
+ 'if [[ -z "$nullius_app" && -f app/nullius/standard/cli.py ]]; then nullius_app=$PWD; fi\n'
79
+ 'if [[ -n "$nullius_app" ]]; then\n'
80
+ ' nullius_app_asked=$nullius_app\n'
81
+ ' nullius_app=$(cd -- "$nullius_app_asked" 2>/dev/null && pwd -P) '
82
+ '|| nullius_die "Cannot open the application directory: $nullius_app_asked"\n'
83
+ ' [[ -f "$nullius_app/app/nullius/standard/cli.py" ]] '
84
+ '|| nullius_die "Required file is missing: $nullius_app/app/nullius/standard/cli.py"\n'
85
+ ' export PYTHONPATH="$nullius_app/app"\n'
86
+ f' exec "${{NULLIUS_PYTHON:-python3}}" -I -S -B -c {shlex.quote(bootstrap)} "$nullius_app/app" {args}\n'
87
+ 'fi\n'
88
+ 'command -v nullius >/dev/null 2>&1 || nullius_die "Install nullius or pass --app PATH"\n'
89
+ f'exec nullius standard {args}\n')
90
+
91
+
92
+ def _all() -> Dict[str, List[dict]]:
93
+ return {k: definitions.load(k, every=True)[0] for k in ("benchmark", "agent", "capability")}
94
+
95
+
96
+ def _bench_page(b: dict) -> dict:
97
+ """A benchmark as a page shows it. ``controls``: the ones it declares,
98
+ each with who wrote it and where it comes from - what the builder offers
99
+ beside the agents or capabilities. Each dataset carries the same controls,
100
+ each with its ``proven`` record on that dataset (None: not proven), and
101
+ ``proofs``, the units and leak checks the same way (``prove.status``)."""
102
+ kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b["family"] == "capability" else []
103
+ status = prove.status(b)
104
+ controls = [{k: c[k] for k in ("key", "label", "source", "from", "expect")}
105
+ for c in (definitions.control(b, w) for w in definitions.CONTROLS) if c]
106
+ versioned = definitions.version_of(b) is not None
107
+ return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": kinds[0] if kinds else "",
108
+ "ref": definitions.ref_of(b) if "_version" in b else b["key"],
109
+ "version": b["_version"] if versioned else "",
110
+ # its versions, each by the ref a recipe names it with (the bare key: the default)
111
+ "versions": [{"ref": b["key"] if v == b.get("_default") else f"{b['key']}@{v}", "version": v, "default": v == b.get("_default")}
112
+ for v in b.get("_versions") or []] if versioned else [],
113
+ "environments": {n: {"summary": e.get("summary", ""), "local_ok": bool(e.get("local_ok")), "network": e.get("network"),
114
+ "needs_secrets": [{"name": s["name"], "to": s["to"], "kind": s.get("kind") or "value",
115
+ "summary": s.get("summary", ""), **{k: s[k] for k in ("path", "env", "provider") if k in s}}
116
+ for s in e.get("needs_secrets") or []]}
117
+ for n, e in (b.get("environments") or {}).items()},
118
+ "needs": list(b.get("needs") or []), "units": b.get("units") or {"noun": "unit", "nouns": "units"},
119
+ "score": (b.get("score") or {}).get("says", ""), "judged": bool((b.get("score") or {}).get("judge")),
120
+ # the upstream it runs: its repository, the release tag when it has one, the commit that is the authority
121
+ "upstream": {k: str((b.get("upstream") or {}).get(k) or "") for k in ("repository", "tag", "tested")},
122
+ "controls": controls,
123
+ # One Original procedure, with separate reference-participant and paper-model metadata.
124
+ "setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"],
125
+ "reference_agent": x.get("reference_agent", ""),
126
+ "comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}),
127
+ **({"models": x["models"], "default": x["default"], "providers": list(x["providers"]),
128
+ "sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}}
129
+ if "models" in x else {"any": True, "providers": list(x["providers"])})}
130
+ for k, x in (b.get("setups") or {}).items() if k == "original"],
131
+ # its own options (a search, a mode): each value with its line and what it needs; and the host requirements
132
+ "simulator": (b.get("simulator") or {}).get("model", ""), "judge": ((b.get("score") or {}).get("judge") or {}).get("model", ""),
133
+ # what the upstream's leaderboard fixes besides the ranked model (Like the leaderboard)
134
+ "leaderboard": b.get("leaderboard") or None,
135
+ "attempts": (b.get("limits") or {}).get("attempts", ""),
136
+ "options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [],
137
+ "setups": o.get("setups") or ["yours"],
138
+ **({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}}
139
+ for n, o in (b.get("options") or {}).items()},
140
+ "host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]}
141
+ for n, r in hostreq.all_of(b).items()},
142
+ "datasets": [{"id": d["id"], "label": d.get("label") or d["id"], "summary": d.get("summary", ""), "units": d.get("units"),
143
+ "environment": d.get("environment") or "",
144
+ "controls": [dict(c, proven=status[d["id"]][c["key"]]["proven"], failed=status[d["id"]][c["key"]]["failed"])
145
+ for c in controls],
146
+ "proofs": {k: status[d["id"]][k]["proven"] for k in ("units", "leak")},
147
+ "failed": {k: status[d["id"]][k]["failed"] for k in ("units", "leak") if status[d["id"]][k]["failed"]}}
148
+ for d in b["datasets"]]}
149
+
150
+
151
+ def protocol(o: dict) -> str:
152
+ """How Nullius talks to it, in a word or two for a page."""
153
+ if o["_kind"] == "control":
154
+ return "no agent"
155
+ if o["_kind"] == "capability":
156
+ return "JSON lines"
157
+ return "ACP" if "acp" in (o.get("start") or {}) else "a command"
158
+
159
+
160
+ def skills_of(port: str, record: Optional[dict]) -> List[dict]:
161
+ """The skills of ``port``'s kit as a page shows them under a proof:
162
+ {id, says, shown, passed} each, ``passed`` the count when the proof
163
+ recorded one. Empty without a proof, for a kit without skills, and for a
164
+ proof on an older kit from before skills were recorded - it says nothing
165
+ of them."""
166
+ have = definitions.port_skills(port)
167
+ if not record or not have or not (isinstance(record.get("skills"), dict) or definitions.on_current_kit(port, record)):
168
+ return []
169
+ shown, counts = definitions.skills_shown(port, record), record.get("skills") or {}
170
+ return [{"id": s, "says": says, "shown": s in shown, "passed": (counts.get(s) or {}).get("passed", "")}
171
+ for s, says in have.items()]
172
+
173
+
174
+ def _other_page(o: dict) -> dict:
175
+ return {"key": o["_ref"], "base": o["key"], "label": definitions.title(o), "summary": o["summary"], "kind": o["_kind"],
176
+ "featureKind": o.get("kind", ""), "protocol": protocol(o), "version": str((o.get("upstream") or {}).get("version") or ""),
177
+ "default": o.get("_version") == o.get("_default"),
178
+ "offers": list(o.get("offers") or []),
179
+ # each proof with the kit version it passed and, on an older kit than the current one, a note saying so
180
+ "proven": {p: {**{k: r.get(k) for k in ("passed", "on", "model")}, "kit": definitions.proof_kit(r),
181
+ "older": definitions.older_kit(p, r), "label": ports.PORTS[p]["label"], "skills": skills_of(p, r)}
182
+ for p, r in (o.get("proven") or {}).items() if ports.known(p)},
183
+ "options": {n: {"summary": x.get("summary", ""), "default": x.get("default"),
184
+ **({"values": list(x["values"])} if "values" in x else {"pattern": x.get("pattern")})}
185
+ for n, x in (o.get("options") or {}).items()},
186
+ **({"fidelity": {k: o["proven"]["fidelity"].get(k) for k in ("verdict", "on", "model")}}
187
+ if isinstance((o.get("proven") or {}).get("fidelity"), dict) else {})}
188
+
189
+
190
+ def choices() -> dict:
191
+ """What the page offers - choices and the reasons a pair does not fit,
192
+ never a command."""
193
+ d = _all()
194
+ # one entry per benchmark, its default version, which lists the others (``versions``); the fits cover every version
195
+ defaults = [b for b in d["benchmark"] if b.get("_version") == b.get("_default")]
196
+ return {"benchmarks": [_bench_page(b) for b in defaults],
197
+ # an example (the reference agent, the example memory) is there to read or copy, never to choose
198
+ "agents": [_other_page(o) for o in d["agent"] if not o.get("example")],
199
+ "capabilities": [_other_page(o) for o in d["capability"] if not o.get("example")],
200
+ "fits": match.table(d["benchmark"], d["agent"], d["capability"]),
201
+ "ports": {p: s["summary"] for p, s in ports.PORTS.items()},
202
+ "providers": list(PROVIDERS)}
203
+
204
+
205
+ def licences_of(key: str) -> List[dict]:
206
+ """The credentials a benchmark's environments need, every version's, as its
207
+ Settings lists them: [{environment, summary, needs: [{name, kind, to,
208
+ summary, path?, env?, provider?, status}]}], ``status`` the store's
209
+ (``secrets.status``: dates, never a value; a data provider's token, its
210
+ provider's) or {"set": False}."""
211
+ from nullius import secrets
212
+ have = secrets.status(key)
213
+ out: Dict[str, dict] = {}
214
+ for b in _all()["benchmark"]:
215
+ if b["key"] != key:
216
+ continue
217
+ for n, e in (b.get("environments") or {}).items():
218
+ env = out.setdefault(n, {"environment": n, "summary": e.get("summary", ""), "needs": []})
219
+ for s in definitions.secrets_of({"environments": {n: e}}):
220
+ if all(x["name"] != s["name"] for x in env["needs"]):
221
+ held = secrets.state(key, s["name"], s["provider"]) if s.get("provider") else have.get(s["name"])
222
+ env["needs"].append(dict(s, status=held or {"set": False}))
223
+ return [e for e in out.values() if e["needs"]]
224
+
225
+
226
+ def page_of(benchmark: str) -> Optional[dict]:
227
+ """One benchmark of the standard as its page shows it (``_bench_page``), by
228
+ key or key@version; None when the standard has no such benchmark."""
229
+ b = definitions.find(_all()["benchmark"], benchmark or "")
230
+ return _bench_page(dict(b, label=definitions.title(b))) if b else None
231
+
232
+
233
+ def chosen_of(recipe: dict) -> List[str]:
234
+ """The agents, capabilities and controls a recipe runs, each once, in its order:
235
+ ``others``, or the one ``other`` of a recipe from before several could be chosen."""
236
+ picked = recipe.get("others")
237
+ if not isinstance(picked, list):
238
+ picked = [recipe.get("other")] if recipe.get("other") else []
239
+ return list(dict.fromkeys(str(x) for x in picked if isinstance(x, str) and x))
240
+
241
+
242
+ def options_for(recipe: dict, ref: str, several: bool):
243
+ """One choice's options: ``options`` is {ref: {name: word}}; a recipe from before, or one with a
244
+ single choice, may give that choice's {name: word} directly."""
245
+ given = recipe.get("options") or {}
246
+ if isinstance(given, dict) and ref in given and isinstance(given[ref], dict):
247
+ return given[ref]
248
+ if not several and isinstance(given, dict) and all(isinstance(v, str) for v in given.values()):
249
+ return given
250
+ return {}
251
+
252
+
253
+ def _int(v, low: int, high: Optional[int] = None) -> Optional[int]:
254
+ if isinstance(v, bool) or not isinstance(v, int) or v < low or (high is not None and v > high):
255
+ return None
256
+ return v
257
+
258
+
259
+ def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *,
260
+ docker_unusable: Optional[Callable[[], Optional[str]]] = None,
261
+ unpriced: Optional[Callable[[List[str]], List[str]]] = None) -> dict:
262
+ """{errors, notes, review, jobs, confirm, script, parts, model}
263
+ for one recipe; errors name the step
264
+ that edits what is wrong. And ``shards``: one per machine of *Where*, each
265
+ {where, index, of, name, script, parts, jobs} - what one run on that
266
+ machine is given (``web/execute.py`` starts one per shard) - and
267
+ ``docker``, whether the runs need docker where they run.
268
+
269
+ ``docker_unusable``: asked, when the container backend is chosen for this
270
+ machine, why docker cannot run here (``container.unusable``) - a reason
271
+ is the Where step's error. Not given, it is not asked: compiling stays a
272
+ function of the definitions alone, and the routes that preview or start
273
+ a run pass it.
274
+
275
+ ``max_usd``: the run's spending cap (``standard.ledger``), one amount for
276
+ the whole run: its jobs that call a model - each dataset with each
277
+ participant, on each machine - take equal shares (``--max-usd``), so
278
+ none spends another's and the amount typed is the most the run spends.
279
+ ``max_usd_per_unit``: each unit's own cap (``--max-usd-per-unit``), given
280
+ whole to every job that calls a model: a unit is a unit wherever it runs.
281
+ ``unpriced``: asked, for a recipe with a cap, which of the models it
282
+ names have no price in the user's list (``nullius.prices``): a run
283
+ with a cap is refused without one, and the Limits step says so first."""
284
+ d = defs or _all()
285
+ errors, notes = [], []
286
+ add = lambda step, message: errors.append({"step": STEP[step], "message": message})
287
+ out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""}
288
+ name = str(recipe.get("name") or "")
289
+ if not _NAME.match(name):
290
+ add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.")
291
+ # the benchmark by key (its default version) or key@version
292
+ b = definitions.find(d["benchmark"], str(recipe.get("benchmark") or "")) if recipe.get("benchmark") else None
293
+ if b is None:
294
+ add("benchmark", "Choose a benchmark of the standard.")
295
+ return out
296
+ b = dict(b, label=definitions.title(b))
297
+ # one dataset or several (``datasets``; a recipe from before names one, ``dataset``): each its own jobs
298
+ wanted = recipe.get("datasets") if isinstance(recipe.get("datasets"), list) else ([recipe["dataset"]] if recipe.get("dataset") else [])
299
+ wanted = list(dict.fromkeys(str(x) for x in wanted if isinstance(x, str)))
300
+ dss = [x for x in b["datasets"] if x["id"] in wanted]
301
+ dss.sort(key=lambda x: wanted.index(x["id"]))
302
+ unknown = [x for x in wanted if x not in {y["id"] for y in b["datasets"]}]
303
+ if not dss or unknown:
304
+ add("dataset", f"Choose one or more of {b['label']}'s datasets" + (f" ({', '.join(unknown)} is not one)" if unknown else "") + ".")
305
+ side = "agent" if b["family"] == "agent" else "capability"
306
+ # A control the benchmark declares (nop, oracle) is a choice beside the agents or capabilities: it runs
307
+ # nothing as you and calls no model - a judged benchmark's judge only - so it asks for neither.
308
+ # Several may be chosen: each is its own job of the one run, one after the other.
309
+ backend = definitions.names.backend(recipe.get("backend") or "local")
310
+ if backend not in BACKENDS:
311
+ add("where", "The backend is local processes or a container.")
312
+ backend = "local"
313
+ what = "an agent" if side == "agent" else f"a {_bench_page(b)['kind'] or 'capability'}"
314
+ refs = chosen_of(recipe)
315
+ if not refs:
316
+ add("agent", f"Choose how {b['label']} runs: " + ("as published, " if b.get("setups") else "") + f"with {what}, or a check.")
317
+ several = len(refs) > 1
318
+ picks = [] # [{o, control, given, opts}]
319
+ provider_ = recipe.get("provider") if recipe.get("provider") in PROVIDERS else ""
320
+ for ref in refs:
321
+ control = ref in definitions.CONTROLS
322
+ given, opts = {}, {}
323
+ if ref == "model":
324
+ add("agent", "Choose Original, then choose the model on the Model step.")
325
+ continue
326
+ if ref == "original":
327
+ orig = (b.get("setups") or {}).get("original") or {}
328
+ picked = str(recipe.get("model") or "").strip()
329
+ if not picked:
330
+ add("model", "Choose a model.")
331
+ continue
332
+ method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original"
333
+ o, why_not = definitions.setup(b, method, picked, provider_)
334
+ if o is None:
335
+ add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".")
336
+ continue
337
+ picks.append({"o": o, "control": False, "setup": True, "given": {}, "opts": {}, "model": picked or o["asked"]})
338
+ continue
339
+ if control:
340
+ o = definitions.control(b, ref)
341
+ if o is None:
342
+ add("agent", f"{b['label']} declares no {ref} control.")
343
+ continue
344
+ else:
345
+ o = definitions.find(d[side], ref)
346
+ if o is None:
347
+ add("agent", f"Choose {what} for {b['label']}: {ref} is not one.")
348
+ continue
349
+ o = dict(o, label=definitions.title(o))
350
+ why = match.fit(b, o)
351
+ if why:
352
+ add("agent", f"{o['label']} cannot run {b['label']}: " + "; ".join(why) + ".")
353
+ given = options_for(recipe, ref, several)
354
+ if not isinstance(given, dict) or not all(isinstance(v, str) for v in given.values()):
355
+ add("agent", "Options are names with a word each.")
356
+ given = {}
357
+ opts, bad = definitions.options_of(o, given)
358
+ for why_not in bad:
359
+ add("agent", f"{o['label']}: " + why_not)
360
+ picks.append({"o": o, "control": control, "given": given, "opts": opts})
361
+ # the benchmark's own options (``bench_options``), each on the datasets it applies to; what the chosen
362
+ # values need: embedding and reranker models through the run's relay, the machine's programs
363
+ chosen_opts = recipe.get("bench_options") or {}
364
+ if not isinstance(chosen_opts, dict) or not all(isinstance(v, str) for v in chosen_opts.values()):
365
+ add("dataset", "The benchmark's options are names with a word each.")
366
+ chosen_opts = {}
367
+ lb = (b.get("leaderboard") or {}) if recipe.get("model_choice") == "leaderboard" else {}
368
+ for k, v in (lb.get("bench_options") or {}).items():
369
+ if chosen_opts.get(k, v) != v:
370
+ add("dataset", f"Like the leaderboard sets {(b.get('options') or {})[k].get('label') or k.replace('_', ' ')} to {v}: choose Any of your choice for {chosen_opts[k]}.")
371
+ chosen_opts = {**(lb.get("bench_options") or {}), **chosen_opts}
372
+ per_ds, needs = {}, {"embeddings": [], "reranker": [], "host": []}
373
+ scopes = list(dict.fromkeys(p["o"]["key"] if p.get("setup") else "yours" for p in picks)) or ["yours"]
374
+ for x in dss:
375
+ # Original's retrievers have machine requirements too; collect the
376
+ # options of every participant that the compiled commands will run.
377
+ vals, bad = {}, []
378
+ for scope in scopes:
379
+ chosen, problems = definitions.bench_options_of(b, x["id"], chosen_opts, scope)
380
+ vals.update(chosen)
381
+ bad.extend(problems)
382
+ for why_not in dict.fromkeys(bad):
383
+ add("dataset", why_not[0].upper() + why_not[1:] + ".")
384
+ per_ds[x["id"]] = vals
385
+ n = definitions.option_needs(b, vals)
386
+ needs["embeddings"] += [m for m in n["embeddings"] if m not in needs["embeddings"]]
387
+ needs["host"] += [h for h in n["host"] if h not in needs["host"]]
388
+ needs["reranker"] += [m for m in n["reranker"] if m not in needs["reranker"]]
389
+ for m in needs["embeddings"]:
390
+ why = providers.embedding_problem(m, recipe.get("provider") or "openai")
391
+ if why:
392
+ add("model", why)
393
+ agents_local = [p["o"]["label"] for p in picks if side == "agent" and not p["control"] and not p.get("setup") and backend == "local"]
394
+ if agents_local and recipe.get("unsandboxed") is not True:
395
+ add("where", f"{', '.join(agents_local)} would run as you, not sandboxed: tick Run it as me, or choose Docker.")
396
+ only_controls = bool(picks) and all(p["control"] for p in picks)
397
+ needs_model = any(not p["control"] for p in picks)
398
+ model_choice = recipe.get("model_choice", "any")
399
+ if model_choice not in ("paper", "leaderboard", "any"):
400
+ add("model", "Choose Like the paper, Like the leaderboard or Any of your choice.")
401
+ if model_choice == "leaderboard" and not b.get("leaderboard"):
402
+ add("model", f"{b['label']} declares no leaderboard; choose Like the paper or Any of your choice.")
403
+ orig = (b.get("setups") or {}).get("original") or {}
404
+ official = {"judge": ((b.get("score") or {}).get("judge") or {}).get("model", ""), "simulator": (b.get("simulator") or {}).get("model", "")}
405
+ fixed = {**official, **{k: lb[k] for k in ("judge", "simulator") if lb.get(k)}}
406
+ if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()):
407
+ add("model", "No paper models are listed for this benchmark; choose Any of your choice.")
408
+ elif model_choice == "paper" and needs_model and orig.get("models"):
409
+ if recipe.get("model") not in orig["models"]:
410
+ add("model", "Choose one of the paper's listed models.")
411
+ elif provider_ not in orig.get("providers", {}):
412
+ add("model", "The paper's models are not listed for this provider.")
413
+ provider = recipe.get("provider")
414
+ if provider not in PROVIDERS:
415
+ add("model", "Choose a provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.")
416
+ model = str(recipe.get("model") or "").strip()
417
+ if not _MODEL.match(model) and (needs_model or model):
418
+ add("model", "Choose a model or enter its ID.")
419
+ if model_choice == "paper" and model in orig.get("models", []) and provider_ in orig.get("providers", {}):
420
+ model = orig["providers"][provider_] + (orig.get("served_as") or {}).get(model, model)
421
+ # each unit tried K times (the upstream's --num-trials: pass^k); the user simulator's and the judge's models
422
+ repeat = _int(recipe.get("repeat") or lb.get("repeat") or 1, 1, 32)
423
+ if repeat is None:
424
+ add("limits", "Attempts per unit: a whole number from 1 to 32.")
425
+ repeat = 1
426
+ if lb and repeat < lb.get("repeat", 1):
427
+ add("limits", f"Like the leaderboard runs each {(b.get('units') or {}).get('noun', 'unit')} at least {lb['repeat']} times.")
428
+ sim_model = str(recipe.get("simulator_model") or "").strip()
429
+ judge_model = str(recipe.get("judge_model") or "").strip()
430
+ like = {"paper": "Like the paper", "leaderboard": "Like the leaderboard"}.get(model_choice)
431
+ for what, role, m in (("the user simulator's", "simulator", sim_model), ("the judge's", "judge", judge_model)):
432
+ if m and not _MODEL.match(m):
433
+ add("model", f"Enter {what} model as the provider lists it, or leave it empty for its official one.")
434
+ elif m and like and m not in (fixed[role], providers.served_as(fixed[role], provider_)):
435
+ add("model", f"{like} uses {what} model {fixed[role]}: choose Any of your choice for {m}.")
436
+ # the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default
437
+ sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model
438
+ judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model
439
+ # the run's spending cap: an amount in USD, or none
440
+ cap = recipe.get("max_usd")
441
+ if cap in ("", None):
442
+ cap = None
443
+ elif isinstance(cap, bool) or not isinstance(cap, (int, float)) or not 0 < cap < float("inf"):
444
+ add("limits", "The spending cap is an amount in USD above zero, or empty for none.")
445
+ cap = None
446
+ noun = (b.get("units") or {}).get("noun", "unit")
447
+ per_unit = recipe.get("max_usd_per_unit")
448
+ if per_unit in ("", None):
449
+ per_unit = None
450
+ elif isinstance(per_unit, bool) or not isinstance(per_unit, (int, float)) or not 0 < per_unit < float("inf"):
451
+ add("limits", f"The spending cap per {noun} is an amount in USD above zero, or empty for none.")
452
+ per_unit = None
453
+ elif per_unit < 1e-6: # what the command line can say: a millionth of a dollar
454
+ add("limits", f"The spending cap per {noun} is too small.")
455
+ per_unit = None
456
+ limit = recipe.get("limit")
457
+ if limit not in ("", None) and _int(limit, 1) is None:
458
+ add("limits", "The number of units is a whole number of at least 1, or empty for all of them.")
459
+ conc = _int(recipe.get("concurrency") or 1, 1, MAX_CONCURRENT) # empty: one at a time
460
+ if conc is None:
461
+ add("limits", f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.")
462
+ targets, why_not = where_of(recipe.get("where", "local"))
463
+ if why_not:
464
+ add("where", why_not)
465
+ elif backend == "container" and "local" in targets and docker_unusable is not None:
466
+ why = docker_unusable()
467
+ if why:
468
+ add("where", f"The container backend cannot run on this machine: {why}. Choose local processes, or a host with Docker or Podman.")
469
+ # a job that calls no model (a control of a benchmark with no judge) spends nothing and takes no share
470
+ spends = lambda p: not (p["control"] and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"])
471
+ paying = len(dss) * sum(1 for p in picks if spends(p)) * len(targets or [None])
472
+ share = None
473
+ if cap is not None and paying:
474
+ share = math.floor(cap / paying * 1e6) / 1e6 # never more than the cap in all
475
+ if not share:
476
+ add("limits", "The spending cap is too small" + (f" to share over {paying} jobs." if paying > 1 else "."))
477
+ if not paying:
478
+ per_unit = None # no job calls a model: nothing to cap
479
+ if (share or per_unit) and unpriced is not None:
480
+ # the models each job names as it sends them, read where its run reads them (ledger.models): each
481
+ # dataset with each choice, the dataset's own embeddings
482
+ judge, sim = (b.get("score") or {}).get("judge"), b.get("simulator")
483
+ called = [m for x in dss for p in picks for m in ledger.models(
484
+ p["o"], None if p["control"] else (p["o"].get("sent") or p.get("model")) if p.get("setup") else model,
485
+ (judge_model or providers.served_as(judge["model"], provider)) if judge else None,
486
+ (sim_model or providers.served_as(sim["model"], provider)) if sim else None,
487
+ providers.served_as((definitions.option_needs(b, per_ds[x["id"]])["embeddings"] or [""])[0], provider), control=p["control"])]
488
+ called += [providers.served_as(m, provider) for m in needs["reranker"]]
489
+ missing = unpriced([m for m in dict.fromkeys(called) if isinstance(m, str) and m])
490
+ if missing:
491
+ add("limits", f"A spending cap needs a price for {', '.join(missing)}: update the price list in Settings, or set a price there.")
492
+ if errors:
493
+ return out
494
+ nouns = (b.get("units") or {}).get("nouns", "units")
495
+ if picks and all(p.get("setup") for p in picks): # only setups: a unit is theirs (a conversation)
496
+ nouns = picks[0]["o"]["units"]["nouns"]
497
+ each = [None if picks and all(p.get("setup") for p in picks) else x.get("units") for x in dss] # a setup's count is not the dataset's
498
+ total = sum(each) if all(each) else None
499
+ n = sum(min(limit, t) for t in each) if isinstance(limit, int) and total else (limit or total)
500
+ of = len(targets)
501
+ many = len(dss) > 1
502
+ # what a run is made of: each dataset with each choice, a job each, in that order
503
+ pairs = [(x, p) for x in dss for p in picks]
504
+
505
+ def word_of(d: dict, p: dict) -> str:
506
+ return "-".join(([d["id"]] if many else []) + ([definitions.job_word(p["o"])] if several else []))
507
+
508
+ def job_of(d: dict, p: dict, index: int) -> str:
509
+ base = "-".join(filter(None, [name, word_of(d, p)]))
510
+ return f"{base}-shard-{index}of{of}" if of > 1 else base
511
+
512
+ def options_for_pick(d: dict, p: dict) -> dict:
513
+ """The benchmark's options for this dataset and this pick: a setup's own, or those of the port."""
514
+ return definitions.bench_options_of(b, d["id"], chosen_opts, p["o"]["key"] if p.get("setup") else "yours")[0]
515
+
516
+ def command_for(d: dict, p: dict, index: int) -> str:
517
+ o, control, given = p["o"], p["control"], p["given"]
518
+ launch = "-".join(filter(None, [name, word_of(d, p)]))
519
+ m = p["model"] if p.get("setup") else model
520
+ words = ["nullius", "standard", "run", definitions.ref_of(b), o["_ref"], *(["--model", m] if m else []), "--provider", provider,
521
+ "--dataset", d["id"], *(["--limit", str(limit)] if isinstance(limit, int) else []), "-j", str(conc),
522
+ *[w for k, v in options_for_pick(d, p).items() for w in ("--bench-option", f"{k}={v}")],
523
+ *(["--repeat", str(repeat)] if repeat > 1 else []),
524
+ *(["--max-usd", f"{share:.6f}".rstrip("0").rstrip(".")] if share and spends(p) else []),
525
+ *(["--max-usd-per-unit", f"{per_unit:.6f}".rstrip("0").rstrip(".")] if per_unit and spends(p) else []),
526
+ *(["--simulator-model", sim_model] if sim_model and not p["control"] else []),
527
+ *(["--judge-model", judge_model] if judge_model else []),
528
+ "--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")],
529
+ *(["--backend", "container"] if backend == "container" else []),
530
+ *(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []),
531
+ *(["--shard", f"{index}/{of}", "--launch", launch] if of > 1 else []), "--from-run"]
532
+ return " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
533
+
534
+ def script_for(d: dict, p: dict, command: str, index: int) -> str:
535
+ return ("#!/usr/bin/env bash\n"
536
+ f"# {b['label']} ({d.get('label') or d['id']}) with {p['o']['label']}, through the Nullius standard (docs/standard.md)"
537
+ + (f": shard {index} of {of}.\n" if of > 1 else ".\n") +
538
+ "# Uses installed nullius or --app PATH ($NULLIUS_STANDARD for an execution payload). The key and\n"
539
+ "# endpoint come from the run's environment: OPENAI_API_KEY, and OPENAI_BASE_URL for an endpoint\n"
540
+ "# other than the provider's own.\n"
541
+ + _script(command))
542
+
543
+ out["shards"] = []
544
+ commands = []
545
+ for i, where in enumerate(targets, 1):
546
+ parts, jobs = [], []
547
+ for d, p in pairs:
548
+ command = command_for(d, p, i)
549
+ commands.append(command)
550
+ parts.append({"id": word_of(d, p) or definitions.job_word(p["o"]),
551
+ "label": " · ".join(([d.get("label") or d["id"]] if many else []) + [p["o"]["label"]]) + (f" · shard {i}/{of}" if of > 1 else ""),
552
+ "script": script_for(d, p, command, i)})
553
+ jobs.append({"name": job_of(d, p, i), "agent": p["o"]["_ref"], "dataset": definitions.ref_of(b)})
554
+ out["shards"].append({"where": where, "index": i, "of": of, "name": f"{name}-shard-{i}of{of}" if of > 1 else name,
555
+ "command": "\n".join(commands[-len(pairs):]), "script": parts[0]["script"], "parts": parts, "jobs": jobs})
556
+ out["script"] = "\n".join(pt["script"] for pt in out["shards"][0]["parts"]) if len(pairs) > 1 else out["shards"][0]["script"]
557
+ out["parts"] = [pt for sh in out["shards"] for pt in sh["parts"]]
558
+ out["jobs"] = [j for sh in out["shards"] for j in sh["jobs"]]
559
+ # docker where it runs: the container backend, or a control run in each unit's own image, whatever the backend
560
+ own = any(definitions.own_image(definitions.environment_of(b, None, x)[1]) for x in dss)
561
+ out["docker"] = backend == "container" or (own and any(p["control"] for p in picks))
562
+ out["model"] = model
563
+ # Nothing in the run asks a model: no key is handed to it, and none need be set.
564
+ out["keyless"] = only_controls and not (b.get("score") or {}).get("judge") and not needs["embeddings"] and not needs["reranker"]
565
+ # what the machines must have for the chosen options (``host``): the Where step and its test show it, Run checks it
566
+ on_host = needs["host"]
567
+ if backend == "container" and any(p.get("setup") for p in picks):
568
+ # a container setup's image installs what has an install; nothing else reaches its container
569
+ imaged = hostreq.in_image(b, on_host)
570
+ cannot = [n for n in hostreq.requirements(b, on_host) if n not in imaged]
571
+ if cannot:
572
+ add("where", f"{', '.join(cannot)} cannot be provided in a container: choose local processes.")
573
+ if all(p.get("setup") for p in picks):
574
+ on_host = [] # an agent's or a check's driver still runs on the machine
575
+ reqs = hostreq.requirements(b, on_host)
576
+ out["host"] = {"benchmark": b["key"], "needs": [{"name": n, "summary": r["summary"], "check": r["check"],
577
+ **({"gpu_memory_mb": r["gpu_memory_mb"]} if "gpu_memory_mb" in r else {})}
578
+ for n, r in reqs.items()],
579
+ "check": [p for r in reqs.values() for p in r["check"]], "install": hostreq.install_lines(b, list(reqs))}
580
+ # The licences its environments may need, by name: what a run on a host is sent of the benchmark's store,
581
+ # a data provider's token of that provider's.
582
+ declared = definitions.secrets_of(b)
583
+ out["licences"] = {"benchmark": b["key"], "names": sorted({s["name"] for s in declared}),
584
+ "providers": {s["name"]: s["provider"] for s in declared if s.get("provider")}}
585
+
586
+
587
+ def row(p: dict) -> list:
588
+ o = p["o"]
589
+ if p["control"]:
590
+ return [STEP["agent"], "Check", f"{o['label']} ({o['key']}) · {o['source']} · {o['from']}"]
591
+ if p.get("setup"):
592
+ original = (b.get("setups") or {}).get("original") or {}
593
+ label = original.get("reference_agent") or ("No agent · Original" if side == "agent" else "Original " + (_bench_page(b)["kind"] or "capability"))
594
+ return [STEP["agent"], "Participant", label]
595
+ proven = ", ".join(f"{q} {r.get('passed', '')}".strip() for q, r in (o.get("proven") or {}).items() if q in b["needs"])
596
+ return [STEP["agent"], "Agent" if side == "agent" else "Capability", f"{o['label']} · {protocol(o)} · proven {proven}"]
597
+ out["review"] = [[STEP["benchmark"], "Benchmark", f"{b['label']} · needs {', '.join(b['needs'])}"],
598
+ [STEP["runner"], "Runner", "Nullius"],
599
+ [STEP["dataset"], "Datasets" if many else "Dataset",
600
+ ", ".join(f"{x.get('label') or x['id']}" + (f" · {x['units']} {nouns}" if x.get("units") else "") for x in dss)],
601
+ *[[STEP["dataset"], (b.get("options") or {})[k].get("label") or k.replace("_", " ").capitalize(),
602
+ f"{v}" + ("" if k in chosen_opts else " (its default)") + f" · {', '.join(x['id'] for x in dss if per_ds[x['id']].get(k) == v)}"]
603
+ for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items()],
604
+ *[row(p) for p in picks],
605
+ *[[STEP["agent"], "Options" + (f" of {p['o']['label']}" if several else ""),
606
+ ", ".join(f"{k} {v}" + ("" if k in p["given"] else " (its default)") for k, v in p["opts"].items())]
607
+ for p in picks if p["opts"]],
608
+ [STEP["model"], "Model", "none: a control calls no model" if out["keyless"]
609
+ else f"{providers.label(providers.choice(provider, str(recipe.get('endpoint') or '')))} · " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only")
610
+ + (f" · user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "")
611
+ + (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")],
612
+ [STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}"
613
+ + (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ")
614
+ + (" of each dataset" if many and isinstance(limit, int) else "")
615
+ + (f" · {len(pairs)} jobs, one after the other" if len(pairs) > 1 else "")
616
+ + (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "")
617
+ + (" · a setup counts " + next(p["o"]["units"]["nouns"] for p in picks if p.get("setup"))
618
+ if any(p.get("setup") for p in picks) and not all(p.get("setup") for p in picks) else "")],
619
+ *([[STEP["limits"], "Spending cap", f"${ledger.usd(cap)}"
620
+ + (f" in all · ${ledger.usd(share)} for each of {paying} jobs" if paying > 1 else "")]] if share else []),
621
+ *([[STEP["limits"], f"Spending cap per {noun}", f"${ledger.usd(per_unit)}"]] if per_unit else []),
622
+ [STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "")
623
+ + ", ".join(where_label(t) for t in targets) + f" · {backend} backend"]]
624
+ if agents_local:
625
+ notes.append(f"{', '.join(agents_local)} {'run' if len(agents_local) > 1 else 'runs'} as you on {'each machine' if of > 1 else where_label(targets[0])}, with its permission requests granted: "
626
+ "the local backend is no sandbox.")
627
+ if of > 1:
628
+ notes.append(f"Each machine runs the units whose shard is its own, by their ids' hash: {of} jobs under one launch, "
629
+ f"{name}, put together by cli merge.")
630
+ docker_only = [n for n, e in (b.get("environments") or {}).items() if not e.get("local_ok")]
631
+ if docker_only and not only_controls and backend == "local":
632
+ notes.append(f"Units in {b['label']}'s environment{'s' if len(docker_only) > 1 else ''} {', '.join(docker_only)} "
633
+ "need the container backend: on local processes they are recorded as not run.")
634
+ if any(p["control"] for p in picks) and any(definitions.own_image(e) for e in (b.get("environments") or {}).values()):
635
+ notes.append(f"Each {(b.get('units') or {}).get('noun', 'unit')} of {b['label']} is its own image: the control "
636
+ "runs in it, so docker is needed where it runs, whatever the backend.")
637
+ if not isinstance(limit, int):
638
+ out["confirm"] = f"Run all {total or ''} {nouns} of {b['label']}{' (' + ', '.join(x.get('label') or x['id'] for x in dss) + ')' if many else ''} with {', '.join(p['o']['label'] for p in picks)}?".replace(" ", " ")
639
+ return out
640
+
641
+
642
+ def where_of(where) -> tuple:
643
+ """*Where* as a list of machines - ``local`` or a host's entry - and why
644
+ it cannot be, or "": a word or a list, each machine once. Whether a host
645
+ is one of Settings' is the route's to check."""
646
+ targets = where if isinstance(where, list) else [where if where not in (None, "") else "local"]
647
+ if not targets:
648
+ return ["local"], "Choose where it runs: this machine, a host, or several."
649
+ out = []
650
+ for t in targets:
651
+ t = str(t).strip() if isinstance(t, str) else ""
652
+ if t != "local":
653
+ try:
654
+ t = hosts.parse(t).entry
655
+ except ValueError as e:
656
+ return ["local"], f"Where: {e}."
657
+ if t in out:
658
+ return ["local"], f"Where names {where_label(t)} twice: each machine runs one shard."
659
+ out.append(t)
660
+ return out, ""
661
+
662
+
663
+ def where_label(target: str) -> str:
664
+ return "this machine" if target == "local" else hosts.parse(target).label
665
+
666
+
667
+ def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: bool, stamp: str, backend: str = "local") -> dict:
668
+ """The check an Agents or Capabilities page starts: every kit the agent or
669
+ capability can take, as one command (``cli conform``), the jobs it will
670
+ write named ahead (``conform.job_name``). {errors, script, jobs, label}."""
671
+ from nullius.standard import conform
672
+ o = definitions.get(kind, key) if kind in ("agent", "capability") else None
673
+ errors = []
674
+ if o is None:
675
+ return {"errors": [f"no {kind} {key!r}"], "script": "", "jobs": [], "label": key}
676
+ o = dict(o, label=definitions.title(o))
677
+ if provider not in PROVIDERS:
678
+ errors.append("Choose a provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.")
679
+ if not _MODEL.match(model or ""):
680
+ errors.append("Choose a model or enter its ID.")
681
+ backend = definitions.names.backend(backend)
682
+ if backend not in BACKENDS:
683
+ errors.append("Choose local processes or a container for the checks.")
684
+ if kind == "agent" and backend == "local" and unsandboxed is not True:
685
+ errors.append(f"{o['label']} runs on this machine as you while its kits run: the local backend is not a sandbox. Say yes to that to check it.")
686
+ if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""):
687
+ errors.append("a time stamp: YYYY-MM-DD__HH-MM-SS")
688
+ kits = [definitions.kit(p) for p in conform.candidates(o)]
689
+ kits = [k for k in kits if k]
690
+ if not kits:
691
+ errors.append(f"{o['label']} has no port with a kit to check.")
692
+ if errors:
693
+ return {"errors": errors, "script": "", "jobs": [], "label": o["label"]}
694
+ words = ["nullius", "standard", "conform", o["_ref"], "--model", model, "--provider", provider,
695
+ "--stamp", stamp, *(["--backend", "container"] if backend == "container" else []),
696
+ *(["--unsandboxed"] if kind == "agent" and backend == "local" else []), "--from-run"]
697
+ command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
698
+ script = ("#!/usr/bin/env bash\n"
699
+ f"# {o['label']}: the conformance kits of the ports it can offer (docs/standard.md, \"Ports\").\n"
700
+ + _script(command))
701
+ return {"errors": [], "script": script, "label": o["label"],
702
+ "jobs": [{"name": conform.job_name(stamp, k["key"], definitions.job_word(o)), "agent": o["_ref"], "dataset": k["key"]}
703
+ for k in kits]}
704
+
705
+
706
+ def proving(benchmark: str, *, dataset: str = "", limit=None, provider: str = "openai", judge_model: str = "",
707
+ stamp: str, concurrency: int = 4) -> dict:
708
+ """The proof a benchmark's page starts, as an Agents page starts a check
709
+ of ports (``conformance``): ``cli prove`` over one dataset or each - its
710
+ units, the leak check and every control it declares - as one command,
711
+ the control jobs it will write named ahead (``prove.job_name``).
712
+ ``keyless``: the controls call no model, so the run is handed no key at
713
+ all - unless the benchmark is judged, whose judge still grades.
714
+ {errors, script, jobs, label, keyless}."""
715
+ errors: List[str] = []
716
+ b = definitions.find(_all()["benchmark"], benchmark or "")
717
+ if b is None:
718
+ return {"errors": [f"no benchmark {benchmark!r}"], "script": "", "jobs": [], "label": benchmark, "keyless": False}
719
+ b = dict(b, label=definitions.title(b))
720
+ judged = bool((b.get("score") or {}).get("judge"))
721
+ sets = [d for d in b["datasets"] if dataset in ("", d["id"])]
722
+ if not sets:
723
+ errors.append(f"Choose one of {b['label']}'s datasets, or each of them.")
724
+ if limit not in ("", None) and _int(limit, 1) is None:
725
+ errors.append("The number of units is a whole number of at least 1, or empty for all of them.")
726
+ if _int(concurrency, 1, MAX_CONCURRENT) is None:
727
+ errors.append(f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.")
728
+ if judged and provider not in PROVIDERS:
729
+ errors.append("Choose the judge's provider: OpenRouter, OpenAI or an OpenAI-compatible endpoint.")
730
+ if judge_model and not _MODEL.match(judge_model):
731
+ errors.append("The judge model, as the provider lists it - or empty for the benchmark's own.")
732
+ if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""):
733
+ errors.append("a time stamp: YYYY-MM-DD__HH-MM-SS")
734
+ if errors:
735
+ return {"errors": errors, "script": "", "jobs": [], "label": b["label"], "keyless": False}
736
+ words = ["nullius", "standard", "prove", definitions.ref_of(b), *(["--dataset", dataset] if dataset else []),
737
+ *(["--limit", str(limit)] if isinstance(limit, int) else []), *(["--provider", provider] if judged else []),
738
+ *(["--judge-model", judge_model] if judged and judge_model else []), "-j", str(concurrency), "--stamp", stamp, "--from-run"]
739
+ command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
740
+ script = ("#!/usr/bin/env bash\n"
741
+ f"# {b['label']}: its proofs - units, no leak, its controls (docs/standard.md, \"Controls and proofs\").\n"
742
+ + _script(command))
743
+ declared = [c for c in definitions.CONTROLS if c in (b.get("controls") or {})]
744
+ return {"errors": [], "script": script, "label": b["label"], "keyless": not judged,
745
+ "jobs": [{"name": prove.job_name(stamp, b["key"], d["id"], c), "agent": c, "dataset": definitions.ref_of(b)}
746
+ for d in sets for c in declared]}