nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,703 @@
1
+ """The standard from a shell.
2
+
3
+ PYTHONPATH=app python3 -m nullius.standard.cli list
4
+ PYTHONPATH=app python3 -m nullius.standard.cli check [PATH]
5
+ PYTHONPATH=app python3 -m nullius.standard.cli fit BENCHMARK[@VERSION]
6
+ PYTHONPATH=app python3 -m nullius.standard.cli pins BENCHMARK[@VERSION] [--tag TAG] [--limit N]
7
+ PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P]
8
+ PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION
9
+ PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container]
10
+ PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--unsandboxed | --backend container]
11
+ PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--limit N]
12
+ PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N]
13
+ PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID]
14
+ PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force]
15
+ PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed
16
+
17
+ An agent runs commands as you on the local backend, which is not a sandbox:
18
+ an agent's run or conformance check there is refused without ``--unsandboxed``.
19
+ ``--backend container`` (``docker``, its earlier name) runs each unit's agent or capability in a container whose
20
+ only way out is the unit's relay, or a public web proxy when its environment
21
+ declares ``network`` (``standard/container.py``), and needs no such yes.
22
+ A benchmark's control (``nop``, ``oracle``: its ``controls``) runs no agent
23
+ and calls no model - only a judged benchmark's judge - so it asks for neither.
24
+ ``prove`` runs a benchmark's controls and checks its units and that nothing
25
+ leaks, and writes what passed into its ``proven`` (``standard/prove.py``).
26
+
27
+ A benchmark, an agent or a capability is named by its key (its default
28
+ version) or ``key@version``. ``list`` shows every definition - each version - and which
29
+ pairs fit; ``check`` checks the
30
+ definitions (or one file); ``new`` writes a definition to fill in, or a new
31
+ version of an agent or a capability (``new version``: the default's pins, with
32
+ the version replaced, and nothing proven); ``fit`` says, for one benchmark, what can run it and
33
+ why the rest cannot; ``conform`` runs the kits and writes what passed into the
34
+ definition - the port, and each skill of its kit shown or not (``--repeat K``:
35
+ each check K times, held by a majority); ``run`` runs a benchmark into a job under ``--jobs`` (``jobs_standard/``),
36
+ which the application opens like any other; ``fidelity`` drives an agent over
37
+ ACP and through its own command, compares the requests its model was sent, and
38
+ writes the verdict into the definition (``standard/fidelity.py``). ``--shard I/N`` runs one
39
+ shard of the dataset's units (``standard/shard.py``: by the unit id's hash, then
40
+ ``--limit``), the job recording its shard and its launch (``--launch``); ``merge``
41
+ puts a launch's shard jobs together into one job (``standard/merge.py``). ``--option``
42
+ sets one of the agent's or capability's options (``options`` in its definition);
43
+ the rest keep the vendor's defaults, and the job's config records them all.
44
+ A run says each stage before its first unit, and each unit's end, as one
45
+ ``[nullius] stage=...`` line on stderr and in the job's ``progress.jsonl``
46
+ (``nullius/progress.py``): what the run's page shows while it goes.
47
+
48
+ Keys come from the process environment, then the saved user settings - the
49
+ file the Settings page edits. Nothing is printed of them. With ``--from-run`` (a run
50
+ the builder or a page started, ``standard/recipe.py``) saved settings are not
51
+ read: the key is the one the run was handed, ``OPENAI_API_KEY``, and the
52
+ endpoint ``OPENAI_BASE_URL`` when it names one (``run_env``). The licences a
53
+ benchmark's environments declare are never in either: the run reads them from
54
+ the benchmark's store (``nullius.secrets``) on this machine, or, on a host, from
55
+ the one line its launcher handed it (``licences``).
56
+ """
57
+ from __future__ import annotations
58
+
59
+ import argparse
60
+ import json
61
+ import os
62
+ import shutil
63
+ import signal
64
+ import sys
65
+ import tempfile
66
+
67
+ from nullius import paths, prices as price_list, progress, secrets, settings
68
+ # The legacy module command needs saved path overrides before install.HOME
69
+ # and other runtime defaults are computed, just like the unified CLI does.
70
+ if __name__ == "__main__" and "--from-run" not in sys.argv[1:]:
71
+ settings.load(paths.settings_file())
72
+
73
+ from nullius.standard import conform, definitions, fidelity, job, match, merge, pins, ports, prove, run, shard
74
+
75
+ ENV_FILE = paths.settings_file()
76
+
77
+
78
+ def _all():
79
+ got, bad = {}, []
80
+ for kind in definitions.KINDS:
81
+ got[kind], b = definitions.load(kind, every=True)
82
+ bad += b
83
+ return got, bad
84
+
85
+
86
+ def _other(key):
87
+ o = definitions.get("agent", key) or definitions.get("capability", key)
88
+ if not o:
89
+ sys.exit(f"no agent or capability {key!r}: {[d['_ref'] for k in ('agent', 'capability') for d in definitions.load(k, every=True)[0]]}")
90
+ return o
91
+
92
+
93
+ def run_env(provider: str, env=os.environ) -> dict:
94
+ """The environment a run started by Nullius reads its provider from:
95
+ ``execution``'s launcher hands every run its one key as ``OPENAI_API_KEY``
96
+ and its endpoint as ``OPENAI_BASE_URL`` (empty for OpenAI's own), so they
97
+ are named here as the provider's own settings and nothing else is."""
98
+ p = run.PROVIDERS[provider]
99
+ out = {k: v for k, v in env.items() if k in run.KEEP}
100
+ if env.get("OPENAI_API_KEY"):
101
+ out[p["key"]] = env["OPENAI_API_KEY"]
102
+ if "base_var" in p and env.get("OPENAI_BASE_URL"):
103
+ out[p["base_var"]] = env["OPENAI_BASE_URL"]
104
+ return out
105
+
106
+
107
+ def licences(bench: dict, env=os.environ):
108
+ """(``run.Licences``, the folder to remove after) for a run on a host,
109
+ whose launcher handed it the benchmark's licences as one line
110
+ (``secrets.HANDED``, ``execution``): a licence file is written to
111
+ ``NULLIUS_LICENCE_DIR`` - in memory, ``/dev/shm`` - and nowhere else.
112
+ The line leaves this process's environment once read. (None, None) on
113
+ this machine: the run reads the benchmark's own store."""
114
+ line = env.pop(secrets.HANDED, None)
115
+ if not line:
116
+ return None, None
117
+ shm = "/dev/shm" if os.path.isdir("/dev/shm") else tempfile.gettempdir()
118
+ spill = env.get("NULLIUS_LICENCE_DIR") or os.path.join(shm, f"nullius-{os.getpid()}")
119
+ return run.Licences.handed(bench, secrets.unpack(line), spill), spill
120
+
121
+
122
+ def older(d: dict, ports_=None) -> list:
123
+ """``agent.tools/v1 proven on kit v1; kit v2 is current``, for each port
124
+ ``d`` offers (of ``ports_``, when given) on a proof older than its kit."""
125
+ proven = d.get("proven") or {}
126
+ out = [(p, definitions.older_kit(p, proven.get(p))) for p in d.get("offers") or [] if ports_ is None or p in ports_]
127
+ return [f"{p} {note}" for p, note in out if note]
128
+
129
+
130
+ def skills(d: dict, ports_=None) -> list:
131
+ """``agent.tools/v1 skills: decoys ✗ · chain ✓ · retry ✗``, for each port
132
+ ``d`` offers (of ``ports_``, when given) whose kit has skills."""
133
+ from nullius.standard.recipe import skills_of
134
+ proven, out = d.get("proven") or {}, []
135
+ for p in d.get("offers") or []:
136
+ got = skills_of(p, proven.get(p)) if ports_ is None or p in ports_ else []
137
+ if got:
138
+ out.append(f"{p} skills: " + " · ".join(f"{s['id']} {'✓' if s['shown'] else '✗'}" for s in got))
139
+ return out
140
+
141
+
142
+ def cmd_pins(a):
143
+ if a.limit is not None and a.limit < 1:
144
+ print("--limit must be a positive whole number", file=sys.stderr)
145
+ return 2
146
+ if a.tag is not None and not pins.TAG.fullmatch(a.tag):
147
+ print("invalid image tag", file=sys.stderr)
148
+ return 2
149
+ bench = definitions.get("benchmark", a.benchmark)
150
+ if bench is None:
151
+ print("no such benchmark", file=sys.stderr)
152
+ return 2
153
+ try:
154
+ refs = pins.references(bench["_here"])
155
+ except (OSError, ValueError, SyntaxError):
156
+ print("not checked: cannot read image pins from images.json or driver.py", file=sys.stderr)
157
+ return 2
158
+ if not refs:
159
+ print("not checked: no image pins found", file=sys.stderr)
160
+ return 2
161
+ code = 0
162
+ for name, ref in refs[:a.limit]:
163
+ state, detail = pins.check(ref, a.tag)
164
+ print(f"{name}: {state}: {detail}", flush=True)
165
+ code = max(code, 2 if state == "not checked" else 0 if state == "same" else 1)
166
+ return code
167
+
168
+
169
+ def cmd_list(a):
170
+ got, bad = _all()
171
+ for kind in definitions.KINDS:
172
+ print(definitions.KINDS[kind]["dir"] + ":")
173
+ for d in got[kind]:
174
+ side = d.get("needs") if kind == "benchmark" else d.get("offers")
175
+ print(f" {d.get('_ref') or d['key']:<18} {('needs ' if kind == 'benchmark' else 'offers ') + ', '.join(side or []) or 'nothing proven yet'}")
176
+ up = d.get("upstream") or {}
177
+ if up.get("repository"):
178
+ print(f" {'':<18} upstream {up['repository'].removeprefix('https://')} @ "
179
+ + (f"{up['tag']} ({str(up.get('tested', ''))[:12]})" if up.get("tag") else str(up.get("tested", ""))[:12]))
180
+ if kind == "benchmark" and d.get("needs_skills"):
181
+ print(f" {'':<18} and the skills " + "; ".join(f"{', '.join(s)} of {p}" for p, s in d["needs_skills"].items()))
182
+ for note in ([] if kind == "benchmark" else older(d) + skills(d)):
183
+ print(f" {'':<18} {note}")
184
+ t = match.table(got["benchmark"], got["agent"], got["capability"])
185
+ print("pairs:")
186
+ for b, row in t.items():
187
+ print(f" {b}: " + ", ".join(f"{o} {'fits' if not why else 'no'}" for o, why in row.items()))
188
+ for b in bad:
189
+ print("problem:", b)
190
+ return 1 if bad else 0
191
+
192
+
193
+ def catalogue_problems(benches) -> list:
194
+ """A standard benchmark is listed by the catalogue only through its entry:
195
+ benchmark.json beside it, naming the standard among its harnesses, and the
196
+ whole catalogue consistent (nullius.catalogue.catalog.validate)."""
197
+ from nullius.catalogue import catalog
198
+ bad = []
199
+ for b in benches:
200
+ entry = os.path.join(b["_here"], "benchmark.json")
201
+ if not os.path.isfile(entry):
202
+ bad.append(f"benchmark {b['key']}: no benchmark.json beside standard.json (the catalogue's entry)")
203
+ continue
204
+ with open(entry) as f:
205
+ if "standard" not in (json.load(f).get("harness") or []):
206
+ bad.append(f"benchmark {b['key']}: benchmark.json does not name 'standard' in harness")
207
+ # each benchmark's settings inventory, against every version of it (definitions.settings_problems)
208
+ every = definitions.load("benchmark", every=True)[0]
209
+ for key in dict.fromkeys(b["key"] for b in benches):
210
+ bad += [f"benchmark {key}: {p}" for p in definitions.settings_problems([v for v in every if v["key"] == key])]
211
+ bad += [f"catalogue: {p}" for p in catalog.validate(catalog.load())]
212
+ return bad
213
+
214
+
215
+ def cmd_check(a):
216
+ if a.path:
217
+ with open(a.path) as f:
218
+ d = definitions.names.definition(json.load(f))
219
+ kind = next((k for k, v in definitions.KINDS.items() if v["schema"] == d.get("schema")), None)
220
+ if not kind:
221
+ print(f"schema: one of {[v['schema'] for v in definitions.KINDS.values()]}")
222
+ return 1
223
+ bad = definitions.problems(dict(d, _here=os.path.dirname(os.path.abspath(a.path))), kind)
224
+ else:
225
+ got, bad = _all()
226
+ for k in definitions.kits():
227
+ bad += [f"kit {k['key']}: {p}" for p in definitions.problems(k, "benchmark")]
228
+ bad += catalogue_problems(got["benchmark"])
229
+ for b in bad:
230
+ print(b)
231
+ print("ok" if not bad else f"{len(bad)} problem(s)")
232
+ return 1 if bad else 0
233
+
234
+
235
+ def cmd_fit(a):
236
+ b = definitions.get("benchmark", a.benchmark)
237
+ if not b:
238
+ sys.exit(f"no benchmark {a.benchmark!r}")
239
+ print(f"{b['label']} ({b['family']} benchmark) needs: " + ", ".join(b["needs"]))
240
+ for p in b["needs"]:
241
+ print(f" {p}: {ports.PORTS[p]['summary']}")
242
+ have = definitions.port_skills(p)
243
+ for s in (b.get("needs_skills") or {}).get(p) or []:
244
+ print(f" and its {s} skill: {have.get(s, '')}")
245
+ others = definitions.load("agent" if b["family"] == "agent" else "capability", every=True)[0]
246
+ for o in others:
247
+ why = match.fit(b, o)
248
+ notes = [] if why else older(o, b["needs"]) + skills(o, b["needs"])
249
+ print(f" {'✓' if not why else '✗'} {o['_ref']}" + ("" if not why else ": " + "; ".join(why))
250
+ + ("" if not notes else " (" + "; ".join(notes) + ")"))
251
+ return 0
252
+
253
+
254
+ TEMPLATES = {
255
+ "benchmark": {
256
+ "standard.json": {
257
+ "schema": "nullius.benchmark/v1", "key": "<KEY>", "label": "<KEY>", "summary": "SAY WHAT ONE UNIT IS AND HOW IT IS SCORED.",
258
+ "family": "{family}", "needs": ["{port}"],
259
+ "upstream": {"repository": "https://github.com/OWNER/REPO", "tested": "FULL 40-CHARACTER COMMIT", "license": "LICENSE"},
260
+ "driver": {"command": ["python3", "{here}/driver.py"]},
261
+ "datasets": [{"id": "main", "label": "main", "path": "PATH/IN/THE/UPSTREAM", "sha256": "SHA256 OF THE FILE", "units": 1}],
262
+ "units": {"noun": "task", "nouns": "tasks"},
263
+ "score": {"kind": "pass", "says": "WHAT A REWARD OF 1 MEANS"},
264
+ },
265
+ "driver.py": '''"""{key}'s driver for the Nullius standard: JSON lines on stdin and stdout.
266
+
267
+ $NULLIUS_DATASET is the dataset file (checked by its sha256), $NULLIUS_UPSTREAM
268
+ the upstream at its tested commit. Each request is one line, {{"op": ...}};
269
+ each reply one line, a JSON object, or {{"error": "..."}}.
270
+ """
271
+ import json
272
+ import os
273
+ import sys
274
+
275
+
276
+ def units(dataset):
277
+ return {{"units": []}} # every unit's id, in order
278
+
279
+
280
+ def setup(unit, dir):
281
+ # fill dir, the unit's workspace; return the prompt (agent family) or
282
+ # {{"memory": {{"add": [...], "recall": [...]}}}} (a memory capability)
283
+ return {{"prompt": ""}}
284
+ {extra}
285
+
286
+ def score(unit, dir, transcript, answer):
287
+ return {{"reward": 0.0, "detail": {{}}}} # 1.0 is a pass; a score benchmark gives 0..1
288
+
289
+
290
+ HANDLERS = {{"units": units, "setup": setup, "score": score{handlers}}}
291
+ for line in sys.stdin:
292
+ try:
293
+ req = json.loads(line)
294
+ out = HANDLERS[req.pop("op")](**req)
295
+ except Exception as e:
296
+ out = {{"error": f"{{type(e).__name__}}: {{e}}"}}
297
+ print(json.dumps(out), flush=True)
298
+ ''',
299
+ },
300
+ "agent": {
301
+ "agent.json": {
302
+ "schema": "nullius.agent/v1", "key": "<KEY>", "label": "<KEY>", "summary": "SAY WHAT THE AGENT IS AND HOW IT IS DRIVEN.",
303
+ "upstream": {"package": "PACKAGE", "version": "VERSION", "license": "LICENSE"},
304
+ "install": {"npm": ["PACKAGE@VERSION"]},
305
+ "start": {"acp": ["{install}/node_modules/.bin/PROGRAM"]},
306
+ "model": {"apis": ["chat"], "env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "MODEL": "{model}"}},
307
+ "offers": [], "proven": {},
308
+ },
309
+ },
310
+ "capability": {
311
+ "capability.json": {
312
+ "schema": "nullius.capability/v1", "key": "<KEY>", "label": "<KEY>", "summary": "SAY WHAT THE CAPABILITY KEEPS AND WHAT IT GIVES BACK.",
313
+ "kind": "memory", "start": {"command": ["python3", "{here}/capability.py"]}, "offers": [], "proven": {},
314
+ },
315
+ "capability.py": '''"""{key}: cap.memory/v1 in JSON lines.
316
+
317
+ add {{session}} keep a session: {{id, date, turns: [{{role, text}}], text}}
318
+ recall {{question}} -> {{"memory": "what was kept that bears on the question"}}
319
+ """
320
+ import json
321
+ import sys
322
+
323
+ for line in sys.stdin:
324
+ req = json.loads(line)
325
+ try:
326
+ if req["op"] == "add":
327
+ out = {{}}
328
+ elif req["op"] == "recall":
329
+ out = {{"memory": ""}}
330
+ else:
331
+ out = {{"error": "unknown op " + req["op"]}}
332
+ except Exception as e:
333
+ out = {{"error": f"{{type(e).__name__}}: {{e}}"}}
334
+ print(json.dumps(out), flush=True)
335
+ ''',
336
+ },
337
+ }
338
+ PRODUCE = '''
339
+
340
+ def produce(unit, dir, recalled, model, base_url, key):
341
+ # a capability benchmark's own step: answer from what the capability recalled,
342
+ # calling the model at base_url (the relay) with key (a placeholder)
343
+ return {{"answer": ""}}
344
+ '''
345
+
346
+
347
+ def new_version(key: str, version: str) -> str:
348
+ """Add ``version`` to an agent's or a capability's file: a copy of its default
349
+ version's block with the old version replaced in its pins, nothing proven.
350
+ A file without ``versions`` is turned into one first, its one version kept
351
+ as it was, proofs included. Returns the file's path."""
352
+ if not definitions._KEY.match(key):
353
+ sys.exit(f"no agent, capability or benchmark {key!r}")
354
+ kind = next((k for k in ("agent", "capability", "benchmark")
355
+ if (folder := definitions.entry_dir(k, key))
356
+ and os.path.isfile(definitions.definition_path(folder, k))), None)
357
+ if not kind:
358
+ sys.exit(f"no agent, capability or benchmark {key!r}")
359
+ if not definitions._VERSION.match(version):
360
+ sys.exit("a version is letters, digits and . + _ -")
361
+ path = definitions.definition_path(definitions.entry_dir(kind, key), kind)
362
+ with open(path) as f:
363
+ d = definitions.names.definition(json.load(f))
364
+ if "versions" not in d:
365
+ up = dict(d.get("upstream") or {})
366
+ old = str(up.pop("version", "") or "")
367
+ if not old:
368
+ sys.exit(f"{path}: upstream.version is not set, so its one version has no name")
369
+ # a benchmark's version holds only its proofs at first: what differs in the new one is the user's to say
370
+ block = {"proven": d.pop("proven", {})} if kind == "benchmark" else {"install": d.pop("install", {}), "proven": d.pop("proven", {})}
371
+ d.pop("offers", None)
372
+ d["upstream"] = up
373
+ d["default"], d["versions"] = old, {old: block}
374
+ if version in d["versions"]:
375
+ sys.exit(f"{key} has a version {version} already")
376
+ old = d["default"]
377
+ base = d["versions"][old]
378
+ copy = json.loads(json.dumps({k: v for k, v in base.items() if k != "proven"}).replace(old, version))
379
+ d["versions"] = {version: dict(copy, proven={}), **d["versions"]}
380
+ path = os.path.join(definitions.writable_dir(kind, key), definitions.KINDS[kind]["file"])
381
+ with open(path, "w") as f:
382
+ json.dump(d, f, indent=2)
383
+ f.write("\n")
384
+ print(f"wrote {path}: {key}@{version} beside {', '.join(v for v in d['versions'] if v != version)} (default still {old})")
385
+ if kind == "benchmark":
386
+ print(f"next: say what differs in it (upstream, datasets, driver, score, limits, controls, environments), then: "
387
+ f"cli check, and cli prove {key}@{version} --limit 3; make it the default by setting \"default\": \"{version}\"")
388
+ return path
389
+ print(f"next: check its pins, then: cli check, and cli conform {key}@{version} --model gpt-5-mini"
390
+ + (" --unsandboxed" if kind == "agent" else "") + f"; make it the default by setting \"default\": \"{version}\"")
391
+ return path
392
+
393
+
394
+ def cmd_new(a):
395
+ """A definition to fill in, in the catalogue folder it belongs in."""
396
+ if a.kind == "version":
397
+ if not a.version:
398
+ sys.exit("new version KEY VERSION")
399
+ new_version(a.key, a.version)
400
+ return 0
401
+ kind, key = a.kind, a.key
402
+ if not definitions._KEY.match(key):
403
+ sys.exit("a key is lower case letters, digits and dashes")
404
+ name = definitions.KINDS[kind]["file"]
405
+ selected_dir = definitions.entry_dir(kind, key)
406
+ selected = os.path.join(selected_dir, name) if selected_dir else None
407
+ if selected and os.path.exists(selected):
408
+ sys.exit(f"{selected} exists already")
409
+ folder = definitions.writable_dir(kind, key)
410
+ os.makedirs(folder, exist_ok=True)
411
+ family = a.family or "agent"
412
+ port = a.port or ("agent.task/v1" if family == "agent" else "cap.memory/v1")
413
+ for fname, body in TEMPLATES[kind].items():
414
+ if isinstance(body, dict):
415
+ text = json.dumps(body, indent=2).replace('"<KEY>"', json.dumps(key)).replace('"{family}"', json.dumps(family)) \
416
+ .replace('"{port}"', json.dumps(port))
417
+ else:
418
+ feat = family == "capability"
419
+ text = body.format(key=key, extra=PRODUCE.format() if feat else "", handlers=', "produce": produce' if feat else "")
420
+ path = os.path.join(folder, fname)
421
+ if os.path.exists(path):
422
+ print("kept", path)
423
+ continue
424
+ with open(path, "w") as f:
425
+ f.write(text + ("\n" if not text.endswith("\n") else ""))
426
+ print("wrote", path)
427
+ if kind == "benchmark":
428
+ then = f"cli run {key} reference --model gpt-5-mini --limit 2 --unsandboxed"
429
+ else:
430
+ then = f"cli conform {key} --model gpt-5-mini" + (" --unsandboxed" if kind == "agent" else "")
431
+ print(f"next: fill in the CAPITALS{', add benchmark.json beside it (the catalogue entry)' if kind == 'benchmark' else ''}, "
432
+ f"then: cli check, and {then}")
433
+ return 0
434
+
435
+
436
+ def options(pairs) -> dict:
437
+ """``--option name=value``, repeatable, as {name: value}."""
438
+ out = {}
439
+ for p in pairs or []:
440
+ name, eq, value = p.partition("=")
441
+ if not eq or not name:
442
+ sys.exit(f"--option {p!r}: NAME=VALUE")
443
+ out[name.strip()] = value.strip()
444
+ return out
445
+
446
+
447
+ def prices(given) -> dict:
448
+ """``--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]``, repeatable: this run's own prices, over the user's list."""
449
+ out = {}
450
+ for p in given or []:
451
+ try:
452
+ name, row = price_list.parse(p)
453
+ except ValueError as e:
454
+ sys.exit(str(e))
455
+ out[name] = row
456
+ return out
457
+
458
+
459
+ def cmd_conform(a):
460
+ o = _other(a.other)
461
+ try:
462
+ got = conform.check(o, model=a.model, provider=a.provider, which=a.port or None, jobs_root=a.jobs, say=print,
463
+ unsandboxed=a.unsandboxed, stamp=a.stamp, backend=a.backend, env=run_env(a.provider) if a.from_run else None,
464
+ options=options(a.option), repeat=a.repeat)
465
+ except run.Refused as e:
466
+ print("refused:", e)
467
+ return 2
468
+ return 0 if all(r["ok"] for r in got.values()) else 1
469
+
470
+
471
+ def _bench(key):
472
+ """A benchmark of the catalogue, or a conformance kit by its key."""
473
+ b = definitions.get("benchmark", key) or next((k for k in definitions.kits() if k["key"] == key), None)
474
+ if not b:
475
+ sys.exit(f"no benchmark {key!r}")
476
+ return b
477
+
478
+
479
+ def cmd_run(a):
480
+ b = _bench(a.benchmark)
481
+ if a.other in definitions.CONTROLS:
482
+ other = definitions.control(b, a.other)
483
+ if other is None:
484
+ sys.exit(f"{b['label']} declares no {a.other} control (controls in its standard.json)")
485
+ elif a.other in definitions.SETUPS:
486
+ other, why = definitions.setup(b, a.other, a.model or "", a.provider)
487
+ if other is None:
488
+ print("refused:", why)
489
+ return 2
490
+ elif not a.model:
491
+ sys.exit("--model: the model the agent or the capability is run with")
492
+ else:
493
+ other = _other(a.other)
494
+ held, spill = licences(b)
495
+ try:
496
+ job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit,
497
+ concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend,
498
+ simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch,
499
+ licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images,
500
+ prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit,
501
+ **({"env": run_env(a.provider)} if a.from_run else {}))
502
+ except run.Refused as e:
503
+ print("refused:", e)
504
+ return 2
505
+ finally:
506
+ if spill:
507
+ shutil.rmtree(spill, ignore_errors=True)
508
+ print(job)
509
+ # no unit reached a verdict (a refusing endpoint, every unit an error): the run failed, and says so
510
+ # by its exit status, so a launch of several agents pauses rather than running the next into the same error.
511
+ # A run its spending cap stopped did what was asked of it: no failure
512
+ return 3 if failed else 0
513
+
514
+
515
+ def _shard(spec):
516
+ if not spec:
517
+ return None
518
+ try:
519
+ return shard.parse(spec)
520
+ except ValueError as e:
521
+ sys.exit(str(e))
522
+
523
+
524
+ def cmd_traces(a):
525
+ said = {"full": "with model calls and tool calls", "partial": "without tool times: the trial kept no ACP events",
526
+ "empty": "nothing recorded", "control": "controls, with nothing to show", "has_trace": "already have a trace, left as they are",
527
+ "not_standard": "not standard runs", "unreadable": "could not be read"}
528
+ counts = {}
529
+ for trial in job.trial_dirs(a.root):
530
+ try:
531
+ got = job.backfill_trace(trial, write=not a.dry_run)
532
+ except (OSError, ValueError, KeyError, TypeError, AttributeError) as e:
533
+ got = "unreadable"
534
+ print(f"{trial}: {e}", file=sys.stderr)
535
+ counts[got] = counts.get(got, 0) + 1
536
+ made = counts.get("full", 0) + counts.get("partial", 0)
537
+ print(f"{'would write' if a.dry_run else 'wrote'} {made} traces")
538
+ for name in ("full", "partial", "empty", "control", "has_trace", "not_standard", "unreadable"):
539
+ if counts.get(name):
540
+ print(f" {counts[name]:>5} {said[name]}")
541
+ return 1 if counts.get("unreadable") else 0
542
+
543
+
544
+ def cmd_merge(a):
545
+ try:
546
+ got = merge.merge(a.jobs, a.out, force=a.force)
547
+ except merge.Refused as e:
548
+ print("refused:", e)
549
+ return 2
550
+ print(f"{got['out']}: {got['trials']} trials from {len(got['merged_from'])} shards")
551
+ for line in got["notes"]:
552
+ print(" " + line)
553
+ return 0
554
+
555
+
556
+ def cmd_fidelity(a):
557
+ o = definitions.get("agent", a.other)
558
+ if not o:
559
+ sys.exit(f"no agent {a.other!r}: {[d['_ref'] for d in definitions.load('agent', every=True)[0]]}")
560
+ try:
561
+ got = fidelity.check(o, model=a.model, provider=a.provider, jobs_root=a.jobs, passes=a.passes, say=print,
562
+ unsandboxed=a.unsandboxed, options=options(a.option),
563
+ env=run_env(a.provider) if a.from_run else None)
564
+ except run.Refused as e:
565
+ print("refused:", e)
566
+ return 2
567
+ return 0 if got["verdict"] == "same" else 1
568
+
569
+
570
+ def cmd_prove(a):
571
+ b = _bench(a.benchmark)
572
+ held, spill = licences(b)
573
+ try:
574
+ got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model,
575
+ stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch,
576
+ env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images)
577
+ except run.Refused as e:
578
+ print("refused:", e)
579
+ return 2
580
+ finally:
581
+ if spill:
582
+ shutil.rmtree(spill, ignore_errors=True)
583
+ return 0 if all(r["ok"] for checks in got.values() for r in checks.values()) else 1
584
+
585
+
586
+ def _exit_on_term():
587
+ """Stop sends SIGTERM, and its default ends the process where it stands:
588
+ Run.start's cleanup never ran, so a docker run's unit and relay
589
+ containers (the relay holding the key) kept running and the job had no
590
+ result.json. The first TERM is an exit, which runs every ``finally``; a
591
+ second one (Stop signals the group, then the process) is ignored so it
592
+ cannot cut that cleanup short."""
593
+ def stopped(signum, frame):
594
+ signal.signal(signal.SIGTERM, signal.SIG_IGN)
595
+ raise SystemExit(128 + signum)
596
+ signal.signal(signal.SIGTERM, stopped)
597
+
598
+
599
+ def main(argv=None):
600
+ ap = argparse.ArgumentParser(prog="nullius.standard.cli", description=__doc__.split("\n\n")[0])
601
+ sub = ap.add_subparsers(dest="cmd", required=True)
602
+ sub.add_parser("list", help="every definition, and which pairs fit")
603
+ c = sub.add_parser("check", help="check the definitions, or one file")
604
+ c.add_argument("path", nargs="?")
605
+ f = sub.add_parser("fit", help="what can run a benchmark, and why the rest cannot")
606
+ f.add_argument("benchmark")
607
+ p = sub.add_parser("pins", help="compare image pins with registry tags without changing them")
608
+ p.add_argument("benchmark")
609
+ p.add_argument("--tag", help="tag to compare (default: embedded tag, or latest)")
610
+ p.add_argument("--limit", type=int, help="check only the first N pins")
611
+ n = sub.add_parser("new", help="a definition to fill in, in its catalogue folder")
612
+ n.add_argument("kind", type=definitions.names.kind, choices=[*definitions.KINDS, "version"])
613
+ n.add_argument("key")
614
+ n.add_argument("version", nargs="?", help="new version: the version to add")
615
+ n.add_argument("--family", type=definitions.names.kind, choices=list(definitions.FAMILIES), help="a benchmark's family (agent by default)")
616
+ n.add_argument("--port", type=definitions.names.port, help="the port a new benchmark needs")
617
+ helps = {"conform": "run the kits, record what passed", "run": "run a benchmark into a job",
618
+ "fidelity": "drive an agent over ACP and through its own command: does its model get the same requests?"}
619
+ for name in ("conform", "run", "fidelity"):
620
+ p = sub.add_parser(name, help=helps[name])
621
+ if name == "run":
622
+ p.add_argument("benchmark")
623
+ p.add_argument("other", help="an agent or a capability: KEY (its default version) or KEY@VERSION"
624
+ + ("; or nop, oracle: a control the benchmark declares; or original, model: one of its setups" if name == "run" else ""))
625
+ p.add_argument("--model", required=name == "conform",
626
+ help="the model (a control needs none; original takes one of the paper's, its first by default)" if name == "run" else None)
627
+ p.add_argument("--provider", default="openai", choices=sorted(run.PROVIDERS))
628
+ p.add_argument("--jobs", default="jobs_standard", help="where jobs are written (default jobs_standard/, gitignored)")
629
+ p.add_argument("--unsandboxed", action="store_true",
630
+ help="allow an agent, or a setup check without Docker, to execute code on this machine as you")
631
+ p.add_argument("--backend", default="local", type=definitions.names.backend, choices=list(run.BACKENDS),
632
+ help="local processes (the default), or container: each unit in a container, its relay beside it "
633
+ "(docker: its earlier name)")
634
+ p.add_argument("--from-run", action="store_true",
635
+ help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read")
636
+ p.add_argument("--option", action="append", metavar="NAME=VALUE",
637
+ help="one of the agent's or capability's options (repeatable); the rest keep their defaults")
638
+ if name == "fidelity":
639
+ p.add_argument("--passes", type=int, default=2, help="runs of each way (default 2: the second shows the noise)")
640
+ elif name == "conform":
641
+ p.add_argument("--port", type=definitions.names.port, action="append", help="only this port's kit (repeatable)")
642
+ p.add_argument("--stamp", help="the time stamp the kit jobs are named with (default: now)")
643
+ p.add_argument("--repeat", type=int, default=1, metavar="K",
644
+ help="run each check K times; a check passes when it passes in most of them (default 1)")
645
+ else:
646
+ p.add_argument("--dataset")
647
+ p.add_argument("--limit", type=int)
648
+ p.add_argument("-j", "--concurrency", type=int, default=4)
649
+ p.add_argument("--judge-model", help="a benchmark scored by a model: this judge instead of its official one")
650
+ p.add_argument("--simulator-model", help="a benchmark whose user a model plays: this model instead of its official one")
651
+ p.add_argument("--repeat", type=int, default=1, metavar="K",
652
+ help="run each unit K times, each an attempt of its own (the upstream's --num-trials, pass^k); default 1")
653
+ p.add_argument("--remove-images", action="store_true",
654
+ help="remove each image this run pulls or builds when its last unit ends (default: kept for the next run)")
655
+ p.add_argument("--name")
656
+ p.add_argument("--shard", metavar="I/N", help="only shard I of N of the units: by the unit id's hash, before --limit")
657
+ p.add_argument("--launch", help="the launch id every shard records (default: the day, the benchmark, the agent, the model)")
658
+ p.add_argument("--bench-option", action="append", metavar="NAME=VALUE",
659
+ help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults")
660
+ p.add_argument("--price", action="append", metavar="MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]",
661
+ help="this run's price for a model in USD per million tokens, over your price list (repeatable; nullius prices)")
662
+ p.add_argument("--max-usd", type=float, metavar="USD",
663
+ help="the run's spending cap: no call goes out once the spend plus the calls in flight would pass it")
664
+ p.add_argument("--max-usd-per-unit", type=float, metavar="USD",
665
+ help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on")
666
+ v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed")
667
+ v.add_argument("benchmark")
668
+ v.add_argument("--dataset", help="one dataset (default: each)")
669
+ v.add_argument("--limit", type=int, help="the first N units of each dataset (the record says so)")
670
+ v.add_argument("--provider", default="openai", choices=sorted(run.PROVIDERS), help="a judged benchmark's: the judge's provider")
671
+ v.add_argument("--judge-model", help="a judged benchmark: this judge instead of its official one")
672
+ v.add_argument("-j", "--concurrency", type=int, default=4)
673
+ v.add_argument("--jobs", default="jobs_standard", help="where the control jobs are written (default jobs_standard/)")
674
+ v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)")
675
+ v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only")
676
+ v.add_argument("--launch", help="the launch id the control jobs of every shard record")
677
+ v.add_argument("--remove-images", action="store_true",
678
+ help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)")
679
+ g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job")
680
+ g.add_argument("jobs", nargs="+", metavar="JOB_DIR")
681
+ g.add_argument("-o", "--out", required=True, help="the merged job's directory (its name is the job's)")
682
+ g.add_argument("--force", action="store_true", help="merge shards of different launches, versions, models or agents")
683
+ tr = sub.add_parser("traces", help="give past standard trials a trace, from the files they kept")
684
+ tr.add_argument("root", metavar="DIR", help="a job, or a folder of jobs")
685
+ tr.add_argument("--dry-run", action="store_true", help="say what it would write, and write nothing")
686
+ v.add_argument("--from-run", action="store_true",
687
+ help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read")
688
+ # Bootstrap decides whether settings may be read before these modules are
689
+ # imported. Keep --from-run exact here too, so --from-r cannot bypass it.
690
+ for parser in (ap, *sub.choices.values()):
691
+ parser.allow_abbrev = False
692
+ a = ap.parse_args(argv)
693
+ if a.cmd in ("run", "prove", "conform", "fidelity"):
694
+ _exit_on_term()
695
+ progress.ECHO = sys.stderr # each stage as a line (nullius/progress.py): stdout stays the job and its units
696
+ if not getattr(a, "from_run", False):
697
+ settings.load(ENV_FILE)
698
+ return {"list": cmd_list, "check": cmd_check, "fit": cmd_fit, "pins": cmd_pins, "new": cmd_new, "conform": cmd_conform, "run": cmd_run,
699
+ "fidelity": cmd_fidelity, "prove": cmd_prove, "merge": cmd_merge, "traces": cmd_traces}[a.cmd](a)
700
+
701
+
702
+ if __name__ == "__main__":
703
+ sys.exit(main())