nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,1651 @@
1
+ """The three definitions, where they live, and what each may say.
2
+
3
+ - a **benchmark**: ``catalogue/benchmarks/<key>/standard.json``
4
+ (``nullius.benchmark/v1``) beside the catalogue's own ``benchmark.json``;
5
+ - an **agent**: ``catalogue/agents/<key>/agent.json`` (``nullius.agent/v1``);
6
+ - a **capability**: ``catalogue/capabilities/<key>/capability.json`` (``nullius.capability/v1``).
7
+
8
+ An agent or a capability may hold several **versions** in its one file:
9
+ ``versions`` maps each version to what differs in it - its ``upstream``
10
+ (package, licence), ``install``, and optionally ``start``, ``model``, ``env``,
11
+ ``options`` (added to the shared ones), ``fidelity`` - and its own ``proven``; ``default`` names the one offered when none is named.
12
+ What every version shares stays at the top. A version is named
13
+ ``<key>@<version>`` (:func:`split`), and :func:`resolve` turns the file into
14
+ one version's definition, as flat as a file without ``versions``: everything
15
+ downstream - the matcher, a run, a kit - sees one version at a time, and a
16
+ version offers exactly the ports it has proven. A file without ``versions``
17
+ is one version, its ``upstream.version``.
18
+
19
+ The conformance kits are benchmarks too, kept inside this package
20
+ (``standard/kits/<name>/standard.json``), so a kit is run the way any
21
+ benchmark is.
22
+
23
+ A benchmark may name its **controls** (``controls``): ``nop``, the null run
24
+ that does nothing and answers "", and ``oracle``, the reference run whose
25
+ answer (and, for a workspace, files) the driver's ``oracle`` op gives. Each
26
+ says who wrote it - ``source``, one of ``SOURCES`` - and where it comes from
27
+ (``from``, a sentence), and may hold it to a bound (``expect``). A control
28
+ is run as the other side of a pair (:func:`control`), with no agent, no
29
+ capability and no model of its own; ``nop`` and ``oracle`` are the words Harbor
30
+ calls its controls by, so the runs pages read the jobs as controls. A
31
+ benchmark's ``proven`` is what ``cli prove`` found, per dataset
32
+ (``standard/prove.py``).
33
+
34
+ A benchmark's **setups** are driver methods behind one Original procedure.
35
+ ``original`` uses the upstream's model selection; ``model`` substitutes the
36
+ chosen model while preserving the ``runs_as`` window and prompt. The driver's
37
+ ``answer`` op uses the unit's relay and may declare conversation-sized ``units``.
38
+ The builder chooses participant and model separately; the resolved setup's
39
+ ``comparable`` flag records whether the run can compare with the paper.
40
+
41
+ A benchmark may hold **versions** the same way (``key@version``): each
42
+ says what differs in it - ``upstream``, ``datasets``, ``driver``, ``score``,
43
+ ``limits``, ``controls``, ``environments`` - and its own ``proven``
44
+ (``BENCH_VERSIONED``). A file without ``versions`` is one version, as an
45
+ agent's is; :func:`load` and :func:`get` resolve a benchmark as they resolve
46
+ an agent.
47
+
48
+ A benchmark may give its units **environments** (``environments``): named
49
+ Dockerfiles in its folder, each a layer on the standard's base image (the
50
+ Dockerfile has no ``FROM``: Nullius writes it), with the files it copies
51
+ listed (``files``), the licences it needs (``needs_secrets``: each a
52
+ ``value`` - a key, given as a variable -, a ``file`` - mounted read-only at
53
+ its ``path`` -, or a ``server`` - the one ``host:port`` the agent may reach -,
54
+ kept per benchmark by ``nullius.secrets``, and where it goes - ``agent`` or
55
+ ``driver``) and whether the local backend may run it (``local_ok``). A unit's setup names its
56
+ environment, or its dataset gives a default; none is the base image
57
+ (:func:`environment_of`). ``container.image_for`` builds and picks the image.
58
+ An environment may instead be an **own image** (``"image": "own"``): no
59
+ Dockerfile here, each unit's setup naming the task's own image - a folder
60
+ of the upstream to build, its ``FROM`` its own, or a reference by digest -
61
+ to which the agent is added at run time (``container.Box``).
62
+
63
+ A definition is a bundled file or a file in the user's catalogue, never a form field, and
64
+ :func:`problems` checks it before anything offers it: command words from a
65
+ safe alphabet, ports that exist, an https upstream at a full commit, every
66
+ dataset with its sha256. The placeholders a command may hold are named here
67
+ (``SLOTS``) and filled by the runner, quoted, never by a shell.
68
+
69
+ An agent or a capability may have **options** (``options``): what a run may set
70
+ of its own behaviour, each with its allowed values and a default that is the
71
+ vendor's own, filled into its templates as ``{opt_<name>}`` (:func:`options_of`).
72
+ An agent may name its own non-interactive command (``fidelity.command``), which
73
+ ``fidelity`` drives beside ACP to show both send the model the same requests.
74
+ """
75
+ from __future__ import annotations
76
+
77
+ import contextlib
78
+ import hashlib
79
+ import json
80
+ import os
81
+ import re
82
+ from typing import Callable, Dict, List, Optional, Tuple
83
+
84
+ from nullius.standard import ports
85
+ from nullius import paths
86
+ from nullius.catalogue import storage
87
+ from . import names
88
+
89
+ HERE = os.path.dirname(os.path.abspath(__file__))
90
+ CATALOGUE = os.path.join(os.path.dirname(HERE), "catalogue")
91
+ KITS = os.path.join(HERE, "kits")
92
+
93
+ KINDS = {
94
+ "benchmark": {"schema": "nullius.benchmark/v1", "dir": "benchmarks", "file": "standard.json"},
95
+ "agent": {"schema": "nullius.agent/v1", "dir": "agents", "file": "agent.json"},
96
+ "capability": {"schema": "nullius.capability/v1", "dir": "capabilities", "file": "capability.json"},
97
+ }
98
+
99
+ # What a command word may be: a literal from a safe alphabet, or one slot.
100
+ # {here} is the definition's folder, {install} the folder its packages were
101
+ # installed into, {prompt} the unit's prompt (a command-template agent only).
102
+ SLOTS = {"here", "install", "prompt"}
103
+ _WORD = re.compile(r"^(?:[A-Za-z0-9_./:=@+-]|\{[a-z]+\})+$")
104
+ _SLOT = re.compile(r"\{([a-z]+)\}")
105
+ _KEY = re.compile(r"^[a-z][a-z0-9-]{0,39}$")
106
+ _ID = re.compile(r"^[A-Za-z0-9][A-Za-z0-9_.-]{0,63}$")
107
+ _TAG = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._+/-]{0,127}$")
108
+ _COMMIT = re.compile(r"^[0-9a-f]{40}$")
109
+ _SHA256 = re.compile(r"^[0-9a-f]{64}$")
110
+ _HTTPS = re.compile(r"^https://[A-Za-z0-9.-]+(?:/[A-Za-z0-9_.%@+-]+)+$")
111
+ _RELATIVE = re.compile(r"^[A-Za-z0-9_-][A-Za-z0-9_.-]*(?:/[A-Za-z0-9_-][A-Za-z0-9_.-]*)*$")
112
+ _NPM = re.compile(r"^(?:@[a-z0-9][a-z0-9._-]*/)?[a-z0-9][a-z0-9._-]*@[0-9][A-Za-z0-9.+-]*$")
113
+ _PIP = re.compile(r"^[A-Za-z0-9_.-]+==[A-Za-z0-9_.+!-]+$")
114
+ _PIP_GIT = re.compile(r"[A-Za-z0-9_.-]+ @ git\+https://[A-Za-z0-9.-]+/(?:[A-Za-z0-9_.-]+/)*[A-Za-z0-9_.-]+@[0-9a-fA-F]{40}")
115
+ FAMILIES = ("agent", "capability")
116
+ _MODEL_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$")
117
+ APIS = ("chat", "responses", "messages")
118
+ SCORES = ("pass", "score")
119
+ # A benchmark's controls, by the words Harbor calls them (``runs.parse.CONTROL_AGENTS``),
120
+ # and who wrote each: the benchmark's authors, a harness adapter's author, or Nullius.
121
+ CONTROLS = ("nop", "oracle")
122
+ SOURCES = ("official", "adapter", "nullius")
123
+ CONTROL_LABEL = {"nop": "Null control", "oracle": "Reference control"}
124
+ # A benchmark's setups (``setups``): the published benchmark as its authors ran it, and the same pipeline with another model.
125
+ SETUPS = ("original", "model")
126
+
127
+
128
+ def setup_label(bench, which):
129
+ """The catalogue has one upstream procedure; model choice is independent."""
130
+ return "Original"
131
+
132
+
133
+ # The bound a control is held to unless it says otherwise: the null run reaches nothing, the reference everything.
134
+ EXPECT = {"nop": ("at_most", 0.0), "oracle": ("at_least", 1.0)}
135
+ # How an agent is told where its model is - the relay - in its definition's
136
+ # own terms: variables of its environment, and files written into the home
137
+ # it is given for the run, each a template with these slots.
138
+ MODEL_SLOTS = {"base_url", "key", "model", "home"}
139
+ _SEG = re.compile(r"^[A-Za-z0-9_.-]{1,64}$") # a segment of a path in the agent's own home: a dot folder is fine
140
+ _ENV_NAME = re.compile(r"^[A-Z_][A-Z0-9_]{0,63}$")
141
+ _TEMPLATE = re.compile(r"^(?:[A-Za-z0-9_./:=@+ -]|\{[a-z_]+\})*$")
142
+ _VERSION = re.compile(r"^[0-9A-Za-z][0-9A-Za-z.+_-]{0,39}$")
143
+ # What a version may say for itself; the rest of the file is shared.
144
+ VERSIONED = ("upstream", "install", "start", "model", "env", "options", "caches", "fidelity", "proven")
145
+ # What a benchmark's version may say for itself: its data, its driver, how it scores, who plays its user, its environments.
146
+ BENCH_VERSIONED = ("upstream", "datasets", "files", "driver", "mcp_servers", "score", "simulator", "limits", "controls", "environments",
147
+ "options", "host", "setups", "proven", "leaderboard")
148
+ # What a benchmark option may need: embedding and reranker models through the relay,
149
+ # machine requirements (host), and digest-pinned auxiliary files.
150
+ NEED_KEYS = ("embeddings", "reranker", "host", "files")
151
+ _PROGRAM = re.compile(r"^[a-z0-9][a-z0-9._+-]{0,39}$")
152
+ _PACKAGE = re.compile(r"^[@a-z0-9][a-z0-9@/._+-]{0,99}$")
153
+ HOST_INSTALLERS = ("apt", "npm")
154
+ # A benchmark's environment (``environments``): its name, what it may say, and where a secret it needs may go -
155
+ # into the agent's container (``agent``) or the driver's process, which scores (``driver``).
156
+ _ENV_KEY = re.compile(r"^[a-z][a-z0-9-]{0,29}$")
157
+ ENV_FIELDS = ("summary", "dockerfile", "files", "needs_secrets", "local_ok", "image", "network")
158
+ OWN_IMAGE = "own" # an environment whose units each bring their own image (``image``)
159
+ SECRET_TO = ("agent", "driver")
160
+ # A licence's shape (``kind``): a key the process gets as a variable, a file mounted read-only where the
161
+ # software looks for it, or a licence server - the one address the agent's container may reach.
162
+ SECRET_KINDS = ("value", "file", "server")
163
+ SECRET_FIELDS = ("name", "to", "summary", "kind", "path", "env", "provider")
164
+ _ABS = re.compile(r"^(?:/[A-Za-z0-9_@+-][A-Za-z0-9_.@+-]*)+$")
165
+ _NOT_UNDER = ("/proc", "/sys", "/dev") # where a licence file is never mounted: the kernel's own
166
+ BASE_ENV = "base" # the name no environment may take: the base image itself
167
+ # An option of an agent or a capability (``options``): a name a template may
168
+ # carry as ``{opt_<name>}``, its values a plain word each - they are written
169
+ # into variables and config files, so never a quote, a space or a newline.
170
+ _OPTION = re.compile(r"^[a-z][a-z_]{0,39}$")
171
+ _OPTION_VALUE = re.compile(r"^[A-Za-z0-9_.:@+-]{1,64}$")
172
+ _OPT_SLOT = re.compile(r"\{opt_([a-z_]+)\}")
173
+
174
+
175
+ def split(ref: str) -> Tuple[str, Optional[str]]:
176
+ """``codex@2.2.2`` -> (``codex``, ``2.2.2``); a bare key -> (key, None)."""
177
+ key, _, version = str(ref or "").partition("@")
178
+ return key, version or None
179
+
180
+
181
+ def versions(d: dict) -> List[str]:
182
+ """Its versions, in the file's order (a file without ``versions``: its one)."""
183
+ vs = d.get("versions")
184
+ if isinstance(vs, dict) and vs:
185
+ return list(vs)
186
+ return [str((d.get("upstream") or {}).get("version") or "")]
187
+
188
+
189
+ def resolve(d: dict, version: Optional[str] = None) -> Optional[dict]:
190
+ """One version of ``d`` as a flat definition: the shared fields, then the
191
+ version's own (its ``upstream`` over the shared one, ``upstream.version``
192
+ set), ``proven`` its own and ``offers`` the ports it proved. It carries
193
+ ``_version``, ``_versions``, ``_default`` and ``_ref`` (what names it).
194
+ None for a version the file does not have."""
195
+ vs = d.get("versions")
196
+ if not isinstance(vs, dict) or not vs:
197
+ v = versions(d)[0]
198
+ if version not in (None, v):
199
+ return None
200
+ return dict(d, _version=v, _versions=[v], _default=v, _ref=d.get("key"))
201
+ v = version or d.get("default")
202
+ if v not in vs or not isinstance(vs[v], dict):
203
+ return None
204
+ block = vs[v]
205
+ bench = d.get("schema") == KINDS["benchmark"]["schema"]
206
+ out = {k: x for k, x in d.items() if k not in ("versions", "default")}
207
+ for k in (BENCH_VERSIONED if bench else VERSIONED):
208
+ if k in block:
209
+ out[k] = block[k]
210
+ if isinstance(d.get("options"), dict) and isinstance(block.get("options"), dict):
211
+ out["options"] = dict(d["options"], **block["options"]) # a version adds an option or restates one
212
+ up = dict(d.get("upstream") or {})
213
+ own = block.get("upstream") or {}
214
+ if ("repository" in own or "tested" in own) and "tag" not in own:
215
+ up.pop("tag", None) # another repository or commit: the shared tag is not its
216
+ out["upstream"] = dict(up, **own, version=v)
217
+ out["proven"] = dict(block.get("proven") or {})
218
+ if not bench: # a benchmark offers nothing: it needs
219
+ out["offers"] = [p for p in ports.PORTS if p in out["proven"]]
220
+ return dict(out, _version=v, _versions=list(vs), _default=d.get("default"), _ref=f"{d.get('key')}@{v}")
221
+
222
+
223
+ def title(d: dict) -> str:
224
+ """Its label, with the version when the file has ``versions`` (``Codex 2.2.2``)."""
225
+ return f"{d['label']} {d['_version']}" if "@" in str(d.get("_ref") or "") else d["label"]
226
+
227
+
228
+ def ref_of(d: dict) -> str:
229
+ """What names a resolved definition where a bare key means its default
230
+ version: its key for the default, ``key@version`` for any other."""
231
+ return d["key"] if d.get("_version") == d.get("_default") else (d.get("_ref") or d["key"])
232
+
233
+
234
+ def version_of(d: dict) -> Optional[str]:
235
+ """The version a resolved definition is when its file has ``versions``; else None."""
236
+ return d.get("_version") if "@" in str(d.get("_ref") or "") else None
237
+
238
+
239
+ def find(others: List[dict], ref: str) -> Optional[dict]:
240
+ """Among resolved definitions, the one ``ref`` names: ``key@version``, or a
241
+ bare key for its default version."""
242
+ return next((o for o in others if o.get("_ref") == ref), None) or \
243
+ next((o for o in others if o["key"] == ref and o.get("_version") == o.get("_default")), None)
244
+
245
+
246
+ def job_word(d: dict) -> str:
247
+ """The resolved definition as a word of a job's name: its key, and its
248
+ version when the file has ``versions`` (``codex-2.2.2``)."""
249
+ return f"{d['key']}-{d['_version']}" if "@" in str(d.get("_ref") or "") else d["key"]
250
+
251
+
252
+ def _in_home(path) -> bool:
253
+ """``{home}/a/b``: a path inside the home a run gives, never the home itself nor a way out of it."""
254
+ rel = path[len("{home}/"):] if isinstance(path, str) and path.startswith("{home}/") else None
255
+ segs = rel.split("/") if rel else []
256
+ return bool(segs) and all(_SEG.match(x) and x not in (".", "..") for x in segs)
257
+
258
+
259
+ def _caches(d: dict) -> List[str]:
260
+ """``caches``: what the agent or capability fills its home with and no one
261
+ reads after the unit (a package cache, a synced repository) - removed
262
+ when the unit ends, so a job keeps its evidence and not a copy of the
263
+ same download per unit. Paths under ``{home}/``, nowhere else."""
264
+ c = d.get("caches")
265
+ if c is None:
266
+ return []
267
+ if not isinstance(c, list) or not all(_in_home(x) for x in c):
268
+ return ["caches: a list of paths under {home}/ (the agent's own home, never outside it)"]
269
+ return []
270
+
271
+
272
+ def _model(m: dict, here: Optional[str]) -> List[str]:
273
+ bad = []
274
+ if not m.get("apis") or any(a not in APIS for a in m.get("apis") or []):
275
+ bad.append(f"model.apis: some of {APIS}")
276
+ env, files = m.get("env") or {}, m.get("files") or {}
277
+ if not env and not files:
278
+ bad.append("model: env or files, so the agent can be pointed at the relay")
279
+ for k, v in env.items():
280
+ if not _ENV_NAME.match(k) or not isinstance(v, str) or not _TEMPLATE.match(v):
281
+ bad.append(f"model.env.{k}: a variable name and a plain template")
282
+ bad += [f"model.env.{k}: {{{s}}} is not a slot" for s in _SLOT.findall(str(v)) if s not in MODEL_SLOTS]
283
+ for dest, src in files.items():
284
+ if not _in_home(dest):
285
+ bad.append(f"model.files: {dest!r} is a path under {{home}}/")
286
+ if not isinstance(src, str) or not _RELATIVE.match(src):
287
+ bad.append(f"model.files.{dest}: a template file beside the definition")
288
+ elif here and not os.path.isfile(os.path.join(here, src)):
289
+ bad.append(f"model.files.{dest}: {src} is not beside the definition")
290
+ side = m.get("side_models")
291
+ if side is not None and (not isinstance(side, list) or len(set(map(str, side))) != len(side)
292
+ or not all(isinstance(x, str) and _MODEL_NAME.match(x) for x in side)):
293
+ bad.append("model.side_models: a list of model names, each once")
294
+ return bad
295
+
296
+
297
+ _PYTHON = re.compile(r"^3\.\d{1,2}$")
298
+ _EXTRA = re.compile(r"^[A-Za-z0-9_-]{1,40}$")
299
+ _PKG = re.compile(r"^[A-Za-z0-9_.-]{1,80}(?:==[A-Za-z0-9_.+!-]{1,40})?$") # a name, or a name pinned
300
+ _HF = re.compile(r"^[A-Za-z0-9_.-]{1,96}/[A-Za-z0-9_.-]{1,96}$")
301
+ # The slots a definition's own variables (``env``) may hold.
302
+ ENV_SLOTS = {"install", "home", "here"}
303
+
304
+
305
+ def _install(d: dict, where: str = "install") -> List[str]:
306
+ """``install``: packages pinned (``npm``, ``pip``), or the definition's own
307
+ upstream as a Python project (``project``, a folder of the repository at
308
+ ``upstream.tested``, with ``extras``), on a ``python`` of its own;
309
+ ``wheels`` for a wheel served by URL only (a model package), each pinned by
310
+ its sha256 and resolved with ``pip``; ``indexes`` for packages served elsewhere (installed first, so the
311
+ project's resolution keeps them; each a name, or a name pinned), and ``prefetch``, models fetched from
312
+ the Hugging Face hub at install time so a run needs no network for them;
313
+ ``locked``, the project's own lock file (``uv.lock``) installed as its
314
+ pins, ``pip`` resolved beside them, rather than its loose requirements.
315
+ A benchmark's driver has the same block, ``driver.install`` (:func:`driver_install`):
316
+ ``where`` is the name its problems are said under."""
317
+ inst = d.get("install") or {}
318
+ if not isinstance(inst, dict):
319
+ return [f"{where}: an object"]
320
+ return [where + p[len("install"):] for p in _install_block(d, inst)]
321
+
322
+
323
+ def driver_install(bench: dict) -> dict:
324
+ """A benchmark's driver as a definition ``install.packages`` installs:
325
+ its ``driver.install`` - the environment the upstream's own code needs,
326
+ which the driver imports - beside the benchmark's ``upstream`` (for
327
+ ``project``), kept under ``drivers/<key>/<digest>/``. Its ``install`` is
328
+ empty when the driver declares none."""
329
+ return {"key": bench.get("key", ""), "_kind": "driver", "_here": bench.get("_here", ""),
330
+ "install": (bench.get("driver") or {}).get("install") or {}, "upstream": bench.get("upstream") or {}}
331
+
332
+
333
+ def _install_block(d: dict, inst: dict) -> List[str]:
334
+ bad = []
335
+ for p in inst.get("npm") or []:
336
+ if not _NPM.match(str(p)):
337
+ bad.append(f"install.npm: {p!r} is not name@version")
338
+ for p in inst.get("pip") or []:
339
+ if not (_PIP.fullmatch(str(p)) or _PIP_GIT.fullmatch(str(p))):
340
+ bad.append(f"install.pip: {p!r} is not name==version or name @ git+https://repository@full-commit")
341
+ if "python" in inst and not _PYTHON.match(str(inst["python"])):
342
+ bad.append("install.python: a version such as 3.12")
343
+ if "project" in inst:
344
+ if not _RELATIVE.match(str(inst["project"])) and inst["project"] != ".":
345
+ bad.append("install.project: a folder of the upstream repository, relative")
346
+ if not (d.get("upstream") or {}).get("repository"):
347
+ bad.append("install.project: needs upstream.repository and upstream.tested, the code it installs")
348
+ if any(not _EXTRA.match(str(x)) for x in inst.get("extras") or []):
349
+ bad.append("install.extras: the project's extras, plain names")
350
+ if inst.get("extras") and "project" not in inst:
351
+ bad.append("install.extras: extras of install.project, which is not set")
352
+ if "locked" in inst and not isinstance(inst["locked"], bool):
353
+ bad.append("install.locked: true or false")
354
+ elif inst.get("locked") and "project" not in inst:
355
+ bad.append("install.locked: the lock file of install.project, which is not set")
356
+ for i, x in enumerate(inst.get("indexes") or []):
357
+ if not isinstance(x, dict) or not _HTTPS.match(str(x.get("url", ""))) or not x.get("packages") \
358
+ or any(not _PKG.match(str(p)) for p in x.get("packages") or []):
359
+ bad.append(f"install.indexes[{i}]: {{url: https, packages: [names]}}")
360
+ for i, x in enumerate(inst.get("wheels") or []):
361
+ if not isinstance(x, dict) or not _HTTPS.match(str(x.get("url", ""))) or not str(x.get("url")).endswith(".whl") \
362
+ or not _SHA256.match(str(x.get("sha256", ""))):
363
+ bad.append(f"install.wheels[{i}]: {{url: https, a .whl, sha256: the file's}}")
364
+ for m in inst.get("prefetch") or []:
365
+ if not _HF.match(str(m)):
366
+ bad.append(f"install.prefetch: {m!r} is not a hub model id (owner/name)")
367
+ if inst.get("prefetch") and not (inst.get("pip") or inst.get("project") or inst.get("indexes") or inst.get("wheels")):
368
+ bad.append("install: prefetch goes with pip, project or indexes, the environment it is installed in")
369
+ return bad
370
+
371
+
372
+ def _env(env) -> List[str]:
373
+ """``env``: the variables the process is started with, beside what Nullius
374
+ gives it - templates of ``{install}``, ``{home}``, ``{here}``."""
375
+ if env is None:
376
+ return []
377
+ if not isinstance(env, dict):
378
+ return ["env: an object of variables"]
379
+ bad = []
380
+ for k, v in env.items():
381
+ if not _ENV_NAME.match(k) or not isinstance(v, str) or not _TEMPLATE.match(v):
382
+ bad.append(f"env.{k}: a variable name and a plain template")
383
+ bad += [f"env.{k}: {{{x}}} is not a slot ({', '.join(sorted(ENV_SLOTS))})" for x in _SLOT.findall(str(v)) if x not in ENV_SLOTS]
384
+ return bad
385
+
386
+
387
+ def option_problem(spec: dict, value) -> Optional[str]:
388
+ """Why ``value`` is not one an option ``spec`` allows; None when it is."""
389
+ if not isinstance(value, str) or not _OPTION_VALUE.match(value):
390
+ return f"{value!r} is not a plain word"
391
+ if "values" in spec:
392
+ return None if value in (spec.get("values") or []) else f"{value!r} is not one of {', '.join(spec.get('values') or [])}"
393
+ try:
394
+ return None if re.fullmatch(str(spec.get("pattern")), value) else f"{value!r} does not match {spec.get('pattern')}"
395
+ except re.error:
396
+ return "its pattern is not a regular expression"
397
+
398
+
399
+ def _option_slots(d: dict) -> List[Tuple[str, str]]:
400
+ """Every ``{opt_<name>}`` a template of ``d`` carries, with where: its
401
+ ``model.env``, its ``env``, and the files ``model.files`` writes."""
402
+ m = d.get("model") if isinstance(d.get("model"), dict) else {}
403
+ found = []
404
+ for where, vals in (("model.env", m.get("env")), ("env", d.get("env"))):
405
+ for k, v in (vals or {}).items() if isinstance(vals, dict) else ():
406
+ found += [(f"{where}.{k}", n) for n in _OPT_SLOT.findall(str(v))]
407
+ for dest, src in (m.get("files") or {}).items() if isinstance(m.get("files"), dict) else ():
408
+ path = os.path.join(d.get("_here") or "", str(src))
409
+ if d.get("_here") and isinstance(src, str) and _RELATIVE.match(src) and os.path.isfile(path):
410
+ with open(path, errors="replace") as f:
411
+ found += [(f"model.files: {src}", n) for n in _OPT_SLOT.findall(f.read())]
412
+ return found
413
+
414
+
415
+ def _options(d: dict) -> List[str]:
416
+ """``options``: what a run may set of the agent's own behaviour, each with
417
+ a ``summary``, its ``values`` (or a ``pattern``) and a ``default`` that is
418
+ the vendor's own - so a run that sets nothing is the agent as it ships.
419
+ Every ``{opt_<name>}`` slot a template carries must be one of them."""
420
+ opts = d.get("options")
421
+ if opts is not None and not isinstance(opts, dict):
422
+ return ["options: an object, name -> {summary, values or pattern, default}"]
423
+ opts, bad = opts or {}, []
424
+ for name, o in opts.items():
425
+ w = f"options.{name}"
426
+ if not _OPTION.match(str(name)):
427
+ bad.append(f"options: {name!r} is not a name (lower case letters and _)")
428
+ continue
429
+ if not isinstance(o, dict):
430
+ bad.append(f"{w}: an object")
431
+ continue
432
+ if not isinstance(o.get("summary"), str) or not o["summary"].strip():
433
+ bad.append(f"{w}.summary: say what it changes, and where its default comes from")
434
+ if ("values" in o) == ("pattern" in o):
435
+ bad.append(f"{w}: values or pattern, one of them")
436
+ continue
437
+ if "values" in o:
438
+ vs = o["values"]
439
+ if not isinstance(vs, list) or not vs or len(set(map(str, vs))) != len(vs) \
440
+ or any(not isinstance(v, str) or not _OPTION_VALUE.match(v) for v in vs):
441
+ bad.append(f"{w}.values: distinct plain words")
442
+ continue
443
+ else:
444
+ try:
445
+ re.compile(str(o["pattern"]))
446
+ except re.error:
447
+ bad.append(f"{w}.pattern: a regular expression")
448
+ continue
449
+ if option_problem(o, o.get("default")):
450
+ bad.append(f"{w}.default: the vendor's own default, a value it allows")
451
+ for where, n in _option_slots(d):
452
+ if n not in opts:
453
+ bad.append(f"{where}: {{opt_{n}}} is not an option (options.{n})")
454
+ return bad
455
+
456
+
457
+ def _when_valid(name: str, when, opts: dict) -> bool:
458
+ return isinstance(when, dict) and all(k != name and k in opts and isinstance(vs, list) and vs
459
+ and all(v in opts[k].get("values", []) for v in vs) for k, vs in when.items())
460
+
461
+
462
+ def _when_applies(when: dict, opts: dict, values: dict) -> bool:
463
+ return all(values.get(k, opts[k].get("default")) in vs for k, vs in when.items())
464
+
465
+
466
+ def _bench_options(d: dict) -> List[str]:
467
+ """A benchmark's ``options``: a choice its upstream offers a run (a search, a mode), each with a
468
+ ``summary``, its ``values``, a ``default`` a run gets when it chooses none, and optionally the
469
+ ``datasets`` it applies to, a line ``about`` each value, what a value ``needs``: an
470
+ ``embeddings`` model called through the run's relay, ``host`` requirements or ``files`` by name - and
471
+ ``when`` it applies: another option's values (a retriever's, only with a retrieval)."""
472
+ opts = d.get("options")
473
+ if opts is None:
474
+ return []
475
+ bad = _options({"options": opts})
476
+ if bad or not isinstance(opts, dict):
477
+ return bad
478
+ ids = {x.get("id") for x in d.get("datasets") or [] if isinstance(x, dict)}
479
+ hosts = d.get("host") or {}
480
+ files = d.get("files") if isinstance(d.get("files"), dict) else {}
481
+ for name, o in opts.items():
482
+ w = f"options.{name}"
483
+ ds = o.get("datasets")
484
+ if ds is not None and (not isinstance(ds, list) or not ds or any(x not in ids for x in ds)):
485
+ bad.append(f"{w}.datasets: some of the benchmark's dataset ids")
486
+ on = o.get("setups")
487
+ if on is not None and (not isinstance(on, list) or not on or any(x not in SETUPS + ("yours",) for x in on)):
488
+ bad.append(f"{w}.setups: where it applies - some of original, model, yours (an agent or a capability)")
489
+ when = o.get("when") or {}
490
+ if not _when_valid(name, when, opts):
491
+ bad.append(f"{w}.when: another option -> the values of it this one applies under")
492
+ about = o.get("about") or {}
493
+ if not isinstance(about, dict) or any(option_problem(o, k) is not None or not isinstance(v, str) for k, v in about.items()):
494
+ bad.append(f"{w}.about: a line for some of its values")
495
+ needs = o.get("needs") or {}
496
+ if not isinstance(needs, dict):
497
+ bad.append(f"{w}.needs: value -> {{embeddings, reranker, host, files}}")
498
+ continue
499
+ for v, n in needs.items():
500
+ if option_problem(o, v) is not None or not isinstance(n, dict) or any(k not in NEED_KEYS + ("when",) for k in n):
501
+ bad.append(f"{w}.needs.{v}: one of its values -> {{embeddings: a model, reranker: a model, host: [requirements], files: [names]}}")
502
+ continue
503
+ if not _when_valid(name, n.get("when") or {}, opts):
504
+ bad.append(f"{w}.needs.{v}.when: another option -> the values that need these requirements")
505
+ for kind in ("embeddings", "reranker"):
506
+ if kind in n and (not isinstance(n[kind], str) or not _MODEL_NAME.fullmatch(n[kind])):
507
+ bad.append(f"{w}.needs.{v}.{kind}: the model, as its provider names it")
508
+ if "host" in n and (not isinstance(n["host"], list) or any(h not in hosts for h in n["host"])):
509
+ bad.append(f"{w}.needs.{v}.host: names of the benchmark's host requirements")
510
+ if "files" in n and (not isinstance(n["files"], list)
511
+ or any(not isinstance(f, str) or f not in files for f in n["files"])):
512
+ bad.append(f"{w}.needs.{v}.files: names of the benchmark's files")
513
+ return bad
514
+
515
+
516
+ def _files(d: dict) -> List[str]:
517
+ """Auxiliary data fetched only when selected options need it, addressed by digest."""
518
+ files = d.get("files", {})
519
+ if not isinstance(files, dict):
520
+ return ["files: name -> {url, sha256}"]
521
+ bad = []
522
+ for name, f in files.items():
523
+ w = f"files.{name}"
524
+ if not _OPTION.fullmatch(str(name)) or not isinstance(f, dict):
525
+ bad.append(f"{w}: a lower case name -> {{url, sha256}}")
526
+ continue
527
+ from urllib.parse import urlsplit
528
+ url = f.get("url", "")
529
+ try:
530
+ parsed = urlsplit(url) if isinstance(url, str) else None
531
+ valid = (parsed and parsed.scheme == "https" and parsed.hostname and not parsed.username
532
+ and not parsed.password and not parsed.fragment and not any(c.isspace() for c in url))
533
+ except ValueError:
534
+ valid = False
535
+ if not valid:
536
+ bad.append(f"{w}.url: an https address without credentials or a fragment")
537
+ if not _SHA256.fullmatch(str(f.get("sha256", ""))):
538
+ bad.append(f"{w}.sha256: the file's digest")
539
+ return bad
540
+
541
+
542
+ def _host(d: dict) -> List[str]:
543
+ """A benchmark's ``host``: what the machine its driver runs on must have for some of its options,
544
+ by name - a ``summary``, the programs that ``check`` it (each on PATH) and how to ``install`` them
545
+ (``apt`` and ``npm`` packages, pinned where the upstream pins them); no ``install`` for the
546
+ machine's own hardware (a GPU, checked by its driver's program), which no package puts there."""
547
+ h = d.get("host")
548
+ if h is None:
549
+ return []
550
+ if not isinstance(h, dict):
551
+ return ["host: name -> {summary, check, install}"]
552
+ bad = []
553
+ for name, r in h.items():
554
+ w = f"host.{name}"
555
+ if not _OPTION.match(str(name)) or not isinstance(r, dict):
556
+ bad.append(f"{w}: a plain name -> an object")
557
+ continue
558
+ if not isinstance(r.get("summary"), str) or not r["summary"].strip():
559
+ bad.append(f"{w}.summary: what it is for")
560
+ if not isinstance(r.get("check"), list) or not r["check"] or any(not _PROGRAM.match(str(x)) for x in r["check"]):
561
+ bad.append(f"{w}.check: the programs it puts on PATH")
562
+ if "gpu_memory_mb" in r and (type(r["gpu_memory_mb"]) is not int or r["gpu_memory_mb"] <= 0):
563
+ bad.append(f"{w}.gpu_memory_mb: a positive whole number of MiB")
564
+ if "install" not in r:
565
+ continue
566
+ inst = r["install"]
567
+ if not isinstance(inst, dict) or not inst or any(k not in HOST_INSTALLERS for k in inst) \
568
+ or any(not isinstance(v, list) or any(not _PACKAGE.match(str(x)) for x in v) for v in inst.values()):
569
+ bad.append(f"{w}.install: {{apt: [packages], npm: [packages]}}")
570
+ return bad
571
+
572
+
573
+ def bench_options_of(bench: dict, dataset: str, given: Optional[dict] = None, setup: str = "yours") -> Tuple[dict, List[str]]:
574
+ """The benchmark's options for one run on ``dataset`` in ``setup`` (original, model, or yours: an
575
+ agent or a capability in its port): each that applies to both, at the value ``given`` or its default;
576
+ and what is wrong with the given ones. An option given for a dataset or a setup it does not apply
577
+ to is ignored - a run of several carries one choice for all. An option applies to an agent's or a
578
+ capability's run unless its ``setups`` say otherwise."""
579
+ opts = {n: o for n, o in (bench.get("options") or {}).items()
580
+ if dataset in (o.get("datasets") or [dataset]) and setup in (o.get("setups") or ["yours"])}
581
+ given = {n: v for n, v in (given or {}).items() if n in opts or n not in (bench.get("options") or {})}
582
+ vals, bad = options_of(dict(options=opts, label=bench.get("label"), key=bench.get("key")), given)
583
+ return vals, bad
584
+
585
+
586
+ def option_needs(bench: dict, values: dict) -> dict:
587
+ """What the chosen option values need: {"embeddings": [models], "reranker": [models], "host": [requirement names], "files": [names]}."""
588
+ out = {k: [] for k in NEED_KEYS}
589
+ opts = bench.get("options") or {}
590
+ for n, v in values.items():
591
+ option = opts.get(n, {})
592
+ if not _when_applies(option.get("when") or {}, opts, values):
593
+ continue
594
+ need = (option.get("needs") or {}).get(v) or {}
595
+ if not _when_applies(need.get("when") or {}, opts, values):
596
+ continue
597
+ for kind in ("embeddings", "reranker"):
598
+ if need.get(kind) and need[kind] not in out[kind]:
599
+ out[kind].append(need[kind])
600
+ for kind in ("host", "files"):
601
+ out[kind] += [x for x in need.get(kind) or [] if x not in out[kind]]
602
+ return out
603
+
604
+
605
+ def options_of(d: dict, given: Optional[dict] = None) -> Tuple[dict, List[str]]:
606
+ """Every option's value for one run - the ones ``given``, the rest at their
607
+ defaults - and what is wrong with the given ones (a name it does not have,
608
+ a value it does not allow)."""
609
+ opts, given = d.get("options") or {}, given or {}
610
+ bad = [f"{d.get('label') or d.get('key')} has no option {n!r}" + (f" (its options: {', '.join(opts)})" if opts else "")
611
+ for n in given if n not in opts]
612
+ for n, v in given.items():
613
+ why = option_problem(opts[n], v) if n in opts else None
614
+ if why:
615
+ bad.append(f"option {n}: {why}")
616
+ return {n: given.get(n, o.get("default")) for n, o in opts.items()}, bad
617
+
618
+
619
+ def _command(where: str, cmd, slots=SLOTS) -> List[str]:
620
+ if not isinstance(cmd, list) or not cmd or not all(isinstance(w, str) for w in cmd):
621
+ return [f"{where}: a list of words"]
622
+ bad = []
623
+ for w in cmd:
624
+ if not _WORD.match(w):
625
+ bad.append(f"{where}: {w!r} is not a plain word")
626
+ for s in _SLOT.findall(w):
627
+ if s not in slots:
628
+ bad.append(f"{where}: {{{s}}} is not a slot ({', '.join(sorted(slots))})")
629
+ return bad
630
+
631
+
632
+ def _ports(where: str, names, side: Optional[str]) -> List[str]:
633
+ if not isinstance(names, list) or not all(isinstance(p, str) for p in names):
634
+ return [f"{where}: a list of port names"]
635
+ bad = []
636
+ for p in names:
637
+ if not ports.known(p):
638
+ bad.append(f"{where}: {p!r} is not a port ({', '.join(ports.PORTS)})")
639
+ elif side and ports.family(p) != side:
640
+ bad.append(f"{where}: {p} is a port of the {ports.family(p)} family, not the {side}")
641
+ return bad
642
+
643
+
644
+ _SERVER = re.compile(r"^[a-z0-9][a-z0-9-]{0,39}$")
645
+
646
+
647
+ def _mcp_servers(servers, has_install: bool) -> List[str]:
648
+ """``mcp_servers``: the benchmark's own MCP servers, run **on the
649
+ machine** in the driver's environment rather than in the agent's - each
650
+ a name, its ``command`` (slots ``{here}``, ``{install}``) and a
651
+ ``summary``. A unit's setup asks for one by name in its ``mcp`` entry
652
+ (``{"name", "server", "args", "env"}``); the run gives the agent a stdio
653
+ command that reaches it (``mcp_machine``)."""
654
+ if servers is None:
655
+ return []
656
+ if not isinstance(servers, dict) or not servers:
657
+ return ["mcp_servers: an object, a name for each server"]
658
+ bad = []
659
+ for name, sv in servers.items():
660
+ w = f"mcp_servers.{name}"
661
+ if not _SERVER.match(str(name)):
662
+ bad.append(f"{w}: a name of lower case letters, digits and dashes")
663
+ if not isinstance(sv, dict):
664
+ bad.append(f"{w}: {{command, summary}}")
665
+ continue
666
+ bad += _command(f"{w}.command", sv.get("command"), {"here", "install"})
667
+ if any("{install}" in x for x in sv.get("command") or [] if isinstance(x, str)) and not has_install:
668
+ bad.append(f"{w}.command: {{install}} names driver.install, which is not set")
669
+ if not isinstance(sv.get("summary"), str) or not sv["summary"].strip():
670
+ bad.append(f"{w}.summary: say what it serves")
671
+ extra = set(sv) - {"command", "summary"}
672
+ if extra:
673
+ bad.append(f"{w}: {', '.join(sorted(extra))} is not a field (command, summary)")
674
+ return bad
675
+
676
+
677
+ def tag_ok(tag) -> bool:
678
+ """A plain git tag name: no spaces, no ``..``, nothing git refuses in a ref."""
679
+ t = str(tag) if isinstance(tag, str) else ""
680
+ return bool(_TAG.match(t)) and ".." not in t and not t.endswith((".lock", "/", ".")) and "//" not in t and "@{" not in t
681
+
682
+
683
+ def _upstream(u) -> List[str]:
684
+ if u is None:
685
+ return []
686
+ if not isinstance(u, dict):
687
+ return ["upstream: an object"]
688
+ bad = []
689
+ if "repository" in u:
690
+ if not _HTTPS.match(str(u["repository"])):
691
+ bad.append("upstream.repository: an https address")
692
+ if not _COMMIT.match(str(u.get("tested", ""))):
693
+ bad.append("upstream.tested: the full commit the definition was checked against")
694
+ if "tag" in u and not tag_ok(u["tag"]):
695
+ bad.append("upstream.tag: the release tag the tested commit is, a plain tag name (v1.0.1)")
696
+ elif "tag" in u and "repository" not in u:
697
+ bad.append("upstream.tag: a tag of upstream.repository, which is not set")
698
+ if not u.get("license"):
699
+ bad.append("upstream.license: say it, or 'unknown'")
700
+ return bad
701
+
702
+
703
+ def _proven(d) -> List[str]:
704
+ offers, proven = d.get("offers") or [], d.get("proven") or {}
705
+ if not isinstance(proven, dict):
706
+ return ["proven: an object, written by the conformance run"]
707
+ return [f"offers {p}, which has no conformance record (proven): run the kit"
708
+ for p in offers if p not in proven]
709
+
710
+
711
+ def _versioned(d: dict, kind: str) -> List[str]:
712
+ """A file with ``versions``: the list itself, then each version resolved
713
+ and checked as a flat definition - a problem every version shares said
714
+ once, one of a single version prefixed with it."""
715
+ vs, bad = d["versions"], []
716
+ own = BENCH_VERSIONED if kind == "benchmark" else VERSIONED
717
+ if not isinstance(vs, dict) or not vs:
718
+ return ["versions: an object, version -> what differs in it"]
719
+ for k in (("proven",) if kind == "benchmark" else ("install", "proven", "offers")):
720
+ if k in d:
721
+ bad.append(f"{k}: goes in each version, under versions")
722
+ if (d.get("upstream") or {}).get("version"):
723
+ bad.append("upstream.version: the versions are the keys of versions")
724
+ if d.get("default") not in vs:
725
+ bad.append(f"default: one of the versions ({', '.join(vs)})")
726
+ per = {}
727
+ for v, block in vs.items():
728
+ if not _VERSION.match(str(v)):
729
+ bad.append(f"versions: {v!r} is not a version (letters, digits, . + _ -)")
730
+ continue
731
+ if not isinstance(block, dict):
732
+ bad.append(f"versions.{v}: an object")
733
+ continue
734
+ extra = [k for k in block if k not in own]
735
+ if extra:
736
+ bad.append(f"versions.{v}: {', '.join(extra)} cannot differ between versions (only {', '.join(own)})")
737
+ if (block.get("upstream") or {}).get("version"):
738
+ bad.append(f"versions.{v}.upstream.version: the version is its key")
739
+ if kind != "benchmark" and not block.get("install"):
740
+ bad.append(f"versions.{v}.install: what this version installs")
741
+ for p, rec in (block.get("proven") or {}).items():
742
+ if isinstance(rec, dict) and rec.get("version") not in (None, "", v):
743
+ bad.append(f"versions.{v}.proven.{p}: proven on {rec['version']}, not {v}")
744
+ per[v] = problems(resolve(d, v), kind)
745
+ if per:
746
+ shared = set.intersection(*(set(x) for x in per.values()))
747
+ bad += [p for p in next(iter(per.values())) if p in shared]
748
+ bad += [f"versions.{v}: {p}" for v, x in per.items() for p in x if p not in shared]
749
+ return bad
750
+
751
+
752
+ def _controls(c) -> List[str]:
753
+ """``controls``: {nop|oracle: {source, from, expect?}}. ``expect`` is the
754
+ one bound a control is held to - ``at_most`` for the null run, ``at_least``
755
+ for the reference - as a pass rate or a mean score, from 0 to 1: a judged
756
+ benchmark's reference may fall short of every unit and still be right.
757
+ The reference may also say why a prompt can name its answer (``in_prompt``:
758
+ a choice question), which the leak check then counts instead of failing."""
759
+ if c is None:
760
+ return []
761
+ if not isinstance(c, dict) or not c:
762
+ return ["controls: an object, nop and/or oracle"]
763
+ bad = []
764
+ for k, x in c.items():
765
+ w = f"controls.{k}"
766
+ if k not in CONTROLS:
767
+ bad.append(f"{w}: a control is one of {CONTROLS}")
768
+ continue
769
+ if not isinstance(x, dict):
770
+ bad.append(f"{w}: an object {{source, from}}")
771
+ continue
772
+ if x.get("source") not in SOURCES:
773
+ bad.append(f"{w}.source: one of {SOURCES} (who wrote it)")
774
+ if not isinstance(x.get("from"), str) or not x["from"].strip():
775
+ bad.append(f"{w}.from: say where it comes from, in a sentence")
776
+ e = x.get("expect")
777
+ if e is not None:
778
+ bound = EXPECT[k][0]
779
+ v = e.get(bound) if isinstance(e, dict) else None
780
+ if not isinstance(e, dict) or set(e) != {bound} or isinstance(v, bool) or not isinstance(v, (int, float)) or not 0 <= v <= 1:
781
+ bad.append(f"{w}.expect: {{{bound}: a number from 0 to 1}}")
782
+ if "in_prompt" in x and (k != "oracle" or not isinstance(x["in_prompt"], str) or not x["in_prompt"].strip()):
783
+ bad.append(f"{w}.in_prompt: the reference's, a sentence on why a prompt may name its answer")
784
+ extra = [f for f in x if f not in ("source", "from", "expect", "in_prompt")]
785
+ if extra:
786
+ bad.append(f"{w}: {', '.join(extra)} is not a field of a control")
787
+ return bad
788
+
789
+
790
+ def dockerfile_lines(text: str) -> List[str]:
791
+ """A Dockerfile's instructions, one per entry: comments and blank lines
792
+ dropped, continuation lines joined."""
793
+ out, cur = [], ""
794
+ for raw in text.splitlines():
795
+ line = raw.strip()
796
+ if not cur and (not line or line.startswith("#")):
797
+ continue
798
+ if line.endswith("\\"):
799
+ cur += line[:-1] + " "
800
+ continue
801
+ out.append((cur + line).strip())
802
+ cur = ""
803
+ if cur.strip():
804
+ out.append(cur.strip())
805
+ return out
806
+
807
+
808
+ def _dockerfile(w: str, text: str, files: List[str]) -> List[str]:
809
+ """An environment's Dockerfile: a layer on the base image, so no ``FROM``
810
+ (Nullius writes it); ``COPY`` only of the files it lists (they are its whole
811
+ build context), and no ``ADD``, which can fetch what nothing pins."""
812
+ bad = []
813
+ for line in dockerfile_lines(text):
814
+ word, _, rest = line.partition(" ")
815
+ word = word.upper()
816
+ if word == "FROM":
817
+ bad.append(f"{w}: no FROM - Nullius writes FROM nullius/base:<hash>, the layer's base")
818
+ elif word == "ADD":
819
+ bad.append(f"{w}: COPY, not ADD")
820
+ elif word == "COPY":
821
+ args = [a for a in rest.split() if not a.startswith("--")]
822
+ if any(a.startswith("--from") for a in rest.split()):
823
+ bad.append(f"{w}: COPY --from: one stage, on the base image")
824
+ elif rest.lstrip().startswith("["):
825
+ bad.append(f"{w}: COPY src... dest, not its JSON form")
826
+ else:
827
+ bad += [f"{w}: COPY {a}, which is not in files" for a in args[:-1] if a not in files]
828
+ return bad
829
+
830
+
831
+ def _environments(envs, here: Optional[str]) -> List[str]:
832
+ """``environments``: name -> {summary, dockerfile, files?, needs_secrets?,
833
+ local_ok?}, or {summary, image: "own", needs_secrets?} for units that
834
+ each bring their own image. ``dockerfile`` and each of ``files`` are paths in the
835
+ benchmark's folder; ``needs_secrets`` is [{name, to, kind, ...}] - a
836
+ credential the user sets in Settings → Data providers (one with ``provider``)
837
+ or Custom keys (``nullius.secrets``), its
838
+ shape (:func:`_secrets`), and where it goes: ``agent``, into the agent's
839
+ container, or ``driver``, into the driver's process, which scores. One
840
+ name is one kind, and one source, in every environment that asks for it."""
841
+ if envs is None:
842
+ return []
843
+ if not isinstance(envs, dict) or not envs:
844
+ return ["environments: an object, name -> {summary, dockerfile, ...}"]
845
+ from nullius import settings # the names Settings keeps for itself, which no secret may take
846
+ bad = []
847
+ for name, e in envs.items():
848
+ w = f"environments.{name}"
849
+ if not _ENV_KEY.match(str(name)) or name == BASE_ENV:
850
+ bad.append(f"environments: {name!r} is not a name (lower case letters, digits, dashes; not {BASE_ENV!r})")
851
+ continue
852
+ if not isinstance(e, dict):
853
+ bad.append(f"{w}: an object")
854
+ continue
855
+ extra = [k for k in e if k not in ENV_FIELDS]
856
+ if extra:
857
+ bad.append(f"{w}: {', '.join(extra)} is not a field of an environment ({', '.join(ENV_FIELDS)})")
858
+ if not isinstance(e.get("summary"), str) or not e["summary"].strip():
859
+ bad.append(f"{w}.summary: say what it holds that the base image does not")
860
+ if "image" in e and e["image"] != OWN_IMAGE:
861
+ bad.append(f"{w}.image: {OWN_IMAGE!r} (each unit's own image, its setup naming it) or left out (a layer on the base)")
862
+ if own_image(e):
863
+ # the task's own image: its setup names it, so there is no Dockerfile here, and no local run
864
+ bad += [f"{w}.{k}: not with an own image - the unit's setup names the image" for k in ("dockerfile", "files") if k in e]
865
+ if e.get("local_ok"):
866
+ bad.append(f"{w}.local_ok: an own image runs only in a container")
867
+ files = [] if own_image(e) else e.get("files") or []
868
+ if not isinstance(files, list) or any(not isinstance(f, str) or not _RELATIVE.match(f) for f in files):
869
+ bad.append(f"{w}.files: paths in the benchmark's folder, relative")
870
+ files = []
871
+ for f in files:
872
+ if here and not os.path.isfile(os.path.join(here, f)):
873
+ bad.append(f"{w}.files: {f} is not in the benchmark's folder")
874
+ df = e.get("dockerfile")
875
+ if own_image(e):
876
+ pass
877
+ elif not isinstance(df, str) or not _RELATIVE.match(df):
878
+ bad.append(f"{w}.dockerfile: a Dockerfile in the benchmark's folder, relative")
879
+ elif here:
880
+ try:
881
+ with open(os.path.join(here, df)) as f:
882
+ bad += _dockerfile(f"{w}.dockerfile", f.read(), files)
883
+ except OSError:
884
+ bad.append(f"{w}.dockerfile: {df} is not in the benchmark's folder")
885
+ if "local_ok" in e and not isinstance(e["local_ok"], bool):
886
+ bad.append(f"{w}.local_ok: true or false")
887
+ if "network" in e:
888
+ from . import network_requirements
889
+ bad += [f"{w}.{message}" for message in network_requirements.problems(e["network"])]
890
+ bad += _secrets(w, e.get("needs_secrets"), bool(e.get("local_ok")), settings)
891
+ kinds: Dict[str, set] = {}
892
+ sources: Dict[str, set] = {}
893
+ for e in envs.values():
894
+ for s in (e.get("needs_secrets") or []) if isinstance(e, dict) and isinstance(e.get("needs_secrets"), list) else []:
895
+ if isinstance(s, dict) and isinstance(s.get("name"), str):
896
+ kinds.setdefault(s["name"], set()).add(s.get("kind") or "value")
897
+ sources.setdefault(s["name"], set()).add(str(s.get("provider") or "the benchmark"))
898
+ bad += [f"environments: {n} is a {' and a '.join(sorted(k))} - one name, one kind, in every environment"
899
+ for n, k in kinds.items() if len(k) > 1]
900
+ bad += [f"environments: {n} comes from {' and from '.join(sorted(k))} - one name, one source, in every environment"
901
+ for n, k in sources.items() if len(k) > 1]
902
+ return bad
903
+
904
+
905
+ def _secrets(w: str, secrets, local_ok: bool, settings) -> List[str]:
906
+ """An environment's ``needs_secrets``: [{name, to, kind?, summary?,
907
+ path?, env?, provider?}]. ``kind`` (``value`` when unsaid): a ``value`` reaches its
908
+ process as the variable ``name``; a ``file`` is mounted read-only at
909
+ ``path`` in the agent's container, and ``env``, when said, names the
910
+ variable that points to it - the only way a driver, or an agent on local
911
+ processes, finds it; a ``server`` (``host:port``) is the variable
912
+ ``name``, and in a container the one address the agent's container may reach.
913
+ ``provider`` (``huggingface``): a ``value`` to the ``driver``, from Settings → Data providers alone."""
914
+ if secrets is None:
915
+ return []
916
+ if not isinstance(secrets, list):
917
+ return [f"{w}.needs_secrets: a list of {{name, to, kind}}"]
918
+ bad, seen, paths = [], set(), set()
919
+ for i, s in enumerate(secrets):
920
+ sw = f"{w}.needs_secrets[{i}]"
921
+ if not isinstance(s, dict) or set(s) - set(SECRET_FIELDS):
922
+ bad.append(f"{sw}: {{{', '.join(SECRET_FIELDS)}}} (name and to always)")
923
+ continue
924
+ n = s.get("name")
925
+ if not isinstance(n, str) or not _ENV_NAME.match(n) or n in seen:
926
+ bad.append(f"{sw}.name: a variable name, each once")
927
+ elif settings.reserved_name(n):
928
+ bad.append(f"{sw}.name: {n} is already the name of something else")
929
+ seen.add(n)
930
+ to, kind = s.get("to"), s.get("kind", "value")
931
+ if to not in SECRET_TO:
932
+ bad.append(f"{sw}.to: one of {SECRET_TO} - where it goes, said explicitly")
933
+ if kind not in SECRET_KINDS:
934
+ bad.append(f"{sw}.kind: one of {SECRET_KINDS}")
935
+ continue
936
+ if "provider" in s:
937
+ from nullius.secrets import PROVIDERS # the data providers Settings keeps a token for
938
+ if not isinstance(s["provider"], str) or s["provider"] not in PROVIDERS:
939
+ bad.append(f"{sw}.provider: one of {', '.join(PROVIDERS)}")
940
+ if to != "driver":
941
+ bad.append(f"{sw}.to: driver - a data provider's token never reaches an agent")
942
+ if kind != "value":
943
+ bad.append(f"{sw}.kind: value - a data provider's token is a key")
944
+ p, v = s.get("path"), s.get("env")
945
+ if kind != "file":
946
+ if p is not None or v is not None:
947
+ bad.append(f"{sw}: path and env are a file's; a {kind} reaches its process as the variable {n}")
948
+ continue
949
+ if v is not None:
950
+ if not isinstance(v, str) or not _ENV_NAME.match(v) or settings.reserved_name(v) or v in seen:
951
+ bad.append(f"{sw}.env: a variable name of its own, which will hold the file's path")
952
+ seen.add(v)
953
+ if to == "driver":
954
+ if p is not None:
955
+ bad.append(f"{sw}.path: the driver runs on the machine, not in a container - it finds the file by env")
956
+ if v is None:
957
+ bad.append(f"{sw}.env: the variable the driver finds the file by")
958
+ continue
959
+ if not isinstance(p, str) or not _ABS.match(p) or "/../" in p + "/" or "/./" in p + "/" \
960
+ or any(p == u or p.startswith(u + "/") for u in _NOT_UNDER):
961
+ bad.append(f"{sw}.path: an absolute path in the agent's container, where the software looks for it "
962
+ f"(not under {', '.join(_NOT_UNDER)})")
963
+ elif p in paths:
964
+ bad.append(f"{sw}.path: {p} is another licence's")
965
+ paths.add(p)
966
+ if local_ok and v is None:
967
+ bad.append(f"{sw}.env: on local processes (local_ok) the agent finds the file by a variable, not at {p}")
968
+ return bad
969
+
970
+
971
+ def environment_of(bench: dict, unit_setup: Optional[dict], dataset: Optional[dict] = None) -> Tuple[Optional[str], Optional[dict]]:
972
+ """The environment a unit runs in: the one its setup names
973
+ (``environment``), else its dataset's default, else none - the base image.
974
+ (name, its definition); (None, None) for the base. A name the benchmark
975
+ does not declare raises ValueError."""
976
+ envs = bench.get("environments") or {}
977
+ name = (unit_setup or {}).get("environment") or (dataset or {}).get("environment")
978
+ if not name or name == BASE_ENV:
979
+ return None, None
980
+ if name not in envs:
981
+ raise ValueError(f"environment {name!r} is not one of {bench.get('label') or bench.get('key')}'s "
982
+ f"({', '.join(envs) or 'it declares none'})")
983
+ return name, envs[name]
984
+
985
+
986
+ def own_image(env_def: Optional[dict]) -> bool:
987
+ """Whether an environment is an own image: each unit's setup names the
988
+ task's own image, and the agent is added to it at run time."""
989
+ return isinstance(env_def, dict) and env_def.get("image") == OWN_IMAGE
990
+
991
+
992
+ def secrets_of(bench: dict) -> List[dict]:
993
+ """Every licence the benchmark's environments need: {name, to, kind,
994
+ environment, summary, path?, env?, provider?}, once per environment that asks."""
995
+ return [dict({k: s[k] for k in ("path", "env", "provider") if k in s}, name=s["name"], to=s["to"], kind=s.get("kind") or "value",
996
+ environment=n, summary=s.get("summary", ""))
997
+ for n, e in (bench.get("environments") or {}).items() if isinstance(e, dict)
998
+ for s in e.get("needs_secrets") or [] if isinstance(s, dict) and s.get("name")]
999
+
1000
+
1001
+ def missing_secrets(env_def: Optional[dict], held) -> List[str]:
1002
+ """The licences an environment needs that ``held`` (name -> what the run
1003
+ holds of it, ``run.Licences``) does not."""
1004
+ return [s["name"] for s in (env_def or {}).get("needs_secrets") or [] if not held.get(s["name"])]
1005
+
1006
+
1007
+ def secret_values(env_def: Optional[dict], held, to: str, *, inside: bool = False) -> dict:
1008
+ """The variables an environment's licences bound for ``to`` (agent,
1009
+ driver) give their process, those that are held: a value or a server as
1010
+ ``name``; a file's path as its ``env`` - where it is mounted
1011
+ (``inside`` a container) or where it is on the machine."""
1012
+ out = {}
1013
+ for s in (env_def or {}).get("needs_secrets") or []:
1014
+ h = held.get(s["name"])
1015
+ if s.get("to") != to or not h:
1016
+ continue
1017
+ if (s.get("kind") or "value") != "file":
1018
+ out[s["name"]] = h["value"]
1019
+ elif s.get("env"):
1020
+ out[s["env"]] = s["path"] if inside and s.get("path") else h["path"]
1021
+ return out
1022
+
1023
+
1024
+ def control(bench: dict, which: str) -> Optional[dict]:
1025
+ """The control ``which`` of ``bench`` as the other side of a run, or None
1026
+ when the benchmark does not declare it. It installs nothing, starts
1027
+ nothing and calls no model: the run answers "" for ``nop`` and asks the
1028
+ driver's ``oracle`` for the reference."""
1029
+ c = (bench.get("controls") or {}).get(which)
1030
+ if which not in CONTROLS or not isinstance(c, dict):
1031
+ return None
1032
+ bound, value = EXPECT[which]
1033
+ return {"key": which, "label": CONTROL_LABEL[which], "summary": c.get("from", ""), "source": c.get("source", ""),
1034
+ "from": c.get("from", ""), "expect": {bound: (c.get("expect") or {}).get(bound, value)}, "upstream": {},
1035
+ "_kind": "control", "_ref": which, "_here": bench.get("_here", "")}
1036
+
1037
+
1038
+ def _leaderboard(d: dict) -> List[str]:
1039
+ """``leaderboard``: {says, simulator?, judge?, bench_options?, repeat?} - what the
1040
+ upstream's leaderboard fixes besides the ranked model, from its own submission
1041
+ rules: the user simulator's model, the judge's, the benchmark's options (a
1042
+ simulator's reasoning effort) and the fewest attempts per unit."""
1043
+ lb = d.get("leaderboard")
1044
+ if lb is None:
1045
+ return []
1046
+ w = "leaderboard"
1047
+ if not isinstance(lb, dict):
1048
+ return [f"{w}: {{says, simulator?, judge?, bench_options?, repeat?}}"]
1049
+ bad = [f"{w}.{k}: not a leaderboard key" for k in lb if k not in ("says", "simulator", "judge", "bench_options", "repeat")]
1050
+ if not isinstance(lb.get("says"), str) or not lb["says"].strip():
1051
+ bad.append(f"{w}.says: which leaderboard, and where its rules say so")
1052
+ for k, has in (("simulator", d.get("simulator")), ("judge", (d.get("score") or {}).get("judge"))):
1053
+ if k in lb and (not has or not _MODEL_NAME.match(str(lb[k]))):
1054
+ bad.append(f"{w}.{k}: the leaderboard's model, as its maker names it, on a benchmark that has one")
1055
+ opts, given = d.get("options") or {}, lb.get("bench_options", {})
1056
+ if not isinstance(given, dict):
1057
+ bad.append(f"{w}.bench_options: an object, option to value")
1058
+ else:
1059
+ bad += [f"{w}.bench_options.{k}: one of the benchmark's options, at one of its values" for k, v in given.items()
1060
+ if k not in opts or v not in (opts[k].get("values") or [])]
1061
+ r = lb.get("repeat", 1)
1062
+ if isinstance(r, bool) or not isinstance(r, int) or not 1 <= r <= 32:
1063
+ bad.append(f"{w}.repeat: the fewest attempts per unit, 1 to 32")
1064
+ return bad
1065
+
1066
+
1067
+ def _setups(s) -> List[str]:
1068
+ """``setups``: {original: {summary, models, default, served_as?, providers,
1069
+ side_models?, units?, turn_seconds?, paper?, comparable?}, model?: {summary, runs_as}}. ``original``'s
1070
+ ``models`` are the paper's, by the names the upstream's code knows them;
1071
+ ``side_models`` the ones its code asks besides the answerer (an extraction,
1072
+ a summary), sent as asked through served_as and the provider's prefix;
1073
+ ``providers`` the ones that serve them, each with the prefix its names
1074
+ carry there; ``model`` needs ``original`` (its pipeline) and asks as one
1075
+ of its models (``runs_as``)."""
1076
+ if s is None:
1077
+ return []
1078
+ if not isinstance(s, dict) or not s:
1079
+ return ["setups: an object, original and/or model"]
1080
+ bad = []
1081
+ for k, x in s.items():
1082
+ w = f"setups.{k}"
1083
+ if k not in SETUPS:
1084
+ bad.append(f"{w}: a setup is one of {SETUPS}")
1085
+ continue
1086
+ if not isinstance(x, dict):
1087
+ bad.append(f"{w}: an object")
1088
+ continue
1089
+ if not isinstance(x.get("summary"), str) or not x["summary"].strip():
1090
+ bad.append(f"{w}.summary: what runs, in a sentence")
1091
+ allowed = {"summary"}
1092
+ if k == "original":
1093
+ allowed |= {"models", "default", "served_as", "providers", "side_models", "units", "turn_seconds", "paper", "comparable", "reference_agent"}
1094
+ if "reference_agent" in x and (not isinstance(x["reference_agent"], str) or not x["reference_agent"].strip()):
1095
+ bad.append(f"{w}.reference_agent: the upstream agent's display name")
1096
+ models = x.get("models")
1097
+ if models is None: # any model: the upstream's leaderboard ranks models, its pipeline fixed
1098
+ models = []
1099
+ if "default" in x or "served_as" in x:
1100
+ bad.append(f"{w}: default and served_as come with models")
1101
+ elif not isinstance(models, list) or not models or not all(isinstance(m, str) and _MODEL_NAME.match(m) for m in models):
1102
+ bad.append(f"{w}.models: the paper's models, by the names the upstream's code knows - or no models: any")
1103
+ models = []
1104
+ elif x.get("default") not in models:
1105
+ bad.append(f"{w}.default: one of its models")
1106
+ side = x.get("side_models", [])
1107
+ if not isinstance(side, list) or not all(isinstance(m, str) and _MODEL_NAME.match(m) for m in side) \
1108
+ or len(set(side)) != len(side) or (side and "models" not in x):
1109
+ bad.append(f"{w}.side_models: the models the upstream's code asks besides its answerer, each once, beside models")
1110
+ side = []
1111
+ sa = x.get("served_as", {})
1112
+ if not isinstance(sa, dict) or not all(m in models + side and isinstance(v, str) and _MODEL_NAME.match(v) for m, v in sa.items()):
1113
+ bad.append(f"{w}.served_as: {{one of its models: the name the provider serves it as}}")
1114
+ pv = x.get("providers")
1115
+ if not isinstance(pv, dict) or not pv or not all(p in ("openrouter", "openai", "custom") and isinstance(v, str) for p, v in pv.items()):
1116
+ bad.append(f"{w}.providers: {{openrouter|openai|custom: the prefix its models' names carry there}}")
1117
+ u = x.get("units")
1118
+ if u is not None and (not isinstance(u, dict) or not all(isinstance(u.get(f), str) and u[f] for f in ("noun", "nouns"))):
1119
+ bad.append(f"{w}.units: {{noun, nouns}}, what one of its units is")
1120
+ t = x.get("turn_seconds")
1121
+ if t is not None and (isinstance(t, bool) or not isinstance(t, int) or t < 1):
1122
+ bad.append(f"{w}.turn_seconds: a whole number of seconds")
1123
+ if "paper" in x and (not isinstance(x["paper"], str) or not x["paper"].strip()):
1124
+ bad.append(f"{w}.paper: where the paper reports its numbers, in a phrase")
1125
+ if "comparable" in x and not isinstance(x["comparable"], bool):
1126
+ bad.append(f"{w}.comparable: false when no served model is the paper's, so its runs compare with each other only")
1127
+ else:
1128
+ allowed |= {"runs_as"}
1129
+ orig = s.get("original") if isinstance(s.get("original"), dict) else None
1130
+ if orig is None:
1131
+ bad.append(f"{w}: needs original, whose pipeline it runs")
1132
+ elif "models" not in orig:
1133
+ bad.append(f"{w}: the original takes any model already")
1134
+ elif x.get("runs_as") not in (orig.get("models") or []):
1135
+ bad.append(f"{w}.runs_as: one of original's models, the one whose window and batching it takes")
1136
+ extra = [f for f in x if f not in allowed]
1137
+ if extra:
1138
+ bad.append(f"{w}: {', '.join(extra)} is not a field of the {k} setup")
1139
+ return bad
1140
+
1141
+
1142
+ # Every setting of a benchmark's upstream, and what Nullius does with it (``upstream-settings.json`` in its folder).
1143
+ SETTINGS_FILE = "upstream-settings.json"
1144
+ _DISPOSITION = re.compile(r"^(option:[a-z0-9_]+|setup:(original|model)|dataset:[A-Za-z0-9_.-]+|fixed:.+ - .+|not offered - .+)$")
1145
+
1146
+
1147
+ def settings_problems(versions: List[dict]) -> List[str]:
1148
+ """What is wrong with a benchmark's settings inventory: every setting of
1149
+ every evaluation entry point of its upstream, one row each,
1150
+ ``{entry, setting, values, default, nullius}``, ``nullius`` exactly one of
1151
+ ``option:<name>``, ``setup:<original|model>``, ``dataset:<id>``,
1152
+ ``fixed:<value> - <why>`` or ``not offered - <why>``; an option, setup or
1153
+ dataset named must be one some version of the benchmark has. ``versions``:
1154
+ the benchmark resolved at each of its versions. Missing, it is a problem:
1155
+ a benchmark's settings are inventoried before it counts as imported."""
1156
+ if not versions:
1157
+ return []
1158
+ here = versions[0].get("_here") or ""
1159
+ path = os.path.join(here, SETTINGS_FILE)
1160
+ if not os.path.isfile(path):
1161
+ return [f"{SETTINGS_FILE}: none - every setting of the upstream's evaluation, and what Nullius does with each"]
1162
+ try:
1163
+ with open(path) as f:
1164
+ rows = json.load(f)
1165
+ except (OSError, ValueError) as e:
1166
+ return [f"{SETTINGS_FILE}: {e}"]
1167
+ if not isinstance(rows, list) or not rows:
1168
+ return [f"{SETTINGS_FILE}: a list of rows, one per setting"]
1169
+ options = {n for v in versions for n in (v.get("options") or {})}
1170
+ setups = {n for v in versions for n in (v.get("setups") or {})}
1171
+ datasets = {d.get("id") for v in versions for d in (v.get("datasets") or [])}
1172
+ bad, seen = [], set()
1173
+ for i, r in enumerate(rows):
1174
+ w = f"{SETTINGS_FILE}[{i}]"
1175
+ if not isinstance(r, dict) or not all(isinstance(r.get(k), str) and r[k].strip() for k in ("entry", "setting", "nullius")):
1176
+ bad.append(f"{w}: {{entry, setting, values, default, nullius}}, the first and last two non-empty")
1177
+ continue
1178
+ key = (r["entry"], r["setting"])
1179
+ if key in seen:
1180
+ bad.append(f"{w}: {r['entry']} {r['setting']} twice")
1181
+ seen.add(key)
1182
+ n = r["nullius"].strip()
1183
+ if not _DISPOSITION.match(n):
1184
+ bad.append(f"{w} ({r['setting']}): nullius is option:<name>, setup:<original|model>, dataset:<id>, "
1185
+ "fixed:<value> - <why> or not offered - <why>")
1186
+ elif n.startswith("option:") and n[7:] not in options:
1187
+ bad.append(f"{w} ({r['setting']}): no option {n[7:]} in any version")
1188
+ elif n.startswith("setup:") and n[6:] not in setups:
1189
+ bad.append(f"{w} ({r['setting']}): no {n[6:]} setup in any version")
1190
+ elif n.startswith("dataset:") and n[8:] not in datasets:
1191
+ bad.append(f"{w} ({r['setting']}): no dataset {n[8:]} in any version")
1192
+ return bad
1193
+
1194
+
1195
+ def setup(bench: dict, which: str, model: str = "", provider: str = "") -> tuple:
1196
+ """The setup ``which`` of ``bench`` as the other side of a run, with
1197
+ ``model`` on ``provider``: (definition, why not). The definition says the
1198
+ model the upstream's code is asked as (``asked``) and the one the relay
1199
+ sends (``sent``). ``original`` takes only the paper's models, on a
1200
+ provider that serves them; ``model`` takes any, asked as its ``runs_as``.
1201
+ (None, why) when the benchmark has no such setup or the model is refused."""
1202
+ s = bench.get("setups") or {}
1203
+ if which not in SETUPS or not isinstance(s.get(which), dict):
1204
+ return None, f"{bench['label']} has no {which} setup"
1205
+ orig, x = s["original"], s[which]
1206
+ model = (model or "").strip()
1207
+ side = []
1208
+ if which == "original" and "models" not in orig: # any model, sent as the provider lists it
1209
+ if not model:
1210
+ return None, f"the original {bench['label']} needs a model: its leaderboard ranks models"
1211
+ if provider and provider not in orig["providers"]:
1212
+ return None, f"the original runs on " + " or ".join(orig["providers"])
1213
+ asked = sent = model
1214
+ elif which == "original":
1215
+ model = model or orig["default"]
1216
+ if model not in orig["models"]:
1217
+ return None, (f"the original {bench['label']} runs only the paper's models ({', '.join(orig['models'])}): "
1218
+ f"{model} is not one - choose the model setup for another")
1219
+ if provider and provider not in orig["providers"]:
1220
+ return None, (f"the paper's models are not served by {provider}: the original runs on "
1221
+ + " or ".join(orig["providers"]))
1222
+ asked = model
1223
+ prefix = orig["providers"].get(provider, "") if provider else ""
1224
+ served = lambda m: prefix + (orig.get("served_as") or {}).get(m, m)
1225
+ sent = served(model)
1226
+ # the other models its code asks, each sent as the provider serves it; one the answerer is served as is its own
1227
+ side = [f"{m}={served(m)}" for m in orig.get("side_models") or [] if served(m) != sent]
1228
+ else:
1229
+ if not model:
1230
+ return None, f"the model setup needs a model: the one to put in {orig['default']}'s place"
1231
+ asked, sent = x["runs_as"], model
1232
+ units = orig.get("units") or bench.get("units") or {"noun": "unit", "nouns": "units"}
1233
+ return {"model": {"side_models": side}, "key": which, "label": f"{bench['label']} {setup_label(bench, which).lower()}",
1234
+ "summary": x["summary"], "asked": asked, "sent": sent, "units": units, "comparable": which == "original" and x.get("comparable", True),
1235
+ "turn_seconds": orig.get("turn_seconds"), "upstream": {}, "_kind": "upstream", "_ref": which,
1236
+ "_here": bench.get("_here", "")}, ""
1237
+
1238
+
1239
+ def problems(d: dict, kind: str) -> List[str]:
1240
+ """What is wrong with definition ``d`` of ``kind``; empty when it may be offered."""
1241
+ kind = names.kind(kind)
1242
+ d = names.definition(d)
1243
+ if kind not in KINDS:
1244
+ return [f"{kind!r} is not a kind of definition"]
1245
+ if not isinstance(d, dict):
1246
+ return ["a JSON object"]
1247
+ if "versions" in d:
1248
+ return _versioned(d, kind)
1249
+ bad = []
1250
+ if d.get("schema") != KINDS[kind]["schema"]:
1251
+ bad.append(f"schema: {KINDS[kind]['schema']}")
1252
+ if not _KEY.match(str(d.get("key", ""))):
1253
+ bad.append("key: lower case letters, digits and dashes")
1254
+ elif kind != "benchmark" and d.get("key") in CONTROLS:
1255
+ bad.append(f"key: {d['key']} is the word for a benchmark's control")
1256
+ elif kind != "benchmark" and d.get("key") in SETUPS:
1257
+ bad.append(f"key: {d['key']} is the word for a benchmark's setup")
1258
+ for f in ("label", "summary"):
1259
+ if not isinstance(d.get(f), str) or not d[f].strip():
1260
+ bad.append(f"{f}: say it")
1261
+ if "example" in d and (kind == "benchmark" or not isinstance(d["example"], bool)):
1262
+ bad.append("example: true or false, an agent's or a capability's")
1263
+ bad += _upstream(d.get("upstream"))
1264
+ if kind == "benchmark":
1265
+ if d.get("family") not in FAMILIES:
1266
+ bad.append(f"family: one of {FAMILIES}")
1267
+ needs = d.get("needs")
1268
+ bad += _ports("needs", needs, d.get("family"))
1269
+ if isinstance(needs, list) and not needs:
1270
+ bad.append("needs: at least one port")
1271
+ bad += _host(d)
1272
+ bad += _files(d)
1273
+ bad += _bench_options(d)
1274
+ drv = d.get("driver") or {}
1275
+ bad += _command("driver.command", drv.get("command") if isinstance(drv, dict) else None, {"here", "install"})
1276
+ if isinstance(drv, dict):
1277
+ if "install" in drv:
1278
+ bad += _install(dict(driver_install(d), install=drv["install"]), "driver.install")
1279
+ elif any("{install}" in w for w in drv.get("command") or [] if isinstance(w, str)):
1280
+ bad.append("driver.command: {install} names driver.install, which is not set")
1281
+ bad += _mcp_servers(d.get("mcp_servers"), isinstance(drv, dict) and "install" in drv)
1282
+ ds = d.get("datasets")
1283
+ if not isinstance(ds, list) or not ds:
1284
+ bad.append("datasets: at least one")
1285
+ else:
1286
+ ids = set()
1287
+ for i, x in enumerate(ds):
1288
+ w = f"datasets[{i}]"
1289
+ if not isinstance(x, dict):
1290
+ bad.append(f"{w}: an object")
1291
+ continue
1292
+ if not _ID.match(str(x.get("id", ""))) or x.get("id") in ids:
1293
+ bad.append(f"{w}.id: a unique plain name")
1294
+ ids.add(x.get("id"))
1295
+ if x.get("path") is not None and x.get("url") is not None:
1296
+ bad.append(f"{w}: path (in the upstream) or url (downloaded), not both")
1297
+ if x.get("path") is not None:
1298
+ if not _RELATIVE.match(str(x["path"])):
1299
+ bad.append(f"{w}.path: a relative path")
1300
+ if x.get("url") is not None and not _HTTPS.match(str(x["url"])):
1301
+ bad.append(f"{w}.url: an https address, pinned (a fixed revision, not main)")
1302
+ if (x.get("path") is not None or x.get("url") is not None) and not _SHA256.match(str(x.get("sha256", ""))):
1303
+ bad.append(f"{w}.sha256: the file's digest")
1304
+ if not isinstance(x.get("units"), int) or x["units"] < 1:
1305
+ bad.append(f"{w}.units: how many units it holds")
1306
+ u = d.get("units") or {}
1307
+ if not (isinstance(u.get("noun"), str) and isinstance(u.get("nouns"), str)):
1308
+ bad.append("units: {noun, nouns}")
1309
+ if "redistribute" in u and not isinstance(u["redistribute"], bool):
1310
+ bad.append("units.redistribute: false when the data may not be redistributed")
1311
+ lim = d.get("limits")
1312
+ if lim is not None:
1313
+ if not isinstance(lim, dict):
1314
+ bad.append("limits: an object")
1315
+ else:
1316
+ for f in ("turn_seconds", "turns"):
1317
+ v = lim.get(f)
1318
+ if v is not None and (isinstance(v, bool) or not isinstance(v, int) or not 1 <= v <= 100000):
1319
+ bad.append(f"limits.{f}: a whole number from 1")
1320
+ a = lim.get("attempts")
1321
+ if a is not None and (not isinstance(a, str) or not a.strip()):
1322
+ bad.append("limits.attempts: how the upstream tells one attempt of a unit from another (its seed per trial), "
1323
+ "in a sentence - the driver is then sent attempt with setup, turn and answer")
1324
+ s = d.get("score") or {}
1325
+ if s.get("kind") not in SCORES or not s.get("says"):
1326
+ bad.append(f"score: {{kind: one of {SCORES}, says}}")
1327
+ if "runs_code" in s and not isinstance(s["runs_code"], bool):
1328
+ bad.append("score.runs_code: true or false")
1329
+ if "kit_version" in d and (isinstance(d["kit_version"], bool) or not isinstance(d["kit_version"], int) or d["kit_version"] < 1):
1330
+ bad.append("kit_version: a kit's version, a whole number from 1")
1331
+ bad += _skills(d)
1332
+ j = s.get("judge")
1333
+ if j is not None and (not isinstance(j, dict) or not _MODEL_NAME.match(str(j.get("model", ""))) or not j.get("says")):
1334
+ bad.append("score.judge: {model: the official judge, as the provider names it; says: what it is asked}")
1335
+ sim = d.get("simulator")
1336
+ if sim is not None:
1337
+ if not isinstance(sim, dict) or not _MODEL_NAME.match(str(sim.get("model", ""))) or not sim.get("says"):
1338
+ bad.append("simulator: {model: the official simulated user's model, as the provider names it; says: whom it plays}")
1339
+ elif "agent.conversation/v1" not in (needs if isinstance(needs, list) else []):
1340
+ bad.append("simulator: only a benchmark that needs agent.conversation/v1 has a user to simulate")
1341
+ bad += _controls(d.get("controls"))
1342
+ bad += _leaderboard(d)
1343
+ bad += _setups(d.get("setups"))
1344
+ bad += _environments(d.get("environments"), d.get("_here"))
1345
+ envs = d.get("environments") if isinstance(d.get("environments"), dict) else {}
1346
+ if d.get("family") == "capability":
1347
+ bad += [f"environments.{n}.image: an own image holds an agent's work; a capability benchmark has none"
1348
+ for n, e in envs.items() if own_image(e)]
1349
+ for i, x in enumerate(ds if isinstance(ds, list) else []):
1350
+ if isinstance(x, dict) and "environment" in x and x["environment"] not in envs:
1351
+ bad.append(f"datasets[{i}].environment: one of environments ({', '.join(envs) or 'none declared'})")
1352
+ if d.get("proven") is not None and not isinstance(d.get("proven"), dict):
1353
+ bad.append("proven: an object, per dataset, written by cli prove")
1354
+ elif kind == "agent":
1355
+ st = d.get("start") or {}
1356
+ if ("acp" in st) == ("command" in st):
1357
+ bad.append("start: acp or command, one of them")
1358
+ elif "acp" in st:
1359
+ bad += _command("start.acp", st["acp"], {"here", "install"})
1360
+ else:
1361
+ bad += _command("start.command", st["command"])
1362
+ if not any("{prompt}" in w for w in st.get("command") or []):
1363
+ bad.append("start.command: carries {prompt}")
1364
+ extra = [p for p in d.get("offers") or [] if p not in ("agent.task/v1", "agent.workspace/v1", "agent.search/v1")]
1365
+ if extra:
1366
+ bad.append(f"offers: a command agent can offer only agent.task/v1, agent.workspace/v1 and agent.search/v1, not {extra}")
1367
+ bad += _ports("offers", d.get("offers") or [], "agent")
1368
+ only = d.get("only_for")
1369
+ if only is not None and (not isinstance(only, list) or not only or len(set(map(str, only))) != len(only)
1370
+ or not all(isinstance(k, str) and _KEY.match(k) for k in only)):
1371
+ bad.append("only_for: the keys of the benchmarks it is the reference agent of, each once")
1372
+ bad += _proven(d)
1373
+ bad += _install(d)
1374
+ bad += _env(d.get("env"))
1375
+ bad += _model(d.get("model") or {}, d.get("_here"))
1376
+ bad += _options(d)
1377
+ bad += _caches(d)
1378
+ fid = d.get("fidelity")
1379
+ if fid is not None:
1380
+ if not isinstance(fid, dict):
1381
+ bad.append("fidelity: {command: the agent's own non-interactive command, carrying {prompt}}")
1382
+ else:
1383
+ bad += _command("fidelity.command", fid.get("command"))
1384
+ if not any("{prompt}" in w for w in fid.get("command") or [] if isinstance(w, str)):
1385
+ bad.append("fidelity.command: carries {prompt}")
1386
+ else:
1387
+ if not _KEY.match(str(d.get("kind", ""))):
1388
+ bad.append("kind: what kind of capability (memory, ...)")
1389
+ bad += _command("start.command", (d.get("start") or {}).get("command"), {"here", "install"})
1390
+ bad += _ports("offers", d.get("offers") or [], "capability")
1391
+ bad += _proven(d)
1392
+ bad += _install(d)
1393
+ bad += _env(d.get("env"))
1394
+ bad += _options(d)
1395
+ bad += _caches(d)
1396
+ if d.get("model") is not None: # a capability that calls a model of its own: the relay, as an agent's
1397
+ bad += _model(d["model"], d.get("_here"))
1398
+ for p in d.get("offers") or []:
1399
+ if ports.known(p) and p.split(".", 1)[1].split("/")[0] != d.get("kind"):
1400
+ bad.append(f"offers {p}, which is not a port of its kind ({d.get('kind')})")
1401
+ return bad
1402
+
1403
+
1404
+ def entry_dir(kind: str, key: str, root: Optional[str] = None) -> Optional[str]:
1405
+ """The selected entry folder, or None; a user folder wins in its entirety."""
1406
+ return storage.entry_dir(KINDS[names.kind(kind)]["dir"], key, CATALOGUE, root)
1407
+
1408
+
1409
+ def writable_dir(kind: str, key: str, root: Optional[str] = None) -> str:
1410
+ """The user folder, copying all bundled files before an edit if needed."""
1411
+ if not _KEY.fullmatch(key):
1412
+ raise ValueError("a key is lower case letters, digits and dashes")
1413
+ return storage.writable_dir(KINDS[names.kind(kind)]["dir"], key, CATALOGUE, root)
1414
+
1415
+
1416
+ def _read(path: str, kind: str, key: str) -> Tuple[Optional[dict], List[str]]:
1417
+ try:
1418
+ with open(path) as f:
1419
+ d = names.definition(json.load(f))
1420
+ except (OSError, ValueError, TypeError, AttributeError) as e:
1421
+ return None, [f"{kind} {key}: {e}"]
1422
+ try:
1423
+ found = problems(dict(d, _here=os.path.dirname(path)) if isinstance(d, dict) else d, kind)
1424
+ except Exception as e: # a field of the wrong type the checks do not expect: left out, said
1425
+ found = [f"malformed ({type(e).__name__}: {e}); a field holds a value of the wrong type"]
1426
+ bad = [f"{kind} {key}: {p}" for p in found]
1427
+ if isinstance(d, dict) and d.get("key") != key:
1428
+ bad.append(f"{kind} {key}: key {d.get('key')!r} is not its folder's name")
1429
+ if bad:
1430
+ return None, bad
1431
+ d = dict(d, _here=os.path.dirname(path), _kind=kind)
1432
+ return d, []
1433
+
1434
+
1435
+ def load(kind: str, root: Optional[str] = None, every: bool = False) -> Tuple[List[dict], List[str]]:
1436
+ """Every definition of ``kind``, in folder order, and errors in those left out.
1437
+ With no root the user catalogue overlays the bundle, by whole folder;
1438
+ an explicit root reads that tree alone. An invalid override never falls
1439
+ back to the bundled definition.
1440
+ A loaded definition carries ``_here``, its folder, and ``_kind``, and is
1441
+ resolved (:func:`resolve`) to its default version, or, with ``every``, to
1442
+ each of its versions in turn - a benchmark as an agent or a capability."""
1443
+ kind = names.kind(kind)
1444
+ name = KINDS[kind]["file"]
1445
+ selected = storage.folders(KINDS[names.kind(kind)]["dir"], CATALOGUE, root)
1446
+ out, bad = [], []
1447
+ for n, folder in selected.items():
1448
+ path = definition_path(folder, kind)
1449
+ if not os.path.isfile(path):
1450
+ if kind != "benchmark": # a Harbor-only benchmark needs no standard definition
1451
+ bad.append(f"{kind} {n}: missing {name}")
1452
+ continue
1453
+ d, why = _read(path, kind, n)
1454
+ bad += why
1455
+ if d:
1456
+ out += [resolve(d, v) for v in versions(d)] if every else [resolve(d)]
1457
+ return out, bad
1458
+
1459
+
1460
+ def get(kind: str, ref: str, root: Optional[str] = None) -> Optional[dict]:
1461
+ """One definition by ``key`` (its default version) or ``key@version``, resolved."""
1462
+ kind = names.kind(kind)
1463
+ key, version = split(ref)
1464
+ if not _KEY.match(key or ""):
1465
+ return None
1466
+ folder = entry_dir(kind, key, root)
1467
+ if folder is None:
1468
+ return None
1469
+ path = definition_path(folder, kind)
1470
+ if not os.path.isfile(path):
1471
+ return None
1472
+ d, _ = _read(path, kind, key)
1473
+ return None if d is None else resolve(d, version)
1474
+
1475
+
1476
+ def kits() -> List[dict]:
1477
+ """The conformance kits: benchmarks kept in this package, one per port."""
1478
+ out = []
1479
+ for n in sorted(os.listdir(KITS)):
1480
+ path = os.path.join(KITS, n, "standard.json")
1481
+ if os.path.isfile(path):
1482
+ d, bad = _read(path, "benchmark", n)
1483
+ if bad:
1484
+ raise ValueError("; ".join(bad))
1485
+ out.append(d)
1486
+ return out
1487
+
1488
+
1489
+ def kit(port: str) -> Optional[dict]:
1490
+ name = (ports.PORTS.get(names.port(port)) or {}).get("kit")
1491
+ return next((k for k in kits() if k["key"] == name), None) if name else None
1492
+
1493
+
1494
+ # A kit is made stronger under the same port - more checks of the same
1495
+ # contract - and says so by its ``kit_version``; each proof records the one it
1496
+ # passed. A proof on an older kit stays a proof (a kit's change un-proves no
1497
+ # one), but it is said to be one, so a re-run can be asked for.
1498
+ def kit_version(k: Optional[dict]) -> int:
1499
+ """A kit's version; 1 for a kit that does not say."""
1500
+ return int((k or {}).get("kit_version") or 1)
1501
+
1502
+
1503
+ def proof_kit(record: Optional[dict]) -> int:
1504
+ """The kit version a ``proven`` record passed; 1 for a record from before
1505
+ kits had versions."""
1506
+ return int((record or {}).get("kit_version") or 1)
1507
+
1508
+
1509
+ def on_current_kit(port: str, record: Optional[dict]) -> bool:
1510
+ """Whether ``record``, a proof of ``port``, was made on its kit's current version."""
1511
+ return proof_kit(record) >= kit_version(kit(port))
1512
+
1513
+
1514
+ def older_kit(port: str, record: Optional[dict]) -> Optional[str]:
1515
+ """``proven on kit v1; kit v2 is current`` for a proof on an older kit, else None."""
1516
+ if not record or on_current_kit(port, record):
1517
+ return None
1518
+ return f"proven on kit v{proof_kit(record)}; kit v{kit_version(kit(port))} is current"
1519
+
1520
+
1521
+ # A kit's checks are of two sorts. Its **basic** checks are the port itself:
1522
+ # the port is proven when every one passes. Its **skills** (``skills``: id ->
1523
+ # what it shows, in plain words) are harder uses of the same contract - an
1524
+ # agent that misses one still has the port, and its proof says which skills
1525
+ # it showed; a benchmark that relies on one says so (``needs_skills``). A
1526
+ # check a kit does not name a skill is basic, so a kit without ``skills``
1527
+ # is every check basic, as before skills existed.
1528
+ def skills_of(k: Optional[dict]) -> dict:
1529
+ """A kit's skills, {id: what it shows}; empty for a kit without."""
1530
+ s = (k or {}).get("skills")
1531
+ return dict(s) if isinstance(s, dict) else {}
1532
+
1533
+
1534
+ def port_skills(port: str) -> dict:
1535
+ """The skills of ``port``'s kit, {id: what it shows}."""
1536
+ return skills_of(kit(port))
1537
+
1538
+
1539
+ def skills_shown(port: str, record: Optional[dict]) -> List[str]:
1540
+ """The skills of ``port`` that ``record``, its proof, showed. A proof from
1541
+ before skills were recorded passed every check of its kit - all of them
1542
+ basic then - so on the current kit it showed every skill the kit has; on
1543
+ an older kit, none can be read from it."""
1544
+ if not record:
1545
+ return []
1546
+ if isinstance(record.get("skills"), dict):
1547
+ return [s for s, r in record["skills"].items() if isinstance(r, dict) and r.get("ok")]
1548
+ return list(port_skills(port)) if on_current_kit(port, record) else []
1549
+
1550
+
1551
+ def _skills(d: dict) -> List[str]:
1552
+ """A kit's ``basic`` and ``skills``, a benchmark's ``needs_skills``."""
1553
+ bad = []
1554
+ basic, skills = d.get("basic"), d.get("skills")
1555
+ if basic is not None and (not isinstance(basic, list) or not all(isinstance(x, str) and _ID.match(x) for x in basic)):
1556
+ bad.append("basic: a list of the kit's check ids that are the port itself")
1557
+ if skills is not None:
1558
+ if not isinstance(skills, dict) or not all(_ID.match(str(k)) and isinstance(v, str) and v.strip() for k, v in skills.items()):
1559
+ bad.append("skills: {check id: what the skill shows, in plain words}")
1560
+ elif isinstance(basic, list) and set(basic) & set(skills):
1561
+ bad.append(f"skills: {', '.join(sorted(set(basic) & set(skills)))} cannot be a basic check and a skill")
1562
+ want = d.get("needs_skills")
1563
+ if want is None:
1564
+ return bad
1565
+ if not isinstance(want, dict):
1566
+ return bad + ["needs_skills: {port: [skill ids of its kit]}"]
1567
+ for p, ids in want.items():
1568
+ w = f"needs_skills.{p}"
1569
+ if p not in (d.get("needs") or []):
1570
+ bad.append(f"{w}: a port the benchmark needs")
1571
+ elif not isinstance(ids, list) or not ids or not all(isinstance(x, str) for x in ids):
1572
+ bad.append(f"{w}: a list of skill ids")
1573
+ else:
1574
+ have = port_skills(p)
1575
+ bad += [f"{w}: {x!r} is not a skill of its kit ({', '.join(have) or 'it has none'})" for x in ids if x not in have]
1576
+ return bad
1577
+
1578
+
1579
+ def definition_path(folder: str, kind: str) -> str:
1580
+ kind = names.kind(kind)
1581
+ path = os.path.join(folder, KINDS[kind]["file"])
1582
+ legacy = os.path.join(folder, "feature.json")
1583
+ return legacy if kind == "capability" and not os.path.exists(path) and os.path.isfile(legacy) else path
1584
+
1585
+
1586
+ def path_of(d: dict) -> str:
1587
+ return definition_path(d["_here"], d["_kind"])
1588
+
1589
+
1590
+ @contextlib.contextmanager
1591
+ def _locked(path: str):
1592
+ """One writer of a definition file at a time, across processes (two
1593
+ proofs, a conformance check and the user's own edit through the app)."""
1594
+ import fcntl
1595
+ where = os.path.join(paths.cache_dir(), "locks")
1596
+ os.makedirs(where, exist_ok=True)
1597
+ with open(os.path.join(where, hashlib.sha256(os.path.realpath(path).encode()).hexdigest()[:24]), "w") as f:
1598
+ fcntl.flock(f, fcntl.LOCK_EX)
1599
+ try:
1600
+ yield
1601
+ finally:
1602
+ fcntl.flock(f, fcntl.LOCK_UN)
1603
+
1604
+
1605
+ def save(d: dict, change: Optional[Callable[[dict], None]] = None) -> None:
1606
+ """Write a loaded definition back (the conformance run's ``offers`` and
1607
+ ``proven``), without the fields loading added. A version of a file with
1608
+ ``versions`` writes only its own ``proven``; the rest of the file is kept.
1609
+
1610
+ ``change``: what the run decided, applied to the definition as it is on
1611
+ disk now, under a lock - not to the copy loaded when a long run started,
1612
+ which would put back what another proof or the user wrote meanwhile. It
1613
+ is given the dict holding ``offers`` and ``proven`` (a version's own, for
1614
+ a version) and changes it in place; ``d`` then holds the result too."""
1615
+ # Explicit fixture roots remain local. Only a loaded bundled entry needs
1616
+ # copying; helpers and model files beside its definition must follow it.
1617
+ folder = d["_here"]
1618
+ bundled = os.path.join(CATALOGUE, KINDS[d["_kind"]]["dir"], d["key"])
1619
+ if os.path.realpath(folder) == os.path.realpath(bundled):
1620
+ d["_here"] = writable_dir(d["_kind"], d["key"])
1621
+ storage.ensure_writable(d["_here"], CATALOGUE)
1622
+ path = path_of(d)
1623
+ with _locked(path):
1624
+ versioned = bool(d.get("_ref") and "@" in str(d["_ref"]))
1625
+ if versioned or change is not None:
1626
+ try:
1627
+ with open(path) as f:
1628
+ clean = names.definition(json.load(f))
1629
+ except (OSError, ValueError):
1630
+ clean = None
1631
+ if not isinstance(clean, dict):
1632
+ clean = {k: v for k, v in d.items() if not k.startswith("_")}
1633
+ if versioned: # a version writes its own proven only, seen beside the offers it resolves to
1634
+ target = clean["versions"][d["_version"]]
1635
+ work = {"offers": d.get("offers") or [], "proven": target.get("proven") or {}}
1636
+ if change is not None:
1637
+ change(work)
1638
+ else:
1639
+ work["proven"] = d.get("proven") or {}
1640
+ target["proven"] = d["proven"] = work["proven"]
1641
+ else:
1642
+ change(clean)
1643
+ for k in ("offers", "proven"):
1644
+ if k in clean:
1645
+ d[k] = clean[k]
1646
+ else:
1647
+ clean = {k: v for k, v in d.items() if not k.startswith("_")}
1648
+ with open(path + ".part", "w") as f:
1649
+ json.dump(clean, f, indent=2)
1650
+ f.write("\n")
1651
+ os.replace(path + ".part", path)