nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,1781 @@
1
+ """A run: a benchmark, an agent or a capability, a model - units, then a job.
2
+
3
+ The plan is units × stages. The driver names the units (``units``); each is
4
+ set up (``setup``: its workspace filled, its prompt and environment
5
+ returned), produced - the agent driven over ACP in that workspace, or the
6
+ capability driven through its port - and scored (``score``). Units run
7
+ ``concurrency`` at a time, each with a driver process per worker, an agent or
8
+ capability process of its own, a home of its own, and a relay of its own.
9
+
10
+ A relay per unit, not one for the run: a shared relay is one more thing whose
11
+ failure stops every unit, and a network hop for a unit sandboxed elsewhere.
12
+ One per unit costs a few megabytes and loses only a shared rate limit - each
13
+ unit backs off on its own, as it always did. Its record sits in the trial's
14
+ ``.work/relay``.
15
+
16
+ Two backends (``backend``). ``local``: processes on this machine, which are
17
+ not a sandbox - what an agent runs, it runs as the user (``docs/standard.md``,
18
+ "Safety"). ``docker``: the agent or the capability in a container of its own on
19
+ an internal network with the unit's relay beside it (``container``); an
20
+ own-image task may grant a public-address proxy with ``internet: true``, and
21
+ an agent whose environment declares ``network`` gets a public web proxy.
22
+ On docker the agent's or capability's packages are installed in the base image, on the libc and tools its container runs on.
23
+ The driver, its environment installed here, stays on this machine either
24
+ way: it owns the answers. So do a benchmark's own MCP servers
25
+ (``mcp_servers``), in the driver's environment: on the local backend the
26
+ agent starts each from it, in a container the run does, behind a door on the
27
+ unit's network (``mcp_machine``).
28
+
29
+ A unit whose environment is an **own image** (``"image": "own"``) runs in
30
+ the task's own container, whatever the backend (a control's too: nothing of
31
+ an agent runs there; an agent's needs the container backend): its setup names
32
+ the image (``image``: a folder of the upstream to build, or a reference by
33
+ digest) and, optionally, the working directory (``workdir``; the image's own
34
+ by default) and a check (``verify``) - files of the upstream copied in, a
35
+ command run, a folder of results copied out - which runs in that container
36
+ after the agent ends and before ``score``, which is sent what it found
37
+ (``verified``: {exit, timed_out, output, dir, collected}) and reads the rest from ``dir``. The
38
+ oracle's reference solution is such a step too (its reply's ``solve``). So
39
+ the driver never needs docker, and the task's tests reach the container only
40
+ once the agent is done and nothing it started still runs there
41
+ (``container.Box``).
42
+
43
+ What a process is given of this machine's environment is a short list
44
+ (``KEEP``) - never the keys Settings holds. The relay holds the real key;
45
+ everything else gets its address and ``relay.PLACEHOLDER_KEY``.
46
+
47
+ An agent run on the local backend starts only on the caller's explicit yes
48
+ (``unsandboxed``); on either backend its permission requests are granted
49
+ (``acp.Client(grant=True)``): no one is there to answer them during a run.
50
+ The job's config records both, and the backend.
51
+ """
52
+ from __future__ import annotations
53
+
54
+ import hashlib
55
+ import json
56
+ import os
57
+ import re
58
+ import shutil
59
+ import signal
60
+ import stat
61
+ import subprocess
62
+ import threading
63
+ import time
64
+ import traceback
65
+ import tempfile
66
+ from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed
67
+ from contextlib import contextmanager
68
+ from typing import Callable, Dict, List, Optional
69
+
70
+ from nullius import paths, prices as price_list, progress, providers
71
+ from nullius.redact import redact
72
+ from nullius.harbor import feed as live_feed
73
+ from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards
74
+
75
+ # the totals a unit's end carries onto the live board, from its trial's agent_result
76
+ FEED_TOTALS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd")
77
+
78
+ # The variables of this machine's environment a run's processes keep: where
79
+ # programs are, the locale, the temporary folder. Not a key among them.
80
+ KEEP = ("PATH", "LANG", "LC_ALL", "TMPDIR", "TERM", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR")
81
+ APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # the folder holding the nullius package
82
+
83
+
84
+ def driver_path() -> str:
85
+ """A driver's ``PYTHONPATH``: a folder whose one entry links to the ``nullius`` package, for the
86
+ kits' drivers that import it. Not ``APP``: installed from a wheel, that is the tool's whole
87
+ site-packages, and every package there would come before the driver's own environment's."""
88
+ pkg = os.path.join(APP, "nullius")
89
+ root = os.path.join(paths.cache_dir(), "driver-path", hashlib.sha256(pkg.encode()).hexdigest()[:16])
90
+ link = os.path.join(root, "nullius")
91
+ if os.path.realpath(link) != os.path.realpath(pkg):
92
+ os.makedirs(root, exist_ok=True)
93
+ # written whole beside the folder, then renamed in: drivers start in parallel, and the folder
94
+ # on their PYTHONPATH never holds anything but the link
95
+ part = f"{root}.{os.getpid()}.{threading.get_ident()}.part"
96
+ os.symlink(pkg, part)
97
+ os.replace(part, link)
98
+ return root
99
+
100
+ PROVIDERS = {
101
+ "openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"},
102
+ "openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"},
103
+ "anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"},
104
+ "custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"},
105
+ }
106
+ TURN_SECONDS = 900
107
+ DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each
108
+ IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image)
109
+ MAX_AGENT_SECONDS = 6 * 3600 # the longest a unit's own agent limit (setup's agent_seconds) may ask
110
+ MAX_TURNS = 20
111
+ BACKENDS = definitions.names.BACKENDS
112
+ _PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe)
113
+
114
+
115
+ class Refused(Exception):
116
+ """The run cannot start: the pair does not fit, or a setting is missing."""
117
+
118
+
119
+ def provider_endpoint(provider: str, env=os.environ) -> tuple:
120
+ p = PROVIDERS.get(provider)
121
+ if not p:
122
+ raise Refused(f"provider {provider!r} is not one of {list(PROVIDERS)}")
123
+ base = p.get("base") or env.get(p["base_var"]) or ""
124
+ key = env.get(p["key"]) or ("local" if provider == "custom" else "")
125
+ if not base:
126
+ raise Refused(f"{p['base_var']} is not set in Settings")
127
+ if not key:
128
+ raise Refused(f"{p['key']} is not set in Settings")
129
+ if not key.isprintable() or any(ord(c) > 255 for c in key) or key != key.strip():
130
+ # http.client refuses such a value, and quotes it whole in its error
131
+ raise Refused(f"{p['key']} holds a character a request header cannot carry (a line break, a space or a "
132
+ "byte-order mark around it, or one outside Latin-1): set it again in Settings")
133
+ return base, key
134
+
135
+
136
+ def _slug(s: str) -> str:
137
+ return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit"
138
+
139
+
140
+ def driver_secrets(bench: dict, licences) -> dict:
141
+ """The set secrets a benchmark's environments send to its driver (``to: driver``)."""
142
+ out = {}
143
+ for e in (bench.get("environments") or {}).values():
144
+ out.update(definitions.secret_values(e, licences, "driver"))
145
+ return out
146
+
147
+
148
+ def option_env(values: dict) -> dict:
149
+ """A benchmark's option values as its driver reads them: ``NULLIUS_OPTION_<NAME>``."""
150
+ return {"NULLIUS_OPTION_" + n.upper(): str(v) for n, v in (values or {}).items()}
151
+
152
+
153
+ # The endpoint refusing the run itself, not one unit: a key it rejects, a model it does not know.
154
+ REFUSALS = ("AgentAuthenticationError", "ModelNotFoundError")
155
+ REFUSED_IN_A_ROW = 3
156
+ # a unit still running when the run is stopped: Harbor's name for a trial a stopped job cut short
157
+ CANCELLED = "CancelledError"
158
+
159
+
160
+ def run_group(cmd: List[str], timeout: float, **kw) -> subprocess.CompletedProcess:
161
+ """``subprocess.run`` for an agent's command, in a session of its own, and
162
+ the whole group ended afterwards - on a timeout, which kills only the
163
+ direct child, and at a normal end, where whatever it left in the
164
+ background would keep running, still writing into the workspace while
165
+ it is scored. Raises ``subprocess.TimeoutExpired`` as ``run`` does.
166
+ The agent ending is what is waited for, not its output's end: a child
167
+ left in the background holding stdout would otherwise hold the unit
168
+ until its timeout."""
169
+ stopped = kw.pop("stopped", None)
170
+ streams = {}
171
+ try:
172
+ for name in ("stdout", "stderr"):
173
+ if kw.get(name) == subprocess.PIPE:
174
+ streams[name] = tempfile.TemporaryFile()
175
+ kw[name] = streams[name]
176
+ p = subprocess.Popen(cmd, start_new_session=True, **kw)
177
+ deadline = time.monotonic() + timeout
178
+ try:
179
+ while p.poll() is None:
180
+ if stopped is not None and stopped.is_set():
181
+ raise CancelledError("the run was stopped")
182
+ remaining = deadline - time.monotonic()
183
+ if remaining <= 0:
184
+ raise subprocess.TimeoutExpired(cmd, timeout)
185
+ try:
186
+ p.wait(timeout=min(remaining, 0.1))
187
+ except subprocess.TimeoutExpired:
188
+ pass
189
+ finally:
190
+ _kill_group(p)
191
+ p.wait()
192
+ output = {}
193
+ for name, stream in streams.items():
194
+ stream.seek(0)
195
+ data = stream.read()
196
+ if kw.get("text") or kw.get("universal_newlines") or kw.get("encoding"):
197
+ data = data.decode(kw.get("encoding") or "utf-8", errors=kw.get("errors") or "strict")
198
+ output[name] = data
199
+ return subprocess.CompletedProcess(cmd, p.returncode, output.get("stdout"), output.get("stderr"))
200
+ finally:
201
+ for stream in streams.values():
202
+ stream.close()
203
+
204
+
205
+ def _kill_group(p: subprocess.Popen) -> None:
206
+ try:
207
+ os.killpg(p.pid, signal.SIGKILL)
208
+ except (ProcessLookupError, PermissionError):
209
+ pass
210
+
211
+
212
+ def base_env(home: str, extra: Optional[dict] = None) -> dict:
213
+ env = {k: os.environ[k] for k in KEEP if k in os.environ}
214
+ env.setdefault("PATH", os.defpath)
215
+ env["HOME"] = home
216
+ env.update(extra or {})
217
+ return env
218
+
219
+
220
+ def _render(template: str, slots: Dict[str, str]) -> str:
221
+ for k, v in slots.items():
222
+ template = template.replace("{" + k + "}", v)
223
+ return template
224
+
225
+
226
+ def agent_env(agent: dict, inst: str, home: str, base_url: str, model: str, options: Optional[dict] = None) -> dict:
227
+ """The environment of an agent - or of a capability that calls a model - and
228
+ its config files written into its home, from its definition's ``model``
229
+ templates; then its own ``env`` (``install.process_env``). ``options``:
230
+ the run's value of each option (``definitions.options_of``), the slots
231
+ ``{opt_<name>}``; an option left out is filled with its default."""
232
+ options = definitions.options_of(agent, options)[0]
233
+ slots = {"base_url": base_url, "key": relay.PLACEHOLDER_KEY, "model": model, "home": home,
234
+ **{"opt_" + k: v for k, v in options.items()}}
235
+ m = agent.get("model") or {}
236
+ for dest, src in (m.get("files") or {}).items():
237
+ path = _render(dest, slots)
238
+ os.makedirs(os.path.dirname(path), exist_ok=True)
239
+ with open(os.path.join(agent["_here"], src)) as f:
240
+ text = _render(f.read(), slots)
241
+ with open(path, "w") as f:
242
+ f.write(text)
243
+ env = base_env(home, {k: _render(v, slots) for k, v in (m.get("env") or {}).items()})
244
+ env.update(install.process_env(agent, inst, home, options))
245
+ env["PATH"] = os.pathsep.join(install.bin_dirs(inst) + [env["PATH"]])
246
+ return env
247
+
248
+
249
+ def _own_dir(path: str) -> None:
250
+ """``path`` as a folder this user alone owns: made 0700 when absent; when
251
+ there already - a shared place like /dev/shm, where another user could
252
+ have made it first, or put a link there - kept only if it is a real
253
+ folder, this user's, open to no one else. Refused otherwise."""
254
+ try:
255
+ os.mkdir(path, 0o700)
256
+ except FileExistsError:
257
+ pass
258
+ st = os.lstat(path)
259
+ if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or st.st_mode & 0o077:
260
+ raise Refused(f"{path} is not a folder of this user's alone: no licence is written there")
261
+
262
+
263
+ class Licences:
264
+ """What a run holds of its benchmark's licences, by name: ``get(name)``
265
+ is {kind, value} for a key or a server, {kind, path} for a file, or None.
266
+
267
+ On this machine they are read from the benchmark's store
268
+ (``nullius.secrets``) when asked - when a unit starts - and a file is
269
+ used where it is kept; a data provider's token (``provider``) from that
270
+ provider's store alone. A run on a host is handed them instead
271
+ (``handed``): its launcher's stdin, a file written to a folder in memory
272
+ (``/dev/shm``) that the run removes when it ends. Only the names the
273
+ benchmark's environments declare are ever looked up, each as the kind
274
+ it is declared."""
275
+
276
+ def __init__(self, bench: dict, given: Optional[Dict[str, dict]] = None):
277
+ self.key = bench["key"]
278
+ declared = definitions.secrets_of(bench)
279
+ self.kinds = {s["name"]: s["kind"] for s in declared}
280
+ self.providers = {s["name"]: s.get("provider") or "" for s in declared}
281
+ self.given = given
282
+
283
+ @classmethod
284
+ def handed(cls, bench: dict, values: Dict[str, bytes], spill: str) -> "Licences":
285
+ """The licences a host's run was sent (name -> bytes), a file among
286
+ them written to ``spill`` (0700) as a 0600 file of its own."""
287
+ me = cls(bench, {})
288
+ for name, data in values.items():
289
+ kind = me.kinds.get(name)
290
+ if kind is None:
291
+ continue
292
+ if kind == "file":
293
+ _own_dir(spill)
294
+ p = os.path.join(spill, name)
295
+ if os.path.lexists(p): # ours: the folder is checked to be this user's alone
296
+ os.unlink(p)
297
+ fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
298
+ with os.fdopen(fd, "wb") as f:
299
+ f.write(data)
300
+ me.given[name] = {"kind": kind, "path": p}
301
+ else:
302
+ me.given[name] = {"kind": kind, "value": data.decode(errors="replace").strip()}
303
+ return me
304
+
305
+ def get(self, name: str) -> Optional[dict]:
306
+ kind = self.kinds.get(name)
307
+ if kind is None:
308
+ return None
309
+ if self.given is not None:
310
+ return self.given.get(name)
311
+ from nullius import secrets
312
+ store, kept = secrets.store_of(self.key, name, self.providers.get(name, ""))
313
+ if kind == "file":
314
+ p = secrets.path(store, kept)
315
+ return {"kind": kind, "path": p} if p else None
316
+ data = secrets.value(store, kept)
317
+ text = data.decode(errors="replace").strip() if data else ""
318
+ return {"kind": kind, "value": text} if text else None
319
+
320
+
321
+ class Run:
322
+ def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None,
323
+ limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None,
324
+ turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ,
325
+ proving: bool = False, unsandboxed: bool = False, judge_model: Optional[str] = None,
326
+ backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None,
327
+ launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None,
328
+ licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which,
329
+ remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None,
330
+ max_usd_per_unit: Optional[float] = None):
331
+ """``proving``: the run is a conformance kit's, the one run that may
332
+ start before the port it checks is proven - the family must still match.
333
+ ``unsandboxed``: consent to execute code as the user: required for
334
+ local agents, and for setup checks that run code without Docker.
335
+ ``backend``: ``local`` or ``container`` (``docker``, its earlier name), which needs
336
+ no such yes and is refused when docker is missing or not usable.
337
+ ``options``: the
338
+ agent's or capability's options set for this run, by name; the rest keep
339
+ their defaults, and the job's config records every value.
340
+ ``shard``: (I, N), only the units of shard I of N (``shard``), before
341
+ ``limit``; ``launch``: the id every shard of one launch records.
342
+ ``repeat``: each unit run that many times (a conformance check run for
343
+ confidence, ``conform``), each run a trial of its own tagged
344
+ ``u0042r2``; once by default, tagged ``u0042`` as always.
345
+ ``simulator_model``: a benchmark whose user a model plays
346
+ (``simulator``): this model instead of its official one.
347
+ ``prices``: this run's own prices, by model, over the user's list
348
+ (``nullius.prices``): what a call's cost is estimated from.
349
+ ``max_usd_per_unit``: each unit's own spending cap, kept in the same
350
+ ledger: its own calls and its simulated user's count toward it, its
351
+ judge's do not. A unit that reaches it ends as ``ledger.UNIT_ERROR``,
352
+ an agent error counted against the agent, and the run goes on.
353
+ ``max_usd``: the run's spending cap (``ledger``): every relay of the
354
+ job books a call before sending it, and once one does not fit no
355
+ further call goes out and nothing more is begun. Refused when a model
356
+ the run calls has no price: a call that cannot be priced cannot be
357
+ booked.
358
+ ``licences``: what the run holds of the benchmark's licences
359
+ (:class:`Licences`); by default its store on this machine.
360
+ ``bench_options``: the benchmark's own options for this run
361
+ (``options`` in its definition: a search, a mode), the rest at their
362
+ defaults; a value that needs an embedding model gets a relay of the
363
+ run's own for it, one that needs something of this machine
364
+ (``host``) is refused without it."""
365
+ backend = definitions.names.backend(backend)
366
+ if backend not in BACKENDS:
367
+ raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}")
368
+ if backend == "local" and other.get("_kind") == "agent" and not unsandboxed:
369
+ raise Refused(f"{other['label']} would run commands as you: the local backend is not a sandbox. "
370
+ "Say so explicitly to go on (--unsandboxed), or run it in a container (--backend container).")
371
+ why = match.fit(bench, other)
372
+ if proving:
373
+ why = [w for w in why if "does not offer" not in w and "has not passed its kit" not in w]
374
+ if why:
375
+ raise Refused(f"{other['label']} cannot run {bench['label']}: " + "; ".join(why))
376
+ if backend == "container":
377
+ why = container.unusable()
378
+ if why:
379
+ raise Refused(f"the container backend cannot run here: {why}")
380
+ self.container_check = False
381
+ if backend == "local" and other.get("_kind") == "upstream" and (bench.get("score") or {}).get("runs_code"):
382
+ why = container.unusable()
383
+ if why and not unsandboxed:
384
+ raise Refused(f"{bench['label']}'s check executes generated code, but Docker is unavailable: {why}. "
385
+ "Make Docker usable, or explicitly allow execution as you (--unsandboxed).")
386
+ self.container_check = not why
387
+ self.bench, self.other, self.model, self.provider, self.backend = bench, other, model, provider, backend
388
+ # A setup (definitions.setup): the upstream's own answerer, asked as one model (``asked``), the relay
389
+ # sending another (``sent``); its units may be its own (a conversation where the paper batches questions).
390
+ self.upstream = other.get("_kind") == "upstream"
391
+ self.setup_driver = None
392
+ self.asked = other.get("asked", "") if self.upstream else ""
393
+ if self.upstream:
394
+ self.model = other["sent"]
395
+ self.bench = bench = dict(bench, units=other["units"])
396
+ self.licences = licences or Licences(bench) # where a benchmark's environments find the licences they need
397
+ self.unsandboxed = bool(unsandboxed)
398
+ self.image = ""
399
+ self.options, bad = definitions.options_of(other, options)
400
+ if bad:
401
+ raise Refused("; ".join(bad))
402
+ # A benchmark scored by a model: its official judge, or the one asked for; its calls go through the relay
403
+ # on a route of their own (<tag>-judge), recorded beside the unit's and never counted as the agent's.
404
+ # The official one is written as its maker spells it; the run's provider may spell it otherwise.
405
+ j = (bench.get("score") or {}).get("judge")
406
+ self.judge_model = (judge_model or providers.served_as(j["model"], provider)) if j else None
407
+ # A benchmark whose user a model plays: its driver's turns ask that model the same way, on <tag>-simulator.
408
+ # A control holds no conversation, so it never asks it.
409
+ sim = bench.get("simulator")
410
+ self.simulator_model = (simulator_model or providers.served_as(sim["model"], provider)) if sim else None
411
+ # A control (definitions.control) calls no model: its run needs a key only for the benchmark's judge.
412
+ ds = [d for d in bench["datasets"] if dataset in (None, d["id"])]
413
+ if not ds:
414
+ raise Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}")
415
+ self.ds = ds[0]
416
+ try:
417
+ _, env_def = definitions.environment_of(bench, None, self.ds)
418
+ except ValueError as e:
419
+ raise Refused(str(e))
420
+ if backend != "container" and other.get("_kind") == "control" and definitions.own_image(env_def):
421
+ why = container.unusable()
422
+ if why:
423
+ raise Refused(f"each {(bench.get('units') or {}).get('noun', 'unit')} of {bench['label']} runs in its own image, "
424
+ f"which needs docker: {why}")
425
+ # The benchmark's own options on this dataset; what their values need: an embedding model the driver
426
+ # calls through a relay of the run's own (bench_relay), programs on this machine.
427
+ # each option where it applies: a setup's own (a retriever, a top-k), or those of the benchmark's port
428
+ self.bench_options, bad = definitions.bench_options_of(bench, self.ds["id"], bench_options,
429
+ other["key"] if self.upstream else "yours")
430
+ if bad:
431
+ raise Refused("; ".join(bad))
432
+ needs = definitions.option_needs(bench, self.bench_options)
433
+ self.files = needs["files"]
434
+ self.file_env: Dict[str, str] = {}
435
+ for m in needs["embeddings"]:
436
+ why = providers.embedding_problem(m, provider)
437
+ if why:
438
+ raise Refused(why)
439
+ self.embed_model = providers.served_as(needs["embeddings"][0], provider) if needs["embeddings"] else None
440
+ self.rerank_model = providers.served_as(needs["reranker"][0], provider) if needs["reranker"] else None
441
+ # What a call's cost is estimated from when the provider does not say it: the user's price list
442
+ # (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price.
443
+ self.run_prices = dict(prices or {})
444
+ self.prices = {**price_list.table(), **self.run_prices}
445
+ self.max_usd, self.unit_max_usd, self.ledger = None, None, None
446
+ self.capped = False # whether the run's spending cap stopped it
447
+ for flag, given, what in (("--max-usd", max_usd, "the run's spending cap"), ("--max-usd-per-unit", max_usd_per_unit, "a unit's spending cap")):
448
+ if given is None:
449
+ continue
450
+ try:
451
+ cap = float(given)
452
+ except (TypeError, ValueError):
453
+ cap = 0.0
454
+ if not 0 < cap < float("inf"):
455
+ raise Refused(f"{flag} {given!r}: {what} in USD, above zero")
456
+ if flag == "--max-usd":
457
+ self.max_usd = cap
458
+ else:
459
+ self.unit_max_usd = cap
460
+ if self.max_usd is not None or self.unit_max_usd is not None:
461
+ unpriced = [m for m in self.models_called() if m not in self.prices]
462
+ if unpriced:
463
+ raise Refused("a spending cap needs a price for every model the run calls, and there is none for "
464
+ + ", ".join(unpriced) + ": nullius prices update, nullius prices set MODEL INPUT OUTPUT, "
465
+ "or --price MODEL=INPUT,OUTPUT")
466
+ on_host = needs["host"]
467
+ if self.upstream and backend == "container":
468
+ # its image installs what has an install; nothing else reaches its container
469
+ imaged = hostreq.in_image(bench, on_host)
470
+ cannot = [n for n in hostreq.requirements(bench, on_host) if n not in imaged]
471
+ if cannot:
472
+ raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs "
473
+ f"{', '.join(cannot)}, which a container setup cannot provide: choose local processes")
474
+ on_host = []
475
+ lacking = hostreq.problems(bench, on_host, which)
476
+ if lacking:
477
+ raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine "
478
+ + "; ".join(f"{n} ({why})" for n, why in lacking.items())
479
+ + (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking)))
480
+ else ". Run it on a machine that has it"))
481
+ self.bench_relay = None
482
+ keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model
483
+ self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env)
484
+ self.limit, self.concurrency = limit, max(1, int(concurrency))
485
+ self.repeat = max(1, int(repeat or 1))
486
+ # an own image this run pulled or built stays for the next run, unless the user asks it removed
487
+ # when its last unit ends (never with a repeat): {image: how many units are using it now}
488
+ self.keep_images = not remove_images or self.repeat > 1
489
+ self._image_users: Dict[str, int] = {}
490
+ self.turn_seconds = (turn_seconds or (other.get("turn_seconds") if self.upstream else None)
491
+ or (bench.get("limits") or {}).get("turn_seconds") or TURN_SECONDS)
492
+ # the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own
493
+ # limits sets this above them (limits.turns), so that its limits, not this one, are the ones met
494
+ self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS
495
+ self.say = say
496
+ stamp = time.strftime("%Y-%m-%d__%H-%M-%S")
497
+ self.shard = tuple(shard) if shard and shard[1] > 1 else None
498
+ self.launch = launch or (shards.launch_id(definitions.job_word(bench), definitions.job_word(other), model)
499
+ if self.shard else None)
500
+ self.name = name or (f"{stamp}__{definitions.job_word(bench)}__{definitions.job_word(other)}"
501
+ + (f"__shard-{self.shard[0]}of{self.shard[1]}" if self.shard else ""))
502
+ self.job_dir = os.path.abspath(os.path.join(jobs_root, self.name))
503
+ # each stage said as it goes, on the cli's stderr and in the job's progress.jsonl (nullius/progress.py)
504
+ self.progress = progress.Reporter(self.job_dir)
505
+ self._said = threading.Lock()
506
+ # the job page's live board (nullius/runs/live.py): written once the units are known
507
+ self.feed: Optional[live_feed.Writer] = None
508
+ self._ended = 0 # units ended so far: the i of the next unit's line
509
+ self._envs: set = set() # environments whose build was said
510
+ self.n_units: Optional[int] = None
511
+ self.local = threading.local()
512
+ self.drivers: List[jsonl.Proc] = []
513
+ self._stopped = threading.Event()
514
+
515
+ def driver(self) -> jsonl.Proc:
516
+ d = getattr(self.local, "driver", None)
517
+ if d is not None and d.p is not None and d.p.poll() is not None:
518
+ d.close()
519
+ d = None # it ended - a reply that never came closed it: the next unit gets a new one
520
+ if d is None:
521
+ if self.setup_driver is not None:
522
+ home = os.path.join(self.job_dir, ".work", "driver-home")
523
+ cwd = os.path.join(self.job_dir, ".work", "driver")
524
+ os.makedirs(home, exist_ok=True)
525
+ os.makedirs(cwd, exist_ok=True)
526
+ env = {"HOME": home, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
527
+ "NULLIUS_DATASET_ID": self.ds["id"], "NULLIUS_VERSION": str(self.bench.get("_version") or ""),
528
+ **option_env(self.bench_options), **self.file_env, **self.embeddings_env(), **self.driver_secrets()}
529
+ files = [h["path"] for s in definitions.secrets_of(self.bench)
530
+ if s.get("to") == "driver" and s.get("kind") == "file"
531
+ for h in [self.licences.get(s["name"])] if h] + list(self.file_env.values())
532
+ d = self.setup_driver.start(env, cwd, os.path.join(self.job_dir, ".work", f"driver-{threading.get_ident()}.log"),
533
+ files=files, stopped=self._stopped,
534
+ servers=[s["name"] for s in definitions.secrets_of(self.bench)
535
+ if s.get("to") == "driver" and s.get("kind") == "server"])
536
+ self.local.driver = d
537
+ self.drivers.append(d)
538
+ return d
539
+ try:
540
+ cmd, extra = install.driver_argv(self.bench) # its driver.install: installed in ready(), found here
541
+ except install.InstallFailed as e:
542
+ raise Refused(f"could not install the driver's environment: {e}")
543
+ env = base_env(os.path.join(self.job_dir, ".work", "driver-home"),
544
+ {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
545
+ "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": driver_path(), "PYTHONDONTWRITEBYTECODE": "1",
546
+ "NULLIUS_VERSION": str(self.bench.get("_version") or ""),
547
+ **({container.CHECK_IMAGE: self.image, **container.check_env()} if self.image else {}),
548
+ **option_env(getattr(self, "bench_options", {})), **self.file_env, **self.embeddings_env(), **self.driver_secrets()})
549
+ os.makedirs(env["HOME"], exist_ok=True)
550
+ # its working directory is the job's, never its catalogue folder: an upstream that writes
551
+ # beside itself (tau2's embeddings cache) would write into the package or the runner snapshot
552
+ cwd = os.path.join(self.job_dir, ".work", "driver")
553
+ os.makedirs(cwd, exist_ok=True)
554
+ d = jsonl.Proc(cmd, cwd, env, os.path.join(self.job_dir, ".work", f"driver-{threading.get_ident()}.log"), stopped=self._stopped).start()
555
+ self.local.driver = d
556
+ self.drivers.append(d)
557
+ return d
558
+
559
+ def start(self) -> str:
560
+ t0 = time.time()
561
+ os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True)
562
+ self.say(f"job {self.job_dir}")
563
+ p, control = self.progress, self.other["_kind"] == "control"
564
+ p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else [])
565
+ + (["upstream"] if install.upstream_dir(self.bench) else [])
566
+ + (["dataset"] if self.ds.get("url") else [])
567
+ + (["driver"] if install.driver_dir(self.bench) else [])
568
+ + ([] if control or self.upstream else ["install"]) + ["units"]))
569
+ if (self.max_usd is not None or self.unit_max_usd is not None) and self.base:
570
+ # one ledger for the job, in a folder of its own: all a unit's relay container is given of the job besides its record
571
+ self.ledger = ledgers.Ledger.create(os.path.join(self.job_dir, ".work", "ledger", "ledger.json"), self.max_usd, self.unit_max_usd)
572
+ try:
573
+ self.ready()
574
+ except Refused as e:
575
+ # said in the job, which holds nothing else: the runs list reads why there (progress.refusal)
576
+ p.emit("refused", detail=str(e))
577
+ raise
578
+ try:
579
+ units = self.driver().request("units", dataset=self.ds["id"], timeout=300,
580
+ **({"setup": self.other["key"]} if self.upstream else {}))["units"]
581
+ except BaseException:
582
+ for driver in self.drivers:
583
+ driver.close()
584
+ if self.bench_relay is not None:
585
+ self.bench_relay.stop()
586
+ if self.backend == "container":
587
+ container.sweep(self.name)
588
+ raise
589
+ everything = [str(u) for u in units]
590
+ # the shard, then the limit; each unit keeps its place in the whole dataset (its trial's tag)
591
+ mine = shards.select(everything, *self.shard) if self.shard else list(enumerate(everything))
592
+ mine = mine[: self.limit] if self.limit else mine
593
+ units = [u for _, u in mine]
594
+ self.n_units = len(units)
595
+ p.emit("units", "start", n=len(units), of=len(everything), noun=self.bench["units"]["nouns"])
596
+ job.write_config(self.job_dir, job=self.name, bench=self.bench, other=self.other, model=self.model,
597
+ provider=self.provider, dataset=self.ds["id"], units=units, concurrency=self.concurrency,
598
+ granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend,
599
+ image=self.setup_driver.image if self.setup_driver else self.image,
600
+ options=self.options, launch=self.launch or "", bench_options=self.bench_options,
601
+ shard=shards.record(*self.shard, len(everything), units, self.limit) if self.shard else None,
602
+ repeat=self.repeat, prices=self.prices_record(),
603
+ simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "")
604
+ # every unit once, then every unit again: a repeat cut short still has each unit's first run
605
+ work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine]
606
+ self.n_units = len(work) # what the progress counts: every run of every unit
607
+ self.feed = live_feed.Writer(os.path.join(self.job_dir, live_feed.FEED), self.name)
608
+ self.feed.emit("job.start", n_total=len(work), n_concurrent=self.concurrency, agents=[self.other["key"]], tasks=units,
609
+ **({"budget": {k: v for k, v in (("max_usd", self.max_usd), ("unit_max_usd", self.unit_max_usd)) if v is not None}}
610
+ if self.ledger is not None else {})) # the live board says the cap
611
+ for pos, (i, u, r) in enumerate(work):
612
+ self.feed.emit("trial.queued", self.trial_name(i, u, r), agent=self.other["key"], position=pos)
613
+ results = []
614
+ u0 = time.time()
615
+ self._halt, self._refused, self.halted = threading.Event(), [], None
616
+ self._cut: List[str] = [] # units the spending cap cut off
617
+ self._over: List[str] = [] # units that reached their own cap: cut off there, the run going on
618
+ self.loopback = None
619
+ self.pace = pacing.Pace(self._stopped, self._halt)
620
+ self._running = 0
621
+ again: Dict[str, dict] = {} # a rate-limited unit's second attempt, by its trial's name
622
+
623
+ def work_unit(iur):
624
+ self.pace.wait() # a unit ended rate-limited: the next waits its pause out
625
+ if self._stopped.is_set():
626
+ return None
627
+ with self._said:
628
+ self._running += 1
629
+ try:
630
+ res = self.unit(*iur)
631
+ except BaseException:
632
+ self._stopped.set()
633
+ self._halt.set()
634
+ raise
635
+ finally:
636
+ with self._said:
637
+ self._running -= 1
638
+ self.paced(res)
639
+ return res
640
+
641
+ pool = ThreadPoolExecutor(self.concurrency)
642
+ futures = []
643
+ try:
644
+ # a model server on this machine's loopback, which the relays' containers cannot reach as written
645
+ if self.backend == "container" and self.base and container.on_loopback(self.base):
646
+ self.loopback = container.Loopback(self.base, image=self.image, job=self.name).start()
647
+ for iur in work:
648
+ futures.append(pool.submit(work_unit, iur))
649
+ for future in as_completed(futures):
650
+ res = future.result()
651
+ if res is not None: # None: a unit not begun once the endpoint refused the run
652
+ results.append(res)
653
+ self.run_again(results, work, work_unit, again)
654
+ except BaseException:
655
+ self._stopped.set()
656
+ self._halt.set()
657
+ if self.backend == "container":
658
+ container.sweep(self.name)
659
+ raise
660
+ finally:
661
+ # a Stop or Ctrl-C: the units not begun are not begun, those running end with their agents
662
+ pool.shutdown(wait=True, cancel_futures=True)
663
+ begun = [not f.cancelled() and f.exception() is None and f.result() is not None for f in futures]
664
+ results = [again.get(f.result()["trial_name"], f.result()) for f, ran in zip(futures, begun) if ran]
665
+ for d in self.drivers:
666
+ d.close()
667
+ if self.bench_relay is not None:
668
+ self.bench_relay.stop()
669
+ if self.backend == "container":
670
+ container.sweep(self.name) # whatever an interrupted unit left
671
+ if self.loopback is not None:
672
+ self.loopback.stop()
673
+ budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]])
674
+ self.capped = bool(budget and budget["stopped"])
675
+ job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget)
676
+ errored = sum(1 for x in results if x.get("exception_info"))
677
+ self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results))
678
+ # the units' stage says why it ended early: a line of its own would be a stage that never starts
679
+ p.emit("units", "done", seconds=round(time.time() - u0, 1),
680
+ **({"detail": f"stopped at the spending cap of {ledgers.usd(budget['max_usd'])} USD: "
681
+ f"{len(budget['cut_off'])} cut off, {len(budget['not_run'])} not run"} if budget and budget["stopped"] else {}))
682
+ self.n_verdicts, self.n_ended = sum(1 for x in results if x.get("verifier_result")), len(results)
683
+ if budget and budget["stopped"]:
684
+ spent = "unknown" if budget["spent_usd"] is None else ledgers.usd(budget["spent_usd"])
685
+ self.say(f"stopped: a call did not fit the spending cap of {ledgers.usd(budget['max_usd'])} USD "
686
+ f"({spent} spent): {len(budget['cut_off'])} cut off, {len(budget['not_run'])} not run")
687
+ elif budget and budget["max_usd"] is not None:
688
+ self.say(f"spent {ledgers.usd(budget['spent_usd'] or 0)} of the {ledgers.usd(budget['max_usd'])} USD cap")
689
+ elif budget:
690
+ self.say(f"spent {ledgers.usd(budget['spent_usd'] or 0)} USD")
691
+ if budget and budget.get("over_unit"):
692
+ self.say(f"{len(budget['over_unit'])} of the {self.bench['units']['nouns']} reached the cap of "
693
+ f"{ledgers.usd(budget['unit_max_usd'])} USD each and were cut off there")
694
+ if self.halted:
695
+ self.say(f"stopped: the endpoint refused {REFUSED_IN_A_ROW} units in a row ({self.halted[0]}: "
696
+ f"{redact(str(self.halted[1]))[:200]}); the {len(work) - len(results)} units not begun were not run")
697
+ rewards = [(r.get("verifier_result") or {}).get("rewards", {}) for r in results]
698
+ if self.bench["score"]["kind"] == "pass":
699
+ ok = sum(1 for w in rewards if w.get("reward") == 1.0)
700
+ p.emit("done", n=len(results), passed=ok, seconds=round(time.time() - t0, 1))
701
+ self.say(f"done: {len(results)} {self.bench['units']['nouns']}, {ok} passed")
702
+ else:
703
+ scores = [w["score"] for w in rewards if "score" in w]
704
+ mean = job.mean_score(results)
705
+ p.emit("done", n=len(results), scored=len(scores), mean=round(mean, 3) if mean is not None else None,
706
+ seconds=round(time.time() - t0, 1))
707
+ self.say(f"done: {len(results)} {self.bench['units']['nouns']}, {len(scores)} scored"
708
+ + (f", mean {mean:.3f}" if mean is not None else ""))
709
+ return self.job_dir
710
+
711
+ def paced(self, res: Optional[dict]) -> None:
712
+ """A unit's end, told to the run's pace: one that ended rate-limited
713
+ holds the next unit back, said with how many units run meanwhile."""
714
+ if res is None:
715
+ return
716
+ pause = self.pace.ended((res.get("exception_info") or {}).get("exception_type"))
717
+ if pause is not None:
718
+ self.progress.emit("units", "progress", detail=f"rate limited: no unit starts for {pause:g} s; "
719
+ f"{self._running} running of {self.concurrency} at once")
720
+
721
+ def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, dict]) -> None:
722
+ """Each unit that ended rate-limited, run once more after the others,
723
+ one at a time and paced as the rest: its first attempt moved into the
724
+ job's ``retried/`` (kept whole, its calls and its cost with it, where
725
+ no reader takes it for a trial of its own), its second written under
726
+ the trial's own name and counted (into ``done``, as each ends). Nothing
727
+ is run again once the run is stopped or halted; the spending cap goes
728
+ on counting every call."""
729
+ by_name = {self.trial_name(*iur): iur for iur in work}
730
+ todo = [by_name[r["trial_name"]] for r in results
731
+ if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name]
732
+ if not todo:
733
+ return
734
+ self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time")
735
+ self.say(f"running again, one at a time: {len(todo)} rate-limited")
736
+ for iur in todo:
737
+ if self._stopped.is_set() or self._halt.is_set():
738
+ break
739
+ name = self.trial_name(*iur)
740
+ keep = os.path.join(self.job_dir, "retried", name)
741
+ os.makedirs(os.path.dirname(keep), exist_ok=True)
742
+ os.replace(os.path.join(self.job_dir, name), keep)
743
+ self.feed.emit("trial.queued", name, agent=self.other["key"], position=0)
744
+ res = work_unit(iur)
745
+ if res is not None:
746
+ done[name] = res
747
+ elif not os.path.exists(os.path.join(self.job_dir, name)):
748
+ os.replace(keep, os.path.join(self.job_dir, name)) # not begun after all: the first attempt stands
749
+
750
+ def ready(self) -> None:
751
+ """What the run needs before its first unit, each a stage said as it
752
+ goes - or as there already: the base image (docker), the benchmark's
753
+ upstream, its dataset when it is downloaded, its driver's environment
754
+ when it declares one (``driver.install``), the agent's or capability's
755
+ packages. Refused when one cannot be had."""
756
+ from . import network_requirements
757
+ why = network_requirements.check(self.bench, self.other)
758
+ if why:
759
+ raise Refused("; ".join(why))
760
+ p = self.progress
761
+ if self.backend == "container" or self.container_check:
762
+ try:
763
+ tag = container.base_tag()
764
+ there = container.present(tag)
765
+ with p.stage("image", cached=there, start=f"building {tag} (first time on this machine)", **({"detail": tag} if there else {})):
766
+ self.image = container.base_image(os.path.join(self.job_dir, ".work", "image.log"))
767
+ except container.Unavailable as e:
768
+ raise Refused(f"the Docker image: {e}")
769
+ # on the container backend the agent's packages are installed in the base image, with the Node, uv and libc it runs on
770
+ image = self.image if self.backend == "container" else None
771
+ run_in = (lambda cmd, where, env=None: container.install_argv(image, where, cmd, env)) if image else None
772
+ try:
773
+ where = install.upstream_dir(self.bench)
774
+ self.up = None
775
+ if where is not None:
776
+ up = self.bench["upstream"]
777
+ with p.stage("upstream", cached=install.ready(where), start=f"fetching {up['repository']} at {str(up['tested'])[:12]}"):
778
+ self.up = install.upstream(self.bench)
779
+ if self.ds.get("url"):
780
+ there = os.path.isfile(install.download_path(self.ds["url"], self.ds["sha256"]))
781
+ with p.stage("dataset", cached=there, start=f"downloading {self.ds['id']}"):
782
+ self.data = install.dataset(self.bench, self.ds, self.up, progress=None if there else p.download("dataset"))
783
+ else:
784
+ self.data = install.dataset(self.bench, self.ds, self.up)
785
+ for name in self.files:
786
+ f = self.bench["files"][name]
787
+ there = os.path.isfile(install.download_path(f["url"], f["sha256"]))
788
+ with p.stage("file", cached=there, start=f"downloading {name}", name=name):
789
+ self.file_env["NULLIUS_FILE_" + name.upper()] = install.download(
790
+ f["url"], f["sha256"], progress=None if there else p.download("file"))
791
+ drv = install.driver_dir(self.bench)
792
+ if self.backend == "container" and self.upstream:
793
+ from nullius.standard import setup_container
794
+ with p.stage("driver", start=f"preparing {self.bench['key']}'s container driver"):
795
+ needs = definitions.option_needs(self.bench, self.bench_options)["host"]
796
+ image = setup_container.image(self.image, self.bench, needs, os.path.join(self.job_dir, ".work", "image.log"))
797
+ self.setup_driver = setup_container.Driver(self.bench, image=image, app=APP, job=self.name,
798
+ work=self.job_dir, upstream=self.up, data=self.data, needs=needs)
799
+ self.setup_driver.prepare()
800
+ elif drv is not None: # the environment the upstream's own code needs, for the driver
801
+ with p.stage("driver", cached=install.ready(drv), start=f"installing {self.bench['key']}'s driver environment"):
802
+ install.driver_packages(self.bench)
803
+ self.inst = ""
804
+ if self.other["_kind"] not in ("control", "upstream"):
805
+ with p.stage("install", cached=install.ready(install.packages_dir(self.other, image=image)), agent=self.other.get("_ref") or self.other["key"]):
806
+ self.inst = install.packages(self.other, run_in=run_in, image=image)
807
+ except (install.InstallFailed, container.Unavailable) as e:
808
+ if self.backend == "container":
809
+ container.sweep(self.name)
810
+ raise Refused(f"could not install what the run needs: {e}")
811
+ # An own image has none of what the agent runs on: it is mounted in, fetched once on this machine.
812
+ self.runtimes: Dict[str, tuple] = {}
813
+ owns = any(definitions.own_image(e) for e in (self.bench.get("environments") or {}).values())
814
+ if owns and self.other["_kind"] == "agent" and self.backend == "container" and self.runtimes_needed():
815
+ try:
816
+ with p.stage("runtime", start="fetching " + ", ".join(self.runtimes_needed()), name=",".join(self.runtimes_needed())):
817
+ self.runtimes = container.runtime_mounts(self.runtimes_needed())
818
+ except container.Unavailable as e:
819
+ raise Refused(f"the agent's runtime for an own image: {e}")
820
+ # An option whose driver embeds or reranks: its calls go through a relay of the run's
821
+ # own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's.
822
+ if (self.embed_model or self.rerank_model) and self.bench_relay is None:
823
+ self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"),
824
+ prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False,
825
+ key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start()
826
+
827
+ def runtimes_needed(self) -> List[str]:
828
+ """What the agent needs in an own image that the image may not have:
829
+ Node for an agent npm installs, Python for one started as ``python3``
830
+ with no environment of its own (a uv environment's interpreter is
831
+ mounted with the install, as on the base image)."""
832
+ a = self.other
833
+ start = (a.get("start") or {}).get("acp") or (a.get("start") or {}).get("command") or [""]
834
+ needs = ["node"] if (a.get("install") or {}).get("npm") else []
835
+ if container.bare_python(start[0]) and not os.path.isfile(os.path.join(self.inst, ".venv", "pyvenv.cfg")):
836
+ needs.append("python")
837
+ return needs
838
+
839
+ def attempt_kw(self) -> dict:
840
+ """What the driver is told of the attempt this unit's run is (1, 2, ...), when the benchmark
841
+ declares that its upstream tells attempts apart (``limits.attempts``: a seed per trial)."""
842
+ a = getattr(self.local, "attempt", None)
843
+ return {"attempt": a} if a and (self.bench.get("limits") or {}).get("attempts") else {}
844
+
845
+ def trial_name(self, i: int, unit: str, r: int = 1) -> str:
846
+ return f"{_slug(unit)}__u{i:04d}" + (f"r{r}" if self.repeat > 1 else "")
847
+
848
+ def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = None) -> None:
849
+ if self.feed is not None:
850
+ self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout)
851
+
852
+ def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]:
853
+ if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()):
854
+ return None # the endpoint refused the run: nothing more is begun
855
+ if getattr(self, "ledger", None) is not None and self.ledger.closed():
856
+ self._halt.set() # the spending cap refused a call: every later one is refused too
857
+ return None
858
+ tag = f"u{i:04d}" + (f"r{r}" if self.repeat > 1 else "")
859
+ self.local.attempt = r
860
+ trial = os.path.join(self.job_dir, self.trial_name(i, unit, r))
861
+ name = os.path.basename(trial)
862
+ ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home")
863
+ os.makedirs(ws, exist_ok=True)
864
+ # The workspace is a repository of its own: agents look up to the git root
865
+ # for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept
866
+ # inside a checkout, would otherwise read that checkout's into every prompt.
867
+ subprocess.run(["git", "init", "-q", ws], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
868
+ os.makedirs(home, exist_ok=True)
869
+ t0 = time.time()
870
+ self.progress.emit("unit", "start", unit=unit)
871
+ self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS)
872
+ produce, score, exc, traj, logs = (None, None), None, None, None, {}
873
+ p0, checking = None, (None, None) # when the agent began; the check's start and end
874
+ setup: dict = {}
875
+ transcript: List[dict] = []
876
+ rl, environment, box, changes = None, None, None, None
877
+ try:
878
+ try:
879
+ # a setup's unit needs none: the upstream's answerer reads its data itself
880
+ setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=DRIVER_SECONDS, **self.attempt_kw())
881
+ except jsonl.Failed as e:
882
+ raise _Ended("EnvironmentSetupError", str(e))
883
+ environment, env_def = self.unit_environment(setup)
884
+ why = self.not_run(environment, env_def)
885
+ if why:
886
+ raise _Ended(*why)
887
+ own = definitions.own_image(env_def)
888
+ if "internet" in setup and (not isinstance(setup["internet"], bool) or not own):
889
+ raise _Ended("EnvironmentSetupError", "the unit's setup: internet is true or false, only in an own image")
890
+ if self.other["_kind"] == "agent" and self.backend == "local" and mcp_machine.wanted(setup):
891
+ try: # the benchmark's own servers: the agent starts them from the driver's environment
892
+ setup = dict(setup, mcp=mcp_machine.entries(self.bench, setup, install.driver_dir(self.bench),
893
+ self.machine_env(trial)))
894
+ except mcp_machine.Unknown as e:
895
+ raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}")
896
+ if "env" in setup and not own:
897
+ raise _Ended("EnvironmentSetupError", "the unit's setup: env requires an own image")
898
+ if setup.get("verify") is not None and not own:
899
+ raise _Ended("EnvironmentSetupError", "the unit's setup: verify runs in an own image, and this unit has none")
900
+ if setup.get("prepare") is not None and not own:
901
+ raise _Ended("EnvironmentSetupError", "the unit's setup: prepare runs in an own image, and this unit has none")
902
+ try:
903
+ rl = self.unit_relay(tag, trial, setup, env_def, own)
904
+ except (container.Unavailable, OSError) as e:
905
+ raise _Ended("EnvironmentSetupError", f"the unit's relay: {e}")
906
+ if rl is not None:
907
+ rl.unit_secrets = self.agent_licences(env_def, rl)
908
+ if own:
909
+ self.phase(name, unit, "env_setup", IMAGE_SECONDS) # its image, pulled or built when not here yet
910
+ box = self.own_box(tag, setup, rl, home)
911
+ if setup.get("prepare") is not None:
912
+ self.prepare(setup["prepare"], box, trial)
913
+ p0 = time.time()
914
+ self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control"
915
+ else DRIVER_SECONDS if self.other["key"] == "oracle" else None)
916
+ try:
917
+ with self.agent_internet(setup, box, rl, trial):
918
+ if self.other["_kind"] == "control":
919
+ answer = self.control_answer(unit, ws, box, name)
920
+ elif self.upstream:
921
+ answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl)
922
+ elif self.other["_kind"] == "agent":
923
+ answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name)
924
+ else:
925
+ answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl)
926
+ finally:
927
+ produce = (p0, time.time())
928
+ if box is not None: # what the agent (or the control) left in the container, before the check
929
+ changes = self.box_changes(box)
930
+ cut = self.cap_reached(rl, tag)
931
+ if cut: # cut off at a spending cap: nothing of it is checked, and its judge is asked nothing
932
+ raise _Ended(cut[0], cut[1], traj)
933
+ step = setup.get("verify") if box is not None else None
934
+ checking = (time.time(), None)
935
+ self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0))
936
+ verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None
937
+ if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read
938
+ container.plain_only(os.path.join(trial, "verifier"))
939
+ try:
940
+ judge = self.judge(rl, tag)
941
+ score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS,
942
+ **judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {}))
943
+ checking = (checking[0], time.time())
944
+ except jsonl.Failed as e:
945
+ raise _Ended("VerifierError", str(e))
946
+ except _Ended as e:
947
+ exc = (e.kind, e.said)
948
+ traj = traj or e.trajectory
949
+ except Exception as e: # anything else is this unit's error, never the whole job's
950
+ exc = (type(e).__name__, f"{type(e).__name__}: {e}\n" + "".join(traceback.format_exc().splitlines(True)[-6:]))
951
+ finally:
952
+ if checking[0] is not None and checking[1] is None:
953
+ checking = (checking[0], time.time())
954
+ if box is not None:
955
+ # the home it wrote as root given back, then the container gone
956
+ box.give_back([home, *getattr(rl, "unit_rw", [])])
957
+ box.stop()
958
+ self.release_image(box.image)
959
+ if rl is not None:
960
+ rl.stop()
961
+ for ln in getattr(rl, "mcp_listeners", None) or []:
962
+ ln.stop()
963
+ calls = rl.calls(tag) if rl is not None else []
964
+ failed = relay.failure(calls)
965
+ if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \
966
+ and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0:
967
+ # a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some
968
+ # models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is
969
+ # the endpoint's, and a cap's refusal is the cap's
970
+ failed = None
971
+ if not failed and rl is not None and exc is not None and exc[0] == "VerifierError":
972
+ # the judge's or the simulated user's model failing (a 429, a refused request) is the
973
+ # unit's infrastructure, as the agent's own would be - not a miss counted against it
974
+ failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None)
975
+ if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)):
976
+ exc = failed # what the endpoint said outranks how the agent took it
977
+ score = None
978
+ cut = self.cap_reached(rl, tag) # at scoring too: its judge's call, or a refusal its agent made something else of
979
+ if cut:
980
+ exc, score = cut, None
981
+ if exc and exc[0] in ("EnvironmentSetupError", "VerifierError") and getattr(self, "ledger", None) is not None \
982
+ and getattr(self, "bench_relay", None) is not None:
983
+ # its driver failed, at setup or at scoring, and the cap refused a call that driver made for this unit (an
984
+ # option's embeddings, on its thread's route of the benchmark's relay): the unit was cut off there. Another
985
+ # unit's refusal is not this one's, and an agent's own error stays the agent's
986
+ late = [c for c in self.bench_relay.calls(self.embeddings_tag()) if c.get("refused") == "cap" and c.get("t0", 0) >= t0]
987
+ if late:
988
+ exc, score = (ledgers.ERROR, late[0].get("error") or ""), None
989
+ if self._stopped.is_set() and score is None:
990
+ # the Stop ended its agent, its relay or its containers under it: what it then failed with is
991
+ # the Stop's doing, not the agent's nor the environment's (Harbor's name for a stopped trial)
992
+ exc = (CANCELLED, "the run was stopped before this unit ended" + (f"; it then failed with {exc[0]}" if exc else ""))
993
+ if traj is None and self.other["_kind"] == "agent":
994
+ traj = job.trajectory_from_events([], calls, self.other, self.model, f"{self.name}/{unit}")
995
+ if traj is not None and setup.get("mcp"): # how a reader tells the benchmark's tools in any agent's spelling
996
+ traj["extra"] = dict(traj.get("extra") or {}, mcp_servers=[str(m.get("name")) for m in setup["mcp"] if m.get("name")])
997
+ res = job.write_trial(trial, job=self.name, bench=self.bench, other=self.other, model=self.model, unit=unit,
998
+ t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls,
999
+ setup_end=p0 or time.time(), checking=checking,
1000
+ trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})),
1001
+ environment=environment or "",
1002
+ image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "")
1003
+ install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now
1004
+ verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}"
1005
+ self.say(f" {unit}: {verdict}")
1006
+ self.said_unit(unit, score, exc)
1007
+ if self.feed is not None:
1008
+ self.feed.emit("trial.end", name, task=unit, agent=self.other["key"],
1009
+ **job.feed_outcome(self.bench["score"]["kind"], score, exc),
1010
+ **{k: v for k, v in (res.get("agent_result") or {}).items() if k in FEED_TOTALS},
1011
+ durations={"agent_run": round(produce[1] - produce[0], 3)} if produce[0] is not None else None)
1012
+ self.note_refusal(exc)
1013
+ if exc and exc[0] == ledgers.ERROR and getattr(self, "_halt", None) is not None:
1014
+ with self._said:
1015
+ self._cut.append(self.attempt_name(unit, r))
1016
+ self._halt.set() # the cap is reached: nothing more is begun
1017
+ elif exc and exc[0] == ledgers.UNIT_ERROR:
1018
+ with self._said:
1019
+ self._over.append(self.attempt_name(unit, r)) # its own cap: the others go on
1020
+ if getattr(self, "ledger", None) is not None and self.unit_max_usd is not None:
1021
+ try:
1022
+ self.ledger.forget(tag) # the unit ended: its own count goes from the ledger
1023
+ except ledgers.Unavailable:
1024
+ pass
1025
+ return res
1026
+
1027
+ def cap_reached(self, rl, tag: str) -> Optional[tuple]:
1028
+ """A spending cap's refusal among a unit's calls - its agent's, its
1029
+ judge's or its simulated user's - as what the unit ends with: cut
1030
+ off by the run's own limit, never a miss, before its own. None
1031
+ without one."""
1032
+ if rl is None or getattr(self, "ledger", None) is None:
1033
+ return None
1034
+ cuts = [f for f in (relay.failure(rl.calls(t)) for t in (tag, f"{tag}-judge", f"{tag}-simulator")) if f]
1035
+ return next((f for f in cuts if f[0] == ledgers.ERROR), None) or next((f for f in cuts if f[0] == ledgers.UNIT_ERROR), None)
1036
+
1037
+ def attempt_name(self, unit: str, r: int = 1) -> str:
1038
+ """A unit as the budget lists it: by its name, and its attempt when the run repeats its units."""
1039
+ return unit if self.repeat <= 1 else f"{unit} (attempt {r})"
1040
+
1041
+ def budget(self, not_run: List[str]) -> Optional[dict]:
1042
+ """What ``result.json`` says of the run's spending cap: the cap,
1043
+ what was spent, what was booked for calls whose cost was never known
1044
+ (``unsettled_usd``, said only when there is some), whether the cap
1045
+ stopped the run, the units it cut off and those never begun. With a
1046
+ cap per unit, that cap and the units that reached it
1047
+ (``unit_max_usd``, ``over_unit``); ``max_usd`` is then None when the
1048
+ run itself had none. None for a run without a cap."""
1049
+ if getattr(self, "ledger", None) is None:
1050
+ return None
1051
+ try:
1052
+ got = self.ledger.state()
1053
+ except ledgers.Unavailable:
1054
+ got = {"spent_usd": None, "unsettled_usd": 0.0, "held_usd": 0.0, "closed": True}
1055
+ unsettled = round(got["unsettled_usd"] + got["held_usd"], 6)
1056
+ stopped = bool(got["closed"]) and self.max_usd is not None # only the run's cap stops a run
1057
+ cut = getattr(self, "_cut", [])
1058
+ return {"max_usd": self.max_usd, "spent_usd": None if got["spent_usd"] is None else round(got["spent_usd"], 6),
1059
+ **({"unsettled_usd": unsettled} if unsettled else {}), "stopped": stopped,
1060
+ "cut_off": list(dict.fromkeys(cut)), "not_run": list(dict.fromkeys(not_run)) if stopped else [],
1061
+ **({"unit_max_usd": self.unit_max_usd, "over_unit": list(dict.fromkeys(getattr(self, "_over", [])))}
1062
+ if self.unit_max_usd is not None else {})}
1063
+
1064
+ def note_refusal(self, exc: Optional[tuple]) -> None:
1065
+ """A unit's end, counted toward the endpoint refusing the run: a wrong
1066
+ key or an unknown model fails every unit the same way, and running
1067
+ the rest only repeats it (with their containers and relays). After
1068
+ ``REFUSED_IN_A_ROW`` such ends in a row, nothing more is begun."""
1069
+ if getattr(self, "_halt", None) is None:
1070
+ return
1071
+ with self._said:
1072
+ if exc and exc[0] in REFUSALS:
1073
+ self._refused.append(exc)
1074
+ if len(self._refused) >= REFUSED_IN_A_ROW and self.halted is None:
1075
+ self.halted = exc
1076
+ self._halt.set()
1077
+ else:
1078
+ self._refused.clear()
1079
+
1080
+ def said_unit(self, unit: str, score: Optional[dict], exc: Optional[tuple]) -> None:
1081
+ """A unit's end, as a progress line: the i-th to end of the run's n,
1082
+ and how - passed or failed (a pass benchmark), scored, or the error."""
1083
+ with self._said:
1084
+ self._ended += 1
1085
+ i = self._ended
1086
+ score = score or {}
1087
+ if exc:
1088
+ how = {"outcome": "error", "error": exc[0]}
1089
+ elif self.bench["score"]["kind"] == "pass":
1090
+ how = {"outcome": "passed" if score.get("reward") == 1.0 else "failed", "reward": score.get("reward")}
1091
+ else:
1092
+ how = {"outcome": "scored", "score": score.get("score", score.get("reward"))}
1093
+ self.progress.emit("unit", "done", i=i, of=self.n_units, unit=unit, **how)
1094
+
1095
+ def unit_environment(self, setup: dict):
1096
+ """(name, definition) of the environment the unit asks for - its
1097
+ setup's ``environment``, else its dataset's - or (None, None), the base
1098
+ image. A name the benchmark does not declare is the driver's error."""
1099
+ try:
1100
+ return definitions.environment_of(self.bench, setup, self.ds)
1101
+ except ValueError as e:
1102
+ raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}")
1103
+
1104
+ def not_run(self, name: Optional[str], env_def: Optional[dict]) -> Optional[tuple]:
1105
+ """Why a unit cannot be run in its environment here, as (kind, said),
1106
+ or None. A secret it needs that is not set: the unit is not run, and
1107
+ that is no one's result (``EnvironmentSecretMissingError``, a setup
1108
+ error to the runs pages). An environment the local backend cannot
1109
+ give - anything but the base image, unless it says ``local_ok`` -
1110
+ refused the same way, for what would run in it (not a control)."""
1111
+ if name is None:
1112
+ return None
1113
+ if self.backend == "local" and self.other["_kind"] not in ("control", "upstream") and not env_def.get("local_ok"):
1114
+ return ("EnvironmentSetupError", f"not run: the environment {name} needs the container backend "
1115
+ "(--backend container); it does not say local_ok")
1116
+ missing = definitions.missing_secrets(env_def, self.licences)
1117
+ if missing:
1118
+ return ("EnvironmentSecretMissingError", "not run: needs " + ", ".join(missing))
1119
+ return None
1120
+
1121
+ def first_probe(self) -> bool:
1122
+ """True once a run: its first unit checks the public web proxy's route,
1123
+ the check before the run having found the public web reachable."""
1124
+ with _PROBED:
1125
+ first = not getattr(self, "_probed", False)
1126
+ self._probed = True
1127
+ return first
1128
+
1129
+ def driver_secrets(self) -> dict:
1130
+ """The secrets the benchmark's environments send to its driver
1131
+ (``to: driver``) that are set: in its process's environment, never in
1132
+ a request, so never in a log."""
1133
+ return driver_secrets(self.bench, self.licences)
1134
+
1135
+ # -- the unit's relay, and where its process runs ------------------------
1136
+ def agent_licences(self, env_def: Optional[dict], rl) -> dict:
1137
+ """The variables the unit's environment gives the agent of its
1138
+ licences (``to: agent``): a key; a file's path, where it is mounted
1139
+ in a container (``rl.unit_binds``) or kept here; a server's address - in a
1140
+ container the one its forwarder answers at (``container.Unit``)."""
1141
+ inside = self.backend == "container"
1142
+ out = definitions.secret_values(env_def, self.licences, "agent", inside=inside)
1143
+ reach = getattr(rl, "reach", None) or {}
1144
+ for s in (env_def or {}).get("needs_secrets") or []:
1145
+ if inside and s.get("to") == "agent" and s.get("kind") == "server" and s["name"] in out:
1146
+ out[s["name"]] = reach.get(out[s["name"]], out[s["name"]])
1147
+ return out
1148
+
1149
+ def agent_files(self, env_def: Optional[dict]) -> List[tuple]:
1150
+ """(where it is kept, its path inside) for each licence file the
1151
+ environment mounts into the agent's container."""
1152
+ out = []
1153
+ for s in (env_def or {}).get("needs_secrets") or []:
1154
+ h = self.licences.get(s["name"]) if s.get("to") == "agent" and s.get("kind") == "file" else None
1155
+ if h and s.get("path"):
1156
+ out.append((h["path"], s["path"]))
1157
+ return out
1158
+
1159
+ def agent_servers(self, env_def: Optional[dict]) -> List[str]:
1160
+ """The licence servers the environment's agent must reach: host:port each."""
1161
+ out = []
1162
+ for s in (env_def or {}).get("needs_secrets") or []:
1163
+ h = self.licences.get(s["name"]) if s.get("to") == "agent" and s.get("kind") == "server" else None
1164
+ if h:
1165
+ out.append(h["value"])
1166
+ return out
1167
+
1168
+ def prices_record(self) -> Optional[dict]:
1169
+ """What the job's config keeps of its prices: the rows of the models
1170
+ it calls, each with its ``source`` (``list`` the public list, ``own``
1171
+ the user's own row, ``run`` this run's ``--price``), and where and
1172
+ when the list was fetched when a row is its. None when it has none."""
1173
+ rows = price_list.given(self.models_called(), self.prices)
1174
+ if not rows:
1175
+ return None
1176
+ doc, mine = price_list.load(), getattr(self, "run_prices", {})
1177
+ rows = {n: {**r, "source": "run" if n in mine else "own" if n in doc["own"] else "list"} for n, r in rows.items()}
1178
+ listed = doc["from"] and any(r["source"] == "list" for r in rows.values())
1179
+ return {**({"from": doc["from"], "on": doc["on"]} if listed else {}), "models": rows}
1180
+
1181
+ def ledger_file(self) -> Optional[str]:
1182
+ """The file of the run's spending cap, for a relay; None without a cap."""
1183
+ led = getattr(self, "ledger", None)
1184
+ return led.path if led is not None else None
1185
+
1186
+ def models_called(self) -> List[str]:
1187
+ """Every model this run's relays may send a call to, as sent: the
1188
+ run's, the judge's, the simulator's, an option's embeddings and the
1189
+ agent's side models. Any other name a caller asks for is replaced by
1190
+ the run's (``relay``), so these are all a run can be priced for."""
1191
+ models = ledgers.models(self.other, *(getattr(self, n, None) for n in ("model", "judge_model", "simulator_model", "embed_model")),
1192
+ control=self.other.get("_kind") == "control")
1193
+ reranker = getattr(self, "rerank_model", None)
1194
+ return models + ([reranker] if reranker and reranker not in models else [])
1195
+
1196
+ def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] = None, own: bool = False):
1197
+ """The unit's own relay, started: in this process on the local
1198
+ backend, else a sidecar container. Either answers ``url`` (the
1199
+ driver's way in, on this machine), ``calls`` and ``stop``;
1200
+ ``unit_url`` is the address the agent or capability is given. ``own``:
1201
+ the unit runs in its own image (:meth:`own_box`), not one of the
1202
+ benchmark's layers on the base."""
1203
+ if not self.base:
1204
+ return None # a control with no judge: no model is called
1205
+ record = os.path.join(trial, ".work", "relay")
1206
+ side = tuple((self.other.get("model") or {}).get("side_models") or ())
1207
+ key_header = PROVIDERS[self.provider].get("key_header", "authorization")
1208
+ if self.backend == "local":
1209
+ rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side,
1210
+ key_header=key_header, ledger=self.ledger_file()).start()
1211
+ rl.unit_url = rl.url(tag)
1212
+ return rl
1213
+ routes = {tag + "-judge": self.judge_model} if self.judge_model else {}
1214
+ if self.simulator_model:
1215
+ routes[tag + "-simulator"] = self.simulator_model
1216
+ doors, listeners = [], []
1217
+ if self.other["_kind"] == "agent" and mcp_machine.wanted(setup):
1218
+ # the benchmark's own servers run here, in the driver's environment; the agent reaches each
1219
+ # through a door of its own on the unit's network (mcp_machine), its entry the bridge
1220
+ work = os.path.join(trial, ".work")
1221
+ try:
1222
+ setup["mcp"], doors, listeners = mcp_machine.doors(self.bench, setup, install.driver_dir(self.bench),
1223
+ self.machine_env(trial), work, work)
1224
+ except mcp_machine.Unknown as e:
1225
+ raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}")
1226
+ from . import web_services
1227
+ try:
1228
+ doors.extend(web_services.doors(setup))
1229
+ except (ValueError, OSError) as e:
1230
+ for ln in listeners:
1231
+ ln.stop()
1232
+ raise _Ended("EnvironmentSetupError", f"the unit's web services: {e}")
1233
+ door = getattr(self, "loopback", None)
1234
+ probe = ((env_def or {}).get("network") or {}).get("probe") if self.other["_kind"] == "agent" else None
1235
+ rl = container.Unit(self.name, tag, record, image=self.image, app=APP, base=door.base if door else self.base, key=self.key,
1236
+ model=self.model, routes=routes, side_models=side, key_header=key_header,
1237
+ prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows
1238
+ servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None,
1239
+ ledger=self.ledger_file(),
1240
+ network_probe=probe, probe_now=bool(probe) and self.first_probe())
1241
+ rl.mcp_listeners = listeners
1242
+ try:
1243
+ try:
1244
+ rl.unit_image = self.setup_driver.image if self.setup_driver else "" if own else self.unit_image(setup)
1245
+ except container.Unavailable as e:
1246
+ raise _Ended("EnvironmentSetupError", f"the unit's environment: {e}")
1247
+ rl.unit_rw, rl.unit_files = container.mcp_mounts(setup.get("mcp") or [], trial)
1248
+ rl.unit_binds = self.agent_files(env_def)
1249
+ rl.start()
1250
+ except BaseException:
1251
+ for ln in listeners:
1252
+ ln.stop()
1253
+ raise
1254
+ rl.unit_url = rl.inside(tag)
1255
+ return rl
1256
+
1257
+ def machine_env(self, trial: str) -> dict:
1258
+ """The environment of a benchmark's MCP server run on this machine
1259
+ (``mcp_machine``): the driver's, less its secrets - the agent may see
1260
+ it on the local backend - with a home of its own in the trial."""
1261
+ home = os.path.join(trial, ".work", "mcp-home")
1262
+ os.makedirs(home, exist_ok=True)
1263
+ drv = install.driver_dir(self.bench)
1264
+ extra = install.process_env(definitions.driver_install(self.bench), drv, home) if drv else {}
1265
+ return base_env(home, {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
1266
+ "NULLIUS_DATASET_ID": self.ds["id"], "PYTHONDONTWRITEBYTECODE": "1",
1267
+ **option_env(getattr(self, "bench_options", {})), **self.embeddings_env()})
1268
+
1269
+ def embeddings_env(self) -> dict:
1270
+ """The driver's (and its MCP servers') search calls: one OpenAI base URL, with embeddings
1271
+ and reranking pinned separately. Nothing when the selected search calls no model."""
1272
+ if getattr(self, "bench_relay", None) is None:
1273
+ return {}
1274
+ return {"OPENAI_BASE_URL": self.bench_relay.url(self.embeddings_tag(), model=self.rerank_model or self.embed_model,
1275
+ embeddings=self.embed_model if self.rerank_model else None),
1276
+ "OPENAI_API_KEY": relay.PLACEHOLDER_KEY}
1277
+
1278
+ def embeddings_tag(self) -> str:
1279
+ """This thread's route on the benchmark's relay. A driver serves one thread, one unit at a time: what
1280
+ it was refused there is that unit's, never one running beside it."""
1281
+ return f"embeddings-{threading.get_ident()}"
1282
+
1283
+ def unit_image(self, setup: dict) -> str:
1284
+ """The image a unit runs in (``container.image_for``); the first unit
1285
+ to ask for an environment says its build - or that it is there
1286
+ already - as a stage of its own."""
1287
+ work = os.path.join(self.job_dir, ".work")
1288
+ name, _ = definitions.environment_of(self.bench, setup, self.ds)
1289
+ with self._said:
1290
+ first = name is not None and name not in self._envs
1291
+ self._envs.add(name)
1292
+ if not first:
1293
+ return container.image_for(self.bench, setup, self.ds, work)
1294
+ tag = container.env_tag(self.bench, name, self.image)
1295
+ with self.progress.stage("environment", cached=container.present(tag), start=f"building {tag}", name=name):
1296
+ return container.image_for(self.bench, setup, self.ds, work)
1297
+
1298
+ def place(self, rl, cmd: List[str], env: dict, workdir: str, home: str, interactive: bool = True, box=None):
1299
+ """A unit's command and environment as they are started: unchanged on
1300
+ the local backend; in a container, ``docker run`` on the unit's network,
1301
+ with the folders it reads at their own paths and the image's ``PATH``
1302
+ behind the install's. The secrets its environment sends the agent
1303
+ (``to: agent``) are added to its environment - in a container passed by
1304
+ name, never on an argument list. In an own image (``box``), ``docker
1305
+ exec`` in the unit's container, the image's own ``PATH`` first - the
1306
+ task's programs are the task's - then the install's and the runtimes'."""
1307
+ env = dict(env, **(getattr(rl, "unit_secrets", None) or {}))
1308
+ env.update(getattr(rl, "network_env", {}))
1309
+ if box is not None:
1310
+ inside = {**box.env, **{k: v for k, v in env.items() if k not in container.HOST_ONLY},
1311
+ **getattr(box, "agent_proxy", {})}
1312
+ path = ":".join(([box.env["PATH"]] if "PATH" in box.env else []) + [box.path]
1313
+ + install.bin_dirs(self.inst) + [d + "/bin" for _, d in box.runtimes.values()])
1314
+ argv = box.exec_argv(container.in_runtime(cmd, box.runtimes), inside, path, interactive=interactive)
1315
+ return argv, container.program_env(inside)
1316
+ if self.backend == "local":
1317
+ return cmd, env
1318
+ ws = os.path.join(os.path.dirname(home), "workspace")
1319
+ # What the agent's container sees: its workspace and home, its own
1320
+ # install and definition, and the files its MCP servers need - never
1321
+ # this application's folder (its .env holds the keys) nor the
1322
+ # benchmark's (its data holds the answers).
1323
+ mounted = container.mounts([ws, home, *getattr(rl, "unit_rw", [])],
1324
+ [self.inst, self.other.get("_here")],
1325
+ getattr(rl, "unit_files", []))
1326
+ env = dict(env, PATH=os.pathsep.join(install.bin_dirs(self.inst) + [container.IMAGE_PATH]))
1327
+ return rl.wrap(list(cmd), env, workdir, mounted, image=rl.unit_image, interactive=interactive,
1328
+ binds=getattr(rl, "unit_binds", []))
1329
+
1330
+ # -- a control: nothing, or the benchmark's own reference -----------------
1331
+ def control_answer(self, unit, ws, box=None, name: str = "") -> str:
1332
+ """``nop`` answers "" and leaves the workspace as setup left it;
1333
+ ``oracle`` is the driver's ``oracle`` - its answer, and whatever it
1334
+ wrote into the workspace, or, in an own image (``box``), the step its
1335
+ reply names (``solve``: the reference solution copied in and run in
1336
+ the unit's container). Either goes to ``score`` as an agent's answer
1337
+ would; a capability benchmark's skips ``produce``."""
1338
+ if self.other["key"] != "oracle":
1339
+ return ""
1340
+ try:
1341
+ reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=DRIVER_SECONDS)
1342
+ except jsonl.Failed as e:
1343
+ raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: " + str(e))
1344
+ if reply.get("solve") is not None:
1345
+ if box is None:
1346
+ raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: solve runs in an own image, and this unit has none")
1347
+ if name: # the reference solution, run under its own limit
1348
+ self.phase(name, unit, "agent_run", float(reply["solve"].get("seconds") or self.turn_seconds))
1349
+ done = self.in_box(box, reply["solve"], collect=False, what="the benchmark's oracle")
1350
+ os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True)
1351
+ with open(os.path.join(os.path.dirname(ws), "agent", "solve.txt"), "w") as f:
1352
+ f.write(done["output"])
1353
+ if done["exit"] != 0:
1354
+ raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: its solution "
1355
+ + ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}"))
1356
+ return str(reply.get("answer") or "")
1357
+
1358
+ # -- an own image: the unit's container ------------------------------------
1359
+ @contextmanager
1360
+ def agent_internet(self, setup: dict, box, rl, trial: str):
1361
+ """The benchmark's task policy, never a run option: a public-address proxy
1362
+ during the agent or reference solution, removed before any verification.
1363
+ Internal relay/server aliases stay direct; failed and cancelled work closes
1364
+ the same door and keeps its traffic record in ``agent/egress.log``."""
1365
+ if not setup.get("internet"):
1366
+ yield
1367
+ return
1368
+ try:
1369
+ try:
1370
+ box.online()
1371
+ box.agent_proxy = container.proxy_env(rl.direct() if isinstance(rl, container.Unit) else [container.RELAY_ALIAS])
1372
+ except (container.Unavailable, ValueError) as e:
1373
+ raise _Ended("EnvironmentSetupError", f"the task's internet access: {e}")
1374
+ yield
1375
+ finally:
1376
+ box.agent_proxy = {}
1377
+ try:
1378
+ try:
1379
+ said = box.egress_said()
1380
+ if said:
1381
+ out = os.path.join(trial, "agent")
1382
+ os.makedirs(out, exist_ok=True)
1383
+ with open(os.path.join(out, "egress.log"), "w") as f:
1384
+ f.write(said)
1385
+ finally:
1386
+ box.offline()
1387
+ except container.Unavailable as e:
1388
+ raise _Ended("VerifierError", f"the task's internet access could not be closed: {e}")
1389
+
1390
+ def own_box(self, tag: str, setup: dict, rl, home: str):
1391
+ """The unit's own container, started (``container.Box``): the image its
1392
+ setup names, built or pulled on first use and said as a stage; for an
1393
+ agent, its install, definition, home and runtimes mounted, and each
1394
+ runtime shown to start in the image - a unit where one cannot (a
1395
+ glibc runtime in a musl image) is not run, which is no one's result."""
1396
+ spec = setup.get("image")
1397
+ bad = container.spec_problems(spec)
1398
+ bad += container.task_env_problems(setup.get("env", {}), (self.other.get("model") or {}).get("env", {}))
1399
+ if setup.get("verify") is not None:
1400
+ bad += ["verify." + p for p in container.step_problems(setup["verify"], collect=True)]
1401
+ if setup.get("prepare") is not None:
1402
+ bad += ["prepare." + p for p in container.step_problems(setup["prepare"], collect=False)]
1403
+ if setup.get("workdir") is not None and not (isinstance(setup["workdir"], str) and setup["workdir"].startswith("/")):
1404
+ bad.append("workdir: an absolute path inside the image")
1405
+ if bad:
1406
+ raise _Ended("EnvironmentSetupError", "the unit's setup: " + "; ".join(bad))
1407
+ why = container.unusable()
1408
+ if why:
1409
+ raise _Ended("EnvironmentSetupError", f"not run: its own image needs docker, and {why}")
1410
+ box = None
1411
+ try:
1412
+ image = self.own_image(spec)
1413
+ cfg = container.image_config(image)
1414
+ mounted, runtimes = [], {}
1415
+ if self.other["_kind"] == "agent":
1416
+ # as on the base: its home, its install and definition, its MCP servers' files -
1417
+ # never this application's folder nor the benchmark's; and what it runs on, under /opt/nullius
1418
+ mounted = container.mounts([home, *getattr(rl, "unit_rw", [])],
1419
+ [self.inst, self.other.get("_here")],
1420
+ getattr(rl, "unit_files", []))
1421
+ runtimes = self.runtimes
1422
+ box = container.Box(self.name, tag, image, workdir=setup.get("workdir") or cfg["workdir"], path=cfg["path"],
1423
+ network=rl.names["network"] if isinstance(rl, container.Unit) else None,
1424
+ mounted=mounted, runtimes=runtimes, memory_mb=spec.get("memory_mb"), cpus=spec.get("cpus"),
1425
+ app=APP, env=setup.get("env")).start()
1426
+ for name in runtimes:
1427
+ why = box.runtime_problem(name)
1428
+ if why:
1429
+ raise _Ended("EnvironmentSetupError", "not run: " + why)
1430
+ return box
1431
+ except (container.Unavailable, ValueError, OSError) as e:
1432
+ if box is not None:
1433
+ box.stop()
1434
+ raise _Ended("EnvironmentSetupError", f"the unit's own image: {e}")
1435
+ except _Ended:
1436
+ box.stop()
1437
+ raise
1438
+
1439
+ @staticmethod
1440
+ def box_changes(box) -> Optional[str]:
1441
+ """What changed in the unit's container since its image - the work
1442
+ itself, where the workspace is the container's: the trial keeps it
1443
+ (``agent/container-changes.txt``)."""
1444
+ try:
1445
+ return box.diff()
1446
+ except container.Unavailable:
1447
+ return None
1448
+
1449
+ def release_image(self, image: str) -> None:
1450
+ """A unit is done with its own image: with ``remove_images``, removed
1451
+ when no other unit is using it and this run fetched it
1452
+ (``container.drop_image``)."""
1453
+ with self._said:
1454
+ left = self._image_users.get(image, 1) - 1
1455
+ if left > 0:
1456
+ self._image_users[image] = left
1457
+ return
1458
+ self._image_users.pop(image, None)
1459
+ if not self.keep_images and container.fetched(image):
1460
+ container.drop_image(image)
1461
+
1462
+ def own_image(self, spec: dict) -> str:
1463
+ """The image an own-image unit names, built or pulled the first time
1464
+ a unit asks for it (``container.own_image``) - said as a stage then, or
1465
+ that it is there already."""
1466
+ root = self.up or self.bench["_here"]
1467
+ tag = spec.get("pull") or container.own_tag(self.bench, spec["build"], container.within(root, spec["build"]))
1468
+ work = os.path.join(self.job_dir, ".work")
1469
+ with self._said:
1470
+ first = tag not in self._envs
1471
+ self._envs.add(tag)
1472
+ self._image_users[tag] = self._image_users.get(tag, 0) + 1
1473
+ if not first:
1474
+ return container.own_image(self.bench, spec, root, work)
1475
+ with self.progress.stage("environment", cached=container.present(tag),
1476
+ start=("pulling " if spec.get("pull") else "building ") + tag, name=tag):
1477
+ return container.own_image(self.bench, spec, root, work)
1478
+
1479
+ def in_box(self, box, step: dict, *, collect: bool, what: str, out: Optional[str] = None) -> dict:
1480
+ """A step (``verify``, ``solve``) run in the unit's container. For the
1481
+ check (``collect``), what the agent left running is stopped first
1482
+ (``container.Box.settle``) - unless the step keeps it
1483
+ (``keep_processes``) - and what was stopped kept in ``out``
1484
+ (``stopped.txt``). Then its files are copied in from the upstream and the ones
1485
+ the driver gives by their text written in (``write``), a
1486
+ way out to public addresses given when it asks for the network
1487
+ (``container.Box.online``: a proxy, named to the command alone), and
1488
+ its command run in the working directory."""
1489
+ bad = container.step_problems(step, collect=collect)
1490
+ if bad:
1491
+ raise _Ended("VerifierError" if collect else "NonZeroAgentExitCodeError", f"{what}: " + "; ".join(bad))
1492
+ root = self.up or self.bench["_here"]
1493
+ try:
1494
+ if collect and not step.get("keep_processes"):
1495
+ stopped = box.settle()
1496
+ if stopped and out:
1497
+ with open(os.path.join(out, "stopped.txt"), "w") as f:
1498
+ f.write("".join(s + "\n" for s in stopped))
1499
+ for src, dest in (step.get("copy") or {}).items():
1500
+ box.copy_in(container.within(root, src), dest)
1501
+ for dest, text in (step.get("write") or {}).items():
1502
+ box.write_in(dest, text)
1503
+ env = box.online() if step.get("network") else (getattr(box, "agent_proxy", None) if not collect else None)
1504
+ if step.get("collect"):
1505
+ box.run(["mkdir", "-p", step["collect"]], 60, user="0")
1506
+ return box.run(step["run"], float(step.get("seconds") or self.turn_seconds), env=env)
1507
+ except (container.Unavailable, ValueError) as e:
1508
+ raise _Ended("VerifierError" if collect else "NonZeroAgentExitCodeError", f"{what}: {e}")
1509
+
1510
+ def prepare(self, step: dict, box, trial: str) -> None:
1511
+ """The benchmark's preparation of the unit's container (``prepare``),
1512
+ before the agent or a control starts: what the task's own harness adds
1513
+ to every image (Terminal-Bench's tmux). Its files copied and written
1514
+ in, its command run as root - with a way out, when it asks for the
1515
+ network, through the check's proxy (``container.Box.online``) - then
1516
+ that proxy removed and shown gone (``Box.offline``) and whatever it
1517
+ left running stopped (``Box.settle``): the agent starts with no way
1518
+ out but its model, as it would have without one. Its output and what
1519
+ the proxy let through are kept (``setup/prepare.txt``,
1520
+ ``setup/egress.log``). Any failure, closing the way out included, is
1521
+ the environment's (``EnvironmentSetupError``), never the agent's."""
1522
+ out = os.path.join(trial, "setup")
1523
+ os.makedirs(out, exist_ok=True)
1524
+ root = self.up or self.bench["_here"]
1525
+ done = None
1526
+ try:
1527
+ for src, dest in (step.get("copy") or {}).items():
1528
+ box.copy_in(container.within(root, src), dest)
1529
+ for dest, text in (step.get("write") or {}).items():
1530
+ box.write_in(dest, text)
1531
+ env = box.online() if step.get("network") else None
1532
+ try:
1533
+ done = box.run(step["run"], float(step.get("seconds") or self.turn_seconds), user="0", env=env)
1534
+ finally:
1535
+ said = box.egress_said()
1536
+ if said:
1537
+ with open(os.path.join(out, "egress.log"), "w") as f:
1538
+ f.write(said)
1539
+ box.offline()
1540
+ box.settle()
1541
+ except (container.Unavailable, ValueError) as e:
1542
+ raise _Ended("EnvironmentSetupError", f"the benchmark's preparation: {e}")
1543
+ finally:
1544
+ if done is not None:
1545
+ with open(os.path.join(out, "prepare.txt"), "w") as f:
1546
+ f.write(done["output"])
1547
+ if done["exit"] != 0:
1548
+ raise _Ended("EnvironmentSetupError", "the benchmark's preparation "
1549
+ + ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}")
1550
+ + ": " + done["output"].strip()[-300:])
1551
+
1552
+ def verify(self, step: dict, box, trial: str) -> dict:
1553
+ """The benchmark's check, run in the unit's container once the agent
1554
+ has ended (``in_box``), what it collects copied out into the trial's
1555
+ ``verifier/`` and its output kept there (``test-stdout.txt``, and what
1556
+ its proxy let through and refused, ``egress.log``): what ``score`` is
1557
+ sent as ``verified``, the folder to read the rest from."""
1558
+ out = os.path.join(trial, "verifier")
1559
+ os.makedirs(out, exist_ok=True)
1560
+ try:
1561
+ done = self.in_box(box, step, collect=True, what="the benchmark's check", out=out)
1562
+ finally:
1563
+ said = box.egress_said()
1564
+ if said:
1565
+ with open(os.path.join(out, "egress.log"), "w") as f:
1566
+ f.write(said)
1567
+ with open(os.path.join(out, "test-stdout.txt"), "w") as f:
1568
+ f.write(done["output"])
1569
+ collected = box.copy_out(step["collect"], out) if step.get("collect") else False
1570
+ return {"exit": done["exit"], "timed_out": done["timed_out"], "output": done["output"][-4000:],
1571
+ "dir": out, "collected": collected}
1572
+
1573
+ # -- an agent, over ACP ----------------------------------------------------
1574
+ def simulator(self, rl, tag: str) -> dict:
1575
+ """What a ``turn`` request carries when a model plays the benchmark's
1576
+ user: that model's route on the unit's relay, recorded beside the
1577
+ unit's calls (``<tag>-simulator``) and never counted as the agent's."""
1578
+ if not self.simulator_model:
1579
+ return {}
1580
+ return {"simulator": {"model": self.simulator_model, "key": relay.PLACEHOLDER_KEY,
1581
+ "base_url": rl.url(tag + "-simulator", model=self.simulator_model)}}
1582
+
1583
+ def judge(self, rl, tag: str) -> dict:
1584
+ """What a ``score`` request carries when the benchmark is graded by a
1585
+ model - and every ``turn`` too, for a driver whose upstream grades the
1586
+ conversation as it ends: that model's route on the unit's relay."""
1587
+ if not self.judge_model:
1588
+ return {}
1589
+ return {"judge": {"model": self.judge_model, "base_url": rl.url(tag + "-judge", model=self.judge_model),
1590
+ "key": relay.PLACEHOLDER_KEY}}
1591
+
1592
+ def turn_limit(self, setup: dict) -> float:
1593
+ """How long one turn of the agent may take: the unit's own limit when its ``setup`` names one
1594
+ (``agent_seconds``: a task that declares its agent timeout, as Terminal-Bench's task.toml does),
1595
+ else the run's ``turn_seconds``."""
1596
+ own = setup.get("agent_seconds") if isinstance(setup, dict) else None
1597
+ if isinstance(own, (int, float)) and not isinstance(own, bool) and 0 < own <= MAX_AGENT_SECONDS:
1598
+ return float(own)
1599
+ return self.turn_seconds
1600
+
1601
+ def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name=""):
1602
+ """The agent over ACP, its session's ``cwd`` the unit's workspace - in
1603
+ an own image (``box``), the container's working directory, which this
1604
+ machine does not see. Files are served (``acp.Client(fs=)``) only to an
1605
+ agent running here: one in a container has its workspace mounted, and
1606
+ serving it from this machine would let it swap a folder for a link
1607
+ between the check and the open and reach any file of the user's."""
1608
+ a = self.other
1609
+ env = agent_env(a, self.inst, home, rl.unit_url, self.model, self.options)
1610
+ log = os.path.join(os.path.dirname(ws), "agent", "agent.log")
1611
+ os.makedirs(os.path.dirname(log), exist_ok=True)
1612
+ if "command" in a["start"]:
1613
+ return self._command_agent(a, env, ws, home, setup, transcript, log, tag, unit, rl, box)
1614
+ cmd = install.words(a["start"]["acp"], {"here": a["_here"], "install": self.inst})
1615
+ if self.backend == "local": # inside a container, its own PATH finds it
1616
+ cmd[0] = shutil.which(cmd[0], path=env["PATH"]) or cmd[0]
1617
+ cmd, env = self.place(rl, cmd, env, ws, home, box=box)
1618
+ c = acp.Client(cmd, env, log, cwd=ws, grant=True, fs=box is None and self.backend == "local", stopped=self._stopped)
1619
+ events: List[dict] = []
1620
+ try:
1621
+ try:
1622
+ try:
1623
+ c.start()
1624
+ sid = c.new_session(box.workdir if box is not None else ws, setup.get("mcp") or [])
1625
+ except TimeoutError as e:
1626
+ # the agent never answered its start: this unit's setup failed, not the run
1627
+ raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})")
1628
+ prompt = setup["prompt"]
1629
+ answer = ""
1630
+ # a conversation's turns carry its simulated user's route and its judge's (each when it has one)
1631
+ sim = dict(self.simulator(rl, tag), **self.judge(rl, tag)) if "agent.conversation/v1" in self.bench["needs"] else {}
1632
+ opened = True
1633
+ if self.simulator_model:
1634
+ # The simulated user speaks first: the first turn, asked with nothing said yet,
1635
+ # follows setup's prompt (what the agent is told before anyone speaks).
1636
+ first = self.driver().request("turn", unit=unit, dir=ws, transcript=[], timeout=600, **sim, **self.attempt_kw())
1637
+ opened = not first.get("done")
1638
+ if opened:
1639
+ prompt = prompt + "\n\n" + first["message"]
1640
+ for n in range(self.max_turns if opened else 0):
1641
+ transcript.append({"role": "user", "text": prompt})
1642
+ if n: # the limit is each turn's: the board's clock starts again with it
1643
+ self.phase(name, unit, "agent_run", self.turn_limit(setup))
1644
+ turn = c.prompt(sid, prompt, timeout=self.turn_limit(setup))
1645
+ answer = turn.text
1646
+ transcript.append({"role": "agent", "text": turn.text, "stop": turn.stop})
1647
+ if "agent.conversation/v1" not in self.bench["needs"]:
1648
+ break
1649
+ nxt = self.driver().request("turn", unit=unit, dir=ws, transcript=transcript, timeout=600, **sim, **self.attempt_kw())
1650
+ if nxt.get("done"):
1651
+ break
1652
+ prompt = nxt["message"]
1653
+ if nxt.get("restart"):
1654
+ # the upstream started its conversation again (its own retry of an attempt that
1655
+ # raised): a new session, told what it was told before anyone spoke
1656
+ sid = c.new_session(box.workdir if box is not None else ws, setup.get("mcp") or [])
1657
+ prompt = setup["prompt"] + "\n\n" + prompt
1658
+ events = c.events
1659
+ except acp.Timeout as e:
1660
+ events = c.events
1661
+ raise _Ended("AgentTimeoutError", str(e), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"))
1662
+ except acp.AgentFailed as e:
1663
+ events = c.events
1664
+ raise _Ended("NonZeroAgentExitCodeError", str(e), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"))
1665
+ except TimeoutError as e:
1666
+ events = c.events # a later request the agent never answered (a new session)
1667
+ raise _Ended("AgentTimeoutError", f"the agent did not answer {e} in time",
1668
+ job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"))
1669
+ except jsonl.Failed as e:
1670
+ raise _Ended("VerifierError", "the benchmark's next turn: " + str(e))
1671
+ finally:
1672
+ c.close()
1673
+ raw = "".join(json.dumps(e) + "\n" for e in events)
1674
+ return answer, job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {"acp.jsonl": raw}
1675
+
1676
+ def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl, box=None):
1677
+ cmd = install.words(a["start"]["command"], {"here": a["_here"], "install": self.inst, "prompt": setup["prompt"]})
1678
+ cmd, env = self.place(rl, cmd, env, ws, home, interactive=False, box=box)
1679
+ transcript.append({"role": "user", "text": setup["prompt"]})
1680
+ t = time.time()
1681
+ try:
1682
+ with open(log, "a") as err:
1683
+ r = run_group(cmd, self.turn_limit(setup), stopped=self._stopped, cwd=ws, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
1684
+ stderr=err, text=True)
1685
+ except subprocess.TimeoutExpired:
1686
+ raise _Ended("AgentTimeoutError", f"the agent did not end in {self.turn_limit(setup)} s")
1687
+ if r.returncode != 0:
1688
+ raise _Ended("NonZeroAgentExitCodeError", f"exit {r.returncode}")
1689
+ transcript.append({"role": "agent", "text": r.stdout})
1690
+ events = [{"t": t, "update": {"sessionUpdate": "user_message_chunk", "content": {"type": "text", "text": setup["prompt"]}}},
1691
+ {"t": time.time(), "update": {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": r.stdout}}}]
1692
+ return r.stdout.strip(), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {}
1693
+
1694
+ # -- a capability, through its port -------------------------------------------
1695
+ def drive_feature(self, unit, ws, home, tag, setup, transcript, rl):
1696
+ f = self.other
1697
+ cmd = install.words(f["start"]["command"], {"here": f["_here"], "install": self.inst})
1698
+ env = agent_env(f, self.inst, home, rl.unit_url, self.model, self.options)
1699
+ env["PYTHONDONTWRITEBYTECODE"] = "1"
1700
+ cmd, env = self.place(rl, cmd, env, f["_here"], home)
1701
+ cwd = os.path.join(os.path.dirname(ws), ".work", "capability") # the unit's, never the catalogue folder
1702
+ os.makedirs(cwd, exist_ok=True)
1703
+ proc = jsonl.Proc(cmd, cwd, env, os.path.join(os.path.dirname(ws), "agent", "capability.log"), stopped=self._stopped)
1704
+ os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True)
1705
+ exchanges: List[dict] = []
1706
+
1707
+ def send(op, **kw):
1708
+ t0 = time.time()
1709
+ reply = proc.request(op, timeout=self.turn_seconds, **kw)
1710
+ exchanges.append({"op": op, "request": kw, "reply": reply, "t0": t0, "t1": time.time()})
1711
+ return reply
1712
+
1713
+ try:
1714
+ try:
1715
+ proc.start()
1716
+ plan = setup.get("memory") or {}
1717
+ for s in plan.get("add") or []:
1718
+ send("add", session=s)
1719
+ recalled = [{"id": q.get("id"), "memory": send("recall", question=q["question"]).get("memory", "")}
1720
+ for q in plan.get("recall") or []]
1721
+ except jsonl.Failed as e:
1722
+ raise _Ended("NonZeroAgentExitCodeError", "the capability: " + str(e),
1723
+ job.trajectory_from_exchanges(exchanges, [], f, self.model, f"{self.name}/{unit}"))
1724
+ finally:
1725
+ proc.close()
1726
+ try:
1727
+ out = self.driver().request("produce", unit=unit, dir=ws, recalled=recalled, model=self.model,
1728
+ base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds)
1729
+ except jsonl.Failed as e:
1730
+ raise _Ended("VerifierError", "the benchmark's reader: " + str(e),
1731
+ job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}"))
1732
+ transcript.append({"role": "capability", "recalled": recalled})
1733
+ return out.get("answer"), job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}")
1734
+
1735
+
1736
+ def drive_upstream(self, unit, ws, tag, transcript, rl):
1737
+ """The setup's unit: the driver runs the upstream's own answerer on it
1738
+ (``answer``), asked as ``asked`` through the unit's relay, which sends
1739
+ the run's model in its place. A reply with no text - a reasoning model
1740
+ that spent the upstream's token budget thinking - is said as such."""
1741
+ o = self.other
1742
+ try:
1743
+ # the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes
1744
+ out = self.driver().request("answer", unit=unit, dir=ws, setup=o["key"], model=self.asked, **self.attempt_kw(),
1745
+ **self.simulator(rl, tag), **self.judge(rl, tag),
1746
+ base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds)
1747
+ except jsonl.Failed as e:
1748
+ calls = rl.calls(tag)
1749
+ silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()]
1750
+ if silent:
1751
+ u = (silent[-1].get("reply") or {}).get("usage") or {}
1752
+ thought = u.get("reasoning") or 0
1753
+ raise _Ended("OutputLengthExceededError",
1754
+ f"{self.model} returned no text" + (f" ({thought} of its {u.get('output')} tokens were reasoning)" if thought else "")
1755
+ + f": {self.bench['label']}'s code asks for a short answer, and a reasoning model can spend it all before it answers"
1756
+ + (" - the original's models do not reason" if o["key"] == "model" else ""),
1757
+ job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}"))
1758
+ raise _Ended("NonZeroAgentExitCodeError", f"{o['label']}: " + str(e),
1759
+ job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}"))
1760
+ transcript.append({"role": "upstream", "setup": o["key"], "asked": self.asked, "sent": self.model})
1761
+ return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}")
1762
+
1763
+
1764
+ class _Ended(Exception):
1765
+ def __init__(self, kind, said, trajectory=None):
1766
+ super().__init__(said)
1767
+ self.kind, self.said, self.trajectory = kind, said, trajectory
1768
+
1769
+
1770
+ def run(bench: dict, other: dict, **kw) -> str:
1771
+ return Run(bench, other, **kw).start()
1772
+
1773
+
1774
+ def run_and_report(bench: dict, other: dict, **kw) -> tuple:
1775
+ """(job directory, whether it failed): it stopped on a refusing endpoint,
1776
+ or units ran and none reached a verdict - what the CLI's exit status says.
1777
+ A run its spending cap stopped ended as it was asked to: no failure,
1778
+ whatever it reached."""
1779
+ r = Run(bench, other, **kw)
1780
+ job_dir = r.start()
1781
+ return job_dir, bool(r.halted) or (r.n_ended > 0 and r.n_verdicts == 0 and not r.capped)