nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/execution.py ADDED
@@ -0,0 +1,1653 @@
1
+ """Running a recipe: on this machine, or on a host from Settings, with the
2
+ job mirrored back as it goes - and surviving Nullius going away.
3
+
4
+ One ``Plan`` is what the builder posts: the run commands compiled for a
5
+ runner (they name ``$NULLIUS_HARBOR``, ``$NULLIUS_TASKS`` and
6
+ ``$NULLIUS_JOBS`` and nothing else about where they run), one part per
7
+ agent in the order they run (controls first), the Harbor the recipe pins
8
+ (``harbor_pin``, ``harbor/source.py``), the task folder they read, and the
9
+ target. ``$NULLIUS_HARBOR`` is a small ``harbor`` script in the run
10
+ directory that runs the pinned Harbor through ``uvx``: a release from PyPI,
11
+ or a repository's commit built on this machine, whose wheels a host is sent
12
+ (only this machine needs access to a private fork; the host resolves the
13
+ rest from PyPI). A plan from before the pin names none, and its
14
+ ``$NULLIUS_HARBOR`` is ``harbor``, whichever is first on the target's PATH.
15
+ The job records which Harbor ran. An ``Execution`` carries the plan out in a
16
+ thread of its own and writes what it does to a log the page follows:
17
+
18
+ - on a host: ssh in; docker checked for, with ``uv`` for a pinned Harbor
19
+ (installed when missing) or ``harbor`` on the host's PATH for an unpinned
20
+ one (``~/.local/bin`` included); the task folder sent over; each agent's
21
+ script started **detached** there (``setsid nohup``, or Python session creation when
22
+ ``setsid`` is absent; its claim, pid and
23
+ exit code on disk beside its log: ``run-<part>.sh``, ``.launch``, ``.pid``,
24
+ ``.log``, ``.exit``, and ``.stop`` once a Stop has been said to it), so a
25
+ dropped connection or a stopped Nullius does not end it; the job
26
+ directory synced back with rsync every few seconds while it runs and once
27
+ more at the end, so each trial appears here as it finishes;
28
+ - on this machine: the same, with the local paths, in ``runs/<name>``; a
29
+ run that needs docker (Harbor's, or the standard's on its container backend)
30
+ first asks that it answer here (``docker_here``) and starts nothing when
31
+ it does not.
32
+
33
+ A plan for the Nullius standard (``runner`` ``standard``,
34
+ ``standard/recipe.py``) runs this repository's own ``cli`` from
35
+ ``$NULLIUS_STANDARD``: this repository here; from an installed wheel or on
36
+ a host, a content-addressed copy of the backend and effective catalogue.
37
+ Only code, definitions and their declared resources travel, never user
38
+ configuration or credentials. Snapshots live in ``standard-app/<content-id>/``
39
+ beside the runs and are sent to ``~/.nullius/app/<content-id>`` the way a task
40
+ folder is. There it needs ``python3`` and ``git``, gets ``uv`` when missing
41
+ (an agent's pip install builds with it), and docker when the plan says so
42
+ (``docker``: a recipe on the container backend) - checked, like a Harbor run's,
43
+ before anything is sent. A run split over several machines is one plan per
44
+ machine, each one shard (``standard/shard.py``).
45
+
46
+ The target reads an explicit catalogue: the user's catalogue for a local
47
+ run; on a host, a writable copy of the snapshot in the run's ``catalogue/``.
48
+ Proofs update that copy, never the shared runner. Both ``jobs/`` and
49
+ ``catalogue/`` are mirrored back beside the run record; remote proofs do
50
+ not replace the local user's active definitions.
51
+
52
+ An agent's run that fails with more to come pauses the run and the page
53
+ asks: continue or stop. Every state change lands in ``runs/<name>/run.json``;
54
+ ``restore`` reads those records when Nullius starts and reattaches to what
55
+ is still running, waits again where it was paused, and settles what ended
56
+ while nobody was looking. A record is checked against the part's files and
57
+ processes on the target before it is trusted (``_settle``): a part whose pid
58
+ was never saved, but which was claimed or started, is followed by the pid it
59
+ wrote itself or by its processes - never started a second time; a record
60
+ from before a part's end was one save, naming the next part with the ended
61
+ one's pid, goes on from that end; and one saved as the next part was
62
+ launched, still holding the ended one's pid, follows the next part by its
63
+ own processes.
64
+
65
+ A pid is a hint, never proof: a part's processes are the ones running its
66
+ script in its run directory (``_part_pids``). A part is alive while it has
67
+ one, gone without an exit code when it has none, and nothing else is
68
+ signalled - not a pid a record kept after its part ended, nor one the system
69
+ has since given to another process. A Stop ends the part it raced, whether
70
+ or not its launcher had answered (``_kill_part``).
71
+
72
+ The model key and endpoint follow the recipe's provider (``Plan.key_var``,
73
+ ``url_var``, ``base_url``): on OpenRouter the Settings' OPENROUTER_API_KEY
74
+ and OpenRouter's own endpoint; on OpenAI, OPENAI_API_KEY and no endpoint, so
75
+ the clients use OpenAI's; on an OpenAI-compatible endpoint, its URL and its
76
+ key (MODEL_API_URL and MODEL_API_KEY, or a named endpoint's own) - the key
77
+ optional: without it the run is sent
78
+ ``settings.MODEL_KEY_PLACEHOLDER``, for a server that checks no key - and a
79
+ run without that endpoint fails before it starts.
80
+ They are read on this machine and handed to the run as environment
81
+ (API_TOKEN, OPENAI_API_KEY, LLM_API_KEY; API_URL, OPENAI_BASE_URL,
82
+ LLM_BASE_URL) - over stdin to the launcher, never in argv (the key to a
83
+ docker relay as a file); a stdin cut short starts nothing and fails the
84
+ run at that part (``_launch``), and one that holds a line break is never sent
85
+ (``_line_break``) - and they are the only credentials it has: its commands get this
86
+ process's environment less every secret in Settings, whichever provider it
87
+ belongs to (``withheld``); a host gets nothing of it but stdin.
88
+
89
+ A standard run on a host is also sent its benchmark's licences
90
+ (``Plan.licences``: the benchmark and the names its environments declare,
91
+ read from its store, ``nullius.secrets``, or a data provider's token from that
92
+ provider's, when each part is launched) as one
93
+ more stdin line, ``NULLIUS_LICENCES``: in the launcher's environment, never
94
+ on disk there - but for a licence file, which the run writes to
95
+ ``/dev/shm/nullius-<run>/`` (0600, in memory) and the part's detached shell
96
+ removes when it ends, a Stop included. On this machine the run reads the
97
+ store itself, and nothing is sent. A record from before ``base_url`` loads with none, and its run
98
+ goes on with the settings it named. Log lines pass through
99
+ ``redact`` before they are kept. Every command runner is a call-time seam,
100
+ so the suite runs a plan with none of ssh, uv, rsync or docker present.
101
+
102
+ What the run is doing is said in stages (``nullius/progress.py``): the steps
103
+ here that get a host ready - connect, uv, the tasks, app/, a pinned Harbor -
104
+ each a ``[nullius] stage=...`` line in the run's log, and the run's own (the
105
+ standard's cli says its image, installs, dataset and units) read from the
106
+ part's log as it comes. ``events`` holds them all, read back from the log on
107
+ a restart, and ``summary``'s ``stages`` is them folded for the page.
108
+ """
109
+ from __future__ import annotations
110
+
111
+ import base64
112
+ import hashlib
113
+ import json
114
+ import os
115
+ import re
116
+ import shlex
117
+ import subprocess
118
+ import sys
119
+ import threading
120
+ import time
121
+ from dataclasses import asdict, dataclass, field
122
+ from datetime import datetime, timezone
123
+ from typing import Callable, Dict, List, Optional
124
+
125
+ from nullius import hosts, paths, prices as price_list, progress, redact, secrets, settings
126
+ from nullius.harbor import source as harbor_source
127
+ from nullius.standard import container, payload as standard_payload
128
+
129
+ REMOTE_HOME = "~/.nullius" # on a host: tasks/, runs/, app/, harbor/ (a pinned Harbor's wheels)
130
+ # The names the launcher gives a run's key and its endpoint (_launch), read
131
+ # from stdin: the only credentials a run's commands get.
132
+ KEY_NAMES = ("API_TOKEN", "OPENAI_API_KEY", "LLM_API_KEY")
133
+ URL_NAMES = ("API_URL", "OPENAI_BASE_URL", "LLM_BASE_URL")
134
+ # The line the launcher's stdin ends with, after the key and the endpoint:
135
+ # either may be empty, so only this line says the two arrived whole.
136
+ STDIN_END = "@end"
137
+ SYNC_EVERY = 10.0 # seconds between looks at a running part (mirror, log, exit)
138
+ STOP_POLL = 0.5 # how often a running command looks for a Stop
139
+ STOP_WAIT = 20.0 # seconds a Stop's own commands on the target take at most: the kill and the last copy back, together
140
+ LAUNCH_GRACE = 5.0 # seconds a restart waits for a part a launcher claimed to start (_settle)
141
+ UV_INSTALL = "curl -LsSf https://astral.sh/uv/install.sh | sh"
142
+ RECORD = "run.json"
143
+ STANDARD = "standard" # the Nullius standard's runner key (standard/recipe.py HARNESS)
144
+ REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # this repository: app/ is in it
145
+ _NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
146
+ _ID = re.compile(r"[^A-Za-z0-9_.-]")
147
+ ALIVE = ("preparing", "running", "paused", "stopping")
148
+ NOT_RUN = "not run"
149
+
150
+
151
+ def runs_root(env=os.environ) -> str:
152
+ return env.get("RUNS_DIR") or paths.runs_dir(env)
153
+
154
+
155
+ def app_root(runs: str) -> str:
156
+ """Content-addressed runner snapshots, in ``standard-app/`` beside runs."""
157
+ return os.path.join(os.path.dirname(os.path.abspath(runs)), "standard-app")
158
+
159
+
160
+ def docker_here(sh: Callable, needs: str = "a Harbor run needs it", program: str = "docker") -> Optional[str]:
161
+ """Why a run cannot use ``program`` on this machine, or None when it can
162
+ (``needs`` says why the run asks): ``docker`` for Harbor, the standard's
163
+ engine (``container.DOCKER``) for its container backend. The command on the
164
+ PATH *and* ``<program> ps`` answered, as a host's probe asks
165
+ (``hosts.PROBE``). The command alone is not enough - Docker Desktop's WSL
166
+ integration leaves ``/usr/bin/docker`` in place while its engine is quit.
167
+ ``sh`` is the run's own (returncode, output) runner; this is looked up at
168
+ call time, so the suite can stand in for docker."""
169
+ rc, out = sh(f"command -v {program} >/dev/null 2>&1 || {{ echo @missing; exit 0; }}; {program} ps 2>&1 >/dev/null || echo @down", 60)
170
+ words = out.split()
171
+ if rc != 0 or "@missing" in words:
172
+ return f"{program} is not on this machine; " + needs
173
+ if "@down" in words:
174
+ said = [l for l in out.strip().splitlines() if l.strip() and l.strip() != "@down"]
175
+ return (f"{program} is on this machine but does not answer ({program} ps failed)"
176
+ + (" - is Docker Desktop or the docker daemon running?" if program == "docker" else "")
177
+ + (": " + said[-1].strip()[-300:] if said else ""))
178
+ return None
179
+
180
+
181
+ def _tasks_place(tasks: str, run: str) -> tuple:
182
+ """Where a run's task folder goes on a host, under ~/.nullius: by the
183
+ folder, not the run, so two runs over the same tasks record the same
184
+ dataset and match on the compare page - the folder's name last, under a
185
+ short digest of its whole path, so /a/tasks and /b/tasks do not share
186
+ one copy that ``rsync --delete`` swaps under a run still reading it."""
187
+ src = os.path.realpath(os.path.expanduser(tasks))
188
+ name = _ID.sub("_", os.path.basename(os.path.normpath(src))) or run
189
+ return ("tasks", hashlib.sha256(src.encode()).hexdigest()[:10], name)
190
+
191
+
192
+ def _rq(path: str) -> str:
193
+ """A remote path quoted for the host's shell: ``~`` becomes ``$HOME`` so it
194
+ still expands inside the quotes. The names inside are ours (validated)."""
195
+ return '"' + (("$HOME" + path[1:]) if path.startswith("~") else path).replace('"', '\\"') + '"'
196
+
197
+
198
+ def name_taken(name: str, where: str = "") -> str:
199
+ """What a new run is told when a run directory of its name is there
200
+ already - here (``web/execute.py``, before it starts) or on a host
201
+ (``where``, asked by the run before it sends anything)."""
202
+ return f"a run named {name} exists already{' on ' + where if where else ''}; press Run again for a new name"
203
+
204
+
205
+ def _base(part: dict) -> str:
206
+ """The stem of a part's files in the run directory: ``run-<part>``."""
207
+ return f"run-{_ID.sub('_', part['id'])}"
208
+
209
+
210
+ def _script_rx(base: str) -> str:
211
+ """An extended regex for a command line that runs ``<base>.sh``: the word
212
+ itself, alone or at the end of a path. Its first letter is bracketed so
213
+ the shell that asks, whose own command line holds the regex, never
214
+ matches it."""
215
+ return "(^|[ /])[" + base[0] + "]" + base[1:].replace(".", r"\.") + r"\.sh( |$)"
216
+
217
+
218
+ # Where a target shows its processes. A target without it (no /proc: not
219
+ # Linux) cannot prove a pid is still the part's, and falls back to asking
220
+ # whether the pid lives at all - what every target was asked before.
221
+ PROC = "/proc"
222
+
223
+
224
+ def _part_pids(base: str, pids=()) -> str:
225
+ """Shell, run in the run directory, that prints the part's processes, one
226
+ pid per line: those whose working directory is the run directory and
227
+ whose command line runs ``<base>.sh`` - the detached shell, the script's
228
+ own bash, or a launcher on its way to them. Asked of ``pids`` (what the
229
+ record holds), of the pid the part wrote, and of whatever ``pgrep`` finds
230
+ by that command line, so a part is found with no pid saved. A pid the
231
+ part held, since given to another process, is never printed, so nothing
232
+ that is not the part's is ever signalled - where the target has ``PROC``
233
+ to check it by; where it has none, a pid that lives is taken as the
234
+ part's, as before these checks."""
235
+ rx = _script_rx(base)
236
+ return (f"for p in {' '.join(str(int(p)) for p in pids if p)} $(head -c 32 {base}.pid 2>/dev/null | tr -cd 0-9) "
237
+ f"$(pgrep -f '{rx}' 2>/dev/null); do [ \"$p\" != $$ ] || continue; "
238
+ f"if [ -d {PROC}/$$ ]; then [ {PROC}/$p/cwd -ef . ] && tr '\\0' ' ' <{PROC}/$p/cmdline 2>/dev/null | grep -Eq '{rx}' && echo $p; "
239
+ f"else kill -0 $p 2>/dev/null && echo $p; fi; "
240
+ f"done | sort -un")
241
+
242
+
243
+ @dataclass
244
+ class Plan:
245
+ name: str # the run's name: a directory here and on the host
246
+ script: str # the whole run, as one script (kept for the record)
247
+ host: Optional[hosts.Host] = None # None: this machine
248
+ tasks: str = "" # a local task folder the recipe reads, or ""
249
+ key_var: str = "MODEL_API_KEY" # the setting the key is read from, on this machine
250
+ url_var: str = "MODEL_API_URL" # same for the endpoint, which a run on it cannot go without; "" for none
251
+ base_url: str = "" # the endpoint when url_var names none: a provider's own (OpenRouter's), or "" for its clients' default
252
+ model: str = ""
253
+ jobs: List[dict] = field(default_factory=list) # the job directories the script writes: [{"name", "agent", "dataset"}]
254
+ parts: List[dict] = field(default_factory=list) # the run, one agent at a time: [{"id", "label", "script"}]; empty: the whole script at once
255
+ recipe: dict = field(default_factory=dict) # the builder's recipe the script was compiled from, so the run can be opened again as one; {} for a plan from before
256
+ runner: str = "harbor" # or "standard" (standard/recipe.py): the Nullius standard's own cli
257
+ harbor_pin: dict = field(default_factory=dict) # the Harbor the recipe pins (harbor/source.py pin_of); {} for a plan from before: the harbor on PATH
258
+ docker: bool = False # a runner that needs docker on its target though it is not Harbor (the standard's container backend)
259
+ keyless: bool = False # nothing in it calls a model (the standard's controls): handed no key and no endpoint at all
260
+ licences: dict = field(default_factory=dict) # the standard's on a host: {"benchmark", "names", "providers"} - what it is sent of that benchmark's licences
261
+
262
+ def steps(self) -> List[dict]:
263
+ return [p for p in self.parts if p.get("script")] or [{"id": "run", "label": "the run", "script": self.script}]
264
+
265
+ def check(self) -> Optional[str]:
266
+ if not _NAME.match(self.name):
267
+ return "the run needs a name: letters, digits, dots, dashes"
268
+ if not self.script.strip() and not any(p.get("script") for p in self.parts):
269
+ return "nothing to run"
270
+ if self.tasks and not os.path.isdir(os.path.expanduser(self.tasks)):
271
+ return f"the task folder {self.tasks} is not a directory here"
272
+ return None
273
+
274
+ def to_dict(self) -> dict:
275
+ d = asdict(self)
276
+ d["host"] = self.host.entry if self.host else None
277
+ return d
278
+
279
+ @classmethod
280
+ def from_dict(cls, d: dict) -> "Plan":
281
+ """A recorded plan; a field this version no longer has (a record
282
+ from before, naming an installed Harbor release) is left out."""
283
+ d = dict(d)
284
+ d["host"] = hosts.parse(d["host"]) if d.get("host") else None
285
+ return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
286
+
287
+
288
+ def subprocess_run(argv: List[str], timeout: float = 600.0, stdin: Optional[str] = None,
289
+ stop: Optional[threading.Event] = None) -> tuple:
290
+ """(returncode, combined output); a timeout is a return code of its own,
291
+ and so is a stop: with ``stop`` given, the child is killed the moment the
292
+ event is set (130, "stopped") rather than when its timeout runs out -
293
+ an ssh that hangs for 120 s, an rsync for 900 s, would otherwise hold
294
+ a Stop for that long."""
295
+ try:
296
+ # read as UTF-8 whatever the locale, a byte that is not one as U+FFFD: a run is never failed by how
297
+ # its target spelt what it printed
298
+ proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
299
+ encoding="utf-8", errors="replace", env=_clean_env())
300
+ except OSError as e:
301
+ return 127, str(e)
302
+ deadline = time.monotonic() + timeout
303
+ got = {}
304
+
305
+ def talk():
306
+ # one communicate, whole: a retried one after a timeout stopped sending
307
+ # input past the pipe's buffer (a 465 KB price list never landed)
308
+ try:
309
+ got["out"] = proc.communicate(input=stdin if stdin is not None else "")[0]
310
+ except (OSError, ValueError) as e:
311
+ got["out"] = str(e)
312
+ talker = threading.Thread(target=talk, daemon=True)
313
+ talker.start()
314
+ while True:
315
+ talker.join(STOP_POLL if stop is not None else max(0.0, deadline - time.monotonic()))
316
+ if not talker.is_alive():
317
+ return proc.returncode, got.get("out") or ""
318
+ if stop is not None and stop.is_set():
319
+ proc.kill(); talker.join()
320
+ return 130, "stopped"
321
+ if time.monotonic() >= deadline:
322
+ proc.kill(); talker.join()
323
+ return 124, f"no answer within {int(timeout)} s"
324
+
325
+
326
+ # A part's log, as a look at the target prints it: between these, then its size in bytes (_log_since, _logged).
327
+ LOG_FROM, LOG_TO = "__log__", "__logged__"
328
+
329
+
330
+ def withheld() -> List[str]:
331
+ """The variables a run's commands never take from this process: every
332
+ secret in Settings (the model keys and endpoints, the judge's key), the
333
+ names other tools give them or this application gave them before
334
+ (``settings.INSTEAD``, those that stand for a secret), the launcher's own
335
+ names for the run's key and endpoint, the agent keys (a Harbor run is
336
+ sent them by its launcher, as the key), the names a standard benchmark's
337
+ own licences go by - kept in its own store (``nullius.secrets``), although
338
+ imported saved settings may still hold a value until it is moved in
339
+ Custom keys - and GitHub tokens from saved settings or the
340
+ shell Nullius was started in. Every named endpoint's settings too, so a
341
+ run gets its own endpoint's key alone. A data provider's token name is not one."""
342
+ secret = {v.name for v in settings.VARIABLES if v.secret or v.kind == "endpoint"} | set(settings.endpoint_names())
343
+ return sorted(secret | {old for old, new in settings.INSTEAD.items() if new in secret}
344
+ | set(KEY_NAMES + URL_NAMES) | {"GITHUB_TOKEN", "GH_TOKEN"} | set(settings.agent_key_names())
345
+ | set(_licence_names()))
346
+
347
+
348
+ def _licence_names() -> List[str]:
349
+ """Every name a standard benchmark's environments give a licence of its
350
+ own; a data provider's token (``provider``) is none, so a shell's reaches
351
+ a Harbor run. A standard driver or agent never inherits it (``run.KEEP``)."""
352
+ try:
353
+ from nullius.standard import definitions
354
+ return sorted({s["name"] for b in definitions.load("benchmark", every=True)[0] for s in definitions.secrets_of(b)
355
+ if not s.get("provider")})
356
+ except Exception:
357
+ return []
358
+
359
+
360
+ def _clean_env() -> Dict[str, str]:
361
+ """The environment a run's commands get: this process's, less what
362
+ ``withheld`` names. ``settings.load`` adds saved settings to this process's
363
+ environment without replacing explicit environment values. A run gets
364
+ its provider's key and endpoint from the launcher alone (``_launch``,
365
+ over stdin); ``--from-run`` prevents the CLI reading saved settings again."""
366
+ gone = set(withheld())
367
+ return {k: v for k, v in os.environ.items() if k not in gone}
368
+
369
+
370
+ class Execution:
371
+ """One run of one plan, in its own thread. ``state`` is one of preparing,
372
+ running, paused, stopping, done, failed, stopped; ``log`` is what
373
+ happened, redacted."""
374
+
375
+ def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callable] = None,
376
+ key: Optional[str] = None, url: Optional[str] = None, env=os.environ, record: Optional[dict] = None,
377
+ agent_keys: Optional[Dict[str, str]] = None, licences: Optional[Dict[str, bytes]] = None):
378
+ self.plan = plan
379
+ self.root = root or runs_root(env)
380
+ self._catalogue = paths.catalogue_dir(env)
381
+ self.dir = os.path.join(self.root, plan.name)
382
+ self.jobs = os.path.join(self.dir, "jobs")
383
+ self.log_path = os.path.join(self.dir, "nullius.log")
384
+ self.pins = harbor_source.root(self.root, env) # where a pinned Harbor is built and kept here
385
+ self._stop = threading.Event()
386
+ # Every command the run makes gives way to a Stop; the kill that a
387
+ # Stop itself sends goes through the plain runner, or it would be
388
+ # the first command cut short.
389
+ self._run_plain = run or subprocess_run
390
+ self._run = run or (lambda argv, timeout=600.0, stdin=None: subprocess_run(argv, timeout, stdin, stop=self._stop))
391
+ # an endpoint of the run's own (url_var: an OpenAI-compatible one) may check no key: it is sent the placeholder
392
+ self._key = (key if key is not None else env.get(plan.key_var, "")) or (settings.MODEL_KEY_PLACEHOLDER if plan.url_var else "")
393
+ self._url = url if url is not None else (env.get(plan.url_var, "") if plan.url_var else plan.base_url)
394
+ if plan.keyless: # a run that calls no model is given nothing it could leak
395
+ self._key, self._url = "", ""
396
+ # the agent keys Settings holds (CURSOR_API_KEY, ...): a Harbor run's agent may need its own vendor's; the standard's never gets one
397
+ self._agent_keys = dict(agent_keys) if agent_keys is not None else (settings.agent_key_values(env) if plan.runner == "harbor" else {})
398
+ # the benchmark's licences a standard run on a host is sent: read from its store at each launch, unless given
399
+ self._licences_given = licences
400
+ self._redact: List[str] = [] # the licences' values sent, as text: never in a log line
401
+ self._decision = threading.Event() # set by continue_() or stop() while paused
402
+ # A Stop (the page's thread) and the run's own thread pausing or ending, each with the record
403
+ # it saves, one at a time: a Stop is never written over, a record never half of each.
404
+ self._turn = threading.RLock()
405
+ self._lock = threading.Lock()
406
+ self.thread: Optional[threading.Thread] = None
407
+ self._lines: List[str] = []
408
+ #: what a part is downloading, while it does: {label, done, total} in bytes, from its `@progress` lines
409
+ self.progress: Optional[dict] = None
410
+ #: every stage line the log holds, parsed (nullius/progress.py): this run's own and its part's
411
+ self._events: List[dict] = []
412
+ self._unread = False # a restored run that ended: its log is read when first asked for
413
+ self._open: Optional[tuple] = None # (stage, monotonic start) of the step here under way, said as a stage
414
+ rec = record or {}
415
+ self.id = rec.get("id") or f"{plan.name}-{int(time.time())}"
416
+ self.state = rec.get("state", "preparing")
417
+ self.step = rec.get("step", "")
418
+ self.part = int(rec.get("part", 0)) # index of the part running or next
419
+ self.pid: Optional[int] = rec.get("pid")
420
+ # the part that may be running on the target, from its launch until its end is recorded: what a Stop ends
421
+ self._live: Optional[dict] = None
422
+ # the last part whose launch began, saved before its script is sent, so a restart tells a Stop that cut its
423
+ # launch short from one before it (_kill_part); "" in a record from before
424
+ self.launched: str = rec.get("launched") or ""
425
+ if self.state == "stopping": # a Stop that a restart interrupted: finish it, do not run on
426
+ self._stop.set()
427
+ self.log_offset = int(rec.get("log_offset", 0))
428
+ self.exit_code: Optional[int] = rec.get("exit_code")
429
+ self.error: Optional[str] = rec.get("error")
430
+ self.question: Optional[str] = rec.get("question")
431
+ #: each part that ran to its end: {part id: {"rc", "at", "label"}} - an agent's own result in a run of several
432
+ self.ends: Dict[str, dict] = dict(rec.get("ends") or {})
433
+ self.started = rec.get("started") or time.time()
434
+ self.finished: Optional[float] = rec.get("finished")
435
+ self.harbor = rec.get("harbor", "") # the harbor executable on the target
436
+ self.tasks_on_target = rec.get("tasks_on_target", "")
437
+ self.app_on_target = rec.get("app_on_target", "") # the standard's source tree or runner snapshot on the target
438
+ self.catalogue_on_target = rec.get("catalogue_on_target", "") # writable definitions, separate from the immutable runner
439
+ # the trials the log has said already, (job, trial), so a restart says each once; none in a record from before
440
+ self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()}
441
+ if record:
442
+ # a run still going reads its log now, to go on from it; one that ended (most of them, and
443
+ # history only grows) when its page asks - start-up read every one, seconds and hundreds of MB
444
+ self._unread = True
445
+ if self.state in ALIVE:
446
+ self._read_log()
447
+
448
+ def _read_log(self) -> None:
449
+ self._unread = False
450
+ try:
451
+ with open(self.log_path, encoding="utf-8") as fh:
452
+ self._lines = fh.read().splitlines()
453
+ except OSError:
454
+ pass
455
+ self._events = [e for e in map(progress.parse, self._lines) if e]
456
+
457
+ @property
458
+ def lines(self) -> List[str]:
459
+ if self._unread:
460
+ self._read_log()
461
+ return self._lines
462
+
463
+ @lines.setter
464
+ def lines(self, value: List[str]) -> None:
465
+ self._unread, self._lines = False, value
466
+
467
+ @property
468
+ def events(self) -> List[dict]:
469
+ if self._unread:
470
+ self._read_log()
471
+ return self._events
472
+
473
+ @events.setter
474
+ def events(self, value: List[dict]) -> None:
475
+ self._unread, self._events = False, value
476
+
477
+
478
+ def _save(self) -> None:
479
+ with self._turn:
480
+ data = {"id": self.id, "plan": self.plan.to_dict(), "state": self.state, "step": self.step, "part": self.part, "pid": self.pid,
481
+ "log_offset": self.log_offset, "exit_code": self.exit_code, "error": self.error, "question": self.question, "ends": self.ends,
482
+ "started": self.started, "finished": self.finished, "harbor": self.harbor, "tasks_on_target": self.tasks_on_target, "app_on_target": self.app_on_target,
483
+ "catalogue_on_target": self.catalogue_on_target,
484
+ "launched": self.launched, "seen_trials": [list(k) for k in sorted(self._seen_trials, key=str)]}
485
+ try:
486
+ os.makedirs(self.dir, exist_ok=True)
487
+ tmp = os.path.join(self.dir, RECORD + ".tmp")
488
+ with open(tmp, "w", encoding="utf-8") as fh:
489
+ json.dump(data, fh, indent=1)
490
+ os.replace(tmp, os.path.join(self.dir, RECORD))
491
+ except OSError:
492
+ pass
493
+
494
+
495
+ def log(self, line: str) -> None:
496
+ if line.startswith("** ") and ("post-quantum" in line or "store now, decrypt later" in line or "openssh.com/pq" in line):
497
+ return # ssh's own warning about the host's key exchange, not the run's
498
+ clean = redact.redact(line.rstrip("\n"))
499
+ if self._key and self._key != settings.MODEL_KEY_PLACEHOLDER: # the placeholder is no secret, and a common word
500
+ clean = clean.replace(self._key, "[REDACTED]")
501
+ for value in list(self._agent_keys.values()) + self._redact:
502
+ if len(value) >= 8:
503
+ clean = clean.replace(value, "[REDACTED]")
504
+ ev = progress.parse(clean)
505
+ with self._lock:
506
+ self.lines.append(clean)
507
+ if ev is not None:
508
+ self.events.append(ev)
509
+ try:
510
+ os.makedirs(self.dir, exist_ok=True)
511
+ with open(self.log_path, "a", encoding="utf-8") as fh:
512
+ fh.write(clean + "\n")
513
+ except OSError:
514
+ pass
515
+
516
+ def _log_since(self, base: str) -> str:
517
+ """Shell that prints a part's log from ``log_offset`` to its size
518
+ now, between markers, and that size after them. The offset then
519
+ moves by the log's own bytes: never by what the transport prints
520
+ around it - OpenSSH warns on stderr at every connection to a host
521
+ without a post-quantum key exchange, 222 bytes ahead of the command's
522
+ output - nor by what reading it as text changes (a \\r\\n read as
523
+ \\n), so a look never skips the log nor starts inside a character."""
524
+ off = self.log_offset
525
+ return (f"n=$( {{ wc -c <{base}.log; }} 2>/dev/null | tr -d ' '); [ -n \"$n\" ] || n=0; printf '\\n{LOG_FROM}\\n'; "
526
+ f"[ \"$n\" -gt {off} ] && tail -c +{off + 1} {base}.log | head -c $((n - {off})); printf '\\n{LOG_TO} %s\\n' \"$n\"")
527
+
528
+ @staticmethod
529
+ def _logged(out: str) -> tuple:
530
+ """(the log text, the log's size in bytes) from what a `_log_since`
531
+ command printed, and nothing printed around it; ("", None) when it
532
+ did not print them."""
533
+ _, start, rest = out.partition(f"\n{LOG_FROM}\n")
534
+ text, end, size = rest.rpartition(f"\n{LOG_TO} ")
535
+ n = size.strip()
536
+ return (text, int(n)) if start and end and n.isdigit() else ("", None)
537
+
538
+ def _read(self, line: str) -> None:
539
+ """One line of a part's log: `@progress <label> <done> <total>` is
540
+ the bar the run's page draws while a part downloads, never a log
541
+ line; any other line ends it and is logged."""
542
+ if line.startswith("@progress "):
543
+ bits = line.split()
544
+ if len(bits) == 4 and bits[2].isdigit() and bits[3].isdigit():
545
+ self.progress = {"label": bits[1], "done": int(bits[2]), "total": int(bits[3])}
546
+ return
547
+ self.progress = None
548
+ self.log(line)
549
+
550
+ def tail(self, since: int = 0) -> List[str]:
551
+ with self._lock:
552
+ return self.lines[since:]
553
+
554
+ def part_for(self, job_id: str) -> Optional[str]:
555
+ """The part that writes a job: the agent it names (an arm of a run of several), or
556
+ the part its name ends with (a Harbor agent in a run of several); the
557
+ only part of a run of one."""
558
+ steps = self.plan.steps()
559
+ job = next((j for j in self.plan.jobs if str(j.get("name")) == job_id), None)
560
+ if job is None:
561
+ return None
562
+ for p in steps:
563
+ if job.get("agent") == p["id"] or job_id == f"{self.plan.name}-{p['id']}":
564
+ return p["id"]
565
+ return steps[0]["id"] if len(steps) == 1 else None
566
+
567
+ def job_state(self, job_id: str) -> Optional[tuple]:
568
+ """``(state, finished_at)`` for one job of this run once it has ended:
569
+ its own part's result when that part ran to its end - an arm that
570
+ answered every question is done though a later one failed - else the
571
+ run's, once the run has ended. None while it is still to come."""
572
+ end = self.ends.get(self.part_for(job_id) or "")
573
+ if end is not None:
574
+ return ("done" if end["rc"] == 0 else "failed"), datetime.fromtimestamp(end["at"], tz=timezone.utc)
575
+ if self.state in ENDED and self.finished:
576
+ return self.state, datetime.fromtimestamp(self.finished, tz=timezone.utc)
577
+ return None
578
+
579
+ def refused(self, job_id: str) -> Optional[str]:
580
+ """Why one job of this run was never written, when the run was refused
581
+ before it: the standard's cli said so in the job's folder
582
+ (``stage=refused`` in its ``progress.jsonl``, the only thing it
583
+ wrote there), or the run failed before any part's launch began - a
584
+ key not set, a host without docker. None for a run that got further,
585
+ whose job is elsewhere or still to come. The folder is kept: it holds
586
+ the refusal and the image's build log, and nothing reads it as a job."""
587
+ if self.state in ALIVE:
588
+ return None
589
+ why = progress.refusal(progress.read(os.path.join(self.jobs, job_id)))
590
+ if why is None and self.state == "failed" and not self.launched and not self.ends:
591
+ why = self.error or "the run failed before it started"
592
+ return why
593
+
594
+ def live_state(self, job_id: str) -> Optional[str]:
595
+ """What one job of this run is doing while the run lives: the run's
596
+ state for the part running now, ``queued`` for a part still to come
597
+ (the parts run one after another), and the run's own while it
598
+ prepares or stops, which is every part's. ``not run`` for one held
599
+ back by a pause: nothing starts it unless the page says continue. None
600
+ once the run ended."""
601
+ if self.state not in ALIVE:
602
+ return None
603
+ part = self.part_for(job_id)
604
+ if self.state in ("preparing", "stopping") or part is None or part == self.launched:
605
+ return self.state
606
+ return NOT_RUN if self.state == "paused" else "queued"
607
+
608
+ def never_ran(self, job_id: str) -> bool:
609
+ """The run ended before this job's part began: nothing of it was launched."""
610
+ ids = [p["id"] for p in self.plan.steps()]
611
+ part = self.part_for(job_id)
612
+ return self.state in ENDED and part in ids and part != self.launched and part not in self.ends and ids.index(part) >= self.part
613
+
614
+ def summary(self) -> dict:
615
+ return {"id": self.id, "name": self.plan.name, "state": self.state, "step": self.step, "host": self.plan.host.label if self.plan.host else "this machine",
616
+ "exit_code": self.exit_code, "error": self.error, "started": self.started, "finished": self.finished, "jobs": self.jobs,
617
+ "job_ids": [str(j.get("name")) for j in self.plan.jobs], "lines": len(self.lines), "question": self.question,
618
+ "progress": self.progress, "stages": self.stages()}
619
+
620
+ def stages(self) -> dict:
621
+ """The stages said so far, folded for the page (``progress.fold``).
622
+ Once the run has failed or been stopped, a stage still running was
623
+ cut short by it, and nothing is waiting any more."""
624
+ with self._lock:
625
+ events = list(self.events)
626
+ got = progress.fold(events)
627
+ if self.state in ("failed", "stopped"):
628
+ got["stages"] = [dict(r, state="failed") if r["state"] == "running" else r for r in got["stages"] if r["state"] != "waiting"]
629
+ got["current"] = None
630
+ return got
631
+
632
+
633
+ def start(self) -> "Execution":
634
+ self._save()
635
+ self.thread = threading.Thread(target=self._main, name=f"run-{self.plan.name}", daemon=True)
636
+ self.thread.start()
637
+ return self
638
+
639
+ def resume(self) -> "Execution":
640
+ """Pick a recorded run up where it was: the part that was running is
641
+ followed again, a pause waits again, the rest runs on."""
642
+ self.thread = threading.Thread(target=self._main_resumed, name=f"run-{self.plan.name}", daemon=True)
643
+ self.thread.start()
644
+ return self
645
+
646
+ def continue_(self) -> None:
647
+ """Answer a pause: go on with the next agent."""
648
+ self.question = None
649
+ self._decision.set()
650
+
651
+ def stop(self) -> None:
652
+ """Say so at once - the page reads `stopping` on its next tick - and
653
+ let the run's own thread do the rest: the command it is inside gives
654
+ way (`subprocess_run`, `stop=`), the follow loop returns, `_done`
655
+ ends the part on the target (`_kill_part`) - one whose launcher it
656
+ cut short included - copies the job back from a host once more, and
657
+ records `stopped`: a part that had ended by itself with its own
658
+ result, a part the Stop ended with 130, the same for a Stop a restart
659
+ finishes (`_done` says the rule). Nothing
660
+ here waits on the host, so a host that does not answer cannot hold
661
+ the button."""
662
+ with self._turn:
663
+ if self.state not in ALIVE:
664
+ return
665
+ self.state = "stopping"
666
+ self.log("— stop requested")
667
+ self._save()
668
+ self._stop.set()
669
+ self._decision.set()
670
+
671
+ def _kill_part(self) -> Optional[int]:
672
+ """The part that may be running on the target (``_live``) ended by
673
+ the Stop - unless it had ended by itself - and the part's result
674
+ returned. One command, through the plain runner since the stop is
675
+ already set, reads the part's exit code before it ends anything: a
676
+ part that has one ended by itself before the Stop reached it -
677
+ between two of ``_follow``'s looks, or while Nullius was away - and
678
+ its end is recorded as the look would have recorded it (the rest of
679
+ its log, its result in ``ends``, the run moved past it); its exit
680
+ code is returned. Otherwise its processes are ended, by their
681
+ process groups, found as ``_part_pids`` finds them, so a part is
682
+ ended though its launcher was cut short before its pid came back,
683
+ and a pid the record kept is signalled only while it is still the
684
+ part's own. Once they are gone - half a second at most - the exit
685
+ code is read again: a part that ended by itself between the first
686
+ read and the kill has it now, and keeps it. One the kill ended has
687
+ none, since its detached shell, which writes it, dies with it - so
688
+ every process group is signalled before any single process, lest
689
+ the script's own bash die alone and its shell write that down.
690
+
691
+ The Stop ended it - 130 - when anything of it was there (its claim,
692
+ pid file, log or a process), the target did not say, or its launch
693
+ had begun (``launched``: its script or its launcher on the way, cut
694
+ short before anything of it reached the target); None when none of
695
+ that, nor any part under way: the Stop ended none.
696
+
697
+ A launcher cut short may yet start the detached shell after this has
698
+ looked - on a host the launcher goes on when the ssh that carried it
699
+ is gone. So the Stop is left in the run directory first
700
+ (``run-<part>.stop``), and the detached shell looks for it after
701
+ writing its pid and before running the script: either it sees the
702
+ Stop and runs nothing, or its pid was on disk, and the shell in the
703
+ process table, before this looked."""
704
+ part = self._live
705
+ if part is None:
706
+ return None
707
+ base = _base(part)
708
+ command = ('export PATH="$HOME/.local/bin:$PATH"; ' + f"cd {self._q(self._run_dir())} 2>/dev/null || exit 0; "
709
+ f"ended() {{ e=$(head -c 32 {base}.exit 2>/dev/null | tr -cd 0-9); [ -z \"$e\" ] || "
710
+ f"{{ {self._log_since(base)}; printf '\\n__part__ %s' \"$e\"; exit 0; }}; }}; ended; : >{base}.stop; "
711
+ f"s=none; [ -e {base}.launch ] || [ -e {base}.pid ] || [ -e {base}.log ] && s=stopped; k=$({_part_pids(base, [self.pid])}); "
712
+ f"for p in $k; do s=stopped; kill -TERM -- -$p 2>/dev/null; done; for p in $k; do kill -TERM $p 2>/dev/null; done; "
713
+ f"for i in 1 2 3 4 5 6 7 8 9 10; do a=; for p in $k; do kill -0 $p 2>/dev/null && a=1; done; [ -n \"$a\" ] || break; sleep 0.05; done; "
714
+ + (f"rm -rf {self._licence_dir()}; " if self._licence_dir() else "")
715
+ + f"ended; printf '\\n__part__ %s' \"$s\"")
716
+ argv = self.plan.host.argv(command, timeout=15) if self.on_host else ["bash", "-c", command]
717
+ rc, out = self._run_plain(argv, STOP_WAIT)
718
+ text, mark, code = out.rpartition("\n__part__ ")
719
+ code = code.strip()
720
+ if rc == 0 and (not mark or code == "none"):
721
+ # nothing of the part there (nor its run directory): the Stop ended none - unless it cut the part's launch short
722
+ return 130 if self.launched == part["id"] else None
723
+ if rc != 0 or not code.isdigit():
724
+ return 130 # the Stop ended it, or the target did not say it had ended
725
+ text, size = self._logged(text)
726
+ for line in text.splitlines():
727
+ self._read(line)
728
+ self._trial_lines()
729
+ with self._turn:
730
+ if size is not None and size > self.log_offset:
731
+ self.log_offset = size
732
+ self.ends[part["id"]] = {"rc": int(code), "at": time.time(), "label": part["label"]}
733
+ self.part = [p["id"] for p in self.plan.steps()].index(part["id"]) + 1
734
+ self.pid, self._live = None, None
735
+ return int(code)
736
+
737
+ def _main(self) -> None:
738
+ try:
739
+ os.makedirs(self.jobs, exist_ok=True)
740
+ with open(os.path.join(self.dir, "run.sh"), "w", encoding="utf-8") as fh:
741
+ fh.write(self.plan.script if self.plan.script.endswith("\n") else self.plan.script + "\n")
742
+ problem = self.plan.check()
743
+ if problem:
744
+ return self._fail("plan", problem)
745
+ harbor_model = "NULLIUS_HARBOR" in (self.plan.script + "".join(p.get("script", "") for p in self.plan.parts)) and self.plan.model
746
+ if not self._key and harbor_model:
747
+ return self._fail("credentials", f"{self.plan.key_var} is not set in Settings; the run would have no model key")
748
+ if not self._key and self.plan.runner != "harbor" and not self.plan.keyless:
749
+ return self._fail("credentials", f"{self.plan.key_var} is not set in Settings; the producers and the judge would have no model key")
750
+ if not self._url and self.plan.url_var and not self.plan.keyless and (harbor_model or self.plan.runner != "harbor"):
751
+ return self._fail("credentials", f"{self.plan.url_var} is not set in Settings; an OpenAI-compatible endpoint has no default, so the run would have nowhere to send its calls")
752
+ broken = self._line_break()
753
+ if broken:
754
+ return self._fail("credentials", broken)
755
+ if not self._prepare(fresh=True):
756
+ return
757
+ self._parts()
758
+ except Exception as e: # a step that raised is a failed run, not a dead thread
759
+ self._fail(self.step or "run", f"{type(e).__name__}: {e}")
760
+
761
+ def _main_resumed(self) -> None:
762
+ try:
763
+ self.log(f"— Nullius is back; picking the run up at {self.plan.steps()[min(self.part, len(self.plan.steps()) - 1)]['label']}")
764
+ if self.state == "preparing":
765
+ if not self._prepare(fresh=False):
766
+ return
767
+ self._parts(resumed=True)
768
+ except Exception as e:
769
+ self._fail(self.step or "run", f"{type(e).__name__}: {e}")
770
+
771
+ def _fail(self, step: str, why: str) -> None:
772
+ if self._stop.is_set(): # a step cut short by a Stop is the stop, not a failure
773
+ return self._done(130)
774
+ self._stage_end("failed", detail=why[:300])
775
+ self.step, self.state, self.error, self.finished = step, "failed", why, time.time()
776
+ self.log(f"✕ {step}: {why}")
777
+ self._save()
778
+
779
+ def _done(self, rc: int) -> None:
780
+ """The run's end, recorded once: ``done`` or ``failed`` by ``rc``,
781
+ or ``stopped`` under a Stop. What a Stop records is the same whether
782
+ it was said to this run or read from a record by a restart, which
783
+ has taken the part from the target first (``_settle``):
784
+
785
+ - a part that ended by itself keeps its result. The part under way
786
+ (``_live``) is asked for its exit code before anything is ended
787
+ (``_kill_part``), and an end found there - since the run's last
788
+ look, or while Nullius was away - is recorded as that look would
789
+ have recorded it: its job reads done or failed.
790
+ - a part the Stop ended - found on the target without an exit code,
791
+ or on a target that did not answer - is 130, the line is
792
+ ``✕ stopped during <part>: exit 130``, and its job reads
793
+ ``stopped``, the run's.
794
+ - the run's exit code is its last part's result: the exit code of a
795
+ part that ended by itself, 130 for one the Stop ended. With no
796
+ part under way - a pause, the moment between two parts, a part
797
+ with nothing of it on the target yet - it is the exit the run
798
+ had: the last part's own (0 before any), or 130 for a step the
799
+ Stop cut short (``_fail``) - a part's launch included, though
800
+ nothing of the part reached the target: the record names the part
801
+ whose launch began (``launched``), so a restart records the 130
802
+ too. A record from before ``launched`` takes it for a Stop before
803
+ the launch.
804
+ - the line says the run was stopped, and where: ``during <part>``
805
+ for a part the Stop ended, else ``before`` the next part, or
806
+ ``after`` the last one - ``✕ stopped before <part>: exit <rc>``,
807
+ never ``✓ finished``, whatever the exit.
808
+ - on a host, the job is copied back once more before the run is
809
+ recorded, once a part has run: the trials that ended there since
810
+ the run's last look are here, and said. The copy goes through the
811
+ plain runner, and the kill and the copy together take at most
812
+ ``STOP_WAIT``; a host that took it all is not asked again.
813
+
814
+ The next part is never started."""
815
+ live, ended = self._live, None
816
+ if self._stop.is_set():
817
+ until = time.monotonic() + STOP_WAIT
818
+ ended = self._kill_part()
819
+ if ended is not None:
820
+ rc = ended
821
+ if self.on_host and (ended is not None or self.ends):
822
+ # the trials that ended there since the run's last look, copied back once more before the run is stopped
823
+ left = until - time.monotonic()
824
+ if left >= 1:
825
+ self._mirror(self._run_plain, left)
826
+ else:
827
+ self.log(f" the job did not sync back: the host took the Stop's {int(STOP_WAIT)} s")
828
+ with self._turn: # a Stop said as the run ends is "stopped", never left "stopping"
829
+ self.exit_code, self.finished, self.pid = rc, time.time(), None
830
+ self.state = "stopped" if self._stop.is_set() else ("done" if rc == 0 else "failed")
831
+ if self.state == "failed":
832
+ failed = [f"{e['label']} exited {e['rc']}" for e in self.ends.values() if e.get("rc")]
833
+ self.error = "; ".join(failed) if failed else f"the run exited {rc}"
834
+ if self.state == "stopped":
835
+ steps = self.plan.steps()
836
+ where = (f"during {live['label']}" if live is not None and ended is not None and live["id"] not in self.ends
837
+ else f"before {steps[self.part]['label']}" if self.part < len(steps) else f"after {steps[-1]['label']}")
838
+ self.log(f"✕ stopped {where}: exit {rc}")
839
+ else:
840
+ self.log(f"{'✓' if rc == 0 else '✕'} finished: exit {rc}")
841
+ self._save()
842
+
843
+ def _begin(self, step: str, what: str, stage: bool = False) -> None:
844
+ """A step of the run, said and saved. ``stage``: one that gets the
845
+ target ready, said as a stage (``progress``) that ends with the next
846
+ such step, the preparation's end (``_stage_end``) or a failure."""
847
+ self.step = step
848
+ if stage:
849
+ self._stage_end()
850
+ self._open = (step, time.monotonic())
851
+ self.log(progress.line(step, "start", detail=what))
852
+ else:
853
+ self.log(f"— {what}")
854
+ self._save()
855
+
856
+ def _stage_end(self, state: str = "done", **fields) -> None:
857
+ """The step under way said ended - done, or failed - with its seconds."""
858
+ if self._open is None:
859
+ return
860
+ step, t0 = self._open
861
+ self._open = None
862
+ self.log(progress.line(step, state, seconds=round(time.monotonic() - t0, 1), **fields))
863
+
864
+ def _pause(self, label: str, rc: int, remaining: List[dict]) -> bool:
865
+ """An agent's run failed with more to come: wait for the page to say
866
+ continue or stop. True to go on; False at once for a Stop said since
867
+ the part ended, which this would otherwise clear and pause over."""
868
+ return self._ask(label, rc, remaining) and self._wait_decision(remaining)
869
+
870
+ def _ask(self, label: str, rc: int, remaining: List[dict]) -> bool:
871
+ """The pause's question, put and saved; False, and nothing asked, once
872
+ a Stop is said."""
873
+ with self._turn:
874
+ if self._stop.is_set():
875
+ return False
876
+ self._decision.clear() # before the page can see "paused" and answer
877
+ self.state = "paused"
878
+ self.question = f"{label} exited {rc}. Continue with {remaining[0]['label']}, or stop? A failure this early usually means the dataset or the environment, not the agent."
879
+ self.log(f"✕ {label} exited {rc} — waiting: continue with {remaining[0]['label']}, or stop?")
880
+ self._save()
881
+ return True
882
+
883
+ def _wait_decision(self, remaining: List[dict]) -> bool:
884
+ self._decision.wait()
885
+ if self._stop.is_set():
886
+ return False
887
+ self.state, self.question = "running", None
888
+ self.log(f"— continuing with {remaining[0]['label']}")
889
+ self._save()
890
+ return True
891
+
892
+ def _trial_lines(self) -> None:
893
+ """One line per trial that ended since the last look, read from the
894
+ jobs' own feeds, so the log says what finished as it finishes. The
895
+ trials said are saved with the record as soon as they are said, so a
896
+ restart, which reads the feeds again from their start, says none of
897
+ them twice."""
898
+ try:
899
+ names = sorted(os.listdir(self.jobs))
900
+ except OSError:
901
+ return
902
+ said = False
903
+ for name in names:
904
+ feed = os.path.join(self.jobs, name, "events.jsonl")
905
+ try:
906
+ with open(feed, encoding="utf-8", errors="replace") as fh:
907
+ for raw in fh:
908
+ if '"trial.end"' not in raw:
909
+ continue
910
+ try:
911
+ ev = json.loads(raw)
912
+ except ValueError:
913
+ continue
914
+ key = (name, ev.get("trial"))
915
+ if ev.get("event") != "trial.end" or key in self._seen_trials:
916
+ continue
917
+ with self._turn: # _save reads the set from the page's thread too
918
+ self._seen_trials.add(key)
919
+ said = True
920
+ d = ev.get("durations") or {}
921
+ secs = sum(v for v in d.values() if isinstance(v, (int, float)))
922
+ reward, score = ev.get("reward"), ev.get("score")
923
+ # a trial its benchmark scored without a pass or fail is neither mark: its score is said
924
+ mark = "✓" if ev.get("outcome") == "resolved" else "·" if ev.get("outcome") == "scored" else "✕"
925
+ self.log(f" {mark} {name} · {ev.get('task') or ev.get('trial')} · {ev.get('outcome')}"
926
+ + (f" · reward {reward}" if reward is not None else f" · score {score:.3g}" if isinstance(score, (int, float)) else "")
927
+ + (f" · {int(secs // 60)}:{int(secs % 60):02d}" if secs else ""))
928
+ except OSError:
929
+ continue
930
+ if said:
931
+ self._save()
932
+
933
+
934
+ @property
935
+ def on_host(self) -> bool:
936
+ return self.plan.host is not None
937
+
938
+ def _remote(self, *parts: str) -> str:
939
+ return "/".join((REMOTE_HOME,) + parts)
940
+
941
+ def _run_dir(self) -> str:
942
+ return self._remote("runs", self.plan.name) if self.on_host else self.dir
943
+
944
+ def _q(self, path: str) -> str:
945
+ return _rq(path) if self.on_host else shlex.quote(path)
946
+
947
+ def _sh(self, command: str, timeout: float = 600.0, stdin: Optional[str] = None) -> tuple:
948
+ """One command on the target, with uv on the PATH; (returncode, output)."""
949
+ command = 'export PATH="$HOME/.local/bin:$PATH"; ' + command
950
+ if self.on_host:
951
+ return self._run(self.plan.host.argv(command, timeout=15), timeout, stdin)
952
+ return self._run(["bash", "-c", command], timeout, stdin)
953
+
954
+ def _ssh_prefix(self) -> str:
955
+ return " ".join(shlex.quote(a) for a in self.plan.host.argv("", timeout=15)[:-2]) # ssh + options, without target and command
956
+
957
+ def _target(self) -> str:
958
+ return self.plan.host.argv("")[-2]
959
+
960
+ def _copy_to(self, src: str, dest: str) -> tuple:
961
+ return self._run(["rsync", "-az", "--delete", "-e", self._ssh_prefix(), src.rstrip("/") + "/", f"{self._target()}:{dest}"], 900)
962
+
963
+ def _copy_back(self, run: Optional[Callable] = None, timeout: float = 900.0) -> tuple:
964
+ started = time.monotonic()
965
+ copy = run or self._run
966
+ result = copy(["rsync", "-az", "-e", self._ssh_prefix(), f"{self._target()}:{self._remote('runs', self.plan.name, 'jobs')}/", self.jobs + "/"], timeout)
967
+ if result[0] == 0 and self.plan.runner == STANDARD and self.catalogue_on_target:
968
+ # Proofs write back to this run's definitions. Keep them beside
969
+ # the job evidence, without replacing this user's catalogue.
970
+ local = os.path.join(self.dir, "catalogue")
971
+ os.makedirs(local, exist_ok=True)
972
+ result = copy(["rsync", "-az", "-e", self._ssh_prefix(),
973
+ f"{self._target()}:{self.catalogue_on_target}/", local + "/"],
974
+ max(0.1, timeout - (time.monotonic() - started)))
975
+ return result
976
+
977
+
978
+ def _prepare(self, fresh: bool = True) -> bool:
979
+ """The target ready for the parts: the tasks, and what the run needs
980
+ there - for a Harbor recipe docker, here or on a host, and the Harbor it pins
981
+ (built here when it is a repository's commit, its wheels sent to a
982
+ host, run through uvx) or, for a plan from before the pin, the harbor
983
+ on the host's PATH; uv and the runner snapshot for the standard. False when a step
984
+ failed and said so.
985
+
986
+ A ``fresh`` run on a host first asks whether its run directory is
987
+ there already, and stops before anything is sent when it is: a
988
+ leftover of the same name would otherwise read as this run's parts,
989
+ claimed or ended already. (Here the route asks, ``web/execute.py``.)
990
+ A restored run's directory is its own."""
991
+ self.harbor = "harbor"
992
+ if fresh and not self.on_host and (self.plan.runner == "harbor" or self.plan.docker):
993
+ # before a pin is built: a run here without docker would start, and every trial fail at its container
994
+ why = (docker_here(self._sh) if self.plan.runner == "harbor"
995
+ else docker_here(self._sh, "this recipe runs its units in containers", container.DOCKER))
996
+ if why:
997
+ self._fail("docker", why)
998
+ return False
999
+ pin = self.plan.harbor_pin if self.plan.runner == "harbor" else {}
1000
+ if fresh and (self.on_host or pin):
1001
+ self.log(progress.line("plan", stages=",".join(
1002
+ (["harbor"] if pin else []) + (["connect", *(["uv"] if pin or self.plan.runner != "harbor" else []),
1003
+ *(["tasks"] if self.plan.tasks else []), *(["app"] if self.plan.runner == STANDARD else [])]
1004
+ if self.on_host else []))))
1005
+ record = self._pin_here(pin) if pin else None
1006
+ if pin and record is None:
1007
+ return False
1008
+ if not self.on_host:
1009
+ if pin:
1010
+ rc, _ = self._sh("command -v uvx", 30)
1011
+ if rc != 0:
1012
+ self._fail("harbor", "uv is not on this machine's PATH; the Harbor this recipe pins runs through uvx")
1013
+ return False
1014
+ os.makedirs(self.dir, exist_ok=True)
1015
+ self.harbor = harbor_source.write_wrapper(os.path.join(self.dir, "harbor"), record,
1016
+ shlex.quote(os.path.join(self.pins, harbor_source.key_of(pin))), pin.get("extras", []))
1017
+ # absolute: the part runs from its run directory, never from where this process started
1018
+ self.tasks_on_target = os.path.abspath(os.path.expanduser(self.plan.tasks)) if self.plan.tasks else ""
1019
+ if self.plan.runner == STANDARD:
1020
+ self.catalogue_on_target = self._catalogue
1021
+ if os.path.isfile(os.path.join(REPO, "app", "nullius", "standard", "cli.py")):
1022
+ self.app_on_target = REPO
1023
+ else:
1024
+ got = self._app_here()
1025
+ if got is None:
1026
+ return False
1027
+ self.app_on_target = got[1]
1028
+ self._stage_end()
1029
+ self._save()
1030
+ return True
1031
+ host = self.plan.host
1032
+ self._begin("connect", f"connecting to {host.label}", stage=True)
1033
+ probe = hosts.test(host, run=lambda argv, t: _probe(self._run, argv[:-1] + ['export PATH="$HOME/.local/bin:$PATH"; ' + argv[-1]], t))
1034
+ if not probe["ok"]:
1035
+ self._fail("connect", probe["error"])
1036
+ return False
1037
+ if fresh:
1038
+ rc, out = self._sh(f"[ -e {_rq(self._run_dir())} ] && echo @taken; true", 30)
1039
+ if rc != 0:
1040
+ self._fail("connect", "the host did not say whether the run's name is free: " + out.strip()[-400:])
1041
+ return False
1042
+ if "@taken" in out.split():
1043
+ self._fail("name", name_taken(self.plan.name, host.label))
1044
+ return False
1045
+ # what this run needs of the host, and nothing it does not
1046
+ if self.plan.runner == "harbor":
1047
+ self.log(f" {probe['hostname'] or host.label}: docker {'yes' if probe['docker'] else 'no'}"
1048
+ + (f" · uv {'yes' if probe['uv'] else 'no'}" if pin else f" · harbor {(probe['harbor_version'] or 'yes') if probe['harbor'] else 'no'}"))
1049
+ if not probe["docker"]:
1050
+ self._fail("connect", "docker is not on the host; a Harbor run needs it")
1051
+ return False
1052
+ if probe["docker_ok"] is False:
1053
+ self._fail("connect", "docker is on the host but this user cannot use it (docker ps failed)")
1054
+ return False
1055
+ if not pin and not probe["harbor"]:
1056
+ self._fail("connect", "harbor is not on the host's PATH (~/.local/bin included); install the Harbor this recipe needs there")
1057
+ return False
1058
+ elif self.plan.runner == STANDARD:
1059
+ # the standard's own cli: python3 and git there, docker or podman when its units run in containers
1060
+ engine = probe["engine"]
1061
+ self.log(f" {probe['hostname'] or host.label}: uv {'yes' if probe['uv'] else 'no'} · {engine or 'docker'} {'yes' if engine else 'no'}")
1062
+ if self.plan.docker and not engine:
1063
+ self._fail("connect", "neither docker nor podman is on the host: this recipe runs its units in containers")
1064
+ return False
1065
+ if self.plan.docker and probe[engine + "_ok"] is False:
1066
+ self._fail("connect", f"{engine} is on the host but this user cannot use it ({engine} ps failed)")
1067
+ return False
1068
+ rc, out = self._sh("for c in python3 git; do command -v $c >/dev/null 2>&1 || echo @missing $c; done; true", 30)
1069
+ gone = [l.split()[1] for l in out.splitlines() if l.startswith("@missing ")]
1070
+ if rc != 0 or gone:
1071
+ self._fail("connect", (" and ".join(gone) + " not on the host; the standard's cli needs them") if gone
1072
+ else "the host did not say whether python3 is there: " + out.strip()[-400:])
1073
+ return False
1074
+ else:
1075
+ # a run record from a runner no longer offered (a benchmark's own scripts now run as the standard's Original setup)
1076
+ self._fail("connect", f"this run's runner, {self.plan.runner}, is no longer offered")
1077
+ return False
1078
+ if (pin or self.plan.runner != "harbor") and probe["uv"]:
1079
+ self._stage_end()
1080
+ self.log(progress.line("uv", "cached", detail="on the host already"))
1081
+ if (pin or self.plan.runner != "harbor") and not probe["uv"]:
1082
+ # the standard's cli builds its environments with uv; a pinned Harbor runs through uvx
1083
+ self._begin("uv", "installing uv on the host", stage=True)
1084
+ rc, out = self._sh(f"command -v uv >/dev/null 2>&1 || ({UV_INSTALL}) >/dev/null 2>&1; command -v uv", 300)
1085
+ found = [l for l in out.strip().splitlines() if not l.startswith("** ")]
1086
+ if rc != 0 or not found:
1087
+ self._fail("uv", "uv could not be installed: " + out.strip()[-400:])
1088
+ return False
1089
+ self.log(" uv: " + found[-1])
1090
+ self.tasks_on_target = ""
1091
+ if self.plan.tasks:
1092
+ self._begin("tasks", f"sending the task folder {self.plan.tasks}", stage=True)
1093
+ self.tasks_on_target = self._remote(*_tasks_place(self.plan.tasks, self.plan.name))
1094
+ rc, out = self._sh(f"mkdir -p {_rq(self.tasks_on_target)}", 30)
1095
+ if rc == 0:
1096
+ rc, out = self._copy_to(os.path.expanduser(self.plan.tasks), self.tasks_on_target)
1097
+ if rc != 0:
1098
+ self._fail("tasks", "the task folder did not copy: " + out.strip()[-400:])
1099
+ return False
1100
+ if self.plan.runner == STANDARD and not self._app_on_host():
1101
+ return False
1102
+ run_dir = self._run_dir()
1103
+ rc, out = self._sh(f"mkdir -p {_rq(run_dir + '/jobs')} && cat > {_rq(run_dir + '/run.sh')}", 30, stdin=self.plan.script)
1104
+ if rc != 0:
1105
+ self._fail("run", "the script did not land on the host: " + out.strip()[-400:])
1106
+ return False
1107
+ if self.plan.runner == STANDARD:
1108
+ catalogue = run_dir + "/catalogue"
1109
+ stage = catalogue + ".part"
1110
+ # A restart keeps the proofs already written by this run.
1111
+ # Copy into staging so an interrupted first copy is not trusted.
1112
+ rc, out = self._sh(f"if [ ! -d {_rq(catalogue)} ]; then rm -rf {_rq(stage)} && "
1113
+ f"cp -R {_rq(self.app_on_target + '/app/nullius/catalogue')} {_rq(stage)} && "
1114
+ f"mv {_rq(stage)} {_rq(catalogue)}; fi", 300)
1115
+ if rc != 0:
1116
+ self._fail("app", "the run's catalogue could not be prepared: " + out.strip()[-400:])
1117
+ return False
1118
+ self.catalogue_on_target = catalogue
1119
+ if pin and not self._pin_on_host(pin, record, run_dir):
1120
+ return False
1121
+ self._stage_end()
1122
+ self._save()
1123
+ return True
1124
+
1125
+ def _app_here(self) -> Optional[tuple]:
1126
+ """(content ID, folder) for this runtime and its effective catalogue.
1127
+
1128
+ The exact code imported here travels, including source edits, so
1129
+ installed wheels and editable checkouts have the same behaviour.
1130
+ Configuration and credential stores are not payload inputs.
1131
+ """
1132
+ try:
1133
+ return standard_payload.build(app_root(self.root), catalogue=self._catalogue)
1134
+ except (OSError, ValueError) as e:
1135
+ self._fail("app", "the standard runner could not be packed: " + str(e)[-300:])
1136
+ return None
1137
+
1138
+ def _app_on_host(self) -> bool:
1139
+ """Send the content-addressed runner and catalogue to the host."""
1140
+ got = self._app_here()
1141
+ if got is None:
1142
+ return False
1143
+ content_id, here = got
1144
+ remote = self._remote("app", content_id)
1145
+ self._begin("app", f"sending the standard runner and catalogue at {content_id[:15]}", stage=True)
1146
+ rc, out = self._sh(f"mkdir -p {_rq(remote)}", 30)
1147
+ if rc == 0:
1148
+ rc, out = self._copy_to(here, remote)
1149
+ if rc != 0:
1150
+ self._fail("app", "app/ did not copy: " + out.strip()[-400:])
1151
+ return False
1152
+ self.app_on_target = remote
1153
+ return True
1154
+
1155
+ def _pin_here(self, pin: dict) -> Optional[dict]:
1156
+ """The pinned Harbor ready on this machine - a repository's commit
1157
+ built here, once (``harbor_source.ensure``) - and its record; None
1158
+ when it could not be, said."""
1159
+ building = pin["kind"] == "repo" and harbor_source.record_of(pin, self.pins) is None
1160
+ self._begin("harbor", harbor_source.label(pin) + (": building it here, once for this commit" if building else ""), stage=True)
1161
+ try:
1162
+ return harbor_source.ensure(pin, self.pins, run=lambda argv, t=600.0: harbor_source.run_command(argv, t, stop=self._stop), log=self.log)
1163
+ except harbor_source.BuildError as e:
1164
+ self._fail("harbor", str(e))
1165
+ return None
1166
+
1167
+ def _pin_on_host(self, pin: dict, record: dict, run_dir: str) -> bool:
1168
+ """The pinned Harbor on the host: a repository's wheels sent to
1169
+ ``~/.nullius/harbor/<pin>/wheels`` (the host resolves the rest from
1170
+ PyPI, so it needs no access to the repository), and the ``harbor``
1171
+ script that runs it written into the run directory."""
1172
+ remote = self._remote("harbor", harbor_source.key_of(pin))
1173
+ if record.get("wheel"):
1174
+ self._begin("harbor", f"sending the wheels built for {harbor_source.label(pin)}", stage=True)
1175
+ rc, out = self._sh(f"mkdir -p {_rq(remote + '/' + harbor_source.WHEELS)}", 30)
1176
+ if rc == 0:
1177
+ rc, out = self._copy_to(os.path.join(self.pins, harbor_source.key_of(pin), harbor_source.WHEELS), remote + "/" + harbor_source.WHEELS)
1178
+ if rc != 0:
1179
+ self._fail("harbor", "the wheels did not copy: " + out.strip()[-400:])
1180
+ return False
1181
+ exe = run_dir + "/harbor"
1182
+ rc, out = self._sh(f"cat > {_rq(exe + harbor_source.FEED_SUFFIX)}", 30, stdin=harbor_source.feed_source()) # the live feed it runs
1183
+ if rc == 0:
1184
+ rc, out = self._sh(f"cat > {_rq(exe)} && chmod +x {_rq(exe)}", 30, stdin=harbor_source.wrapper(record, _rq(remote), pin.get("extras", [])))
1185
+ if rc != 0:
1186
+ self._fail("harbor", "the harbor script did not land on the host: " + out.strip()[-400:])
1187
+ return False
1188
+ self.harbor = exe
1189
+ return True
1190
+
1191
+ # --- the parts, one agent at a time -----------------------------------------
1192
+
1193
+ def _parts(self, resumed: bool = False) -> None:
1194
+ steps = self.plan.steps()
1195
+ follow = resumed and self._settle(steps) # a restored record, checked against the target first
1196
+ i = self.part
1197
+ # picked up after a part ended: that part's own exit, as the run had it, not a 0 for the restart
1198
+ last = self.ends.get(steps[i - 1]["id"], {}).get("rc", 0) if 0 < i <= len(steps) else 0
1199
+ while i < len(steps):
1200
+ part = steps[i]
1201
+ if resumed and self.state == "paused":
1202
+ if not self._wait_decision(steps[i:]):
1203
+ break
1204
+ launched = self._launch(part)
1205
+ if launched is None:
1206
+ break
1207
+ if not launched:
1208
+ return
1209
+ elif follow and self.state == "running":
1210
+ self.log(f"— following {part['label']} again on {self.plan.host.label if self.on_host else 'this machine'}")
1211
+ else:
1212
+ launched = self._launch(part)
1213
+ if launched is None:
1214
+ break
1215
+ if not launched:
1216
+ return
1217
+ resumed = follow = False
1218
+ last = self._follow(part)
1219
+ if self._stop.is_set():
1220
+ break
1221
+ with self._turn:
1222
+ # Ended, and recorded so before anything else is: its exit, the next part, and no pid -
1223
+ # a restart launches the next part, never follows it by this one's pid.
1224
+ self.ends[part["id"]] = {"rc": last, "at": time.time(), "label": part["label"]}
1225
+ i += 1
1226
+ self.part, self.pid, self._live = i, None, None
1227
+ if last != 0 and i < len(steps):
1228
+ if not self._pause(part["label"], last, steps[i:]): # its one save is the end and the question: a restart asks again
1229
+ break
1230
+ else:
1231
+ self._save()
1232
+ self._done(last)
1233
+
1234
+ def _settle(self, steps: List[dict]) -> bool:
1235
+ """A restored record's part and pid, checked against the target
1236
+ before either is trusted; True when that part is to be followed, not
1237
+ launched. A part has started when its run directory there holds its
1238
+ pid file (its detached shell's first act), its log or its exit code,
1239
+ or when a process runs its script there (``_part_pids``: the launcher
1240
+ before c69f527 wrote no pid file).
1241
+
1242
+ - No pid in the record: it was saved before the pid was. Started, the
1243
+ part is followed - by the pid it wrote, else one of its processes,
1244
+ else none: its exit code ends it, or, with no process left, it is
1245
+ gone without one - its log read from the start. Claimed by a
1246
+ launcher (``.launch``) and nothing more: the launcher is on its way
1247
+ to starting it, so it is looked at again for ``LAUNCH_GRACE``; still
1248
+ nothing - no pid, no process - and the launcher died first: the
1249
+ claim is cleared and the part launched. Not claimed: launched.
1250
+ - A pid, a part before it, and this part started: the pid is checked
1251
+ against the part - the one it wrote, or one of its processes. Not
1252
+ its own, it is the part before's: a record an older version saved
1253
+ as this part was launched. It is dropped, the part
1254
+ before's end taken as below, and this part followed as if no pid
1255
+ had been saved.
1256
+ - A pid, and the part not started - or, past the last part, no end
1257
+ recorded for it: a record from before a part's end was one save,
1258
+ saved as the previous part ended. The pid is that part's;
1259
+ its end is the record's or, from before ``ends``, its ``.exit``
1260
+ (none: gone without one). The run goes on as it would have: asks
1261
+ after a failure with more to come, else launches the part or ends
1262
+ with that exit.
1263
+ - A pause: it follows a part's end, so nothing of the run is running.
1264
+ A pid it kept (an older record's: the ended part's) is
1265
+ nobody's now, and is dropped.
1266
+ - A Stop the restart cut short: the part is taken as above, and the
1267
+ Stop is then finished as a live one is (``_done``) - a part that
1268
+ ended by itself meanwhile keeps its own result, read through the
1269
+ plain runner, since the Stop, set already, would cut a slow host's
1270
+ answer short.
1271
+
1272
+ A pid with no part before it stands as recorded: it can be no other
1273
+ part's, and it is followed and killed only while it runs the part."""
1274
+ i = self.part
1275
+ if self.state == "paused":
1276
+ if self.pid:
1277
+ with self._turn:
1278
+ self.pid = None
1279
+ self._save()
1280
+ return False
1281
+ if self.state not in ("running", "stopping") or i > len(steps):
1282
+ return bool(self.pid)
1283
+ cur, prev = (steps[i] if i < len(steps) else None), (steps[i - 1] if i else None)
1284
+ self._live = cur # running, or being launched, as far as the record knows
1285
+ # a pid with no part before it can be no other part's: it stands (followed and killed only while it runs the part)
1286
+ check = cur is not None and (not self.pid or prev is not None)
1287
+ mine = [f"{_base(cur)}.{ext}" for ext in ("launch", "pid", "log", "exit")] if check else []
1288
+ before = [f"{_base(prev)}.exit"] if prev and prev["id"] not in self.ends else []
1289
+ if not mine and not before:
1290
+ return bool(self.pid)
1291
+ look = self._look(mine + before, cur if check else None, [self.pid])
1292
+ if look is None:
1293
+ return bool(self.pid)
1294
+ found, procs = look
1295
+ started = lambda: bool(procs) or any(n in found for n in mine[1:]) # its pid file, log or exit code, or its processes
1296
+ if check and mine[0] in found and not started():
1297
+ # claimed by a launcher and not started yet: the launcher is on its way to it, or died first
1298
+ look = self._await_start(mine + before, cur)
1299
+ if look is None:
1300
+ return bool(self.pid)
1301
+ found, procs = look
1302
+ if not started():
1303
+ self.log(f" {cur['label']} was claimed by a launcher that never started it; starting it again")
1304
+ self._sh(f"cd {self._q(self._run_dir())} && rm -f {mine[0]}", 30)
1305
+ if check and started():
1306
+ written = int(found.get(mine[1]) or 0) or None
1307
+ if self.pid and prev is not None and self.pid != written and self.pid not in procs:
1308
+ # Not this part's: a record an older version saved as it was launched kept the part
1309
+ # before's pid - and this part, started by the launcher before its pid file, is found by its processes.
1310
+ with self._turn:
1311
+ if before:
1312
+ rc = found.get(before[0], "")
1313
+ self.ends[prev["id"]] = {"rc": int(rc) if rc else 1, "at": time.time(), "label": prev["label"]}
1314
+ self.pid = None
1315
+ if not self.pid:
1316
+ with self._turn:
1317
+ self.pid, self.log_offset = written or (procs[0] if procs else None), 0
1318
+ self._save()
1319
+ return True
1320
+ old = bool(self.pid) # a pid, and no part of its own: the part before's
1321
+ if not old and not before:
1322
+ return False # never started: launched
1323
+ with self._turn:
1324
+ if before:
1325
+ rc = found.get(before[0], "")
1326
+ self.ends[prev["id"]] = {"rc": int(rc) if rc else 1, "at": time.time(), "label": prev["label"]}
1327
+ self.pid = None
1328
+ rc = self.ends[prev["id"]]["rc"]
1329
+ if not (old and cur and rc != 0 and self._ask(prev["label"], rc, steps[i:])): # asked: that save is the record
1330
+ self._save()
1331
+ return False
1332
+
1333
+ def _look(self, names: List[str], part: Optional[dict] = None, pids=()) -> Optional[tuple]:
1334
+ """Which of ``names`` the run directory on the target holds, each with
1335
+ the digits it starts with (a pid, an exit code), and - for ``part`` -
1336
+ the pids of its processes there (``_part_pids``, asked of ``pids``
1337
+ too): ({name: digits}, [pid]). Asked again while the target does not
1338
+ answer, as ``_follow`` does; None for a Stop said meanwhile."""
1339
+ files = " ".join(shlex.quote(n) for n in names)
1340
+ procs = f"for p in $({_part_pids(_base(part), pids)}); do echo \"@proc $p\"; done; " if part else ""
1341
+ while True:
1342
+ rc, out = self._sh(f"cd {self._q(self._run_dir())} 2>/dev/null || exit 0; for f in {files}; do "
1343
+ f"[ -f \"$f\" ] && printf '@file %s %s\\n' \"$f\" \"$(head -c 32 \"$f\" | tr -cd 0-9)\"; done; {procs}true", 60)
1344
+ if rc == 0:
1345
+ lines = [l.split() for l in out.splitlines()]
1346
+ return ({b[1]: "".join(b[2:]) for b in lines if len(b) >= 2 and b[0] == "@file"},
1347
+ [int(b[1]) for b in lines if len(b) == 2 and b[0] == "@proc" and b[1].isdigit()])
1348
+ if self._stop.wait(SYNC_EVERY):
1349
+ return None
1350
+
1351
+ def _await_start(self, names: List[str], part: dict) -> Optional[tuple]:
1352
+ """A part its launcher claimed and nothing after: looked at again
1353
+ for up to ``LAUNCH_GRACE`` seconds, until its pid file, log or exit
1354
+ code is there or a process runs it. A launcher that has claimed a
1355
+ part runs in its run directory until it has started the part, so
1356
+ nothing there, all that time, means the launcher died first. The
1357
+ last look; None for a Stop said meanwhile."""
1358
+ look = None
1359
+ for _ in range(max(1, int(LAUNCH_GRACE / STOP_POLL))):
1360
+ if self._stop.wait(min(STOP_POLL, LAUNCH_GRACE)):
1361
+ return None
1362
+ look = self._look(names, part, [self.pid])
1363
+ if look is None or look[1] or any(n in look[0] for n in names[1:4]):
1364
+ return look
1365
+ return look
1366
+
1367
+ def _stdin(self) -> str:
1368
+ """What the launcher reads: the key, the endpoint, the agent keys as
1369
+ NAME=value, the benchmark's licences as one more (a standard run on a
1370
+ host), the end line."""
1371
+ lic = self._licence_line()
1372
+ return "".join(f"{line}\n" for line in [self._key, self._url, *(f"{n}={v}" for n, v in self._agent_keys.items()),
1373
+ *([f"{secrets.HANDED}={lic}"] if lic else []), STDIN_END])
1374
+
1375
+ def _licence_dir(self) -> str:
1376
+ """Where a standard run on a host writes a licence file: in memory, the run's own folder."""
1377
+ return f"/dev/shm/nullius-{self.plan.name}" if self.on_host and (self.plan.licences or {}).get("names") else ""
1378
+
1379
+ def _licence_trap(self) -> str:
1380
+ """The detached shell's first words when it may hold a licence file:
1381
+ the folder removed when it ends, by itself or by a Stop's signal."""
1382
+ d = self._licence_dir()
1383
+ return f'trap "rm -rf {d}" EXIT; trap "exit 143" TERM HUP INT; ' if d else ""
1384
+
1385
+ def _licence_line(self) -> str:
1386
+ """The benchmark's licences, as the one line a host's run reads
1387
+ (base64 of {name: base64 bytes}): only the names its environments declare, only
1388
+ those set - a data provider's token from that provider's store - and nothing
1389
+ at all for a run on this machine, which reads the store itself."""
1390
+ lic = self.plan.licences or {}
1391
+ if not self._licence_dir():
1392
+ return ""
1393
+ names = [n for n in lic.get("names") or [] if isinstance(n, str)]
1394
+ providers = lic.get("providers") if isinstance(lic.get("providers"), dict) else {}
1395
+ if self._licences_given is not None:
1396
+ got = {n: v for n, v in self._licences_given.items() if n in names and v}
1397
+ else:
1398
+ try:
1399
+ got = {n: secrets.value(*secrets.store_of(lic.get("benchmark") or "", n, providers.get(n) or "")) for n in names}
1400
+ except ValueError:
1401
+ got = {}
1402
+ got = {n: v for n, v in got.items() if v}
1403
+ for v in got.values():
1404
+ try:
1405
+ text = v.decode().strip()
1406
+ except UnicodeDecodeError:
1407
+ continue
1408
+ if "\n" not in text and text not in self._redact:
1409
+ self._redact.append(text)
1410
+ if not got:
1411
+ return ""
1412
+ return base64.b64encode(json.dumps({n: base64.b64encode(v).decode() for n, v in sorted(got.items())},
1413
+ separators=(",", ":")).encode()).decode()
1414
+
1415
+ def _line_break(self) -> Optional[str]:
1416
+ """Why the run's key or endpoint cannot be sent, or None: one that
1417
+ holds a line break, which the launcher, reading each as one line
1418
+ (``_launch``), would take in pieces. Settings refuses one, so only
1419
+ the environment Nullius was started in can hold it. The variable is
1420
+ named, never its value."""
1421
+ for value, var in ((self._key, self.plan.key_var), (self._url, self.plan.url_var or "the endpoint"), *((v, n) for n, v in self._agent_keys.items())):
1422
+ if "\n" in value or "\r" in value:
1423
+ return (f"{var} holds a line break; a key or an endpoint is one line, and the run's launcher would take this one "
1424
+ f"in pieces. Set it again on one line in the environment Nullius was started in (Settings refuses a line break)")
1425
+ return None
1426
+
1427
+ def _launch(self, part: dict) -> Optional[bool]:
1428
+ """Start one part detached on the target and keep its pid. The key and
1429
+ endpoint arrive on stdin, become environment for the launcher, and the
1430
+ detached shell inherits them - nothing in argv, nothing on disk. They
1431
+ are two lines and ``STDIN_END`` a third: either may be empty (a
1432
+ recipe of controls needs no key, OpenAI's clients no endpoint), so a
1433
+ stdin cut short - Nullius gone as it sent them, a dropped ssh - is
1434
+ told by the end line missing, never by an empty value. Without it the
1435
+ launcher answers ``@cut`` and does nothing more, the part neither
1436
+ claimed nor started, and the run fails at that part. The
1437
+ launcher claims the part first (``run-<part>.launch``, created only
1438
+ where there is none), and the detached shell writes its own pid
1439
+ beside its log before it runs the script: Nullius gone before the pid
1440
+ is saved here, a restart finds the claim, the pid or the process
1441
+ there (``_settle``) and never starts the part a second time - nor
1442
+ does a launcher that reaches the target after the restart's: it finds
1443
+ the part claimed, says so and starts nothing, and a restart that
1444
+ meets that answer follows the part.
1445
+
1446
+ A key or endpoint that holds a line break is never sent: it would
1447
+ reach the launcher in pieces and read as a stdin cut short. A new
1448
+ run refuses it before anything is sent (``_main``); a restored one,
1449
+ whose key and endpoint are read again, here, before the part.
1450
+
1451
+ True once launched, False when the launch failed (the run is failed),
1452
+ None when a Stop was said before it began: nothing is started, and the
1453
+ caller ends the run as the Stop records it. The check and the
1454
+ ``running`` it sets hold the lock ``stop`` takes, so a Stop is never
1455
+ written over."""
1456
+ broken = self._line_break()
1457
+ if broken:
1458
+ self._fail("credentials", broken)
1459
+ return False
1460
+ with self._turn: # the Stop takes this lock too: said before this, nothing is launched
1461
+ if self._stop.is_set():
1462
+ return None
1463
+ self.state, self._live, self.launched = "running", part, part["id"]
1464
+ where = self.plan.host.label if self.on_host else "this machine"
1465
+ self._begin("run", f"running {part['label']} on {where}")
1466
+ run_dir = self._run_dir()
1467
+ script = _base(part) + ".sh"
1468
+ rc, out = self._sh(f"cat > {self._q(run_dir + '/' + script)}", 30, stdin=part["script"] if part["script"].endswith("\n") else part["script"] + "\n")
1469
+ if rc != 0:
1470
+ self._fail("run", f"the script did not land on {where}: " + out.strip()[-400:])
1471
+ return False
1472
+ prices_on_target = ""
1473
+ if self.plan.runner == STANDARD and self.on_host:
1474
+ # the price list is this machine's user's, so the run takes it along (nullius.prices) - an empty one
1475
+ # too: a host's own list, if it has one, never prices a run started here
1476
+ prices_on_target = run_dir + "/prices.json"
1477
+ rc, out = self._sh(f"cat > {self._q(prices_on_target)}", 60, stdin=json.dumps(price_list.load()) + "\n")
1478
+ if rc != 0:
1479
+ self._fail("run", f"the price list did not land on {where}: " + out.strip()[-400:])
1480
+ return False
1481
+ exports = " ".join(f"{k}={self._q(v) if v.startswith('~') else shlex.quote(v)}" for k, v in
1482
+ [("NULLIUS_HARBOR", self.harbor), ("NULLIUS_TASKS", self.tasks_on_target), ("NULLIUS_JOBS", run_dir + "/jobs"),
1483
+ ("NULLIUS_PROGRESS", "1")]
1484
+ + ([("NULLIUS_STANDARD", self.app_on_target or REPO), ("PYTHONDONTWRITEBYTECODE", "1")]
1485
+ if self.plan.runner == STANDARD else [])
1486
+ + ([("NULLIUS_CATALOGUE", self.catalogue_on_target)]
1487
+ if self.plan.runner == STANDARD and self.catalogue_on_target else [])
1488
+ + ([("NULLIUS_PRICES", prices_on_target)] if prices_on_target else [])
1489
+ + ([("NULLIUS_PYTHON", sys.executable)] if self.plan.runner == STANDARD and not self.on_host else [])
1490
+ + ([("NULLIUS_LICENCE_DIR", self._licence_dir())] if self._licence_dir() else [])
1491
+ + ([("API_MODEL_NAME", self.plan.model)] if self.plan.model else []))
1492
+ base = script[:-3]
1493
+ # macOS has no setsid executable. Python can create the same session and
1494
+ # exec in place, keeping $!, the pid file and the process group identical.
1495
+ python = shlex.quote("python3" if self.on_host else sys.executable)
1496
+ session = shlex.quote("import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])")
1497
+ detach = (f"if ! command -v setsid >/dev/null 2>&1; then "
1498
+ f"command -v {python} >/dev/null 2>&1 || {{ echo 'detached runs need setsid or Python 3'; exit 1; }}; "
1499
+ f"setsid() {{ exec {python} -c {session} \"$@\"; }}; fi; ")
1500
+ # stdin: the key, the endpoint, then one NAME=value line per agent key, then the end line - which alone says it all arrived
1501
+ launcher = ('IFS= read -r K && IFS= read -r U || { echo @cut; exit 1; }; E=; '
1502
+ 'while IFS= read -r L; do [ "$L" = ' + STDIN_END + ' ] && { E=1; break; }; '
1503
+ 'case "$L" in [A-Z_]*=*) export "$L";; *) echo @cut; exit 1;; esac; done; [ -n "$E" ] || { echo @cut; exit 1; }; unset E L; '
1504
+ '[ -n "$K" ] && export ' + " ".join(f'{n}="$K"' for n in KEY_NAMES) + '; '
1505
+ '[ -n "$U" ] && export ' + " ".join(f'{n}="$U"' for n in URL_NAMES) + '; unset K U; '
1506
+ f'export {exports}; cd {self._q(run_dir)} || exit 1; '
1507
+ + detach +
1508
+ # The part claimed before anything else is done for it, created only where there is none (noclobber:
1509
+ # O_EXCL): of two launchers - one a Nullius gone since sent, late to arrive, and the restart's - one
1510
+ # starts the part and the other says so and starts nothing. From the claim on, this launcher runs
1511
+ # in the run directory until the part has started, which is how a restart tells it is on its way.
1512
+ f'( set -C; : >{base}.launch ) 2>/dev/null || {{ echo @claimed; exit 0; }}; rm -f {base}.exit {base}.pid; '
1513
+ # `&` binds to the setsid command alone (hence `;` before it): $! is the detached shell, a session
1514
+ # leader of its own, with nothing of the launcher's pipes, so the launcher returns at once. Its
1515
+ # first act is its pid ($$, the same $!) on disk, before the log opens; its second, a look for a
1516
+ # Stop that came while its launcher had not answered (_kill_part).
1517
+ f"setsid nohup bash -c '{self._licence_trap()}echo $$ >{base}.pid; [ -e {base}.stop ] && exit 130; bash {script} </dev/null >{base}.log 2>&1; echo $? >{base}.exit' "
1518
+ f"</dev/null >/dev/null 2>&1 & echo $!")
1519
+ rc, out = self._sh(launcher, 60, stdin=self._stdin())
1520
+ if rc == 124 and not self._stop.is_set():
1521
+ # An ssh that stalled says nothing of whether the launcher ran. Asking again is safe: the claim above
1522
+ # lets one launcher start the part, and a second that finds it says @claimed and starts nothing.
1523
+ self.log(f" {where} did not answer the launch within 60 s; asking once more")
1524
+ rc, out = self._sh(launcher, 60, stdin=self._stdin())
1525
+ if rc == 124:
1526
+ out = f"{where} did not answer the launch, twice, within 60 s each; nothing is known to have started there"
1527
+ if "@cut" in out.split():
1528
+ self._fail("run", f"{part['label']} did not start on {where}: the key and endpoint reached its launcher cut short; nothing was started")
1529
+ return False
1530
+ if rc == 0 and "@claimed" in out.split():
1531
+ self.log(f" {part['label']} was started already, by a launcher sent before Nullius was gone; following it")
1532
+ self.pid, self.log_offset = None, 0
1533
+ self._save()
1534
+ return True
1535
+ pid = next((int(l) for l in out.strip().splitlines() if l.strip().isdigit()), None)
1536
+ if rc != 0 or not pid:
1537
+ self._fail("run", f"{part['label']} did not start on {where}: " + out.strip()[-400:])
1538
+ return False
1539
+ self.pid, self.log_offset = pid, 0
1540
+ self._save()
1541
+ return True
1542
+
1543
+ def _follow(self, part: dict) -> int:
1544
+ """Look at the running part every few seconds - its log, the mirror,
1545
+ the trials - until its exit code lands; a part with no process left
1546
+ and no exit code is gone, a failure of its own."""
1547
+ base = _base(part)
1548
+ # Alive while a process runs its script in its run directory (_part_pids): its pid when known, else the
1549
+ # one it wrote or one found by its command line - a part an older launcher started has no pid file - and
1550
+ # never a pid that has since gone to another process.
1551
+ alive = f"[ -n \"$({_part_pids(base, [self.pid])})\" ] && echo alive || echo gone"
1552
+ while True:
1553
+ # Read in the reverse of the order the target writes them (the log, the exit code, the process
1554
+ # ends): an exit code seen means the whole log is there, a process gone means none is coming.
1555
+ rc, out = self._sh(f"cd {self._q(self._run_dir())} 2>/dev/null; s=$({alive}); [ -f {base}.exit ] && s=$(cat {base}.exit); "
1556
+ f"{self._log_since(base)}; printf '\\n__nullius__ %s' \"$s\"", 120)
1557
+ head, _, status = out.rpartition("\n__nullius__ ")
1558
+ if rc == 0:
1559
+ text, size = self._logged(head)
1560
+ if size is not None and size > self.log_offset:
1561
+ self.log_offset = size
1562
+ for line in text.splitlines():
1563
+ self._read(line)
1564
+ self._save()
1565
+ status = status.strip()
1566
+ if self.on_host:
1567
+ self._mirror()
1568
+ else:
1569
+ self._trial_lines()
1570
+ if status.isdigit():
1571
+ return int(status)
1572
+ if status == "gone":
1573
+ self.log(f" {part['label']} is gone without an exit code")
1574
+ return 1
1575
+ if self._stop.wait(SYNC_EVERY):
1576
+ return 130
1577
+
1578
+ def _mirror(self, run: Optional[Callable] = None, timeout: float = 900.0) -> None:
1579
+ """The job copied back from the host, and the trials that ended said;
1580
+ ``run`` and ``timeout`` for the last copy a Stop makes (``_done``)."""
1581
+ rc, out = self._copy_back(run, timeout)
1582
+ if rc != 0:
1583
+ self.log(" the job did not sync back: " + out.strip()[-300:])
1584
+ self._trial_lines()
1585
+
1586
+
1587
+ def _probe(run: Callable, argv: List[str], timeout: float) -> tuple:
1588
+ """hosts.test wants (rc, stdout, stderr); the execution runner gives (rc, output)."""
1589
+ rc, out = run(argv, timeout)
1590
+ return rc, out, ""
1591
+
1592
+
1593
+ # --- the registry the page reads --------------------------------------------
1594
+
1595
+ EXECUTIONS: Dict[str, Execution] = {}
1596
+
1597
+
1598
+ def start(plan: Plan, **kw) -> Execution:
1599
+ ex = Execution(plan, **kw)
1600
+ EXECUTIONS[ex.id] = ex
1601
+ return ex.start()
1602
+
1603
+
1604
+ def active() -> List[Execution]:
1605
+ return [e for e in EXECUTIONS.values() if e.state in ALIVE]
1606
+
1607
+
1608
+ def for_job(job_id: str) -> Optional[Execution]:
1609
+ """The newest execution whose script writes this job, if any - the one
1610
+ the job page shows beside the job, or instead of it while Harbor has
1611
+ not written it yet."""
1612
+ found = [e for e in EXECUTIONS.values() if any(str(j.get("name")) == job_id for j in e.plan.jobs)]
1613
+ return max(found, key=lambda e: e.started) if found else None
1614
+
1615
+
1616
+ ENDED = ("stopped", "failed", "done")
1617
+
1618
+
1619
+ def ended_for(job_id: str) -> Optional[tuple]:
1620
+ """``(state, finished_at)`` when the run that writes this job has ended
1621
+ - stopped, failed or done - by the runner's own record; None while it
1622
+ is alive or when no run of this application wrote the job. A Harbor
1623
+ killed by a Stop leaves its feed without ``job.end`` and its result
1624
+ record with trials "running", so a page reading either alone would say
1625
+ the job goes on; this is the fact that says otherwise."""
1626
+ ex = for_job(job_id)
1627
+ return ex.job_state(job_id) if ex is not None else None
1628
+
1629
+
1630
+ def restore(root: Optional[str] = None, **kw) -> List[Execution]:
1631
+ """At start-up: every recorded run under the root. The ones still alive
1632
+ are reattached (their thread follows the part that was running, or
1633
+ waits at the pause it was at); the finished ones are listed as they
1634
+ ended. Returns the ones reattached."""
1635
+ root = root or runs_root(kw.get("env", os.environ))
1636
+ back = []
1637
+ try:
1638
+ names = sorted(os.listdir(root))
1639
+ except OSError:
1640
+ return back
1641
+ for name in names:
1642
+ path = os.path.join(root, name, RECORD)
1643
+ try:
1644
+ with open(path, encoding="utf-8") as fh:
1645
+ rec = json.load(fh)
1646
+ plan = Plan.from_dict(rec["plan"])
1647
+ except (OSError, ValueError, KeyError, TypeError):
1648
+ continue
1649
+ ex = Execution(plan, root=root, record=rec, **kw)
1650
+ EXECUTIONS[ex.id] = ex
1651
+ if ex.state in ALIVE:
1652
+ back.append(ex.resume())
1653
+ return back