nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,1870 @@
1
+ """The container backend: each unit's agent, or capability, in a container of its
2
+ own, and its relay in another beside it. This module shells out to the
3
+ engine's program (:data:`DOCKER`) and imports nothing outside the standard
4
+ library. That is ``docker``, or ``podman`` on a machine without Docker or
5
+ where ``docker`` is podman's own (:func:`engine`). Podman needs no Docker API
6
+ socket and no settings of its own for Nullius.
7
+
8
+ **One base image** (``nullius/base:<hash>``), built once and named by the
9
+ hash of its Dockerfile, so a change to it builds a new one beside the old:
10
+ Ubuntu 24.04 with git, python3, curl, CA certificates, tmux (Terminus's
11
+ terminal), Node 22 from its official tarball, checked by its sha256, and uv
12
+ from its official image, pinned by digest. A unit's image is chosen by
13
+ :func:`image_for`: the base image, or the benchmark's environment the unit
14
+ asks for - a layer on the base, its Dockerfile in the benchmark's folder,
15
+ named by the hash of what it is built from
16
+ (``nullius/env-<benchmark>-<name>:<hash>``) and built the first time a unit
17
+ uses it (:func:`env_image`).
18
+
19
+ **The same absolute paths inside.** A unit's workspace and home are mounted
20
+ read-write; the agent's install and its definition's folder read-only, and
21
+ each file a unit's stdio MCP server is started from, alone (:func:`mcp_mounts`)
22
+ - never this repository's ``app/`` (it holds the keys' file; ``Unit.wrap``
23
+ refuses it) nor the benchmark's folder or upstream (their data holds the
24
+ answers). All where they are on this machine, so every path a
25
+ definition's templates rendered (``{home}``, ``{install}``, ``{here}``) holds
26
+ inside unchanged. A uv environment's interpreter is in the install
27
+ (``.python``), installed with it in the base image (:func:`install_argv`).
28
+ The container runs as this machine's user, with no capabilities and no way to gain
29
+ privileges, so what it writes in the workspace is the user's to read and
30
+ remove. That is ``--user uid:gid``, with ``--userns=keep-id`` on rootless
31
+ podman, which maps that uid to this user, or ``--user 0:0`` on rootless
32
+ Docker, where only a container's root is this machine's user. On a Docker with
33
+ ``userns-remap``, a container that uses this machine's user's files also
34
+ runs in this machine's user namespace (:func:`_userns`). On podman it
35
+ runs with no SELinux label (:func:`_security_opt`), as on Docker's default
36
+ daemon.
37
+
38
+ **The relay is a sidecar, and the agent's only way out** but for the public
39
+ web proxy of an environment that declares ``network``. Per unit, an
40
+ internal network (``docker network create --internal``: no route off it); a
41
+ relay container (``python3 -m nullius.standard.relay serve``) on the default
42
+ bridge *and* on that network as ``relay``; the agent's container on the
43
+ internal network alone. Its model endpoint is ``http://relay:8080/<tag>/v1``
44
+ and nothing else answers it. The real key reaches the relay's container
45
+ only, as an owner-only file under :func:`key_root`, mounted read-only and
46
+ removed from this machine once the relay listens: never on an argument list
47
+ (``ps``) or in the container's environment (``docker inspect``). The relay
48
+ publishes its port on 127.0.0.1 so the driver (on the machine for
49
+ agent/capability runs) reaches the judge's route and a capability
50
+ benchmark's reader. Its record folder is mounted read-write, so the host
51
+ reads each call as the local relay's are read. On podman the default bridge
52
+ is Nullius's own (:data:`BRIDGE`, :func:`way_out`). A model endpoint on this
53
+ machine's loopback is reached by the relays alone (:class:`Loopback`):
54
+ through a forwarder on the default bridge's gateway, or, on a rootless
55
+ engine or podman, a door container on the engine's default network.
56
+
57
+ **A licence server is one more door.** A unit whose environment needs a
58
+ licence of kind ``server`` (``host:port``) gets a forwarder beside its relay
59
+ (``forward.py``, mounted alone into the base image): on the default bridge
60
+ (``BRIDGE`` on podman), and on the unit's network under the server's own name, listening on its
61
+ port and passing every connection to that one address. The agent reaches
62
+ the server by the address it was given; nothing else is opened. A server
63
+ named by an IPv4 address is reached as ``licence-server-<i>`` instead, a
64
+ network alias being a name; ``host.docker.internal`` is the machine the run
65
+ is on. On a rootless engine or podman on Linux, a server at
66
+ ``host.docker.internal`` is reached through a :class:`SocketDoor` instead, as ``licence-server-<i>``
67
+ on the unit's network alone. A licence file is mounted read-only, alone, at
68
+ the path its environment declares (``wrap``'s ``binds``).
69
+
70
+ **A benchmark's MCP server on the machine is one door more.** A server the
71
+ benchmark declares in ``mcp_servers`` runs on this machine, in the driver's
72
+ environment, behind a Unix socket (``mcp_machine.Listener``). The unit gets
73
+ a forwarder for it (``doors``): on the unit's internal network *only* - no
74
+ bridge, no route anywhere - under the alias the agent's entry names, the
75
+ socket alone mounted into it, passing every connection to that socket and
76
+ nothing else. On macOS, where the socket cannot be mounted, the forwarder
77
+ is on the default bridge too and goes to a loopback TCP bridge here, which
78
+ wants the door's nonce first, given as a key file (``forward.py``). The
79
+ agent starts ``mcp_bridge.py`` (mounted alone, read-only), which joins its
80
+ stdio to that door.
81
+
82
+ **An own image** (an environment with ``"image": "own"``) is the other way
83
+ round: the task's image, built from its own Dockerfile - its own ``FROM``,
84
+ its own folder as the build context (:func:`own_image`) - or pulled by its
85
+ digest, and the agent added to it at run time, never built in. A unit's
86
+ :class:`Box` is that image kept running (``sh -c 'sleep'``) for the whole
87
+ unit: the agent is started in it with ``docker exec``, so it works on the
88
+ container's own state in the image's working directory, and the benchmark's
89
+ check runs in it after the agent ends - what the agent left running stopped
90
+ first, its files copied in only then, and a way out, when it asks for one,
91
+ through a proxy to public addresses only (:meth:`Box.settle`,
92
+ :meth:`Box.online`). What
93
+ the agent needs that the image may not have is mounted read-only under
94
+ ``/opt/nullius``: Node, unpacked once from the same pinned tarball the base
95
+ image is built from (:func:`node_runtime`), and a standalone Python
96
+ (:func:`python_runtime`); its install, its definition and its home are
97
+ mounted at their own paths, as on the base. A runtime that cannot start in
98
+ the image - a glibc build in a musl one - is found before the agent starts
99
+ (:meth:`Box.runtime_problem`), and the unit is not run.
100
+
101
+ Everything a unit starts carries the label ``nullius.job=<job>`` and is
102
+ removed when the unit ends, after an error or an interrupt too; :func:`sweep`
103
+ removes whatever a killed run left behind.
104
+ Docker Original setup drivers are handled by ``setup_container.py``.
105
+ """
106
+ from __future__ import annotations
107
+
108
+ import functools
109
+ import glob
110
+ import hashlib
111
+ import json
112
+ import os
113
+ import platform
114
+ import re
115
+ import shutil
116
+ import socket
117
+ import socketserver
118
+ import stat
119
+ import subprocess
120
+ import tarfile
121
+ import tempfile
122
+ import threading
123
+ import time
124
+ import urllib.parse
125
+ import urllib.request
126
+ from secrets import compare_digest, token_hex
127
+ from typing import Dict, Iterable, List, Optional, Tuple
128
+
129
+ from nullius import paths
130
+ from nullius.standard import definitions, install
131
+
132
+ def _runs_podman(path: str) -> bool:
133
+ """Whether the program at ``path`` is podman under another name: a link to
134
+ it, or a script that runs it (``podman-docker``'s ``docker``)."""
135
+ if os.path.basename(os.path.realpath(path)) == "podman":
136
+ return True
137
+ try:
138
+ with open(path, "rb") as f:
139
+ head = f.read(4096)
140
+ except OSError:
141
+ return False
142
+ return head.startswith(b"#!") and re.search(rb"^\s*exec\s+\S*podman\b", head, re.M) is not None
143
+
144
+
145
+ def engine() -> str:
146
+ """``podman`` when it is on the PATH and ``docker`` is not, or is podman
147
+ under that name; else ``docker``."""
148
+ docker, podman = shutil.which("docker"), shutil.which("podman")
149
+ return "podman" if podman and (not docker or _runs_podman(docker)) else "docker"
150
+
151
+
152
+ DOCKER = engine() # the engine's program, chosen once
153
+ LABEL = "nullius.job"
154
+ RELAY_PORT = 8080
155
+ RELAY_ALIAS = "relay"
156
+ FORWARD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "forward.py")
157
+ FORWARD_IN = "/opt/nullius/forward.py" # where the forwarder's one file is mounted in its container
158
+ FORWARD_NONCE_IN = "/opt/nullius/forward-nonce" # a door's nonce's one file, in its forwarder's container
159
+ EGRESS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "egress.py")
160
+ EGRESS_IN = "/opt/nullius/egress.py" # the check's proxy's one file, in its container
161
+ EGRESS_PORT = 3128
162
+ EGRESS_ALIAS = "egress"
163
+ BRIDGE = "nullius-bridge" # podman: the network a container with a way out starts on (way_out)
164
+ RELAY_KEY_IN = "/opt/nullius/relay-key" # the key's one file, in the relay's container
165
+ KEY_STALE = 600 # older than any start can take: a killed run left it
166
+ IMAGE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
167
+
168
+ # uv and PYTHON_VERSION move together: this uv must know that Python
169
+ UV_VERSION = "0.13.5"
170
+ UV_DIGEST = "sha256:32a7bb39b496babbd997b0a39a4d66c126bdaf5661b302bf6f06abda8deca6ff" # its index: every platform
171
+ NODE_VERSION = "22.23.3"
172
+ NODE_SHA256 = {
173
+ "x64": "df450af89261115ef9f9e3830c3eeb2cc9213b63c720b1af623cb5dcbe2e02de",
174
+ "arm64": "a44aeb94849a299b22df10b9e622ec2f605c2183501bc40590705131de7c740f",
175
+ }
176
+ _ARCH = {"x86_64": "x64", "amd64": "x64", "aarch64": "arm64", "arm64": "arm64"}
177
+
178
+ # FROM in full: podman resolves a short name by this machine's registries
179
+ DOCKERFILE = """\
180
+ FROM docker.io/library/ubuntu:24.04
181
+ COPY --from=ghcr.io/astral-sh/uv:{uv_version}@{uv_digest} /uv /uvx /usr/local/bin/
182
+ RUN apt-get update \\
183
+ && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git python3 ca-certificates curl tmux xz-utils \\
184
+ && rm -rf /var/lib/apt/lists/*
185
+ RUN curl -fsSLo /tmp/node.tar.xz https://nodejs.org/dist/v{node}/node-v{node}-linux-{arch}.tar.xz \\
186
+ && echo "{sha} /tmp/node.tar.xz" | sha256sum -c - \\
187
+ && tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner \\
188
+ && rm /tmp/node.tar.xz
189
+ ENV PATH={path}
190
+ """
191
+
192
+ # What a unit's process holds of this machine's environment that names a
193
+ # place on this machine: meaningless, or wrong, inside.
194
+ HOST_ONLY = ("TMPDIR", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR")
195
+ # What the engine's program keeps of this process's environment, to find the
196
+ # daemon, or podman its storage and settings. Not a key among them, though
197
+ # CONTAINER_HOST may name an ssh URL with a password: it is never on an argument list.
198
+ CLIENT_KEEP = ("PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY",
199
+ "XDG_RUNTIME_DIR", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "CONTAINERS_CONF", "CONTAINERS_STORAGE_CONF",
200
+ "CONTAINERS_REGISTRIES_CONF", "REGISTRY_AUTH_FILE", "CONTAINER_HOST", "CONTAINER_CONNECTION")
201
+
202
+
203
+ class Unavailable(Exception):
204
+ """The engine is missing, not usable by this user, or failed at something."""
205
+
206
+
207
+ def podman() -> bool:
208
+ return DOCKER == "podman"
209
+
210
+
211
+ def _docker(args: List[str], *, env: Optional[dict] = None, input: Optional[str] = None, timeout: float = 120,
212
+ check: bool = True) -> subprocess.CompletedProcess:
213
+ try:
214
+ r = subprocess.run([DOCKER] + args, env=env if env is not None else client_env(), input=input, text=True,
215
+ stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
216
+ except FileNotFoundError:
217
+ raise Unavailable(f"{DOCKER} is not installed on this machine")
218
+ except subprocess.TimeoutExpired:
219
+ raise Unavailable(f"{DOCKER} {args[0]} did not answer in {int(timeout)} s")
220
+ if check and r.returncode != 0:
221
+ raise Unavailable(f"{DOCKER} {' '.join(args[:2])}: {(r.stderr or r.stdout).strip()[-300:]}")
222
+ return r
223
+
224
+
225
+ def client_env(extra: Optional[dict] = None) -> dict:
226
+ """The engine program's environment: what it needs to find the daemon, or
227
+ podman its storage and settings (``CLIENT_KEEP``), and the values of the
228
+ variables a ``-e NAME`` passes on."""
229
+ env = {**_context(), **{k: os.environ[k] for k in CLIENT_KEEP if k in os.environ}}
230
+ env.setdefault("PATH", os.defpath)
231
+ env.update(extra or {})
232
+ return env
233
+
234
+
235
+ @functools.cache
236
+ def _context() -> Dict[str, str]:
237
+ """The daemon that ``docker context use`` selected, as the variables that
238
+ name it. The user's ``~/.docker`` keeps that selection, and a unit or a
239
+ driver has a home of its own. Only the endpoint goes on: the registry
240
+ logins in ``~/.docker`` do not. Read once per process."""
241
+ if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"):
242
+ return {}
243
+ try:
244
+ r = subprocess.run([DOCKER, "context", "inspect"], capture_output=True, text=True, timeout=30)
245
+ c = json.loads(r.stdout)[0]
246
+ ep = c["Endpoints"]["docker"]
247
+ except (OSError, subprocess.SubprocessError, ValueError, LookupError):
248
+ return {}
249
+ if c.get("Name") == "default":
250
+ return {}
251
+ env = {"DOCKER_HOST": ep["Host"]}
252
+ tls = bool(c.get("TLSMaterial", {}).get("docker"))
253
+ if tls:
254
+ env["DOCKER_CERT_PATH"] = os.path.join(c["Storage"]["TLSPath"], "docker")
255
+ # shortcut: docker reads a ca.pem from DOCKER_CERT_PATH, so a skip-verify context with a client certificate and no CA is not reached
256
+ if ep.get("SkipTLSVerify"):
257
+ env["DOCKER_TLS"] = "1"
258
+ elif tls:
259
+ env["DOCKER_TLS_VERIFY"] = "1"
260
+ return env
261
+
262
+
263
+ def unusable() -> Optional[str]:
264
+ """Why the container backend cannot run here, or None when it can."""
265
+ try:
266
+ r = _docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
267
+ except Unavailable as e:
268
+ return str(e)
269
+ if r.returncode != 0:
270
+ said = (r.stderr or r.stdout).strip().splitlines()
271
+ hint = (" (add this user to the docker group, then log in again)"
272
+ if not podman() and "permission denied" in r.stderr.lower() else "")
273
+ if not podman() and shutil.which("podman"): # docker wins (engine), so say podman is there
274
+ hint += ". podman is on this machine: Nullius uses it when docker is not on the PATH"
275
+ return f"{DOCKER} is installed but not usable by this user: " + (said[-1][:200] if said else f"exit {r.returncode}") + hint
276
+ if podman():
277
+ try:
278
+ net = _docker(["info", "--format", "{{.Host.NetworkBackend}}"], timeout=30, check=False).stdout.strip()
279
+ except Unavailable as e:
280
+ return str(e)
281
+ if net != "netavark":
282
+ return (f"podman's network backend is {net or 'unknown'}: Nullius needs netavark"
283
+ ' (podman 4.7 or later, network_backend = "netavark")')
284
+ return None
285
+
286
+
287
+ def dockerfile(machine: Optional[str] = None) -> str:
288
+ arch = _ARCH.get((machine or platform.machine()).lower())
289
+ if not arch:
290
+ raise Unavailable(f"no Node build pinned for {machine or platform.machine()}")
291
+ return DOCKERFILE.format(uv_version=UV_VERSION, uv_digest=UV_DIGEST, node=NODE_VERSION, arch=arch, sha=NODE_SHA256[arch], path=IMAGE_PATH)
292
+
293
+
294
+ def image_tag(text: str) -> str:
295
+ return "nullius/base:" + hashlib.sha256(text.encode()).hexdigest()[:12]
296
+
297
+
298
+ _built: Dict[str, bool] = {}
299
+ # the own images this process pulled or built itself - never one that was there already, which is the user's
300
+ _fetched: set = set()
301
+
302
+
303
+ def base_tag() -> str:
304
+ """The base image's tag on this machine, built or not."""
305
+ return image_tag(dockerfile())
306
+
307
+
308
+ def present(tag: str) -> bool:
309
+ """Whether this machine has an image by that tag: a run says it is there
310
+ already, or that it builds it now (``progress``)."""
311
+ if _built.get(tag) or _docker(["image", "inspect", tag], check=False).returncode == 0:
312
+ _built[tag] = True
313
+ return bool(_built.get(tag))
314
+
315
+
316
+ def base_image(log: Optional[str] = None) -> str:
317
+ """The base image's tag, building it first when this machine has not."""
318
+ text = dockerfile()
319
+ tag = image_tag(text)
320
+ if present(tag):
321
+ return tag
322
+ r = _docker(["build", "--label", "nullius.base=1", "-t", tag, "-"], input=text, timeout=1800, check=False)
323
+ if log:
324
+ with open(log, "a") as f:
325
+ f.write(r.stdout + r.stderr)
326
+ if r.returncode != 0:
327
+ raise Unavailable(f"building {tag}: {(r.stderr or r.stdout).strip()[-300:]}")
328
+ _built[tag] = True
329
+ return tag
330
+
331
+
332
+ def env_context(bench: dict, name: str, base: str) -> Tuple[str, Dict[str, bytes]]:
333
+ """An environment's Dockerfile as it is built - ``FROM <base>``, then the
334
+ benchmark's text - and its build context: the files it lists, by their
335
+ path in the benchmark's folder. Nothing else of that folder goes in."""
336
+ e = bench["environments"][name]
337
+ with open(os.path.join(bench["_here"], e["dockerfile"])) as f:
338
+ text = f"FROM {base}\n" + f.read()
339
+ files = {}
340
+ for rel in sorted(e.get("files") or []):
341
+ with open(os.path.join(bench["_here"], rel), "rb") as f:
342
+ files[rel] = f.read()
343
+ return text, files
344
+
345
+
346
+ def env_tag(bench: dict, name: str, base: str) -> str:
347
+ """``nullius/env-<benchmark>-<name>:<hash>``: the sha256 of the Dockerfile
348
+ as built (its base's tag included, so a new base is a new layer) and of
349
+ each file it copies, path and bytes."""
350
+ text, files = env_context(bench, name, base)
351
+ h = hashlib.sha256(text.encode())
352
+ for rel, data in files.items():
353
+ h.update(b"\0" + rel.encode() + b"\0" + hashlib.sha256(data).digest())
354
+ return f"nullius/env-{bench['key']}-{name}:{h.hexdigest()[:12]}"
355
+
356
+
357
+ _locks: Dict[str, threading.Lock] = {}
358
+ _locks_lock = threading.Lock()
359
+
360
+
361
+ def env_image(bench: dict, name: str, log_dir: Optional[str] = None) -> str:
362
+ """An environment's image, built the first time a unit asks for it - so a
363
+ run builds only the environments its units use - and kept by its tag
364
+ after. Two units asking at once wait for one build. Its build output is
365
+ ``env-<name>.log`` in ``log_dir`` (the job's ``.work``). A secret is never
366
+ given to a build: an image is shared, a secret is the run's."""
367
+ base = base_image(os.path.join(log_dir, "image.log") if log_dir else None)
368
+ tag = env_tag(bench, name, base)
369
+ with _locks_lock:
370
+ lock = _locks.setdefault(tag, threading.Lock())
371
+ with lock:
372
+ if _built.get(tag) or _docker(["image", "inspect", tag], check=False).returncode == 0:
373
+ _built[tag] = True
374
+ return tag
375
+ text, files = env_context(bench, name, base)
376
+ ctx = tempfile.mkdtemp(prefix="nullius-env-")
377
+ try:
378
+ with open(os.path.join(ctx, "Dockerfile"), "w") as f:
379
+ f.write(text)
380
+ for rel, data in files.items():
381
+ os.makedirs(os.path.dirname(os.path.join(ctx, rel)), exist_ok=True)
382
+ with open(os.path.join(ctx, rel), "wb") as f:
383
+ f.write(data)
384
+ r = _docker(["build", "--label", f"nullius.env={bench['key']}/{name}", "-t", tag, ctx], timeout=3600, check=False)
385
+ finally:
386
+ shutil.rmtree(ctx, ignore_errors=True)
387
+ if log_dir:
388
+ os.makedirs(log_dir, exist_ok=True)
389
+ with open(os.path.join(log_dir, f"env-{name}.log"), "a") as f:
390
+ f.write(f"# {tag}\n" + r.stdout + r.stderr)
391
+ if r.returncode != 0:
392
+ raise Unavailable(f"building the environment {name} ({tag}): {(r.stderr or r.stdout).strip()[-300:]}")
393
+ _built[tag] = True
394
+ return tag
395
+
396
+
397
+ def image_for(bench: dict, unit_setup: Optional[dict], dataset: Optional[dict] = None,
398
+ log_dir: Optional[str] = None) -> str:
399
+ """The image a unit runs in: the environment its setup names, else its
400
+ dataset's default (``definitions.environment_of``), built on first use;
401
+ the base image when it has none. An environment the benchmark does not
402
+ declare raises ValueError, as does an own image, which the unit's setup
403
+ names itself (:func:`own_image`)."""
404
+ name, e = definitions.environment_of(bench, unit_setup, dataset)
405
+ if name is None:
406
+ return base_image(os.path.join(log_dir, "image.log") if log_dir else None)
407
+ if definitions.own_image(e):
408
+ raise ValueError(f"environment {name!r} is an own image: the unit's setup names it")
409
+ return env_image(bench, name, log_dir)
410
+
411
+
412
+ # -- names, mounts, argument lists -----------------------------------------------
413
+ def slug(job: str) -> str:
414
+ s = re.sub(r"[^A-Za-z0-9_.-]+", "-", job).strip("-.") or "job"
415
+ if len(s) > 80:
416
+ s = s[:72] + "-" + hashlib.sha256(job.encode()).hexdigest()[:7]
417
+ return s
418
+
419
+
420
+ def names(job: str, tag: str) -> Dict[str, str]:
421
+ base = f"nullius-{slug(job)}-{tag}"
422
+ return {"network": base, "agent": base, "relay": base + "-relay"}
423
+
424
+
425
+ def mcp_mounts(servers: Iterable[dict], trial: str) -> Tuple[List[str], List[str]]:
426
+ """What a unit's stdio MCP servers need inside the agent's container,
427
+ which starts them: each file their command, arguments or environment
428
+ name (read-only, the file alone - never the folder it sits in, which may
429
+ hold the benchmark's data and answers), and each folder inside the trial
430
+ they will write to (read-write: a server's call log). (rw, ro_files)."""
431
+ rw, ro = [], []
432
+ trial = os.path.abspath(trial)
433
+ for s in servers or []:
434
+ if not isinstance(s, dict):
435
+ continue
436
+ env = s.get("env") or {}
437
+ values = [s.get("command")] + list(s.get("args") or []) + \
438
+ [v for v in (env.values() if isinstance(env, dict) else (e.get("value") for e in env if isinstance(e, dict)))]
439
+ for v in values:
440
+ if not isinstance(v, str) or not os.path.isabs(v):
441
+ continue
442
+ v = os.path.abspath(v)
443
+ if os.path.isfile(v) and not v.startswith(trial + os.sep):
444
+ ro.append(v)
445
+ elif v.startswith(trial + os.sep):
446
+ d = v if os.path.isdir(v) else os.path.dirname(v)
447
+ os.makedirs(d, exist_ok=True)
448
+ rw.append(d)
449
+ return rw, ro
450
+
451
+
452
+ def mounts(rw: Iterable[str], ro: Iterable[str], files: Iterable[str] = ()) -> List[Tuple[str, bool]]:
453
+ """Each folder once, at its own path; a read-only folder already inside
454
+ another read-only one is not mounted twice. ``files``: single files,
455
+ read-only, unless a folder mounted already holds them."""
456
+ out: List[Tuple[str, bool]] = []
457
+ seen = set()
458
+ for p in rw:
459
+ p = os.path.abspath(p)
460
+ if p not in seen:
461
+ seen.add(p)
462
+ out.append((p, False))
463
+ ros = sorted({os.path.abspath(p) for p in ro if p and os.path.isdir(p)} - seen, key=len)
464
+ kept: List[str] = []
465
+ for p in ros:
466
+ if not any(p == k or p.startswith(k + os.sep) for k in kept):
467
+ kept.append(p)
468
+ out.append((p, True))
469
+ held = [p for p, _ in out]
470
+ for f in sorted({os.path.abspath(f) for f in files if f and os.path.isfile(f)}):
471
+ if not any(f.startswith(h + os.sep) for h in held):
472
+ out.append((f, True))
473
+ return out
474
+
475
+
476
+ _security: Optional[str] = None
477
+
478
+
479
+ def _security_options() -> str:
480
+ """The engine's security options (``docker info``), or podman's answer in
481
+ their terms (``name=rootless``): podman has no ``SecurityOptions``. The
482
+ first answer is kept. An engine that does not answer gives none and is
483
+ asked again."""
484
+ global _security
485
+ if _security is None:
486
+ asked = "{{if .Host.Security.Rootless}}name=rootless{{end}}" if podman() else "{{.SecurityOptions}}"
487
+ try:
488
+ r = _docker(["info", "--format", asked], check=False, timeout=30)
489
+ except Unavailable:
490
+ return ""
491
+ if r.returncode != 0:
492
+ return ""
493
+ _security = r.stdout
494
+ return _security
495
+
496
+
497
+ def rootless() -> bool:
498
+ """Whether the engine is rootless (Docker or podman): its uid 0 is this
499
+ machine's user, every other uid a sub-uid."""
500
+ return "name=rootless" in _security_options()
501
+
502
+
503
+ def userns_remap() -> bool:
504
+ """Whether a rootful Docker remaps its containers' users (``userns-remap``):
505
+ every uid inside, root too, is then a sub-uid of this machine."""
506
+ return "name=userns" in _security_options()
507
+
508
+
509
+ def _keep_id() -> List[str]:
510
+ """``--userns=keep-id`` on rootless podman, which maps :func:`_user`'s uid
511
+ to this machine's user, as on rootful Docker. Rootless Docker cannot map
512
+ a uid, and runs its containers as root, which is this user there."""
513
+ return ["--userns=keep-id"] if podman() and rootless() else []
514
+
515
+
516
+ def _userns() -> List[str]:
517
+ """The user namespace of a container that uses this machine's user's files:
518
+ :func:`_keep_id` on podman, ``--userns=host`` on a Docker with
519
+ ``userns-remap``, where a container's uids are otherwise sub-uids that
520
+ cannot use those files."""
521
+ if podman():
522
+ return _keep_id()
523
+ return ["--userns=host"] if userns_remap() else []
524
+
525
+
526
+ def make_bridge() -> None:
527
+ """On podman, make ``BRIDGE`` when it is missing (``--ignore``: asked each
528
+ time, so a network removed between units is made again): a bridge with
529
+ DNS, as podman's default network has DNS off. Docker has its default bridge."""
530
+ if podman():
531
+ _docker(["network", "create", "--ignore", "--label", "nullius.bridge=1", BRIDGE])
532
+
533
+
534
+ def way_out() -> List[str]:
535
+ """The network of a container that reaches out, or that the relays reach
536
+ (the relay, a licence server's forwarder, the check's proxy, the door to a
537
+ model server): the engine's default, or ``BRIDGE`` on podman
538
+ (:func:`make_bridge`). A rootless podman container starts on pasta, which
539
+ cannot join a network after it starts."""
540
+ return ["--network", BRIDGE] if podman() else []
541
+
542
+
543
+ def _user() -> str:
544
+ """The ``--user`` that gives a container's files to this machine's user:
545
+ ``0:0`` on rootless Docker, ``uid:gid`` otherwise (:func:`_keep_id`)."""
546
+ return "0:0" if rootless() and not podman() else f"{os.getuid()}:{os.getgid()}"
547
+
548
+
549
+ def _security_opt() -> List[str]:
550
+ """Every container's ``--security-opt``: no way to gain privileges, and on
551
+ podman no SELinux label. Docker's default daemon (``selinux-enabled`` off)
552
+ gives none, so a container reads the folders it mounts, which are never
553
+ relabelled (``:z`` would relabel the user's). Labels are future work."""
554
+ return ["--security-opt", "no-new-privileges", *(["--security-opt", "label=disable"] if podman() else [])]
555
+
556
+
557
+ def by_value(env: Dict[str, str]) -> Dict[str, str]:
558
+ """What of a unit's environment podman itself reads (``CLIENT_KEEP``:
559
+ ``HOME``, the XDG folders...) but ``PATH``: on podman passed by value
560
+ (``-e NAME=value``), so the podman program keeps this machine's own and
561
+ finds its images. Nothing on Docker."""
562
+ return {k: v for k, v in env.items() if k in CLIENT_KEEP and k != "PATH"} if podman() else {}
563
+
564
+
565
+ def program_env(env: Dict[str, str]) -> dict:
566
+ """The engine program's environment for a unit's process given ``env``:
567
+ the values of what goes by name (``-e NAME``), never ``PATH`` nor
568
+ :func:`by_value`'s."""
569
+ given = by_value(env)
570
+ return client_env({k: v for k, v in env.items() if k != "PATH" and k not in given})
571
+
572
+
573
+ def env_args(env: Dict[str, str], path: str) -> Tuple[List[str], List[str]]:
574
+ """A unit's process's ``-e`` words: ``PATH``, the container's and no
575
+ secret, and :func:`by_value`'s by value; then the rest by name, the value
576
+ in the engine program's own environment (:func:`program_env`), never on
577
+ an argument list."""
578
+ given = by_value(env)
579
+ return (["-e", "PATH=" + path, *[w for k, v in sorted(given.items()) for w in ("-e", f"{k}={v}")]],
580
+ [w for k in sorted(set(env) - {"PATH"} - set(given)) for w in ("-e", k)])
581
+
582
+
583
+ def run_argv(image: str, cmd: List[str], *, name: str, network: str, job: str, workdir: str,
584
+ mounted: List[Tuple[str, bool]], env: Dict[str, str], path: str, interactive: bool = True,
585
+ binds: Iterable[Tuple[str, str]] = ()) -> List[str]:
586
+ """``docker run`` for a unit's agent or capability, its environment as
587
+ :func:`env_args` gives it. ``binds``: (a file on this machine, its path
588
+ inside), each read-only - a licence file where its environment says the
589
+ software looks for it."""
590
+ values, named = env_args(env, path)
591
+ a = [DOCKER, "run", "--rm", "--name", name, "--network", network, "--label", f"{LABEL}={job}",
592
+ *_userns(), "--user", _user(), "--cap-drop", "ALL", *_security_opt(), "--init",
593
+ "-w", workdir, *values]
594
+ if interactive:
595
+ a.insert(2, "-i")
596
+ for p, ro in mounted:
597
+ a += ["-v", f"{p}:{p}:{'ro' if ro else 'rw'}"]
598
+ for src, dest in binds:
599
+ a += ["-v", f"{src}:{dest}:ro"]
600
+ return a + named + [image] + list(cmd)
601
+
602
+
603
+ def plain_only(root: str) -> List[str]:
604
+ """Remove from ``root`` everything that is not a regular file or a folder -
605
+ links above all - and return what went. What the agent could write (a
606
+ folder mounted into its container, what a check collects out of it) is
607
+ then read and written on this machine, where a link it planted would
608
+ reach any file the user can (verifier/reward.txt -> ~/.bashrc)."""
609
+ gone = []
610
+ for here, dirs, files in os.walk(root, followlinks=False):
611
+ for name in list(dirs) + files:
612
+ path = os.path.join(here, name)
613
+ mode = os.lstat(path).st_mode
614
+ if stat.S_ISREG(mode) or (stat.S_ISDIR(mode) and name in dirs):
615
+ continue
616
+ os.unlink(path)
617
+ gone.append(os.path.relpath(path, root))
618
+ if name in dirs:
619
+ dirs.remove(name)
620
+ return gone
621
+
622
+
623
+ CHECK_IMAGE = "NULLIUS_CHECK_IMAGE" # driver checks: the unit's image, or the base image for a local setup
624
+ CHECK_ENGINE = "NULLIUS_CHECK_ENGINE" # beside it on podman: check_start as podman runs from the driver, JSON
625
+ _XDG = (("XDG_DATA_HOME", ".local/share"), ("XDG_CONFIG_HOME", ".config")) # where podman keeps its images and settings
626
+
627
+
628
+ def check_start() -> List[str]:
629
+ """A check's container command up to its mounts (:func:`check_argv`): no
630
+ network, no capabilities, as this machine's user."""
631
+ return [DOCKER, "run", "--rm", "--network", "none", *_userns(), "--user", _user(), "--cap-drop", "ALL",
632
+ *_security_opt()]
633
+
634
+
635
+ def check_env() -> dict:
636
+ """What a driver, which runs with a home of its own, gets to run
637
+ :func:`check_argv` on the same engine as the units. On Docker: the
638
+ ``DOCKER_*`` variables of :func:`client_env`. On podman: the variables of
639
+ ``CLIENT_KEEP`` (``CONTAINER_HOST``, ``REGISTRY_AUTH_FILE``...) except ``PATH``, ``HOME``,
640
+ ``XDG_DATA_HOME`` and ``XDG_CONFIG_HOME``, and ``CHECK_ENGINE``. That is
641
+ :func:`check_start` behind ``env``, which gives podman this user's XDG data
642
+ and config folders. The driver's own processes do not use these folders."""
643
+ if not podman():
644
+ return {k: v for k, v in client_env().items() if k.startswith("DOCKER")}
645
+ home = os.path.expanduser("~")
646
+ xdg = [f"{k}={os.environ.get(k) or os.path.join(home, d)}" for k, d in _XDG]
647
+ return {**{k: os.environ[k] for k in CLIENT_KEEP if k not in ("PATH", "HOME", *dict(_XDG)) and k in os.environ},
648
+ CHECK_ENGINE: json.dumps(["env", *xdg, *check_start()])}
649
+
650
+
651
+ def check_argv(workdir: str, here: List[str], inside: List[str]) -> List[str]:
652
+ """With CHECK_IMAGE, run in a throwaway container with no network or
653
+ capabilities and only the workspace mounted. Otherwise return ``here``."""
654
+ image = os.environ.get(CHECK_IMAGE)
655
+ if not image:
656
+ return list(here)
657
+ start = json.loads(os.environ[CHECK_ENGINE]) if os.environ.get(CHECK_ENGINE) else check_start()
658
+ return [*start, "-v", f"{workdir}:{workdir}:rw", "-w", workdir, image, *inside]
659
+
660
+
661
+ # where this machine gets its packages from, passed by name: a proxy's URL may hold its password
662
+ INSTALL_KEEP = ("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "no_proxy", "all_proxy",
663
+ "UV_INDEX", "UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL", "UV_PYTHON_INSTALL_MIRROR",
664
+ "PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "npm_config_registry", "NPM_CONFIG_REGISTRY", "HF_ENDPOINT")
665
+
666
+
667
+ def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = None) -> List[str]:
668
+ """``docker run`` for an install step (``install.packages``' ``run_in``):
669
+ in the base image, as this machine's user, with the install folder alone
670
+ mounted, at its own path. The home and npm's and uv's caches are inside
671
+ it, removed after (``install.SCRATCH``). uv's Python is inside it too
672
+ (``.python``), and kept: the environment links to it, and the agent's
673
+ container mounts the folder. ``env``: the step's own variables. This
674
+ machine's proxy and package indexes (``INSTALL_KEEP``) are passed by name.
675
+ It has the network - it downloads the packages - and nothing of a run: no
676
+ key, no unit."""
677
+ home, npm, uv = (f"{where}/{s}" for s in install.SCRATCH)
678
+ env = {"HOME": home, "npm_config_cache": npm, "UV_CACHE_DIR": uv,
679
+ "UV_PYTHON_INSTALL_DIR": f"{where}/.python", "UV_PYTHON_DOWNLOADS": "automatic", **(env or {})}
680
+ return [DOCKER, "run", "--rm", "--label", "nullius.install=1", *_userns(), "--user", _user(), "--cap-drop", "ALL",
681
+ *_security_opt(), "-v", f"{where}:{where}:rw", "-w", where,
682
+ *[w for k, v in env.items() for w in ("-e", f"{k}={v}")],
683
+ *[w for k in INSTALL_KEEP if k in os.environ for w in ("-e", k)], image] + list(cmd)
684
+
685
+
686
+ def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str,
687
+ model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(),
688
+ key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None,
689
+ ledger: Optional[str] = None) -> List[str]:
690
+ """``docker run`` for a unit's relay: on the default bridge (its way to the
691
+ provider), its port published on 127.0.0.1 only. The key is never given:
692
+ ``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with
693
+ ``--mount``, which fails when the engine cannot see it (``-v`` would mount
694
+ an empty folder). ``hosts``: names it resolves
695
+ (``--add-host``), the machine's own for an endpoint on its loopback.
696
+ ``ledger``: the file of the run's spending cap, its folder mounted so
697
+ this relay books its calls where every other relay of the job does."""
698
+ a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_userns(), "--user", _user(),
699
+ "--cap-drop", "ALL", *_security_opt(), "--init",
700
+ *[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")],
701
+ "-p", f"127.0.0.1::{RELAY_PORT}", "-v", f"{app}:{app}:ro", "-v", f"{record}:{record}:rw",
702
+ *(["-v", f"{os.path.dirname(ledger)}:{os.path.dirname(ledger)}:rw"] if ledger else []),
703
+ "--mount", f"type=bind,source={key_file},target={RELAY_KEY_IN},readonly", "-w", app,
704
+ "-e", "PYTHONPATH=" + app, "-e", "PYTHONDONTWRITEBYTECODE=1",
705
+ image, "python3", "-m", "nullius.standard.relay", "serve", "--listen", f"0.0.0.0:{RELAY_PORT}",
706
+ "--base", base, "--record", record, "--key-file", RELAY_KEY_IN, "--public", f"http://{RELAY_ALIAS}:{RELAY_PORT}"]
707
+ if model:
708
+ a += ["--model", model]
709
+ for tag, m in sorted(routes.items()):
710
+ a += ["--route", f"{tag}={m}"]
711
+ if prices:
712
+ a += ["--prices", json.dumps(prices, separators=(",", ":"))]
713
+ if ledger:
714
+ a += ["--ledger", ledger]
715
+ for m in side_models:
716
+ a += ["--side-model", m]
717
+ if key_header != "authorization":
718
+ a += ["--key-header", key_header]
719
+ return a
720
+
721
+
722
+ _IPV4 = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
723
+
724
+
725
+ def on_loopback(base: str) -> bool:
726
+ """Whether a model endpoint is on this machine's loopback (a local model
727
+ server: ``http://localhost:11434/v1``), which a container cannot reach as
728
+ written - there, 127.0.0.1 is the container itself."""
729
+ host = (urllib.parse.urlsplit(base or "").hostname or "").lower()
730
+ return host == "localhost" or host == "::1" or host.startswith("127.")
731
+
732
+
733
+ def bridge_gateway() -> str:
734
+ """This machine's address on docker's default bridge, or ""."""
735
+ r = _docker(["network", "inspect", "bridge", "-f", "{{range .IPAM.Config}}{{.Gateway}} {{end}}"], check=False)
736
+ return next((w for w in r.stdout.split() if _IPV4.match(w)), "")
737
+
738
+
739
+ def _pump(src: socket.socket, dst: socket.socket) -> None:
740
+ try:
741
+ while data := src.recv(65536):
742
+ dst.sendall(data)
743
+ except OSError:
744
+ pass
745
+ finally:
746
+ for sk in (src, dst):
747
+ try:
748
+ sk.shutdown(socket.SHUT_RDWR)
749
+ except OSError:
750
+ pass
751
+
752
+
753
+ def _pass_each(server: socket.socket, target: Tuple[str, int]) -> None:
754
+ """Pass each connection ``server`` accepts to ``target``, until it is closed."""
755
+ while True:
756
+ try:
757
+ inside, _ = server.accept()
758
+ except OSError:
759
+ return
760
+ try:
761
+ outside = socket.create_connection(target, timeout=10)
762
+ outside.settimeout(None)
763
+ except OSError:
764
+ inside.close()
765
+ continue
766
+ for a, b in ((inside, outside), (outside, inside)):
767
+ threading.Thread(target=_pump, args=(a, b), daemon=True).start()
768
+
769
+
770
+ def _close(server: socket.socket) -> None:
771
+ """Close a listening socket. The shutdown wakes the thread blocked in its
772
+ ``accept``, which a close alone does not do on Linux."""
773
+ try:
774
+ server.shutdown(socket.SHUT_RDWR)
775
+ except OSError:
776
+ pass
777
+ server.close()
778
+
779
+
780
+ def use_socket_doors() -> bool:
781
+ """Whether a server on this machine's loopback is reached through a
782
+ :class:`SocketDoor`: on a rootless engine on Linux, or on podman there,
783
+ whose bridge has no address on this machine until a container joins it.
784
+ Elsewhere the engine runs in a virtual machine, which this machine's
785
+ sockets may not reach."""
786
+ return (rootless() or podman()) and platform.system() == "Linux"
787
+
788
+
789
+ class SocketDoor:
790
+ """A Unix socket in a folder of its own that passes each connection to one
791
+ ``host:port`` here: a rootless engine's door container mounts it to reach
792
+ this machine's loopback. ``stop`` closes it and removes the folder."""
793
+
794
+ def __init__(self, host: str, port: int):
795
+ self.dir = tempfile.mkdtemp(prefix="nullius-door-")
796
+ self.path, self.target = os.path.join(self.dir, "s"), (host, port) # short: a socket path has ~108 bytes
797
+ self._server: Optional[socket.socket] = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
798
+ try:
799
+ self._server.bind(self.path)
800
+ self._server.listen(64)
801
+ except OSError:
802
+ self.stop()
803
+ raise
804
+ threading.Thread(target=_pass_each, args=(self._server, self.target), daemon=True).start()
805
+
806
+ def stop(self) -> None:
807
+ server, self._server = self._server, None
808
+ if server is not None:
809
+ _close(server)
810
+ shutil.rmtree(self.dir, ignore_errors=True)
811
+
812
+
813
+ class Loopback:
814
+ """An endpoint on this machine's loopback, as the relays' containers reach
815
+ it: ``base`` and the ``hosts`` they are given. On Linux, a forwarder for
816
+ the run, listening on the default bridge's gateway alone - the address a
817
+ container there has for this machine, reached by the relays and by no
818
+ agent, whose networks are internal - and passing each connection to the
819
+ endpoint; the relays call it ``host.docker.internal``. Where the bridge
820
+ is not on this machine and the engine runs as root (Docker Desktop's
821
+ virtual machine), that name already reaches its loopback: only the name
822
+ changes. On Linux, a rootless engine's gateway is in its own network
823
+ namespace, and podman's bridge has no address here until a container
824
+ joins it, so a :class:`SocketDoor` (``door``) passes each connection
825
+ instead, and one door container for the run reaches it, on the engine's
826
+ default network as the relays are. The relays call it
827
+ ``host.docker.internal`` there too, at the door's address, on port 8000."""
828
+
829
+ NAME = "host.docker.internal"
830
+ DOOR_PORT = 8000 # not the endpoint's: below 1024 needs a capability on podman
831
+
832
+ def __init__(self, base: str, *, image: str = "", job: str = ""):
833
+ self.target = urllib.parse.urlsplit(base)
834
+ self.base, self.hosts = base, {}
835
+ self.image, self.job = image, job # the door container's (use_socket_doors)
836
+ self._server: Optional[socket.socket] = None
837
+ self.door: Optional[SocketDoor] = None
838
+ self.container: Optional[str] = None
839
+
840
+ def _url(self, port: Optional[int]) -> str:
841
+ t = self.target
842
+ netloc = (t.netloc.rpartition("@")[0] + "@" if "@" in t.netloc else "") + self.NAME + (f":{port}" if port else "")
843
+ return urllib.parse.urlunsplit((t.scheme, netloc, t.path, t.query, t.fragment))
844
+
845
+ def start(self) -> "Loopback":
846
+ if use_socket_doors():
847
+ self.door = SocketDoor(*self._endpoint())
848
+ self.container = f"nullius-{slug(self.job)}-model"
849
+ try:
850
+ make_bridge()
851
+ _docker(door_argv(self.image, name=self.container, job=self.job, port=self.DOOR_PORT,
852
+ socket_path=self.door.path)[1:],
853
+ env=client_env({"NULLIUS_FORWARD_TO": "unix:" + self.door.path}))
854
+ wait_listening(self.container, "the door to the model server")
855
+ ip = _docker(["inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}",
856
+ self.container]).stdout.split()
857
+ if not ip:
858
+ raise Unavailable("the door to the model server has no address on the default network")
859
+ except Exception:
860
+ self.stop()
861
+ raise
862
+ self.base, self.hosts = self._url(self.DOOR_PORT), {self.NAME: ip[0]}
863
+ return self
864
+ gateway = bridge_gateway()
865
+ try:
866
+ if not gateway:
867
+ raise OSError("no bridge gateway")
868
+ self._server = socket.create_server((gateway, 0))
869
+ except OSError:
870
+ self.base, self.hosts = self._url(self.target.port), {self.NAME: "host-gateway"}
871
+ return self
872
+ self.base, self.hosts = self._url(self._server.getsockname()[1]), {self.NAME: gateway}
873
+ threading.Thread(target=_pass_each, args=(self._server, self._endpoint()), daemon=True).start()
874
+ return self
875
+
876
+ def _endpoint(self) -> Tuple[str, int]:
877
+ t = self.target
878
+ return t.hostname, t.port or (443 if t.scheme == "https" else 80)
879
+
880
+ def stop(self) -> None:
881
+ server, self._server = self._server, None
882
+ if server is not None:
883
+ _close(server)
884
+ name, self.container = self.container, None
885
+ if name is not None:
886
+ try:
887
+ _docker(["rm", "-f", name], check=False, timeout=60)
888
+ except Unavailable:
889
+ pass
890
+ if self.door is not None:
891
+ self.door.stop()
892
+
893
+
894
+ def server_alias(address: str, i: int = 1) -> Tuple[str, str, int]:
895
+ """(the name the agent reaches it by, host, port) for a licence server
896
+ ``host:port``: its own name, or ``licence-server-<i>`` for an IPv4
897
+ address, which cannot be a network alias."""
898
+ host, _, port = address.strip().rpartition(":")
899
+ return (f"licence-server-{i}" if _IPV4.match(host) else host), host, int(port)
900
+
901
+
902
+ def forward_argv(image: str, *, name: str, job: str, port: int, nonce_file: Optional[str] = None) -> List[str]:
903
+ """``docker run`` for a licence server's forwarder: on the default bridge
904
+ (``BRIDGE`` on podman), its one file mounted, listening on the server's
905
+ port. Where it forwards is named, never given: ``-e NULLIUS_FORWARD_TO``.
906
+ ``host.docker.internal`` is the machine the run is on, for a server there
907
+ (where :func:`use_socket_doors`, ``Unit._forward`` gives such a server a
908
+ door instead). A macOS door mounts its nonce read-only."""
909
+ mount = ["--mount", f"type=bind,source={nonce_file},target={FORWARD_NONCE_IN},readonly"] if nonce_file else []
910
+ return [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_keep_id(), "--user", _user(),
911
+ "--cap-drop", "ALL", *_security_opt(), "--init",
912
+ "--add-host", "host.docker.internal:host-gateway", "-v", f"{FORWARD}:{FORWARD_IN}:ro", *mount,
913
+ "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "NULLIUS_FORWARD_TO",
914
+ image, "python3", FORWARD_IN, "--listen", f"0.0.0.0:{port}",
915
+ *(["--nonce-file", FORWARD_NONCE_IN] if nonce_file else [])]
916
+
917
+
918
+ def door_argv(image: str, *, name: str, job: str, port: int, socket_path: str,
919
+ network: Optional[str] = None, alias: Optional[str] = None) -> List[str]:
920
+ """``docker run`` for a door to a server on this machine (``mcp_machine``,
921
+ a :class:`SocketDoor`): on one internal network alone, as ``alias``, or
922
+ without ``network`` on the default one (``BRIDGE`` on podman: a
923
+ :class:`Loopback`'s), its one
924
+ file and the server's socket mounted, each alone and read-only. Where it
925
+ forwards is named, never given: ``-e NULLIUS_FORWARD_TO``
926
+ (``unix:<socket>``)."""
927
+ return [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *_userns(), "--user", _user(),
928
+ "--cap-drop", "ALL", *_security_opt(), "--init",
929
+ *(["--network", network, "--network-alias", alias] if network else way_out()),
930
+ "-v", f"{FORWARD}:{FORWARD_IN}:ro", "-v", f"{socket_path}:{socket_path}:ro",
931
+ "-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "NULLIUS_FORWARD_TO",
932
+ image, "python3", FORWARD_IN, "--listen", f"0.0.0.0:{port}"]
933
+
934
+
935
+ def proxy_env(direct: Iterable[str] = ()) -> Dict[str, str]:
936
+ """The variables that send HTTP(S) through the ``egress`` proxy; this
937
+ machine's loopback and ``direct`` (the unit's relay, doors and servers) are
938
+ reached as they are."""
939
+ proxy = f"http://{EGRESS_ALIAS}:{EGRESS_PORT}"
940
+ local = ",".join(dict.fromkeys(["localhost", "127.0.0.1", "::1", *direct]))
941
+ return {"HTTP_PROXY": proxy, "HTTPS_PROXY": proxy, "http_proxy": proxy, "https_proxy": proxy,
942
+ "NO_PROXY": local, "no_proxy": local}
943
+
944
+
945
+ def egress_argv(image: str, *, name: str, job: str) -> List[str]:
946
+ """``docker run`` for an own image's check's proxy (``egress.py``): on the
947
+ default bridge (``BRIDGE`` on podman), its one file mounted, nothing else given."""
948
+ return [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_keep_id(), "--user", _user(),
949
+ "--cap-drop", "ALL", *_security_opt(), "--init",
950
+ "-v", f"{EGRESS}:{EGRESS_IN}:ro", "-e", "PYTHONDONTWRITEBYTECODE=1",
951
+ image, "python3", EGRESS_IN, "--listen", f"0.0.0.0:{EGRESS_PORT}"]
952
+
953
+
954
+ def wait_listening(name: str, what: str, timeout: float = 30):
955
+ """Until a sidecar says ``listening`` on its output; Unavailable when it
956
+ ends first, or does not in ``timeout``."""
957
+ end = time.time() + timeout
958
+ while time.time() < end:
959
+ r = _docker(["logs", name], check=False)
960
+ if '"listening"' in r.stdout:
961
+ return
962
+ state = _docker(["inspect", "-f", "{{.State.Running}}", name], check=False).stdout.strip()
963
+ if state == "false":
964
+ raise Unavailable(f"{what} ended: " + (r.stdout + r.stderr).strip()[-300:])
965
+ time.sleep(0.2)
966
+ raise Unavailable(f"{what} did not listen in {int(timeout)} s")
967
+
968
+
969
+ class Unit:
970
+ """One unit's network and relay; what wraps its agent's command.
971
+
972
+ It answers as a relay does - ``url`` (the host's way in, for the driver),
973
+ ``calls``, ``stop`` - and ``inside``, the address the agent is given."""
974
+
975
+ def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, base: str, key: str,
976
+ model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None,
977
+ side_models=(), key_header: str = "authorization", servers: Iterable[str] = (),
978
+ doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None,
979
+ ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True):
980
+ self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app
981
+ self.ledger = ledger # the file of the run's spending cap, when it has one
982
+ self.servers = list(servers) # licence servers, host:port each: a forwarder apiece
983
+ self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece
984
+ self.side_models, self.key_header = tuple(side_models), key_header
985
+ self.base, self.key, self.model, self.routes, self.prices = base, key, model, dict(routes or {}), prices
986
+ self.hosts = dict(hosts or {})
987
+ self.names = names(job, tag)
988
+ self.host: Optional[str] = None
989
+ self.socket_bridges = []
990
+ self.started: List[str] = [] # containers to remove
991
+ self.network = False
992
+ self.key_dir: Optional[str] = None # its secret files' folder, until what reads them listens
993
+ self.network_probe = network_probe
994
+ self.probe_now = probe_now # the run's first unit checks the proxy route; the rest trust it
995
+ self.network_env = {}
996
+ self.socket_doors: List[SocketDoor] = []
997
+ os.makedirs(record_dir, exist_ok=True)
998
+
999
+ def start(self) -> "Unit":
1000
+ try:
1001
+ make_bridge()
1002
+ _docker(["network", "create", "--internal", "--label", f"{LABEL}={self.job}", self.names["network"]])
1003
+ self.network = True
1004
+ argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir,
1005
+ key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes,
1006
+ prices=self.prices, side_models=self.side_models, key_header=self.key_header,
1007
+ hosts=self.hosts, ledger=self.ledger)
1008
+ self.started.append(self.names["relay"])
1009
+ _docker(argv[1:])
1010
+ _docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]])
1011
+ self._wait(self.names["relay"], "the relay's container")
1012
+ self._forget_key()
1013
+ port = _docker(["port", self.names["relay"], f"{RELAY_PORT}/tcp"]).stdout.strip().splitlines()[0]
1014
+ self.host = "http://" + port.replace("0.0.0.0", "127.0.0.1")
1015
+ self.reach = {}
1016
+ for i, address in enumerate(self.servers, 1):
1017
+ self.reach[address] = self._forward(i, address)
1018
+ for i, (alias, port, sock) in enumerate(self.doors, 1):
1019
+ self._door(f"{self.names['agent']}-mcp{i}", alias, port, sock, "the door to the benchmark's MCP server")
1020
+ if self.network_probe:
1021
+ self._online()
1022
+ except Exception:
1023
+ self.stop()
1024
+ raise
1025
+ return self
1026
+
1027
+ def direct(self) -> List[str]:
1028
+ """What the agent reaches without the public web proxy: its relay, its
1029
+ doors and its licence servers' forwarders."""
1030
+ return [RELAY_ALIAS, *[alias for alias, _, _ in self.doors],
1031
+ *[address.rsplit(":", 1)[0] for address in getattr(self, "reach", {}).values()]]
1032
+
1033
+ def _key_file(self, name: str = "key", text: Optional[str] = None) -> str:
1034
+ """Write the key, or ``text``, to an owner-only file in the unit's key
1035
+ folder, a new one under :func:`key_root` when it has none, and return
1036
+ its path. Not the record folder: it is kept."""
1037
+ if not self.key_dir:
1038
+ self.key_dir = key_folder(self.job)
1039
+ return secret_file(os.path.join(self.key_dir, name), self.key if text is None else text)
1040
+
1041
+ def _forget_key(self):
1042
+ if self.key_dir:
1043
+ shutil.rmtree(self.key_dir, ignore_errors=True)
1044
+ self.key_dir = None
1045
+
1046
+ def _online(self):
1047
+ """Opt-in public HTTP(S), with private destinations blocked by the proxy."""
1048
+ name = self.names["agent"] + "-egress"
1049
+ self.started.append(name)
1050
+ _docker(egress_argv(self.image, name=name, job=self.job)[1:])
1051
+ _docker(["network", "connect", "--alias", EGRESS_ALIAS, self.names["network"], name])
1052
+ self._wait(name, "the agent's public web proxy")
1053
+ self.network_env = proxy_env(self.direct())
1054
+ if not self.probe_now:
1055
+ return
1056
+ # Check the actual network path, without mounts or credentials.
1057
+ code = ("import sys,urllib.request; "
1058
+ "r=urllib.request.urlopen(urllib.request.Request(sys.argv[1], headers={'User-Agent': 'Nullius-connectivity-check'}), timeout=10); "
1059
+ "assert 200 <= r.status < 400")
1060
+ probe = self.names["agent"] + "-network-check"
1061
+ # Docker startup competes with other units for daemon/network locks.
1062
+ # Its deadline must not consume the HTTP check's own execution budget.
1063
+ # Track the name before starting so cancellation also removes it.
1064
+ self.started.append(probe)
1065
+ args = ["run", "-d", "--name", probe, "--label", f"{LABEL}={self.job}",
1066
+ "--init", "--network", self.names["network"],
1067
+ "--cap-drop", "ALL", "--security-opt", "no-new-privileges"]
1068
+ for key in self.network_env:
1069
+ args += ["-e", key]
1070
+ try:
1071
+ _docker(args + [self.image, "python3", "-c", code, self.network_probe],
1072
+ env=client_env(self.network_env), timeout=120)
1073
+ result = _docker(["wait", probe], timeout=30)
1074
+ if result.stdout.strip() != "0":
1075
+ logs = _docker(["logs", probe], check=False, timeout=10)
1076
+ raise Unavailable((logs.stderr or logs.stdout).strip()[-500:] or "probe exited unsuccessfully")
1077
+ except Unavailable as error:
1078
+ raise Unavailable(f"the public HTTPS connectivity check: {error}") from error
1079
+
1080
+ def _forward(self, i: int, address: str) -> str:
1081
+ """A licence server's forwarder, started and on the unit's network
1082
+ under the server's name: the address the agent reaches it by. It joins
1083
+ that network once it listens, after it resolves the server's name. One
1084
+ at ``host.docker.internal``, where :func:`use_socket_doors`, is a door
1085
+ to a :class:`SocketDoor` instead, as ``licence-server-<i>``."""
1086
+ alias, host, port = server_alias(address, i)
1087
+ name = f"{self.names['agent']}-lic{i}"
1088
+ if use_socket_doors() and host == Loopback.NAME:
1089
+ door = SocketDoor("127.0.0.1", port)
1090
+ self.socket_doors.append(door)
1091
+ alias = f"licence-server-{i}"
1092
+ self._door(name, alias, port, door.path, "the licence server's forwarder")
1093
+ return f"{alias}:{port}"
1094
+ self.started.append(name)
1095
+ _docker(forward_argv(self.image, name=name, job=self.job, port=port)[1:],
1096
+ env=client_env({"NULLIUS_FORWARD_TO": f"{host}:{port}"}))
1097
+ self._wait(name, "the licence server's forwarder")
1098
+ _docker(["network", "connect", "--alias", alias, self.names["network"], name])
1099
+ return f"{alias}:{port}"
1100
+
1101
+ def _door(self, name: str, alias: str, port: int, sock: str, what: str) -> None:
1102
+ """A door to a server on this machine, on the unit's network alone. On
1103
+ macOS, a forwarder to a loopback TCP bridge here that serves a
1104
+ connection only after the door's nonce line, given as a key file."""
1105
+ self.started.append(name)
1106
+ if platform.system() == "Darwin":
1107
+ # Docker Desktop cannot connect to host Unix sockets through a
1108
+ # file mount. Expose only this socket via a loopback TCP bridge.
1109
+ from nullius.standard import forward
1110
+ nonce = token_hex(32)
1111
+ class Handler(socketserver.BaseRequestHandler):
1112
+ def handle(self):
1113
+ self.request.settimeout(5)
1114
+ expected = (nonce + "\n").encode()
1115
+ received = b""
1116
+ try:
1117
+ while len(received) < len(expected):
1118
+ chunk = self.request.recv(len(expected) - len(received))
1119
+ if not chunk:
1120
+ return
1121
+ received += chunk
1122
+ except OSError:
1123
+ return
1124
+ if not compare_digest(received, expected):
1125
+ return
1126
+ self.request.settimeout(None)
1127
+ forward._serve(self.request, [(socket.AF_UNIX, sock)])
1128
+ bridge = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
1129
+ bridge.daemon_threads = True
1130
+ self.socket_bridges.append(bridge)
1131
+ threading.Thread(target=bridge.serve_forever, daemon=True).start()
1132
+ _docker(forward_argv(self.image, name=name, job=self.job, port=port,
1133
+ nonce_file=self._key_file(name, nonce))[1:],
1134
+ env=client_env({"NULLIUS_FORWARD_TO": f"host.docker.internal:{bridge.server_address[1]}"}))
1135
+ _docker(["network", "connect", "--alias", alias, self.names["network"], name])
1136
+ else:
1137
+ _docker(door_argv(self.image, name=name, job=self.job, network=self.names["network"], alias=alias,
1138
+ port=port, socket_path=sock)[1:],
1139
+ env=client_env({"NULLIUS_FORWARD_TO": "unix:" + sock}))
1140
+ self._wait(name, what)
1141
+ self._forget_key() # read before it listens
1142
+
1143
+ def _wait(self, name: str, what: str, timeout: float = 30):
1144
+ wait_listening(name, what, timeout)
1145
+
1146
+ def url(self, tag: str, model: Optional[str] = None) -> str:
1147
+ """The host's way to a route. A route's model is fixed when the relay
1148
+ starts (``routes``); asking for another is an error, not a silent switch."""
1149
+ if model and self.routes.get(tag) != model:
1150
+ raise ValueError(f"route {tag} was not started as {model}")
1151
+ return f"{self.host}/{tag}/v1"
1152
+
1153
+ def inside(self, tag: str) -> str:
1154
+ return f"http://{RELAY_ALIAS}:{RELAY_PORT}/{tag}/v1"
1155
+
1156
+ def calls(self, tag: str) -> List[dict]:
1157
+ try:
1158
+ with open(os.path.join(self.dir, tag + ".jsonl")) as f:
1159
+ return [json.loads(l) for l in f if l.strip()]
1160
+ except OSError:
1161
+ return []
1162
+
1163
+ def wrap(self, cmd: List[str], env: dict, workdir: str, mounted: List[Tuple[str, bool]], *,
1164
+ image: Optional[str] = None, interactive: bool = True,
1165
+ binds: Iterable[Tuple[str, str]] = ()) -> Tuple[List[str], dict]:
1166
+ """A command and its environment turned into ``docker run`` and the
1167
+ engine program's environment (:func:`env_args`, :func:`program_env`);
1168
+ ``binds``, licence files mounted read-only where they are declared."""
1169
+ app = os.path.abspath(self.app)
1170
+ binds = [(os.path.abspath(src), dest) for src, dest in binds]
1171
+ for p, _ in mounted: # the application's folder holds the keys' file: never it, nor a folder above it
1172
+ if app == p or app.startswith(p.rstrip(os.sep) + os.sep):
1173
+ raise ValueError(f"{p} would show the agent this application's folder and its keys")
1174
+ for src, _ in binds: # a licence comes from its store, never from in here
1175
+ if src.startswith(app + os.sep):
1176
+ raise ValueError(f"{src} is in this application's folder: a licence is never mounted from there")
1177
+ inside = {k: v for k, v in {**env, **self.network_env}.items() if k not in HOST_ONLY}
1178
+ argv = run_argv(image or self.image, cmd, name=self.names["agent"], network=self.names["network"], job=self.job,
1179
+ workdir=workdir, mounted=mounted, env=inside, path=inside.get("PATH") or IMAGE_PATH,
1180
+ interactive=interactive, binds=binds)
1181
+ self.started.append(self.names["agent"])
1182
+ return argv, program_env(inside)
1183
+
1184
+ def stop(self):
1185
+ """Remove the unit's containers and network, and the key's file if the
1186
+ relay has not read it; the relay's log is kept beside its record first."""
1187
+ self._forget_key()
1188
+ if self.names["relay"] in self.started:
1189
+ try:
1190
+ r = _docker(["logs", self.names["relay"]], check=False, timeout=30)
1191
+ with open(os.path.join(self.dir, "relay.log"), "w") as f:
1192
+ f.write(r.stdout + r.stderr)
1193
+ except (Unavailable, OSError):
1194
+ pass
1195
+ if self.network_probe and self.names["agent"] + "-egress" in self.started:
1196
+ try:
1197
+ r = _docker(["logs", self.names["agent"] + "-egress"], check=False, timeout=30)
1198
+ with open(os.path.join(self.dir, "egress.log"), "w") as f:
1199
+ f.write(r.stdout + r.stderr)
1200
+ except (Unavailable, OSError):
1201
+ pass
1202
+ for n in dict.fromkeys(self.started):
1203
+ try:
1204
+ _docker(["rm", "-f", n], check=False, timeout=60)
1205
+ except Unavailable:
1206
+ pass
1207
+ self.started = []
1208
+ for bridge in self.socket_bridges:
1209
+ bridge.shutdown()
1210
+ bridge.server_close()
1211
+ self.socket_bridges = []
1212
+
1213
+ for d in self.socket_doors:
1214
+ d.stop()
1215
+ self.socket_doors = []
1216
+ if self.network:
1217
+ try:
1218
+ _docker(["network", "rm", self.names["network"]], check=False, timeout=60)
1219
+ except Unavailable:
1220
+ pass
1221
+ self.network = False
1222
+
1223
+
1224
+ # -- an own image: the task's container, the agent added at run time -------------
1225
+ RUNTIMES = "/opt/nullius" # where an own image is given what the agent runs on
1226
+ PYTHON_VERSION = "3.12.14" # the standalone Python an agent started as python3 runs on there; moves with UV_VERSION
1227
+ _PULL = re.compile(r"^[a-z0-9][a-z0-9._/:-]*@sha256:[0-9a-f]{64}$")
1228
+ IMAGE_SPEC = ("build", "pull", "memory_mb", "cpus")
1229
+ STEP = ("copy", "write", "run", "collect", "seconds", "network", "keep_processes")
1230
+ KEEP_ALIVE = ["sh", "-c", "while :; do sleep 3600; done"]
1231
+ INIT = "/usr/sbin/docker-init" # what --init mounts: docker's, not the unit's
1232
+
1233
+ # The box's processes, read from /proc with the shell's own builtins only - a
1234
+ # task's image may have no ps, no pkill, not even tr - and with no subshell,
1235
+ # so the script never sees a child of its own. A line of /proc/<pid>/stat is
1236
+ # "pid (comm) state ppid ...", comm free to hold spaces and parentheses: what
1237
+ # follows its last ") " is the state, then the parent.
1238
+ _SCAN = r"""
1239
+ line=
1240
+ read -r line 2>/dev/null < "$d/stat" || [ -n "$line" ] || continue
1241
+ set -- ${line##*) }
1242
+ """
1243
+ # The keep-alive: the one child of PID 1 (docker's init) right after the start.
1244
+ KEEPER = r"""for d in /proc/[0-9]*; do
1245
+ p=${d#/proc/}
1246
+ [ "$p" = "$$" ] && continue""" + _SCAN + r""" [ "$2" = 1 ] && echo "$p"
1247
+ done
1248
+ """
1249
+ # Every process but PID 1, the keep-alive ($1) and its own sleep, and this
1250
+ # script: stopped first (SIGSTOP - a stopped process cannot fork), then
1251
+ # killed, round after round until a scan finds none; a zombie is init's to
1252
+ # reap and is passed over. Says each it stopped; exit 1 with what is left when
1253
+ # twenty rounds were not enough.
1254
+ SETTLE = r"""keep=$1; n=0
1255
+ while :; do
1256
+ left=
1257
+ for d in /proc/[0-9]*; do
1258
+ p=${d#/proc/}
1259
+ case " 1 $keep $$ " in *" $p "*) continue ;; esac""" + _SCAN + r""" case $1 in Z|X|x) continue ;; esac
1260
+ [ "$2" = "$keep" ] && continue
1261
+ left="$left $p"
1262
+ done
1263
+ [ -z "$left" ] && exit 0
1264
+ n=$((n+1))
1265
+ if [ "$n" -gt 20 ]; then echo "still running:$left"; exit 1; fi
1266
+ for p in $left; do kill -STOP "$p" 2>/dev/null; done
1267
+ for p in $left; do
1268
+ c=; read -r c 2>/dev/null < "/proc/$p/comm"
1269
+ echo "stopped $p $c"
1270
+ kill -KILL "$p" 2>/dev/null
1271
+ done
1272
+ sleep 1
1273
+ done
1274
+ """
1275
+
1276
+
1277
+ def runtimes_dir() -> str:
1278
+ return os.path.join(install.HOME, "runtimes")
1279
+
1280
+
1281
+ def within(root: str, rel) -> str:
1282
+ """A path a driver named relative to its upstream (or its own folder),
1283
+ resolved, and refused unless it stays inside: a driver names the task's
1284
+ files, never anything else of this machine."""
1285
+ if not isinstance(rel, str) or not rel or os.path.isabs(rel):
1286
+ raise ValueError(f"{rel!r}: a path relative to the benchmark's upstream")
1287
+ r = os.path.realpath(root)
1288
+ p = os.path.realpath(os.path.join(r, rel))
1289
+ if not p.startswith(r + os.sep):
1290
+ raise ValueError(f"{rel} is outside the benchmark's upstream")
1291
+ if not os.path.exists(p):
1292
+ raise ValueError(f"{rel} is not in the benchmark's upstream")
1293
+ return p
1294
+
1295
+
1296
+ def spec_problems(spec) -> List[str]:
1297
+ """What is wrong with a unit's ``image`` (its setup's): ``build``, a
1298
+ folder holding a Dockerfile, or ``pull``, a reference pinned by digest;
1299
+ ``memory_mb`` and ``cpus``, the limits the task asks for."""
1300
+ if not isinstance(spec, dict):
1301
+ return ["image: {build: <folder in the upstream>} or {pull: <name>@sha256:<digest>}"]
1302
+ bad = [f"image.{k}: not a field ({', '.join(IMAGE_SPEC)})" for k in spec if k not in IMAGE_SPEC]
1303
+ if ("build" in spec) == ("pull" in spec):
1304
+ bad.append("image: one of build (a folder with a Dockerfile) or pull (a reference by digest)")
1305
+ if "pull" in spec and not _PULL.match(str(spec["pull"])):
1306
+ bad.append(f"image.pull: {spec['pull']!r} is not pinned by digest (<name>@sha256:<64 hex>)")
1307
+ for k in ("memory_mb", "cpus"):
1308
+ if k in spec and (not isinstance(spec[k], (int, float)) or isinstance(spec[k], bool) or spec[k] <= 0):
1309
+ bad.append(f"image.{k}: a positive number")
1310
+ return bad
1311
+
1312
+
1313
+ def task_env_problems(env, reserved: Iterable[str] = ()) -> List[str]:
1314
+ """Task variables for an agent in its own image, never its model configuration."""
1315
+ if not isinstance(env, dict):
1316
+ return ["env: an object of variable names and string values"]
1317
+ bad = []
1318
+ for name, value in env.items():
1319
+ if not isinstance(name, str) or not re.fullmatch(r"[A-Z_][A-Z0-9_]*", name):
1320
+ bad.append("env: variable names use capitals, digits and underscores")
1321
+ continue
1322
+ if (name in reserved or name.startswith(("NULLIUS_", "DOCKER_"))
1323
+ or re.search(r"(^|_)(API|AUTH|KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?|BASE_URL|ENDPOINT|MODEL)(_|$)", name)):
1324
+ bad.append(f"env.{name}: model settings and credentials belong to the run")
1325
+ if not isinstance(value, str) or "\0" in value:
1326
+ bad.append(f"env.{name}: a string without NUL")
1327
+ elif name == "PATH" and any(not part.startswith("/") for part in value.split(":")):
1328
+ bad.append("env.PATH: absolute folders separated by ':'")
1329
+ return bad
1330
+
1331
+
1332
+ def step_problems(step, *, collect: bool) -> List[str]:
1333
+ """What is wrong with a step run in a unit's box (``verify``, ``solve``):
1334
+ ``copy`` {relative path in the upstream: absolute path inside}, ``write``
1335
+ {absolute path inside: its text - a file the driver has by content, an
1336
+ eval script or a patch the dataset carries}, ``run``
1337
+ (argv), ``collect`` (a folder inside, ``verify`` only), ``seconds``,
1338
+ ``network`` (true: a way out to public addresses, :meth:`Box.online`),
1339
+ ``keep_processes`` (``verify`` only - true: what the agent left running is
1340
+ not stopped before the check, :meth:`Box.settle`)."""
1341
+ if not isinstance(step, dict):
1342
+ return ["a step: {copy?, write?, run, collect?, seconds?, network?, keep_processes?}"]
1343
+ bad = [f"{k}: not a field ({', '.join(STEP)})" for k in step
1344
+ if k not in STEP or (k in ("collect", "keep_processes") and not collect)]
1345
+ run = step.get("run")
1346
+ if not isinstance(run, list) or not run or not all(isinstance(w, str) and w for w in run):
1347
+ bad.append("run: the command, a list of words")
1348
+ copy = step.get("copy") or {}
1349
+ if not isinstance(copy, dict) or not all(isinstance(d, str) and d.startswith("/") for d in copy.values()):
1350
+ bad.append("copy: {path in the upstream: absolute path inside}")
1351
+ write = step.get("write") or {}
1352
+ if not isinstance(write, dict) or not all(isinstance(d, str) and d.startswith("/") and isinstance(t, str)
1353
+ for d, t in write.items()):
1354
+ bad.append("write: {absolute path inside: the file's text}")
1355
+ if "collect" in step and not (isinstance(step["collect"], str) and step["collect"].startswith("/")):
1356
+ bad.append("collect: an absolute path inside")
1357
+ if "seconds" in step and (not isinstance(step["seconds"], (int, float)) or step["seconds"] <= 0):
1358
+ bad.append("seconds: a positive number")
1359
+ if "network" in step and not isinstance(step["network"], bool):
1360
+ bad.append("network: true or false")
1361
+ if "keep_processes" in step and not isinstance(step["keep_processes"], bool):
1362
+ bad.append("keep_processes: true or false")
1363
+ return bad
1364
+
1365
+
1366
+ def own_context(path: str) -> Dict[str, str]:
1367
+ """An own image's build context: every file of its folder, by its path
1368
+ there, and its sha256."""
1369
+ out = {}
1370
+ for d, dirs, files in os.walk(path):
1371
+ dirs.sort()
1372
+ for f in sorted(files):
1373
+ p = os.path.join(d, f)
1374
+ out[os.path.relpath(p, path)] = "link:" + os.readlink(p) if os.path.islink(p) else install.sha256_of(p)
1375
+ return out
1376
+
1377
+
1378
+ def own_tag(bench: dict, rel: str, path: str) -> str:
1379
+ """``nullius/own-<benchmark>-<folder>:<hash>``: the sha256 of every file
1380
+ the build reads, so the same folder is the same image, and a changed one a
1381
+ new image beside it. Its ``FROM`` is the task's, as published."""
1382
+ h = hashlib.sha256()
1383
+ for f, digest in own_context(path).items():
1384
+ h.update(f.encode() + b"\0" + digest.encode() + b"\0")
1385
+ word = re.sub(r"[^a-z0-9]+", "-", rel.lower()).strip("-")[:60] or "task"
1386
+ return f"nullius/own-{bench['key']}-{word}:{h.hexdigest()[:12]}"
1387
+
1388
+
1389
+ def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None) -> str:
1390
+ """A unit's own image: ``pull``ed by its digest, or built from the folder
1391
+ ``build`` names in the benchmark's upstream (``root``) - the task's own
1392
+ Dockerfile, its own ``FROM``, its folder the whole build context - the
1393
+ first time a unit asks for it, and kept by its tag after. Two units
1394
+ asking at once wait for one build. No secret is given to a build."""
1395
+ if spec.get("pull"):
1396
+ ref = spec["pull"]
1397
+ if not present(ref):
1398
+ r = _docker(["pull", ref], timeout=3600, check=False)
1399
+ if r.returncode != 0:
1400
+ raise Unavailable(f"pulling {ref}: {(r.stderr or r.stdout).strip()[-300:]}")
1401
+ _built[ref] = True
1402
+ _fetched.add(ref)
1403
+ return ref
1404
+ path = within(root, spec["build"])
1405
+ if not os.path.isfile(os.path.join(path, "Dockerfile")):
1406
+ raise Unavailable(f"{spec['build']} holds no Dockerfile")
1407
+ tag = own_tag(bench, spec["build"], path)
1408
+ with _locks_lock:
1409
+ lock = _locks.setdefault(tag, threading.Lock())
1410
+ with lock:
1411
+ if present(tag):
1412
+ return tag
1413
+ # --force-rm: a build that fails (a task's Dockerfile fetching what is gone) leaves no container behind
1414
+ r = _docker(["build", "--force-rm", "--label", f"nullius.own={bench['key']}", "-t", tag, path], timeout=3600, check=False)
1415
+ if log_dir:
1416
+ os.makedirs(log_dir, exist_ok=True)
1417
+ with open(os.path.join(log_dir, "own-" + tag.split(":")[0].split("/")[-1] + ".log"), "a") as f:
1418
+ f.write(f"# {tag}\n" + r.stdout + r.stderr)
1419
+ if r.returncode != 0:
1420
+ raise Unavailable(f"building {spec['build']} ({tag}): {(r.stderr or r.stdout).strip()[-300:]}")
1421
+ _built[tag] = True
1422
+ _fetched.add(tag)
1423
+ return tag
1424
+
1425
+
1426
+ def fetched(image: str) -> bool:
1427
+ """Whether this process pulled or built ``image`` (``own_image``): an image
1428
+ that was there before it is the user's, never removed here. ``_built``
1429
+ is not the answer: it also remembers every image found present."""
1430
+ return image in _fetched
1431
+
1432
+
1433
+ def drop_image(image: str) -> None:
1434
+ """Remove an own image this process fetched, once no unit needs it: one
1435
+ per SWE-bench Verified instance, kept, filled docker's disk part-way
1436
+ through a run (the upstream harness removes its instance images too)."""
1437
+ if image in _fetched:
1438
+ _fetched.discard(image)
1439
+ _built.pop(image, None)
1440
+ _docker(["rmi", image], timeout=600, check=False)
1441
+
1442
+
1443
+ def image_config(image: str) -> dict:
1444
+ """What an image says of itself that a box keeps: its working directory,
1445
+ its user and its PATH."""
1446
+ c = json.loads(_docker(["image", "inspect", "-f", "{{json .Config}}", image]).stdout or "null") or {}
1447
+ variables = dict(e.split("=", 1) for e in c.get("Env") or [] if "=" in e) # the image's, not this machine's
1448
+ return {"workdir": c.get("WorkingDir") or "/", "user": c.get("User") or "", "path": variables.get("PATH") or IMAGE_PATH}
1449
+
1450
+
1451
+ def node_runtime(machine: Optional[str] = None) -> str:
1452
+ """Node for an own image: the official tarball the base image is built
1453
+ from, checked by the same sha256, unpacked once into
1454
+ ``<STANDARD_DIR>/runtimes/node-v<version>-linux-<arch>``. A glibc build:
1455
+ a musl image cannot run it (:meth:`Box.runtime_problem`)."""
1456
+ arch = _ARCH.get((machine or platform.machine()).lower())
1457
+ if not arch:
1458
+ raise Unavailable(f"no Node build pinned for {machine or platform.machine()}")
1459
+ name = f"node-v{NODE_VERSION}-linux-{arch}"
1460
+ where = os.path.join(runtimes_dir(), name)
1461
+ if install.ready(where):
1462
+ return where
1463
+ with install._locked(where):
1464
+ if install.ready(where):
1465
+ return where
1466
+ tmp = tempfile.mkdtemp(prefix="node-", dir=runtimes_dir())
1467
+ try:
1468
+ tarball = os.path.join(tmp, name + ".tar.xz")
1469
+ with urllib.request.urlopen(f"https://nodejs.org/dist/v{NODE_VERSION}/{name}.tar.xz", timeout=60) as r, \
1470
+ open(tarball, "wb") as f:
1471
+ shutil.copyfileobj(r, f)
1472
+ if install.sha256_of(tarball) != NODE_SHA256[arch]:
1473
+ raise Unavailable(f"{name}.tar.xz is not the one pinned (sha256)")
1474
+ with tarfile.open(tarball, "r:xz") as t:
1475
+ t.extractall(tmp, filter="data")
1476
+ if os.path.isdir(where):
1477
+ shutil.rmtree(where)
1478
+ os.rename(os.path.join(tmp, name), where)
1479
+ open(os.path.join(where, install.MARK), "w").close()
1480
+ except OSError as e:
1481
+ raise Unavailable(f"fetching Node {NODE_VERSION}: {e}")
1482
+ finally:
1483
+ shutil.rmtree(tmp, ignore_errors=True)
1484
+ return where
1485
+
1486
+
1487
+ def python_runtime() -> str:
1488
+ """A standalone Python for an own image (python-build-standalone, by
1489
+ ``uv python install``, which checks it by its own pinned digests),
1490
+ installed once into ``<STANDARD_DIR>/runtimes/python-<version>``: the
1491
+ interpreter's folder. Installed in the base image, so it is a glibc
1492
+ build, as Node's is, whatever this machine's libc."""
1493
+ where = os.path.join(runtimes_dir(), f"python-{PYTHON_VERSION}")
1494
+ found = lambda: sorted(glob.glob(os.path.join(where, f"cpython-{PYTHON_VERSION}-linux-*-gnu")))
1495
+ if install.ready(where) and found():
1496
+ return found()[0]
1497
+ with install._locked(where):
1498
+ if not (install.ready(where) and found()):
1499
+ os.makedirs(where, exist_ok=True)
1500
+ try:
1501
+ r = subprocess.run(install_argv(base_image(), where, ["uv", "python", "install", PYTHON_VERSION,
1502
+ "--install-dir", where, "--no-bin"]),
1503
+ env=client_env({k: os.environ[k] for k in INSTALL_KEEP if k in os.environ}),
1504
+ capture_output=True, text=True, timeout=1800)
1505
+ except (OSError, subprocess.TimeoutExpired) as e:
1506
+ raise Unavailable(f"installing Python {PYTHON_VERSION}: {e}")
1507
+ finally:
1508
+ install.remove_scratch(where)
1509
+ if r.returncode != 0 or not found():
1510
+ raise Unavailable(f"installing Python {PYTHON_VERSION}: {(r.stderr or r.stdout).strip()[-300:]}")
1511
+ open(os.path.join(where, install.MARK), "w").close()
1512
+ return found()[0]
1513
+
1514
+
1515
+ def runtime_mounts(names: Iterable[str]) -> Dict[str, Tuple[str, str]]:
1516
+ """Each runtime an agent needs in an own image: name -> (its folder on
1517
+ this machine, where it is mounted inside)."""
1518
+ out = {}
1519
+ for n in names:
1520
+ out[n] = ({"node": node_runtime, "python": python_runtime}[n](), f"{RUNTIMES}/{n}")
1521
+ return out
1522
+
1523
+
1524
+ def bare_python(word: str) -> bool:
1525
+ """``python3`` or ``python`` with no environment of its own - not an
1526
+ environment's interpreter (``{install}/.venv/bin/python``), which must
1527
+ stay itself or the agent runs without its packages."""
1528
+ return os.path.basename(word) in ("python3", "python") and "/.venv/" not in word
1529
+
1530
+
1531
+ def in_runtime(cmd: List[str], runtimes: Dict[str, Tuple[str, str]]) -> List[str]:
1532
+ """An agent's start command as an own image runs it: ``python3`` or
1533
+ ``node`` - the word, or a script's ``#!/usr/bin/env`` line - becomes the
1534
+ runtime under ``/opt/nullius``, so the agent runs on what it was installed
1535
+ for while the image's own ``python3`` and ``node`` stay the task's."""
1536
+ cmd = list(cmd)
1537
+ first = os.path.basename(cmd[0]) if cmd else ""
1538
+ if cmd and bare_python(cmd[0]) and "python" in runtimes:
1539
+ return [runtimes["python"][1] + "/bin/python3"] + cmd[1:]
1540
+ if first == "node" and "node" in runtimes:
1541
+ return [runtimes["node"][1] + "/bin/node"] + cmd[1:]
1542
+ try:
1543
+ with open(cmd[0], "rb") as f:
1544
+ line = f.readline(200).decode("utf-8", "replace").split()
1545
+ except (OSError, IndexError):
1546
+ return cmd
1547
+ if line[:1] == ["#!/usr/bin/env"] and len(line) > 1:
1548
+ word = {"node": "node", "python3": "python", "python": "python"}.get(line[1])
1549
+ if word in runtimes:
1550
+ return [runtimes[word][1] + ("/bin/node" if word == "node" else "/bin/python3")] + cmd
1551
+ return cmd
1552
+
1553
+
1554
+ class Box:
1555
+ """A unit's own container: the task's image, kept running for the whole
1556
+ unit so the agent, a control's reference solution and the benchmark's
1557
+ check all act on one container's state, one after the other.
1558
+
1559
+ It runs as the image's own user, as the task was written for - usually
1560
+ root, which its tasks need (a package to install, a file in ``/etc``) -
1561
+ with docker's default capabilities, no way to gain more
1562
+ (``no-new-privileges``), and on the unit's internal network (or one of its
1563
+ own, for a control): the relay, when there is one, is its only way out.
1564
+ The folders it writes on this machine (the agent's home) are given back to
1565
+ this machine's user before it is removed (``give_back``).
1566
+
1567
+ **Before the check, nothing of the agent runs** (:meth:`settle`). A
1568
+ process the agent started and left - a server, a loop, a watcher - would
1569
+ otherwise still run while the check's files are copied in and while the
1570
+ check has a way out: free to rewrite the tests, or to reach what the
1571
+ check's network reaches. So every process but docker's init and the
1572
+ keep-alive is stopped, as root in the box, and the box is confirmed empty
1573
+ of them, before anything is copied in. A task about a service the agent
1574
+ must leave running opts out with its ``verify`` step's ``keep_processes``;
1575
+ what that keeps runs on beside the tests, which is the task's trade.
1576
+
1577
+ **The check's way out is a proxy, not a bridge** (:meth:`online`). The
1578
+ box stays on its internal network; a proxy container joins it as
1579
+ ``egress`` (``egress.py``) and reaches public addresses only - never
1580
+ link-local (cloud metadata), private ranges, the shared range some clouds
1581
+ keep their metadata in, nor the machine itself through the docker
1582
+ bridge. A benchmark may also grant it for the agent's work with
1583
+ ``internet: true``; it is removed before the check. Each phase is told
1584
+ of its proxy through ``HTTP_PROXY`` and ``HTTPS_PROXY``. It needs no root
1585
+ on the machine, so every host runs the check the same way; ``egress.py`` says why that beat firewall rules.
1586
+
1587
+ **The agent's mounts stay for the check.** A running container's mounts
1588
+ cannot be changed, and taking its home away would mean a second container
1589
+ for the check, without the state the agent left in this one. They are
1590
+ harmless there: the install, definition and runtimes are read-only, the
1591
+ home is the trial's own folder and nothing else of this machine, the check
1592
+ does not read them, and nothing of the agent is left running to use them
1593
+ (unless the task keeps its processes)."""
1594
+
1595
+ def __init__(self, job: str, tag: str, image: str, *, workdir: str, path: str = IMAGE_PATH,
1596
+ network: Optional[str] = None, mounted: Iterable[Tuple[str, bool]] = (),
1597
+ runtimes: Optional[Dict[str, Tuple[str, str]]] = None, memory_mb=None, cpus=None,
1598
+ app: Optional[str] = None, env: Optional[dict] = None):
1599
+ n = names(job, tag)
1600
+ self.job, self.image, self.workdir, self.path = job, image, workdir, path
1601
+ self.env = dict(env or {})
1602
+ self.name = n["agent"]
1603
+ self.net = network or n["network"]
1604
+ self.own_net = network is None
1605
+ self.mounted, self.runtimes = list(mounted), dict(runtimes or {})
1606
+ self.memory_mb, self.cpus, self.app = memory_mb, cpus, app
1607
+ self.up = False
1608
+ self.keep: Optional[str] = None # the keep-alive's PID inside, found at the start
1609
+ self.egress: Optional[str] = None # the check's proxy container, once online
1610
+
1611
+ def start(self) -> "Box":
1612
+ if self.app:
1613
+ app = os.path.abspath(self.app)
1614
+ for p, _ in self.mounted: # the application's folder holds the keys' file
1615
+ if app == p or app.startswith(p.rstrip(os.sep) + os.sep):
1616
+ raise ValueError(f"{p} would show the agent this application's folder and its keys")
1617
+ try:
1618
+ if self.own_net:
1619
+ _docker(["network", "create", "--internal", "--label", f"{LABEL}={self.job}", self.net])
1620
+ # in the machine's user namespace on podman (its root stays this user on rootless podman whatever
1621
+ # containers.conf sets: keep-id makes it a sub-uid) and on a Docker with userns-remap
1622
+ a = ["run", "-d", "--name", self.name, "--label", f"{LABEL}={self.job}", "--network", self.net, "--init",
1623
+ *_security_opt(), "-w", self.workdir, *(["--userns=host"] if podman() else _userns())]
1624
+ if self.memory_mb:
1625
+ a += ["--memory", f"{int(self.memory_mb)}m"]
1626
+ if self.cpus:
1627
+ a += ["--cpus", str(self.cpus)]
1628
+ for p, ro in self.mounted:
1629
+ a += ["-v", f"{p}:{p}:{'ro' if ro else 'rw'}"]
1630
+ for host, inside in self.runtimes.values():
1631
+ a += ["-v", f"{host}:{inside}:ro"]
1632
+ self.up = True
1633
+ _docker(a + ["--entrypoint", KEEP_ALIVE[0], self.image] + KEEP_ALIVE[1:], timeout=300)
1634
+ self.keep = self.keeper()
1635
+ except Exception:
1636
+ self.stop()
1637
+ raise
1638
+ return self
1639
+
1640
+ def exec_argv(self, cmd: List[str], env: Dict[str, str], path: str, *, interactive: bool = True,
1641
+ workdir: Optional[str] = None) -> List[str]:
1642
+ """``docker exec`` of a command in the box, its environment as
1643
+ :func:`env_args` gives it."""
1644
+ values, named = env_args(env, path)
1645
+ return [DOCKER, "exec", *(["-i"] if interactive else []), "-w", workdir or self.workdir,
1646
+ *values, *named, self.name, *cmd]
1647
+
1648
+ def run(self, cmd: List[str], seconds: float, *, user: Optional[str] = None, env: Optional[dict] = None) -> dict:
1649
+ """A command run in the box, to its end or ``seconds``: {exit, output,
1650
+ timed_out}. ``exit`` is None when it timed out."""
1651
+ a = ["exec", "-w", self.workdir] + (["-u", user] if user else [])
1652
+ for k in sorted(env or {}):
1653
+ a += ["-e", k]
1654
+ try:
1655
+ r = subprocess.run([DOCKER] + a + [self.name] + list(cmd), env=client_env(env), stdin=subprocess.DEVNULL,
1656
+ stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=seconds)
1657
+ except subprocess.TimeoutExpired as e:
1658
+ out = e.stdout or b""
1659
+ return {"exit": None, "output": out.decode("utf-8", "replace") if isinstance(out, bytes) else out, "timed_out": True}
1660
+ except FileNotFoundError:
1661
+ raise Unavailable(f"{DOCKER} is not installed on this machine")
1662
+ return {"exit": r.returncode, "output": r.stdout.decode("utf-8", "replace"), "timed_out": False}
1663
+
1664
+ def copy_in(self, src: str, dest: str):
1665
+ """A file or a folder of this machine into the box at ``dest``: a
1666
+ folder's contents become ``dest``'s."""
1667
+ self.run(["mkdir", "-p", dest if os.path.isdir(src) else os.path.dirname(dest) or "/"], 60, user="0")
1668
+ _docker(["cp", src + ("/." if os.path.isdir(src) else ""), f"{self.name}:{dest}"], timeout=600)
1669
+
1670
+ def write_in(self, dest: str, text: str):
1671
+ """A file given by its text into the box at ``dest`` (a step's
1672
+ ``write``): written here first, then copied in as ``copy_in`` does."""
1673
+ tmp = tempfile.mkdtemp(prefix="write-")
1674
+ try:
1675
+ path = os.path.join(tmp, os.path.basename(dest) or "file")
1676
+ with open(path, "w") as f:
1677
+ f.write(text)
1678
+ self.copy_in(path, dest)
1679
+ finally:
1680
+ shutil.rmtree(tmp, ignore_errors=True)
1681
+
1682
+ def copy_out(self, src: str, dest: str) -> bool:
1683
+ """The contents of a folder in the box into ``dest`` on this machine;
1684
+ False when the box has no such folder."""
1685
+ os.makedirs(dest, exist_ok=True)
1686
+ ok = _docker(["cp", f"{self.name}:{src.rstrip('/')}/.", dest], timeout=600, check=False).returncode == 0
1687
+ plain_only(dest) # docker cp keeps the box's links, absolute targets included
1688
+ return ok
1689
+
1690
+ def keeper(self, timeout: float = 10) -> str:
1691
+ """The keep-alive's PID inside: the one child of docker's init, read
1692
+ right after the start, before anything else runs in the box."""
1693
+ end = time.time() + timeout
1694
+ while True:
1695
+ r = self.run(["sh", "-c", KEEPER], 30, user="0")
1696
+ pids = r["output"].split()
1697
+ if r["exit"] == 0 and len(pids) == 1 and pids[0].isdigit():
1698
+ return pids[0]
1699
+ if time.time() > end:
1700
+ raise Unavailable("the container's keep-alive was not found: " + r["output"].strip()[-200:])
1701
+ time.sleep(0.2)
1702
+
1703
+ @staticmethod
1704
+ def settle_argv(keep: str) -> List[str]:
1705
+ """The command that stops every process but init and the keep-alive."""
1706
+ return ["sh", "-c", SETTLE, "settle", keep]
1707
+
1708
+ def settle(self) -> List[str]:
1709
+ """Every process the agent (or a control's solution) left in the box
1710
+ stopped, as root: "<pid> <name>" for each. Unavailable when some are
1711
+ still running after it - the check is not run beside them."""
1712
+ if not self.keep:
1713
+ raise Unavailable("the container's keep-alive is not known: its processes cannot be told apart")
1714
+ r = self.run(self.settle_argv(self.keep), 120, user="0")
1715
+ said = r["output"].splitlines()
1716
+ if r["exit"] != 0:
1717
+ raise Unavailable("what the agent left running could not be stopped: " + (said[-1] if said else
1718
+ ("timed out" if r["timed_out"] else f"exit {r['exit']}")))
1719
+ return [l[len("stopped "):] for l in said if l.startswith("stopped ")]
1720
+
1721
+ def online(self) -> Dict[str, str]:
1722
+ """A way out for the check: its proxy (``egress.py``, in the base image)
1723
+ started on the default bridge (``BRIDGE`` on podman) and joined to the box's network as
1724
+ ``egress``. The box itself stays on its internal network. The
1725
+ variables the check is run with, which name the proxy - and leave
1726
+ the box's own services (``localhost``) to be reached directly."""
1727
+ if self.egress is None:
1728
+ image = base_image()
1729
+ name = self.name + "-egress"
1730
+ self.egress = name
1731
+ make_bridge()
1732
+ _docker(egress_argv(image, name=name, job=self.job)[1:])
1733
+ _docker(["network", "connect", "--alias", EGRESS_ALIAS, self.net, name])
1734
+ wait_listening(name, "the task's proxy")
1735
+ return proxy_env()
1736
+
1737
+ def offline(self):
1738
+ """The way out :meth:`online` gave taken away again: its proxy
1739
+ container removed, then shown gone - no container of that name, and
1740
+ none answering to ``egress`` on the box's network. Unavailable when
1741
+ either is still there: the agent is never started beside it."""
1742
+ name = self.egress
1743
+ if not name:
1744
+ return
1745
+ _docker(["rm", "-f", name], check=False, timeout=60)
1746
+ left = _docker(["ps", "-aq", "--filter", f"name=^/{name}$"], check=False, timeout=30)
1747
+ if left.returncode != 0 or left.stdout.strip():
1748
+ raise Unavailable(f"the way out ({name}) could not be removed")
1749
+ on = _docker(["network", "inspect", "-f", "{{range .Containers}}{{.Name}} {{end}}", self.net], check=False, timeout=30)
1750
+ if on.returncode != 0 or name in on.stdout.split():
1751
+ raise Unavailable(f"the way out ({name}) is still on the unit's network")
1752
+ self.egress = None
1753
+
1754
+ def egress_said(self) -> str:
1755
+ """What the check's proxy let through and refused, a JSON line each."""
1756
+ if not self.egress:
1757
+ return ""
1758
+ try:
1759
+ r = _docker(["logs", self.egress], check=False, timeout=30)
1760
+ except Unavailable:
1761
+ return ""
1762
+ return r.stdout + r.stderr
1763
+
1764
+ def runtime_problem(self, name: str) -> Optional[str]:
1765
+ """Why the agent's runtime ``name`` cannot start in this image, or
1766
+ None. A glibc build in a musl image is the usual reason, said as such."""
1767
+ inside = self.runtimes[name][1] + ("/bin/node" if name == "node" else "/bin/python3")
1768
+ r = self.run([inside, "--version"], 60)
1769
+ if r["exit"] == 0:
1770
+ return None
1771
+ musl = self.run(["sh", "-c", "ls /lib/ld-musl-* 2>/dev/null"], 30)
1772
+ if musl["exit"] == 0 and musl["output"].strip():
1773
+ return f"the agent's runtime ({name}) needs glibc; this task's image is musl-based (Alpine, say)"
1774
+ return f"the agent's runtime ({name}) does not start in this task's image: " + r["output"].strip()[-200:]
1775
+
1776
+ def diff(self, limit: int = 2000) -> str:
1777
+ """What changed in the box's file system since its image (``docker
1778
+ diff``), less the folders Nullius mounted and the ones above them."""
1779
+ r = _docker(["diff", self.name], check=False, timeout=120)
1780
+ ours = [p for p, _ in self.mounted] + [inside for _, inside in self.runtimes.values()] + [INIT]
1781
+ mine = lambda path: any(t == path or t.startswith(path.rstrip("/") + "/") or path.startswith(t + "/") for t in ours)
1782
+ lines = [l for l in r.stdout.splitlines() if not mine(l[2:])]
1783
+ return "\n".join(lines[:limit]) + (f"\n... {len(lines) - limit} more\n" if len(lines) > limit else "\n")
1784
+
1785
+ def give_back(self, paths: Iterable[str]):
1786
+ """The folders of this machine the box wrote into, given back to this
1787
+ machine's user: what root wrote there is the user's to read and remove.
1788
+ On a rootless engine the box's root is that user."""
1789
+ paths = [p for p in paths if p]
1790
+ if self.up and paths:
1791
+ try:
1792
+ self.run(["chown", "-R", "0:0" if rootless() else _user()] + paths, 120, user="0")
1793
+ except Unavailable:
1794
+ pass
1795
+
1796
+ def stop(self):
1797
+ if self.egress:
1798
+ try:
1799
+ _docker(["rm", "-f", self.egress], check=False, timeout=60)
1800
+ except Unavailable:
1801
+ pass
1802
+ self.egress = None
1803
+ if self.up:
1804
+ try:
1805
+ _docker(["rm", "-f", self.name], check=False, timeout=60)
1806
+ except Unavailable:
1807
+ pass
1808
+ self.up = False
1809
+ if self.own_net:
1810
+ try:
1811
+ _docker(["network", "rm", self.net], check=False, timeout=60)
1812
+ except Unavailable:
1813
+ pass
1814
+ self.own_net = False
1815
+
1816
+
1817
+ def key_root() -> str:
1818
+ """The key folders (a relay's key, a door's nonce, a setup driver's
1819
+ credentials), in the user's cache: Colima and Lima share the home with
1820
+ their engine, not the temporary folder."""
1821
+ return os.path.join(paths.cache_dir(), "relay-keys")
1822
+
1823
+
1824
+ def key_folder(job: str) -> str:
1825
+ """A new owner-only folder for a job's secret files under :func:`key_root`."""
1826
+ root = key_root()
1827
+ os.makedirs(root, exist_ok=True)
1828
+ os.chmod(root, 0o700)
1829
+ forget_stale_keys()
1830
+ return tempfile.mkdtemp(prefix=key_prefix(job), dir=root)
1831
+
1832
+
1833
+ def secret_file(path: str, text: str) -> str:
1834
+ """``text`` in a new file at ``path`` that only its owner reads."""
1835
+ with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as f:
1836
+ f.write(text)
1837
+ return path
1838
+
1839
+
1840
+ def forget_stale_keys():
1841
+ """Remove every job's key folders older than ``KEY_STALE``."""
1842
+ old = time.time() - KEY_STALE
1843
+ for d in glob.glob(os.path.join(glob.escape(key_root()), "*")):
1844
+ try:
1845
+ if os.lstat(d).st_mtime < old:
1846
+ shutil.rmtree(d, ignore_errors=True)
1847
+ except OSError:
1848
+ pass
1849
+
1850
+
1851
+ def key_prefix(job: str) -> str:
1852
+ """A job's key folder prefix: a hash, so no job's prefix starts another's."""
1853
+ return f"nullius-relay-key-{hashlib.sha256(job.encode()).hexdigest()[:12]}-"
1854
+
1855
+
1856
+ def sweep(job: str):
1857
+ """Remove every container and network a job's units left, by its label,
1858
+ its key folders, and stale key folders."""
1859
+ for d in glob.glob(os.path.join(glob.escape(key_root()), glob.escape(key_prefix(job)) + "*")):
1860
+ shutil.rmtree(d, ignore_errors=True)
1861
+ forget_stale_keys()
1862
+ try:
1863
+ ids = _docker(["ps", "-aq", "--filter", f"label={LABEL}={job}"], check=False).stdout.split()
1864
+ if ids:
1865
+ _docker(["rm", "-f"] + ids, check=False, timeout=120)
1866
+ nets = _docker(["network", "ls", "-q", "--filter", f"label={LABEL}={job}"], check=False).stdout.split()
1867
+ if nets:
1868
+ _docker(["network", "rm"] + nets, check=False, timeout=120)
1869
+ except Unavailable:
1870
+ pass