nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,1870 @@
|
|
|
1
|
+
"""The container backend: each unit's agent, or capability, in a container of its
|
|
2
|
+
own, and its relay in another beside it. This module shells out to the
|
|
3
|
+
engine's program (:data:`DOCKER`) and imports nothing outside the standard
|
|
4
|
+
library. That is ``docker``, or ``podman`` on a machine without Docker or
|
|
5
|
+
where ``docker`` is podman's own (:func:`engine`). Podman needs no Docker API
|
|
6
|
+
socket and no settings of its own for Nullius.
|
|
7
|
+
|
|
8
|
+
**One base image** (``nullius/base:<hash>``), built once and named by the
|
|
9
|
+
hash of its Dockerfile, so a change to it builds a new one beside the old:
|
|
10
|
+
Ubuntu 24.04 with git, python3, curl, CA certificates, tmux (Terminus's
|
|
11
|
+
terminal), Node 22 from its official tarball, checked by its sha256, and uv
|
|
12
|
+
from its official image, pinned by digest. A unit's image is chosen by
|
|
13
|
+
:func:`image_for`: the base image, or the benchmark's environment the unit
|
|
14
|
+
asks for - a layer on the base, its Dockerfile in the benchmark's folder,
|
|
15
|
+
named by the hash of what it is built from
|
|
16
|
+
(``nullius/env-<benchmark>-<name>:<hash>``) and built the first time a unit
|
|
17
|
+
uses it (:func:`env_image`).
|
|
18
|
+
|
|
19
|
+
**The same absolute paths inside.** A unit's workspace and home are mounted
|
|
20
|
+
read-write; the agent's install and its definition's folder read-only, and
|
|
21
|
+
each file a unit's stdio MCP server is started from, alone (:func:`mcp_mounts`)
|
|
22
|
+
- never this repository's ``app/`` (it holds the keys' file; ``Unit.wrap``
|
|
23
|
+
refuses it) nor the benchmark's folder or upstream (their data holds the
|
|
24
|
+
answers). All where they are on this machine, so every path a
|
|
25
|
+
definition's templates rendered (``{home}``, ``{install}``, ``{here}``) holds
|
|
26
|
+
inside unchanged. A uv environment's interpreter is in the install
|
|
27
|
+
(``.python``), installed with it in the base image (:func:`install_argv`).
|
|
28
|
+
The container runs as this machine's user, with no capabilities and no way to gain
|
|
29
|
+
privileges, so what it writes in the workspace is the user's to read and
|
|
30
|
+
remove. That is ``--user uid:gid``, with ``--userns=keep-id`` on rootless
|
|
31
|
+
podman, which maps that uid to this user, or ``--user 0:0`` on rootless
|
|
32
|
+
Docker, where only a container's root is this machine's user. On a Docker with
|
|
33
|
+
``userns-remap``, a container that uses this machine's user's files also
|
|
34
|
+
runs in this machine's user namespace (:func:`_userns`). On podman it
|
|
35
|
+
runs with no SELinux label (:func:`_security_opt`), as on Docker's default
|
|
36
|
+
daemon.
|
|
37
|
+
|
|
38
|
+
**The relay is a sidecar, and the agent's only way out** but for the public
|
|
39
|
+
web proxy of an environment that declares ``network``. Per unit, an
|
|
40
|
+
internal network (``docker network create --internal``: no route off it); a
|
|
41
|
+
relay container (``python3 -m nullius.standard.relay serve``) on the default
|
|
42
|
+
bridge *and* on that network as ``relay``; the agent's container on the
|
|
43
|
+
internal network alone. Its model endpoint is ``http://relay:8080/<tag>/v1``
|
|
44
|
+
and nothing else answers it. The real key reaches the relay's container
|
|
45
|
+
only, as an owner-only file under :func:`key_root`, mounted read-only and
|
|
46
|
+
removed from this machine once the relay listens: never on an argument list
|
|
47
|
+
(``ps``) or in the container's environment (``docker inspect``). The relay
|
|
48
|
+
publishes its port on 127.0.0.1 so the driver (on the machine for
|
|
49
|
+
agent/capability runs) reaches the judge's route and a capability
|
|
50
|
+
benchmark's reader. Its record folder is mounted read-write, so the host
|
|
51
|
+
reads each call as the local relay's are read. On podman the default bridge
|
|
52
|
+
is Nullius's own (:data:`BRIDGE`, :func:`way_out`). A model endpoint on this
|
|
53
|
+
machine's loopback is reached by the relays alone (:class:`Loopback`):
|
|
54
|
+
through a forwarder on the default bridge's gateway, or, on a rootless
|
|
55
|
+
engine or podman, a door container on the engine's default network.
|
|
56
|
+
|
|
57
|
+
**A licence server is one more door.** A unit whose environment needs a
|
|
58
|
+
licence of kind ``server`` (``host:port``) gets a forwarder beside its relay
|
|
59
|
+
(``forward.py``, mounted alone into the base image): on the default bridge
|
|
60
|
+
(``BRIDGE`` on podman), and on the unit's network under the server's own name, listening on its
|
|
61
|
+
port and passing every connection to that one address. The agent reaches
|
|
62
|
+
the server by the address it was given; nothing else is opened. A server
|
|
63
|
+
named by an IPv4 address is reached as ``licence-server-<i>`` instead, a
|
|
64
|
+
network alias being a name; ``host.docker.internal`` is the machine the run
|
|
65
|
+
is on. On a rootless engine or podman on Linux, a server at
|
|
66
|
+
``host.docker.internal`` is reached through a :class:`SocketDoor` instead, as ``licence-server-<i>``
|
|
67
|
+
on the unit's network alone. A licence file is mounted read-only, alone, at
|
|
68
|
+
the path its environment declares (``wrap``'s ``binds``).
|
|
69
|
+
|
|
70
|
+
**A benchmark's MCP server on the machine is one door more.** A server the
|
|
71
|
+
benchmark declares in ``mcp_servers`` runs on this machine, in the driver's
|
|
72
|
+
environment, behind a Unix socket (``mcp_machine.Listener``). The unit gets
|
|
73
|
+
a forwarder for it (``doors``): on the unit's internal network *only* - no
|
|
74
|
+
bridge, no route anywhere - under the alias the agent's entry names, the
|
|
75
|
+
socket alone mounted into it, passing every connection to that socket and
|
|
76
|
+
nothing else. On macOS, where the socket cannot be mounted, the forwarder
|
|
77
|
+
is on the default bridge too and goes to a loopback TCP bridge here, which
|
|
78
|
+
wants the door's nonce first, given as a key file (``forward.py``). The
|
|
79
|
+
agent starts ``mcp_bridge.py`` (mounted alone, read-only), which joins its
|
|
80
|
+
stdio to that door.
|
|
81
|
+
|
|
82
|
+
**An own image** (an environment with ``"image": "own"``) is the other way
|
|
83
|
+
round: the task's image, built from its own Dockerfile - its own ``FROM``,
|
|
84
|
+
its own folder as the build context (:func:`own_image`) - or pulled by its
|
|
85
|
+
digest, and the agent added to it at run time, never built in. A unit's
|
|
86
|
+
:class:`Box` is that image kept running (``sh -c 'sleep'``) for the whole
|
|
87
|
+
unit: the agent is started in it with ``docker exec``, so it works on the
|
|
88
|
+
container's own state in the image's working directory, and the benchmark's
|
|
89
|
+
check runs in it after the agent ends - what the agent left running stopped
|
|
90
|
+
first, its files copied in only then, and a way out, when it asks for one,
|
|
91
|
+
through a proxy to public addresses only (:meth:`Box.settle`,
|
|
92
|
+
:meth:`Box.online`). What
|
|
93
|
+
the agent needs that the image may not have is mounted read-only under
|
|
94
|
+
``/opt/nullius``: Node, unpacked once from the same pinned tarball the base
|
|
95
|
+
image is built from (:func:`node_runtime`), and a standalone Python
|
|
96
|
+
(:func:`python_runtime`); its install, its definition and its home are
|
|
97
|
+
mounted at their own paths, as on the base. A runtime that cannot start in
|
|
98
|
+
the image - a glibc build in a musl one - is found before the agent starts
|
|
99
|
+
(:meth:`Box.runtime_problem`), and the unit is not run.
|
|
100
|
+
|
|
101
|
+
Everything a unit starts carries the label ``nullius.job=<job>`` and is
|
|
102
|
+
removed when the unit ends, after an error or an interrupt too; :func:`sweep`
|
|
103
|
+
removes whatever a killed run left behind.
|
|
104
|
+
Docker Original setup drivers are handled by ``setup_container.py``.
|
|
105
|
+
"""
|
|
106
|
+
from __future__ import annotations
|
|
107
|
+
|
|
108
|
+
import functools
|
|
109
|
+
import glob
|
|
110
|
+
import hashlib
|
|
111
|
+
import json
|
|
112
|
+
import os
|
|
113
|
+
import platform
|
|
114
|
+
import re
|
|
115
|
+
import shutil
|
|
116
|
+
import socket
|
|
117
|
+
import socketserver
|
|
118
|
+
import stat
|
|
119
|
+
import subprocess
|
|
120
|
+
import tarfile
|
|
121
|
+
import tempfile
|
|
122
|
+
import threading
|
|
123
|
+
import time
|
|
124
|
+
import urllib.parse
|
|
125
|
+
import urllib.request
|
|
126
|
+
from secrets import compare_digest, token_hex
|
|
127
|
+
from typing import Dict, Iterable, List, Optional, Tuple
|
|
128
|
+
|
|
129
|
+
from nullius import paths
|
|
130
|
+
from nullius.standard import definitions, install
|
|
131
|
+
|
|
132
|
+
def _runs_podman(path: str) -> bool:
|
|
133
|
+
"""Whether the program at ``path`` is podman under another name: a link to
|
|
134
|
+
it, or a script that runs it (``podman-docker``'s ``docker``)."""
|
|
135
|
+
if os.path.basename(os.path.realpath(path)) == "podman":
|
|
136
|
+
return True
|
|
137
|
+
try:
|
|
138
|
+
with open(path, "rb") as f:
|
|
139
|
+
head = f.read(4096)
|
|
140
|
+
except OSError:
|
|
141
|
+
return False
|
|
142
|
+
return head.startswith(b"#!") and re.search(rb"^\s*exec\s+\S*podman\b", head, re.M) is not None
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def engine() -> str:
|
|
146
|
+
"""``podman`` when it is on the PATH and ``docker`` is not, or is podman
|
|
147
|
+
under that name; else ``docker``."""
|
|
148
|
+
docker, podman = shutil.which("docker"), shutil.which("podman")
|
|
149
|
+
return "podman" if podman and (not docker or _runs_podman(docker)) else "docker"
|
|
150
|
+
|
|
151
|
+
|
|
152
|
+
DOCKER = engine() # the engine's program, chosen once
|
|
153
|
+
LABEL = "nullius.job"
|
|
154
|
+
RELAY_PORT = 8080
|
|
155
|
+
RELAY_ALIAS = "relay"
|
|
156
|
+
FORWARD = os.path.join(os.path.dirname(os.path.abspath(__file__)), "forward.py")
|
|
157
|
+
FORWARD_IN = "/opt/nullius/forward.py" # where the forwarder's one file is mounted in its container
|
|
158
|
+
FORWARD_NONCE_IN = "/opt/nullius/forward-nonce" # a door's nonce's one file, in its forwarder's container
|
|
159
|
+
EGRESS = os.path.join(os.path.dirname(os.path.abspath(__file__)), "egress.py")
|
|
160
|
+
EGRESS_IN = "/opt/nullius/egress.py" # the check's proxy's one file, in its container
|
|
161
|
+
EGRESS_PORT = 3128
|
|
162
|
+
EGRESS_ALIAS = "egress"
|
|
163
|
+
BRIDGE = "nullius-bridge" # podman: the network a container with a way out starts on (way_out)
|
|
164
|
+
RELAY_KEY_IN = "/opt/nullius/relay-key" # the key's one file, in the relay's container
|
|
165
|
+
KEY_STALE = 600 # older than any start can take: a killed run left it
|
|
166
|
+
IMAGE_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
|
|
167
|
+
|
|
168
|
+
# uv and PYTHON_VERSION move together: this uv must know that Python
|
|
169
|
+
UV_VERSION = "0.13.5"
|
|
170
|
+
UV_DIGEST = "sha256:32a7bb39b496babbd997b0a39a4d66c126bdaf5661b302bf6f06abda8deca6ff" # its index: every platform
|
|
171
|
+
NODE_VERSION = "22.23.3"
|
|
172
|
+
NODE_SHA256 = {
|
|
173
|
+
"x64": "df450af89261115ef9f9e3830c3eeb2cc9213b63c720b1af623cb5dcbe2e02de",
|
|
174
|
+
"arm64": "a44aeb94849a299b22df10b9e622ec2f605c2183501bc40590705131de7c740f",
|
|
175
|
+
}
|
|
176
|
+
_ARCH = {"x86_64": "x64", "amd64": "x64", "aarch64": "arm64", "arm64": "arm64"}
|
|
177
|
+
|
|
178
|
+
# FROM in full: podman resolves a short name by this machine's registries
|
|
179
|
+
DOCKERFILE = """\
|
|
180
|
+
FROM docker.io/library/ubuntu:24.04
|
|
181
|
+
COPY --from=ghcr.io/astral-sh/uv:{uv_version}@{uv_digest} /uv /uvx /usr/local/bin/
|
|
182
|
+
RUN apt-get update \\
|
|
183
|
+
&& DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends git python3 ca-certificates curl tmux xz-utils \\
|
|
184
|
+
&& rm -rf /var/lib/apt/lists/*
|
|
185
|
+
RUN curl -fsSLo /tmp/node.tar.xz https://nodejs.org/dist/v{node}/node-v{node}-linux-{arch}.tar.xz \\
|
|
186
|
+
&& echo "{sha} /tmp/node.tar.xz" | sha256sum -c - \\
|
|
187
|
+
&& tar -xJf /tmp/node.tar.xz -C /usr/local --strip-components=1 --no-same-owner \\
|
|
188
|
+
&& rm /tmp/node.tar.xz
|
|
189
|
+
ENV PATH={path}
|
|
190
|
+
"""
|
|
191
|
+
|
|
192
|
+
# What a unit's process holds of this machine's environment that names a
|
|
193
|
+
# place on this machine: meaningless, or wrong, inside.
|
|
194
|
+
HOST_ONLY = ("TMPDIR", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR")
|
|
195
|
+
# What the engine's program keeps of this process's environment, to find the
|
|
196
|
+
# daemon, or podman its storage and settings. Not a key among them, though
|
|
197
|
+
# CONTAINER_HOST may name an ssh URL with a password: it is never on an argument list.
|
|
198
|
+
CLIENT_KEEP = ("PATH", "HOME", "DOCKER_HOST", "DOCKER_CONTEXT", "DOCKER_CONFIG", "DOCKER_CERT_PATH", "DOCKER_TLS_VERIFY",
|
|
199
|
+
"XDG_RUNTIME_DIR", "XDG_DATA_HOME", "XDG_CONFIG_HOME", "CONTAINERS_CONF", "CONTAINERS_STORAGE_CONF",
|
|
200
|
+
"CONTAINERS_REGISTRIES_CONF", "REGISTRY_AUTH_FILE", "CONTAINER_HOST", "CONTAINER_CONNECTION")
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
class Unavailable(Exception):
|
|
204
|
+
"""The engine is missing, not usable by this user, or failed at something."""
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
def podman() -> bool:
|
|
208
|
+
return DOCKER == "podman"
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _docker(args: List[str], *, env: Optional[dict] = None, input: Optional[str] = None, timeout: float = 120,
|
|
212
|
+
check: bool = True) -> subprocess.CompletedProcess:
|
|
213
|
+
try:
|
|
214
|
+
r = subprocess.run([DOCKER] + args, env=env if env is not None else client_env(), input=input, text=True,
|
|
215
|
+
stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout)
|
|
216
|
+
except FileNotFoundError:
|
|
217
|
+
raise Unavailable(f"{DOCKER} is not installed on this machine")
|
|
218
|
+
except subprocess.TimeoutExpired:
|
|
219
|
+
raise Unavailable(f"{DOCKER} {args[0]} did not answer in {int(timeout)} s")
|
|
220
|
+
if check and r.returncode != 0:
|
|
221
|
+
raise Unavailable(f"{DOCKER} {' '.join(args[:2])}: {(r.stderr or r.stdout).strip()[-300:]}")
|
|
222
|
+
return r
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def client_env(extra: Optional[dict] = None) -> dict:
|
|
226
|
+
"""The engine program's environment: what it needs to find the daemon, or
|
|
227
|
+
podman its storage and settings (``CLIENT_KEEP``), and the values of the
|
|
228
|
+
variables a ``-e NAME`` passes on."""
|
|
229
|
+
env = {**_context(), **{k: os.environ[k] for k in CLIENT_KEEP if k in os.environ}}
|
|
230
|
+
env.setdefault("PATH", os.defpath)
|
|
231
|
+
env.update(extra or {})
|
|
232
|
+
return env
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
@functools.cache
|
|
236
|
+
def _context() -> Dict[str, str]:
|
|
237
|
+
"""The daemon that ``docker context use`` selected, as the variables that
|
|
238
|
+
name it. The user's ``~/.docker`` keeps that selection, and a unit or a
|
|
239
|
+
driver has a home of its own. Only the endpoint goes on: the registry
|
|
240
|
+
logins in ``~/.docker`` do not. Read once per process."""
|
|
241
|
+
if os.environ.get("DOCKER_HOST") or os.environ.get("DOCKER_CONTEXT"):
|
|
242
|
+
return {}
|
|
243
|
+
try:
|
|
244
|
+
r = subprocess.run([DOCKER, "context", "inspect"], capture_output=True, text=True, timeout=30)
|
|
245
|
+
c = json.loads(r.stdout)[0]
|
|
246
|
+
ep = c["Endpoints"]["docker"]
|
|
247
|
+
except (OSError, subprocess.SubprocessError, ValueError, LookupError):
|
|
248
|
+
return {}
|
|
249
|
+
if c.get("Name") == "default":
|
|
250
|
+
return {}
|
|
251
|
+
env = {"DOCKER_HOST": ep["Host"]}
|
|
252
|
+
tls = bool(c.get("TLSMaterial", {}).get("docker"))
|
|
253
|
+
if tls:
|
|
254
|
+
env["DOCKER_CERT_PATH"] = os.path.join(c["Storage"]["TLSPath"], "docker")
|
|
255
|
+
# shortcut: docker reads a ca.pem from DOCKER_CERT_PATH, so a skip-verify context with a client certificate and no CA is not reached
|
|
256
|
+
if ep.get("SkipTLSVerify"):
|
|
257
|
+
env["DOCKER_TLS"] = "1"
|
|
258
|
+
elif tls:
|
|
259
|
+
env["DOCKER_TLS_VERIFY"] = "1"
|
|
260
|
+
return env
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def unusable() -> Optional[str]:
|
|
264
|
+
"""Why the container backend cannot run here, or None when it can."""
|
|
265
|
+
try:
|
|
266
|
+
r = _docker(["version", "--format", "{{.Server.Version}}"], timeout=30, check=False)
|
|
267
|
+
except Unavailable as e:
|
|
268
|
+
return str(e)
|
|
269
|
+
if r.returncode != 0:
|
|
270
|
+
said = (r.stderr or r.stdout).strip().splitlines()
|
|
271
|
+
hint = (" (add this user to the docker group, then log in again)"
|
|
272
|
+
if not podman() and "permission denied" in r.stderr.lower() else "")
|
|
273
|
+
if not podman() and shutil.which("podman"): # docker wins (engine), so say podman is there
|
|
274
|
+
hint += ". podman is on this machine: Nullius uses it when docker is not on the PATH"
|
|
275
|
+
return f"{DOCKER} is installed but not usable by this user: " + (said[-1][:200] if said else f"exit {r.returncode}") + hint
|
|
276
|
+
if podman():
|
|
277
|
+
try:
|
|
278
|
+
net = _docker(["info", "--format", "{{.Host.NetworkBackend}}"], timeout=30, check=False).stdout.strip()
|
|
279
|
+
except Unavailable as e:
|
|
280
|
+
return str(e)
|
|
281
|
+
if net != "netavark":
|
|
282
|
+
return (f"podman's network backend is {net or 'unknown'}: Nullius needs netavark"
|
|
283
|
+
' (podman 4.7 or later, network_backend = "netavark")')
|
|
284
|
+
return None
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def dockerfile(machine: Optional[str] = None) -> str:
|
|
288
|
+
arch = _ARCH.get((machine or platform.machine()).lower())
|
|
289
|
+
if not arch:
|
|
290
|
+
raise Unavailable(f"no Node build pinned for {machine or platform.machine()}")
|
|
291
|
+
return DOCKERFILE.format(uv_version=UV_VERSION, uv_digest=UV_DIGEST, node=NODE_VERSION, arch=arch, sha=NODE_SHA256[arch], path=IMAGE_PATH)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def image_tag(text: str) -> str:
|
|
295
|
+
return "nullius/base:" + hashlib.sha256(text.encode()).hexdigest()[:12]
|
|
296
|
+
|
|
297
|
+
|
|
298
|
+
_built: Dict[str, bool] = {}
|
|
299
|
+
# the own images this process pulled or built itself - never one that was there already, which is the user's
|
|
300
|
+
_fetched: set = set()
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def base_tag() -> str:
|
|
304
|
+
"""The base image's tag on this machine, built or not."""
|
|
305
|
+
return image_tag(dockerfile())
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def present(tag: str) -> bool:
|
|
309
|
+
"""Whether this machine has an image by that tag: a run says it is there
|
|
310
|
+
already, or that it builds it now (``progress``)."""
|
|
311
|
+
if _built.get(tag) or _docker(["image", "inspect", tag], check=False).returncode == 0:
|
|
312
|
+
_built[tag] = True
|
|
313
|
+
return bool(_built.get(tag))
|
|
314
|
+
|
|
315
|
+
|
|
316
|
+
def base_image(log: Optional[str] = None) -> str:
|
|
317
|
+
"""The base image's tag, building it first when this machine has not."""
|
|
318
|
+
text = dockerfile()
|
|
319
|
+
tag = image_tag(text)
|
|
320
|
+
if present(tag):
|
|
321
|
+
return tag
|
|
322
|
+
r = _docker(["build", "--label", "nullius.base=1", "-t", tag, "-"], input=text, timeout=1800, check=False)
|
|
323
|
+
if log:
|
|
324
|
+
with open(log, "a") as f:
|
|
325
|
+
f.write(r.stdout + r.stderr)
|
|
326
|
+
if r.returncode != 0:
|
|
327
|
+
raise Unavailable(f"building {tag}: {(r.stderr or r.stdout).strip()[-300:]}")
|
|
328
|
+
_built[tag] = True
|
|
329
|
+
return tag
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def env_context(bench: dict, name: str, base: str) -> Tuple[str, Dict[str, bytes]]:
|
|
333
|
+
"""An environment's Dockerfile as it is built - ``FROM <base>``, then the
|
|
334
|
+
benchmark's text - and its build context: the files it lists, by their
|
|
335
|
+
path in the benchmark's folder. Nothing else of that folder goes in."""
|
|
336
|
+
e = bench["environments"][name]
|
|
337
|
+
with open(os.path.join(bench["_here"], e["dockerfile"])) as f:
|
|
338
|
+
text = f"FROM {base}\n" + f.read()
|
|
339
|
+
files = {}
|
|
340
|
+
for rel in sorted(e.get("files") or []):
|
|
341
|
+
with open(os.path.join(bench["_here"], rel), "rb") as f:
|
|
342
|
+
files[rel] = f.read()
|
|
343
|
+
return text, files
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def env_tag(bench: dict, name: str, base: str) -> str:
|
|
347
|
+
"""``nullius/env-<benchmark>-<name>:<hash>``: the sha256 of the Dockerfile
|
|
348
|
+
as built (its base's tag included, so a new base is a new layer) and of
|
|
349
|
+
each file it copies, path and bytes."""
|
|
350
|
+
text, files = env_context(bench, name, base)
|
|
351
|
+
h = hashlib.sha256(text.encode())
|
|
352
|
+
for rel, data in files.items():
|
|
353
|
+
h.update(b"\0" + rel.encode() + b"\0" + hashlib.sha256(data).digest())
|
|
354
|
+
return f"nullius/env-{bench['key']}-{name}:{h.hexdigest()[:12]}"
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
_locks: Dict[str, threading.Lock] = {}
|
|
358
|
+
_locks_lock = threading.Lock()
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def env_image(bench: dict, name: str, log_dir: Optional[str] = None) -> str:
|
|
362
|
+
"""An environment's image, built the first time a unit asks for it - so a
|
|
363
|
+
run builds only the environments its units use - and kept by its tag
|
|
364
|
+
after. Two units asking at once wait for one build. Its build output is
|
|
365
|
+
``env-<name>.log`` in ``log_dir`` (the job's ``.work``). A secret is never
|
|
366
|
+
given to a build: an image is shared, a secret is the run's."""
|
|
367
|
+
base = base_image(os.path.join(log_dir, "image.log") if log_dir else None)
|
|
368
|
+
tag = env_tag(bench, name, base)
|
|
369
|
+
with _locks_lock:
|
|
370
|
+
lock = _locks.setdefault(tag, threading.Lock())
|
|
371
|
+
with lock:
|
|
372
|
+
if _built.get(tag) or _docker(["image", "inspect", tag], check=False).returncode == 0:
|
|
373
|
+
_built[tag] = True
|
|
374
|
+
return tag
|
|
375
|
+
text, files = env_context(bench, name, base)
|
|
376
|
+
ctx = tempfile.mkdtemp(prefix="nullius-env-")
|
|
377
|
+
try:
|
|
378
|
+
with open(os.path.join(ctx, "Dockerfile"), "w") as f:
|
|
379
|
+
f.write(text)
|
|
380
|
+
for rel, data in files.items():
|
|
381
|
+
os.makedirs(os.path.dirname(os.path.join(ctx, rel)), exist_ok=True)
|
|
382
|
+
with open(os.path.join(ctx, rel), "wb") as f:
|
|
383
|
+
f.write(data)
|
|
384
|
+
r = _docker(["build", "--label", f"nullius.env={bench['key']}/{name}", "-t", tag, ctx], timeout=3600, check=False)
|
|
385
|
+
finally:
|
|
386
|
+
shutil.rmtree(ctx, ignore_errors=True)
|
|
387
|
+
if log_dir:
|
|
388
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
389
|
+
with open(os.path.join(log_dir, f"env-{name}.log"), "a") as f:
|
|
390
|
+
f.write(f"# {tag}\n" + r.stdout + r.stderr)
|
|
391
|
+
if r.returncode != 0:
|
|
392
|
+
raise Unavailable(f"building the environment {name} ({tag}): {(r.stderr or r.stdout).strip()[-300:]}")
|
|
393
|
+
_built[tag] = True
|
|
394
|
+
return tag
|
|
395
|
+
|
|
396
|
+
|
|
397
|
+
def image_for(bench: dict, unit_setup: Optional[dict], dataset: Optional[dict] = None,
|
|
398
|
+
log_dir: Optional[str] = None) -> str:
|
|
399
|
+
"""The image a unit runs in: the environment its setup names, else its
|
|
400
|
+
dataset's default (``definitions.environment_of``), built on first use;
|
|
401
|
+
the base image when it has none. An environment the benchmark does not
|
|
402
|
+
declare raises ValueError, as does an own image, which the unit's setup
|
|
403
|
+
names itself (:func:`own_image`)."""
|
|
404
|
+
name, e = definitions.environment_of(bench, unit_setup, dataset)
|
|
405
|
+
if name is None:
|
|
406
|
+
return base_image(os.path.join(log_dir, "image.log") if log_dir else None)
|
|
407
|
+
if definitions.own_image(e):
|
|
408
|
+
raise ValueError(f"environment {name!r} is an own image: the unit's setup names it")
|
|
409
|
+
return env_image(bench, name, log_dir)
|
|
410
|
+
|
|
411
|
+
|
|
412
|
+
# -- names, mounts, argument lists -----------------------------------------------
|
|
413
|
+
def slug(job: str) -> str:
|
|
414
|
+
s = re.sub(r"[^A-Za-z0-9_.-]+", "-", job).strip("-.") or "job"
|
|
415
|
+
if len(s) > 80:
|
|
416
|
+
s = s[:72] + "-" + hashlib.sha256(job.encode()).hexdigest()[:7]
|
|
417
|
+
return s
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def names(job: str, tag: str) -> Dict[str, str]:
|
|
421
|
+
base = f"nullius-{slug(job)}-{tag}"
|
|
422
|
+
return {"network": base, "agent": base, "relay": base + "-relay"}
|
|
423
|
+
|
|
424
|
+
|
|
425
|
+
def mcp_mounts(servers: Iterable[dict], trial: str) -> Tuple[List[str], List[str]]:
|
|
426
|
+
"""What a unit's stdio MCP servers need inside the agent's container,
|
|
427
|
+
which starts them: each file their command, arguments or environment
|
|
428
|
+
name (read-only, the file alone - never the folder it sits in, which may
|
|
429
|
+
hold the benchmark's data and answers), and each folder inside the trial
|
|
430
|
+
they will write to (read-write: a server's call log). (rw, ro_files)."""
|
|
431
|
+
rw, ro = [], []
|
|
432
|
+
trial = os.path.abspath(trial)
|
|
433
|
+
for s in servers or []:
|
|
434
|
+
if not isinstance(s, dict):
|
|
435
|
+
continue
|
|
436
|
+
env = s.get("env") or {}
|
|
437
|
+
values = [s.get("command")] + list(s.get("args") or []) + \
|
|
438
|
+
[v for v in (env.values() if isinstance(env, dict) else (e.get("value") for e in env if isinstance(e, dict)))]
|
|
439
|
+
for v in values:
|
|
440
|
+
if not isinstance(v, str) or not os.path.isabs(v):
|
|
441
|
+
continue
|
|
442
|
+
v = os.path.abspath(v)
|
|
443
|
+
if os.path.isfile(v) and not v.startswith(trial + os.sep):
|
|
444
|
+
ro.append(v)
|
|
445
|
+
elif v.startswith(trial + os.sep):
|
|
446
|
+
d = v if os.path.isdir(v) else os.path.dirname(v)
|
|
447
|
+
os.makedirs(d, exist_ok=True)
|
|
448
|
+
rw.append(d)
|
|
449
|
+
return rw, ro
|
|
450
|
+
|
|
451
|
+
|
|
452
|
+
def mounts(rw: Iterable[str], ro: Iterable[str], files: Iterable[str] = ()) -> List[Tuple[str, bool]]:
|
|
453
|
+
"""Each folder once, at its own path; a read-only folder already inside
|
|
454
|
+
another read-only one is not mounted twice. ``files``: single files,
|
|
455
|
+
read-only, unless a folder mounted already holds them."""
|
|
456
|
+
out: List[Tuple[str, bool]] = []
|
|
457
|
+
seen = set()
|
|
458
|
+
for p in rw:
|
|
459
|
+
p = os.path.abspath(p)
|
|
460
|
+
if p not in seen:
|
|
461
|
+
seen.add(p)
|
|
462
|
+
out.append((p, False))
|
|
463
|
+
ros = sorted({os.path.abspath(p) for p in ro if p and os.path.isdir(p)} - seen, key=len)
|
|
464
|
+
kept: List[str] = []
|
|
465
|
+
for p in ros:
|
|
466
|
+
if not any(p == k or p.startswith(k + os.sep) for k in kept):
|
|
467
|
+
kept.append(p)
|
|
468
|
+
out.append((p, True))
|
|
469
|
+
held = [p for p, _ in out]
|
|
470
|
+
for f in sorted({os.path.abspath(f) for f in files if f and os.path.isfile(f)}):
|
|
471
|
+
if not any(f.startswith(h + os.sep) for h in held):
|
|
472
|
+
out.append((f, True))
|
|
473
|
+
return out
|
|
474
|
+
|
|
475
|
+
|
|
476
|
+
_security: Optional[str] = None
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def _security_options() -> str:
|
|
480
|
+
"""The engine's security options (``docker info``), or podman's answer in
|
|
481
|
+
their terms (``name=rootless``): podman has no ``SecurityOptions``. The
|
|
482
|
+
first answer is kept. An engine that does not answer gives none and is
|
|
483
|
+
asked again."""
|
|
484
|
+
global _security
|
|
485
|
+
if _security is None:
|
|
486
|
+
asked = "{{if .Host.Security.Rootless}}name=rootless{{end}}" if podman() else "{{.SecurityOptions}}"
|
|
487
|
+
try:
|
|
488
|
+
r = _docker(["info", "--format", asked], check=False, timeout=30)
|
|
489
|
+
except Unavailable:
|
|
490
|
+
return ""
|
|
491
|
+
if r.returncode != 0:
|
|
492
|
+
return ""
|
|
493
|
+
_security = r.stdout
|
|
494
|
+
return _security
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def rootless() -> bool:
|
|
498
|
+
"""Whether the engine is rootless (Docker or podman): its uid 0 is this
|
|
499
|
+
machine's user, every other uid a sub-uid."""
|
|
500
|
+
return "name=rootless" in _security_options()
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def userns_remap() -> bool:
|
|
504
|
+
"""Whether a rootful Docker remaps its containers' users (``userns-remap``):
|
|
505
|
+
every uid inside, root too, is then a sub-uid of this machine."""
|
|
506
|
+
return "name=userns" in _security_options()
|
|
507
|
+
|
|
508
|
+
|
|
509
|
+
def _keep_id() -> List[str]:
|
|
510
|
+
"""``--userns=keep-id`` on rootless podman, which maps :func:`_user`'s uid
|
|
511
|
+
to this machine's user, as on rootful Docker. Rootless Docker cannot map
|
|
512
|
+
a uid, and runs its containers as root, which is this user there."""
|
|
513
|
+
return ["--userns=keep-id"] if podman() and rootless() else []
|
|
514
|
+
|
|
515
|
+
|
|
516
|
+
def _userns() -> List[str]:
|
|
517
|
+
"""The user namespace of a container that uses this machine's user's files:
|
|
518
|
+
:func:`_keep_id` on podman, ``--userns=host`` on a Docker with
|
|
519
|
+
``userns-remap``, where a container's uids are otherwise sub-uids that
|
|
520
|
+
cannot use those files."""
|
|
521
|
+
if podman():
|
|
522
|
+
return _keep_id()
|
|
523
|
+
return ["--userns=host"] if userns_remap() else []
|
|
524
|
+
|
|
525
|
+
|
|
526
|
+
def make_bridge() -> None:
|
|
527
|
+
"""On podman, make ``BRIDGE`` when it is missing (``--ignore``: asked each
|
|
528
|
+
time, so a network removed between units is made again): a bridge with
|
|
529
|
+
DNS, as podman's default network has DNS off. Docker has its default bridge."""
|
|
530
|
+
if podman():
|
|
531
|
+
_docker(["network", "create", "--ignore", "--label", "nullius.bridge=1", BRIDGE])
|
|
532
|
+
|
|
533
|
+
|
|
534
|
+
def way_out() -> List[str]:
|
|
535
|
+
"""The network of a container that reaches out, or that the relays reach
|
|
536
|
+
(the relay, a licence server's forwarder, the check's proxy, the door to a
|
|
537
|
+
model server): the engine's default, or ``BRIDGE`` on podman
|
|
538
|
+
(:func:`make_bridge`). A rootless podman container starts on pasta, which
|
|
539
|
+
cannot join a network after it starts."""
|
|
540
|
+
return ["--network", BRIDGE] if podman() else []
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _user() -> str:
|
|
544
|
+
"""The ``--user`` that gives a container's files to this machine's user:
|
|
545
|
+
``0:0`` on rootless Docker, ``uid:gid`` otherwise (:func:`_keep_id`)."""
|
|
546
|
+
return "0:0" if rootless() and not podman() else f"{os.getuid()}:{os.getgid()}"
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def _security_opt() -> List[str]:
|
|
550
|
+
"""Every container's ``--security-opt``: no way to gain privileges, and on
|
|
551
|
+
podman no SELinux label. Docker's default daemon (``selinux-enabled`` off)
|
|
552
|
+
gives none, so a container reads the folders it mounts, which are never
|
|
553
|
+
relabelled (``:z`` would relabel the user's). Labels are future work."""
|
|
554
|
+
return ["--security-opt", "no-new-privileges", *(["--security-opt", "label=disable"] if podman() else [])]
|
|
555
|
+
|
|
556
|
+
|
|
557
|
+
def by_value(env: Dict[str, str]) -> Dict[str, str]:
|
|
558
|
+
"""What of a unit's environment podman itself reads (``CLIENT_KEEP``:
|
|
559
|
+
``HOME``, the XDG folders...) but ``PATH``: on podman passed by value
|
|
560
|
+
(``-e NAME=value``), so the podman program keeps this machine's own and
|
|
561
|
+
finds its images. Nothing on Docker."""
|
|
562
|
+
return {k: v for k, v in env.items() if k in CLIENT_KEEP and k != "PATH"} if podman() else {}
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
def program_env(env: Dict[str, str]) -> dict:
|
|
566
|
+
"""The engine program's environment for a unit's process given ``env``:
|
|
567
|
+
the values of what goes by name (``-e NAME``), never ``PATH`` nor
|
|
568
|
+
:func:`by_value`'s."""
|
|
569
|
+
given = by_value(env)
|
|
570
|
+
return client_env({k: v for k, v in env.items() if k != "PATH" and k not in given})
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
def env_args(env: Dict[str, str], path: str) -> Tuple[List[str], List[str]]:
|
|
574
|
+
"""A unit's process's ``-e`` words: ``PATH``, the container's and no
|
|
575
|
+
secret, and :func:`by_value`'s by value; then the rest by name, the value
|
|
576
|
+
in the engine program's own environment (:func:`program_env`), never on
|
|
577
|
+
an argument list."""
|
|
578
|
+
given = by_value(env)
|
|
579
|
+
return (["-e", "PATH=" + path, *[w for k, v in sorted(given.items()) for w in ("-e", f"{k}={v}")]],
|
|
580
|
+
[w for k in sorted(set(env) - {"PATH"} - set(given)) for w in ("-e", k)])
|
|
581
|
+
|
|
582
|
+
|
|
583
|
+
def run_argv(image: str, cmd: List[str], *, name: str, network: str, job: str, workdir: str,
|
|
584
|
+
mounted: List[Tuple[str, bool]], env: Dict[str, str], path: str, interactive: bool = True,
|
|
585
|
+
binds: Iterable[Tuple[str, str]] = ()) -> List[str]:
|
|
586
|
+
"""``docker run`` for a unit's agent or capability, its environment as
|
|
587
|
+
:func:`env_args` gives it. ``binds``: (a file on this machine, its path
|
|
588
|
+
inside), each read-only - a licence file where its environment says the
|
|
589
|
+
software looks for it."""
|
|
590
|
+
values, named = env_args(env, path)
|
|
591
|
+
a = [DOCKER, "run", "--rm", "--name", name, "--network", network, "--label", f"{LABEL}={job}",
|
|
592
|
+
*_userns(), "--user", _user(), "--cap-drop", "ALL", *_security_opt(), "--init",
|
|
593
|
+
"-w", workdir, *values]
|
|
594
|
+
if interactive:
|
|
595
|
+
a.insert(2, "-i")
|
|
596
|
+
for p, ro in mounted:
|
|
597
|
+
a += ["-v", f"{p}:{p}:{'ro' if ro else 'rw'}"]
|
|
598
|
+
for src, dest in binds:
|
|
599
|
+
a += ["-v", f"{src}:{dest}:ro"]
|
|
600
|
+
return a + named + [image] + list(cmd)
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
def plain_only(root: str) -> List[str]:
|
|
604
|
+
"""Remove from ``root`` everything that is not a regular file or a folder -
|
|
605
|
+
links above all - and return what went. What the agent could write (a
|
|
606
|
+
folder mounted into its container, what a check collects out of it) is
|
|
607
|
+
then read and written on this machine, where a link it planted would
|
|
608
|
+
reach any file the user can (verifier/reward.txt -> ~/.bashrc)."""
|
|
609
|
+
gone = []
|
|
610
|
+
for here, dirs, files in os.walk(root, followlinks=False):
|
|
611
|
+
for name in list(dirs) + files:
|
|
612
|
+
path = os.path.join(here, name)
|
|
613
|
+
mode = os.lstat(path).st_mode
|
|
614
|
+
if stat.S_ISREG(mode) or (stat.S_ISDIR(mode) and name in dirs):
|
|
615
|
+
continue
|
|
616
|
+
os.unlink(path)
|
|
617
|
+
gone.append(os.path.relpath(path, root))
|
|
618
|
+
if name in dirs:
|
|
619
|
+
dirs.remove(name)
|
|
620
|
+
return gone
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
CHECK_IMAGE = "NULLIUS_CHECK_IMAGE" # driver checks: the unit's image, or the base image for a local setup
|
|
624
|
+
CHECK_ENGINE = "NULLIUS_CHECK_ENGINE" # beside it on podman: check_start as podman runs from the driver, JSON
|
|
625
|
+
_XDG = (("XDG_DATA_HOME", ".local/share"), ("XDG_CONFIG_HOME", ".config")) # where podman keeps its images and settings
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def check_start() -> List[str]:
|
|
629
|
+
"""A check's container command up to its mounts (:func:`check_argv`): no
|
|
630
|
+
network, no capabilities, as this machine's user."""
|
|
631
|
+
return [DOCKER, "run", "--rm", "--network", "none", *_userns(), "--user", _user(), "--cap-drop", "ALL",
|
|
632
|
+
*_security_opt()]
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def check_env() -> dict:
|
|
636
|
+
"""What a driver, which runs with a home of its own, gets to run
|
|
637
|
+
:func:`check_argv` on the same engine as the units. On Docker: the
|
|
638
|
+
``DOCKER_*`` variables of :func:`client_env`. On podman: the variables of
|
|
639
|
+
``CLIENT_KEEP`` (``CONTAINER_HOST``, ``REGISTRY_AUTH_FILE``...) except ``PATH``, ``HOME``,
|
|
640
|
+
``XDG_DATA_HOME`` and ``XDG_CONFIG_HOME``, and ``CHECK_ENGINE``. That is
|
|
641
|
+
:func:`check_start` behind ``env``, which gives podman this user's XDG data
|
|
642
|
+
and config folders. The driver's own processes do not use these folders."""
|
|
643
|
+
if not podman():
|
|
644
|
+
return {k: v for k, v in client_env().items() if k.startswith("DOCKER")}
|
|
645
|
+
home = os.path.expanduser("~")
|
|
646
|
+
xdg = [f"{k}={os.environ.get(k) or os.path.join(home, d)}" for k, d in _XDG]
|
|
647
|
+
return {**{k: os.environ[k] for k in CLIENT_KEEP if k not in ("PATH", "HOME", *dict(_XDG)) and k in os.environ},
|
|
648
|
+
CHECK_ENGINE: json.dumps(["env", *xdg, *check_start()])}
|
|
649
|
+
|
|
650
|
+
|
|
651
|
+
def check_argv(workdir: str, here: List[str], inside: List[str]) -> List[str]:
|
|
652
|
+
"""With CHECK_IMAGE, run in a throwaway container with no network or
|
|
653
|
+
capabilities and only the workspace mounted. Otherwise return ``here``."""
|
|
654
|
+
image = os.environ.get(CHECK_IMAGE)
|
|
655
|
+
if not image:
|
|
656
|
+
return list(here)
|
|
657
|
+
start = json.loads(os.environ[CHECK_ENGINE]) if os.environ.get(CHECK_ENGINE) else check_start()
|
|
658
|
+
return [*start, "-v", f"{workdir}:{workdir}:rw", "-w", workdir, image, *inside]
|
|
659
|
+
|
|
660
|
+
|
|
661
|
+
# where this machine gets its packages from, passed by name: a proxy's URL may hold its password
|
|
662
|
+
INSTALL_KEEP = ("HTTP_PROXY", "HTTPS_PROXY", "NO_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "no_proxy", "all_proxy",
|
|
663
|
+
"UV_INDEX", "UV_INDEX_URL", "UV_DEFAULT_INDEX", "UV_EXTRA_INDEX_URL", "UV_PYTHON_INSTALL_MIRROR",
|
|
664
|
+
"PIP_INDEX_URL", "PIP_EXTRA_INDEX_URL", "npm_config_registry", "NPM_CONFIG_REGISTRY", "HF_ENDPOINT")
|
|
665
|
+
|
|
666
|
+
|
|
667
|
+
def install_argv(image: str, where: str, cmd: List[str], env: Optional[dict] = None) -> List[str]:
|
|
668
|
+
"""``docker run`` for an install step (``install.packages``' ``run_in``):
|
|
669
|
+
in the base image, as this machine's user, with the install folder alone
|
|
670
|
+
mounted, at its own path. The home and npm's and uv's caches are inside
|
|
671
|
+
it, removed after (``install.SCRATCH``). uv's Python is inside it too
|
|
672
|
+
(``.python``), and kept: the environment links to it, and the agent's
|
|
673
|
+
container mounts the folder. ``env``: the step's own variables. This
|
|
674
|
+
machine's proxy and package indexes (``INSTALL_KEEP``) are passed by name.
|
|
675
|
+
It has the network - it downloads the packages - and nothing of a run: no
|
|
676
|
+
key, no unit."""
|
|
677
|
+
home, npm, uv = (f"{where}/{s}" for s in install.SCRATCH)
|
|
678
|
+
env = {"HOME": home, "npm_config_cache": npm, "UV_CACHE_DIR": uv,
|
|
679
|
+
"UV_PYTHON_INSTALL_DIR": f"{where}/.python", "UV_PYTHON_DOWNLOADS": "automatic", **(env or {})}
|
|
680
|
+
return [DOCKER, "run", "--rm", "--label", "nullius.install=1", *_userns(), "--user", _user(), "--cap-drop", "ALL",
|
|
681
|
+
*_security_opt(), "-v", f"{where}:{where}:rw", "-w", where,
|
|
682
|
+
*[w for k, v in env.items() for w in ("-e", f"{k}={v}")],
|
|
683
|
+
*[w for k in INSTALL_KEEP if k in os.environ for w in ("-e", k)], image] + list(cmd)
|
|
684
|
+
|
|
685
|
+
|
|
686
|
+
def relay_argv(image: str, *, name: str, job: str, app: str, record: str, key_file: str, base: str,
|
|
687
|
+
model: Optional[str], routes: Dict[str, str], prices: Optional[dict] = None, side_models=(),
|
|
688
|
+
key_header: str = "authorization", hosts: Optional[Dict[str, str]] = None,
|
|
689
|
+
ledger: Optional[str] = None) -> List[str]:
|
|
690
|
+
"""``docker run`` for a unit's relay: on the default bridge (its way to the
|
|
691
|
+
provider), its port published on 127.0.0.1 only. The key is never given:
|
|
692
|
+
``key_file``, its file, is mounted read-only at ``RELAY_KEY_IN`` with
|
|
693
|
+
``--mount``, which fails when the engine cannot see it (``-v`` would mount
|
|
694
|
+
an empty folder). ``hosts``: names it resolves
|
|
695
|
+
(``--add-host``), the machine's own for an endpoint on its loopback.
|
|
696
|
+
``ledger``: the file of the run's spending cap, its folder mounted so
|
|
697
|
+
this relay books its calls where every other relay of the job does."""
|
|
698
|
+
a = [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_userns(), "--user", _user(),
|
|
699
|
+
"--cap-drop", "ALL", *_security_opt(), "--init",
|
|
700
|
+
*[w for n, ip in sorted((hosts or {}).items()) for w in ("--add-host", f"{n}:{ip}")],
|
|
701
|
+
"-p", f"127.0.0.1::{RELAY_PORT}", "-v", f"{app}:{app}:ro", "-v", f"{record}:{record}:rw",
|
|
702
|
+
*(["-v", f"{os.path.dirname(ledger)}:{os.path.dirname(ledger)}:rw"] if ledger else []),
|
|
703
|
+
"--mount", f"type=bind,source={key_file},target={RELAY_KEY_IN},readonly", "-w", app,
|
|
704
|
+
"-e", "PYTHONPATH=" + app, "-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
705
|
+
image, "python3", "-m", "nullius.standard.relay", "serve", "--listen", f"0.0.0.0:{RELAY_PORT}",
|
|
706
|
+
"--base", base, "--record", record, "--key-file", RELAY_KEY_IN, "--public", f"http://{RELAY_ALIAS}:{RELAY_PORT}"]
|
|
707
|
+
if model:
|
|
708
|
+
a += ["--model", model]
|
|
709
|
+
for tag, m in sorted(routes.items()):
|
|
710
|
+
a += ["--route", f"{tag}={m}"]
|
|
711
|
+
if prices:
|
|
712
|
+
a += ["--prices", json.dumps(prices, separators=(",", ":"))]
|
|
713
|
+
if ledger:
|
|
714
|
+
a += ["--ledger", ledger]
|
|
715
|
+
for m in side_models:
|
|
716
|
+
a += ["--side-model", m]
|
|
717
|
+
if key_header != "authorization":
|
|
718
|
+
a += ["--key-header", key_header]
|
|
719
|
+
return a
|
|
720
|
+
|
|
721
|
+
|
|
722
|
+
_IPV4 = re.compile(r"^\d{1,3}(?:\.\d{1,3}){3}$")
|
|
723
|
+
|
|
724
|
+
|
|
725
|
+
def on_loopback(base: str) -> bool:
|
|
726
|
+
"""Whether a model endpoint is on this machine's loopback (a local model
|
|
727
|
+
server: ``http://localhost:11434/v1``), which a container cannot reach as
|
|
728
|
+
written - there, 127.0.0.1 is the container itself."""
|
|
729
|
+
host = (urllib.parse.urlsplit(base or "").hostname or "").lower()
|
|
730
|
+
return host == "localhost" or host == "::1" or host.startswith("127.")
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
def bridge_gateway() -> str:
|
|
734
|
+
"""This machine's address on docker's default bridge, or ""."""
|
|
735
|
+
r = _docker(["network", "inspect", "bridge", "-f", "{{range .IPAM.Config}}{{.Gateway}} {{end}}"], check=False)
|
|
736
|
+
return next((w for w in r.stdout.split() if _IPV4.match(w)), "")
|
|
737
|
+
|
|
738
|
+
|
|
739
|
+
def _pump(src: socket.socket, dst: socket.socket) -> None:
|
|
740
|
+
try:
|
|
741
|
+
while data := src.recv(65536):
|
|
742
|
+
dst.sendall(data)
|
|
743
|
+
except OSError:
|
|
744
|
+
pass
|
|
745
|
+
finally:
|
|
746
|
+
for sk in (src, dst):
|
|
747
|
+
try:
|
|
748
|
+
sk.shutdown(socket.SHUT_RDWR)
|
|
749
|
+
except OSError:
|
|
750
|
+
pass
|
|
751
|
+
|
|
752
|
+
|
|
753
|
+
def _pass_each(server: socket.socket, target: Tuple[str, int]) -> None:
|
|
754
|
+
"""Pass each connection ``server`` accepts to ``target``, until it is closed."""
|
|
755
|
+
while True:
|
|
756
|
+
try:
|
|
757
|
+
inside, _ = server.accept()
|
|
758
|
+
except OSError:
|
|
759
|
+
return
|
|
760
|
+
try:
|
|
761
|
+
outside = socket.create_connection(target, timeout=10)
|
|
762
|
+
outside.settimeout(None)
|
|
763
|
+
except OSError:
|
|
764
|
+
inside.close()
|
|
765
|
+
continue
|
|
766
|
+
for a, b in ((inside, outside), (outside, inside)):
|
|
767
|
+
threading.Thread(target=_pump, args=(a, b), daemon=True).start()
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _close(server: socket.socket) -> None:
|
|
771
|
+
"""Close a listening socket. The shutdown wakes the thread blocked in its
|
|
772
|
+
``accept``, which a close alone does not do on Linux."""
|
|
773
|
+
try:
|
|
774
|
+
server.shutdown(socket.SHUT_RDWR)
|
|
775
|
+
except OSError:
|
|
776
|
+
pass
|
|
777
|
+
server.close()
|
|
778
|
+
|
|
779
|
+
|
|
780
|
+
def use_socket_doors() -> bool:
|
|
781
|
+
"""Whether a server on this machine's loopback is reached through a
|
|
782
|
+
:class:`SocketDoor`: on a rootless engine on Linux, or on podman there,
|
|
783
|
+
whose bridge has no address on this machine until a container joins it.
|
|
784
|
+
Elsewhere the engine runs in a virtual machine, which this machine's
|
|
785
|
+
sockets may not reach."""
|
|
786
|
+
return (rootless() or podman()) and platform.system() == "Linux"
|
|
787
|
+
|
|
788
|
+
|
|
789
|
+
class SocketDoor:
|
|
790
|
+
"""A Unix socket in a folder of its own that passes each connection to one
|
|
791
|
+
``host:port`` here: a rootless engine's door container mounts it to reach
|
|
792
|
+
this machine's loopback. ``stop`` closes it and removes the folder."""
|
|
793
|
+
|
|
794
|
+
def __init__(self, host: str, port: int):
|
|
795
|
+
self.dir = tempfile.mkdtemp(prefix="nullius-door-")
|
|
796
|
+
self.path, self.target = os.path.join(self.dir, "s"), (host, port) # short: a socket path has ~108 bytes
|
|
797
|
+
self._server: Optional[socket.socket] = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
|
798
|
+
try:
|
|
799
|
+
self._server.bind(self.path)
|
|
800
|
+
self._server.listen(64)
|
|
801
|
+
except OSError:
|
|
802
|
+
self.stop()
|
|
803
|
+
raise
|
|
804
|
+
threading.Thread(target=_pass_each, args=(self._server, self.target), daemon=True).start()
|
|
805
|
+
|
|
806
|
+
def stop(self) -> None:
|
|
807
|
+
server, self._server = self._server, None
|
|
808
|
+
if server is not None:
|
|
809
|
+
_close(server)
|
|
810
|
+
shutil.rmtree(self.dir, ignore_errors=True)
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
class Loopback:
|
|
814
|
+
"""An endpoint on this machine's loopback, as the relays' containers reach
|
|
815
|
+
it: ``base`` and the ``hosts`` they are given. On Linux, a forwarder for
|
|
816
|
+
the run, listening on the default bridge's gateway alone - the address a
|
|
817
|
+
container there has for this machine, reached by the relays and by no
|
|
818
|
+
agent, whose networks are internal - and passing each connection to the
|
|
819
|
+
endpoint; the relays call it ``host.docker.internal``. Where the bridge
|
|
820
|
+
is not on this machine and the engine runs as root (Docker Desktop's
|
|
821
|
+
virtual machine), that name already reaches its loopback: only the name
|
|
822
|
+
changes. On Linux, a rootless engine's gateway is in its own network
|
|
823
|
+
namespace, and podman's bridge has no address here until a container
|
|
824
|
+
joins it, so a :class:`SocketDoor` (``door``) passes each connection
|
|
825
|
+
instead, and one door container for the run reaches it, on the engine's
|
|
826
|
+
default network as the relays are. The relays call it
|
|
827
|
+
``host.docker.internal`` there too, at the door's address, on port 8000."""
|
|
828
|
+
|
|
829
|
+
NAME = "host.docker.internal"
|
|
830
|
+
DOOR_PORT = 8000 # not the endpoint's: below 1024 needs a capability on podman
|
|
831
|
+
|
|
832
|
+
def __init__(self, base: str, *, image: str = "", job: str = ""):
|
|
833
|
+
self.target = urllib.parse.urlsplit(base)
|
|
834
|
+
self.base, self.hosts = base, {}
|
|
835
|
+
self.image, self.job = image, job # the door container's (use_socket_doors)
|
|
836
|
+
self._server: Optional[socket.socket] = None
|
|
837
|
+
self.door: Optional[SocketDoor] = None
|
|
838
|
+
self.container: Optional[str] = None
|
|
839
|
+
|
|
840
|
+
def _url(self, port: Optional[int]) -> str:
|
|
841
|
+
t = self.target
|
|
842
|
+
netloc = (t.netloc.rpartition("@")[0] + "@" if "@" in t.netloc else "") + self.NAME + (f":{port}" if port else "")
|
|
843
|
+
return urllib.parse.urlunsplit((t.scheme, netloc, t.path, t.query, t.fragment))
|
|
844
|
+
|
|
845
|
+
def start(self) -> "Loopback":
|
|
846
|
+
if use_socket_doors():
|
|
847
|
+
self.door = SocketDoor(*self._endpoint())
|
|
848
|
+
self.container = f"nullius-{slug(self.job)}-model"
|
|
849
|
+
try:
|
|
850
|
+
make_bridge()
|
|
851
|
+
_docker(door_argv(self.image, name=self.container, job=self.job, port=self.DOOR_PORT,
|
|
852
|
+
socket_path=self.door.path)[1:],
|
|
853
|
+
env=client_env({"NULLIUS_FORWARD_TO": "unix:" + self.door.path}))
|
|
854
|
+
wait_listening(self.container, "the door to the model server")
|
|
855
|
+
ip = _docker(["inspect", "-f", "{{range .NetworkSettings.Networks}}{{.IPAddress}} {{end}}",
|
|
856
|
+
self.container]).stdout.split()
|
|
857
|
+
if not ip:
|
|
858
|
+
raise Unavailable("the door to the model server has no address on the default network")
|
|
859
|
+
except Exception:
|
|
860
|
+
self.stop()
|
|
861
|
+
raise
|
|
862
|
+
self.base, self.hosts = self._url(self.DOOR_PORT), {self.NAME: ip[0]}
|
|
863
|
+
return self
|
|
864
|
+
gateway = bridge_gateway()
|
|
865
|
+
try:
|
|
866
|
+
if not gateway:
|
|
867
|
+
raise OSError("no bridge gateway")
|
|
868
|
+
self._server = socket.create_server((gateway, 0))
|
|
869
|
+
except OSError:
|
|
870
|
+
self.base, self.hosts = self._url(self.target.port), {self.NAME: "host-gateway"}
|
|
871
|
+
return self
|
|
872
|
+
self.base, self.hosts = self._url(self._server.getsockname()[1]), {self.NAME: gateway}
|
|
873
|
+
threading.Thread(target=_pass_each, args=(self._server, self._endpoint()), daemon=True).start()
|
|
874
|
+
return self
|
|
875
|
+
|
|
876
|
+
def _endpoint(self) -> Tuple[str, int]:
|
|
877
|
+
t = self.target
|
|
878
|
+
return t.hostname, t.port or (443 if t.scheme == "https" else 80)
|
|
879
|
+
|
|
880
|
+
def stop(self) -> None:
|
|
881
|
+
server, self._server = self._server, None
|
|
882
|
+
if server is not None:
|
|
883
|
+
_close(server)
|
|
884
|
+
name, self.container = self.container, None
|
|
885
|
+
if name is not None:
|
|
886
|
+
try:
|
|
887
|
+
_docker(["rm", "-f", name], check=False, timeout=60)
|
|
888
|
+
except Unavailable:
|
|
889
|
+
pass
|
|
890
|
+
if self.door is not None:
|
|
891
|
+
self.door.stop()
|
|
892
|
+
|
|
893
|
+
|
|
894
|
+
def server_alias(address: str, i: int = 1) -> Tuple[str, str, int]:
|
|
895
|
+
"""(the name the agent reaches it by, host, port) for a licence server
|
|
896
|
+
``host:port``: its own name, or ``licence-server-<i>`` for an IPv4
|
|
897
|
+
address, which cannot be a network alias."""
|
|
898
|
+
host, _, port = address.strip().rpartition(":")
|
|
899
|
+
return (f"licence-server-{i}" if _IPV4.match(host) else host), host, int(port)
|
|
900
|
+
|
|
901
|
+
|
|
902
|
+
def forward_argv(image: str, *, name: str, job: str, port: int, nonce_file: Optional[str] = None) -> List[str]:
|
|
903
|
+
"""``docker run`` for a licence server's forwarder: on the default bridge
|
|
904
|
+
(``BRIDGE`` on podman), its one file mounted, listening on the server's
|
|
905
|
+
port. Where it forwards is named, never given: ``-e NULLIUS_FORWARD_TO``.
|
|
906
|
+
``host.docker.internal`` is the machine the run is on, for a server there
|
|
907
|
+
(where :func:`use_socket_doors`, ``Unit._forward`` gives such a server a
|
|
908
|
+
door instead). A macOS door mounts its nonce read-only."""
|
|
909
|
+
mount = ["--mount", f"type=bind,source={nonce_file},target={FORWARD_NONCE_IN},readonly"] if nonce_file else []
|
|
910
|
+
return [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_keep_id(), "--user", _user(),
|
|
911
|
+
"--cap-drop", "ALL", *_security_opt(), "--init",
|
|
912
|
+
"--add-host", "host.docker.internal:host-gateway", "-v", f"{FORWARD}:{FORWARD_IN}:ro", *mount,
|
|
913
|
+
"-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "NULLIUS_FORWARD_TO",
|
|
914
|
+
image, "python3", FORWARD_IN, "--listen", f"0.0.0.0:{port}",
|
|
915
|
+
*(["--nonce-file", FORWARD_NONCE_IN] if nonce_file else [])]
|
|
916
|
+
|
|
917
|
+
|
|
918
|
+
def door_argv(image: str, *, name: str, job: str, port: int, socket_path: str,
|
|
919
|
+
network: Optional[str] = None, alias: Optional[str] = None) -> List[str]:
|
|
920
|
+
"""``docker run`` for a door to a server on this machine (``mcp_machine``,
|
|
921
|
+
a :class:`SocketDoor`): on one internal network alone, as ``alias``, or
|
|
922
|
+
without ``network`` on the default one (``BRIDGE`` on podman: a
|
|
923
|
+
:class:`Loopback`'s), its one
|
|
924
|
+
file and the server's socket mounted, each alone and read-only. Where it
|
|
925
|
+
forwards is named, never given: ``-e NULLIUS_FORWARD_TO``
|
|
926
|
+
(``unix:<socket>``)."""
|
|
927
|
+
return [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *_userns(), "--user", _user(),
|
|
928
|
+
"--cap-drop", "ALL", *_security_opt(), "--init",
|
|
929
|
+
*(["--network", network, "--network-alias", alias] if network else way_out()),
|
|
930
|
+
"-v", f"{FORWARD}:{FORWARD_IN}:ro", "-v", f"{socket_path}:{socket_path}:ro",
|
|
931
|
+
"-e", "PYTHONDONTWRITEBYTECODE=1", "-e", "NULLIUS_FORWARD_TO",
|
|
932
|
+
image, "python3", FORWARD_IN, "--listen", f"0.0.0.0:{port}"]
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def proxy_env(direct: Iterable[str] = ()) -> Dict[str, str]:
|
|
936
|
+
"""The variables that send HTTP(S) through the ``egress`` proxy; this
|
|
937
|
+
machine's loopback and ``direct`` (the unit's relay, doors and servers) are
|
|
938
|
+
reached as they are."""
|
|
939
|
+
proxy = f"http://{EGRESS_ALIAS}:{EGRESS_PORT}"
|
|
940
|
+
local = ",".join(dict.fromkeys(["localhost", "127.0.0.1", "::1", *direct]))
|
|
941
|
+
return {"HTTP_PROXY": proxy, "HTTPS_PROXY": proxy, "http_proxy": proxy, "https_proxy": proxy,
|
|
942
|
+
"NO_PROXY": local, "no_proxy": local}
|
|
943
|
+
|
|
944
|
+
|
|
945
|
+
def egress_argv(image: str, *, name: str, job: str) -> List[str]:
|
|
946
|
+
"""``docker run`` for an own image's check's proxy (``egress.py``): on the
|
|
947
|
+
default bridge (``BRIDGE`` on podman), its one file mounted, nothing else given."""
|
|
948
|
+
return [DOCKER, "run", "-d", "--name", name, "--label", f"{LABEL}={job}", *way_out(), *_keep_id(), "--user", _user(),
|
|
949
|
+
"--cap-drop", "ALL", *_security_opt(), "--init",
|
|
950
|
+
"-v", f"{EGRESS}:{EGRESS_IN}:ro", "-e", "PYTHONDONTWRITEBYTECODE=1",
|
|
951
|
+
image, "python3", EGRESS_IN, "--listen", f"0.0.0.0:{EGRESS_PORT}"]
|
|
952
|
+
|
|
953
|
+
|
|
954
|
+
def wait_listening(name: str, what: str, timeout: float = 30):
|
|
955
|
+
"""Until a sidecar says ``listening`` on its output; Unavailable when it
|
|
956
|
+
ends first, or does not in ``timeout``."""
|
|
957
|
+
end = time.time() + timeout
|
|
958
|
+
while time.time() < end:
|
|
959
|
+
r = _docker(["logs", name], check=False)
|
|
960
|
+
if '"listening"' in r.stdout:
|
|
961
|
+
return
|
|
962
|
+
state = _docker(["inspect", "-f", "{{.State.Running}}", name], check=False).stdout.strip()
|
|
963
|
+
if state == "false":
|
|
964
|
+
raise Unavailable(f"{what} ended: " + (r.stdout + r.stderr).strip()[-300:])
|
|
965
|
+
time.sleep(0.2)
|
|
966
|
+
raise Unavailable(f"{what} did not listen in {int(timeout)} s")
|
|
967
|
+
|
|
968
|
+
|
|
969
|
+
class Unit:
|
|
970
|
+
"""One unit's network and relay; what wraps its agent's command.
|
|
971
|
+
|
|
972
|
+
It answers as a relay does - ``url`` (the host's way in, for the driver),
|
|
973
|
+
``calls``, ``stop`` - and ``inside``, the address the agent is given."""
|
|
974
|
+
|
|
975
|
+
def __init__(self, job: str, tag: str, record_dir: str, *, image: str, app: str, base: str, key: str,
|
|
976
|
+
model: Optional[str] = None, routes: Optional[Dict[str, str]] = None, prices: Optional[dict] = None,
|
|
977
|
+
side_models=(), key_header: str = "authorization", servers: Iterable[str] = (),
|
|
978
|
+
doors: Iterable[Tuple[str, int, str]] = (), hosts: Optional[Dict[str, str]] = None,
|
|
979
|
+
ledger: Optional[str] = None, network_probe: Optional[str] = None, probe_now: bool = True):
|
|
980
|
+
self.job, self.tag, self.dir, self.image, self.app = job, tag, record_dir, image, app
|
|
981
|
+
self.ledger = ledger # the file of the run's spending cap, when it has one
|
|
982
|
+
self.servers = list(servers) # licence servers, host:port each: a forwarder apiece
|
|
983
|
+
self.doors = list(doors) # servers on this machine: (alias, port, socket), a door apiece
|
|
984
|
+
self.side_models, self.key_header = tuple(side_models), key_header
|
|
985
|
+
self.base, self.key, self.model, self.routes, self.prices = base, key, model, dict(routes or {}), prices
|
|
986
|
+
self.hosts = dict(hosts or {})
|
|
987
|
+
self.names = names(job, tag)
|
|
988
|
+
self.host: Optional[str] = None
|
|
989
|
+
self.socket_bridges = []
|
|
990
|
+
self.started: List[str] = [] # containers to remove
|
|
991
|
+
self.network = False
|
|
992
|
+
self.key_dir: Optional[str] = None # its secret files' folder, until what reads them listens
|
|
993
|
+
self.network_probe = network_probe
|
|
994
|
+
self.probe_now = probe_now # the run's first unit checks the proxy route; the rest trust it
|
|
995
|
+
self.network_env = {}
|
|
996
|
+
self.socket_doors: List[SocketDoor] = []
|
|
997
|
+
os.makedirs(record_dir, exist_ok=True)
|
|
998
|
+
|
|
999
|
+
def start(self) -> "Unit":
|
|
1000
|
+
try:
|
|
1001
|
+
make_bridge()
|
|
1002
|
+
_docker(["network", "create", "--internal", "--label", f"{LABEL}={self.job}", self.names["network"]])
|
|
1003
|
+
self.network = True
|
|
1004
|
+
argv = relay_argv(self.image, name=self.names["relay"], job=self.job, app=self.app, record=self.dir,
|
|
1005
|
+
key_file=self._key_file(), base=self.base, model=self.model, routes=self.routes,
|
|
1006
|
+
prices=self.prices, side_models=self.side_models, key_header=self.key_header,
|
|
1007
|
+
hosts=self.hosts, ledger=self.ledger)
|
|
1008
|
+
self.started.append(self.names["relay"])
|
|
1009
|
+
_docker(argv[1:])
|
|
1010
|
+
_docker(["network", "connect", "--alias", RELAY_ALIAS, self.names["network"], self.names["relay"]])
|
|
1011
|
+
self._wait(self.names["relay"], "the relay's container")
|
|
1012
|
+
self._forget_key()
|
|
1013
|
+
port = _docker(["port", self.names["relay"], f"{RELAY_PORT}/tcp"]).stdout.strip().splitlines()[0]
|
|
1014
|
+
self.host = "http://" + port.replace("0.0.0.0", "127.0.0.1")
|
|
1015
|
+
self.reach = {}
|
|
1016
|
+
for i, address in enumerate(self.servers, 1):
|
|
1017
|
+
self.reach[address] = self._forward(i, address)
|
|
1018
|
+
for i, (alias, port, sock) in enumerate(self.doors, 1):
|
|
1019
|
+
self._door(f"{self.names['agent']}-mcp{i}", alias, port, sock, "the door to the benchmark's MCP server")
|
|
1020
|
+
if self.network_probe:
|
|
1021
|
+
self._online()
|
|
1022
|
+
except Exception:
|
|
1023
|
+
self.stop()
|
|
1024
|
+
raise
|
|
1025
|
+
return self
|
|
1026
|
+
|
|
1027
|
+
def direct(self) -> List[str]:
|
|
1028
|
+
"""What the agent reaches without the public web proxy: its relay, its
|
|
1029
|
+
doors and its licence servers' forwarders."""
|
|
1030
|
+
return [RELAY_ALIAS, *[alias for alias, _, _ in self.doors],
|
|
1031
|
+
*[address.rsplit(":", 1)[0] for address in getattr(self, "reach", {}).values()]]
|
|
1032
|
+
|
|
1033
|
+
def _key_file(self, name: str = "key", text: Optional[str] = None) -> str:
|
|
1034
|
+
"""Write the key, or ``text``, to an owner-only file in the unit's key
|
|
1035
|
+
folder, a new one under :func:`key_root` when it has none, and return
|
|
1036
|
+
its path. Not the record folder: it is kept."""
|
|
1037
|
+
if not self.key_dir:
|
|
1038
|
+
self.key_dir = key_folder(self.job)
|
|
1039
|
+
return secret_file(os.path.join(self.key_dir, name), self.key if text is None else text)
|
|
1040
|
+
|
|
1041
|
+
def _forget_key(self):
|
|
1042
|
+
if self.key_dir:
|
|
1043
|
+
shutil.rmtree(self.key_dir, ignore_errors=True)
|
|
1044
|
+
self.key_dir = None
|
|
1045
|
+
|
|
1046
|
+
def _online(self):
|
|
1047
|
+
"""Opt-in public HTTP(S), with private destinations blocked by the proxy."""
|
|
1048
|
+
name = self.names["agent"] + "-egress"
|
|
1049
|
+
self.started.append(name)
|
|
1050
|
+
_docker(egress_argv(self.image, name=name, job=self.job)[1:])
|
|
1051
|
+
_docker(["network", "connect", "--alias", EGRESS_ALIAS, self.names["network"], name])
|
|
1052
|
+
self._wait(name, "the agent's public web proxy")
|
|
1053
|
+
self.network_env = proxy_env(self.direct())
|
|
1054
|
+
if not self.probe_now:
|
|
1055
|
+
return
|
|
1056
|
+
# Check the actual network path, without mounts or credentials.
|
|
1057
|
+
code = ("import sys,urllib.request; "
|
|
1058
|
+
"r=urllib.request.urlopen(urllib.request.Request(sys.argv[1], headers={'User-Agent': 'Nullius-connectivity-check'}), timeout=10); "
|
|
1059
|
+
"assert 200 <= r.status < 400")
|
|
1060
|
+
probe = self.names["agent"] + "-network-check"
|
|
1061
|
+
# Docker startup competes with other units for daemon/network locks.
|
|
1062
|
+
# Its deadline must not consume the HTTP check's own execution budget.
|
|
1063
|
+
# Track the name before starting so cancellation also removes it.
|
|
1064
|
+
self.started.append(probe)
|
|
1065
|
+
args = ["run", "-d", "--name", probe, "--label", f"{LABEL}={self.job}",
|
|
1066
|
+
"--init", "--network", self.names["network"],
|
|
1067
|
+
"--cap-drop", "ALL", "--security-opt", "no-new-privileges"]
|
|
1068
|
+
for key in self.network_env:
|
|
1069
|
+
args += ["-e", key]
|
|
1070
|
+
try:
|
|
1071
|
+
_docker(args + [self.image, "python3", "-c", code, self.network_probe],
|
|
1072
|
+
env=client_env(self.network_env), timeout=120)
|
|
1073
|
+
result = _docker(["wait", probe], timeout=30)
|
|
1074
|
+
if result.stdout.strip() != "0":
|
|
1075
|
+
logs = _docker(["logs", probe], check=False, timeout=10)
|
|
1076
|
+
raise Unavailable((logs.stderr or logs.stdout).strip()[-500:] or "probe exited unsuccessfully")
|
|
1077
|
+
except Unavailable as error:
|
|
1078
|
+
raise Unavailable(f"the public HTTPS connectivity check: {error}") from error
|
|
1079
|
+
|
|
1080
|
+
def _forward(self, i: int, address: str) -> str:
|
|
1081
|
+
"""A licence server's forwarder, started and on the unit's network
|
|
1082
|
+
under the server's name: the address the agent reaches it by. It joins
|
|
1083
|
+
that network once it listens, after it resolves the server's name. One
|
|
1084
|
+
at ``host.docker.internal``, where :func:`use_socket_doors`, is a door
|
|
1085
|
+
to a :class:`SocketDoor` instead, as ``licence-server-<i>``."""
|
|
1086
|
+
alias, host, port = server_alias(address, i)
|
|
1087
|
+
name = f"{self.names['agent']}-lic{i}"
|
|
1088
|
+
if use_socket_doors() and host == Loopback.NAME:
|
|
1089
|
+
door = SocketDoor("127.0.0.1", port)
|
|
1090
|
+
self.socket_doors.append(door)
|
|
1091
|
+
alias = f"licence-server-{i}"
|
|
1092
|
+
self._door(name, alias, port, door.path, "the licence server's forwarder")
|
|
1093
|
+
return f"{alias}:{port}"
|
|
1094
|
+
self.started.append(name)
|
|
1095
|
+
_docker(forward_argv(self.image, name=name, job=self.job, port=port)[1:],
|
|
1096
|
+
env=client_env({"NULLIUS_FORWARD_TO": f"{host}:{port}"}))
|
|
1097
|
+
self._wait(name, "the licence server's forwarder")
|
|
1098
|
+
_docker(["network", "connect", "--alias", alias, self.names["network"], name])
|
|
1099
|
+
return f"{alias}:{port}"
|
|
1100
|
+
|
|
1101
|
+
def _door(self, name: str, alias: str, port: int, sock: str, what: str) -> None:
|
|
1102
|
+
"""A door to a server on this machine, on the unit's network alone. On
|
|
1103
|
+
macOS, a forwarder to a loopback TCP bridge here that serves a
|
|
1104
|
+
connection only after the door's nonce line, given as a key file."""
|
|
1105
|
+
self.started.append(name)
|
|
1106
|
+
if platform.system() == "Darwin":
|
|
1107
|
+
# Docker Desktop cannot connect to host Unix sockets through a
|
|
1108
|
+
# file mount. Expose only this socket via a loopback TCP bridge.
|
|
1109
|
+
from nullius.standard import forward
|
|
1110
|
+
nonce = token_hex(32)
|
|
1111
|
+
class Handler(socketserver.BaseRequestHandler):
|
|
1112
|
+
def handle(self):
|
|
1113
|
+
self.request.settimeout(5)
|
|
1114
|
+
expected = (nonce + "\n").encode()
|
|
1115
|
+
received = b""
|
|
1116
|
+
try:
|
|
1117
|
+
while len(received) < len(expected):
|
|
1118
|
+
chunk = self.request.recv(len(expected) - len(received))
|
|
1119
|
+
if not chunk:
|
|
1120
|
+
return
|
|
1121
|
+
received += chunk
|
|
1122
|
+
except OSError:
|
|
1123
|
+
return
|
|
1124
|
+
if not compare_digest(received, expected):
|
|
1125
|
+
return
|
|
1126
|
+
self.request.settimeout(None)
|
|
1127
|
+
forward._serve(self.request, [(socket.AF_UNIX, sock)])
|
|
1128
|
+
bridge = socketserver.ThreadingTCPServer(("127.0.0.1", 0), Handler)
|
|
1129
|
+
bridge.daemon_threads = True
|
|
1130
|
+
self.socket_bridges.append(bridge)
|
|
1131
|
+
threading.Thread(target=bridge.serve_forever, daemon=True).start()
|
|
1132
|
+
_docker(forward_argv(self.image, name=name, job=self.job, port=port,
|
|
1133
|
+
nonce_file=self._key_file(name, nonce))[1:],
|
|
1134
|
+
env=client_env({"NULLIUS_FORWARD_TO": f"host.docker.internal:{bridge.server_address[1]}"}))
|
|
1135
|
+
_docker(["network", "connect", "--alias", alias, self.names["network"], name])
|
|
1136
|
+
else:
|
|
1137
|
+
_docker(door_argv(self.image, name=name, job=self.job, network=self.names["network"], alias=alias,
|
|
1138
|
+
port=port, socket_path=sock)[1:],
|
|
1139
|
+
env=client_env({"NULLIUS_FORWARD_TO": "unix:" + sock}))
|
|
1140
|
+
self._wait(name, what)
|
|
1141
|
+
self._forget_key() # read before it listens
|
|
1142
|
+
|
|
1143
|
+
def _wait(self, name: str, what: str, timeout: float = 30):
|
|
1144
|
+
wait_listening(name, what, timeout)
|
|
1145
|
+
|
|
1146
|
+
def url(self, tag: str, model: Optional[str] = None) -> str:
|
|
1147
|
+
"""The host's way to a route. A route's model is fixed when the relay
|
|
1148
|
+
starts (``routes``); asking for another is an error, not a silent switch."""
|
|
1149
|
+
if model and self.routes.get(tag) != model:
|
|
1150
|
+
raise ValueError(f"route {tag} was not started as {model}")
|
|
1151
|
+
return f"{self.host}/{tag}/v1"
|
|
1152
|
+
|
|
1153
|
+
def inside(self, tag: str) -> str:
|
|
1154
|
+
return f"http://{RELAY_ALIAS}:{RELAY_PORT}/{tag}/v1"
|
|
1155
|
+
|
|
1156
|
+
def calls(self, tag: str) -> List[dict]:
|
|
1157
|
+
try:
|
|
1158
|
+
with open(os.path.join(self.dir, tag + ".jsonl")) as f:
|
|
1159
|
+
return [json.loads(l) for l in f if l.strip()]
|
|
1160
|
+
except OSError:
|
|
1161
|
+
return []
|
|
1162
|
+
|
|
1163
|
+
def wrap(self, cmd: List[str], env: dict, workdir: str, mounted: List[Tuple[str, bool]], *,
|
|
1164
|
+
image: Optional[str] = None, interactive: bool = True,
|
|
1165
|
+
binds: Iterable[Tuple[str, str]] = ()) -> Tuple[List[str], dict]:
|
|
1166
|
+
"""A command and its environment turned into ``docker run`` and the
|
|
1167
|
+
engine program's environment (:func:`env_args`, :func:`program_env`);
|
|
1168
|
+
``binds``, licence files mounted read-only where they are declared."""
|
|
1169
|
+
app = os.path.abspath(self.app)
|
|
1170
|
+
binds = [(os.path.abspath(src), dest) for src, dest in binds]
|
|
1171
|
+
for p, _ in mounted: # the application's folder holds the keys' file: never it, nor a folder above it
|
|
1172
|
+
if app == p or app.startswith(p.rstrip(os.sep) + os.sep):
|
|
1173
|
+
raise ValueError(f"{p} would show the agent this application's folder and its keys")
|
|
1174
|
+
for src, _ in binds: # a licence comes from its store, never from in here
|
|
1175
|
+
if src.startswith(app + os.sep):
|
|
1176
|
+
raise ValueError(f"{src} is in this application's folder: a licence is never mounted from there")
|
|
1177
|
+
inside = {k: v for k, v in {**env, **self.network_env}.items() if k not in HOST_ONLY}
|
|
1178
|
+
argv = run_argv(image or self.image, cmd, name=self.names["agent"], network=self.names["network"], job=self.job,
|
|
1179
|
+
workdir=workdir, mounted=mounted, env=inside, path=inside.get("PATH") or IMAGE_PATH,
|
|
1180
|
+
interactive=interactive, binds=binds)
|
|
1181
|
+
self.started.append(self.names["agent"])
|
|
1182
|
+
return argv, program_env(inside)
|
|
1183
|
+
|
|
1184
|
+
def stop(self):
|
|
1185
|
+
"""Remove the unit's containers and network, and the key's file if the
|
|
1186
|
+
relay has not read it; the relay's log is kept beside its record first."""
|
|
1187
|
+
self._forget_key()
|
|
1188
|
+
if self.names["relay"] in self.started:
|
|
1189
|
+
try:
|
|
1190
|
+
r = _docker(["logs", self.names["relay"]], check=False, timeout=30)
|
|
1191
|
+
with open(os.path.join(self.dir, "relay.log"), "w") as f:
|
|
1192
|
+
f.write(r.stdout + r.stderr)
|
|
1193
|
+
except (Unavailable, OSError):
|
|
1194
|
+
pass
|
|
1195
|
+
if self.network_probe and self.names["agent"] + "-egress" in self.started:
|
|
1196
|
+
try:
|
|
1197
|
+
r = _docker(["logs", self.names["agent"] + "-egress"], check=False, timeout=30)
|
|
1198
|
+
with open(os.path.join(self.dir, "egress.log"), "w") as f:
|
|
1199
|
+
f.write(r.stdout + r.stderr)
|
|
1200
|
+
except (Unavailable, OSError):
|
|
1201
|
+
pass
|
|
1202
|
+
for n in dict.fromkeys(self.started):
|
|
1203
|
+
try:
|
|
1204
|
+
_docker(["rm", "-f", n], check=False, timeout=60)
|
|
1205
|
+
except Unavailable:
|
|
1206
|
+
pass
|
|
1207
|
+
self.started = []
|
|
1208
|
+
for bridge in self.socket_bridges:
|
|
1209
|
+
bridge.shutdown()
|
|
1210
|
+
bridge.server_close()
|
|
1211
|
+
self.socket_bridges = []
|
|
1212
|
+
|
|
1213
|
+
for d in self.socket_doors:
|
|
1214
|
+
d.stop()
|
|
1215
|
+
self.socket_doors = []
|
|
1216
|
+
if self.network:
|
|
1217
|
+
try:
|
|
1218
|
+
_docker(["network", "rm", self.names["network"]], check=False, timeout=60)
|
|
1219
|
+
except Unavailable:
|
|
1220
|
+
pass
|
|
1221
|
+
self.network = False
|
|
1222
|
+
|
|
1223
|
+
|
|
1224
|
+
# -- an own image: the task's container, the agent added at run time -------------
|
|
1225
|
+
RUNTIMES = "/opt/nullius" # where an own image is given what the agent runs on
|
|
1226
|
+
PYTHON_VERSION = "3.12.14" # the standalone Python an agent started as python3 runs on there; moves with UV_VERSION
|
|
1227
|
+
_PULL = re.compile(r"^[a-z0-9][a-z0-9._/:-]*@sha256:[0-9a-f]{64}$")
|
|
1228
|
+
IMAGE_SPEC = ("build", "pull", "memory_mb", "cpus")
|
|
1229
|
+
STEP = ("copy", "write", "run", "collect", "seconds", "network", "keep_processes")
|
|
1230
|
+
KEEP_ALIVE = ["sh", "-c", "while :; do sleep 3600; done"]
|
|
1231
|
+
INIT = "/usr/sbin/docker-init" # what --init mounts: docker's, not the unit's
|
|
1232
|
+
|
|
1233
|
+
# The box's processes, read from /proc with the shell's own builtins only - a
|
|
1234
|
+
# task's image may have no ps, no pkill, not even tr - and with no subshell,
|
|
1235
|
+
# so the script never sees a child of its own. A line of /proc/<pid>/stat is
|
|
1236
|
+
# "pid (comm) state ppid ...", comm free to hold spaces and parentheses: what
|
|
1237
|
+
# follows its last ") " is the state, then the parent.
|
|
1238
|
+
_SCAN = r"""
|
|
1239
|
+
line=
|
|
1240
|
+
read -r line 2>/dev/null < "$d/stat" || [ -n "$line" ] || continue
|
|
1241
|
+
set -- ${line##*) }
|
|
1242
|
+
"""
|
|
1243
|
+
# The keep-alive: the one child of PID 1 (docker's init) right after the start.
|
|
1244
|
+
KEEPER = r"""for d in /proc/[0-9]*; do
|
|
1245
|
+
p=${d#/proc/}
|
|
1246
|
+
[ "$p" = "$$" ] && continue""" + _SCAN + r""" [ "$2" = 1 ] && echo "$p"
|
|
1247
|
+
done
|
|
1248
|
+
"""
|
|
1249
|
+
# Every process but PID 1, the keep-alive ($1) and its own sleep, and this
|
|
1250
|
+
# script: stopped first (SIGSTOP - a stopped process cannot fork), then
|
|
1251
|
+
# killed, round after round until a scan finds none; a zombie is init's to
|
|
1252
|
+
# reap and is passed over. Says each it stopped; exit 1 with what is left when
|
|
1253
|
+
# twenty rounds were not enough.
|
|
1254
|
+
SETTLE = r"""keep=$1; n=0
|
|
1255
|
+
while :; do
|
|
1256
|
+
left=
|
|
1257
|
+
for d in /proc/[0-9]*; do
|
|
1258
|
+
p=${d#/proc/}
|
|
1259
|
+
case " 1 $keep $$ " in *" $p "*) continue ;; esac""" + _SCAN + r""" case $1 in Z|X|x) continue ;; esac
|
|
1260
|
+
[ "$2" = "$keep" ] && continue
|
|
1261
|
+
left="$left $p"
|
|
1262
|
+
done
|
|
1263
|
+
[ -z "$left" ] && exit 0
|
|
1264
|
+
n=$((n+1))
|
|
1265
|
+
if [ "$n" -gt 20 ]; then echo "still running:$left"; exit 1; fi
|
|
1266
|
+
for p in $left; do kill -STOP "$p" 2>/dev/null; done
|
|
1267
|
+
for p in $left; do
|
|
1268
|
+
c=; read -r c 2>/dev/null < "/proc/$p/comm"
|
|
1269
|
+
echo "stopped $p $c"
|
|
1270
|
+
kill -KILL "$p" 2>/dev/null
|
|
1271
|
+
done
|
|
1272
|
+
sleep 1
|
|
1273
|
+
done
|
|
1274
|
+
"""
|
|
1275
|
+
|
|
1276
|
+
|
|
1277
|
+
def runtimes_dir() -> str:
|
|
1278
|
+
return os.path.join(install.HOME, "runtimes")
|
|
1279
|
+
|
|
1280
|
+
|
|
1281
|
+
def within(root: str, rel) -> str:
|
|
1282
|
+
"""A path a driver named relative to its upstream (or its own folder),
|
|
1283
|
+
resolved, and refused unless it stays inside: a driver names the task's
|
|
1284
|
+
files, never anything else of this machine."""
|
|
1285
|
+
if not isinstance(rel, str) or not rel or os.path.isabs(rel):
|
|
1286
|
+
raise ValueError(f"{rel!r}: a path relative to the benchmark's upstream")
|
|
1287
|
+
r = os.path.realpath(root)
|
|
1288
|
+
p = os.path.realpath(os.path.join(r, rel))
|
|
1289
|
+
if not p.startswith(r + os.sep):
|
|
1290
|
+
raise ValueError(f"{rel} is outside the benchmark's upstream")
|
|
1291
|
+
if not os.path.exists(p):
|
|
1292
|
+
raise ValueError(f"{rel} is not in the benchmark's upstream")
|
|
1293
|
+
return p
|
|
1294
|
+
|
|
1295
|
+
|
|
1296
|
+
def spec_problems(spec) -> List[str]:
|
|
1297
|
+
"""What is wrong with a unit's ``image`` (its setup's): ``build``, a
|
|
1298
|
+
folder holding a Dockerfile, or ``pull``, a reference pinned by digest;
|
|
1299
|
+
``memory_mb`` and ``cpus``, the limits the task asks for."""
|
|
1300
|
+
if not isinstance(spec, dict):
|
|
1301
|
+
return ["image: {build: <folder in the upstream>} or {pull: <name>@sha256:<digest>}"]
|
|
1302
|
+
bad = [f"image.{k}: not a field ({', '.join(IMAGE_SPEC)})" for k in spec if k not in IMAGE_SPEC]
|
|
1303
|
+
if ("build" in spec) == ("pull" in spec):
|
|
1304
|
+
bad.append("image: one of build (a folder with a Dockerfile) or pull (a reference by digest)")
|
|
1305
|
+
if "pull" in spec and not _PULL.match(str(spec["pull"])):
|
|
1306
|
+
bad.append(f"image.pull: {spec['pull']!r} is not pinned by digest (<name>@sha256:<64 hex>)")
|
|
1307
|
+
for k in ("memory_mb", "cpus"):
|
|
1308
|
+
if k in spec and (not isinstance(spec[k], (int, float)) or isinstance(spec[k], bool) or spec[k] <= 0):
|
|
1309
|
+
bad.append(f"image.{k}: a positive number")
|
|
1310
|
+
return bad
|
|
1311
|
+
|
|
1312
|
+
|
|
1313
|
+
def task_env_problems(env, reserved: Iterable[str] = ()) -> List[str]:
|
|
1314
|
+
"""Task variables for an agent in its own image, never its model configuration."""
|
|
1315
|
+
if not isinstance(env, dict):
|
|
1316
|
+
return ["env: an object of variable names and string values"]
|
|
1317
|
+
bad = []
|
|
1318
|
+
for name, value in env.items():
|
|
1319
|
+
if not isinstance(name, str) or not re.fullmatch(r"[A-Z_][A-Z0-9_]*", name):
|
|
1320
|
+
bad.append("env: variable names use capitals, digits and underscores")
|
|
1321
|
+
continue
|
|
1322
|
+
if (name in reserved or name.startswith(("NULLIUS_", "DOCKER_"))
|
|
1323
|
+
or re.search(r"(^|_)(API|AUTH|KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?|BASE_URL|ENDPOINT|MODEL)(_|$)", name)):
|
|
1324
|
+
bad.append(f"env.{name}: model settings and credentials belong to the run")
|
|
1325
|
+
if not isinstance(value, str) or "\0" in value:
|
|
1326
|
+
bad.append(f"env.{name}: a string without NUL")
|
|
1327
|
+
elif name == "PATH" and any(not part.startswith("/") for part in value.split(":")):
|
|
1328
|
+
bad.append("env.PATH: absolute folders separated by ':'")
|
|
1329
|
+
return bad
|
|
1330
|
+
|
|
1331
|
+
|
|
1332
|
+
def step_problems(step, *, collect: bool) -> List[str]:
|
|
1333
|
+
"""What is wrong with a step run in a unit's box (``verify``, ``solve``):
|
|
1334
|
+
``copy`` {relative path in the upstream: absolute path inside}, ``write``
|
|
1335
|
+
{absolute path inside: its text - a file the driver has by content, an
|
|
1336
|
+
eval script or a patch the dataset carries}, ``run``
|
|
1337
|
+
(argv), ``collect`` (a folder inside, ``verify`` only), ``seconds``,
|
|
1338
|
+
``network`` (true: a way out to public addresses, :meth:`Box.online`),
|
|
1339
|
+
``keep_processes`` (``verify`` only - true: what the agent left running is
|
|
1340
|
+
not stopped before the check, :meth:`Box.settle`)."""
|
|
1341
|
+
if not isinstance(step, dict):
|
|
1342
|
+
return ["a step: {copy?, write?, run, collect?, seconds?, network?, keep_processes?}"]
|
|
1343
|
+
bad = [f"{k}: not a field ({', '.join(STEP)})" for k in step
|
|
1344
|
+
if k not in STEP or (k in ("collect", "keep_processes") and not collect)]
|
|
1345
|
+
run = step.get("run")
|
|
1346
|
+
if not isinstance(run, list) or not run or not all(isinstance(w, str) and w for w in run):
|
|
1347
|
+
bad.append("run: the command, a list of words")
|
|
1348
|
+
copy = step.get("copy") or {}
|
|
1349
|
+
if not isinstance(copy, dict) or not all(isinstance(d, str) and d.startswith("/") for d in copy.values()):
|
|
1350
|
+
bad.append("copy: {path in the upstream: absolute path inside}")
|
|
1351
|
+
write = step.get("write") or {}
|
|
1352
|
+
if not isinstance(write, dict) or not all(isinstance(d, str) and d.startswith("/") and isinstance(t, str)
|
|
1353
|
+
for d, t in write.items()):
|
|
1354
|
+
bad.append("write: {absolute path inside: the file's text}")
|
|
1355
|
+
if "collect" in step and not (isinstance(step["collect"], str) and step["collect"].startswith("/")):
|
|
1356
|
+
bad.append("collect: an absolute path inside")
|
|
1357
|
+
if "seconds" in step and (not isinstance(step["seconds"], (int, float)) or step["seconds"] <= 0):
|
|
1358
|
+
bad.append("seconds: a positive number")
|
|
1359
|
+
if "network" in step and not isinstance(step["network"], bool):
|
|
1360
|
+
bad.append("network: true or false")
|
|
1361
|
+
if "keep_processes" in step and not isinstance(step["keep_processes"], bool):
|
|
1362
|
+
bad.append("keep_processes: true or false")
|
|
1363
|
+
return bad
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def own_context(path: str) -> Dict[str, str]:
|
|
1367
|
+
"""An own image's build context: every file of its folder, by its path
|
|
1368
|
+
there, and its sha256."""
|
|
1369
|
+
out = {}
|
|
1370
|
+
for d, dirs, files in os.walk(path):
|
|
1371
|
+
dirs.sort()
|
|
1372
|
+
for f in sorted(files):
|
|
1373
|
+
p = os.path.join(d, f)
|
|
1374
|
+
out[os.path.relpath(p, path)] = "link:" + os.readlink(p) if os.path.islink(p) else install.sha256_of(p)
|
|
1375
|
+
return out
|
|
1376
|
+
|
|
1377
|
+
|
|
1378
|
+
def own_tag(bench: dict, rel: str, path: str) -> str:
|
|
1379
|
+
"""``nullius/own-<benchmark>-<folder>:<hash>``: the sha256 of every file
|
|
1380
|
+
the build reads, so the same folder is the same image, and a changed one a
|
|
1381
|
+
new image beside it. Its ``FROM`` is the task's, as published."""
|
|
1382
|
+
h = hashlib.sha256()
|
|
1383
|
+
for f, digest in own_context(path).items():
|
|
1384
|
+
h.update(f.encode() + b"\0" + digest.encode() + b"\0")
|
|
1385
|
+
word = re.sub(r"[^a-z0-9]+", "-", rel.lower()).strip("-")[:60] or "task"
|
|
1386
|
+
return f"nullius/own-{bench['key']}-{word}:{h.hexdigest()[:12]}"
|
|
1387
|
+
|
|
1388
|
+
|
|
1389
|
+
def own_image(bench: dict, spec: dict, root: str, log_dir: Optional[str] = None) -> str:
|
|
1390
|
+
"""A unit's own image: ``pull``ed by its digest, or built from the folder
|
|
1391
|
+
``build`` names in the benchmark's upstream (``root``) - the task's own
|
|
1392
|
+
Dockerfile, its own ``FROM``, its folder the whole build context - the
|
|
1393
|
+
first time a unit asks for it, and kept by its tag after. Two units
|
|
1394
|
+
asking at once wait for one build. No secret is given to a build."""
|
|
1395
|
+
if spec.get("pull"):
|
|
1396
|
+
ref = spec["pull"]
|
|
1397
|
+
if not present(ref):
|
|
1398
|
+
r = _docker(["pull", ref], timeout=3600, check=False)
|
|
1399
|
+
if r.returncode != 0:
|
|
1400
|
+
raise Unavailable(f"pulling {ref}: {(r.stderr or r.stdout).strip()[-300:]}")
|
|
1401
|
+
_built[ref] = True
|
|
1402
|
+
_fetched.add(ref)
|
|
1403
|
+
return ref
|
|
1404
|
+
path = within(root, spec["build"])
|
|
1405
|
+
if not os.path.isfile(os.path.join(path, "Dockerfile")):
|
|
1406
|
+
raise Unavailable(f"{spec['build']} holds no Dockerfile")
|
|
1407
|
+
tag = own_tag(bench, spec["build"], path)
|
|
1408
|
+
with _locks_lock:
|
|
1409
|
+
lock = _locks.setdefault(tag, threading.Lock())
|
|
1410
|
+
with lock:
|
|
1411
|
+
if present(tag):
|
|
1412
|
+
return tag
|
|
1413
|
+
# --force-rm: a build that fails (a task's Dockerfile fetching what is gone) leaves no container behind
|
|
1414
|
+
r = _docker(["build", "--force-rm", "--label", f"nullius.own={bench['key']}", "-t", tag, path], timeout=3600, check=False)
|
|
1415
|
+
if log_dir:
|
|
1416
|
+
os.makedirs(log_dir, exist_ok=True)
|
|
1417
|
+
with open(os.path.join(log_dir, "own-" + tag.split(":")[0].split("/")[-1] + ".log"), "a") as f:
|
|
1418
|
+
f.write(f"# {tag}\n" + r.stdout + r.stderr)
|
|
1419
|
+
if r.returncode != 0:
|
|
1420
|
+
raise Unavailable(f"building {spec['build']} ({tag}): {(r.stderr or r.stdout).strip()[-300:]}")
|
|
1421
|
+
_built[tag] = True
|
|
1422
|
+
_fetched.add(tag)
|
|
1423
|
+
return tag
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def fetched(image: str) -> bool:
|
|
1427
|
+
"""Whether this process pulled or built ``image`` (``own_image``): an image
|
|
1428
|
+
that was there before it is the user's, never removed here. ``_built``
|
|
1429
|
+
is not the answer: it also remembers every image found present."""
|
|
1430
|
+
return image in _fetched
|
|
1431
|
+
|
|
1432
|
+
|
|
1433
|
+
def drop_image(image: str) -> None:
|
|
1434
|
+
"""Remove an own image this process fetched, once no unit needs it: one
|
|
1435
|
+
per SWE-bench Verified instance, kept, filled docker's disk part-way
|
|
1436
|
+
through a run (the upstream harness removes its instance images too)."""
|
|
1437
|
+
if image in _fetched:
|
|
1438
|
+
_fetched.discard(image)
|
|
1439
|
+
_built.pop(image, None)
|
|
1440
|
+
_docker(["rmi", image], timeout=600, check=False)
|
|
1441
|
+
|
|
1442
|
+
|
|
1443
|
+
def image_config(image: str) -> dict:
|
|
1444
|
+
"""What an image says of itself that a box keeps: its working directory,
|
|
1445
|
+
its user and its PATH."""
|
|
1446
|
+
c = json.loads(_docker(["image", "inspect", "-f", "{{json .Config}}", image]).stdout or "null") or {}
|
|
1447
|
+
variables = dict(e.split("=", 1) for e in c.get("Env") or [] if "=" in e) # the image's, not this machine's
|
|
1448
|
+
return {"workdir": c.get("WorkingDir") or "/", "user": c.get("User") or "", "path": variables.get("PATH") or IMAGE_PATH}
|
|
1449
|
+
|
|
1450
|
+
|
|
1451
|
+
def node_runtime(machine: Optional[str] = None) -> str:
|
|
1452
|
+
"""Node for an own image: the official tarball the base image is built
|
|
1453
|
+
from, checked by the same sha256, unpacked once into
|
|
1454
|
+
``<STANDARD_DIR>/runtimes/node-v<version>-linux-<arch>``. A glibc build:
|
|
1455
|
+
a musl image cannot run it (:meth:`Box.runtime_problem`)."""
|
|
1456
|
+
arch = _ARCH.get((machine or platform.machine()).lower())
|
|
1457
|
+
if not arch:
|
|
1458
|
+
raise Unavailable(f"no Node build pinned for {machine or platform.machine()}")
|
|
1459
|
+
name = f"node-v{NODE_VERSION}-linux-{arch}"
|
|
1460
|
+
where = os.path.join(runtimes_dir(), name)
|
|
1461
|
+
if install.ready(where):
|
|
1462
|
+
return where
|
|
1463
|
+
with install._locked(where):
|
|
1464
|
+
if install.ready(where):
|
|
1465
|
+
return where
|
|
1466
|
+
tmp = tempfile.mkdtemp(prefix="node-", dir=runtimes_dir())
|
|
1467
|
+
try:
|
|
1468
|
+
tarball = os.path.join(tmp, name + ".tar.xz")
|
|
1469
|
+
with urllib.request.urlopen(f"https://nodejs.org/dist/v{NODE_VERSION}/{name}.tar.xz", timeout=60) as r, \
|
|
1470
|
+
open(tarball, "wb") as f:
|
|
1471
|
+
shutil.copyfileobj(r, f)
|
|
1472
|
+
if install.sha256_of(tarball) != NODE_SHA256[arch]:
|
|
1473
|
+
raise Unavailable(f"{name}.tar.xz is not the one pinned (sha256)")
|
|
1474
|
+
with tarfile.open(tarball, "r:xz") as t:
|
|
1475
|
+
t.extractall(tmp, filter="data")
|
|
1476
|
+
if os.path.isdir(where):
|
|
1477
|
+
shutil.rmtree(where)
|
|
1478
|
+
os.rename(os.path.join(tmp, name), where)
|
|
1479
|
+
open(os.path.join(where, install.MARK), "w").close()
|
|
1480
|
+
except OSError as e:
|
|
1481
|
+
raise Unavailable(f"fetching Node {NODE_VERSION}: {e}")
|
|
1482
|
+
finally:
|
|
1483
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
1484
|
+
return where
|
|
1485
|
+
|
|
1486
|
+
|
|
1487
|
+
def python_runtime() -> str:
|
|
1488
|
+
"""A standalone Python for an own image (python-build-standalone, by
|
|
1489
|
+
``uv python install``, which checks it by its own pinned digests),
|
|
1490
|
+
installed once into ``<STANDARD_DIR>/runtimes/python-<version>``: the
|
|
1491
|
+
interpreter's folder. Installed in the base image, so it is a glibc
|
|
1492
|
+
build, as Node's is, whatever this machine's libc."""
|
|
1493
|
+
where = os.path.join(runtimes_dir(), f"python-{PYTHON_VERSION}")
|
|
1494
|
+
found = lambda: sorted(glob.glob(os.path.join(where, f"cpython-{PYTHON_VERSION}-linux-*-gnu")))
|
|
1495
|
+
if install.ready(where) and found():
|
|
1496
|
+
return found()[0]
|
|
1497
|
+
with install._locked(where):
|
|
1498
|
+
if not (install.ready(where) and found()):
|
|
1499
|
+
os.makedirs(where, exist_ok=True)
|
|
1500
|
+
try:
|
|
1501
|
+
r = subprocess.run(install_argv(base_image(), where, ["uv", "python", "install", PYTHON_VERSION,
|
|
1502
|
+
"--install-dir", where, "--no-bin"]),
|
|
1503
|
+
env=client_env({k: os.environ[k] for k in INSTALL_KEEP if k in os.environ}),
|
|
1504
|
+
capture_output=True, text=True, timeout=1800)
|
|
1505
|
+
except (OSError, subprocess.TimeoutExpired) as e:
|
|
1506
|
+
raise Unavailable(f"installing Python {PYTHON_VERSION}: {e}")
|
|
1507
|
+
finally:
|
|
1508
|
+
install.remove_scratch(where)
|
|
1509
|
+
if r.returncode != 0 or not found():
|
|
1510
|
+
raise Unavailable(f"installing Python {PYTHON_VERSION}: {(r.stderr or r.stdout).strip()[-300:]}")
|
|
1511
|
+
open(os.path.join(where, install.MARK), "w").close()
|
|
1512
|
+
return found()[0]
|
|
1513
|
+
|
|
1514
|
+
|
|
1515
|
+
def runtime_mounts(names: Iterable[str]) -> Dict[str, Tuple[str, str]]:
|
|
1516
|
+
"""Each runtime an agent needs in an own image: name -> (its folder on
|
|
1517
|
+
this machine, where it is mounted inside)."""
|
|
1518
|
+
out = {}
|
|
1519
|
+
for n in names:
|
|
1520
|
+
out[n] = ({"node": node_runtime, "python": python_runtime}[n](), f"{RUNTIMES}/{n}")
|
|
1521
|
+
return out
|
|
1522
|
+
|
|
1523
|
+
|
|
1524
|
+
def bare_python(word: str) -> bool:
|
|
1525
|
+
"""``python3`` or ``python`` with no environment of its own - not an
|
|
1526
|
+
environment's interpreter (``{install}/.venv/bin/python``), which must
|
|
1527
|
+
stay itself or the agent runs without its packages."""
|
|
1528
|
+
return os.path.basename(word) in ("python3", "python") and "/.venv/" not in word
|
|
1529
|
+
|
|
1530
|
+
|
|
1531
|
+
def in_runtime(cmd: List[str], runtimes: Dict[str, Tuple[str, str]]) -> List[str]:
|
|
1532
|
+
"""An agent's start command as an own image runs it: ``python3`` or
|
|
1533
|
+
``node`` - the word, or a script's ``#!/usr/bin/env`` line - becomes the
|
|
1534
|
+
runtime under ``/opt/nullius``, so the agent runs on what it was installed
|
|
1535
|
+
for while the image's own ``python3`` and ``node`` stay the task's."""
|
|
1536
|
+
cmd = list(cmd)
|
|
1537
|
+
first = os.path.basename(cmd[0]) if cmd else ""
|
|
1538
|
+
if cmd and bare_python(cmd[0]) and "python" in runtimes:
|
|
1539
|
+
return [runtimes["python"][1] + "/bin/python3"] + cmd[1:]
|
|
1540
|
+
if first == "node" and "node" in runtimes:
|
|
1541
|
+
return [runtimes["node"][1] + "/bin/node"] + cmd[1:]
|
|
1542
|
+
try:
|
|
1543
|
+
with open(cmd[0], "rb") as f:
|
|
1544
|
+
line = f.readline(200).decode("utf-8", "replace").split()
|
|
1545
|
+
except (OSError, IndexError):
|
|
1546
|
+
return cmd
|
|
1547
|
+
if line[:1] == ["#!/usr/bin/env"] and len(line) > 1:
|
|
1548
|
+
word = {"node": "node", "python3": "python", "python": "python"}.get(line[1])
|
|
1549
|
+
if word in runtimes:
|
|
1550
|
+
return [runtimes[word][1] + ("/bin/node" if word == "node" else "/bin/python3")] + cmd
|
|
1551
|
+
return cmd
|
|
1552
|
+
|
|
1553
|
+
|
|
1554
|
+
class Box:
|
|
1555
|
+
"""A unit's own container: the task's image, kept running for the whole
|
|
1556
|
+
unit so the agent, a control's reference solution and the benchmark's
|
|
1557
|
+
check all act on one container's state, one after the other.
|
|
1558
|
+
|
|
1559
|
+
It runs as the image's own user, as the task was written for - usually
|
|
1560
|
+
root, which its tasks need (a package to install, a file in ``/etc``) -
|
|
1561
|
+
with docker's default capabilities, no way to gain more
|
|
1562
|
+
(``no-new-privileges``), and on the unit's internal network (or one of its
|
|
1563
|
+
own, for a control): the relay, when there is one, is its only way out.
|
|
1564
|
+
The folders it writes on this machine (the agent's home) are given back to
|
|
1565
|
+
this machine's user before it is removed (``give_back``).
|
|
1566
|
+
|
|
1567
|
+
**Before the check, nothing of the agent runs** (:meth:`settle`). A
|
|
1568
|
+
process the agent started and left - a server, a loop, a watcher - would
|
|
1569
|
+
otherwise still run while the check's files are copied in and while the
|
|
1570
|
+
check has a way out: free to rewrite the tests, or to reach what the
|
|
1571
|
+
check's network reaches. So every process but docker's init and the
|
|
1572
|
+
keep-alive is stopped, as root in the box, and the box is confirmed empty
|
|
1573
|
+
of them, before anything is copied in. A task about a service the agent
|
|
1574
|
+
must leave running opts out with its ``verify`` step's ``keep_processes``;
|
|
1575
|
+
what that keeps runs on beside the tests, which is the task's trade.
|
|
1576
|
+
|
|
1577
|
+
**The check's way out is a proxy, not a bridge** (:meth:`online`). The
|
|
1578
|
+
box stays on its internal network; a proxy container joins it as
|
|
1579
|
+
``egress`` (``egress.py``) and reaches public addresses only - never
|
|
1580
|
+
link-local (cloud metadata), private ranges, the shared range some clouds
|
|
1581
|
+
keep their metadata in, nor the machine itself through the docker
|
|
1582
|
+
bridge. A benchmark may also grant it for the agent's work with
|
|
1583
|
+
``internet: true``; it is removed before the check. Each phase is told
|
|
1584
|
+
of its proxy through ``HTTP_PROXY`` and ``HTTPS_PROXY``. It needs no root
|
|
1585
|
+
on the machine, so every host runs the check the same way; ``egress.py`` says why that beat firewall rules.
|
|
1586
|
+
|
|
1587
|
+
**The agent's mounts stay for the check.** A running container's mounts
|
|
1588
|
+
cannot be changed, and taking its home away would mean a second container
|
|
1589
|
+
for the check, without the state the agent left in this one. They are
|
|
1590
|
+
harmless there: the install, definition and runtimes are read-only, the
|
|
1591
|
+
home is the trial's own folder and nothing else of this machine, the check
|
|
1592
|
+
does not read them, and nothing of the agent is left running to use them
|
|
1593
|
+
(unless the task keeps its processes)."""
|
|
1594
|
+
|
|
1595
|
+
def __init__(self, job: str, tag: str, image: str, *, workdir: str, path: str = IMAGE_PATH,
|
|
1596
|
+
network: Optional[str] = None, mounted: Iterable[Tuple[str, bool]] = (),
|
|
1597
|
+
runtimes: Optional[Dict[str, Tuple[str, str]]] = None, memory_mb=None, cpus=None,
|
|
1598
|
+
app: Optional[str] = None, env: Optional[dict] = None):
|
|
1599
|
+
n = names(job, tag)
|
|
1600
|
+
self.job, self.image, self.workdir, self.path = job, image, workdir, path
|
|
1601
|
+
self.env = dict(env or {})
|
|
1602
|
+
self.name = n["agent"]
|
|
1603
|
+
self.net = network or n["network"]
|
|
1604
|
+
self.own_net = network is None
|
|
1605
|
+
self.mounted, self.runtimes = list(mounted), dict(runtimes or {})
|
|
1606
|
+
self.memory_mb, self.cpus, self.app = memory_mb, cpus, app
|
|
1607
|
+
self.up = False
|
|
1608
|
+
self.keep: Optional[str] = None # the keep-alive's PID inside, found at the start
|
|
1609
|
+
self.egress: Optional[str] = None # the check's proxy container, once online
|
|
1610
|
+
|
|
1611
|
+
def start(self) -> "Box":
|
|
1612
|
+
if self.app:
|
|
1613
|
+
app = os.path.abspath(self.app)
|
|
1614
|
+
for p, _ in self.mounted: # the application's folder holds the keys' file
|
|
1615
|
+
if app == p or app.startswith(p.rstrip(os.sep) + os.sep):
|
|
1616
|
+
raise ValueError(f"{p} would show the agent this application's folder and its keys")
|
|
1617
|
+
try:
|
|
1618
|
+
if self.own_net:
|
|
1619
|
+
_docker(["network", "create", "--internal", "--label", f"{LABEL}={self.job}", self.net])
|
|
1620
|
+
# in the machine's user namespace on podman (its root stays this user on rootless podman whatever
|
|
1621
|
+
# containers.conf sets: keep-id makes it a sub-uid) and on a Docker with userns-remap
|
|
1622
|
+
a = ["run", "-d", "--name", self.name, "--label", f"{LABEL}={self.job}", "--network", self.net, "--init",
|
|
1623
|
+
*_security_opt(), "-w", self.workdir, *(["--userns=host"] if podman() else _userns())]
|
|
1624
|
+
if self.memory_mb:
|
|
1625
|
+
a += ["--memory", f"{int(self.memory_mb)}m"]
|
|
1626
|
+
if self.cpus:
|
|
1627
|
+
a += ["--cpus", str(self.cpus)]
|
|
1628
|
+
for p, ro in self.mounted:
|
|
1629
|
+
a += ["-v", f"{p}:{p}:{'ro' if ro else 'rw'}"]
|
|
1630
|
+
for host, inside in self.runtimes.values():
|
|
1631
|
+
a += ["-v", f"{host}:{inside}:ro"]
|
|
1632
|
+
self.up = True
|
|
1633
|
+
_docker(a + ["--entrypoint", KEEP_ALIVE[0], self.image] + KEEP_ALIVE[1:], timeout=300)
|
|
1634
|
+
self.keep = self.keeper()
|
|
1635
|
+
except Exception:
|
|
1636
|
+
self.stop()
|
|
1637
|
+
raise
|
|
1638
|
+
return self
|
|
1639
|
+
|
|
1640
|
+
def exec_argv(self, cmd: List[str], env: Dict[str, str], path: str, *, interactive: bool = True,
|
|
1641
|
+
workdir: Optional[str] = None) -> List[str]:
|
|
1642
|
+
"""``docker exec`` of a command in the box, its environment as
|
|
1643
|
+
:func:`env_args` gives it."""
|
|
1644
|
+
values, named = env_args(env, path)
|
|
1645
|
+
return [DOCKER, "exec", *(["-i"] if interactive else []), "-w", workdir or self.workdir,
|
|
1646
|
+
*values, *named, self.name, *cmd]
|
|
1647
|
+
|
|
1648
|
+
def run(self, cmd: List[str], seconds: float, *, user: Optional[str] = None, env: Optional[dict] = None) -> dict:
|
|
1649
|
+
"""A command run in the box, to its end or ``seconds``: {exit, output,
|
|
1650
|
+
timed_out}. ``exit`` is None when it timed out."""
|
|
1651
|
+
a = ["exec", "-w", self.workdir] + (["-u", user] if user else [])
|
|
1652
|
+
for k in sorted(env or {}):
|
|
1653
|
+
a += ["-e", k]
|
|
1654
|
+
try:
|
|
1655
|
+
r = subprocess.run([DOCKER] + a + [self.name] + list(cmd), env=client_env(env), stdin=subprocess.DEVNULL,
|
|
1656
|
+
stdout=subprocess.PIPE, stderr=subprocess.STDOUT, timeout=seconds)
|
|
1657
|
+
except subprocess.TimeoutExpired as e:
|
|
1658
|
+
out = e.stdout or b""
|
|
1659
|
+
return {"exit": None, "output": out.decode("utf-8", "replace") if isinstance(out, bytes) else out, "timed_out": True}
|
|
1660
|
+
except FileNotFoundError:
|
|
1661
|
+
raise Unavailable(f"{DOCKER} is not installed on this machine")
|
|
1662
|
+
return {"exit": r.returncode, "output": r.stdout.decode("utf-8", "replace"), "timed_out": False}
|
|
1663
|
+
|
|
1664
|
+
def copy_in(self, src: str, dest: str):
|
|
1665
|
+
"""A file or a folder of this machine into the box at ``dest``: a
|
|
1666
|
+
folder's contents become ``dest``'s."""
|
|
1667
|
+
self.run(["mkdir", "-p", dest if os.path.isdir(src) else os.path.dirname(dest) or "/"], 60, user="0")
|
|
1668
|
+
_docker(["cp", src + ("/." if os.path.isdir(src) else ""), f"{self.name}:{dest}"], timeout=600)
|
|
1669
|
+
|
|
1670
|
+
def write_in(self, dest: str, text: str):
|
|
1671
|
+
"""A file given by its text into the box at ``dest`` (a step's
|
|
1672
|
+
``write``): written here first, then copied in as ``copy_in`` does."""
|
|
1673
|
+
tmp = tempfile.mkdtemp(prefix="write-")
|
|
1674
|
+
try:
|
|
1675
|
+
path = os.path.join(tmp, os.path.basename(dest) or "file")
|
|
1676
|
+
with open(path, "w") as f:
|
|
1677
|
+
f.write(text)
|
|
1678
|
+
self.copy_in(path, dest)
|
|
1679
|
+
finally:
|
|
1680
|
+
shutil.rmtree(tmp, ignore_errors=True)
|
|
1681
|
+
|
|
1682
|
+
def copy_out(self, src: str, dest: str) -> bool:
|
|
1683
|
+
"""The contents of a folder in the box into ``dest`` on this machine;
|
|
1684
|
+
False when the box has no such folder."""
|
|
1685
|
+
os.makedirs(dest, exist_ok=True)
|
|
1686
|
+
ok = _docker(["cp", f"{self.name}:{src.rstrip('/')}/.", dest], timeout=600, check=False).returncode == 0
|
|
1687
|
+
plain_only(dest) # docker cp keeps the box's links, absolute targets included
|
|
1688
|
+
return ok
|
|
1689
|
+
|
|
1690
|
+
def keeper(self, timeout: float = 10) -> str:
|
|
1691
|
+
"""The keep-alive's PID inside: the one child of docker's init, read
|
|
1692
|
+
right after the start, before anything else runs in the box."""
|
|
1693
|
+
end = time.time() + timeout
|
|
1694
|
+
while True:
|
|
1695
|
+
r = self.run(["sh", "-c", KEEPER], 30, user="0")
|
|
1696
|
+
pids = r["output"].split()
|
|
1697
|
+
if r["exit"] == 0 and len(pids) == 1 and pids[0].isdigit():
|
|
1698
|
+
return pids[0]
|
|
1699
|
+
if time.time() > end:
|
|
1700
|
+
raise Unavailable("the container's keep-alive was not found: " + r["output"].strip()[-200:])
|
|
1701
|
+
time.sleep(0.2)
|
|
1702
|
+
|
|
1703
|
+
@staticmethod
|
|
1704
|
+
def settle_argv(keep: str) -> List[str]:
|
|
1705
|
+
"""The command that stops every process but init and the keep-alive."""
|
|
1706
|
+
return ["sh", "-c", SETTLE, "settle", keep]
|
|
1707
|
+
|
|
1708
|
+
def settle(self) -> List[str]:
|
|
1709
|
+
"""Every process the agent (or a control's solution) left in the box
|
|
1710
|
+
stopped, as root: "<pid> <name>" for each. Unavailable when some are
|
|
1711
|
+
still running after it - the check is not run beside them."""
|
|
1712
|
+
if not self.keep:
|
|
1713
|
+
raise Unavailable("the container's keep-alive is not known: its processes cannot be told apart")
|
|
1714
|
+
r = self.run(self.settle_argv(self.keep), 120, user="0")
|
|
1715
|
+
said = r["output"].splitlines()
|
|
1716
|
+
if r["exit"] != 0:
|
|
1717
|
+
raise Unavailable("what the agent left running could not be stopped: " + (said[-1] if said else
|
|
1718
|
+
("timed out" if r["timed_out"] else f"exit {r['exit']}")))
|
|
1719
|
+
return [l[len("stopped "):] for l in said if l.startswith("stopped ")]
|
|
1720
|
+
|
|
1721
|
+
def online(self) -> Dict[str, str]:
|
|
1722
|
+
"""A way out for the check: its proxy (``egress.py``, in the base image)
|
|
1723
|
+
started on the default bridge (``BRIDGE`` on podman) and joined to the box's network as
|
|
1724
|
+
``egress``. The box itself stays on its internal network. The
|
|
1725
|
+
variables the check is run with, which name the proxy - and leave
|
|
1726
|
+
the box's own services (``localhost``) to be reached directly."""
|
|
1727
|
+
if self.egress is None:
|
|
1728
|
+
image = base_image()
|
|
1729
|
+
name = self.name + "-egress"
|
|
1730
|
+
self.egress = name
|
|
1731
|
+
make_bridge()
|
|
1732
|
+
_docker(egress_argv(image, name=name, job=self.job)[1:])
|
|
1733
|
+
_docker(["network", "connect", "--alias", EGRESS_ALIAS, self.net, name])
|
|
1734
|
+
wait_listening(name, "the task's proxy")
|
|
1735
|
+
return proxy_env()
|
|
1736
|
+
|
|
1737
|
+
def offline(self):
|
|
1738
|
+
"""The way out :meth:`online` gave taken away again: its proxy
|
|
1739
|
+
container removed, then shown gone - no container of that name, and
|
|
1740
|
+
none answering to ``egress`` on the box's network. Unavailable when
|
|
1741
|
+
either is still there: the agent is never started beside it."""
|
|
1742
|
+
name = self.egress
|
|
1743
|
+
if not name:
|
|
1744
|
+
return
|
|
1745
|
+
_docker(["rm", "-f", name], check=False, timeout=60)
|
|
1746
|
+
left = _docker(["ps", "-aq", "--filter", f"name=^/{name}$"], check=False, timeout=30)
|
|
1747
|
+
if left.returncode != 0 or left.stdout.strip():
|
|
1748
|
+
raise Unavailable(f"the way out ({name}) could not be removed")
|
|
1749
|
+
on = _docker(["network", "inspect", "-f", "{{range .Containers}}{{.Name}} {{end}}", self.net], check=False, timeout=30)
|
|
1750
|
+
if on.returncode != 0 or name in on.stdout.split():
|
|
1751
|
+
raise Unavailable(f"the way out ({name}) is still on the unit's network")
|
|
1752
|
+
self.egress = None
|
|
1753
|
+
|
|
1754
|
+
def egress_said(self) -> str:
|
|
1755
|
+
"""What the check's proxy let through and refused, a JSON line each."""
|
|
1756
|
+
if not self.egress:
|
|
1757
|
+
return ""
|
|
1758
|
+
try:
|
|
1759
|
+
r = _docker(["logs", self.egress], check=False, timeout=30)
|
|
1760
|
+
except Unavailable:
|
|
1761
|
+
return ""
|
|
1762
|
+
return r.stdout + r.stderr
|
|
1763
|
+
|
|
1764
|
+
def runtime_problem(self, name: str) -> Optional[str]:
|
|
1765
|
+
"""Why the agent's runtime ``name`` cannot start in this image, or
|
|
1766
|
+
None. A glibc build in a musl image is the usual reason, said as such."""
|
|
1767
|
+
inside = self.runtimes[name][1] + ("/bin/node" if name == "node" else "/bin/python3")
|
|
1768
|
+
r = self.run([inside, "--version"], 60)
|
|
1769
|
+
if r["exit"] == 0:
|
|
1770
|
+
return None
|
|
1771
|
+
musl = self.run(["sh", "-c", "ls /lib/ld-musl-* 2>/dev/null"], 30)
|
|
1772
|
+
if musl["exit"] == 0 and musl["output"].strip():
|
|
1773
|
+
return f"the agent's runtime ({name}) needs glibc; this task's image is musl-based (Alpine, say)"
|
|
1774
|
+
return f"the agent's runtime ({name}) does not start in this task's image: " + r["output"].strip()[-200:]
|
|
1775
|
+
|
|
1776
|
+
def diff(self, limit: int = 2000) -> str:
|
|
1777
|
+
"""What changed in the box's file system since its image (``docker
|
|
1778
|
+
diff``), less the folders Nullius mounted and the ones above them."""
|
|
1779
|
+
r = _docker(["diff", self.name], check=False, timeout=120)
|
|
1780
|
+
ours = [p for p, _ in self.mounted] + [inside for _, inside in self.runtimes.values()] + [INIT]
|
|
1781
|
+
mine = lambda path: any(t == path or t.startswith(path.rstrip("/") + "/") or path.startswith(t + "/") for t in ours)
|
|
1782
|
+
lines = [l for l in r.stdout.splitlines() if not mine(l[2:])]
|
|
1783
|
+
return "\n".join(lines[:limit]) + (f"\n... {len(lines) - limit} more\n" if len(lines) > limit else "\n")
|
|
1784
|
+
|
|
1785
|
+
def give_back(self, paths: Iterable[str]):
|
|
1786
|
+
"""The folders of this machine the box wrote into, given back to this
|
|
1787
|
+
machine's user: what root wrote there is the user's to read and remove.
|
|
1788
|
+
On a rootless engine the box's root is that user."""
|
|
1789
|
+
paths = [p for p in paths if p]
|
|
1790
|
+
if self.up and paths:
|
|
1791
|
+
try:
|
|
1792
|
+
self.run(["chown", "-R", "0:0" if rootless() else _user()] + paths, 120, user="0")
|
|
1793
|
+
except Unavailable:
|
|
1794
|
+
pass
|
|
1795
|
+
|
|
1796
|
+
def stop(self):
|
|
1797
|
+
if self.egress:
|
|
1798
|
+
try:
|
|
1799
|
+
_docker(["rm", "-f", self.egress], check=False, timeout=60)
|
|
1800
|
+
except Unavailable:
|
|
1801
|
+
pass
|
|
1802
|
+
self.egress = None
|
|
1803
|
+
if self.up:
|
|
1804
|
+
try:
|
|
1805
|
+
_docker(["rm", "-f", self.name], check=False, timeout=60)
|
|
1806
|
+
except Unavailable:
|
|
1807
|
+
pass
|
|
1808
|
+
self.up = False
|
|
1809
|
+
if self.own_net:
|
|
1810
|
+
try:
|
|
1811
|
+
_docker(["network", "rm", self.net], check=False, timeout=60)
|
|
1812
|
+
except Unavailable:
|
|
1813
|
+
pass
|
|
1814
|
+
self.own_net = False
|
|
1815
|
+
|
|
1816
|
+
|
|
1817
|
+
def key_root() -> str:
|
|
1818
|
+
"""The key folders (a relay's key, a door's nonce, a setup driver's
|
|
1819
|
+
credentials), in the user's cache: Colima and Lima share the home with
|
|
1820
|
+
their engine, not the temporary folder."""
|
|
1821
|
+
return os.path.join(paths.cache_dir(), "relay-keys")
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
def key_folder(job: str) -> str:
|
|
1825
|
+
"""A new owner-only folder for a job's secret files under :func:`key_root`."""
|
|
1826
|
+
root = key_root()
|
|
1827
|
+
os.makedirs(root, exist_ok=True)
|
|
1828
|
+
os.chmod(root, 0o700)
|
|
1829
|
+
forget_stale_keys()
|
|
1830
|
+
return tempfile.mkdtemp(prefix=key_prefix(job), dir=root)
|
|
1831
|
+
|
|
1832
|
+
|
|
1833
|
+
def secret_file(path: str, text: str) -> str:
|
|
1834
|
+
"""``text`` in a new file at ``path`` that only its owner reads."""
|
|
1835
|
+
with os.fdopen(os.open(path, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "w") as f:
|
|
1836
|
+
f.write(text)
|
|
1837
|
+
return path
|
|
1838
|
+
|
|
1839
|
+
|
|
1840
|
+
def forget_stale_keys():
|
|
1841
|
+
"""Remove every job's key folders older than ``KEY_STALE``."""
|
|
1842
|
+
old = time.time() - KEY_STALE
|
|
1843
|
+
for d in glob.glob(os.path.join(glob.escape(key_root()), "*")):
|
|
1844
|
+
try:
|
|
1845
|
+
if os.lstat(d).st_mtime < old:
|
|
1846
|
+
shutil.rmtree(d, ignore_errors=True)
|
|
1847
|
+
except OSError:
|
|
1848
|
+
pass
|
|
1849
|
+
|
|
1850
|
+
|
|
1851
|
+
def key_prefix(job: str) -> str:
|
|
1852
|
+
"""A job's key folder prefix: a hash, so no job's prefix starts another's."""
|
|
1853
|
+
return f"nullius-relay-key-{hashlib.sha256(job.encode()).hexdigest()[:12]}-"
|
|
1854
|
+
|
|
1855
|
+
|
|
1856
|
+
def sweep(job: str):
|
|
1857
|
+
"""Remove every container and network a job's units left, by its label,
|
|
1858
|
+
its key folders, and stale key folders."""
|
|
1859
|
+
for d in glob.glob(os.path.join(glob.escape(key_root()), glob.escape(key_prefix(job)) + "*")):
|
|
1860
|
+
shutil.rmtree(d, ignore_errors=True)
|
|
1861
|
+
forget_stale_keys()
|
|
1862
|
+
try:
|
|
1863
|
+
ids = _docker(["ps", "-aq", "--filter", f"label={LABEL}={job}"], check=False).stdout.split()
|
|
1864
|
+
if ids:
|
|
1865
|
+
_docker(["rm", "-f"] + ids, check=False, timeout=120)
|
|
1866
|
+
nets = _docker(["network", "ls", "-q", "--filter", f"label={LABEL}={job}"], check=False).stdout.split()
|
|
1867
|
+
if nets:
|
|
1868
|
+
_docker(["network", "rm"] + nets, check=False, timeout=120)
|
|
1869
|
+
except Unavailable:
|
|
1870
|
+
pass
|