nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/execution.py
ADDED
|
@@ -0,0 +1,1653 @@
|
|
|
1
|
+
"""Running a recipe: on this machine, or on a host from Settings, with the
|
|
2
|
+
job mirrored back as it goes - and surviving Nullius going away.
|
|
3
|
+
|
|
4
|
+
One ``Plan`` is what the builder posts: the run commands compiled for a
|
|
5
|
+
runner (they name ``$NULLIUS_HARBOR``, ``$NULLIUS_TASKS`` and
|
|
6
|
+
``$NULLIUS_JOBS`` and nothing else about where they run), one part per
|
|
7
|
+
agent in the order they run (controls first), the Harbor the recipe pins
|
|
8
|
+
(``harbor_pin``, ``harbor/source.py``), the task folder they read, and the
|
|
9
|
+
target. ``$NULLIUS_HARBOR`` is a small ``harbor`` script in the run
|
|
10
|
+
directory that runs the pinned Harbor through ``uvx``: a release from PyPI,
|
|
11
|
+
or a repository's commit built on this machine, whose wheels a host is sent
|
|
12
|
+
(only this machine needs access to a private fork; the host resolves the
|
|
13
|
+
rest from PyPI). A plan from before the pin names none, and its
|
|
14
|
+
``$NULLIUS_HARBOR`` is ``harbor``, whichever is first on the target's PATH.
|
|
15
|
+
The job records which Harbor ran. An ``Execution`` carries the plan out in a
|
|
16
|
+
thread of its own and writes what it does to a log the page follows:
|
|
17
|
+
|
|
18
|
+
- on a host: ssh in; docker checked for, with ``uv`` for a pinned Harbor
|
|
19
|
+
(installed when missing) or ``harbor`` on the host's PATH for an unpinned
|
|
20
|
+
one (``~/.local/bin`` included); the task folder sent over; each agent's
|
|
21
|
+
script started **detached** there (``setsid nohup``, or Python session creation when
|
|
22
|
+
``setsid`` is absent; its claim, pid and
|
|
23
|
+
exit code on disk beside its log: ``run-<part>.sh``, ``.launch``, ``.pid``,
|
|
24
|
+
``.log``, ``.exit``, and ``.stop`` once a Stop has been said to it), so a
|
|
25
|
+
dropped connection or a stopped Nullius does not end it; the job
|
|
26
|
+
directory synced back with rsync every few seconds while it runs and once
|
|
27
|
+
more at the end, so each trial appears here as it finishes;
|
|
28
|
+
- on this machine: the same, with the local paths, in ``runs/<name>``; a
|
|
29
|
+
run that needs docker (Harbor's, or the standard's on its container backend)
|
|
30
|
+
first asks that it answer here (``docker_here``) and starts nothing when
|
|
31
|
+
it does not.
|
|
32
|
+
|
|
33
|
+
A plan for the Nullius standard (``runner`` ``standard``,
|
|
34
|
+
``standard/recipe.py``) runs this repository's own ``cli`` from
|
|
35
|
+
``$NULLIUS_STANDARD``: this repository here; from an installed wheel or on
|
|
36
|
+
a host, a content-addressed copy of the backend and effective catalogue.
|
|
37
|
+
Only code, definitions and their declared resources travel, never user
|
|
38
|
+
configuration or credentials. Snapshots live in ``standard-app/<content-id>/``
|
|
39
|
+
beside the runs and are sent to ``~/.nullius/app/<content-id>`` the way a task
|
|
40
|
+
folder is. There it needs ``python3`` and ``git``, gets ``uv`` when missing
|
|
41
|
+
(an agent's pip install builds with it), and docker when the plan says so
|
|
42
|
+
(``docker``: a recipe on the container backend) - checked, like a Harbor run's,
|
|
43
|
+
before anything is sent. A run split over several machines is one plan per
|
|
44
|
+
machine, each one shard (``standard/shard.py``).
|
|
45
|
+
|
|
46
|
+
The target reads an explicit catalogue: the user's catalogue for a local
|
|
47
|
+
run; on a host, a writable copy of the snapshot in the run's ``catalogue/``.
|
|
48
|
+
Proofs update that copy, never the shared runner. Both ``jobs/`` and
|
|
49
|
+
``catalogue/`` are mirrored back beside the run record; remote proofs do
|
|
50
|
+
not replace the local user's active definitions.
|
|
51
|
+
|
|
52
|
+
An agent's run that fails with more to come pauses the run and the page
|
|
53
|
+
asks: continue or stop. Every state change lands in ``runs/<name>/run.json``;
|
|
54
|
+
``restore`` reads those records when Nullius starts and reattaches to what
|
|
55
|
+
is still running, waits again where it was paused, and settles what ended
|
|
56
|
+
while nobody was looking. A record is checked against the part's files and
|
|
57
|
+
processes on the target before it is trusted (``_settle``): a part whose pid
|
|
58
|
+
was never saved, but which was claimed or started, is followed by the pid it
|
|
59
|
+
wrote itself or by its processes - never started a second time; a record
|
|
60
|
+
from before a part's end was one save, naming the next part with the ended
|
|
61
|
+
one's pid, goes on from that end; and one saved as the next part was
|
|
62
|
+
launched, still holding the ended one's pid, follows the next part by its
|
|
63
|
+
own processes.
|
|
64
|
+
|
|
65
|
+
A pid is a hint, never proof: a part's processes are the ones running its
|
|
66
|
+
script in its run directory (``_part_pids``). A part is alive while it has
|
|
67
|
+
one, gone without an exit code when it has none, and nothing else is
|
|
68
|
+
signalled - not a pid a record kept after its part ended, nor one the system
|
|
69
|
+
has since given to another process. A Stop ends the part it raced, whether
|
|
70
|
+
or not its launcher had answered (``_kill_part``).
|
|
71
|
+
|
|
72
|
+
The model key and endpoint follow the recipe's provider (``Plan.key_var``,
|
|
73
|
+
``url_var``, ``base_url``): on OpenRouter the Settings' OPENROUTER_API_KEY
|
|
74
|
+
and OpenRouter's own endpoint; on OpenAI, OPENAI_API_KEY and no endpoint, so
|
|
75
|
+
the clients use OpenAI's; on an OpenAI-compatible endpoint, its URL and its
|
|
76
|
+
key (MODEL_API_URL and MODEL_API_KEY, or a named endpoint's own) - the key
|
|
77
|
+
optional: without it the run is sent
|
|
78
|
+
``settings.MODEL_KEY_PLACEHOLDER``, for a server that checks no key - and a
|
|
79
|
+
run without that endpoint fails before it starts.
|
|
80
|
+
They are read on this machine and handed to the run as environment
|
|
81
|
+
(API_TOKEN, OPENAI_API_KEY, LLM_API_KEY; API_URL, OPENAI_BASE_URL,
|
|
82
|
+
LLM_BASE_URL) - over stdin to the launcher, never in argv (the key to a
|
|
83
|
+
docker relay as a file); a stdin cut short starts nothing and fails the
|
|
84
|
+
run at that part (``_launch``), and one that holds a line break is never sent
|
|
85
|
+
(``_line_break``) - and they are the only credentials it has: its commands get this
|
|
86
|
+
process's environment less every secret in Settings, whichever provider it
|
|
87
|
+
belongs to (``withheld``); a host gets nothing of it but stdin.
|
|
88
|
+
|
|
89
|
+
A standard run on a host is also sent its benchmark's licences
|
|
90
|
+
(``Plan.licences``: the benchmark and the names its environments declare,
|
|
91
|
+
read from its store, ``nullius.secrets``, or a data provider's token from that
|
|
92
|
+
provider's, when each part is launched) as one
|
|
93
|
+
more stdin line, ``NULLIUS_LICENCES``: in the launcher's environment, never
|
|
94
|
+
on disk there - but for a licence file, which the run writes to
|
|
95
|
+
``/dev/shm/nullius-<run>/`` (0600, in memory) and the part's detached shell
|
|
96
|
+
removes when it ends, a Stop included. On this machine the run reads the
|
|
97
|
+
store itself, and nothing is sent. A record from before ``base_url`` loads with none, and its run
|
|
98
|
+
goes on with the settings it named. Log lines pass through
|
|
99
|
+
``redact`` before they are kept. Every command runner is a call-time seam,
|
|
100
|
+
so the suite runs a plan with none of ssh, uv, rsync or docker present.
|
|
101
|
+
|
|
102
|
+
What the run is doing is said in stages (``nullius/progress.py``): the steps
|
|
103
|
+
here that get a host ready - connect, uv, the tasks, app/, a pinned Harbor -
|
|
104
|
+
each a ``[nullius] stage=...`` line in the run's log, and the run's own (the
|
|
105
|
+
standard's cli says its image, installs, dataset and units) read from the
|
|
106
|
+
part's log as it comes. ``events`` holds them all, read back from the log on
|
|
107
|
+
a restart, and ``summary``'s ``stages`` is them folded for the page.
|
|
108
|
+
"""
|
|
109
|
+
from __future__ import annotations
|
|
110
|
+
|
|
111
|
+
import base64
|
|
112
|
+
import hashlib
|
|
113
|
+
import json
|
|
114
|
+
import os
|
|
115
|
+
import re
|
|
116
|
+
import shlex
|
|
117
|
+
import subprocess
|
|
118
|
+
import sys
|
|
119
|
+
import threading
|
|
120
|
+
import time
|
|
121
|
+
from dataclasses import asdict, dataclass, field
|
|
122
|
+
from datetime import datetime, timezone
|
|
123
|
+
from typing import Callable, Dict, List, Optional
|
|
124
|
+
|
|
125
|
+
from nullius import hosts, paths, prices as price_list, progress, redact, secrets, settings
|
|
126
|
+
from nullius.harbor import source as harbor_source
|
|
127
|
+
from nullius.standard import container, payload as standard_payload
|
|
128
|
+
|
|
129
|
+
REMOTE_HOME = "~/.nullius" # on a host: tasks/, runs/, app/, harbor/ (a pinned Harbor's wheels)
|
|
130
|
+
# The names the launcher gives a run's key and its endpoint (_launch), read
|
|
131
|
+
# from stdin: the only credentials a run's commands get.
|
|
132
|
+
KEY_NAMES = ("API_TOKEN", "OPENAI_API_KEY", "LLM_API_KEY")
|
|
133
|
+
URL_NAMES = ("API_URL", "OPENAI_BASE_URL", "LLM_BASE_URL")
|
|
134
|
+
# The line the launcher's stdin ends with, after the key and the endpoint:
|
|
135
|
+
# either may be empty, so only this line says the two arrived whole.
|
|
136
|
+
STDIN_END = "@end"
|
|
137
|
+
SYNC_EVERY = 10.0 # seconds between looks at a running part (mirror, log, exit)
|
|
138
|
+
STOP_POLL = 0.5 # how often a running command looks for a Stop
|
|
139
|
+
STOP_WAIT = 20.0 # seconds a Stop's own commands on the target take at most: the kill and the last copy back, together
|
|
140
|
+
LAUNCH_GRACE = 5.0 # seconds a restart waits for a part a launcher claimed to start (_settle)
|
|
141
|
+
UV_INSTALL = "curl -LsSf https://astral.sh/uv/install.sh | sh"
|
|
142
|
+
RECORD = "run.json"
|
|
143
|
+
STANDARD = "standard" # the Nullius standard's runner key (standard/recipe.py HARNESS)
|
|
144
|
+
REPO = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # this repository: app/ is in it
|
|
145
|
+
_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
|
|
146
|
+
_ID = re.compile(r"[^A-Za-z0-9_.-]")
|
|
147
|
+
ALIVE = ("preparing", "running", "paused", "stopping")
|
|
148
|
+
NOT_RUN = "not run"
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def runs_root(env=os.environ) -> str:
|
|
152
|
+
return env.get("RUNS_DIR") or paths.runs_dir(env)
|
|
153
|
+
|
|
154
|
+
|
|
155
|
+
def app_root(runs: str) -> str:
|
|
156
|
+
"""Content-addressed runner snapshots, in ``standard-app/`` beside runs."""
|
|
157
|
+
return os.path.join(os.path.dirname(os.path.abspath(runs)), "standard-app")
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def docker_here(sh: Callable, needs: str = "a Harbor run needs it", program: str = "docker") -> Optional[str]:
|
|
161
|
+
"""Why a run cannot use ``program`` on this machine, or None when it can
|
|
162
|
+
(``needs`` says why the run asks): ``docker`` for Harbor, the standard's
|
|
163
|
+
engine (``container.DOCKER``) for its container backend. The command on the
|
|
164
|
+
PATH *and* ``<program> ps`` answered, as a host's probe asks
|
|
165
|
+
(``hosts.PROBE``). The command alone is not enough - Docker Desktop's WSL
|
|
166
|
+
integration leaves ``/usr/bin/docker`` in place while its engine is quit.
|
|
167
|
+
``sh`` is the run's own (returncode, output) runner; this is looked up at
|
|
168
|
+
call time, so the suite can stand in for docker."""
|
|
169
|
+
rc, out = sh(f"command -v {program} >/dev/null 2>&1 || {{ echo @missing; exit 0; }}; {program} ps 2>&1 >/dev/null || echo @down", 60)
|
|
170
|
+
words = out.split()
|
|
171
|
+
if rc != 0 or "@missing" in words:
|
|
172
|
+
return f"{program} is not on this machine; " + needs
|
|
173
|
+
if "@down" in words:
|
|
174
|
+
said = [l for l in out.strip().splitlines() if l.strip() and l.strip() != "@down"]
|
|
175
|
+
return (f"{program} is on this machine but does not answer ({program} ps failed)"
|
|
176
|
+
+ (" - is Docker Desktop or the docker daemon running?" if program == "docker" else "")
|
|
177
|
+
+ (": " + said[-1].strip()[-300:] if said else ""))
|
|
178
|
+
return None
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
def _tasks_place(tasks: str, run: str) -> tuple:
|
|
182
|
+
"""Where a run's task folder goes on a host, under ~/.nullius: by the
|
|
183
|
+
folder, not the run, so two runs over the same tasks record the same
|
|
184
|
+
dataset and match on the compare page - the folder's name last, under a
|
|
185
|
+
short digest of its whole path, so /a/tasks and /b/tasks do not share
|
|
186
|
+
one copy that ``rsync --delete`` swaps under a run still reading it."""
|
|
187
|
+
src = os.path.realpath(os.path.expanduser(tasks))
|
|
188
|
+
name = _ID.sub("_", os.path.basename(os.path.normpath(src))) or run
|
|
189
|
+
return ("tasks", hashlib.sha256(src.encode()).hexdigest()[:10], name)
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _rq(path: str) -> str:
|
|
193
|
+
"""A remote path quoted for the host's shell: ``~`` becomes ``$HOME`` so it
|
|
194
|
+
still expands inside the quotes. The names inside are ours (validated)."""
|
|
195
|
+
return '"' + (("$HOME" + path[1:]) if path.startswith("~") else path).replace('"', '\\"') + '"'
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def name_taken(name: str, where: str = "") -> str:
|
|
199
|
+
"""What a new run is told when a run directory of its name is there
|
|
200
|
+
already - here (``web/execute.py``, before it starts) or on a host
|
|
201
|
+
(``where``, asked by the run before it sends anything)."""
|
|
202
|
+
return f"a run named {name} exists already{' on ' + where if where else ''}; press Run again for a new name"
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _base(part: dict) -> str:
|
|
206
|
+
"""The stem of a part's files in the run directory: ``run-<part>``."""
|
|
207
|
+
return f"run-{_ID.sub('_', part['id'])}"
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _script_rx(base: str) -> str:
|
|
211
|
+
"""An extended regex for a command line that runs ``<base>.sh``: the word
|
|
212
|
+
itself, alone or at the end of a path. Its first letter is bracketed so
|
|
213
|
+
the shell that asks, whose own command line holds the regex, never
|
|
214
|
+
matches it."""
|
|
215
|
+
return "(^|[ /])[" + base[0] + "]" + base[1:].replace(".", r"\.") + r"\.sh( |$)"
|
|
216
|
+
|
|
217
|
+
|
|
218
|
+
# Where a target shows its processes. A target without it (no /proc: not
|
|
219
|
+
# Linux) cannot prove a pid is still the part's, and falls back to asking
|
|
220
|
+
# whether the pid lives at all - what every target was asked before.
|
|
221
|
+
PROC = "/proc"
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _part_pids(base: str, pids=()) -> str:
|
|
225
|
+
"""Shell, run in the run directory, that prints the part's processes, one
|
|
226
|
+
pid per line: those whose working directory is the run directory and
|
|
227
|
+
whose command line runs ``<base>.sh`` - the detached shell, the script's
|
|
228
|
+
own bash, or a launcher on its way to them. Asked of ``pids`` (what the
|
|
229
|
+
record holds), of the pid the part wrote, and of whatever ``pgrep`` finds
|
|
230
|
+
by that command line, so a part is found with no pid saved. A pid the
|
|
231
|
+
part held, since given to another process, is never printed, so nothing
|
|
232
|
+
that is not the part's is ever signalled - where the target has ``PROC``
|
|
233
|
+
to check it by; where it has none, a pid that lives is taken as the
|
|
234
|
+
part's, as before these checks."""
|
|
235
|
+
rx = _script_rx(base)
|
|
236
|
+
return (f"for p in {' '.join(str(int(p)) for p in pids if p)} $(head -c 32 {base}.pid 2>/dev/null | tr -cd 0-9) "
|
|
237
|
+
f"$(pgrep -f '{rx}' 2>/dev/null); do [ \"$p\" != $$ ] || continue; "
|
|
238
|
+
f"if [ -d {PROC}/$$ ]; then [ {PROC}/$p/cwd -ef . ] && tr '\\0' ' ' <{PROC}/$p/cmdline 2>/dev/null | grep -Eq '{rx}' && echo $p; "
|
|
239
|
+
f"else kill -0 $p 2>/dev/null && echo $p; fi; "
|
|
240
|
+
f"done | sort -un")
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
@dataclass
|
|
244
|
+
class Plan:
|
|
245
|
+
name: str # the run's name: a directory here and on the host
|
|
246
|
+
script: str # the whole run, as one script (kept for the record)
|
|
247
|
+
host: Optional[hosts.Host] = None # None: this machine
|
|
248
|
+
tasks: str = "" # a local task folder the recipe reads, or ""
|
|
249
|
+
key_var: str = "MODEL_API_KEY" # the setting the key is read from, on this machine
|
|
250
|
+
url_var: str = "MODEL_API_URL" # same for the endpoint, which a run on it cannot go without; "" for none
|
|
251
|
+
base_url: str = "" # the endpoint when url_var names none: a provider's own (OpenRouter's), or "" for its clients' default
|
|
252
|
+
model: str = ""
|
|
253
|
+
jobs: List[dict] = field(default_factory=list) # the job directories the script writes: [{"name", "agent", "dataset"}]
|
|
254
|
+
parts: List[dict] = field(default_factory=list) # the run, one agent at a time: [{"id", "label", "script"}]; empty: the whole script at once
|
|
255
|
+
recipe: dict = field(default_factory=dict) # the builder's recipe the script was compiled from, so the run can be opened again as one; {} for a plan from before
|
|
256
|
+
runner: str = "harbor" # or "standard" (standard/recipe.py): the Nullius standard's own cli
|
|
257
|
+
harbor_pin: dict = field(default_factory=dict) # the Harbor the recipe pins (harbor/source.py pin_of); {} for a plan from before: the harbor on PATH
|
|
258
|
+
docker: bool = False # a runner that needs docker on its target though it is not Harbor (the standard's container backend)
|
|
259
|
+
keyless: bool = False # nothing in it calls a model (the standard's controls): handed no key and no endpoint at all
|
|
260
|
+
licences: dict = field(default_factory=dict) # the standard's on a host: {"benchmark", "names", "providers"} - what it is sent of that benchmark's licences
|
|
261
|
+
|
|
262
|
+
def steps(self) -> List[dict]:
|
|
263
|
+
return [p for p in self.parts if p.get("script")] or [{"id": "run", "label": "the run", "script": self.script}]
|
|
264
|
+
|
|
265
|
+
def check(self) -> Optional[str]:
|
|
266
|
+
if not _NAME.match(self.name):
|
|
267
|
+
return "the run needs a name: letters, digits, dots, dashes"
|
|
268
|
+
if not self.script.strip() and not any(p.get("script") for p in self.parts):
|
|
269
|
+
return "nothing to run"
|
|
270
|
+
if self.tasks and not os.path.isdir(os.path.expanduser(self.tasks)):
|
|
271
|
+
return f"the task folder {self.tasks} is not a directory here"
|
|
272
|
+
return None
|
|
273
|
+
|
|
274
|
+
def to_dict(self) -> dict:
|
|
275
|
+
d = asdict(self)
|
|
276
|
+
d["host"] = self.host.entry if self.host else None
|
|
277
|
+
return d
|
|
278
|
+
|
|
279
|
+
@classmethod
|
|
280
|
+
def from_dict(cls, d: dict) -> "Plan":
|
|
281
|
+
"""A recorded plan; a field this version no longer has (a record
|
|
282
|
+
from before, naming an installed Harbor release) is left out."""
|
|
283
|
+
d = dict(d)
|
|
284
|
+
d["host"] = hosts.parse(d["host"]) if d.get("host") else None
|
|
285
|
+
return cls(**{k: v for k, v in d.items() if k in cls.__dataclass_fields__})
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def subprocess_run(argv: List[str], timeout: float = 600.0, stdin: Optional[str] = None,
|
|
289
|
+
stop: Optional[threading.Event] = None) -> tuple:
|
|
290
|
+
"""(returncode, combined output); a timeout is a return code of its own,
|
|
291
|
+
and so is a stop: with ``stop`` given, the child is killed the moment the
|
|
292
|
+
event is set (130, "stopped") rather than when its timeout runs out -
|
|
293
|
+
an ssh that hangs for 120 s, an rsync for 900 s, would otherwise hold
|
|
294
|
+
a Stop for that long."""
|
|
295
|
+
try:
|
|
296
|
+
# read as UTF-8 whatever the locale, a byte that is not one as U+FFFD: a run is never failed by how
|
|
297
|
+
# its target spelt what it printed
|
|
298
|
+
proc = subprocess.Popen(argv, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
|
|
299
|
+
encoding="utf-8", errors="replace", env=_clean_env())
|
|
300
|
+
except OSError as e:
|
|
301
|
+
return 127, str(e)
|
|
302
|
+
deadline = time.monotonic() + timeout
|
|
303
|
+
got = {}
|
|
304
|
+
|
|
305
|
+
def talk():
|
|
306
|
+
# one communicate, whole: a retried one after a timeout stopped sending
|
|
307
|
+
# input past the pipe's buffer (a 465 KB price list never landed)
|
|
308
|
+
try:
|
|
309
|
+
got["out"] = proc.communicate(input=stdin if stdin is not None else "")[0]
|
|
310
|
+
except (OSError, ValueError) as e:
|
|
311
|
+
got["out"] = str(e)
|
|
312
|
+
talker = threading.Thread(target=talk, daemon=True)
|
|
313
|
+
talker.start()
|
|
314
|
+
while True:
|
|
315
|
+
talker.join(STOP_POLL if stop is not None else max(0.0, deadline - time.monotonic()))
|
|
316
|
+
if not talker.is_alive():
|
|
317
|
+
return proc.returncode, got.get("out") or ""
|
|
318
|
+
if stop is not None and stop.is_set():
|
|
319
|
+
proc.kill(); talker.join()
|
|
320
|
+
return 130, "stopped"
|
|
321
|
+
if time.monotonic() >= deadline:
|
|
322
|
+
proc.kill(); talker.join()
|
|
323
|
+
return 124, f"no answer within {int(timeout)} s"
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
# A part's log, as a look at the target prints it: between these, then its size in bytes (_log_since, _logged).
|
|
327
|
+
LOG_FROM, LOG_TO = "__log__", "__logged__"
|
|
328
|
+
|
|
329
|
+
|
|
330
|
+
def withheld() -> List[str]:
|
|
331
|
+
"""The variables a run's commands never take from this process: every
|
|
332
|
+
secret in Settings (the model keys and endpoints, the judge's key), the
|
|
333
|
+
names other tools give them or this application gave them before
|
|
334
|
+
(``settings.INSTEAD``, those that stand for a secret), the launcher's own
|
|
335
|
+
names for the run's key and endpoint, the agent keys (a Harbor run is
|
|
336
|
+
sent them by its launcher, as the key), the names a standard benchmark's
|
|
337
|
+
own licences go by - kept in its own store (``nullius.secrets``), although
|
|
338
|
+
imported saved settings may still hold a value until it is moved in
|
|
339
|
+
Custom keys - and GitHub tokens from saved settings or the
|
|
340
|
+
shell Nullius was started in. Every named endpoint's settings too, so a
|
|
341
|
+
run gets its own endpoint's key alone. A data provider's token name is not one."""
|
|
342
|
+
secret = {v.name for v in settings.VARIABLES if v.secret or v.kind == "endpoint"} | set(settings.endpoint_names())
|
|
343
|
+
return sorted(secret | {old for old, new in settings.INSTEAD.items() if new in secret}
|
|
344
|
+
| set(KEY_NAMES + URL_NAMES) | {"GITHUB_TOKEN", "GH_TOKEN"} | set(settings.agent_key_names())
|
|
345
|
+
| set(_licence_names()))
|
|
346
|
+
|
|
347
|
+
|
|
348
|
+
def _licence_names() -> List[str]:
|
|
349
|
+
"""Every name a standard benchmark's environments give a licence of its
|
|
350
|
+
own; a data provider's token (``provider``) is none, so a shell's reaches
|
|
351
|
+
a Harbor run. A standard driver or agent never inherits it (``run.KEEP``)."""
|
|
352
|
+
try:
|
|
353
|
+
from nullius.standard import definitions
|
|
354
|
+
return sorted({s["name"] for b in definitions.load("benchmark", every=True)[0] for s in definitions.secrets_of(b)
|
|
355
|
+
if not s.get("provider")})
|
|
356
|
+
except Exception:
|
|
357
|
+
return []
|
|
358
|
+
|
|
359
|
+
|
|
360
|
+
def _clean_env() -> Dict[str, str]:
|
|
361
|
+
"""The environment a run's commands get: this process's, less what
|
|
362
|
+
``withheld`` names. ``settings.load`` adds saved settings to this process's
|
|
363
|
+
environment without replacing explicit environment values. A run gets
|
|
364
|
+
its provider's key and endpoint from the launcher alone (``_launch``,
|
|
365
|
+
over stdin); ``--from-run`` prevents the CLI reading saved settings again."""
|
|
366
|
+
gone = set(withheld())
|
|
367
|
+
return {k: v for k, v in os.environ.items() if k not in gone}
|
|
368
|
+
|
|
369
|
+
|
|
370
|
+
class Execution:
|
|
371
|
+
"""One run of one plan, in its own thread. ``state`` is one of preparing,
|
|
372
|
+
running, paused, stopping, done, failed, stopped; ``log`` is what
|
|
373
|
+
happened, redacted."""
|
|
374
|
+
|
|
375
|
+
def __init__(self, plan: Plan, root: Optional[str] = None, run: Optional[Callable] = None,
|
|
376
|
+
key: Optional[str] = None, url: Optional[str] = None, env=os.environ, record: Optional[dict] = None,
|
|
377
|
+
agent_keys: Optional[Dict[str, str]] = None, licences: Optional[Dict[str, bytes]] = None):
|
|
378
|
+
self.plan = plan
|
|
379
|
+
self.root = root or runs_root(env)
|
|
380
|
+
self._catalogue = paths.catalogue_dir(env)
|
|
381
|
+
self.dir = os.path.join(self.root, plan.name)
|
|
382
|
+
self.jobs = os.path.join(self.dir, "jobs")
|
|
383
|
+
self.log_path = os.path.join(self.dir, "nullius.log")
|
|
384
|
+
self.pins = harbor_source.root(self.root, env) # where a pinned Harbor is built and kept here
|
|
385
|
+
self._stop = threading.Event()
|
|
386
|
+
# Every command the run makes gives way to a Stop; the kill that a
|
|
387
|
+
# Stop itself sends goes through the plain runner, or it would be
|
|
388
|
+
# the first command cut short.
|
|
389
|
+
self._run_plain = run or subprocess_run
|
|
390
|
+
self._run = run or (lambda argv, timeout=600.0, stdin=None: subprocess_run(argv, timeout, stdin, stop=self._stop))
|
|
391
|
+
# an endpoint of the run's own (url_var: an OpenAI-compatible one) may check no key: it is sent the placeholder
|
|
392
|
+
self._key = (key if key is not None else env.get(plan.key_var, "")) or (settings.MODEL_KEY_PLACEHOLDER if plan.url_var else "")
|
|
393
|
+
self._url = url if url is not None else (env.get(plan.url_var, "") if plan.url_var else plan.base_url)
|
|
394
|
+
if plan.keyless: # a run that calls no model is given nothing it could leak
|
|
395
|
+
self._key, self._url = "", ""
|
|
396
|
+
# the agent keys Settings holds (CURSOR_API_KEY, ...): a Harbor run's agent may need its own vendor's; the standard's never gets one
|
|
397
|
+
self._agent_keys = dict(agent_keys) if agent_keys is not None else (settings.agent_key_values(env) if plan.runner == "harbor" else {})
|
|
398
|
+
# the benchmark's licences a standard run on a host is sent: read from its store at each launch, unless given
|
|
399
|
+
self._licences_given = licences
|
|
400
|
+
self._redact: List[str] = [] # the licences' values sent, as text: never in a log line
|
|
401
|
+
self._decision = threading.Event() # set by continue_() or stop() while paused
|
|
402
|
+
# A Stop (the page's thread) and the run's own thread pausing or ending, each with the record
|
|
403
|
+
# it saves, one at a time: a Stop is never written over, a record never half of each.
|
|
404
|
+
self._turn = threading.RLock()
|
|
405
|
+
self._lock = threading.Lock()
|
|
406
|
+
self.thread: Optional[threading.Thread] = None
|
|
407
|
+
self._lines: List[str] = []
|
|
408
|
+
#: what a part is downloading, while it does: {label, done, total} in bytes, from its `@progress` lines
|
|
409
|
+
self.progress: Optional[dict] = None
|
|
410
|
+
#: every stage line the log holds, parsed (nullius/progress.py): this run's own and its part's
|
|
411
|
+
self._events: List[dict] = []
|
|
412
|
+
self._unread = False # a restored run that ended: its log is read when first asked for
|
|
413
|
+
self._open: Optional[tuple] = None # (stage, monotonic start) of the step here under way, said as a stage
|
|
414
|
+
rec = record or {}
|
|
415
|
+
self.id = rec.get("id") or f"{plan.name}-{int(time.time())}"
|
|
416
|
+
self.state = rec.get("state", "preparing")
|
|
417
|
+
self.step = rec.get("step", "")
|
|
418
|
+
self.part = int(rec.get("part", 0)) # index of the part running or next
|
|
419
|
+
self.pid: Optional[int] = rec.get("pid")
|
|
420
|
+
# the part that may be running on the target, from its launch until its end is recorded: what a Stop ends
|
|
421
|
+
self._live: Optional[dict] = None
|
|
422
|
+
# the last part whose launch began, saved before its script is sent, so a restart tells a Stop that cut its
|
|
423
|
+
# launch short from one before it (_kill_part); "" in a record from before
|
|
424
|
+
self.launched: str = rec.get("launched") or ""
|
|
425
|
+
if self.state == "stopping": # a Stop that a restart interrupted: finish it, do not run on
|
|
426
|
+
self._stop.set()
|
|
427
|
+
self.log_offset = int(rec.get("log_offset", 0))
|
|
428
|
+
self.exit_code: Optional[int] = rec.get("exit_code")
|
|
429
|
+
self.error: Optional[str] = rec.get("error")
|
|
430
|
+
self.question: Optional[str] = rec.get("question")
|
|
431
|
+
#: each part that ran to its end: {part id: {"rc", "at", "label"}} - an agent's own result in a run of several
|
|
432
|
+
self.ends: Dict[str, dict] = dict(rec.get("ends") or {})
|
|
433
|
+
self.started = rec.get("started") or time.time()
|
|
434
|
+
self.finished: Optional[float] = rec.get("finished")
|
|
435
|
+
self.harbor = rec.get("harbor", "") # the harbor executable on the target
|
|
436
|
+
self.tasks_on_target = rec.get("tasks_on_target", "")
|
|
437
|
+
self.app_on_target = rec.get("app_on_target", "") # the standard's source tree or runner snapshot on the target
|
|
438
|
+
self.catalogue_on_target = rec.get("catalogue_on_target", "") # writable definitions, separate from the immutable runner
|
|
439
|
+
# the trials the log has said already, (job, trial), so a restart says each once; none in a record from before
|
|
440
|
+
self._seen_trials: set = {(k[0], k[1]) for k in rec.get("seen_trials") or ()}
|
|
441
|
+
if record:
|
|
442
|
+
# a run still going reads its log now, to go on from it; one that ended (most of them, and
|
|
443
|
+
# history only grows) when its page asks - start-up read every one, seconds and hundreds of MB
|
|
444
|
+
self._unread = True
|
|
445
|
+
if self.state in ALIVE:
|
|
446
|
+
self._read_log()
|
|
447
|
+
|
|
448
|
+
def _read_log(self) -> None:
|
|
449
|
+
self._unread = False
|
|
450
|
+
try:
|
|
451
|
+
with open(self.log_path, encoding="utf-8") as fh:
|
|
452
|
+
self._lines = fh.read().splitlines()
|
|
453
|
+
except OSError:
|
|
454
|
+
pass
|
|
455
|
+
self._events = [e for e in map(progress.parse, self._lines) if e]
|
|
456
|
+
|
|
457
|
+
@property
|
|
458
|
+
def lines(self) -> List[str]:
|
|
459
|
+
if self._unread:
|
|
460
|
+
self._read_log()
|
|
461
|
+
return self._lines
|
|
462
|
+
|
|
463
|
+
@lines.setter
|
|
464
|
+
def lines(self, value: List[str]) -> None:
|
|
465
|
+
self._unread, self._lines = False, value
|
|
466
|
+
|
|
467
|
+
@property
|
|
468
|
+
def events(self) -> List[dict]:
|
|
469
|
+
if self._unread:
|
|
470
|
+
self._read_log()
|
|
471
|
+
return self._events
|
|
472
|
+
|
|
473
|
+
@events.setter
|
|
474
|
+
def events(self, value: List[dict]) -> None:
|
|
475
|
+
self._unread, self._events = False, value
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
def _save(self) -> None:
|
|
479
|
+
with self._turn:
|
|
480
|
+
data = {"id": self.id, "plan": self.plan.to_dict(), "state": self.state, "step": self.step, "part": self.part, "pid": self.pid,
|
|
481
|
+
"log_offset": self.log_offset, "exit_code": self.exit_code, "error": self.error, "question": self.question, "ends": self.ends,
|
|
482
|
+
"started": self.started, "finished": self.finished, "harbor": self.harbor, "tasks_on_target": self.tasks_on_target, "app_on_target": self.app_on_target,
|
|
483
|
+
"catalogue_on_target": self.catalogue_on_target,
|
|
484
|
+
"launched": self.launched, "seen_trials": [list(k) for k in sorted(self._seen_trials, key=str)]}
|
|
485
|
+
try:
|
|
486
|
+
os.makedirs(self.dir, exist_ok=True)
|
|
487
|
+
tmp = os.path.join(self.dir, RECORD + ".tmp")
|
|
488
|
+
with open(tmp, "w", encoding="utf-8") as fh:
|
|
489
|
+
json.dump(data, fh, indent=1)
|
|
490
|
+
os.replace(tmp, os.path.join(self.dir, RECORD))
|
|
491
|
+
except OSError:
|
|
492
|
+
pass
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def log(self, line: str) -> None:
|
|
496
|
+
if line.startswith("** ") and ("post-quantum" in line or "store now, decrypt later" in line or "openssh.com/pq" in line):
|
|
497
|
+
return # ssh's own warning about the host's key exchange, not the run's
|
|
498
|
+
clean = redact.redact(line.rstrip("\n"))
|
|
499
|
+
if self._key and self._key != settings.MODEL_KEY_PLACEHOLDER: # the placeholder is no secret, and a common word
|
|
500
|
+
clean = clean.replace(self._key, "[REDACTED]")
|
|
501
|
+
for value in list(self._agent_keys.values()) + self._redact:
|
|
502
|
+
if len(value) >= 8:
|
|
503
|
+
clean = clean.replace(value, "[REDACTED]")
|
|
504
|
+
ev = progress.parse(clean)
|
|
505
|
+
with self._lock:
|
|
506
|
+
self.lines.append(clean)
|
|
507
|
+
if ev is not None:
|
|
508
|
+
self.events.append(ev)
|
|
509
|
+
try:
|
|
510
|
+
os.makedirs(self.dir, exist_ok=True)
|
|
511
|
+
with open(self.log_path, "a", encoding="utf-8") as fh:
|
|
512
|
+
fh.write(clean + "\n")
|
|
513
|
+
except OSError:
|
|
514
|
+
pass
|
|
515
|
+
|
|
516
|
+
def _log_since(self, base: str) -> str:
|
|
517
|
+
"""Shell that prints a part's log from ``log_offset`` to its size
|
|
518
|
+
now, between markers, and that size after them. The offset then
|
|
519
|
+
moves by the log's own bytes: never by what the transport prints
|
|
520
|
+
around it - OpenSSH warns on stderr at every connection to a host
|
|
521
|
+
without a post-quantum key exchange, 222 bytes ahead of the command's
|
|
522
|
+
output - nor by what reading it as text changes (a \\r\\n read as
|
|
523
|
+
\\n), so a look never skips the log nor starts inside a character."""
|
|
524
|
+
off = self.log_offset
|
|
525
|
+
return (f"n=$( {{ wc -c <{base}.log; }} 2>/dev/null | tr -d ' '); [ -n \"$n\" ] || n=0; printf '\\n{LOG_FROM}\\n'; "
|
|
526
|
+
f"[ \"$n\" -gt {off} ] && tail -c +{off + 1} {base}.log | head -c $((n - {off})); printf '\\n{LOG_TO} %s\\n' \"$n\"")
|
|
527
|
+
|
|
528
|
+
@staticmethod
|
|
529
|
+
def _logged(out: str) -> tuple:
|
|
530
|
+
"""(the log text, the log's size in bytes) from what a `_log_since`
|
|
531
|
+
command printed, and nothing printed around it; ("", None) when it
|
|
532
|
+
did not print them."""
|
|
533
|
+
_, start, rest = out.partition(f"\n{LOG_FROM}\n")
|
|
534
|
+
text, end, size = rest.rpartition(f"\n{LOG_TO} ")
|
|
535
|
+
n = size.strip()
|
|
536
|
+
return (text, int(n)) if start and end and n.isdigit() else ("", None)
|
|
537
|
+
|
|
538
|
+
def _read(self, line: str) -> None:
|
|
539
|
+
"""One line of a part's log: `@progress <label> <done> <total>` is
|
|
540
|
+
the bar the run's page draws while a part downloads, never a log
|
|
541
|
+
line; any other line ends it and is logged."""
|
|
542
|
+
if line.startswith("@progress "):
|
|
543
|
+
bits = line.split()
|
|
544
|
+
if len(bits) == 4 and bits[2].isdigit() and bits[3].isdigit():
|
|
545
|
+
self.progress = {"label": bits[1], "done": int(bits[2]), "total": int(bits[3])}
|
|
546
|
+
return
|
|
547
|
+
self.progress = None
|
|
548
|
+
self.log(line)
|
|
549
|
+
|
|
550
|
+
def tail(self, since: int = 0) -> List[str]:
|
|
551
|
+
with self._lock:
|
|
552
|
+
return self.lines[since:]
|
|
553
|
+
|
|
554
|
+
def part_for(self, job_id: str) -> Optional[str]:
|
|
555
|
+
"""The part that writes a job: the agent it names (an arm of a run of several), or
|
|
556
|
+
the part its name ends with (a Harbor agent in a run of several); the
|
|
557
|
+
only part of a run of one."""
|
|
558
|
+
steps = self.plan.steps()
|
|
559
|
+
job = next((j for j in self.plan.jobs if str(j.get("name")) == job_id), None)
|
|
560
|
+
if job is None:
|
|
561
|
+
return None
|
|
562
|
+
for p in steps:
|
|
563
|
+
if job.get("agent") == p["id"] or job_id == f"{self.plan.name}-{p['id']}":
|
|
564
|
+
return p["id"]
|
|
565
|
+
return steps[0]["id"] if len(steps) == 1 else None
|
|
566
|
+
|
|
567
|
+
def job_state(self, job_id: str) -> Optional[tuple]:
|
|
568
|
+
"""``(state, finished_at)`` for one job of this run once it has ended:
|
|
569
|
+
its own part's result when that part ran to its end - an arm that
|
|
570
|
+
answered every question is done though a later one failed - else the
|
|
571
|
+
run's, once the run has ended. None while it is still to come."""
|
|
572
|
+
end = self.ends.get(self.part_for(job_id) or "")
|
|
573
|
+
if end is not None:
|
|
574
|
+
return ("done" if end["rc"] == 0 else "failed"), datetime.fromtimestamp(end["at"], tz=timezone.utc)
|
|
575
|
+
if self.state in ENDED and self.finished:
|
|
576
|
+
return self.state, datetime.fromtimestamp(self.finished, tz=timezone.utc)
|
|
577
|
+
return None
|
|
578
|
+
|
|
579
|
+
def refused(self, job_id: str) -> Optional[str]:
|
|
580
|
+
"""Why one job of this run was never written, when the run was refused
|
|
581
|
+
before it: the standard's cli said so in the job's folder
|
|
582
|
+
(``stage=refused`` in its ``progress.jsonl``, the only thing it
|
|
583
|
+
wrote there), or the run failed before any part's launch began - a
|
|
584
|
+
key not set, a host without docker. None for a run that got further,
|
|
585
|
+
whose job is elsewhere or still to come. The folder is kept: it holds
|
|
586
|
+
the refusal and the image's build log, and nothing reads it as a job."""
|
|
587
|
+
if self.state in ALIVE:
|
|
588
|
+
return None
|
|
589
|
+
why = progress.refusal(progress.read(os.path.join(self.jobs, job_id)))
|
|
590
|
+
if why is None and self.state == "failed" and not self.launched and not self.ends:
|
|
591
|
+
why = self.error or "the run failed before it started"
|
|
592
|
+
return why
|
|
593
|
+
|
|
594
|
+
def live_state(self, job_id: str) -> Optional[str]:
|
|
595
|
+
"""What one job of this run is doing while the run lives: the run's
|
|
596
|
+
state for the part running now, ``queued`` for a part still to come
|
|
597
|
+
(the parts run one after another), and the run's own while it
|
|
598
|
+
prepares or stops, which is every part's. ``not run`` for one held
|
|
599
|
+
back by a pause: nothing starts it unless the page says continue. None
|
|
600
|
+
once the run ended."""
|
|
601
|
+
if self.state not in ALIVE:
|
|
602
|
+
return None
|
|
603
|
+
part = self.part_for(job_id)
|
|
604
|
+
if self.state in ("preparing", "stopping") or part is None or part == self.launched:
|
|
605
|
+
return self.state
|
|
606
|
+
return NOT_RUN if self.state == "paused" else "queued"
|
|
607
|
+
|
|
608
|
+
def never_ran(self, job_id: str) -> bool:
|
|
609
|
+
"""The run ended before this job's part began: nothing of it was launched."""
|
|
610
|
+
ids = [p["id"] for p in self.plan.steps()]
|
|
611
|
+
part = self.part_for(job_id)
|
|
612
|
+
return self.state in ENDED and part in ids and part != self.launched and part not in self.ends and ids.index(part) >= self.part
|
|
613
|
+
|
|
614
|
+
def summary(self) -> dict:
|
|
615
|
+
return {"id": self.id, "name": self.plan.name, "state": self.state, "step": self.step, "host": self.plan.host.label if self.plan.host else "this machine",
|
|
616
|
+
"exit_code": self.exit_code, "error": self.error, "started": self.started, "finished": self.finished, "jobs": self.jobs,
|
|
617
|
+
"job_ids": [str(j.get("name")) for j in self.plan.jobs], "lines": len(self.lines), "question": self.question,
|
|
618
|
+
"progress": self.progress, "stages": self.stages()}
|
|
619
|
+
|
|
620
|
+
def stages(self) -> dict:
|
|
621
|
+
"""The stages said so far, folded for the page (``progress.fold``).
|
|
622
|
+
Once the run has failed or been stopped, a stage still running was
|
|
623
|
+
cut short by it, and nothing is waiting any more."""
|
|
624
|
+
with self._lock:
|
|
625
|
+
events = list(self.events)
|
|
626
|
+
got = progress.fold(events)
|
|
627
|
+
if self.state in ("failed", "stopped"):
|
|
628
|
+
got["stages"] = [dict(r, state="failed") if r["state"] == "running" else r for r in got["stages"] if r["state"] != "waiting"]
|
|
629
|
+
got["current"] = None
|
|
630
|
+
return got
|
|
631
|
+
|
|
632
|
+
|
|
633
|
+
def start(self) -> "Execution":
|
|
634
|
+
self._save()
|
|
635
|
+
self.thread = threading.Thread(target=self._main, name=f"run-{self.plan.name}", daemon=True)
|
|
636
|
+
self.thread.start()
|
|
637
|
+
return self
|
|
638
|
+
|
|
639
|
+
def resume(self) -> "Execution":
|
|
640
|
+
"""Pick a recorded run up where it was: the part that was running is
|
|
641
|
+
followed again, a pause waits again, the rest runs on."""
|
|
642
|
+
self.thread = threading.Thread(target=self._main_resumed, name=f"run-{self.plan.name}", daemon=True)
|
|
643
|
+
self.thread.start()
|
|
644
|
+
return self
|
|
645
|
+
|
|
646
|
+
def continue_(self) -> None:
|
|
647
|
+
"""Answer a pause: go on with the next agent."""
|
|
648
|
+
self.question = None
|
|
649
|
+
self._decision.set()
|
|
650
|
+
|
|
651
|
+
def stop(self) -> None:
|
|
652
|
+
"""Say so at once - the page reads `stopping` on its next tick - and
|
|
653
|
+
let the run's own thread do the rest: the command it is inside gives
|
|
654
|
+
way (`subprocess_run`, `stop=`), the follow loop returns, `_done`
|
|
655
|
+
ends the part on the target (`_kill_part`) - one whose launcher it
|
|
656
|
+
cut short included - copies the job back from a host once more, and
|
|
657
|
+
records `stopped`: a part that had ended by itself with its own
|
|
658
|
+
result, a part the Stop ended with 130, the same for a Stop a restart
|
|
659
|
+
finishes (`_done` says the rule). Nothing
|
|
660
|
+
here waits on the host, so a host that does not answer cannot hold
|
|
661
|
+
the button."""
|
|
662
|
+
with self._turn:
|
|
663
|
+
if self.state not in ALIVE:
|
|
664
|
+
return
|
|
665
|
+
self.state = "stopping"
|
|
666
|
+
self.log("— stop requested")
|
|
667
|
+
self._save()
|
|
668
|
+
self._stop.set()
|
|
669
|
+
self._decision.set()
|
|
670
|
+
|
|
671
|
+
def _kill_part(self) -> Optional[int]:
|
|
672
|
+
"""The part that may be running on the target (``_live``) ended by
|
|
673
|
+
the Stop - unless it had ended by itself - and the part's result
|
|
674
|
+
returned. One command, through the plain runner since the stop is
|
|
675
|
+
already set, reads the part's exit code before it ends anything: a
|
|
676
|
+
part that has one ended by itself before the Stop reached it -
|
|
677
|
+
between two of ``_follow``'s looks, or while Nullius was away - and
|
|
678
|
+
its end is recorded as the look would have recorded it (the rest of
|
|
679
|
+
its log, its result in ``ends``, the run moved past it); its exit
|
|
680
|
+
code is returned. Otherwise its processes are ended, by their
|
|
681
|
+
process groups, found as ``_part_pids`` finds them, so a part is
|
|
682
|
+
ended though its launcher was cut short before its pid came back,
|
|
683
|
+
and a pid the record kept is signalled only while it is still the
|
|
684
|
+
part's own. Once they are gone - half a second at most - the exit
|
|
685
|
+
code is read again: a part that ended by itself between the first
|
|
686
|
+
read and the kill has it now, and keeps it. One the kill ended has
|
|
687
|
+
none, since its detached shell, which writes it, dies with it - so
|
|
688
|
+
every process group is signalled before any single process, lest
|
|
689
|
+
the script's own bash die alone and its shell write that down.
|
|
690
|
+
|
|
691
|
+
The Stop ended it - 130 - when anything of it was there (its claim,
|
|
692
|
+
pid file, log or a process), the target did not say, or its launch
|
|
693
|
+
had begun (``launched``: its script or its launcher on the way, cut
|
|
694
|
+
short before anything of it reached the target); None when none of
|
|
695
|
+
that, nor any part under way: the Stop ended none.
|
|
696
|
+
|
|
697
|
+
A launcher cut short may yet start the detached shell after this has
|
|
698
|
+
looked - on a host the launcher goes on when the ssh that carried it
|
|
699
|
+
is gone. So the Stop is left in the run directory first
|
|
700
|
+
(``run-<part>.stop``), and the detached shell looks for it after
|
|
701
|
+
writing its pid and before running the script: either it sees the
|
|
702
|
+
Stop and runs nothing, or its pid was on disk, and the shell in the
|
|
703
|
+
process table, before this looked."""
|
|
704
|
+
part = self._live
|
|
705
|
+
if part is None:
|
|
706
|
+
return None
|
|
707
|
+
base = _base(part)
|
|
708
|
+
command = ('export PATH="$HOME/.local/bin:$PATH"; ' + f"cd {self._q(self._run_dir())} 2>/dev/null || exit 0; "
|
|
709
|
+
f"ended() {{ e=$(head -c 32 {base}.exit 2>/dev/null | tr -cd 0-9); [ -z \"$e\" ] || "
|
|
710
|
+
f"{{ {self._log_since(base)}; printf '\\n__part__ %s' \"$e\"; exit 0; }}; }}; ended; : >{base}.stop; "
|
|
711
|
+
f"s=none; [ -e {base}.launch ] || [ -e {base}.pid ] || [ -e {base}.log ] && s=stopped; k=$({_part_pids(base, [self.pid])}); "
|
|
712
|
+
f"for p in $k; do s=stopped; kill -TERM -- -$p 2>/dev/null; done; for p in $k; do kill -TERM $p 2>/dev/null; done; "
|
|
713
|
+
f"for i in 1 2 3 4 5 6 7 8 9 10; do a=; for p in $k; do kill -0 $p 2>/dev/null && a=1; done; [ -n \"$a\" ] || break; sleep 0.05; done; "
|
|
714
|
+
+ (f"rm -rf {self._licence_dir()}; " if self._licence_dir() else "")
|
|
715
|
+
+ f"ended; printf '\\n__part__ %s' \"$s\"")
|
|
716
|
+
argv = self.plan.host.argv(command, timeout=15) if self.on_host else ["bash", "-c", command]
|
|
717
|
+
rc, out = self._run_plain(argv, STOP_WAIT)
|
|
718
|
+
text, mark, code = out.rpartition("\n__part__ ")
|
|
719
|
+
code = code.strip()
|
|
720
|
+
if rc == 0 and (not mark or code == "none"):
|
|
721
|
+
# nothing of the part there (nor its run directory): the Stop ended none - unless it cut the part's launch short
|
|
722
|
+
return 130 if self.launched == part["id"] else None
|
|
723
|
+
if rc != 0 or not code.isdigit():
|
|
724
|
+
return 130 # the Stop ended it, or the target did not say it had ended
|
|
725
|
+
text, size = self._logged(text)
|
|
726
|
+
for line in text.splitlines():
|
|
727
|
+
self._read(line)
|
|
728
|
+
self._trial_lines()
|
|
729
|
+
with self._turn:
|
|
730
|
+
if size is not None and size > self.log_offset:
|
|
731
|
+
self.log_offset = size
|
|
732
|
+
self.ends[part["id"]] = {"rc": int(code), "at": time.time(), "label": part["label"]}
|
|
733
|
+
self.part = [p["id"] for p in self.plan.steps()].index(part["id"]) + 1
|
|
734
|
+
self.pid, self._live = None, None
|
|
735
|
+
return int(code)
|
|
736
|
+
|
|
737
|
+
def _main(self) -> None:
|
|
738
|
+
try:
|
|
739
|
+
os.makedirs(self.jobs, exist_ok=True)
|
|
740
|
+
with open(os.path.join(self.dir, "run.sh"), "w", encoding="utf-8") as fh:
|
|
741
|
+
fh.write(self.plan.script if self.plan.script.endswith("\n") else self.plan.script + "\n")
|
|
742
|
+
problem = self.plan.check()
|
|
743
|
+
if problem:
|
|
744
|
+
return self._fail("plan", problem)
|
|
745
|
+
harbor_model = "NULLIUS_HARBOR" in (self.plan.script + "".join(p.get("script", "") for p in self.plan.parts)) and self.plan.model
|
|
746
|
+
if not self._key and harbor_model:
|
|
747
|
+
return self._fail("credentials", f"{self.plan.key_var} is not set in Settings; the run would have no model key")
|
|
748
|
+
if not self._key and self.plan.runner != "harbor" and not self.plan.keyless:
|
|
749
|
+
return self._fail("credentials", f"{self.plan.key_var} is not set in Settings; the producers and the judge would have no model key")
|
|
750
|
+
if not self._url and self.plan.url_var and not self.plan.keyless and (harbor_model or self.plan.runner != "harbor"):
|
|
751
|
+
return self._fail("credentials", f"{self.plan.url_var} is not set in Settings; an OpenAI-compatible endpoint has no default, so the run would have nowhere to send its calls")
|
|
752
|
+
broken = self._line_break()
|
|
753
|
+
if broken:
|
|
754
|
+
return self._fail("credentials", broken)
|
|
755
|
+
if not self._prepare(fresh=True):
|
|
756
|
+
return
|
|
757
|
+
self._parts()
|
|
758
|
+
except Exception as e: # a step that raised is a failed run, not a dead thread
|
|
759
|
+
self._fail(self.step or "run", f"{type(e).__name__}: {e}")
|
|
760
|
+
|
|
761
|
+
def _main_resumed(self) -> None:
|
|
762
|
+
try:
|
|
763
|
+
self.log(f"— Nullius is back; picking the run up at {self.plan.steps()[min(self.part, len(self.plan.steps()) - 1)]['label']}")
|
|
764
|
+
if self.state == "preparing":
|
|
765
|
+
if not self._prepare(fresh=False):
|
|
766
|
+
return
|
|
767
|
+
self._parts(resumed=True)
|
|
768
|
+
except Exception as e:
|
|
769
|
+
self._fail(self.step or "run", f"{type(e).__name__}: {e}")
|
|
770
|
+
|
|
771
|
+
def _fail(self, step: str, why: str) -> None:
|
|
772
|
+
if self._stop.is_set(): # a step cut short by a Stop is the stop, not a failure
|
|
773
|
+
return self._done(130)
|
|
774
|
+
self._stage_end("failed", detail=why[:300])
|
|
775
|
+
self.step, self.state, self.error, self.finished = step, "failed", why, time.time()
|
|
776
|
+
self.log(f"✕ {step}: {why}")
|
|
777
|
+
self._save()
|
|
778
|
+
|
|
779
|
+
def _done(self, rc: int) -> None:
|
|
780
|
+
"""The run's end, recorded once: ``done`` or ``failed`` by ``rc``,
|
|
781
|
+
or ``stopped`` under a Stop. What a Stop records is the same whether
|
|
782
|
+
it was said to this run or read from a record by a restart, which
|
|
783
|
+
has taken the part from the target first (``_settle``):
|
|
784
|
+
|
|
785
|
+
- a part that ended by itself keeps its result. The part under way
|
|
786
|
+
(``_live``) is asked for its exit code before anything is ended
|
|
787
|
+
(``_kill_part``), and an end found there - since the run's last
|
|
788
|
+
look, or while Nullius was away - is recorded as that look would
|
|
789
|
+
have recorded it: its job reads done or failed.
|
|
790
|
+
- a part the Stop ended - found on the target without an exit code,
|
|
791
|
+
or on a target that did not answer - is 130, the line is
|
|
792
|
+
``✕ stopped during <part>: exit 130``, and its job reads
|
|
793
|
+
``stopped``, the run's.
|
|
794
|
+
- the run's exit code is its last part's result: the exit code of a
|
|
795
|
+
part that ended by itself, 130 for one the Stop ended. With no
|
|
796
|
+
part under way - a pause, the moment between two parts, a part
|
|
797
|
+
with nothing of it on the target yet - it is the exit the run
|
|
798
|
+
had: the last part's own (0 before any), or 130 for a step the
|
|
799
|
+
Stop cut short (``_fail``) - a part's launch included, though
|
|
800
|
+
nothing of the part reached the target: the record names the part
|
|
801
|
+
whose launch began (``launched``), so a restart records the 130
|
|
802
|
+
too. A record from before ``launched`` takes it for a Stop before
|
|
803
|
+
the launch.
|
|
804
|
+
- the line says the run was stopped, and where: ``during <part>``
|
|
805
|
+
for a part the Stop ended, else ``before`` the next part, or
|
|
806
|
+
``after`` the last one - ``✕ stopped before <part>: exit <rc>``,
|
|
807
|
+
never ``✓ finished``, whatever the exit.
|
|
808
|
+
- on a host, the job is copied back once more before the run is
|
|
809
|
+
recorded, once a part has run: the trials that ended there since
|
|
810
|
+
the run's last look are here, and said. The copy goes through the
|
|
811
|
+
plain runner, and the kill and the copy together take at most
|
|
812
|
+
``STOP_WAIT``; a host that took it all is not asked again.
|
|
813
|
+
|
|
814
|
+
The next part is never started."""
|
|
815
|
+
live, ended = self._live, None
|
|
816
|
+
if self._stop.is_set():
|
|
817
|
+
until = time.monotonic() + STOP_WAIT
|
|
818
|
+
ended = self._kill_part()
|
|
819
|
+
if ended is not None:
|
|
820
|
+
rc = ended
|
|
821
|
+
if self.on_host and (ended is not None or self.ends):
|
|
822
|
+
# the trials that ended there since the run's last look, copied back once more before the run is stopped
|
|
823
|
+
left = until - time.monotonic()
|
|
824
|
+
if left >= 1:
|
|
825
|
+
self._mirror(self._run_plain, left)
|
|
826
|
+
else:
|
|
827
|
+
self.log(f" the job did not sync back: the host took the Stop's {int(STOP_WAIT)} s")
|
|
828
|
+
with self._turn: # a Stop said as the run ends is "stopped", never left "stopping"
|
|
829
|
+
self.exit_code, self.finished, self.pid = rc, time.time(), None
|
|
830
|
+
self.state = "stopped" if self._stop.is_set() else ("done" if rc == 0 else "failed")
|
|
831
|
+
if self.state == "failed":
|
|
832
|
+
failed = [f"{e['label']} exited {e['rc']}" for e in self.ends.values() if e.get("rc")]
|
|
833
|
+
self.error = "; ".join(failed) if failed else f"the run exited {rc}"
|
|
834
|
+
if self.state == "stopped":
|
|
835
|
+
steps = self.plan.steps()
|
|
836
|
+
where = (f"during {live['label']}" if live is not None and ended is not None and live["id"] not in self.ends
|
|
837
|
+
else f"before {steps[self.part]['label']}" if self.part < len(steps) else f"after {steps[-1]['label']}")
|
|
838
|
+
self.log(f"✕ stopped {where}: exit {rc}")
|
|
839
|
+
else:
|
|
840
|
+
self.log(f"{'✓' if rc == 0 else '✕'} finished: exit {rc}")
|
|
841
|
+
self._save()
|
|
842
|
+
|
|
843
|
+
def _begin(self, step: str, what: str, stage: bool = False) -> None:
|
|
844
|
+
"""A step of the run, said and saved. ``stage``: one that gets the
|
|
845
|
+
target ready, said as a stage (``progress``) that ends with the next
|
|
846
|
+
such step, the preparation's end (``_stage_end``) or a failure."""
|
|
847
|
+
self.step = step
|
|
848
|
+
if stage:
|
|
849
|
+
self._stage_end()
|
|
850
|
+
self._open = (step, time.monotonic())
|
|
851
|
+
self.log(progress.line(step, "start", detail=what))
|
|
852
|
+
else:
|
|
853
|
+
self.log(f"— {what}")
|
|
854
|
+
self._save()
|
|
855
|
+
|
|
856
|
+
def _stage_end(self, state: str = "done", **fields) -> None:
|
|
857
|
+
"""The step under way said ended - done, or failed - with its seconds."""
|
|
858
|
+
if self._open is None:
|
|
859
|
+
return
|
|
860
|
+
step, t0 = self._open
|
|
861
|
+
self._open = None
|
|
862
|
+
self.log(progress.line(step, state, seconds=round(time.monotonic() - t0, 1), **fields))
|
|
863
|
+
|
|
864
|
+
def _pause(self, label: str, rc: int, remaining: List[dict]) -> bool:
|
|
865
|
+
"""An agent's run failed with more to come: wait for the page to say
|
|
866
|
+
continue or stop. True to go on; False at once for a Stop said since
|
|
867
|
+
the part ended, which this would otherwise clear and pause over."""
|
|
868
|
+
return self._ask(label, rc, remaining) and self._wait_decision(remaining)
|
|
869
|
+
|
|
870
|
+
def _ask(self, label: str, rc: int, remaining: List[dict]) -> bool:
|
|
871
|
+
"""The pause's question, put and saved; False, and nothing asked, once
|
|
872
|
+
a Stop is said."""
|
|
873
|
+
with self._turn:
|
|
874
|
+
if self._stop.is_set():
|
|
875
|
+
return False
|
|
876
|
+
self._decision.clear() # before the page can see "paused" and answer
|
|
877
|
+
self.state = "paused"
|
|
878
|
+
self.question = f"{label} exited {rc}. Continue with {remaining[0]['label']}, or stop? A failure this early usually means the dataset or the environment, not the agent."
|
|
879
|
+
self.log(f"✕ {label} exited {rc} — waiting: continue with {remaining[0]['label']}, or stop?")
|
|
880
|
+
self._save()
|
|
881
|
+
return True
|
|
882
|
+
|
|
883
|
+
def _wait_decision(self, remaining: List[dict]) -> bool:
|
|
884
|
+
self._decision.wait()
|
|
885
|
+
if self._stop.is_set():
|
|
886
|
+
return False
|
|
887
|
+
self.state, self.question = "running", None
|
|
888
|
+
self.log(f"— continuing with {remaining[0]['label']}")
|
|
889
|
+
self._save()
|
|
890
|
+
return True
|
|
891
|
+
|
|
892
|
+
def _trial_lines(self) -> None:
|
|
893
|
+
"""One line per trial that ended since the last look, read from the
|
|
894
|
+
jobs' own feeds, so the log says what finished as it finishes. The
|
|
895
|
+
trials said are saved with the record as soon as they are said, so a
|
|
896
|
+
restart, which reads the feeds again from their start, says none of
|
|
897
|
+
them twice."""
|
|
898
|
+
try:
|
|
899
|
+
names = sorted(os.listdir(self.jobs))
|
|
900
|
+
except OSError:
|
|
901
|
+
return
|
|
902
|
+
said = False
|
|
903
|
+
for name in names:
|
|
904
|
+
feed = os.path.join(self.jobs, name, "events.jsonl")
|
|
905
|
+
try:
|
|
906
|
+
with open(feed, encoding="utf-8", errors="replace") as fh:
|
|
907
|
+
for raw in fh:
|
|
908
|
+
if '"trial.end"' not in raw:
|
|
909
|
+
continue
|
|
910
|
+
try:
|
|
911
|
+
ev = json.loads(raw)
|
|
912
|
+
except ValueError:
|
|
913
|
+
continue
|
|
914
|
+
key = (name, ev.get("trial"))
|
|
915
|
+
if ev.get("event") != "trial.end" or key in self._seen_trials:
|
|
916
|
+
continue
|
|
917
|
+
with self._turn: # _save reads the set from the page's thread too
|
|
918
|
+
self._seen_trials.add(key)
|
|
919
|
+
said = True
|
|
920
|
+
d = ev.get("durations") or {}
|
|
921
|
+
secs = sum(v for v in d.values() if isinstance(v, (int, float)))
|
|
922
|
+
reward, score = ev.get("reward"), ev.get("score")
|
|
923
|
+
# a trial its benchmark scored without a pass or fail is neither mark: its score is said
|
|
924
|
+
mark = "✓" if ev.get("outcome") == "resolved" else "·" if ev.get("outcome") == "scored" else "✕"
|
|
925
|
+
self.log(f" {mark} {name} · {ev.get('task') or ev.get('trial')} · {ev.get('outcome')}"
|
|
926
|
+
+ (f" · reward {reward}" if reward is not None else f" · score {score:.3g}" if isinstance(score, (int, float)) else "")
|
|
927
|
+
+ (f" · {int(secs // 60)}:{int(secs % 60):02d}" if secs else ""))
|
|
928
|
+
except OSError:
|
|
929
|
+
continue
|
|
930
|
+
if said:
|
|
931
|
+
self._save()
|
|
932
|
+
|
|
933
|
+
|
|
934
|
+
@property
|
|
935
|
+
def on_host(self) -> bool:
|
|
936
|
+
return self.plan.host is not None
|
|
937
|
+
|
|
938
|
+
def _remote(self, *parts: str) -> str:
|
|
939
|
+
return "/".join((REMOTE_HOME,) + parts)
|
|
940
|
+
|
|
941
|
+
def _run_dir(self) -> str:
|
|
942
|
+
return self._remote("runs", self.plan.name) if self.on_host else self.dir
|
|
943
|
+
|
|
944
|
+
def _q(self, path: str) -> str:
|
|
945
|
+
return _rq(path) if self.on_host else shlex.quote(path)
|
|
946
|
+
|
|
947
|
+
def _sh(self, command: str, timeout: float = 600.0, stdin: Optional[str] = None) -> tuple:
|
|
948
|
+
"""One command on the target, with uv on the PATH; (returncode, output)."""
|
|
949
|
+
command = 'export PATH="$HOME/.local/bin:$PATH"; ' + command
|
|
950
|
+
if self.on_host:
|
|
951
|
+
return self._run(self.plan.host.argv(command, timeout=15), timeout, stdin)
|
|
952
|
+
return self._run(["bash", "-c", command], timeout, stdin)
|
|
953
|
+
|
|
954
|
+
def _ssh_prefix(self) -> str:
|
|
955
|
+
return " ".join(shlex.quote(a) for a in self.plan.host.argv("", timeout=15)[:-2]) # ssh + options, without target and command
|
|
956
|
+
|
|
957
|
+
def _target(self) -> str:
|
|
958
|
+
return self.plan.host.argv("")[-2]
|
|
959
|
+
|
|
960
|
+
def _copy_to(self, src: str, dest: str) -> tuple:
|
|
961
|
+
return self._run(["rsync", "-az", "--delete", "-e", self._ssh_prefix(), src.rstrip("/") + "/", f"{self._target()}:{dest}"], 900)
|
|
962
|
+
|
|
963
|
+
def _copy_back(self, run: Optional[Callable] = None, timeout: float = 900.0) -> tuple:
|
|
964
|
+
started = time.monotonic()
|
|
965
|
+
copy = run or self._run
|
|
966
|
+
result = copy(["rsync", "-az", "-e", self._ssh_prefix(), f"{self._target()}:{self._remote('runs', self.plan.name, 'jobs')}/", self.jobs + "/"], timeout)
|
|
967
|
+
if result[0] == 0 and self.plan.runner == STANDARD and self.catalogue_on_target:
|
|
968
|
+
# Proofs write back to this run's definitions. Keep them beside
|
|
969
|
+
# the job evidence, without replacing this user's catalogue.
|
|
970
|
+
local = os.path.join(self.dir, "catalogue")
|
|
971
|
+
os.makedirs(local, exist_ok=True)
|
|
972
|
+
result = copy(["rsync", "-az", "-e", self._ssh_prefix(),
|
|
973
|
+
f"{self._target()}:{self.catalogue_on_target}/", local + "/"],
|
|
974
|
+
max(0.1, timeout - (time.monotonic() - started)))
|
|
975
|
+
return result
|
|
976
|
+
|
|
977
|
+
|
|
978
|
+
def _prepare(self, fresh: bool = True) -> bool:
|
|
979
|
+
"""The target ready for the parts: the tasks, and what the run needs
|
|
980
|
+
there - for a Harbor recipe docker, here or on a host, and the Harbor it pins
|
|
981
|
+
(built here when it is a repository's commit, its wheels sent to a
|
|
982
|
+
host, run through uvx) or, for a plan from before the pin, the harbor
|
|
983
|
+
on the host's PATH; uv and the runner snapshot for the standard. False when a step
|
|
984
|
+
failed and said so.
|
|
985
|
+
|
|
986
|
+
A ``fresh`` run on a host first asks whether its run directory is
|
|
987
|
+
there already, and stops before anything is sent when it is: a
|
|
988
|
+
leftover of the same name would otherwise read as this run's parts,
|
|
989
|
+
claimed or ended already. (Here the route asks, ``web/execute.py``.)
|
|
990
|
+
A restored run's directory is its own."""
|
|
991
|
+
self.harbor = "harbor"
|
|
992
|
+
if fresh and not self.on_host and (self.plan.runner == "harbor" or self.plan.docker):
|
|
993
|
+
# before a pin is built: a run here without docker would start, and every trial fail at its container
|
|
994
|
+
why = (docker_here(self._sh) if self.plan.runner == "harbor"
|
|
995
|
+
else docker_here(self._sh, "this recipe runs its units in containers", container.DOCKER))
|
|
996
|
+
if why:
|
|
997
|
+
self._fail("docker", why)
|
|
998
|
+
return False
|
|
999
|
+
pin = self.plan.harbor_pin if self.plan.runner == "harbor" else {}
|
|
1000
|
+
if fresh and (self.on_host or pin):
|
|
1001
|
+
self.log(progress.line("plan", stages=",".join(
|
|
1002
|
+
(["harbor"] if pin else []) + (["connect", *(["uv"] if pin or self.plan.runner != "harbor" else []),
|
|
1003
|
+
*(["tasks"] if self.plan.tasks else []), *(["app"] if self.plan.runner == STANDARD else [])]
|
|
1004
|
+
if self.on_host else []))))
|
|
1005
|
+
record = self._pin_here(pin) if pin else None
|
|
1006
|
+
if pin and record is None:
|
|
1007
|
+
return False
|
|
1008
|
+
if not self.on_host:
|
|
1009
|
+
if pin:
|
|
1010
|
+
rc, _ = self._sh("command -v uvx", 30)
|
|
1011
|
+
if rc != 0:
|
|
1012
|
+
self._fail("harbor", "uv is not on this machine's PATH; the Harbor this recipe pins runs through uvx")
|
|
1013
|
+
return False
|
|
1014
|
+
os.makedirs(self.dir, exist_ok=True)
|
|
1015
|
+
self.harbor = harbor_source.write_wrapper(os.path.join(self.dir, "harbor"), record,
|
|
1016
|
+
shlex.quote(os.path.join(self.pins, harbor_source.key_of(pin))), pin.get("extras", []))
|
|
1017
|
+
# absolute: the part runs from its run directory, never from where this process started
|
|
1018
|
+
self.tasks_on_target = os.path.abspath(os.path.expanduser(self.plan.tasks)) if self.plan.tasks else ""
|
|
1019
|
+
if self.plan.runner == STANDARD:
|
|
1020
|
+
self.catalogue_on_target = self._catalogue
|
|
1021
|
+
if os.path.isfile(os.path.join(REPO, "app", "nullius", "standard", "cli.py")):
|
|
1022
|
+
self.app_on_target = REPO
|
|
1023
|
+
else:
|
|
1024
|
+
got = self._app_here()
|
|
1025
|
+
if got is None:
|
|
1026
|
+
return False
|
|
1027
|
+
self.app_on_target = got[1]
|
|
1028
|
+
self._stage_end()
|
|
1029
|
+
self._save()
|
|
1030
|
+
return True
|
|
1031
|
+
host = self.plan.host
|
|
1032
|
+
self._begin("connect", f"connecting to {host.label}", stage=True)
|
|
1033
|
+
probe = hosts.test(host, run=lambda argv, t: _probe(self._run, argv[:-1] + ['export PATH="$HOME/.local/bin:$PATH"; ' + argv[-1]], t))
|
|
1034
|
+
if not probe["ok"]:
|
|
1035
|
+
self._fail("connect", probe["error"])
|
|
1036
|
+
return False
|
|
1037
|
+
if fresh:
|
|
1038
|
+
rc, out = self._sh(f"[ -e {_rq(self._run_dir())} ] && echo @taken; true", 30)
|
|
1039
|
+
if rc != 0:
|
|
1040
|
+
self._fail("connect", "the host did not say whether the run's name is free: " + out.strip()[-400:])
|
|
1041
|
+
return False
|
|
1042
|
+
if "@taken" in out.split():
|
|
1043
|
+
self._fail("name", name_taken(self.plan.name, host.label))
|
|
1044
|
+
return False
|
|
1045
|
+
# what this run needs of the host, and nothing it does not
|
|
1046
|
+
if self.plan.runner == "harbor":
|
|
1047
|
+
self.log(f" {probe['hostname'] or host.label}: docker {'yes' if probe['docker'] else 'no'}"
|
|
1048
|
+
+ (f" · uv {'yes' if probe['uv'] else 'no'}" if pin else f" · harbor {(probe['harbor_version'] or 'yes') if probe['harbor'] else 'no'}"))
|
|
1049
|
+
if not probe["docker"]:
|
|
1050
|
+
self._fail("connect", "docker is not on the host; a Harbor run needs it")
|
|
1051
|
+
return False
|
|
1052
|
+
if probe["docker_ok"] is False:
|
|
1053
|
+
self._fail("connect", "docker is on the host but this user cannot use it (docker ps failed)")
|
|
1054
|
+
return False
|
|
1055
|
+
if not pin and not probe["harbor"]:
|
|
1056
|
+
self._fail("connect", "harbor is not on the host's PATH (~/.local/bin included); install the Harbor this recipe needs there")
|
|
1057
|
+
return False
|
|
1058
|
+
elif self.plan.runner == STANDARD:
|
|
1059
|
+
# the standard's own cli: python3 and git there, docker or podman when its units run in containers
|
|
1060
|
+
engine = probe["engine"]
|
|
1061
|
+
self.log(f" {probe['hostname'] or host.label}: uv {'yes' if probe['uv'] else 'no'} · {engine or 'docker'} {'yes' if engine else 'no'}")
|
|
1062
|
+
if self.plan.docker and not engine:
|
|
1063
|
+
self._fail("connect", "neither docker nor podman is on the host: this recipe runs its units in containers")
|
|
1064
|
+
return False
|
|
1065
|
+
if self.plan.docker and probe[engine + "_ok"] is False:
|
|
1066
|
+
self._fail("connect", f"{engine} is on the host but this user cannot use it ({engine} ps failed)")
|
|
1067
|
+
return False
|
|
1068
|
+
rc, out = self._sh("for c in python3 git; do command -v $c >/dev/null 2>&1 || echo @missing $c; done; true", 30)
|
|
1069
|
+
gone = [l.split()[1] for l in out.splitlines() if l.startswith("@missing ")]
|
|
1070
|
+
if rc != 0 or gone:
|
|
1071
|
+
self._fail("connect", (" and ".join(gone) + " not on the host; the standard's cli needs them") if gone
|
|
1072
|
+
else "the host did not say whether python3 is there: " + out.strip()[-400:])
|
|
1073
|
+
return False
|
|
1074
|
+
else:
|
|
1075
|
+
# a run record from a runner no longer offered (a benchmark's own scripts now run as the standard's Original setup)
|
|
1076
|
+
self._fail("connect", f"this run's runner, {self.plan.runner}, is no longer offered")
|
|
1077
|
+
return False
|
|
1078
|
+
if (pin or self.plan.runner != "harbor") and probe["uv"]:
|
|
1079
|
+
self._stage_end()
|
|
1080
|
+
self.log(progress.line("uv", "cached", detail="on the host already"))
|
|
1081
|
+
if (pin or self.plan.runner != "harbor") and not probe["uv"]:
|
|
1082
|
+
# the standard's cli builds its environments with uv; a pinned Harbor runs through uvx
|
|
1083
|
+
self._begin("uv", "installing uv on the host", stage=True)
|
|
1084
|
+
rc, out = self._sh(f"command -v uv >/dev/null 2>&1 || ({UV_INSTALL}) >/dev/null 2>&1; command -v uv", 300)
|
|
1085
|
+
found = [l for l in out.strip().splitlines() if not l.startswith("** ")]
|
|
1086
|
+
if rc != 0 or not found:
|
|
1087
|
+
self._fail("uv", "uv could not be installed: " + out.strip()[-400:])
|
|
1088
|
+
return False
|
|
1089
|
+
self.log(" uv: " + found[-1])
|
|
1090
|
+
self.tasks_on_target = ""
|
|
1091
|
+
if self.plan.tasks:
|
|
1092
|
+
self._begin("tasks", f"sending the task folder {self.plan.tasks}", stage=True)
|
|
1093
|
+
self.tasks_on_target = self._remote(*_tasks_place(self.plan.tasks, self.plan.name))
|
|
1094
|
+
rc, out = self._sh(f"mkdir -p {_rq(self.tasks_on_target)}", 30)
|
|
1095
|
+
if rc == 0:
|
|
1096
|
+
rc, out = self._copy_to(os.path.expanduser(self.plan.tasks), self.tasks_on_target)
|
|
1097
|
+
if rc != 0:
|
|
1098
|
+
self._fail("tasks", "the task folder did not copy: " + out.strip()[-400:])
|
|
1099
|
+
return False
|
|
1100
|
+
if self.plan.runner == STANDARD and not self._app_on_host():
|
|
1101
|
+
return False
|
|
1102
|
+
run_dir = self._run_dir()
|
|
1103
|
+
rc, out = self._sh(f"mkdir -p {_rq(run_dir + '/jobs')} && cat > {_rq(run_dir + '/run.sh')}", 30, stdin=self.plan.script)
|
|
1104
|
+
if rc != 0:
|
|
1105
|
+
self._fail("run", "the script did not land on the host: " + out.strip()[-400:])
|
|
1106
|
+
return False
|
|
1107
|
+
if self.plan.runner == STANDARD:
|
|
1108
|
+
catalogue = run_dir + "/catalogue"
|
|
1109
|
+
stage = catalogue + ".part"
|
|
1110
|
+
# A restart keeps the proofs already written by this run.
|
|
1111
|
+
# Copy into staging so an interrupted first copy is not trusted.
|
|
1112
|
+
rc, out = self._sh(f"if [ ! -d {_rq(catalogue)} ]; then rm -rf {_rq(stage)} && "
|
|
1113
|
+
f"cp -R {_rq(self.app_on_target + '/app/nullius/catalogue')} {_rq(stage)} && "
|
|
1114
|
+
f"mv {_rq(stage)} {_rq(catalogue)}; fi", 300)
|
|
1115
|
+
if rc != 0:
|
|
1116
|
+
self._fail("app", "the run's catalogue could not be prepared: " + out.strip()[-400:])
|
|
1117
|
+
return False
|
|
1118
|
+
self.catalogue_on_target = catalogue
|
|
1119
|
+
if pin and not self._pin_on_host(pin, record, run_dir):
|
|
1120
|
+
return False
|
|
1121
|
+
self._stage_end()
|
|
1122
|
+
self._save()
|
|
1123
|
+
return True
|
|
1124
|
+
|
|
1125
|
+
def _app_here(self) -> Optional[tuple]:
|
|
1126
|
+
"""(content ID, folder) for this runtime and its effective catalogue.
|
|
1127
|
+
|
|
1128
|
+
The exact code imported here travels, including source edits, so
|
|
1129
|
+
installed wheels and editable checkouts have the same behaviour.
|
|
1130
|
+
Configuration and credential stores are not payload inputs.
|
|
1131
|
+
"""
|
|
1132
|
+
try:
|
|
1133
|
+
return standard_payload.build(app_root(self.root), catalogue=self._catalogue)
|
|
1134
|
+
except (OSError, ValueError) as e:
|
|
1135
|
+
self._fail("app", "the standard runner could not be packed: " + str(e)[-300:])
|
|
1136
|
+
return None
|
|
1137
|
+
|
|
1138
|
+
def _app_on_host(self) -> bool:
|
|
1139
|
+
"""Send the content-addressed runner and catalogue to the host."""
|
|
1140
|
+
got = self._app_here()
|
|
1141
|
+
if got is None:
|
|
1142
|
+
return False
|
|
1143
|
+
content_id, here = got
|
|
1144
|
+
remote = self._remote("app", content_id)
|
|
1145
|
+
self._begin("app", f"sending the standard runner and catalogue at {content_id[:15]}", stage=True)
|
|
1146
|
+
rc, out = self._sh(f"mkdir -p {_rq(remote)}", 30)
|
|
1147
|
+
if rc == 0:
|
|
1148
|
+
rc, out = self._copy_to(here, remote)
|
|
1149
|
+
if rc != 0:
|
|
1150
|
+
self._fail("app", "app/ did not copy: " + out.strip()[-400:])
|
|
1151
|
+
return False
|
|
1152
|
+
self.app_on_target = remote
|
|
1153
|
+
return True
|
|
1154
|
+
|
|
1155
|
+
def _pin_here(self, pin: dict) -> Optional[dict]:
|
|
1156
|
+
"""The pinned Harbor ready on this machine - a repository's commit
|
|
1157
|
+
built here, once (``harbor_source.ensure``) - and its record; None
|
|
1158
|
+
when it could not be, said."""
|
|
1159
|
+
building = pin["kind"] == "repo" and harbor_source.record_of(pin, self.pins) is None
|
|
1160
|
+
self._begin("harbor", harbor_source.label(pin) + (": building it here, once for this commit" if building else ""), stage=True)
|
|
1161
|
+
try:
|
|
1162
|
+
return harbor_source.ensure(pin, self.pins, run=lambda argv, t=600.0: harbor_source.run_command(argv, t, stop=self._stop), log=self.log)
|
|
1163
|
+
except harbor_source.BuildError as e:
|
|
1164
|
+
self._fail("harbor", str(e))
|
|
1165
|
+
return None
|
|
1166
|
+
|
|
1167
|
+
def _pin_on_host(self, pin: dict, record: dict, run_dir: str) -> bool:
|
|
1168
|
+
"""The pinned Harbor on the host: a repository's wheels sent to
|
|
1169
|
+
``~/.nullius/harbor/<pin>/wheels`` (the host resolves the rest from
|
|
1170
|
+
PyPI, so it needs no access to the repository), and the ``harbor``
|
|
1171
|
+
script that runs it written into the run directory."""
|
|
1172
|
+
remote = self._remote("harbor", harbor_source.key_of(pin))
|
|
1173
|
+
if record.get("wheel"):
|
|
1174
|
+
self._begin("harbor", f"sending the wheels built for {harbor_source.label(pin)}", stage=True)
|
|
1175
|
+
rc, out = self._sh(f"mkdir -p {_rq(remote + '/' + harbor_source.WHEELS)}", 30)
|
|
1176
|
+
if rc == 0:
|
|
1177
|
+
rc, out = self._copy_to(os.path.join(self.pins, harbor_source.key_of(pin), harbor_source.WHEELS), remote + "/" + harbor_source.WHEELS)
|
|
1178
|
+
if rc != 0:
|
|
1179
|
+
self._fail("harbor", "the wheels did not copy: " + out.strip()[-400:])
|
|
1180
|
+
return False
|
|
1181
|
+
exe = run_dir + "/harbor"
|
|
1182
|
+
rc, out = self._sh(f"cat > {_rq(exe + harbor_source.FEED_SUFFIX)}", 30, stdin=harbor_source.feed_source()) # the live feed it runs
|
|
1183
|
+
if rc == 0:
|
|
1184
|
+
rc, out = self._sh(f"cat > {_rq(exe)} && chmod +x {_rq(exe)}", 30, stdin=harbor_source.wrapper(record, _rq(remote), pin.get("extras", [])))
|
|
1185
|
+
if rc != 0:
|
|
1186
|
+
self._fail("harbor", "the harbor script did not land on the host: " + out.strip()[-400:])
|
|
1187
|
+
return False
|
|
1188
|
+
self.harbor = exe
|
|
1189
|
+
return True
|
|
1190
|
+
|
|
1191
|
+
# --- the parts, one agent at a time -----------------------------------------
|
|
1192
|
+
|
|
1193
|
+
def _parts(self, resumed: bool = False) -> None:
|
|
1194
|
+
steps = self.plan.steps()
|
|
1195
|
+
follow = resumed and self._settle(steps) # a restored record, checked against the target first
|
|
1196
|
+
i = self.part
|
|
1197
|
+
# picked up after a part ended: that part's own exit, as the run had it, not a 0 for the restart
|
|
1198
|
+
last = self.ends.get(steps[i - 1]["id"], {}).get("rc", 0) if 0 < i <= len(steps) else 0
|
|
1199
|
+
while i < len(steps):
|
|
1200
|
+
part = steps[i]
|
|
1201
|
+
if resumed and self.state == "paused":
|
|
1202
|
+
if not self._wait_decision(steps[i:]):
|
|
1203
|
+
break
|
|
1204
|
+
launched = self._launch(part)
|
|
1205
|
+
if launched is None:
|
|
1206
|
+
break
|
|
1207
|
+
if not launched:
|
|
1208
|
+
return
|
|
1209
|
+
elif follow and self.state == "running":
|
|
1210
|
+
self.log(f"— following {part['label']} again on {self.plan.host.label if self.on_host else 'this machine'}")
|
|
1211
|
+
else:
|
|
1212
|
+
launched = self._launch(part)
|
|
1213
|
+
if launched is None:
|
|
1214
|
+
break
|
|
1215
|
+
if not launched:
|
|
1216
|
+
return
|
|
1217
|
+
resumed = follow = False
|
|
1218
|
+
last = self._follow(part)
|
|
1219
|
+
if self._stop.is_set():
|
|
1220
|
+
break
|
|
1221
|
+
with self._turn:
|
|
1222
|
+
# Ended, and recorded so before anything else is: its exit, the next part, and no pid -
|
|
1223
|
+
# a restart launches the next part, never follows it by this one's pid.
|
|
1224
|
+
self.ends[part["id"]] = {"rc": last, "at": time.time(), "label": part["label"]}
|
|
1225
|
+
i += 1
|
|
1226
|
+
self.part, self.pid, self._live = i, None, None
|
|
1227
|
+
if last != 0 and i < len(steps):
|
|
1228
|
+
if not self._pause(part["label"], last, steps[i:]): # its one save is the end and the question: a restart asks again
|
|
1229
|
+
break
|
|
1230
|
+
else:
|
|
1231
|
+
self._save()
|
|
1232
|
+
self._done(last)
|
|
1233
|
+
|
|
1234
|
+
def _settle(self, steps: List[dict]) -> bool:
|
|
1235
|
+
"""A restored record's part and pid, checked against the target
|
|
1236
|
+
before either is trusted; True when that part is to be followed, not
|
|
1237
|
+
launched. A part has started when its run directory there holds its
|
|
1238
|
+
pid file (its detached shell's first act), its log or its exit code,
|
|
1239
|
+
or when a process runs its script there (``_part_pids``: the launcher
|
|
1240
|
+
before c69f527 wrote no pid file).
|
|
1241
|
+
|
|
1242
|
+
- No pid in the record: it was saved before the pid was. Started, the
|
|
1243
|
+
part is followed - by the pid it wrote, else one of its processes,
|
|
1244
|
+
else none: its exit code ends it, or, with no process left, it is
|
|
1245
|
+
gone without one - its log read from the start. Claimed by a
|
|
1246
|
+
launcher (``.launch``) and nothing more: the launcher is on its way
|
|
1247
|
+
to starting it, so it is looked at again for ``LAUNCH_GRACE``; still
|
|
1248
|
+
nothing - no pid, no process - and the launcher died first: the
|
|
1249
|
+
claim is cleared and the part launched. Not claimed: launched.
|
|
1250
|
+
- A pid, a part before it, and this part started: the pid is checked
|
|
1251
|
+
against the part - the one it wrote, or one of its processes. Not
|
|
1252
|
+
its own, it is the part before's: a record an older version saved
|
|
1253
|
+
as this part was launched. It is dropped, the part
|
|
1254
|
+
before's end taken as below, and this part followed as if no pid
|
|
1255
|
+
had been saved.
|
|
1256
|
+
- A pid, and the part not started - or, past the last part, no end
|
|
1257
|
+
recorded for it: a record from before a part's end was one save,
|
|
1258
|
+
saved as the previous part ended. The pid is that part's;
|
|
1259
|
+
its end is the record's or, from before ``ends``, its ``.exit``
|
|
1260
|
+
(none: gone without one). The run goes on as it would have: asks
|
|
1261
|
+
after a failure with more to come, else launches the part or ends
|
|
1262
|
+
with that exit.
|
|
1263
|
+
- A pause: it follows a part's end, so nothing of the run is running.
|
|
1264
|
+
A pid it kept (an older record's: the ended part's) is
|
|
1265
|
+
nobody's now, and is dropped.
|
|
1266
|
+
- A Stop the restart cut short: the part is taken as above, and the
|
|
1267
|
+
Stop is then finished as a live one is (``_done``) - a part that
|
|
1268
|
+
ended by itself meanwhile keeps its own result, read through the
|
|
1269
|
+
plain runner, since the Stop, set already, would cut a slow host's
|
|
1270
|
+
answer short.
|
|
1271
|
+
|
|
1272
|
+
A pid with no part before it stands as recorded: it can be no other
|
|
1273
|
+
part's, and it is followed and killed only while it runs the part."""
|
|
1274
|
+
i = self.part
|
|
1275
|
+
if self.state == "paused":
|
|
1276
|
+
if self.pid:
|
|
1277
|
+
with self._turn:
|
|
1278
|
+
self.pid = None
|
|
1279
|
+
self._save()
|
|
1280
|
+
return False
|
|
1281
|
+
if self.state not in ("running", "stopping") or i > len(steps):
|
|
1282
|
+
return bool(self.pid)
|
|
1283
|
+
cur, prev = (steps[i] if i < len(steps) else None), (steps[i - 1] if i else None)
|
|
1284
|
+
self._live = cur # running, or being launched, as far as the record knows
|
|
1285
|
+
# a pid with no part before it can be no other part's: it stands (followed and killed only while it runs the part)
|
|
1286
|
+
check = cur is not None and (not self.pid or prev is not None)
|
|
1287
|
+
mine = [f"{_base(cur)}.{ext}" for ext in ("launch", "pid", "log", "exit")] if check else []
|
|
1288
|
+
before = [f"{_base(prev)}.exit"] if prev and prev["id"] not in self.ends else []
|
|
1289
|
+
if not mine and not before:
|
|
1290
|
+
return bool(self.pid)
|
|
1291
|
+
look = self._look(mine + before, cur if check else None, [self.pid])
|
|
1292
|
+
if look is None:
|
|
1293
|
+
return bool(self.pid)
|
|
1294
|
+
found, procs = look
|
|
1295
|
+
started = lambda: bool(procs) or any(n in found for n in mine[1:]) # its pid file, log or exit code, or its processes
|
|
1296
|
+
if check and mine[0] in found and not started():
|
|
1297
|
+
# claimed by a launcher and not started yet: the launcher is on its way to it, or died first
|
|
1298
|
+
look = self._await_start(mine + before, cur)
|
|
1299
|
+
if look is None:
|
|
1300
|
+
return bool(self.pid)
|
|
1301
|
+
found, procs = look
|
|
1302
|
+
if not started():
|
|
1303
|
+
self.log(f" {cur['label']} was claimed by a launcher that never started it; starting it again")
|
|
1304
|
+
self._sh(f"cd {self._q(self._run_dir())} && rm -f {mine[0]}", 30)
|
|
1305
|
+
if check and started():
|
|
1306
|
+
written = int(found.get(mine[1]) or 0) or None
|
|
1307
|
+
if self.pid and prev is not None and self.pid != written and self.pid not in procs:
|
|
1308
|
+
# Not this part's: a record an older version saved as it was launched kept the part
|
|
1309
|
+
# before's pid - and this part, started by the launcher before its pid file, is found by its processes.
|
|
1310
|
+
with self._turn:
|
|
1311
|
+
if before:
|
|
1312
|
+
rc = found.get(before[0], "")
|
|
1313
|
+
self.ends[prev["id"]] = {"rc": int(rc) if rc else 1, "at": time.time(), "label": prev["label"]}
|
|
1314
|
+
self.pid = None
|
|
1315
|
+
if not self.pid:
|
|
1316
|
+
with self._turn:
|
|
1317
|
+
self.pid, self.log_offset = written or (procs[0] if procs else None), 0
|
|
1318
|
+
self._save()
|
|
1319
|
+
return True
|
|
1320
|
+
old = bool(self.pid) # a pid, and no part of its own: the part before's
|
|
1321
|
+
if not old and not before:
|
|
1322
|
+
return False # never started: launched
|
|
1323
|
+
with self._turn:
|
|
1324
|
+
if before:
|
|
1325
|
+
rc = found.get(before[0], "")
|
|
1326
|
+
self.ends[prev["id"]] = {"rc": int(rc) if rc else 1, "at": time.time(), "label": prev["label"]}
|
|
1327
|
+
self.pid = None
|
|
1328
|
+
rc = self.ends[prev["id"]]["rc"]
|
|
1329
|
+
if not (old and cur and rc != 0 and self._ask(prev["label"], rc, steps[i:])): # asked: that save is the record
|
|
1330
|
+
self._save()
|
|
1331
|
+
return False
|
|
1332
|
+
|
|
1333
|
+
def _look(self, names: List[str], part: Optional[dict] = None, pids=()) -> Optional[tuple]:
|
|
1334
|
+
"""Which of ``names`` the run directory on the target holds, each with
|
|
1335
|
+
the digits it starts with (a pid, an exit code), and - for ``part`` -
|
|
1336
|
+
the pids of its processes there (``_part_pids``, asked of ``pids``
|
|
1337
|
+
too): ({name: digits}, [pid]). Asked again while the target does not
|
|
1338
|
+
answer, as ``_follow`` does; None for a Stop said meanwhile."""
|
|
1339
|
+
files = " ".join(shlex.quote(n) for n in names)
|
|
1340
|
+
procs = f"for p in $({_part_pids(_base(part), pids)}); do echo \"@proc $p\"; done; " if part else ""
|
|
1341
|
+
while True:
|
|
1342
|
+
rc, out = self._sh(f"cd {self._q(self._run_dir())} 2>/dev/null || exit 0; for f in {files}; do "
|
|
1343
|
+
f"[ -f \"$f\" ] && printf '@file %s %s\\n' \"$f\" \"$(head -c 32 \"$f\" | tr -cd 0-9)\"; done; {procs}true", 60)
|
|
1344
|
+
if rc == 0:
|
|
1345
|
+
lines = [l.split() for l in out.splitlines()]
|
|
1346
|
+
return ({b[1]: "".join(b[2:]) for b in lines if len(b) >= 2 and b[0] == "@file"},
|
|
1347
|
+
[int(b[1]) for b in lines if len(b) == 2 and b[0] == "@proc" and b[1].isdigit()])
|
|
1348
|
+
if self._stop.wait(SYNC_EVERY):
|
|
1349
|
+
return None
|
|
1350
|
+
|
|
1351
|
+
def _await_start(self, names: List[str], part: dict) -> Optional[tuple]:
|
|
1352
|
+
"""A part its launcher claimed and nothing after: looked at again
|
|
1353
|
+
for up to ``LAUNCH_GRACE`` seconds, until its pid file, log or exit
|
|
1354
|
+
code is there or a process runs it. A launcher that has claimed a
|
|
1355
|
+
part runs in its run directory until it has started the part, so
|
|
1356
|
+
nothing there, all that time, means the launcher died first. The
|
|
1357
|
+
last look; None for a Stop said meanwhile."""
|
|
1358
|
+
look = None
|
|
1359
|
+
for _ in range(max(1, int(LAUNCH_GRACE / STOP_POLL))):
|
|
1360
|
+
if self._stop.wait(min(STOP_POLL, LAUNCH_GRACE)):
|
|
1361
|
+
return None
|
|
1362
|
+
look = self._look(names, part, [self.pid])
|
|
1363
|
+
if look is None or look[1] or any(n in look[0] for n in names[1:4]):
|
|
1364
|
+
return look
|
|
1365
|
+
return look
|
|
1366
|
+
|
|
1367
|
+
def _stdin(self) -> str:
|
|
1368
|
+
"""What the launcher reads: the key, the endpoint, the agent keys as
|
|
1369
|
+
NAME=value, the benchmark's licences as one more (a standard run on a
|
|
1370
|
+
host), the end line."""
|
|
1371
|
+
lic = self._licence_line()
|
|
1372
|
+
return "".join(f"{line}\n" for line in [self._key, self._url, *(f"{n}={v}" for n, v in self._agent_keys.items()),
|
|
1373
|
+
*([f"{secrets.HANDED}={lic}"] if lic else []), STDIN_END])
|
|
1374
|
+
|
|
1375
|
+
def _licence_dir(self) -> str:
|
|
1376
|
+
"""Where a standard run on a host writes a licence file: in memory, the run's own folder."""
|
|
1377
|
+
return f"/dev/shm/nullius-{self.plan.name}" if self.on_host and (self.plan.licences or {}).get("names") else ""
|
|
1378
|
+
|
|
1379
|
+
def _licence_trap(self) -> str:
|
|
1380
|
+
"""The detached shell's first words when it may hold a licence file:
|
|
1381
|
+
the folder removed when it ends, by itself or by a Stop's signal."""
|
|
1382
|
+
d = self._licence_dir()
|
|
1383
|
+
return f'trap "rm -rf {d}" EXIT; trap "exit 143" TERM HUP INT; ' if d else ""
|
|
1384
|
+
|
|
1385
|
+
def _licence_line(self) -> str:
|
|
1386
|
+
"""The benchmark's licences, as the one line a host's run reads
|
|
1387
|
+
(base64 of {name: base64 bytes}): only the names its environments declare, only
|
|
1388
|
+
those set - a data provider's token from that provider's store - and nothing
|
|
1389
|
+
at all for a run on this machine, which reads the store itself."""
|
|
1390
|
+
lic = self.plan.licences or {}
|
|
1391
|
+
if not self._licence_dir():
|
|
1392
|
+
return ""
|
|
1393
|
+
names = [n for n in lic.get("names") or [] if isinstance(n, str)]
|
|
1394
|
+
providers = lic.get("providers") if isinstance(lic.get("providers"), dict) else {}
|
|
1395
|
+
if self._licences_given is not None:
|
|
1396
|
+
got = {n: v for n, v in self._licences_given.items() if n in names and v}
|
|
1397
|
+
else:
|
|
1398
|
+
try:
|
|
1399
|
+
got = {n: secrets.value(*secrets.store_of(lic.get("benchmark") or "", n, providers.get(n) or "")) for n in names}
|
|
1400
|
+
except ValueError:
|
|
1401
|
+
got = {}
|
|
1402
|
+
got = {n: v for n, v in got.items() if v}
|
|
1403
|
+
for v in got.values():
|
|
1404
|
+
try:
|
|
1405
|
+
text = v.decode().strip()
|
|
1406
|
+
except UnicodeDecodeError:
|
|
1407
|
+
continue
|
|
1408
|
+
if "\n" not in text and text not in self._redact:
|
|
1409
|
+
self._redact.append(text)
|
|
1410
|
+
if not got:
|
|
1411
|
+
return ""
|
|
1412
|
+
return base64.b64encode(json.dumps({n: base64.b64encode(v).decode() for n, v in sorted(got.items())},
|
|
1413
|
+
separators=(",", ":")).encode()).decode()
|
|
1414
|
+
|
|
1415
|
+
def _line_break(self) -> Optional[str]:
|
|
1416
|
+
"""Why the run's key or endpoint cannot be sent, or None: one that
|
|
1417
|
+
holds a line break, which the launcher, reading each as one line
|
|
1418
|
+
(``_launch``), would take in pieces. Settings refuses one, so only
|
|
1419
|
+
the environment Nullius was started in can hold it. The variable is
|
|
1420
|
+
named, never its value."""
|
|
1421
|
+
for value, var in ((self._key, self.plan.key_var), (self._url, self.plan.url_var or "the endpoint"), *((v, n) for n, v in self._agent_keys.items())):
|
|
1422
|
+
if "\n" in value or "\r" in value:
|
|
1423
|
+
return (f"{var} holds a line break; a key or an endpoint is one line, and the run's launcher would take this one "
|
|
1424
|
+
f"in pieces. Set it again on one line in the environment Nullius was started in (Settings refuses a line break)")
|
|
1425
|
+
return None
|
|
1426
|
+
|
|
1427
|
+
def _launch(self, part: dict) -> Optional[bool]:
|
|
1428
|
+
"""Start one part detached on the target and keep its pid. The key and
|
|
1429
|
+
endpoint arrive on stdin, become environment for the launcher, and the
|
|
1430
|
+
detached shell inherits them - nothing in argv, nothing on disk. They
|
|
1431
|
+
are two lines and ``STDIN_END`` a third: either may be empty (a
|
|
1432
|
+
recipe of controls needs no key, OpenAI's clients no endpoint), so a
|
|
1433
|
+
stdin cut short - Nullius gone as it sent them, a dropped ssh - is
|
|
1434
|
+
told by the end line missing, never by an empty value. Without it the
|
|
1435
|
+
launcher answers ``@cut`` and does nothing more, the part neither
|
|
1436
|
+
claimed nor started, and the run fails at that part. The
|
|
1437
|
+
launcher claims the part first (``run-<part>.launch``, created only
|
|
1438
|
+
where there is none), and the detached shell writes its own pid
|
|
1439
|
+
beside its log before it runs the script: Nullius gone before the pid
|
|
1440
|
+
is saved here, a restart finds the claim, the pid or the process
|
|
1441
|
+
there (``_settle``) and never starts the part a second time - nor
|
|
1442
|
+
does a launcher that reaches the target after the restart's: it finds
|
|
1443
|
+
the part claimed, says so and starts nothing, and a restart that
|
|
1444
|
+
meets that answer follows the part.
|
|
1445
|
+
|
|
1446
|
+
A key or endpoint that holds a line break is never sent: it would
|
|
1447
|
+
reach the launcher in pieces and read as a stdin cut short. A new
|
|
1448
|
+
run refuses it before anything is sent (``_main``); a restored one,
|
|
1449
|
+
whose key and endpoint are read again, here, before the part.
|
|
1450
|
+
|
|
1451
|
+
True once launched, False when the launch failed (the run is failed),
|
|
1452
|
+
None when a Stop was said before it began: nothing is started, and the
|
|
1453
|
+
caller ends the run as the Stop records it. The check and the
|
|
1454
|
+
``running`` it sets hold the lock ``stop`` takes, so a Stop is never
|
|
1455
|
+
written over."""
|
|
1456
|
+
broken = self._line_break()
|
|
1457
|
+
if broken:
|
|
1458
|
+
self._fail("credentials", broken)
|
|
1459
|
+
return False
|
|
1460
|
+
with self._turn: # the Stop takes this lock too: said before this, nothing is launched
|
|
1461
|
+
if self._stop.is_set():
|
|
1462
|
+
return None
|
|
1463
|
+
self.state, self._live, self.launched = "running", part, part["id"]
|
|
1464
|
+
where = self.plan.host.label if self.on_host else "this machine"
|
|
1465
|
+
self._begin("run", f"running {part['label']} on {where}")
|
|
1466
|
+
run_dir = self._run_dir()
|
|
1467
|
+
script = _base(part) + ".sh"
|
|
1468
|
+
rc, out = self._sh(f"cat > {self._q(run_dir + '/' + script)}", 30, stdin=part["script"] if part["script"].endswith("\n") else part["script"] + "\n")
|
|
1469
|
+
if rc != 0:
|
|
1470
|
+
self._fail("run", f"the script did not land on {where}: " + out.strip()[-400:])
|
|
1471
|
+
return False
|
|
1472
|
+
prices_on_target = ""
|
|
1473
|
+
if self.plan.runner == STANDARD and self.on_host:
|
|
1474
|
+
# the price list is this machine's user's, so the run takes it along (nullius.prices) - an empty one
|
|
1475
|
+
# too: a host's own list, if it has one, never prices a run started here
|
|
1476
|
+
prices_on_target = run_dir + "/prices.json"
|
|
1477
|
+
rc, out = self._sh(f"cat > {self._q(prices_on_target)}", 60, stdin=json.dumps(price_list.load()) + "\n")
|
|
1478
|
+
if rc != 0:
|
|
1479
|
+
self._fail("run", f"the price list did not land on {where}: " + out.strip()[-400:])
|
|
1480
|
+
return False
|
|
1481
|
+
exports = " ".join(f"{k}={self._q(v) if v.startswith('~') else shlex.quote(v)}" for k, v in
|
|
1482
|
+
[("NULLIUS_HARBOR", self.harbor), ("NULLIUS_TASKS", self.tasks_on_target), ("NULLIUS_JOBS", run_dir + "/jobs"),
|
|
1483
|
+
("NULLIUS_PROGRESS", "1")]
|
|
1484
|
+
+ ([("NULLIUS_STANDARD", self.app_on_target or REPO), ("PYTHONDONTWRITEBYTECODE", "1")]
|
|
1485
|
+
if self.plan.runner == STANDARD else [])
|
|
1486
|
+
+ ([("NULLIUS_CATALOGUE", self.catalogue_on_target)]
|
|
1487
|
+
if self.plan.runner == STANDARD and self.catalogue_on_target else [])
|
|
1488
|
+
+ ([("NULLIUS_PRICES", prices_on_target)] if prices_on_target else [])
|
|
1489
|
+
+ ([("NULLIUS_PYTHON", sys.executable)] if self.plan.runner == STANDARD and not self.on_host else [])
|
|
1490
|
+
+ ([("NULLIUS_LICENCE_DIR", self._licence_dir())] if self._licence_dir() else [])
|
|
1491
|
+
+ ([("API_MODEL_NAME", self.plan.model)] if self.plan.model else []))
|
|
1492
|
+
base = script[:-3]
|
|
1493
|
+
# macOS has no setsid executable. Python can create the same session and
|
|
1494
|
+
# exec in place, keeping $!, the pid file and the process group identical.
|
|
1495
|
+
python = shlex.quote("python3" if self.on_host else sys.executable)
|
|
1496
|
+
session = shlex.quote("import os, sys; os.setsid(); os.execvp(sys.argv[1], sys.argv[1:])")
|
|
1497
|
+
detach = (f"if ! command -v setsid >/dev/null 2>&1; then "
|
|
1498
|
+
f"command -v {python} >/dev/null 2>&1 || {{ echo 'detached runs need setsid or Python 3'; exit 1; }}; "
|
|
1499
|
+
f"setsid() {{ exec {python} -c {session} \"$@\"; }}; fi; ")
|
|
1500
|
+
# stdin: the key, the endpoint, then one NAME=value line per agent key, then the end line - which alone says it all arrived
|
|
1501
|
+
launcher = ('IFS= read -r K && IFS= read -r U || { echo @cut; exit 1; }; E=; '
|
|
1502
|
+
'while IFS= read -r L; do [ "$L" = ' + STDIN_END + ' ] && { E=1; break; }; '
|
|
1503
|
+
'case "$L" in [A-Z_]*=*) export "$L";; *) echo @cut; exit 1;; esac; done; [ -n "$E" ] || { echo @cut; exit 1; }; unset E L; '
|
|
1504
|
+
'[ -n "$K" ] && export ' + " ".join(f'{n}="$K"' for n in KEY_NAMES) + '; '
|
|
1505
|
+
'[ -n "$U" ] && export ' + " ".join(f'{n}="$U"' for n in URL_NAMES) + '; unset K U; '
|
|
1506
|
+
f'export {exports}; cd {self._q(run_dir)} || exit 1; '
|
|
1507
|
+
+ detach +
|
|
1508
|
+
# The part claimed before anything else is done for it, created only where there is none (noclobber:
|
|
1509
|
+
# O_EXCL): of two launchers - one a Nullius gone since sent, late to arrive, and the restart's - one
|
|
1510
|
+
# starts the part and the other says so and starts nothing. From the claim on, this launcher runs
|
|
1511
|
+
# in the run directory until the part has started, which is how a restart tells it is on its way.
|
|
1512
|
+
f'( set -C; : >{base}.launch ) 2>/dev/null || {{ echo @claimed; exit 0; }}; rm -f {base}.exit {base}.pid; '
|
|
1513
|
+
# `&` binds to the setsid command alone (hence `;` before it): $! is the detached shell, a session
|
|
1514
|
+
# leader of its own, with nothing of the launcher's pipes, so the launcher returns at once. Its
|
|
1515
|
+
# first act is its pid ($$, the same $!) on disk, before the log opens; its second, a look for a
|
|
1516
|
+
# Stop that came while its launcher had not answered (_kill_part).
|
|
1517
|
+
f"setsid nohup bash -c '{self._licence_trap()}echo $$ >{base}.pid; [ -e {base}.stop ] && exit 130; bash {script} </dev/null >{base}.log 2>&1; echo $? >{base}.exit' "
|
|
1518
|
+
f"</dev/null >/dev/null 2>&1 & echo $!")
|
|
1519
|
+
rc, out = self._sh(launcher, 60, stdin=self._stdin())
|
|
1520
|
+
if rc == 124 and not self._stop.is_set():
|
|
1521
|
+
# An ssh that stalled says nothing of whether the launcher ran. Asking again is safe: the claim above
|
|
1522
|
+
# lets one launcher start the part, and a second that finds it says @claimed and starts nothing.
|
|
1523
|
+
self.log(f" {where} did not answer the launch within 60 s; asking once more")
|
|
1524
|
+
rc, out = self._sh(launcher, 60, stdin=self._stdin())
|
|
1525
|
+
if rc == 124:
|
|
1526
|
+
out = f"{where} did not answer the launch, twice, within 60 s each; nothing is known to have started there"
|
|
1527
|
+
if "@cut" in out.split():
|
|
1528
|
+
self._fail("run", f"{part['label']} did not start on {where}: the key and endpoint reached its launcher cut short; nothing was started")
|
|
1529
|
+
return False
|
|
1530
|
+
if rc == 0 and "@claimed" in out.split():
|
|
1531
|
+
self.log(f" {part['label']} was started already, by a launcher sent before Nullius was gone; following it")
|
|
1532
|
+
self.pid, self.log_offset = None, 0
|
|
1533
|
+
self._save()
|
|
1534
|
+
return True
|
|
1535
|
+
pid = next((int(l) for l in out.strip().splitlines() if l.strip().isdigit()), None)
|
|
1536
|
+
if rc != 0 or not pid:
|
|
1537
|
+
self._fail("run", f"{part['label']} did not start on {where}: " + out.strip()[-400:])
|
|
1538
|
+
return False
|
|
1539
|
+
self.pid, self.log_offset = pid, 0
|
|
1540
|
+
self._save()
|
|
1541
|
+
return True
|
|
1542
|
+
|
|
1543
|
+
def _follow(self, part: dict) -> int:
|
|
1544
|
+
"""Look at the running part every few seconds - its log, the mirror,
|
|
1545
|
+
the trials - until its exit code lands; a part with no process left
|
|
1546
|
+
and no exit code is gone, a failure of its own."""
|
|
1547
|
+
base = _base(part)
|
|
1548
|
+
# Alive while a process runs its script in its run directory (_part_pids): its pid when known, else the
|
|
1549
|
+
# one it wrote or one found by its command line - a part an older launcher started has no pid file - and
|
|
1550
|
+
# never a pid that has since gone to another process.
|
|
1551
|
+
alive = f"[ -n \"$({_part_pids(base, [self.pid])})\" ] && echo alive || echo gone"
|
|
1552
|
+
while True:
|
|
1553
|
+
# Read in the reverse of the order the target writes them (the log, the exit code, the process
|
|
1554
|
+
# ends): an exit code seen means the whole log is there, a process gone means none is coming.
|
|
1555
|
+
rc, out = self._sh(f"cd {self._q(self._run_dir())} 2>/dev/null; s=$({alive}); [ -f {base}.exit ] && s=$(cat {base}.exit); "
|
|
1556
|
+
f"{self._log_since(base)}; printf '\\n__nullius__ %s' \"$s\"", 120)
|
|
1557
|
+
head, _, status = out.rpartition("\n__nullius__ ")
|
|
1558
|
+
if rc == 0:
|
|
1559
|
+
text, size = self._logged(head)
|
|
1560
|
+
if size is not None and size > self.log_offset:
|
|
1561
|
+
self.log_offset = size
|
|
1562
|
+
for line in text.splitlines():
|
|
1563
|
+
self._read(line)
|
|
1564
|
+
self._save()
|
|
1565
|
+
status = status.strip()
|
|
1566
|
+
if self.on_host:
|
|
1567
|
+
self._mirror()
|
|
1568
|
+
else:
|
|
1569
|
+
self._trial_lines()
|
|
1570
|
+
if status.isdigit():
|
|
1571
|
+
return int(status)
|
|
1572
|
+
if status == "gone":
|
|
1573
|
+
self.log(f" {part['label']} is gone without an exit code")
|
|
1574
|
+
return 1
|
|
1575
|
+
if self._stop.wait(SYNC_EVERY):
|
|
1576
|
+
return 130
|
|
1577
|
+
|
|
1578
|
+
def _mirror(self, run: Optional[Callable] = None, timeout: float = 900.0) -> None:
|
|
1579
|
+
"""The job copied back from the host, and the trials that ended said;
|
|
1580
|
+
``run`` and ``timeout`` for the last copy a Stop makes (``_done``)."""
|
|
1581
|
+
rc, out = self._copy_back(run, timeout)
|
|
1582
|
+
if rc != 0:
|
|
1583
|
+
self.log(" the job did not sync back: " + out.strip()[-300:])
|
|
1584
|
+
self._trial_lines()
|
|
1585
|
+
|
|
1586
|
+
|
|
1587
|
+
def _probe(run: Callable, argv: List[str], timeout: float) -> tuple:
|
|
1588
|
+
"""hosts.test wants (rc, stdout, stderr); the execution runner gives (rc, output)."""
|
|
1589
|
+
rc, out = run(argv, timeout)
|
|
1590
|
+
return rc, out, ""
|
|
1591
|
+
|
|
1592
|
+
|
|
1593
|
+
# --- the registry the page reads --------------------------------------------
|
|
1594
|
+
|
|
1595
|
+
EXECUTIONS: Dict[str, Execution] = {}
|
|
1596
|
+
|
|
1597
|
+
|
|
1598
|
+
def start(plan: Plan, **kw) -> Execution:
|
|
1599
|
+
ex = Execution(plan, **kw)
|
|
1600
|
+
EXECUTIONS[ex.id] = ex
|
|
1601
|
+
return ex.start()
|
|
1602
|
+
|
|
1603
|
+
|
|
1604
|
+
def active() -> List[Execution]:
|
|
1605
|
+
return [e for e in EXECUTIONS.values() if e.state in ALIVE]
|
|
1606
|
+
|
|
1607
|
+
|
|
1608
|
+
def for_job(job_id: str) -> Optional[Execution]:
|
|
1609
|
+
"""The newest execution whose script writes this job, if any - the one
|
|
1610
|
+
the job page shows beside the job, or instead of it while Harbor has
|
|
1611
|
+
not written it yet."""
|
|
1612
|
+
found = [e for e in EXECUTIONS.values() if any(str(j.get("name")) == job_id for j in e.plan.jobs)]
|
|
1613
|
+
return max(found, key=lambda e: e.started) if found else None
|
|
1614
|
+
|
|
1615
|
+
|
|
1616
|
+
ENDED = ("stopped", "failed", "done")
|
|
1617
|
+
|
|
1618
|
+
|
|
1619
|
+
def ended_for(job_id: str) -> Optional[tuple]:
|
|
1620
|
+
"""``(state, finished_at)`` when the run that writes this job has ended
|
|
1621
|
+
- stopped, failed or done - by the runner's own record; None while it
|
|
1622
|
+
is alive or when no run of this application wrote the job. A Harbor
|
|
1623
|
+
killed by a Stop leaves its feed without ``job.end`` and its result
|
|
1624
|
+
record with trials "running", so a page reading either alone would say
|
|
1625
|
+
the job goes on; this is the fact that says otherwise."""
|
|
1626
|
+
ex = for_job(job_id)
|
|
1627
|
+
return ex.job_state(job_id) if ex is not None else None
|
|
1628
|
+
|
|
1629
|
+
|
|
1630
|
+
def restore(root: Optional[str] = None, **kw) -> List[Execution]:
|
|
1631
|
+
"""At start-up: every recorded run under the root. The ones still alive
|
|
1632
|
+
are reattached (their thread follows the part that was running, or
|
|
1633
|
+
waits at the pause it was at); the finished ones are listed as they
|
|
1634
|
+
ended. Returns the ones reattached."""
|
|
1635
|
+
root = root or runs_root(kw.get("env", os.environ))
|
|
1636
|
+
back = []
|
|
1637
|
+
try:
|
|
1638
|
+
names = sorted(os.listdir(root))
|
|
1639
|
+
except OSError:
|
|
1640
|
+
return back
|
|
1641
|
+
for name in names:
|
|
1642
|
+
path = os.path.join(root, name, RECORD)
|
|
1643
|
+
try:
|
|
1644
|
+
with open(path, encoding="utf-8") as fh:
|
|
1645
|
+
rec = json.load(fh)
|
|
1646
|
+
plan = Plan.from_dict(rec["plan"])
|
|
1647
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
1648
|
+
continue
|
|
1649
|
+
ex = Execution(plan, root=root, record=rec, **kw)
|
|
1650
|
+
EXECUTIONS[ex.id] = ex
|
|
1651
|
+
if ex.state in ALIVE:
|
|
1652
|
+
back.append(ex.resume())
|
|
1653
|
+
return back
|