nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/standard/run.py
ADDED
|
@@ -0,0 +1,1781 @@
|
|
|
1
|
+
"""A run: a benchmark, an agent or a capability, a model - units, then a job.
|
|
2
|
+
|
|
3
|
+
The plan is units × stages. The driver names the units (``units``); each is
|
|
4
|
+
set up (``setup``: its workspace filled, its prompt and environment
|
|
5
|
+
returned), produced - the agent driven over ACP in that workspace, or the
|
|
6
|
+
capability driven through its port - and scored (``score``). Units run
|
|
7
|
+
``concurrency`` at a time, each with a driver process per worker, an agent or
|
|
8
|
+
capability process of its own, a home of its own, and a relay of its own.
|
|
9
|
+
|
|
10
|
+
A relay per unit, not one for the run: a shared relay is one more thing whose
|
|
11
|
+
failure stops every unit, and a network hop for a unit sandboxed elsewhere.
|
|
12
|
+
One per unit costs a few megabytes and loses only a shared rate limit - each
|
|
13
|
+
unit backs off on its own, as it always did. Its record sits in the trial's
|
|
14
|
+
``.work/relay``.
|
|
15
|
+
|
|
16
|
+
Two backends (``backend``). ``local``: processes on this machine, which are
|
|
17
|
+
not a sandbox - what an agent runs, it runs as the user (``docs/standard.md``,
|
|
18
|
+
"Safety"). ``docker``: the agent or the capability in a container of its own on
|
|
19
|
+
an internal network with the unit's relay beside it (``container``); an
|
|
20
|
+
own-image task may grant a public-address proxy with ``internet: true``, and
|
|
21
|
+
an agent whose environment declares ``network`` gets a public web proxy.
|
|
22
|
+
On docker the agent's or capability's packages are installed in the base image, on the libc and tools its container runs on.
|
|
23
|
+
The driver, its environment installed here, stays on this machine either
|
|
24
|
+
way: it owns the answers. So do a benchmark's own MCP servers
|
|
25
|
+
(``mcp_servers``), in the driver's environment: on the local backend the
|
|
26
|
+
agent starts each from it, in a container the run does, behind a door on the
|
|
27
|
+
unit's network (``mcp_machine``).
|
|
28
|
+
|
|
29
|
+
A unit whose environment is an **own image** (``"image": "own"``) runs in
|
|
30
|
+
the task's own container, whatever the backend (a control's too: nothing of
|
|
31
|
+
an agent runs there; an agent's needs the container backend): its setup names
|
|
32
|
+
the image (``image``: a folder of the upstream to build, or a reference by
|
|
33
|
+
digest) and, optionally, the working directory (``workdir``; the image's own
|
|
34
|
+
by default) and a check (``verify``) - files of the upstream copied in, a
|
|
35
|
+
command run, a folder of results copied out - which runs in that container
|
|
36
|
+
after the agent ends and before ``score``, which is sent what it found
|
|
37
|
+
(``verified``: {exit, timed_out, output, dir, collected}) and reads the rest from ``dir``. The
|
|
38
|
+
oracle's reference solution is such a step too (its reply's ``solve``). So
|
|
39
|
+
the driver never needs docker, and the task's tests reach the container only
|
|
40
|
+
once the agent is done and nothing it started still runs there
|
|
41
|
+
(``container.Box``).
|
|
42
|
+
|
|
43
|
+
What a process is given of this machine's environment is a short list
|
|
44
|
+
(``KEEP``) - never the keys Settings holds. The relay holds the real key;
|
|
45
|
+
everything else gets its address and ``relay.PLACEHOLDER_KEY``.
|
|
46
|
+
|
|
47
|
+
An agent run on the local backend starts only on the caller's explicit yes
|
|
48
|
+
(``unsandboxed``); on either backend its permission requests are granted
|
|
49
|
+
(``acp.Client(grant=True)``): no one is there to answer them during a run.
|
|
50
|
+
The job's config records both, and the backend.
|
|
51
|
+
"""
|
|
52
|
+
from __future__ import annotations
|
|
53
|
+
|
|
54
|
+
import hashlib
|
|
55
|
+
import json
|
|
56
|
+
import os
|
|
57
|
+
import re
|
|
58
|
+
import shutil
|
|
59
|
+
import signal
|
|
60
|
+
import stat
|
|
61
|
+
import subprocess
|
|
62
|
+
import threading
|
|
63
|
+
import time
|
|
64
|
+
import traceback
|
|
65
|
+
import tempfile
|
|
66
|
+
from concurrent.futures import CancelledError, ThreadPoolExecutor, as_completed
|
|
67
|
+
from contextlib import contextmanager
|
|
68
|
+
from typing import Callable, Dict, List, Optional
|
|
69
|
+
|
|
70
|
+
from nullius import paths, prices as price_list, progress, providers
|
|
71
|
+
from nullius.redact import redact
|
|
72
|
+
from nullius.harbor import feed as live_feed
|
|
73
|
+
from nullius.standard import acp, container, definitions, hostreq, install, job, jsonl, ledger as ledgers, match, mcp_machine, pace as pacing, relay, shard as shards
|
|
74
|
+
|
|
75
|
+
# the totals a unit's end carries onto the live board, from its trial's agent_result
|
|
76
|
+
FEED_TOTALS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd")
|
|
77
|
+
|
|
78
|
+
# The variables of this machine's environment a run's processes keep: where
|
|
79
|
+
# programs are, the locale, the temporary folder. Not a key among them.
|
|
80
|
+
KEEP = ("PATH", "LANG", "LC_ALL", "TMPDIR", "TERM", "NVM_DIR", "NODE_PATH", "SSL_CERT_FILE", "SSL_CERT_DIR")
|
|
81
|
+
APP = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) # the folder holding the nullius package
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def driver_path() -> str:
|
|
85
|
+
"""A driver's ``PYTHONPATH``: a folder whose one entry links to the ``nullius`` package, for the
|
|
86
|
+
kits' drivers that import it. Not ``APP``: installed from a wheel, that is the tool's whole
|
|
87
|
+
site-packages, and every package there would come before the driver's own environment's."""
|
|
88
|
+
pkg = os.path.join(APP, "nullius")
|
|
89
|
+
root = os.path.join(paths.cache_dir(), "driver-path", hashlib.sha256(pkg.encode()).hexdigest()[:16])
|
|
90
|
+
link = os.path.join(root, "nullius")
|
|
91
|
+
if os.path.realpath(link) != os.path.realpath(pkg):
|
|
92
|
+
os.makedirs(root, exist_ok=True)
|
|
93
|
+
# written whole beside the folder, then renamed in: drivers start in parallel, and the folder
|
|
94
|
+
# on their PYTHONPATH never holds anything but the link
|
|
95
|
+
part = f"{root}.{os.getpid()}.{threading.get_ident()}.part"
|
|
96
|
+
os.symlink(pkg, part)
|
|
97
|
+
os.replace(part, link)
|
|
98
|
+
return root
|
|
99
|
+
|
|
100
|
+
PROVIDERS = {
|
|
101
|
+
"openai": {"base": "https://api.openai.com/v1", "key": "OPENAI_API_KEY"},
|
|
102
|
+
"openrouter": {"base": "https://openrouter.ai/api/v1", "key": "OPENROUTER_API_KEY"},
|
|
103
|
+
"anthropic": {"base": "https://api.anthropic.com/v1", "key": "ANTHROPIC_API_KEY", "key_header": "x-api-key"},
|
|
104
|
+
"custom": {"base_var": "MODEL_API_URL", "key": "MODEL_API_KEY"},
|
|
105
|
+
}
|
|
106
|
+
TURN_SECONDS = 900
|
|
107
|
+
DRIVER_SECONDS = 600 # a driver's setup, oracle and score requests, each
|
|
108
|
+
IMAGE_SECONDS = 3600 # an own image's pull or build (container.own_image)
|
|
109
|
+
MAX_AGENT_SECONDS = 6 * 3600 # the longest a unit's own agent limit (setup's agent_seconds) may ask
|
|
110
|
+
MAX_TURNS = 20
|
|
111
|
+
BACKENDS = definitions.names.BACKENDS
|
|
112
|
+
_PROBED = threading.Lock() # a run's first unit, claimed once (Run.first_probe)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
class Refused(Exception):
|
|
116
|
+
"""The run cannot start: the pair does not fit, or a setting is missing."""
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def provider_endpoint(provider: str, env=os.environ) -> tuple:
|
|
120
|
+
p = PROVIDERS.get(provider)
|
|
121
|
+
if not p:
|
|
122
|
+
raise Refused(f"provider {provider!r} is not one of {list(PROVIDERS)}")
|
|
123
|
+
base = p.get("base") or env.get(p["base_var"]) or ""
|
|
124
|
+
key = env.get(p["key"]) or ("local" if provider == "custom" else "")
|
|
125
|
+
if not base:
|
|
126
|
+
raise Refused(f"{p['base_var']} is not set in Settings")
|
|
127
|
+
if not key:
|
|
128
|
+
raise Refused(f"{p['key']} is not set in Settings")
|
|
129
|
+
if not key.isprintable() or any(ord(c) > 255 for c in key) or key != key.strip():
|
|
130
|
+
# http.client refuses such a value, and quotes it whole in its error
|
|
131
|
+
raise Refused(f"{p['key']} holds a character a request header cannot carry (a line break, a space or a "
|
|
132
|
+
"byte-order mark around it, or one outside Latin-1): set it again in Settings")
|
|
133
|
+
return base, key
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def _slug(s: str) -> str:
|
|
137
|
+
return re.sub(r"[^A-Za-z0-9_.-]+", "_", s).strip("_")[:60] or "unit"
|
|
138
|
+
|
|
139
|
+
|
|
140
|
+
def driver_secrets(bench: dict, licences) -> dict:
|
|
141
|
+
"""The set secrets a benchmark's environments send to its driver (``to: driver``)."""
|
|
142
|
+
out = {}
|
|
143
|
+
for e in (bench.get("environments") or {}).values():
|
|
144
|
+
out.update(definitions.secret_values(e, licences, "driver"))
|
|
145
|
+
return out
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def option_env(values: dict) -> dict:
|
|
149
|
+
"""A benchmark's option values as its driver reads them: ``NULLIUS_OPTION_<NAME>``."""
|
|
150
|
+
return {"NULLIUS_OPTION_" + n.upper(): str(v) for n, v in (values or {}).items()}
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# The endpoint refusing the run itself, not one unit: a key it rejects, a model it does not know.
|
|
154
|
+
REFUSALS = ("AgentAuthenticationError", "ModelNotFoundError")
|
|
155
|
+
REFUSED_IN_A_ROW = 3
|
|
156
|
+
# a unit still running when the run is stopped: Harbor's name for a trial a stopped job cut short
|
|
157
|
+
CANCELLED = "CancelledError"
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def run_group(cmd: List[str], timeout: float, **kw) -> subprocess.CompletedProcess:
|
|
161
|
+
"""``subprocess.run`` for an agent's command, in a session of its own, and
|
|
162
|
+
the whole group ended afterwards - on a timeout, which kills only the
|
|
163
|
+
direct child, and at a normal end, where whatever it left in the
|
|
164
|
+
background would keep running, still writing into the workspace while
|
|
165
|
+
it is scored. Raises ``subprocess.TimeoutExpired`` as ``run`` does.
|
|
166
|
+
The agent ending is what is waited for, not its output's end: a child
|
|
167
|
+
left in the background holding stdout would otherwise hold the unit
|
|
168
|
+
until its timeout."""
|
|
169
|
+
stopped = kw.pop("stopped", None)
|
|
170
|
+
streams = {}
|
|
171
|
+
try:
|
|
172
|
+
for name in ("stdout", "stderr"):
|
|
173
|
+
if kw.get(name) == subprocess.PIPE:
|
|
174
|
+
streams[name] = tempfile.TemporaryFile()
|
|
175
|
+
kw[name] = streams[name]
|
|
176
|
+
p = subprocess.Popen(cmd, start_new_session=True, **kw)
|
|
177
|
+
deadline = time.monotonic() + timeout
|
|
178
|
+
try:
|
|
179
|
+
while p.poll() is None:
|
|
180
|
+
if stopped is not None and stopped.is_set():
|
|
181
|
+
raise CancelledError("the run was stopped")
|
|
182
|
+
remaining = deadline - time.monotonic()
|
|
183
|
+
if remaining <= 0:
|
|
184
|
+
raise subprocess.TimeoutExpired(cmd, timeout)
|
|
185
|
+
try:
|
|
186
|
+
p.wait(timeout=min(remaining, 0.1))
|
|
187
|
+
except subprocess.TimeoutExpired:
|
|
188
|
+
pass
|
|
189
|
+
finally:
|
|
190
|
+
_kill_group(p)
|
|
191
|
+
p.wait()
|
|
192
|
+
output = {}
|
|
193
|
+
for name, stream in streams.items():
|
|
194
|
+
stream.seek(0)
|
|
195
|
+
data = stream.read()
|
|
196
|
+
if kw.get("text") or kw.get("universal_newlines") or kw.get("encoding"):
|
|
197
|
+
data = data.decode(kw.get("encoding") or "utf-8", errors=kw.get("errors") or "strict")
|
|
198
|
+
output[name] = data
|
|
199
|
+
return subprocess.CompletedProcess(cmd, p.returncode, output.get("stdout"), output.get("stderr"))
|
|
200
|
+
finally:
|
|
201
|
+
for stream in streams.values():
|
|
202
|
+
stream.close()
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def _kill_group(p: subprocess.Popen) -> None:
|
|
206
|
+
try:
|
|
207
|
+
os.killpg(p.pid, signal.SIGKILL)
|
|
208
|
+
except (ProcessLookupError, PermissionError):
|
|
209
|
+
pass
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def base_env(home: str, extra: Optional[dict] = None) -> dict:
|
|
213
|
+
env = {k: os.environ[k] for k in KEEP if k in os.environ}
|
|
214
|
+
env.setdefault("PATH", os.defpath)
|
|
215
|
+
env["HOME"] = home
|
|
216
|
+
env.update(extra or {})
|
|
217
|
+
return env
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
def _render(template: str, slots: Dict[str, str]) -> str:
|
|
221
|
+
for k, v in slots.items():
|
|
222
|
+
template = template.replace("{" + k + "}", v)
|
|
223
|
+
return template
|
|
224
|
+
|
|
225
|
+
|
|
226
|
+
def agent_env(agent: dict, inst: str, home: str, base_url: str, model: str, options: Optional[dict] = None) -> dict:
|
|
227
|
+
"""The environment of an agent - or of a capability that calls a model - and
|
|
228
|
+
its config files written into its home, from its definition's ``model``
|
|
229
|
+
templates; then its own ``env`` (``install.process_env``). ``options``:
|
|
230
|
+
the run's value of each option (``definitions.options_of``), the slots
|
|
231
|
+
``{opt_<name>}``; an option left out is filled with its default."""
|
|
232
|
+
options = definitions.options_of(agent, options)[0]
|
|
233
|
+
slots = {"base_url": base_url, "key": relay.PLACEHOLDER_KEY, "model": model, "home": home,
|
|
234
|
+
**{"opt_" + k: v for k, v in options.items()}}
|
|
235
|
+
m = agent.get("model") or {}
|
|
236
|
+
for dest, src in (m.get("files") or {}).items():
|
|
237
|
+
path = _render(dest, slots)
|
|
238
|
+
os.makedirs(os.path.dirname(path), exist_ok=True)
|
|
239
|
+
with open(os.path.join(agent["_here"], src)) as f:
|
|
240
|
+
text = _render(f.read(), slots)
|
|
241
|
+
with open(path, "w") as f:
|
|
242
|
+
f.write(text)
|
|
243
|
+
env = base_env(home, {k: _render(v, slots) for k, v in (m.get("env") or {}).items()})
|
|
244
|
+
env.update(install.process_env(agent, inst, home, options))
|
|
245
|
+
env["PATH"] = os.pathsep.join(install.bin_dirs(inst) + [env["PATH"]])
|
|
246
|
+
return env
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _own_dir(path: str) -> None:
|
|
250
|
+
"""``path`` as a folder this user alone owns: made 0700 when absent; when
|
|
251
|
+
there already - a shared place like /dev/shm, where another user could
|
|
252
|
+
have made it first, or put a link there - kept only if it is a real
|
|
253
|
+
folder, this user's, open to no one else. Refused otherwise."""
|
|
254
|
+
try:
|
|
255
|
+
os.mkdir(path, 0o700)
|
|
256
|
+
except FileExistsError:
|
|
257
|
+
pass
|
|
258
|
+
st = os.lstat(path)
|
|
259
|
+
if not stat.S_ISDIR(st.st_mode) or st.st_uid != os.getuid() or st.st_mode & 0o077:
|
|
260
|
+
raise Refused(f"{path} is not a folder of this user's alone: no licence is written there")
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
class Licences:
|
|
264
|
+
"""What a run holds of its benchmark's licences, by name: ``get(name)``
|
|
265
|
+
is {kind, value} for a key or a server, {kind, path} for a file, or None.
|
|
266
|
+
|
|
267
|
+
On this machine they are read from the benchmark's store
|
|
268
|
+
(``nullius.secrets``) when asked - when a unit starts - and a file is
|
|
269
|
+
used where it is kept; a data provider's token (``provider``) from that
|
|
270
|
+
provider's store alone. A run on a host is handed them instead
|
|
271
|
+
(``handed``): its launcher's stdin, a file written to a folder in memory
|
|
272
|
+
(``/dev/shm``) that the run removes when it ends. Only the names the
|
|
273
|
+
benchmark's environments declare are ever looked up, each as the kind
|
|
274
|
+
it is declared."""
|
|
275
|
+
|
|
276
|
+
def __init__(self, bench: dict, given: Optional[Dict[str, dict]] = None):
|
|
277
|
+
self.key = bench["key"]
|
|
278
|
+
declared = definitions.secrets_of(bench)
|
|
279
|
+
self.kinds = {s["name"]: s["kind"] for s in declared}
|
|
280
|
+
self.providers = {s["name"]: s.get("provider") or "" for s in declared}
|
|
281
|
+
self.given = given
|
|
282
|
+
|
|
283
|
+
@classmethod
|
|
284
|
+
def handed(cls, bench: dict, values: Dict[str, bytes], spill: str) -> "Licences":
|
|
285
|
+
"""The licences a host's run was sent (name -> bytes), a file among
|
|
286
|
+
them written to ``spill`` (0700) as a 0600 file of its own."""
|
|
287
|
+
me = cls(bench, {})
|
|
288
|
+
for name, data in values.items():
|
|
289
|
+
kind = me.kinds.get(name)
|
|
290
|
+
if kind is None:
|
|
291
|
+
continue
|
|
292
|
+
if kind == "file":
|
|
293
|
+
_own_dir(spill)
|
|
294
|
+
p = os.path.join(spill, name)
|
|
295
|
+
if os.path.lexists(p): # ours: the folder is checked to be this user's alone
|
|
296
|
+
os.unlink(p)
|
|
297
|
+
fd = os.open(p, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
|
|
298
|
+
with os.fdopen(fd, "wb") as f:
|
|
299
|
+
f.write(data)
|
|
300
|
+
me.given[name] = {"kind": kind, "path": p}
|
|
301
|
+
else:
|
|
302
|
+
me.given[name] = {"kind": kind, "value": data.decode(errors="replace").strip()}
|
|
303
|
+
return me
|
|
304
|
+
|
|
305
|
+
def get(self, name: str) -> Optional[dict]:
|
|
306
|
+
kind = self.kinds.get(name)
|
|
307
|
+
if kind is None:
|
|
308
|
+
return None
|
|
309
|
+
if self.given is not None:
|
|
310
|
+
return self.given.get(name)
|
|
311
|
+
from nullius import secrets
|
|
312
|
+
store, kept = secrets.store_of(self.key, name, self.providers.get(name, ""))
|
|
313
|
+
if kind == "file":
|
|
314
|
+
p = secrets.path(store, kept)
|
|
315
|
+
return {"kind": kind, "path": p} if p else None
|
|
316
|
+
data = secrets.value(store, kept)
|
|
317
|
+
text = data.decode(errors="replace").strip() if data else ""
|
|
318
|
+
return {"kind": kind, "value": text} if text else None
|
|
319
|
+
|
|
320
|
+
|
|
321
|
+
class Run:
|
|
322
|
+
def __init__(self, bench: dict, other: dict, *, model: str, provider: str, dataset: Optional[str] = None,
|
|
323
|
+
limit: Optional[int] = None, concurrency: int = 4, jobs_root: str = "jobs", name: Optional[str] = None,
|
|
324
|
+
turn_seconds: Optional[int] = None, say: Callable[[str], None] = print, env=os.environ,
|
|
325
|
+
proving: bool = False, unsandboxed: bool = False, judge_model: Optional[str] = None,
|
|
326
|
+
backend: str = "local", options: Optional[dict] = None, shard: Optional[tuple] = None,
|
|
327
|
+
launch: Optional[str] = None, repeat: int = 1, simulator_model: Optional[str] = None,
|
|
328
|
+
licences: Optional[Licences] = None, bench_options: Optional[dict] = None, which=shutil.which,
|
|
329
|
+
remove_images: bool = False, prices: Optional[dict] = None, max_usd: Optional[float] = None,
|
|
330
|
+
max_usd_per_unit: Optional[float] = None):
|
|
331
|
+
"""``proving``: the run is a conformance kit's, the one run that may
|
|
332
|
+
start before the port it checks is proven - the family must still match.
|
|
333
|
+
``unsandboxed``: consent to execute code as the user: required for
|
|
334
|
+
local agents, and for setup checks that run code without Docker.
|
|
335
|
+
``backend``: ``local`` or ``container`` (``docker``, its earlier name), which needs
|
|
336
|
+
no such yes and is refused when docker is missing or not usable.
|
|
337
|
+
``options``: the
|
|
338
|
+
agent's or capability's options set for this run, by name; the rest keep
|
|
339
|
+
their defaults, and the job's config records every value.
|
|
340
|
+
``shard``: (I, N), only the units of shard I of N (``shard``), before
|
|
341
|
+
``limit``; ``launch``: the id every shard of one launch records.
|
|
342
|
+
``repeat``: each unit run that many times (a conformance check run for
|
|
343
|
+
confidence, ``conform``), each run a trial of its own tagged
|
|
344
|
+
``u0042r2``; once by default, tagged ``u0042`` as always.
|
|
345
|
+
``simulator_model``: a benchmark whose user a model plays
|
|
346
|
+
(``simulator``): this model instead of its official one.
|
|
347
|
+
``prices``: this run's own prices, by model, over the user's list
|
|
348
|
+
(``nullius.prices``): what a call's cost is estimated from.
|
|
349
|
+
``max_usd_per_unit``: each unit's own spending cap, kept in the same
|
|
350
|
+
ledger: its own calls and its simulated user's count toward it, its
|
|
351
|
+
judge's do not. A unit that reaches it ends as ``ledger.UNIT_ERROR``,
|
|
352
|
+
an agent error counted against the agent, and the run goes on.
|
|
353
|
+
``max_usd``: the run's spending cap (``ledger``): every relay of the
|
|
354
|
+
job books a call before sending it, and once one does not fit no
|
|
355
|
+
further call goes out and nothing more is begun. Refused when a model
|
|
356
|
+
the run calls has no price: a call that cannot be priced cannot be
|
|
357
|
+
booked.
|
|
358
|
+
``licences``: what the run holds of the benchmark's licences
|
|
359
|
+
(:class:`Licences`); by default its store on this machine.
|
|
360
|
+
``bench_options``: the benchmark's own options for this run
|
|
361
|
+
(``options`` in its definition: a search, a mode), the rest at their
|
|
362
|
+
defaults; a value that needs an embedding model gets a relay of the
|
|
363
|
+
run's own for it, one that needs something of this machine
|
|
364
|
+
(``host``) is refused without it."""
|
|
365
|
+
backend = definitions.names.backend(backend)
|
|
366
|
+
if backend not in BACKENDS:
|
|
367
|
+
raise Refused(f"backend {backend!r} is not one of {list(BACKENDS)}")
|
|
368
|
+
if backend == "local" and other.get("_kind") == "agent" and not unsandboxed:
|
|
369
|
+
raise Refused(f"{other['label']} would run commands as you: the local backend is not a sandbox. "
|
|
370
|
+
"Say so explicitly to go on (--unsandboxed), or run it in a container (--backend container).")
|
|
371
|
+
why = match.fit(bench, other)
|
|
372
|
+
if proving:
|
|
373
|
+
why = [w for w in why if "does not offer" not in w and "has not passed its kit" not in w]
|
|
374
|
+
if why:
|
|
375
|
+
raise Refused(f"{other['label']} cannot run {bench['label']}: " + "; ".join(why))
|
|
376
|
+
if backend == "container":
|
|
377
|
+
why = container.unusable()
|
|
378
|
+
if why:
|
|
379
|
+
raise Refused(f"the container backend cannot run here: {why}")
|
|
380
|
+
self.container_check = False
|
|
381
|
+
if backend == "local" and other.get("_kind") == "upstream" and (bench.get("score") or {}).get("runs_code"):
|
|
382
|
+
why = container.unusable()
|
|
383
|
+
if why and not unsandboxed:
|
|
384
|
+
raise Refused(f"{bench['label']}'s check executes generated code, but Docker is unavailable: {why}. "
|
|
385
|
+
"Make Docker usable, or explicitly allow execution as you (--unsandboxed).")
|
|
386
|
+
self.container_check = not why
|
|
387
|
+
self.bench, self.other, self.model, self.provider, self.backend = bench, other, model, provider, backend
|
|
388
|
+
# A setup (definitions.setup): the upstream's own answerer, asked as one model (``asked``), the relay
|
|
389
|
+
# sending another (``sent``); its units may be its own (a conversation where the paper batches questions).
|
|
390
|
+
self.upstream = other.get("_kind") == "upstream"
|
|
391
|
+
self.setup_driver = None
|
|
392
|
+
self.asked = other.get("asked", "") if self.upstream else ""
|
|
393
|
+
if self.upstream:
|
|
394
|
+
self.model = other["sent"]
|
|
395
|
+
self.bench = bench = dict(bench, units=other["units"])
|
|
396
|
+
self.licences = licences or Licences(bench) # where a benchmark's environments find the licences they need
|
|
397
|
+
self.unsandboxed = bool(unsandboxed)
|
|
398
|
+
self.image = ""
|
|
399
|
+
self.options, bad = definitions.options_of(other, options)
|
|
400
|
+
if bad:
|
|
401
|
+
raise Refused("; ".join(bad))
|
|
402
|
+
# A benchmark scored by a model: its official judge, or the one asked for; its calls go through the relay
|
|
403
|
+
# on a route of their own (<tag>-judge), recorded beside the unit's and never counted as the agent's.
|
|
404
|
+
# The official one is written as its maker spells it; the run's provider may spell it otherwise.
|
|
405
|
+
j = (bench.get("score") or {}).get("judge")
|
|
406
|
+
self.judge_model = (judge_model or providers.served_as(j["model"], provider)) if j else None
|
|
407
|
+
# A benchmark whose user a model plays: its driver's turns ask that model the same way, on <tag>-simulator.
|
|
408
|
+
# A control holds no conversation, so it never asks it.
|
|
409
|
+
sim = bench.get("simulator")
|
|
410
|
+
self.simulator_model = (simulator_model or providers.served_as(sim["model"], provider)) if sim else None
|
|
411
|
+
# A control (definitions.control) calls no model: its run needs a key only for the benchmark's judge.
|
|
412
|
+
ds = [d for d in bench["datasets"] if dataset in (None, d["id"])]
|
|
413
|
+
if not ds:
|
|
414
|
+
raise Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}")
|
|
415
|
+
self.ds = ds[0]
|
|
416
|
+
try:
|
|
417
|
+
_, env_def = definitions.environment_of(bench, None, self.ds)
|
|
418
|
+
except ValueError as e:
|
|
419
|
+
raise Refused(str(e))
|
|
420
|
+
if backend != "container" and other.get("_kind") == "control" and definitions.own_image(env_def):
|
|
421
|
+
why = container.unusable()
|
|
422
|
+
if why:
|
|
423
|
+
raise Refused(f"each {(bench.get('units') or {}).get('noun', 'unit')} of {bench['label']} runs in its own image, "
|
|
424
|
+
f"which needs docker: {why}")
|
|
425
|
+
# The benchmark's own options on this dataset; what their values need: an embedding model the driver
|
|
426
|
+
# calls through a relay of the run's own (bench_relay), programs on this machine.
|
|
427
|
+
# each option where it applies: a setup's own (a retriever, a top-k), or those of the benchmark's port
|
|
428
|
+
self.bench_options, bad = definitions.bench_options_of(bench, self.ds["id"], bench_options,
|
|
429
|
+
other["key"] if self.upstream else "yours")
|
|
430
|
+
if bad:
|
|
431
|
+
raise Refused("; ".join(bad))
|
|
432
|
+
needs = definitions.option_needs(bench, self.bench_options)
|
|
433
|
+
self.files = needs["files"]
|
|
434
|
+
self.file_env: Dict[str, str] = {}
|
|
435
|
+
for m in needs["embeddings"]:
|
|
436
|
+
why = providers.embedding_problem(m, provider)
|
|
437
|
+
if why:
|
|
438
|
+
raise Refused(why)
|
|
439
|
+
self.embed_model = providers.served_as(needs["embeddings"][0], provider) if needs["embeddings"] else None
|
|
440
|
+
self.rerank_model = providers.served_as(needs["reranker"][0], provider) if needs["reranker"] else None
|
|
441
|
+
# What a call's cost is estimated from when the provider does not say it: the user's price list
|
|
442
|
+
# (nullius.prices), this run's own prices over it. Read once, here: a run never fetches a price.
|
|
443
|
+
self.run_prices = dict(prices or {})
|
|
444
|
+
self.prices = {**price_list.table(), **self.run_prices}
|
|
445
|
+
self.max_usd, self.unit_max_usd, self.ledger = None, None, None
|
|
446
|
+
self.capped = False # whether the run's spending cap stopped it
|
|
447
|
+
for flag, given, what in (("--max-usd", max_usd, "the run's spending cap"), ("--max-usd-per-unit", max_usd_per_unit, "a unit's spending cap")):
|
|
448
|
+
if given is None:
|
|
449
|
+
continue
|
|
450
|
+
try:
|
|
451
|
+
cap = float(given)
|
|
452
|
+
except (TypeError, ValueError):
|
|
453
|
+
cap = 0.0
|
|
454
|
+
if not 0 < cap < float("inf"):
|
|
455
|
+
raise Refused(f"{flag} {given!r}: {what} in USD, above zero")
|
|
456
|
+
if flag == "--max-usd":
|
|
457
|
+
self.max_usd = cap
|
|
458
|
+
else:
|
|
459
|
+
self.unit_max_usd = cap
|
|
460
|
+
if self.max_usd is not None or self.unit_max_usd is not None:
|
|
461
|
+
unpriced = [m for m in self.models_called() if m not in self.prices]
|
|
462
|
+
if unpriced:
|
|
463
|
+
raise Refused("a spending cap needs a price for every model the run calls, and there is none for "
|
|
464
|
+
+ ", ".join(unpriced) + ": nullius prices update, nullius prices set MODEL INPUT OUTPUT, "
|
|
465
|
+
"or --price MODEL=INPUT,OUTPUT")
|
|
466
|
+
on_host = needs["host"]
|
|
467
|
+
if self.upstream and backend == "container":
|
|
468
|
+
# its image installs what has an install; nothing else reaches its container
|
|
469
|
+
imaged = hostreq.in_image(bench, on_host)
|
|
470
|
+
cannot = [n for n in hostreq.requirements(bench, on_host) if n not in imaged]
|
|
471
|
+
if cannot:
|
|
472
|
+
raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs "
|
|
473
|
+
f"{', '.join(cannot)}, which a container setup cannot provide: choose local processes")
|
|
474
|
+
on_host = []
|
|
475
|
+
lacking = hostreq.problems(bench, on_host, which)
|
|
476
|
+
if lacking:
|
|
477
|
+
raise Refused(f"{', '.join(f'{k}={v}' for k, v in self.bench_options.items())} needs on this machine "
|
|
478
|
+
+ "; ".join(f"{n} ({why})" for n, why in lacking.items())
|
|
479
|
+
+ (". Install with: " + " && ".join(lines) if (lines := hostreq.install_lines(bench, list(lacking)))
|
|
480
|
+
else ". Run it on a machine that has it"))
|
|
481
|
+
self.bench_relay = None
|
|
482
|
+
keyless = other.get("_kind") == "control" and not self.judge_model and not self.embed_model and not self.rerank_model
|
|
483
|
+
self.base, self.key = (None, None) if keyless else provider_endpoint(provider, env)
|
|
484
|
+
self.limit, self.concurrency = limit, max(1, int(concurrency))
|
|
485
|
+
self.repeat = max(1, int(repeat or 1))
|
|
486
|
+
# an own image this run pulled or built stays for the next run, unless the user asks it removed
|
|
487
|
+
# when its last unit ends (never with a repeat): {image: how many units are using it now}
|
|
488
|
+
self.keep_images = not remove_images or self.repeat > 1
|
|
489
|
+
self._image_users: Dict[str, int] = {}
|
|
490
|
+
self.turn_seconds = (turn_seconds or (other.get("turn_seconds") if self.upstream else None)
|
|
491
|
+
or (bench.get("limits") or {}).get("turn_seconds") or TURN_SECONDS)
|
|
492
|
+
# the most agent turns a conversation runs; a driver whose upstream ends the conversation by its own
|
|
493
|
+
# limits sets this above them (limits.turns), so that its limits, not this one, are the ones met
|
|
494
|
+
self.max_turns = (bench.get("limits") or {}).get("turns") or MAX_TURNS
|
|
495
|
+
self.say = say
|
|
496
|
+
stamp = time.strftime("%Y-%m-%d__%H-%M-%S")
|
|
497
|
+
self.shard = tuple(shard) if shard and shard[1] > 1 else None
|
|
498
|
+
self.launch = launch or (shards.launch_id(definitions.job_word(bench), definitions.job_word(other), model)
|
|
499
|
+
if self.shard else None)
|
|
500
|
+
self.name = name or (f"{stamp}__{definitions.job_word(bench)}__{definitions.job_word(other)}"
|
|
501
|
+
+ (f"__shard-{self.shard[0]}of{self.shard[1]}" if self.shard else ""))
|
|
502
|
+
self.job_dir = os.path.abspath(os.path.join(jobs_root, self.name))
|
|
503
|
+
# each stage said as it goes, on the cli's stderr and in the job's progress.jsonl (nullius/progress.py)
|
|
504
|
+
self.progress = progress.Reporter(self.job_dir)
|
|
505
|
+
self._said = threading.Lock()
|
|
506
|
+
# the job page's live board (nullius/runs/live.py): written once the units are known
|
|
507
|
+
self.feed: Optional[live_feed.Writer] = None
|
|
508
|
+
self._ended = 0 # units ended so far: the i of the next unit's line
|
|
509
|
+
self._envs: set = set() # environments whose build was said
|
|
510
|
+
self.n_units: Optional[int] = None
|
|
511
|
+
self.local = threading.local()
|
|
512
|
+
self.drivers: List[jsonl.Proc] = []
|
|
513
|
+
self._stopped = threading.Event()
|
|
514
|
+
|
|
515
|
+
def driver(self) -> jsonl.Proc:
|
|
516
|
+
d = getattr(self.local, "driver", None)
|
|
517
|
+
if d is not None and d.p is not None and d.p.poll() is not None:
|
|
518
|
+
d.close()
|
|
519
|
+
d = None # it ended - a reply that never came closed it: the next unit gets a new one
|
|
520
|
+
if d is None:
|
|
521
|
+
if self.setup_driver is not None:
|
|
522
|
+
home = os.path.join(self.job_dir, ".work", "driver-home")
|
|
523
|
+
cwd = os.path.join(self.job_dir, ".work", "driver")
|
|
524
|
+
os.makedirs(home, exist_ok=True)
|
|
525
|
+
os.makedirs(cwd, exist_ok=True)
|
|
526
|
+
env = {"HOME": home, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
|
|
527
|
+
"NULLIUS_DATASET_ID": self.ds["id"], "NULLIUS_VERSION": str(self.bench.get("_version") or ""),
|
|
528
|
+
**option_env(self.bench_options), **self.file_env, **self.embeddings_env(), **self.driver_secrets()}
|
|
529
|
+
files = [h["path"] for s in definitions.secrets_of(self.bench)
|
|
530
|
+
if s.get("to") == "driver" and s.get("kind") == "file"
|
|
531
|
+
for h in [self.licences.get(s["name"])] if h] + list(self.file_env.values())
|
|
532
|
+
d = self.setup_driver.start(env, cwd, os.path.join(self.job_dir, ".work", f"driver-{threading.get_ident()}.log"),
|
|
533
|
+
files=files, stopped=self._stopped,
|
|
534
|
+
servers=[s["name"] for s in definitions.secrets_of(self.bench)
|
|
535
|
+
if s.get("to") == "driver" and s.get("kind") == "server"])
|
|
536
|
+
self.local.driver = d
|
|
537
|
+
self.drivers.append(d)
|
|
538
|
+
return d
|
|
539
|
+
try:
|
|
540
|
+
cmd, extra = install.driver_argv(self.bench) # its driver.install: installed in ready(), found here
|
|
541
|
+
except install.InstallFailed as e:
|
|
542
|
+
raise Refused(f"could not install the driver's environment: {e}")
|
|
543
|
+
env = base_env(os.path.join(self.job_dir, ".work", "driver-home"),
|
|
544
|
+
{**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
|
|
545
|
+
"NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": driver_path(), "PYTHONDONTWRITEBYTECODE": "1",
|
|
546
|
+
"NULLIUS_VERSION": str(self.bench.get("_version") or ""),
|
|
547
|
+
**({container.CHECK_IMAGE: self.image, **container.check_env()} if self.image else {}),
|
|
548
|
+
**option_env(getattr(self, "bench_options", {})), **self.file_env, **self.embeddings_env(), **self.driver_secrets()})
|
|
549
|
+
os.makedirs(env["HOME"], exist_ok=True)
|
|
550
|
+
# its working directory is the job's, never its catalogue folder: an upstream that writes
|
|
551
|
+
# beside itself (tau2's embeddings cache) would write into the package or the runner snapshot
|
|
552
|
+
cwd = os.path.join(self.job_dir, ".work", "driver")
|
|
553
|
+
os.makedirs(cwd, exist_ok=True)
|
|
554
|
+
d = jsonl.Proc(cmd, cwd, env, os.path.join(self.job_dir, ".work", f"driver-{threading.get_ident()}.log"), stopped=self._stopped).start()
|
|
555
|
+
self.local.driver = d
|
|
556
|
+
self.drivers.append(d)
|
|
557
|
+
return d
|
|
558
|
+
|
|
559
|
+
def start(self) -> str:
|
|
560
|
+
t0 = time.time()
|
|
561
|
+
os.makedirs(os.path.join(self.job_dir, ".work"), exist_ok=True)
|
|
562
|
+
self.say(f"job {self.job_dir}")
|
|
563
|
+
p, control = self.progress, self.other["_kind"] == "control"
|
|
564
|
+
p.emit("plan", stages=",".join((["image"] if self.backend == "container" or self.container_check else [])
|
|
565
|
+
+ (["upstream"] if install.upstream_dir(self.bench) else [])
|
|
566
|
+
+ (["dataset"] if self.ds.get("url") else [])
|
|
567
|
+
+ (["driver"] if install.driver_dir(self.bench) else [])
|
|
568
|
+
+ ([] if control or self.upstream else ["install"]) + ["units"]))
|
|
569
|
+
if (self.max_usd is not None or self.unit_max_usd is not None) and self.base:
|
|
570
|
+
# one ledger for the job, in a folder of its own: all a unit's relay container is given of the job besides its record
|
|
571
|
+
self.ledger = ledgers.Ledger.create(os.path.join(self.job_dir, ".work", "ledger", "ledger.json"), self.max_usd, self.unit_max_usd)
|
|
572
|
+
try:
|
|
573
|
+
self.ready()
|
|
574
|
+
except Refused as e:
|
|
575
|
+
# said in the job, which holds nothing else: the runs list reads why there (progress.refusal)
|
|
576
|
+
p.emit("refused", detail=str(e))
|
|
577
|
+
raise
|
|
578
|
+
try:
|
|
579
|
+
units = self.driver().request("units", dataset=self.ds["id"], timeout=300,
|
|
580
|
+
**({"setup": self.other["key"]} if self.upstream else {}))["units"]
|
|
581
|
+
except BaseException:
|
|
582
|
+
for driver in self.drivers:
|
|
583
|
+
driver.close()
|
|
584
|
+
if self.bench_relay is not None:
|
|
585
|
+
self.bench_relay.stop()
|
|
586
|
+
if self.backend == "container":
|
|
587
|
+
container.sweep(self.name)
|
|
588
|
+
raise
|
|
589
|
+
everything = [str(u) for u in units]
|
|
590
|
+
# the shard, then the limit; each unit keeps its place in the whole dataset (its trial's tag)
|
|
591
|
+
mine = shards.select(everything, *self.shard) if self.shard else list(enumerate(everything))
|
|
592
|
+
mine = mine[: self.limit] if self.limit else mine
|
|
593
|
+
units = [u for _, u in mine]
|
|
594
|
+
self.n_units = len(units)
|
|
595
|
+
p.emit("units", "start", n=len(units), of=len(everything), noun=self.bench["units"]["nouns"])
|
|
596
|
+
job.write_config(self.job_dir, job=self.name, bench=self.bench, other=self.other, model=self.model,
|
|
597
|
+
provider=self.provider, dataset=self.ds["id"], units=units, concurrency=self.concurrency,
|
|
598
|
+
granted=self.other["_kind"] == "agent", judge=self.judge_model or "", backend=self.backend,
|
|
599
|
+
image=self.setup_driver.image if self.setup_driver else self.image,
|
|
600
|
+
options=self.options, launch=self.launch or "", bench_options=self.bench_options,
|
|
601
|
+
shard=shards.record(*self.shard, len(everything), units, self.limit) if self.shard else None,
|
|
602
|
+
repeat=self.repeat, prices=self.prices_record(),
|
|
603
|
+
simulator=self.simulator_model if self.other["_kind"] in ("agent", "upstream") and self.simulator_model else "")
|
|
604
|
+
# every unit once, then every unit again: a repeat cut short still has each unit's first run
|
|
605
|
+
work = [(i, u, r) for r in range(1, self.repeat + 1) for i, u in mine]
|
|
606
|
+
self.n_units = len(work) # what the progress counts: every run of every unit
|
|
607
|
+
self.feed = live_feed.Writer(os.path.join(self.job_dir, live_feed.FEED), self.name)
|
|
608
|
+
self.feed.emit("job.start", n_total=len(work), n_concurrent=self.concurrency, agents=[self.other["key"]], tasks=units,
|
|
609
|
+
**({"budget": {k: v for k, v in (("max_usd", self.max_usd), ("unit_max_usd", self.unit_max_usd)) if v is not None}}
|
|
610
|
+
if self.ledger is not None else {})) # the live board says the cap
|
|
611
|
+
for pos, (i, u, r) in enumerate(work):
|
|
612
|
+
self.feed.emit("trial.queued", self.trial_name(i, u, r), agent=self.other["key"], position=pos)
|
|
613
|
+
results = []
|
|
614
|
+
u0 = time.time()
|
|
615
|
+
self._halt, self._refused, self.halted = threading.Event(), [], None
|
|
616
|
+
self._cut: List[str] = [] # units the spending cap cut off
|
|
617
|
+
self._over: List[str] = [] # units that reached their own cap: cut off there, the run going on
|
|
618
|
+
self.loopback = None
|
|
619
|
+
self.pace = pacing.Pace(self._stopped, self._halt)
|
|
620
|
+
self._running = 0
|
|
621
|
+
again: Dict[str, dict] = {} # a rate-limited unit's second attempt, by its trial's name
|
|
622
|
+
|
|
623
|
+
def work_unit(iur):
|
|
624
|
+
self.pace.wait() # a unit ended rate-limited: the next waits its pause out
|
|
625
|
+
if self._stopped.is_set():
|
|
626
|
+
return None
|
|
627
|
+
with self._said:
|
|
628
|
+
self._running += 1
|
|
629
|
+
try:
|
|
630
|
+
res = self.unit(*iur)
|
|
631
|
+
except BaseException:
|
|
632
|
+
self._stopped.set()
|
|
633
|
+
self._halt.set()
|
|
634
|
+
raise
|
|
635
|
+
finally:
|
|
636
|
+
with self._said:
|
|
637
|
+
self._running -= 1
|
|
638
|
+
self.paced(res)
|
|
639
|
+
return res
|
|
640
|
+
|
|
641
|
+
pool = ThreadPoolExecutor(self.concurrency)
|
|
642
|
+
futures = []
|
|
643
|
+
try:
|
|
644
|
+
# a model server on this machine's loopback, which the relays' containers cannot reach as written
|
|
645
|
+
if self.backend == "container" and self.base and container.on_loopback(self.base):
|
|
646
|
+
self.loopback = container.Loopback(self.base, image=self.image, job=self.name).start()
|
|
647
|
+
for iur in work:
|
|
648
|
+
futures.append(pool.submit(work_unit, iur))
|
|
649
|
+
for future in as_completed(futures):
|
|
650
|
+
res = future.result()
|
|
651
|
+
if res is not None: # None: a unit not begun once the endpoint refused the run
|
|
652
|
+
results.append(res)
|
|
653
|
+
self.run_again(results, work, work_unit, again)
|
|
654
|
+
except BaseException:
|
|
655
|
+
self._stopped.set()
|
|
656
|
+
self._halt.set()
|
|
657
|
+
if self.backend == "container":
|
|
658
|
+
container.sweep(self.name)
|
|
659
|
+
raise
|
|
660
|
+
finally:
|
|
661
|
+
# a Stop or Ctrl-C: the units not begun are not begun, those running end with their agents
|
|
662
|
+
pool.shutdown(wait=True, cancel_futures=True)
|
|
663
|
+
begun = [not f.cancelled() and f.exception() is None and f.result() is not None for f in futures]
|
|
664
|
+
results = [again.get(f.result()["trial_name"], f.result()) for f, ran in zip(futures, begun) if ran]
|
|
665
|
+
for d in self.drivers:
|
|
666
|
+
d.close()
|
|
667
|
+
if self.bench_relay is not None:
|
|
668
|
+
self.bench_relay.stop()
|
|
669
|
+
if self.backend == "container":
|
|
670
|
+
container.sweep(self.name) # whatever an interrupted unit left
|
|
671
|
+
if self.loopback is not None:
|
|
672
|
+
self.loopback.stop()
|
|
673
|
+
budget = self.budget([self.attempt_name(u, r) for n, (_, u, r) in enumerate(work) if n >= len(begun) or not begun[n]])
|
|
674
|
+
self.capped = bool(budget and budget["stopped"])
|
|
675
|
+
job.write_result(self.job_dir, self.name, t0, time.time(), results, len(work), budget=budget)
|
|
676
|
+
errored = sum(1 for x in results if x.get("exception_info"))
|
|
677
|
+
self.feed.emit("job.end", n_completed=len(results) - errored, n_errored=errored, n_cancelled=len(work) - len(results))
|
|
678
|
+
# the units' stage says why it ended early: a line of its own would be a stage that never starts
|
|
679
|
+
p.emit("units", "done", seconds=round(time.time() - u0, 1),
|
|
680
|
+
**({"detail": f"stopped at the spending cap of {ledgers.usd(budget['max_usd'])} USD: "
|
|
681
|
+
f"{len(budget['cut_off'])} cut off, {len(budget['not_run'])} not run"} if budget and budget["stopped"] else {}))
|
|
682
|
+
self.n_verdicts, self.n_ended = sum(1 for x in results if x.get("verifier_result")), len(results)
|
|
683
|
+
if budget and budget["stopped"]:
|
|
684
|
+
spent = "unknown" if budget["spent_usd"] is None else ledgers.usd(budget["spent_usd"])
|
|
685
|
+
self.say(f"stopped: a call did not fit the spending cap of {ledgers.usd(budget['max_usd'])} USD "
|
|
686
|
+
f"({spent} spent): {len(budget['cut_off'])} cut off, {len(budget['not_run'])} not run")
|
|
687
|
+
elif budget and budget["max_usd"] is not None:
|
|
688
|
+
self.say(f"spent {ledgers.usd(budget['spent_usd'] or 0)} of the {ledgers.usd(budget['max_usd'])} USD cap")
|
|
689
|
+
elif budget:
|
|
690
|
+
self.say(f"spent {ledgers.usd(budget['spent_usd'] or 0)} USD")
|
|
691
|
+
if budget and budget.get("over_unit"):
|
|
692
|
+
self.say(f"{len(budget['over_unit'])} of the {self.bench['units']['nouns']} reached the cap of "
|
|
693
|
+
f"{ledgers.usd(budget['unit_max_usd'])} USD each and were cut off there")
|
|
694
|
+
if self.halted:
|
|
695
|
+
self.say(f"stopped: the endpoint refused {REFUSED_IN_A_ROW} units in a row ({self.halted[0]}: "
|
|
696
|
+
f"{redact(str(self.halted[1]))[:200]}); the {len(work) - len(results)} units not begun were not run")
|
|
697
|
+
rewards = [(r.get("verifier_result") or {}).get("rewards", {}) for r in results]
|
|
698
|
+
if self.bench["score"]["kind"] == "pass":
|
|
699
|
+
ok = sum(1 for w in rewards if w.get("reward") == 1.0)
|
|
700
|
+
p.emit("done", n=len(results), passed=ok, seconds=round(time.time() - t0, 1))
|
|
701
|
+
self.say(f"done: {len(results)} {self.bench['units']['nouns']}, {ok} passed")
|
|
702
|
+
else:
|
|
703
|
+
scores = [w["score"] for w in rewards if "score" in w]
|
|
704
|
+
mean = job.mean_score(results)
|
|
705
|
+
p.emit("done", n=len(results), scored=len(scores), mean=round(mean, 3) if mean is not None else None,
|
|
706
|
+
seconds=round(time.time() - t0, 1))
|
|
707
|
+
self.say(f"done: {len(results)} {self.bench['units']['nouns']}, {len(scores)} scored"
|
|
708
|
+
+ (f", mean {mean:.3f}" if mean is not None else ""))
|
|
709
|
+
return self.job_dir
|
|
710
|
+
|
|
711
|
+
def paced(self, res: Optional[dict]) -> None:
|
|
712
|
+
"""A unit's end, told to the run's pace: one that ended rate-limited
|
|
713
|
+
holds the next unit back, said with how many units run meanwhile."""
|
|
714
|
+
if res is None:
|
|
715
|
+
return
|
|
716
|
+
pause = self.pace.ended((res.get("exception_info") or {}).get("exception_type"))
|
|
717
|
+
if pause is not None:
|
|
718
|
+
self.progress.emit("units", "progress", detail=f"rate limited: no unit starts for {pause:g} s; "
|
|
719
|
+
f"{self._running} running of {self.concurrency} at once")
|
|
720
|
+
|
|
721
|
+
def run_again(self, results: List[dict], work: list, work_unit, done: Dict[str, dict]) -> None:
|
|
722
|
+
"""Each unit that ended rate-limited, run once more after the others,
|
|
723
|
+
one at a time and paced as the rest: its first attempt moved into the
|
|
724
|
+
job's ``retried/`` (kept whole, its calls and its cost with it, where
|
|
725
|
+
no reader takes it for a trial of its own), its second written under
|
|
726
|
+
the trial's own name and counted (into ``done``, as each ends). Nothing
|
|
727
|
+
is run again once the run is stopped or halted; the spending cap goes
|
|
728
|
+
on counting every call."""
|
|
729
|
+
by_name = {self.trial_name(*iur): iur for iur in work}
|
|
730
|
+
todo = [by_name[r["trial_name"]] for r in results
|
|
731
|
+
if (r.get("exception_info") or {}).get("exception_type") == pacing.RATE_LIMITED and r["trial_name"] in by_name]
|
|
732
|
+
if not todo:
|
|
733
|
+
return
|
|
734
|
+
self.progress.emit("units", "start", n=len(todo), detail=f"running {len(todo)} rate-limited again, one at a time")
|
|
735
|
+
self.say(f"running again, one at a time: {len(todo)} rate-limited")
|
|
736
|
+
for iur in todo:
|
|
737
|
+
if self._stopped.is_set() or self._halt.is_set():
|
|
738
|
+
break
|
|
739
|
+
name = self.trial_name(*iur)
|
|
740
|
+
keep = os.path.join(self.job_dir, "retried", name)
|
|
741
|
+
os.makedirs(os.path.dirname(keep), exist_ok=True)
|
|
742
|
+
os.replace(os.path.join(self.job_dir, name), keep)
|
|
743
|
+
self.feed.emit("trial.queued", name, agent=self.other["key"], position=0)
|
|
744
|
+
res = work_unit(iur)
|
|
745
|
+
if res is not None:
|
|
746
|
+
done[name] = res
|
|
747
|
+
elif not os.path.exists(os.path.join(self.job_dir, name)):
|
|
748
|
+
os.replace(keep, os.path.join(self.job_dir, name)) # not begun after all: the first attempt stands
|
|
749
|
+
|
|
750
|
+
def ready(self) -> None:
|
|
751
|
+
"""What the run needs before its first unit, each a stage said as it
|
|
752
|
+
goes - or as there already: the base image (docker), the benchmark's
|
|
753
|
+
upstream, its dataset when it is downloaded, its driver's environment
|
|
754
|
+
when it declares one (``driver.install``), the agent's or capability's
|
|
755
|
+
packages. Refused when one cannot be had."""
|
|
756
|
+
from . import network_requirements
|
|
757
|
+
why = network_requirements.check(self.bench, self.other)
|
|
758
|
+
if why:
|
|
759
|
+
raise Refused("; ".join(why))
|
|
760
|
+
p = self.progress
|
|
761
|
+
if self.backend == "container" or self.container_check:
|
|
762
|
+
try:
|
|
763
|
+
tag = container.base_tag()
|
|
764
|
+
there = container.present(tag)
|
|
765
|
+
with p.stage("image", cached=there, start=f"building {tag} (first time on this machine)", **({"detail": tag} if there else {})):
|
|
766
|
+
self.image = container.base_image(os.path.join(self.job_dir, ".work", "image.log"))
|
|
767
|
+
except container.Unavailable as e:
|
|
768
|
+
raise Refused(f"the Docker image: {e}")
|
|
769
|
+
# on the container backend the agent's packages are installed in the base image, with the Node, uv and libc it runs on
|
|
770
|
+
image = self.image if self.backend == "container" else None
|
|
771
|
+
run_in = (lambda cmd, where, env=None: container.install_argv(image, where, cmd, env)) if image else None
|
|
772
|
+
try:
|
|
773
|
+
where = install.upstream_dir(self.bench)
|
|
774
|
+
self.up = None
|
|
775
|
+
if where is not None:
|
|
776
|
+
up = self.bench["upstream"]
|
|
777
|
+
with p.stage("upstream", cached=install.ready(where), start=f"fetching {up['repository']} at {str(up['tested'])[:12]}"):
|
|
778
|
+
self.up = install.upstream(self.bench)
|
|
779
|
+
if self.ds.get("url"):
|
|
780
|
+
there = os.path.isfile(install.download_path(self.ds["url"], self.ds["sha256"]))
|
|
781
|
+
with p.stage("dataset", cached=there, start=f"downloading {self.ds['id']}"):
|
|
782
|
+
self.data = install.dataset(self.bench, self.ds, self.up, progress=None if there else p.download("dataset"))
|
|
783
|
+
else:
|
|
784
|
+
self.data = install.dataset(self.bench, self.ds, self.up)
|
|
785
|
+
for name in self.files:
|
|
786
|
+
f = self.bench["files"][name]
|
|
787
|
+
there = os.path.isfile(install.download_path(f["url"], f["sha256"]))
|
|
788
|
+
with p.stage("file", cached=there, start=f"downloading {name}", name=name):
|
|
789
|
+
self.file_env["NULLIUS_FILE_" + name.upper()] = install.download(
|
|
790
|
+
f["url"], f["sha256"], progress=None if there else p.download("file"))
|
|
791
|
+
drv = install.driver_dir(self.bench)
|
|
792
|
+
if self.backend == "container" and self.upstream:
|
|
793
|
+
from nullius.standard import setup_container
|
|
794
|
+
with p.stage("driver", start=f"preparing {self.bench['key']}'s container driver"):
|
|
795
|
+
needs = definitions.option_needs(self.bench, self.bench_options)["host"]
|
|
796
|
+
image = setup_container.image(self.image, self.bench, needs, os.path.join(self.job_dir, ".work", "image.log"))
|
|
797
|
+
self.setup_driver = setup_container.Driver(self.bench, image=image, app=APP, job=self.name,
|
|
798
|
+
work=self.job_dir, upstream=self.up, data=self.data, needs=needs)
|
|
799
|
+
self.setup_driver.prepare()
|
|
800
|
+
elif drv is not None: # the environment the upstream's own code needs, for the driver
|
|
801
|
+
with p.stage("driver", cached=install.ready(drv), start=f"installing {self.bench['key']}'s driver environment"):
|
|
802
|
+
install.driver_packages(self.bench)
|
|
803
|
+
self.inst = ""
|
|
804
|
+
if self.other["_kind"] not in ("control", "upstream"):
|
|
805
|
+
with p.stage("install", cached=install.ready(install.packages_dir(self.other, image=image)), agent=self.other.get("_ref") or self.other["key"]):
|
|
806
|
+
self.inst = install.packages(self.other, run_in=run_in, image=image)
|
|
807
|
+
except (install.InstallFailed, container.Unavailable) as e:
|
|
808
|
+
if self.backend == "container":
|
|
809
|
+
container.sweep(self.name)
|
|
810
|
+
raise Refused(f"could not install what the run needs: {e}")
|
|
811
|
+
# An own image has none of what the agent runs on: it is mounted in, fetched once on this machine.
|
|
812
|
+
self.runtimes: Dict[str, tuple] = {}
|
|
813
|
+
owns = any(definitions.own_image(e) for e in (self.bench.get("environments") or {}).values())
|
|
814
|
+
if owns and self.other["_kind"] == "agent" and self.backend == "container" and self.runtimes_needed():
|
|
815
|
+
try:
|
|
816
|
+
with p.stage("runtime", start="fetching " + ", ".join(self.runtimes_needed()), name=",".join(self.runtimes_needed())):
|
|
817
|
+
self.runtimes = container.runtime_mounts(self.runtimes_needed())
|
|
818
|
+
except container.Unavailable as e:
|
|
819
|
+
raise Refused(f"the agent's runtime for an own image: {e}")
|
|
820
|
+
# An option whose driver embeds or reranks: its calls go through a relay of the run's
|
|
821
|
+
# own, recorded in the job's .work/relay-benchmark beside the units', never counted as the agent's.
|
|
822
|
+
if (self.embed_model or self.rerank_model) and self.bench_relay is None:
|
|
823
|
+
self.bench_relay = relay.Relay(self.base, self.key, os.path.join(self.job_dir, ".work", "relay-benchmark"),
|
|
824
|
+
prices=self.prices, model=self.rerank_model or self.embed_model, ledger=self.ledger_file(), unit_caps=False,
|
|
825
|
+
key_header=PROVIDERS[self.provider].get("key_header", "authorization")).start()
|
|
826
|
+
|
|
827
|
+
def runtimes_needed(self) -> List[str]:
|
|
828
|
+
"""What the agent needs in an own image that the image may not have:
|
|
829
|
+
Node for an agent npm installs, Python for one started as ``python3``
|
|
830
|
+
with no environment of its own (a uv environment's interpreter is
|
|
831
|
+
mounted with the install, as on the base image)."""
|
|
832
|
+
a = self.other
|
|
833
|
+
start = (a.get("start") or {}).get("acp") or (a.get("start") or {}).get("command") or [""]
|
|
834
|
+
needs = ["node"] if (a.get("install") or {}).get("npm") else []
|
|
835
|
+
if container.bare_python(start[0]) and not os.path.isfile(os.path.join(self.inst, ".venv", "pyvenv.cfg")):
|
|
836
|
+
needs.append("python")
|
|
837
|
+
return needs
|
|
838
|
+
|
|
839
|
+
def attempt_kw(self) -> dict:
|
|
840
|
+
"""What the driver is told of the attempt this unit's run is (1, 2, ...), when the benchmark
|
|
841
|
+
declares that its upstream tells attempts apart (``limits.attempts``: a seed per trial)."""
|
|
842
|
+
a = getattr(self.local, "attempt", None)
|
|
843
|
+
return {"attempt": a} if a and (self.bench.get("limits") or {}).get("attempts") else {}
|
|
844
|
+
|
|
845
|
+
def trial_name(self, i: int, unit: str, r: int = 1) -> str:
|
|
846
|
+
return f"{_slug(unit)}__u{i:04d}" + (f"r{r}" if self.repeat > 1 else "")
|
|
847
|
+
|
|
848
|
+
def phase(self, name: str, unit: str, phase: str, timeout: Optional[float] = None) -> None:
|
|
849
|
+
if self.feed is not None:
|
|
850
|
+
self.feed.emit("trial.phase", name, task=unit, agent=self.other["key"], phase=phase, timeout_sec=timeout)
|
|
851
|
+
|
|
852
|
+
def unit(self, i: int, unit: str, r: int = 1) -> Optional[dict]:
|
|
853
|
+
if self._stopped.is_set() or (getattr(self, "_halt", None) is not None and self._halt.is_set()):
|
|
854
|
+
return None # the endpoint refused the run: nothing more is begun
|
|
855
|
+
if getattr(self, "ledger", None) is not None and self.ledger.closed():
|
|
856
|
+
self._halt.set() # the spending cap refused a call: every later one is refused too
|
|
857
|
+
return None
|
|
858
|
+
tag = f"u{i:04d}" + (f"r{r}" if self.repeat > 1 else "")
|
|
859
|
+
self.local.attempt = r
|
|
860
|
+
trial = os.path.join(self.job_dir, self.trial_name(i, unit, r))
|
|
861
|
+
name = os.path.basename(trial)
|
|
862
|
+
ws, home = os.path.join(trial, "workspace"), os.path.join(trial, ".home")
|
|
863
|
+
os.makedirs(ws, exist_ok=True)
|
|
864
|
+
# The workspace is a repository of its own: agents look up to the git root
|
|
865
|
+
# for instructions (AGENTS.md, CLAUDE.md, their config) and, with jobs kept
|
|
866
|
+
# inside a checkout, would otherwise read that checkout's into every prompt.
|
|
867
|
+
subprocess.run(["git", "init", "-q", ws], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL, check=False)
|
|
868
|
+
os.makedirs(home, exist_ok=True)
|
|
869
|
+
t0 = time.time()
|
|
870
|
+
self.progress.emit("unit", "start", unit=unit)
|
|
871
|
+
self.phase(name, unit, "env_setup", None if self.upstream else DRIVER_SECONDS)
|
|
872
|
+
produce, score, exc, traj, logs = (None, None), None, None, None, {}
|
|
873
|
+
p0, checking = None, (None, None) # when the agent began; the check's start and end
|
|
874
|
+
setup: dict = {}
|
|
875
|
+
transcript: List[dict] = []
|
|
876
|
+
rl, environment, box, changes = None, None, None, None
|
|
877
|
+
try:
|
|
878
|
+
try:
|
|
879
|
+
# a setup's unit needs none: the upstream's answerer reads its data itself
|
|
880
|
+
setup = {} if self.upstream else self.driver().request("setup", unit=unit, dir=ws, timeout=DRIVER_SECONDS, **self.attempt_kw())
|
|
881
|
+
except jsonl.Failed as e:
|
|
882
|
+
raise _Ended("EnvironmentSetupError", str(e))
|
|
883
|
+
environment, env_def = self.unit_environment(setup)
|
|
884
|
+
why = self.not_run(environment, env_def)
|
|
885
|
+
if why:
|
|
886
|
+
raise _Ended(*why)
|
|
887
|
+
own = definitions.own_image(env_def)
|
|
888
|
+
if "internet" in setup and (not isinstance(setup["internet"], bool) or not own):
|
|
889
|
+
raise _Ended("EnvironmentSetupError", "the unit's setup: internet is true or false, only in an own image")
|
|
890
|
+
if self.other["_kind"] == "agent" and self.backend == "local" and mcp_machine.wanted(setup):
|
|
891
|
+
try: # the benchmark's own servers: the agent starts them from the driver's environment
|
|
892
|
+
setup = dict(setup, mcp=mcp_machine.entries(self.bench, setup, install.driver_dir(self.bench),
|
|
893
|
+
self.machine_env(trial)))
|
|
894
|
+
except mcp_machine.Unknown as e:
|
|
895
|
+
raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}")
|
|
896
|
+
if "env" in setup and not own:
|
|
897
|
+
raise _Ended("EnvironmentSetupError", "the unit's setup: env requires an own image")
|
|
898
|
+
if setup.get("verify") is not None and not own:
|
|
899
|
+
raise _Ended("EnvironmentSetupError", "the unit's setup: verify runs in an own image, and this unit has none")
|
|
900
|
+
if setup.get("prepare") is not None and not own:
|
|
901
|
+
raise _Ended("EnvironmentSetupError", "the unit's setup: prepare runs in an own image, and this unit has none")
|
|
902
|
+
try:
|
|
903
|
+
rl = self.unit_relay(tag, trial, setup, env_def, own)
|
|
904
|
+
except (container.Unavailable, OSError) as e:
|
|
905
|
+
raise _Ended("EnvironmentSetupError", f"the unit's relay: {e}")
|
|
906
|
+
if rl is not None:
|
|
907
|
+
rl.unit_secrets = self.agent_licences(env_def, rl)
|
|
908
|
+
if own:
|
|
909
|
+
self.phase(name, unit, "env_setup", IMAGE_SECONDS) # its image, pulled or built when not here yet
|
|
910
|
+
box = self.own_box(tag, setup, rl, home)
|
|
911
|
+
if setup.get("prepare") is not None:
|
|
912
|
+
self.prepare(setup["prepare"], box, trial)
|
|
913
|
+
p0 = time.time()
|
|
914
|
+
self.phase(name, unit, "agent_run", self.turn_limit(setup) if self.other["_kind"] != "control"
|
|
915
|
+
else DRIVER_SECONDS if self.other["key"] == "oracle" else None)
|
|
916
|
+
try:
|
|
917
|
+
with self.agent_internet(setup, box, rl, trial):
|
|
918
|
+
if self.other["_kind"] == "control":
|
|
919
|
+
answer = self.control_answer(unit, ws, box, name)
|
|
920
|
+
elif self.upstream:
|
|
921
|
+
answer, traj = self.drive_upstream(unit, ws, tag, transcript, rl)
|
|
922
|
+
elif self.other["_kind"] == "agent":
|
|
923
|
+
answer, traj, logs = self.drive_agent(unit, ws, home, tag, setup, transcript, rl, box, name)
|
|
924
|
+
else:
|
|
925
|
+
answer, traj = self.drive_feature(unit, ws, home, tag, setup, transcript, rl)
|
|
926
|
+
finally:
|
|
927
|
+
produce = (p0, time.time())
|
|
928
|
+
if box is not None: # what the agent (or the control) left in the container, before the check
|
|
929
|
+
changes = self.box_changes(box)
|
|
930
|
+
cut = self.cap_reached(rl, tag)
|
|
931
|
+
if cut: # cut off at a spending cap: nothing of it is checked, and its judge is asked nothing
|
|
932
|
+
raise _Ended(cut[0], cut[1], traj)
|
|
933
|
+
step = setup.get("verify") if box is not None else None
|
|
934
|
+
checking = (time.time(), None)
|
|
935
|
+
self.phase(name, unit, "verify", DRIVER_SECONDS + (float(step.get("seconds") or self.turn_seconds) if step else 0))
|
|
936
|
+
verified = self.verify(setup["verify"], box, trial) if box is not None and setup.get("verify") else None
|
|
937
|
+
if self.backend == "container": # the agent could write here (an MCP server's log folder): no link it left is read
|
|
938
|
+
container.plain_only(os.path.join(trial, "verifier"))
|
|
939
|
+
try:
|
|
940
|
+
judge = self.judge(rl, tag)
|
|
941
|
+
score = self.driver().request("score", unit=unit, dir=ws, transcript=transcript, answer=answer, timeout=DRIVER_SECONDS,
|
|
942
|
+
**judge, **({"setup": self.other["key"]} if self.upstream else {}), **({"verified": verified} if verified is not None else {}))
|
|
943
|
+
checking = (checking[0], time.time())
|
|
944
|
+
except jsonl.Failed as e:
|
|
945
|
+
raise _Ended("VerifierError", str(e))
|
|
946
|
+
except _Ended as e:
|
|
947
|
+
exc = (e.kind, e.said)
|
|
948
|
+
traj = traj or e.trajectory
|
|
949
|
+
except Exception as e: # anything else is this unit's error, never the whole job's
|
|
950
|
+
exc = (type(e).__name__, f"{type(e).__name__}: {e}\n" + "".join(traceback.format_exc().splitlines(True)[-6:]))
|
|
951
|
+
finally:
|
|
952
|
+
if checking[0] is not None and checking[1] is None:
|
|
953
|
+
checking = (checking[0], time.time())
|
|
954
|
+
if box is not None:
|
|
955
|
+
# the home it wrote as root given back, then the container gone
|
|
956
|
+
box.give_back([home, *getattr(rl, "unit_rw", [])])
|
|
957
|
+
box.stop()
|
|
958
|
+
self.release_image(box.image)
|
|
959
|
+
if rl is not None:
|
|
960
|
+
rl.stop()
|
|
961
|
+
for ln in getattr(rl, "mcp_listeners", None) or []:
|
|
962
|
+
ln.stop()
|
|
963
|
+
calls = rl.calls(tag) if rl is not None else []
|
|
964
|
+
failed = relay.failure(calls)
|
|
965
|
+
if failed and failed[0] not in (ledgers.ERROR, ledgers.UNIT_ERROR) and exc is None \
|
|
966
|
+
and self.bench["score"]["kind"] == "pass" and score.get("reward") == 1.0:
|
|
967
|
+
# a passed unit's failed last call came after its answer (jiuwenswarm's image probe, which some
|
|
968
|
+
# models refuse): the pass stands, the call stays in its record. A failed or score-only unit's is
|
|
969
|
+
# the endpoint's, and a cap's refusal is the cap's
|
|
970
|
+
failed = None
|
|
971
|
+
if not failed and rl is not None and exc is not None and exc[0] == "VerifierError":
|
|
972
|
+
# the judge's or the simulated user's model failing (a 429, a refused request) is the
|
|
973
|
+
# unit's infrastructure, as the agent's own would be - not a miss counted against it
|
|
974
|
+
failed = next(filter(None, (relay.failure(rl.calls(f"{tag}-{side}")) for side in ("judge", "simulator"))), None)
|
|
975
|
+
if failed and (exc is None or exc[0] not in ("EnvironmentSetupError",)):
|
|
976
|
+
exc = failed # what the endpoint said outranks how the agent took it
|
|
977
|
+
score = None
|
|
978
|
+
cut = self.cap_reached(rl, tag) # at scoring too: its judge's call, or a refusal its agent made something else of
|
|
979
|
+
if cut:
|
|
980
|
+
exc, score = cut, None
|
|
981
|
+
if exc and exc[0] in ("EnvironmentSetupError", "VerifierError") and getattr(self, "ledger", None) is not None \
|
|
982
|
+
and getattr(self, "bench_relay", None) is not None:
|
|
983
|
+
# its driver failed, at setup or at scoring, and the cap refused a call that driver made for this unit (an
|
|
984
|
+
# option's embeddings, on its thread's route of the benchmark's relay): the unit was cut off there. Another
|
|
985
|
+
# unit's refusal is not this one's, and an agent's own error stays the agent's
|
|
986
|
+
late = [c for c in self.bench_relay.calls(self.embeddings_tag()) if c.get("refused") == "cap" and c.get("t0", 0) >= t0]
|
|
987
|
+
if late:
|
|
988
|
+
exc, score = (ledgers.ERROR, late[0].get("error") or ""), None
|
|
989
|
+
if self._stopped.is_set() and score is None:
|
|
990
|
+
# the Stop ended its agent, its relay or its containers under it: what it then failed with is
|
|
991
|
+
# the Stop's doing, not the agent's nor the environment's (Harbor's name for a stopped trial)
|
|
992
|
+
exc = (CANCELLED, "the run was stopped before this unit ended" + (f"; it then failed with {exc[0]}" if exc else ""))
|
|
993
|
+
if traj is None and self.other["_kind"] == "agent":
|
|
994
|
+
traj = job.trajectory_from_events([], calls, self.other, self.model, f"{self.name}/{unit}")
|
|
995
|
+
if traj is not None and setup.get("mcp"): # how a reader tells the benchmark's tools in any agent's spelling
|
|
996
|
+
traj["extra"] = dict(traj.get("extra") or {}, mcp_servers=[str(m.get("name")) for m in setup["mcp"] if m.get("name")])
|
|
997
|
+
res = job.write_trial(trial, job=self.name, bench=self.bench, other=self.other, model=self.model, unit=unit,
|
|
998
|
+
t0=t0, t1=time.time(), produce=produce, score=score, exception=exc, calls=calls,
|
|
999
|
+
setup_end=p0 or time.time(), checking=checking,
|
|
1000
|
+
trajectory=traj, extra_logs=dict(logs or {}, **({"container-changes.txt": changes} if changes else {})),
|
|
1001
|
+
environment=environment or "",
|
|
1002
|
+
image=(box.image if box is not None else getattr(rl, "unit_image", "")) or "")
|
|
1003
|
+
install.prune(self.other, home) # its declared caches: the trial is written, no one reads them now
|
|
1004
|
+
verdict = "error " + exc[0] if exc else f"reward {score.get('reward')}"
|
|
1005
|
+
self.say(f" {unit}: {verdict}")
|
|
1006
|
+
self.said_unit(unit, score, exc)
|
|
1007
|
+
if self.feed is not None:
|
|
1008
|
+
self.feed.emit("trial.end", name, task=unit, agent=self.other["key"],
|
|
1009
|
+
**job.feed_outcome(self.bench["score"]["kind"], score, exc),
|
|
1010
|
+
**{k: v for k, v in (res.get("agent_result") or {}).items() if k in FEED_TOTALS},
|
|
1011
|
+
durations={"agent_run": round(produce[1] - produce[0], 3)} if produce[0] is not None else None)
|
|
1012
|
+
self.note_refusal(exc)
|
|
1013
|
+
if exc and exc[0] == ledgers.ERROR and getattr(self, "_halt", None) is not None:
|
|
1014
|
+
with self._said:
|
|
1015
|
+
self._cut.append(self.attempt_name(unit, r))
|
|
1016
|
+
self._halt.set() # the cap is reached: nothing more is begun
|
|
1017
|
+
elif exc and exc[0] == ledgers.UNIT_ERROR:
|
|
1018
|
+
with self._said:
|
|
1019
|
+
self._over.append(self.attempt_name(unit, r)) # its own cap: the others go on
|
|
1020
|
+
if getattr(self, "ledger", None) is not None and self.unit_max_usd is not None:
|
|
1021
|
+
try:
|
|
1022
|
+
self.ledger.forget(tag) # the unit ended: its own count goes from the ledger
|
|
1023
|
+
except ledgers.Unavailable:
|
|
1024
|
+
pass
|
|
1025
|
+
return res
|
|
1026
|
+
|
|
1027
|
+
def cap_reached(self, rl, tag: str) -> Optional[tuple]:
|
|
1028
|
+
"""A spending cap's refusal among a unit's calls - its agent's, its
|
|
1029
|
+
judge's or its simulated user's - as what the unit ends with: cut
|
|
1030
|
+
off by the run's own limit, never a miss, before its own. None
|
|
1031
|
+
without one."""
|
|
1032
|
+
if rl is None or getattr(self, "ledger", None) is None:
|
|
1033
|
+
return None
|
|
1034
|
+
cuts = [f for f in (relay.failure(rl.calls(t)) for t in (tag, f"{tag}-judge", f"{tag}-simulator")) if f]
|
|
1035
|
+
return next((f for f in cuts if f[0] == ledgers.ERROR), None) or next((f for f in cuts if f[0] == ledgers.UNIT_ERROR), None)
|
|
1036
|
+
|
|
1037
|
+
def attempt_name(self, unit: str, r: int = 1) -> str:
|
|
1038
|
+
"""A unit as the budget lists it: by its name, and its attempt when the run repeats its units."""
|
|
1039
|
+
return unit if self.repeat <= 1 else f"{unit} (attempt {r})"
|
|
1040
|
+
|
|
1041
|
+
def budget(self, not_run: List[str]) -> Optional[dict]:
|
|
1042
|
+
"""What ``result.json`` says of the run's spending cap: the cap,
|
|
1043
|
+
what was spent, what was booked for calls whose cost was never known
|
|
1044
|
+
(``unsettled_usd``, said only when there is some), whether the cap
|
|
1045
|
+
stopped the run, the units it cut off and those never begun. With a
|
|
1046
|
+
cap per unit, that cap and the units that reached it
|
|
1047
|
+
(``unit_max_usd``, ``over_unit``); ``max_usd`` is then None when the
|
|
1048
|
+
run itself had none. None for a run without a cap."""
|
|
1049
|
+
if getattr(self, "ledger", None) is None:
|
|
1050
|
+
return None
|
|
1051
|
+
try:
|
|
1052
|
+
got = self.ledger.state()
|
|
1053
|
+
except ledgers.Unavailable:
|
|
1054
|
+
got = {"spent_usd": None, "unsettled_usd": 0.0, "held_usd": 0.0, "closed": True}
|
|
1055
|
+
unsettled = round(got["unsettled_usd"] + got["held_usd"], 6)
|
|
1056
|
+
stopped = bool(got["closed"]) and self.max_usd is not None # only the run's cap stops a run
|
|
1057
|
+
cut = getattr(self, "_cut", [])
|
|
1058
|
+
return {"max_usd": self.max_usd, "spent_usd": None if got["spent_usd"] is None else round(got["spent_usd"], 6),
|
|
1059
|
+
**({"unsettled_usd": unsettled} if unsettled else {}), "stopped": stopped,
|
|
1060
|
+
"cut_off": list(dict.fromkeys(cut)), "not_run": list(dict.fromkeys(not_run)) if stopped else [],
|
|
1061
|
+
**({"unit_max_usd": self.unit_max_usd, "over_unit": list(dict.fromkeys(getattr(self, "_over", [])))}
|
|
1062
|
+
if self.unit_max_usd is not None else {})}
|
|
1063
|
+
|
|
1064
|
+
def note_refusal(self, exc: Optional[tuple]) -> None:
|
|
1065
|
+
"""A unit's end, counted toward the endpoint refusing the run: a wrong
|
|
1066
|
+
key or an unknown model fails every unit the same way, and running
|
|
1067
|
+
the rest only repeats it (with their containers and relays). After
|
|
1068
|
+
``REFUSED_IN_A_ROW`` such ends in a row, nothing more is begun."""
|
|
1069
|
+
if getattr(self, "_halt", None) is None:
|
|
1070
|
+
return
|
|
1071
|
+
with self._said:
|
|
1072
|
+
if exc and exc[0] in REFUSALS:
|
|
1073
|
+
self._refused.append(exc)
|
|
1074
|
+
if len(self._refused) >= REFUSED_IN_A_ROW and self.halted is None:
|
|
1075
|
+
self.halted = exc
|
|
1076
|
+
self._halt.set()
|
|
1077
|
+
else:
|
|
1078
|
+
self._refused.clear()
|
|
1079
|
+
|
|
1080
|
+
def said_unit(self, unit: str, score: Optional[dict], exc: Optional[tuple]) -> None:
|
|
1081
|
+
"""A unit's end, as a progress line: the i-th to end of the run's n,
|
|
1082
|
+
and how - passed or failed (a pass benchmark), scored, or the error."""
|
|
1083
|
+
with self._said:
|
|
1084
|
+
self._ended += 1
|
|
1085
|
+
i = self._ended
|
|
1086
|
+
score = score or {}
|
|
1087
|
+
if exc:
|
|
1088
|
+
how = {"outcome": "error", "error": exc[0]}
|
|
1089
|
+
elif self.bench["score"]["kind"] == "pass":
|
|
1090
|
+
how = {"outcome": "passed" if score.get("reward") == 1.0 else "failed", "reward": score.get("reward")}
|
|
1091
|
+
else:
|
|
1092
|
+
how = {"outcome": "scored", "score": score.get("score", score.get("reward"))}
|
|
1093
|
+
self.progress.emit("unit", "done", i=i, of=self.n_units, unit=unit, **how)
|
|
1094
|
+
|
|
1095
|
+
def unit_environment(self, setup: dict):
|
|
1096
|
+
"""(name, definition) of the environment the unit asks for - its
|
|
1097
|
+
setup's ``environment``, else its dataset's - or (None, None), the base
|
|
1098
|
+
image. A name the benchmark does not declare is the driver's error."""
|
|
1099
|
+
try:
|
|
1100
|
+
return definitions.environment_of(self.bench, setup, self.ds)
|
|
1101
|
+
except ValueError as e:
|
|
1102
|
+
raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}")
|
|
1103
|
+
|
|
1104
|
+
def not_run(self, name: Optional[str], env_def: Optional[dict]) -> Optional[tuple]:
|
|
1105
|
+
"""Why a unit cannot be run in its environment here, as (kind, said),
|
|
1106
|
+
or None. A secret it needs that is not set: the unit is not run, and
|
|
1107
|
+
that is no one's result (``EnvironmentSecretMissingError``, a setup
|
|
1108
|
+
error to the runs pages). An environment the local backend cannot
|
|
1109
|
+
give - anything but the base image, unless it says ``local_ok`` -
|
|
1110
|
+
refused the same way, for what would run in it (not a control)."""
|
|
1111
|
+
if name is None:
|
|
1112
|
+
return None
|
|
1113
|
+
if self.backend == "local" and self.other["_kind"] not in ("control", "upstream") and not env_def.get("local_ok"):
|
|
1114
|
+
return ("EnvironmentSetupError", f"not run: the environment {name} needs the container backend "
|
|
1115
|
+
"(--backend container); it does not say local_ok")
|
|
1116
|
+
missing = definitions.missing_secrets(env_def, self.licences)
|
|
1117
|
+
if missing:
|
|
1118
|
+
return ("EnvironmentSecretMissingError", "not run: needs " + ", ".join(missing))
|
|
1119
|
+
return None
|
|
1120
|
+
|
|
1121
|
+
def first_probe(self) -> bool:
|
|
1122
|
+
"""True once a run: its first unit checks the public web proxy's route,
|
|
1123
|
+
the check before the run having found the public web reachable."""
|
|
1124
|
+
with _PROBED:
|
|
1125
|
+
first = not getattr(self, "_probed", False)
|
|
1126
|
+
self._probed = True
|
|
1127
|
+
return first
|
|
1128
|
+
|
|
1129
|
+
def driver_secrets(self) -> dict:
|
|
1130
|
+
"""The secrets the benchmark's environments send to its driver
|
|
1131
|
+
(``to: driver``) that are set: in its process's environment, never in
|
|
1132
|
+
a request, so never in a log."""
|
|
1133
|
+
return driver_secrets(self.bench, self.licences)
|
|
1134
|
+
|
|
1135
|
+
# -- the unit's relay, and where its process runs ------------------------
|
|
1136
|
+
def agent_licences(self, env_def: Optional[dict], rl) -> dict:
|
|
1137
|
+
"""The variables the unit's environment gives the agent of its
|
|
1138
|
+
licences (``to: agent``): a key; a file's path, where it is mounted
|
|
1139
|
+
in a container (``rl.unit_binds``) or kept here; a server's address - in a
|
|
1140
|
+
container the one its forwarder answers at (``container.Unit``)."""
|
|
1141
|
+
inside = self.backend == "container"
|
|
1142
|
+
out = definitions.secret_values(env_def, self.licences, "agent", inside=inside)
|
|
1143
|
+
reach = getattr(rl, "reach", None) or {}
|
|
1144
|
+
for s in (env_def or {}).get("needs_secrets") or []:
|
|
1145
|
+
if inside and s.get("to") == "agent" and s.get("kind") == "server" and s["name"] in out:
|
|
1146
|
+
out[s["name"]] = reach.get(out[s["name"]], out[s["name"]])
|
|
1147
|
+
return out
|
|
1148
|
+
|
|
1149
|
+
def agent_files(self, env_def: Optional[dict]) -> List[tuple]:
|
|
1150
|
+
"""(where it is kept, its path inside) for each licence file the
|
|
1151
|
+
environment mounts into the agent's container."""
|
|
1152
|
+
out = []
|
|
1153
|
+
for s in (env_def or {}).get("needs_secrets") or []:
|
|
1154
|
+
h = self.licences.get(s["name"]) if s.get("to") == "agent" and s.get("kind") == "file" else None
|
|
1155
|
+
if h and s.get("path"):
|
|
1156
|
+
out.append((h["path"], s["path"]))
|
|
1157
|
+
return out
|
|
1158
|
+
|
|
1159
|
+
def agent_servers(self, env_def: Optional[dict]) -> List[str]:
|
|
1160
|
+
"""The licence servers the environment's agent must reach: host:port each."""
|
|
1161
|
+
out = []
|
|
1162
|
+
for s in (env_def or {}).get("needs_secrets") or []:
|
|
1163
|
+
h = self.licences.get(s["name"]) if s.get("to") == "agent" and s.get("kind") == "server" else None
|
|
1164
|
+
if h:
|
|
1165
|
+
out.append(h["value"])
|
|
1166
|
+
return out
|
|
1167
|
+
|
|
1168
|
+
def prices_record(self) -> Optional[dict]:
|
|
1169
|
+
"""What the job's config keeps of its prices: the rows of the models
|
|
1170
|
+
it calls, each with its ``source`` (``list`` the public list, ``own``
|
|
1171
|
+
the user's own row, ``run`` this run's ``--price``), and where and
|
|
1172
|
+
when the list was fetched when a row is its. None when it has none."""
|
|
1173
|
+
rows = price_list.given(self.models_called(), self.prices)
|
|
1174
|
+
if not rows:
|
|
1175
|
+
return None
|
|
1176
|
+
doc, mine = price_list.load(), getattr(self, "run_prices", {})
|
|
1177
|
+
rows = {n: {**r, "source": "run" if n in mine else "own" if n in doc["own"] else "list"} for n, r in rows.items()}
|
|
1178
|
+
listed = doc["from"] and any(r["source"] == "list" for r in rows.values())
|
|
1179
|
+
return {**({"from": doc["from"], "on": doc["on"]} if listed else {}), "models": rows}
|
|
1180
|
+
|
|
1181
|
+
def ledger_file(self) -> Optional[str]:
|
|
1182
|
+
"""The file of the run's spending cap, for a relay; None without a cap."""
|
|
1183
|
+
led = getattr(self, "ledger", None)
|
|
1184
|
+
return led.path if led is not None else None
|
|
1185
|
+
|
|
1186
|
+
def models_called(self) -> List[str]:
|
|
1187
|
+
"""Every model this run's relays may send a call to, as sent: the
|
|
1188
|
+
run's, the judge's, the simulator's, an option's embeddings and the
|
|
1189
|
+
agent's side models. Any other name a caller asks for is replaced by
|
|
1190
|
+
the run's (``relay``), so these are all a run can be priced for."""
|
|
1191
|
+
models = ledgers.models(self.other, *(getattr(self, n, None) for n in ("model", "judge_model", "simulator_model", "embed_model")),
|
|
1192
|
+
control=self.other.get("_kind") == "control")
|
|
1193
|
+
reranker = getattr(self, "rerank_model", None)
|
|
1194
|
+
return models + ([reranker] if reranker and reranker not in models else [])
|
|
1195
|
+
|
|
1196
|
+
def unit_relay(self, tag: str, trial: str, setup: dict, env_def: Optional[dict] = None, own: bool = False):
|
|
1197
|
+
"""The unit's own relay, started: in this process on the local
|
|
1198
|
+
backend, else a sidecar container. Either answers ``url`` (the
|
|
1199
|
+
driver's way in, on this machine), ``calls`` and ``stop``;
|
|
1200
|
+
``unit_url`` is the address the agent or capability is given. ``own``:
|
|
1201
|
+
the unit runs in its own image (:meth:`own_box`), not one of the
|
|
1202
|
+
benchmark's layers on the base."""
|
|
1203
|
+
if not self.base:
|
|
1204
|
+
return None # a control with no judge: no model is called
|
|
1205
|
+
record = os.path.join(trial, ".work", "relay")
|
|
1206
|
+
side = tuple((self.other.get("model") or {}).get("side_models") or ())
|
|
1207
|
+
key_header = PROVIDERS[self.provider].get("key_header", "authorization")
|
|
1208
|
+
if self.backend == "local":
|
|
1209
|
+
rl = relay.Relay(self.base, self.key, record, prices=getattr(self, "prices", None), model=self.model, side_models=side,
|
|
1210
|
+
key_header=key_header, ledger=self.ledger_file()).start()
|
|
1211
|
+
rl.unit_url = rl.url(tag)
|
|
1212
|
+
return rl
|
|
1213
|
+
routes = {tag + "-judge": self.judge_model} if self.judge_model else {}
|
|
1214
|
+
if self.simulator_model:
|
|
1215
|
+
routes[tag + "-simulator"] = self.simulator_model
|
|
1216
|
+
doors, listeners = [], []
|
|
1217
|
+
if self.other["_kind"] == "agent" and mcp_machine.wanted(setup):
|
|
1218
|
+
# the benchmark's own servers run here, in the driver's environment; the agent reaches each
|
|
1219
|
+
# through a door of its own on the unit's network (mcp_machine), its entry the bridge
|
|
1220
|
+
work = os.path.join(trial, ".work")
|
|
1221
|
+
try:
|
|
1222
|
+
setup["mcp"], doors, listeners = mcp_machine.doors(self.bench, setup, install.driver_dir(self.bench),
|
|
1223
|
+
self.machine_env(trial), work, work)
|
|
1224
|
+
except mcp_machine.Unknown as e:
|
|
1225
|
+
raise _Ended("EnvironmentSetupError", f"the unit's setup: {e}")
|
|
1226
|
+
from . import web_services
|
|
1227
|
+
try:
|
|
1228
|
+
doors.extend(web_services.doors(setup))
|
|
1229
|
+
except (ValueError, OSError) as e:
|
|
1230
|
+
for ln in listeners:
|
|
1231
|
+
ln.stop()
|
|
1232
|
+
raise _Ended("EnvironmentSetupError", f"the unit's web services: {e}")
|
|
1233
|
+
door = getattr(self, "loopback", None)
|
|
1234
|
+
probe = ((env_def or {}).get("network") or {}).get("probe") if self.other["_kind"] == "agent" else None
|
|
1235
|
+
rl = container.Unit(self.name, tag, record, image=self.image, app=APP, base=door.base if door else self.base, key=self.key,
|
|
1236
|
+
model=self.model, routes=routes, side_models=side, key_header=key_header,
|
|
1237
|
+
prices=price_list.given(self.models_called(), getattr(self, "prices", None)), # a sidecar's argv: its models' rows
|
|
1238
|
+
servers=self.agent_servers(env_def), doors=doors, hosts=door.hosts if door else None,
|
|
1239
|
+
ledger=self.ledger_file(),
|
|
1240
|
+
network_probe=probe, probe_now=bool(probe) and self.first_probe())
|
|
1241
|
+
rl.mcp_listeners = listeners
|
|
1242
|
+
try:
|
|
1243
|
+
try:
|
|
1244
|
+
rl.unit_image = self.setup_driver.image if self.setup_driver else "" if own else self.unit_image(setup)
|
|
1245
|
+
except container.Unavailable as e:
|
|
1246
|
+
raise _Ended("EnvironmentSetupError", f"the unit's environment: {e}")
|
|
1247
|
+
rl.unit_rw, rl.unit_files = container.mcp_mounts(setup.get("mcp") or [], trial)
|
|
1248
|
+
rl.unit_binds = self.agent_files(env_def)
|
|
1249
|
+
rl.start()
|
|
1250
|
+
except BaseException:
|
|
1251
|
+
for ln in listeners:
|
|
1252
|
+
ln.stop()
|
|
1253
|
+
raise
|
|
1254
|
+
rl.unit_url = rl.inside(tag)
|
|
1255
|
+
return rl
|
|
1256
|
+
|
|
1257
|
+
def machine_env(self, trial: str) -> dict:
|
|
1258
|
+
"""The environment of a benchmark's MCP server run on this machine
|
|
1259
|
+
(``mcp_machine``): the driver's, less its secrets - the agent may see
|
|
1260
|
+
it on the local backend - with a home of its own in the trial."""
|
|
1261
|
+
home = os.path.join(trial, ".work", "mcp-home")
|
|
1262
|
+
os.makedirs(home, exist_ok=True)
|
|
1263
|
+
drv = install.driver_dir(self.bench)
|
|
1264
|
+
extra = install.process_env(definitions.driver_install(self.bench), drv, home) if drv else {}
|
|
1265
|
+
return base_env(home, {**extra, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
|
|
1266
|
+
"NULLIUS_DATASET_ID": self.ds["id"], "PYTHONDONTWRITEBYTECODE": "1",
|
|
1267
|
+
**option_env(getattr(self, "bench_options", {})), **self.embeddings_env()})
|
|
1268
|
+
|
|
1269
|
+
def embeddings_env(self) -> dict:
|
|
1270
|
+
"""The driver's (and its MCP servers') search calls: one OpenAI base URL, with embeddings
|
|
1271
|
+
and reranking pinned separately. Nothing when the selected search calls no model."""
|
|
1272
|
+
if getattr(self, "bench_relay", None) is None:
|
|
1273
|
+
return {}
|
|
1274
|
+
return {"OPENAI_BASE_URL": self.bench_relay.url(self.embeddings_tag(), model=self.rerank_model or self.embed_model,
|
|
1275
|
+
embeddings=self.embed_model if self.rerank_model else None),
|
|
1276
|
+
"OPENAI_API_KEY": relay.PLACEHOLDER_KEY}
|
|
1277
|
+
|
|
1278
|
+
def embeddings_tag(self) -> str:
|
|
1279
|
+
"""This thread's route on the benchmark's relay. A driver serves one thread, one unit at a time: what
|
|
1280
|
+
it was refused there is that unit's, never one running beside it."""
|
|
1281
|
+
return f"embeddings-{threading.get_ident()}"
|
|
1282
|
+
|
|
1283
|
+
def unit_image(self, setup: dict) -> str:
|
|
1284
|
+
"""The image a unit runs in (``container.image_for``); the first unit
|
|
1285
|
+
to ask for an environment says its build - or that it is there
|
|
1286
|
+
already - as a stage of its own."""
|
|
1287
|
+
work = os.path.join(self.job_dir, ".work")
|
|
1288
|
+
name, _ = definitions.environment_of(self.bench, setup, self.ds)
|
|
1289
|
+
with self._said:
|
|
1290
|
+
first = name is not None and name not in self._envs
|
|
1291
|
+
self._envs.add(name)
|
|
1292
|
+
if not first:
|
|
1293
|
+
return container.image_for(self.bench, setup, self.ds, work)
|
|
1294
|
+
tag = container.env_tag(self.bench, name, self.image)
|
|
1295
|
+
with self.progress.stage("environment", cached=container.present(tag), start=f"building {tag}", name=name):
|
|
1296
|
+
return container.image_for(self.bench, setup, self.ds, work)
|
|
1297
|
+
|
|
1298
|
+
def place(self, rl, cmd: List[str], env: dict, workdir: str, home: str, interactive: bool = True, box=None):
|
|
1299
|
+
"""A unit's command and environment as they are started: unchanged on
|
|
1300
|
+
the local backend; in a container, ``docker run`` on the unit's network,
|
|
1301
|
+
with the folders it reads at their own paths and the image's ``PATH``
|
|
1302
|
+
behind the install's. The secrets its environment sends the agent
|
|
1303
|
+
(``to: agent``) are added to its environment - in a container passed by
|
|
1304
|
+
name, never on an argument list. In an own image (``box``), ``docker
|
|
1305
|
+
exec`` in the unit's container, the image's own ``PATH`` first - the
|
|
1306
|
+
task's programs are the task's - then the install's and the runtimes'."""
|
|
1307
|
+
env = dict(env, **(getattr(rl, "unit_secrets", None) or {}))
|
|
1308
|
+
env.update(getattr(rl, "network_env", {}))
|
|
1309
|
+
if box is not None:
|
|
1310
|
+
inside = {**box.env, **{k: v for k, v in env.items() if k not in container.HOST_ONLY},
|
|
1311
|
+
**getattr(box, "agent_proxy", {})}
|
|
1312
|
+
path = ":".join(([box.env["PATH"]] if "PATH" in box.env else []) + [box.path]
|
|
1313
|
+
+ install.bin_dirs(self.inst) + [d + "/bin" for _, d in box.runtimes.values()])
|
|
1314
|
+
argv = box.exec_argv(container.in_runtime(cmd, box.runtimes), inside, path, interactive=interactive)
|
|
1315
|
+
return argv, container.program_env(inside)
|
|
1316
|
+
if self.backend == "local":
|
|
1317
|
+
return cmd, env
|
|
1318
|
+
ws = os.path.join(os.path.dirname(home), "workspace")
|
|
1319
|
+
# What the agent's container sees: its workspace and home, its own
|
|
1320
|
+
# install and definition, and the files its MCP servers need - never
|
|
1321
|
+
# this application's folder (its .env holds the keys) nor the
|
|
1322
|
+
# benchmark's (its data holds the answers).
|
|
1323
|
+
mounted = container.mounts([ws, home, *getattr(rl, "unit_rw", [])],
|
|
1324
|
+
[self.inst, self.other.get("_here")],
|
|
1325
|
+
getattr(rl, "unit_files", []))
|
|
1326
|
+
env = dict(env, PATH=os.pathsep.join(install.bin_dirs(self.inst) + [container.IMAGE_PATH]))
|
|
1327
|
+
return rl.wrap(list(cmd), env, workdir, mounted, image=rl.unit_image, interactive=interactive,
|
|
1328
|
+
binds=getattr(rl, "unit_binds", []))
|
|
1329
|
+
|
|
1330
|
+
# -- a control: nothing, or the benchmark's own reference -----------------
|
|
1331
|
+
def control_answer(self, unit, ws, box=None, name: str = "") -> str:
|
|
1332
|
+
"""``nop`` answers "" and leaves the workspace as setup left it;
|
|
1333
|
+
``oracle`` is the driver's ``oracle`` - its answer, and whatever it
|
|
1334
|
+
wrote into the workspace, or, in an own image (``box``), the step its
|
|
1335
|
+
reply names (``solve``: the reference solution copied in and run in
|
|
1336
|
+
the unit's container). Either goes to ``score`` as an agent's answer
|
|
1337
|
+
would; a capability benchmark's skips ``produce``."""
|
|
1338
|
+
if self.other["key"] != "oracle":
|
|
1339
|
+
return ""
|
|
1340
|
+
try:
|
|
1341
|
+
reply = self.driver().request("oracle", unit=unit, dir=ws, timeout=DRIVER_SECONDS)
|
|
1342
|
+
except jsonl.Failed as e:
|
|
1343
|
+
raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: " + str(e))
|
|
1344
|
+
if reply.get("solve") is not None:
|
|
1345
|
+
if box is None:
|
|
1346
|
+
raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: solve runs in an own image, and this unit has none")
|
|
1347
|
+
if name: # the reference solution, run under its own limit
|
|
1348
|
+
self.phase(name, unit, "agent_run", float(reply["solve"].get("seconds") or self.turn_seconds))
|
|
1349
|
+
done = self.in_box(box, reply["solve"], collect=False, what="the benchmark's oracle")
|
|
1350
|
+
os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True)
|
|
1351
|
+
with open(os.path.join(os.path.dirname(ws), "agent", "solve.txt"), "w") as f:
|
|
1352
|
+
f.write(done["output"])
|
|
1353
|
+
if done["exit"] != 0:
|
|
1354
|
+
raise _Ended("NonZeroAgentExitCodeError", "the benchmark's oracle: its solution "
|
|
1355
|
+
+ ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}"))
|
|
1356
|
+
return str(reply.get("answer") or "")
|
|
1357
|
+
|
|
1358
|
+
# -- an own image: the unit's container ------------------------------------
|
|
1359
|
+
@contextmanager
|
|
1360
|
+
def agent_internet(self, setup: dict, box, rl, trial: str):
|
|
1361
|
+
"""The benchmark's task policy, never a run option: a public-address proxy
|
|
1362
|
+
during the agent or reference solution, removed before any verification.
|
|
1363
|
+
Internal relay/server aliases stay direct; failed and cancelled work closes
|
|
1364
|
+
the same door and keeps its traffic record in ``agent/egress.log``."""
|
|
1365
|
+
if not setup.get("internet"):
|
|
1366
|
+
yield
|
|
1367
|
+
return
|
|
1368
|
+
try:
|
|
1369
|
+
try:
|
|
1370
|
+
box.online()
|
|
1371
|
+
box.agent_proxy = container.proxy_env(rl.direct() if isinstance(rl, container.Unit) else [container.RELAY_ALIAS])
|
|
1372
|
+
except (container.Unavailable, ValueError) as e:
|
|
1373
|
+
raise _Ended("EnvironmentSetupError", f"the task's internet access: {e}")
|
|
1374
|
+
yield
|
|
1375
|
+
finally:
|
|
1376
|
+
box.agent_proxy = {}
|
|
1377
|
+
try:
|
|
1378
|
+
try:
|
|
1379
|
+
said = box.egress_said()
|
|
1380
|
+
if said:
|
|
1381
|
+
out = os.path.join(trial, "agent")
|
|
1382
|
+
os.makedirs(out, exist_ok=True)
|
|
1383
|
+
with open(os.path.join(out, "egress.log"), "w") as f:
|
|
1384
|
+
f.write(said)
|
|
1385
|
+
finally:
|
|
1386
|
+
box.offline()
|
|
1387
|
+
except container.Unavailable as e:
|
|
1388
|
+
raise _Ended("VerifierError", f"the task's internet access could not be closed: {e}")
|
|
1389
|
+
|
|
1390
|
+
def own_box(self, tag: str, setup: dict, rl, home: str):
|
|
1391
|
+
"""The unit's own container, started (``container.Box``): the image its
|
|
1392
|
+
setup names, built or pulled on first use and said as a stage; for an
|
|
1393
|
+
agent, its install, definition, home and runtimes mounted, and each
|
|
1394
|
+
runtime shown to start in the image - a unit where one cannot (a
|
|
1395
|
+
glibc runtime in a musl image) is not run, which is no one's result."""
|
|
1396
|
+
spec = setup.get("image")
|
|
1397
|
+
bad = container.spec_problems(spec)
|
|
1398
|
+
bad += container.task_env_problems(setup.get("env", {}), (self.other.get("model") or {}).get("env", {}))
|
|
1399
|
+
if setup.get("verify") is not None:
|
|
1400
|
+
bad += ["verify." + p for p in container.step_problems(setup["verify"], collect=True)]
|
|
1401
|
+
if setup.get("prepare") is not None:
|
|
1402
|
+
bad += ["prepare." + p for p in container.step_problems(setup["prepare"], collect=False)]
|
|
1403
|
+
if setup.get("workdir") is not None and not (isinstance(setup["workdir"], str) and setup["workdir"].startswith("/")):
|
|
1404
|
+
bad.append("workdir: an absolute path inside the image")
|
|
1405
|
+
if bad:
|
|
1406
|
+
raise _Ended("EnvironmentSetupError", "the unit's setup: " + "; ".join(bad))
|
|
1407
|
+
why = container.unusable()
|
|
1408
|
+
if why:
|
|
1409
|
+
raise _Ended("EnvironmentSetupError", f"not run: its own image needs docker, and {why}")
|
|
1410
|
+
box = None
|
|
1411
|
+
try:
|
|
1412
|
+
image = self.own_image(spec)
|
|
1413
|
+
cfg = container.image_config(image)
|
|
1414
|
+
mounted, runtimes = [], {}
|
|
1415
|
+
if self.other["_kind"] == "agent":
|
|
1416
|
+
# as on the base: its home, its install and definition, its MCP servers' files -
|
|
1417
|
+
# never this application's folder nor the benchmark's; and what it runs on, under /opt/nullius
|
|
1418
|
+
mounted = container.mounts([home, *getattr(rl, "unit_rw", [])],
|
|
1419
|
+
[self.inst, self.other.get("_here")],
|
|
1420
|
+
getattr(rl, "unit_files", []))
|
|
1421
|
+
runtimes = self.runtimes
|
|
1422
|
+
box = container.Box(self.name, tag, image, workdir=setup.get("workdir") or cfg["workdir"], path=cfg["path"],
|
|
1423
|
+
network=rl.names["network"] if isinstance(rl, container.Unit) else None,
|
|
1424
|
+
mounted=mounted, runtimes=runtimes, memory_mb=spec.get("memory_mb"), cpus=spec.get("cpus"),
|
|
1425
|
+
app=APP, env=setup.get("env")).start()
|
|
1426
|
+
for name in runtimes:
|
|
1427
|
+
why = box.runtime_problem(name)
|
|
1428
|
+
if why:
|
|
1429
|
+
raise _Ended("EnvironmentSetupError", "not run: " + why)
|
|
1430
|
+
return box
|
|
1431
|
+
except (container.Unavailable, ValueError, OSError) as e:
|
|
1432
|
+
if box is not None:
|
|
1433
|
+
box.stop()
|
|
1434
|
+
raise _Ended("EnvironmentSetupError", f"the unit's own image: {e}")
|
|
1435
|
+
except _Ended:
|
|
1436
|
+
box.stop()
|
|
1437
|
+
raise
|
|
1438
|
+
|
|
1439
|
+
@staticmethod
|
|
1440
|
+
def box_changes(box) -> Optional[str]:
|
|
1441
|
+
"""What changed in the unit's container since its image - the work
|
|
1442
|
+
itself, where the workspace is the container's: the trial keeps it
|
|
1443
|
+
(``agent/container-changes.txt``)."""
|
|
1444
|
+
try:
|
|
1445
|
+
return box.diff()
|
|
1446
|
+
except container.Unavailable:
|
|
1447
|
+
return None
|
|
1448
|
+
|
|
1449
|
+
def release_image(self, image: str) -> None:
|
|
1450
|
+
"""A unit is done with its own image: with ``remove_images``, removed
|
|
1451
|
+
when no other unit is using it and this run fetched it
|
|
1452
|
+
(``container.drop_image``)."""
|
|
1453
|
+
with self._said:
|
|
1454
|
+
left = self._image_users.get(image, 1) - 1
|
|
1455
|
+
if left > 0:
|
|
1456
|
+
self._image_users[image] = left
|
|
1457
|
+
return
|
|
1458
|
+
self._image_users.pop(image, None)
|
|
1459
|
+
if not self.keep_images and container.fetched(image):
|
|
1460
|
+
container.drop_image(image)
|
|
1461
|
+
|
|
1462
|
+
def own_image(self, spec: dict) -> str:
|
|
1463
|
+
"""The image an own-image unit names, built or pulled the first time
|
|
1464
|
+
a unit asks for it (``container.own_image``) - said as a stage then, or
|
|
1465
|
+
that it is there already."""
|
|
1466
|
+
root = self.up or self.bench["_here"]
|
|
1467
|
+
tag = spec.get("pull") or container.own_tag(self.bench, spec["build"], container.within(root, spec["build"]))
|
|
1468
|
+
work = os.path.join(self.job_dir, ".work")
|
|
1469
|
+
with self._said:
|
|
1470
|
+
first = tag not in self._envs
|
|
1471
|
+
self._envs.add(tag)
|
|
1472
|
+
self._image_users[tag] = self._image_users.get(tag, 0) + 1
|
|
1473
|
+
if not first:
|
|
1474
|
+
return container.own_image(self.bench, spec, root, work)
|
|
1475
|
+
with self.progress.stage("environment", cached=container.present(tag),
|
|
1476
|
+
start=("pulling " if spec.get("pull") else "building ") + tag, name=tag):
|
|
1477
|
+
return container.own_image(self.bench, spec, root, work)
|
|
1478
|
+
|
|
1479
|
+
def in_box(self, box, step: dict, *, collect: bool, what: str, out: Optional[str] = None) -> dict:
|
|
1480
|
+
"""A step (``verify``, ``solve``) run in the unit's container. For the
|
|
1481
|
+
check (``collect``), what the agent left running is stopped first
|
|
1482
|
+
(``container.Box.settle``) - unless the step keeps it
|
|
1483
|
+
(``keep_processes``) - and what was stopped kept in ``out``
|
|
1484
|
+
(``stopped.txt``). Then its files are copied in from the upstream and the ones
|
|
1485
|
+
the driver gives by their text written in (``write``), a
|
|
1486
|
+
way out to public addresses given when it asks for the network
|
|
1487
|
+
(``container.Box.online``: a proxy, named to the command alone), and
|
|
1488
|
+
its command run in the working directory."""
|
|
1489
|
+
bad = container.step_problems(step, collect=collect)
|
|
1490
|
+
if bad:
|
|
1491
|
+
raise _Ended("VerifierError" if collect else "NonZeroAgentExitCodeError", f"{what}: " + "; ".join(bad))
|
|
1492
|
+
root = self.up or self.bench["_here"]
|
|
1493
|
+
try:
|
|
1494
|
+
if collect and not step.get("keep_processes"):
|
|
1495
|
+
stopped = box.settle()
|
|
1496
|
+
if stopped and out:
|
|
1497
|
+
with open(os.path.join(out, "stopped.txt"), "w") as f:
|
|
1498
|
+
f.write("".join(s + "\n" for s in stopped))
|
|
1499
|
+
for src, dest in (step.get("copy") or {}).items():
|
|
1500
|
+
box.copy_in(container.within(root, src), dest)
|
|
1501
|
+
for dest, text in (step.get("write") or {}).items():
|
|
1502
|
+
box.write_in(dest, text)
|
|
1503
|
+
env = box.online() if step.get("network") else (getattr(box, "agent_proxy", None) if not collect else None)
|
|
1504
|
+
if step.get("collect"):
|
|
1505
|
+
box.run(["mkdir", "-p", step["collect"]], 60, user="0")
|
|
1506
|
+
return box.run(step["run"], float(step.get("seconds") or self.turn_seconds), env=env)
|
|
1507
|
+
except (container.Unavailable, ValueError) as e:
|
|
1508
|
+
raise _Ended("VerifierError" if collect else "NonZeroAgentExitCodeError", f"{what}: {e}")
|
|
1509
|
+
|
|
1510
|
+
def prepare(self, step: dict, box, trial: str) -> None:
|
|
1511
|
+
"""The benchmark's preparation of the unit's container (``prepare``),
|
|
1512
|
+
before the agent or a control starts: what the task's own harness adds
|
|
1513
|
+
to every image (Terminal-Bench's tmux). Its files copied and written
|
|
1514
|
+
in, its command run as root - with a way out, when it asks for the
|
|
1515
|
+
network, through the check's proxy (``container.Box.online``) - then
|
|
1516
|
+
that proxy removed and shown gone (``Box.offline``) and whatever it
|
|
1517
|
+
left running stopped (``Box.settle``): the agent starts with no way
|
|
1518
|
+
out but its model, as it would have without one. Its output and what
|
|
1519
|
+
the proxy let through are kept (``setup/prepare.txt``,
|
|
1520
|
+
``setup/egress.log``). Any failure, closing the way out included, is
|
|
1521
|
+
the environment's (``EnvironmentSetupError``), never the agent's."""
|
|
1522
|
+
out = os.path.join(trial, "setup")
|
|
1523
|
+
os.makedirs(out, exist_ok=True)
|
|
1524
|
+
root = self.up or self.bench["_here"]
|
|
1525
|
+
done = None
|
|
1526
|
+
try:
|
|
1527
|
+
for src, dest in (step.get("copy") or {}).items():
|
|
1528
|
+
box.copy_in(container.within(root, src), dest)
|
|
1529
|
+
for dest, text in (step.get("write") or {}).items():
|
|
1530
|
+
box.write_in(dest, text)
|
|
1531
|
+
env = box.online() if step.get("network") else None
|
|
1532
|
+
try:
|
|
1533
|
+
done = box.run(step["run"], float(step.get("seconds") or self.turn_seconds), user="0", env=env)
|
|
1534
|
+
finally:
|
|
1535
|
+
said = box.egress_said()
|
|
1536
|
+
if said:
|
|
1537
|
+
with open(os.path.join(out, "egress.log"), "w") as f:
|
|
1538
|
+
f.write(said)
|
|
1539
|
+
box.offline()
|
|
1540
|
+
box.settle()
|
|
1541
|
+
except (container.Unavailable, ValueError) as e:
|
|
1542
|
+
raise _Ended("EnvironmentSetupError", f"the benchmark's preparation: {e}")
|
|
1543
|
+
finally:
|
|
1544
|
+
if done is not None:
|
|
1545
|
+
with open(os.path.join(out, "prepare.txt"), "w") as f:
|
|
1546
|
+
f.write(done["output"])
|
|
1547
|
+
if done["exit"] != 0:
|
|
1548
|
+
raise _Ended("EnvironmentSetupError", "the benchmark's preparation "
|
|
1549
|
+
+ ("did not end in time" if done["timed_out"] else f"ended with exit {done['exit']}")
|
|
1550
|
+
+ ": " + done["output"].strip()[-300:])
|
|
1551
|
+
|
|
1552
|
+
def verify(self, step: dict, box, trial: str) -> dict:
|
|
1553
|
+
"""The benchmark's check, run in the unit's container once the agent
|
|
1554
|
+
has ended (``in_box``), what it collects copied out into the trial's
|
|
1555
|
+
``verifier/`` and its output kept there (``test-stdout.txt``, and what
|
|
1556
|
+
its proxy let through and refused, ``egress.log``): what ``score`` is
|
|
1557
|
+
sent as ``verified``, the folder to read the rest from."""
|
|
1558
|
+
out = os.path.join(trial, "verifier")
|
|
1559
|
+
os.makedirs(out, exist_ok=True)
|
|
1560
|
+
try:
|
|
1561
|
+
done = self.in_box(box, step, collect=True, what="the benchmark's check", out=out)
|
|
1562
|
+
finally:
|
|
1563
|
+
said = box.egress_said()
|
|
1564
|
+
if said:
|
|
1565
|
+
with open(os.path.join(out, "egress.log"), "w") as f:
|
|
1566
|
+
f.write(said)
|
|
1567
|
+
with open(os.path.join(out, "test-stdout.txt"), "w") as f:
|
|
1568
|
+
f.write(done["output"])
|
|
1569
|
+
collected = box.copy_out(step["collect"], out) if step.get("collect") else False
|
|
1570
|
+
return {"exit": done["exit"], "timed_out": done["timed_out"], "output": done["output"][-4000:],
|
|
1571
|
+
"dir": out, "collected": collected}
|
|
1572
|
+
|
|
1573
|
+
# -- an agent, over ACP ----------------------------------------------------
|
|
1574
|
+
def simulator(self, rl, tag: str) -> dict:
|
|
1575
|
+
"""What a ``turn`` request carries when a model plays the benchmark's
|
|
1576
|
+
user: that model's route on the unit's relay, recorded beside the
|
|
1577
|
+
unit's calls (``<tag>-simulator``) and never counted as the agent's."""
|
|
1578
|
+
if not self.simulator_model:
|
|
1579
|
+
return {}
|
|
1580
|
+
return {"simulator": {"model": self.simulator_model, "key": relay.PLACEHOLDER_KEY,
|
|
1581
|
+
"base_url": rl.url(tag + "-simulator", model=self.simulator_model)}}
|
|
1582
|
+
|
|
1583
|
+
def judge(self, rl, tag: str) -> dict:
|
|
1584
|
+
"""What a ``score`` request carries when the benchmark is graded by a
|
|
1585
|
+
model - and every ``turn`` too, for a driver whose upstream grades the
|
|
1586
|
+
conversation as it ends: that model's route on the unit's relay."""
|
|
1587
|
+
if not self.judge_model:
|
|
1588
|
+
return {}
|
|
1589
|
+
return {"judge": {"model": self.judge_model, "base_url": rl.url(tag + "-judge", model=self.judge_model),
|
|
1590
|
+
"key": relay.PLACEHOLDER_KEY}}
|
|
1591
|
+
|
|
1592
|
+
def turn_limit(self, setup: dict) -> float:
|
|
1593
|
+
"""How long one turn of the agent may take: the unit's own limit when its ``setup`` names one
|
|
1594
|
+
(``agent_seconds``: a task that declares its agent timeout, as Terminal-Bench's task.toml does),
|
|
1595
|
+
else the run's ``turn_seconds``."""
|
|
1596
|
+
own = setup.get("agent_seconds") if isinstance(setup, dict) else None
|
|
1597
|
+
if isinstance(own, (int, float)) and not isinstance(own, bool) and 0 < own <= MAX_AGENT_SECONDS:
|
|
1598
|
+
return float(own)
|
|
1599
|
+
return self.turn_seconds
|
|
1600
|
+
|
|
1601
|
+
def drive_agent(self, unit, ws, home, tag, setup, transcript, rl, box=None, name=""):
|
|
1602
|
+
"""The agent over ACP, its session's ``cwd`` the unit's workspace - in
|
|
1603
|
+
an own image (``box``), the container's working directory, which this
|
|
1604
|
+
machine does not see. Files are served (``acp.Client(fs=)``) only to an
|
|
1605
|
+
agent running here: one in a container has its workspace mounted, and
|
|
1606
|
+
serving it from this machine would let it swap a folder for a link
|
|
1607
|
+
between the check and the open and reach any file of the user's."""
|
|
1608
|
+
a = self.other
|
|
1609
|
+
env = agent_env(a, self.inst, home, rl.unit_url, self.model, self.options)
|
|
1610
|
+
log = os.path.join(os.path.dirname(ws), "agent", "agent.log")
|
|
1611
|
+
os.makedirs(os.path.dirname(log), exist_ok=True)
|
|
1612
|
+
if "command" in a["start"]:
|
|
1613
|
+
return self._command_agent(a, env, ws, home, setup, transcript, log, tag, unit, rl, box)
|
|
1614
|
+
cmd = install.words(a["start"]["acp"], {"here": a["_here"], "install": self.inst})
|
|
1615
|
+
if self.backend == "local": # inside a container, its own PATH finds it
|
|
1616
|
+
cmd[0] = shutil.which(cmd[0], path=env["PATH"]) or cmd[0]
|
|
1617
|
+
cmd, env = self.place(rl, cmd, env, ws, home, box=box)
|
|
1618
|
+
c = acp.Client(cmd, env, log, cwd=ws, grant=True, fs=box is None and self.backend == "local", stopped=self._stopped)
|
|
1619
|
+
events: List[dict] = []
|
|
1620
|
+
try:
|
|
1621
|
+
try:
|
|
1622
|
+
try:
|
|
1623
|
+
c.start()
|
|
1624
|
+
sid = c.new_session(box.workdir if box is not None else ws, setup.get("mcp") or [])
|
|
1625
|
+
except TimeoutError as e:
|
|
1626
|
+
# the agent never answered its start: this unit's setup failed, not the run
|
|
1627
|
+
raise _Ended("AgentSetupTimeoutError", f"the agent did not answer {e} in time ({c.tail()})")
|
|
1628
|
+
prompt = setup["prompt"]
|
|
1629
|
+
answer = ""
|
|
1630
|
+
# a conversation's turns carry its simulated user's route and its judge's (each when it has one)
|
|
1631
|
+
sim = dict(self.simulator(rl, tag), **self.judge(rl, tag)) if "agent.conversation/v1" in self.bench["needs"] else {}
|
|
1632
|
+
opened = True
|
|
1633
|
+
if self.simulator_model:
|
|
1634
|
+
# The simulated user speaks first: the first turn, asked with nothing said yet,
|
|
1635
|
+
# follows setup's prompt (what the agent is told before anyone speaks).
|
|
1636
|
+
first = self.driver().request("turn", unit=unit, dir=ws, transcript=[], timeout=600, **sim, **self.attempt_kw())
|
|
1637
|
+
opened = not first.get("done")
|
|
1638
|
+
if opened:
|
|
1639
|
+
prompt = prompt + "\n\n" + first["message"]
|
|
1640
|
+
for n in range(self.max_turns if opened else 0):
|
|
1641
|
+
transcript.append({"role": "user", "text": prompt})
|
|
1642
|
+
if n: # the limit is each turn's: the board's clock starts again with it
|
|
1643
|
+
self.phase(name, unit, "agent_run", self.turn_limit(setup))
|
|
1644
|
+
turn = c.prompt(sid, prompt, timeout=self.turn_limit(setup))
|
|
1645
|
+
answer = turn.text
|
|
1646
|
+
transcript.append({"role": "agent", "text": turn.text, "stop": turn.stop})
|
|
1647
|
+
if "agent.conversation/v1" not in self.bench["needs"]:
|
|
1648
|
+
break
|
|
1649
|
+
nxt = self.driver().request("turn", unit=unit, dir=ws, transcript=transcript, timeout=600, **sim, **self.attempt_kw())
|
|
1650
|
+
if nxt.get("done"):
|
|
1651
|
+
break
|
|
1652
|
+
prompt = nxt["message"]
|
|
1653
|
+
if nxt.get("restart"):
|
|
1654
|
+
# the upstream started its conversation again (its own retry of an attempt that
|
|
1655
|
+
# raised): a new session, told what it was told before anyone spoke
|
|
1656
|
+
sid = c.new_session(box.workdir if box is not None else ws, setup.get("mcp") or [])
|
|
1657
|
+
prompt = setup["prompt"] + "\n\n" + prompt
|
|
1658
|
+
events = c.events
|
|
1659
|
+
except acp.Timeout as e:
|
|
1660
|
+
events = c.events
|
|
1661
|
+
raise _Ended("AgentTimeoutError", str(e), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"))
|
|
1662
|
+
except acp.AgentFailed as e:
|
|
1663
|
+
events = c.events
|
|
1664
|
+
raise _Ended("NonZeroAgentExitCodeError", str(e), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"))
|
|
1665
|
+
except TimeoutError as e:
|
|
1666
|
+
events = c.events # a later request the agent never answered (a new session)
|
|
1667
|
+
raise _Ended("AgentTimeoutError", f"the agent did not answer {e} in time",
|
|
1668
|
+
job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"))
|
|
1669
|
+
except jsonl.Failed as e:
|
|
1670
|
+
raise _Ended("VerifierError", "the benchmark's next turn: " + str(e))
|
|
1671
|
+
finally:
|
|
1672
|
+
c.close()
|
|
1673
|
+
raw = "".join(json.dumps(e) + "\n" for e in events)
|
|
1674
|
+
return answer, job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {"acp.jsonl": raw}
|
|
1675
|
+
|
|
1676
|
+
def _command_agent(self, a, env, ws, home, setup, transcript, log, tag, unit, rl, box=None):
|
|
1677
|
+
cmd = install.words(a["start"]["command"], {"here": a["_here"], "install": self.inst, "prompt": setup["prompt"]})
|
|
1678
|
+
cmd, env = self.place(rl, cmd, env, ws, home, interactive=False, box=box)
|
|
1679
|
+
transcript.append({"role": "user", "text": setup["prompt"]})
|
|
1680
|
+
t = time.time()
|
|
1681
|
+
try:
|
|
1682
|
+
with open(log, "a") as err:
|
|
1683
|
+
r = run_group(cmd, self.turn_limit(setup), stopped=self._stopped, cwd=ws, env=env, stdin=subprocess.DEVNULL, stdout=subprocess.PIPE,
|
|
1684
|
+
stderr=err, text=True)
|
|
1685
|
+
except subprocess.TimeoutExpired:
|
|
1686
|
+
raise _Ended("AgentTimeoutError", f"the agent did not end in {self.turn_limit(setup)} s")
|
|
1687
|
+
if r.returncode != 0:
|
|
1688
|
+
raise _Ended("NonZeroAgentExitCodeError", f"exit {r.returncode}")
|
|
1689
|
+
transcript.append({"role": "agent", "text": r.stdout})
|
|
1690
|
+
events = [{"t": t, "update": {"sessionUpdate": "user_message_chunk", "content": {"type": "text", "text": setup["prompt"]}}},
|
|
1691
|
+
{"t": time.time(), "update": {"sessionUpdate": "agent_message_chunk", "content": {"type": "text", "text": r.stdout}}}]
|
|
1692
|
+
return r.stdout.strip(), job.trajectory_from_events(events, rl.calls(tag), a, self.model, f"{self.name}/{unit}"), {}
|
|
1693
|
+
|
|
1694
|
+
# -- a capability, through its port -------------------------------------------
|
|
1695
|
+
def drive_feature(self, unit, ws, home, tag, setup, transcript, rl):
|
|
1696
|
+
f = self.other
|
|
1697
|
+
cmd = install.words(f["start"]["command"], {"here": f["_here"], "install": self.inst})
|
|
1698
|
+
env = agent_env(f, self.inst, home, rl.unit_url, self.model, self.options)
|
|
1699
|
+
env["PYTHONDONTWRITEBYTECODE"] = "1"
|
|
1700
|
+
cmd, env = self.place(rl, cmd, env, f["_here"], home)
|
|
1701
|
+
cwd = os.path.join(os.path.dirname(ws), ".work", "capability") # the unit's, never the catalogue folder
|
|
1702
|
+
os.makedirs(cwd, exist_ok=True)
|
|
1703
|
+
proc = jsonl.Proc(cmd, cwd, env, os.path.join(os.path.dirname(ws), "agent", "capability.log"), stopped=self._stopped)
|
|
1704
|
+
os.makedirs(os.path.join(os.path.dirname(ws), "agent"), exist_ok=True)
|
|
1705
|
+
exchanges: List[dict] = []
|
|
1706
|
+
|
|
1707
|
+
def send(op, **kw):
|
|
1708
|
+
t0 = time.time()
|
|
1709
|
+
reply = proc.request(op, timeout=self.turn_seconds, **kw)
|
|
1710
|
+
exchanges.append({"op": op, "request": kw, "reply": reply, "t0": t0, "t1": time.time()})
|
|
1711
|
+
return reply
|
|
1712
|
+
|
|
1713
|
+
try:
|
|
1714
|
+
try:
|
|
1715
|
+
proc.start()
|
|
1716
|
+
plan = setup.get("memory") or {}
|
|
1717
|
+
for s in plan.get("add") or []:
|
|
1718
|
+
send("add", session=s)
|
|
1719
|
+
recalled = [{"id": q.get("id"), "memory": send("recall", question=q["question"]).get("memory", "")}
|
|
1720
|
+
for q in plan.get("recall") or []]
|
|
1721
|
+
except jsonl.Failed as e:
|
|
1722
|
+
raise _Ended("NonZeroAgentExitCodeError", "the capability: " + str(e),
|
|
1723
|
+
job.trajectory_from_exchanges(exchanges, [], f, self.model, f"{self.name}/{unit}"))
|
|
1724
|
+
finally:
|
|
1725
|
+
proc.close()
|
|
1726
|
+
try:
|
|
1727
|
+
out = self.driver().request("produce", unit=unit, dir=ws, recalled=recalled, model=self.model,
|
|
1728
|
+
base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds)
|
|
1729
|
+
except jsonl.Failed as e:
|
|
1730
|
+
raise _Ended("VerifierError", "the benchmark's reader: " + str(e),
|
|
1731
|
+
job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}"))
|
|
1732
|
+
transcript.append({"role": "capability", "recalled": recalled})
|
|
1733
|
+
return out.get("answer"), job.trajectory_from_exchanges(exchanges, rl.calls(tag), f, self.model, f"{self.name}/{unit}")
|
|
1734
|
+
|
|
1735
|
+
|
|
1736
|
+
def drive_upstream(self, unit, ws, tag, transcript, rl):
|
|
1737
|
+
"""The setup's unit: the driver runs the upstream's own answerer on it
|
|
1738
|
+
(``answer``), asked as ``asked`` through the unit's relay, which sends
|
|
1739
|
+
the run's model in its place. A reply with no text - a reasoning model
|
|
1740
|
+
that spent the upstream's token budget thinking - is said as such."""
|
|
1741
|
+
o = self.other
|
|
1742
|
+
try:
|
|
1743
|
+
# the benchmark's other models reach the upstream's pipeline as they reach a turn or a score: their own routes
|
|
1744
|
+
out = self.driver().request("answer", unit=unit, dir=ws, setup=o["key"], model=self.asked, **self.attempt_kw(),
|
|
1745
|
+
**self.simulator(rl, tag), **self.judge(rl, tag),
|
|
1746
|
+
base_url=rl.url(tag), key=relay.PLACEHOLDER_KEY, timeout=self.turn_seconds)
|
|
1747
|
+
except jsonl.Failed as e:
|
|
1748
|
+
calls = rl.calls(tag)
|
|
1749
|
+
silent = [c for c in calls if c.get("status") == 200 and not ((c.get("reply") or {}).get("text") or "").strip()]
|
|
1750
|
+
if silent:
|
|
1751
|
+
u = (silent[-1].get("reply") or {}).get("usage") or {}
|
|
1752
|
+
thought = u.get("reasoning") or 0
|
|
1753
|
+
raise _Ended("OutputLengthExceededError",
|
|
1754
|
+
f"{self.model} returned no text" + (f" ({thought} of its {u.get('output')} tokens were reasoning)" if thought else "")
|
|
1755
|
+
+ f": {self.bench['label']}'s code asks for a short answer, and a reasoning model can spend it all before it answers"
|
|
1756
|
+
+ (" - the original's models do not reason" if o["key"] == "model" else ""),
|
|
1757
|
+
job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}"))
|
|
1758
|
+
raise _Ended("NonZeroAgentExitCodeError", f"{o['label']}: " + str(e),
|
|
1759
|
+
job.trajectory_from_exchanges([], calls, o, self.model, f"{self.name}/{unit}"))
|
|
1760
|
+
transcript.append({"role": "upstream", "setup": o["key"], "asked": self.asked, "sent": self.model})
|
|
1761
|
+
return out.get("answer"), job.trajectory_from_exchanges([], rl.calls(tag), o, self.model, f"{self.name}/{unit}")
|
|
1762
|
+
|
|
1763
|
+
|
|
1764
|
+
class _Ended(Exception):
|
|
1765
|
+
def __init__(self, kind, said, trajectory=None):
|
|
1766
|
+
super().__init__(said)
|
|
1767
|
+
self.kind, self.said, self.trajectory = kind, said, trajectory
|
|
1768
|
+
|
|
1769
|
+
|
|
1770
|
+
def run(bench: dict, other: dict, **kw) -> str:
|
|
1771
|
+
return Run(bench, other, **kw).start()
|
|
1772
|
+
|
|
1773
|
+
|
|
1774
|
+
def run_and_report(bench: dict, other: dict, **kw) -> tuple:
|
|
1775
|
+
"""(job directory, whether it failed): it stopped on a refusing endpoint,
|
|
1776
|
+
or units ran and none reached a verdict - what the CLI's exit status says.
|
|
1777
|
+
A run its spending cap stopped ended as it was asked to: no failure,
|
|
1778
|
+
whatever it reached."""
|
|
1779
|
+
r = Run(bench, other, **kw)
|
|
1780
|
+
job_dir = r.start()
|
|
1781
|
+
return job_dir, bool(r.halted) or (r.n_ended > 0 and r.n_verdicts == 0 and not r.capped)
|