nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Docker execution for benchmark setups, whose answerer lives in the driver.
|
|
2
|
+
|
|
3
|
+
Unlike an evaluated agent, this trusted process owns the reference data. It gets
|
|
4
|
+
only the selected benchmark, its upstream/data, an isolated Linux install cache,
|
|
5
|
+
the job workspace and backend Python code. Its network can fetch benchmark data;
|
|
6
|
+
model credentials remain in the relay. No host virtual environment is reused.
|
|
7
|
+
Its declared credentials come as an owner-only file in a key folder, mounted
|
|
8
|
+
read-only; :func:`main` loads them into the driver's environment, never the
|
|
9
|
+
container's, which ``docker inspect`` shows. The folder is removed once the
|
|
10
|
+
driver answers or ends.
|
|
11
|
+
"""
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import contextlib
|
|
15
|
+
import hashlib
|
|
16
|
+
import json
|
|
17
|
+
import os
|
|
18
|
+
import shutil
|
|
19
|
+
import sys
|
|
20
|
+
import uuid
|
|
21
|
+
from urllib.parse import urlsplit
|
|
22
|
+
|
|
23
|
+
from nullius.standard import container, definitions, hostreq, install, jsonl
|
|
24
|
+
|
|
25
|
+
UV_VERSION = "0.13.4"
|
|
26
|
+
CREDENTIALS_IN = "/opt/nullius/credentials" # the driver's credentials' one file, in its container
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def door_base(door):
|
|
30
|
+
# A rootless door is another container. The driver may already be running,
|
|
31
|
+
# so use its bridge address rather than changing the driver's hosts file.
|
|
32
|
+
return door.base.replace("host.docker.internal", door.hosts["host.docker.internal"], 1) if door.container else door.base
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def credential_names(bench: dict) -> set:
|
|
36
|
+
"""The variables the benchmark's credentials for its driver set
|
|
37
|
+
(``needs_secrets``, ``to: driver``): a value's or a server's name, a
|
|
38
|
+
file's ``env``."""
|
|
39
|
+
return {s.get("env") if s["kind"] == "file" else s["name"]
|
|
40
|
+
for s in definitions.secrets_of(bench) if s["to"] == "driver"} - {None}
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def image(base: str, bench: dict, needs: list, log: str) -> str:
|
|
44
|
+
text = f"FROM {base}\nCOPY --from=ghcr.io/astral-sh/uv:{UV_VERSION} /uv /uvx /usr/local/bin/\n"
|
|
45
|
+
commands = hostreq.install_lines(bench, needs, sudo="")
|
|
46
|
+
if commands:
|
|
47
|
+
text += "RUN apt-get update && " + " && ".join(c.strip() for c in commands) + " && rm -rf /var/lib/apt/lists/*\n"
|
|
48
|
+
tag = "nullius/setup:" + hashlib.sha256(text.encode()).hexdigest()[:12]
|
|
49
|
+
if not container.present(tag):
|
|
50
|
+
result = container._docker(["build", "-t", tag, "-"], input=text, timeout=1800, check=False)
|
|
51
|
+
with open(log, "a") as f:
|
|
52
|
+
f.write(result.stdout + result.stderr)
|
|
53
|
+
if result.returncode:
|
|
54
|
+
raise container.Unavailable(f"building {tag}: {(result.stderr or result.stdout)[-300:]}")
|
|
55
|
+
return tag
|
|
56
|
+
|
|
57
|
+
|
|
58
|
+
class Driver:
|
|
59
|
+
def __init__(self, bench, *, image, app, job, work, upstream, data, needs):
|
|
60
|
+
self.bench, self.image, self.app, self.job = bench, image, app, job
|
|
61
|
+
self.work, self.upstream, self.data = work, upstream, data
|
|
62
|
+
self.cache = os.path.join(install.HOME, "docker-setups", image.rsplit(":", 1)[-1])
|
|
63
|
+
os.makedirs(self.cache, exist_ok=True)
|
|
64
|
+
self.config = os.path.join(work, ".work", "setup-driver.json")
|
|
65
|
+
with open(self.config, "w") as f:
|
|
66
|
+
json.dump({"bench": bench, "upstream": upstream, "needs": needs}, f)
|
|
67
|
+
|
|
68
|
+
def command(self, env, cwd, *, prepare=False, files=(), credentials=None):
|
|
69
|
+
# Only the backend package, not app/ or any user configuration directory.
|
|
70
|
+
mounts = [(os.path.join(self.app, "nullius"), True), (self.bench["_here"], True),
|
|
71
|
+
(self.cache, False), (self.work, False)]
|
|
72
|
+
if self.upstream:
|
|
73
|
+
mounts.append((self.upstream, False)) # upstreams may cache data beside their code
|
|
74
|
+
if self.data and not any(self.data == p or self.data.startswith(p + os.sep) for p, _ in mounts):
|
|
75
|
+
mounts.append((self.data, True))
|
|
76
|
+
mounts += [(p, True) for p in dict.fromkeys(files) if p not in {m[0] for m in mounts}]
|
|
77
|
+
values = {k: v for k, v in env.items() if k not in container.HOST_ONLY}
|
|
78
|
+
values.update(PATH=container.IMAGE_PATH, PYTHONPATH=self.app, PYTHONDONTWRITEBYTECODE="1",
|
|
79
|
+
UV_CACHE_DIR=os.path.join(self.cache, "uv-cache"),
|
|
80
|
+
UV_PYTHON_INSTALL_DIR=os.path.join(self.cache, "python"))
|
|
81
|
+
name = "nullius-setup-" + uuid.uuid4().hex[:20]
|
|
82
|
+
cmd = ["python3", "-m", "nullius.standard.setup_container", self.config, self.cache,
|
|
83
|
+
"prepare" if prepare else "run"] + ([CREDENTIALS_IN] if credentials else [])
|
|
84
|
+
argv = container.run_argv(self.image, cmd, name=name, network=container.BRIDGE if container.podman() else "bridge", job=self.job,
|
|
85
|
+
workdir=cwd, mounted=mounts, env=values, path=container.IMAGE_PATH)
|
|
86
|
+
argv[3:3] = ["--add-host", "host.docker.internal:host-gateway"]
|
|
87
|
+
if credentials: # as the relay's key: fails when the engine cannot see it
|
|
88
|
+
argv[3:3] = ["--mount", f"type=bind,source={credentials},target={CREDENTIALS_IN},readonly"]
|
|
89
|
+
return argv, container.program_env(values), name
|
|
90
|
+
|
|
91
|
+
def prepare(self):
|
|
92
|
+
container.make_bridge()
|
|
93
|
+
home = os.path.join(self.work, ".work", "setup-install-home")
|
|
94
|
+
os.makedirs(home, exist_ok=True)
|
|
95
|
+
argv, env, name = self.command({"HOME": home}, self.work, prepare=True)
|
|
96
|
+
try:
|
|
97
|
+
result = container._docker(argv[1:], env=env, timeout=1800, check=False)
|
|
98
|
+
finally:
|
|
99
|
+
with contextlib.suppress(container.Unavailable):
|
|
100
|
+
container._docker(["rm", "-f", name], check=False)
|
|
101
|
+
with open(os.path.join(self.work, ".work", "setup-install.log"), "a") as f:
|
|
102
|
+
f.write(result.stdout + result.stderr)
|
|
103
|
+
if result.returncode:
|
|
104
|
+
raise install.InstallFailed("container driver install: " + (result.stderr or result.stdout)[-500:])
|
|
105
|
+
|
|
106
|
+
def start(self, env, cwd, log, *, files=(), servers=(), stopped=None):
|
|
107
|
+
# Embedding routes are environment variables; their forwarders live as
|
|
108
|
+
# long as the process. Per-request answer/judge/simulator routes below
|
|
109
|
+
# live only until that protocol reply returns.
|
|
110
|
+
doors, keys, credentials = [], None, None
|
|
111
|
+
env = dict(env)
|
|
112
|
+
try:
|
|
113
|
+
for key, value in env.items():
|
|
114
|
+
target = "tcp://" + value if key in servers else value
|
|
115
|
+
if (key.endswith("BASE_URL") or key in servers) and container.on_loopback(target):
|
|
116
|
+
door = container.Loopback(target, image=self.image, job=f"{self.job}-{uuid.uuid4().hex[:12]}").start()
|
|
117
|
+
doors.append(door)
|
|
118
|
+
env[key] = urlsplit(door_base(door)).netloc if key in servers else door_base(door)
|
|
119
|
+
held = {k: env.pop(k) for k in sorted(credential_names(self.bench)) if k in env}
|
|
120
|
+
if held:
|
|
121
|
+
keys = container.key_folder(self.job)
|
|
122
|
+
credentials = container.secret_file(os.path.join(keys, "credentials"), json.dumps(held))
|
|
123
|
+
argv, client, name = self.command(env, cwd, files=files, credentials=credentials)
|
|
124
|
+
process = Process(argv, cwd, client, log, stopped=stopped, name=name, image=self.image, doors=doors, keys=keys)
|
|
125
|
+
try:
|
|
126
|
+
return process.start()
|
|
127
|
+
except BaseException:
|
|
128
|
+
process.close()
|
|
129
|
+
raise
|
|
130
|
+
except BaseException:
|
|
131
|
+
for door in doors:
|
|
132
|
+
door.stop()
|
|
133
|
+
if keys:
|
|
134
|
+
shutil.rmtree(keys, ignore_errors=True)
|
|
135
|
+
raise
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
class Process(jsonl.Proc):
|
|
139
|
+
def __init__(self, *args, name, doors, image="", keys=None, **kwargs):
|
|
140
|
+
super().__init__(*args, **kwargs)
|
|
141
|
+
self.name, self.doors, self.image = name, doors, image
|
|
142
|
+
self.keys = keys # the credentials' folder, until the driver answers or ends
|
|
143
|
+
|
|
144
|
+
def forget(self):
|
|
145
|
+
keys, self.keys = self.keys, None
|
|
146
|
+
if keys:
|
|
147
|
+
shutil.rmtree(keys, ignore_errors=True)
|
|
148
|
+
|
|
149
|
+
def request(self, op, timeout=600, **kw):
|
|
150
|
+
doors = []
|
|
151
|
+
def rewrite(value):
|
|
152
|
+
if isinstance(value, dict):
|
|
153
|
+
return {k: url(v) if k == "base_url" and isinstance(v, str) else rewrite(v) for k, v in value.items()}
|
|
154
|
+
if isinstance(value, list):
|
|
155
|
+
return [rewrite(v) for v in value]
|
|
156
|
+
return value
|
|
157
|
+
def url(value):
|
|
158
|
+
if not container.on_loopback(value):
|
|
159
|
+
return value
|
|
160
|
+
door = container.Loopback(value, image=self.image, job=f"{self.name}-{len(doors)}").start()
|
|
161
|
+
doors.append(door)
|
|
162
|
+
return door_base(door)
|
|
163
|
+
try:
|
|
164
|
+
return super().request(op, timeout=timeout, **rewrite(kw))
|
|
165
|
+
finally:
|
|
166
|
+
for door in doors:
|
|
167
|
+
door.stop()
|
|
168
|
+
self.forget() # answered or ended: main() read them, or never will
|
|
169
|
+
|
|
170
|
+
def close(self):
|
|
171
|
+
try:
|
|
172
|
+
super().close()
|
|
173
|
+
finally:
|
|
174
|
+
self.forget()
|
|
175
|
+
with contextlib.suppress(container.Unavailable):
|
|
176
|
+
container._docker(["rm", "-f", self.name], check=False)
|
|
177
|
+
for door in self.doors:
|
|
178
|
+
door.stop()
|
|
179
|
+
self.doors = []
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def main():
|
|
183
|
+
config, cache, mode, *credentials = sys.argv[1:]
|
|
184
|
+
with open(config) as f:
|
|
185
|
+
record = json.load(f)
|
|
186
|
+
bench = record["bench"]
|
|
187
|
+
definition = definitions.driver_install(bench)
|
|
188
|
+
where = install.packages_dir(definition, cache) if definition["install"] else None
|
|
189
|
+
if mode == "prepare":
|
|
190
|
+
missing = hostreq.missing(bench, record["needs"])
|
|
191
|
+
if missing:
|
|
192
|
+
raise install.InstallFailed(f"setup container lacks required programs: {missing}")
|
|
193
|
+
if where:
|
|
194
|
+
install.packages(definition, cache, src=record["upstream"])
|
|
195
|
+
return
|
|
196
|
+
if where and not install.ready(where):
|
|
197
|
+
raise install.InstallFailed("setup container's driver environment has not been prepared")
|
|
198
|
+
env = dict(os.environ)
|
|
199
|
+
for path in credentials: # a JSON object: no shell reads a value
|
|
200
|
+
with open(path) as f:
|
|
201
|
+
env.update(json.load(f))
|
|
202
|
+
command, extra = install.driver_command(bench, where, cache)
|
|
203
|
+
env.update(extra)
|
|
204
|
+
os.execvpe(command[0], command, env)
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
if __name__ == "__main__":
|
|
208
|
+
main()
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
"""One run over many machines: shards of a dataset's units, and the launch they share.
|
|
2
|
+
|
|
3
|
+
``--shard I/N`` runs the units whose shard is ``I`` of ``N``. A unit's shard
|
|
4
|
+
is read off its id, not its position: ``sha256(id) mod N`` (``PARTITION``).
|
|
5
|
+
So every machine computes the same split from the ids alone, with no list to
|
|
6
|
+
hand around, and a dataset that gains units leaves every unit it had in the
|
|
7
|
+
shard it was in - only the new ones land somewhere. Changing ``N`` moves
|
|
8
|
+
most units; a launch keeps one ``N``.
|
|
9
|
+
|
|
10
|
+
The order inside a run is: the driver's ``units``, then the shard, then
|
|
11
|
+
``--limit`` - so ``--shard 1/2 --limit 6`` is the first six units of shard 1,
|
|
12
|
+
and the two shards of a ``--limit 6`` launch run up to twelve units between
|
|
13
|
+
them. The limit is a shard's own: a machine is told how much it takes on.
|
|
14
|
+
|
|
15
|
+
A trial keeps its unit's place in the whole dataset as its tag (``u0042`` is
|
|
16
|
+
the 43rd unit in every shard), so the shards' trials never share a name and
|
|
17
|
+
merge without renaming (``merge``).
|
|
18
|
+
|
|
19
|
+
Every shard of one launch records the same ``launch`` id: ``--launch``, or by
|
|
20
|
+
default the day (UTC), the benchmark, the agent or capability and the model, so
|
|
21
|
+
shards started separately - on other machines, minutes apart - still say
|
|
22
|
+
they belong together. Two launches of the same pair on the same day need
|
|
23
|
+
``--launch`` to stay apart.
|
|
24
|
+
"""
|
|
25
|
+
from __future__ import annotations
|
|
26
|
+
|
|
27
|
+
import hashlib
|
|
28
|
+
import re
|
|
29
|
+
import time
|
|
30
|
+
from typing import List, Optional, Tuple
|
|
31
|
+
|
|
32
|
+
PARTITION = "sha256(unit id) mod N"
|
|
33
|
+
_SPEC = re.compile(r"^\s*(\d+)\s*/\s*(\d+)\s*$")
|
|
34
|
+
_WORD = re.compile(r"[^A-Za-z0-9_.-]+")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def parse(spec: str) -> Tuple[int, int]:
|
|
38
|
+
"""``"I/N"`` as (I, N), 1 <= I <= N; ValueError otherwise."""
|
|
39
|
+
m = _SPEC.match(str(spec or ""))
|
|
40
|
+
if not m:
|
|
41
|
+
raise ValueError(f"--shard {spec!r}: I/N, such as 1/4")
|
|
42
|
+
i, n = int(m.group(1)), int(m.group(2))
|
|
43
|
+
if n < 1 or not 1 <= i <= n:
|
|
44
|
+
raise ValueError(f"--shard {spec!r}: I is from 1 to N")
|
|
45
|
+
return i, n
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def of(unit: str, n: int) -> int:
|
|
49
|
+
"""The shard (1..n) a unit falls in: by its id alone."""
|
|
50
|
+
return int.from_bytes(hashlib.sha256(str(unit).encode("utf-8")).digest()[:8], "big") % n + 1
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def select(units: List[str], index: int, n: int) -> List[Tuple[int, str]]:
|
|
54
|
+
"""The units of shard ``index`` of ``n``, each with its place in the whole
|
|
55
|
+
list (what its trial's tag is made of), in the dataset's order."""
|
|
56
|
+
return [(i, u) for i, u in enumerate(units) if n == 1 or of(u, n) == index]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def launch_id(bench_word: str, other_word: str, model: str, day: Optional[str] = None) -> str:
|
|
60
|
+
"""The default launch id shards started separately agree on."""
|
|
61
|
+
day = day or time.strftime("%Y-%m-%d", time.gmtime())
|
|
62
|
+
words = [day, bench_word, other_word] + ([model] if model else [])
|
|
63
|
+
return "__".join(_WORD.sub("_", w).strip("_") for w in words)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def record(index: int, n: int, total: int, planned: List[str], limit: Optional[int] = None) -> dict:
|
|
67
|
+
"""What a shard's config says of it (``standard.shard``): which shard, of
|
|
68
|
+
how many, over how many units, split how; the units it planned to run, so
|
|
69
|
+
a merge can name a unit planned and never run; and its own ``limit``."""
|
|
70
|
+
return {"index": index, "of": n, "units_total": total, "partition": PARTITION, "planned": list(planned),
|
|
71
|
+
**({"limit": limit} if limit else {})}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
"""HTTP fixture doors declared by a driver's setup, without injecting agent tools.
|
|
2
|
+
|
|
3
|
+
Each service is an existing Unix socket owned by the driver. Docker mounts only
|
|
4
|
+
that socket into a forwarder on the unit's internal network. A local run uses
|
|
5
|
+
the loopback URL supplied by the driver in its prompt.
|
|
6
|
+
"""
|
|
7
|
+
import os
|
|
8
|
+
import re
|
|
9
|
+
import stat
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def doors(setup):
|
|
13
|
+
out = []
|
|
14
|
+
services = setup.get('web_services', [])
|
|
15
|
+
if not isinstance(services, list):
|
|
16
|
+
raise ValueError('web_services must be a list')
|
|
17
|
+
for entry in services:
|
|
18
|
+
if not isinstance(entry, dict) or set(entry) != {'name', 'port', 'socket'}:
|
|
19
|
+
raise ValueError('web_services entries need name, port and socket')
|
|
20
|
+
name, port, path = entry['name'], entry['port'], entry['socket']
|
|
21
|
+
if not isinstance(name, str) or not re.fullmatch(r'web-[a-z0-9-]{1,40}', name):
|
|
22
|
+
raise ValueError('web service names must start with web-')
|
|
23
|
+
if type(port) is not int or not 1024 <= port <= 65535:
|
|
24
|
+
raise ValueError('web service port must be between 1024 and 65535')
|
|
25
|
+
if not isinstance(path, str) or not os.path.isabs(path) or not stat.S_ISSOCK(os.lstat(path).st_mode):
|
|
26
|
+
raise ValueError('web service socket must be an existing absolute Unix socket, not a link')
|
|
27
|
+
if name in [n for n, _, _ in out]:
|
|
28
|
+
raise ValueError('duplicate web service name')
|
|
29
|
+
out.append((name, port, path))
|
|
30
|
+
return out
|
|
File without changes
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
"""Behavioral metrics over a parsed trajectory.
|
|
2
|
+
|
|
3
|
+
The outcome says *whether* the agent solved the task; these numbers say *how*
|
|
4
|
+
it worked — how long before it committed to an edit, whether it re-read what
|
|
5
|
+
it had already seen, whether it re-ran commands that had already failed. Two
|
|
6
|
+
resolved trials can hide very different amounts of wasted work, and the waste
|
|
7
|
+
is what token cost scales with.
|
|
8
|
+
|
|
9
|
+
Everything here is a pure function of data the app already loads; nothing
|
|
10
|
+
reaches back into the job directory.
|
|
11
|
+
"""
|
|
12
|
+
|
|
13
|
+
from __future__ import annotations
|
|
14
|
+
|
|
15
|
+
from dataclasses import dataclass
|
|
16
|
+
|
|
17
|
+
# Public in trajectory.py so this module need not reach into an underscore.
|
|
18
|
+
from nullius.traces.trajectory import PATCH_PATHS as _PATCH_PATHS
|
|
19
|
+
from nullius.traces.trajectory import parse_ts as _parse_ts
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
@dataclass
|
|
23
|
+
class Behavior:
|
|
24
|
+
# 1-based position of the first edit among the run's tool calls, and the
|
|
25
|
+
# wall-clock seconds it took to get there (None without a step clock).
|
|
26
|
+
first_edit_call: int | None = None
|
|
27
|
+
time_to_first_edit: float | None = None
|
|
28
|
+
# read/search calls whose exact target had already been fetched before.
|
|
29
|
+
n_redundant_reads: int = 0
|
|
30
|
+
# calls repeating a (name, target) pair that had already failed — retrying
|
|
31
|
+
# the same thing and hoping is the signature of a flailing run.
|
|
32
|
+
n_repeated_failed_calls: int = 0
|
|
33
|
+
# completed edit→execute alternations: how many times an edit was actually
|
|
34
|
+
# followed by running something (tests, the repro, anything).
|
|
35
|
+
edit_test_cycles: int = 0
|
|
36
|
+
n_search: int = 0
|
|
37
|
+
n_read: int = 0
|
|
38
|
+
n_edit: int = 0
|
|
39
|
+
# (search + read) per edit — how much looking preceded each change.
|
|
40
|
+
exploration_per_edit: float | None = None
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def analyze(traj) -> Behavior:
|
|
44
|
+
"""Compute a Behavior from a loaded Trajectory."""
|
|
45
|
+
b = Behavior()
|
|
46
|
+
|
|
47
|
+
seen_fetches: set = set()
|
|
48
|
+
failed_keys: set = set()
|
|
49
|
+
edit_pending = False
|
|
50
|
+
first_edit_step_id = None
|
|
51
|
+
|
|
52
|
+
for i, call in enumerate(traj.calls, start=1):
|
|
53
|
+
cat = call["category"]
|
|
54
|
+
target = call["target"]
|
|
55
|
+
key = (call["name"], target)
|
|
56
|
+
|
|
57
|
+
if cat == "search":
|
|
58
|
+
b.n_search += 1
|
|
59
|
+
elif cat == "read":
|
|
60
|
+
b.n_read += 1
|
|
61
|
+
elif cat == "edit":
|
|
62
|
+
b.n_edit += 1
|
|
63
|
+
|
|
64
|
+
if cat in ("read", "search") and target:
|
|
65
|
+
fetch = (cat, target)
|
|
66
|
+
if fetch in seen_fetches:
|
|
67
|
+
b.n_redundant_reads += 1
|
|
68
|
+
seen_fetches.add(fetch)
|
|
69
|
+
|
|
70
|
+
if key in failed_keys:
|
|
71
|
+
b.n_repeated_failed_calls += 1
|
|
72
|
+
if call["failure"]:
|
|
73
|
+
failed_keys.add(key)
|
|
74
|
+
|
|
75
|
+
if cat == "edit":
|
|
76
|
+
edit_pending = True
|
|
77
|
+
if b.first_edit_call is None:
|
|
78
|
+
b.first_edit_call = i
|
|
79
|
+
first_edit_step_id = call["step_id"]
|
|
80
|
+
elif cat == "execute" and edit_pending:
|
|
81
|
+
b.edit_test_cycles += 1
|
|
82
|
+
edit_pending = False
|
|
83
|
+
|
|
84
|
+
if b.n_edit:
|
|
85
|
+
b.exploration_per_edit = round((b.n_search + b.n_read) / b.n_edit, 2)
|
|
86
|
+
|
|
87
|
+
if traj.has_step_clock and first_edit_step_id is not None:
|
|
88
|
+
start = next((t for t in (_parse_ts(s.timestamp) for s in traj.steps) if t), None)
|
|
89
|
+
at_edit = next(
|
|
90
|
+
(_parse_ts(s.timestamp) for s in traj.steps if s.step_id == first_edit_step_id),
|
|
91
|
+
None,
|
|
92
|
+
)
|
|
93
|
+
if start and at_edit and at_edit >= start:
|
|
94
|
+
b.time_to_first_edit = (at_edit - start).total_seconds()
|
|
95
|
+
|
|
96
|
+
return b
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def gold_files(solve_text: str) -> list:
|
|
100
|
+
"""File paths named in a task's gold patch (``solution/solve.sh``).
|
|
101
|
+
|
|
102
|
+
A solve script that embeds the gold diff in a heredoc is read with the
|
|
103
|
+
diff-header patterns used on agent patch payloads, unchanged.
|
|
104
|
+
"""
|
|
105
|
+
found = []
|
|
106
|
+
for pattern in _PATCH_PATHS:
|
|
107
|
+
for path in pattern.findall(solve_text or ""):
|
|
108
|
+
if path not in found and path != "/dev/null":
|
|
109
|
+
found.append(path)
|
|
110
|
+
return found
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
@dataclass
|
|
114
|
+
class Localization:
|
|
115
|
+
hit: list # gold files the agent edited
|
|
116
|
+
missed: list # gold files it never touched
|
|
117
|
+
extra: list # files it edited that the gold patch doesn't name
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def _same_file(a: str, b: str) -> bool:
|
|
121
|
+
# Agent paths may be container-absolute (/testbed/sklearn/...), gold paths
|
|
122
|
+
# repo-relative (sklearn/...). Suffix match on a path boundary covers both.
|
|
123
|
+
return a == b or a.endswith("/" + b) or b.endswith("/" + a)
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def localization(touched, gold) -> Localization:
|
|
127
|
+
"""Compare the files the agent edited against the gold patch's files."""
|
|
128
|
+
hit = [g for g in gold if any(_same_file(t, g) for t in touched)]
|
|
129
|
+
missed = [g for g in gold if g not in hit]
|
|
130
|
+
extra = [t for t in touched if not any(_same_file(t, g) for g in gold)]
|
|
131
|
+
return Localization(hit=hit, missed=missed, extra=extra)
|