nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,428 @@
|
|
|
1
|
+
"""Where what a run needs is installed, and installing it once.
|
|
2
|
+
|
|
3
|
+
Everything goes under ``STANDARD_DIR`` (``standard/`` in the persistent user data directory):
|
|
4
|
+
|
|
5
|
+
- ``agents/<key>/<digest>/`` and ``capabilities/<key>/<digest>/``: the packages a
|
|
6
|
+
definition's ``install`` names (``npm`` into ``node_modules``, ``pip`` and
|
|
7
|
+
``wheels`` - a wheel by its URL, checked by its sha256 - into a ``uv`` environment), the digest taken over that list so a re-pin installs
|
|
8
|
+
beside the old one rather than over it;
|
|
9
|
+
- ``drivers/<key>/<digest>/``: a benchmark's ``driver.install`` the same way -
|
|
10
|
+
the environment its upstream's own code needs, which its driver imports
|
|
11
|
+
(``driver_argv``), never this application's and never an agent's. Its
|
|
12
|
+
``project`` is installed from the benchmark's upstream clone (below), the
|
|
13
|
+
same commit, rather than fetched a second time;
|
|
14
|
+
- ``upstreams/<key>/<commit>/``: a benchmark's upstream repository at its
|
|
15
|
+
tested commit;
|
|
16
|
+
- ``datasets/<sha256>/<file>``: a dataset or auxiliary file served outside its upstream, under
|
|
17
|
+
``STANDARD_DATA_DIR`` instead when that is set - datasets can be gigabytes
|
|
18
|
+
where installs are not, so they may sit on a bigger disk than the installs.
|
|
19
|
+
|
|
20
|
+
On the container backend an agent's or a capability's packages are installed in
|
|
21
|
+
the base image (``container.install_argv``), and the base image's tag is part
|
|
22
|
+
of their folder's digest. A driver's packages, and all packages on the local
|
|
23
|
+
backend, are installed with this machine's ``npm`` and ``uv``. Git always runs
|
|
24
|
+
on this machine.
|
|
25
|
+
|
|
26
|
+
A folder is complete when its ``.installed`` marker is; a failed install
|
|
27
|
+
leaves no marker and is tried again from scratch. Two runs installing the same
|
|
28
|
+
thing at once take turns (a lock file beside the folder): the second finds the
|
|
29
|
+
first one's marker rather than removing its half-made folder. A dataset file is checked
|
|
30
|
+
against its sha256 before a run reads it.
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import contextlib
|
|
35
|
+
import fcntl
|
|
36
|
+
import hashlib
|
|
37
|
+
import json
|
|
38
|
+
import os
|
|
39
|
+
import re
|
|
40
|
+
import shutil
|
|
41
|
+
import subprocess
|
|
42
|
+
from typing import Callable, Dict, List, Optional
|
|
43
|
+
|
|
44
|
+
from nullius.standard import definitions
|
|
45
|
+
|
|
46
|
+
from nullius import paths
|
|
47
|
+
|
|
48
|
+
HOME = os.environ.get("STANDARD_DIR") or os.path.join(paths.data_dir(), "standard")
|
|
49
|
+
MARK = ".installed"
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def data_dir(home: str = HOME, env=os.environ) -> str:
|
|
53
|
+
"""Where downloaded datasets are kept: ``STANDARD_DATA_DIR``, else
|
|
54
|
+
``datasets/`` in the standard's home. A dataset already downloaded
|
|
55
|
+
elsewhere is not moved: setting the variable downloads it once more."""
|
|
56
|
+
return env.get("STANDARD_DATA_DIR") or os.path.join(home, "datasets")
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
class InstallFailed(Exception):
|
|
60
|
+
pass
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
# A variable an install step is never given: a package's install script runs
|
|
64
|
+
# as published, and a run's process holds the model key (under --from-run as
|
|
65
|
+
# OPENAI_API_KEY, API_TOKEN, LLM_API_KEY) - the relay is the only holder meant.
|
|
66
|
+
SECRET = re.compile(r"(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?)$", re.I) # the last word: GIT_CONFIG_KEY_0 is git's own
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def install_env(env=None) -> dict:
|
|
70
|
+
"""This process's environment less every variable named like a secret."""
|
|
71
|
+
return {k: v for k, v in (os.environ if env is None else env).items() if not SECRET.search(k)}
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def git_env() -> dict:
|
|
75
|
+
"""What a fetch of an upstream runs with: the installer's environment and
|
|
76
|
+
no prompt. A private repository - a fork of the user's - is fetched with
|
|
77
|
+
this machine's own git credentials (its credential helper, its ssh agent,
|
|
78
|
+
in its HOME); without them the fetch fails, said, rather than waiting on a
|
|
79
|
+
terminal for a password."""
|
|
80
|
+
return dict(install_env(), GIT_TERMINAL_PROMPT="0")
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def _run(cmd: List[str], cwd: str, log: str, env: Optional[dict] = None, shown: Optional[List[str]] = None):
|
|
84
|
+
"""Run one install step, its output to ``log``. ``shown``: the step as a
|
|
85
|
+
failure names it, when ``cmd`` wraps it (``docker run ...``)."""
|
|
86
|
+
with open(log, "a") as f:
|
|
87
|
+
f.write("$ " + " ".join(cmd) + "\n")
|
|
88
|
+
f.flush()
|
|
89
|
+
try:
|
|
90
|
+
# uv may download its Python over a system setting against it (Fedora's), unless this environment says otherwise
|
|
91
|
+
env = {"UV_PYTHON_DOWNLOADS": "automatic", **(install_env() if env is None else env)}
|
|
92
|
+
r = subprocess.run(cmd, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, env=env)
|
|
93
|
+
except FileNotFoundError:
|
|
94
|
+
raise InstallFailed(f"{cmd[0]} is not on this machine")
|
|
95
|
+
if r.returncode != 0:
|
|
96
|
+
with open(log, errors="replace") as f:
|
|
97
|
+
tail = f.read().strip().splitlines()[-3:]
|
|
98
|
+
raise InstallFailed(f"{' '.join((shown or cmd)[:3])}...: exit {r.returncode}: {' | '.join(tail)[-300:]}")
|
|
99
|
+
|
|
100
|
+
|
|
101
|
+
@contextlib.contextmanager
|
|
102
|
+
def _locked(where: str):
|
|
103
|
+
os.makedirs(os.path.dirname(where), exist_ok=True)
|
|
104
|
+
with open(where + ".lock", "w") as f:
|
|
105
|
+
fcntl.flock(f, fcntl.LOCK_EX)
|
|
106
|
+
try:
|
|
107
|
+
yield
|
|
108
|
+
finally:
|
|
109
|
+
fcntl.flock(f, fcntl.LOCK_UN)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _digest(obj) -> str:
|
|
113
|
+
return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest()[:12]
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
def packages(d: dict, home: str = HOME, run_in: Optional[Callable[..., List[str]]] = None,
|
|
117
|
+
src: Optional[str] = None, image: Optional[str] = None) -> str:
|
|
118
|
+
"""The folder an agent's or a capability's packages are installed in,
|
|
119
|
+
installing them first when they are not. A definition installing nothing
|
|
120
|
+
gets an empty folder all the same, so ``{install}`` always names one.
|
|
121
|
+
|
|
122
|
+
``run_in``: where every ``npm``, ``uv`` and Python step runs instead of
|
|
123
|
+
this machine: the container backend's base image, which carries Node and uv
|
|
124
|
+
(``container.install_argv``). Given the command, the folder and the
|
|
125
|
+
step's own variables, it returns the argument list that runs it there.
|
|
126
|
+
The install is then made on the libc and the tools the agent's container
|
|
127
|
+
runs on, uv's Python inside the folder (``.python``). Git stays on this
|
|
128
|
+
machine, with its credentials. ``image``: that base image's tag, part of
|
|
129
|
+
the folder's digest, so an install made in it never serves a run on this
|
|
130
|
+
machine. Without ``run_in``, a machine with no ``npm`` is refused, said.
|
|
131
|
+
|
|
132
|
+
``src``: a checkout of the definition's upstream at ``upstream.tested``
|
|
133
|
+
already on this machine, which ``project`` is installed from instead of
|
|
134
|
+
a clone of its own (a benchmark's driver: its upstream clone)."""
|
|
135
|
+
assert (run_in is None) == (image is None), "an install in the base image is named by its tag"
|
|
136
|
+
inst = d.get("install") or {}
|
|
137
|
+
up = (d.get("upstream") or {}) if inst.get("project") else {}
|
|
138
|
+
where = packages_dir(d, home, image)
|
|
139
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
140
|
+
return where
|
|
141
|
+
with _locked(where):
|
|
142
|
+
return _packages(inst, where, up, run_in, src)
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
def packages_dir(d: dict, home: str = HOME, image: Optional[str] = None) -> str:
|
|
146
|
+
"""Where ``packages`` installs a definition's packages, installed or not."""
|
|
147
|
+
inst = d.get("install") or {}
|
|
148
|
+
kind = {"agent": "agents", "driver": "drivers"}.get(d.get("_kind"), "capabilities")
|
|
149
|
+
up = (d.get("upstream") or {}) if inst.get("project") else {}
|
|
150
|
+
key = dict(inst, _at=up.get("tested")) if up else inst
|
|
151
|
+
return os.path.join(home, kind, d["key"], _digest(dict(key, _image=image) if image else key))
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def ready(where: Optional[str]) -> bool:
|
|
155
|
+
"""Whether an install folder is complete (its marker is there): a run says
|
|
156
|
+
the stage was there already rather than that it took no time."""
|
|
157
|
+
return bool(where) and os.path.exists(os.path.join(where, MARK))
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
HF = "hf" # inside an install: the models prefetched from the hub (HF_HUB_CACHE at run time)
|
|
161
|
+
WHEELS = "wheels" # inside an install: the wheels it names by URL, as downloaded
|
|
162
|
+
SRC = "src" # inside an install: the definition's upstream, at its tested commit, for install.project
|
|
163
|
+
SCRATCH = (".home", ".npm-cache", ".uv-cache") # what an install in the base image keeps of its own (container.install_argv)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def remove_scratch(where: str) -> None:
|
|
167
|
+
for scratch in SCRATCH:
|
|
168
|
+
shutil.rmtree(os.path.join(where, scratch), ignore_errors=True)
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _packages(inst: dict, where: str, up: Optional[dict] = None, run_in: Optional[Callable] = None,
|
|
172
|
+
src: Optional[str] = None) -> str:
|
|
173
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
174
|
+
return where # another run finished it while this one waited
|
|
175
|
+
if os.path.isdir(where):
|
|
176
|
+
shutil.rmtree(where)
|
|
177
|
+
os.makedirs(where)
|
|
178
|
+
log = os.path.join(where, "install.log")
|
|
179
|
+
if run_in:
|
|
180
|
+
step = lambda cmd, env=None: _run(run_in(cmd, where, env), where, log, shown=cmd)
|
|
181
|
+
else:
|
|
182
|
+
step = lambda cmd, env=None: _run(cmd, where, log, env=dict(install_env(), **(env or {})))
|
|
183
|
+
try:
|
|
184
|
+
_steps(inst, where, up, src, log, step, boxed=run_in is not None)
|
|
185
|
+
finally:
|
|
186
|
+
remove_scratch(where)
|
|
187
|
+
with open(os.path.join(where, MARK), "w") as f:
|
|
188
|
+
json.dump(inst, f)
|
|
189
|
+
return where
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
def _steps(inst: dict, where: str, up: Optional[dict], src: Optional[str], log: str, step: Callable, boxed: bool):
|
|
193
|
+
if inst.get("npm"):
|
|
194
|
+
if not boxed and not shutil.which("npm"):
|
|
195
|
+
raise InstallFailed("npm is not on this machine: install Node.js 22 here, or run on the container backend, "
|
|
196
|
+
"which installs with the Node in its base image")
|
|
197
|
+
step(["npm", "install", "--no-audit", "--no-fund", "--prefix", where] + list(inst["npm"]))
|
|
198
|
+
if inst.get("pip") or inst.get("project") or inst.get("indexes") or inst.get("wheels"):
|
|
199
|
+
py = os.path.join(where, ".venv", "bin", "python")
|
|
200
|
+
# uv's own interpreter, never the system's: on the container backend in the install (.python), mounted with it; /usr/bin is the image's
|
|
201
|
+
step(["uv", "venv", "--quiet", "--python-preference", "only-managed", os.path.join(where, ".venv")]
|
|
202
|
+
+ (["--python", inst["python"]] if inst.get("python") else []))
|
|
203
|
+
for x in inst.get("indexes") or []: # first: the project's resolution then keeps them
|
|
204
|
+
step(["uv", "pip", "install", "--quiet", "--python", py, "--index-url", x["url"]] + list(x["packages"]))
|
|
205
|
+
wheels = [] # each checked by its sha256 before it is installed
|
|
206
|
+
for x in inst.get("wheels") or []:
|
|
207
|
+
os.makedirs(os.path.join(where, WHEELS), exist_ok=True)
|
|
208
|
+
wheels.append(os.path.join(where, WHEELS, x["url"].rsplit("/", 1)[1]))
|
|
209
|
+
_fetch(x["url"], x["sha256"], wheels[-1], "wheel")
|
|
210
|
+
locked = bool(inst.get("project") and inst.get("locked"))
|
|
211
|
+
if (inst.get("pip") or wheels) and not locked:
|
|
212
|
+
# resolved together: a wheel's dependencies are pip's pins
|
|
213
|
+
step(["uv", "pip", "install", "--quiet", "--python", py] + list(inst.get("pip") or []) + wheels)
|
|
214
|
+
if inst.get("project"):
|
|
215
|
+
if src is None:
|
|
216
|
+
src = os.path.join(where, SRC)
|
|
217
|
+
os.makedirs(src)
|
|
218
|
+
for cmd in (["git", "init", "--quiet"], ["git", "fetch", "--quiet", "--depth", "1", up["repository"], up["tested"]],
|
|
219
|
+
["git", "checkout", "--quiet", "FETCH_HEAD"]):
|
|
220
|
+
_run(cmd, src, log, env=git_env()) # on this machine, with its git credentials
|
|
221
|
+
proj = os.path.normpath(os.path.join(src, inst["project"]))
|
|
222
|
+
extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else ""
|
|
223
|
+
if locked:
|
|
224
|
+
# the project's own lock file, its pins as published, and pip's packages
|
|
225
|
+
# resolved beside them in one go: a conflict fails the install, said
|
|
226
|
+
lock = os.path.join(where, "requirements.lock")
|
|
227
|
+
step(["uv", "export", "--frozen", "--no-dev", "--no-emit-project", "--no-hashes", "--quiet",
|
|
228
|
+
"--project", proj, "-o", lock] + [w for x in inst.get("extras") or [] for w in ("--extra", x)])
|
|
229
|
+
step(["uv", "pip", "install", "--quiet", "--python", py, "-r", lock] + list(inst.get("pip") or []) + wheels)
|
|
230
|
+
step(["uv", "pip", "install", "--quiet", "--python", py, "--no-deps", "-e", proj + extras])
|
|
231
|
+
else:
|
|
232
|
+
step(["uv", "pip", "install", "--quiet", "--python", py, "-e", proj + extras])
|
|
233
|
+
if inst.get("prefetch"):
|
|
234
|
+
step([py, "-c", "import sys\nfrom huggingface_hub import snapshot_download\nfor m in sys.argv[1:]: snapshot_download(m)"]
|
|
235
|
+
+ list(inst["prefetch"]), env={"HF_HUB_CACHE": os.path.join(where, HF)})
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def process_env(d: dict, where: str, home: str, options: Optional[dict] = None) -> dict:
|
|
239
|
+
"""What an install adds to its process's environment: the definition's own
|
|
240
|
+
``env`` rendered (``{opt_<name>}`` from ``options``, the run's values), and
|
|
241
|
+
``HF_HUB_CACHE`` when it prefetched models."""
|
|
242
|
+
slots = {"install": where, "home": home, "here": d.get("_here", ""),
|
|
243
|
+
**{"opt_" + k: v for k, v in (options or {}).items()}}
|
|
244
|
+
out = {}
|
|
245
|
+
if (d.get("install") or {}).get("prefetch"):
|
|
246
|
+
out["HF_HUB_CACHE"] = os.path.join(where, HF)
|
|
247
|
+
for k, v in (d.get("env") or {}).items():
|
|
248
|
+
for s, val in slots.items():
|
|
249
|
+
v = v.replace("{" + s + "}", val)
|
|
250
|
+
out[k] = v
|
|
251
|
+
return out
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
def prune(d: dict, home: str) -> List[str]:
|
|
255
|
+
"""Remove what the definition declares as ``caches`` from a unit's home,
|
|
256
|
+
once the unit has ended and its trial is written: the paths it removed.
|
|
257
|
+
A path is resolved under ``home`` and refused when it would leave it
|
|
258
|
+
(a symlink the agent made included); the definition's check already
|
|
259
|
+
holds them to ``{home}/...``."""
|
|
260
|
+
root, gone = os.path.realpath(home), []
|
|
261
|
+
for c in d.get("caches") or []:
|
|
262
|
+
if not isinstance(c, str) or not c.startswith("{home}/"):
|
|
263
|
+
continue
|
|
264
|
+
path = os.path.realpath(os.path.join(home, c[len("{home}/"):]))
|
|
265
|
+
if not path.startswith(root + os.sep) or not os.path.lexists(path):
|
|
266
|
+
continue
|
|
267
|
+
if os.path.isdir(path):
|
|
268
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
269
|
+
else:
|
|
270
|
+
with contextlib.suppress(OSError):
|
|
271
|
+
os.remove(path)
|
|
272
|
+
gone.append(path)
|
|
273
|
+
return gone
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def bin_dirs(where: str) -> List[str]:
|
|
277
|
+
"""The folders of an install whose programs a start command may name."""
|
|
278
|
+
return [p for p in (os.path.join(where, "node_modules", ".bin"), os.path.join(where, ".venv", "bin")) if os.path.isdir(p)]
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def driver_packages(bench: dict, home: str = HOME) -> Optional[str]:
|
|
282
|
+
"""The folder a benchmark's ``driver.install`` is installed in, installing
|
|
283
|
+
it first when it is not (once per digest, as an agent's packages); None
|
|
284
|
+
for a driver that declares none and runs on what the machine has. Its
|
|
285
|
+
``project`` comes from the benchmark's upstream clone, cloned first when
|
|
286
|
+
it is not there: the same commit, fetched once."""
|
|
287
|
+
d = definitions.driver_install(bench)
|
|
288
|
+
if not d["install"]:
|
|
289
|
+
return None
|
|
290
|
+
src = upstream(bench, home) if d["install"].get("project") else None
|
|
291
|
+
return packages(d, home, src=src)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
def driver_dir(bench: dict, home: str = HOME) -> Optional[str]:
|
|
295
|
+
"""Where ``driver_packages`` installs, installed or not; None when it declares nothing."""
|
|
296
|
+
d = definitions.driver_install(bench)
|
|
297
|
+
return packages_dir(d, home) if d["install"] else None
|
|
298
|
+
|
|
299
|
+
|
|
300
|
+
def driver_argv(bench: dict, home: str = HOME) -> tuple:
|
|
301
|
+
"""How a benchmark's driver is started: its ``driver.command`` with
|
|
302
|
+
``{here}`` (the definition's folder) and ``{install}`` (its
|
|
303
|
+
``driver.install`` folder, installed first when it is not) filled, and
|
|
304
|
+
what that install adds to the driver's environment (``HF_HUB_CACHE``
|
|
305
|
+
when it prefetched models). Every place that starts a driver - a run, a
|
|
306
|
+
shard, a proof's checks - starts it here; a container setup, whose install
|
|
307
|
+
is made in its image, through :func:`driver_command`."""
|
|
308
|
+
return driver_command(bench, driver_packages(bench, home), home)
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def driver_command(bench: dict, where: Optional[str], home: str = HOME) -> tuple:
|
|
312
|
+
"""A driver's command and what its install in ``where`` adds to its
|
|
313
|
+
environment; ``where`` None for a driver that declares no install."""
|
|
314
|
+
extra = process_env(definitions.driver_install(bench), where, home) if where else {}
|
|
315
|
+
return words(bench["driver"]["command"], {"here": bench["_here"], "install": where or ""}), extra
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def upstream(bench: dict, home: str = HOME) -> Optional[str]:
|
|
319
|
+
"""A benchmark's upstream at its tested commit, cloned once; None when the
|
|
320
|
+
definition has no upstream repository (its data sits beside it)."""
|
|
321
|
+
u = bench.get("upstream") or {}
|
|
322
|
+
where = upstream_dir(bench, home)
|
|
323
|
+
if where is None:
|
|
324
|
+
return None
|
|
325
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
326
|
+
return where
|
|
327
|
+
with _locked(where):
|
|
328
|
+
return _upstream(u, where)
|
|
329
|
+
|
|
330
|
+
|
|
331
|
+
def upstream_dir(bench: dict, home: str = HOME) -> Optional[str]:
|
|
332
|
+
"""Where ``upstream`` clones a benchmark's upstream; None when it has none."""
|
|
333
|
+
u = bench.get("upstream") or {}
|
|
334
|
+
return os.path.join(home, "upstreams", bench["key"], u["tested"]) if u.get("repository") else None
|
|
335
|
+
|
|
336
|
+
|
|
337
|
+
def _upstream(u: dict, where: str) -> str:
|
|
338
|
+
if os.path.exists(os.path.join(where, MARK)):
|
|
339
|
+
return where
|
|
340
|
+
if os.path.isdir(where):
|
|
341
|
+
shutil.rmtree(where)
|
|
342
|
+
os.makedirs(where)
|
|
343
|
+
log = os.path.join(os.path.dirname(where), u["tested"] + ".log")
|
|
344
|
+
_run(["git", "init", "--quiet"], where, log)
|
|
345
|
+
_run(["git", "fetch", "--quiet", "--depth", "1", u["repository"], u["tested"]], where, log, env=git_env())
|
|
346
|
+
_run(["git", "checkout", "--quiet", "FETCH_HEAD"], where, log)
|
|
347
|
+
open(os.path.join(where, MARK), "w").close()
|
|
348
|
+
return where
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def sha256_of(path: str) -> str:
|
|
352
|
+
h = hashlib.sha256()
|
|
353
|
+
with open(path, "rb") as f:
|
|
354
|
+
for chunk in iter(lambda: f.read(1 << 20), b""):
|
|
355
|
+
h.update(chunk)
|
|
356
|
+
return h.hexdigest()
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
def dataset(bench: dict, ds: dict, up: Optional[str], progress: Optional[Callable] = None) -> Optional[str]:
|
|
360
|
+
"""The dataset's file, checked by its sha256: in the upstream when there
|
|
361
|
+
is one, else beside the definition. None for a dataset with no file (its
|
|
362
|
+
driver makes its units). ``progress``: told a download's bytes as they
|
|
363
|
+
come (``download``)."""
|
|
364
|
+
if ds.get("url"):
|
|
365
|
+
return download(ds["url"], ds["sha256"], progress=progress)
|
|
366
|
+
if not ds.get("path"):
|
|
367
|
+
return None
|
|
368
|
+
path = os.path.join(up or bench["_here"], ds["path"])
|
|
369
|
+
if not os.path.isfile(path):
|
|
370
|
+
raise InstallFailed(f"dataset {ds['id']}: {ds['path']} is not in {'the upstream' if up else 'the definition folder'}")
|
|
371
|
+
got = sha256_of(path)
|
|
372
|
+
if got != ds["sha256"]:
|
|
373
|
+
raise InstallFailed(f"dataset {ds['id']}: sha256 {got[:12]}..., the definition says {ds['sha256'][:12]}...")
|
|
374
|
+
return path
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
def download_path(url: str, sha256: str, home: str = HOME) -> str:
|
|
378
|
+
"""Where ``download`` keeps a dataset, downloaded or not."""
|
|
379
|
+
return os.path.join(data_dir(home), sha256, os.path.basename(url.split("?", 1)[0]) or "data")
|
|
380
|
+
|
|
381
|
+
|
|
382
|
+
def download(url: str, sha256: str, home: str = HOME, progress: Optional[Callable] = None) -> str:
|
|
383
|
+
"""A dataset served outside its upstream, downloaded once under its
|
|
384
|
+
digest (``data_dir``), and checked by it. ``progress(done, total)`` is
|
|
385
|
+
told the bytes so far after each chunk (``total`` from the server's
|
|
386
|
+
length, None when it gives none), and once more with ``final=True``."""
|
|
387
|
+
where = download_path(url, sha256, home)
|
|
388
|
+
if os.path.isfile(where):
|
|
389
|
+
return where
|
|
390
|
+
with _locked(where):
|
|
391
|
+
if not os.path.isfile(where):
|
|
392
|
+
_fetch(url, sha256, where, "dataset", progress)
|
|
393
|
+
return where
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _fetch(url: str, sha256: str, where: str, what: str, progress: Optional[Callable] = None) -> None:
|
|
397
|
+
"""``url`` downloaded to ``where``, kept only when it is the file ``sha256`` names."""
|
|
398
|
+
import urllib.request
|
|
399
|
+
part = where + ".part"
|
|
400
|
+
try:
|
|
401
|
+
with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "nullius"}), timeout=60) as r, open(part, "wb") as f:
|
|
402
|
+
total = int(r.headers.get("Content-Length") or 0) or None
|
|
403
|
+
done = 0
|
|
404
|
+
for chunk in iter(lambda: r.read(1 << 20), b""):
|
|
405
|
+
f.write(chunk)
|
|
406
|
+
done += len(chunk)
|
|
407
|
+
if progress:
|
|
408
|
+
progress(done, total)
|
|
409
|
+
if progress:
|
|
410
|
+
progress(done, total, final=True)
|
|
411
|
+
except OSError as e:
|
|
412
|
+
raise InstallFailed(f"{what} {url}: {e}")
|
|
413
|
+
got = sha256_of(part)
|
|
414
|
+
if got != sha256:
|
|
415
|
+
os.remove(part)
|
|
416
|
+
raise InstallFailed(f"{what} {url}: sha256 {got[:12]}..., the definition says {sha256[:12]}...")
|
|
417
|
+
os.replace(part, where)
|
|
418
|
+
|
|
419
|
+
|
|
420
|
+
def words(cmd: List[str], slots: Dict[str, str]) -> List[str]:
|
|
421
|
+
"""A definition's command with its slots filled: each slot is replaced
|
|
422
|
+
inside its word, and the word stays one argument - no shell reads it."""
|
|
423
|
+
out = []
|
|
424
|
+
for w in cmd:
|
|
425
|
+
for k, v in slots.items():
|
|
426
|
+
w = w.replace("{" + k + "}", v)
|
|
427
|
+
out.append(w)
|
|
428
|
+
return out
|