nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,428 @@
1
+ """Where what a run needs is installed, and installing it once.
2
+
3
+ Everything goes under ``STANDARD_DIR`` (``standard/`` in the persistent user data directory):
4
+
5
+ - ``agents/<key>/<digest>/`` and ``capabilities/<key>/<digest>/``: the packages a
6
+ definition's ``install`` names (``npm`` into ``node_modules``, ``pip`` and
7
+ ``wheels`` - a wheel by its URL, checked by its sha256 - into a ``uv`` environment), the digest taken over that list so a re-pin installs
8
+ beside the old one rather than over it;
9
+ - ``drivers/<key>/<digest>/``: a benchmark's ``driver.install`` the same way -
10
+ the environment its upstream's own code needs, which its driver imports
11
+ (``driver_argv``), never this application's and never an agent's. Its
12
+ ``project`` is installed from the benchmark's upstream clone (below), the
13
+ same commit, rather than fetched a second time;
14
+ - ``upstreams/<key>/<commit>/``: a benchmark's upstream repository at its
15
+ tested commit;
16
+ - ``datasets/<sha256>/<file>``: a dataset or auxiliary file served outside its upstream, under
17
+ ``STANDARD_DATA_DIR`` instead when that is set - datasets can be gigabytes
18
+ where installs are not, so they may sit on a bigger disk than the installs.
19
+
20
+ On the container backend an agent's or a capability's packages are installed in
21
+ the base image (``container.install_argv``), and the base image's tag is part
22
+ of their folder's digest. A driver's packages, and all packages on the local
23
+ backend, are installed with this machine's ``npm`` and ``uv``. Git always runs
24
+ on this machine.
25
+
26
+ A folder is complete when its ``.installed`` marker is; a failed install
27
+ leaves no marker and is tried again from scratch. Two runs installing the same
28
+ thing at once take turns (a lock file beside the folder): the second finds the
29
+ first one's marker rather than removing its half-made folder. A dataset file is checked
30
+ against its sha256 before a run reads it.
31
+ """
32
+ from __future__ import annotations
33
+
34
+ import contextlib
35
+ import fcntl
36
+ import hashlib
37
+ import json
38
+ import os
39
+ import re
40
+ import shutil
41
+ import subprocess
42
+ from typing import Callable, Dict, List, Optional
43
+
44
+ from nullius.standard import definitions
45
+
46
+ from nullius import paths
47
+
48
+ HOME = os.environ.get("STANDARD_DIR") or os.path.join(paths.data_dir(), "standard")
49
+ MARK = ".installed"
50
+
51
+
52
+ def data_dir(home: str = HOME, env=os.environ) -> str:
53
+ """Where downloaded datasets are kept: ``STANDARD_DATA_DIR``, else
54
+ ``datasets/`` in the standard's home. A dataset already downloaded
55
+ elsewhere is not moved: setting the variable downloads it once more."""
56
+ return env.get("STANDARD_DATA_DIR") or os.path.join(home, "datasets")
57
+
58
+
59
+ class InstallFailed(Exception):
60
+ pass
61
+
62
+
63
+ # A variable an install step is never given: a package's install script runs
64
+ # as published, and a run's process holds the model key (under --from-run as
65
+ # OPENAI_API_KEY, API_TOKEN, LLM_API_KEY) - the relay is the only holder meant.
66
+ SECRET = re.compile(r"(?:^|_)(?:KEY|TOKEN|SECRET|PASSWORD|CREDENTIALS?)$", re.I) # the last word: GIT_CONFIG_KEY_0 is git's own
67
+
68
+
69
+ def install_env(env=None) -> dict:
70
+ """This process's environment less every variable named like a secret."""
71
+ return {k: v for k, v in (os.environ if env is None else env).items() if not SECRET.search(k)}
72
+
73
+
74
+ def git_env() -> dict:
75
+ """What a fetch of an upstream runs with: the installer's environment and
76
+ no prompt. A private repository - a fork of the user's - is fetched with
77
+ this machine's own git credentials (its credential helper, its ssh agent,
78
+ in its HOME); without them the fetch fails, said, rather than waiting on a
79
+ terminal for a password."""
80
+ return dict(install_env(), GIT_TERMINAL_PROMPT="0")
81
+
82
+
83
+ def _run(cmd: List[str], cwd: str, log: str, env: Optional[dict] = None, shown: Optional[List[str]] = None):
84
+ """Run one install step, its output to ``log``. ``shown``: the step as a
85
+ failure names it, when ``cmd`` wraps it (``docker run ...``)."""
86
+ with open(log, "a") as f:
87
+ f.write("$ " + " ".join(cmd) + "\n")
88
+ f.flush()
89
+ try:
90
+ # uv may download its Python over a system setting against it (Fedora's), unless this environment says otherwise
91
+ env = {"UV_PYTHON_DOWNLOADS": "automatic", **(install_env() if env is None else env)}
92
+ r = subprocess.run(cmd, cwd=cwd, stdout=f, stderr=subprocess.STDOUT, env=env)
93
+ except FileNotFoundError:
94
+ raise InstallFailed(f"{cmd[0]} is not on this machine")
95
+ if r.returncode != 0:
96
+ with open(log, errors="replace") as f:
97
+ tail = f.read().strip().splitlines()[-3:]
98
+ raise InstallFailed(f"{' '.join((shown or cmd)[:3])}...: exit {r.returncode}: {' | '.join(tail)[-300:]}")
99
+
100
+
101
+ @contextlib.contextmanager
102
+ def _locked(where: str):
103
+ os.makedirs(os.path.dirname(where), exist_ok=True)
104
+ with open(where + ".lock", "w") as f:
105
+ fcntl.flock(f, fcntl.LOCK_EX)
106
+ try:
107
+ yield
108
+ finally:
109
+ fcntl.flock(f, fcntl.LOCK_UN)
110
+
111
+
112
+ def _digest(obj) -> str:
113
+ return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest()[:12]
114
+
115
+
116
+ def packages(d: dict, home: str = HOME, run_in: Optional[Callable[..., List[str]]] = None,
117
+ src: Optional[str] = None, image: Optional[str] = None) -> str:
118
+ """The folder an agent's or a capability's packages are installed in,
119
+ installing them first when they are not. A definition installing nothing
120
+ gets an empty folder all the same, so ``{install}`` always names one.
121
+
122
+ ``run_in``: where every ``npm``, ``uv`` and Python step runs instead of
123
+ this machine: the container backend's base image, which carries Node and uv
124
+ (``container.install_argv``). Given the command, the folder and the
125
+ step's own variables, it returns the argument list that runs it there.
126
+ The install is then made on the libc and the tools the agent's container
127
+ runs on, uv's Python inside the folder (``.python``). Git stays on this
128
+ machine, with its credentials. ``image``: that base image's tag, part of
129
+ the folder's digest, so an install made in it never serves a run on this
130
+ machine. Without ``run_in``, a machine with no ``npm`` is refused, said.
131
+
132
+ ``src``: a checkout of the definition's upstream at ``upstream.tested``
133
+ already on this machine, which ``project`` is installed from instead of
134
+ a clone of its own (a benchmark's driver: its upstream clone)."""
135
+ assert (run_in is None) == (image is None), "an install in the base image is named by its tag"
136
+ inst = d.get("install") or {}
137
+ up = (d.get("upstream") or {}) if inst.get("project") else {}
138
+ where = packages_dir(d, home, image)
139
+ if os.path.exists(os.path.join(where, MARK)):
140
+ return where
141
+ with _locked(where):
142
+ return _packages(inst, where, up, run_in, src)
143
+
144
+
145
+ def packages_dir(d: dict, home: str = HOME, image: Optional[str] = None) -> str:
146
+ """Where ``packages`` installs a definition's packages, installed or not."""
147
+ inst = d.get("install") or {}
148
+ kind = {"agent": "agents", "driver": "drivers"}.get(d.get("_kind"), "capabilities")
149
+ up = (d.get("upstream") or {}) if inst.get("project") else {}
150
+ key = dict(inst, _at=up.get("tested")) if up else inst
151
+ return os.path.join(home, kind, d["key"], _digest(dict(key, _image=image) if image else key))
152
+
153
+
154
+ def ready(where: Optional[str]) -> bool:
155
+ """Whether an install folder is complete (its marker is there): a run says
156
+ the stage was there already rather than that it took no time."""
157
+ return bool(where) and os.path.exists(os.path.join(where, MARK))
158
+
159
+
160
+ HF = "hf" # inside an install: the models prefetched from the hub (HF_HUB_CACHE at run time)
161
+ WHEELS = "wheels" # inside an install: the wheels it names by URL, as downloaded
162
+ SRC = "src" # inside an install: the definition's upstream, at its tested commit, for install.project
163
+ SCRATCH = (".home", ".npm-cache", ".uv-cache") # what an install in the base image keeps of its own (container.install_argv)
164
+
165
+
166
+ def remove_scratch(where: str) -> None:
167
+ for scratch in SCRATCH:
168
+ shutil.rmtree(os.path.join(where, scratch), ignore_errors=True)
169
+
170
+
171
+ def _packages(inst: dict, where: str, up: Optional[dict] = None, run_in: Optional[Callable] = None,
172
+ src: Optional[str] = None) -> str:
173
+ if os.path.exists(os.path.join(where, MARK)):
174
+ return where # another run finished it while this one waited
175
+ if os.path.isdir(where):
176
+ shutil.rmtree(where)
177
+ os.makedirs(where)
178
+ log = os.path.join(where, "install.log")
179
+ if run_in:
180
+ step = lambda cmd, env=None: _run(run_in(cmd, where, env), where, log, shown=cmd)
181
+ else:
182
+ step = lambda cmd, env=None: _run(cmd, where, log, env=dict(install_env(), **(env or {})))
183
+ try:
184
+ _steps(inst, where, up, src, log, step, boxed=run_in is not None)
185
+ finally:
186
+ remove_scratch(where)
187
+ with open(os.path.join(where, MARK), "w") as f:
188
+ json.dump(inst, f)
189
+ return where
190
+
191
+
192
+ def _steps(inst: dict, where: str, up: Optional[dict], src: Optional[str], log: str, step: Callable, boxed: bool):
193
+ if inst.get("npm"):
194
+ if not boxed and not shutil.which("npm"):
195
+ raise InstallFailed("npm is not on this machine: install Node.js 22 here, or run on the container backend, "
196
+ "which installs with the Node in its base image")
197
+ step(["npm", "install", "--no-audit", "--no-fund", "--prefix", where] + list(inst["npm"]))
198
+ if inst.get("pip") or inst.get("project") or inst.get("indexes") or inst.get("wheels"):
199
+ py = os.path.join(where, ".venv", "bin", "python")
200
+ # uv's own interpreter, never the system's: on the container backend in the install (.python), mounted with it; /usr/bin is the image's
201
+ step(["uv", "venv", "--quiet", "--python-preference", "only-managed", os.path.join(where, ".venv")]
202
+ + (["--python", inst["python"]] if inst.get("python") else []))
203
+ for x in inst.get("indexes") or []: # first: the project's resolution then keeps them
204
+ step(["uv", "pip", "install", "--quiet", "--python", py, "--index-url", x["url"]] + list(x["packages"]))
205
+ wheels = [] # each checked by its sha256 before it is installed
206
+ for x in inst.get("wheels") or []:
207
+ os.makedirs(os.path.join(where, WHEELS), exist_ok=True)
208
+ wheels.append(os.path.join(where, WHEELS, x["url"].rsplit("/", 1)[1]))
209
+ _fetch(x["url"], x["sha256"], wheels[-1], "wheel")
210
+ locked = bool(inst.get("project") and inst.get("locked"))
211
+ if (inst.get("pip") or wheels) and not locked:
212
+ # resolved together: a wheel's dependencies are pip's pins
213
+ step(["uv", "pip", "install", "--quiet", "--python", py] + list(inst.get("pip") or []) + wheels)
214
+ if inst.get("project"):
215
+ if src is None:
216
+ src = os.path.join(where, SRC)
217
+ os.makedirs(src)
218
+ for cmd in (["git", "init", "--quiet"], ["git", "fetch", "--quiet", "--depth", "1", up["repository"], up["tested"]],
219
+ ["git", "checkout", "--quiet", "FETCH_HEAD"]):
220
+ _run(cmd, src, log, env=git_env()) # on this machine, with its git credentials
221
+ proj = os.path.normpath(os.path.join(src, inst["project"]))
222
+ extras = "[" + ",".join(inst["extras"]) + "]" if inst.get("extras") else ""
223
+ if locked:
224
+ # the project's own lock file, its pins as published, and pip's packages
225
+ # resolved beside them in one go: a conflict fails the install, said
226
+ lock = os.path.join(where, "requirements.lock")
227
+ step(["uv", "export", "--frozen", "--no-dev", "--no-emit-project", "--no-hashes", "--quiet",
228
+ "--project", proj, "-o", lock] + [w for x in inst.get("extras") or [] for w in ("--extra", x)])
229
+ step(["uv", "pip", "install", "--quiet", "--python", py, "-r", lock] + list(inst.get("pip") or []) + wheels)
230
+ step(["uv", "pip", "install", "--quiet", "--python", py, "--no-deps", "-e", proj + extras])
231
+ else:
232
+ step(["uv", "pip", "install", "--quiet", "--python", py, "-e", proj + extras])
233
+ if inst.get("prefetch"):
234
+ step([py, "-c", "import sys\nfrom huggingface_hub import snapshot_download\nfor m in sys.argv[1:]: snapshot_download(m)"]
235
+ + list(inst["prefetch"]), env={"HF_HUB_CACHE": os.path.join(where, HF)})
236
+
237
+
238
+ def process_env(d: dict, where: str, home: str, options: Optional[dict] = None) -> dict:
239
+ """What an install adds to its process's environment: the definition's own
240
+ ``env`` rendered (``{opt_<name>}`` from ``options``, the run's values), and
241
+ ``HF_HUB_CACHE`` when it prefetched models."""
242
+ slots = {"install": where, "home": home, "here": d.get("_here", ""),
243
+ **{"opt_" + k: v for k, v in (options or {}).items()}}
244
+ out = {}
245
+ if (d.get("install") or {}).get("prefetch"):
246
+ out["HF_HUB_CACHE"] = os.path.join(where, HF)
247
+ for k, v in (d.get("env") or {}).items():
248
+ for s, val in slots.items():
249
+ v = v.replace("{" + s + "}", val)
250
+ out[k] = v
251
+ return out
252
+
253
+
254
+ def prune(d: dict, home: str) -> List[str]:
255
+ """Remove what the definition declares as ``caches`` from a unit's home,
256
+ once the unit has ended and its trial is written: the paths it removed.
257
+ A path is resolved under ``home`` and refused when it would leave it
258
+ (a symlink the agent made included); the definition's check already
259
+ holds them to ``{home}/...``."""
260
+ root, gone = os.path.realpath(home), []
261
+ for c in d.get("caches") or []:
262
+ if not isinstance(c, str) or not c.startswith("{home}/"):
263
+ continue
264
+ path = os.path.realpath(os.path.join(home, c[len("{home}/"):]))
265
+ if not path.startswith(root + os.sep) or not os.path.lexists(path):
266
+ continue
267
+ if os.path.isdir(path):
268
+ shutil.rmtree(path, ignore_errors=True)
269
+ else:
270
+ with contextlib.suppress(OSError):
271
+ os.remove(path)
272
+ gone.append(path)
273
+ return gone
274
+
275
+
276
+ def bin_dirs(where: str) -> List[str]:
277
+ """The folders of an install whose programs a start command may name."""
278
+ return [p for p in (os.path.join(where, "node_modules", ".bin"), os.path.join(where, ".venv", "bin")) if os.path.isdir(p)]
279
+
280
+
281
+ def driver_packages(bench: dict, home: str = HOME) -> Optional[str]:
282
+ """The folder a benchmark's ``driver.install`` is installed in, installing
283
+ it first when it is not (once per digest, as an agent's packages); None
284
+ for a driver that declares none and runs on what the machine has. Its
285
+ ``project`` comes from the benchmark's upstream clone, cloned first when
286
+ it is not there: the same commit, fetched once."""
287
+ d = definitions.driver_install(bench)
288
+ if not d["install"]:
289
+ return None
290
+ src = upstream(bench, home) if d["install"].get("project") else None
291
+ return packages(d, home, src=src)
292
+
293
+
294
+ def driver_dir(bench: dict, home: str = HOME) -> Optional[str]:
295
+ """Where ``driver_packages`` installs, installed or not; None when it declares nothing."""
296
+ d = definitions.driver_install(bench)
297
+ return packages_dir(d, home) if d["install"] else None
298
+
299
+
300
+ def driver_argv(bench: dict, home: str = HOME) -> tuple:
301
+ """How a benchmark's driver is started: its ``driver.command`` with
302
+ ``{here}`` (the definition's folder) and ``{install}`` (its
303
+ ``driver.install`` folder, installed first when it is not) filled, and
304
+ what that install adds to the driver's environment (``HF_HUB_CACHE``
305
+ when it prefetched models). Every place that starts a driver - a run, a
306
+ shard, a proof's checks - starts it here; a container setup, whose install
307
+ is made in its image, through :func:`driver_command`."""
308
+ return driver_command(bench, driver_packages(bench, home), home)
309
+
310
+
311
+ def driver_command(bench: dict, where: Optional[str], home: str = HOME) -> tuple:
312
+ """A driver's command and what its install in ``where`` adds to its
313
+ environment; ``where`` None for a driver that declares no install."""
314
+ extra = process_env(definitions.driver_install(bench), where, home) if where else {}
315
+ return words(bench["driver"]["command"], {"here": bench["_here"], "install": where or ""}), extra
316
+
317
+
318
+ def upstream(bench: dict, home: str = HOME) -> Optional[str]:
319
+ """A benchmark's upstream at its tested commit, cloned once; None when the
320
+ definition has no upstream repository (its data sits beside it)."""
321
+ u = bench.get("upstream") or {}
322
+ where = upstream_dir(bench, home)
323
+ if where is None:
324
+ return None
325
+ if os.path.exists(os.path.join(where, MARK)):
326
+ return where
327
+ with _locked(where):
328
+ return _upstream(u, where)
329
+
330
+
331
+ def upstream_dir(bench: dict, home: str = HOME) -> Optional[str]:
332
+ """Where ``upstream`` clones a benchmark's upstream; None when it has none."""
333
+ u = bench.get("upstream") or {}
334
+ return os.path.join(home, "upstreams", bench["key"], u["tested"]) if u.get("repository") else None
335
+
336
+
337
+ def _upstream(u: dict, where: str) -> str:
338
+ if os.path.exists(os.path.join(where, MARK)):
339
+ return where
340
+ if os.path.isdir(where):
341
+ shutil.rmtree(where)
342
+ os.makedirs(where)
343
+ log = os.path.join(os.path.dirname(where), u["tested"] + ".log")
344
+ _run(["git", "init", "--quiet"], where, log)
345
+ _run(["git", "fetch", "--quiet", "--depth", "1", u["repository"], u["tested"]], where, log, env=git_env())
346
+ _run(["git", "checkout", "--quiet", "FETCH_HEAD"], where, log)
347
+ open(os.path.join(where, MARK), "w").close()
348
+ return where
349
+
350
+
351
+ def sha256_of(path: str) -> str:
352
+ h = hashlib.sha256()
353
+ with open(path, "rb") as f:
354
+ for chunk in iter(lambda: f.read(1 << 20), b""):
355
+ h.update(chunk)
356
+ return h.hexdigest()
357
+
358
+
359
+ def dataset(bench: dict, ds: dict, up: Optional[str], progress: Optional[Callable] = None) -> Optional[str]:
360
+ """The dataset's file, checked by its sha256: in the upstream when there
361
+ is one, else beside the definition. None for a dataset with no file (its
362
+ driver makes its units). ``progress``: told a download's bytes as they
363
+ come (``download``)."""
364
+ if ds.get("url"):
365
+ return download(ds["url"], ds["sha256"], progress=progress)
366
+ if not ds.get("path"):
367
+ return None
368
+ path = os.path.join(up or bench["_here"], ds["path"])
369
+ if not os.path.isfile(path):
370
+ raise InstallFailed(f"dataset {ds['id']}: {ds['path']} is not in {'the upstream' if up else 'the definition folder'}")
371
+ got = sha256_of(path)
372
+ if got != ds["sha256"]:
373
+ raise InstallFailed(f"dataset {ds['id']}: sha256 {got[:12]}..., the definition says {ds['sha256'][:12]}...")
374
+ return path
375
+
376
+
377
+ def download_path(url: str, sha256: str, home: str = HOME) -> str:
378
+ """Where ``download`` keeps a dataset, downloaded or not."""
379
+ return os.path.join(data_dir(home), sha256, os.path.basename(url.split("?", 1)[0]) or "data")
380
+
381
+
382
+ def download(url: str, sha256: str, home: str = HOME, progress: Optional[Callable] = None) -> str:
383
+ """A dataset served outside its upstream, downloaded once under its
384
+ digest (``data_dir``), and checked by it. ``progress(done, total)`` is
385
+ told the bytes so far after each chunk (``total`` from the server's
386
+ length, None when it gives none), and once more with ``final=True``."""
387
+ where = download_path(url, sha256, home)
388
+ if os.path.isfile(where):
389
+ return where
390
+ with _locked(where):
391
+ if not os.path.isfile(where):
392
+ _fetch(url, sha256, where, "dataset", progress)
393
+ return where
394
+
395
+
396
+ def _fetch(url: str, sha256: str, where: str, what: str, progress: Optional[Callable] = None) -> None:
397
+ """``url`` downloaded to ``where``, kept only when it is the file ``sha256`` names."""
398
+ import urllib.request
399
+ part = where + ".part"
400
+ try:
401
+ with urllib.request.urlopen(urllib.request.Request(url, headers={"User-Agent": "nullius"}), timeout=60) as r, open(part, "wb") as f:
402
+ total = int(r.headers.get("Content-Length") or 0) or None
403
+ done = 0
404
+ for chunk in iter(lambda: r.read(1 << 20), b""):
405
+ f.write(chunk)
406
+ done += len(chunk)
407
+ if progress:
408
+ progress(done, total)
409
+ if progress:
410
+ progress(done, total, final=True)
411
+ except OSError as e:
412
+ raise InstallFailed(f"{what} {url}: {e}")
413
+ got = sha256_of(part)
414
+ if got != sha256:
415
+ os.remove(part)
416
+ raise InstallFailed(f"{what} {url}: sha256 {got[:12]}..., the definition says {sha256[:12]}...")
417
+ os.replace(part, where)
418
+
419
+
420
+ def words(cmd: List[str], slots: Dict[str, str]) -> List[str]:
421
+ """A definition's command with its slots filled: each slot is replaced
422
+ inside its word, and the word stays one argument - no shell reads it."""
423
+ out = []
424
+ for w in cmd:
425
+ for k, v in slots.items():
426
+ w = w.replace("{" + k + "}", v)
427
+ out.append(w)
428
+ return out