nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,208 @@
1
+ """Docker execution for benchmark setups, whose answerer lives in the driver.
2
+
3
+ Unlike an evaluated agent, this trusted process owns the reference data. It gets
4
+ only the selected benchmark, its upstream/data, an isolated Linux install cache,
5
+ the job workspace and backend Python code. Its network can fetch benchmark data;
6
+ model credentials remain in the relay. No host virtual environment is reused.
7
+ Its declared credentials come as an owner-only file in a key folder, mounted
8
+ read-only; :func:`main` loads them into the driver's environment, never the
9
+ container's, which ``docker inspect`` shows. The folder is removed once the
10
+ driver answers or ends.
11
+ """
12
+ from __future__ import annotations
13
+
14
+ import contextlib
15
+ import hashlib
16
+ import json
17
+ import os
18
+ import shutil
19
+ import sys
20
+ import uuid
21
+ from urllib.parse import urlsplit
22
+
23
+ from nullius.standard import container, definitions, hostreq, install, jsonl
24
+
25
+ UV_VERSION = "0.13.4"
26
+ CREDENTIALS_IN = "/opt/nullius/credentials" # the driver's credentials' one file, in its container
27
+
28
+
29
+ def door_base(door):
30
+ # A rootless door is another container. The driver may already be running,
31
+ # so use its bridge address rather than changing the driver's hosts file.
32
+ return door.base.replace("host.docker.internal", door.hosts["host.docker.internal"], 1) if door.container else door.base
33
+
34
+
35
+ def credential_names(bench: dict) -> set:
36
+ """The variables the benchmark's credentials for its driver set
37
+ (``needs_secrets``, ``to: driver``): a value's or a server's name, a
38
+ file's ``env``."""
39
+ return {s.get("env") if s["kind"] == "file" else s["name"]
40
+ for s in definitions.secrets_of(bench) if s["to"] == "driver"} - {None}
41
+
42
+
43
+ def image(base: str, bench: dict, needs: list, log: str) -> str:
44
+ text = f"FROM {base}\nCOPY --from=ghcr.io/astral-sh/uv:{UV_VERSION} /uv /uvx /usr/local/bin/\n"
45
+ commands = hostreq.install_lines(bench, needs, sudo="")
46
+ if commands:
47
+ text += "RUN apt-get update && " + " && ".join(c.strip() for c in commands) + " && rm -rf /var/lib/apt/lists/*\n"
48
+ tag = "nullius/setup:" + hashlib.sha256(text.encode()).hexdigest()[:12]
49
+ if not container.present(tag):
50
+ result = container._docker(["build", "-t", tag, "-"], input=text, timeout=1800, check=False)
51
+ with open(log, "a") as f:
52
+ f.write(result.stdout + result.stderr)
53
+ if result.returncode:
54
+ raise container.Unavailable(f"building {tag}: {(result.stderr or result.stdout)[-300:]}")
55
+ return tag
56
+
57
+
58
+ class Driver:
59
+ def __init__(self, bench, *, image, app, job, work, upstream, data, needs):
60
+ self.bench, self.image, self.app, self.job = bench, image, app, job
61
+ self.work, self.upstream, self.data = work, upstream, data
62
+ self.cache = os.path.join(install.HOME, "docker-setups", image.rsplit(":", 1)[-1])
63
+ os.makedirs(self.cache, exist_ok=True)
64
+ self.config = os.path.join(work, ".work", "setup-driver.json")
65
+ with open(self.config, "w") as f:
66
+ json.dump({"bench": bench, "upstream": upstream, "needs": needs}, f)
67
+
68
+ def command(self, env, cwd, *, prepare=False, files=(), credentials=None):
69
+ # Only the backend package, not app/ or any user configuration directory.
70
+ mounts = [(os.path.join(self.app, "nullius"), True), (self.bench["_here"], True),
71
+ (self.cache, False), (self.work, False)]
72
+ if self.upstream:
73
+ mounts.append((self.upstream, False)) # upstreams may cache data beside their code
74
+ if self.data and not any(self.data == p or self.data.startswith(p + os.sep) for p, _ in mounts):
75
+ mounts.append((self.data, True))
76
+ mounts += [(p, True) for p in dict.fromkeys(files) if p not in {m[0] for m in mounts}]
77
+ values = {k: v for k, v in env.items() if k not in container.HOST_ONLY}
78
+ values.update(PATH=container.IMAGE_PATH, PYTHONPATH=self.app, PYTHONDONTWRITEBYTECODE="1",
79
+ UV_CACHE_DIR=os.path.join(self.cache, "uv-cache"),
80
+ UV_PYTHON_INSTALL_DIR=os.path.join(self.cache, "python"))
81
+ name = "nullius-setup-" + uuid.uuid4().hex[:20]
82
+ cmd = ["python3", "-m", "nullius.standard.setup_container", self.config, self.cache,
83
+ "prepare" if prepare else "run"] + ([CREDENTIALS_IN] if credentials else [])
84
+ argv = container.run_argv(self.image, cmd, name=name, network=container.BRIDGE if container.podman() else "bridge", job=self.job,
85
+ workdir=cwd, mounted=mounts, env=values, path=container.IMAGE_PATH)
86
+ argv[3:3] = ["--add-host", "host.docker.internal:host-gateway"]
87
+ if credentials: # as the relay's key: fails when the engine cannot see it
88
+ argv[3:3] = ["--mount", f"type=bind,source={credentials},target={CREDENTIALS_IN},readonly"]
89
+ return argv, container.program_env(values), name
90
+
91
+ def prepare(self):
92
+ container.make_bridge()
93
+ home = os.path.join(self.work, ".work", "setup-install-home")
94
+ os.makedirs(home, exist_ok=True)
95
+ argv, env, name = self.command({"HOME": home}, self.work, prepare=True)
96
+ try:
97
+ result = container._docker(argv[1:], env=env, timeout=1800, check=False)
98
+ finally:
99
+ with contextlib.suppress(container.Unavailable):
100
+ container._docker(["rm", "-f", name], check=False)
101
+ with open(os.path.join(self.work, ".work", "setup-install.log"), "a") as f:
102
+ f.write(result.stdout + result.stderr)
103
+ if result.returncode:
104
+ raise install.InstallFailed("container driver install: " + (result.stderr or result.stdout)[-500:])
105
+
106
+ def start(self, env, cwd, log, *, files=(), servers=(), stopped=None):
107
+ # Embedding routes are environment variables; their forwarders live as
108
+ # long as the process. Per-request answer/judge/simulator routes below
109
+ # live only until that protocol reply returns.
110
+ doors, keys, credentials = [], None, None
111
+ env = dict(env)
112
+ try:
113
+ for key, value in env.items():
114
+ target = "tcp://" + value if key in servers else value
115
+ if (key.endswith("BASE_URL") or key in servers) and container.on_loopback(target):
116
+ door = container.Loopback(target, image=self.image, job=f"{self.job}-{uuid.uuid4().hex[:12]}").start()
117
+ doors.append(door)
118
+ env[key] = urlsplit(door_base(door)).netloc if key in servers else door_base(door)
119
+ held = {k: env.pop(k) for k in sorted(credential_names(self.bench)) if k in env}
120
+ if held:
121
+ keys = container.key_folder(self.job)
122
+ credentials = container.secret_file(os.path.join(keys, "credentials"), json.dumps(held))
123
+ argv, client, name = self.command(env, cwd, files=files, credentials=credentials)
124
+ process = Process(argv, cwd, client, log, stopped=stopped, name=name, image=self.image, doors=doors, keys=keys)
125
+ try:
126
+ return process.start()
127
+ except BaseException:
128
+ process.close()
129
+ raise
130
+ except BaseException:
131
+ for door in doors:
132
+ door.stop()
133
+ if keys:
134
+ shutil.rmtree(keys, ignore_errors=True)
135
+ raise
136
+
137
+
138
+ class Process(jsonl.Proc):
139
+ def __init__(self, *args, name, doors, image="", keys=None, **kwargs):
140
+ super().__init__(*args, **kwargs)
141
+ self.name, self.doors, self.image = name, doors, image
142
+ self.keys = keys # the credentials' folder, until the driver answers or ends
143
+
144
+ def forget(self):
145
+ keys, self.keys = self.keys, None
146
+ if keys:
147
+ shutil.rmtree(keys, ignore_errors=True)
148
+
149
+ def request(self, op, timeout=600, **kw):
150
+ doors = []
151
+ def rewrite(value):
152
+ if isinstance(value, dict):
153
+ return {k: url(v) if k == "base_url" and isinstance(v, str) else rewrite(v) for k, v in value.items()}
154
+ if isinstance(value, list):
155
+ return [rewrite(v) for v in value]
156
+ return value
157
+ def url(value):
158
+ if not container.on_loopback(value):
159
+ return value
160
+ door = container.Loopback(value, image=self.image, job=f"{self.name}-{len(doors)}").start()
161
+ doors.append(door)
162
+ return door_base(door)
163
+ try:
164
+ return super().request(op, timeout=timeout, **rewrite(kw))
165
+ finally:
166
+ for door in doors:
167
+ door.stop()
168
+ self.forget() # answered or ended: main() read them, or never will
169
+
170
+ def close(self):
171
+ try:
172
+ super().close()
173
+ finally:
174
+ self.forget()
175
+ with contextlib.suppress(container.Unavailable):
176
+ container._docker(["rm", "-f", self.name], check=False)
177
+ for door in self.doors:
178
+ door.stop()
179
+ self.doors = []
180
+
181
+
182
+ def main():
183
+ config, cache, mode, *credentials = sys.argv[1:]
184
+ with open(config) as f:
185
+ record = json.load(f)
186
+ bench = record["bench"]
187
+ definition = definitions.driver_install(bench)
188
+ where = install.packages_dir(definition, cache) if definition["install"] else None
189
+ if mode == "prepare":
190
+ missing = hostreq.missing(bench, record["needs"])
191
+ if missing:
192
+ raise install.InstallFailed(f"setup container lacks required programs: {missing}")
193
+ if where:
194
+ install.packages(definition, cache, src=record["upstream"])
195
+ return
196
+ if where and not install.ready(where):
197
+ raise install.InstallFailed("setup container's driver environment has not been prepared")
198
+ env = dict(os.environ)
199
+ for path in credentials: # a JSON object: no shell reads a value
200
+ with open(path) as f:
201
+ env.update(json.load(f))
202
+ command, extra = install.driver_command(bench, where, cache)
203
+ env.update(extra)
204
+ os.execvpe(command[0], command, env)
205
+
206
+
207
+ if __name__ == "__main__":
208
+ main()
@@ -0,0 +1,71 @@
1
+ """One run over many machines: shards of a dataset's units, and the launch they share.
2
+
3
+ ``--shard I/N`` runs the units whose shard is ``I`` of ``N``. A unit's shard
4
+ is read off its id, not its position: ``sha256(id) mod N`` (``PARTITION``).
5
+ So every machine computes the same split from the ids alone, with no list to
6
+ hand around, and a dataset that gains units leaves every unit it had in the
7
+ shard it was in - only the new ones land somewhere. Changing ``N`` moves
8
+ most units; a launch keeps one ``N``.
9
+
10
+ The order inside a run is: the driver's ``units``, then the shard, then
11
+ ``--limit`` - so ``--shard 1/2 --limit 6`` is the first six units of shard 1,
12
+ and the two shards of a ``--limit 6`` launch run up to twelve units between
13
+ them. The limit is a shard's own: a machine is told how much it takes on.
14
+
15
+ A trial keeps its unit's place in the whole dataset as its tag (``u0042`` is
16
+ the 43rd unit in every shard), so the shards' trials never share a name and
17
+ merge without renaming (``merge``).
18
+
19
+ Every shard of one launch records the same ``launch`` id: ``--launch``, or by
20
+ default the day (UTC), the benchmark, the agent or capability and the model, so
21
+ shards started separately - on other machines, minutes apart - still say
22
+ they belong together. Two launches of the same pair on the same day need
23
+ ``--launch`` to stay apart.
24
+ """
25
+ from __future__ import annotations
26
+
27
+ import hashlib
28
+ import re
29
+ import time
30
+ from typing import List, Optional, Tuple
31
+
32
+ PARTITION = "sha256(unit id) mod N"
33
+ _SPEC = re.compile(r"^\s*(\d+)\s*/\s*(\d+)\s*$")
34
+ _WORD = re.compile(r"[^A-Za-z0-9_.-]+")
35
+
36
+
37
+ def parse(spec: str) -> Tuple[int, int]:
38
+ """``"I/N"`` as (I, N), 1 <= I <= N; ValueError otherwise."""
39
+ m = _SPEC.match(str(spec or ""))
40
+ if not m:
41
+ raise ValueError(f"--shard {spec!r}: I/N, such as 1/4")
42
+ i, n = int(m.group(1)), int(m.group(2))
43
+ if n < 1 or not 1 <= i <= n:
44
+ raise ValueError(f"--shard {spec!r}: I is from 1 to N")
45
+ return i, n
46
+
47
+
48
+ def of(unit: str, n: int) -> int:
49
+ """The shard (1..n) a unit falls in: by its id alone."""
50
+ return int.from_bytes(hashlib.sha256(str(unit).encode("utf-8")).digest()[:8], "big") % n + 1
51
+
52
+
53
+ def select(units: List[str], index: int, n: int) -> List[Tuple[int, str]]:
54
+ """The units of shard ``index`` of ``n``, each with its place in the whole
55
+ list (what its trial's tag is made of), in the dataset's order."""
56
+ return [(i, u) for i, u in enumerate(units) if n == 1 or of(u, n) == index]
57
+
58
+
59
+ def launch_id(bench_word: str, other_word: str, model: str, day: Optional[str] = None) -> str:
60
+ """The default launch id shards started separately agree on."""
61
+ day = day or time.strftime("%Y-%m-%d", time.gmtime())
62
+ words = [day, bench_word, other_word] + ([model] if model else [])
63
+ return "__".join(_WORD.sub("_", w).strip("_") for w in words)
64
+
65
+
66
+ def record(index: int, n: int, total: int, planned: List[str], limit: Optional[int] = None) -> dict:
67
+ """What a shard's config says of it (``standard.shard``): which shard, of
68
+ how many, over how many units, split how; the units it planned to run, so
69
+ a merge can name a unit planned and never run; and its own ``limit``."""
70
+ return {"index": index, "of": n, "units_total": total, "partition": PARTITION, "planned": list(planned),
71
+ **({"limit": limit} if limit else {})}
@@ -0,0 +1,30 @@
1
+ """HTTP fixture doors declared by a driver's setup, without injecting agent tools.
2
+
3
+ Each service is an existing Unix socket owned by the driver. Docker mounts only
4
+ that socket into a forwarder on the unit's internal network. A local run uses
5
+ the loopback URL supplied by the driver in its prompt.
6
+ """
7
+ import os
8
+ import re
9
+ import stat
10
+
11
+
12
+ def doors(setup):
13
+ out = []
14
+ services = setup.get('web_services', [])
15
+ if not isinstance(services, list):
16
+ raise ValueError('web_services must be a list')
17
+ for entry in services:
18
+ if not isinstance(entry, dict) or set(entry) != {'name', 'port', 'socket'}:
19
+ raise ValueError('web_services entries need name, port and socket')
20
+ name, port, path = entry['name'], entry['port'], entry['socket']
21
+ if not isinstance(name, str) or not re.fullmatch(r'web-[a-z0-9-]{1,40}', name):
22
+ raise ValueError('web service names must start with web-')
23
+ if type(port) is not int or not 1024 <= port <= 65535:
24
+ raise ValueError('web service port must be between 1024 and 65535')
25
+ if not isinstance(path, str) or not os.path.isabs(path) or not stat.S_ISSOCK(os.lstat(path).st_mode):
26
+ raise ValueError('web service socket must be an existing absolute Unix socket, not a link')
27
+ if name in [n for n, _, _ in out]:
28
+ raise ValueError('duplicate web service name')
29
+ out.append((name, port, path))
30
+ return out
File without changes
@@ -0,0 +1,131 @@
1
+ """Behavioral metrics over a parsed trajectory.
2
+
3
+ The outcome says *whether* the agent solved the task; these numbers say *how*
4
+ it worked — how long before it committed to an edit, whether it re-read what
5
+ it had already seen, whether it re-ran commands that had already failed. Two
6
+ resolved trials can hide very different amounts of wasted work, and the waste
7
+ is what token cost scales with.
8
+
9
+ Everything here is a pure function of data the app already loads; nothing
10
+ reaches back into the job directory.
11
+ """
12
+
13
+ from __future__ import annotations
14
+
15
+ from dataclasses import dataclass
16
+
17
+ # Public in trajectory.py so this module need not reach into an underscore.
18
+ from nullius.traces.trajectory import PATCH_PATHS as _PATCH_PATHS
19
+ from nullius.traces.trajectory import parse_ts as _parse_ts
20
+
21
+
22
+ @dataclass
23
+ class Behavior:
24
+ # 1-based position of the first edit among the run's tool calls, and the
25
+ # wall-clock seconds it took to get there (None without a step clock).
26
+ first_edit_call: int | None = None
27
+ time_to_first_edit: float | None = None
28
+ # read/search calls whose exact target had already been fetched before.
29
+ n_redundant_reads: int = 0
30
+ # calls repeating a (name, target) pair that had already failed — retrying
31
+ # the same thing and hoping is the signature of a flailing run.
32
+ n_repeated_failed_calls: int = 0
33
+ # completed edit→execute alternations: how many times an edit was actually
34
+ # followed by running something (tests, the repro, anything).
35
+ edit_test_cycles: int = 0
36
+ n_search: int = 0
37
+ n_read: int = 0
38
+ n_edit: int = 0
39
+ # (search + read) per edit — how much looking preceded each change.
40
+ exploration_per_edit: float | None = None
41
+
42
+
43
+ def analyze(traj) -> Behavior:
44
+ """Compute a Behavior from a loaded Trajectory."""
45
+ b = Behavior()
46
+
47
+ seen_fetches: set = set()
48
+ failed_keys: set = set()
49
+ edit_pending = False
50
+ first_edit_step_id = None
51
+
52
+ for i, call in enumerate(traj.calls, start=1):
53
+ cat = call["category"]
54
+ target = call["target"]
55
+ key = (call["name"], target)
56
+
57
+ if cat == "search":
58
+ b.n_search += 1
59
+ elif cat == "read":
60
+ b.n_read += 1
61
+ elif cat == "edit":
62
+ b.n_edit += 1
63
+
64
+ if cat in ("read", "search") and target:
65
+ fetch = (cat, target)
66
+ if fetch in seen_fetches:
67
+ b.n_redundant_reads += 1
68
+ seen_fetches.add(fetch)
69
+
70
+ if key in failed_keys:
71
+ b.n_repeated_failed_calls += 1
72
+ if call["failure"]:
73
+ failed_keys.add(key)
74
+
75
+ if cat == "edit":
76
+ edit_pending = True
77
+ if b.first_edit_call is None:
78
+ b.first_edit_call = i
79
+ first_edit_step_id = call["step_id"]
80
+ elif cat == "execute" and edit_pending:
81
+ b.edit_test_cycles += 1
82
+ edit_pending = False
83
+
84
+ if b.n_edit:
85
+ b.exploration_per_edit = round((b.n_search + b.n_read) / b.n_edit, 2)
86
+
87
+ if traj.has_step_clock and first_edit_step_id is not None:
88
+ start = next((t for t in (_parse_ts(s.timestamp) for s in traj.steps) if t), None)
89
+ at_edit = next(
90
+ (_parse_ts(s.timestamp) for s in traj.steps if s.step_id == first_edit_step_id),
91
+ None,
92
+ )
93
+ if start and at_edit and at_edit >= start:
94
+ b.time_to_first_edit = (at_edit - start).total_seconds()
95
+
96
+ return b
97
+
98
+
99
+ def gold_files(solve_text: str) -> list:
100
+ """File paths named in a task's gold patch (``solution/solve.sh``).
101
+
102
+ A solve script that embeds the gold diff in a heredoc is read with the
103
+ diff-header patterns used on agent patch payloads, unchanged.
104
+ """
105
+ found = []
106
+ for pattern in _PATCH_PATHS:
107
+ for path in pattern.findall(solve_text or ""):
108
+ if path not in found and path != "/dev/null":
109
+ found.append(path)
110
+ return found
111
+
112
+
113
+ @dataclass
114
+ class Localization:
115
+ hit: list # gold files the agent edited
116
+ missed: list # gold files it never touched
117
+ extra: list # files it edited that the gold patch doesn't name
118
+
119
+
120
+ def _same_file(a: str, b: str) -> bool:
121
+ # Agent paths may be container-absolute (/testbed/sklearn/...), gold paths
122
+ # repo-relative (sklearn/...). Suffix match on a path boundary covers both.
123
+ return a == b or a.endswith("/" + b) or b.endswith("/" + a)
124
+
125
+
126
+ def localization(touched, gold) -> Localization:
127
+ """Compare the files the agent edited against the gold patch's files."""
128
+ hit = [g for g in gold if any(_same_file(t, g) for t in touched)]
129
+ missed = [g for g in gold if g not in hit]
130
+ extra = [t for t in touched if not any(_same_file(t, g) for g in gold)]
131
+ return Localization(hit=hit, missed=missed, extra=extra)