nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,406 @@
1
+ """The run builder: the recipe vocabulary as a page, directory and registry lookups."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import re
7
+ import shlex
8
+ import subprocess
9
+ import urllib.request
10
+ from concurrent.futures import ThreadPoolExecutor
11
+ from flask import abort, jsonify, render_template, request
12
+ from nullius import settings, execution, gitrefs, hosts, prerequisites, providers
13
+ from nullius.harbor import source as harbor_source
14
+ from nullius.catalogue import catalog
15
+ from nullius.recipes import launch as command_builder
16
+ from nullius.standard import definitions, hostreq
17
+ from nullius.standard import recipe as standard_recipe
18
+
19
+ from .core import app
20
+
21
+ # Harbor agents that call their vendor with a key of their own, beside the model's (read from Harbor's agents): each
22
+ # entry's names are alternatives, any one of them will do. A recipe naming one whose key Settings lacks says so on Review.
23
+ AGENT_NEEDS = {"cursor-cli": ["CURSOR_API_KEY"], "copilot-cli": ["COPILOT_GITHUB_TOKEN", "GH_TOKEN", "GITHUB_TOKEN"],
24
+ "rovodev-cli": ["ROVODEV_USER_API_TOKEN"]}
25
+ from . import shared
26
+ from . import execute
27
+
28
+
29
+ @app.route("/new-run")
30
+ def new_run():
31
+ """Build a Harbor recipe; never execute it from this route."""
32
+ active = shared.STATE.get("workspace")
33
+ data = command_builder.payload(catalog.load(), shared._load_jobs(), active.name if active else "")
34
+ # Execution sets these credential variables from Settings.
35
+ data["credentialVars"] = {"key": "API_TOKEN", "url": "API_URL"}
36
+ # Where a recipe can run: this machine, or the hosts named in Settings.
37
+ # Their entries only - a key is a path, never read here.
38
+ try:
39
+ data["hosts"] = [{"entry": h.entry, "label": h.label} for h in hosts.parse_list(os.environ.get("HOSTS", ""))]
40
+ except ValueError:
41
+ data["hosts"] = []
42
+ # The providers a run can start on, whose key (an OpenAI-compatible endpoint: its URL) is set in Settings, in the
43
+ # order the Model step prefers them, a named endpoint as custom:<id>, with its name; and whether the endpoint at
44
+ # MODEL_API_URL has a key of its own. Presence and names only.
45
+ data["credentials"] = {"available": providers.available(os.environ), "model_key": bool(os.environ.get("MODEL_API_KEY")),
46
+ "endpoints": {e["id"]: e["name"] for e in providers.endpoints(os.environ) if e["id"]},
47
+ "broken": providers.broken(os.environ), # a URL that holds a credential: fixed in Settings first
48
+ # the agent keys Settings holds, by name only: a Harbor agent that needs one it lacks is named on Review
49
+ "agent_keys": sorted(settings.agent_key_values(os.environ))}
50
+ data["agentNeeds"] = AGENT_NEEDS
51
+ # The standard's Where step: why its container backend cannot run on this machine (None: it can); and, per
52
+ # benchmark, the licences its store holds (nullius.secrets) - set, when, until when; never a value.
53
+ # what this machine lacks of each benchmark's host requirements: {benchmark: {requirement: [reasons]}}
54
+ host = {b["key"]: {n: [why] for n, why in hostreq.problems(b, list(b["host"])).items()}
55
+ for b in definitions.load("benchmark", every=True)[0] if b.get("host")}
56
+ data["standardHere"] = {"docker": execute.docker_problem(), "licences": _licence_states(), "host": host}
57
+ data["csrf"] = shared._csrf_token()
58
+ data["local"] = shared_local_client()
59
+ # ?run=<job id>: the recipe a run of this application was started from,
60
+ # opened as it was. ?like=<job id>, the job page's "New run like
61
+ # this": that recipe when the run kept one, else what the job recorded -
62
+ # its launch, its Harbor and what its trials say ran - for the recipe core
63
+ # to rebuild a recipe from (fromJob). Individual query fields also prefill the form.
64
+ like = request.args.get("like")
65
+ run = request.args.get("run") or (like if like and _kept_recipe(like) else None)
66
+ if run:
67
+ ex = execution.for_job(run)
68
+ if ex is None or not ex.plan.recipe:
69
+ abort(404)
70
+ data["loadRecipe"] = ex.plan.recipe
71
+ data["loadRun"] = run
72
+ elif like:
73
+ job = shared._find_job(shared._jobs_for_request(), like)
74
+ if job is None:
75
+ abort(404)
76
+ data["loadLike"] = {"job": job.job_id, "agent": job.agent_name, "launch": job.launch,
77
+ "harbor": {"version": job.harbor_version, "editable": job.harbor_editable},
78
+ "env": next((t.env_type for t in job.trials if t.env_type), "")}
79
+ return render_template("builder/new_run.html", builder_data=data)
80
+
81
+
82
+ @app.route("/new-run/runner", methods=["POST"])
83
+ def new_run_runner():
84
+ """A recipe for the Nullius standard, compiled from its definitions: what
85
+ the builder shows while one is edited. Read-only; the same compile happens
86
+ again in /run, from the definitions, never from the page. Any other
87
+ runner is refused: a benchmark's own scripts are its Original setup."""
88
+ shared._require_csrf()
89
+ body = request.get_json(silent=True) or {}
90
+ recipe = body.get("recipe") if isinstance(body.get("recipe"), dict) else {}
91
+ if len(json.dumps(recipe)) > 200_000:
92
+ abort(413)
93
+ if recipe.get("harness") == standard_recipe.HARNESS:
94
+ out = standard_recipe.compile(recipe, docker_unusable=execute.docker_problem, unpriced=execute.unpriced)
95
+ return {k: out[k] for k in ("errors", "notes", "review", "jobs", "confirm")}
96
+ return {"errors": [{"step": providers.STEP["runner"], "message": providers.GONE}],
97
+ "notes": [], "review": []}, 404
98
+
99
+
100
+ @app.route("/new-run/prerequisites", methods=["POST"])
101
+ def new_run_prerequisites():
102
+ """What the recipe needs of each machine it runs on, and whether that
103
+ machine has it (``nullius/prerequisites.py``): {machines: [{where,
104
+ label, rows}], ready, notes}. The Review step asks on opening and on *Check
105
+ again*, and Run waits for ``ready``; the run asks again as it starts.
106
+ A host is asked over ssh, so this is never asked on a page load."""
107
+ shared._require_csrf()
108
+ if not shared_local_client():
109
+ abort(403, description="Machines are checked from a browser on the application host only.")
110
+ body = request.get_json(silent=True) or {}
111
+ recipe = body.get("recipe") if isinstance(body.get("recipe"), dict) else {}
112
+ if len(json.dumps(recipe)) > 200_000:
113
+ abort(413)
114
+ setups = {}
115
+ standard = recipe.get("harness") == standard_recipe.HARNESS
116
+ if standard:
117
+ out = standard_recipe.compile(recipe)
118
+ if out["errors"]:
119
+ return jsonify(machines=[], ready=False, errors=[e["message"] for e in out["errors"]])
120
+ wheres = [sh["where"] for sh in out["shards"]]
121
+ needs = ["engine"] if out["docker"] else []
122
+ h = out.get("host") or {}
123
+ b = definitions.get("benchmark", h.get("benchmark", "")) if h.get("needs") else None
124
+ setups = {n["name"]: {**n, "install": hostreq.install_lines(b, [n["name"]])}
125
+ for n in h.get("needs") or []} if b else {}
126
+ host_needs = needs + ["python3", "git"]
127
+ elif recipe.get("harness", "harbor") != "harbor":
128
+ return jsonify(machines=[], ready=False, errors=[providers.GONE])
129
+ else:
130
+ pin, problem = harbor_source.pin_of(recipe.get("harbor"))
131
+ if problem:
132
+ return jsonify(machines=[], ready=False, errors=[problem])
133
+ wheres, needs = [str(recipe.get("where") or "local")], ["docker", "uv" if pin else "harbor"]
134
+ host_needs = needs
135
+ known = {h.entry: h for h in execute._hosts()}
136
+
137
+ def machine(w):
138
+ if w == "local":
139
+ return {"where": w, "label": "this machine", "rows": prerequisites.check_here(needs, setups, standard=standard)}
140
+ if w not in known:
141
+ return {"where": w, "label": w, "rows": [prerequisites.row("connect", "missing", f"{w} is not one of the hosts in Settings", label="Connection")]}
142
+ return {"where": w, "label": known[w].label, "rows": prerequisites.check_host(known[w], host_needs, setups, standard=standard)}
143
+ # one ssh probe per host, all at once: one after another, Review waited for their sum
144
+ # (an unreachable host alone holds a probe 8 to 12 s)
145
+ with ThreadPoolExecutor(max(1, min(8, len(wheres)))) as pool:
146
+ machines = list(pool.map(machine, wheres))
147
+ # a notice, not a refusal: the provider's last test in Settings failed (providers.test)
148
+ choice = str(body.get("provider") or "")
149
+ last = providers.tests().get(choice)
150
+ notes = [f"{providers.label(choice)} failed its last test in Settings: {providers.said(last)}"] if isinstance(last, dict) and not last.get("ok") else []
151
+ return jsonify(machines=machines, ready=prerequisites.ready(machines), errors=[], notes=notes)
152
+
153
+
154
+ # The Harbor a recipe pins (nullius/harbor/source.py): PyPI's releases and a
155
+ # repository's refs, each read once per process when the read succeeds (refs
156
+ # again on ?fresh=1, since a branch moves); the agents a pin registers, kept
157
+ # beside its wheels. Nothing here builds on a GET.
158
+ _HARBOR_RELEASES: dict = {}
159
+ _HARBOR_REFS: dict = {}
160
+
161
+
162
+ @app.route("/new-run/harbor/releases")
163
+ def new_run_harbor_releases():
164
+ """The official Harbor's releases on PyPI, newest first, for the
165
+ builder's Harbor panel; read after the page loads, never blocking it."""
166
+ if not _HARBOR_RELEASES:
167
+ found = harbor_source.releases()
168
+ if found["error"]:
169
+ return found
170
+ _HARBOR_RELEASES.update(found)
171
+ return _HARBOR_RELEASES
172
+
173
+
174
+ @app.route("/new-run/harbor/refs")
175
+ def new_run_harbor_refs():
176
+ """A repository's branches and tags, each at its commit, for a Harbor
177
+ pinned from a repository - a private fork included: git ls-remote with
178
+ this machine's own credentials, for a browser on this machine only."""
179
+ if not shared_local_client():
180
+ abort(403, description="Repositories are listed for a browser on the application host only.")
181
+ url = gitrefs.url_of(request.args.get("repo", ""))
182
+ if not url:
183
+ return {"error": "not a repository: owner/name on GitHub, or a git URL (https, ssh, user@host:path, file)"}, 400
184
+ if url not in _HARBOR_REFS or request.args.get("fresh"):
185
+ found = dict(gitrefs.ls_remote(url), url=url, label=gitrefs.label_of(url))
186
+ if found["error"]:
187
+ return dict(found, error=f"could not list refs: {found['error']}; type a commit")
188
+ _HARBOR_REFS[url] = found
189
+ return _HARBOR_REFS[url]
190
+
191
+
192
+ @app.route("/new-run/harbor/agents", methods=["GET", "POST"])
193
+ def new_run_harbor_agents():
194
+ """The agents a pinned Harbor registers. GET reads what a scan kept
195
+ (``state`` ``absent`` when none did); POST - the builder's explicit scan,
196
+ with the form token, from this machine - builds the pin here when it is a
197
+ repository, installs it in a throwaway environment and asks it, once."""
198
+ if request.method == "POST":
199
+ shared._require_csrf()
200
+ if not shared_local_client():
201
+ abort(403, description="A Harbor is built from a browser on the application host only.")
202
+ raw = (request.get_json(silent=True) or {}).get("harbor")
203
+ else:
204
+ raw = {k: request.args.get(k) for k in ("kind", "version", "repo", "revisionKind", "revision", "commit")}
205
+ pin, problem = harbor_source.pin_of(raw)
206
+ if pin is None:
207
+ return {"state": "failed", "agents": None, "error": problem or "no Harbor chosen"}, 400
208
+ return harbor_source.scan(pin, harbor_source.root(execution.runs_root()), build=request.method == "POST")
209
+
210
+
211
+ def _licence_states() -> dict:
212
+ """{benchmark: {name: {set, on, expires, expired}}} for each standard
213
+ benchmark whose environments need licences: what is held of each name
214
+ they declare, a data provider's token in that provider's store."""
215
+ from nullius import secrets
216
+ from nullius.standard import definitions
217
+ out: dict = {}
218
+ for b in definitions.load("benchmark", every=True)[0]:
219
+ for s in definitions.secrets_of(b):
220
+ held = out.setdefault(b["key"], {})
221
+ got = secrets.state(b["key"], s["name"], s.get("provider") or "")
222
+ if got and s["name"] not in held:
223
+ held[s["name"]] = {k: got[k] for k in ("set", "on", "expires", "expired")}
224
+ return out
225
+
226
+
227
+ def _kept_recipe(job_id: str) -> bool:
228
+ ex = execution.for_job(job_id)
229
+ return bool(ex and ex.plan.recipe)
230
+
231
+
232
+ def shared_local_client() -> bool:
233
+ from . import settings
234
+ return settings._local_client()
235
+
236
+
237
+ def _host_target():
238
+ """The benchmark and its requirements a host check or install is about, and where: 'local' or a host of Settings."""
239
+ shared._require_csrf()
240
+ if not shared_local_client():
241
+ abort(403, description="Machines are checked and set up from a browser on the application host only.")
242
+ b = definitions.get("benchmark", request.form.get("benchmark", ""))
243
+ if b is None:
244
+ abort(400, description="no such benchmark")
245
+ names = [n for n in request.form.get("requirements", "").split(",") if n in (b.get("host") or {})]
246
+ entry = request.form.get("entry", "local")
247
+ if entry == "local":
248
+ return b, names, None
249
+ known = {h.entry: h for h in execute._hosts()}
250
+ if entry not in known:
251
+ abort(400, description=f"{entry} is not one of the hosts in Settings")
252
+ return b, names, known[entry]
253
+
254
+
255
+ @app.route("/new-run/host/check", methods=["POST"])
256
+ def new_run_host_check():
257
+ """What a machine lacks: {ok, missing: {requirement: [reasons]}, install}.
258
+ Remote GPU memory is checked by the run on its host, not by the SSH probe."""
259
+ b, names, host = _host_target()
260
+ if host is None:
261
+ gone = {n: [why] for n, why in hostreq.problems(b, names).items()}
262
+ else:
263
+ programs = [p for r in hostreq.requirements(b, names).values() for p in r["check"]]
264
+ got = hosts.test(host, programs=programs)
265
+ if not got["ok"]:
266
+ return jsonify(ok=False, error=got["error"])
267
+ gone = {n: [p for p in r["check"] if p in got["missing"]] for n, r in hostreq.requirements(b, names).items()}
268
+ gone = {n: [", ".join(ps) + " not on PATH"] for n, ps in gone.items() if ps}
269
+ return jsonify(ok=True, missing=gone, install=hostreq.install_lines(b, list(gone)))
270
+
271
+
272
+ @app.route("/new-run/host/install", methods=["POST"])
273
+ def new_run_host_install():
274
+ """The benchmark's install commands for the requirements a machine lacks, run there on an explicit
275
+ choice: apt with ``sudo -n``, so a machine whose sudo asks for a password is refused and told the
276
+ commands to run by hand. The commands come from the definition, never from the form."""
277
+ b, names, host = _host_target()
278
+ script = hostreq.install_script(b, names)
279
+ if script is None:
280
+ return jsonify(ok=True, said="nothing to install")
281
+ argv = ["bash", "-c", script] if host is None else host.argv("bash -c " + shlex.quote(script), timeout=10)
282
+ try:
283
+ p = subprocess.run(argv, capture_output=True, text=True, timeout=600, stdin=subprocess.DEVNULL)
284
+ rc, out = p.returncode, (p.stdout + p.stderr)[-2000:]
285
+ except subprocess.TimeoutExpired:
286
+ rc, out = 124, "no end within 10 minutes"
287
+ if rc != 0:
288
+ why = "sudo needs a password there" if "a password is required" in out or "a terminal is required" in out else out.strip().splitlines()[-1:] or ["failed"]
289
+ return jsonify(ok=False, error=why if isinstance(why, str) else why[0], manual=hostreq.install_lines(b, names))
290
+ return jsonify(ok=True, said="installed")
291
+
292
+
293
+ @app.route("/new-run/dirs")
294
+ def new_run_dirs():
295
+ """Directory completions for the local-checkout field. Names only.
296
+
297
+ Read-only like everything else here, and no more than a shell tab-complete
298
+ reveals: subdirectory names under the typed prefix, never file contents.
299
+ The typed spelling is preserved in the suggestions (a `~` or a relative
300
+ path stays as typed) because the shell, not this app, will resolve it.
301
+ """
302
+ prefix = request.args.get("path", "")
303
+ head, part = os.path.split(os.path.expanduser(prefix))
304
+ try:
305
+ names = sorted(
306
+ n for n in os.listdir(head or ".")
307
+ if n.startswith(part) and (n[:1] != "." or part[:1] == ".")
308
+ and os.path.isdir(os.path.join(head, n))
309
+ )[:20]
310
+ except OSError:
311
+ names = []
312
+ typed_head = os.path.split(prefix)[0]
313
+ return {"dirs": [os.path.join(typed_head, n) if typed_head else n
314
+ for n in names]}
315
+
316
+
317
+ # Where `harbor run -d name@version` resolves names: the registry file the
318
+ # CLI itself defaults to (harbor.constants.DEFAULT_REGISTRY_URL). The CLI's
319
+ # own `datasets list` points at the hub
320
+ # website, whose org/name namespace has no public JSON — so this file is the
321
+ # one machine-readable source of the name@version entries.
322
+ RUN_REGISTRY_URL = (
323
+ "https://raw.githubusercontent.com/laude-institute/harbor/main/registry.json"
324
+ )
325
+
326
+ # One fetch per process — every app start re-fetches on the first page view,
327
+ # so the list is as fresh as the launch, and a dead network is not retried on
328
+ # every view. The page fetches this endpoint after load, so a slow or failed
329
+ # fetch never delays rendering.
330
+ _REGISTRY_SCAN: dict = {}
331
+
332
+
333
+ def _fetch_registry_names():
334
+ """The registry's name@version entries, or None when the fetch failed."""
335
+ try:
336
+ req = urllib.request.Request(
337
+ RUN_REGISTRY_URL, headers={"User-Agent": "nullius"})
338
+ with urllib.request.urlopen(req, timeout=15) as resp:
339
+ data = json.load(resp)
340
+ entries = data.get("datasets", []) if isinstance(data, dict) else data
341
+ return sorted({f"{d['name']}@{d['version']}" for d in entries
342
+ if d.get("name") and d.get("version")})
343
+ except (OSError, ValueError, KeyError, TypeError):
344
+ return None
345
+
346
+
347
+ # The hub is the newer, larger catalog (org/name, resolves latest unless
348
+ # pinned). It has no JSON API — probed 2026-08-18, /api/* all 404 — so its
349
+ # names are collected from the paginated HTML listing.
350
+ RUN_HUB_URL = "https://hub.harborframework.com/datasets"
351
+
352
+
353
+ def _fetch_hub_names():
354
+ """org/name entries from the hub's paginated listing, or None.
355
+
356
+ Walks ?page=N collecting /datasets/org/name links until a page adds
357
+ nothing new; a mid-walk network error keeps what was already collected.
358
+ """
359
+ names: set[str] = set()
360
+ try:
361
+ for page in range(1, 21):
362
+ req = urllib.request.Request(f"{RUN_HUB_URL}?page={page}",
363
+ headers={"User-Agent": "nullius"})
364
+ with urllib.request.urlopen(req, timeout=10) as resp:
365
+ html = resp.read().decode("utf-8", "replace")
366
+ # The links sit inside a React Server Components payload with
367
+ # escaped quotes, so match the path itself, not an href attribute.
368
+ found = set(re.findall(r"/datasets/([\w.-]+/[\w.-]+)", html))
369
+ if not found - names:
370
+ break
371
+ names |= found
372
+ except OSError:
373
+ pass
374
+ return sorted(names) or None
375
+
376
+
377
+ @app.route("/new-run/registry")
378
+ def new_run_registry():
379
+ """Dataset names from Harbor's two catalogs, best-effort.
380
+
381
+ `names` is the registry file's name@version entries (pinned, what the
382
+ README documents); `hub` is the hub website's org/name entries (resolves
383
+ latest unless pinned). The page shows them as two groups of one select —
384
+ only one value ever reaches the command. Model catalogues are fetched
385
+ separately through /new-run/models using the configured provider.
386
+ """
387
+ if "names" not in _REGISTRY_SCAN:
388
+ _REGISTRY_SCAN["names"] = _fetch_registry_names() or []
389
+ _REGISTRY_SCAN["hub"] = _fetch_hub_names() or []
390
+ return _REGISTRY_SCAN
391
+
392
+
393
+ @app.route("/new-run/models", methods=["POST"])
394
+ def new_run_models():
395
+ """Return the selected provider's catalogue, keeping its key on this machine."""
396
+ shared._require_csrf()
397
+ if not shared_local_client():
398
+ abort(403)
399
+ body = request.get_json(silent=True)
400
+ provider = body.get("provider") if isinstance(body, dict) else None
401
+ if providers.split(provider)[0] not in providers.PROVIDERS:
402
+ return jsonify(ok=False, models=[], message="Choose a model provider."), 400
403
+ try:
404
+ return jsonify(ok=True, models=providers.list_models(provider))
405
+ except ValueError as error:
406
+ return jsonify(ok=False, models=[], message=str(error)), 502