nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,223 @@
1
+ """Running a recipe from the builder - here or on a host - and following it."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import re
7
+ import time
8
+ from flask import abort, jsonify, request, url_for
9
+ from nullius import execution, hosts, prerequisites, prices, providers, settings
10
+ from nullius.harbor import source as harbor_source
11
+ from nullius.standard import recipe as standard_recipe
12
+
13
+ from .core import app
14
+ from . import shared
15
+ from . import settings as settings_web
16
+
17
+
18
+ @app.route("/run", methods=["POST"])
19
+ def run_start():
20
+ """Start a plan. The builder posts the runner script and what it needs;
21
+ the target is this machine or one of the Settings' hosts. Same guards
22
+ as every mutation: form token, same origin, and a loopback client -
23
+ running is writing to a machine."""
24
+ shared._require_csrf()
25
+ if not settings_web._local_client():
26
+ abort(403, description="Runs are started from a browser on the application host only.")
27
+ body = request.get_json(silent=True) or {}
28
+ recipe = body.get("recipe") if isinstance(body.get("recipe"), dict) else {}
29
+ if recipe.get("harness") == standard_recipe.HARNESS:
30
+ return _start_standard(recipe)
31
+ where = str(body.get("where") or "local")
32
+ host = None
33
+ if where != "local":
34
+ try:
35
+ host = hosts.parse(where)
36
+ except ValueError as e:
37
+ return jsonify(ok=False, error=str(e)), 400
38
+ if host.entry not in {h.entry for h in _hosts()}:
39
+ return jsonify(ok=False, error=f"{where} is not one of the hosts in Settings"), 400
40
+ if len(json.dumps(recipe)) > 200_000:
41
+ return jsonify(ok=False, error="the recipe is too large to keep with the run"), 400
42
+ if recipe.get("harness", "harbor") != "harbor":
43
+ return jsonify(ok=False, error=providers.GONE), 400
44
+ # A Harbor recipe runs its pin (nullius/harbor/source.py), or harbor on the target's PATH when unpinned.
45
+ # One with a model to call needs its provider's key here (controls alone call none).
46
+ missing = provider_problem(recipe.get("provider") or "openrouter", recipe.get("endpoint")) if body.get("model") else ""
47
+ if missing:
48
+ return jsonify(ok=False, error=missing), 400
49
+ pin, problem = harbor_source.pin_of(recipe.get("harbor"))
50
+ if problem:
51
+ return jsonify(ok=False, error=problem), 400
52
+ # a local task folder as the server reads it, absolute: the check runs from here, the part from its run directory
53
+ tasks = str(body.get("tasks") or "")
54
+ tasks = os.path.abspath(os.path.expanduser(tasks)) if tasks else ""
55
+ plan = execution.Plan(name=str(body.get("name") or ""), script=str(body.get("script") or ""), host=host, tasks=tasks, harbor_pin=pin or {},
56
+ **credentials_for(recipe.get("provider") or "openrouter", recipe.get("endpoint")), # none named: OpenRouter, as the builder reads it
57
+ model=str(body.get("model") or ""), jobs=[j for j in (body.get("jobs") or []) if isinstance(j, dict)],
58
+ parts=[{"id": re.sub(r"[^A-Za-z0-9_.-]", "_", str(p.get("id") or "")), "label": str(p.get("label") or ""), "script": str(p.get("script") or "")}
59
+ for p in (body.get("parts") or []) if isinstance(p, dict)],
60
+ recipe=recipe)
61
+ return _start(plan)
62
+
63
+
64
+ def credentials_for(provider, endpoint=None) -> dict:
65
+ """Where a run's key and endpoint come from, as ``Plan`` fields: a recipe
66
+ on OpenAI is sent to OpenAI's own endpoint, fixed, with OPENAI_API_KEY;
67
+ one on OpenRouter to OpenRouter's own, fixed, with OPENROUTER_API_KEY;
68
+ one on an OpenAI-compatible endpoint - or a provider this does not
69
+ know - with that endpoint's URL and key (``endpoint``, its id: none,
70
+ MODEL_API_URL and MODEL_API_KEY), which such a run cannot go
71
+ without. Every run is given its endpoint: a Harbor part's script requires
72
+ API_URL, and an OpenAI run left without one exited before Harbor began."""
73
+ if provider == "openai":
74
+ return {"key_var": providers.PROVIDER_KEY["openai"], "url_var": "", "base_url": providers.PROVIDER_BASE["openai"]}
75
+ if provider == "openrouter":
76
+ return {"key_var": providers.PROVIDER_KEY["openrouter"], "url_var": "", "base_url": providers.PROVIDER_BASE["openrouter"]}
77
+ url_var, key_var = providers.endpoint_vars(str(endpoint or "") if provider == "custom" else "")
78
+ return {"key_var": key_var, "url_var": url_var, "base_url": ""}
79
+
80
+
81
+ def provider_problem(provider, endpoint=None) -> str:
82
+ """Why a run cannot start here on this provider - its key, or for an
83
+ OpenAI-compatible endpoint its URL, is not set in Settings, or the
84
+ endpoint is gone - or "" when it can. The builder says the same before
85
+ Run (command-builder-core.js, providerBlocker)."""
86
+ if provider == "custom" and endpoint and not settings.ENDPOINT_ID.match(str(endpoint)):
87
+ return "The recipe's endpoint is not an endpoint id: choose an endpoint on the Model step."
88
+ p = providers.choice(provider, str(endpoint or "")) if provider in providers.PROVIDERS else "custom" # as credentials_for reads one it does not know
89
+ have = providers.available(os.environ)
90
+ if p in have:
91
+ return ""
92
+ why = (f"{providers.label(p)} cannot run here: its URL, {providers.needs(p)}, holds a credential or is not an http(s) URL; "
93
+ "put the key in the Key field in Settings." if p in providers.broken(os.environ)
94
+ else f"{providers.label(p)} cannot run here: {providers.needs(p)} is not set in Settings.")
95
+ if not have:
96
+ return why + " Set a model key there to run the recipe."
97
+ return why + " Choose " + " or ".join(providers.label(x) for x in have) + " on the Model step, or set it in Settings."
98
+
99
+
100
+ def standard_plans(recipe: dict, out: dict, known) -> tuple:
101
+ """([Plan], error): one plan per machine the standard recipe's *Where*
102
+ names - its shard there - each a run of its own, or the error that keeps
103
+ every one of them from starting. ``known``: the entries of Settings'
104
+ hosts; a machine not among them is refused."""
105
+ plans = []
106
+ for sh in out["shards"]:
107
+ host = None
108
+ if sh["where"] != "local":
109
+ host = hosts.parse(sh["where"])
110
+ if host.entry not in known:
111
+ return [], f"{sh['where']} is not one of the hosts in Settings"
112
+ plans.append(execution.Plan(name=sh["name"], script=sh["script"], host=host, runner=standard_recipe.HARNESS,
113
+ **credentials_for(recipe.get("provider"), recipe.get("endpoint")), model=out["model"], jobs=sh["jobs"],
114
+ parts=sh["parts"], recipe=recipe, docker=bool(out.get("docker")), keyless=bool(out.get("keyless")),
115
+ licences=dict(out.get("licences") or {}) if host else {}))
116
+ return plans, ""
117
+
118
+
119
+ def _start_standard(recipe: dict):
120
+ """The standard: compiled here from its definitions, like a runner's; on
121
+ this machine, a host, or split over several - one run per machine, every
122
+ one checked before any starts."""
123
+ if len(json.dumps(recipe)) > 200_000:
124
+ return jsonify(ok=False, error="the recipe is too large to keep with the run"), 400
125
+ out = standard_recipe.compile(recipe, docker_unusable=docker_problem, unpriced=unpriced)
126
+ if out["errors"]:
127
+ return jsonify(ok=False, error=" ".join(e["message"] for e in out["errors"])), 400
128
+ # a run of controls calls no model: it is handed no key, so none need be set
129
+ missing = "" if out.get("keyless") else provider_problem(recipe.get("provider"), recipe.get("endpoint"))
130
+ if missing:
131
+ return jsonify(ok=False, error=missing), 400
132
+ plans, problem = standard_plans(recipe, out, {h.entry for h in _hosts()})
133
+ if problem:
134
+ return jsonify(ok=False, error=problem), 400
135
+ if len(plans) == 1:
136
+ return _start(plans[0])
137
+ for plan in plans:
138
+ problem = plan.check()
139
+ if problem:
140
+ return jsonify(ok=False, error=problem), 400
141
+ if any(e.plan.name == plan.name for e in execution.active()):
142
+ return jsonify(ok=False, error=f"{plan.name} is running already"), 409
143
+ if os.path.exists(os.path.join(execution.runs_root(), plan.name)):
144
+ return jsonify(ok=False, error=execution.name_taken(plan.name)), 409
145
+ started = [execution.start(plan) for plan in plans]
146
+ shared.adopt_roots([ex.jobs for ex in started])
147
+ return jsonify(ok=True, open=url_for("index", view="launch"), runs=[ex.summary() for ex in started],
148
+ **started[0].summary())
149
+
150
+
151
+ _PRICED = {"at": float("-inf"), "rows": {}}
152
+
153
+
154
+ def unpriced(models: list) -> list:
155
+ """Which of ``models`` have no price in the user's list (``nullius.prices``): what a recipe with a spending
156
+ cap is told at its Limits step, since a run with a cap is refused without a price for each model it calls."""
157
+ if time.monotonic() - _PRICED["at"] > 5: # the builder asks at every keystroke: the list is read at most every five seconds
158
+ _PRICED.update(at=time.monotonic(), rows=prices.table())
159
+ return [m for m in models if m not in _PRICED["rows"]]
160
+
161
+
162
+ def docker_problem():
163
+ """Why docker cannot run containers on this machine, or None; kept for a
164
+ minute (``prerequisites.docker_here``)."""
165
+ return prerequisites.docker_here()
166
+
167
+
168
+ def _start(plan: execution.Plan):
169
+ problem = plan.check()
170
+ if problem:
171
+ return jsonify(ok=False, error=problem), 400
172
+ if any(e.plan.name == plan.name for e in execution.active()):
173
+ return jsonify(ok=False, error=f"{plan.name} is running already"), 409
174
+ if os.path.exists(os.path.join(execution.runs_root(), plan.name)):
175
+ return jsonify(ok=False, error=execution.name_taken(plan.name)), 409
176
+ ex = execution.start(plan)
177
+ # The job lands (or is mirrored) under runs/<name>/jobs: the workspace
178
+ # reads it from now on. The Runs page, by launch, shows it from the first
179
+ # second: its runs queued, preparing, then running, each opening its job.
180
+ shared.adopt_roots([ex.jobs])
181
+ return jsonify(ok=True, open=url_for("index", view="launch"), **ex.summary())
182
+
183
+
184
+ @app.route("/run/<run_id>")
185
+ def run_status(run_id):
186
+ """State and log lines from ``since`` on; the page polls this."""
187
+ ex = execution.EXECUTIONS.get(run_id)
188
+ if ex is None:
189
+ abort(404)
190
+ since = request.args.get("since", type=int) or 0
191
+ return jsonify(**ex.summary(), log=ex.tail(since))
192
+
193
+
194
+ @app.route("/run/<run_id>/continue", methods=["POST"])
195
+ def run_continue(run_id):
196
+ """The answer to a pause: go on with the next agent."""
197
+ shared._require_csrf()
198
+ if not settings_web._local_client():
199
+ abort(403)
200
+ ex = execution.EXECUTIONS.get(run_id)
201
+ if ex is None:
202
+ abort(404)
203
+ ex.continue_()
204
+ return jsonify(ok=True, **ex.summary())
205
+
206
+
207
+ @app.route("/run/<run_id>/stop", methods=["POST"])
208
+ def run_stop(run_id):
209
+ shared._require_csrf()
210
+ if not settings_web._local_client():
211
+ abort(403)
212
+ ex = execution.EXECUTIONS.get(run_id)
213
+ if ex is None:
214
+ abort(404)
215
+ ex.stop()
216
+ return jsonify(ok=True, **ex.summary())
217
+
218
+
219
+ def _hosts():
220
+ try:
221
+ return hosts.parse_list(os.environ.get("HOSTS", ""))
222
+ except ValueError:
223
+ return []
@@ -0,0 +1,60 @@
1
+ """The feasibility report of a run: what its nop and oracle controls said,
2
+ task by task. Read from recorded jobs; runs nothing.
3
+
4
+ Beside each side that did not behave, the judge's triage
5
+ (`control_failure_cause`, advisory): why the control misbehaved, over that
6
+ control's own small state. The verdict is `feasibility.verdict`'s and is
7
+ never touched; `/judge/feasibility/<job_id>` asks the unjudged sides on the
8
+ user's click (POST)."""
9
+ from __future__ import annotations
10
+
11
+ from flask import abort, flash, redirect, render_template, url_for
12
+ from nullius.judge import client
13
+ from nullius.runs import feasibility
14
+
15
+ from .core import app
16
+ from . import shared
17
+
18
+
19
+ @app.route("/job/<job_id>/feasibility")
20
+ def feasibility_page(job_id):
21
+ from . import judge
22
+ jobs = shared._jobs_for_request()
23
+ j = shared._find_job(jobs, job_id)
24
+ if not j:
25
+ abort(404)
26
+ report = feasibility.report(j, jobs)
27
+ return render_template("runs/feasibility.html", job=j, report=report,
28
+ triage=judge.triage_summary(report, jobs), csrf=shared._csrf_token(),
29
+ F_ORDER=feasibility.ORDER, F_LABELS=feasibility.LABELS, F_DESCRIPTIONS=feasibility.DESCRIPTIONS)
30
+
31
+
32
+ @app.route("/judge/feasibility/<job_id>", methods=["POST"])
33
+ def judge_feasibility_route(job_id):
34
+ """Ask why each misbehaving control side misbehaved, for every side not yet judged, on the click of the button."""
35
+ from . import judge
36
+ jobs = shared._jobs_for_request()
37
+ j = shared._find_job(jobs, job_id)
38
+ if not j:
39
+ abort(404)
40
+ report = feasibility.report(j, jobs)
41
+ summary = judge.triage_summary(report, jobs)
42
+ todo = summary["todo"]
43
+ judge._guard()
44
+ done, failed = 0, []
45
+ for row, side in todo:
46
+ try:
47
+ judge.judge_control(side, jobs, judge.key()); done += 1
48
+ except client.JudgeError as e:
49
+ failed.append(f"{row.task_base} ({side.control}): {e}")
50
+ if e.status in (401, 0):
51
+ break
52
+ flash(f"Judged {done} of {len(todo)} control side(s)." + (" Failed: " + "; ".join(failed[:3]) if failed else ""), "error" if failed else "")
53
+ return redirect(url_for("feasibility_page", job_id=job_id) + "#triage")
54
+
55
+
56
+ @app.template_global()
57
+ def feasibility_controls(job) -> list:
58
+ """The controls recorded with a job, for the link the job page shows
59
+ only when there is a report to read — empty when neither control is."""
60
+ return sorted(feasibility.related(job, shared._jobs_for_request()))
@@ -0,0 +1,231 @@
1
+ """Template filters and the context every page renders with."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import re
6
+ from pathlib import Path
7
+
8
+ from nullius import importers as harnesses
9
+ from nullius.runs import classify
10
+ from nullius import workspace
11
+ from nullius.standard import ledger, ports, definitions
12
+
13
+ from .core import app
14
+ from . import shared
15
+
16
+
17
+ AGENT_MARKS = json.loads((Path(app.static_folder) / "catalogue/agents/marks.json").read_text())
18
+
19
+
20
+
21
+ @app.template_filter("pct")
22
+ def _pct(value):
23
+ return "—" if value is None else f"{value * 100:.0f}%"
24
+
25
+
26
+ @app.template_filter("runtime")
27
+ def _runtime(minutes):
28
+ """A run's start to finish: seconds under a minute, so a run of forty seconds is not "0m"."""
29
+ if minutes is None:
30
+ return "—"
31
+ return f"{minutes * 60:.0f}s" if minutes < 1 else f"{minutes:.0f}m"
32
+
33
+
34
+ @app.template_filter("hitrate")
35
+ def _hitrate(value):
36
+ """Cache hit share. One decimal — two when rounding would claim 100%.
37
+
38
+ 99.97% is the signature of an Anthropic-style denominator;
39
+ flattening it to 100.0% would hide exactly the artifact worth seeing.
40
+ """
41
+ if value is None:
42
+ return "—"
43
+ text = f"{value * 100:.1f}%"
44
+ return f"{value * 100:.2f}%" if text == "100.0%" and value < 1 else text
45
+
46
+
47
+ @app.template_filter("share")
48
+ def _share(value):
49
+ """A segment's share of the wall clock.
50
+
51
+ Never a bare "0%" for time that was really spent: a span worth 40µs of a
52
+ six-second run is 0.0007% and rounding it away would read as "not measured".
53
+ """
54
+ if value is None:
55
+ return "—"
56
+ if value <= 0:
57
+ return "0%"
58
+ if value < 1:
59
+ return "<1%"
60
+ return f"{value:.1f}%" if value < 10 else f"{value:.0f}%"
61
+
62
+
63
+ @app.template_filter("nsdelta")
64
+ def _nsdelta(value):
65
+ """A signed change in nanoseconds — "" when nothing moved.
66
+
67
+ A true minus sign, not a hyphen: these sit next to durations in a legend and
68
+ a hyphen there reads as part of the number.
69
+ """
70
+ if not value:
71
+ return ""
72
+ return ("−" if value < 0 else "+") + _nsdur(abs(value))
73
+
74
+
75
+ @app.template_filter("tokens")
76
+ def _tokens(value, kind="agent"):
77
+ """Token count. For a control the zero is real, so render it as 0.
78
+
79
+ For a real agent, 0/None means the run happened and we failed to capture
80
+ it — that stays "—". Same glyph for both would make a control look broken
81
+ and a broken measurement look deliberate.
82
+ """
83
+ if not value:
84
+ return "0" if kind == "control" else "—"
85
+ if value >= 1_000_000:
86
+ return f"{value / 1_000_000:.2f}M"
87
+ if value >= 1_000:
88
+ return f"{value / 1_000:.1f}k"
89
+ return str(value)
90
+
91
+
92
+ @app.template_filter("usd")
93
+ def _usd(value, kind="agent"):
94
+ """Cost. A control genuinely spent $0.00; an agent with no figure shows —."""
95
+ if value is None:
96
+ return "$0.00" if kind == "control" else "—"
97
+ return f"${value:.4f}" if value < 0.01 else f"${value:.2f}"
98
+
99
+
100
+ @app.template_filter("cap")
101
+ def _cap(value):
102
+ """A spending cap as it was given, and what was spent of it: cents, and the digits past them it has
103
+ ($0.395, $0.00005). The cost filter rounds to the cent, which would show a cap the run never had."""
104
+ return "$" + ledger.usd(value)
105
+
106
+
107
+ @app.template_filter("dsname")
108
+ def _dsname(value):
109
+ return shared.dataset_title(value)
110
+
111
+
112
+ @app.template_filter("dt")
113
+ def _dt(value):
114
+ return value.strftime("%b %d %H:%M") if value else "—"
115
+
116
+
117
+ @app.template_filter("ts")
118
+ def _ts(value):
119
+ """Sort key for a datetime column; missing values sort first."""
120
+ return f"{value.timestamp():.0f}" if value else "-1"
121
+
122
+
123
+ @app.template_filter("phase")
124
+ def _phase(value):
125
+ """Flex weight for a phase segment, or 0 when the phase is effectively nil.
126
+
127
+ Harbor records a control's agent setup as 4.2e-05s — truthy, but it renders
128
+ as "0s" and would still cost a flex gap and a sub-pixel sliver, implying
129
+ setup time that never happened. Anything under the 0.05s that `secs` rounds
130
+ to zero paints nothing at all.
131
+ """
132
+ value = value or 0
133
+ return value if value >= 0.05 else 0
134
+
135
+
136
+ @app.template_filter("secs")
137
+ def _secs(value):
138
+ """Seconds with the precision the clock behind them has.
139
+
140
+ Show tenths below ten seconds, whole seconds below a minute, and tenths
141
+ of minutes above that.
142
+ """
143
+ if value is None:
144
+ return "—"
145
+ if value < 9.95: # what .1f would still show as one digit
146
+ return f"{value:.1f}s"
147
+ return f"{value:.0f}s" if value < 60 else f"{value / 60:.1f}m"
148
+
149
+
150
+ @app.template_filter("nsround")
151
+ def _nsround(value):
152
+ """A duration for a chart's axis or headline: ``100ms``, not ``100.00ms``."""
153
+ return re.sub(r"\.0+(?=\D|$)", "", _nsdur(value))
154
+
155
+
156
+ @app.template_filter("nsdur")
157
+ def _nsdur(value):
158
+ """A span duration, scaled to its own magnitude.
159
+
160
+ None is not zero: it means the span recorded one instant and has no known
161
+ end. Rendering that as "0.00ms" would claim a measurement nobody made.
162
+ """
163
+ if value is None:
164
+ return "—"
165
+ if value < 1_000:
166
+ return f"{value}ns"
167
+ if value < 1_000_000:
168
+ return f"{value / 1_000:.0f}µs"
169
+ if value < 1_000_000_000:
170
+ return f"{value / 1_000_000:.2f}ms"
171
+ if value < 60_000_000_000:
172
+ return f"{value / 1_000_000_000:.2f}s"
173
+ # Hands over to `secs`'s scale at the same minute mark, so a trace covering
174
+ # an hour reads like the phase bar above it instead of as "3600.00s".
175
+ return f"{value / 60_000_000_000:.1f}m"
176
+
177
+
178
+ @app.template_filter("nsoff")
179
+ def _nsoff(value):
180
+ """An offset from the start of the trace — always relative.
181
+
182
+ Never render the absolute epoch nanoseconds these are derived from: the
183
+ trace's own clock anchor exists precisely because the container clock and
184
+ the host clock disagree, so an absolute instant is not the exact thing its
185
+ 19 digits imply. A difference between two stamps from the same clock is.
186
+ """
187
+ if value is None:
188
+ return "—"
189
+ if not value:
190
+ return "0" # the axis origin, not "+0ns"
191
+ return "+" + _nsdur(value)
192
+
193
+
194
+ @app.context_processor
195
+ def _inject():
196
+ return {
197
+ "LABELS": classify.LABELS,
198
+ "agent_marks": AGENT_MARKS,
199
+ "DESCRIPTIONS": classify.DESCRIPTIONS,
200
+ "INFRA": classify.INFRA_OUTCOMES,
201
+ "ORDER": classify.ORDER,
202
+ "workspace": shared.STATE["workspace"],
203
+ "record_workspace": shared._record_workspace(),
204
+ # For the rail's switcher: the most recent few, and a stable hue per
205
+ # name so a workspace keeps its colour across pages and restarts.
206
+ "workspaces": workspace.list_all(shared.WS_HOME)[:8],
207
+ "ws_hue": lambda name: sum(ord(c) * (i + 7) for i, c in enumerate(name or "")) % 360,
208
+ # The open workspace's runs, for the rail's tree — from the cache index,
209
+ # so listing them costs one small file, not a parse.
210
+ "nav_runs": sorted(
211
+ workspace.job_index(shared.STATE["workspace"]).values(), key=lambda e: e["job_id"]
212
+ )[:12] if shared.STATE["workspace"] else [],
213
+ # The catalogue's two trees in the rail: one small JSON read, cached
214
+ # on its mtime so a page does not re-read it.
215
+ "nav_catalog": shared._catalog_cached(),
216
+ "nav_kits": [{"key": p["kit"], "label": p["label"]}
217
+ for p in ports.PORTS.values() if p.get("kit")],
218
+ "nav_definitions": {
219
+ kind: sorted(({"key": d["key"], "label": definitions.title(d), "example": bool(d.get("example"))}
220
+ for d in definitions.load(kind)[0]), key=lambda d: (not d["example"], d["label"].casefold()))
221
+ for kind in ("agent", "capability")
222
+ },
223
+ # Named on the import page so what is readable stays discoverable
224
+ # without asking anyone to choose it. Empty when no importer is
225
+ # installed, and the sentence disappears with it.
226
+ "IMPORTERS": [
227
+ getattr(m, "LABEL", n.upper()) for n, m in harnesses.available().items()
228
+ ],
229
+ # The runs list's Columns menu; the template reads it by this name.
230
+ "COLUMNS": shared.COLUMNS,
231
+ }