nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,20 @@
1
+ """Getting started: the guide's checklist, read fresh on each ask
2
+ (`nullius.onboarding`). Booleans only: no key or path leaves in it."""
3
+ from __future__ import annotations
4
+
5
+ import os
6
+
7
+ from flask import jsonify, url_for
8
+
9
+ from nullius import onboarding, workspace
10
+
11
+ from .core import app
12
+ from . import shared
13
+
14
+
15
+ @app.route("/onboarding")
16
+ def onboarding_state():
17
+ ws = shared.STATE["workspace"]
18
+ jobs = workspace.job_index(ws).values() if ws else ()
19
+ found = onboarding.facts(os.environ, jobs, onboarding.any_judged(ws.dir if ws else None))
20
+ return jsonify(sections=onboarding.checklist(found, url_for))
@@ -0,0 +1,47 @@
1
+ """Reconciliation: the classifier's outcome beside the judge's
2
+ failure_cause, per trial of a job, from the workspace store - where they
3
+ disagree, ranked strongest first, and a suggestion for the trials the
4
+ classifier could not place. Reads the store; asks the judge nothing;
5
+ writes nothing. Advisory beside the recorded evidence, never in its
6
+ place."""
7
+ from __future__ import annotations
8
+
9
+ from flask import abort, render_template, request
10
+ from nullius.judge import questions, reconcile, store
11
+ from nullius.runs import classify
12
+
13
+ from .core import app
14
+ from . import shared
15
+
16
+
17
+ def ws_dir():
18
+ ws = shared.STATE.get("workspace")
19
+ return ws.dir if ws else None
20
+
21
+
22
+ def threshold_requested() -> float:
23
+ """`?threshold=` as a number in 0..1, else the module's default."""
24
+ raw = request.args.get("threshold")
25
+ try:
26
+ t = float(raw)
27
+ except (TypeError, ValueError):
28
+ return reconcile.THRESHOLD
29
+ return min(1.0, max(0.0, t))
30
+
31
+
32
+ def view_of(job, threshold: float) -> dict:
33
+ d = ws_dir()
34
+ records = store.for_job(d, job.job_id) if d else {}
35
+ return reconcile.reconcile(job, records, threshold, key=lambda name: store._SAFE.sub("_", name))
36
+
37
+
38
+ @app.route("/job/<job_id>/reconcile")
39
+ def reconcile_page(job_id):
40
+ j = shared._find_job(shared._jobs_for_request(), job_id)
41
+ if not j:
42
+ abort(404)
43
+ return render_template("runs/reconcile.html", job=j, view=view_of(j, threshold_requested()),
44
+ STATUSES=reconcile.STATUSES, STATUS_LABELS=reconcile.STATUS_LABELS,
45
+ COMPATIBLE=reconcile.COMPATIBLE, SUGGESTS=reconcile.SUGGESTS,
46
+ C_ORDER=classify.ORDER, C_LABELS=classify.LABELS, default_threshold=reconcile.THRESHOLD,
47
+ QUESTIONS_VERSION=questions.VERSION)
@@ -0,0 +1,464 @@
1
+ """The runs list, a job's page and its live board, the workspace picker and upload."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import tempfile
8
+ from flask import (
9
+ abort,
10
+ flash,
11
+ jsonify,
12
+ redirect,
13
+ render_template,
14
+ request,
15
+ url_for,
16
+ )
17
+ from markupsafe import Markup, escape
18
+ from nullius import execution
19
+ from nullius import importers as harnesses
20
+ from nullius.redact import redact
21
+ from nullius.runs import classify, feasibility, launches, live, parse
22
+ from nullius import workspace
23
+
24
+ from .core import app
25
+ from . import shared
26
+
27
+
28
+
29
+ @app.route("/")
30
+ def home():
31
+ """Open the application on Runs, with its current workspace or first steps."""
32
+ return redirect(url_for("index"))
33
+
34
+
35
+ @app.route("/runs")
36
+ def index():
37
+ if shared.STATE["workspace"] is None:
38
+ return render_template("runs/start.html", workspaces=workspace.list_all(shared.WS_HOME))
39
+ jobs = shared._load_jobs()
40
+ # Runs started from the builder whose job Harbor has not written yet:
41
+ # listed above the table, each a link to the page that follows it.
42
+ have = {j.job_id for j in jobs}
43
+ # The run records are the app's, not the workspace's: only those launched
44
+ # into this workspace's roots, or with a job already written here, belong.
45
+ mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have)]
46
+ merged_sources = launches.merged_sources(jobs)
47
+ jobs = [j for j in jobs if j.job_id not in merged_sources]
48
+ preparing = [(jid, e) for e in mine if e.state in execution.ALIVE for jid in e.summary()["job_ids"] if jid not in have]
49
+ # a launch whose runs wrote no job yet (or never will: refused, stopped) is still shown
50
+ if not jobs and not mine:
51
+ return render_template("runs/setup.html", scanned=True, mode="add",
52
+ workspaces=workspace.list_all(shared.WS_HOME))
53
+ # The header, the chips and the mix bar describe the agents' trials. A
54
+ # control (nop, oracle) is a check on the tasks, not a competitor: it is
55
+ # listed on its own line as behaved or not (`feasibility.summary_of`),
56
+ # and its trials never enter the totals — a nop rejected on every task
57
+ # is the expected result, not five failed verifications.
58
+ verdicts = {j.job_id: feasibility.summary_of(j) for j in jobs}
59
+ # A Harbor killed by a Stop (or dead with the run) leaves its result
60
+ # record saying trials are running; the runner's record says otherwise.
61
+ ended = {j.job_id: execution.ended_for(j.job_id) for j in jobs}
62
+ agents = [j for j in jobs if verdicts[j.job_id] is None]
63
+ controls = [(j, verdicts[j.job_id]) for j in jobs if verdicts[j.job_id] is not None]
64
+ totals = {k: 0 for k in classify.ORDER}
65
+ n_trials = 0
66
+ for j in agents:
67
+ for k, v in j.counts.items():
68
+ totals[k] += v
69
+ n_trials += len(j.trials)
70
+
71
+ # Only describe datasets these runs actually used, most-used first.
72
+ used = {}
73
+ for j in jobs:
74
+ entry = used.setdefault(j.dataset, {"runs": 0, "trials": 0})
75
+ entry["runs"] += 1
76
+ entry["trials"] += len(j.trials)
77
+ datasets = [
78
+ {"name": name, "info": shared.dataset_info(name), **counts}
79
+ for name, counts in sorted(used.items(), key=lambda kv: -kv[1]["trials"])
80
+ ]
81
+
82
+ # Distinct values behind the per-column filter menus, as (value, label).
83
+ # Value is what the row carries; label is what the menu shows.
84
+ choices = {
85
+ "dataset": sorted(
86
+ {(j.dataset, shared.dataset_title(j.dataset)) for j in jobs}, key=lambda p: p[1]
87
+ ),
88
+ "model": sorted({(j.model_name, j.model_name) for j in jobs}),
89
+ "agent": sorted({(j.agent_label, j.agent_label or "—") for j in jobs}),
90
+ }
91
+ # The page's first view: the runs grouped by the launch that made them
92
+ # (nullius.runs.launches) - the builder's run record, else the same dataset
93
+ # within half an hour - each with its task checks and its agents side by side.
94
+ records = {e.id: (e.plan.name, [str(x.get("name")) for x in e.plan.jobs
95
+ if str(x.get("name")) not in merged_sources], e.started) for e in mine}
96
+ by_id = {j.job_id: j for j in jobs}
97
+ groups = [_launch_view(g, by_id, verdicts, ended) for g in launches.group(jobs, records)]
98
+ view = request.args.get("view") if request.args.get("view") in ("launch", "all") else "launch"
99
+ states = {}
100
+ for g in groups:
101
+ for row in g["agents"]:
102
+ states[row["state"]] = states.get(row["state"], 0) + 1
103
+ return render_template(
104
+ "runs/index.html", jobs=jobs, totals=totals, n_trials=n_trials,
105
+ n_agent_runs=len(agents), controls=controls, verdicts=verdicts, ended=ended,
106
+ datasets=datasets, choices=choices, pulse=_roots_pulse(), preparing=preparing,
107
+ launches=groups, view=view, states=states,
108
+ n_checked=sum(1 for g in groups if g["checked"]),
109
+ )
110
+
111
+
112
+ def _launched_here(ex, have) -> bool:
113
+ """A run record belongs to the open workspace when its job root is one of
114
+ the workspace's roots (a Run adopts it there) or one of its jobs is here."""
115
+ if any(str(x.get("name")) in have for x in ex.plan.jobs):
116
+ return True
117
+ jobs_dir = getattr(ex, "jobs", None)
118
+ roots = list(shared.STATE.get("roots") or []) + list(shared.STATE.get("job_roots") or [])
119
+ return bool(jobs_dir) and os.path.realpath(jobs_dir) in {os.path.realpath(r) for r in roots}
120
+
121
+
122
+ # A launch's runs, in the order the block lists them: finished first, best
123
+ # adjusted rate on top; then those still going; then the ones that stopped.
124
+ _STATE_ORDER = {"finished": 0, "running": 1, "preparing": 2, "queued": 3, "stopping": 4, "paused": 5, "failed": 6, "refused": 7, "stopped": 8, execution.NOT_RUN: 8.5,
125
+ "not in this workspace": 9}
126
+
127
+
128
+ def _state_of(job_id, j, ended):
129
+ """What a run is doing now, in the words the block shows. The runner's
130
+ record wins over Harbor's result record, which a Stop leaves saying
131
+ "running"."""
132
+ if j is not None and ended.get(job_id):
133
+ how = ended[job_id][0]
134
+ return "finished" if how == "done" else how
135
+ ex = execution.for_job(job_id)
136
+ if ex is not None and ex.state in execution.ALIVE:
137
+ return ex.live_state(job_id)
138
+ if j is None:
139
+ # named by a run record and not under this workspace's roots: still to be written while the run lives, refused
140
+ # before it started (its folder says so, or no part was launched), else elsewhere
141
+ if ex is not None and ex.state not in execution.ALIVE:
142
+ return "refused" if ex.refused(job_id) else execution.NOT_RUN if ex.never_ran(job_id) else "not in this workspace"
143
+ return ex.state if ex is not None else "preparing"
144
+ if j.is_preparing:
145
+ return "preparing"
146
+ if j.is_running:
147
+ return "running"
148
+ return "finished"
149
+
150
+
151
+ def _control_planned(job_id) -> str:
152
+ """``nop`` or ``oracle`` when a job Harbor has not written yet is one of
153
+ its controls by the run's plan, else ''. A runner's arm is never one."""
154
+ ex = execution.for_job(job_id)
155
+ if ex is None or ex.plan.runner != "harbor":
156
+ return ""
157
+ agent = next((str(j.get("agent") or "") for j in ex.plan.jobs if str(j.get("name")) == job_id), "").lower()
158
+ return agent if agent in parse.CONTROL_AGENTS else ""
159
+
160
+
161
+ def _launch_view(g, by_id, verdicts, ended) -> dict:
162
+ """One launch as its block draws it: the task checks, the agents ranked on
163
+ one scale, the runs its record named that Harbor has not written yet."""
164
+ jobs = [by_id[i] for i in g.job_ids]
165
+ checks = [(j, verdicts[j.job_id]) for j in jobs if verdicts[j.job_id] is not None]
166
+ rows = [{"job": j, "job_id": j.job_id, "state": _state_of(j.job_id, j, ended)} for j in jobs if verdicts[j.job_id] is None]
167
+ waiting = [(i, c) for i, c in ((i, _control_planned(i)) for i in g.pending) if c]
168
+ rows += [{"job": None, "job_id": i, "state": _state_of(i, None, ended)} for i in g.pending if i not in dict(waiting)]
169
+ for r in rows:
170
+ if r["state"] == "refused":
171
+ r["why"] = execution.for_job(r["job_id"]).refused(r["job_id"])
172
+ rate = lambda r: r["job"].adjusted_solve_rate if r["job"] is not None and r["job"].adjusted_solve_rate is not None else -1.0
173
+ rows.sort(key=lambda r: (_STATE_ORDER.get(r["state"], 9), -rate(r), r["job_id"]))
174
+ ok = {v.control for _, v in checks if v.state == "ok"}
175
+ if waiting:
176
+ verdict, checked = "Not checked yet: a control has not finished.", False
177
+ elif not checks:
178
+ verdict, checked = None, False
179
+ elif any(v.state == "warn" for _, v in checks):
180
+ verdict, checked = "A control did not behave: some tasks may be broken. See Feasibility.", False
181
+ elif any(v.state == "pending" for _, v in checks):
182
+ verdict, checked = "Not checked yet: a control has not finished.", False
183
+ elif {"nop", "oracle"} <= ok:
184
+ verdict, checked = "Every task is feasible: an empty answer fails and the reference passes.", True
185
+ elif "nop" in ok:
186
+ verdict, checked = "An empty answer fails every task; no oracle ran to show each can pass.", True
187
+ else:
188
+ verdict, checked = "The reference passes every task; no nop ran to show an empty answer fails.", True
189
+ n_tasks = max([j.n_planned or len(j.trials) for j in jobs] + [0]) or None
190
+ finished = [r["job_id"] for r in rows if r["state"] == "finished" and r["job"] is not None and r["job"].has_verdict]
191
+ # The controls are rows of their own at the top of the table, written or still to come, nop before oracle:
192
+ # a check on the tasks, never ranked with the agents nor offered to Compare, and kept once they end.
193
+ controls = [{"job": j, "job_id": j.job_id, "state": _state_of(j.job_id, j, ended), "control": v.control, "verdict": v} for j, v in checks]
194
+ controls += [{"job": None, "job_id": i, "state": _state_of(i, None, ended), "control": c, "verdict": None} for i, c in waiting]
195
+ controls.sort(key=lambda r: (feasibility.CONTROLS.index(r["control"]) if r["control"] in feasibility.CONTROLS else 9, r["job_id"]))
196
+ counts = {}
197
+ for r in controls + rows:
198
+ counts[r["state"]] = counts.get(r["state"], 0) + 1
199
+ ex = execution.for_job((g.job_ids + g.pending)[0])
200
+ return {"launch": g, "checks": checks, "verdict": verdict,
201
+ "controls": controls, "checked": checked, "agents": rows,
202
+ "n_tasks": n_tasks, "compare": finished,
203
+ "paused": next((i for i in g.job_ids + g.pending
204
+ if (execution.for_job(i) is not None and execution.for_job(i).state == "paused")), None),
205
+ "summary": " · ".join(f"{n} {s}" for s, n in sorted(counts.items(), key=lambda kv: _STATE_ORDER.get(kv[0], 9))),
206
+ "recipe": (g.job_ids + g.pending)[0] if g.source == "record" and ex is not None and ex.plan.recipe else None,
207
+ "feasibility": checks[0][0].job_id if checks else None,
208
+ "dataset_title": shared.dataset_title(g.dataset) if g.dataset and g.dataset != "mixed" else (g.dataset or "—")}
209
+
210
+
211
+ def _roots_pulse() -> str:
212
+ """A short signature of what the job roots hold right now: each job
213
+ directory's name, whether it has a result record and a feed, and the
214
+ feed's size. It changes when a run is launched (a directory appears),
215
+ while it runs (the feed grows) and when it ends (the result lands) — and
216
+ at no other time, so a page that polls it reloads only for a reason.
217
+ Nothing is parsed; this is a handful of stats per job."""
218
+ dirs = parse.find_job_dirs(shared.STATE["job_roots"] or [])
219
+ have = {os.path.basename(d) for d in dirs}
220
+ marks = [(f"run:{e.id}", e.state, 0) for e in execution.EXECUTIONS.values() if _launched_here(e, have)]
221
+ for d in dirs:
222
+ feed = os.path.join(d, "events.jsonl")
223
+ marks.append((os.path.basename(d), os.path.isfile(os.path.join(d, "result.json")),
224
+ os.path.getsize(feed) if os.path.isfile(feed) else -1))
225
+ return hashlib.sha1(json.dumps(sorted(marks)).encode()).hexdigest()[:16]
226
+
227
+
228
+ @app.route("/runs/pulse")
229
+ def runs_pulse():
230
+ """What the runs list polls. ``signature`` is :func:`_roots_pulse`; a
231
+ change means the list is stale."""
232
+ response = jsonify({"signature": _roots_pulse()})
233
+ response.headers["Cache-Control"] = "no-store"
234
+ return response
235
+
236
+
237
+ def _readable(roots, shown=lambda root: root):
238
+ """Split *roots* into the ones a workspace can adopt and the ones it must
239
+ not, each of those with why.
240
+
241
+ A root is refused when the importer that claims it cannot read it — a
242
+ ``run.json`` that is not JSON, a log of the wrong shape — or when nothing
243
+ claims it and nothing in it is job-shaped either. Adopting either would
244
+ have been reported as success: the first as a job with no trials, the
245
+ second as a workspace with nothing new in it, and neither says what was
246
+ wrong with the folder. The message is redacted before it is shown: a
247
+ file's path or content is arbitrary text, and a credential in a bad one
248
+ must not reach the page because the file was bad.
249
+ """
250
+ kept, refused = [], []
251
+ for root in roots:
252
+ why = harnesses.problem(root)
253
+ if why is None and not parse.find_job_dirs([root]) and harnesses.detect(root) is None:
254
+ why = (f"nothing job-shaped here, and no importer recognises it "
255
+ f"(known: {harnesses.labels()})")
256
+ if why is None:
257
+ kept.append(root)
258
+ else:
259
+ refused.append((shown(root), why))
260
+ return kept, refused
261
+
262
+
263
+ def _refused_message(refused):
264
+ """One line per refused root, the root in monospace like `_bad_paths_message`."""
265
+ return Markup("<br>".join(
266
+ f'Not imported: <span class="mono">{escape(redact(root))}</span> — {escape(why)}'
267
+ for root, why in refused))
268
+
269
+
270
+ def _after_import(good, refused, new):
271
+ """Where a submission lands: the runs list — or, when something was
272
+ refused, the picker, which is the page that shows the message. The runs
273
+ list does not, and a refusal nobody sees is a silent success."""
274
+ if refused:
275
+ flash(_refused_message(refused), "error")
276
+ return redirect(url_for("import_page", new="1" if new and not good else None))
277
+ return redirect(url_for("index"))
278
+
279
+
280
+ @app.route("/use-path", methods=["POST"])
281
+ def use_path():
282
+ shared._require_csrf()
283
+ good, bad = shared.resolve_submitted_paths(request.form.getlist("path"))
284
+ if bad:
285
+ flash(shared._bad_paths_message(bad), "error")
286
+ good, refused = _readable(good)
287
+ new = request.form.get("new") == "1"
288
+ if good:
289
+ shared.adopt_roots(good, label=(request.form.get("name") or "").strip(), new=new)
290
+ return _after_import(good, refused, new)
291
+
292
+
293
+ @app.route("/api/folders", methods=["POST"])
294
+ def folder_suggestions():
295
+ shared._require_csrf()
296
+ response = jsonify(paths=shared.suggest_directories(request.form.get("path", "")))
297
+ response.headers["Cache-Control"] = "no-store"
298
+ return response
299
+
300
+
301
+ @app.route("/change-source", methods=["POST"])
302
+ def change_source():
303
+ """Close the workspace and return to the Runs page's first steps."""
304
+ shared._require_csrf()
305
+ shared.close_workspace()
306
+ return redirect(url_for("index"))
307
+
308
+
309
+ @app.route("/import")
310
+ def import_page():
311
+ """The picker: add runs to the open workspace, or start a new one."""
312
+ mode = "new" if request.args.get("new") == "1" or shared.STATE["workspace"] is None else "add"
313
+ return render_template("runs/setup.html", scanned=False, mode=mode,
314
+ workspaces=workspace.list_all(shared.WS_HOME))
315
+
316
+
317
+ @app.route("/upload", methods=["POST"])
318
+ def upload():
319
+ shared._require_csrf()
320
+ files = request.files.getlist("files")
321
+ files = [f for f in files if f.filename]
322
+ # One Parse button serves both halves of the form, so typed paths can
323
+ # arrive alongside uploaded folders. Both end up in the same root list.
324
+ typed, bad = shared.resolve_submitted_paths(request.form.getlist("path"))
325
+ if bad:
326
+ flash(shared._bad_paths_message(bad), "error")
327
+ if not files:
328
+ if typed:
329
+ shared.set_source(typed)
330
+ else:
331
+ flash("No files received — nothing to scan.", "error")
332
+ return redirect(url_for("index"))
333
+
334
+ tmp = tempfile.mkdtemp(prefix="nullius_jobs_")
335
+ saved = 0
336
+ for f in files:
337
+ dest = shared._safe_dest(tmp, f.filename)
338
+ if dest is None:
339
+ continue
340
+ os.makedirs(os.path.dirname(dest), exist_ok=True)
341
+ f.save(dest)
342
+ saved += 1
343
+ if not saved:
344
+ flash("Upload failed — no files could be saved.", "error")
345
+ return redirect(url_for("index"))
346
+
347
+ # Scan uploaded folders without also scanning tmp, which would import them twice.
348
+ # Use tmp only for loose files; name sources and errors after the uploaded paths.
349
+ top = sorted(n for n in os.listdir(tmp) if os.path.isdir(os.path.join(tmp, n)))
350
+ dropped, refused = _readable([os.path.join(tmp, n) for n in top] or [tmp],
351
+ shown=lambda root: os.path.relpath(root, tmp))
352
+ typed, refused_typed = _readable(typed)
353
+ refused += refused_typed
354
+ kept_top = [os.path.basename(d) for d in dropped if d != tmp]
355
+ roots = dropped + typed
356
+ if len(kept_top) == 1:
357
+ label = f"{kept_top[0]} (uploaded)"
358
+ elif kept_top:
359
+ label = f"{len(kept_top)} folders (uploaded)"
360
+ else:
361
+ label = f"{saved} files (uploaded)"
362
+ if typed:
363
+ label += f" +{len(typed)} local"
364
+ new = request.form.get("new") == "1"
365
+ if roots:
366
+ shared.adopt_roots(roots, label=(request.form.get("name") or "").strip() or label,
367
+ new=new)
368
+ return _after_import(roots, refused, new)
369
+
370
+
371
+ @app.route("/job/<job_id>", methods=["GET", "HEAD"])
372
+ def job(job_id):
373
+ j = shared._find_job(shared._jobs_for_request(), job_id)
374
+ ex = execution.for_job(job_id)
375
+ # this job's own state beside the run's: ended, or a part still to come while another runs
376
+ own = ((execution.NOT_RUN, None) if ex is not None and ex.state not in execution.ALIVE and ex.never_ran(job_id)
377
+ else execution.ended_for(job_id) or ((ex.live_state(job_id), None)
378
+ if ex is not None and ex.live_state(job_id) in ("queued", execution.NOT_RUN) else None))
379
+ if not j:
380
+ # Started from the builder and not written by Harbor yet: the run's
381
+ # own page, which becomes this one when the job appears.
382
+ if ex is not None and request.method == "GET":
383
+ return render_template("runs/preparing.html", job_id=job_id, ex=ex, csrf=shared._csrf_token(), own=own)
384
+ abort(404)
385
+ from . import judge
386
+ board = _live_board_state(j)
387
+ return render_template("runs/job.html", job=j, live_board=board, view=_view_of(board, j),
388
+ execution=ex, csrf=shared._csrf_token(),
389
+ judgement=judge.job_summary(j) if j.trials else None,
390
+ control=feasibility.summary_of(j), ended=execution.ended_for(job_id), own=own)
391
+
392
+
393
+ def _view_of(board, j=None) -> str | None:
394
+ """Which tab the job page opens on, or None when it has no tabs.
395
+
396
+ A job with a board has two views of one page: Live, the feed folded as it
397
+ grows, and Results, the page rendered from the job's files. ``?view=`` wins,
398
+ so a link opens the view it was copied from; otherwise Live while the feed
399
+ says the job is going, Results once it ended - unless there are no results
400
+ to show: a job with a feed and no result record yet (a runner's arm whose
401
+ importer has not written one) stays on Live. A job with no feed, a broken
402
+ one or an empty one has no Live view, and no tabs.
403
+ """
404
+ if not board or not board.get("initial") or board["initial"]["state"] == "unknown":
405
+ return None
406
+ asked = request.args.get("view")
407
+ if asked in ("live", "results"):
408
+ return asked
409
+ return "live" if board["running"] or (j is not None and j.is_preparing) else "results"
410
+
411
+
412
+ def _live_board_state(j):
413
+ """Whether the job page has a live board, and from what.
414
+
415
+ The board is the job page's Live view, not a second page: it polls while
416
+ the feed says the job has not ended, and once it ended it stays as the
417
+ record of how the run went, beside Results. A job without a feed has no
418
+ live board.
419
+ """
420
+ if not live.has_feed(j.dir):
421
+ return None
422
+ try:
423
+ snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True)
424
+ except live.FeedError as e:
425
+ # A malformed line in the middle of the feed is a broken writer, not a
426
+ # race. The page must still render — the settled sections read
427
+ # result.json, not the feed — so the board steps aside and says why.
428
+ return {"running": False, "initial": None, "error": str(e)}
429
+ # "running" here means "show the board and keep polling": the preparation
430
+ # window and the starting state are on the board too, so it is every
431
+ # state but finished — and unknown, an empty feed, which has nothing to show.
432
+ return {"running": snap["state"] in ("preparing", "starting", "running"), "initial": snap}
433
+
434
+
435
+ @app.route("/job/<job_id>/live")
436
+ def job_live(job_id):
437
+ """The board's snapshot, folded fresh from the feed on every call.
438
+
439
+ Deliberately not through the workspace cache: the cache is keyed by the
440
+ files parse reads, and a feed that grows every second would either miss
441
+ the cache on every request or, worse, hit it and show a board that stopped.
442
+ Folding a few thousand lines is cheap enough to do every second.
443
+ """
444
+ j = shared._find_job(shared._jobs_for_request(), job_id)
445
+ if not j:
446
+ abort(404)
447
+ if not live.has_feed(j.dir):
448
+ ended = execution.ended_for(job_id)
449
+ if ended:
450
+ # The preparing page must settle even if the runner never wrote a feed.
451
+ response = jsonify({"state": "finished", "running": False, "ended_by": ended[0]})
452
+ response.headers["Cache-Control"] = "no-store"
453
+ return response
454
+ abort(404)
455
+ try:
456
+ snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True)
457
+ except live.FeedError as e:
458
+ response = jsonify({"error": f"events.jsonl is not a feed: {e}"})
459
+ response.status_code = 422
460
+ response.headers["Cache-Control"] = "no-store"
461
+ return response
462
+ response = jsonify(snap)
463
+ response.headers["Cache-Control"] = "no-store"
464
+ return response