nullius-eval 0.1.0rc1__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (268) hide show
  1. _nullius_app/.env.example +54 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +16 -0
  5. _nullius_app/static/brand/favicon.svg +1 -0
  6. _nullius_app/static/brand/mark-dark.svg +1 -0
  7. _nullius_app/static/brand/marks.svg +34 -0
  8. _nullius_app/static/brand/tokens.json +51 -0
  9. _nullius_app/static/brand/typesafe.png +0 -0
  10. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  11. _nullius_app/static/builder/command-builder-core.js +744 -0
  12. _nullius_app/static/builder/command-builder.css +248 -0
  13. _nullius_app/static/builder/command-builder.js +994 -0
  14. _nullius_app/static/builder/model-picker.js +113 -0
  15. _nullius_app/static/catalogue/add.js +46 -0
  16. _nullius_app/static/catalogue/agents/README.md +19 -0
  17. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  18. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  19. _nullius_app/static/catalogue/agents/marks.json +8 -0
  20. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  21. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  22. _nullius_app/static/catalogue/catalogue.css +453 -0
  23. _nullius_app/static/catalogue/catalogue.js +87 -0
  24. _nullius_app/static/catalogue/marks.svg +25 -0
  25. _nullius_app/static/catalogue/new.js +78 -0
  26. _nullius_app/static/catalogue/versions.js +27 -0
  27. _nullius_app/static/compare/comparison.css +117 -0
  28. _nullius_app/static/controls.js +117 -0
  29. _nullius_app/static/disclosures.js +70 -0
  30. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  31. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  32. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  33. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  36. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/README.md +11 -0
  38. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  39. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  40. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  41. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  42. _nullius_app/static/fonts.css +49 -0
  43. _nullius_app/static/notifications.js +34 -0
  44. _nullius_app/static/onboarding.css +78 -0
  45. _nullius_app/static/onboarding.js +158 -0
  46. _nullius_app/static/runs/board.css +354 -0
  47. _nullius_app/static/runs/board.js +127 -0
  48. _nullius_app/static/runs/launches.css +87 -0
  49. _nullius_app/static/runs/live-board.css +186 -0
  50. _nullius_app/static/runs/live-board.js +387 -0
  51. _nullius_app/static/runs/path-suggestions.js +119 -0
  52. _nullius_app/static/runs/start.css +38 -0
  53. _nullius_app/static/settings/autosave.js +145 -0
  54. _nullius_app/static/settings/judges.js +61 -0
  55. _nullius_app/static/settings/modal.css +32 -0
  56. _nullius_app/static/settings/modal.js +91 -0
  57. _nullius_app/static/settings/navigation.js +55 -0
  58. _nullius_app/static/settings/providers.js +78 -0
  59. _nullius_app/static/settings/sandbox.js +83 -0
  60. _nullius_app/static/settings/settings.css +229 -0
  61. _nullius_app/static/style.css +1751 -0
  62. _nullius_app/static/traces/conversation.css +49 -0
  63. _nullius_app/static/traces/conversation.js +215 -0
  64. _nullius_app/templates/agents/entry.html +145 -0
  65. _nullius_app/templates/agents/kits.html +55 -0
  66. _nullius_app/templates/agents/standard.html +58 -0
  67. _nullius_app/templates/base.html +477 -0
  68. _nullius_app/templates/builder/new_run.html +55 -0
  69. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  70. _nullius_app/templates/catalogue/benchmark.html +301 -0
  71. _nullius_app/templates/catalogue/benchmarks.html +92 -0
  72. _nullius_app/templates/catalogue/new.html +141 -0
  73. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  74. _nullius_app/templates/compare/_detail.html +37 -0
  75. _nullius_app/templates/compare/compare.html +298 -0
  76. _nullius_app/templates/compare/compare_picker.html +51 -0
  77. _nullius_app/templates/compare/compare_task.html +329 -0
  78. _nullius_app/templates/runs/_execution.html +131 -0
  79. _nullius_app/templates/runs/_runtime.html +5 -0
  80. _nullius_app/templates/runs/ascii-mark.html +485 -0
  81. _nullius_app/templates/runs/feasibility.html +127 -0
  82. _nullius_app/templates/runs/index.html +609 -0
  83. _nullius_app/templates/runs/job.html +446 -0
  84. _nullius_app/templates/runs/preparing.html +14 -0
  85. _nullius_app/templates/runs/reconcile.html +124 -0
  86. _nullius_app/templates/runs/setup.html +272 -0
  87. _nullius_app/templates/runs/start.html +51 -0
  88. _nullius_app/templates/settings/settings.html +508 -0
  89. _nullius_app/templates/traces/_charts.html +165 -0
  90. _nullius_app/templates/traces/_conversation.html +37 -0
  91. _nullius_app/templates/traces/_judgement.html +72 -0
  92. _nullius_app/templates/traces/artifact.html +15 -0
  93. _nullius_app/templates/traces/trial.html +838 -0
  94. _nullius_app/templates/workspaces/workspaces.html +54 -0
  95. _nullius_app/web/__init__.py +1 -0
  96. _nullius_app/web/agents.py +230 -0
  97. _nullius_app/web/builder.py +392 -0
  98. _nullius_app/web/catalogue.py +374 -0
  99. _nullius_app/web/compare.py +489 -0
  100. _nullius_app/web/core.py +17 -0
  101. _nullius_app/web/execute.py +205 -0
  102. _nullius_app/web/feasibility.py +60 -0
  103. _nullius_app/web/filters.py +224 -0
  104. _nullius_app/web/judge.py +623 -0
  105. _nullius_app/web/judge_tasks.py +33 -0
  106. _nullius_app/web/onboarding.py +20 -0
  107. _nullius_app/web/reconcile.py +47 -0
  108. _nullius_app/web/runs.py +461 -0
  109. _nullius_app/web/settings.py +210 -0
  110. _nullius_app/web/shared.py +468 -0
  111. _nullius_app/web/traces.py +306 -0
  112. _nullius_app/web/workspaces.py +84 -0
  113. nullius/__init__.py +32 -0
  114. nullius/__main__.py +5 -0
  115. nullius/catalogue/__init__.py +0 -0
  116. nullius/catalogue/agents/codex/agent.json +202 -0
  117. nullius/catalogue/agents/codex/config.toml +25 -0
  118. nullius/catalogue/agents/goose/agent.json +120 -0
  119. nullius/catalogue/agents/goose/config.yaml +20 -0
  120. nullius/catalogue/agents/jiuwenswarm/agent.json +115 -0
  121. nullius/catalogue/agents/jiuwenswarm/agent.py +316 -0
  122. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  123. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  124. nullius/catalogue/agents/opencode/agent.json +124 -0
  125. nullius/catalogue/agents/opencode/opencode.json +32 -0
  126. nullius/catalogue/agents/reference/agent.json +89 -0
  127. nullius/catalogue/agents/reference/agent.py +209 -0
  128. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  129. nullius/catalogue/agents/terminus-2/run.py +139 -0
  130. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  131. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  132. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  133. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  134. nullius/catalogue/benchmarks/humaneval/benchmark.json +37 -0
  135. nullius/catalogue/benchmarks/humaneval/driver.py +208 -0
  136. nullius/catalogue/benchmarks/humaneval/standard.json +114 -0
  137. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  138. nullius/catalogue/benchmarks/locomo/benchmark.json +42 -0
  139. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  140. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  141. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  142. nullius/catalogue/benchmarks/longmemeval/benchmark.json +64 -0
  143. nullius/catalogue/benchmarks/longmemeval/driver.py +323 -0
  144. nullius/catalogue/benchmarks/longmemeval/standard.json +551 -0
  145. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +74 -0
  146. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  147. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +105 -0
  148. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  149. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  150. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  151. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  152. nullius/catalogue/benchmarks/tau3-bench/driver.py +605 -0
  153. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +111 -0
  154. nullius/catalogue/benchmarks/tau3-bench/standard.json +495 -0
  155. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  156. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +70 -0
  157. nullius/catalogue/benchmarks/terminal-bench/driver.py +189 -0
  158. nullius/catalogue/benchmarks/terminal-bench/standard.json +160 -0
  159. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  160. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  161. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  162. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  163. nullius/catalogue/capabilities/mem0/capability.py +170 -0
  164. nullius/catalogue/catalog.py +413 -0
  165. nullius/catalogue/citation.py +116 -0
  166. nullius/catalogue/harbor_entry.py +367 -0
  167. nullius/catalogue/sources.py +85 -0
  168. nullius/catalogue/storage.py +125 -0
  169. nullius/cli.py +83 -0
  170. nullius/compare/__init__.py +0 -0
  171. nullius/compare/cli.py +265 -0
  172. nullius/compare/comparison.py +184 -0
  173. nullius/execution.py +1624 -0
  174. nullius/gitrefs.py +128 -0
  175. nullius/harbor/__init__.py +0 -0
  176. nullius/harbor/feed.py +228 -0
  177. nullius/harbor/source.py +582 -0
  178. nullius/hosts.py +339 -0
  179. nullius/importers/__init__.py +140 -0
  180. nullius/importers/atif.py +387 -0
  181. nullius/importers/cli.py +135 -0
  182. nullius/importers/otel.py +274 -0
  183. nullius/judge/__init__.py +14 -0
  184. nullius/judge/batch.py +122 -0
  185. nullius/judge/client.py +416 -0
  186. nullius/judge/floors.json +86 -0
  187. nullius/judge/lanes.py +89 -0
  188. nullius/judge/questions.py +309 -0
  189. nullius/judge/reconcile.py +172 -0
  190. nullius/judge/state.py +224 -0
  191. nullius/judge/store.py +167 -0
  192. nullius/migrate.py +268 -0
  193. nullius/onboarding.py +112 -0
  194. nullius/paths.py +84 -0
  195. nullius/prerequisites.py +111 -0
  196. nullius/progress.py +294 -0
  197. nullius/providers.py +115 -0
  198. nullius/recipes/__init__.py +0 -0
  199. nullius/recipes/launch.py +78 -0
  200. nullius/redact.py +72 -0
  201. nullius/runs/__init__.py +0 -0
  202. nullius/runs/artifacts.py +67 -0
  203. nullius/runs/classify.py +189 -0
  204. nullius/runs/cli.py +316 -0
  205. nullius/runs/feasibility.py +442 -0
  206. nullius/runs/launches.py +139 -0
  207. nullius/runs/live.py +705 -0
  208. nullius/runs/parse.py +991 -0
  209. nullius/runs/verifier.py +241 -0
  210. nullius/sandbox.py +146 -0
  211. nullius/secrets.py +192 -0
  212. nullius/settings.py +456 -0
  213. nullius/standard/__init__.py +15 -0
  214. nullius/standard/acp.py +301 -0
  215. nullius/standard/cancellation.py +19 -0
  216. nullius/standard/cli.py +650 -0
  217. nullius/standard/conform.py +149 -0
  218. nullius/standard/container.py +1381 -0
  219. nullius/standard/definitions.py +1577 -0
  220. nullius/standard/egress.py +192 -0
  221. nullius/standard/fidelity.py +255 -0
  222. nullius/standard/forward.py +92 -0
  223. nullius/standard/hostreq.py +65 -0
  224. nullius/standard/howto.py +220 -0
  225. nullius/standard/install.py +397 -0
  226. nullius/standard/job.py +541 -0
  227. nullius/standard/jsonl.py +129 -0
  228. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  229. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  230. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  231. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  232. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  233. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  234. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  235. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  236. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  237. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  238. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  239. nullius/standard/match.py +66 -0
  240. nullius/standard/mcp_bridge.py +62 -0
  241. nullius/standard/mcp_machine.py +193 -0
  242. nullius/standard/merge.py +209 -0
  243. nullius/standard/names.py +30 -0
  244. nullius/standard/payload.py +255 -0
  245. nullius/standard/ports.py +64 -0
  246. nullius/standard/prove.py +357 -0
  247. nullius/standard/recipe.py +663 -0
  248. nullius/standard/relay.py +629 -0
  249. nullius/standard/run.py +1367 -0
  250. nullius/standard/shard.py +71 -0
  251. nullius/traces/__init__.py +0 -0
  252. nullius/traces/behavior.py +131 -0
  253. nullius/traces/charts.py +898 -0
  254. nullius/traces/conversation.py +159 -0
  255. nullius/traces/otel.py +608 -0
  256. nullius/traces/trace.py +1477 -0
  257. nullius/traces/trajectory.py +586 -0
  258. nullius/workspace.py +409 -0
  259. nullius/workspace_cli.py +119 -0
  260. nullius_eval-0.1.0rc1.dist-info/METADATA +93 -0
  261. nullius_eval-0.1.0rc1.dist-info/RECORD +268 -0
  262. nullius_eval-0.1.0rc1.dist-info/WHEEL +5 -0
  263. nullius_eval-0.1.0rc1.dist-info/entry_points.txt +2 -0
  264. nullius_eval-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
  265. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  266. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  267. nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  268. nullius_eval-0.1.0rc1.dist-info/top_level.txt +2 -0
@@ -0,0 +1,20 @@
1
+ """Getting started: the guide's checklist, read fresh on each ask
2
+ (`nullius.onboarding`). Booleans only: no key or path leaves in it."""
3
+ from __future__ import annotations
4
+
5
+ import os
6
+
7
+ from flask import jsonify, url_for
8
+
9
+ from nullius import onboarding, workspace
10
+
11
+ from .core import app
12
+ from . import shared
13
+
14
+
15
+ @app.route("/onboarding")
16
+ def onboarding_state():
17
+ ws = shared.STATE["workspace"]
18
+ jobs = workspace.job_index(ws).values() if ws else ()
19
+ found = onboarding.facts(os.environ, jobs, onboarding.any_judged(ws.dir if ws else None))
20
+ return jsonify(sections=onboarding.checklist(found, url_for))
@@ -0,0 +1,47 @@
1
+ """Reconciliation: the classifier's outcome beside the judge's
2
+ failure_cause, per trial of a job, from the workspace store - where they
3
+ disagree, ranked strongest first, and a suggestion for the trials the
4
+ classifier could not place. Reads the store; asks the judge nothing;
5
+ writes nothing. Advisory beside the recorded evidence, never in its
6
+ place."""
7
+ from __future__ import annotations
8
+
9
+ from flask import abort, render_template, request
10
+ from nullius.judge import questions, reconcile, store
11
+ from nullius.runs import classify
12
+
13
+ from .core import app
14
+ from . import shared
15
+
16
+
17
+ def ws_dir():
18
+ ws = shared.STATE.get("workspace")
19
+ return ws.dir if ws else None
20
+
21
+
22
+ def threshold_requested() -> float:
23
+ """`?threshold=` as a number in 0..1, else the module's default."""
24
+ raw = request.args.get("threshold")
25
+ try:
26
+ t = float(raw)
27
+ except (TypeError, ValueError):
28
+ return reconcile.THRESHOLD
29
+ return min(1.0, max(0.0, t))
30
+
31
+
32
+ def view_of(job, threshold: float) -> dict:
33
+ d = ws_dir()
34
+ records = store.for_job(d, job.job_id) if d else {}
35
+ return reconcile.reconcile(job, records, threshold, key=lambda name: store._SAFE.sub("_", name))
36
+
37
+
38
+ @app.route("/job/<job_id>/reconcile")
39
+ def reconcile_page(job_id):
40
+ j = shared._find_job(shared._jobs_for_request(), job_id)
41
+ if not j:
42
+ abort(404)
43
+ return render_template("runs/reconcile.html", job=j, view=view_of(j, threshold_requested()),
44
+ STATUSES=reconcile.STATUSES, STATUS_LABELS=reconcile.STATUS_LABELS,
45
+ COMPATIBLE=reconcile.COMPATIBLE, SUGGESTS=reconcile.SUGGESTS,
46
+ C_ORDER=classify.ORDER, C_LABELS=classify.LABELS, default_threshold=reconcile.THRESHOLD,
47
+ QUESTIONS_VERSION=questions.VERSION)
@@ -0,0 +1,461 @@
1
+ """The runs list, a job's page and its live board, the workspace picker and upload."""
2
+ from __future__ import annotations
3
+
4
+ import hashlib
5
+ import json
6
+ import os
7
+ import tempfile
8
+ from flask import (
9
+ abort,
10
+ flash,
11
+ jsonify,
12
+ redirect,
13
+ render_template,
14
+ request,
15
+ url_for,
16
+ )
17
+ from markupsafe import Markup, escape
18
+ from nullius import execution
19
+ from nullius import importers as harnesses
20
+ from nullius.redact import redact
21
+ from nullius.runs import classify, feasibility, launches, live, parse
22
+ from nullius import workspace
23
+
24
+ from .core import app
25
+ from . import shared
26
+
27
+
28
+
29
+ @app.route("/")
30
+ def home():
31
+ """Open the application on Runs, with its current workspace or first steps."""
32
+ return redirect(url_for("index"))
33
+
34
+
35
+ @app.route("/runs")
36
+ def index():
37
+ if shared.STATE["workspace"] is None:
38
+ return render_template("runs/start.html", workspaces=workspace.list_all(shared.WS_HOME))
39
+ jobs = shared._load_jobs()
40
+ # Runs started from the builder whose job Harbor has not written yet:
41
+ # listed above the table, each a link to the page that follows it.
42
+ have = {j.job_id for j in jobs}
43
+ # The run records are the app's, not the workspace's: only those launched
44
+ # into this workspace's roots, or with a job already written here, belong.
45
+ mine = [e for e in execution.EXECUTIONS.values() if _launched_here(e, have)]
46
+ preparing = [(jid, e) for e in mine if e.state in execution.ALIVE for jid in e.summary()["job_ids"] if jid not in have]
47
+ # a launch whose runs wrote no job yet (or never will: refused, stopped) is still shown
48
+ if not jobs and not mine:
49
+ return render_template("runs/setup.html", scanned=True, mode="add",
50
+ workspaces=workspace.list_all(shared.WS_HOME))
51
+ # The header, the chips and the mix bar describe the agents' trials. A
52
+ # control (nop, oracle) is a check on the tasks, not a competitor: it is
53
+ # listed on its own line as behaved or not (`feasibility.summary_of`),
54
+ # and its trials never enter the totals — a nop rejected on every task
55
+ # is the expected result, not five failed verifications.
56
+ verdicts = {j.job_id: feasibility.summary_of(j) for j in jobs}
57
+ # A Harbor killed by a Stop (or dead with the run) leaves its result
58
+ # record saying trials are running; the runner's record says otherwise.
59
+ ended = {j.job_id: execution.ended_for(j.job_id) for j in jobs}
60
+ agents = [j for j in jobs if verdicts[j.job_id] is None]
61
+ controls = [(j, verdicts[j.job_id]) for j in jobs if verdicts[j.job_id] is not None]
62
+ totals = {k: 0 for k in classify.ORDER}
63
+ n_trials = 0
64
+ for j in agents:
65
+ for k, v in j.counts.items():
66
+ totals[k] += v
67
+ n_trials += len(j.trials)
68
+
69
+ # Only describe datasets these runs actually used, most-used first.
70
+ used = {}
71
+ for j in jobs:
72
+ entry = used.setdefault(j.dataset, {"runs": 0, "trials": 0})
73
+ entry["runs"] += 1
74
+ entry["trials"] += len(j.trials)
75
+ datasets = [
76
+ {"name": name, "info": shared.dataset_info(name), **counts}
77
+ for name, counts in sorted(used.items(), key=lambda kv: -kv[1]["trials"])
78
+ ]
79
+
80
+ # Distinct values behind the per-column filter menus, as (value, label).
81
+ # Value is what the row carries; label is what the menu shows.
82
+ choices = {
83
+ "dataset": sorted(
84
+ {(j.dataset, shared.dataset_title(j.dataset)) for j in jobs}, key=lambda p: p[1]
85
+ ),
86
+ "model": sorted({(j.model_name, j.model_name) for j in jobs}),
87
+ "agent": sorted({(j.agent_label, j.agent_label or "—") for j in jobs}),
88
+ }
89
+ # The page's first view: the runs grouped by the launch that made them
90
+ # (nullius.runs.launches) - the builder's run record, else the same dataset
91
+ # within half an hour - each with its task checks and its agents side by side.
92
+ records = {e.id: (e.plan.name, [str(x.get("name")) for x in e.plan.jobs], e.started) for e in mine}
93
+ by_id = {j.job_id: j for j in jobs}
94
+ groups = [_launch_view(g, by_id, verdicts, ended) for g in launches.group(jobs, records)]
95
+ view = request.args.get("view") if request.args.get("view") in ("launch", "all") else "launch"
96
+ states = {}
97
+ for g in groups:
98
+ for row in g["agents"]:
99
+ states[row["state"]] = states.get(row["state"], 0) + 1
100
+ return render_template(
101
+ "runs/index.html", jobs=jobs, totals=totals, n_trials=n_trials,
102
+ n_agent_runs=len(agents), controls=controls, verdicts=verdicts, ended=ended,
103
+ datasets=datasets, choices=choices, pulse=_roots_pulse(), preparing=preparing,
104
+ launches=groups, view=view, states=states,
105
+ n_checked=sum(1 for g in groups if g["checked"]),
106
+ )
107
+
108
+
109
+ def _launched_here(ex, have) -> bool:
110
+ """A run record belongs to the open workspace when its job root is one of
111
+ the workspace's roots (a Run adopts it there) or one of its jobs is here."""
112
+ if any(str(x.get("name")) in have for x in ex.plan.jobs):
113
+ return True
114
+ jobs_dir = getattr(ex, "jobs", None)
115
+ roots = list(shared.STATE.get("roots") or []) + list(shared.STATE.get("job_roots") or [])
116
+ return bool(jobs_dir) and os.path.realpath(jobs_dir) in {os.path.realpath(r) for r in roots}
117
+
118
+
119
+ # A launch's runs, in the order the block lists them: finished first, best
120
+ # adjusted rate on top; then those still going; then the ones that stopped.
121
+ _STATE_ORDER = {"finished": 0, "running": 1, "preparing": 2, "queued": 3, "stopping": 4, "paused": 5, "failed": 6, "refused": 7, "stopped": 8, execution.NOT_RUN: 8.5,
122
+ "not in this workspace": 9}
123
+
124
+
125
+ def _state_of(job_id, j, ended):
126
+ """What a run is doing now, in the words the block shows. The runner's
127
+ record wins over Harbor's result record, which a Stop leaves saying
128
+ "running"."""
129
+ if j is not None and ended.get(job_id):
130
+ how = ended[job_id][0]
131
+ return "finished" if how == "done" else how
132
+ ex = execution.for_job(job_id)
133
+ if ex is not None and ex.state in execution.ALIVE:
134
+ return ex.live_state(job_id)
135
+ if j is None:
136
+ # named by a run record and not under this workspace's roots: still to be written while the run lives, refused
137
+ # before it started (its folder says so, or no part was launched), else elsewhere
138
+ if ex is not None and ex.state not in execution.ALIVE:
139
+ return "refused" if ex.refused(job_id) else execution.NOT_RUN if ex.never_ran(job_id) else "not in this workspace"
140
+ return ex.state if ex is not None else "preparing"
141
+ if j.is_preparing:
142
+ return "preparing"
143
+ if j.is_running:
144
+ return "running"
145
+ return "finished"
146
+
147
+
148
+ def _control_planned(job_id) -> str:
149
+ """``nop`` or ``oracle`` when a job Harbor has not written yet is one of
150
+ its controls by the run's plan, else ''. A runner's arm is never one."""
151
+ ex = execution.for_job(job_id)
152
+ if ex is None or ex.plan.runner != "harbor":
153
+ return ""
154
+ agent = next((str(j.get("agent") or "") for j in ex.plan.jobs if str(j.get("name")) == job_id), "").lower()
155
+ return agent if agent in parse.CONTROL_AGENTS else ""
156
+
157
+
158
+ def _launch_view(g, by_id, verdicts, ended) -> dict:
159
+ """One launch as its block draws it: the task checks, the agents ranked on
160
+ one scale, the runs its record named that Harbor has not written yet."""
161
+ jobs = [by_id[i] for i in g.job_ids]
162
+ checks = [(j, verdicts[j.job_id]) for j in jobs if verdicts[j.job_id] is not None]
163
+ rows = [{"job": j, "job_id": j.job_id, "state": _state_of(j.job_id, j, ended)} for j in jobs if verdicts[j.job_id] is None]
164
+ waiting = [(i, c) for i, c in ((i, _control_planned(i)) for i in g.pending) if c]
165
+ rows += [{"job": None, "job_id": i, "state": _state_of(i, None, ended)} for i in g.pending if i not in dict(waiting)]
166
+ for r in rows:
167
+ if r["state"] == "refused":
168
+ r["why"] = execution.for_job(r["job_id"]).refused(r["job_id"])
169
+ rate = lambda r: r["job"].adjusted_solve_rate if r["job"] is not None and r["job"].adjusted_solve_rate is not None else -1.0
170
+ rows.sort(key=lambda r: (_STATE_ORDER.get(r["state"], 9), -rate(r), r["job_id"]))
171
+ ok = {v.control for _, v in checks if v.state == "ok"}
172
+ if waiting:
173
+ verdict, checked = "Not checked yet: a control has not finished.", False
174
+ elif not checks:
175
+ verdict, checked = None, False
176
+ elif any(v.state == "warn" for _, v in checks):
177
+ verdict, checked = "A control did not behave: some tasks may be broken. See Feasibility.", False
178
+ elif any(v.state == "pending" for _, v in checks):
179
+ verdict, checked = "Not checked yet: a control has not finished.", False
180
+ elif {"nop", "oracle"} <= ok:
181
+ verdict, checked = "Every task is feasible: an empty answer fails and the reference passes.", True
182
+ elif "nop" in ok:
183
+ verdict, checked = "An empty answer fails every task; no oracle ran to show each can pass.", True
184
+ else:
185
+ verdict, checked = "The reference passes every task; no nop ran to show an empty answer fails.", True
186
+ n_tasks = max([j.n_planned or len(j.trials) for j in jobs] + [0]) or None
187
+ finished = [r["job_id"] for r in rows if r["state"] == "finished" and r["job"] is not None and r["job"].has_verdict]
188
+ # The controls are rows of their own at the top of the table, written or still to come, nop before oracle:
189
+ # a check on the tasks, never ranked with the agents nor offered to Compare, and kept once they end.
190
+ controls = [{"job": j, "job_id": j.job_id, "state": _state_of(j.job_id, j, ended), "control": v.control, "verdict": v} for j, v in checks]
191
+ controls += [{"job": None, "job_id": i, "state": _state_of(i, None, ended), "control": c, "verdict": None} for i, c in waiting]
192
+ controls.sort(key=lambda r: (feasibility.CONTROLS.index(r["control"]) if r["control"] in feasibility.CONTROLS else 9, r["job_id"]))
193
+ counts = {}
194
+ for r in controls + rows:
195
+ counts[r["state"]] = counts.get(r["state"], 0) + 1
196
+ ex = execution.for_job((g.job_ids + g.pending)[0])
197
+ return {"launch": g, "checks": checks, "verdict": verdict,
198
+ "controls": controls, "checked": checked, "agents": rows,
199
+ "n_tasks": n_tasks, "compare": finished,
200
+ "paused": next((i for i in g.job_ids + g.pending
201
+ if (execution.for_job(i) is not None and execution.for_job(i).state == "paused")), None),
202
+ "summary": " · ".join(f"{n} {s}" for s, n in sorted(counts.items(), key=lambda kv: _STATE_ORDER.get(kv[0], 9))),
203
+ "recipe": (g.job_ids + g.pending)[0] if g.source == "record" and ex is not None and ex.plan.recipe else None,
204
+ "feasibility": checks[0][0].job_id if checks else None,
205
+ "dataset_title": shared.dataset_title(g.dataset) if g.dataset and g.dataset != "mixed" else (g.dataset or "—")}
206
+
207
+
208
+ def _roots_pulse() -> str:
209
+ """A short signature of what the job roots hold right now: each job
210
+ directory's name, whether it has a result record and a feed, and the
211
+ feed's size. It changes when a run is launched (a directory appears),
212
+ while it runs (the feed grows) and when it ends (the result lands) — and
213
+ at no other time, so a page that polls it reloads only for a reason.
214
+ Nothing is parsed; this is a handful of stats per job."""
215
+ dirs = parse.find_job_dirs(shared.STATE["job_roots"] or [])
216
+ have = {os.path.basename(d) for d in dirs}
217
+ marks = [(f"run:{e.id}", e.state, 0) for e in execution.EXECUTIONS.values() if _launched_here(e, have)]
218
+ for d in dirs:
219
+ feed = os.path.join(d, "events.jsonl")
220
+ marks.append((os.path.basename(d), os.path.isfile(os.path.join(d, "result.json")),
221
+ os.path.getsize(feed) if os.path.isfile(feed) else -1))
222
+ return hashlib.sha1(json.dumps(sorted(marks)).encode()).hexdigest()[:16]
223
+
224
+
225
+ @app.route("/runs/pulse")
226
+ def runs_pulse():
227
+ """What the runs list polls. ``signature`` is :func:`_roots_pulse`; a
228
+ change means the list is stale."""
229
+ response = jsonify({"signature": _roots_pulse()})
230
+ response.headers["Cache-Control"] = "no-store"
231
+ return response
232
+
233
+
234
+ def _readable(roots, shown=lambda root: root):
235
+ """Split *roots* into the ones a workspace can adopt and the ones it must
236
+ not, each of those with why.
237
+
238
+ A root is refused when the importer that claims it cannot read it — a
239
+ ``run.json`` that is not JSON, a log of the wrong shape — or when nothing
240
+ claims it and nothing in it is job-shaped either. Adopting either would
241
+ have been reported as success: the first as a job with no trials, the
242
+ second as a workspace with nothing new in it, and neither says what was
243
+ wrong with the folder. The message is redacted before it is shown: a
244
+ file's path or content is arbitrary text, and a credential in a bad one
245
+ must not reach the page because the file was bad.
246
+ """
247
+ kept, refused = [], []
248
+ for root in roots:
249
+ why = harnesses.problem(root)
250
+ if why is None and not parse.find_job_dirs([root]) and harnesses.detect(root) is None:
251
+ why = (f"nothing job-shaped here, and no importer recognises it "
252
+ f"(known: {harnesses.labels()})")
253
+ if why is None:
254
+ kept.append(root)
255
+ else:
256
+ refused.append((shown(root), why))
257
+ return kept, refused
258
+
259
+
260
+ def _refused_message(refused):
261
+ """One line per refused root, the root in monospace like `_bad_paths_message`."""
262
+ return Markup("<br>".join(
263
+ f'Not imported: <span class="mono">{escape(redact(root))}</span> — {escape(why)}'
264
+ for root, why in refused))
265
+
266
+
267
+ def _after_import(good, refused, new):
268
+ """Where a submission lands: the runs list — or, when something was
269
+ refused, the picker, which is the page that shows the message. The runs
270
+ list does not, and a refusal nobody sees is a silent success."""
271
+ if refused:
272
+ flash(_refused_message(refused), "error")
273
+ return redirect(url_for("import_page", new="1" if new and not good else None))
274
+ return redirect(url_for("index"))
275
+
276
+
277
+ @app.route("/use-path", methods=["POST"])
278
+ def use_path():
279
+ shared._require_csrf()
280
+ good, bad = shared.resolve_submitted_paths(request.form.getlist("path"))
281
+ if bad:
282
+ flash(shared._bad_paths_message(bad), "error")
283
+ good, refused = _readable(good)
284
+ new = request.form.get("new") == "1"
285
+ if good:
286
+ shared.adopt_roots(good, label=(request.form.get("name") or "").strip(), new=new)
287
+ return _after_import(good, refused, new)
288
+
289
+
290
+ @app.route("/api/folders", methods=["POST"])
291
+ def folder_suggestions():
292
+ shared._require_csrf()
293
+ response = jsonify(paths=shared.suggest_directories(request.form.get("path", "")))
294
+ response.headers["Cache-Control"] = "no-store"
295
+ return response
296
+
297
+
298
+ @app.route("/change-source", methods=["POST"])
299
+ def change_source():
300
+ """Close the workspace and return to the Runs page's first steps."""
301
+ shared._require_csrf()
302
+ shared.close_workspace()
303
+ return redirect(url_for("index"))
304
+
305
+
306
+ @app.route("/import")
307
+ def import_page():
308
+ """The picker: add runs to the open workspace, or start a new one."""
309
+ mode = "new" if request.args.get("new") == "1" or shared.STATE["workspace"] is None else "add"
310
+ return render_template("runs/setup.html", scanned=False, mode=mode,
311
+ workspaces=workspace.list_all(shared.WS_HOME))
312
+
313
+
314
+ @app.route("/upload", methods=["POST"])
315
+ def upload():
316
+ shared._require_csrf()
317
+ files = request.files.getlist("files")
318
+ files = [f for f in files if f.filename]
319
+ # One Parse button serves both halves of the form, so typed paths can
320
+ # arrive alongside uploaded folders. Both end up in the same root list.
321
+ typed, bad = shared.resolve_submitted_paths(request.form.getlist("path"))
322
+ if bad:
323
+ flash(shared._bad_paths_message(bad), "error")
324
+ if not files:
325
+ if typed:
326
+ shared.set_source(typed)
327
+ else:
328
+ flash("No files received — nothing to scan.", "error")
329
+ return redirect(url_for("index"))
330
+
331
+ tmp = tempfile.mkdtemp(prefix="nullius_jobs_")
332
+ saved = 0
333
+ for f in files:
334
+ dest = shared._safe_dest(tmp, f.filename)
335
+ if dest is None:
336
+ continue
337
+ os.makedirs(os.path.dirname(dest), exist_ok=True)
338
+ f.save(dest)
339
+ saved += 1
340
+ if not saved:
341
+ flash("Upload failed — no files could be saved.", "error")
342
+ return redirect(url_for("index"))
343
+
344
+ # Scan uploaded folders without also scanning tmp, which would import them twice.
345
+ # Use tmp only for loose files; name sources and errors after the uploaded paths.
346
+ top = sorted(n for n in os.listdir(tmp) if os.path.isdir(os.path.join(tmp, n)))
347
+ dropped, refused = _readable([os.path.join(tmp, n) for n in top] or [tmp],
348
+ shown=lambda root: os.path.relpath(root, tmp))
349
+ typed, refused_typed = _readable(typed)
350
+ refused += refused_typed
351
+ kept_top = [os.path.basename(d) for d in dropped if d != tmp]
352
+ roots = dropped + typed
353
+ if len(kept_top) == 1:
354
+ label = f"{kept_top[0]} (uploaded)"
355
+ elif kept_top:
356
+ label = f"{len(kept_top)} folders (uploaded)"
357
+ else:
358
+ label = f"{saved} files (uploaded)"
359
+ if typed:
360
+ label += f" +{len(typed)} local"
361
+ new = request.form.get("new") == "1"
362
+ if roots:
363
+ shared.adopt_roots(roots, label=(request.form.get("name") or "").strip() or label,
364
+ new=new)
365
+ return _after_import(roots, refused, new)
366
+
367
+
368
+ @app.route("/job/<job_id>", methods=["GET", "HEAD"])
369
+ def job(job_id):
370
+ j = shared._find_job(shared._jobs_for_request(), job_id)
371
+ ex = execution.for_job(job_id)
372
+ # this job's own state beside the run's: ended, or a part still to come while another runs
373
+ own = ((execution.NOT_RUN, None) if ex is not None and ex.state not in execution.ALIVE and ex.never_ran(job_id)
374
+ else execution.ended_for(job_id) or ((ex.live_state(job_id), None)
375
+ if ex is not None and ex.live_state(job_id) in ("queued", execution.NOT_RUN) else None))
376
+ if not j:
377
+ # Started from the builder and not written by Harbor yet: the run's
378
+ # own page, which becomes this one when the job appears.
379
+ if ex is not None and request.method == "GET":
380
+ return render_template("runs/preparing.html", job_id=job_id, ex=ex, csrf=shared._csrf_token(), own=own)
381
+ abort(404)
382
+ from . import judge
383
+ board = _live_board_state(j)
384
+ return render_template("runs/job.html", job=j, live_board=board, view=_view_of(board, j),
385
+ execution=ex, csrf=shared._csrf_token(),
386
+ judgement=judge.job_summary(j) if j.trials else None,
387
+ control=feasibility.summary_of(j), ended=execution.ended_for(job_id), own=own)
388
+
389
+
390
+ def _view_of(board, j=None) -> str | None:
391
+ """Which tab the job page opens on, or None when it has no tabs.
392
+
393
+ A job with a board has two views of one page: Live, the feed folded as it
394
+ grows, and Results, the page rendered from the job's files. ``?view=`` wins,
395
+ so a link opens the view it was copied from; otherwise Live while the feed
396
+ says the job is going, Results once it ended - unless there are no results
397
+ to show: a job with a feed and no result record yet (a runner's arm whose
398
+ importer has not written one) stays on Live. A job with no feed, a broken
399
+ one or an empty one has no Live view, and no tabs.
400
+ """
401
+ if not board or not board.get("initial") or board["initial"]["state"] == "unknown":
402
+ return None
403
+ asked = request.args.get("view")
404
+ if asked in ("live", "results"):
405
+ return asked
406
+ return "live" if board["running"] or (j is not None and j.is_preparing) else "results"
407
+
408
+
409
+ def _live_board_state(j):
410
+ """Whether the job page has a live board, and from what.
411
+
412
+ The board is the job page's Live view, not a second page: it polls while
413
+ the feed says the job has not ended, and once it ended it stays as the
414
+ record of how the run went, beside Results. A job without a feed has no
415
+ live board.
416
+ """
417
+ if not live.has_feed(j.dir):
418
+ return None
419
+ try:
420
+ snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True)
421
+ except live.FeedError as e:
422
+ # A malformed line in the middle of the feed is a broken writer, not a
423
+ # race. The page must still render — the settled sections read
424
+ # result.json, not the feed — so the board steps aside and says why.
425
+ return {"running": False, "initial": None, "error": str(e)}
426
+ # "running" here means "show the board and keep polling": the preparation
427
+ # window and the starting state are on the board too, so it is every
428
+ # state but finished — and unknown, an empty feed, which has nothing to show.
429
+ return {"running": snap["state"] in ("preparing", "starting", "running"), "initial": snap}
430
+
431
+
432
+ @app.route("/job/<job_id>/live")
433
+ def job_live(job_id):
434
+ """The board's snapshot, folded fresh from the feed on every call.
435
+
436
+ Deliberately not through the workspace cache: the cache is keyed by the
437
+ files parse reads, and a feed that grows every second would either miss
438
+ the cache on every request or, worse, hit it and show a board that stopped.
439
+ Folding a few thousand lines is cheap enough to do every second.
440
+ """
441
+ j = shared._find_job(shared._jobs_for_request(), job_id)
442
+ if not j:
443
+ abort(404)
444
+ if not live.has_feed(j.dir):
445
+ ended = execution.ended_for(job_id)
446
+ if ended:
447
+ # The preparing page must settle even if the runner never wrote a feed.
448
+ response = jsonify({"state": "finished", "running": False, "ended_by": ended[0]})
449
+ response.headers["Cache-Control"] = "no-store"
450
+ return response
451
+ abort(404)
452
+ try:
453
+ snap = live.snapshot(j.dir, ended=execution.ended_for(j.job_id), with_timeline=True)
454
+ except live.FeedError as e:
455
+ response = jsonify({"error": f"events.jsonl is not a feed: {e}"})
456
+ response.status_code = 422
457
+ response.headers["Cache-Control"] = "no-store"
458
+ return response
459
+ response = jsonify(snap)
460
+ response.headers["Cache-Control"] = "no-store"
461
+ return response