nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,507 @@
1
+ """What the pages share: the open workspace, its jobs, the catalogue cache, path resolution, the query helpers, the form-token guard."""
2
+ from __future__ import annotations
3
+
4
+ import glob
5
+ import hashlib
6
+ import ipaddress
7
+ from itertools import islice
8
+ import secrets
9
+ import os
10
+ from urllib.parse import urlsplit
11
+ from flask import abort, flash, g, has_request_context, request, session
12
+ from markupsafe import Markup, escape
13
+ from werkzeug.exceptions import Forbidden
14
+ from nullius import importers as harnesses
15
+ from nullius import execution
16
+ from nullius.catalogue import catalog
17
+ from nullius import workspace
18
+ from nullius import settings, paths
19
+ from nullius.redact import redact
20
+
21
+ from .core import app
22
+
23
+
24
+ _APP_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # app/, one up from this package
25
+ # User settings, loaded at start and read again before any request once it has
26
+ # changed on disk (settings.sync), so a key written by hand while the app
27
+ # runs counts from the next request, without a restart; existing environment
28
+ # variables win. The Settings page edits the same file (nullius.settings
29
+ # knows every variable read here).
30
+ ENV_FILE = paths.settings_file()
31
+ settings.load(ENV_FILE)
32
+
33
+
34
+ def _host_name(host: str) -> str:
35
+ """A Host header's name without its port: "[::1]:5057" is "::1"."""
36
+ if host.startswith("["):
37
+ return host[1:host.find("]")] if "]" in host else host
38
+ return host.rsplit(":", 1)[0] if host.count(":") == 1 else host
39
+
40
+
41
+ def _served_name(name: str) -> bool:
42
+ """A name this server answers to. A page on a name that resolves here
43
+ (DNS rebinding) would otherwise be same-origin with the app and pass the
44
+ loopback, Origin and form-token guards alike; an address literal or
45
+ localhost cannot be such a page's origin. The --host name it was started
46
+ with is accepted as well."""
47
+ name = name.lower().rstrip(".")
48
+ if name == "localhost" or name.endswith(".localhost") or name == str(app.config.get("SERVED_HOST", "")).lower():
49
+ return True
50
+ try:
51
+ ipaddress.ip_address(name)
52
+ return True
53
+ except ValueError:
54
+ return False
55
+
56
+
57
+ @app.before_request
58
+ def _refuse_foreign_host():
59
+ if not _served_name(_host_name(request.host)):
60
+ abort(421, description="This server answers on its own address only (localhost, an IP address or its --host name).")
61
+
62
+
63
+ @app.before_request
64
+ def _sync_env_file():
65
+ """The file as it is now: a hand edit applies to this request and every
66
+ one after it. A file that cannot be read leaves the settings as loaded."""
67
+ try:
68
+ settings.sync(ENV_FILE)
69
+ except (OSError, ValueError, UnicodeDecodeError) as e:
70
+ app.logger.warning("%s not read: %s", os.path.basename(ENV_FILE), getattr(e, "strerror", None) or e)
71
+
72
+
73
+ # What each benchmark actually measures. Keyed by the dataset name Harbor
74
+ # records in the job config; unknown datasets fall back to a stub so a new
75
+ # benchmark shows up in the UI rather than vanishing.
76
+ DATASETS = {
77
+ "swe-bench-verified": {
78
+ "title": "SWE-bench Verified",
79
+ "what": (
80
+ "500 real GitHub issues drawn from 12 widely used Python projects — Django, "
81
+ "SymPy, scikit-learn, matplotlib, requests and others. Each task hands the agent "
82
+ "a repository at the commit before the fix, plus the issue text."
83
+ ),
84
+ "scored": (
85
+ "The agent must produce a patch. It is applied to a clean checkout and the test "
86
+ "suite is run: FAIL_TO_PASS tests must flip from failing to passing, and "
87
+ "PASS_TO_PASS tests must stay green — so a patch cannot pass by breaking "
88
+ "unrelated behaviour."
89
+ ),
90
+ "note": (
91
+ "The “Verified” subset was human-annotated to remove tasks that were unsolvable "
92
+ "or had broken tests, so a failure here reflects the agent, not the benchmark."
93
+ ),
94
+ },
95
+ "swe-bench": {
96
+ "title": "SWE-bench",
97
+ "what": (
98
+ "Real GitHub issues from open-source Python projects, each paired with the "
99
+ "repository at the commit before the fix. The agent reads the issue and must "
100
+ "produce a working patch."
101
+ ),
102
+ "scored": (
103
+ "The patch is applied to a clean checkout and the project's tests are run: "
104
+ "FAIL_TO_PASS tests must flip from failing to passing, and PASS_TO_PASS tests "
105
+ "must stay green."
106
+ ),
107
+ "note": (
108
+ "Recorded simply as “swe-bench” by the harness, so the exact split "
109
+ "(full, Lite or Verified) isn't distinguished here."
110
+ ),
111
+ },
112
+ "terminal-bench": {
113
+ "title": "Terminal-Bench",
114
+ "what": (
115
+ "Tasks that exercise a real terminal inside an isolated container — building and "
116
+ "configuring software, debugging a broken environment, driving command-line tools "
117
+ "end to end. From the same team as Harbor."
118
+ ),
119
+ "scored": (
120
+ "Each task ships a test script that runs after the agent finishes. The task counts "
121
+ "as solved only if that script exits clean, so the agent is judged on the final "
122
+ "state of the machine rather than on what it said it did."
123
+ ),
124
+ "note": "",
125
+ },
126
+ }
127
+
128
+
129
+ def dataset_title(name):
130
+ """Human name for a dataset slug, falling back to the slug itself."""
131
+ entry = DATASETS.get(name)
132
+ return entry["title"] if entry else name
133
+
134
+
135
+ def dataset_info(name):
136
+ return DATASETS.get(name) or {
137
+ "title": name,
138
+ "what": "No description recorded for this dataset yet.",
139
+ "scored": "",
140
+ "note": "",
141
+ }
142
+
143
+
144
+ # Runs-table columns, in render order. Drives the show/hide menu; the template
145
+ # tags every th/td with the matching data-col.
146
+ COLUMNS = [
147
+ ("run", "Run"), ("agent", "Agent"), ("version", "Version"), ("start", "Start"), ("end", "End"), ("time", "Time"),
148
+ ("model", "Model"), ("dataset", "Dataset"), ("mix", "Mix"),
149
+ ("trials", "Trials"), ("solved", "Solved"), ("raw", "Raw"),
150
+ ("adjusted", "Adjusted"), ("infra", "Infra errors"), ("broke", "Broke tests"),
151
+ ("tokens", "Tokens"), ("cost", "Cost"),
152
+ ]
153
+
154
+
155
+ # Active roots, mutable at runtime via the workspace picker. Seeded at
156
+ # startup from CLI args / env; roots may be empty, in which case the UI asks.
157
+ # `label` is what the header shows — roots can be an internal detail (an upload
158
+ # scans a temp dir *and* its child, which is meaningless to read).
159
+ # The open workspace, and what it resolved to. `roots` and `label` are what
160
+ # the user pointed at; `job_roots` is after the foreign trees among them were
161
+ # converted — the directories parse actually walks. Conversion happens when
162
+ # the workspace is opened or refreshed, not on every request.
163
+ STATE: dict = {"roots": [], "label": "", "workspace": None, "job_roots": []}
164
+ WS_HOME = workspace.home()
165
+
166
+
167
+ # A path containing any of these is treated as a glob, so one entry can name
168
+ # several run directories: jobs_*, current/jobs_*/2026-08-*, **/jobs.
169
+ GLOB_CHARS = "*?["
170
+
171
+
172
+ def is_pattern(value) -> bool:
173
+ return any(ch in (value or "") for ch in GLOB_CHARS)
174
+
175
+
176
+ def resolve_submitted_paths(values):
177
+ """Split submitted paths into (existing dirs, unusable), preserving order.
178
+
179
+ Entries containing glob metacharacters expand to every directory they
180
+ match, so `.../jobs_*` picks up all of them in one field. Non-matching
181
+ patterns are reported like a missing directory rather than silently
182
+ resolving to nothing.
183
+ """
184
+ good, bad = [], []
185
+ for raw in values:
186
+ raw = (raw or "").strip()
187
+ if not raw:
188
+ continue
189
+ resolved = os.path.abspath(os.path.expanduser(raw))
190
+ if is_pattern(raw):
191
+ matches = sorted(p for p in glob.glob(resolved, recursive=True) if os.path.isdir(p))
192
+ if not matches:
193
+ bad.append(raw)
194
+ for p in matches:
195
+ if p not in good:
196
+ good.append(p)
197
+ elif os.path.isdir(resolved):
198
+ if resolved not in good:
199
+ good.append(resolved)
200
+ else:
201
+ bad.append(raw)
202
+ return good, bad
203
+
204
+
205
+ def suggest_directories(value):
206
+ """Complete one path component, including linked and mounted directories."""
207
+ value = (value or "").lstrip()
208
+ if not value or len(value) > 4096 or "\0" in value or is_pattern(value):
209
+ return []
210
+ expanded = os.path.expanduser(value)
211
+ if value == "~":
212
+ expanded += os.sep
213
+ parent, prefix = os.path.split(expanded)
214
+ parent = os.path.abspath(parent or ".")
215
+ matches = []
216
+ try:
217
+ with os.scandir(parent) as entries:
218
+ # Suggestions never recurse or enumerate an unbounded directory.
219
+ for entry in islice(entries, 5000):
220
+ if not os.path.normcase(entry.name).startswith(os.path.normcase(prefix)):
221
+ continue
222
+ if entry.name.startswith(".") and not prefix.startswith("."):
223
+ continue
224
+ try:
225
+ if entry.is_dir():
226
+ matches.append(os.path.join(parent, entry.name) + os.sep)
227
+ except OSError:
228
+ continue
229
+ except OSError:
230
+ return []
231
+ return sorted(matches, key=os.path.normcase)[:20]
232
+
233
+
234
+ def _bad_paths_message(bad):
235
+ """Patterns that matched nothing read differently from a missing path."""
236
+ patterns = [p for p in bad if is_pattern(p)]
237
+ plain = [p for p in bad if not is_pattern(p)]
238
+ parts = []
239
+ if plain:
240
+ listed = ", ".join(f'<span class="mono">{escape(p)}</span>' for p in plain)
241
+ parts.append(f"No {'directory' if len(plain) == 1 else 'directories'} at {listed}")
242
+ if patterns:
243
+ listed = ", ".join(f'<span class="mono">{escape(p)}</span>' for p in patterns)
244
+ parts.append(f"Nothing matches {listed}")
245
+ return Markup(" · ".join(parts))
246
+
247
+
248
+ def set_source(roots, label=None) -> None:
249
+ """Point the app at *roots*, with *label* as the human-facing name."""
250
+ STATE["roots"] = roots
251
+ if label:
252
+ STATE["label"] = label
253
+ elif len(roots) == 1:
254
+ STATE["label"] = roots[0]
255
+ else:
256
+ STATE["label"] = f"{roots[0]} +{len(roots) - 1}" if roots else ""
257
+
258
+
259
+ def _import_roots(roots, import_root):
260
+ """Convert any root a foreign harness recognises, returning job roots.
261
+
262
+ Which harness wrote a run is a property of the directory, not a question
263
+ for whoever opens it: the shapes are distinguishable, and someone handed a
264
+ results folder often cannot answer. Detection identifies it, and
265
+ a harness added to `nullius/importers/` is readable here without a UI
266
+ change — the point of the registry.
267
+
268
+ What one job is under a claimed root is the importer's answer
269
+ (`harnesses.trees`), matching the CLI's job boundaries. A root nothing
270
+ claims goes through as it is: a Harbor job directory or a parent of
271
+ them, which `find_job_dirs` walks.
272
+
273
+ A tree the importer claims but refuses is left out, so one bad tree
274
+ never costs the others or the process — but not silently: the reason,
275
+ redacted where it was raised, is flashed to the page being served, where
276
+ the picker shows it in place of "Nothing job-shaped", and logged when
277
+ there is no page (a workspace reopened at start-up).
278
+ """
279
+ resolved, seen = [], set()
280
+
281
+ def take(path):
282
+ if path not in seen:
283
+ seen.add(path)
284
+ resolved.append(path)
285
+
286
+ for root in roots:
287
+ module = harnesses.detect(root) if os.path.isdir(root) else None
288
+ if module is None:
289
+ take(root)
290
+ continue
291
+ for tree in harnesses.trees(module, root):
292
+ try:
293
+ for job_dir in module.convert(tree, import_root):
294
+ take(job_dir)
295
+ except (OSError, SystemExit) as e:
296
+ why = redact(str(e) or f"{module.NAME} could not read it")
297
+ app.logger.warning("Not imported: %s — %s", redact(tree), why)
298
+ if has_request_context():
299
+ flash(Markup(f'Not imported: <span class="mono">{escape(redact(tree))}</span> '
300
+ f'— {escape(why)}'), "error")
301
+ return resolved
302
+
303
+
304
+ def open_workspace(ws, refresh=False):
305
+ """Make *ws* the one the app reads: convert its foreign roots once, then
306
+ every request walks the result and reads parsed jobs from its cache."""
307
+ STATE["workspace"] = ws
308
+ set_source(ws.roots, ws.label)
309
+ if refresh:
310
+ workspace.clear_cache(ws)
311
+ STATE["job_roots"] = _import_roots(ws.roots, ws.imports_dir)
312
+ ws.job_roots = list(STATE["job_roots"])
313
+ workspace.touch(ws)
314
+
315
+
316
+ def close_workspace():
317
+ STATE.update({"workspace": None, "roots": [], "label": "", "job_roots": []})
318
+
319
+
320
+ def adopt_roots(roots, label="", new=False):
321
+ """Roots the user just pointed at: into the open workspace, or a new one.
322
+
323
+ Without a label, roots that already have a workspace reopen it — so a
324
+ restart with the same argv lands in the same place, cache and all.
325
+ """
326
+ ws = STATE["workspace"]
327
+ if ws and not new:
328
+ ws.roots = list(ws.roots) + [r for r in roots if r not in ws.roots]
329
+ workspace.save(ws)
330
+ open_workspace(ws)
331
+ return ws
332
+ ws = None if label else workspace.find_by_roots(WS_HOME, roots, besides=execution.runs_root())
333
+ ws = ws or workspace.create(WS_HOME, roots, label)
334
+ open_workspace(ws)
335
+ return ws
336
+
337
+
338
+ def _load_jobs():
339
+ ws = STATE["workspace"]
340
+ if ws is None:
341
+ return []
342
+ return workspace.load_jobs(ws, STATE["job_roots"])
343
+
344
+
345
+ def _jobs_for_workspace(name=None):
346
+ """Read an indexed workspace without making it active or converting sources.
347
+ Once per request and workspace: a page that asks for the jobs several
348
+ times (the job page's controls, the rail, a summary) walks them once."""
349
+ memo = g.setdefault("_jobs", {}) if has_request_context() else {}
350
+ key = name if name and not (STATE["workspace"] and name == STATE["workspace"].name) else ""
351
+ if key in memo:
352
+ return memo[key]
353
+ if not key:
354
+ jobs = _load_jobs()
355
+ else:
356
+ ws = workspace.load(WS_HOME, name)
357
+ if ws is None:
358
+ abort(404)
359
+ jobs = workspace.load_jobs(ws, ws.job_roots or ws.roots)
360
+ memo[key] = jobs
361
+ return jobs
362
+
363
+
364
+ def _record_workspace():
365
+ name = request.args.get("workspace") if has_request_context() else None
366
+ if not name:
367
+ return STATE["workspace"]
368
+ ws = workspace.load(WS_HOME, name)
369
+ if ws is None:
370
+ abort(404)
371
+ return ws
372
+
373
+
374
+ def _jobs_for_request():
375
+ return _jobs_for_workspace(request.args.get("workspace"))
376
+
377
+
378
+ @app.url_defaults
379
+ def _preserve_record_workspace(endpoint, values):
380
+ """Carry evidence provenance through links, forms, and lazy detail requests."""
381
+ record_endpoints = {"job", "trial", "save_trace", "artifact_page", "conversation_data",
382
+ "trace_span_data"}
383
+ if has_request_context() and endpoint in record_endpoints and "workspace" not in values:
384
+ name = request.args.get("workspace")
385
+ if name:
386
+ values["workspace"] = name
387
+
388
+
389
+ def _find_job(jobs, job_id):
390
+ return next((j for j in jobs if j.job_id == job_id), None)
391
+
392
+
393
+ def _safe_dest(base: str, rel: str) -> str | None:
394
+ """Join *rel* onto *base*, refusing anything that escapes *base*."""
395
+ dest = os.path.normpath(os.path.join(base, rel))
396
+ if os.path.commonpath([base, dest]) != base:
397
+ return None
398
+ return dest
399
+
400
+
401
+ @app.template_global()
402
+ def view_query(filt, chart_filt):
403
+ """Every argument a trial view is defined by, as one set of `url_for` kwargs.
404
+
405
+ The two filters answer different questions — `trace.Filter` prunes spans out
406
+ of the whole view, `charts.ChartFilter` only says which kinds one chart
407
+ draws — but a link has to carry both, or clicking either control silently
408
+ discards the other. Each still owns the naming of its own query keys.
409
+ """
410
+ return {**filt.query, **chart_filt.query}
411
+
412
+
413
+ @app.template_global()
414
+ def task_labels_for(job):
415
+ """Compact labels for this run only; IDs and ambiguous names stay intact."""
416
+ dataset = os.path.basename((job.dataset or "").rstrip("/")).split("@", 1)[0]
417
+ prefix = f"{dataset}__" if dataset and dataset != "mixed" else ""
418
+ labels, originals = {}, {}
419
+ for trial in job.trials:
420
+ full = trial.task_base
421
+ short = full[len(prefix):] if prefix and full.startswith(prefix) else full
422
+ short = short or full
423
+ labels[trial.trial_name] = short
424
+ originals.setdefault(short, set()).add(full)
425
+ return {trial.trial_name: (trial.task_base if len(originals[labels[trial.trial_name]]) > 1
426
+ else labels[trial.trial_name])
427
+ for trial in job.trials}
428
+
429
+
430
+ _CATALOG_CACHE: dict = {"signature": None, "cat": None}
431
+
432
+
433
+ def _catalog_cached():
434
+ # read again when any of its files changes, or a benchmark folder comes or goes
435
+ signature = catalog.signature()
436
+ if all(mtime is None for _, mtime in signature):
437
+ return None
438
+ if _CATALOG_CACHE["signature"] != signature:
439
+ _CATALOG_CACHE.update(signature=signature, cat=catalog.load())
440
+ return _CATALOG_CACHE["cat"]
441
+
442
+
443
+ def _csrf_token() -> str:
444
+ token = session.get("csrf")
445
+ if not token:
446
+ token = secrets.token_urlsafe(32)
447
+ session["csrf"] = token
448
+ return token
449
+
450
+
451
+ def _origin_of(url: str) -> str:
452
+ """scheme://host[:port] of an Origin or Referer, compared whole: a prefix
453
+ test would let http://127.0.0.1:50570 pass for :5057."""
454
+ parts = urlsplit(url)
455
+ return f"{parts.scheme}://{parts.netloc}".lower()
456
+
457
+
458
+ # Every page's forms send it: the workspace menu in base.html posts from anywhere.
459
+ app.jinja_env.globals["csrf_token"] = lambda: _csrf_token()
460
+
461
+
462
+ def _interface_id() -> str:
463
+ """A digest of the templates, static files and web routes a page is made of
464
+ and reads. A page rendered under another digest runs code this process no
465
+ longer serves."""
466
+ digest = hashlib.sha256()
467
+ for root in (os.path.join(app.root_path, app.template_folder), app.static_folder, os.path.join(app.root_path, "web")):
468
+ for folder, subfolders, names in os.walk(root):
469
+ subfolders[:] = sorted(d for d in subfolders if d != "__pycache__") # written at import, not code
470
+ for name in sorted(names):
471
+ path = os.path.join(folder, name)
472
+ digest.update(os.path.relpath(path, root).encode() + b"\0")
473
+ with open(path, "rb") as f:
474
+ digest.update(f.read())
475
+ return digest.hexdigest()[:16]
476
+
477
+
478
+ INTERFACE = _interface_id()
479
+ app.jinja_env.globals["interface_id"] = lambda: INTERFACE
480
+
481
+
482
+ @app.after_request
483
+ def _send_interface(response):
484
+ response.headers["X-Nullius-Interface"] = INTERFACE
485
+ return response
486
+
487
+
488
+ class StaleFormToken(Forbidden):
489
+ """A token the session does not hold, as from a page loaded before a restart.
490
+ Its header tells the page to ask for a reload instead of showing the error."""
491
+ description = "Stale form token; reload the page and try again."
492
+
493
+ def get_headers(self, environ=None, scope=None):
494
+ return [*super().get_headers(environ, scope), ("X-Nullius-Reason", "stale-form-token")]
495
+
496
+
497
+ def _require_csrf():
498
+ """The one guard every mutation of the environments takes: the session's
499
+ token, sent back by the form, and a same-origin request."""
500
+ sent = request.form.get("csrf") or request.headers.get("X-CSRF-Token")
501
+ if not sent:
502
+ abort(403, description="Missing form token; reload the page and try again.")
503
+ if not secrets.compare_digest(sent, session.get("csrf", "")):
504
+ raise StaleFormToken()
505
+ origin = request.headers.get("Origin") or request.headers.get("Referer")
506
+ if origin and _origin_of(origin) != request.host_url.rstrip("/").lower():
507
+ abort(403, description="Cross-origin requests cannot change environments.")