nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,489 @@
1
+ """Runs against runs: whole runs, matched tasks, one task side by side."""
2
+ from __future__ import annotations
3
+
4
+ from functools import partial
5
+ from itertools import zip_longest
6
+ from flask import abort, flash, redirect, render_template, request, url_for
7
+ from nullius.traces import behavior, charts, trace, trajectory
8
+ from nullius.runs import classify
9
+ from nullius.runs import verifier
10
+ from nullius import workspace
11
+
12
+ from .core import app
13
+ from . import shared
14
+ from . import filters
15
+
16
+
17
+ def _dash(value, suffix=""):
18
+ return "—" if value in (None, "") else f"{value}{suffix}"
19
+
20
+
21
+ # Every cost this application shows is priced by the agent from its price
22
+ # table over the usage it recorded — the ATIF RFC formula, never a provider's
23
+ # invoice (the field guide: derived, not published). The row says so in its
24
+ # name, and a run whose agent priced nothing shows a dash, not a free run.
25
+ COST_LABEL = "Cost (estimated)"
26
+ COST_HINT = ("An estimate: priced by the agent from its price table over the recorded usage, "
27
+ "not a figure from the provider. — where the agent recorded none")
28
+
29
+
30
+ def _trial_stats(t):
31
+ """Trajectory-derived numbers for one trial (empty dict when unavailable)."""
32
+ if t is None or not t.has_trajectory:
33
+ return {}
34
+ tr = trajectory.load(t.trajectory_path)
35
+ if tr.error:
36
+ return {}
37
+ return {
38
+ "steps": tr.n_steps,
39
+ "calls": len(tr.calls),
40
+ "failed": tr.n_failed_calls,
41
+ "cats": dict(tr.category_freq),
42
+ "files": tr.files_touched,
43
+ "beh": behavior.analyze(tr),
44
+ }
45
+
46
+
47
+ def rank_cells(raw, direction, allow_single=False, cells=None, exclude=None):
48
+ """Per-cell classes: 'best' for the winner(s), 'tie' when all sides match.
49
+
50
+ *direction* is 'lower', 'higher' or None (no meaningful winner). Cells with
51
+ no value are never marked — a missing measurement isn't a loss. Normally a
52
+ metric needs two values to have a winner; *allow_single* marks the only
53
+ reported value anyway (used for cost, which not every agent records yet).
54
+
55
+ Ranking uses the raw numbers, but *cells* (the rendered text) decides
56
+ equality: two runs showing "$0.01" or "100%" must both win, even though
57
+ their underlying floats differ in a digit the reader can't see.
58
+
59
+ *exclude* marks columns that don't compete — Harbor's oracle/nop controls.
60
+ They are never marked, and they don't count toward "needs two values to
61
+ have a winner": a control at $0.00 and ~0s would take every cost and speed
62
+ row, which says nothing about the agents being compared.
63
+ """
64
+ n = len(raw)
65
+ exclude = list(exclude) if exclude else [False] * n
66
+ entrants = [i for i in range(n) if not exclude[i]]
67
+ reported = [i for i in entrants if raw[i] is not None]
68
+ values = [raw[i] for i in reported]
69
+ if not direction or len(values) < (1 if allow_single else 2):
70
+ return [""] * n
71
+
72
+ shown = list(cells) if cells else [None] * n
73
+
74
+ # Everyone displaying the same thing is a draw — but only if every entrant
75
+ # reported. If some are missing the measurement, those that have it are
76
+ # joint winners rather than a no-contest draw.
77
+ all_reported = len(reported) == len(entrants)
78
+ if cells and all_reported and len(values) > 1 and len({shown[i] for i in reported}) == 1:
79
+ return ["tie" if not exclude[i] else "" for i in range(n)]
80
+
81
+ best = min(values) if direction == "lower" else max(values)
82
+ if not cells and len(values) > 1 and best == (max(values) if direction == "lower" else min(values)):
83
+ return ["tie" if i in reported else "" for i in range(n)]
84
+
85
+ winners = {i for i in reported if raw[i] == best}
86
+ # Anything rendering identically to a winner wins too.
87
+ if cells:
88
+ texts = {shown[i] for i in winners}
89
+ winners |= {i for i in reported if shown[i] in texts}
90
+ return ["best" if i in winners else "" for i in range(n)]
91
+
92
+
93
+ def _row(label, cells, raw=None, direction=None, hint="", allow_single=False, exclude=None):
94
+ raw = raw if raw is not None else [None] * len(cells)
95
+ return {
96
+ "label": label, "hint": hint, "cells": cells,
97
+ "classes": rank_cells(raw, direction, allow_single, cells, exclude),
98
+ }
99
+
100
+
101
+ def _ratio(part, whole):
102
+ return (part / whole) if (part and whole) else None
103
+
104
+
105
+ def summary_rows(jobs):
106
+ """The numeric half of the side-by-side table, with winners marked.
107
+
108
+ Control runs (oracle/nop) are excluded from every ranking, and their token
109
+ and cost cells render a real 0 rather than the "—" used for a missing
110
+ measurement.
111
+ """
112
+ controls = [j.is_control for j in jobs]
113
+ kinds = [j.kind for j in jobs]
114
+
115
+ def row(label, cells, raw=None, direction=None, hint="", allow_single=False):
116
+ return _row(label, cells, raw, direction, hint, allow_single, exclude=controls)
117
+
118
+ def tok(values):
119
+ return [filters._tokens(v, k) for v, k in zip(values, kinds)]
120
+
121
+ # A benchmark that scores (an F1, partial credit) is compared on its mean
122
+ # score - over every part when a trial is scored on its parts - and a run
123
+ # with a score and no pass or fail has nothing solved to count.
124
+ scored = [
125
+ row("Mean score", ["—" if j.mean_score is None else f"{j.mean_score:.2f}" for j in jobs],
126
+ [j.mean_score for j in jobs], "higher",
127
+ "The mean of the benchmark's own score, over every part scored when a trial is scored on its parts"),
128
+ ] if any(j.mean_score is not None for j in jobs) else []
129
+ # and by the field the benchmark breaks it down by (a question's category), a row a value
130
+ by = next((j.parts_by for j in jobs if j.parts_by), "")
131
+ groups = {}
132
+ for j in jobs:
133
+ for g in j.score_by_group:
134
+ groups.setdefault(g["group"], None)
135
+ for g in groups:
136
+ means = [next((x["mean"] for x in j.score_by_group if x["group"] == g), None) for j in jobs]
137
+ scored.append(row(f"{by.capitalize()} {g}", ["—" if m is None else f"{m:.2f}" for m in means], means, "higher",
138
+ f"The mean score of the parts with {by} {g}"))
139
+ return [
140
+ row("Trials", [len(j.trials) for j in jobs]),
141
+ *scored,
142
+ row("Solved", ["—" if j.scored_only else f"{j.n_resolved}/{len(j.trials)}" for j in jobs],
143
+ [None if j.scored_only else j.n_resolved for j in jobs], "higher"),
144
+ row("Raw rate", [filters._pct(j.raw_solve_rate) for j in jobs],
145
+ [j.raw_solve_rate for j in jobs], "higher", "Solved ÷ all trials"),
146
+ row("Adjusted rate", [filters._pct(j.adjusted_solve_rate) for j in jobs],
147
+ [j.adjusted_solve_rate for j in jobs], "higher",
148
+ "Solved ÷ trials that actually ran"),
149
+ row("Infra errors", [j.n_infra or "—" for j in jobs],
150
+ [j.n_infra for j in jobs], "lower",
151
+ "Trials that never got a fair attempt"),
152
+ row("Broke tests", [j.n_regressions or "—" for j in jobs],
153
+ [j.n_regressions for j in jobs], "lower",
154
+ "Patches that broke previously-passing tests"),
155
+ # Token block, ordered so the arithmetic reads down the column:
156
+ # prompt size = fresh + cached, then the ratio those two imply.
157
+ row("Prompt size", tok([j.total_input_tokens for j in jobs]),
158
+ [j.total_input_tokens or None for j in jobs], "lower",
159
+ "Total prompt tokens sent, cached ones included"),
160
+ row("Fresh prompt",
161
+ tok([j.total_input_tokens - j.total_cache_tokens for j in jobs]),
162
+ [(j.total_input_tokens - j.total_cache_tokens) or None for j in jobs], "lower",
163
+ "Prompt tokens NOT served from cache — the expensive part"),
164
+ # Ranked on the ratio, not the absolute: a bigger cached count only
165
+ # means a bigger prompt. The share of the prompt reused is the claim.
166
+ row("Cached", tok([j.total_cache_tokens for j in jobs]),
167
+ [_ratio(j.total_cache_tokens, j.total_input_tokens) for j in jobs], "higher",
168
+ "Prompt tokens served from cache — ranked by share of the prompt, not the raw count"),
169
+ row("Cached ratio",
170
+ [filters._pct(_ratio(j.total_cache_tokens, j.total_input_tokens)) for j in jobs],
171
+ [_ratio(j.total_cache_tokens, j.total_input_tokens) for j in jobs], "higher",
172
+ "Cached ÷ prompt size — higher means less of the prompt was re-sent"),
173
+ row("Output tokens", tok([j.total_output_tokens for j in jobs]),
174
+ [j.total_output_tokens or None for j in jobs], "lower"),
175
+ row(COST_LABEL, [filters._usd(j.total_cost_usd, k) for j, k in zip(jobs, kinds)],
176
+ [j.total_cost_usd for j in jobs], "lower", COST_HINT, allow_single=True),
177
+ row("Wall time",
178
+ [f"{j.duration_min:.0f}m" if j.duration_min else "—" for j in jobs],
179
+ [j.duration_min for j in jobs], "lower"),
180
+ ]
181
+
182
+
183
+ def compare_detail(row, jobs):
184
+ """Line up the same measurements across each run's trial for one task.
185
+
186
+ Returns {"rows": [{label, hint, cells, classes}], "files": [...],
187
+ "broke": [...]}, with "—" wherever a run didn't cover the task. *jobs* is
188
+ the matching run per column, needed to tell a control (real zeros, never
189
+ ranked) from an agent that simply failed to report.
190
+ """
191
+ stats = [_trial_stats(t) for t in row]
192
+ controls = [j.is_control for j in jobs]
193
+ kinds = [j.kind for j in jobs]
194
+
195
+ # Controls never compete, so every row in this block excludes them.
196
+ field = partial(_row, exclude=controls)
197
+
198
+ def fmt_time(t):
199
+ return "—" if t is None or t.dur_agent is None else filters._secs(t.dur_agent)
200
+
201
+ def fmt_phase(t):
202
+ return "—" if t is None else f"{filters._secs(t.dur_env)} / {filters._secs(t.dur_verifier)}"
203
+
204
+ def fmt_tests(t, attr_pass, attr_fail):
205
+ if t is None:
206
+ return "—"
207
+ good, bad = getattr(t, attr_pass), getattr(t, attr_fail)
208
+ return "—" if not (good or bad) else f"{good} passed, {bad} failed"
209
+
210
+ def fmt_cats(s):
211
+ if not s.get("cats"):
212
+ return "—"
213
+ order = ("lookup", "change", "search", "read", "edit", "execute", "other")
214
+ parts = [f"{k} {s['cats'][k]}" for k in order if s["cats"].get(k)]
215
+ return " · ".join(parts)
216
+
217
+ def fmt_first_edit(s):
218
+ b = s.get("beh")
219
+ if not b or b.first_edit_call is None:
220
+ return "—"
221
+ out = f"call {b.first_edit_call}"
222
+ if b.time_to_first_edit is not None:
223
+ out += f" · {filters._secs(b.time_to_first_edit)}"
224
+ return out
225
+
226
+ def phase_total(t):
227
+ return None if t is None else (t.dur_env or 0) + (t.dur_verifier or 0)
228
+
229
+ def order_rank(t):
230
+ """Position in classify.ORDER — lower is a better outcome."""
231
+ return None if t is None else classify.ORDER.index(t.outcome)
232
+
233
+ # One task, so one dataset: the first present trial's vocabulary names the
234
+ # check rows, and a category the dataset doesn't have emits no row at all.
235
+ vocab = next((t.vocab for t in row if t), verifier.GENERIC_VOCABULARY)
236
+
237
+ check_rows = []
238
+ if vocab.gained:
239
+ check_rows.append(
240
+ field(vocab.gained, [fmt_tests(t, "f2p_pass", "f2p_fail") for t in row],
241
+ [t.f2p_fail if t else None for t in row], "lower",
242
+ "Checks the attempt had to make pass — fewer failures is better"))
243
+ if vocab.held:
244
+ check_rows.append(
245
+ field(vocab.held, [fmt_tests(t, "p2p_pass", "p2p_fail") for t in row],
246
+ [t.p2p_fail if t else None for t in row], "lower",
247
+ "Tests that had to keep passing — failures here are regressions"))
248
+
249
+ rows = [
250
+ field("Outcome", [t.outcome_label if t else "—" for t in row],
251
+ [order_rank(t) for t in row], "lower"),
252
+ *([field("Score", ["—" if t is None or t.score is None else f"{t.score:.2f}" for t in row],
253
+ [None if t is None else t.score for t in row], "higher",
254
+ "The benchmark's own score for this task" + (", the mean of its parts" if any(t and t.n_parts for t in row) else ""))]
255
+ if any(t and t.score is not None for t in row) else []),
256
+ *check_rows,
257
+ field("Agent time", [fmt_time(t) for t in row],
258
+ [t.dur_agent if t else None for t in row], "lower",
259
+ "Time the agent spent working"),
260
+ field("Env / Verify", [fmt_phase(t) for t in row],
261
+ [phase_total(t) for t in row], "lower",
262
+ "Container build / verifier run"),
263
+ field("Steps", [_dash(s.get("steps")) for s in stats],
264
+ hint="Trajectory steps — not ranked, fewer isn't inherently better"),
265
+ field("Tool calls", [_dash(s.get("calls")) for s in stats],
266
+ hint="Real calls; writer placeholders excluded — not ranked"),
267
+ field("Failed calls", [_dash(s.get("failed")) for s in stats],
268
+ [s.get("failed") for s in stats], "lower",
269
+ "Tool calls that returned an error"),
270
+ field("Tool mix", [fmt_cats(s) for s in stats],
271
+ hint="lookup / change / search / read / edit / execute"),
272
+ field("First edit", [fmt_first_edit(s) for s in stats],
273
+ hint="Which tool call first changed a file, and how far into the run"),
274
+ field("Edit → test cycles", [_dash(s["beh"].edit_test_cycles) if s.get("beh") else "—" for s in stats],
275
+ hint="Times an edit was followed by running something — the iterate loop"),
276
+ field("Redundant reads", [_dash(s["beh"].n_redundant_reads) if s.get("beh") else "—" for s in stats],
277
+ [s["beh"].n_redundant_reads if s.get("beh") else None for s in stats], "lower",
278
+ "Re-fetches of a target already read or searched — paid for twice"),
279
+ field("Retried failed calls", [_dash(s["beh"].n_repeated_failed_calls) if s.get("beh") else "—" for s in stats],
280
+ [s["beh"].n_repeated_failed_calls if s.get("beh") else None for s in stats], "lower",
281
+ "Calls repeating something that had already failed — flailing"),
282
+ field("Exploration per edit", [_dash(s["beh"].exploration_per_edit) if s.get("beh") else "—" for s in stats],
283
+ hint="(search + read) ÷ edits — not ranked, more isn't inherently worse"),
284
+ field("Prompt size", [filters._tokens(t.n_input_tokens, k) if t else "—" for t, k in zip(row, kinds)],
285
+ [(t.n_input_tokens or None) if t else None for t in row], "lower",
286
+ "Total prompt tokens sent, cached ones included"),
287
+ field("Fresh prompt",
288
+ [filters._tokens(t.n_input_tokens - t.n_cache_tokens, k) if t else "—" for t, k in zip(row, kinds)],
289
+ [((t.n_input_tokens - t.n_cache_tokens) or None) if t else None for t in row],
290
+ "lower", "Prompt tokens NOT served from cache — the expensive part"),
291
+ field("Cached", [filters._tokens(t.n_cache_tokens, k) if t else "—" for t, k in zip(row, kinds)],
292
+ [_ratio(t.n_cache_tokens, t.n_input_tokens) if t else None for t in row], "higher",
293
+ "Prompt tokens served from cache — ranked by share of the prompt, not the raw count"),
294
+ field("Cached ratio",
295
+ [filters._pct(_ratio(t.n_cache_tokens, t.n_input_tokens)) if t else "—" for t in row],
296
+ [_ratio(t.n_cache_tokens, t.n_input_tokens) if t else None for t in row], "higher",
297
+ "Cached ÷ prompt size — higher means less of the prompt was re-sent"),
298
+ field("Output tokens", [filters._tokens(t.n_output_tokens, k) if t else "—" for t, k in zip(row, kinds)],
299
+ [(t.n_output_tokens or None) if t else None for t in row], "lower",
300
+ "Completion tokens"),
301
+ field(COST_LABEL, [filters._usd(t.cost_usd, k) if t else "—" for t, k in zip(row, kinds)],
302
+ [t.cost_usd if t else None for t in row], "lower", COST_HINT, allow_single=True),
303
+ ]
304
+ return {
305
+ "rows": rows,
306
+ "files": [s.get("files") or [] for s in stats],
307
+ "broke": [(t.p2p_failures if t else []) for t in row],
308
+ }
309
+
310
+
311
+ @app.route("/compare")
312
+ def compare():
313
+ """Qualified, ordered run columns over whole runs or exact shared tasks."""
314
+ view = _comparison_requested()
315
+ if len(view.selected) < 2:
316
+ if view.selected:
317
+ flash("Choose two different runs to compare.", "error")
318
+ return compare_picker()
319
+ chosen = view.jobs
320
+ by_agent: dict = {}
321
+ for selection in view.selected:
322
+ by_agent.setdefault(selection.job.agent_label or "—", []).append(selection.value)
323
+ grouped = [ref for group in by_agent.values() for ref in group]
324
+ alternating = [
325
+ ref
326
+ for tier in zip_longest(*by_agent.values())
327
+ for ref in tier
328
+ if ref is not None
329
+ ]
330
+ summary = summary_rows(chosen) if view.cohort == "whole" or view.shared_tasks else []
331
+ # There is no truthful wall-clock interval for a subset of parallel trials.
332
+ # Whole-run intervals keep their existing ranking, explicitly labelled.
333
+ if view.cohort == "matched":
334
+ summary = [row for row in summary if row["label"] != "Wall time"]
335
+ else:
336
+ for row in summary:
337
+ if row["label"] == "Wall time":
338
+ row.update(label="Run wall time", hint="The recorded wall-clock interval of the full run")
339
+ return render_template(
340
+ "compare/compare.html", comparison=view, selections=view.selected, jobs=chosen,
341
+ tasks=view.tasks, summary=summary,
342
+ n_differing=view.n_differing, n_shared=view.shared_tasks,
343
+ picks=view.references, cohort=view.cohort,
344
+ order_grouped=grouped, order_alternating=alternating,
345
+ n_agents=len(by_agent),
346
+ judgement=_judgement(view), csrf=shared._csrf_token(),
347
+ )
348
+
349
+
350
+ @app.route("/compare/detail/<int:index>")
351
+ def compare_detail_row(index):
352
+ """One task's detail row, built when it is opened: building every row of
353
+ the page loaded and analysed every trajectory of every run compared
354
+ (2 x 500 trials took 10 to 12 s and 4.8 MB of HTML)."""
355
+ view = _comparison_requested()
356
+ if len(view.selected) < 2 or not 0 <= index < len(view.tasks):
357
+ abort(404)
358
+ return render_template("compare/_detail.html", jobs=view.jobs, d=compare_detail(view.tasks[index].trials, view.jobs))
359
+
360
+
361
+ def _judgement(view):
362
+ from . import judge
363
+ return judge.compare_summary(view) if view.cohort == "matched" else None
364
+
365
+
366
+ def _comparison_requested():
367
+ """Resolve only the workspaces named by this URL, without switching one."""
368
+ from nullius.compare import comparison
369
+ active = shared.STATE["workspace"]
370
+ active_name = active.name if active else ""
371
+ values = request.args.getlist("pick")
372
+ if not values:
373
+ # Qualify Runs-page selections with the active workspace for generated links.
374
+ values = [comparison.RunRef(active_name, job_id).value
375
+ for job_id in request.args.getlist("job")]
376
+ try:
377
+ refs = [comparison.RunRef.parse(value) for value in values]
378
+ names = list(dict.fromkeys(ref.workspace for ref in refs))
379
+ jobs = {name: shared._jobs_for_workspace(name) for name in names}
380
+ labels = {}
381
+ for name in names:
382
+ ws = active if name == active_name else workspace.load(shared.WS_HOME, name)
383
+ labels[name] = ws.label if ws else "Active source"
384
+ selected = comparison.resolve(values, jobs, labels)
385
+ return comparison.build(selected, request.args.get("cohort", "whole"))
386
+ except ValueError as exc:
387
+ abort(400, description=str(exc))
388
+
389
+
390
+ @app.route("/compare/pick")
391
+ def compare_picker():
392
+ """Choose real runs from two saved workspaces; reads never open or refresh."""
393
+ from nullius.compare import comparison
394
+ saved = workspace.list_all(shared.WS_HOME)
395
+ active = shared.STATE["workspace"]
396
+ options = [{"name": ws.name, "label": ws.label} for ws in saved]
397
+ if active and active.name not in {option["name"] for option in options}:
398
+ options.insert(0, {"name": active.name, "label": active.label})
399
+ if not options or (not active and shared.STATE["roots"]):
400
+ options.insert(0, {"name": "", "label": "Active source"})
401
+ values = request.args.getlist("pick")
402
+ try:
403
+ refs = [comparison.RunRef.parse(value) for value in values[:2]]
404
+ except ValueError as exc:
405
+ abort(400, description=str(exc))
406
+ default_left = refs[0].workspace if refs else (active.name if active else options[0]["name"])
407
+ left = request.args.get("left", default_left)
408
+ default_right = refs[1].workspace if len(refs) > 1 else next(
409
+ (option["name"] for option in options if option["name"] != left), left)
410
+ right = request.args.get("right", default_right)
411
+ choices, loaded = [], {}
412
+ for index, name in enumerate((left, right)):
413
+ if name not in {option["name"] for option in options}:
414
+ abort(404, description="That workspace is unavailable.")
415
+ if name not in loaded:
416
+ loaded[name] = shared._jobs_for_workspace(name)
417
+ jobs = loaded[name]
418
+ selected = refs[index].value if len(refs) > index and refs[index].workspace == name else ""
419
+ if not selected and jobs:
420
+ run_index = 1 if index == 1 and left == right and len(jobs) > 1 else 0
421
+ selected = comparison.RunRef(name, jobs[run_index].job_id).value
422
+ choices.append({"name": name, "label": next(option["label"] for option in options if option["name"] == name),
423
+ "jobs": jobs, "selected": selected})
424
+ cohort = request.args.get("cohort", "whole")
425
+ if cohort not in {"whole", "matched"}:
426
+ abort(400, description="Choose whole runs or matched tasks as the comparison cohort.")
427
+ return render_template("compare/compare_picker.html", workspace_options=options,
428
+ choices=choices, cohort=cohort)
429
+
430
+
431
+ def trace_panel(t, chart_filter=None):
432
+ """One side's trace on the task page: its charts and strips, or why not.
433
+
434
+ Reads the producer's own ``trace.json`` unfiltered — the compare page has no
435
+ ``hide`` control, so there is nothing for a saved view to differ by. The
436
+ charts go through :func:`charts.charts_for`, which is :func:`charts.charts_of`
437
+ over that trace's own samples, so a side here shows the numbers the trial's
438
+ own page shows for it (`test_comparison_routes` pins the two as one
439
+ drawing). Each side is loaded on its own and never pooled: two trials never
440
+ shared a clock, so nothing here carries a wall clock across them.
441
+
442
+ Returns ``{"trace": Trace | None, "charts": Charts | None, "absent": str}``
443
+ — ``absent`` says, in words, why a side has nothing to draw. Exactly one of
444
+ ``charts`` and ``absent`` is set: a side is never silently empty.
445
+ """
446
+ if t is None:
447
+ return {"trace": None, "charts": None, "absent": "Didn't include this task."}
448
+ if not t.has_trace:
449
+ return {"trace": None, "charts": None,
450
+ "absent": "No trace was captured for this trial — the agent emitted no spans."}
451
+ tr = trace.load(t.trace_path)
452
+ if tr.error:
453
+ return {"trace": tr, "charts": None, "absent": f"Trace unreadable: {tr.error}"}
454
+ if not tr.has_spans:
455
+ return {"trace": tr, "charts": None, "absent": "The trace has no spans to plot."}
456
+ return {"trace": tr, "charts": charts.charts_for(tr.roots, chart_filter), "absent": ""}
457
+
458
+
459
+ @app.route("/compare/task/<path:task>")
460
+ def compare_task(task):
461
+ """Full detail of the same last indexed attempt shown in the task table."""
462
+ view = _comparison_requested()
463
+ if len(view.selected) < 2:
464
+ return redirect(url_for("compare_picker", cohort=view.cohort))
465
+ dataset = request.args.get("dataset")
466
+ matches = [row for row in view.tasks if row.task == task and (dataset is None or row.dataset == dataset)]
467
+ if len(matches) != 1:
468
+ if len(matches) > 1:
469
+ flash("That task name occurs in different datasets. Choose its dataset row first.", "error")
470
+ return redirect(url_for("compare", pick=view.references, cohort=view.cohort))
471
+ abort(404, description="This task is not in the selected comparison cohort.")
472
+ row = matches[0]
473
+ # One legend state for every side: a kind switched off in a chart is off in
474
+ # that chart on both sides, so the two stay drawn over the same kinds. The
475
+ # trace filter (`hide`) is deliberately not offered here — a filtered view
476
+ # belongs to one trial, and is saved from its own page.
477
+ cfilt = charts.ChartFilter.of(request.args.getlist("chart_off"))
478
+ panels = []
479
+ for selection, job, t, attempts in zip(view.selected, view.jobs, row.trials, row.attempts):
480
+ traj = trajectory.load(t.trajectory_path) if (t and t.has_trajectory) else None
481
+ panels.append({"job": job, "selection": selection, "trial": t,
482
+ "attempts": attempts,
483
+ "traj": traj if traj and not traj.error else None,
484
+ **trace_panel(t, cfilt)})
485
+ return render_template(
486
+ "compare/compare_task.html", task=task, dataset=row.dataset, panels=panels,
487
+ comparison=view, picks=view.references, cohort=view.cohort,
488
+ chart_filter=cfilt, no_filter=trace.Filter(),
489
+ )
@@ -0,0 +1,17 @@
1
+ """The Flask application object and its configuration. Every capability module registers its routes on it; nothing else lives here."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+ from flask import Flask
6
+
7
+
8
+
9
+ # The source app/ directory, or the wheel's _nullius_app/: templates and
10
+ # static stay beside the web package in both layouts.
11
+ APP_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
12
+ app = Flask(__name__, root_path=APP_DIR)
13
+ app.config["MAX_CONTENT_LENGTH"] = 1024 * 1024 * 1024 # 1 GiB per folder upload
14
+ # Signs the session cookie (flash messages, the form token). A new random key
15
+ # per process: nothing here outlives the run, and nobody needs to set one.
16
+ app.secret_key = os.urandom(32)
17
+