nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,442 @@
1
+ """The feasibility sweep, read back from what the controls recorded.
2
+
3
+ A sweep runs Harbor's two control agents over a task set to ask one question
4
+ per task — *is the task feasible*: can an empty solution fail, and can a
5
+ reference solution pass? ``nop`` does nothing, so the verifier must reject
6
+ it; ``oracle`` applies the task's reference solution, so the verifier must
7
+ accept it. Environment setup, data loading and the verifier are all
8
+ exercised on the way. It is feasibility evidence and nothing else: not a
9
+ configuration or hyperparameter search, and no promise that an agent will
10
+ succeed on a feasible task.
11
+
12
+ This module builds the report from recorded jobs. It never runs anything.
13
+
14
+ Where the controls are on disk: a run started from the builder with several
15
+ agents writes one job per agent, ``<name>-nop``, ``<name>-oracle`` and
16
+ ``<name>-<agent>``; the portable bundle's ``--bracket`` writes
17
+ ``<name>-bracket-nop`` and ``<name>-bracket-oracle``; a sweep may also be
18
+ a control job on its own. ``related`` finds the controls that belong with a
19
+ job by those names, within the one workspace, and ``report`` reads them.
20
+
21
+ Per task the report keeps **both** control outcomes and classifies from the
22
+ pair, in this order — the first that applies wins, so a task is called
23
+ feasible only when both sides ran and said the right thing:
24
+
25
+ unexpected_reward nop was accepted: the verifier rejects nothing, and
26
+ nothing else about the task can be trusted
27
+ task_broken nop was rejected, but the verifier says the empty
28
+ solution failed tests that were passing before it —
29
+ held (PASS_TO_PASS) checks. Nothing was changed, so
30
+ the task was already broken as shipped, or its
31
+ verifier is unstable; whatever oracle said, the
32
+ task is not feasible
33
+ missing a side is absent (never ran, or not in this
34
+ workspace) or recorded no outcome
35
+ setup_failure a side failed before the verifier ran — an
36
+ environment, image, API or agent error — so the
37
+ task was not exercised
38
+ reference_rejected oracle ran and the verifier rejected the reference
39
+ solution: the task cannot be passed as shipped
40
+ feasible oracle accepted, nop rejected, both by the verifier
41
+
42
+ One side alone never validates a task: oracle-only evidence cannot show that
43
+ a verifier rejects an empty solution, nop-only evidence cannot show that a
44
+ solution can pass. The header says which controls actually executed, and
45
+ over what scope, so a report over one control reads as exactly that.
46
+
47
+ ``task_broken`` is the same facet the trial page shows as *⚠ Broke tests*
48
+ (`Trial.is_regression`, from `p2p_fail`), read on the control instead of the
49
+ agent: on the agent's trial a held failure is a regression the patch caused;
50
+ on ``nop``, which applies nothing, it is a failure the task brought with it.
51
+ It is benchmark-agnostic — any reader that fills ``held`` gets it — and a
52
+ benchmark whose verifier reports no held checks simply never
53
+ produces the class, rather than a plausible "not broken".
54
+ """
55
+ from __future__ import annotations
56
+
57
+ import re
58
+ from dataclasses import dataclass, field
59
+
60
+ from . import classify
61
+ from .parse import CONTROL_AGENTS
62
+
63
+ NOP, ORACLE = "nop", "oracle"
64
+ CONTROLS = (NOP, ORACLE)
65
+
66
+ FEASIBLE = "feasible"
67
+ UNEXPECTED_REWARD = "unexpected_reward"
68
+ TASK_BROKEN = "task_broken"
69
+ SETUP_FAILURE = "setup_failure"
70
+ REFERENCE_REJECTED = "reference_rejected"
71
+ MISSING = "missing"
72
+ ORDER = [FEASIBLE, UNEXPECTED_REWARD, TASK_BROKEN, REFERENCE_REJECTED, SETUP_FAILURE, MISSING]
73
+ LABELS = {
74
+ FEASIBLE: "Feasible",
75
+ UNEXPECTED_REWARD: "Unexpected reward",
76
+ TASK_BROKEN: "Task broken",
77
+ REFERENCE_REJECTED: "Reference rejected",
78
+ SETUP_FAILURE: "Setup/data failure",
79
+ MISSING: "Missing result",
80
+ }
81
+ DESCRIPTIONS = {
82
+ FEASIBLE: "The verifier rejected the empty solution and accepted the reference one. Both controls ran; the task is feasible.",
83
+ UNEXPECTED_REWARD: "The verifier accepted the empty solution. It rejects nothing, so a pass on this task means nothing — whatever the oracle did.",
84
+ TASK_BROKEN: "The empty solution failed tests that were passing before it. nop changes nothing, so the task was already broken as shipped, or its verifier is unstable — the failures are named. Not feasible, whatever the oracle did.",
85
+ REFERENCE_REJECTED: "Both controls reached the verifier, and it rejected the reference solution. The task cannot be passed as shipped.",
86
+ SETUP_FAILURE: "A control failed before the verifier ran — environment build, image, API or agent error — so the task was not exercised on that side.",
87
+ MISSING: "A control has no recorded outcome for this task: it did not run, ran elsewhere, or recorded nothing. Not validated.",
88
+ }
89
+
90
+ # Outcomes that mean a control reached the verifier and was rejected.
91
+ _REJECTED = {classify.FAILED_VERIFICATION, classify.PATCH_FAILED_TESTS, classify.PATCH_NOT_APPLIED, classify.NO_PATCH}
92
+ # Outcomes that mean the control failed before the verifier could say anything.
93
+ _FAILED_BEFORE_VERIFIER = classify.INFRA_OUTCOMES | {classify.AGENT_ERROR}
94
+
95
+ _SUFFIX = re.compile(r"-(?:bracket-)?(?:nop|oracle)$")
96
+
97
+
98
+ @dataclass
99
+ class Side:
100
+ """One control's outcome on one task. ``present`` is False when the
101
+ control never recorded this task — that is the missing case, kept apart
102
+ from a recorded outcome of ``unknown``."""
103
+
104
+ control: str
105
+ present: bool = False
106
+ job_id: str = ""
107
+ trial_name: str = ""
108
+ outcome: str = ""
109
+ reward: float | None = None
110
+ exception_type: str = ""
111
+ # Held (PASS_TO_PASS) checks the verifier saw fail on this control, as
112
+ # `Trial.p2p_fail` / `p2p_failures` record them — the same reading the
113
+ # trial page flags as *⚠ Broke tests*.
114
+ p2p_fail: int = 0
115
+ p2p_failures: list = field(default_factory=list)
116
+
117
+ @property
118
+ def label(self) -> str:
119
+ if not self.present:
120
+ return "not run"
121
+ return classify.LABELS.get(self.outcome, self.outcome)
122
+
123
+ @property
124
+ def accepted(self) -> bool:
125
+ return self.present and self.outcome == classify.RESOLVED
126
+
127
+ @property
128
+ def rejected(self) -> bool:
129
+ return self.present and self.outcome in _REJECTED
130
+
131
+ @property
132
+ def failed(self) -> bool:
133
+ return self.present and self.outcome in _FAILED_BEFORE_VERIFIER
134
+
135
+ @property
136
+ def unscored(self) -> bool:
137
+ return not self.present or self.outcome == classify.UNKNOWN
138
+
139
+ @property
140
+ def broke_tests(self) -> bool:
141
+ """The verifier ran and saw previously-passing tests fail."""
142
+ return self.present and self.p2p_fail > 0
143
+
144
+
145
+ @dataclass
146
+ class TaskRow:
147
+ task_base: str
148
+ task_name: str
149
+ nop: Side
150
+ oracle: Side
151
+ verdict: str = MISSING
152
+
153
+ @property
154
+ def label(self) -> str:
155
+ return LABELS[self.verdict]
156
+
157
+ @property
158
+ def validated(self) -> bool:
159
+ """True only for `feasible`: both sides ran and the verifier answered
160
+ both the right way. Never true with a side missing or failed."""
161
+ return self.verdict == FEASIBLE
162
+
163
+ @property
164
+ def why(self) -> str:
165
+ """The row's reason in one phrase, naming the side that decided it."""
166
+ if self.verdict == UNEXPECTED_REWARD:
167
+ return "nop accepted"
168
+ if self.verdict == TASK_BROKEN:
169
+ n = self.nop.p2p_fail
170
+ return f"nop broke {n} passing test{'' if n == 1 else 's'}: " + ", ".join(self.nop.p2p_failures)
171
+ if self.verdict == MISSING:
172
+ sides = [s.control for s in (self.nop, self.oracle) if s.unscored]
173
+ return ", ".join(f"{c} {'not run' if not getattr(self, c).present else 'unscored'}" for c in sides)
174
+ if self.verdict == SETUP_FAILURE:
175
+ sides = [s for s in (self.nop, self.oracle) if s.failed]
176
+ return ", ".join(f"{s.control}: {s.exception_type or s.label}" for s in sides)
177
+ if self.verdict == REFERENCE_REJECTED:
178
+ return f"oracle {self.oracle.label.lower()}"
179
+ return "nop rejected, oracle accepted"
180
+
181
+
182
+ @dataclass
183
+ class ControlRun:
184
+ """One control as it actually executed: the job it is, and how much of
185
+ the scope it covered."""
186
+
187
+ control: str
188
+ job_id: str
189
+ n_trials: int
190
+ n_planned: int
191
+ state: str # preparing | running | finished
192
+
193
+ @property
194
+ def short(self) -> str:
195
+ """The control and its coverage, without the job id: what the page shows."""
196
+ planned = f" of {self.n_planned} planned" if self.n_planned and self.n_planned != self.n_trials else ""
197
+ return f"{self.control} · {self.n_trials} trial{'s' if self.n_trials != 1 else ''}{planned}{' · still ' + self.state if self.state != 'finished' else ''}"
198
+
199
+ @property
200
+ def summary(self) -> str:
201
+ """The same with the job it is: the hover, and the log line."""
202
+ return self.short.replace(f"{self.control} · ", f"{self.control} · {self.job_id} · ", 1)
203
+
204
+
205
+ @dataclass
206
+ class ControlSummary:
207
+ """One control job read as what it is: a check that behaved or did not,
208
+ never a score.
209
+
210
+ ``nop`` behaves when the verifier rejects it, ``oracle`` when the
211
+ verifier accepts it; the other answer is the feasibility class for that
212
+ side — ``unexpected reward`` for a resolved nop, ``reference rejected``
213
+ for a failed oracle. A trial that failed before the verifier ran is a
214
+ setup failure, and one with no verdict is unscored; neither says the
215
+ task is feasible, so neither counts as behaved. The runs list shows
216
+ this in place of the control's Solved/Raw/Adjusted cells and keeps its
217
+ trials out of the agents' totals — a nop that failed every task is the
218
+ expected result, not "Failed verification".
219
+ """
220
+
221
+ control: str
222
+ n: int
223
+ n_behaved: int
224
+ n_unexpected: int
225
+ n_failed: int
226
+ n_unscored: int
227
+ running: bool = False
228
+
229
+ @property
230
+ def expected(self) -> str:
231
+ """What the verifier must say for this control to have behaved."""
232
+ return "rejected" if self.control == NOP else "passed"
233
+
234
+ @property
235
+ def problem(self) -> str:
236
+ """The feasibility class of the side that did not behave, or ''."""
237
+ if self.n_unexpected:
238
+ return "unexpected reward" if self.control == NOP else "reference rejected"
239
+ if self.n_failed:
240
+ return "setup failure"
241
+ return ""
242
+
243
+ @property
244
+ def ok(self) -> bool:
245
+ return self.n > 0 and self.n_behaved == self.n
246
+
247
+ @property
248
+ def state(self) -> str:
249
+ """`ok`, `warn` or `pending` — the tone of the line and the cells."""
250
+ if self.n_unexpected or self.n_failed:
251
+ return "warn"
252
+ if self.ok and not self.running:
253
+ return "ok"
254
+ return "pending"
255
+
256
+ @property
257
+ def label(self) -> str:
258
+ """`5/5 rejected ✓`, `1 unexpected reward ⚠`, `3/5 passed · running`."""
259
+ if not self.n:
260
+ return "no trials yet" if self.running else "no trials"
261
+ if self.problem:
262
+ n = self.n_unexpected or self.n_failed
263
+ return f"{n} {self.problem} ⚠"
264
+ if self.n_unscored:
265
+ return f"{self.n_behaved}/{self.n} {self.expected} · {self.n_unscored} unscored"
266
+ if self.running:
267
+ return f"{self.n_behaved}/{self.n} {self.expected} · running"
268
+ return f"{self.n_behaved}/{self.n} {self.expected} ✓"
269
+
270
+
271
+ def summary_of(job) -> ControlSummary | None:
272
+ """`job` read as a control, or None when it is an agent's run."""
273
+ control = control_of(job)
274
+ if not control:
275
+ return None
276
+ sides = [Side(control, True, job.job_id, t.trial_name, _outcome(t), t.reward, t.exception_type,
277
+ t.p2p_fail, list(t.p2p_failures)) for t in job.trials]
278
+ behaved = (lambda s: s.rejected) if control == NOP else (lambda s: s.accepted)
279
+ wrong = (lambda s: s.accepted) if control == NOP else (lambda s: s.rejected)
280
+ return ControlSummary(
281
+ control, len(sides),
282
+ sum(1 for s in sides if behaved(s)),
283
+ sum(1 for s in sides if wrong(s)),
284
+ sum(1 for s in sides if s.failed),
285
+ sum(1 for s in sides if s.unscored),
286
+ running=bool(job.is_running or job.is_preparing),
287
+ )
288
+
289
+
290
+ @dataclass
291
+ class Report:
292
+ job_id: str
293
+ dataset: str
294
+ n_tasks: int # the task limit the job recorded; 0 is the full dataset
295
+ executed: list = field(default_factory=list) # [ControlRun], in CONTROLS order
296
+ rows: list = field(default_factory=list) # [TaskRow], by task name
297
+
298
+ @property
299
+ def not_run(self) -> list:
300
+ ran = {c.control for c in self.executed}
301
+ return [c for c in CONTROLS if c not in ran]
302
+
303
+ @property
304
+ def scope(self) -> str:
305
+ """The task scope the job recorded, as the recipe asked for it."""
306
+ ds = self.dataset or "the dataset"
307
+ if self.n_tasks:
308
+ return f"{self.n_tasks} task{'s' if self.n_tasks != 1 else ''} of {ds} (--n-tasks {self.n_tasks})"
309
+ return f"every task of {ds} (no task limit recorded)"
310
+
311
+ @property
312
+ def counts(self) -> dict:
313
+ c = {k: 0 for k in ORDER}
314
+ for r in self.rows:
315
+ c[r.verdict] += 1
316
+ return c
317
+
318
+ @property
319
+ def n_feasible(self) -> int:
320
+ return self.counts[FEASIBLE]
321
+
322
+ @property
323
+ def caveat(self) -> str:
324
+ """What this evidence cannot say, when one control is absent."""
325
+ missing = self.not_run
326
+ if not missing:
327
+ return ""
328
+ if missing == [NOP]:
329
+ return "Only oracle ran: this cannot establish that the verifier rejects an empty solution. No task is validated."
330
+ if missing == [ORACLE]:
331
+ return "Only nop ran: this cannot establish that a solution can pass. No task is validated."
332
+ return "Neither control ran: nothing here is feasibility evidence."
333
+
334
+
335
+ def control_of(job) -> str:
336
+ """`nop` or `oracle` when the job is one of Harbor's controls, or a
337
+ standard job that ran a control its benchmark declares, else ''. A
338
+ runner's arm is never one, whatever it is called (`Job.kind`)."""
339
+ name = job.agent_label.lower()
340
+ return name if name in CONTROL_AGENTS and job.is_control else ""
341
+
342
+
343
+ def _outcome(t) -> str:
344
+ """A control's outcome on one task. A benchmark that scores and does not
345
+ pass or fail leaves *scored*: at the ceiling (1) that is the reference
346
+ accepted, at the floor (0) the answer rejected, in between neither."""
347
+ if t.outcome == classify.SCORED and t.score is not None:
348
+ if t.score >= 1.0:
349
+ return classify.RESOLVED
350
+ if t.score <= 0.0:
351
+ return classify.FAILED_VERIFICATION
352
+ return t.outcome
353
+
354
+
355
+ def base_name(job_id: str) -> str:
356
+ """The recipe name a control job belongs to: the id without its control
357
+ suffix, in either spelling the builder writes."""
358
+ return _SUFFIX.sub("", job_id)
359
+
360
+
361
+ def related(job, jobs) -> dict:
362
+ """The control jobs that belong with `job`, by control name.
363
+
364
+ Matched by the builder's naming: a job's own base name, and — for an
365
+ agent job named `<base>-<agent>` — the base with its last segment
366
+ dropped. A control job is always its own side. The dataset must agree
367
+ where both record one, so a control over another dataset that happens
368
+ to share a name is not read as this job's.
369
+ """
370
+ out = {}
371
+ own = control_of(job)
372
+ if own:
373
+ out[own] = job
374
+ bases = [base_name(job.job_id)]
375
+ if not own and "-" in job.job_id:
376
+ bases.append(job.job_id.rsplit("-", 1)[0])
377
+ wanted = {f"{b}{infix}{c}": c for b in bases for infix in ("-", "-bracket-") for c in CONTROLS}
378
+ for other in jobs:
379
+ c = wanted.get(other.job_id)
380
+ if not c or c in out or control_of(other) != c:
381
+ continue
382
+ if job.dataset and other.dataset and job.dataset != other.dataset:
383
+ continue
384
+ out[c] = other
385
+ return out
386
+
387
+
388
+ def _last_by_task(job) -> dict:
389
+ """The last recorded attempt per task, as the run pages show one."""
390
+ by_task = {}
391
+ for t in job.trials:
392
+ by_task[t.task_base] = t
393
+ return by_task
394
+
395
+
396
+ def _side(control, job, task_base) -> Side:
397
+ if job is None:
398
+ return Side(control)
399
+ t = _last_by_task(job).get(task_base)
400
+ if t is None:
401
+ return Side(control, job_id=job.job_id)
402
+ return Side(control, True, job.job_id, t.trial_name, _outcome(t), t.reward, t.exception_type,
403
+ t.p2p_fail, list(t.p2p_failures))
404
+
405
+
406
+ def verdict(nop: Side, oracle: Side) -> str:
407
+ """The class of a task from its two sides — see the module docstring for
408
+ the order and why a single side never validates."""
409
+ if nop.accepted:
410
+ return UNEXPECTED_REWARD
411
+ if nop.broke_tests:
412
+ return TASK_BROKEN
413
+ if nop.unscored or oracle.unscored:
414
+ return MISSING
415
+ if nop.failed or oracle.failed:
416
+ return SETUP_FAILURE
417
+ if not oracle.accepted:
418
+ return REFERENCE_REJECTED
419
+ return FEASIBLE
420
+
421
+
422
+ def report(job, jobs) -> Report:
423
+ """The feasibility report for `job`: its controls, the scope they ran
424
+ over, and every task either control recorded with both outcomes."""
425
+ controls = related(job, jobs)
426
+ scoped = job if not control_of(job) else next((controls[c] for c in CONTROLS if c in controls), job)
427
+ rep = Report(job_id=job.job_id, dataset=scoped.dataset, n_tasks=scoped.n_tasks)
428
+ for c in CONTROLS:
429
+ cj = controls.get(c)
430
+ if cj is not None:
431
+ rep.executed.append(ControlRun(c, cj.job_id, len(cj.trials), cj.n_planned, cj.state))
432
+ # Every task any side recorded, and — for an agent job — every task the
433
+ # agent ran, so a task the bracket never covered shows as missing rather
434
+ # than disappearing from the report.
435
+ names = {}
436
+ for source in [job, *controls.values()]:
437
+ for t in source.trials:
438
+ names.setdefault(t.task_base, t.task_name)
439
+ for task_base in sorted(names):
440
+ nop, oracle = _side(NOP, controls.get(NOP), task_base), _side(ORACLE, controls.get(ORACLE), task_base)
441
+ rep.rows.append(TaskRow(task_base, names[task_base], nop, oracle, verdict(nop, oracle)))
442
+ return rep
@@ -0,0 +1,149 @@
1
+ """Runs grouped by the launch that made them.
2
+
3
+ Runs started together ran the same tasks: the agents a recipe named, and the
4
+ nop and oracle controls that check those tasks. The Runs page shows each launch
5
+ as one block, so it needs to know which runs belong together. Two sources, in
6
+ this order:
7
+
8
+ - **The run record.** A run started from the builder's Run writes
9
+ ``runs/<name>/run.json``, and its plan names every job it launches
10
+ (``execution.Plan.jobs``). Those jobs are one launch, including the ones
11
+ Harbor has not written yet.
12
+ - **Inferred, for everything else** — an imported tree, a job run by hand, a run
13
+ from before the record existed: runs on the same dataset whose start is within
14
+ ``WINDOW_SEC`` of the launch's first run are one launch. A run with no start
15
+ time at all is a launch of its own.
16
+
17
+ Either way, the shards of one launch of the Nullius standard - jobs whose
18
+ config names the same ``launch`` id (``Job.launch_id``), each run on a machine
19
+ of its own, perhaps by a run record of its own, perhaps hours apart - are one
20
+ launch: a shard joins the launch that holds another shard of its id before
21
+ any other rule is asked.
22
+
23
+ Nothing here reads a file: it takes the parsed jobs and the records' job lists,
24
+ and groups. Newest launch first.
25
+ """
26
+ from __future__ import annotations
27
+
28
+ import os
29
+ from dataclasses import dataclass, field
30
+ from datetime import datetime, timezone
31
+
32
+ WINDOW_SEC = 30 * 60
33
+
34
+
35
+ @dataclass
36
+ class Launch:
37
+ key: str # "run:<execution id>" or "inferred:<first job id>"
38
+ source: str # "record" or "inferred"
39
+ dataset: str | None
40
+ started: datetime | None # the earliest start of its runs; none written yet, when it was launched
41
+ job_ids: list = field(default_factory=list) # written by Harbor, in start order
42
+ pending: list = field(default_factory=list) # named by the record, not written yet
43
+ name: str | None = None # the run record's name
44
+ launch_ids: set = field(default_factory=set) # the standard's launch ids among its runs (its shards)
45
+
46
+ @property
47
+ def title(self) -> str:
48
+ """What the launch is called on the Runs page: the standard's launch id
49
+ when its runs are the shards of one launch - what every shard
50
+ records, and what ``cli merge`` puts together - else ``prefix``."""
51
+ return next(iter(self.launch_ids)) if len(self.launch_ids) == 1 else self.prefix
52
+
53
+ @property
54
+ def prefix(self) -> str:
55
+ """What every run of the launch is called, up to the part that differs:
56
+ ``tau-latest3-0919-132944-*`` over ``…-agent`` and ``…-nop``. The
57
+ common prefix of the ids, cut back to a separator; the id itself for
58
+ a launch of one run; the first id "and N more" when they share nothing."""
59
+ ids = self.job_ids + self.pending
60
+ if len(ids) == 1:
61
+ return ids[0]
62
+ common = os.path.commonprefix(ids)
63
+ cut = max(common.rfind("-"), common.rfind("_"))
64
+ if cut <= 0: # nothing shared worth a wildcard: name the first and count the rest
65
+ return f"{ids[0]} and {len(ids) - 1} more"
66
+ return common[:cut + 1] + "*"
67
+
68
+
69
+ def merged_sources(jobs) -> set[str]:
70
+ """Source jobs represented by a finished merge in this workspace.
71
+
72
+ Use the recorded names, never a shared launch id: another agent or an
73
+ unmerged shard may belong to the same launch and must still count.
74
+ """
75
+ return {source for j in jobs if not j.is_running and not j.is_preparing
76
+ for source in getattr(j, "merged_from", []) if source != j.job_id}
77
+
78
+
79
+ def group(jobs, records: dict | None = None, window: float = WINDOW_SEC) -> list[Launch]:
80
+ """``jobs``: the parsed jobs (``job_id``, ``dataset``, ``started_at``).
81
+ ``records``: ``{execution id: (name, [job ids], launched at)}``, the builder's
82
+ run records, the time in seconds. Returns the launches, newest first, each job in exactly one."""
83
+ records = records or {}
84
+ by_id = {j.job_id: j for j in jobs}
85
+ placed: set[str] = set()
86
+ out: list[Launch] = []
87
+ for key, (name, ids, launched) in records.items():
88
+ mine = [i for i in ids if i in by_id and i not in placed]
89
+ pending = [i for i in ids if i not in by_id and i not in placed]
90
+ if not mine and not pending:
91
+ continue
92
+ placed.update(mine + pending)
93
+ written = sorted((by_id[i] for i in mine), key=_start_key)
94
+ ids = {_launch_id(j) for j in written} - {""}
95
+ home = next((l for l in out if l.launch_ids & ids), None)
96
+ if home is not None: # another shard of a launch a record placed already
97
+ _join(home, written, pending)
98
+ continue
99
+ out.append(Launch(f"run:{key}", "record", _dataset_of(written), _earliest(written) or _launched_at(launched),
100
+ [j.job_id for j in written], pending, name, ids))
101
+ rest = sorted((j for j in jobs if j.job_id not in placed), key=_start_key)
102
+ for j in rest:
103
+ dataset = getattr(j, "questions_from", "") or j.dataset
104
+ lid = _launch_id(j)
105
+ home = next((l for l in out if lid and lid in l.launch_ids), None)
106
+ if home is not None: # a shard of a launch placed already
107
+ _join(home, [j], [])
108
+ continue
109
+ home = next((l for l in out if l.source == "inferred" and l.dataset == dataset and l.started and j.started_at
110
+ and 0 <= (j.started_at - l.started).total_seconds() <= window), None)
111
+ if home is None:
112
+ out.append(Launch(f"inferred:{j.job_id}", "inferred", dataset, j.started_at, [j.job_id], launch_ids={lid} - {""}))
113
+ else:
114
+ home.job_ids.append(j.job_id)
115
+ home.launch_ids |= {lid} - {""}
116
+ return sorted(out, key=lambda l: (l.started is not None, l.started.timestamp() if l.started else 0.0), reverse=True)
117
+
118
+
119
+ def _launch_id(j) -> str:
120
+ return str(getattr(j, "launch_id", "") or "")
121
+
122
+
123
+ def _join(home: Launch, jobs, pending) -> None:
124
+ """Shards of a launch placed already join it; its start is the earliest of them all."""
125
+ home.job_ids += [j.job_id for j in jobs]
126
+ home.pending += [p for p in pending if p not in home.pending]
127
+ starts = [s for s in [home.started] + [j.started_at for j in jobs] if s]
128
+ home.started = min(starts) if starts else None
129
+ home.launch_ids |= {_launch_id(j) for j in jobs} - {""}
130
+
131
+
132
+ def _start_key(j):
133
+ s = j.started_at
134
+ return (s is None, s.timestamp() if s else 0.0, j.job_id)
135
+
136
+
137
+ def _launched_at(seconds):
138
+ return datetime.fromtimestamp(seconds, timezone.utc) if seconds else None # aware, as a job's stamps are (parse._iso)
139
+
140
+
141
+ def _earliest(jobs):
142
+ starts = [j.started_at for j in jobs if j.started_at]
143
+ return min(starts) if starts else None
144
+
145
+
146
+ def _dataset_of(jobs):
147
+ # a runner's arms answer the same questions, the oracle's from a file of its own
148
+ names = {getattr(j, "questions_from", "") or j.dataset for j in jobs if j.dataset}
149
+ return names.pop() if len(names) == 1 else (None if not names else "mixed")