nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/runs/parse.py ADDED
@@ -0,0 +1,1016 @@
1
+ """Walk job directories and build the normalized data model.
2
+
3
+ A *job* is one Harbor run (a timestamped dir holding a ``result.json`` and one
4
+ subdir per *trial*). A *trial* is one task attempt. We read only what we need
5
+ and redact on the way in.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import os
11
+ import re
12
+ import shlex
13
+ from dataclasses import dataclass, field, replace
14
+ from datetime import datetime, timezone
15
+
16
+ from nullius.runs import verifier
17
+ from nullius.runs import classify
18
+ from nullius.redact import redact
19
+
20
+ # What a trace document in a trial's `agent/` may be called: the producer's own
21
+ # capture, or a filtered view saved from the trial page. A bare filename, so
22
+ # there is no path to traverse in the first place, and no leading dot to hide the
23
+ # file it writes.
24
+ #
25
+ # One definition, two readers: the trial page validates a name before writing it,
26
+ # and one arriving in a query string (?trace=) before reading it.
27
+ TRACE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}\.json$")
28
+ DEFAULT_TRACE_NAME = "trace.json"
29
+
30
+ # The shape this module produces. A workspace caches parsed Jobs keyed on it,
31
+ # so bump it when a Trial or Job field is added, renamed or means something
32
+ # new — otherwise a cache written by the old parse is read back as current.
33
+ SCHEMA = "15"
34
+
35
+ # What a rate-limit record in `agent/gateway.log` looks like. The agent's SDK
36
+ # retries a 429 below the harness (an agent's `max_retries` reaching its
37
+ # OpenAI client), so a limit the run survived leaves nothing in result.json —
38
+ # the outcome says `resolved` and the only evidence is the log. Three shapes
39
+ # are counted: the exception class or the API's error code (`RateLimitError`,
40
+ # `rate_limit_exceeded`, "Rate limit reached"), the HTTP reason phrase ("Too
41
+ # Many Requests"), and a bare 429 *as a status* — `status_code=429`, `HTTP 429`,
42
+ # `Error code: 429`. A bare 429 alone is not enough: event ids, hashes and
43
+ # token counts in the same log can carry those digits by chance.
44
+ # What is not counted: the SDK's "Retrying request …" line, which names no
45
+ # status, so a 5xx retry is not mistaken for a rate limit.
46
+ RATE_LIMIT_RE = re.compile(
47
+ rb"rate.?limit|too many requests"
48
+ rb"|(?:status(?:[_ -]?code)?|error code|code|http(?:/\d(?:\.\d)?)?)\W{0,4}429(?!\w|\.\d)",
49
+ re.IGNORECASE)
50
+ # How much of a gateway log the scan reads: the count is over the first
51
+ # GATEWAY_SCAN_BYTES only. Scanning is a bounded parse-time cost; for a log
52
+ # that outgrows the bound, the count is a floor.
53
+ GATEWAY_SCAN_BYTES = 8 * 1024 * 1024
54
+
55
+
56
+ def count_rate_limits(path: str) -> int | None:
57
+ """Rate-limit records in a gateway log, or None when there is no log.
58
+
59
+ None and 0 are different answers: an agent that wrote no gateway log
60
+ (a control, an imported run, the direct transport) cannot be said to have
61
+ hit no limits, only that nothing recorded it. Lines are counted, bounded by
62
+ GATEWAY_SCAN_BYTES, and the file is never held whole.
63
+ """
64
+ try:
65
+ handle = open(path, "rb")
66
+ except OSError:
67
+ return None
68
+ count, budget = 0, GATEWAY_SCAN_BYTES
69
+ with handle:
70
+ for line in handle:
71
+ if RATE_LIMIT_RE.search(line):
72
+ count += 1
73
+ budget -= len(line)
74
+ if budget <= 0:
75
+ break
76
+ return count
77
+
78
+
79
+ def _iso(ts: str | None):
80
+ """A stamp as an aware datetime. Harbor writes its stamps without a zone,
81
+ in UTC; the standard, OpenTelemetry and zoned ATIF imports write one. A
82
+ naive and an aware datetime cannot be compared or subtracted, so a page
83
+ that ordered or spanned both returned 500: every stamp is given UTC."""
84
+ if not ts or not isinstance(ts, str):
85
+ return None
86
+ try:
87
+ when = datetime.fromisoformat(ts.replace("Z", "+00:00"))
88
+ except ValueError:
89
+ return None
90
+ return when.replace(tzinfo=timezone.utc) if when.tzinfo is None else when
91
+
92
+
93
+ def _dur(a: dict | None) -> float | None:
94
+ """Seconds between started_at/finished_at of a phase block."""
95
+ if not a:
96
+ return None
97
+ s, f = _iso(a.get("started_at")), _iso(a.get("finished_at"))
98
+ if s and f:
99
+ return (f - s).total_seconds()
100
+ return None
101
+
102
+
103
+ @dataclass
104
+ class Trial:
105
+ job_id: str
106
+ dir: str
107
+ task_name: str
108
+ trial_name: str
109
+ task_base: str # e.g. django__django-10554
110
+ source: str
111
+ model_name: str
112
+ outcome: str
113
+ reward: float | None
114
+ exception_type: str
115
+ exception_message: str
116
+ # verifier detail. The patch pair is `None` for a benchmark that has no
117
+ # patches to report on — templates show absence, not "No".
118
+ patch_exists: bool | None
119
+ patch_applied: bool | None
120
+ f2p_pass: int
121
+ f2p_fail: int
122
+ p2p_pass: int
123
+ p2p_fail: int
124
+ n_input_tokens: int
125
+ n_output_tokens: int
126
+ n_cache_tokens: int
127
+ cost_usd: float | None
128
+ # timings (seconds)
129
+ dur_total: float | None
130
+ dur_env: float | None
131
+ dur_agent_setup: float | None
132
+ dur_agent: float | None
133
+ dur_verifier: float | None
134
+ has_trajectory: bool = False
135
+ # The resolved agent version recorded in agent_info —
136
+ # what makes two runs of "the same" agent actually comparable.
137
+ agent_version: str = ""
138
+ # The agent the trial's own result.json names (agent_info.name). A job
139
+ # config written with defaults left out names no agent when the agent is
140
+ # Harbor's default (oracle); the trials still say who ran.
141
+ agent_info_name: str = ""
142
+ reasoning_effort: str = ""
143
+ # The environment the trial's own result.json records (`config.environment
144
+ # .type`, e.g. "docker") — the job-level config does not carry it.
145
+ env_type: str = ""
146
+ has_trace: bool = False
147
+ # Rate-limit records the agent's own log carries — 429s its SDK retried and
148
+ # the run survived, which no result.json field records (`count_rate_limits`).
149
+ # None when the trial has no `agent/gateway.log`: not measured, not zero.
150
+ n_rate_limited_retries: int | None = None
151
+ # A graded verdict (an F1, partial credit), beside the pass or fail the
152
+ # reward says: the runner records it when its benchmark scores that way.
153
+ # A benchmark that gives a score and no pass or fail leaves the reward
154
+ # out: the trial is then scored, not resolved or failed.
155
+ score: float | None = None
156
+ # How many parts that score is the mean of (the questions of a
157
+ # conversation), when the benchmark scores an item part by part; each
158
+ # part is in verifier/parts.json. None: the score is the item's own.
159
+ n_parts: int | None = None
160
+ # Those parts' scores by the field the benchmark breaks its score down by
161
+ # (a question's category): {value: [sum of scores, parts]}, a part with no
162
+ # score counting nothing, as in the trial's own mean; and that field's word.
163
+ part_groups: dict = field(default_factory=dict)
164
+ parts_by: str = ""
165
+ # Names of the tests that failed, kept so a regression can be shown, not
166
+ # just counted.
167
+ f2p_failures: list = field(default_factory=list)
168
+ p2p_failures: list = field(default_factory=list)
169
+ # Absolute clocks, for anything that places this trial on a timeline
170
+ # rather than measuring it — an exporter drawing spans needs where a phase
171
+ # *sat*, not just how long it took. `phases` maps a Harbor phase name
172
+ # (environment_setup, agent_setup, agent_execution, verifier) to its
173
+ # (started, finished) pair; a phase the run never recorded is absent, so a
174
+ # reader sees nothing rather than a zero-length span.
175
+ started_at: datetime | None = None
176
+ finished_at: datetime | None = None
177
+ phases: dict = field(default_factory=dict)
178
+
179
+ @property
180
+ def dur_phases(self) -> float:
181
+ """Sum of the four phases Harbor breaks out."""
182
+ return sum(
183
+ v or 0
184
+ for v in (self.dur_env, self.dur_agent_setup, self.dur_agent, self.dur_verifier)
185
+ )
186
+
187
+ @property
188
+ def dur_other(self) -> float:
189
+ """Trial time Harbor doesn't attribute to a phase.
190
+
191
+ Image pull, container start and teardown, artifact collection. Clamped
192
+ at 0: if the phases ever exceed the total — clock skew, or phases that
193
+ overlap — a negative value would corrupt the bar's geometry silently
194
+ rather than showing an error.
195
+ """
196
+ return max(0.0, (self.dur_total or 0) - self.dur_phases)
197
+
198
+ @property
199
+ def is_regression(self) -> bool:
200
+ """The patch broke tests that were passing before it was applied."""
201
+ return self.p2p_fail > 0
202
+
203
+ @property
204
+ def outcome_label(self) -> str:
205
+ return classify.LABELS.get(self.outcome, self.outcome)
206
+
207
+ @property
208
+ def vocab(self):
209
+ """The dataset's own names for its checks — see `verifier.Vocabulary`."""
210
+ return verifier.vocabulary_for(self.source)
211
+
212
+ @property
213
+ def is_infra(self) -> bool:
214
+ return self.outcome in classify.INFRA_OUTCOMES
215
+
216
+ @property
217
+ def trajectory_path(self) -> str:
218
+ return os.path.join(self.dir, "agent", "trajectory.json")
219
+
220
+ @property
221
+ def trace_path(self) -> str:
222
+ """The timing sidecar, written beside the trajectory when captured."""
223
+ return os.path.join(self.dir, "agent", DEFAULT_TRACE_NAME)
224
+
225
+ # Harbor's control agents. Neither calls a model: `oracle` applies the task's
226
+ # gold patch, `nop` does nothing. Their zero tokens and zero cost are true
227
+ # measurements, not missing ones — which is the whole reason to tell them apart.
228
+ CONTROL_AGENTS = {"oracle", "nop"}
229
+
230
+
231
+ @dataclass
232
+ class Job:
233
+ job_id: str
234
+ dir: str
235
+ model_name: str
236
+ dataset: str
237
+ n_tasks: int
238
+ started_at: datetime | None
239
+ finished_at: datetime | None
240
+ trials: list = field(default_factory=list) # list[Trial]
241
+ agent_name: str = "" # e.g. "claude-code", "opencode"
242
+ reasoning_effort: str = ""
243
+ # Harbor's own bookkeeping from the job-level result.json stats block.
244
+ # n_running/n_pending distinguish a finished run from one still going —
245
+ # without them an in-progress sweep reads as a run full of misses.
246
+ n_total_trials: int = 0
247
+ n_errored_trials: int = 0
248
+ n_running_trials: int = 0
249
+ n_pending_trials: int = 0
250
+ n_retries: int = 0
251
+ preparing: bool = False # launched, no result.json yet — see is_preparing_job_dir
252
+ # The dataset path exactly as the config recorded it (may be relative to
253
+ # wherever harbor ran, and empty for registry fetches). Kept because it's
254
+ # the only lead for finding the checkout — and its gold patches — locally.
255
+ dataset_path: str = ""
256
+ # The `harbor run` invocation reconstructed from this job's recorded
257
+ # config — what a copy-and-rerun needs. Empty when config.json is absent.
258
+ harbor_command: str = ""
259
+ #: the Harbor fork version the job recorded in its lock.json (`harbor.version`),
260
+ #: and whether that Harbor was an editable checkout — what actually ran,
261
+ #: as distinct from what a recipe intended; "" when the tree carries none
262
+ harbor_version: str = ""
263
+ harbor_editable: bool | None = None
264
+ #: a job whose config names `runner`: its runner and resolved upstream
265
+ #: commit, in place of Harbor metadata
266
+ runner: str = ""
267
+ runner_commit: str = ""
268
+ runner_judge: str = "" # the benchmark's own judge, as the run named it: "official gpt-4o · evaluate_qa.py"
269
+ questions_from: str = "" # a runner's recipe dataset: the items every arm answers, a control's with a file of its own included
270
+ runner_items: list = field(default_factory=list) # what the runner calls one item and many, from its definition: ["question", "questions"]
271
+ # A Nullius standard job that ran one of its benchmark's declared controls
272
+ # (config ``standard.control``): who wrote it - official, adapter, nullius.
273
+ # "" for every other job.
274
+ control_source: str = ""
275
+ # A Nullius standard job (config ``standard``): its trials are written as each unit ends, so the
276
+ # live board links one as soon as it has - unlike a runner's, imported at its end.
277
+ standard: bool = False
278
+ # The launch a Nullius standard job was one shard of (config
279
+ # ``standard.launch``): the shards of one launch, run on several machines,
280
+ # are grouped as one on the runs page (launches.group). "" for every other job.
281
+ launch_id: str = ""
282
+ # Source job names explicitly recorded by standard merge.
283
+ merged_from: list[str] = field(default_factory=list)
284
+ # How the job was launched, as its config.json recorded it with Harbor's
285
+ # defaults filled in (launch_of): what "New run like this" rebuilds a
286
+ # recipe from when the run kept none. Empty when config.json is absent.
287
+ launch: dict = field(default_factory=dict)
288
+ # A run with a spending cap (a Nullius standard job's ``result.json``, ``budget``): the cap and what was
289
+ # spent in USD, whether the cap stopped the run, the units it cut off and those never begun; a cap per unit
290
+ # (``unit_max_usd``) and the units that reached it (``over_unit``). {} without one.
291
+ budget: dict = field(default_factory=dict)
292
+ # A Harbor job run with several agents or models (`-m a -m b`, or a config
293
+ # listing several): each one's {name, model, effort}, which split_agents
294
+ # turns into one view per agent. [] for a job of one agent.
295
+ agents: list = field(default_factory=list)
296
+
297
+ @property
298
+ def agent_label(self) -> str:
299
+ """Readable half of the agent id — the class, not the module path."""
300
+ return self.agent_name.split(":")[-1] if self.agent_name else ""
301
+
302
+ @property
303
+ def kind(self) -> str:
304
+ """'control' for Harbor's oracle/nop baselines, else 'agent'. A
305
+ runner's arm is never Harbor's control, whatever it is called:
306
+ a runner's `oracle` arm is its own ceiling, not a reference solution.
307
+ A standard job is one when its config says it ran a declared control."""
308
+ named = self.agent_label.lower() in CONTROL_AGENTS
309
+ return "control" if named and (not self.runner or self.control_source) else "agent"
310
+
311
+ @property
312
+ def is_control(self) -> bool:
313
+ return self.kind == "control"
314
+
315
+ @property
316
+ def n_planned(self) -> int:
317
+ """How many trials the run set out to do — Harbor's own count when it
318
+ recorded one, else the task count from the config. Comparing this with
319
+ the trials actually on disk exposes a partial or interrupted run."""
320
+ return self.n_total_trials or self.n_tasks
321
+
322
+ @property
323
+ def is_running(self) -> bool:
324
+ """Trials still executing or queued — the run isn't over yet."""
325
+ return (self.n_running_trials + self.n_pending_trials) > 0
326
+
327
+ @property
328
+ def harbor_label(self) -> str:
329
+ """The recorded Harbor: version, and `editable` when the lock says the
330
+ checkout was — a development Harbor, not a release; empty when unrecorded."""
331
+ if not self.harbor_version:
332
+ return ""
333
+ return self.harbor_version + (" · editable checkout" if self.harbor_editable else "")
334
+
335
+ @property
336
+ def is_preparing(self) -> bool:
337
+ """Launched, not started: Harbor has written the job's config and
338
+ nothing else yet — no result record, no trial. The runs list shows the
339
+ row now rather than when the first event lands."""
340
+ return self.preparing
341
+
342
+ @property
343
+ def state(self) -> str:
344
+ """``preparing`` → ``running`` → ``finished``, for a badge."""
345
+ return "preparing" if self.preparing else "running" if self.is_running else "finished"
346
+
347
+ @property
348
+ def agent_version(self) -> str:
349
+ """The resolved agent version, taken from the first trial that has one."""
350
+ return next((t.agent_version for t in self.trials if t.agent_version), "")
351
+
352
+ @property
353
+ def vocab(self):
354
+ """The check vocabulary of the dataset this job ran, from its trials.
355
+
356
+ Every job on disk runs one dataset, so the first trial that names its
357
+ checks speaks for the run. A job of only unknown datasets gets the
358
+ empty vocabulary, and the table drops the check columns rather than
359
+ labelling them with another benchmark's words.
360
+ """
361
+ for t in self.trials:
362
+ v = t.vocab
363
+ if v.gained or v.held or v.has_patch:
364
+ return v
365
+ return verifier.GENERIC_VOCABULARY
366
+
367
+ @property
368
+ def duration_min(self) -> float | None:
369
+ if self.started_at and self.finished_at:
370
+ return (self.finished_at - self.started_at).total_seconds() / 60.0
371
+ return None
372
+
373
+ @property
374
+ def counts(self) -> dict:
375
+ c = {k: 0 for k in classify.ORDER}
376
+ for t in self.trials:
377
+ c[t.outcome] = c.get(t.outcome, 0) + 1
378
+ return c
379
+
380
+ @property
381
+ def total_input_tokens(self) -> int:
382
+ return sum(t.n_input_tokens or 0 for t in self.trials)
383
+
384
+ @property
385
+ def total_output_tokens(self) -> int:
386
+ return sum(t.n_output_tokens or 0 for t in self.trials)
387
+
388
+ @property
389
+ def total_cache_tokens(self) -> int:
390
+ return sum(t.n_cache_tokens or 0 for t in self.trials)
391
+
392
+ @property
393
+ def total_cost_usd(self):
394
+ """None when no trial reported a cost — not every agent does."""
395
+ costs = [t.cost_usd for t in self.trials if t.cost_usd is not None]
396
+ return sum(costs) if costs else None
397
+
398
+ @property
399
+ def mean_score(self):
400
+ """The mean of the trials' graded scores, None when none was graded.
401
+ A trial scored part by part weighs as many as it has: the mean is
402
+ then over every part - every question, not every conversation - which
403
+ is the number a paper that scores questions reports."""
404
+ scored = [(t.score, t.n_parts or 1) for t in self.trials if t.score is not None]
405
+ return sum(s * n for s, n in scored) / sum(n for _, n in scored) if scored else None
406
+
407
+ @property
408
+ def score_by_group(self) -> list:
409
+ """The mean score by the field the benchmark breaks it down by - a
410
+ question's category - over every part of every trial: [{group, mean,
411
+ n}], in the order the values sort (numbers as numbers); [] when no
412
+ trial was scored that way."""
413
+ total = {}
414
+ for t in self.trials:
415
+ for g, (s, n) in (t.part_groups or {}).items():
416
+ acc = total.setdefault(g, [0.0, 0])
417
+ acc[0] += s
418
+ acc[1] += n
419
+ key = lambda g: (0, float(g), "") if g.replace(".", "", 1).lstrip("-").isdigit() else (1, 0.0, g)
420
+ return [{"group": g, "mean": s / n, "n": n} for g, (s, n) in sorted(total.items(), key=lambda kv: key(kv[0])) if n]
421
+
422
+ @property
423
+ def parts_by(self) -> str:
424
+ """The word for what the score is broken down by, '' when it is not."""
425
+ return next((t.parts_by for t in self.trials if t.parts_by), "")
426
+
427
+ @property
428
+ def n_scored_parts(self):
429
+ """How many parts the mean score is over, None when no trial has parts."""
430
+ counts = [t.n_parts for t in self.trials if t.score is not None and t.n_parts]
431
+ return sum(counts) if counts else None
432
+
433
+ @property
434
+ def scored_only(self) -> bool:
435
+ """Scores and no pass or fail anywhere: the mean score is the
436
+ headline, and the solve rates stay absent rather than 0% - also when
437
+ a trial failed on an exception, which is a verdict of its own."""
438
+ return any(t.score is not None for t in self.trials) and not any(t.reward is not None for t in self.trials)
439
+
440
+ @property
441
+ def n_regressions(self) -> int:
442
+ """Trials whose patch broke previously-passing tests."""
443
+ return sum(1 for t in self.trials if t.is_regression)
444
+
445
+ @property
446
+ def reports_regressions(self) -> bool:
447
+ """Can this job's benchmark say whether anything broke?
448
+
449
+ `is_regression` reads `p2p_fail`, which a benchmark with no reader for
450
+ held checks leaves at 0 — indistinguishable from "nothing broke"
451
+ unless the question of whether it was asked is kept separate. A job
452
+ where some trial reported held checks at all can be counted; one where
453
+ none did shows absence, per the rule that a field a benchmark cannot
454
+ answer is never rendered as a plausible zero.
455
+ """
456
+ return any(t.p2p_pass or t.p2p_fail for t in self.trials)
457
+
458
+ @property
459
+ def n_resolved(self) -> int:
460
+ return sum(1 for t in self.trials if t.outcome == classify.RESOLVED)
461
+
462
+ @property
463
+ def has_verdict(self) -> bool:
464
+ """Did anything score this job?
465
+
466
+ A trial's outcome is `unknown` only when it carries no verdict at all
467
+ — no verifier report, no reward and no exception (`classify
468
+ .from_verdict`). A standalone ATIF trajectory or an OTel export is
469
+ nothing but such trials, and "nothing was scored"
470
+ is a different claim from "everything scored missed": the rates
471
+ below are `None` here, and the pages show absence rather than 0%.
472
+ One verified trial is enough — from then on the unknowns stay in the
473
+ denominator, the rule `.claude/rules/running.md` states.
474
+ """
475
+ return any(t.outcome != classify.UNKNOWN for t in self.trials)
476
+
477
+ @property
478
+ def raw_solve_rate(self) -> float | None:
479
+ """Resolved ÷ all trials; `None` when no trial carries a verdict."""
480
+ if not self.trials or not self.has_verdict or self.scored_only:
481
+ return None
482
+ return self.n_resolved / len(self.trials)
483
+
484
+ @property
485
+ def adjusted_solve_rate(self) -> float | None:
486
+ """Solve rate over trials that actually ran (excludes infra failures);
487
+ `None` when no trial carries a verdict."""
488
+ if not self.has_verdict or self.scored_only:
489
+ return None
490
+ eligible = [t for t in self.trials if not t.is_infra]
491
+ return (self.n_resolved / len(eligible)) if eligible else None
492
+
493
+ @property
494
+ def n_infra(self) -> int:
495
+ return sum(1 for t in self.trials if t.is_infra)
496
+
497
+
498
+ def is_preparing_job_dir(path: str) -> bool:
499
+ """A job directory Harbor has created but not started: its job-level
500
+ ``config.json`` (the one naming ``agents`` or ``datasets`` — a trial's names
501
+ a ``task``) with no ``result.json`` yet. Everything between the launch and
502
+ ``job.start`` looks like this on disk, and it is the state a person who
503
+ just pressed Run is waiting to see."""
504
+ if os.path.isfile(os.path.join(path, "result.json")):
505
+ return False
506
+ cfg = verifier._read_json(os.path.join(path, "config.json"))
507
+ return isinstance(cfg, dict) and ("agents" in cfg or "datasets" in cfg) and "task" not in cfg
508
+
509
+
510
+ def looks_like_job_dir(path: str) -> bool:
511
+ """A ``result.json`` and either a trial that has one of its own or a live
512
+ feed — or a job that has been launched and not started yet. Harbor writes a
513
+ trial's ``result.json`` when the trial ends, so a job whose first trial is
514
+ still running has none, and that is exactly the job the live board exists
515
+ for: its ``events.jsonl`` is Harbor's own sign that the job is underway, so
516
+ it counts. Before even that, the job-level ``config.json`` alone means
517
+ *preparing* (see :func:`is_preparing_job_dir`). A bare ``result.json``
518
+ still does not count.
519
+ """
520
+ if not os.path.isfile(os.path.join(path, "result.json")):
521
+ return is_preparing_job_dir(path)
522
+ if os.path.isfile(os.path.join(path, "events.jsonl")):
523
+ return True
524
+ return any(
525
+ os.path.isdir(os.path.join(path, d))
526
+ and os.path.isfile(os.path.join(path, d, "result.json"))
527
+ for d in os.listdir(path)
528
+ )
529
+
530
+
531
+ def find_job_dirs(roots) -> list[str]:
532
+ """Return every job directory found under the given root paths.
533
+
534
+ A root may itself be a job dir, or a parent containing job dirs (one level
535
+ down). Both are handled, and results are de-duplicated while preserving
536
+ order, so overlapping roots are safe.
537
+ """
538
+ found: list[str] = []
539
+ seen: set[str] = set()
540
+
541
+ def add(path: str) -> None:
542
+ real = os.path.realpath(path)
543
+ if real not in seen:
544
+ seen.add(real)
545
+ found.append(path)
546
+
547
+ for root in roots:
548
+ if not os.path.isdir(root):
549
+ continue
550
+ if looks_like_job_dir(root):
551
+ add(root)
552
+ for name in sorted(os.listdir(root)):
553
+ p = os.path.join(root, name)
554
+ if os.path.isdir(p) and looks_like_job_dir(p):
555
+ add(p)
556
+ return found
557
+
558
+
559
+ def _token_totals(res: dict) -> dict:
560
+ """Token/cost totals for a trial, whichever shape Harbor wrote.
561
+
562
+ Single-step trials carry one AgentContext on ``agent_result``; multi-step
563
+ trials leave that null and record one per entry of ``step_results``.
564
+ Mirrors Harbor's ``TrialResult.compute_token_cost_totals``: cost stays
565
+ None unless some step actually reported one — "unpriced" and "free" are
566
+ different claims.
567
+ """
568
+ if res.get("agent_result"):
569
+ contexts = [res["agent_result"]]
570
+ else:
571
+ contexts = [
572
+ sr.get("agent_result")
573
+ for sr in res.get("step_results") or []
574
+ if isinstance(sr, dict) and sr.get("agent_result")
575
+ ]
576
+ totals = {"n_input_tokens": 0, "n_output_tokens": 0, "n_cache_tokens": 0, "cost_usd": None}
577
+ for ctx in contexts:
578
+ for key in ("n_input_tokens", "n_output_tokens", "n_cache_tokens"):
579
+ totals[key] += ctx.get(key) or 0
580
+ if ctx.get("cost_usd") is not None:
581
+ totals["cost_usd"] = (totals["cost_usd"] or 0.0) + ctx["cost_usd"]
582
+ return totals
583
+
584
+
585
+ def parse_trial(job_id: str, trial_dir: str) -> Trial | None:
586
+ res = verifier._read_json(os.path.join(trial_dir, "result.json"))
587
+ if not res:
588
+ return None
589
+
590
+ task_name = res.get("task_name", os.path.basename(trial_dir))
591
+ task_base = task_name.split("/")[-1]
592
+ exc = res.get("exception_info") or {}
593
+ exc_type = exc.get("exception_type", "") or ""
594
+
595
+ agent_res = _token_totals(res)
596
+ ver_res = res.get("verifier_result") or {}
597
+ reward = None
598
+ if isinstance(ver_res, dict):
599
+ reward = (ver_res.get("rewards") or {}).get("reward")
600
+
601
+ # Older jobs record `source`; newer ones only carry it as the prefix of
602
+ # a "<dataset>/<task>" task name. It selects the verifier loader and
603
+ # reader, so it has to be resolved before the report is touched.
604
+ source = res.get("source") or (task_name.rsplit("/", 1)[0] if "/" in task_name else "")
605
+
606
+ report = verifier.load(source, trial_dir, task_base)
607
+ verdict = verifier.read(source, report, reward)
608
+ outcome = classify.from_verdict(exc_type, verdict)
609
+ rewards = (ver_res.get("rewards") or {}) if isinstance(ver_res, dict) else {}
610
+ score = rewards.get("score") if isinstance(rewards.get("score"), (int, float)) and not isinstance(rewards.get("score"), bool) else None
611
+ # the parts' scores by the field the benchmark breaks its score down by (verifier/parts.json)
612
+ part_groups, parts_by = {}, ""
613
+ pj = verifier._read_json(os.path.join(trial_dir, "verifier", "parts.json"))
614
+ if isinstance(pj, dict) and pj.get("by") and isinstance(pj.get("parts"), list):
615
+ parts_by = str(pj["by"])
616
+ for x in pj["parts"]:
617
+ if isinstance(x, dict) and "group" in x:
618
+ acc = part_groups.setdefault(str(x["group"]), [0.0, 0])
619
+ acc[0] += x["score"] if isinstance(x.get("score"), (int, float)) and not isinstance(x.get("score"), bool) else 0.0
620
+ acc[1] += 1
621
+ # a score and no pass or fail, no exception: the benchmark measured it and gave no verdict to classify
622
+ if outcome == classify.UNKNOWN and score is not None and reward is None and not exc_type:
623
+ outcome = classify.SCORED
624
+
625
+ traj_path = os.path.join(trial_dir, "agent", "trajectory.json")
626
+ trace_path = os.path.join(trial_dir, "agent", "trace.json")
627
+
628
+ phases = {}
629
+ for name in ("environment_setup", "agent_setup", "agent_execution", "verifier"):
630
+ block = res.get(name) or {}
631
+ started, finished = _iso(block.get("started_at")), _iso(block.get("finished_at"))
632
+ if started and finished:
633
+ phases[name] = (started, finished)
634
+
635
+ return Trial(
636
+ job_id=job_id,
637
+ dir=trial_dir,
638
+ task_name=task_name,
639
+ trial_name=res.get("trial_name", os.path.basename(trial_dir)),
640
+ task_base=task_base,
641
+ source=source,
642
+ model_name=(res.get("agent_info", {}).get("model_info", {}) or {}).get("name", ""),
643
+ agent_version=(res.get("agent_info") or {}).get("version") or "",
644
+ reasoning_effort=str((((res.get("config") or {}).get("agent") or {}).get("kwargs") or {}).get("reasoning_effort", "")),
645
+ agent_info_name=str((res.get("agent_info") or {}).get("name") or ""),
646
+ outcome=outcome,
647
+ reward=reward,
648
+ score=score,
649
+ part_groups=part_groups,
650
+ parts_by=parts_by,
651
+ n_parts=(ver_res.get("parts") or {}).get("n") if isinstance(ver_res, dict) and isinstance(ver_res.get("parts"), dict)
652
+ and isinstance((ver_res.get("parts") or {}).get("n"), int) and not isinstance((ver_res.get("parts") or {}).get("n"), bool) else None,
653
+ exception_type=exc_type,
654
+ exception_message=redact(exc.get("exception_message", "")),
655
+ patch_exists=verdict.produced,
656
+ patch_applied=verdict.usable,
657
+ f2p_pass=verdict.gained_pass,
658
+ f2p_fail=verdict.gained_fail,
659
+ p2p_pass=verdict.held_pass,
660
+ p2p_fail=verdict.held_fail,
661
+ f2p_failures=list(verdict.gained_failures),
662
+ p2p_failures=list(verdict.held_failures),
663
+ n_input_tokens=agent_res.get("n_input_tokens") or 0,
664
+ n_output_tokens=agent_res.get("n_output_tokens") or 0,
665
+ n_cache_tokens=agent_res.get("n_cache_tokens") or 0,
666
+ cost_usd=agent_res.get("cost_usd"),
667
+ dur_total=_dur(
668
+ {
669
+ "started_at": res.get("started_at"),
670
+ "finished_at": res.get("finished_at"),
671
+ }
672
+ ),
673
+ dur_env=_dur(res.get("environment_setup")),
674
+ dur_agent_setup=_dur(res.get("agent_setup")),
675
+ dur_agent=_dur(res.get("agent_execution")),
676
+ dur_verifier=_dur(res.get("verifier")),
677
+ has_trajectory=os.path.isfile(traj_path),
678
+ has_trace=os.path.isfile(trace_path),
679
+ n_rate_limited_retries=count_rate_limits(os.path.join(trial_dir, "agent", "gateway.log")),
680
+ env_type=((res.get("config") or {}).get("environment") or {}).get("type", ""),
681
+ started_at=_iso(res.get("started_at")),
682
+ finished_at=_iso(res.get("finished_at")),
683
+ phases=phases,
684
+ )
685
+
686
+
687
+ def harbor_command(cfg: dict, env_type: str = "") -> str:
688
+ """Reconstruct the `harbor run` invocation a job's config.json records.
689
+
690
+ Faithful to the file: only flags the harness actually recorded appear, so
691
+ a copied command reruns what ran rather than what a form would suggest —
692
+ including the `--ak` kwargs, which no other surface reproduces. The one
693
+ outside fact is `--env`, which the job-level config does not carry; the
694
+ caller passes it from a trial's recorded environment when it has one.
695
+
696
+ A job that another harness produced records its own invocation under
697
+ ``command`` and it is returned verbatim: reconstructing a ``harbor run``
698
+ for a run Harbor never performed would print a line that cannot reproduce
699
+ it, which is the one thing this function exists not to do. No Harbor
700
+ config carries that key, so this cannot shadow a real one.
701
+ """
702
+ recorded = cfg.get("command")
703
+ if isinstance(recorded, str) and recorded.strip():
704
+ return recorded.strip()
705
+
706
+ agents = cfg.get("agents") or [{}]
707
+ agent = agents[0]
708
+ name = agent.get("name", "")
709
+ if not name:
710
+ return ""
711
+
712
+ # No PYTHONPATH is reconstructed, whatever the recorded name looks like: the
713
+ # lane in service is Harbor's built-in agent, and a module:Class name in an
714
+ # old job is a fact about that job, not an instruction to rebuild the
715
+ # adapter checkout it once resolved in. The --agent value is what the job
716
+ # recorded, verbatim.
717
+ parts = ["harbor", "run"]
718
+
719
+ n_tasks = 0
720
+ datasets = cfg.get("datasets") or []
721
+ if datasets:
722
+ d0 = datasets[0]
723
+ if d0.get("path"):
724
+ parts += ["-p", shlex.quote(d0["path"])]
725
+ elif d0.get("name"):
726
+ ref = d0["name"] + (f"@{d0['version']}" if d0.get("version") else "")
727
+ parts += ["-d", shlex.quote(ref)]
728
+ n_tasks = d0.get("n_tasks") or 0
729
+
730
+ if env_type:
731
+ parts += ["--env", shlex.quote(env_type)]
732
+ if agent.get("model_name"):
733
+ parts += ["--model", shlex.quote(agent["model_name"])]
734
+ parts += ["--agent", shlex.quote(name)]
735
+ for key, value in (agent.get("kwargs") or {}).items():
736
+ parts += ["--ak", shlex.quote(f"{key}={value}")]
737
+ if n_tasks:
738
+ parts += ["--n-tasks", str(n_tasks)]
739
+ if cfg.get("n_concurrent_trials"):
740
+ parts += ["--n-concurrent", str(cfg["n_concurrent_trials"])]
741
+ if (cfg.get("retry") or {}).get("max_retries"):
742
+ parts += ["--max-retries", str(cfg["retry"]["max_retries"])]
743
+ return " ".join(parts)
744
+
745
+
746
+ # Harbor writes config.json with `exclude_defaults=True` (harbor/job.py,
747
+ # 0.22): a key it leaves out held Harbor's default, which is not the
748
+ # builder's (concurrency 4 and no retries here, 1 and 3 there). Read from
749
+ # harbor/models/job/config.py at 0.22; a launch says which ones it filled.
750
+ HARBOR_DEFAULTS = {"n_concurrent_trials": 4, "max_retries": 0, "n_attempts": 1, "timeout_multiplier": 1.0}
751
+ _PHASE_MULTIPLIERS = ("agent_timeout_multiplier", "agent_setup_timeout_multiplier",
752
+ "verifier_timeout_multiplier", "environment_build_timeout_multiplier")
753
+ # A launch carries only the name of a value filed under a name like these: it
754
+ # is a credential, and the record is embedded in the builder's page.
755
+ _SECRET_KEY = re.compile(r"(?:api[-_]?key|token|password|authorization|credential|secret)", re.I)
756
+ # What a recipe can say; any other key a config recorded is named, not dropped.
757
+ _LAUNCH_TOP = {"job_name", "jobs_dir", "quiet", "debug", "agents", "datasets", "tasks", "retry",
758
+ "n_concurrent_trials", "n_attempts", "timeout_multiplier", "environment", *_PHASE_MULTIPLIERS}
759
+ _LAUNCH_AGENT = {"name", "import_path", "model_name", "kwargs", "env"}
760
+ _LAUNCH_DATASET = {"path", "name", "version", "task_names", "exclude_task_names", "n_tasks"}
761
+
762
+
763
+ def _scrub(values, at: str = "") -> tuple:
764
+ """(kept, secret names): string values redacted, credential-named ones left
765
+ out - at any depth, since a kwarg may nest them (``llm_kwargs.api_key``,
766
+ an MCP server's ``headers.Authorization``); a nested one is named by its path."""
767
+ kept, secret = {}, []
768
+ for key, value in (values if isinstance(values, dict) else {}).items():
769
+ name = f"{at}{key}"
770
+ if _SECRET_KEY.search(str(key)):
771
+ secret.append(name)
772
+ else:
773
+ kept[str(key)], inner = _scrub_value(value, name)
774
+ secret += inner
775
+ return kept, secret
776
+
777
+
778
+ def _scrub_value(value, at: str) -> tuple:
779
+ if isinstance(value, str):
780
+ return redact(value), []
781
+ if isinstance(value, dict):
782
+ return _scrub(value, at + ".")
783
+ if isinstance(value, list):
784
+ out, secret = [], []
785
+ for i, v in enumerate(value):
786
+ v, inner = _scrub_value(v, f"{at}[{i}]")
787
+ out.append(v); secret += inner
788
+ return out, secret
789
+ return value, []
790
+
791
+
792
+ def launch_of(cfg: dict) -> dict:
793
+ """How a job was launched, from its config.json: the agents with their
794
+ models, kwargs and environment, the first dataset and its task filters,
795
+ the limits and the timeout multipliers - Harbor's defaults filled in for
796
+ the keys it left out (``defaulted`` names them), credential-named values
797
+ left out (``secret`` names them per agent), and every recorded key a
798
+ builder recipe has no field for named in ``unexpressed``. Empty for a
799
+ config another harness wrote (``command``) and for no config at all."""
800
+ if not isinstance(cfg, dict) or not cfg or cfg.get("command"):
801
+ return {}
802
+ unexpressed = sorted(k for k in cfg if k not in _LAUNCH_TOP)
803
+ agents = []
804
+ for i, a in enumerate(cfg.get("agents") or []):
805
+ if not isinstance(a, dict):
806
+ continue
807
+ kwargs, secret = _scrub(a.get("kwargs"))
808
+ env, secret_env = _scrub(a.get("env"))
809
+ agents.append({"name": a.get("name") or a.get("import_path") or "", "model": a.get("model_name") or "",
810
+ "kwargs": kwargs, "env": env, "secret": secret + secret_env})
811
+ unexpressed += [f"agents[{i}].{k}" for k in sorted(a) if k not in _LAUNCH_AGENT]
812
+ datasets = [d for d in cfg.get("datasets") or [] if isinstance(d, dict)]
813
+ d0 = datasets[0] if datasets else {}
814
+ dataset = ({"path": redact(str(d0["path"]))} if d0.get("path") else
815
+ {"name": str(d0["name"]), "version": str(d0.get("version") or "")} if d0.get("name") else {})
816
+ unexpressed += [f"datasets[0].{k}" for k in sorted(d0) if k not in _LAUNCH_DATASET]
817
+ unexpressed += [f"datasets[{i}]" for i in range(1, len(datasets))]
818
+ retry = cfg.get("retry") if isinstance(cfg.get("retry"), dict) else {}
819
+ unexpressed += [f"retry.{k}" for k in sorted(retry) if k != "max_retries"]
820
+ environment = cfg.get("environment") if isinstance(cfg.get("environment"), dict) else {}
821
+ unexpressed += [f"environment.{k}" for k in sorted(environment) if k != "type"]
822
+ recorded = {"n_concurrent_trials": cfg.get("n_concurrent_trials"), "max_retries": retry.get("max_retries"),
823
+ "n_attempts": cfg.get("n_attempts"), "timeout_multiplier": cfg.get("timeout_multiplier")}
824
+ return {
825
+ "agents": agents,
826
+ "dataset": dataset,
827
+ "tasks": len(cfg.get("tasks") or []),
828
+ "task_names": [str(t) for t in d0.get("task_names") or []],
829
+ "exclude_task_names": [str(t) for t in d0.get("exclude_task_names") or []],
830
+ "n_tasks": d0.get("n_tasks") or None,
831
+ **{k: (HARBOR_DEFAULTS[k] if v is None else v) for k, v in recorded.items()},
832
+ **{k: cfg.get(k) for k in _PHASE_MULTIPLIERS},
833
+ "environment": str(environment.get("type") or ""),
834
+ "defaulted": [k for k, v in recorded.items() if v is None],
835
+ "unexpressed": unexpressed,
836
+ }
837
+
838
+
839
+ def _budget(b) -> dict:
840
+ """A job's recorded spending cap, read leniently: {} unless it names a cap; an amount that is no number is
841
+ None (not reported), never zero."""
842
+ num = lambda v: v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
843
+ if not isinstance(b, dict) or not (num(b.get("max_usd")) or num(b.get("unit_max_usd"))):
844
+ return {}
845
+ names = lambda v: [str(x) for x in v] if isinstance(v, list) else []
846
+ return {"max_usd": num(b.get("max_usd")), "spent_usd": num(b.get("spent_usd")), "unsettled_usd": num(b.get("unsettled_usd")),
847
+ # only the run's cap stops a run: without one there is no stop to say
848
+ "stopped": b.get("stopped") is True and num(b.get("max_usd")) is not None,
849
+ "cut_off": names(b.get("cut_off")), "not_run": names(b.get("not_run")),
850
+ "unit_max_usd": num(b.get("unit_max_usd")), "over_unit": names(b.get("over_unit")),
851
+ # a merged job: its shards, and how many stopped at their share
852
+ "shards": num(b.get("shards")) or 0, "shards_stopped": num(b.get("shards_stopped")) or 0}
853
+
854
+
855
+ def parse_job(job_dir: str, reuse=None) -> Job:
856
+ """One job directory. ``reuse(trial_dir)``: a Trial already parsed from
857
+ files that have not changed since (a workspace's cache, for a running
858
+ job), or None to parse it."""
859
+ res = verifier._read_json(os.path.join(job_dir, "result.json")) or {}
860
+ cfg = verifier._read_json(os.path.join(job_dir, "config.json")) or {}
861
+ stats = res.get("stats") or {}
862
+
863
+ agents = cfg.get("agents") or [{}]
864
+ model = agents[0].get("model_name", "")
865
+
866
+ # Three config shapes in the wild: `datasets: [{path, n_tasks}]` for a
867
+ # local checkout, `datasets: [{name, version, n_tasks}]` for a registry
868
+ # fetch (no `path` key at all), and `tasks: [{name: "<dataset>/<task>"}]`
869
+ # where the dataset is only a prefix.
870
+ datasets = cfg.get("datasets") or []
871
+ tasks = cfg.get("tasks") or []
872
+ dataset_path = ""
873
+ if datasets:
874
+ d0 = datasets[0]
875
+ dataset = dataset_path = d0.get("path", "")
876
+ if not dataset:
877
+ name, version = d0.get("name", ""), d0.get("version", "")
878
+ dataset = f"{name}@{version}" if name and version else name
879
+ n_tasks = d0.get("n_tasks") or 0
880
+ elif tasks:
881
+ prefixes = {
882
+ t["name"].rsplit("/", 1)[0]
883
+ for t in tasks
884
+ if isinstance(t.get("name"), str) and "/" in t["name"]
885
+ }
886
+ dataset = prefixes.pop() if len(prefixes) == 1 else ("mixed" if prefixes else "")
887
+ n_tasks = len(tasks)
888
+ else:
889
+ dataset, n_tasks = "", 0
890
+
891
+ job = Job(
892
+ job_id=os.path.basename(job_dir),
893
+ dir=job_dir,
894
+ model_name=model,
895
+ dataset=os.path.basename(dataset.rstrip("/")) if dataset else "",
896
+ n_tasks=n_tasks,
897
+ started_at=_iso(res.get("started_at")),
898
+ finished_at=_iso(res.get("finished_at")),
899
+ agent_name=agents[0].get("name", ""),
900
+ reasoning_effort=str((agents[0].get("kwargs") or {}).get("reasoning_effort", "")),
901
+ n_total_trials=res.get("n_total_trials") or 0,
902
+ n_errored_trials=stats.get("n_errored_trials") or 0,
903
+ n_running_trials=stats.get("n_running_trials") or 0,
904
+ n_pending_trials=stats.get("n_pending_trials") or 0,
905
+ n_retries=stats.get("n_retries") or 0,
906
+ preparing=is_preparing_job_dir(job_dir),
907
+ # what a job recorded about itself is shown and copied verbatim, so a
908
+ # key that leaked into a path or an importer's --command is scrubbed here
909
+ dataset_path=redact(dataset_path),
910
+ )
911
+
912
+ for name in sorted(os.listdir(job_dir)):
913
+ sub = os.path.join(job_dir, name)
914
+ if os.path.isdir(sub) and os.path.isfile(os.path.join(sub, "result.json")):
915
+ t = (reuse(sub) if reuse else None) or parse_trial(job.job_id, sub)
916
+ if t:
917
+ job.trials.append(t)
918
+ if not job.agent_name:
919
+ # a config written with defaults left out names no agent for Harbor's default one; the trials do
920
+ job.agent_name = next((t.agent_info_name for t in job.trials if t.agent_info_name), "")
921
+ job.runner = str(cfg.get("runner_label") or cfg.get("runner") or "") if isinstance(cfg.get("runner"), str) else ""
922
+ job.runner_commit = str(cfg.get("runner_commit") or "")[:12] if job.runner else ""
923
+ job.runner_judge = str(cfg.get("runner_judge") or "") if job.runner else ""
924
+ job.questions_from = str(cfg.get("questions_from") or "") if job.runner else ""
925
+ words = cfg.get("runner_items") if job.runner else None
926
+ job.runner_items = [str(w) for w in words[:2]] if isinstance(words, list) and len(words) >= 2 else []
927
+ std = cfg.get("standard") if isinstance(cfg.get("standard"), dict) else {}
928
+ ctl = std.get("control") if isinstance(std.get("control"), dict) else {}
929
+ job.control_source = str(ctl.get("source") or "control") if ctl.get("name") == job.agent_label.lower() else ""
930
+ job.launch_id = str(std.get("launch") or "")
931
+ merged = std.get("merged_from")
932
+ job.merged_from = [s["job"] for s in merged if isinstance(s, dict) and isinstance(s.get("job"), str) and s["job"]] if isinstance(merged, list) else []
933
+ job.budget = _budget(res.get("budget"))
934
+ job.standard = bool(std)
935
+ # a runner's job was never a `harbor run`: there is no command to rebuild
936
+ job.harbor_command = "" if job.runner else redact(harbor_command(
937
+ cfg, next((t.env_type for t in job.trials if t.env_type), "")))
938
+ job.launch = launch_of(cfg)
939
+ if len(agents) > 1 and not job.runner:
940
+ job.agents = [{"name": a.get("name") or a.get("import_path") or "", "model": a.get("model_name") or "",
941
+ "effort": str((a.get("kwargs") or {}).get("reasoning_effort", "")),
942
+ "command": redact(harbor_command(dict(cfg, agents=[a]),
943
+ next((t.env_type for t in job.trials if t.env_type), "")))}
944
+ for a in agents if isinstance(a, dict)]
945
+ lock = verifier._read_json(os.path.join(job_dir, "lock.json")) or {}
946
+ harbor = lock.get("harbor") if isinstance(lock.get("harbor"), dict) else {}
947
+ job.harbor_version = str(harbor.get("version") or "")
948
+ job.harbor_editable = harbor.get("is_editable") if isinstance(harbor.get("is_editable"), bool) else None
949
+ return job
950
+
951
+
952
+ def _agent_of(t, agents: list) -> int:
953
+ """Which of a job's agents ran trial ``t``: by its model, then its name."""
954
+ def named(a):
955
+ return t.agent_info_name and t.agent_info_name in (a["name"], a["name"].split(":")[-1])
956
+ by_model = [i for i, a in enumerate(agents) if a["model"] == t.model_name]
957
+ if len(by_model) == 1:
958
+ return by_model[0]
959
+ pool = by_model or list(range(len(agents)))
960
+ named_pool = [i for i in pool if named(agents[i])]
961
+ pool = named_pool or pool
962
+ if len(pool) == 1:
963
+ return pool[0]
964
+ exact = [i for i in pool if agents[i]["effort"] == t.reasoning_effort]
965
+ return exact[0] if len(exact) == 1 else -1
966
+
967
+
968
+ def split_agents(jobs: list) -> list:
969
+ """A Harbor job of several agents or models as one view per agent, in
970
+ their config's order: each its own trials, rate, control or not, and
971
+ command. Pooled under its first agent, model-b's misses were shown as
972
+ model-a's and an oracle listed first made the real agent a control. A
973
+ view's id is the job's with the agent after ``~``; its trials link to it.
974
+ Harbor's job-level counts are shared evenly; the rest is the view's own."""
975
+ out = []
976
+ for job in jobs:
977
+ if len(job.agents) < 2:
978
+ out.append(job)
979
+ continue
980
+ k = len(job.agents)
981
+ groups = {i: [] for i in range(k)}
982
+ for t in job.trials:
983
+ groups.setdefault(_agent_of(t, job.agents), []).append(t)
984
+ labels = set()
985
+ for i, a in enumerate(job.agents):
986
+ label = re.sub(r"[^A-Za-z0-9._-]+", "-", f"{a['name'].split(':')[-1]}-{a['model']}".strip("-")) or str(i + 1)
987
+ if label in labels:
988
+ label += f"-{i + 1}"
989
+ labels.add(label)
990
+ vid = f"{job.job_id}~{label}"
991
+ trials = [replace(t, job_id=vid) for t in groups[i]]
992
+ out.append(replace(
993
+ job, job_id=vid, model_name=a["model"], agent_name=a["name"], reasoning_effort=a["effort"],
994
+ trials=trials, harbor_command=a["command"], agents=[],
995
+ n_total_trials=job.n_total_trials // k if job.n_total_trials % k == 0 else len(trials),
996
+ n_errored_trials=sum(1 for t in trials if t.exception_type),
997
+ n_running_trials=job.n_running_trials // k + (i < job.n_running_trials % k),
998
+ n_pending_trials=job.n_pending_trials // k + (i < job.n_pending_trials % k),
999
+ n_retries=job.n_retries // k + (i < job.n_retries % k)))
1000
+ orphans = groups.get(-1)
1001
+ if orphans: # a trial of an agent the config does not list: kept, under the job's own id
1002
+ out.append(replace(job, trials=orphans, agents=[]))
1003
+ return out
1004
+
1005
+
1006
+ def load_all(roots) -> list[Job]:
1007
+ jobs = split_agents([parse_job(d) for d in find_job_dirs(roots)])
1008
+ # Newest first. Compared as epoch numbers, not datetimes: a job whose
1009
+ # result.json never got a started_at (a crashed run) yields None, and
1010
+ # datetime.min is naive where parsed stamps are aware — mixing the two
1011
+ # in one sort raises. -inf sorts the undated to the end instead.
1012
+ jobs.sort(
1013
+ key=lambda j: j.started_at.timestamp() if j.started_at else float("-inf"),
1014
+ reverse=True,
1015
+ )
1016
+ return jobs