nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/runs/cli.py ADDED
@@ -0,0 +1,316 @@
1
+ """Read the runs under a job root from a shell: what /runs and /job/<id> show.
2
+
3
+ PYTHONPATH=app python3 -m nullius.runs.cli list <root>... # job ids, one per line
4
+ PYTHONPATH=app python3 -m nullius.runs.cli list <root> --json # the runs table
5
+ PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> # the trials and their outcomes
6
+ PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> --json
7
+ PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> <trial> # one trial: what /trial/<job>/<trial> shows
8
+ PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> <trial> --json
9
+
10
+ ``<root>`` is anything ``app.py`` accepts as a job root: a Harbor job
11
+ directory, a parent of several, or a tree the import CLI has already
12
+ converted. The same reader the pages use (``parse.load_all``) walks it, so a
13
+ job id printed here is the one ``/job/<id>`` opens, and the outcomes are the
14
+ ones its table shows — from the files on disk, never from a workspace cache.
15
+ With a trial name, ``show`` reads that trial the way the trial page does —
16
+ the outcome and checks, the usage and cost, the exception, and the steps of
17
+ ``agent/trajectory.json`` through ``traces.trajectory`` (already redacted on
18
+ the way in; the calls' arguments and observations, which can be the whole
19
+ patch, are left to the page's conversation browser).
20
+
21
+ Reading only: nothing is written, anywhere, and no socket is opened. The
22
+ exit codes are the contract in ``docs/cli.md``, shared by every CLI here.
23
+ """
24
+ from __future__ import annotations
25
+
26
+ import argparse
27
+ import json
28
+ import os
29
+ import sys
30
+ from dataclasses import asdict
31
+ from datetime import datetime
32
+
33
+ from nullius.runs import parse
34
+ from nullius.traces import trajectory
35
+
36
+ # Exit codes — docs/cli.md is the contract; the other two CLIs use the same
37
+ # numbers. 1 is left to Python itself: an uncaught exception is a bug, and
38
+ # its traceback is the report.
39
+ EXIT_OK = 0
40
+ EXIT_NOT_FOUND = 3
41
+
42
+ EPILOG = """\
43
+ examples:
44
+ PYTHONPATH=app python3 -m nullius.runs.cli list jobs
45
+ PYTHONPATH=app python3 -m nullius.runs.cli list jobs jobs_imported --json
46
+ PYTHONPATH=app python3 -m nullius.runs.cli show jobs 2026-09-12__22-53-55 --json
47
+ PYTHONPATH=app python3 -m nullius.runs.cli show jobs 2026-09-12__22-53-55 astropy__astropy-8707__agWFDYq
48
+
49
+ exit codes:
50
+ 0 done
51
+ 1 unexpected error (a traceback: report it)
52
+ 2 usage: a bad flag, a missing argument
53
+ 3 input not found: no such root, no job under it, no job of that id, no trial of that name
54
+ 4 the operation could not be completed
55
+ With --json, stdout is exactly one JSON document; everything else goes to stderr.
56
+ """
57
+
58
+
59
+ def _stamp(value):
60
+ return value.isoformat() if isinstance(value, datetime) else value
61
+
62
+
63
+ def job_row(job: parse.Job) -> dict:
64
+ """One job as the runs table shows it: identity, state, the outcome mix
65
+ and the two solve rates. Derived properties are included by name because
66
+ ``asdict`` cannot see them, and they are what the page renders."""
67
+ return {
68
+ "job_id": job.job_id,
69
+ "dir": job.dir,
70
+ "dataset": job.dataset,
71
+ "model": job.model_name,
72
+ "agent": job.agent_label,
73
+ "kind": job.kind,
74
+ "state": job.state,
75
+ "n_trials": len(job.trials),
76
+ "n_planned": job.n_planned,
77
+ "counts": job.counts,
78
+ "n_resolved": job.n_resolved,
79
+ "n_infra": job.n_infra,
80
+ "raw_solve_rate": job.raw_solve_rate,
81
+ "adjusted_solve_rate": job.adjusted_solve_rate,
82
+ "started_at": _stamp(job.started_at),
83
+ "finished_at": _stamp(job.finished_at),
84
+ }
85
+
86
+
87
+ def job_detail(job: parse.Job) -> dict:
88
+ """The whole parsed job — every field the workspace cache keeps, plus the
89
+ derived ones the job page renders, per trial too. ``exception_message``
90
+ was redacted on the way in (``parse.parse_trial``), so this is as safe to
91
+ print as the page is to render."""
92
+ data = asdict(job)
93
+ data.update(job_row(job))
94
+ data["harbor_label"] = job.harbor_label
95
+ for trial, raw in zip(job.trials, data["trials"]):
96
+ raw["outcome_label"] = trial.outcome_label
97
+ raw["is_infra"] = trial.is_infra
98
+ raw["is_regression"] = trial.is_regression
99
+ raw["has_trace"] = trial.has_trace
100
+ raw["started_at"] = _stamp(trial.started_at)
101
+ raw["finished_at"] = _stamp(trial.finished_at)
102
+ raw["phases"] = {k: [_stamp(a), _stamp(b)] for k, (a, b) in trial.phases.items()}
103
+ return data
104
+
105
+
106
+ def _trial_record(trial: parse.Trial) -> dict:
107
+ """One trial as ``job_detail`` carries it: every parsed field plus the
108
+ derived ones the pages render, timestamps as ISO strings."""
109
+ raw = asdict(trial)
110
+ raw["outcome_label"] = trial.outcome_label
111
+ raw["is_infra"] = trial.is_infra
112
+ raw["is_regression"] = trial.is_regression
113
+ raw["has_trace"] = trial.has_trace
114
+ raw["started_at"] = _stamp(trial.started_at)
115
+ raw["finished_at"] = _stamp(trial.finished_at)
116
+ raw["phases"] = {k: [_stamp(a), _stamp(b)] for k, (a, b) in trial.phases.items()}
117
+ return raw
118
+
119
+
120
+ def trial_detail(job: parse.Job, trial: parse.Trial) -> dict:
121
+ """What the trial page shows: the trial's record, the check vocabulary
122
+ the dataset names its cards with, and the trajectory as ``trajectory.load``
123
+ reads it — steps with their tool calls by name, category, target and
124
+ failure, and the totals — or the reason it could not be read. Every
125
+ string came through ``redact`` in the reader; nothing is added here.
126
+ Arguments and observations are left out: they are the conversation
127
+ browser's, paged and bounded there, and can be the whole patch."""
128
+ vocab = trial.vocab
129
+ doc = {"job_id": job.job_id, "trial": _trial_record(trial),
130
+ "vocab": {"gained": vocab.gained, "held": vocab.held, "has_patch": vocab.has_patch},
131
+ "trajectory": None, "steps": []}
132
+ if not trial.has_trajectory:
133
+ return doc
134
+ traj = trajectory.load(trial.trajectory_path)
135
+ doc["trajectory"] = {
136
+ "error": traj.error, "n_steps": traj.n_steps, "n_tool_calls": traj.n_tool_calls,
137
+ "n_failed_calls": traj.n_failed_calls, "category_freq": dict(traj.category_freq),
138
+ "files_touched": list(traj.files_touched), "final_metrics": traj.final_metrics,
139
+ "has_step_clock": traj.has_step_clock, "last_agent_message": traj.last_agent_message,
140
+ "total_reasoning_tokens": traj.total_reasoning_tokens,
141
+ }
142
+ doc["steps"] = [{
143
+ "step_id": s.step_id, "source": s.source, "timestamp": s.timestamp, "dur": s.dur,
144
+ "message": s.message,
145
+ "tool_calls": [{"name": c["name"], "category": c["category"], "target": c["target"],
146
+ "failure": c["failure"], "obs_bytes": c["obs_bytes"]} for c in s.tool_calls],
147
+ } for s in traj.steps]
148
+ return doc
149
+
150
+
151
+ def _first_line(text: str, width: int = 96) -> str:
152
+ line = (text or "").strip().splitlines()[0] if (text or "").strip() else ""
153
+ return line if len(line) <= width else line[:width - 1] + "…"
154
+
155
+
156
+ def _secs(value) -> str:
157
+ return "—" if value is None else f"{value:.1f}s"
158
+
159
+
160
+ def _print_trial(job: parse.Job, trial: parse.Trial) -> None:
161
+ doc = trial_detail(job, trial)
162
+ t, vocab = trial, doc["vocab"]
163
+ print(f"{job.job_id}/{t.trial_name}")
164
+ print(f" task {t.task_base} · {t.outcome_label} ({t.outcome}) · "
165
+ f"model {t.model_name or job.model_name or '—'} · reward "
166
+ f"{'—' if t.reward is None else t.reward}")
167
+ checks = []
168
+ if vocab["gained"]:
169
+ checks.append(f"{vocab['gained']} {t.f2p_pass} passed, {t.f2p_fail} failed")
170
+ if vocab["held"]:
171
+ checks.append(f"{vocab['held']} {t.p2p_pass} passed, {t.p2p_fail} failed")
172
+ if vocab["has_patch"]:
173
+ checks.append("patch " + ("—" if t.patch_exists is None else
174
+ ("applied" if t.patch_applied else "not applied")))
175
+ if checks:
176
+ print(" checks: " + " · ".join(checks))
177
+ cost = "—" if t.cost_usd is None else f"${t.cost_usd:.4f}"
178
+ print(f" usage: input {t.n_input_tokens} · cached {t.n_cache_tokens} · "
179
+ f"output {t.n_output_tokens} · cost {cost} (estimated)")
180
+ print(f" time: total {_secs(t.dur_total)} · env {_secs(t.dur_env)} · "
181
+ f"setup {_secs(t.dur_agent_setup)} · agent {_secs(t.dur_agent)} · "
182
+ f"verifier {_secs(t.dur_verifier)}")
183
+ print(f" evidence: trajectory {'recorded' if t.has_trajectory else 'absent'} · "
184
+ f"trace {'recorded' if t.has_trace else 'absent'} · "
185
+ f"reward {'recorded' if t.reward is not None else 'absent'}")
186
+ if t.exception_type:
187
+ print(f" exception: {t.exception_type}: {_first_line(t.exception_message)}")
188
+ for name in t.p2p_failures:
189
+ print(f" broke: {name}")
190
+ traj = doc["trajectory"]
191
+ if traj is None:
192
+ return
193
+ if traj["error"]:
194
+ print(f" trajectory unreadable: {_first_line(traj['error'])}")
195
+ return
196
+ mix = " · ".join(f"{k} {v}" for k, v in sorted(traj["category_freq"].items()))
197
+ print(f" steps: {traj['n_steps']} · tool calls {traj['n_tool_calls']} · "
198
+ f"failed {traj['n_failed_calls']}" + (f" · {mix}" if mix else ""))
199
+ width = len(str(max((s["step_id"] for s in doc["steps"]), default=0)))
200
+ for s in doc["steps"]:
201
+ calls = ", ".join(c["name"] + (f" ✗ {c['failure']}" if c["failure"] else "")
202
+ for c in s["tool_calls"])
203
+ text = _first_line(s["message"]) if s["message"] and s["message"] != "(tool use)" else ""
204
+ body = " · ".join(x for x in (text, calls) if x)
205
+ dur = "" if s["dur"] is None else f" {s['dur']:.1f}s"
206
+ print(f" {s['step_id']:>{width}} {s['source']:<6} {body}{dur}")
207
+
208
+
209
+ def _emit(doc) -> None:
210
+ json.dump(doc, sys.stdout, ensure_ascii=False, indent=1, default=_stamp)
211
+ sys.stdout.write("\n")
212
+
213
+
214
+ def _load(roots) -> list | None:
215
+ """The jobs under *roots*, or None (said on stderr) when a root is missing."""
216
+ missing = [r for r in roots if not os.path.isdir(r)]
217
+ if missing:
218
+ for r in missing:
219
+ print(f"no such directory: {r}", file=sys.stderr)
220
+ return None
221
+ return parse.load_all(roots)
222
+
223
+
224
+ def _list(args) -> int:
225
+ jobs = _load(args.roots)
226
+ if jobs is None:
227
+ return EXIT_NOT_FOUND
228
+ if not jobs:
229
+ print(f"no job directory found under {', '.join(args.roots)}", file=sys.stderr)
230
+ return EXIT_NOT_FOUND
231
+ if args.json:
232
+ _emit([job_row(j) for j in jobs])
233
+ else:
234
+ for j in jobs:
235
+ print(j.job_id)
236
+ return EXIT_OK
237
+
238
+
239
+ def _show(args) -> int:
240
+ jobs = _load([args.root])
241
+ if jobs is None:
242
+ return EXIT_NOT_FOUND
243
+ job = next((j for j in jobs if j.job_id == args.job_id), None)
244
+ if job is None:
245
+ known = ", ".join(j.job_id for j in jobs) or "none"
246
+ print(f"no job named {args.job_id!r} under {args.root} (jobs there: {known})",
247
+ file=sys.stderr)
248
+ return EXIT_NOT_FOUND
249
+ if args.trial_name:
250
+ trial = next((t for t in job.trials if t.trial_name == args.trial_name), None)
251
+ if trial is None:
252
+ known = ", ".join(t.trial_name for t in job.trials) or "none"
253
+ print(f"no trial named {args.trial_name!r} in {job.job_id} (trials there: {known})",
254
+ file=sys.stderr)
255
+ return EXIT_NOT_FOUND
256
+ if args.json:
257
+ _emit(trial_detail(job, trial))
258
+ else:
259
+ _print_trial(job, trial)
260
+ return EXIT_OK
261
+ if args.json:
262
+ _emit(job_detail(job))
263
+ return EXIT_OK
264
+ row = job_row(job)
265
+ rate = lambda v: "—" if v is None else f"{100 * v:.0f}%"
266
+ print(f"{job.job_id}")
267
+ print(f" dataset {row['dataset'] or '—'} · model {row['model'] or '—'} · "
268
+ f"agent {row['agent'] or '—'} · {row['state']}")
269
+ print(f" {row['n_trials']} trial(s) of {row['n_planned']} planned: "
270
+ f"resolved {row['n_resolved']} · infra {row['n_infra']} · "
271
+ f"raw {rate(row['raw_solve_rate'])} · adjusted {rate(row['adjusted_solve_rate'])}")
272
+ mix = " · ".join(f"{k} {v}" for k, v in row["counts"].items() if v)
273
+ if mix:
274
+ print(f" outcomes: {mix}")
275
+ width = max((len(t.trial_name) for t in job.trials), default=0)
276
+ for t in job.trials:
277
+ line = f" {t.trial_name:<{width}} {t.outcome}"
278
+ if t.exception_type:
279
+ line += f" {t.exception_type}"
280
+ print(line)
281
+ return EXIT_OK
282
+
283
+
284
+ def main(argv=None) -> int:
285
+ ap = argparse.ArgumentParser(
286
+ prog="python3 -m nullius.runs.cli", description=__doc__.splitlines()[0],
287
+ epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
288
+ sub = ap.add_subparsers(dest="command", metavar="COMMAND")
289
+ ls = sub.add_parser("list", help="the job ids under one or more roots, newest first",
290
+ epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
291
+ ls.add_argument("roots", nargs="+", metavar="ROOT",
292
+ help="a job directory, a parent of job directories, or an imported root")
293
+ ls.add_argument("--json", action="store_true",
294
+ help="one JSON array of job rows (what the runs table shows) instead of ids")
295
+ ls.set_defaults(run=_list)
296
+ sh = sub.add_parser("show", help="one job: its trials and their outcomes; or one trial",
297
+ epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
298
+ sh.add_argument("root", metavar="ROOT", help="the root the job is under")
299
+ sh.add_argument("job_id", metavar="JOB_ID", help="a job id, as `list` prints it")
300
+ sh.add_argument("trial_name", metavar="TRIAL", nargs="?", default="",
301
+ help="a trial name, as `show` on the job prints it: that trial's outcome, "
302
+ "checks, usage, exception and steps, as /trial/<job>/<trial> shows them")
303
+ sh.add_argument("--json", action="store_true",
304
+ help="the whole parsed job (or the one trial, with its steps) as one JSON document")
305
+ sh.set_defaults(run=_show)
306
+ args = ap.parse_args(argv)
307
+ if not args.command:
308
+ ap.error("a command is required: list or show")
309
+ return args.run(args)
310
+
311
+
312
+ if __name__ == "__main__": # pragma: no cover - entry point
313
+ # Run as `PYTHONPATH=app python3 -m nullius.runs.cli`. A module, not a
314
+ # script, like its two siblings: without this guard `-m` imports the
315
+ # module, runs nothing and exits 0, which reads exactly like success.
316
+ raise SystemExit(main())