nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,306 @@
1
+ """A trial: its trace, trajectory, conversation, artifacts and saved views."""
2
+ from __future__ import annotations
3
+
4
+ import json
5
+ import os
6
+ import threading
7
+ from collections import OrderedDict
8
+ from flask import abort, flash, jsonify, redirect, render_template, request, url_for
9
+ from markupsafe import Markup, escape
10
+ from nullius.traces import behavior, charts, conversation, trace, trajectory
11
+ from nullius.runs import artifacts, parse
12
+
13
+ from .core import app
14
+ from . import shared
15
+
16
+
17
+ # What a saved trace view may be called. Defined in the data layer, because two
18
+ # things depend on agreeing about it: this route writes one, and ?trace= reads one.
19
+ SAVED_TRACE_RE = parse.TRACE_NAME_RE
20
+ SAVED_TRACE_DEFAULT = "trace-filtered.json"
21
+
22
+
23
+ def trace_file_path(trial, name: str) -> str | None:
24
+ """Absolute path to a trace file inside *trial*'s ``agent/`` directory.
25
+
26
+ *name* arrives from a query string or a form field, so it is validated as a
27
+ filename rather than trusted as a path: the regex admits no separator and
28
+ `shared._safe_dest` refuses anything that still manages to leave the directory.
29
+ Returns None when the name is not one this app will read or write.
30
+ """
31
+ if not SAVED_TRACE_RE.match((name or "").strip()):
32
+ return None
33
+ return shared._safe_dest(os.path.join(trial.dir, "agent"), name.strip())
34
+
35
+
36
+ def _dataset_dirs(job):
37
+ """Candidate dataset checkouts for a job's tasks, best lead first.
38
+
39
+ The job config's dataset path (absolute, or relative to wherever harbor
40
+ ran), then DATASET_ROOTS. Registry-fetched runs carry no path, so
41
+ the env var is the only way to point those at a local checkout.
42
+ """
43
+ candidates = [job.dataset_path] + os.environ.get(
44
+ "DATASET_ROOTS", ""
45
+ ).split(os.pathsep)
46
+ dirs = []
47
+ for c in candidates:
48
+ if not c:
49
+ continue
50
+ p = os.path.abspath(os.path.expanduser(c))
51
+ if os.path.isdir(p) and p not in dirs:
52
+ dirs.append(p)
53
+ return dirs
54
+
55
+
56
+ def _gold_localization(job, trial, traj):
57
+ """Agent-edited files vs the task's gold patch, or None when no local
58
+ dataset checkout (or no parsable gold diff) can be found."""
59
+ if traj is None:
60
+ return None
61
+ for root in _dataset_dirs(job):
62
+ solve = os.path.join(root, trial.task_base, "solution", "solve.sh")
63
+ if not os.path.isfile(solve):
64
+ continue
65
+ try:
66
+ with open(solve, encoding="utf-8") as fh:
67
+ gold = behavior.gold_files(fh.read())
68
+ except OSError:
69
+ return None
70
+ if gold:
71
+ return behavior.localization(traj.files_touched, gold)
72
+ return None
73
+
74
+
75
+ def _find_trial(job, trial_name):
76
+ return next((x for x in job.trials if x.trial_name == trial_name), None)
77
+
78
+
79
+ def _trial_evidence(job_id, trial_name):
80
+ """Resolve URL identifiers through parsed runs, never as filesystem paths."""
81
+ job = shared._find_job(shared._jobs_for_request(), job_id)
82
+ if job is None:
83
+ abort(404)
84
+ trial = _find_trial(job, trial_name)
85
+ if trial is None:
86
+ abort(404)
87
+ return job, trial
88
+
89
+
90
+ def _requested_trace(trial, *, notify=False):
91
+ """One trace/filter resolution for the page and its read-only detail API.
92
+
93
+ `zoom` is parsed beside `hide`/`hide_span` because it is part of the same
94
+ URL-defined view — but it re-bases the waterfall's axis on one subtree and
95
+ hides nothing (`trace.Filter`).
96
+ """
97
+ filt = trace.Filter.of(request.args.getlist("hide"), request.args.getlist("hide_span"),
98
+ request.args.get("zoom"))
99
+ wanted = (request.args.get("trace") or "").strip()
100
+ path, viewing = (trial.trace_path if trial.has_trace else None), ""
101
+ if wanted:
102
+ alt = trace_file_path(trial, wanted)
103
+ if alt and os.path.isfile(alt):
104
+ path, viewing = alt, os.path.basename(alt)
105
+ elif notify:
106
+ flash(Markup(f"No saved trace called <span class=\"mono\">{escape(wanted)}</span> "
107
+ "in this trial — showing the producer's own."), "error")
108
+ else:
109
+ abort(404)
110
+ return (_parsed_trace(path, filt) if path else None), viewing
111
+
112
+
113
+ # Traces parsed for a view: every span-detail click, filter and zoom link asked
114
+ # for the whole fold, prune, redaction and breakdowns again (about a second at
115
+ # 6k spans). Kept by file (path, size, mtime) and view, a few at a time; a Trace
116
+ # is not changed once parsed.
117
+ _PARSED: "OrderedDict" = OrderedDict()
118
+ _PARSED_MAX = 8
119
+ _PARSED_LOCK = threading.Lock()
120
+
121
+
122
+ def _parsed_trace(path, filt):
123
+ try:
124
+ st = os.stat(path)
125
+ except OSError:
126
+ return trace.load(path, filt)
127
+ key = (os.path.realpath(path), st.st_size, st.st_mtime_ns, filt)
128
+ with _PARSED_LOCK:
129
+ if key in _PARSED:
130
+ _PARSED.move_to_end(key)
131
+ return _PARSED[key]
132
+ tr = trace.load(path, filt)
133
+ with _PARSED_LOCK:
134
+ _PARSED[key] = tr
135
+ while len(_PARSED) > _PARSED_MAX:
136
+ _PARSED.popitem(last=False)
137
+ return tr
138
+
139
+
140
+ @app.route("/trial/<job_id>/<trial_name>/artifact/<key>")
141
+ def artifact_page(job_id, trial_name, key):
142
+ job, trial = _trial_evidence(job_id, trial_name)
143
+ try:
144
+ record = artifacts.read(trial.dir, key, request.args.get("page", 1))
145
+ except (LookupError, OSError):
146
+ abort(404)
147
+ response = app.make_response(render_template("traces/artifact.html", job=job, trial=trial,
148
+ artifact=record))
149
+ response.headers["Cache-Control"] = "no-store"
150
+ return response
151
+
152
+
153
+ @app.route("/trial/<job_id>/<trial_name>/conversation")
154
+ def conversation_data(job_id, trial_name):
155
+ _, t = _trial_evidence(job_id, trial_name)
156
+ traj = trajectory.load(t.trajectory_path) if t.has_trajectory else None
157
+ payload = conversation.page(
158
+ traj, query=request.args.get("q", ""), kind=request.args.get("kind", "all"),
159
+ page=request.args.get("page", 1), position=request.args.get("position"),
160
+ step=request.args.get("step"), section=request.args.get("section"),
161
+ chunk=request.args.get("chunk", 0),
162
+ )
163
+ response = jsonify(payload)
164
+ response.headers["Cache-Control"] = "no-store"
165
+ return response
166
+
167
+
168
+ @app.route("/trial/<job_id>/<trial_name>/trace-span")
169
+ def trace_span_data(job_id, trial_name):
170
+ _, t = _trial_evidence(job_id, trial_name)
171
+ tr, _ = _requested_trace(t)
172
+ try:
173
+ payload = conversation.span_detail(
174
+ tr, request.args.get("span", ""), section=request.args.get("section"),
175
+ chunk=request.args.get("chunk", 0),
176
+ )
177
+ except LookupError:
178
+ abort(404)
179
+ response = jsonify(payload)
180
+ response.headers["Cache-Control"] = "no-store"
181
+ return response
182
+
183
+
184
+ @app.route("/trial/<job_id>/<trial_name>")
185
+ def trial(job_id, trial_name):
186
+ j, t = _trial_evidence(job_id, trial_name)
187
+ traj = trajectory.load(t.trajectory_path) if t.has_trajectory else None
188
+ ok = traj is not None and not traj.error
189
+ bench_judge = bench_parts = None
190
+ if j.runner: # the benchmark's own judge: what it was asked, what it said
191
+ try:
192
+ with open(os.path.join(t.dir, "verifier", "judge.json"), encoding="utf-8") as fh:
193
+ bench_judge = json.load(fh)
194
+ except (OSError, ValueError):
195
+ pass
196
+ try: # an item scored part by part: each part's text, answer and score
197
+ with open(os.path.join(t.dir, "verifier", "parts.json"), encoding="utf-8") as fh:
198
+ got = json.load(fh)
199
+ if isinstance(got, dict) and isinstance(got.get("parts"), list):
200
+ bench_parts = {"says": str(got.get("says") or ""), "nouns": str(got.get("nouns") or "parts"),
201
+ "parts": [x for x in got["parts"] if isinstance(x, dict)]}
202
+ # what the definition chose to show of a part, under its own words, in its order
203
+ bench_parts["shows"] = list(dict.fromkeys(k for x in bench_parts["parts"] if isinstance(x.get("shows"), dict) for k in x["shows"]))
204
+ except (OSError, ValueError):
205
+ pass
206
+ loc = _gold_localization(j, t, traj) if ok else None
207
+
208
+ # `hide`/`hide_span` make a filtered view a plain link — shareable,
209
+ # bookmarkable, and the same thing `save_trace` writes out. `trace` opens a
210
+ # view saved earlier instead of the producer's own file.
211
+ # A second, narrower query key: which kinds each chart is drawing. Separate
212
+ # from `hide` because it changes one chart and not the trace — the waterfall,
213
+ # the breakdown and what `save_trace` writes are all blind to it.
214
+ cfilt = charts.ChartFilter.of(request.args.getlist("chart_off"))
215
+ tr, viewing = _requested_trace(t, notify=True)
216
+ # Over `tr.roots`, which `parse` has already pruned — the charts describe the
217
+ # filtered view by construction, so there is one definition of "which spans
218
+ # are in this view" and it is still the query string.
219
+ ch = charts.charts_for(tr.roots, cfilt) if tr and tr.visible else None
220
+ records = artifacts.listing(t.dir)
221
+ for record in records:
222
+ record["url"] = url_for("artifact_page", job_id=job_id, trial_name=trial_name,
223
+ key=record["key"], workspace=request.args.get("workspace"))
224
+ return render_template(
225
+ "traces/trial.html", job=j, trial=t, traj=traj, loc=loc, bench_judge=bench_judge, bench_parts=bench_parts,
226
+ trace=tr, charts=ch, viewing=viewing,
227
+ chart_filter=cfilt, default_save_name=SAVED_TRACE_DEFAULT,
228
+ artifacts=records,
229
+ judgement=_judgement_panel(j, t), llm_judgement=_judgement_panel(j, t, "llm"), csrf=shared._csrf_token(),
230
+ )
231
+
232
+
233
+ def _judgement_panel(job, trial, lane="system-one"):
234
+ from . import judge
235
+ return judge.panel(job, trial, lane)
236
+
237
+
238
+ @app.route("/trial/<job_id>/<trial_name>/save-trace", methods=["POST"])
239
+ def save_trace(job_id, trial_name):
240
+ """Write the filtered view to a new trace file beside the trial's own.
241
+
242
+ The only route in this app that writes into a job directory. Three refusals
243
+ keep that safe: the name must be a plain `*.json` filename, the destination
244
+ must not exist, and it must not be the file being read — a producer's capture
245
+ is not reproducible, so nothing here may overwrite one.
246
+ """
247
+ shared._require_csrf()
248
+ _, t = _trial_evidence(job_id, trial_name)
249
+
250
+ # The zoom rides along only so the page comes back as it was; the file written
251
+ # is the filtered tree whole — a zoom is a way of looking, not a change.
252
+ filt = trace.Filter.of(request.form.getlist("hide"), request.form.getlist("hide_span"),
253
+ request.form.get("zoom"))
254
+ # Filtering a saved view again is allowed, so the source is whichever file the
255
+ # page was rendering — falling back to the producer's if that one has gone.
256
+ source_name = (request.form.get("trace") or "").strip()
257
+ source = trace_file_path(t, source_name) if source_name else t.trace_path
258
+ if not source or not os.path.isfile(source):
259
+ source, source_name = t.trace_path, ""
260
+ back = url_for(
261
+ "trial", job_id=job_id, trial_name=trial_name,
262
+ trace=source_name or None, **filt.query,
263
+ )
264
+
265
+ if not os.path.isfile(source):
266
+ flash("The trace this view was filtered from is no longer there.", "error")
267
+ return redirect(back)
268
+ if not filt.active:
269
+ flash("Nothing is filtered out — a saved view would just copy the original.", "error")
270
+ return redirect(back)
271
+
272
+ name = (request.form.get("filename") or "").strip() or SAVED_TRACE_DEFAULT
273
+ if not name.lower().endswith(".json"):
274
+ name += ".json"
275
+ dest = trace_file_path(t, name)
276
+ if not dest:
277
+ flash("Name it as a plain file ending in .json — letters, digits, dot, dash "
278
+ "and underscore, no directories.", "error")
279
+ return redirect(back)
280
+ shown = Markup(f'<span class="mono">{escape(os.path.basename(dest))}</span>')
281
+ if os.path.realpath(dest) == os.path.realpath(source) or os.path.basename(dest) == "trace.json":
282
+ flash(Markup(f"{shown} is the trace being read — save the view under another name."), "error")
283
+ return redirect(back)
284
+ if os.path.exists(dest):
285
+ flash(Markup(f"{shown} already exists; nothing was overwritten."), "error")
286
+ return redirect(back)
287
+
288
+ try:
289
+ with open(source, encoding="utf-8") as fh:
290
+ data = json.load(fh)
291
+ # Written from the raw document, so the saved file keeps every field the
292
+ # producer wrote — including the content this app redacts at render. It is
293
+ # exactly as sensitive as its source.
294
+ out = trace.filtered_document(data, filt)
295
+ with open(dest, "w", encoding="utf-8") as fh:
296
+ json.dump(out, fh, ensure_ascii=False, indent=1)
297
+ except (OSError, json.JSONDecodeError, ValueError) as exc:
298
+ flash(Markup(f"Could not write {shown}: {escape(str(exc))}"), "error")
299
+ return redirect(back)
300
+
301
+ view = url_for("trial", job_id=job_id, trial_name=trial_name, trace=os.path.basename(dest))
302
+ flash(Markup(
303
+ f"Saved {shown} beside the trial's own trace — {out['filtered']['spans_removed']} "
304
+ f'span(s) removed, totals recomputed. <a href="{view}">Open it</a>.'
305
+ ), "")
306
+ return redirect(back)
@@ -0,0 +1,84 @@
1
+ """Workspaces: list, open, refresh, rename, delete."""
2
+ from __future__ import annotations
3
+
4
+ from flask import flash, redirect, render_template, request, url_for
5
+ from markupsafe import Markup, escape
6
+ from nullius.runs import parse
7
+ from nullius import workspace
8
+
9
+ from .core import app
10
+ from . import shared
11
+
12
+
13
+
14
+ @app.route("/workspaces")
15
+ def workspaces_page():
16
+ return render_template("workspaces/workspaces.html", workspaces=workspace.list_all(shared.WS_HOME),
17
+ home=shared.WS_HOME, csrf=shared._csrf_token())
18
+
19
+
20
+ @app.route("/workspaces/open/<name>", methods=["POST"])
21
+ def workspace_open(name):
22
+ shared._require_csrf()
23
+ ws = workspace.load(shared.WS_HOME, name)
24
+ if ws is None:
25
+ flash(Markup(f'No workspace called <span class="mono">{escape(name)}</span>.'), "error")
26
+ return redirect(url_for("workspaces_page"))
27
+ shared.open_workspace(ws)
28
+ return redirect(url_for("index"))
29
+
30
+
31
+ @app.route("/workspaces/refresh", methods=["POST"])
32
+ def workspace_refresh():
33
+ """Throw the parse cache away and re-convert imported trees. The only way
34
+ a changed *foreign* run tree is noticed; a changed Harbor job is noticed
35
+ by its signature without this."""
36
+ shared._require_csrf()
37
+ ws = shared.STATE["workspace"]
38
+ if ws is None:
39
+ return redirect(url_for("index"))
40
+ shared.open_workspace(ws, refresh=True)
41
+ n = len(parse.find_job_dirs(shared.STATE["job_roots"]))
42
+ flash(Markup(f"Refreshed <span class=\"mono\">{escape(ws.label)}</span> — {n} run(s) "
43
+ "re-read; imported trees converted again."), "")
44
+ return redirect(request.referrer or url_for("index"))
45
+
46
+
47
+ @app.route("/workspaces/rename", methods=["POST"])
48
+ def workspace_rename():
49
+ """Rename the workspace a row names, open or not: its label, and its
50
+ directory moved to the label's slug (``workspace.rename``). The open one
51
+ is read from where it is now. Takes the form token: a rename moves a
52
+ directory."""
53
+ shared._require_csrf()
54
+ open_ws = shared.STATE["workspace"]
55
+ name = request.form.get("name") or (open_ws.name if open_ws is not None else "")
56
+ label = (request.form.get("label") or "").strip()
57
+ is_open = open_ws is not None and open_ws.name == name
58
+ ws = open_ws if is_open else workspace.load(shared.WS_HOME, name)
59
+ if ws is None:
60
+ flash(Markup(f'No workspace called <span class="mono">{escape(name)}</span>.'), "error")
61
+ elif label:
62
+ try:
63
+ workspace.rename(shared.WS_HOME, ws, label)
64
+ except OSError as exc:
65
+ flash(Markup(f'<span class="mono">{escape(ws.label)}</span> could not be renamed: {escape(exc.strerror or str(exc))}.'), "error")
66
+ return redirect(url_for("workspaces_page"))
67
+ if is_open:
68
+ shared.open_workspace(ws) # its imports moved with it: read from where it is now
69
+ return redirect(url_for("workspaces_page"))
70
+
71
+
72
+ @app.route("/workspaces/delete/<name>", methods=["POST"])
73
+ def workspace_delete(name):
74
+ """Remove a workspace - its cache and imports, never the job roots it
75
+ read. The open one closes first. Takes the form token."""
76
+ shared._require_csrf()
77
+ ws = shared.STATE["workspace"]
78
+ if ws is not None and ws.name == name:
79
+ shared.close_workspace()
80
+ if workspace.delete(shared.WS_HOME, name):
81
+ flash(Markup(f'Removed workspace <span class="mono">{escape(name)}</span>; its job roots are untouched.'), "")
82
+ else:
83
+ flash(Markup(f'No workspace called <span class="mono">{escape(name)}</span>.'), "error")
84
+ return redirect(url_for("workspaces_page"))
nullius/__init__.py ADDED
@@ -0,0 +1,32 @@
1
+ """Nullius's own backend: everything the application does with a run.
2
+
3
+ This package owns the whole path from a directory of benchmark output to a
4
+ rendered page, one package per purpose:
5
+
6
+ - `runs/` — reading one Harbor job tree: `parse`, `classify`, `live` (the
7
+ feed a running job appends), `artifacts`, `feasibility`, `launches`,
8
+ `verifier` (a verifier's report, read as its catalogue entry describes it),
9
+ and the `cli`.
10
+ - `traces/` — what a trial recorded: `trace`, `trajectory`,
11
+ `conversation`, `charts`, `behavior`, `otel`.
12
+ - `compare/` — runs against runs.
13
+ - `catalogue/` — benchmarks and datasets: a folder per benchmark under
14
+ `benchmarks/` (`benchmark.json`, its datasets and report block nested in it,
15
+ and its `standard.json` when it runs on the Nullius standard).
16
+ - `recipes/` — `launch`, the vocabulary the run builder composes.
17
+ - `providers` — where a run's model calls go, and the builder's steps.
18
+ - `harbor/` — `source`: the Harbor a recipe pins, built here, kept, scanned.
19
+ - `importers/` — run trees other harnesses write: `atif`, `otel`, and the
20
+ `cli`.
21
+ - `judge/` — judgement: `questions`, `state`, `client`, `store`, `batch`,
22
+ `reconcile`, `lanes`.
23
+ - `execution`, `hosts`, `gitrefs` — running a recipe here or on a host, the
24
+ hosts and their keys, a repository's branches and tags.
25
+ - `workspace`, `settings`, `redact` — cross-cutting, at the top.
26
+
27
+ **These are Nullius's own implementations**, each with the tests that pin it;
28
+ nothing in this application installs another package to run.
29
+
30
+ One name, one implementation: `app/tests/test_dependency_surface.py` counts
31
+ providers so a local package cannot shadow an installed distribution.
32
+ """
nullius/__main__.py ADDED
@@ -0,0 +1,5 @@
1
+ """``python -m nullius`` has the same commands as the installed executable."""
2
+ from nullius.cli import main
3
+
4
+ if __name__ == "__main__":
5
+ raise SystemExit(main())
File without changes
@@ -0,0 +1,222 @@
1
+ {
2
+ "schema": "nullius.agent/v1",
3
+ "key": "codex",
4
+ "label": "Codex",
5
+ "summary": "OpenAI's coding agent.",
6
+ "upstream": {
7
+ "license": "Apache-2.0"
8
+ },
9
+ "start": {
10
+ "acp": [
11
+ "{install}/node_modules/.bin/codex-acp"
12
+ ]
13
+ },
14
+ "model": {
15
+ "apis": [
16
+ "responses"
17
+ ],
18
+ "env": {
19
+ "CODEX_HOME": "{home}/.codex",
20
+ "NULLIUS_RELAY_KEY": "{key}"
21
+ },
22
+ "files": {
23
+ "{home}/.codex/config.toml": "config.toml"
24
+ }
25
+ },
26
+ "caches": [
27
+ "{home}/.codex/.tmp"
28
+ ],
29
+ "options": {
30
+ "reasoning_effort": {
31
+ "summary": "How much reasoning the model uses before answering.",
32
+ "values": [
33
+ "minimal",
34
+ "low",
35
+ "medium",
36
+ "high"
37
+ ],
38
+ "default": "medium"
39
+ }
40
+ },
41
+ "default": "2.2.2",
42
+ "versions": {
43
+ "2.2.2": {
44
+ "upstream": {
45
+ "package": "@agentclientprotocol/codex-acp",
46
+ "core": "@openai/codex 0.160.1, a dependency of the adapter (^0.160.1), pinned in install"
47
+ },
48
+ "install": {
49
+ "npm": [
50
+ "@agentclientprotocol/codex-acp@2.2.2",
51
+ "@openai/codex@0.160.1"
52
+ ]
53
+ },
54
+ "env": {
55
+ "INITIAL_AGENT_MODE": "agent-full-access"
56
+ },
57
+ "fidelity": {
58
+ "command": [
59
+ "{install}/node_modules/.bin/codex",
60
+ "exec",
61
+ "--skip-git-repo-check",
62
+ "--json",
63
+ "--",
64
+ "{prompt}"
65
+ ]
66
+ },
67
+ "proven": {
68
+ "agent.task/v1": {
69
+ "passed": "3/3",
70
+ "on": "2026-11-02",
71
+ "model": "openai/gpt-5-mini",
72
+ "provider": "openrouter",
73
+ "version": "2.2.2",
74
+ "kit_version": 1,
75
+ "backend": "docker",
76
+ "job": "2026-11-02__13-16-33__kit__agent-task-v1__codex-2.2.2"
77
+ },
78
+ "agent.workspace/v1": {
79
+ "passed": "2/2",
80
+ "on": "2026-11-02",
81
+ "model": "openai/gpt-5-mini",
82
+ "provider": "openrouter",
83
+ "version": "2.2.2",
84
+ "kit_version": 1,
85
+ "backend": "docker",
86
+ "job": "2026-11-02__13-17-34__kit__agent-workspace-v1__codex-2.2.2"
87
+ },
88
+ "agent.conversation/v1": {
89
+ "passed": "2/2",
90
+ "on": "2026-11-02",
91
+ "model": "openai/gpt-5-mini",
92
+ "provider": "openrouter",
93
+ "version": "2.2.2",
94
+ "kit_version": 1,
95
+ "backend": "docker",
96
+ "job": "2026-11-02__13-20-14__kit__agent-conversation-v1__codex-2.2.2"
97
+ },
98
+ "fidelity": {
99
+ "verdict": "same",
100
+ "on": "2026-10-12",
101
+ "model": "gpt-5-mini",
102
+ "provider": "openai",
103
+ "version": "2.2.2",
104
+ "job": "2026-10-12__16-11-01__fidelity__codex-2.2.2",
105
+ "passes": 2,
106
+ "differs": []
107
+ },
108
+ "agent.tools/v1": {
109
+ "passed": "2/2",
110
+ "on": "2026-11-02",
111
+ "model": "openai/gpt-5-mini",
112
+ "provider": "openrouter",
113
+ "version": "2.2.2",
114
+ "kit_version": 2,
115
+ "backend": "docker",
116
+ "job": "2026-11-02__13-22-34__kit__agent-tools-v1__codex-2.2.2",
117
+ "skills": {
118
+ "decoys": {
119
+ "passed": "1/1",
120
+ "ok": true
121
+ },
122
+ "chain": {
123
+ "passed": "1/1",
124
+ "ok": true
125
+ },
126
+ "retry": {
127
+ "passed": "1/1",
128
+ "ok": true
129
+ }
130
+ }
131
+ },
132
+ "agent.search/v1": {
133
+ "passed": "3/3",
134
+ "on": "2026-11-12",
135
+ "model": "meta/muse-spark-1.3-contributor",
136
+ "provider": "openrouter",
137
+ "version": "2.2.2",
138
+ "kit_version": 1,
139
+ "backend": "docker",
140
+ "job": "2026-11-12__10-24-26__kit__agent-search-v1__codex-2.2.2"
141
+ }
142
+ }
143
+ },
144
+ "0.16.0": {
145
+ "install": {
146
+ "npm": [
147
+ "@zed-industries/codex-acp@0.16.0"
148
+ ]
149
+ },
150
+ "proven": {
151
+ "agent.task/v1": {
152
+ "passed": "3/3",
153
+ "on": "2026-11-02",
154
+ "model": "openai/gpt-5-mini",
155
+ "provider": "openrouter",
156
+ "version": "0.16.0",
157
+ "kit_version": 1,
158
+ "backend": "docker",
159
+ "job": "2026-11-02__13-20-41__kit__agent-task-v1__codex-0.16.0"
160
+ },
161
+ "agent.workspace/v1": {
162
+ "passed": "2/2",
163
+ "on": "2026-11-02",
164
+ "model": "openai/gpt-5-mini",
165
+ "provider": "openrouter",
166
+ "version": "0.16.0",
167
+ "kit_version": 1,
168
+ "backend": "docker",
169
+ "job": "2026-11-02__13-21-58__kit__agent-workspace-v1__codex-0.16.0"
170
+ },
171
+ "agent.conversation/v1": {
172
+ "passed": "2/2",
173
+ "on": "2026-11-02",
174
+ "model": "openai/gpt-5-mini",
175
+ "provider": "openrouter",
176
+ "version": "0.16.0",
177
+ "kit_version": 1,
178
+ "backend": "docker",
179
+ "job": "2026-11-02__13-24-39__kit__agent-conversation-v1__codex-0.16.0"
180
+ },
181
+ "agent.tools/v1": {
182
+ "passed": "2/2",
183
+ "on": "2026-11-02",
184
+ "model": "openai/gpt-5-mini",
185
+ "provider": "openrouter",
186
+ "version": "0.16.0",
187
+ "kit_version": 2,
188
+ "backend": "docker",
189
+ "job": "2026-11-02__13-22-38__kit__agent-tools-v1__codex-0.16.0",
190
+ "skills": {
191
+ "decoys": {
192
+ "passed": "1/1",
193
+ "ok": true
194
+ },
195
+ "chain": {
196
+ "passed": "1/1",
197
+ "ok": true
198
+ },
199
+ "retry": {
200
+ "passed": "1/1",
201
+ "ok": true
202
+ }
203
+ }
204
+ },
205
+ "agent.search/v1": {
206
+ "passed": "3/3",
207
+ "on": "2026-11-12",
208
+ "model": "meta/muse-spark-1.3-contributor",
209
+ "provider": "openrouter",
210
+ "version": "0.16.0",
211
+ "kit_version": 1,
212
+ "backend": "docker",
213
+ "job": "2026-11-12__11-08-33__kit__agent-search-v1__codex-0.16.0"
214
+ }
215
+ },
216
+ "upstream": {
217
+ "package": "@zed-industries/codex-acp",
218
+ "core": "openai/codex rust-v0.137.0 (f221438b691b8f749d98f22077c93ebe01923fbe), compiled into the adapter's binary"
219
+ }
220
+ }
221
+ }
222
+ }
@@ -0,0 +1,25 @@
1
+ # Codex's configuration for a Nullius run, written into the home it is given.
2
+ # Its model calls go to the relay, a provider of its own: Codex's built-in
3
+ # OpenAI provider does not follow OPENAI_BASE_URL and would call OpenAI
4
+ # directly. The relay holds the key. Codex runs without its own sandbox
5
+ # (which needs bubblewrap) and without asking: the run is the isolation, and
6
+ # no one is there to answer. The reasoning effort is always written (the
7
+ # option reasoning_effort, its default Codex's own): left out, codex-acp and
8
+ # Codex's CLI would each fill it in their own way. Its plugins are off: with
9
+ # them on, Codex 0.160 fetches OpenAI's curated plugins repository (about
10
+ # 103 MB) into .codex/.tmp/plugins at every start - a download per unit, into
11
+ # every unit's home - and a run gives it no plugin to use.
12
+ model = "{model}"
13
+ model_reasoning_effort = "{opt_reasoning_effort}"
14
+ model_provider = "nullius"
15
+ approval_policy = "never"
16
+ sandbox_mode = "danger-full-access"
17
+
18
+ [features]
19
+ plugins = false
20
+
21
+ [model_providers.nullius]
22
+ name = "Nullius relay"
23
+ base_url = "{base_url}"
24
+ env_key = "NULLIUS_RELAY_KEY"
25
+ wire_api = "responses"