nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,78 @@
1
+ """Read-only launch vocabulary for Harbor recipes; no harness imports.
2
+
3
+ The catalogue owns benchmark facts. This module owns only launch choices and
4
+ parameter metadata: the agents the builder offers by name and the parameters
5
+ it knows for them. The browser compiles recipes locally; this module never
6
+ executes commands, and nothing here imports a harness or a runtime.
7
+
8
+ Every name here has to match what Harbor accepts - an agent's import path, the
9
+ ``--ak`` a parameter is sent as - and this repository has no Harbor source to
10
+ check them against. What is checked here is the other half:
11
+ ``app/ui_tests/test_command_builder.py`` pins the recipes this module
12
+ produces, so a change made *here* cannot silently alter what the builder emits.
13
+ """
14
+ from dataclasses import asdict
15
+
16
+ # `kind` groups the builder's agent list: the reference agent a benchmark
17
+ # brings and the controls that bracket it. Harbor's own agents join them once
18
+ # the pinned Harbor is scanned (`builtin`); an agent under evaluation is
19
+ # typed per recipe, as an import path.
20
+ AGENTS = [
21
+ # A benchmark's own reference agent: lives in Harbor's source tree, not its package,
22
+ # so it needs the adapter directory on PYTHONPATH (see running.md). The path is
23
+ # anchored on the run target's home, not on a sibling of a checkout: a run from
24
+ # the builder starts in its own run directory, on this machine or a host, and
25
+ # `../harbor-src` from there pointed nowhere. A recipe may name another.
26
+ {"id": "tau3-llm-agent", "label": "tau3-llm-agent (τ³ reference)", "arg": "tau3_llm_agent:Tau3LLMAgent",
27
+ "control": False, "benchmarks": ["tau3-bench"], "pythonpath": "~/harbor-src/adapters/tau3-bench", "kind": "reference",
28
+ "summary": "τ³'s own reference agent, from Harbor's source tree."},
29
+ {"id": "oracle", "label": "oracle", "arg": "oracle", "control": True, "kind": "control",
30
+ "summary": "Applies the task's reference solution; never calls a model. A ceiling, never ranked."},
31
+ {"id": "nop", "label": "nop", "arg": "nop", "control": True, "kind": "control",
32
+ "summary": "Does nothing. The floor a verifier must reject."},
33
+ ]
34
+
35
+
36
+ def parameter(key, label, group, kind, default=None, *, help="", **extra):
37
+ return dict(key=key, label=label, group=group, type=kind, default=default, help=help, **extra)
38
+
39
+
40
+ PARAMETERS = [
41
+ # Sent to the agent its group names: the τ³ reference agent's tool timeout,
42
+ # emitted as `--ak tool_timeout_sec=`.
43
+ parameter("mcp_timeout_s", "Tool call timeout · seconds", "tau3-llm-agent", "integer", 300, min=0,
44
+ help="How long one MCP tool call may take."),
45
+ ]
46
+
47
+
48
+ # Local launch paths are examples from this repository, not claims about disk state.
49
+ LAUNCH = {
50
+ "swe-bench-verified": {"local": "../harbor-datasets/swe-bench-verified"},
51
+ "terminal-bench": {"local": "../harbor-datasets/terminal-bench"},
52
+ # A generator: the program `uv run` starts in a checkout of the repository that holds it, writing to --output-dir.
53
+ "tau3-bench": {"local": "../harbor-datasets/tau3-bench",
54
+ "generate": {"program": "tau3-bench", "checkout": "tau2-bench checkout · dev/tau3 branch", "example": "../tau2-bench"}},
55
+ }
56
+
57
+
58
+ def payload(cat, jobs, workspace_name=""):
59
+ from nullius import providers
60
+ from nullius.standard import recipe as standard
61
+ models = sorted({j.model_name for j in jobs if j.model_name})
62
+ dated = [j for j in jobs if j.model_name and j.started_at]
63
+ latest = max(dated, key=lambda j: j.started_at.timestamp()).model_name if dated else ""
64
+ return {
65
+ "version": 1, "workspace": workspace_name or "unassigned", "agents": AGENTS,
66
+ # Harbor and the Nullius standard: the page offers choices, the core and standard/recipe.py compile them.
67
+ "runners": [standard.RUNNER, providers.HARBOR],
68
+ # why a saved recipe of a runner no longer offered does not open (a benchmark's own scripts: its Original setup)
69
+ "runnerGone": providers.GONE,
70
+ # the standard's benchmarks, agents and capabilities, and why a pair does not fit (standard/recipe.py, choices)
71
+ "standard": standard.choices(),
72
+ "parameters": PARAMETERS, "benchmarks": [asdict(b) | {"label": b.label} for b in cat.benchmarks],
73
+ "datasets": [asdict(d) | LAUNCH.get(d.key, {}) for d in cat.datasets],
74
+ "models": models, "defaultModel": latest,
75
+ "seenLocal": sorted({j.dataset_path for j in jobs if j.dataset_path}),
76
+ "seenRegistry": sorted({j.dataset for j in jobs if "@" in j.dataset}),
77
+ }
78
+
nullius/redact.py ADDED
@@ -0,0 +1,72 @@
1
+ """Secret redaction.
2
+
3
+ Job directories are arbitrary text from wherever an exception was raised, and
4
+ this app renders that text into a browser. Never surface raw text from a job
5
+ directory without passing it through :func:`redact` first.
6
+
7
+ ``app/tests/test_redact.py`` pins what the patterns below catch and what they
8
+ leave alone.
9
+
10
+ Two properties matter:
11
+
12
+ * **Anchored.** ``sk-`` must start at a token boundary, or the pattern fires
13
+ inside base64 payloads and eats the rest of a diff.
14
+ * **Separator-agnostic.** Credentials arrive as ``KEY=v`` from shell commands
15
+ *and* as ``"key": "v"`` from JSON — ``trajectory.py`` renders tool arguments
16
+ with ``json.dumps``, so the JSON form is the common one here.
17
+ """
18
+
19
+ from __future__ import annotations
20
+
21
+ import re
22
+
23
+ _PATTERNS = (
24
+ # OpenAI-style keys. The lookbehind is the whole point: without it this
25
+ # matches "sk-" appearing mid-base64 and swallows everything after it.
26
+ re.compile(r"(?<![A-Za-z0-9_-])sk-[A-Za-z0-9_-]{8,}"),
27
+ # Bearer tokens in headers. Keeps the word "bearer" so the line still reads.
28
+ re.compile(r"(?i)\b(bearer)\s+[A-Za-z0-9._~+/=-]{8,}"),
29
+ # KEY=value and "key": "value", quoted or not, escaped or not. Longer
30
+ # alternatives lead: `access_token` must win before `token` can match it.
31
+ # The name may carry a prefix joined by `_` or `-` (GH_TOKEN,
32
+ # CURSOR_API_KEY, AWS_SECRET_ACCESS_KEY), which `\b` would refuse since `_`
33
+ # is a word character; a letter or digit before it still does not count.
34
+ re.compile(
35
+ r"(?i)(?<![A-Za-z0-9])(api[_-]?key|access[_-]?token|access[_-]?key|authorization|password|secret|token)"
36
+ r"(\s*[=:]\s*|\\?[\"']?\s*[=:]\s*\\?[\"']?)"
37
+ r"[A-Za-z0-9._~+/=-]{8,}"
38
+ ),
39
+ )
40
+
41
+ # Credentials that need no name beside them: their prefix says what they are
42
+ # (GitHub, Hugging Face, AWS access key ids, Google API keys, Slack).
43
+ _BARE = re.compile(
44
+ r"(?<![A-Za-z0-9_-])(?:gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|hf_[A-Za-z0-9]{20,}"
45
+ r"|AKIA[0-9A-Z]{16}|AIza[0-9A-Za-z_-]{30,}|xox[abeoprs]-[A-Za-z0-9-]{10,})"
46
+ )
47
+ # "Authorization: token X" and "Authorization: Basic X": the scheme word is
48
+ # kept, as with Bearer. Only after the header name, where "basic" is no word.
49
+ _SCHEME = re.compile(r"(?i)(authorization\\?[\"']?\s*[=:]\s*\\?[\"']?\s*(?:token|basic)\s+)[A-Za-z0-9._~+/=-]{8,}")
50
+ # user:password@ in a URL: the user stays, the password goes.
51
+ _USERINFO = re.compile(r"(?<=://)([^/\s:@\[\]]+):([^/\s@\[\]]+)@")
52
+
53
+ # Angle brackets would survive Jinja's autoescaping as &lt;…&gt;. Square
54
+ # brackets read the same in a browser as they do in a terminal.
55
+ _REPLACEMENT = "[REDACTED]"
56
+
57
+
58
+ def redact(text: str | None) -> str:
59
+ """Return *text* with anything credential-shaped replaced.
60
+
61
+ The key name survives; only the value goes. An error message that reads
62
+ "auth failed for [REDACTED]" is still diagnostic, and over-redaction
63
+ destroys exactly the text someone opened the page to read.
64
+ """
65
+ if not text:
66
+ return ""
67
+ out = _PATTERNS[0].sub(_REPLACEMENT, text)
68
+ out = _BARE.sub(_REPLACEMENT, out)
69
+ out = _USERINFO.sub(lambda m: m.group(0) if m.group(2) == _REPLACEMENT else f"{m.group(1)}:{_REPLACEMENT}@", out)
70
+ out = _SCHEME.sub(rf"\1{_REPLACEMENT}", out)
71
+ out = _PATTERNS[1].sub(rf"\1 {_REPLACEMENT}", out)
72
+ return _PATTERNS[2].sub(rf"\1\2{_REPLACEMENT}", out)
File without changes
@@ -0,0 +1,67 @@
1
+ """Bounded, redacted views of named trial records. No writes or arbitrary paths."""
2
+ from __future__ import annotations
3
+
4
+ import os
5
+
6
+ from nullius.redact import redact
7
+
8
+ FILES = {
9
+ "result": ("Result record", "result.json"),
10
+ "config": ("Run configuration", "config.json"),
11
+ "exception": ("Exception", "exception.txt"),
12
+ "trial-log": ("Trial log", "trial.log"),
13
+ "gateway": ("Gateway log", "agent/gateway.log"),
14
+ "reward": ("Recorded reward", "verifier/reward.txt"),
15
+ "reward-json": ("Recorded reward (rewardkit)", "verifier/reward.json"),
16
+ "reward-details": ("Reward breakdown (rewardkit)", "verifier/reward-details.json"),
17
+ # Harbor's record of what it copied out of the container after the run —
18
+ # one entry per requested path with its status (`ok`, `failed`, `empty`,
19
+ # `skipped` — Harbor's `ArtifactManifestEntry`). The last of the nine
20
+ # files a trial writes to reach a page: the others were already
21
+ # here, or are read at parse.
22
+ "manifest": ("Artifact manifest", "artifacts/manifest.json"),
23
+ }
24
+ CHUNK_SIZE = 8_000
25
+ MAX_READ = 16 * 1024 * 1024
26
+
27
+
28
+ def path_for(trial_dir, key):
29
+ if key not in FILES:
30
+ raise LookupError(key)
31
+ base = os.path.realpath(trial_dir)
32
+ path = os.path.realpath(os.path.join(base, FILES[key][1]))
33
+ if os.path.commonpath([base, path]) != base:
34
+ raise LookupError(key)
35
+ return path
36
+
37
+
38
+ def listing(trial_dir):
39
+ records = []
40
+ for key, (label, filename) in FILES.items():
41
+ try:
42
+ path = path_for(trial_dir, key)
43
+ exists = os.path.isfile(path)
44
+ size = os.path.getsize(path) if exists else None
45
+ except (OSError, LookupError):
46
+ exists, size = False, None
47
+ records.append(dict(key=key, label=label, filename=filename, exists=exists, size=size))
48
+ return records
49
+
50
+
51
+ def read(trial_dir, key, page=1):
52
+ path = path_for(trial_dir, key)
53
+ with open(path, encoding="utf-8", errors="replace") as handle:
54
+ raw = handle.read(MAX_READ + 1)
55
+ # Redact before slicing so credentials cannot straddle visible chunks.
56
+ text = redact(raw[:MAX_READ])
57
+ pages = max(1, (len(text) + CHUNK_SIZE - 1) // CHUNK_SIZE)
58
+ try:
59
+ page = int(page)
60
+ except (TypeError, ValueError):
61
+ page = 1
62
+ page = min(pages, max(1, page))
63
+ start = (page - 1) * CHUNK_SIZE
64
+ return dict(key=key, label=FILES[key][0], filename=FILES[key][1], page=page,
65
+ pages=pages, text=text[start:start + CHUNK_SIZE], empty=not text,
66
+ truncated=len(raw) > MAX_READ, characters=len(text), start=start,
67
+ end=min(start + CHUNK_SIZE, len(text)))
@@ -0,0 +1,216 @@
1
+ """Outcome classification.
2
+
3
+ The whole point of the app: separate *infrastructure* failures (rate limits,
4
+ setup crashes) from genuine *agent* outcomes (solved / wrong patch / no patch),
5
+ so the solve rate reflects the agent and not the harness.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ # Outcome labels, ordered from best to worst for display.
11
+ RESOLVED = "resolved"
12
+ # A benchmark that gives a trial a score and no pass or fail (a token F1 over a
13
+ # conversation's questions): the trial was answered and measured, neither
14
+ # resolved nor failed. Its score is the measure; the run has no solve rate.
15
+ SCORED = "scored"
16
+ PATCH_FAILED_TESTS = "patch_failed_tests"
17
+ # The benchmark-neutral failure: the verifier said no, and the benchmark
18
+ # reports nothing finer than the score. What a failed trial of a dataset
19
+ # whose verifier reports no patch detail, or that the app knows only by its
20
+ # reward, classifies as.
21
+ FAILED_VERIFICATION = "failed_verification"
22
+ PATCH_NOT_APPLIED = "patch_not_applied"
23
+ NO_PATCH = "no_patch"
24
+ RATE_LIMITED = "rate_limited"
25
+ API_ERROR = "api_error"
26
+ SETUP_ERROR = "setup_error"
27
+ # The run's own spending cap refused a model call the unit needed (the
28
+ # standard's relay, ``standard.ledger``): the unit was cut off. Nothing
29
+ # failed, and no attempt was had: the money ran out, by the user's own word.
30
+ SPENDING_CAP = "spending_cap"
31
+ CANCELLED = "cancelled"
32
+ AGENT_ERROR = "agent_error"
33
+ UNKNOWN = "unknown"
34
+
35
+ # Outcomes excluded from the adjusted rate: infrastructure failures or
36
+ # interruptions imposed by the run, not the agent's ability.
37
+ INFRA_OUTCOMES = {RATE_LIMITED, API_ERROR, SETUP_ERROR, SPENDING_CAP, CANCELLED}
38
+
39
+ ORDER = [
40
+ RESOLVED,
41
+ SCORED,
42
+ PATCH_FAILED_TESTS,
43
+ FAILED_VERIFICATION,
44
+ PATCH_NOT_APPLIED,
45
+ NO_PATCH,
46
+ AGENT_ERROR,
47
+ RATE_LIMITED,
48
+ API_ERROR,
49
+ SETUP_ERROR,
50
+ CANCELLED,
51
+ SPENDING_CAP,
52
+ UNKNOWN,
53
+ ]
54
+
55
+ # Harbor names its provider failures Api*, all descending from ApiError, so the
56
+ # prefix follows their hierarchy instead of guessing at it — and a class they
57
+ # add later is classified correctly without us noticing it exists.
58
+ #
59
+ # The two below are the exceptions to that rule: they are provider or config
60
+ # failures that don't carry the prefix. Everything Harbor ships is pinned by a
61
+ # test, so this list cannot quietly fall behind theirs.
62
+ _API_PREFIX = "api"
63
+ _INFRA_EXCEPTIONS = {
64
+ "agentauthenticationerror", # bad or missing credentials — config, not skill
65
+ "modelnotfounderror", # the model name is wrong; the agent never ran
66
+ "networkconnectionerror", # DNS, refused connection, TLS — transport, not skill
67
+ "unknownapierror", # Harbor's own "some provider error"; an ApiError without the prefix
68
+ }
69
+ # What a unit the run's spending cap cut off carries (``standard.ledger.ERROR``): recorded by the relay that
70
+ # refused the call, never inferred.
71
+ _CAP_EXCEPTION = "spendingcaperror"
72
+
73
+ # What Harbor's agents raise when the agent itself fails — its budget spent,
74
+ # its own context filled, its own output truncated, a refusal, its process
75
+ # exiting non-zero. Each is the agent's behaviour under conditions it was
76
+ # given, so each is a result and belongs in the denominator, charged to the
77
+ # agent. Deliberately *not* infra; move one only with a reason written down.
78
+ #
79
+ # The list is explicit and short on purpose. An exception on none of these
80
+ # lists is UNKNOWN: still in the denominator, charged to nobody. The
81
+ # alternative — everything unmatched is the agent's fault — made a crash in
82
+ # the harness, the verifier or this reader read as poor coding. The trial
83
+ # page shows the recorded type beside the label, so a reader who meets a new
84
+ # one can place it here.
85
+ _AGENT_EXCEPTIONS = {
86
+ "agenttimeouterror", # the agent spent its budget — that is a result
87
+ "unitspendingcaperror", # the unit spent its own spending cap (the standard's relay): the same, in money
88
+ "agentsafetyrefusalerror", # the agent (or its provider) chose to refuse
89
+ "contextwindowexceedederror", # the agent filled its own context
90
+ "contextlengthexceedederror", # same, from Harbor's LLM layer
91
+ "outputlengthexceedederror", # the agent's reply hit max_tokens
92
+ "outputtokenexceedederror", # same, seen from an installed agent's output
93
+ "nonzeroagentexitcodeerror", # the agent's process died and Harbor found no finer reason
94
+ }
95
+
96
+ LABELS = {
97
+ RESOLVED: "Resolved",
98
+ SCORED: "Scored",
99
+ PATCH_FAILED_TESTS: "Patch failed tests",
100
+ FAILED_VERIFICATION: "Failed verification",
101
+ PATCH_NOT_APPLIED: "Patch did not apply",
102
+ NO_PATCH: "No patch produced",
103
+ AGENT_ERROR: "Agent error",
104
+ RATE_LIMITED: "Rate limited",
105
+ API_ERROR: "API error",
106
+ SETUP_ERROR: "Setup error",
107
+ SPENDING_CAP: "Spending cap",
108
+ CANCELLED: "Run stopped",
109
+ UNKNOWN: "Unknown",
110
+ }
111
+
112
+
113
+ # One-line meaning of each outcome, for the glossary the UI renders. Lives here
114
+ # rather than in a template so the wording follows the classifier that produces
115
+ # it — change the rule below, change the sentence with it.
116
+ DESCRIPTIONS = {
117
+ RESOLVED: "The patch applied and the verifier's tests all passed. A genuine solve.",
118
+ SCORED: (
119
+ "The benchmark gave the trial a score and no pass or fail - a token F1 over a "
120
+ "conversation's questions, partial credit. Answered and measured, neither "
121
+ "solved nor failed: the score is the result, and the run has no solve rate."
122
+ ),
123
+ PATCH_FAILED_TESTS: (
124
+ "The agent produced a patch that applied cleanly, but the tests still failed. "
125
+ "A real attempt that got the answer wrong — the most informative failure."
126
+ ),
127
+ FAILED_VERIFICATION: (
128
+ "The verifier ran and scored the attempt below success, on a benchmark that "
129
+ "reports no patch detail, or that the app knows only "
130
+ "by its reward. A real attempt that didn't pass; counted against the agent."
131
+ ),
132
+ PATCH_NOT_APPLIED: (
133
+ "A patch was produced but wouldn't apply to the repository — usually malformed "
134
+ "context lines or edits against the wrong revision."
135
+ ),
136
+ NO_PATCH: "The agent finished without producing a patch at all.",
137
+ AGENT_ERROR: (
138
+ "The agent itself failed — it timed out, filled its context, truncated its "
139
+ "own output, refused, spent its own spending cap, or its process exited non-zero. Counted against the "
140
+ "agent, because each of those is a result rather than an interruption."
141
+ ),
142
+ RATE_LIMITED: (
143
+ "The model API returned a rate-limit error (HTTP 429) and the run died. "
144
+ "Infrastructure, not capability — excluded from the adjusted rate."
145
+ ),
146
+ API_ERROR: (
147
+ "The model API failed some other way — overloaded, unreachable, out of "
148
+ "quota, or the key was rejected. The agent never got a fair attempt, so "
149
+ "this is infrastructure too and is excluded from the adjusted rate."
150
+ ),
151
+ SETUP_ERROR: (
152
+ "The environment or image failed to build or start, so the agent never ran. "
153
+ "Infrastructure, not capability — excluded from the adjusted rate."
154
+ ),
155
+ SPENDING_CAP: (
156
+ "The run reached its spending cap: its relay refused a model call this "
157
+ "trial needed, so the trial was cut off. Nothing failed and no attempt "
158
+ "was had, so it is excluded from the adjusted rate."
159
+ ),
160
+ CANCELLED: (
161
+ "The run was stopped while this trial was in progress. Its attempt was interrupted, "
162
+ "so it is excluded from the adjusted rate."
163
+ ),
164
+ UNKNOWN: (
165
+ "No verifier report and no exception, or an exception this reader does not "
166
+ "recognise — a crash in the harness, the verifier or the reader itself. The "
167
+ "outcome could not be determined; it stays in the denominator, charged to "
168
+ "nobody, and the recorded type is shown so the classifier can be extended."
169
+ ),
170
+ }
171
+
172
+
173
+ def from_verdict(exception_type, verdict):
174
+ """Return an outcome label from the exception type and a `Verdict`.
175
+
176
+ An exception outranks the verifier: a trial that crashed did not earn
177
+ whatever the verifier managed to say about it. Which side it is charged
178
+ to comes from the lists above; an exception on none of them is `UNKNOWN`.
179
+
180
+ The patch-shaped labels are reachable only for a benchmark whose reader
181
+ fills `produced` / `usable`. A verdict that failed without that detail is
182
+ `FAILED_VERIFICATION` — the verifier said no, and this benchmark has
183
+ nothing further to say about why — rather than borrowing a label about
184
+ patches that would not be true.
185
+ """
186
+ if exception_type:
187
+ et = exception_type.lower().rpartition(".")[2] # tolerate dotted paths
188
+ if et == "cancellederror":
189
+ return CANCELLED
190
+ if et == _CAP_EXCEPTION:
191
+ return SPENDING_CAP
192
+ # Rate limiting first: it is an Api* class too, but common enough to be
193
+ # worth seeing on its own rather than folded into the generic bucket.
194
+ if "ratelimit" in et or "rate_limit" in et or "429" in et:
195
+ return RATE_LIMITED
196
+ if "setup" in et or "environment" in et or "build" in et:
197
+ return SETUP_ERROR
198
+ if et.startswith(_API_PREFIX) or et in _INFRA_EXCEPTIONS:
199
+ return API_ERROR
200
+ if et in _AGENT_EXCEPTIONS:
201
+ return AGENT_ERROR
202
+ # Neither side claims it. Not the agent's fault by default: the
203
+ # trial stays counted and nobody is charged.
204
+ return UNKNOWN
205
+
206
+ if verdict is None or verdict.succeeded is None:
207
+ return UNKNOWN
208
+ if verdict.succeeded:
209
+ return RESOLVED
210
+ if verdict.produced is None:
211
+ return FAILED_VERIFICATION
212
+ if not verdict.produced:
213
+ return NO_PATCH
214
+ if verdict.usable is False:
215
+ return PATCH_NOT_APPLIED
216
+ return PATCH_FAILED_TESTS