nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,551 @@
1
+ """A run written as a job in Harbor's shape, so every page reads it unchanged.
2
+
3
+ ``<jobs>/<job>/``: ``config.json`` (what was run: the benchmark, the agent or
4
+ capability, the model), ``result.json`` (the counts), and one folder per unit -
5
+ a trial - holding:
6
+
7
+ - ``result.json``: the unit, the agent, the timings, the verdict
8
+ (``verifier_result.rewards``) or the exception that ended it, the tokens and
9
+ the cost the relay recorded;
10
+ - ``agent/trajectory.json``: ATIF, built from what Nullius saw - the session's
11
+ events over ACP (the prompt, the agent's messages and thoughts, each tool
12
+ call with its result) or, for a capability, each request it was sent and its
13
+ reply - with the model calls' tokens and cost from the relay;
14
+ - ``agent/trace.json``: the same run as a timeline, ``harbor-trace-v1``: each
15
+ model call as the relay timed it, each tool call as Nullius saw it start and
16
+ end, under the agent's turns. What the agent does inside is not seen;
17
+ - ``agent/calls.jsonl``: the relay's record of each model call, and
18
+ ``agent/gateway.log``: one line per call with its status, where the runs
19
+ pages count rate limits;
20
+ - ``verifier/reward.txt`` and ``verifier/score.json``: the driver's verdict
21
+ and its detail.
22
+ """
23
+ from __future__ import annotations
24
+
25
+ import json
26
+ import shutil
27
+ import os
28
+ import re
29
+ from datetime import datetime, timezone
30
+ from typing import Dict, List, Optional
31
+
32
+ from nullius.standard import ledger
33
+ from nullius.traces import otel, trace
34
+
35
+
36
+ def iso(t: Optional[float]) -> Optional[str]:
37
+ return datetime.fromtimestamp(t, timezone.utc).isoformat() if t else None
38
+
39
+
40
+ def _text(content) -> str:
41
+ """ACP content: a block, a list of blocks, or a tool call's content list."""
42
+ if content is None:
43
+ return ""
44
+ if isinstance(content, str):
45
+ return content
46
+ if isinstance(content, dict):
47
+ if content.get("type") == "text":
48
+ return content.get("text") or ""
49
+ if content.get("type") == "content":
50
+ return _text(content.get("content"))
51
+ if content.get("type") == "diff":
52
+ return f"diff {content.get('path')}\n{content.get('newText') or ''}"[:4000]
53
+ return json.dumps(content)[:4000]
54
+ if isinstance(content, list):
55
+ return "\n".join(_text(c) for c in content)
56
+ return str(content)
57
+
58
+
59
+ def _usage_total(calls: List[dict]) -> dict:
60
+ ok = [c for c in calls if c.get("status") == 200]
61
+ u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok)
62
+ costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))]
63
+ return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"),
64
+ "cost": sum(costs) if costs else None,
65
+ "cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None),
66
+ "n_calls": sum(1 for c in calls if not c.get("refused"))} # a call its cap refused was never sent
67
+
68
+
69
+ def _tool_clock(extra: dict, status: Optional[str], t: Optional[float]):
70
+ """When a tool call began to run (announced earlier, it may still wait for the
71
+ model's reply to end) and when it ended, for the trace."""
72
+ if status == "in_progress":
73
+ extra.setdefault("running_at", iso(t))
74
+ elif status in ("completed", "failed"):
75
+ extra["ended_at"] = iso(t)
76
+
77
+
78
+ def trajectory_from_events(events: List[dict], calls: List[dict], agent: dict, model: str, session: str) -> dict:
79
+ """ATIF from an ACP session's events: a user step per prompt, an agent
80
+ step per stretch of message, and one per tool call with its result as the
81
+ observation. The relay's calls give the totals."""
82
+ steps: List[dict] = []
83
+ tools = {} # toolCallId -> its step
84
+
85
+ def add(source, t, **kw):
86
+ steps.append(dict({"step_id": len(steps) + 1, "timestamp": iso(t), "source": source}, **kw))
87
+ return steps[-1]
88
+
89
+ open_msg = None
90
+ message_id = None
91
+ for e in events:
92
+ u, t = e.get("update") or {}, e.get("t")
93
+ kind = u.get("sessionUpdate")
94
+ if kind == "user_message_chunk":
95
+ open_msg = None
96
+ add("user", t, message=_text(u.get("content")))
97
+ elif kind in ("agent_message_chunk", "agent_thought_chunk"):
98
+ chunk_id = u.get("messageId") if kind == "agent_message_chunk" else None
99
+ if open_msg is None or (chunk_id and message_id and chunk_id != message_id):
100
+ open_msg = add("agent", t, message="", model_name=model)
101
+ message_id = None
102
+ message_id = chunk_id or message_id
103
+ if kind == "agent_message_chunk":
104
+ open_msg["message"] += _text(u.get("content"))
105
+ else:
106
+ open_msg["reasoning_content"] = open_msg.get("reasoning_content", "") + _text(u.get("content"))
107
+ elif kind == "tool_call":
108
+ open_msg = None
109
+ cid = u.get("toolCallId") or f"call-{len(tools) + 1}"
110
+ st = add("agent", t, message="", model_name=model,
111
+ tool_calls=[{"tool_call_id": cid, "function_name": u.get("title") or u.get("kind") or "tool",
112
+ "arguments": u.get("rawInput") if isinstance(u.get("rawInput"), dict) else {"input": u.get("rawInput")}}],
113
+ extra={"kind": u.get("kind"), "status": u.get("status")})
114
+ tools[cid] = st
115
+ _tool_clock(st["extra"], u.get("status"), t)
116
+ if u.get("content") or u.get("rawOutput") is not None:
117
+ st["observation"] = {"results": [{"source_call_id": cid, "content": _text(u.get("content")) or json.dumps(u.get("rawOutput"))[:8000]}]}
118
+ elif kind == "tool_call_update":
119
+ st = tools.get(u.get("toolCallId"))
120
+ if st is None:
121
+ continue
122
+ if u.get("status"):
123
+ st["extra"]["status"] = u["status"]
124
+ _tool_clock(st["extra"], u["status"], t)
125
+ out = _text(u.get("content")) if u.get("content") else (json.dumps(u.get("rawOutput"))[:8000] if u.get("rawOutput") is not None else "")
126
+ if out:
127
+ if u.get("status") == "failed" and not out.lower().startswith(("error", "failed")):
128
+ out = "Error: " + out
129
+ st["observation"] = {"results": [{"source_call_id": u.get("toolCallId"), "content": out}]}
130
+ elif kind == "permission":
131
+ open_msg = None
132
+ add("system", t, message=("granted" if u.get("granted") else "refused") + " permission: " +
133
+ _text((u.get("toolCall") or {}).get("title") or ""))
134
+ elif kind == "notice":
135
+ # an adapter's warning, sent apart from the agent's message: the client takes notices (acp)
136
+ open_msg = None
137
+ add("system", t, message=f"{u.get('severity') or 'notice'}: {u.get('title') or ''}"
138
+ + (f" - {u['description']}" if u.get("description") else ""))
139
+ elif kind == "plan":
140
+ add("agent", t, message="plan:\n" + "\n".join(f"- [{x.get('status')}] {x.get('content')}" for x in u.get("entries") or []),
141
+ model_name=model)
142
+ tot = _usage_total(calls)
143
+ return {"schema_version": "ATIF-v1.6", "session_id": session,
144
+ "agent": {"name": agent.get("key"), "version": str((agent.get("upstream") or {}).get("version") or ""), "model_name": model},
145
+ "steps": steps,
146
+ "final_metrics": {"total_prompt_tokens": tot["input"], "total_completion_tokens": tot["output"],
147
+ "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}}
148
+
149
+
150
+ REPLY_CHARS = 100_000 # a capability's reply in its trajectory: its memory and its trace, one request's
151
+
152
+
153
+ def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capability: dict, model: str, session: str) -> dict:
154
+ """ATIF for a capability: each request Nullius sent it (``add``, ``recall``)
155
+ as a tool call and its reply as the observation, whole up to ``REPLY_CHARS``
156
+ (a reply's ``trace`` is what the library did); then the benchmark's own
157
+ model calls, from the relay, as the reader's steps."""
158
+ steps = []
159
+ for i, x in enumerate(exchanges):
160
+ cid = f"{x['op']}-{i + 1}"
161
+ steps.append({"step_id": len(steps) + 1, "timestamp": iso(x.get("t0")), "source": "agent",
162
+ "message": "", "tool_calls": [{"tool_call_id": cid, "function_name": f"{capability['key']}.{x['op']}",
163
+ "arguments": x.get("request") or {}}],
164
+ "extra": {"ended_at": iso(x.get("t1"))},
165
+ "observation": {"results": [{"source_call_id": cid, "content": json.dumps(x.get("reply"))[:REPLY_CHARS]}]}})
166
+ for c in calls:
167
+ last = ((c.get("request") or {}).get("last") or [{}])[-1]
168
+ steps.append({"step_id": len(steps) + 1, "timestamp": iso(c.get("t0")), "source": "user", "message": last.get("content") or ""})
169
+ reply = c.get("reply") or {}
170
+ st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"),
171
+ "message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"}
172
+ if reply.get("usage"):
173
+ st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"],
174
+ "cached_tokens": reply["usage"]["cached"]}
175
+ if isinstance(c.get("cost_usd"), (int, float)):
176
+ st["metrics"]["cost_usd"] = c["cost_usd"]
177
+ steps.append(st)
178
+ tot = _usage_total(calls)
179
+ return {"schema_version": "ATIF-v1.6", "session_id": session,
180
+ "agent": {"name": capability.get("key"), "version": "", "model_name": model},
181
+ "steps": steps,
182
+ "final_metrics": {"total_prompt_tokens": tot["input"], "total_completion_tokens": tot["output"],
183
+ "total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}}
184
+
185
+
186
+ def _control(other: dict) -> dict:
187
+ """A control's run says so, and who wrote the control: the runs pages read
188
+ a standard job named ``nop`` or ``oracle`` as a control only by this."""
189
+ if other.get("_kind") != "control":
190
+ return {}
191
+ return {"control": {"name": other["key"], "source": other.get("source", ""), "from": other.get("from", "")}}
192
+
193
+
194
+ def _setup(other: dict) -> dict:
195
+ """A setup's run says which one it is, the model the upstream's code was
196
+ asked as and the one sent in its place, and whether its scores compare with
197
+ the paper's: only the original's do."""
198
+ if other.get("_kind") != "upstream":
199
+ return {}
200
+ return {"setup": {"name": other["key"], "asked": other.get("asked", ""), "sent": other.get("sent", ""),
201
+ "comparable_with_paper": bool(other.get("comparable"))}}
202
+
203
+
204
+ def _version(bench: dict) -> dict:
205
+ """A benchmark with versions says which one ran (``benchmark_version``), and the
206
+ ``key@version`` that names it; one without says nothing more than its key."""
207
+ v = bench.get("_version") if "@" in str(bench.get("_ref") or "") else None
208
+ return {"benchmark_version": v, "benchmark_ref": bench["_ref"]} if v else {}
209
+
210
+
211
+ def _write(path: str, text: str) -> None:
212
+ """A trial file written as a new regular file: whatever is at the path -
213
+ a link the agent planted in a folder it could write, above all - is
214
+ replaced, never written through (verifier/score.json -> ~/.bashrc)."""
215
+ if os.path.islink(path) or (os.path.lexists(path) and not os.path.isfile(path)):
216
+ if os.path.isdir(path) and not os.path.islink(path):
217
+ shutil.rmtree(path)
218
+ else:
219
+ os.unlink(path)
220
+ fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0), 0o644)
221
+ with os.fdopen(fd, "w") as f:
222
+ f.write(text)
223
+
224
+
225
+ def mean_score(results: List[dict]):
226
+ """The trials' mean score as the pages read it (parse.Job.mean_score): a
227
+ trial scored over several parts weighs as many. None when none was scored."""
228
+ scored = []
229
+ for r in results:
230
+ v = r.get("verifier_result") or {}
231
+ s = (v.get("rewards") or {}).get("score")
232
+ if isinstance(s, (int, float)) and not isinstance(s, bool):
233
+ scored.append((s, (v.get("parts") or {}).get("n") or 1))
234
+ return sum(s * n for s, n in scored) / sum(n for _, n in scored) if scored else None
235
+
236
+
237
+ def _ns(stamp) -> Optional[int]:
238
+ """A time as the trace's nanoseconds: a clock's seconds, or a step's ISO time."""
239
+ try:
240
+ if isinstance(stamp, str):
241
+ stamp = datetime.fromisoformat(stamp).timestamp()
242
+ return int(stamp * 1e9) if stamp else None
243
+ except (TypeError, ValueError, OverflowError, OSError):
244
+ return None
245
+
246
+
247
+ def _span(span_id: str, name: str, kind: str, source: str, start: int, end: Optional[int], **fields) -> dict:
248
+ """One span. No end: a moment when it says so (``measured``), else a call seen to start and never to end."""
249
+ moment = fields.pop("moment", False)
250
+ span = {"span_id": span_id, "name": name, "kind": kind, "start_ns": start, "source": source,
251
+ "timing_quality": "measured" if end is not None or moment else "unknown", "attributes": {}, "children": [], **fields}
252
+ if end is not None:
253
+ span["end_ns"], span["duration_ns"] = end, end - start
254
+ return span
255
+
256
+
257
+ def _short_title(title: str) -> str:
258
+ """A tool's title for its row and its filter chip: each path as its last part
259
+ (``Fetching file:///var/tmp/run/calc.py``: ``Fetching calc.py``). The whole title stays in the span's attributes."""
260
+ return re.sub(r"[^\s'\"]+", lambda word: word.group().rsplit("/", 1)[-1] or word.group(), title)
261
+
262
+
263
+ def _call_span(n: int, c: dict) -> Optional[dict]:
264
+ """A model call, as the relay timed it."""
265
+ start, end = _ns(c.get("t0")), _ns(c.get("t1"))
266
+ if start is None:
267
+ return None
268
+ reply = c.get("reply") or {}
269
+ usage = reply.get("usage") or {}
270
+ tokens = {name: usage[k] for k, name in (("input", "input_tokens"), ("output", "output_tokens"), ("cached", "cache_read_tokens"),
271
+ ("cache_write", "cache_write_tokens"), ("reasoning", "reasoning_tokens")) if usage.get(k)}
272
+ attributes = {k: c[k] for k in ("model", "model_asked", "path", "status", "error", "cost_usd", "cost_from") if c.get(k) not in (None, "")}
273
+ named = [str(x.get("name")) for x in reply.get("tool_calls") or [] if isinstance(x, dict) and x.get("name")]
274
+ if named:
275
+ attributes["tool_calls"] = ", ".join(named)
276
+ last = ((c.get("request") or {}).get("last") or [{}])[-1]
277
+ content = {"input": _text(last.get("content")) if isinstance(last, dict) else "", "output": _text(reply.get("text"))}
278
+ # named for its model: under the name ``llm_request`` the reader would call it by what it produced, as an agent's message reads
279
+ return _span(f"llm-{n}", str(c.get("model") or "model call"), "llm_request", "relay", start, end, tokens=tokens, content=content, attributes=attributes)
280
+
281
+
282
+ def _step_spans(n: int, st: dict, source: str) -> List[dict]:
283
+ """What a trajectory step adds to the relay's calls: its tool call, or what the agent said at that moment."""
284
+ at = _ns(st.get("timestamp"))
285
+ if at is None:
286
+ return []
287
+ extra = st.get("extra") or {}
288
+ if st.get("tool_calls"):
289
+ call = st["tool_calls"][0]
290
+ said = ((st.get("observation") or {}).get("results") or [{}])[0].get("content")
291
+ name = str(call.get("function_name") or "tool")
292
+ return [_span(f"tool-{n}", name, "tool", source, _ns(extra.get("running_at")) or at, _ns(extra.get("ended_at")),
293
+ content={"arguments": call.get("arguments"), "result": said},
294
+ attributes={"tool_name": _short_title(name), "title": name, **{k: extra[k] for k in ("kind", "status") if extra.get(k)}})]
295
+ spans = []
296
+ if st.get("reasoning_content"):
297
+ spans.append(_span(f"reasoning-{n}", "reasoning", "reasoning", source, at, None, moment=True, content={"reasoning": st["reasoning_content"]}))
298
+ if st.get("message"):
299
+ kind = "system" if st.get("source") == "system" else "output"
300
+ spans.append(_span(f"{kind}-{n}", kind, kind, source, at, None, moment=True, content={"output": st["message"]}))
301
+ return spans
302
+
303
+
304
+ def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tuple) -> Optional[dict]:
305
+ """``agent/trace.json``: the agent's window as one session, its turns in it
306
+ (an agent's only: a capability's and a setup's user steps are the benchmark's
307
+ own model calls), each holding the model calls and tool calls that began in
308
+ it. Nothing recorded, no trace: a control has none to show."""
309
+ agent = other.get("_kind") == "agent"
310
+ source = "acp" if agent else "driver"
311
+ steps = trajectory.get("steps") or []
312
+ spans = [s for s in (_call_span(i + 1, c) for i, c in enumerate(calls)) if s is not None]
313
+ turns = []
314
+ for i, st in enumerate(steps):
315
+ if agent and st.get("source") == "user":
316
+ at = _ns(st.get("timestamp"))
317
+ if at is not None:
318
+ turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True,
319
+ content={"input": st.get("message")}))
320
+ elif agent or st.get("tool_calls"):
321
+ spans.extend(_step_spans(i + 1, st, source))
322
+ if not spans and not turns:
323
+ return None
324
+ ends = lambda rows: [s["end_ns"] if "end_ns" in s else s["start_ns"] for s in rows]
325
+ first = min(s["start_ns"] for s in spans + turns)
326
+ begin, finish = produce if produce else (None, None)
327
+ begin = min(_ns(begin) or first, first)
328
+ finish = max([_ns(finish) or 0] + ends(spans + turns))
329
+ session = _span("session", str(other.get("key") or "agent"), "session", "nullius", begin, finish)
330
+ for span in sorted(spans, key=lambda s: s["start_ns"]):
331
+ holder = next((t for t in reversed(turns) if t["start_ns"] <= span["start_ns"]), session)
332
+ span["parent_span_id"] = holder["span_id"]
333
+ holder["children"].append(span)
334
+ for turn in turns:
335
+ turn["parent_span_id"] = "session"
336
+ if turn["children"]:
337
+ end = max(ends(turn["children"]))
338
+ turn.update(end_ns=end, duration_ns=end - turn["start_ns"])
339
+ session["children"] = sorted(session["children"] + turns, key=lambda s: s["start_ns"])
340
+ return {"schema_version": otel.SCHEMA, "producer": "app/nullius/standard/job.py", "trial_name": trajectory.get("session_id") or "",
341
+ "session_id": trajectory.get("session_id") or "", "agent": trajectory.get("agent") or {},
342
+ "clock": {"source": "this machine's clock, as the relay and the ACP client saw each call",
343
+ "note": "wall_ns is the agent's window; a tool call's time is what the agent reported"},
344
+ "spans": [session], "totals": trace.breakdown([session], finish - begin).as_totals()}
345
+
346
+
347
+ def _jsonl(path: str) -> List[dict]:
348
+ with open(path, errors="replace") as f:
349
+ return [json.loads(line) for line in f if line.strip()]
350
+
351
+
352
+ def trial_dirs(root: str) -> List[str]:
353
+ """The trials under a job, or under a folder of jobs. Two levels, no deeper: a trial holds its agent's own files."""
354
+ jobs = [root] if os.path.isfile(os.path.join(root, "config.json")) else [e.path for e in os.scandir(root) if e.is_dir(follow_symlinks=False)]
355
+ return sorted(e.path for j in jobs for e in os.scandir(j) if e.is_dir(follow_symlinks=False) and os.path.isfile(os.path.join(e.path, "result.json")))
356
+
357
+
358
+ def backfill_trace(trial_dir: str, write: bool = False) -> str:
359
+ """Give a standard trial written before traces its ``agent/trace.json``, from
360
+ the files it kept. Says which: ``full`` (it kept its ACP events: tool calls
361
+ with their times), ``partial`` (none kept: the relay's model calls, and any
362
+ tool call at its start, with no duration), ``empty`` (nothing recorded),
363
+ ``control``, ``has_trace`` (never replaced: a capture cannot be taken again),
364
+ or ``not_standard``."""
365
+ with open(os.path.join(trial_dir, "result.json")) as f:
366
+ res = json.load(f)
367
+ if "standard" not in res:
368
+ return "not_standard"
369
+ agent_dir = os.path.join(trial_dir, "agent")
370
+ target = os.path.join(agent_dir, "trace.json")
371
+ if os.path.lexists(target):
372
+ return "has_trace"
373
+ if "control" in res["standard"]:
374
+ return "control"
375
+ with open(os.path.join(os.path.dirname(trial_dir), "config.json")) as f:
376
+ kind = (json.load(f).get("standard") or {}).get("kind")
377
+ info = res.get("agent_info") or {}
378
+ other = {"key": info.get("name"), "_kind": kind, "upstream": {"version": info.get("version")}}
379
+ calls_at = os.path.join(agent_dir, "calls.jsonl")
380
+ calls = _jsonl(calls_at) if os.path.isfile(calls_at) else []
381
+ session = f"{res['standard'].get('job') or ''}/{res['standard'].get('unit') or ''}"
382
+ events_at, kept_at = os.path.join(agent_dir, "acp.jsonl"), os.path.join(agent_dir, "trajectory.json")
383
+ full = kind == "agent" and os.path.isfile(events_at)
384
+ if full:
385
+ trajectory = trajectory_from_events(_jsonl(events_at), calls, other, (info.get("model_info") or {}).get("name") or "", session)
386
+ elif os.path.isfile(kept_at):
387
+ with open(kept_at) as f:
388
+ trajectory = json.load(f)
389
+ else:
390
+ trajectory = {"session_id": session}
391
+ window = res.get("agent_execution") or {}
392
+ clock = lambda name: datetime.fromisoformat(window[name]).timestamp() if window.get(name) else None
393
+ timeline = trace_from_run(trajectory, calls, other, (clock("started_at"), clock("finished_at")))
394
+ if timeline is None:
395
+ return "empty"
396
+ if write:
397
+ fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o644)
398
+ with os.fdopen(fd, "w") as f:
399
+ json.dump(timeline, f, indent=1)
400
+ return "full" if full else "partial"
401
+
402
+
403
+ def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float,
404
+ produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict],
405
+ trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "",
406
+ image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict:
407
+ """Write one unit's trial and return its result. ``environment``: the
408
+ benchmark's environment the unit asked for (none: the base image), and
409
+ ``image``, the one it ran in on the container backend - names only.
410
+ ``setup_end``: when the unit's environment was ready and the agent began
411
+ (its setup, relay and container from ``t0``); ``checking``: the check's
412
+ start and end (its own step in the container, then the driver's score)."""
413
+ os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True)
414
+ os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True)
415
+ tot = _usage_total(calls)
416
+ # the pages read a task's name after its last slash: a unit's own slashes
417
+ # (a unit id like "set/12") would leave only its last part
418
+ res = {"task_name": f"{bench['key']}/{unit.replace('/', '-')}","trial_name": os.path.basename(trial_dir), "source": bench["key"],
419
+ "started_at": iso(t0), "finished_at": iso(t1),
420
+ "agent_info": {"name": other["key"], "version": str((other.get("upstream") or {}).get("version") or ""),
421
+ "model_info": {"name": model}},
422
+ "environment_setup": {"started_at": iso(t0), "finished_at": iso(setup_end)} if setup_end else None,
423
+ "agent_execution": {"started_at": iso(produce[0]), "finished_at": iso(produce[1])} if produce and produce[0] else None,
424
+ "verifier": {"started_at": iso(checking[0]), "finished_at": iso(checking[1])} if checking and checking[0] and checking[1] else None,
425
+ "agent_result": {"n_input_tokens": tot["input"], "n_output_tokens": tot["output"], "n_cache_tokens": tot["cached"],
426
+ "cost_usd": tot["cost"], "cost_from": tot["cost_from"], "n_model_calls": tot["n_calls"]},
427
+ "standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit,
428
+ "plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other),
429
+ **({"environment": environment} if environment else {}), **({"image": image} if image else {})}}
430
+ if score is not None:
431
+ value = float(score.get("reward") or 0.0)
432
+ # a pass benchmark leaves a reward, as a verifier does; a score
433
+ # benchmark a score and no pass or fail - the trial reads as Scored
434
+ rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value}
435
+ res["verifier_result"] = {"rewards": rewards}
436
+ parts = score.get("parts")
437
+ if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0:
438
+ # a score over several parts (a conversation's questions): the job's mean weighs it by them
439
+ res["verifier_result"]["parts"] = {"n": parts["n"]}
440
+ if parts.get("by") and isinstance(parts.get("parts"), list):
441
+ _write(os.path.join(trial_dir, "verifier", "parts.json"),
442
+ json.dumps({"by": parts["by"], "parts": parts["parts"]}, indent=1))
443
+ if "reward" in rewards:
444
+ _write(os.path.join(trial_dir, "verifier", "reward.txt"), str(value) + "\n")
445
+ _write(os.path.join(trial_dir, "verifier", "score.json"),
446
+ json.dumps(dict({k: v for k, v in score.items() if k != "parts"}, says=bench["score"]["says"]), indent=1))
447
+ if exception:
448
+ res["exception_info"] = {"exception_type": exception[0], "exception_message": exception[1][:2000]}
449
+ _write(os.path.join(trial_dir, "agent", "calls.jsonl"), "".join(json.dumps(c) + "\n" for c in calls))
450
+ _write(os.path.join(trial_dir, "agent", "gateway.log"),
451
+ "".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls))
452
+ if trajectory is not None:
453
+ _write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1))
454
+ timeline = trace_from_run(trajectory, calls, other, produce)
455
+ if timeline is not None:
456
+ _write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1))
457
+ for name, text in (extra_logs or {}).items():
458
+ _write(os.path.join(trial_dir, "agent", name), text)
459
+ _write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1))
460
+ return res
461
+
462
+
463
+ def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str, provider: str, dataset: str, units: List[str],
464
+ concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "",
465
+ options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1,
466
+ bench_options: Optional[dict] = None,
467
+ simulator: str = "", prices: Optional[dict] = None):
468
+ """``unsandboxed`` is true for an agent on the local backend only: in a
469
+ container (``backend``, ``image``) it is not. ``standard.options`` holds
470
+ every option's value, the defaults included, so two runs read side by side
471
+ say what differed. ``launch`` and ``shard``: a shard of a launch run over
472
+ several machines (``shard``), which ``merge`` reads back. ``repeat``: how
473
+ many times each unit was run, said only when more than once. ``judge`` and
474
+ ``simulator``: the models that graded and that played the user, when the
475
+ benchmark has them. ``prices``: the rows its models' costs are
476
+ estimated from and where the user's list came from, when it has any."""
477
+ with open(os.path.join(job_dir, "config.json"), "w") as f:
478
+ json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius",
479
+ "standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider,
480
+ "needs": bench["needs"], "permissions_granted": granted,
481
+ "unsandboxed": granted and backend == "local", "backend": backend,
482
+ **({"image": image} if image else {}), **({"judge": judge} if judge else {}),
483
+ **({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other),
484
+ **({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}),
485
+ **({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {})},
486
+ "runner_items": [bench["units"]["noun"], bench["units"]["nouns"]],
487
+ "n_concurrent_trials": concurrency,
488
+ "agents": [{"name": other["key"], "model_name": model}],
489
+ "datasets": [{"name": f"{bench['key']}/{dataset}", "n_tasks": len(units)}]}, f, indent=2)
490
+
491
+
492
+ #: an error kind naming the machine or the benchmark's side rather than the agent: the live board's infra_error
493
+ _INFRA = ("Setup", "Verifier", "Api", "RateLimit", "Environment", "Docker", "Build", "Connection", "Network", "Authentication",
494
+ "ModelNotFound")
495
+
496
+
497
+ def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict:
498
+ """A unit's end in the live feed's words (``nullius.runs.live``): its
499
+ outcome, with the reward (a pass benchmark) or the score, and the error's
500
+ kind as evidence. A unit the run's spending cap cut off (``ledger.ERROR``)
501
+ is the feed's ``budget``."""
502
+ score = score or {}
503
+ if exception:
504
+ k = exception[0]
505
+ return {"outcome": "cancelled" if k == "CancelledError" else "budget" if k == ledger.ERROR else "timeout" if "Timeout" in k else "infra_error" if any(w in k for w in _INFRA) else "agent_error",
506
+ "exc": k, "evidence": (exception[1] or k)[:200]}
507
+ if kind == "pass":
508
+ reward = score.get("reward")
509
+ return {"outcome": "resolved" if reward == 1.0 else "unresolved", "reward": reward, "evidence": f"reward {reward}"}
510
+ value = score.get("score", score.get("reward"))
511
+ return {"outcome": "scored", "score": value, "evidence": f"score {value}"}
512
+
513
+
514
+ def pass_hat(results: List[dict]) -> Optional[dict]:
515
+ """pass^k of a run that tried each unit several times (``repeat``), as τ-bench defines it: over
516
+ the units, the chance that k attempts drawn from its n all passed, C(c, k) / C(n, k), averaged -
517
+ pass^1 is the pass rate; and pass@k (``at``), the chance that one of them did, as HumanEval reads it. A unit with an attempt that ended in an error, with no verdict, is left
518
+ out, as the upstreams leave an infrastructure error out. None for a run of one attempt each."""
519
+ from math import comb
520
+ per: Dict[str, List[bool]] = {}
521
+ broken = set()
522
+ for r in results:
523
+ u = (r.get("standard") or {}).get("unit")
524
+ rewards = (r.get("verifier_result") or {}).get("rewards") or {}
525
+ if u is None:
526
+ continue
527
+ if r.get("exception_info") or "reward" not in rewards:
528
+ broken.add(u)
529
+ continue
530
+ per.setdefault(u, []).append(rewards["reward"] == 1.0)
531
+ whole = {u: v for u, v in per.items() if u not in broken}
532
+ n = min((len(v) for v in whole.values()), default=0)
533
+ if n < 2:
534
+ return None
535
+ return {"units": len(whole), "attempts": n,
536
+ **{str(k): round(sum(comb(sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)},
537
+ # pass@k, as HumanEval's estimate_pass_at_k reads n attempts with c passed: 1 - C(n-c, k) / C(n, k), averaged
538
+ "at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}}
539
+
540
+
541
+ def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None):
542
+ """``budget``: a run with a spending cap (``run.Run.budget``): the cap,
543
+ what was spent, whether it stopped the run, the units it cut off and
544
+ those never begun."""
545
+ errored = sum(1 for r in results if r.get("exception_info"))
546
+ hat = pass_hat(results)
547
+ with open(os.path.join(job_dir, "result.json"), "w") as f:
548
+ json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned,
549
+ "stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored,
550
+ "n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0},
551
+ **({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1)