nullius-eval 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (291) hide show
  1. _nullius_app/.env.example +55 -0
  2. _nullius_app/app.py +96 -0
  3. _nullius_app/requirements.txt +1 -0
  4. _nullius_app/static/brand/README.md +20 -0
  5. _nullius_app/static/brand/docker.svg +1 -0
  6. _nullius_app/static/brand/favicon.svg +1 -0
  7. _nullius_app/static/brand/huggingface.svg +37 -0
  8. _nullius_app/static/brand/mark-dark.svg +1 -0
  9. _nullius_app/static/brand/marks.svg +34 -0
  10. _nullius_app/static/brand/tokens.json +51 -0
  11. _nullius_app/static/brand/typesafe.png +0 -0
  12. _nullius_app/static/brand/wordmark-dark.svg +1 -0
  13. _nullius_app/static/builder/command-builder-core.js +763 -0
  14. _nullius_app/static/builder/command-builder.css +248 -0
  15. _nullius_app/static/builder/command-builder.js +997 -0
  16. _nullius_app/static/builder/model-picker.js +113 -0
  17. _nullius_app/static/catalogue/add.js +46 -0
  18. _nullius_app/static/catalogue/agents/README.md +19 -0
  19. _nullius_app/static/catalogue/agents/goose.svg +1 -0
  20. _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
  21. _nullius_app/static/catalogue/agents/marks.json +8 -0
  22. _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
  23. _nullius_app/static/catalogue/agents/opencode.svg +7 -0
  24. _nullius_app/static/catalogue/catalogue.css +436 -0
  25. _nullius_app/static/catalogue/catalogue.js +87 -0
  26. _nullius_app/static/catalogue/marks.svg +29 -0
  27. _nullius_app/static/catalogue/new.js +78 -0
  28. _nullius_app/static/catalogue/versions.js +27 -0
  29. _nullius_app/static/compare/comparison.css +117 -0
  30. _nullius_app/static/controls.js +117 -0
  31. _nullius_app/static/disclosures.js +70 -0
  32. _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
  33. _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
  34. _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
  35. _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
  36. _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
  37. _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
  38. _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
  39. _nullius_app/static/fonts/README.md +11 -0
  40. _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
  41. _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
  42. _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
  43. _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
  44. _nullius_app/static/fonts.css +49 -0
  45. _nullius_app/static/notifications.js +49 -0
  46. _nullius_app/static/onboarding.css +78 -0
  47. _nullius_app/static/onboarding.js +158 -0
  48. _nullius_app/static/runs/board.css +358 -0
  49. _nullius_app/static/runs/board.js +127 -0
  50. _nullius_app/static/runs/launches.css +88 -0
  51. _nullius_app/static/runs/live-board.css +186 -0
  52. _nullius_app/static/runs/live-board.js +394 -0
  53. _nullius_app/static/runs/path-suggestions.js +119 -0
  54. _nullius_app/static/runs/start.css +38 -0
  55. _nullius_app/static/settings/autosave.js +162 -0
  56. _nullius_app/static/settings/custom-keys.js +100 -0
  57. _nullius_app/static/settings/data-providers.js +88 -0
  58. _nullius_app/static/settings/judges.js +61 -0
  59. _nullius_app/static/settings/modal.css +32 -0
  60. _nullius_app/static/settings/modal.js +91 -0
  61. _nullius_app/static/settings/navigation.js +55 -0
  62. _nullius_app/static/settings/prices.js +68 -0
  63. _nullius_app/static/settings/providers.js +134 -0
  64. _nullius_app/static/settings/sandbox.js +20 -0
  65. _nullius_app/static/settings/settings.css +288 -0
  66. _nullius_app/static/style.css +1752 -0
  67. _nullius_app/static/traces/conversation.css +49 -0
  68. _nullius_app/static/traces/conversation.js +215 -0
  69. _nullius_app/templates/agents/entry.html +145 -0
  70. _nullius_app/templates/agents/kits.html +55 -0
  71. _nullius_app/templates/agents/standard.html +58 -0
  72. _nullius_app/templates/base.html +479 -0
  73. _nullius_app/templates/builder/new_run.html +55 -0
  74. _nullius_app/templates/catalogue/add_dialog.html +32 -0
  75. _nullius_app/templates/catalogue/benchmark.html +270 -0
  76. _nullius_app/templates/catalogue/benchmarks.html +91 -0
  77. _nullius_app/templates/catalogue/new.html +141 -0
  78. _nullius_app/templates/catalogue/use_bundle.html +11 -0
  79. _nullius_app/templates/compare/_detail.html +37 -0
  80. _nullius_app/templates/compare/compare.html +298 -0
  81. _nullius_app/templates/compare/compare_picker.html +51 -0
  82. _nullius_app/templates/compare/compare_task.html +329 -0
  83. _nullius_app/templates/runs/_execution.html +131 -0
  84. _nullius_app/templates/runs/_runtime.html +5 -0
  85. _nullius_app/templates/runs/ascii-mark.html +485 -0
  86. _nullius_app/templates/runs/feasibility.html +127 -0
  87. _nullius_app/templates/runs/index.html +609 -0
  88. _nullius_app/templates/runs/job.html +460 -0
  89. _nullius_app/templates/runs/preparing.html +14 -0
  90. _nullius_app/templates/runs/reconcile.html +124 -0
  91. _nullius_app/templates/runs/setup.html +272 -0
  92. _nullius_app/templates/runs/start.html +51 -0
  93. _nullius_app/templates/settings/custom_keys.html +82 -0
  94. _nullius_app/templates/settings/data_providers.html +27 -0
  95. _nullius_app/templates/settings/settings.html +564 -0
  96. _nullius_app/templates/traces/_charts.html +165 -0
  97. _nullius_app/templates/traces/_conversation.html +37 -0
  98. _nullius_app/templates/traces/_judgement.html +72 -0
  99. _nullius_app/templates/traces/artifact.html +15 -0
  100. _nullius_app/templates/traces/trial.html +838 -0
  101. _nullius_app/templates/workspaces/workspaces.html +54 -0
  102. _nullius_app/web/__init__.py +1 -0
  103. _nullius_app/web/agents.py +230 -0
  104. _nullius_app/web/builder.py +406 -0
  105. _nullius_app/web/catalogue.py +370 -0
  106. _nullius_app/web/compare.py +489 -0
  107. _nullius_app/web/core.py +17 -0
  108. _nullius_app/web/execute.py +223 -0
  109. _nullius_app/web/feasibility.py +60 -0
  110. _nullius_app/web/filters.py +231 -0
  111. _nullius_app/web/judge.py +623 -0
  112. _nullius_app/web/judge_tasks.py +33 -0
  113. _nullius_app/web/onboarding.py +20 -0
  114. _nullius_app/web/reconcile.py +47 -0
  115. _nullius_app/web/runs.py +464 -0
  116. _nullius_app/web/settings.py +324 -0
  117. _nullius_app/web/shared.py +507 -0
  118. _nullius_app/web/traces.py +306 -0
  119. _nullius_app/web/workspaces.py +84 -0
  120. nullius/__init__.py +32 -0
  121. nullius/__main__.py +5 -0
  122. nullius/catalogue/__init__.py +0 -0
  123. nullius/catalogue/agents/codex/agent.json +222 -0
  124. nullius/catalogue/agents/codex/config.toml +25 -0
  125. nullius/catalogue/agents/goose/agent.json +131 -0
  126. nullius/catalogue/agents/goose/config.yaml +20 -0
  127. nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
  128. nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
  129. nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
  130. nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
  131. nullius/catalogue/agents/opencode/agent.json +135 -0
  132. nullius/catalogue/agents/opencode/opencode.json +32 -0
  133. nullius/catalogue/agents/reference/agent.json +89 -0
  134. nullius/catalogue/agents/reference/agent.py +209 -0
  135. nullius/catalogue/agents/terminus-2/agent.json +64 -0
  136. nullius/catalogue/agents/terminus-2/run.py +139 -0
  137. nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
  138. nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
  139. nullius/catalogue/benchmarks/gaia/driver.py +188 -0
  140. nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
  141. nullius/catalogue/benchmarks/gaia/standard.json +278 -0
  142. nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
  143. nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
  144. nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
  145. nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
  146. nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
  147. nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
  148. nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
  149. nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
  150. nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
  151. nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
  152. nullius/catalogue/benchmarks/locomo/driver.py +270 -0
  153. nullius/catalogue/benchmarks/locomo/standard.json +265 -0
  154. nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
  155. nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
  156. nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
  157. nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
  158. nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
  159. nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
  160. nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
  161. nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
  162. nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
  163. nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
  164. nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
  165. nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
  166. nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
  167. nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
  168. nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
  169. nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
  170. nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
  171. nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
  172. nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
  173. nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
  174. nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
  175. nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
  176. nullius/catalogue/capabilities/mem0/capability.json +135 -0
  177. nullius/catalogue/capabilities/mem0/capability.py +173 -0
  178. nullius/catalogue/catalog.py +426 -0
  179. nullius/catalogue/citation.py +116 -0
  180. nullius/catalogue/harbor_entry.py +367 -0
  181. nullius/catalogue/sources.py +107 -0
  182. nullius/catalogue/storage.py +125 -0
  183. nullius/cli.py +85 -0
  184. nullius/compare/__init__.py +0 -0
  185. nullius/compare/cli.py +265 -0
  186. nullius/compare/comparison.py +184 -0
  187. nullius/execution.py +1653 -0
  188. nullius/gitrefs.py +128 -0
  189. nullius/harbor/__init__.py +0 -0
  190. nullius/harbor/feed.py +228 -0
  191. nullius/harbor/source.py +582 -0
  192. nullius/hosts.py +368 -0
  193. nullius/importers/__init__.py +140 -0
  194. nullius/importers/atif.py +387 -0
  195. nullius/importers/cli.py +135 -0
  196. nullius/importers/otel.py +274 -0
  197. nullius/judge/__init__.py +14 -0
  198. nullius/judge/batch.py +122 -0
  199. nullius/judge/client.py +433 -0
  200. nullius/judge/floors.json +86 -0
  201. nullius/judge/lanes.py +89 -0
  202. nullius/judge/questions.py +309 -0
  203. nullius/judge/reconcile.py +176 -0
  204. nullius/judge/state.py +224 -0
  205. nullius/judge/store.py +167 -0
  206. nullius/migrate.py +268 -0
  207. nullius/onboarding.py +116 -0
  208. nullius/paths.py +89 -0
  209. nullius/prerequisites.py +129 -0
  210. nullius/prices.py +283 -0
  211. nullius/progress.py +294 -0
  212. nullius/providers.py +335 -0
  213. nullius/recipes/__init__.py +0 -0
  214. nullius/recipes/launch.py +78 -0
  215. nullius/redact.py +72 -0
  216. nullius/runs/__init__.py +0 -0
  217. nullius/runs/artifacts.py +67 -0
  218. nullius/runs/classify.py +216 -0
  219. nullius/runs/cli.py +316 -0
  220. nullius/runs/feasibility.py +442 -0
  221. nullius/runs/launches.py +149 -0
  222. nullius/runs/live.py +715 -0
  223. nullius/runs/parse.py +1016 -0
  224. nullius/runs/verifier.py +241 -0
  225. nullius/sandbox.py +109 -0
  226. nullius/secrets.py +218 -0
  227. nullius/settings.py +544 -0
  228. nullius/standard/__init__.py +15 -0
  229. nullius/standard/acp.py +305 -0
  230. nullius/standard/cancellation.py +19 -0
  231. nullius/standard/cli.py +703 -0
  232. nullius/standard/conform.py +149 -0
  233. nullius/standard/container.py +1870 -0
  234. nullius/standard/definitions.py +1651 -0
  235. nullius/standard/egress.py +192 -0
  236. nullius/standard/fidelity.py +256 -0
  237. nullius/standard/forward.py +139 -0
  238. nullius/standard/hostreq.py +105 -0
  239. nullius/standard/howto.py +220 -0
  240. nullius/standard/install.py +428 -0
  241. nullius/standard/job.py +551 -0
  242. nullius/standard/jsonl.py +129 -0
  243. nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
  244. nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
  245. nullius/standard/kits/agent-search-v1/driver.py +171 -0
  246. nullius/standard/kits/agent-search-v1/standard.json +99 -0
  247. nullius/standard/kits/agent-task-v1/driver.py +25 -0
  248. nullius/standard/kits/agent-task-v1/standard.json +100 -0
  249. nullius/standard/kits/agent-tools-v1/driver.py +206 -0
  250. nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
  251. nullius/standard/kits/agent-tools-v1/standard.json +119 -0
  252. nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
  253. nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
  254. nullius/standard/kits/cap-memory-v1/driver.py +39 -0
  255. nullius/standard/kits/cap-memory-v1/standard.json +91 -0
  256. nullius/standard/ledger.py +326 -0
  257. nullius/standard/match.py +66 -0
  258. nullius/standard/mcp_bridge.py +62 -0
  259. nullius/standard/mcp_machine.py +193 -0
  260. nullius/standard/merge.py +239 -0
  261. nullius/standard/names.py +37 -0
  262. nullius/standard/network_requirements.py +44 -0
  263. nullius/standard/pace.py +53 -0
  264. nullius/standard/payload.py +278 -0
  265. nullius/standard/pins.py +130 -0
  266. nullius/standard/ports.py +70 -0
  267. nullius/standard/prove.py +373 -0
  268. nullius/standard/recipe.py +746 -0
  269. nullius/standard/relay.py +785 -0
  270. nullius/standard/run.py +1781 -0
  271. nullius/standard/setup_container.py +208 -0
  272. nullius/standard/shard.py +71 -0
  273. nullius/standard/web_services.py +30 -0
  274. nullius/traces/__init__.py +0 -0
  275. nullius/traces/behavior.py +131 -0
  276. nullius/traces/charts.py +898 -0
  277. nullius/traces/conversation.py +159 -0
  278. nullius/traces/otel.py +608 -0
  279. nullius/traces/trace.py +1477 -0
  280. nullius/traces/trajectory.py +586 -0
  281. nullius/workspace.py +417 -0
  282. nullius/workspace_cli.py +119 -0
  283. nullius_eval-0.1.0.dist-info/METADATA +93 -0
  284. nullius_eval-0.1.0.dist-info/RECORD +291 -0
  285. nullius_eval-0.1.0.dist-info/WHEEL +5 -0
  286. nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
  287. nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
  288. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
  289. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
  290. nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
  291. nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,586 @@
1
+ """Parse an ATIF trajectory (``agent/trajectory.json``) into steps + stats.
2
+
3
+ Each step may carry ``tool_calls`` (a list of {function_name, arguments,
4
+ tool_call_id}) and an ``observation`` (results keyed by source_call_id). We
5
+ flatten those into a per-step view and compute run-level tool statistics.
6
+
7
+ One rule about those statistics: a tool call's ``category`` is the only
8
+ key that compares across agents; its ``name`` never is. ``function_name`` is
9
+ each agent's own vocabulary - OpenCode writes ``bash``, Claude Code ``Bash``,
10
+ another agent a sentence, so name frequencies are not comparable. Every summary a
11
+ page or a comparison reads (``category_freq``, ``behavior.analyze``, the
12
+ ``Tool mix`` row) is therefore keyed by category, and the names are kept
13
+ beside it (``name_freq``) as the evidence of what the category folded, never
14
+ as a key of their own.
15
+
16
+ A benchmark's own tools reach the agent over MCP, where each agent spells the
17
+ same tool its own way (``mcp.kit.get_code``, ``Tool: kit/get_code``,
18
+ ``kit: get code``). :func:`mcp_tool` reads the server and the tool back from any
19
+ of them, and such a tool's category comes from what its name says it does:
20
+ ``lookup`` or ``change``. No benchmark is named here, and the coding rules below
21
+ are left to the agent's own tools.
22
+ """
23
+
24
+ from __future__ import annotations
25
+
26
+ import json
27
+ import re
28
+ from collections import Counter
29
+ from dataclasses import dataclass, field
30
+ from datetime import datetime
31
+
32
+ from nullius.redact import redact
33
+
34
+ # Coarse tool categories, matched against the (verbose) function_name label.
35
+ # Order matters: the first rule to match wins. `plan` comes first because
36
+ # opencode's `todowrite` contains "write" but updates a plan rather than code.
37
+ _CATEGORY_RULES = [
38
+ ("plan", re.compile(r"(?i)todo|plan|think|note")),
39
+ ("search", re.compile(r"(?i)search|grep|glob|find|list")),
40
+ ("read", re.compile(r"(?i)read|cat|view|open|inspect")),
41
+ ("edit", re.compile(r"(?i)edit|write|patch|replace|apply|create")),
42
+ ("execute", re.compile(r"(?i)execute|run|bash|shell|command|test|python")),
43
+ ]
44
+
45
+
46
+ # The spellings of an MCP tool that say so themselves. Any other (``kit: get code``,
47
+ # ``kit_get_code``) reads as one only for a server the run says it gave the agent.
48
+ _MCP_SPELLINGS = (
49
+ re.compile(r"^mcp(?:\.|__)([^\s.]+?)(?:\.|__)(\S+)$"),
50
+ re.compile(r"^Tool: ([^\s/]+)/(\S+)$"),
51
+ re.compile(r"^([A-Za-z0-9-]+)__(\S+)$"),
52
+ )
53
+ # What an MCP tool's first word says it does. Anything else stays ``other``: a guess would read as a measurement.
54
+ _LOOKUP_WORDS = frozenset("get list search find read check fetch lookup query show view retrieve describe count".split())
55
+ _CHANGE_WORDS = frozenset("create add update set edit delete remove cancel book send write modify replace move".split())
56
+ # An MCP tool over files or a shell is the coding work the rules below count: the edit-to-test loop reads those categories.
57
+ _CODE_WORDS = frozenset("file files dir directory patch shell bash command test tests grep glob editor python".split())
58
+
59
+
60
+ def mcp_tool(function_name: str, servers=()) -> tuple | None:
61
+ """``(server, tool)`` when *function_name* is an MCP tool, in any agent's spelling."""
62
+ name = (function_name or "").strip()
63
+ for pat in _MCP_SPELLINGS:
64
+ got = pat.match(name)
65
+ if got:
66
+ return got.group(1), got.group(2)
67
+ for server in servers:
68
+ for joint in ("__", "_", ": ", ":", "/", "."):
69
+ if name.startswith(server + joint) and name[len(server) + len(joint):].strip():
70
+ return server, re.sub(r"\s+", "_", name[len(server) + len(joint):].strip())
71
+ return None
72
+
73
+
74
+ def categorize(function_name: str, servers=()) -> str:
75
+ """The coarse category of a tool call - the one key comparable across agents.
76
+
77
+ The name it is derived from is not: agents name the same tool differently
78
+ (module docstring). A name no rule matches is ``other``, which is a
79
+ category like the rest and is summarised like the rest.
80
+ """
81
+ tool = mcp_tool(function_name, servers)
82
+ if tool:
83
+ words = tool[1].lower().split("_")
84
+ if not _CODE_WORDS.intersection(words):
85
+ return "lookup" if words[0] in _LOOKUP_WORDS else "change" if words[0] in _CHANGE_WORDS else "other"
86
+ for name, pat in _CATEGORY_RULES:
87
+ if pat.search(tool[1] if tool else function_name or ""):
88
+ return name
89
+ return "other"
90
+
91
+
92
+ @dataclass
93
+ class Step:
94
+ step_id: int
95
+ source: str
96
+ timestamp: str
97
+ message: str
98
+ tool_calls: list = field(default_factory=list) # [{name, category, arguments, observation}]
99
+ # Seconds until the next step opened — None on the last step, when either
100
+ # timestamp is missing, or when the whole trajectory's clock is degenerate.
101
+ dur: float | None = None
102
+
103
+
104
+ # Observations are serialised as `success=<bool> data={...} error=<repr>`, so
105
+ # status is explicit — no need to guess from the text. Fall back to scanning
106
+ # for tracebacks only when that prefix is absent (older/other writers).
107
+ _SUCCESS_RE = re.compile(r"^success=(True|False)\b")
108
+ _ERROR_RE = re.compile(r"error='((?:[^'\\]|\\.)*)'|error=\"((?:[^\"\\]|\\.)*)\"")
109
+ _FAILURE_MARKERS = [
110
+ (re.compile(r"Traceback \(most recent call last\)"), "Traceback"),
111
+ (re.compile(r"\b(\w*(?:Error|Exception))\b(?=:)"), None),
112
+ (re.compile(r"command not found"), "command not found"),
113
+ (re.compile(r"No such file or directory"), "No such file or directory"),
114
+ ]
115
+
116
+
117
+ def _first_line(text: str) -> str:
118
+ """Most informative line of an error blob.
119
+
120
+ Skips the exit-code header, and for a traceback takes the *last* line —
121
+ "Traceback (most recent call last):" says nothing, the final
122
+ "ModuleNotFoundError: ..." says everything.
123
+ """
124
+ # The blob arrives as a Python repr, so quotes and newlines are escaped.
125
+ text = text.replace("\\n", "\n").replace("\\'", "'").replace('\\"', '"')
126
+ lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
127
+ lines = [ln for ln in lines if not ln.lower().startswith("exit code")]
128
+ if not lines:
129
+ return ""
130
+ if any(ln.startswith("Traceback (most recent call last)") for ln in lines):
131
+ return lines[-1][:160]
132
+ return lines[0][:160]
133
+
134
+ # Argument keys worth showing as the call's "target", most specific first.
135
+ _TARGET_KEYS = ("command", "cmd", "file_path", "path", "pattern", "query", "glob")
136
+
137
+ # Some agents don't name the file in an argument at all — opencode's
138
+ # `apply_patch` takes only `patchText`, with the path inside the payload. Both
139
+ # the codex-style header and a plain unified diff appear in the wild.
140
+ _PATCH_KEYS = ("patchText", "patch", "diff", "content")
141
+ PATCH_PATHS = [
142
+ re.compile(r"(?m)^\*\*\* (?:Update|Add|Delete) File:\s*(\S+)"),
143
+ re.compile(r"(?m)^\+\+\+ (?:b/)?(\S+)"),
144
+ ]
145
+ # Public for the same reason as trace.clean: behavior.py reads both this and
146
+ # parse_ts, and it stayed with the application when the reader was packaged.
147
+ _PATCH_PATHS = PATCH_PATHS
148
+
149
+
150
+ # Container WORKDIRs an agent runs in, such as /testbed or /app. Whether a path arrives absolute or relative is the model's
151
+ # choice, not the harness's — one agent passed relative paths on two runs of
152
+ # one task and absolute on a third, same model, same task. Left alone, the same
153
+ # file is two different strings and every set operation over them lies:
154
+ # "which files did both agents touch" answered 0 when the answer was 2.
155
+ #
156
+ # Normalising here rather than at each comparison keeps one canonical form. The
157
+ # agent's original string stays on the tool call, because *that* difference is
158
+ # real signal about how the agent behaved — it just must not reach set
159
+ # arithmetic. `behavior._same_file` still suffix-matches for gold patches,
160
+ # which covers workspace roots this list doesn't know.
161
+ _WORKSPACE_ROOTS = ("/testbed/", "/app/")
162
+
163
+
164
+ def workspace_relative(path: str) -> str:
165
+ """Strip a known container workspace root so paths compare equal."""
166
+ for root in _WORKSPACE_ROOTS:
167
+ if path.startswith(root):
168
+ return path[len(root):]
169
+ return path
170
+
171
+
172
+ def patched_files(arguments: dict) -> list:
173
+ """File paths named inside a patch payload, in first-seen order."""
174
+ if not isinstance(arguments, dict):
175
+ return []
176
+ body = next(
177
+ (arguments[k] for k in _PATCH_KEYS if isinstance(arguments.get(k), str)), ""
178
+ )
179
+ found = []
180
+ for pattern in _PATCH_PATHS:
181
+ for path in pattern.findall(body):
182
+ if path not in found and path != "/dev/null":
183
+ found.append(path)
184
+ return found
185
+
186
+
187
+ def failure_of(observation: str) -> str:
188
+ """Describe the failure in *observation*, or '' if the call succeeded."""
189
+ if not observation:
190
+ return ""
191
+ status = _SUCCESS_RE.match(observation)
192
+ if status:
193
+ if status.group(1) == "True":
194
+ return ""
195
+ err = _ERROR_RE.search(observation)
196
+ message = _first_line(err.group(1) or err.group(2) or "") if err else ""
197
+ return message or "failed"
198
+ for pattern, label in _FAILURE_MARKERS:
199
+ hit = pattern.search(observation)
200
+ if hit:
201
+ return label or hit.group(1)
202
+ return ""
203
+
204
+
205
+ def target_of(arguments: dict) -> str:
206
+ """A short, human-readable subject for a tool call."""
207
+ if not isinstance(arguments, dict):
208
+ return ""
209
+ for key in _TARGET_KEYS:
210
+ value = arguments.get(key)
211
+ if isinstance(value, str) and value.strip():
212
+ return " ".join(value.split())
213
+ # No named argument: a patch payload still tells us what was touched.
214
+ files = patched_files(arguments)
215
+ if files:
216
+ return files[0] if len(files) == 1 else f"{files[0]} +{len(files) - 1}"
217
+ return ""
218
+
219
+
220
+ # Two usage conventions reach this parser. OpenAI-style counts every input
221
+ # token in prompt_tokens and the cached subset again in cached_tokens, so
222
+ # cached ÷ prompt is a real hit rate. Anthropic-style reports cache *reads*
223
+ # (plus a few-token fresh remainder) as prompt_tokens and the new tokens as a
224
+ # cache write in a field of its own that never enters the total — cached ÷
225
+ # prompt is ~100% there by construction, however the cache performed.
226
+ # The number worth comparing across agents is therefore
227
+ # reads ÷ every input token, with the writes added back in.
228
+ _REMAINDER_CEILING = 16 # max per-call prompt−cached that reads as Anthropic-style
229
+
230
+
231
+ @dataclass
232
+ class CacheAccounting:
233
+ convention: str # "openai" | "anthropic"
234
+ reads: int # tokens served from cache
235
+ new: int # every other input token: cache writes + fresh remainder
236
+ reported_prompt: int # final_metrics' own total_prompt_tokens
237
+ reported_cached: int
238
+
239
+ @property
240
+ def total(self) -> int:
241
+ return self.reads + self.new
242
+
243
+ @property
244
+ def hit(self) -> float | None:
245
+ return self.reads / self.total if self.total else None
246
+
247
+ @property
248
+ def reported_hit(self) -> float | None:
249
+ return self.reported_cached / self.reported_prompt if self.reported_prompt else None
250
+
251
+ @property
252
+ def reported_diverges(self) -> bool:
253
+ """Whether the file's own ratio would mislead next to the real one."""
254
+ if self.hit is None or self.reported_hit is None:
255
+ return False
256
+ return abs(self.hit - self.reported_hit) > 0.005
257
+
258
+
259
+ def _extension(container, key):
260
+ """A field ATIF does not name. Newer writers put it in the
261
+ `extra` slot of its level; earlier artifacts carry it beside the named
262
+ fields. The slot wins when both are present, the two are never summed,
263
+ and a null in the slot stays None - the slot said so. A non-dict `extra`
264
+ is no slot: the old position is read."""
265
+ if not isinstance(container, dict):
266
+ return None
267
+ extra = container.get("extra")
268
+ if isinstance(extra, dict) and key in extra:
269
+ return extra[key]
270
+ return container.get(key)
271
+
272
+
273
+ def _cache_accounting(series: list, final_metrics: dict) -> CacheAccounting | None:
274
+ """Fold the token series into reads and new, whichever convention wrote it.
275
+
276
+ Also annotates every series entry with ``sent`` and ``new`` so the growth
277
+ strip can draw each call's whole input — under the Anthropic convention a
278
+ call's prompt_tokens is missing its write, which belongs to the call that
279
+ paid for it. The write of the final call never surfaces (nothing read it
280
+ back), so it is counted as zero rather than guessed.
281
+ """
282
+ for e in series:
283
+ e["new"] = max(e["prompt"] - e["cached"], 0)
284
+ e["sent"] = e["prompt"]
285
+ usable = [e for e in series if e["prompt"] > 0]
286
+
287
+ reported_prompt = final_metrics.get("total_prompt_tokens")
288
+ reported_cached = final_metrics.get("total_cached_tokens")
289
+ if not isinstance(reported_prompt, (int, float)):
290
+ reported_prompt = sum(e["prompt"] for e in usable)
291
+ reported_cached = sum(e["cached"] for e in usable)
292
+ if not isinstance(reported_cached, (int, float)):
293
+ reported_cached = 0
294
+ reported_prompt, reported_cached = int(reported_prompt), int(reported_cached)
295
+
296
+ if len(usable) >= 2 and all(
297
+ e["prompt"] - e["cached"] <= _REMAINDER_CEILING for e in usable
298
+ ):
299
+ for cur, nxt in zip(usable, usable[1:]):
300
+ write = max(nxt["prompt"] - cur["prompt"], 0)
301
+ cur["new"] += write
302
+ cur["sent"] = cur["cached"] + cur["new"]
303
+ reads = sum(e["cached"] for e in usable)
304
+ new = sum(e["new"] for e in usable)
305
+ return CacheAccounting("anthropic", reads, new, reported_prompt, reported_cached)
306
+
307
+ if usable:
308
+ reads = sum(e["cached"] for e in usable)
309
+ new = sum(e["new"] for e in usable)
310
+ return CacheAccounting("openai", reads, new, reported_prompt, reported_cached)
311
+
312
+ if reported_prompt:
313
+ reads = min(reported_cached, reported_prompt)
314
+ return CacheAccounting("openai", reads, reported_prompt - reads,
315
+ reported_prompt, reported_cached)
316
+ return None
317
+
318
+
319
+ @dataclass
320
+ class Trajectory:
321
+ n_steps: int
322
+ steps: list # list[Step]
323
+ category_freq: Counter # by coarse category - the cross-agent key
324
+ n_tool_calls: int
325
+ last_agent_message: str
326
+ final_metrics: dict
327
+ error: str = "" # set if the file could not be parsed
328
+ calls: list = field(default_factory=list) # flat, in order
329
+ files_touched: list = field(default_factory=list) # paths from edit calls
330
+ # False when the writer stamped every step at the same instant (older
331
+ # trajectories) — durations would be noise, so none are set.
332
+ has_step_clock: bool = False
333
+ # One entry per step that carries per-call metrics (ATIF v1.7 writers, and
334
+ # runs where a gateway-log matcher attached them):
335
+ # {step_id, prompt, cached, completion}. Prompt growth across this series
336
+ # is the caching story of the whole session in one curve.
337
+ token_series: list = field(default_factory=list)
338
+ # Reads/new fold over the series, convention detected — None when the
339
+ # file carries no usage at all.
340
+ cache: CacheAccounting | None = None
341
+ # Summed from metrics.extra.reasoning_tokens; None when no step has it —
342
+ # "not reported" and "zero reasoning" are different claims.
343
+ total_reasoning_tokens: int | None = None
344
+ # {category: Counter(name)} over the same calls as category_freq: what each
345
+ # category folded, in the agent's own words. Evidence beside the key, so a
346
+ # reader can check the fold - never a key itself (module docstring).
347
+ name_freq: dict = field(default_factory=dict)
348
+
349
+ @property
350
+ def n_failed_calls(self) -> int:
351
+ return sum(1 for c in self.calls if c["failure"])
352
+
353
+ @property
354
+ def tools(self) -> list:
355
+ """Each tool once: ``{server, name, category, calls, failed, ns, spellings}``,
356
+ by server, the most called first. ``ns`` is ``None`` when no call has a clock."""
357
+ rows: dict = {}
358
+ for c in self.calls:
359
+ row = rows.setdefault((c["server"], c["tool"]), {"server": c["server"], "name": c["tool"], "category": c["category"],
360
+ "calls": 0, "failed": 0, "ns": None, "spellings": []})
361
+ row["calls"] += 1
362
+ row["failed"] += 1 if c["failure"] else 0
363
+ if c["ns"] is not None:
364
+ row["ns"] = (row["ns"] or 0) + c["ns"]
365
+ if c["name"] not in row["spellings"]:
366
+ row["spellings"].append(c["name"])
367
+ return sorted(rows.values(), key=lambda r: (bool(r["server"]) is False, r["server"], -r["calls"], r["name"]))
368
+
369
+ def names_of(self, category: str) -> list:
370
+ """``[(name, count), …]`` the agent used for *category*, most frequent first."""
371
+ return self.name_freq.get(category, Counter()).most_common()
372
+
373
+
374
+ def parse_ts(value: str):
375
+ """An ISO timestamp, or None — a record with an unreadable clock is not
376
+ an error here, it is a record whose clock we decline to guess at."""
377
+ if not value:
378
+ return None
379
+ try:
380
+ return datetime.fromisoformat(value.replace("Z", "+00:00"))
381
+ except ValueError:
382
+ return None
383
+
384
+
385
+ _parse_ts = parse_ts
386
+
387
+
388
+ def _tool_ns(step: dict) -> int | None:
389
+ """How long a step's tool call ran, when the step says when it ended (a standard run's do)."""
390
+ extra = step.get("extra") if isinstance(step.get("extra"), dict) else {}
391
+ start, end = parse_ts(extra.get("running_at") or step.get("timestamp")), parse_ts(extra.get("ended_at"))
392
+ return max(0, int((end - start).total_seconds() * 1e9)) if start and end else None
393
+
394
+
395
+ # Below this, the trajectory's whole clock spread is treated as one write-time
396
+ # stamp, not a real per-step clock. Same refuse-don't-guess policy as the
397
+ # agent-side metrics matcher.
398
+ _MIN_CLOCK_SPAN_SEC = 1.0
399
+
400
+
401
+ def _attach_durations(steps: list) -> bool:
402
+ """Set each step's ``dur`` from consecutive timestamps.
403
+
404
+ Returns whether the trajectory has a usable step clock at all; when it
405
+ does not, every ``dur`` is left None.
406
+ """
407
+ stamps = [_parse_ts(s.timestamp) for s in steps]
408
+ known = [t for t in stamps if t]
409
+ if len(known) < 2 or (max(known) - min(known)).total_seconds() < _MIN_CLOCK_SPAN_SEC:
410
+ return False
411
+ for i, step in enumerate(steps[:-1]):
412
+ here, there = stamps[i], stamps[i + 1]
413
+ if here and there and there >= here:
414
+ step.dur = (there - here).total_seconds()
415
+ return True
416
+
417
+
418
+ def _extract_observation(step: dict, call_id: str) -> str:
419
+ obs = step.get("observation")
420
+ if not isinstance(obs, dict):
421
+ return ""
422
+ for r in obs.get("results", []):
423
+ if r.get("source_call_id") == call_id:
424
+ content = r.get("content", "")
425
+ # content is often a JSON-encoded list of {type, content:{text}}
426
+ try:
427
+ parsed = json.loads(content)
428
+ texts = []
429
+ for item in parsed:
430
+ c = item.get("content", {})
431
+ if isinstance(c, dict) and "text" in c:
432
+ texts.append(c["text"])
433
+ if texts:
434
+ return redact("\n".join(texts))
435
+ except Exception:
436
+ pass
437
+ return redact(str(content))
438
+ return ""
439
+
440
+
441
+ def message_text(message) -> str:
442
+ """A step's message as text. ATIF v1.6 lets it be a list of content parts
443
+ ({"type": "text", "text": ...}, an image, ...): the text parts joined, a
444
+ placeholder for each other part, so a multimodal prompt reads rather
445
+ than failing the page."""
446
+ if isinstance(message, str):
447
+ return message
448
+ if isinstance(message, dict):
449
+ message = [message]
450
+ if not isinstance(message, list):
451
+ return "" if message is None else str(message)
452
+ out = []
453
+ for part in message:
454
+ if isinstance(part, str):
455
+ out.append(part)
456
+ elif isinstance(part, dict) and isinstance(part.get("text"), str):
457
+ out.append(part["text"])
458
+ elif isinstance(part, dict):
459
+ out.append(f"[{part.get('type') or 'part'}]")
460
+ return "\n".join(out)
461
+
462
+
463
+ def load(path: str) -> Trajectory:
464
+ try:
465
+ with open(path, encoding="utf-8") as fh:
466
+ data = json.load(fh)
467
+ except (OSError, json.JSONDecodeError) as exc:
468
+ return Trajectory(0, [], Counter(), 0, "", {}, error=str(exc))
469
+
470
+ steps_out: list[Step] = []
471
+ category_freq: Counter = Counter()
472
+ name_freq: dict = {}
473
+ n_tool_calls = 0
474
+ last_agent_message = ""
475
+
476
+ all_calls: list = []
477
+ files_touched: list = []
478
+ token_series: list = []
479
+ reasoning_total: int | None = None
480
+ servers = (data.get("extra") or {}).get("mcp_servers") or ()
481
+
482
+ for raw in data.get("steps", []):
483
+ metrics = raw.get("metrics")
484
+ if isinstance(metrics, dict):
485
+ token_series.append(
486
+ {
487
+ "step_id": raw.get("step_id", 0),
488
+ "prompt": metrics.get("prompt_tokens") or 0,
489
+ "cached": metrics.get("cached_tokens") or 0,
490
+ "completion": metrics.get("completion_tokens") or 0,
491
+ }
492
+ )
493
+ reasoning = (metrics.get("extra") or {}).get("reasoning_tokens")
494
+ if reasoning is not None:
495
+ reasoning_total = (reasoning_total or 0) + reasoning
496
+ calls_out = []
497
+ for tc in raw.get("tool_calls") or []:
498
+ name = tc.get("function_name") or tc.get("name") or "(tool)"
499
+ cat = categorize(name, servers)
500
+ server, tool = mcp_tool(name, servers) or ("", name)
501
+ args = tc.get("arguments", {})
502
+ observation = _extract_observation(raw, tc.get("tool_call_id"))
503
+ call = {
504
+ "step_id": raw.get("step_id", 0),
505
+ # An ATIF id is source evidence, not the call's position. Keep
506
+ # it so parallel tools and their observations stay identifiable.
507
+ "tool_call_id": tc.get("tool_call_id"),
508
+ "name": redact(name),
509
+ "server": redact(server),
510
+ "tool": redact(tool),
511
+ "ns": _tool_ns(raw),
512
+ "category": cat,
513
+ "target": redact(target_of(args)),
514
+ "arguments": redact(json.dumps(args, ensure_ascii=False)),
515
+ "observation": observation,
516
+ # Size of what came back — the closest thing to "work done" this
517
+ # schema offers, since the timestamps are not per-step.
518
+ "obs_bytes": len(observation),
519
+ "failure": failure_of(observation),
520
+ }
521
+ calls_out.append(call)
522
+ # The writer emits placeholder entries — no name, no arguments, no
523
+ # observation — which would otherwise double the tool-call count.
524
+ # The summaries count the same calls the list keeps, so the chips,
525
+ # the names behind them and the total describe one population.
526
+ if args or observation or name != "Using tool":
527
+ all_calls.append(call)
528
+ n_tool_calls += 1
529
+ category_freq[cat] += 1
530
+ name_freq.setdefault(cat, Counter())[call["name"]] += 1
531
+ if cat == "edit":
532
+ # Prefer a named argument; fall back to paths inside the patch
533
+ # body for agents that pass only a diff.
534
+ touched = patched_files(args) or ([target_of(args)] if target_of(args) else [])
535
+ for path in touched:
536
+ path = workspace_relative(path)
537
+ if path and path not in files_touched:
538
+ files_touched.append(path)
539
+
540
+ msg = redact(message_text(raw.get("message")))
541
+ src = raw.get("source", "")
542
+ if src == "agent" and msg and msg != "(tool use)":
543
+ last_agent_message = msg
544
+
545
+ steps_out.append(
546
+ Step(
547
+ step_id=raw.get("step_id", 0),
548
+ source=src,
549
+ timestamp=raw.get("timestamp", ""),
550
+ message=msg,
551
+ tool_calls=calls_out,
552
+ )
553
+ )
554
+
555
+ has_clock = _attach_durations(steps_out)
556
+ # A call inherits its step's span (None without a clock) — coarse, but the
557
+ # schema gives one timestamp per step, not per call.
558
+ for step in steps_out:
559
+ for call in step.tool_calls:
560
+ call["dur"] = step.dur
561
+
562
+ # Agents whose LLM calls cannot be matched to steps leave per-step metrics
563
+ # unset and report the total in final_metrics instead. Reading only the
564
+ # per-step path renders nothing while the number sits in the file.
565
+ if reasoning_total is None:
566
+ fallback = _extension(data.get("final_metrics") or {}, "total_reasoning_tokens")
567
+ if isinstance(fallback, (int, float)):
568
+ reasoning_total = int(fallback)
569
+
570
+ cache = _cache_accounting(token_series, data.get("final_metrics") or {})
571
+
572
+ return Trajectory(
573
+ n_steps=len(steps_out),
574
+ steps=steps_out,
575
+ category_freq=category_freq,
576
+ n_tool_calls=n_tool_calls,
577
+ last_agent_message=last_agent_message,
578
+ final_metrics=data.get("final_metrics") or {},
579
+ calls=all_calls,
580
+ files_touched=files_touched,
581
+ has_step_clock=has_clock,
582
+ token_series=token_series,
583
+ cache=cache,
584
+ total_reasoning_tokens=reasoning_total,
585
+ name_freq=name_freq,
586
+ )