nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,586 @@
|
|
|
1
|
+
"""Parse an ATIF trajectory (``agent/trajectory.json``) into steps + stats.
|
|
2
|
+
|
|
3
|
+
Each step may carry ``tool_calls`` (a list of {function_name, arguments,
|
|
4
|
+
tool_call_id}) and an ``observation`` (results keyed by source_call_id). We
|
|
5
|
+
flatten those into a per-step view and compute run-level tool statistics.
|
|
6
|
+
|
|
7
|
+
One rule about those statistics: a tool call's ``category`` is the only
|
|
8
|
+
key that compares across agents; its ``name`` never is. ``function_name`` is
|
|
9
|
+
each agent's own vocabulary - OpenCode writes ``bash``, Claude Code ``Bash``,
|
|
10
|
+
another agent a sentence, so name frequencies are not comparable. Every summary a
|
|
11
|
+
page or a comparison reads (``category_freq``, ``behavior.analyze``, the
|
|
12
|
+
``Tool mix`` row) is therefore keyed by category, and the names are kept
|
|
13
|
+
beside it (``name_freq``) as the evidence of what the category folded, never
|
|
14
|
+
as a key of their own.
|
|
15
|
+
|
|
16
|
+
A benchmark's own tools reach the agent over MCP, where each agent spells the
|
|
17
|
+
same tool its own way (``mcp.kit.get_code``, ``Tool: kit/get_code``,
|
|
18
|
+
``kit: get code``). :func:`mcp_tool` reads the server and the tool back from any
|
|
19
|
+
of them, and such a tool's category comes from what its name says it does:
|
|
20
|
+
``lookup`` or ``change``. No benchmark is named here, and the coding rules below
|
|
21
|
+
are left to the agent's own tools.
|
|
22
|
+
"""
|
|
23
|
+
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import json
|
|
27
|
+
import re
|
|
28
|
+
from collections import Counter
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from datetime import datetime
|
|
31
|
+
|
|
32
|
+
from nullius.redact import redact
|
|
33
|
+
|
|
34
|
+
# Coarse tool categories, matched against the (verbose) function_name label.
|
|
35
|
+
# Order matters: the first rule to match wins. `plan` comes first because
|
|
36
|
+
# opencode's `todowrite` contains "write" but updates a plan rather than code.
|
|
37
|
+
_CATEGORY_RULES = [
|
|
38
|
+
("plan", re.compile(r"(?i)todo|plan|think|note")),
|
|
39
|
+
("search", re.compile(r"(?i)search|grep|glob|find|list")),
|
|
40
|
+
("read", re.compile(r"(?i)read|cat|view|open|inspect")),
|
|
41
|
+
("edit", re.compile(r"(?i)edit|write|patch|replace|apply|create")),
|
|
42
|
+
("execute", re.compile(r"(?i)execute|run|bash|shell|command|test|python")),
|
|
43
|
+
]
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
# The spellings of an MCP tool that say so themselves. Any other (``kit: get code``,
|
|
47
|
+
# ``kit_get_code``) reads as one only for a server the run says it gave the agent.
|
|
48
|
+
_MCP_SPELLINGS = (
|
|
49
|
+
re.compile(r"^mcp(?:\.|__)([^\s.]+?)(?:\.|__)(\S+)$"),
|
|
50
|
+
re.compile(r"^Tool: ([^\s/]+)/(\S+)$"),
|
|
51
|
+
re.compile(r"^([A-Za-z0-9-]+)__(\S+)$"),
|
|
52
|
+
)
|
|
53
|
+
# What an MCP tool's first word says it does. Anything else stays ``other``: a guess would read as a measurement.
|
|
54
|
+
_LOOKUP_WORDS = frozenset("get list search find read check fetch lookup query show view retrieve describe count".split())
|
|
55
|
+
_CHANGE_WORDS = frozenset("create add update set edit delete remove cancel book send write modify replace move".split())
|
|
56
|
+
# An MCP tool over files or a shell is the coding work the rules below count: the edit-to-test loop reads those categories.
|
|
57
|
+
_CODE_WORDS = frozenset("file files dir directory patch shell bash command test tests grep glob editor python".split())
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def mcp_tool(function_name: str, servers=()) -> tuple | None:
|
|
61
|
+
"""``(server, tool)`` when *function_name* is an MCP tool, in any agent's spelling."""
|
|
62
|
+
name = (function_name or "").strip()
|
|
63
|
+
for pat in _MCP_SPELLINGS:
|
|
64
|
+
got = pat.match(name)
|
|
65
|
+
if got:
|
|
66
|
+
return got.group(1), got.group(2)
|
|
67
|
+
for server in servers:
|
|
68
|
+
for joint in ("__", "_", ": ", ":", "/", "."):
|
|
69
|
+
if name.startswith(server + joint) and name[len(server) + len(joint):].strip():
|
|
70
|
+
return server, re.sub(r"\s+", "_", name[len(server) + len(joint):].strip())
|
|
71
|
+
return None
|
|
72
|
+
|
|
73
|
+
|
|
74
|
+
def categorize(function_name: str, servers=()) -> str:
|
|
75
|
+
"""The coarse category of a tool call - the one key comparable across agents.
|
|
76
|
+
|
|
77
|
+
The name it is derived from is not: agents name the same tool differently
|
|
78
|
+
(module docstring). A name no rule matches is ``other``, which is a
|
|
79
|
+
category like the rest and is summarised like the rest.
|
|
80
|
+
"""
|
|
81
|
+
tool = mcp_tool(function_name, servers)
|
|
82
|
+
if tool:
|
|
83
|
+
words = tool[1].lower().split("_")
|
|
84
|
+
if not _CODE_WORDS.intersection(words):
|
|
85
|
+
return "lookup" if words[0] in _LOOKUP_WORDS else "change" if words[0] in _CHANGE_WORDS else "other"
|
|
86
|
+
for name, pat in _CATEGORY_RULES:
|
|
87
|
+
if pat.search(tool[1] if tool else function_name or ""):
|
|
88
|
+
return name
|
|
89
|
+
return "other"
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass
|
|
93
|
+
class Step:
|
|
94
|
+
step_id: int
|
|
95
|
+
source: str
|
|
96
|
+
timestamp: str
|
|
97
|
+
message: str
|
|
98
|
+
tool_calls: list = field(default_factory=list) # [{name, category, arguments, observation}]
|
|
99
|
+
# Seconds until the next step opened — None on the last step, when either
|
|
100
|
+
# timestamp is missing, or when the whole trajectory's clock is degenerate.
|
|
101
|
+
dur: float | None = None
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
# Observations are serialised as `success=<bool> data={...} error=<repr>`, so
|
|
105
|
+
# status is explicit — no need to guess from the text. Fall back to scanning
|
|
106
|
+
# for tracebacks only when that prefix is absent (older/other writers).
|
|
107
|
+
_SUCCESS_RE = re.compile(r"^success=(True|False)\b")
|
|
108
|
+
_ERROR_RE = re.compile(r"error='((?:[^'\\]|\\.)*)'|error=\"((?:[^\"\\]|\\.)*)\"")
|
|
109
|
+
_FAILURE_MARKERS = [
|
|
110
|
+
(re.compile(r"Traceback \(most recent call last\)"), "Traceback"),
|
|
111
|
+
(re.compile(r"\b(\w*(?:Error|Exception))\b(?=:)"), None),
|
|
112
|
+
(re.compile(r"command not found"), "command not found"),
|
|
113
|
+
(re.compile(r"No such file or directory"), "No such file or directory"),
|
|
114
|
+
]
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def _first_line(text: str) -> str:
|
|
118
|
+
"""Most informative line of an error blob.
|
|
119
|
+
|
|
120
|
+
Skips the exit-code header, and for a traceback takes the *last* line —
|
|
121
|
+
"Traceback (most recent call last):" says nothing, the final
|
|
122
|
+
"ModuleNotFoundError: ..." says everything.
|
|
123
|
+
"""
|
|
124
|
+
# The blob arrives as a Python repr, so quotes and newlines are escaped.
|
|
125
|
+
text = text.replace("\\n", "\n").replace("\\'", "'").replace('\\"', '"')
|
|
126
|
+
lines = [ln.strip() for ln in text.splitlines() if ln.strip()]
|
|
127
|
+
lines = [ln for ln in lines if not ln.lower().startswith("exit code")]
|
|
128
|
+
if not lines:
|
|
129
|
+
return ""
|
|
130
|
+
if any(ln.startswith("Traceback (most recent call last)") for ln in lines):
|
|
131
|
+
return lines[-1][:160]
|
|
132
|
+
return lines[0][:160]
|
|
133
|
+
|
|
134
|
+
# Argument keys worth showing as the call's "target", most specific first.
|
|
135
|
+
_TARGET_KEYS = ("command", "cmd", "file_path", "path", "pattern", "query", "glob")
|
|
136
|
+
|
|
137
|
+
# Some agents don't name the file in an argument at all — opencode's
|
|
138
|
+
# `apply_patch` takes only `patchText`, with the path inside the payload. Both
|
|
139
|
+
# the codex-style header and a plain unified diff appear in the wild.
|
|
140
|
+
_PATCH_KEYS = ("patchText", "patch", "diff", "content")
|
|
141
|
+
PATCH_PATHS = [
|
|
142
|
+
re.compile(r"(?m)^\*\*\* (?:Update|Add|Delete) File:\s*(\S+)"),
|
|
143
|
+
re.compile(r"(?m)^\+\+\+ (?:b/)?(\S+)"),
|
|
144
|
+
]
|
|
145
|
+
# Public for the same reason as trace.clean: behavior.py reads both this and
|
|
146
|
+
# parse_ts, and it stayed with the application when the reader was packaged.
|
|
147
|
+
_PATCH_PATHS = PATCH_PATHS
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
# Container WORKDIRs an agent runs in, such as /testbed or /app. Whether a path arrives absolute or relative is the model's
|
|
151
|
+
# choice, not the harness's — one agent passed relative paths on two runs of
|
|
152
|
+
# one task and absolute on a third, same model, same task. Left alone, the same
|
|
153
|
+
# file is two different strings and every set operation over them lies:
|
|
154
|
+
# "which files did both agents touch" answered 0 when the answer was 2.
|
|
155
|
+
#
|
|
156
|
+
# Normalising here rather than at each comparison keeps one canonical form. The
|
|
157
|
+
# agent's original string stays on the tool call, because *that* difference is
|
|
158
|
+
# real signal about how the agent behaved — it just must not reach set
|
|
159
|
+
# arithmetic. `behavior._same_file` still suffix-matches for gold patches,
|
|
160
|
+
# which covers workspace roots this list doesn't know.
|
|
161
|
+
_WORKSPACE_ROOTS = ("/testbed/", "/app/")
|
|
162
|
+
|
|
163
|
+
|
|
164
|
+
def workspace_relative(path: str) -> str:
|
|
165
|
+
"""Strip a known container workspace root so paths compare equal."""
|
|
166
|
+
for root in _WORKSPACE_ROOTS:
|
|
167
|
+
if path.startswith(root):
|
|
168
|
+
return path[len(root):]
|
|
169
|
+
return path
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def patched_files(arguments: dict) -> list:
|
|
173
|
+
"""File paths named inside a patch payload, in first-seen order."""
|
|
174
|
+
if not isinstance(arguments, dict):
|
|
175
|
+
return []
|
|
176
|
+
body = next(
|
|
177
|
+
(arguments[k] for k in _PATCH_KEYS if isinstance(arguments.get(k), str)), ""
|
|
178
|
+
)
|
|
179
|
+
found = []
|
|
180
|
+
for pattern in _PATCH_PATHS:
|
|
181
|
+
for path in pattern.findall(body):
|
|
182
|
+
if path not in found and path != "/dev/null":
|
|
183
|
+
found.append(path)
|
|
184
|
+
return found
|
|
185
|
+
|
|
186
|
+
|
|
187
|
+
def failure_of(observation: str) -> str:
|
|
188
|
+
"""Describe the failure in *observation*, or '' if the call succeeded."""
|
|
189
|
+
if not observation:
|
|
190
|
+
return ""
|
|
191
|
+
status = _SUCCESS_RE.match(observation)
|
|
192
|
+
if status:
|
|
193
|
+
if status.group(1) == "True":
|
|
194
|
+
return ""
|
|
195
|
+
err = _ERROR_RE.search(observation)
|
|
196
|
+
message = _first_line(err.group(1) or err.group(2) or "") if err else ""
|
|
197
|
+
return message or "failed"
|
|
198
|
+
for pattern, label in _FAILURE_MARKERS:
|
|
199
|
+
hit = pattern.search(observation)
|
|
200
|
+
if hit:
|
|
201
|
+
return label or hit.group(1)
|
|
202
|
+
return ""
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
def target_of(arguments: dict) -> str:
|
|
206
|
+
"""A short, human-readable subject for a tool call."""
|
|
207
|
+
if not isinstance(arguments, dict):
|
|
208
|
+
return ""
|
|
209
|
+
for key in _TARGET_KEYS:
|
|
210
|
+
value = arguments.get(key)
|
|
211
|
+
if isinstance(value, str) and value.strip():
|
|
212
|
+
return " ".join(value.split())
|
|
213
|
+
# No named argument: a patch payload still tells us what was touched.
|
|
214
|
+
files = patched_files(arguments)
|
|
215
|
+
if files:
|
|
216
|
+
return files[0] if len(files) == 1 else f"{files[0]} +{len(files) - 1}"
|
|
217
|
+
return ""
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
# Two usage conventions reach this parser. OpenAI-style counts every input
|
|
221
|
+
# token in prompt_tokens and the cached subset again in cached_tokens, so
|
|
222
|
+
# cached ÷ prompt is a real hit rate. Anthropic-style reports cache *reads*
|
|
223
|
+
# (plus a few-token fresh remainder) as prompt_tokens and the new tokens as a
|
|
224
|
+
# cache write in a field of its own that never enters the total — cached ÷
|
|
225
|
+
# prompt is ~100% there by construction, however the cache performed.
|
|
226
|
+
# The number worth comparing across agents is therefore
|
|
227
|
+
# reads ÷ every input token, with the writes added back in.
|
|
228
|
+
_REMAINDER_CEILING = 16 # max per-call prompt−cached that reads as Anthropic-style
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@dataclass
|
|
232
|
+
class CacheAccounting:
|
|
233
|
+
convention: str # "openai" | "anthropic"
|
|
234
|
+
reads: int # tokens served from cache
|
|
235
|
+
new: int # every other input token: cache writes + fresh remainder
|
|
236
|
+
reported_prompt: int # final_metrics' own total_prompt_tokens
|
|
237
|
+
reported_cached: int
|
|
238
|
+
|
|
239
|
+
@property
|
|
240
|
+
def total(self) -> int:
|
|
241
|
+
return self.reads + self.new
|
|
242
|
+
|
|
243
|
+
@property
|
|
244
|
+
def hit(self) -> float | None:
|
|
245
|
+
return self.reads / self.total if self.total else None
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def reported_hit(self) -> float | None:
|
|
249
|
+
return self.reported_cached / self.reported_prompt if self.reported_prompt else None
|
|
250
|
+
|
|
251
|
+
@property
|
|
252
|
+
def reported_diverges(self) -> bool:
|
|
253
|
+
"""Whether the file's own ratio would mislead next to the real one."""
|
|
254
|
+
if self.hit is None or self.reported_hit is None:
|
|
255
|
+
return False
|
|
256
|
+
return abs(self.hit - self.reported_hit) > 0.005
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _extension(container, key):
|
|
260
|
+
"""A field ATIF does not name. Newer writers put it in the
|
|
261
|
+
`extra` slot of its level; earlier artifacts carry it beside the named
|
|
262
|
+
fields. The slot wins when both are present, the two are never summed,
|
|
263
|
+
and a null in the slot stays None - the slot said so. A non-dict `extra`
|
|
264
|
+
is no slot: the old position is read."""
|
|
265
|
+
if not isinstance(container, dict):
|
|
266
|
+
return None
|
|
267
|
+
extra = container.get("extra")
|
|
268
|
+
if isinstance(extra, dict) and key in extra:
|
|
269
|
+
return extra[key]
|
|
270
|
+
return container.get(key)
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
def _cache_accounting(series: list, final_metrics: dict) -> CacheAccounting | None:
|
|
274
|
+
"""Fold the token series into reads and new, whichever convention wrote it.
|
|
275
|
+
|
|
276
|
+
Also annotates every series entry with ``sent`` and ``new`` so the growth
|
|
277
|
+
strip can draw each call's whole input — under the Anthropic convention a
|
|
278
|
+
call's prompt_tokens is missing its write, which belongs to the call that
|
|
279
|
+
paid for it. The write of the final call never surfaces (nothing read it
|
|
280
|
+
back), so it is counted as zero rather than guessed.
|
|
281
|
+
"""
|
|
282
|
+
for e in series:
|
|
283
|
+
e["new"] = max(e["prompt"] - e["cached"], 0)
|
|
284
|
+
e["sent"] = e["prompt"]
|
|
285
|
+
usable = [e for e in series if e["prompt"] > 0]
|
|
286
|
+
|
|
287
|
+
reported_prompt = final_metrics.get("total_prompt_tokens")
|
|
288
|
+
reported_cached = final_metrics.get("total_cached_tokens")
|
|
289
|
+
if not isinstance(reported_prompt, (int, float)):
|
|
290
|
+
reported_prompt = sum(e["prompt"] for e in usable)
|
|
291
|
+
reported_cached = sum(e["cached"] for e in usable)
|
|
292
|
+
if not isinstance(reported_cached, (int, float)):
|
|
293
|
+
reported_cached = 0
|
|
294
|
+
reported_prompt, reported_cached = int(reported_prompt), int(reported_cached)
|
|
295
|
+
|
|
296
|
+
if len(usable) >= 2 and all(
|
|
297
|
+
e["prompt"] - e["cached"] <= _REMAINDER_CEILING for e in usable
|
|
298
|
+
):
|
|
299
|
+
for cur, nxt in zip(usable, usable[1:]):
|
|
300
|
+
write = max(nxt["prompt"] - cur["prompt"], 0)
|
|
301
|
+
cur["new"] += write
|
|
302
|
+
cur["sent"] = cur["cached"] + cur["new"]
|
|
303
|
+
reads = sum(e["cached"] for e in usable)
|
|
304
|
+
new = sum(e["new"] for e in usable)
|
|
305
|
+
return CacheAccounting("anthropic", reads, new, reported_prompt, reported_cached)
|
|
306
|
+
|
|
307
|
+
if usable:
|
|
308
|
+
reads = sum(e["cached"] for e in usable)
|
|
309
|
+
new = sum(e["new"] for e in usable)
|
|
310
|
+
return CacheAccounting("openai", reads, new, reported_prompt, reported_cached)
|
|
311
|
+
|
|
312
|
+
if reported_prompt:
|
|
313
|
+
reads = min(reported_cached, reported_prompt)
|
|
314
|
+
return CacheAccounting("openai", reads, reported_prompt - reads,
|
|
315
|
+
reported_prompt, reported_cached)
|
|
316
|
+
return None
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
@dataclass
|
|
320
|
+
class Trajectory:
|
|
321
|
+
n_steps: int
|
|
322
|
+
steps: list # list[Step]
|
|
323
|
+
category_freq: Counter # by coarse category - the cross-agent key
|
|
324
|
+
n_tool_calls: int
|
|
325
|
+
last_agent_message: str
|
|
326
|
+
final_metrics: dict
|
|
327
|
+
error: str = "" # set if the file could not be parsed
|
|
328
|
+
calls: list = field(default_factory=list) # flat, in order
|
|
329
|
+
files_touched: list = field(default_factory=list) # paths from edit calls
|
|
330
|
+
# False when the writer stamped every step at the same instant (older
|
|
331
|
+
# trajectories) — durations would be noise, so none are set.
|
|
332
|
+
has_step_clock: bool = False
|
|
333
|
+
# One entry per step that carries per-call metrics (ATIF v1.7 writers, and
|
|
334
|
+
# runs where a gateway-log matcher attached them):
|
|
335
|
+
# {step_id, prompt, cached, completion}. Prompt growth across this series
|
|
336
|
+
# is the caching story of the whole session in one curve.
|
|
337
|
+
token_series: list = field(default_factory=list)
|
|
338
|
+
# Reads/new fold over the series, convention detected — None when the
|
|
339
|
+
# file carries no usage at all.
|
|
340
|
+
cache: CacheAccounting | None = None
|
|
341
|
+
# Summed from metrics.extra.reasoning_tokens; None when no step has it —
|
|
342
|
+
# "not reported" and "zero reasoning" are different claims.
|
|
343
|
+
total_reasoning_tokens: int | None = None
|
|
344
|
+
# {category: Counter(name)} over the same calls as category_freq: what each
|
|
345
|
+
# category folded, in the agent's own words. Evidence beside the key, so a
|
|
346
|
+
# reader can check the fold - never a key itself (module docstring).
|
|
347
|
+
name_freq: dict = field(default_factory=dict)
|
|
348
|
+
|
|
349
|
+
@property
|
|
350
|
+
def n_failed_calls(self) -> int:
|
|
351
|
+
return sum(1 for c in self.calls if c["failure"])
|
|
352
|
+
|
|
353
|
+
@property
|
|
354
|
+
def tools(self) -> list:
|
|
355
|
+
"""Each tool once: ``{server, name, category, calls, failed, ns, spellings}``,
|
|
356
|
+
by server, the most called first. ``ns`` is ``None`` when no call has a clock."""
|
|
357
|
+
rows: dict = {}
|
|
358
|
+
for c in self.calls:
|
|
359
|
+
row = rows.setdefault((c["server"], c["tool"]), {"server": c["server"], "name": c["tool"], "category": c["category"],
|
|
360
|
+
"calls": 0, "failed": 0, "ns": None, "spellings": []})
|
|
361
|
+
row["calls"] += 1
|
|
362
|
+
row["failed"] += 1 if c["failure"] else 0
|
|
363
|
+
if c["ns"] is not None:
|
|
364
|
+
row["ns"] = (row["ns"] or 0) + c["ns"]
|
|
365
|
+
if c["name"] not in row["spellings"]:
|
|
366
|
+
row["spellings"].append(c["name"])
|
|
367
|
+
return sorted(rows.values(), key=lambda r: (bool(r["server"]) is False, r["server"], -r["calls"], r["name"]))
|
|
368
|
+
|
|
369
|
+
def names_of(self, category: str) -> list:
|
|
370
|
+
"""``[(name, count), …]`` the agent used for *category*, most frequent first."""
|
|
371
|
+
return self.name_freq.get(category, Counter()).most_common()
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
def parse_ts(value: str):
|
|
375
|
+
"""An ISO timestamp, or None — a record with an unreadable clock is not
|
|
376
|
+
an error here, it is a record whose clock we decline to guess at."""
|
|
377
|
+
if not value:
|
|
378
|
+
return None
|
|
379
|
+
try:
|
|
380
|
+
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
381
|
+
except ValueError:
|
|
382
|
+
return None
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
_parse_ts = parse_ts
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _tool_ns(step: dict) -> int | None:
|
|
389
|
+
"""How long a step's tool call ran, when the step says when it ended (a standard run's do)."""
|
|
390
|
+
extra = step.get("extra") if isinstance(step.get("extra"), dict) else {}
|
|
391
|
+
start, end = parse_ts(extra.get("running_at") or step.get("timestamp")), parse_ts(extra.get("ended_at"))
|
|
392
|
+
return max(0, int((end - start).total_seconds() * 1e9)) if start and end else None
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
# Below this, the trajectory's whole clock spread is treated as one write-time
|
|
396
|
+
# stamp, not a real per-step clock. Same refuse-don't-guess policy as the
|
|
397
|
+
# agent-side metrics matcher.
|
|
398
|
+
_MIN_CLOCK_SPAN_SEC = 1.0
|
|
399
|
+
|
|
400
|
+
|
|
401
|
+
def _attach_durations(steps: list) -> bool:
|
|
402
|
+
"""Set each step's ``dur`` from consecutive timestamps.
|
|
403
|
+
|
|
404
|
+
Returns whether the trajectory has a usable step clock at all; when it
|
|
405
|
+
does not, every ``dur`` is left None.
|
|
406
|
+
"""
|
|
407
|
+
stamps = [_parse_ts(s.timestamp) for s in steps]
|
|
408
|
+
known = [t for t in stamps if t]
|
|
409
|
+
if len(known) < 2 or (max(known) - min(known)).total_seconds() < _MIN_CLOCK_SPAN_SEC:
|
|
410
|
+
return False
|
|
411
|
+
for i, step in enumerate(steps[:-1]):
|
|
412
|
+
here, there = stamps[i], stamps[i + 1]
|
|
413
|
+
if here and there and there >= here:
|
|
414
|
+
step.dur = (there - here).total_seconds()
|
|
415
|
+
return True
|
|
416
|
+
|
|
417
|
+
|
|
418
|
+
def _extract_observation(step: dict, call_id: str) -> str:
|
|
419
|
+
obs = step.get("observation")
|
|
420
|
+
if not isinstance(obs, dict):
|
|
421
|
+
return ""
|
|
422
|
+
for r in obs.get("results", []):
|
|
423
|
+
if r.get("source_call_id") == call_id:
|
|
424
|
+
content = r.get("content", "")
|
|
425
|
+
# content is often a JSON-encoded list of {type, content:{text}}
|
|
426
|
+
try:
|
|
427
|
+
parsed = json.loads(content)
|
|
428
|
+
texts = []
|
|
429
|
+
for item in parsed:
|
|
430
|
+
c = item.get("content", {})
|
|
431
|
+
if isinstance(c, dict) and "text" in c:
|
|
432
|
+
texts.append(c["text"])
|
|
433
|
+
if texts:
|
|
434
|
+
return redact("\n".join(texts))
|
|
435
|
+
except Exception:
|
|
436
|
+
pass
|
|
437
|
+
return redact(str(content))
|
|
438
|
+
return ""
|
|
439
|
+
|
|
440
|
+
|
|
441
|
+
def message_text(message) -> str:
|
|
442
|
+
"""A step's message as text. ATIF v1.6 lets it be a list of content parts
|
|
443
|
+
({"type": "text", "text": ...}, an image, ...): the text parts joined, a
|
|
444
|
+
placeholder for each other part, so a multimodal prompt reads rather
|
|
445
|
+
than failing the page."""
|
|
446
|
+
if isinstance(message, str):
|
|
447
|
+
return message
|
|
448
|
+
if isinstance(message, dict):
|
|
449
|
+
message = [message]
|
|
450
|
+
if not isinstance(message, list):
|
|
451
|
+
return "" if message is None else str(message)
|
|
452
|
+
out = []
|
|
453
|
+
for part in message:
|
|
454
|
+
if isinstance(part, str):
|
|
455
|
+
out.append(part)
|
|
456
|
+
elif isinstance(part, dict) and isinstance(part.get("text"), str):
|
|
457
|
+
out.append(part["text"])
|
|
458
|
+
elif isinstance(part, dict):
|
|
459
|
+
out.append(f"[{part.get('type') or 'part'}]")
|
|
460
|
+
return "\n".join(out)
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def load(path: str) -> Trajectory:
|
|
464
|
+
try:
|
|
465
|
+
with open(path, encoding="utf-8") as fh:
|
|
466
|
+
data = json.load(fh)
|
|
467
|
+
except (OSError, json.JSONDecodeError) as exc:
|
|
468
|
+
return Trajectory(0, [], Counter(), 0, "", {}, error=str(exc))
|
|
469
|
+
|
|
470
|
+
steps_out: list[Step] = []
|
|
471
|
+
category_freq: Counter = Counter()
|
|
472
|
+
name_freq: dict = {}
|
|
473
|
+
n_tool_calls = 0
|
|
474
|
+
last_agent_message = ""
|
|
475
|
+
|
|
476
|
+
all_calls: list = []
|
|
477
|
+
files_touched: list = []
|
|
478
|
+
token_series: list = []
|
|
479
|
+
reasoning_total: int | None = None
|
|
480
|
+
servers = (data.get("extra") or {}).get("mcp_servers") or ()
|
|
481
|
+
|
|
482
|
+
for raw in data.get("steps", []):
|
|
483
|
+
metrics = raw.get("metrics")
|
|
484
|
+
if isinstance(metrics, dict):
|
|
485
|
+
token_series.append(
|
|
486
|
+
{
|
|
487
|
+
"step_id": raw.get("step_id", 0),
|
|
488
|
+
"prompt": metrics.get("prompt_tokens") or 0,
|
|
489
|
+
"cached": metrics.get("cached_tokens") or 0,
|
|
490
|
+
"completion": metrics.get("completion_tokens") or 0,
|
|
491
|
+
}
|
|
492
|
+
)
|
|
493
|
+
reasoning = (metrics.get("extra") or {}).get("reasoning_tokens")
|
|
494
|
+
if reasoning is not None:
|
|
495
|
+
reasoning_total = (reasoning_total or 0) + reasoning
|
|
496
|
+
calls_out = []
|
|
497
|
+
for tc in raw.get("tool_calls") or []:
|
|
498
|
+
name = tc.get("function_name") or tc.get("name") or "(tool)"
|
|
499
|
+
cat = categorize(name, servers)
|
|
500
|
+
server, tool = mcp_tool(name, servers) or ("", name)
|
|
501
|
+
args = tc.get("arguments", {})
|
|
502
|
+
observation = _extract_observation(raw, tc.get("tool_call_id"))
|
|
503
|
+
call = {
|
|
504
|
+
"step_id": raw.get("step_id", 0),
|
|
505
|
+
# An ATIF id is source evidence, not the call's position. Keep
|
|
506
|
+
# it so parallel tools and their observations stay identifiable.
|
|
507
|
+
"tool_call_id": tc.get("tool_call_id"),
|
|
508
|
+
"name": redact(name),
|
|
509
|
+
"server": redact(server),
|
|
510
|
+
"tool": redact(tool),
|
|
511
|
+
"ns": _tool_ns(raw),
|
|
512
|
+
"category": cat,
|
|
513
|
+
"target": redact(target_of(args)),
|
|
514
|
+
"arguments": redact(json.dumps(args, ensure_ascii=False)),
|
|
515
|
+
"observation": observation,
|
|
516
|
+
# Size of what came back — the closest thing to "work done" this
|
|
517
|
+
# schema offers, since the timestamps are not per-step.
|
|
518
|
+
"obs_bytes": len(observation),
|
|
519
|
+
"failure": failure_of(observation),
|
|
520
|
+
}
|
|
521
|
+
calls_out.append(call)
|
|
522
|
+
# The writer emits placeholder entries — no name, no arguments, no
|
|
523
|
+
# observation — which would otherwise double the tool-call count.
|
|
524
|
+
# The summaries count the same calls the list keeps, so the chips,
|
|
525
|
+
# the names behind them and the total describe one population.
|
|
526
|
+
if args or observation or name != "Using tool":
|
|
527
|
+
all_calls.append(call)
|
|
528
|
+
n_tool_calls += 1
|
|
529
|
+
category_freq[cat] += 1
|
|
530
|
+
name_freq.setdefault(cat, Counter())[call["name"]] += 1
|
|
531
|
+
if cat == "edit":
|
|
532
|
+
# Prefer a named argument; fall back to paths inside the patch
|
|
533
|
+
# body for agents that pass only a diff.
|
|
534
|
+
touched = patched_files(args) or ([target_of(args)] if target_of(args) else [])
|
|
535
|
+
for path in touched:
|
|
536
|
+
path = workspace_relative(path)
|
|
537
|
+
if path and path not in files_touched:
|
|
538
|
+
files_touched.append(path)
|
|
539
|
+
|
|
540
|
+
msg = redact(message_text(raw.get("message")))
|
|
541
|
+
src = raw.get("source", "")
|
|
542
|
+
if src == "agent" and msg and msg != "(tool use)":
|
|
543
|
+
last_agent_message = msg
|
|
544
|
+
|
|
545
|
+
steps_out.append(
|
|
546
|
+
Step(
|
|
547
|
+
step_id=raw.get("step_id", 0),
|
|
548
|
+
source=src,
|
|
549
|
+
timestamp=raw.get("timestamp", ""),
|
|
550
|
+
message=msg,
|
|
551
|
+
tool_calls=calls_out,
|
|
552
|
+
)
|
|
553
|
+
)
|
|
554
|
+
|
|
555
|
+
has_clock = _attach_durations(steps_out)
|
|
556
|
+
# A call inherits its step's span (None without a clock) — coarse, but the
|
|
557
|
+
# schema gives one timestamp per step, not per call.
|
|
558
|
+
for step in steps_out:
|
|
559
|
+
for call in step.tool_calls:
|
|
560
|
+
call["dur"] = step.dur
|
|
561
|
+
|
|
562
|
+
# Agents whose LLM calls cannot be matched to steps leave per-step metrics
|
|
563
|
+
# unset and report the total in final_metrics instead. Reading only the
|
|
564
|
+
# per-step path renders nothing while the number sits in the file.
|
|
565
|
+
if reasoning_total is None:
|
|
566
|
+
fallback = _extension(data.get("final_metrics") or {}, "total_reasoning_tokens")
|
|
567
|
+
if isinstance(fallback, (int, float)):
|
|
568
|
+
reasoning_total = int(fallback)
|
|
569
|
+
|
|
570
|
+
cache = _cache_accounting(token_series, data.get("final_metrics") or {})
|
|
571
|
+
|
|
572
|
+
return Trajectory(
|
|
573
|
+
n_steps=len(steps_out),
|
|
574
|
+
steps=steps_out,
|
|
575
|
+
category_freq=category_freq,
|
|
576
|
+
n_tool_calls=n_tool_calls,
|
|
577
|
+
last_agent_message=last_agent_message,
|
|
578
|
+
final_metrics=data.get("final_metrics") or {},
|
|
579
|
+
calls=all_calls,
|
|
580
|
+
files_touched=files_touched,
|
|
581
|
+
has_step_clock=has_clock,
|
|
582
|
+
token_series=token_series,
|
|
583
|
+
cache=cache,
|
|
584
|
+
total_reasoning_tokens=reasoning_total,
|
|
585
|
+
name_freq=name_freq,
|
|
586
|
+
)
|