nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/standard/job.py
ADDED
|
@@ -0,0 +1,551 @@
|
|
|
1
|
+
"""A run written as a job in Harbor's shape, so every page reads it unchanged.
|
|
2
|
+
|
|
3
|
+
``<jobs>/<job>/``: ``config.json`` (what was run: the benchmark, the agent or
|
|
4
|
+
capability, the model), ``result.json`` (the counts), and one folder per unit -
|
|
5
|
+
a trial - holding:
|
|
6
|
+
|
|
7
|
+
- ``result.json``: the unit, the agent, the timings, the verdict
|
|
8
|
+
(``verifier_result.rewards``) or the exception that ended it, the tokens and
|
|
9
|
+
the cost the relay recorded;
|
|
10
|
+
- ``agent/trajectory.json``: ATIF, built from what Nullius saw - the session's
|
|
11
|
+
events over ACP (the prompt, the agent's messages and thoughts, each tool
|
|
12
|
+
call with its result) or, for a capability, each request it was sent and its
|
|
13
|
+
reply - with the model calls' tokens and cost from the relay;
|
|
14
|
+
- ``agent/trace.json``: the same run as a timeline, ``harbor-trace-v1``: each
|
|
15
|
+
model call as the relay timed it, each tool call as Nullius saw it start and
|
|
16
|
+
end, under the agent's turns. What the agent does inside is not seen;
|
|
17
|
+
- ``agent/calls.jsonl``: the relay's record of each model call, and
|
|
18
|
+
``agent/gateway.log``: one line per call with its status, where the runs
|
|
19
|
+
pages count rate limits;
|
|
20
|
+
- ``verifier/reward.txt`` and ``verifier/score.json``: the driver's verdict
|
|
21
|
+
and its detail.
|
|
22
|
+
"""
|
|
23
|
+
from __future__ import annotations
|
|
24
|
+
|
|
25
|
+
import json
|
|
26
|
+
import shutil
|
|
27
|
+
import os
|
|
28
|
+
import re
|
|
29
|
+
from datetime import datetime, timezone
|
|
30
|
+
from typing import Dict, List, Optional
|
|
31
|
+
|
|
32
|
+
from nullius.standard import ledger
|
|
33
|
+
from nullius.traces import otel, trace
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def iso(t: Optional[float]) -> Optional[str]:
|
|
37
|
+
return datetime.fromtimestamp(t, timezone.utc).isoformat() if t else None
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _text(content) -> str:
|
|
41
|
+
"""ACP content: a block, a list of blocks, or a tool call's content list."""
|
|
42
|
+
if content is None:
|
|
43
|
+
return ""
|
|
44
|
+
if isinstance(content, str):
|
|
45
|
+
return content
|
|
46
|
+
if isinstance(content, dict):
|
|
47
|
+
if content.get("type") == "text":
|
|
48
|
+
return content.get("text") or ""
|
|
49
|
+
if content.get("type") == "content":
|
|
50
|
+
return _text(content.get("content"))
|
|
51
|
+
if content.get("type") == "diff":
|
|
52
|
+
return f"diff {content.get('path')}\n{content.get('newText') or ''}"[:4000]
|
|
53
|
+
return json.dumps(content)[:4000]
|
|
54
|
+
if isinstance(content, list):
|
|
55
|
+
return "\n".join(_text(c) for c in content)
|
|
56
|
+
return str(content)
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _usage_total(calls: List[dict]) -> dict:
|
|
60
|
+
ok = [c for c in calls if c.get("status") == 200]
|
|
61
|
+
u = lambda k: sum(((c.get("reply") or {}).get("usage") or {}).get(k) or 0 for c in ok)
|
|
62
|
+
costs = [c["cost_usd"] for c in ok if isinstance(c.get("cost_usd"), (int, float))]
|
|
63
|
+
return {"input": u("input"), "output": u("output"), "cached": u("cached"), "reasoning": u("reasoning"),
|
|
64
|
+
"cost": sum(costs) if costs else None,
|
|
65
|
+
"cost_from": "estimate" if any(c.get("cost_from") == "estimate" for c in ok) else ("provider" if costs else None),
|
|
66
|
+
"n_calls": sum(1 for c in calls if not c.get("refused"))} # a call its cap refused was never sent
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def _tool_clock(extra: dict, status: Optional[str], t: Optional[float]):
|
|
70
|
+
"""When a tool call began to run (announced earlier, it may still wait for the
|
|
71
|
+
model's reply to end) and when it ended, for the trace."""
|
|
72
|
+
if status == "in_progress":
|
|
73
|
+
extra.setdefault("running_at", iso(t))
|
|
74
|
+
elif status in ("completed", "failed"):
|
|
75
|
+
extra["ended_at"] = iso(t)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def trajectory_from_events(events: List[dict], calls: List[dict], agent: dict, model: str, session: str) -> dict:
|
|
79
|
+
"""ATIF from an ACP session's events: a user step per prompt, an agent
|
|
80
|
+
step per stretch of message, and one per tool call with its result as the
|
|
81
|
+
observation. The relay's calls give the totals."""
|
|
82
|
+
steps: List[dict] = []
|
|
83
|
+
tools = {} # toolCallId -> its step
|
|
84
|
+
|
|
85
|
+
def add(source, t, **kw):
|
|
86
|
+
steps.append(dict({"step_id": len(steps) + 1, "timestamp": iso(t), "source": source}, **kw))
|
|
87
|
+
return steps[-1]
|
|
88
|
+
|
|
89
|
+
open_msg = None
|
|
90
|
+
message_id = None
|
|
91
|
+
for e in events:
|
|
92
|
+
u, t = e.get("update") or {}, e.get("t")
|
|
93
|
+
kind = u.get("sessionUpdate")
|
|
94
|
+
if kind == "user_message_chunk":
|
|
95
|
+
open_msg = None
|
|
96
|
+
add("user", t, message=_text(u.get("content")))
|
|
97
|
+
elif kind in ("agent_message_chunk", "agent_thought_chunk"):
|
|
98
|
+
chunk_id = u.get("messageId") if kind == "agent_message_chunk" else None
|
|
99
|
+
if open_msg is None or (chunk_id and message_id and chunk_id != message_id):
|
|
100
|
+
open_msg = add("agent", t, message="", model_name=model)
|
|
101
|
+
message_id = None
|
|
102
|
+
message_id = chunk_id or message_id
|
|
103
|
+
if kind == "agent_message_chunk":
|
|
104
|
+
open_msg["message"] += _text(u.get("content"))
|
|
105
|
+
else:
|
|
106
|
+
open_msg["reasoning_content"] = open_msg.get("reasoning_content", "") + _text(u.get("content"))
|
|
107
|
+
elif kind == "tool_call":
|
|
108
|
+
open_msg = None
|
|
109
|
+
cid = u.get("toolCallId") or f"call-{len(tools) + 1}"
|
|
110
|
+
st = add("agent", t, message="", model_name=model,
|
|
111
|
+
tool_calls=[{"tool_call_id": cid, "function_name": u.get("title") or u.get("kind") or "tool",
|
|
112
|
+
"arguments": u.get("rawInput") if isinstance(u.get("rawInput"), dict) else {"input": u.get("rawInput")}}],
|
|
113
|
+
extra={"kind": u.get("kind"), "status": u.get("status")})
|
|
114
|
+
tools[cid] = st
|
|
115
|
+
_tool_clock(st["extra"], u.get("status"), t)
|
|
116
|
+
if u.get("content") or u.get("rawOutput") is not None:
|
|
117
|
+
st["observation"] = {"results": [{"source_call_id": cid, "content": _text(u.get("content")) or json.dumps(u.get("rawOutput"))[:8000]}]}
|
|
118
|
+
elif kind == "tool_call_update":
|
|
119
|
+
st = tools.get(u.get("toolCallId"))
|
|
120
|
+
if st is None:
|
|
121
|
+
continue
|
|
122
|
+
if u.get("status"):
|
|
123
|
+
st["extra"]["status"] = u["status"]
|
|
124
|
+
_tool_clock(st["extra"], u["status"], t)
|
|
125
|
+
out = _text(u.get("content")) if u.get("content") else (json.dumps(u.get("rawOutput"))[:8000] if u.get("rawOutput") is not None else "")
|
|
126
|
+
if out:
|
|
127
|
+
if u.get("status") == "failed" and not out.lower().startswith(("error", "failed")):
|
|
128
|
+
out = "Error: " + out
|
|
129
|
+
st["observation"] = {"results": [{"source_call_id": u.get("toolCallId"), "content": out}]}
|
|
130
|
+
elif kind == "permission":
|
|
131
|
+
open_msg = None
|
|
132
|
+
add("system", t, message=("granted" if u.get("granted") else "refused") + " permission: " +
|
|
133
|
+
_text((u.get("toolCall") or {}).get("title") or ""))
|
|
134
|
+
elif kind == "notice":
|
|
135
|
+
# an adapter's warning, sent apart from the agent's message: the client takes notices (acp)
|
|
136
|
+
open_msg = None
|
|
137
|
+
add("system", t, message=f"{u.get('severity') or 'notice'}: {u.get('title') or ''}"
|
|
138
|
+
+ (f" - {u['description']}" if u.get("description") else ""))
|
|
139
|
+
elif kind == "plan":
|
|
140
|
+
add("agent", t, message="plan:\n" + "\n".join(f"- [{x.get('status')}] {x.get('content')}" for x in u.get("entries") or []),
|
|
141
|
+
model_name=model)
|
|
142
|
+
tot = _usage_total(calls)
|
|
143
|
+
return {"schema_version": "ATIF-v1.6", "session_id": session,
|
|
144
|
+
"agent": {"name": agent.get("key"), "version": str((agent.get("upstream") or {}).get("version") or ""), "model_name": model},
|
|
145
|
+
"steps": steps,
|
|
146
|
+
"final_metrics": {"total_prompt_tokens": tot["input"], "total_completion_tokens": tot["output"],
|
|
147
|
+
"total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}}
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
REPLY_CHARS = 100_000 # a capability's reply in its trajectory: its memory and its trace, one request's
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def trajectory_from_exchanges(exchanges: List[dict], calls: List[dict], capability: dict, model: str, session: str) -> dict:
|
|
154
|
+
"""ATIF for a capability: each request Nullius sent it (``add``, ``recall``)
|
|
155
|
+
as a tool call and its reply as the observation, whole up to ``REPLY_CHARS``
|
|
156
|
+
(a reply's ``trace`` is what the library did); then the benchmark's own
|
|
157
|
+
model calls, from the relay, as the reader's steps."""
|
|
158
|
+
steps = []
|
|
159
|
+
for i, x in enumerate(exchanges):
|
|
160
|
+
cid = f"{x['op']}-{i + 1}"
|
|
161
|
+
steps.append({"step_id": len(steps) + 1, "timestamp": iso(x.get("t0")), "source": "agent",
|
|
162
|
+
"message": "", "tool_calls": [{"tool_call_id": cid, "function_name": f"{capability['key']}.{x['op']}",
|
|
163
|
+
"arguments": x.get("request") or {}}],
|
|
164
|
+
"extra": {"ended_at": iso(x.get("t1"))},
|
|
165
|
+
"observation": {"results": [{"source_call_id": cid, "content": json.dumps(x.get("reply"))[:REPLY_CHARS]}]}})
|
|
166
|
+
for c in calls:
|
|
167
|
+
last = ((c.get("request") or {}).get("last") or [{}])[-1]
|
|
168
|
+
steps.append({"step_id": len(steps) + 1, "timestamp": iso(c.get("t0")), "source": "user", "message": last.get("content") or ""})
|
|
169
|
+
reply = c.get("reply") or {}
|
|
170
|
+
st = {"step_id": len(steps) + 1, "timestamp": iso(c.get("t1")), "source": "agent", "model_name": c.get("model"),
|
|
171
|
+
"message": reply.get("text") if c.get("status") == 200 else "(" + (c.get("error") or "failed") + ")"}
|
|
172
|
+
if reply.get("usage"):
|
|
173
|
+
st["metrics"] = {"prompt_tokens": reply["usage"]["input"], "completion_tokens": reply["usage"]["output"],
|
|
174
|
+
"cached_tokens": reply["usage"]["cached"]}
|
|
175
|
+
if isinstance(c.get("cost_usd"), (int, float)):
|
|
176
|
+
st["metrics"]["cost_usd"] = c["cost_usd"]
|
|
177
|
+
steps.append(st)
|
|
178
|
+
tot = _usage_total(calls)
|
|
179
|
+
return {"schema_version": "ATIF-v1.6", "session_id": session,
|
|
180
|
+
"agent": {"name": capability.get("key"), "version": "", "model_name": model},
|
|
181
|
+
"steps": steps,
|
|
182
|
+
"final_metrics": {"total_prompt_tokens": tot["input"], "total_completion_tokens": tot["output"],
|
|
183
|
+
"total_cached_tokens": tot["cached"], "total_cost_usd": tot["cost"], "total_steps": len(steps)}}
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
def _control(other: dict) -> dict:
|
|
187
|
+
"""A control's run says so, and who wrote the control: the runs pages read
|
|
188
|
+
a standard job named ``nop`` or ``oracle`` as a control only by this."""
|
|
189
|
+
if other.get("_kind") != "control":
|
|
190
|
+
return {}
|
|
191
|
+
return {"control": {"name": other["key"], "source": other.get("source", ""), "from": other.get("from", "")}}
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
def _setup(other: dict) -> dict:
|
|
195
|
+
"""A setup's run says which one it is, the model the upstream's code was
|
|
196
|
+
asked as and the one sent in its place, and whether its scores compare with
|
|
197
|
+
the paper's: only the original's do."""
|
|
198
|
+
if other.get("_kind") != "upstream":
|
|
199
|
+
return {}
|
|
200
|
+
return {"setup": {"name": other["key"], "asked": other.get("asked", ""), "sent": other.get("sent", ""),
|
|
201
|
+
"comparable_with_paper": bool(other.get("comparable"))}}
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def _version(bench: dict) -> dict:
|
|
205
|
+
"""A benchmark with versions says which one ran (``benchmark_version``), and the
|
|
206
|
+
``key@version`` that names it; one without says nothing more than its key."""
|
|
207
|
+
v = bench.get("_version") if "@" in str(bench.get("_ref") or "") else None
|
|
208
|
+
return {"benchmark_version": v, "benchmark_ref": bench["_ref"]} if v else {}
|
|
209
|
+
|
|
210
|
+
|
|
211
|
+
def _write(path: str, text: str) -> None:
|
|
212
|
+
"""A trial file written as a new regular file: whatever is at the path -
|
|
213
|
+
a link the agent planted in a folder it could write, above all - is
|
|
214
|
+
replaced, never written through (verifier/score.json -> ~/.bashrc)."""
|
|
215
|
+
if os.path.islink(path) or (os.path.lexists(path) and not os.path.isfile(path)):
|
|
216
|
+
if os.path.isdir(path) and not os.path.islink(path):
|
|
217
|
+
shutil.rmtree(path)
|
|
218
|
+
else:
|
|
219
|
+
os.unlink(path)
|
|
220
|
+
fd = os.open(path, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | getattr(os, "O_NOFOLLOW", 0), 0o644)
|
|
221
|
+
with os.fdopen(fd, "w") as f:
|
|
222
|
+
f.write(text)
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def mean_score(results: List[dict]):
|
|
226
|
+
"""The trials' mean score as the pages read it (parse.Job.mean_score): a
|
|
227
|
+
trial scored over several parts weighs as many. None when none was scored."""
|
|
228
|
+
scored = []
|
|
229
|
+
for r in results:
|
|
230
|
+
v = r.get("verifier_result") or {}
|
|
231
|
+
s = (v.get("rewards") or {}).get("score")
|
|
232
|
+
if isinstance(s, (int, float)) and not isinstance(s, bool):
|
|
233
|
+
scored.append((s, (v.get("parts") or {}).get("n") or 1))
|
|
234
|
+
return sum(s * n for s, n in scored) / sum(n for _, n in scored) if scored else None
|
|
235
|
+
|
|
236
|
+
|
|
237
|
+
def _ns(stamp) -> Optional[int]:
|
|
238
|
+
"""A time as the trace's nanoseconds: a clock's seconds, or a step's ISO time."""
|
|
239
|
+
try:
|
|
240
|
+
if isinstance(stamp, str):
|
|
241
|
+
stamp = datetime.fromisoformat(stamp).timestamp()
|
|
242
|
+
return int(stamp * 1e9) if stamp else None
|
|
243
|
+
except (TypeError, ValueError, OverflowError, OSError):
|
|
244
|
+
return None
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _span(span_id: str, name: str, kind: str, source: str, start: int, end: Optional[int], **fields) -> dict:
|
|
248
|
+
"""One span. No end: a moment when it says so (``measured``), else a call seen to start and never to end."""
|
|
249
|
+
moment = fields.pop("moment", False)
|
|
250
|
+
span = {"span_id": span_id, "name": name, "kind": kind, "start_ns": start, "source": source,
|
|
251
|
+
"timing_quality": "measured" if end is not None or moment else "unknown", "attributes": {}, "children": [], **fields}
|
|
252
|
+
if end is not None:
|
|
253
|
+
span["end_ns"], span["duration_ns"] = end, end - start
|
|
254
|
+
return span
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def _short_title(title: str) -> str:
|
|
258
|
+
"""A tool's title for its row and its filter chip: each path as its last part
|
|
259
|
+
(``Fetching file:///var/tmp/run/calc.py``: ``Fetching calc.py``). The whole title stays in the span's attributes."""
|
|
260
|
+
return re.sub(r"[^\s'\"]+", lambda word: word.group().rsplit("/", 1)[-1] or word.group(), title)
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _call_span(n: int, c: dict) -> Optional[dict]:
|
|
264
|
+
"""A model call, as the relay timed it."""
|
|
265
|
+
start, end = _ns(c.get("t0")), _ns(c.get("t1"))
|
|
266
|
+
if start is None:
|
|
267
|
+
return None
|
|
268
|
+
reply = c.get("reply") or {}
|
|
269
|
+
usage = reply.get("usage") or {}
|
|
270
|
+
tokens = {name: usage[k] for k, name in (("input", "input_tokens"), ("output", "output_tokens"), ("cached", "cache_read_tokens"),
|
|
271
|
+
("cache_write", "cache_write_tokens"), ("reasoning", "reasoning_tokens")) if usage.get(k)}
|
|
272
|
+
attributes = {k: c[k] for k in ("model", "model_asked", "path", "status", "error", "cost_usd", "cost_from") if c.get(k) not in (None, "")}
|
|
273
|
+
named = [str(x.get("name")) for x in reply.get("tool_calls") or [] if isinstance(x, dict) and x.get("name")]
|
|
274
|
+
if named:
|
|
275
|
+
attributes["tool_calls"] = ", ".join(named)
|
|
276
|
+
last = ((c.get("request") or {}).get("last") or [{}])[-1]
|
|
277
|
+
content = {"input": _text(last.get("content")) if isinstance(last, dict) else "", "output": _text(reply.get("text"))}
|
|
278
|
+
# named for its model: under the name ``llm_request`` the reader would call it by what it produced, as an agent's message reads
|
|
279
|
+
return _span(f"llm-{n}", str(c.get("model") or "model call"), "llm_request", "relay", start, end, tokens=tokens, content=content, attributes=attributes)
|
|
280
|
+
|
|
281
|
+
|
|
282
|
+
def _step_spans(n: int, st: dict, source: str) -> List[dict]:
|
|
283
|
+
"""What a trajectory step adds to the relay's calls: its tool call, or what the agent said at that moment."""
|
|
284
|
+
at = _ns(st.get("timestamp"))
|
|
285
|
+
if at is None:
|
|
286
|
+
return []
|
|
287
|
+
extra = st.get("extra") or {}
|
|
288
|
+
if st.get("tool_calls"):
|
|
289
|
+
call = st["tool_calls"][0]
|
|
290
|
+
said = ((st.get("observation") or {}).get("results") or [{}])[0].get("content")
|
|
291
|
+
name = str(call.get("function_name") or "tool")
|
|
292
|
+
return [_span(f"tool-{n}", name, "tool", source, _ns(extra.get("running_at")) or at, _ns(extra.get("ended_at")),
|
|
293
|
+
content={"arguments": call.get("arguments"), "result": said},
|
|
294
|
+
attributes={"tool_name": _short_title(name), "title": name, **{k: extra[k] for k in ("kind", "status") if extra.get(k)}})]
|
|
295
|
+
spans = []
|
|
296
|
+
if st.get("reasoning_content"):
|
|
297
|
+
spans.append(_span(f"reasoning-{n}", "reasoning", "reasoning", source, at, None, moment=True, content={"reasoning": st["reasoning_content"]}))
|
|
298
|
+
if st.get("message"):
|
|
299
|
+
kind = "system" if st.get("source") == "system" else "output"
|
|
300
|
+
spans.append(_span(f"{kind}-{n}", kind, kind, source, at, None, moment=True, content={"output": st["message"]}))
|
|
301
|
+
return spans
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def trace_from_run(trajectory: dict, calls: List[dict], other: dict, produce: tuple) -> Optional[dict]:
|
|
305
|
+
"""``agent/trace.json``: the agent's window as one session, its turns in it
|
|
306
|
+
(an agent's only: a capability's and a setup's user steps are the benchmark's
|
|
307
|
+
own model calls), each holding the model calls and tool calls that began in
|
|
308
|
+
it. Nothing recorded, no trace: a control has none to show."""
|
|
309
|
+
agent = other.get("_kind") == "agent"
|
|
310
|
+
source = "acp" if agent else "driver"
|
|
311
|
+
steps = trajectory.get("steps") or []
|
|
312
|
+
spans = [s for s in (_call_span(i + 1, c) for i, c in enumerate(calls)) if s is not None]
|
|
313
|
+
turns = []
|
|
314
|
+
for i, st in enumerate(steps):
|
|
315
|
+
if agent and st.get("source") == "user":
|
|
316
|
+
at = _ns(st.get("timestamp"))
|
|
317
|
+
if at is not None:
|
|
318
|
+
turns.append(_span(f"turn-{len(turns) + 1}", f"turn {len(turns) + 1}", "user_message", source, at, None, moment=True,
|
|
319
|
+
content={"input": st.get("message")}))
|
|
320
|
+
elif agent or st.get("tool_calls"):
|
|
321
|
+
spans.extend(_step_spans(i + 1, st, source))
|
|
322
|
+
if not spans and not turns:
|
|
323
|
+
return None
|
|
324
|
+
ends = lambda rows: [s["end_ns"] if "end_ns" in s else s["start_ns"] for s in rows]
|
|
325
|
+
first = min(s["start_ns"] for s in spans + turns)
|
|
326
|
+
begin, finish = produce if produce else (None, None)
|
|
327
|
+
begin = min(_ns(begin) or first, first)
|
|
328
|
+
finish = max([_ns(finish) or 0] + ends(spans + turns))
|
|
329
|
+
session = _span("session", str(other.get("key") or "agent"), "session", "nullius", begin, finish)
|
|
330
|
+
for span in sorted(spans, key=lambda s: s["start_ns"]):
|
|
331
|
+
holder = next((t for t in reversed(turns) if t["start_ns"] <= span["start_ns"]), session)
|
|
332
|
+
span["parent_span_id"] = holder["span_id"]
|
|
333
|
+
holder["children"].append(span)
|
|
334
|
+
for turn in turns:
|
|
335
|
+
turn["parent_span_id"] = "session"
|
|
336
|
+
if turn["children"]:
|
|
337
|
+
end = max(ends(turn["children"]))
|
|
338
|
+
turn.update(end_ns=end, duration_ns=end - turn["start_ns"])
|
|
339
|
+
session["children"] = sorted(session["children"] + turns, key=lambda s: s["start_ns"])
|
|
340
|
+
return {"schema_version": otel.SCHEMA, "producer": "app/nullius/standard/job.py", "trial_name": trajectory.get("session_id") or "",
|
|
341
|
+
"session_id": trajectory.get("session_id") or "", "agent": trajectory.get("agent") or {},
|
|
342
|
+
"clock": {"source": "this machine's clock, as the relay and the ACP client saw each call",
|
|
343
|
+
"note": "wall_ns is the agent's window; a tool call's time is what the agent reported"},
|
|
344
|
+
"spans": [session], "totals": trace.breakdown([session], finish - begin).as_totals()}
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def _jsonl(path: str) -> List[dict]:
|
|
348
|
+
with open(path, errors="replace") as f:
|
|
349
|
+
return [json.loads(line) for line in f if line.strip()]
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def trial_dirs(root: str) -> List[str]:
|
|
353
|
+
"""The trials under a job, or under a folder of jobs. Two levels, no deeper: a trial holds its agent's own files."""
|
|
354
|
+
jobs = [root] if os.path.isfile(os.path.join(root, "config.json")) else [e.path for e in os.scandir(root) if e.is_dir(follow_symlinks=False)]
|
|
355
|
+
return sorted(e.path for j in jobs for e in os.scandir(j) if e.is_dir(follow_symlinks=False) and os.path.isfile(os.path.join(e.path, "result.json")))
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def backfill_trace(trial_dir: str, write: bool = False) -> str:
|
|
359
|
+
"""Give a standard trial written before traces its ``agent/trace.json``, from
|
|
360
|
+
the files it kept. Says which: ``full`` (it kept its ACP events: tool calls
|
|
361
|
+
with their times), ``partial`` (none kept: the relay's model calls, and any
|
|
362
|
+
tool call at its start, with no duration), ``empty`` (nothing recorded),
|
|
363
|
+
``control``, ``has_trace`` (never replaced: a capture cannot be taken again),
|
|
364
|
+
or ``not_standard``."""
|
|
365
|
+
with open(os.path.join(trial_dir, "result.json")) as f:
|
|
366
|
+
res = json.load(f)
|
|
367
|
+
if "standard" not in res:
|
|
368
|
+
return "not_standard"
|
|
369
|
+
agent_dir = os.path.join(trial_dir, "agent")
|
|
370
|
+
target = os.path.join(agent_dir, "trace.json")
|
|
371
|
+
if os.path.lexists(target):
|
|
372
|
+
return "has_trace"
|
|
373
|
+
if "control" in res["standard"]:
|
|
374
|
+
return "control"
|
|
375
|
+
with open(os.path.join(os.path.dirname(trial_dir), "config.json")) as f:
|
|
376
|
+
kind = (json.load(f).get("standard") or {}).get("kind")
|
|
377
|
+
info = res.get("agent_info") or {}
|
|
378
|
+
other = {"key": info.get("name"), "_kind": kind, "upstream": {"version": info.get("version")}}
|
|
379
|
+
calls_at = os.path.join(agent_dir, "calls.jsonl")
|
|
380
|
+
calls = _jsonl(calls_at) if os.path.isfile(calls_at) else []
|
|
381
|
+
session = f"{res['standard'].get('job') or ''}/{res['standard'].get('unit') or ''}"
|
|
382
|
+
events_at, kept_at = os.path.join(agent_dir, "acp.jsonl"), os.path.join(agent_dir, "trajectory.json")
|
|
383
|
+
full = kind == "agent" and os.path.isfile(events_at)
|
|
384
|
+
if full:
|
|
385
|
+
trajectory = trajectory_from_events(_jsonl(events_at), calls, other, (info.get("model_info") or {}).get("name") or "", session)
|
|
386
|
+
elif os.path.isfile(kept_at):
|
|
387
|
+
with open(kept_at) as f:
|
|
388
|
+
trajectory = json.load(f)
|
|
389
|
+
else:
|
|
390
|
+
trajectory = {"session_id": session}
|
|
391
|
+
window = res.get("agent_execution") or {}
|
|
392
|
+
clock = lambda name: datetime.fromisoformat(window[name]).timestamp() if window.get(name) else None
|
|
393
|
+
timeline = trace_from_run(trajectory, calls, other, (clock("started_at"), clock("finished_at")))
|
|
394
|
+
if timeline is None:
|
|
395
|
+
return "empty"
|
|
396
|
+
if write:
|
|
397
|
+
fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o644)
|
|
398
|
+
with os.fdopen(fd, "w") as f:
|
|
399
|
+
json.dump(timeline, f, indent=1)
|
|
400
|
+
return "full" if full else "partial"
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
def write_trial(trial_dir: str, *, job: str, bench: dict, other: dict, model: str, unit: str, t0: float, t1: float,
|
|
404
|
+
produce: tuple, score: Optional[dict], exception: Optional[tuple], calls: List[dict],
|
|
405
|
+
trajectory: Optional[dict], extra_logs: Optional[dict] = None, environment: str = "",
|
|
406
|
+
image: str = "", setup_end: Optional[float] = None, checking: tuple = (None, None)) -> dict:
|
|
407
|
+
"""Write one unit's trial and return its result. ``environment``: the
|
|
408
|
+
benchmark's environment the unit asked for (none: the base image), and
|
|
409
|
+
``image``, the one it ran in on the container backend - names only.
|
|
410
|
+
``setup_end``: when the unit's environment was ready and the agent began
|
|
411
|
+
(its setup, relay and container from ``t0``); ``checking``: the check's
|
|
412
|
+
start and end (its own step in the container, then the driver's score)."""
|
|
413
|
+
os.makedirs(os.path.join(trial_dir, "agent"), exist_ok=True)
|
|
414
|
+
os.makedirs(os.path.join(trial_dir, "verifier"), exist_ok=True)
|
|
415
|
+
tot = _usage_total(calls)
|
|
416
|
+
# the pages read a task's name after its last slash: a unit's own slashes
|
|
417
|
+
# (a unit id like "set/12") would leave only its last part
|
|
418
|
+
res = {"task_name": f"{bench['key']}/{unit.replace('/', '-')}","trial_name": os.path.basename(trial_dir), "source": bench["key"],
|
|
419
|
+
"started_at": iso(t0), "finished_at": iso(t1),
|
|
420
|
+
"agent_info": {"name": other["key"], "version": str((other.get("upstream") or {}).get("version") or ""),
|
|
421
|
+
"model_info": {"name": model}},
|
|
422
|
+
"environment_setup": {"started_at": iso(t0), "finished_at": iso(setup_end)} if setup_end else None,
|
|
423
|
+
"agent_execution": {"started_at": iso(produce[0]), "finished_at": iso(produce[1])} if produce and produce[0] else None,
|
|
424
|
+
"verifier": {"started_at": iso(checking[0]), "finished_at": iso(checking[1])} if checking and checking[0] and checking[1] else None,
|
|
425
|
+
"agent_result": {"n_input_tokens": tot["input"], "n_output_tokens": tot["output"], "n_cache_tokens": tot["cached"],
|
|
426
|
+
"cost_usd": tot["cost"], "cost_from": tot["cost_from"], "n_model_calls": tot["n_calls"]},
|
|
427
|
+
"standard": {"benchmark": bench["key"], "family": bench["family"], "needs": bench["needs"], "unit": unit,
|
|
428
|
+
"plugged": other.get("_ref") or other["key"], "job": job, **_version(bench), **_control(other), **_setup(other),
|
|
429
|
+
**({"environment": environment} if environment else {}), **({"image": image} if image else {})}}
|
|
430
|
+
if score is not None:
|
|
431
|
+
value = float(score.get("reward") or 0.0)
|
|
432
|
+
# a pass benchmark leaves a reward, as a verifier does; a score
|
|
433
|
+
# benchmark a score and no pass or fail - the trial reads as Scored
|
|
434
|
+
rewards = {"reward": value} if bench["score"]["kind"] == "pass" else {"score": value}
|
|
435
|
+
res["verifier_result"] = {"rewards": rewards}
|
|
436
|
+
parts = score.get("parts")
|
|
437
|
+
if isinstance(parts, dict) and isinstance(parts.get("n"), int) and not isinstance(parts["n"], bool) and parts["n"] > 0:
|
|
438
|
+
# a score over several parts (a conversation's questions): the job's mean weighs it by them
|
|
439
|
+
res["verifier_result"]["parts"] = {"n": parts["n"]}
|
|
440
|
+
if parts.get("by") and isinstance(parts.get("parts"), list):
|
|
441
|
+
_write(os.path.join(trial_dir, "verifier", "parts.json"),
|
|
442
|
+
json.dumps({"by": parts["by"], "parts": parts["parts"]}, indent=1))
|
|
443
|
+
if "reward" in rewards:
|
|
444
|
+
_write(os.path.join(trial_dir, "verifier", "reward.txt"), str(value) + "\n")
|
|
445
|
+
_write(os.path.join(trial_dir, "verifier", "score.json"),
|
|
446
|
+
json.dumps(dict({k: v for k, v in score.items() if k != "parts"}, says=bench["score"]["says"]), indent=1))
|
|
447
|
+
if exception:
|
|
448
|
+
res["exception_info"] = {"exception_type": exception[0], "exception_message": exception[1][:2000]}
|
|
449
|
+
_write(os.path.join(trial_dir, "agent", "calls.jsonl"), "".join(json.dumps(c) + "\n" for c in calls))
|
|
450
|
+
_write(os.path.join(trial_dir, "agent", "gateway.log"),
|
|
451
|
+
"".join(f"{iso(c.get('t0'))} {c.get('model')} status {c.get('status')} {c.get('error') or ''}\n" for c in calls))
|
|
452
|
+
if trajectory is not None:
|
|
453
|
+
_write(os.path.join(trial_dir, "agent", "trajectory.json"), json.dumps(trajectory, indent=1))
|
|
454
|
+
timeline = trace_from_run(trajectory, calls, other, produce)
|
|
455
|
+
if timeline is not None:
|
|
456
|
+
_write(os.path.join(trial_dir, "agent", "trace.json"), json.dumps(timeline, indent=1))
|
|
457
|
+
for name, text in (extra_logs or {}).items():
|
|
458
|
+
_write(os.path.join(trial_dir, "agent", name), text)
|
|
459
|
+
_write(os.path.join(trial_dir, "result.json"), json.dumps(res, indent=1))
|
|
460
|
+
return res
|
|
461
|
+
|
|
462
|
+
|
|
463
|
+
def write_config(job_dir: str, *, job: str, bench: dict, other: dict, model: str, provider: str, dataset: str, units: List[str],
|
|
464
|
+
concurrency: int, granted: bool, judge: str = "", backend: str = "local", image: str = "",
|
|
465
|
+
options: Optional[dict] = None, launch: str = "", shard: Optional[dict] = None, repeat: int = 1,
|
|
466
|
+
bench_options: Optional[dict] = None,
|
|
467
|
+
simulator: str = "", prices: Optional[dict] = None):
|
|
468
|
+
"""``unsandboxed`` is true for an agent on the local backend only: in a
|
|
469
|
+
container (``backend``, ``image``) it is not. ``standard.options`` holds
|
|
470
|
+
every option's value, the defaults included, so two runs read side by side
|
|
471
|
+
say what differed. ``launch`` and ``shard``: a shard of a launch run over
|
|
472
|
+
several machines (``shard``), which ``merge`` reads back. ``repeat``: how
|
|
473
|
+
many times each unit was run, said only when more than once. ``judge`` and
|
|
474
|
+
``simulator``: the models that graded and that played the user, when the
|
|
475
|
+
benchmark has them. ``prices``: the rows its models' costs are
|
|
476
|
+
estimated from and where the user's list came from, when it has any."""
|
|
477
|
+
with open(os.path.join(job_dir, "config.json"), "w") as f:
|
|
478
|
+
json.dump({"job_name": job, "runner": "standard", "runner_label": "Nullius",
|
|
479
|
+
"standard": {"benchmark": bench["key"], **_version(bench), "plugged": other.get("_ref") or other["key"], "kind": other["_kind"], "provider": provider,
|
|
480
|
+
"needs": bench["needs"], "permissions_granted": granted,
|
|
481
|
+
"unsandboxed": granted and backend == "local", "backend": backend,
|
|
482
|
+
**({"image": image} if image else {}), **({"judge": judge} if judge else {}),
|
|
483
|
+
**({"simulator": simulator} if simulator else {}), **({"options": options} if options else {}), **({"benchmark_options": bench_options} if bench_options else {}), **_control(other), **_setup(other),
|
|
484
|
+
**({"launch": launch} if launch else {}), **({"shard": shard} if shard else {}),
|
|
485
|
+
**({"repeat": repeat} if repeat > 1 else {}), **({"prices": prices} if prices else {})},
|
|
486
|
+
"runner_items": [bench["units"]["noun"], bench["units"]["nouns"]],
|
|
487
|
+
"n_concurrent_trials": concurrency,
|
|
488
|
+
"agents": [{"name": other["key"], "model_name": model}],
|
|
489
|
+
"datasets": [{"name": f"{bench['key']}/{dataset}", "n_tasks": len(units)}]}, f, indent=2)
|
|
490
|
+
|
|
491
|
+
|
|
492
|
+
#: an error kind naming the machine or the benchmark's side rather than the agent: the live board's infra_error
|
|
493
|
+
_INFRA = ("Setup", "Verifier", "Api", "RateLimit", "Environment", "Docker", "Build", "Connection", "Network", "Authentication",
|
|
494
|
+
"ModelNotFound")
|
|
495
|
+
|
|
496
|
+
|
|
497
|
+
def feed_outcome(kind: str, score: Optional[dict], exception: Optional[tuple]) -> dict:
|
|
498
|
+
"""A unit's end in the live feed's words (``nullius.runs.live``): its
|
|
499
|
+
outcome, with the reward (a pass benchmark) or the score, and the error's
|
|
500
|
+
kind as evidence. A unit the run's spending cap cut off (``ledger.ERROR``)
|
|
501
|
+
is the feed's ``budget``."""
|
|
502
|
+
score = score or {}
|
|
503
|
+
if exception:
|
|
504
|
+
k = exception[0]
|
|
505
|
+
return {"outcome": "cancelled" if k == "CancelledError" else "budget" if k == ledger.ERROR else "timeout" if "Timeout" in k else "infra_error" if any(w in k for w in _INFRA) else "agent_error",
|
|
506
|
+
"exc": k, "evidence": (exception[1] or k)[:200]}
|
|
507
|
+
if kind == "pass":
|
|
508
|
+
reward = score.get("reward")
|
|
509
|
+
return {"outcome": "resolved" if reward == 1.0 else "unresolved", "reward": reward, "evidence": f"reward {reward}"}
|
|
510
|
+
value = score.get("score", score.get("reward"))
|
|
511
|
+
return {"outcome": "scored", "score": value, "evidence": f"score {value}"}
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def pass_hat(results: List[dict]) -> Optional[dict]:
|
|
515
|
+
"""pass^k of a run that tried each unit several times (``repeat``), as τ-bench defines it: over
|
|
516
|
+
the units, the chance that k attempts drawn from its n all passed, C(c, k) / C(n, k), averaged -
|
|
517
|
+
pass^1 is the pass rate; and pass@k (``at``), the chance that one of them did, as HumanEval reads it. A unit with an attempt that ended in an error, with no verdict, is left
|
|
518
|
+
out, as the upstreams leave an infrastructure error out. None for a run of one attempt each."""
|
|
519
|
+
from math import comb
|
|
520
|
+
per: Dict[str, List[bool]] = {}
|
|
521
|
+
broken = set()
|
|
522
|
+
for r in results:
|
|
523
|
+
u = (r.get("standard") or {}).get("unit")
|
|
524
|
+
rewards = (r.get("verifier_result") or {}).get("rewards") or {}
|
|
525
|
+
if u is None:
|
|
526
|
+
continue
|
|
527
|
+
if r.get("exception_info") or "reward" not in rewards:
|
|
528
|
+
broken.add(u)
|
|
529
|
+
continue
|
|
530
|
+
per.setdefault(u, []).append(rewards["reward"] == 1.0)
|
|
531
|
+
whole = {u: v for u, v in per.items() if u not in broken}
|
|
532
|
+
n = min((len(v) for v in whole.values()), default=0)
|
|
533
|
+
if n < 2:
|
|
534
|
+
return None
|
|
535
|
+
return {"units": len(whole), "attempts": n,
|
|
536
|
+
**{str(k): round(sum(comb(sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)},
|
|
537
|
+
# pass@k, as HumanEval's estimate_pass_at_k reads n attempts with c passed: 1 - C(n-c, k) / C(n, k), averaged
|
|
538
|
+
"at": {str(k): round(sum(1 - comb(n - sum(v[:n]), k) / comb(n, k) for v in whole.values()) / len(whole), 4) for k in range(1, n + 1)}}
|
|
539
|
+
|
|
540
|
+
|
|
541
|
+
def write_result(job_dir: str, job: str, t0: float, t1: float, results: List[dict], planned: int, budget: Optional[dict] = None):
|
|
542
|
+
"""``budget``: a run with a spending cap (``run.Run.budget``): the cap,
|
|
543
|
+
what was spent, whether it stopped the run, the units it cut off and
|
|
544
|
+
those never begun."""
|
|
545
|
+
errored = sum(1 for r in results if r.get("exception_info"))
|
|
546
|
+
hat = pass_hat(results)
|
|
547
|
+
with open(os.path.join(job_dir, "result.json"), "w") as f:
|
|
548
|
+
json.dump({"id": job, "started_at": iso(t0), "finished_at": iso(t1), "n_total_trials": planned,
|
|
549
|
+
"stats": {"n_completed_trials": len(results) - errored, "n_errored_trials": errored,
|
|
550
|
+
"n_cancelled_trials": planned - len(results), "n_running_trials": 0, "n_pending_trials": 0},
|
|
551
|
+
**({"pass_hat_k": hat} if hat else {}), **({"budget": budget} if budget else {})}, f, indent=1)
|