nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,838 @@
|
|
|
1
|
+
{% from 'runs/_runtime.html' import what_ran %}
|
|
2
|
+
{% extends "base.html" %}
|
|
3
|
+
{% block title %}{{ trial.task_base }} — Nullius{% endblock %}
|
|
4
|
+
{% block crumb %}/ <a href="{{ url_for('job', job_id=job.job_id) }}">{{ job.job_id }}</a> / {{ trial.task_base }}{% endblock %}
|
|
5
|
+
|
|
6
|
+
{# The chart grid and the label chips are shared with the compare page's
|
|
7
|
+
sides, so there is one definition of each and the two cannot drift apart.
|
|
8
|
+
They take `view_url` below as their link builder. #}
|
|
9
|
+
{% import "traces/_charts.html" as ui %}
|
|
10
|
+
|
|
11
|
+
{# Every filter control is a link to this same page with a different `hide` set,
|
|
12
|
+
so a filtered view has a URL — shareable, bookmarkable, and exactly what
|
|
13
|
+
save-trace writes out. `viewing` rides along so toggling a chip while reading a
|
|
14
|
+
saved view does not silently drop back to the producer's capture.
|
|
15
|
+
|
|
16
|
+
`cf` is the charts' own legend state, which is a different query key and a
|
|
17
|
+
different thing: it changes what one chart draws, never what the trace
|
|
18
|
+
contains. It defaults to the current one so a control that only means to
|
|
19
|
+
change `hide` carries the legend through untouched, and vice versa. #}
|
|
20
|
+
{% macro view_url(f, cf=none) -%}
|
|
21
|
+
{{ url_for('trial', job_id=job.job_id, trial_name=trial.trial_name, trace=viewing or none, tab='trace',
|
|
22
|
+
**view_query(f, cf if cf is not none else chart_filter)) }}
|
|
23
|
+
{%- endmacro %}
|
|
24
|
+
|
|
25
|
+
{% from 'traces/_judgement.html' import judgement_panel %}
|
|
26
|
+
{% block body %}
|
|
27
|
+
<link rel="stylesheet" href="{{ url_for('static', filename='traces/conversation.css') }}">
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
<div class="page-title">
|
|
31
|
+
<h1>{{ trial.task_base }}</h1>
|
|
32
|
+
<span class="tag oc-{{ trial.outcome }}">{{ trial.outcome_label }}</span>
|
|
33
|
+
{# The recorded exception type, beside the label that came from it. For an
|
|
34
|
+
Unknown it says which class the classifier has no side for, so a reader
|
|
35
|
+
can place it (classify._AGENT_EXCEPTIONS / _INFRA_EXCEPTIONS) instead of
|
|
36
|
+
guessing from the colour. #}
|
|
37
|
+
{% if trial.exception_type %}<span class="mono faint exception-type"
|
|
38
|
+
title="{% if trial.outcome == 'unknown' %}Harbor recorded this exception on the trial and the classifier has no side for it, so the trial counts and is charged to nobody. To place it, add the name to _AGENT_EXCEPTIONS or _INFRA_EXCEPTIONS in app/nullius/runs/classify.py.{% else %}Harbor recorded this exception on the trial; the outcome label follows from it.{% endif %}">{{ trial.exception_type }}</span>{% endif %}
|
|
39
|
+
</div>
|
|
40
|
+
{# The trial records a bare model name ("gpt-5.6-luna"); the job keeps the
|
|
41
|
+
provider-qualified one ("openai/gpt-5.6-luna"). Prefer the fuller id. #}
|
|
42
|
+
{% set model = job.model_name if job.model_name and job.model_name.endswith(trial.model_name or '~')
|
|
43
|
+
else (trial.model_name or job.model_name) %}
|
|
44
|
+
<p class="subtitle">
|
|
45
|
+
<span class="mono">{{ model }}</span><span class="dot">·</span>
|
|
46
|
+
<span title="{{ trial.source }}">{{ trial.source|dsname }}</span>
|
|
47
|
+
{%- if trial.agent_version %}<span class="dot">·</span><span class="mono faint"
|
|
48
|
+
title="The agent's version as the trial's agent_info reports it">v{{ trial.agent_version }}</span>{% endif %}
|
|
49
|
+
{%- if job.runner %}<span class="dot">·</span>{{ what_ran(job) }}{% endif %}
|
|
50
|
+
</p>
|
|
51
|
+
|
|
52
|
+
<div class="cards">
|
|
53
|
+
{# The check cards speak the dataset's language: labels come from its
|
|
54
|
+
Vocabulary, and a category it doesn't have (a verifier with no
|
|
55
|
+
regression guard and no patches; a session-derived trial has neither)
|
|
56
|
+
is not drawn at all rather than dashed under a foreign name. The dash
|
|
57
|
+
inside a drawn card still means "this trial reported nothing". #}
|
|
58
|
+
{% set vocab = trial.vocab %}
|
|
59
|
+
{% if vocab.gained %}
|
|
60
|
+
<div class="card"><div class="k">{{ vocab.gained }}</div><div class="v">{% if trial.f2p_pass or trial.f2p_fail %}<span class="ok">{{ trial.f2p_pass }}</span><small>/{{ trial.f2p_fail }}</small>{% else %}—{% endif %}</div></div>
|
|
61
|
+
{% endif %}
|
|
62
|
+
{% if vocab.held %}
|
|
63
|
+
<div class="card"><div class="k">{{ vocab.held }}</div><div class="v">{% if trial.p2p_pass or trial.p2p_fail %}<span class="ok">{{ trial.p2p_pass }}</span><small>/{{ trial.p2p_fail }}</small>{% else %}—{% endif %}</div></div>
|
|
64
|
+
{% endif %}
|
|
65
|
+
{% if vocab.has_patch %}
|
|
66
|
+
<div class="card"><div class="k">Patch applied</div><div class="v">{% if trial.patch_exists is none %}—{% else %}{{ 'Yes' if trial.patch_applied else 'No' }}{% endif %}</div></div>
|
|
67
|
+
{% endif %}
|
|
68
|
+
{# The verifier's numeric score. Mostly redundant with the outcome tag on
|
|
69
|
+
a pass/fail verifier (0 or 1), but it's the number Harbor's mean is built from, and
|
|
70
|
+
partial-credit verifiers make it a real scalar. #}
|
|
71
|
+
<div class="card" title="{{ "The judge's verdict on this answer: 1 when it said the answer is right" if job.runner else "The verifier's reward for this trial — what Harbor's headline mean averages" }}">
|
|
72
|
+
<div class="k">Reward</div>
|
|
73
|
+
<div class="v">{{ trial.reward if trial.reward is not none else '—' }}</div>
|
|
74
|
+
</div>
|
|
75
|
+
<div class="card"><div class="k">Agent time</div><div class="v">{{ trial.dur_agent|secs }}</div></div>
|
|
76
|
+
{% if traj and not traj.error %}
|
|
77
|
+
<div class="card"><div class="k">Steps</div><div class="v">{{ traj.n_steps }}</div></div>
|
|
78
|
+
<div class="card"><div class="k">Tool calls</div><div class="v">{{ traj.n_tool_calls }}</div></div>
|
|
79
|
+
{% endif %}
|
|
80
|
+
</div>
|
|
81
|
+
|
|
82
|
+
<div class="evidence-strip" aria-label="Evidence recorded">
|
|
83
|
+
<span>Trajectory · {{ 'recorded' if trial.has_trajectory else 'absent' }}</span>
|
|
84
|
+
<span>Trace · {{ 'recorded' if trial.has_trace else 'absent' }}</span>
|
|
85
|
+
<span>Reward · {{ 'recorded' if trial.reward is not none else 'absent' }}</span>
|
|
86
|
+
{# Rate limits the agent's SDK retried and the run survived. They are in the
|
|
87
|
+
gateway log and nowhere else — result.json records only the 429 that
|
|
88
|
+
killed a run — so a trial without that log reads as unmeasured, not clean. #}
|
|
89
|
+
{% if trial.n_rate_limited_retries is none %}
|
|
90
|
+
<span class="rate-limits" title="No agent/gateway.log in this trial, so retried rate limits cannot be counted">Rate limits · not recorded</span>
|
|
91
|
+
{% elif trial.n_rate_limited_retries %}
|
|
92
|
+
<span class="rate-limits" title="Rate-limit records in agent/gateway.log — 429s the agent's SDK retried below the harness. The run survived them; the outcome does not show them."><b class="warn">{{ trial.n_rate_limited_retries }} rate-limited call{{ '' if trial.n_rate_limited_retries == 1 else 's' }} retried</b></span>
|
|
93
|
+
{% else %}
|
|
94
|
+
<span class="rate-limits" title="agent/gateway.log carries no rate-limit record">Rate limits · none logged</span>
|
|
95
|
+
{% endif %}
|
|
96
|
+
</div>
|
|
97
|
+
<nav class="trial-tabs" role="tablist" aria-label="Trial evidence">
|
|
98
|
+
{% for name, label in [('overview', 'Overview'), ('judgement', 'System One judge'), ('llm-judge', 'LLM judge'), ('task', 'Task instruction'), ('trace', 'Trace'), ('conversation', 'Conversation'), ('artifacts', 'Artifacts')] %}
|
|
99
|
+
<button id="tab-{{ name }}" role="tab" type="button" data-tab="{{ name }}" aria-controls="trial-{{ name }}" aria-selected="{{ 'true' if name == 'overview' else 'false' }}" tabindex="{{ '0' if name == 'overview' else '-1' }}">{{ label }}</button>
|
|
100
|
+
{% endfor %}
|
|
101
|
+
</nav>
|
|
102
|
+
<section class="trial-tab" id="trial-overview" role="tabpanel" aria-labelledby="tab-overview">
|
|
103
|
+
{% if trial.is_regression %}
|
|
104
|
+
<div class="panel danger-panel">
|
|
105
|
+
<h3 style="margin-top:0">⚠ This patch broke {{ trial.p2p_fail }} previously-passing test{{ '' if trial.p2p_fail == 1 else 's' }}</h3>
|
|
106
|
+
<p class="muted small" style="margin-top:0">These passed before the patch was applied and fail after it — a
|
|
107
|
+
regression, not just a missed fix.</p>
|
|
108
|
+
<ul class="test-list">
|
|
109
|
+
{% for name in trial.p2p_failures %}<li class="mono bad">{{ name }}</li>{% endfor %}
|
|
110
|
+
</ul>
|
|
111
|
+
</div>
|
|
112
|
+
{% endif %}
|
|
113
|
+
|
|
114
|
+
{% if trial.f2p_failures %}
|
|
115
|
+
<div class="panel">
|
|
116
|
+
<h3 style="margin-top:0">Tests that had to pass, still failing</h3>
|
|
117
|
+
<ul class="test-list">
|
|
118
|
+
{% for name in trial.f2p_failures %}<li class="mono faint">{{ name }}</li>{% endfor %}
|
|
119
|
+
</ul>
|
|
120
|
+
</div>
|
|
121
|
+
{% endif %}
|
|
122
|
+
|
|
123
|
+
{% if trial.exception_type %}
|
|
124
|
+
<div class="panel err">
|
|
125
|
+
<h3 style="margin-top:0">{{ trial.exception_type }}</h3>
|
|
126
|
+
<pre>{{ trial.exception_message }}</pre>
|
|
127
|
+
</div>
|
|
128
|
+
{% endif %}
|
|
129
|
+
|
|
130
|
+
<h2>Shape of the run</h2>
|
|
131
|
+
|
|
132
|
+
{# The phase bar reads only trial.*, so it renders whether or not a trajectory
|
|
133
|
+
was recorded: a trial that failed before writing one still shows its timing. #}
|
|
134
|
+
{% set total = trial.dur_phases + trial.dur_other %}
|
|
135
|
+
{% if total %}
|
|
136
|
+
<div class="panel">
|
|
137
|
+
<h3 style="margin-top:0">Where the time went</h3>
|
|
138
|
+
{# All four phases Harbor records. Leaving agent_setup out understated the
|
|
139
|
+
bar by a third on some agents, and it is the most actionable number in
|
|
140
|
+
the set — installing the agent can cost more than solving the task. #}
|
|
141
|
+
<div class="bar phases">
|
|
142
|
+
{# A 0s phase emits no segment at all, so a control's empty setup leaves
|
|
143
|
+
no stray sliver. #}
|
|
144
|
+
{% if trial.dur_env|phase %}<span class="seg ph-env" style="flex:{{ trial.dur_env|phase }}" title="Environment build: {{ trial.dur_env|secs }}"></span>{% endif %}
|
|
145
|
+
{% if trial.dur_agent_setup|phase %}<span class="seg ph-setup" style="flex:{{ trial.dur_agent_setup|phase }}" title="Agent setup (installing the agent): {{ trial.dur_agent_setup|secs }}"></span>{% endif %}
|
|
146
|
+
{% if trial.dur_agent|phase %}<span class="seg ph-agent" style="flex:{{ trial.dur_agent|phase }}" title="Agent working: {{ trial.dur_agent|secs }}"></span>{% endif %}
|
|
147
|
+
{% if trial.dur_verifier|phase %}<span class="seg ph-verify" style="flex:{{ trial.dur_verifier|phase }}" title="Verifier: {{ trial.dur_verifier|secs }}"></span>{% endif %}
|
|
148
|
+
{% set items = job.runner_items[1] if job.runner_items else 'items' %}
|
|
149
|
+
{% set other_is = 'the wait between the answer and its judgement: the judge reads the answers once its share of the ' ~ items ~ ' is answered' if job.runner else 'time Harbor does not attribute to a phase: image pull, container start and teardown, artifact collection' %}
|
|
150
|
+
{% if trial.dur_other|phase %}<span class="seg ph-other" style="flex:{{ trial.dur_other|phase }}" title="Other: {{ trial.dur_other|secs }} — {{ other_is }}"></span>{% endif %}
|
|
151
|
+
</div>
|
|
152
|
+
<div class="legend" style="margin-bottom:0">
|
|
153
|
+
<span class="chip ph-env">Env <b>{{ trial.dur_env|secs }}</b></span>
|
|
154
|
+
<span class="chip ph-setup">Setup <b>{{ trial.dur_agent_setup|secs }}</b></span>
|
|
155
|
+
<span class="chip ph-agent">Agent <b>{{ trial.dur_agent|secs }}</b></span>
|
|
156
|
+
<span class="chip ph-verify">Verify <b>{{ trial.dur_verifier|secs }}</b></span>
|
|
157
|
+
<span class="chip ph-other"
|
|
158
|
+
title="{{ other_is|capitalize }}">Other <b>{{ trial.dur_other|secs }}</b></span>
|
|
159
|
+
</div>
|
|
160
|
+
<p class="muted small">The bar spans the trial's full {{ trial.dur_total|secs }}.
|
|
161
|
+
{% if job.runner %}“Other” is the wait for the judge, which reads the answers once its share of the {{ items }} is answered.{% else %}“Other” is harness overhead Harbor doesn't break out — the same tax for every agent.{% endif %}
|
|
162
|
+
{% if trace and trace.has_spans %}The Trace tab shows recorded span timing and its
|
|
163
|
+
measured or inferred provenance.{% elif traj and traj.has_step_clock %}Conversation
|
|
164
|
+
reports time between source steps; these are not per-tool durations.{% else %}No
|
|
165
|
+
per-step clock is available. Conversation preserves the recorded order.{% endif %}</p>
|
|
166
|
+
</div>
|
|
167
|
+
{% endif %}
|
|
168
|
+
|
|
169
|
+
{% if traj and traj.error %}
|
|
170
|
+
<div class="panel err"><h3 style="margin-top:0">Trajectory unreadable</h3><pre>{{ traj.error }}</pre></div>
|
|
171
|
+
{% elif traj %}
|
|
172
|
+
<div class="panel">
|
|
173
|
+
{% set has_growth = traj.token_series|length > 1 %}
|
|
174
|
+
{% if has_growth %}
|
|
175
|
+
<h3 style="margin-top:0">Prompt growth</h3>
|
|
176
|
+
{% set max_sent = traj.token_series|map(attribute='sent')|max %}
|
|
177
|
+
<div class="spark">
|
|
178
|
+
{% for p in traj.token_series %}
|
|
179
|
+
{% set h = (56 * p.sent / max_sent)|round(1) if max_sent else 0 %}
|
|
180
|
+
{% set ch = (56 * p.cached / max_sent)|round(1) if max_sent else 0 %}
|
|
181
|
+
<span class="spark-col"
|
|
182
|
+
title="step {{ p.step_id }} · input {{ p.sent|tokens('control') }} ({{ p.cached|tokens('control') }} cached, {{ p.new|tokens('control') }} new) · output {{ p.completion|tokens('control') }}">
|
|
183
|
+
<span class="sp-fresh" style="height:{{ [h - ch, 1]|max }}px"></span>
|
|
184
|
+
{% if ch >= 1 %}<span class="sp-cached" style="height:{{ ch }}px"></span>{% endif %}
|
|
185
|
+
</span>
|
|
186
|
+
{% endfor %}
|
|
187
|
+
</div>
|
|
188
|
+
<div class="legend" style="margin:.6rem 0 0">
|
|
189
|
+
<span class="chip sp-fresh-chip">new tokens</span>
|
|
190
|
+
<span class="chip sp-cached-chip">served from cache</span>
|
|
191
|
+
{% if traj.cache and traj.cache.hit is not none %}
|
|
192
|
+
<span class="chip" title="Reads ÷ every input token, both usage conventions on the same ruler: {{ traj.cache.reads|tokens('control') }} read from cache, {{ traj.cache.new|tokens('control') }} new (cache writes + fresh)">
|
|
193
|
+
cache hit <b>{{ traj.cache.hit|hitrate }}</b></span>
|
|
194
|
+
{% if traj.cache.reported_diverges %}
|
|
195
|
+
<span class="chip" title="{% if traj.cache.convention == 'anthropic' %}The file's own cached ÷ prompt: this agent reports Anthropic-style usage, whose prompt_tokens excludes cache writes, so the ratio is ~100% by construction{% else %}The file's own cached ÷ prompt: its final_metrics disagree with the per-call series{% endif %}">
|
|
196
|
+
reported {{ traj.cache.reported_hit|hitrate }}</span>
|
|
197
|
+
{% endif %}
|
|
198
|
+
{% endif %}
|
|
199
|
+
{% if traj.total_reasoning_tokens is not none %}
|
|
200
|
+
<span class="chip" title="Summed from per-step metrics, or the trajectory's final metrics">reasoning <b>{{ traj.total_reasoning_tokens|tokens('control') }}</b></span>
|
|
201
|
+
{% endif %}
|
|
202
|
+
</div>
|
|
203
|
+
<p class="muted small">One column per LLM step, height ∝ input tokens; the gray band is what came from cache.
|
|
204
|
+
{%- if traj.cache and traj.cache.convention == 'anthropic' %} Anthropic-style usage: cache writes are added back onto the step that paid for them.{%- endif %}</p>
|
|
205
|
+
{% elif traj.token_series or traj.total_reasoning_tokens is not none %}
|
|
206
|
+
{# one call: no growth to draw, so the call itself - what came from cache, what was new, what the model wrote #}
|
|
207
|
+
{% set p = traj.token_series[0] if traj.token_series else none %}
|
|
208
|
+
<h3 style="margin-top:0">Usage</h3>
|
|
209
|
+
{% if p and p.sent %}
|
|
210
|
+
<div class="mix-bar" id="usage-mix" role="img" aria-label="Input {{ p.sent }} tokens, {{ p.cached }} from cache; output {{ p.completion }} tokens">
|
|
211
|
+
{% if p.cached %}<span class="mix-cached" style="flex:{{ p.cached }}" title="served from cache: {{ p.cached|tokens('control') }}"></span>{% endif %}
|
|
212
|
+
{% if p.new %}<span class="mix-new" style="flex:{{ p.new }}" title="new input: {{ p.new|tokens('control') }}"></span>{% endif %}
|
|
213
|
+
{% if p.completion %}<span class="mix-out" style="flex:{{ p.completion }}" title="output: {{ p.completion|tokens('control') }}"></span>{% endif %}
|
|
214
|
+
</div>
|
|
215
|
+
{% endif %}
|
|
216
|
+
<div class="legend">
|
|
217
|
+
{% if p %}<span class="chip sp-cached-chip">served from cache <b>{{ p.cached|tokens('control') }}</b></span>
|
|
218
|
+
<span class="chip sp-fresh-chip">new input <b>{{ p.new|tokens('control') }}</b></span>
|
|
219
|
+
<span class="chip mix-out-chip">output <b>{{ p.completion|tokens('control') }}</b></span>
|
|
220
|
+
{% if p.sent %}<span class="chip" title="Served from cache ÷ input">cache hit <b>{{ (p.cached / p.sent)|hitrate }}</b></span>{% endif %}{% endif %}
|
|
221
|
+
{% if traj.total_reasoning_tokens is not none %}<span class="chip" title="From the trajectory's final metrics">reasoning <b>{{ traj.total_reasoning_tokens|tokens('control') }}</b></span>{% endif %}
|
|
222
|
+
</div>
|
|
223
|
+
<p class="muted small">{% if job.runner %}{{ job.runner }} asks its model once for each of its {{ job.runner_items[1] if job.runner_items else 'items' }}, so the bar is that one call: its input, from cache or new, and what it wrote. An agent's run charts its calls one after another instead, as its context grows.{% else %}One step carried usage: the bar is that call; the growth strip needs two.{% endif %}</p>
|
|
224
|
+
{% endif %}
|
|
225
|
+
|
|
226
|
+
{% if traj.calls %}
|
|
227
|
+
<h3{% if not has_growth %} style="margin-top:0"{% endif %}>Tool calls</h3>
|
|
228
|
+
<div class="flow">
|
|
229
|
+
{% for c in traj.calls %}
|
|
230
|
+
<span class="flow-cell cat-{{ c.category }}{% if c.failure %} failed{% endif %}"
|
|
231
|
+
title="#{{ c.step_id }} {{ c.category }}{% if c.target %} · {{ c.target[:70] }}{% endif %}{% if c.failure %} — FAILED{% endif %}"></span>
|
|
232
|
+
{% endfor %}
|
|
233
|
+
</div>
|
|
234
|
+
{# Tool summary. Grouped by category, never by the agent's own tool names:
|
|
235
|
+
OpenCode writes `bash`, Claude Code `Bash`, another agent a
|
|
236
|
+
sentence, so a count by name compares nothing across agents. The names
|
|
237
|
+
stay beside each category as the evidence of what it folded — in the
|
|
238
|
+
chip's hover and in the table below — so the fold can be checked. #}
|
|
239
|
+
<div class="legend tool-summary" style="margin:.6rem 0 0" aria-label="Tool calls by category">
|
|
240
|
+
{% for cat, n in traj.category_freq.most_common() %}
|
|
241
|
+
<span class="chip cat-{{ cat }}" data-category="{{ cat }}" data-count="{{ n }}"
|
|
242
|
+
title="{{ cat }}: {% for name, k in traj.names_of(cat) %}{{ name[:80] }} ×{{ k }}{{ ', ' if not loop.last }}{% endfor %}">{{ cat }} <b>{{ n }}</b></span>
|
|
243
|
+
{% endfor %}
|
|
244
|
+
{% if traj.n_failed_calls %}<span class="chip failed-chip">failed <b>{{ traj.n_failed_calls }}</b></span>{% endif %}
|
|
245
|
+
</div>
|
|
246
|
+
{% if traj.name_freq %}
|
|
247
|
+
{# Each tool once, under the server that offers it. An agent's spellings of a name are its hover: evidence, never a comparison key. #}
|
|
248
|
+
<table class="grid compact tool-names" style="margin-top:.6rem">
|
|
249
|
+
<thead><tr><th>Tool</th><th>Category</th><th class="num">Calls</th><th class="num">Failed</th><th class="num">Time</th></tr></thead>
|
|
250
|
+
<tbody>
|
|
251
|
+
{% for t in traj.tools %}
|
|
252
|
+
{% if t.server and (loop.first or traj.tools[loop.index0 - 1].server != t.server) %}<tr class="tool-server"><td colspan="5" class="muted small">{{ t.server }}</td></tr>{% endif %}
|
|
253
|
+
<tr data-category="{{ t.category }}" data-name="{{ t.name }}">
|
|
254
|
+
<td class="mono clip" title="{{ t.spellings|join(', ') }}">{{ t.name[:80] }}{% if t.name|length > 80 %}…{% endif %}</td>
|
|
255
|
+
<td><span class="cat-tag cat-{{ t.category }}"><span class="cat-dot"></span>{{ t.category }}</span></td>
|
|
256
|
+
<td class="num">{{ t.calls }}</td>
|
|
257
|
+
<td class="num{{ ' bad' if t.failed }}">{{ t.failed or '—' }}</td>
|
|
258
|
+
<td class="num">{{ t.ns|nsdur if t.ns is not none else '—' }}</td>
|
|
259
|
+
</tr>
|
|
260
|
+
{% endfor %}
|
|
261
|
+
</tbody>
|
|
262
|
+
</table>
|
|
263
|
+
{% endif %}
|
|
264
|
+
{% endif %}
|
|
265
|
+
</div>
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
{% if loc %}
|
|
270
|
+
<div class="panel">
|
|
271
|
+
<h3 style="margin-top:0">Against the gold patch</h3>
|
|
272
|
+
<ul class="files">
|
|
273
|
+
{% for f in loc.hit %}<li class="mono"><span class="ok">✓</span> {{ f }}</li>{% endfor %}
|
|
274
|
+
{% for f in loc.missed %}<li class="mono"><span class="bad">×</span> {{ f }} <span class="faint">— in the gold patch, never edited</span></li>{% endfor %}
|
|
275
|
+
{% for f in loc.extra %}<li class="mono faint">{{ f }} <span class="faint">— edited, not in the gold patch</span></li>{% endfor %}
|
|
276
|
+
</ul>
|
|
277
|
+
<p class="muted small">The gold patch is <em>a</em> correct answer, not the only one.</p>
|
|
278
|
+
</div>
|
|
279
|
+
{% endif %}
|
|
280
|
+
|
|
281
|
+
{% if traj.files_touched %}
|
|
282
|
+
<div class="panel">
|
|
283
|
+
<h3 style="margin-top:0">Files the agent edited</h3>
|
|
284
|
+
<ul class="files">
|
|
285
|
+
{% for f in traj.files_touched %}<li class="mono">{{ f }}</li>{% endfor %}
|
|
286
|
+
</ul>
|
|
287
|
+
</div>
|
|
288
|
+
{% endif %}
|
|
289
|
+
|
|
290
|
+
{% else %}
|
|
291
|
+
<div class="panel"><p class="muted" style="margin:0">No trajectory recorded for this trial.</p></div>
|
|
292
|
+
{% endif %}
|
|
293
|
+
</section>
|
|
294
|
+
<section class="trial-tab" id="trial-judgement" role="tabpanel" aria-labelledby="tab-judgement">
|
|
295
|
+
{% if job.runner and bench_parts %}
|
|
296
|
+
{#- scored part by part by the benchmark's own script: the score is the mean of these, each as the script wrote it #}
|
|
297
|
+
<div class="panel" id="runner-verdict">
|
|
298
|
+
<h3 style="margin-top:0">{{ job.runner }}'s score <span class="badge">gives the outcome</span></h3>
|
|
299
|
+
<p class="small muted">{% if bench_parts.says %}{{ bench_parts.says }}. {% endif %}{% if trial.score is not none %}This one scored <b class="mono">{{ '%.3f'|format(trial.score) }}</b>: the mean over its {{ bench_parts.parts|length }} {{ bench_parts.nouns }}, one without a score counting nothing.{% endif %}</p>
|
|
300
|
+
<details class="tc" open><summary class="small">Its {{ bench_parts.parts|length }} {{ bench_parts.nouns }}</summary>
|
|
301
|
+
<table class="grid compact" id="runner-parts" style="margin-top:.4rem">
|
|
302
|
+
<thead><tr><th class="num">#</th><th>Asked</th><th>Answered</th>{% for k in bench_parts.shows %}<th>{{ k|capitalize }}</th>{% endfor %}<th class="num">Score</th></tr></thead>
|
|
303
|
+
<tbody>
|
|
304
|
+
{% for x in bench_parts.parts %}
|
|
305
|
+
<tr><td class="num mono">{{ x.n }}</td><td>{{ x.text }}</td><td>{% if x.said is not none %}{{ x.said }}{% else %}<span class="faint">no answer</span>{% endif %}</td>
|
|
306
|
+
{%- for k in bench_parts.shows %}<td>{% if x.shows and x.shows[k] is not none %}{{ x.shows[k] }}{% endif %}</td>{% endfor -%}
|
|
307
|
+
<td class="num mono">{% if x.score is not none %}{{ '%.3f'|format(x.score) }}{% else %}<span class="faint">—</span>{% endif %}</td></tr>
|
|
308
|
+
{% endfor %}
|
|
309
|
+
</tbody>
|
|
310
|
+
</table></details>
|
|
311
|
+
<p class="small muted" style="margin-bottom:0">Below, Nullius's own trajectory judge: experimental, asked on a click.</p>
|
|
312
|
+
</div>
|
|
313
|
+
{% elif job.runner %}
|
|
314
|
+
<div class="panel" id="runner-verdict">
|
|
315
|
+
<h3 style="margin-top:0">{{ job.runner }}'s judge <span class="badge">gives the outcome</span></h3>
|
|
316
|
+
{% if bench_judge %}
|
|
317
|
+
<p class="small muted">{{ bench_judge.judge or job.runner_judge }}{% if bench_judge.model %} · sent as <span class="mono">{{ bench_judge.model }}</span>{% endif %}</p>
|
|
318
|
+
<p>Said <b class="mono">{{ (bench_judge.said or '—')[:200] }}</b>: {{ 'resolved' if trial.outcome == 'resolved' else trial.outcome|replace('_', ' ') }}.</p>
|
|
319
|
+
<details class="tc"><summary class="small">What it was asked</summary><pre class="wrap">{{ bench_judge.asked }}</pre></details>
|
|
320
|
+
{% else %}
|
|
321
|
+
<p class="small muted">{% if job.runner_judge %}{{ job.runner_judge }}. {% endif %}No record of its call for this one{% if trial.outcome not in ('resolved', 'failed_verification') %}: it never got to judge it{% endif %}.</p>
|
|
322
|
+
{% endif %}
|
|
323
|
+
<p class="small muted" style="margin-bottom:0">Below, Nullius's own trajectory judge: experimental, asked on a click.</p>
|
|
324
|
+
</div>
|
|
325
|
+
{% endif %}
|
|
326
|
+
{{ judgement_panel(judgement, job, trial, csrf) }}
|
|
327
|
+
</section>
|
|
328
|
+
{#- the second judgement: the same questions, asked of the LLM judge Settings names, its records apart -#}
|
|
329
|
+
<section class="trial-tab" id="trial-llm-judge" role="tabpanel" aria-labelledby="tab-llm-judge">
|
|
330
|
+
{{ judgement_panel(llm_judgement, job, trial, csrf) }}
|
|
331
|
+
</section>
|
|
332
|
+
<section class="trial-tab" id="trial-task" role="tabpanel" aria-labelledby="tab-task">
|
|
333
|
+
{# The task's instruction, scored on its own text: once per dataset and task, the same for every attempt,
|
|
334
|
+
nothing of any run read. An unclear instruction caps every agent on this task - a finding about the
|
|
335
|
+
benchmark, not this trial. The score changes no outcome. #}
|
|
336
|
+
{% set tt = task_tab(job, trial, traj) %}
|
|
337
|
+
{% set tq = tt.quality %}
|
|
338
|
+
<div class="judgement" id="task-quality">
|
|
339
|
+
<div class="judgement-head">
|
|
340
|
+
<div>
|
|
341
|
+
<h2>Task instruction <span class="badge">experimental</span></h2>
|
|
342
|
+
<p class="muted small">How clearly the task says what is wanted, judged from its text alone: once per dataset and task, shared by every trial of it.
|
|
343
|
+
An unclear instruction caps every agent's score on this task, a finding about the benchmark, not about this agent. It changes no outcome.</p>
|
|
344
|
+
</div>
|
|
345
|
+
</div>
|
|
346
|
+
{% if job.is_control %}
|
|
347
|
+
<p class="small faint">A control run: its trials are a check on the tasks, so the instruction is scored from an agent's run of the same dataset.</p>
|
|
348
|
+
{% endif %}
|
|
349
|
+
<div class="jcards">
|
|
350
|
+
<div class="jcard tone-{{ 'bad' if tq and tq.level == 0 else ('warn' if tq and tq.level == 1 else 'none') }}" data-question="instruction_quality">
|
|
351
|
+
<div class="jcard-title">Instruction quality</div>
|
|
352
|
+
{% if tq %}
|
|
353
|
+
<div class="jcard-value"><span class="jnum"><strong>{{ '%.1f'|format(tq.score) }}</strong></span><span class="junit">/ {{ tq.top }}</span></div>
|
|
354
|
+
<div class="jcard-blurb muted small">{{ tq.label }}</div>
|
|
355
|
+
<div class="jcard-foot small">{% if tq.confidence is not none %}<span class="muted">Confidence</span><span class="mono">{{ '%.0f'|format(tq.confidence * 100) }}%</span>{% endif %}<span class="muted">{{ tq.model }} · asked {{ tq.asked_at[:16]|replace('T', ' ') }}Z</span></div>
|
|
356
|
+
{% else %}
|
|
357
|
+
<div class="jcard-value"><span class="faint">not scored</span></div>
|
|
358
|
+
<div class="jcard-blurb muted small"><em>Score instructions</em> on the <a href="{{ url_for('job', job_id=job.job_id) }}#task-quality">run page</a> asks the judge about each task's text once.</div>
|
|
359
|
+
{% endif %}
|
|
360
|
+
</div>
|
|
361
|
+
</div>
|
|
362
|
+
<h3>Rubric</h3>
|
|
363
|
+
<ol class="rubric" start="0">
|
|
364
|
+
{% for level in tt.rubric %}<li class="rubric-level{{ ' current' if tq and tq.level == loop.index0 }}"><span class="mono small">{{ loop.index0 }}</span> {{ level }}</li>{% endfor %}
|
|
365
|
+
</ol>
|
|
366
|
+
<h3>The instruction, as the agent received it</h3>
|
|
367
|
+
{% if tt.instruction %}<pre class="instruction" id="task-instruction-text">{{ tt.instruction }}</pre>
|
|
368
|
+
{% else %}<p class="small faint" id="task-instruction-text">No instruction can be read from this trial's trajectory.</p>{% endif %}
|
|
369
|
+
</div>
|
|
370
|
+
</section>
|
|
371
|
+
<section class="trial-tab" id="trial-trace" role="tabpanel" aria-labelledby="tab-trace">
|
|
372
|
+
{% if trace and trace.error %}
|
|
373
|
+
<div class="panel err">
|
|
374
|
+
<h3 style="margin-top:0">Trace unreadable</h3>
|
|
375
|
+
<pre>{{ trace.error }}</pre>
|
|
376
|
+
</div>
|
|
377
|
+
{# `visible`, not `has_spans`: a filter may legitimately hide every span, and the
|
|
378
|
+
panel that vanished would take the control to undo it with. #}
|
|
379
|
+
{% elif trace and trace.visible %}
|
|
380
|
+
{% set filt = trace.filter %}
|
|
381
|
+
{# A trace whose producer measured nothing — a session transcript reconstructed
|
|
382
|
+
from record arrival times — must not have its totals captioned as measured
|
|
383
|
+
anywhere below: the banner says so once, and this word says so wherever the
|
|
384
|
+
wall clock is named. #}
|
|
385
|
+
{% set clocked = 'inferred' if trace.zero_measured else 'measured' %}
|
|
386
|
+
|
|
387
|
+
{% if trace.zero_measured %}
|
|
388
|
+
<div class="panel warn-panel">
|
|
389
|
+
<h3 style="margin-top:0">⚠ Nothing was measured</h3>
|
|
390
|
+
<p class="muted small" style="margin:0">All {{ trace.totals.span_count }} spans are
|
|
391
|
+
<b>inferred</b> — the exporter captured nothing and the producer fell back to reading
|
|
392
|
+
the agent's own logs. Every duration below is approximate, and simply wrong wherever
|
|
393
|
+
tool calls ran in parallel.</p>
|
|
394
|
+
</div>
|
|
395
|
+
{% endif %}
|
|
396
|
+
|
|
397
|
+
<div class="panel">
|
|
398
|
+
<h3 style="margin-top:0">Filter</h3>
|
|
399
|
+
{% if viewing %}
|
|
400
|
+
<p class="muted small" style="margin:-.15rem 0 .8rem">Rendering
|
|
401
|
+
<span class="mono">{{ viewing }}</span> — a view saved from this trial, not the producer's capture.
|
|
402
|
+
<a class="linkish" href="{{ url_for('trial', job_id=job.job_id, trial_name=trial.trial_name) }}">Back to trace.json</a></p>
|
|
403
|
+
{% endif %}
|
|
404
|
+
{{ ui.facet_chips(trace.facets, filt, chart_filter, view_url) }}
|
|
405
|
+
{% if trace.hidden_spans %}
|
|
406
|
+
{# Hidden one at a time rather than by label — these need their own controls,
|
|
407
|
+
since no chip above represents them. #}
|
|
408
|
+
<div class="legend facets" style="margin:.55rem 0 0">
|
|
409
|
+
{% for sid, label in trace.hidden_spans %}
|
|
410
|
+
<a class="chip off" href="{{ view_url(filt.toggled_span(sid)) }}" title="Show this span again">
|
|
411
|
+
{{ label }} <span class="mono faint">{{ sid[:12] }}</span></a>
|
|
412
|
+
{% endfor %}
|
|
413
|
+
</div>
|
|
414
|
+
{% endif %}
|
|
415
|
+
<p class="muted small" style="margin:.75rem 0 0">
|
|
416
|
+
{% if filt.active %}
|
|
417
|
+
{# `shown` counts the filtered view whole — a zoom narrows the waterfall, not
|
|
418
|
+
the filter, so this sentence must not shrink with it. #}
|
|
419
|
+
<b>{{ trace.shown }}</b> span{{ '' if trace.shown == 1 else 's' }} shown,
|
|
420
|
+
<b>{{ trace.hidden_count }}</b> left out.
|
|
421
|
+
<a class="linkish" href="{{ view_url(filt.cleared()) }}">Clear ×</a>
|
|
422
|
+
{% else %}
|
|
423
|
+
Click a label to leave out every span carrying it, or the <span class="mono">×</span> on a
|
|
424
|
+
waterfall row — alt-click its bar — to leave out one call. Either way what it called goes with
|
|
425
|
+
it, and the breakdown is recomputed without it. The producer's own totals stay as reported.
|
|
426
|
+
{% endif %}
|
|
427
|
+
</p>
|
|
428
|
+
{% if filt.active %}
|
|
429
|
+
<form class="wf-save" method="post"
|
|
430
|
+
action="{{ url_for('save_trace', job_id=job.job_id, trial_name=trial.trial_name) }}">
|
|
431
|
+
<input type="hidden" name="csrf" value="{{ csrf_token() }}">
|
|
432
|
+
{# The view is reproduced from these, not from a session — the file written is
|
|
433
|
+
the file this page is showing. #}
|
|
434
|
+
{% for name in filt.names %}<input type="hidden" name="hide" value="{{ name }}">{% endfor %}
|
|
435
|
+
{% for sid in filt.span_ids %}<input type="hidden" name="hide_span" value="{{ sid }}">{% endfor %}
|
|
436
|
+
{% if filt.zoom %}<input type="hidden" name="zoom" value="{{ filt.zoom }}">{% endif %}
|
|
437
|
+
{% if viewing %}<input type="hidden" name="trace" value="{{ viewing }}">{% endif %}
|
|
438
|
+
<label for="save-as">Save this view as</label>
|
|
439
|
+
<input id="save-as" type="text" name="filename" value="{{ default_save_name }}" required
|
|
440
|
+
pattern="[A-Za-z0-9][A-Za-z0-9._-]*" spellcheck="false" autocomplete="off"
|
|
441
|
+
title="A plain filename — letters, digits, dot, dash and underscore">
|
|
442
|
+
<button class="btn sm" type="submit">Save</button>
|
|
443
|
+
</form>
|
|
444
|
+
<p class="muted small" style="margin:.55rem 0 0">Writes a new
|
|
445
|
+
<span class="mono">harbor-trace-v1</span> file beside the trial's own, in
|
|
446
|
+
<span class="mono">agent/</span>: the hidden spans gone, <span class="mono">totals</span>
|
|
447
|
+
recomputed over what is left, and the producer's numbers kept under
|
|
448
|
+
<span class="mono">filtered.source_totals</span>. Nothing is overwritten — least of all
|
|
449
|
+
<span class="mono">trace.json</span>, which is a capture that cannot be taken again.
|
|
450
|
+
{% if trace.zoomed %}The zoom is not written: it is a way of looking at the file, and the
|
|
451
|
+
link carries it.{% endif %}</p>
|
|
452
|
+
{% endif %}
|
|
453
|
+
</div>
|
|
454
|
+
|
|
455
|
+
{# ---- distributions ----
|
|
456
|
+
The population of calls, not the timeline: how long a typical call runs and
|
|
457
|
+
where the summed time sits. Deliberately above the waterfall, since it is the
|
|
458
|
+
summary you read before scrolling one. It draws from `trace.roots`, which is
|
|
459
|
+
already pruned by the same query string the page rendered from.
|
|
460
|
+
|
|
461
|
+
The same macro draws each side of the compare page. Only the closing
|
|
462
|
+
sentence differs, because only this page has a Time breakdown strip to point
|
|
463
|
+
at — a page without one must not claim it. #}
|
|
464
|
+
{% if charts %}
|
|
465
|
+
{% set breakdown_tail %}The waterfall below is the timeline. <b>Time breakdown</b> is the accounting of where the seconds went; this is the shape of the calls that spent them.{% endset %}
|
|
466
|
+
{{ ui.charts_panel(charts, chart_filter, filt, view_url, tail=breakdown_tail) }}
|
|
467
|
+
{% endif %}
|
|
468
|
+
|
|
469
|
+
<div class="panel">
|
|
470
|
+
<h3 style="margin-top:0">Time breakdown
|
|
471
|
+
<span class="muted" style="font-weight:400">· not a timeline{% if trace.zero_measured %} · inferred, not measured{% endif %}</span></h3>
|
|
472
|
+
{% if trace.totals.segments %}
|
|
473
|
+
<div class="bar strip">
|
|
474
|
+
{% for s in trace.totals.segments %}
|
|
475
|
+
<span class="seg tt-{{ s.key }}" style="flex:{{ s.ns }}"
|
|
476
|
+
title="{{ s.label }}: {{ s.ns|nsdur }} · {{ s.pct|share }} of the {{ trace.totals.wall_ns|nsdur }} {{ clocked }}"></span>
|
|
477
|
+
{% endfor %}
|
|
478
|
+
</div>
|
|
479
|
+
{# Shares are of the wall clock, not of the segments' own sum, so categories
|
|
480
|
+
that don't close read as short of 100% instead of being rescaled to fill it. #}
|
|
481
|
+
<div class="legend" style="margin-bottom:0">
|
|
482
|
+
{% for s in trace.totals.segments %}
|
|
483
|
+
<span class="chip tt-{{ s.key }}">{{ s.label }} <b>{{ s.ns|nsdur }}</b> <span class="faint">{{ s.pct|share }}</span></span>
|
|
484
|
+
{% endfor %}
|
|
485
|
+
</div>
|
|
486
|
+
<p class="muted small">Categories, not a sequence. Reported by the trace, never summed from the bars below: spans nest.{% if trace.totals.unaccounted_ns %} Unaccounted is wall clock outside the instrumented spans.{% endif %}</p>
|
|
487
|
+
{% else %}
|
|
488
|
+
<p class="muted" style="margin:0">The trace reports no totals.</p>
|
|
489
|
+
{% endif %}
|
|
490
|
+
|
|
491
|
+
{% if filt.active %}
|
|
492
|
+
<h3>Without the {{ trace.hidden_count }} span{{ '' if trace.hidden_count == 1 else 's' }} left out
|
|
493
|
+
<span class="muted" style="font-weight:400">· recomputed here</span></h3>
|
|
494
|
+
{# Each Δ is against the same partition run over the unfiltered tree, so it is
|
|
495
|
+
the filter's effect alone — not the gap between this arithmetic and the
|
|
496
|
+
producer's, which is a different quantity and can be much larger. #}
|
|
497
|
+
{% set changed = trace.view.compared_to(trace.baseline) %}
|
|
498
|
+
{% if changed %}
|
|
499
|
+
<div class="bar strip">
|
|
500
|
+
{% for s in changed %}{% if s.ns %}
|
|
501
|
+
<span class="seg tt-{{ s.key }}" style="flex:{{ s.ns }}" title="{{ s.label }}: {{ s.ns|nsdur }} · {{ s.pct|share }}{% if s.delta_ns %} · {{ s.delta_ns|nsdelta }} against the unfiltered run{% endif %}"></span>
|
|
502
|
+
{% endif %}{% endfor %}
|
|
503
|
+
</div>
|
|
504
|
+
<div class="legend" style="margin-bottom:0">
|
|
505
|
+
{% for s in changed %}
|
|
506
|
+
<span class="chip tt-{{ s.key }}">{{ s.label }} <b>{{ s.ns|nsdur }}</b>
|
|
507
|
+
<span class="faint">{{ s.pct|share }}</span>
|
|
508
|
+
{% if s.delta_ns %}<span class="delta">{{ s.delta_ns|nsdelta }}</span>{% endif %}</span>
|
|
509
|
+
{% endfor %}
|
|
510
|
+
</div>
|
|
511
|
+
<p class="muted small">Derived from the {{ trace.view.span_count }} remaining span{{ '' if trace.view.span_count == 1 else 's' }}.
|
|
512
|
+
Each Δ is measured against this same arithmetic over the whole trace, so it is what the filter
|
|
513
|
+
took out — read the absolute numbers against the strip above instead, which is the producer's own
|
|
514
|
+
accounting and can differ from this one by a lot: time no span covers is “Unaccounted” here,
|
|
515
|
+
where a producer may know it was a wait for a prompt and call it idle.
|
|
516
|
+
Every instant is charged to exactly one category (waiting on a human beats real work, which beats
|
|
517
|
+
the call it ran inside), because summing spans that nest or overlap attributes more than the run.
|
|
518
|
+
So hiding a span moves its time into “Unaccounted” rather than out of the total: the run took just
|
|
519
|
+
as long.</p>
|
|
520
|
+
{% else %}
|
|
521
|
+
<p class="muted" style="margin:0">Nothing is left to break down.</p>
|
|
522
|
+
{% endif %}
|
|
523
|
+
{% endif %}
|
|
524
|
+
|
|
525
|
+
{% if trace.coverage_pct %}
|
|
526
|
+
{% set uncovered = trace.totals.wall_ns - trace.instrumented_ns %}
|
|
527
|
+
<h3>Coverage</h3>
|
|
528
|
+
<div class="bar strip">
|
|
529
|
+
<span class="seg cov-in" style="flex:{{ trace.instrumented_ns }}"
|
|
530
|
+
title="Instrumented: {{ trace.instrumented_ns|nsdur }}"></span>
|
|
531
|
+
{% if uncovered > 0 %}
|
|
532
|
+
<span class="seg cov-out" style="flex:{{ uncovered }}"
|
|
533
|
+
title="Not instrumented: {{ uncovered|nsdur }} — startup, queueing and teardown the agent emitted no spans for"></span>
|
|
534
|
+
{% endif %}
|
|
535
|
+
</div>
|
|
536
|
+
{# `wall_ns` is the run the trace itself measures — the agent's own window,
|
|
537
|
+
not the trial total, which also carries image pull, setup and the verifier. #}
|
|
538
|
+
<p class="muted small">The waterfall covers <b>{{ trace.instrumented_ns|nsdur }}</b> of the
|
|
539
|
+
{{ trace.totals.wall_ns|nsdur }} the trace {{ 'infers' if trace.zero_measured else 'measures' }} — {{ '%.1f'|format(trace.coverage_pct) }}%. The axis is the captured window, not the run.</p>
|
|
540
|
+
{% endif %}
|
|
541
|
+
|
|
542
|
+
{% if trace.has_spans %}
|
|
543
|
+
<h3>Waterfall</h3>
|
|
544
|
+
<style>
|
|
545
|
+
/* The zoom control mirrors `.wf-hide` (style.css): present in the DOM at
|
|
546
|
+
all times so the keyboard path exists, visible on hover or focus. */
|
|
547
|
+
.wf-zoom {
|
|
548
|
+
flex: none; padding: 0 .1rem; margin-left: auto;
|
|
549
|
+
color: var(--text-faint); font-size: .72rem; line-height: 1;
|
|
550
|
+
text-decoration: none; opacity: 0; transition: opacity .12s;
|
|
551
|
+
}
|
|
552
|
+
.wf-zoom + .wf-hide { margin-left: 0; }
|
|
553
|
+
.wf-row:hover .wf-zoom, .wf-zoom:focus-visible { opacity: 1; }
|
|
554
|
+
@media (prefers-reduced-motion: reduce) { .wf-zoom { transition: none; } }
|
|
555
|
+
.wf-zoom:hover { color: var(--accent); }
|
|
556
|
+
.wf-zoomchip { display: flex; flex-wrap: wrap; gap: .5rem; align-items: center; margin: 0 0 .6rem; }
|
|
557
|
+
</style>
|
|
558
|
+
{# ---- zoom ----
|
|
559
|
+
A second, explicit window on the axis (`?zoom=<span id>`), for the trace
|
|
560
|
+
where one global axis over an hour makes a 20ms Read a hairline. It is
|
|
561
|
+
not the filter and not a re-scaling of it: the rows are the zoomed span's
|
|
562
|
+
subtree alone, placed on that span's own extent, and everything outside it
|
|
563
|
+
is left out rather than squeezed in. The strips above are untouched — the
|
|
564
|
+
totals and the breakdown are over the whole filtered view, the coverage
|
|
565
|
+
strip describes the capture. Offsets on the axis and in the rows count from
|
|
566
|
+
the zoomed span's start; the chip says where that is in the capture. #}
|
|
567
|
+
{% if trace.zoomed %}
|
|
568
|
+
<p class="wf-zoomchip">
|
|
569
|
+
<span class="chip on">Zoomed to <b>{{ trace.zoomed.label }}</b>
|
|
570
|
+
<span class="mono faint">{{ trace.zoomed.span_id[:12] }}</span></span>
|
|
571
|
+
<span class="muted small">{{ trace.rows|length }} of {{ trace.shown }} span{{ '' if trace.shown == 1 else 's' }}
|
|
572
|
+
· starts <b>{{ trace.zoom_offset_ns|nsoff }}</b> into the capture
|
|
573
|
+
· <b>{{ trace.axis.span_ns|nsdur }}</b> wide — offsets below count from its start</span>
|
|
574
|
+
<a class="linkish" href="{{ view_url(filt.unzoomed()) }}">Reset zoom ×</a>
|
|
575
|
+
</p>
|
|
576
|
+
{% elif trace.zoom_lost %}
|
|
577
|
+
<p class="wf-zoomchip">
|
|
578
|
+
<span class="chip off">No span <span class="mono">{{ trace.zoom_lost[:24] }}</span> in this view</span>
|
|
579
|
+
<span class="muted small">— nothing to zoom to, so this is the whole trace. It may be hidden by the
|
|
580
|
+
filter, or belong to another trace file.</span>
|
|
581
|
+
<a class="linkish" href="{{ view_url(filt.unzoomed()) }}">Clear ×</a>
|
|
582
|
+
</p>
|
|
583
|
+
{% endif %}
|
|
584
|
+
<div class="wf-toolbar">
|
|
585
|
+
<button class="linkish" id="wf-toggle-all" type="button">Collapse all</button>
|
|
586
|
+
<span class="muted small">{{ trace.rows|length }} span{{ '' if trace.rows|length == 1 else 's' }}{% if trace.run_id %} · <span class="mono">{{ trace.run_id }}</span>{% endif %}
|
|
587
|
+
· alt-click a bar to leave it out · shift-click, or <span class="mono">⤢</span>, to zoom the axis to it</span>
|
|
588
|
+
</div>
|
|
589
|
+
<div class="wf-head">
|
|
590
|
+
<div>Span</div>
|
|
591
|
+
<div class="wf-axis">
|
|
592
|
+
{% for t in trace.ticks %}<span class="wf-tick" style="left:{{ t.pct }}%">{{ t.ns|nsoff }}</span>{% endfor %}
|
|
593
|
+
</div>
|
|
594
|
+
<div class="wf-dur">Duration</div>
|
|
595
|
+
</div>
|
|
596
|
+
{# One global axis: every bar is positioned against the whole trace, so a
|
|
597
|
+
child that starts before its parent renders early instead of being
|
|
598
|
+
clamped into a tidy lie. Zoomed, the axis is the one span's extent and
|
|
599
|
+
the rows are its subtree — still one axis for every row shown. #}
|
|
600
|
+
<div class="wf" id="wf">
|
|
601
|
+
{% for r in trace.rows %}
|
|
602
|
+
<div class="wf-row k-{{ r.span.kind }} q-{{ r.span.timing_quality }}{% if r.is_instant %} instant{% endif %}"
|
|
603
|
+
data-span="{{ r.span.span_id }}"
|
|
604
|
+
data-parent="{{ r.span.parent_span_id }}"
|
|
605
|
+
data-label-filter="{{ view_url(filt.toggled_name(r.span.label)) }}"
|
|
606
|
+
data-tip="{{ r.span.label }} · {{ r.span.kind }} · {{ r.span.timing_quality }} · {{ r.span.source }} · at {{ r.offset_ns|nsoff }} · {{ r.span.duration_ns|nsdur }}"
|
|
607
|
+
style="--depth:{{ r.depth }};--left:{{ r.left_pct }}%;--width:{{ r.width_pct }}%">
|
|
608
|
+
<div class="wf-gutter">
|
|
609
|
+
{% if r.has_children %}<button class="wf-chev" type="button" aria-expanded="true" aria-label="Collapse subtree"></button>{% else %}<span class="wf-chev-gap"></span>{% endif %}
|
|
610
|
+
<span class="wf-dot"></span>
|
|
611
|
+
<button class="wf-label" type="button" title="{{ r.span.label }}">{{ r.span.label }}</button>
|
|
612
|
+
{# Links, not buttons: the filter and the zoom live in the URL, so these
|
|
613
|
+
work with middle-click and keyboard navigation and need no script. #}
|
|
614
|
+
{% if r.has_children and r.span.span_id != filt.zoom %}
|
|
615
|
+
<a class="wf-zoom" href="{{ view_url(filt.zoomed_to(r.span.span_id)) }}"
|
|
616
|
+
title="Zoom the axis to this span and what it called" aria-label="Zoom to this span">⤢</a>
|
|
617
|
+
{% endif %}
|
|
618
|
+
<a class="wf-hide" href="{{ view_url(filt.toggled_span(r.span.span_id)) }}"
|
|
619
|
+
title="Leave this span, and what it called, out" aria-label="Leave this span out">×</a>
|
|
620
|
+
</div>
|
|
621
|
+
<div class="wf-track"><span class="wf-bar"></span></div>
|
|
622
|
+
<div class="wf-dur">{{ r.span.duration_ns|nsdur }}</div>
|
|
623
|
+
</div>
|
|
624
|
+
{% endfor %}
|
|
625
|
+
</div>
|
|
626
|
+
{% if trace.truncated %}
|
|
627
|
+
<p class="muted small">{{ trace.truncated }} further span{{ '' if trace.truncated == 1 else 's' }} not shown — the trace exceeds the render cap.</p>
|
|
628
|
+
{% endif %}
|
|
629
|
+
|
|
630
|
+
<div class="legend" style="margin:.7rem 0 .2rem">
|
|
631
|
+
{% for k in trace.kinds_present %}<span class="chip k-{{ k }}">{{ k }}</span>{% endfor %}
|
|
632
|
+
</div>
|
|
633
|
+
{# Said once, where the kind first appears. A reader who does not know what the
|
|
634
|
+
suffix means reads the wrapper and the call inside it as two calls, which is
|
|
635
|
+
the misreading the re-kinding exists to stop. #}
|
|
636
|
+
{% if 'group' in trace.kinds_present %}
|
|
637
|
+
<p class="muted small" style="margin:.3rem 0 0">A <b>group</b> only brackets what
|
|
638
|
+
is inside it — <span class="mono">claude_code.tool</span> holds the wait for
|
|
639
|
+
permission and the execution, and a request whose whole response was one block
|
|
640
|
+
holds that block — so it is charged no time of its own and the row below it is
|
|
641
|
+
the same measurement. Hiding a group takes its contents with it; hiding the
|
|
642
|
+
contents leaves the bracket.</p>
|
|
643
|
+
{% endif %}
|
|
644
|
+
<div class="legend" style="margin:0">
|
|
645
|
+
{% for q in trace.qualities_present %}<span class="chip q-{{ q }}">{{ q }}</span>{% endfor %}
|
|
646
|
+
</div>
|
|
647
|
+
{% else %}
|
|
648
|
+
<h3>Waterfall</h3>
|
|
649
|
+
<p class="muted" style="margin:0">Every span is filtered out — nothing left to draw.</p>
|
|
650
|
+
{% endif %}
|
|
651
|
+
</div>
|
|
652
|
+
|
|
653
|
+
{# Content is fetched for the selected normalised span only. A large trace
|
|
654
|
+
must not inline every prompt/result into hidden DOM (A15). #}
|
|
655
|
+
{% if trace.has_spans %}
|
|
656
|
+
<div class="panel wf-detail" id="wf-detail"
|
|
657
|
+
data-endpoint="{{ url_for('trace_span_data', job_id=job.job_id, trial_name=trial.trial_name) }}">
|
|
658
|
+
<p class="muted" style="margin:0">Select a span to load its recorded detail.</p>
|
|
659
|
+
</div>
|
|
660
|
+
{% endif %}
|
|
661
|
+
{% endif %}
|
|
662
|
+
{% if not trace %}<div class="panel"><h2>No trace recorded</h2><p class="muted">This trial has no trace sidecar. Its recorded trajectory, when available, is in Conversation. Missing span clocks are not reconstructed here.</p></div>{% endif %}
|
|
663
|
+
{% if trace and not trace.error and not trace.visible %}<div class="panel"><h2>No spans recorded</h2><p class="muted">The trace file contains no recorded spans to display.</p></div>{% endif %}
|
|
664
|
+
</section>
|
|
665
|
+
<section class="trial-tab" id="trial-conversation" role="tabpanel" aria-labelledby="tab-conversation">
|
|
666
|
+
{% include 'traces/_conversation.html' %}
|
|
667
|
+
</section>
|
|
668
|
+
<section class="trial-tab" id="trial-artifacts" role="tabpanel" aria-labelledby="tab-artifacts">
|
|
669
|
+
<div class="reader-heading"><div><h2>Artifacts</h2><p class="muted small">Read the recorded source files. Content is redacted and paged; source files remain intact.</p></div></div>
|
|
670
|
+
<div class="panel">
|
|
671
|
+
<table class="grid"><thead><tr><th>Record</th><th>Availability</th><th class="num">Size</th><th></th></tr></thead><tbody>
|
|
672
|
+
{% for artifact in artifacts %}
|
|
673
|
+
<tr><td>{{ artifact.label }}</td><td class="muted">{{ 'Recorded' if artifact.exists else 'Not recorded' }}</td><td class="num mono muted">{{ ((artifact.size / 1024)|round(1)) ~ ' KB' if artifact.exists else '—' }}</td><td>{% if artifact.exists %}<a href="{{ artifact.url }}">Read →</a>{% endif %}</td></tr>
|
|
674
|
+
{% endfor %}
|
|
675
|
+
</tbody></table>
|
|
676
|
+
</div>
|
|
677
|
+
</section>
|
|
678
|
+
<script src="{{ url_for('static', filename='traces/conversation.js') }}"></script>
|
|
679
|
+
<script>
|
|
680
|
+
(() => {
|
|
681
|
+
/* Every filter and zoom control is a link to this page with another query (the
|
|
682
|
+
filter lives in the URL: the top of this file). Followed here, it puts in the
|
|
683
|
+
Trace tab that URL renders and keeps the page around it: no reload, no jump. */
|
|
684
|
+
const panel = document.getElementById('trial-trace');
|
|
685
|
+
async function showView(url, push = true) {
|
|
686
|
+
let fresh = null;
|
|
687
|
+
try {
|
|
688
|
+
const r = await nulliusFetch(url, {cache: 'no-store'});
|
|
689
|
+
if (r.ok) fresh = new DOMParser().parseFromString(await r.text(), 'text/html').getElementById('trial-trace');
|
|
690
|
+
} catch {}
|
|
691
|
+
if (!fresh) { location.href = url; return; }
|
|
692
|
+
const y = scrollY;
|
|
693
|
+
panel.replaceChildren(...fresh.childNodes);
|
|
694
|
+
if (push) history.pushState(null, '', url);
|
|
695
|
+
bindWaterfall();
|
|
696
|
+
scrollTo(0, y);
|
|
697
|
+
}
|
|
698
|
+
panel.addEventListener('click', (e) => {
|
|
699
|
+
const a = e.target.closest('a[href]');
|
|
700
|
+
if (!a || e.ctrlKey || e.metaKey || e.shiftKey || a.target) return;
|
|
701
|
+
if (a.pathname !== location.pathname || a.search === location.search) return;
|
|
702
|
+
e.preventDefault();
|
|
703
|
+
showView(a.href);
|
|
704
|
+
});
|
|
705
|
+
addEventListener('popstate', () => showView(location.href, false));
|
|
706
|
+
|
|
707
|
+
// One tooltip for the table, not one per row: a per-row component with
|
|
708
|
+
// hover-intent delay is unusable on a dense timeline.
|
|
709
|
+
const tip = document.createElement('div');
|
|
710
|
+
tip.className = 'wf-tip';
|
|
711
|
+
tip.hidden = true;
|
|
712
|
+
document.body.appendChild(tip);
|
|
713
|
+
|
|
714
|
+
function bindWaterfall() {
|
|
715
|
+
tip.hidden = true;
|
|
716
|
+
const wf = document.getElementById('wf');
|
|
717
|
+
if (!wf) return;
|
|
718
|
+
|
|
719
|
+
const rows = Array.from(wf.querySelectorAll('.wf-row'));
|
|
720
|
+
// Span ids are opaque: log-derived ones look like `block:<uuid>:0`. A colon is
|
|
721
|
+
// legal in an id and in a fragment but breaks a CSS selector, so every lookup
|
|
722
|
+
// here goes through this Map — never querySelector('#...').
|
|
723
|
+
const byId = new Map(rows.map(r => [r.dataset.span, r]));
|
|
724
|
+
const kids = new Map();
|
|
725
|
+
rows.forEach(r => {
|
|
726
|
+
const p = r.dataset.parent;
|
|
727
|
+
if (!p) return;
|
|
728
|
+
if (!kids.has(p)) kids.set(p, []);
|
|
729
|
+
kids.get(p).push(r);
|
|
730
|
+
});
|
|
731
|
+
|
|
732
|
+
const descendants = (row) => {
|
|
733
|
+
const out = [];
|
|
734
|
+
const stack = (kids.get(row.dataset.span) || []).slice();
|
|
735
|
+
while (stack.length) {
|
|
736
|
+
const node = stack.pop();
|
|
737
|
+
out.push(node);
|
|
738
|
+
const next = kids.get(node.dataset.span);
|
|
739
|
+
if (next) stack.push(...next);
|
|
740
|
+
}
|
|
741
|
+
return out;
|
|
742
|
+
};
|
|
743
|
+
|
|
744
|
+
const applyVisibility = () => {
|
|
745
|
+
rows.forEach(r => { r.hidden = false; });
|
|
746
|
+
rows.forEach(r => {
|
|
747
|
+
if (r.classList.contains('collapsed')) descendants(r).forEach(d => { d.hidden = true; });
|
|
748
|
+
});
|
|
749
|
+
};
|
|
750
|
+
|
|
751
|
+
const setCollapsed = (row, collapsed) => {
|
|
752
|
+
row.classList.toggle('collapsed', collapsed);
|
|
753
|
+
const chev = row.querySelector('.wf-chev');
|
|
754
|
+
if (chev) chev.setAttribute('aria-expanded', String(!collapsed));
|
|
755
|
+
};
|
|
756
|
+
|
|
757
|
+
const detail = document.getElementById('wf-detail');
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
const select = (id, push) => {
|
|
761
|
+
const row = byId.get(id);
|
|
762
|
+
if (!row) return;
|
|
763
|
+
// A deep link can point inside a collapsed subtree, which would otherwise
|
|
764
|
+
// show nothing at all. Walk up and open every ancestor first.
|
|
765
|
+
let parent = byId.get(row.dataset.parent);
|
|
766
|
+
while (parent) {
|
|
767
|
+
setCollapsed(parent, false);
|
|
768
|
+
parent = byId.get(parent.dataset.parent);
|
|
769
|
+
}
|
|
770
|
+
applyVisibility();
|
|
771
|
+
rows.forEach(r => r.classList.toggle('selected', r === row));
|
|
772
|
+
window.NulliusEvidence.selectSpan(detail, id);
|
|
773
|
+
if (push !== false) history.replaceState(null, '', '#span=' + encodeURIComponent(id));
|
|
774
|
+
};
|
|
775
|
+
|
|
776
|
+
wf.addEventListener('click', (e) => {
|
|
777
|
+
// The × and alt-click both navigate to a filtered URL. The href is read off
|
|
778
|
+
// the row's own link rather than rebuilt here, so there is one place where a
|
|
779
|
+
// filter URL is constructed and it is the server's.
|
|
780
|
+
if (e.target.closest('.wf-hide') || e.target.closest('.wf-zoom')) return;
|
|
781
|
+
const hide = (e.altKey || e.metaKey) && e.target.closest('.wf-track')
|
|
782
|
+
? e.target.closest('.wf-row').querySelector('.wf-hide') : null;
|
|
783
|
+
if (hide) { e.preventDefault(); showView(hide.href); return; }
|
|
784
|
+
// Shift-click zooms the same way: the row's own link, built by the server.
|
|
785
|
+
// A leaf has no zoom link — its extent is its bar — so the click falls
|
|
786
|
+
// through to selection.
|
|
787
|
+
const zoom = e.shiftKey && e.target.closest('.wf-track')
|
|
788
|
+
? e.target.closest('.wf-row').querySelector('.wf-zoom') : null;
|
|
789
|
+
if (zoom) { e.preventDefault(); showView(zoom.href); return; }
|
|
790
|
+
|
|
791
|
+
const chev = e.target.closest('.wf-chev');
|
|
792
|
+
if (chev) {
|
|
793
|
+
const row = chev.closest('.wf-row');
|
|
794
|
+
setCollapsed(row, !row.classList.contains('collapsed'));
|
|
795
|
+
applyVisibility();
|
|
796
|
+
return;
|
|
797
|
+
}
|
|
798
|
+
const row = e.target.closest('.wf-row');
|
|
799
|
+
if (row && (e.target.closest('.wf-label') || e.target.closest('.wf-track'))) {
|
|
800
|
+
select(row.dataset.span);
|
|
801
|
+
}
|
|
802
|
+
});
|
|
803
|
+
|
|
804
|
+
const toggleAll = document.getElementById('wf-toggle-all');
|
|
805
|
+
let allCollapsed = false;
|
|
806
|
+
toggleAll.addEventListener('click', () => {
|
|
807
|
+
allCollapsed = !allCollapsed;
|
|
808
|
+
rows.forEach(r => { if (kids.has(r.dataset.span)) setCollapsed(r, allCollapsed); });
|
|
809
|
+
applyVisibility();
|
|
810
|
+
toggleAll.textContent = allCollapsed ? 'Expand all' : 'Collapse all';
|
|
811
|
+
});
|
|
812
|
+
|
|
813
|
+
wf.addEventListener('mousemove', (e) => {
|
|
814
|
+
const row = e.target.closest('.wf-row');
|
|
815
|
+
if (!row) { tip.hidden = true; return; }
|
|
816
|
+
tip.textContent = row.dataset.tip;
|
|
817
|
+
tip.hidden = false;
|
|
818
|
+
const box = row.getBoundingClientRect();
|
|
819
|
+
const width = tip.offsetWidth;
|
|
820
|
+
let x = e.clientX + 14;
|
|
821
|
+
if (x + width > window.innerWidth - 8) x = Math.max(8, window.innerWidth - width - 8);
|
|
822
|
+
tip.style.left = x + 'px';
|
|
823
|
+
tip.style.top = (box.bottom + 6) + 'px';
|
|
824
|
+
});
|
|
825
|
+
wf.addEventListener('mouseleave', () => { tip.hidden = true; });
|
|
826
|
+
|
|
827
|
+
if (location.hash.startsWith('#span=')) {
|
|
828
|
+
const id = decodeURIComponent(location.hash.slice('#span='.length));
|
|
829
|
+
if (byId.has(id)) {
|
|
830
|
+
select(id, false);
|
|
831
|
+
byId.get(id).scrollIntoView({ block: 'center' });
|
|
832
|
+
}
|
|
833
|
+
}
|
|
834
|
+
}
|
|
835
|
+
bindWaterfall();
|
|
836
|
+
})();
|
|
837
|
+
</script>
|
|
838
|
+
{% endblock %}
|