nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,298 @@
|
|
|
1
|
+
{% from 'runs/_runtime.html' import what_ran %}
|
|
2
|
+
{% extends "base.html" %}
|
|
3
|
+
{% block title %}Compare {{ jobs|length }} runs — Nullius{% endblock %}
|
|
4
|
+
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='compare/comparison.css') }}">{% endblock %}
|
|
5
|
+
{% block crumb %}/ compare{% endblock %}
|
|
6
|
+
{% block body %}
|
|
7
|
+
{% macro comparison_columns() %}
|
|
8
|
+
<colgroup>
|
|
9
|
+
<col class="compare-label-col">
|
|
10
|
+
{% for j in jobs %}<col>{% endfor %}
|
|
11
|
+
</colgroup>
|
|
12
|
+
{% endmacro %}
|
|
13
|
+
|
|
14
|
+
<div class="page-title compare-title"><h1>Comparing {{ jobs|length }} runs</h1><a class="btn ghost sm" href="{{ url_for('compare_picker', pick=picks, cohort=cohort) }}">Choose runs</a></div>
|
|
15
|
+
<p class="subtitle">
|
|
16
|
+
{{ n_shared }} task{{ '' if n_shared == 1 else 's' }} run by every side<span class="dot">·</span>
|
|
17
|
+
<strong class="{{ 'warn' if n_differing else 'muted' }}">{{ n_differing }} where they disagree</strong>
|
|
18
|
+
</p>
|
|
19
|
+
<div class="cohort-tools">
|
|
20
|
+
<nav class="cohort-switch" aria-label="Comparison cohort">
|
|
21
|
+
<a href="{{ url_for('compare', pick=picks, cohort='whole') }}" aria-current="{{ 'true' if cohort == 'whole' else 'false' }}">Whole runs</a>
|
|
22
|
+
<a href="{{ url_for('compare', pick=picks, cohort='matched') }}" aria-current="{{ 'true' if cohort == 'matched' else 'false' }}">Matched tasks</a>
|
|
23
|
+
</nav>
|
|
24
|
+
</div>
|
|
25
|
+
<div class="cohort-note">
|
|
26
|
+
{% if cohort == 'matched' %}
|
|
27
|
+
<p><strong>{{ n_shared }} matched task{{ '' if n_shared == 1 else 's' }}.</strong> Rates, cost and tokens include every attempt on those tasks. The full run’s wall time is omitted.</p>
|
|
28
|
+
{% else %}
|
|
29
|
+
<p>These numbers describe each full run, including tasks the other runs did not cover. Switch to matched tasks to compare the same recorded task IDs.</p>
|
|
30
|
+
{% endif %}
|
|
31
|
+
{% if not comparison.datasets_known %}<p>A run has no recorded dataset identity. Its task names are kept separate because their identity cannot be verified.</p>
|
|
32
|
+
{% elif not comparison.datasets_match %}<p>The recorded datasets differ. Matching requires exact dataset names and versions; identical task names from different datasets are kept separate.</p>{% endif %}
|
|
33
|
+
{% if cohort == 'matched' and not n_shared %}<p>No shared cohort is available. Choose runs with overlapping coverage or return to whole runs.</p>{% endif %}
|
|
34
|
+
{% if comparison.repeated_trials|sum %}<p>Some tasks have repeated attempts. All attempts count in the summaries; each task row and its detail show the <strong>last indexed attempt</strong>, not necessarily the latest by timestamp.</p>{% endif %}
|
|
35
|
+
<p>Adjusted rate excludes only rate limits, API errors, setup errors and spending caps. Unknown outcomes stay in its denominator. Controls are never ranked.</p>
|
|
36
|
+
</div>
|
|
37
|
+
|
|
38
|
+
<h2>Side by side</h2>
|
|
39
|
+
{% if n_agents > 1 %}
|
|
40
|
+
<div class="table-tools">
|
|
41
|
+
<span class="table-note">Order columns:</span>
|
|
42
|
+
<a class="btn ghost sm" href="{{ url_for('compare', pick=order_grouped, cohort=cohort) }}">Group by agent</a>
|
|
43
|
+
<a class="btn ghost sm" href="{{ url_for('compare', pick=order_alternating, cohort=cohort) }}">Alternate agents</a>
|
|
44
|
+
</div>
|
|
45
|
+
{% endif %}
|
|
46
|
+
<div class="table-wrap compare-table-wrap" role="region" aria-label="Run comparison" tabindex="0">
|
|
47
|
+
<table class="grid compare" style="--cols: {{ jobs|length }}">
|
|
48
|
+
{{ comparison_columns() }}
|
|
49
|
+
<thead>
|
|
50
|
+
<tr>
|
|
51
|
+
<th></th>
|
|
52
|
+
{% for j in jobs %}
|
|
53
|
+
{% set selection = selections[loop.index0] %}
|
|
54
|
+
<th class="draggable-col" draggable="true" data-job="{{ selection.value }}"
|
|
55
|
+
title="Drag to reorder">
|
|
56
|
+
<div class="col-head">
|
|
57
|
+
<span class="strong">{{ j.agent_label or '—' }}
|
|
58
|
+
{%- if j.is_control %} <span class="chip-control"
|
|
59
|
+
title="Harbor control: applies the gold patch or does nothing. Never calls a model, and never ranked.">control</span>{% endif %}</span>
|
|
60
|
+
<a class="faint mono col-id" href="{{ url_for('job', job_id=j.job_id, workspace=selection.workspace) }}">{{ j.job_id }}</a>
|
|
61
|
+
<span class="compare-workspace">{{ selection.workspace_label }}</span>
|
|
62
|
+
{% if cohort == 'matched' %}<span class="compare-excluded">{{ comparison.excluded_trials[loop.index0] }} of {{ selection.job.trials|length }} trials outside this cohort</span>{% endif %}
|
|
63
|
+
</div>
|
|
64
|
+
</th>
|
|
65
|
+
{% endfor %}
|
|
66
|
+
</tr>
|
|
67
|
+
</thead>
|
|
68
|
+
<tbody>
|
|
69
|
+
{% macro row(label, values, hint='') %}
|
|
70
|
+
<tr>
|
|
71
|
+
<th class="rowhead" {% if hint %}title="{{ hint }}"{% endif %}>{{ label }}</th>
|
|
72
|
+
{% for v in values %}<td>{{ v }}</td>{% endfor %}
|
|
73
|
+
</tr>
|
|
74
|
+
{% endmacro %}
|
|
75
|
+
|
|
76
|
+
<tr>
|
|
77
|
+
<th class="rowhead">Agent</th>
|
|
78
|
+
{% for j in jobs %}
|
|
79
|
+
<td><span class="strong">{{ j.agent_label or '—' }}</span>
|
|
80
|
+
{%- if j.is_control %} <span class="chip-control"
|
|
81
|
+
title="Harbor control — excluded from every ranking below">control</span>{% endif %}
|
|
82
|
+
{%- if j.reasoning_effort %} <span class="faint">· effort {{ j.reasoning_effort }}</span>{% endif %}</td>
|
|
83
|
+
{% endfor %}
|
|
84
|
+
</tr>
|
|
85
|
+
{{ row('Model', jobs|map(attribute='model_name')|list) }}
|
|
86
|
+
<tr>
|
|
87
|
+
<th class="rowhead">Dataset</th>
|
|
88
|
+
{% for j in jobs %}<td title="{{ j.dataset }}">{{ j.dataset|dsname }}</td>{% endfor %}
|
|
89
|
+
</tr>
|
|
90
|
+
{% if jobs|selectattr('runner')|list %}
|
|
91
|
+
<tr>
|
|
92
|
+
<th class="rowhead" title="What ran: the runner, and the upstream commit its install resolved">Runner</th>
|
|
93
|
+
{% for j in jobs %}<td>{% if j.runner %}{{ what_ran(j) }}{% else %}<span class="faint">—</span>{% endif %}</td>{% endfor %}
|
|
94
|
+
</tr>
|
|
95
|
+
{% endif %}
|
|
96
|
+
<tr>
|
|
97
|
+
<th class="rowhead">Outcome mix</th>
|
|
98
|
+
{% for j in jobs %}
|
|
99
|
+
<td><span class="minibar">
|
|
100
|
+
{%- for k in ORDER %}{% if j.counts[k] %}<span class="seg oc-{{ k }}" style="flex:{{ j.counts[k] }}"
|
|
101
|
+
title="{{ LABELS[k] }}: {{ j.counts[k] }}"></span>{% endif %}{% endfor -%}
|
|
102
|
+
</span></td>
|
|
103
|
+
{% endfor %}
|
|
104
|
+
</tr>
|
|
105
|
+
{% for r in summary %}
|
|
106
|
+
<tr>
|
|
107
|
+
<th class="rowhead" {% if r.hint %}title="{{ r.hint }}"{% endif %}>{{ r.label }}</th>
|
|
108
|
+
{% for v in r.cells %}<td class="{{ r.classes[loop.index0] }}">{{ v }}</td>{% endfor %}
|
|
109
|
+
</tr>
|
|
110
|
+
{% endfor %}
|
|
111
|
+
<tr>
|
|
112
|
+
<th class="rowhead" title="The recorded invocation — a foreign harness's own command verbatim, or the harbor run reconstructed from this run's recorded config — copy it to run the same benchmark again">Rerun</th>
|
|
113
|
+
{% for j in jobs %}
|
|
114
|
+
<td>{% if j.harbor_command %}<button class="btn ghost sm" data-copy="{{ j.harbor_command }}"
|
|
115
|
+
title="{{ j.harbor_command }}">copy command</button>{% else %}<span class="faint">—</span>{% endif %}</td>
|
|
116
|
+
{% endfor %}
|
|
117
|
+
</tr>
|
|
118
|
+
</tbody>
|
|
119
|
+
</table>
|
|
120
|
+
</div>
|
|
121
|
+
|
|
122
|
+
<h2>Task by task</h2>
|
|
123
|
+
<p class="subtitle">Tasks where the runs disagree are listed first. A blank cell means that run
|
|
124
|
+
didn't include the task.</p>
|
|
125
|
+
{% if judgement %}
|
|
126
|
+
<div class="judgement-job" id="judgement">
|
|
127
|
+
<span class="small"><strong>Judgement</strong> <span class="badge">experimental</span>
|
|
128
|
+
· {{ judgement.judged }} of {{ judgement.pairs }} matched pair{{ '' if judgement.pairs == 1 else 's' }} judged{% if judgement.unjudged %}, {{ judgement.unjudged }} not judged{% endif %}{% if judgement.earlier %} ({{ judgement.earlier }} under an earlier vocabulary){% endif %}</span>
|
|
129
|
+
{% if judgement.unjudged and judgement.keyed %}
|
|
130
|
+
<form method="post" action="{{ url_for('judge_compare_route', pick=picks, cohort=cohort) }}" class="header-form" id="judge-all">
|
|
131
|
+
<input type="hidden" name="csrf" value="{{ csrf }}">
|
|
132
|
+
<button class="btn sm ghost" type="submit">Judge {{ judgement.unjudged }} pair{{ '' if judgement.unjudged == 1 else 's' }}</button>
|
|
133
|
+
</form>
|
|
134
|
+
{% elif judgement.unjudged %}<span class="small faint">System One not configured — <a href="{{ url_for('settings_page') }}#judge">set its key in Settings</a></span><button class="btn sm ghost" type="button" disabled title="System One not configured: Settings, Judge">Judge all</button>{% endif %}
|
|
135
|
+
</div>
|
|
136
|
+
{% endif %}
|
|
137
|
+
<div class="table-wrap compare-table-wrap" role="region" aria-label="Task comparison" tabindex="0">
|
|
138
|
+
<table class="grid compare tasks" style="--cols: {{ jobs|length }}">
|
|
139
|
+
{{ comparison_columns() }}
|
|
140
|
+
<thead>
|
|
141
|
+
<tr>
|
|
142
|
+
<th>Task</th>
|
|
143
|
+
{% for j in jobs %}
|
|
144
|
+
<th class="draggable-col" draggable="true" data-job="{{ selections[loop.index0].value }}" title="Drag to reorder">
|
|
145
|
+
{{ j.agent_label or j.job_id }}
|
|
146
|
+
{%- if j.is_control %} <span class="chip-control">control</span>{% endif %}
|
|
147
|
+
</th>
|
|
148
|
+
{% endfor %}
|
|
149
|
+
{% if judgement %}<th class="judgement-col" title="Each side scored alone on {{ judgement.question }}, 0 to 3; the higher score by at least {{ judgement.margin }} wins, else a tie">Better of two</th>{% endif %}
|
|
150
|
+
</tr>
|
|
151
|
+
</thead>
|
|
152
|
+
<tbody>
|
|
153
|
+
{% for taskrow in tasks %}
|
|
154
|
+
{% set task_index = loop.index0 %}
|
|
155
|
+
{% set row = taskrow.trials %}
|
|
156
|
+
{% set outcomes = row|select|map(attribute='outcome')|list %}
|
|
157
|
+
<tr class="compare-task-row {% if outcomes|unique|list|length > 1 %}row-differs{% endif %}">
|
|
158
|
+
<td class="mono">
|
|
159
|
+
<div class="compare-task-name">
|
|
160
|
+
{# One glyph, rotated by CSS on aria-expanded. #}
|
|
161
|
+
<button type="button" class="disclose" aria-expanded="false"
|
|
162
|
+
data-task="{{ loop.index0 }}" aria-controls="compare-detail-{{ loop.index0 }}"
|
|
163
|
+
aria-label="Compare details for {{ taskrow.task }}" title="Compare details">▼</button>
|
|
164
|
+
<span class="compare-task-label" title="{{ taskrow.task }}">{{ taskrow.task }}</span>
|
|
165
|
+
</div>
|
|
166
|
+
{% if not comparison.datasets_match or not comparison.datasets_known %}<span class="compare-task-dataset">{{ 'Dataset not recorded' if taskrow.dataset.startswith('[unidentified:') else taskrow.dataset|dsname }}</span>{% endif %}
|
|
167
|
+
</td>
|
|
168
|
+
{% for t in row %}
|
|
169
|
+
<td>
|
|
170
|
+
{% if t %}
|
|
171
|
+
<div class="compare-run-cell">
|
|
172
|
+
<div class="compare-cell-summary">
|
|
173
|
+
<div class="compare-cell-outcome">
|
|
174
|
+
<a class="cell-outcome" href="{{ url_for('trial', job_id=t.job_id, trial_name=t.trial_name, workspace=selections[loop.index0].workspace) }}">
|
|
175
|
+
<span class="tag oc-{{ t.outcome }}">{{ t.outcome_label }}</span>
|
|
176
|
+
</a>
|
|
177
|
+
{% if t.score is not none %}<span class="badge trial-score" title="{% if t.n_parts %}The mean score of its {{ t.n_parts }} parts{% else %}Its score{% endif %}, as the benchmark's own scorer gave it">{{ '%.2f'|format(t.score) }}</span>{% endif %}
|
|
178
|
+
{% if t.f2p_pass or t.f2p_fail %}
|
|
179
|
+
<span class="faint small">{{ t.f2p_pass }}/{{ t.f2p_pass + t.f2p_fail }} {{ t.vocab.gained_abbr or t.vocab.gained }}</span>
|
|
180
|
+
{% endif %}
|
|
181
|
+
{% if t.is_regression %}<span class="badge danger" title="Broke {{ t.p2p_fail }} passing tests">⚠</span>{% endif %}
|
|
182
|
+
</div>
|
|
183
|
+
<span class="compare-quick-metric" title="Recorded total trial duration"><span>Total time</span><strong>{{ t.dur_total|secs }}</strong></span>
|
|
184
|
+
{% set used_tokens = (t.n_input_tokens or 0) + (t.n_output_tokens or 0) %}
|
|
185
|
+
<span class="compare-quick-metric" title="{% if used_tokens or jobs[loop.index0].is_control %}Input {{ t.n_input_tokens or 0 }} · Output {{ t.n_output_tokens or 0 }} · Cached input {{ t.n_cache_tokens or 0 }} (included in input){% else %}Token usage not recorded{% endif %}"><span>Tokens</span><strong>{{ used_tokens|tokens(jobs[loop.index0].kind) }}</strong></span>
|
|
186
|
+
</div>
|
|
187
|
+
</div>
|
|
188
|
+
{% if taskrow.attempts[loop.index0] > 1 %}<span class="compare-attempts">{{ taskrow.attempts[loop.index0] }} attempts · last indexed shown</span>{% endif %}
|
|
189
|
+
{% else %}
|
|
190
|
+
<span class="faint">—</span>
|
|
191
|
+
{% endif %}
|
|
192
|
+
</td>
|
|
193
|
+
{% endfor %}
|
|
194
|
+
{% if judgement %}<td class="judgement-cell">{% set jr = judgement.rows.get(taskrow.task) %}
|
|
195
|
+
{% if jr and jr.earlier %}<strong>{{ {'A': jobs[0].agent_label or 'A', 'B': jobs[1].agent_label or 'B', 'same': 'same'}[jr.choice] if jr.choice in ('A', 'B', 'same') else jr.choice }}</strong> <span class="mono small" title="{% for k, v in jr.probabilities.items() %}{{ k }} {{ '%.0f'|format(v * 100) }}%{{ ' · ' if not loop.last }}{% endfor %}">{{ '%.0f'|format((jr.p or 0) * 100) }}%</span>
|
|
196
|
+
<span class="badge" title="Asked under {{ jr.questions }} as one pair state, the pick the model's own; the current vocabulary scores each side alone. Judging again scores both sides.">earlier vocabulary</span>
|
|
197
|
+
{% elif jr %}
|
|
198
|
+
{% if jr.verdict == 'same' %}<strong title="The two scores are within {{ judgement.margin }} of a level: a tie, not a pick">tie</strong>
|
|
199
|
+
{% elif jr.verdict %}<strong>{{ (jobs[0] if jr.verdict == 'A' else jobs[1]).agent_label or jr.verdict }}</strong>
|
|
200
|
+
{% else %}<span class="faint">one side judged</span>{% endif %}
|
|
201
|
+
<div class="judgement-sides">
|
|
202
|
+
{% for side, j in ((jr.a, jobs[0]), (jr.b, jobs[1])) %}
|
|
203
|
+
<span class="small" title="{% if side %}{{ j.agent_label or j.job_id }}: {{ side.label }} — {{ '%.2f'|format(side.score) }} of {{ side.top }}{% if side.confidence is not none %}, confidence {{ '%.0f'|format(side.confidence * 100) }}%{% endif %} · {{ side.model }}{% else %}{{ j.agent_label or j.job_id }}: not judged{% endif %}">
|
|
204
|
+
<span class="muted">{{ 'A' if loop.first else 'B' }}</span>
|
|
205
|
+
{% if side %}<span class="mono">{{ '%.1f'|format(side.score) }}</span><span class="faint">/{{ side.top }}</span>{% if side.confidence is not none %} <span class="mono faint">{{ '%.0f'|format(side.confidence * 100) }}%</span>{% endif %}
|
|
206
|
+
{% else %}<span class="faint">not judged</span>{% endif %}</span>
|
|
207
|
+
{% endfor %}
|
|
208
|
+
</div>
|
|
209
|
+
{% else %}<span class="faint">not judged</span>{% endif %}</td>{% endif %}
|
|
210
|
+
</tr>
|
|
211
|
+
|
|
212
|
+
{# detail rows: the same metrics for each side, so differences line up #}
|
|
213
|
+
<tr class="detail-row" id="compare-detail-{{ loop.index0 }}" data-detail="{{ loop.index0 }}" hidden>
|
|
214
|
+
<td colspan="{{ jobs|length + 1 }}">
|
|
215
|
+
<div class="detail-body" data-detail-url="{{ url_for('compare_detail_row', index=task_index, pick=picks, cohort=cohort) }}"><p class="faint small">Loading…</p></div>
|
|
216
|
+
<p class="detail-more">
|
|
217
|
+
<a class="btn ghost sm" href="{{ url_for('compare_task', task=taskrow.task, dataset=taskrow.dataset, pick=picks, cohort=cohort) }}">Open trials side by side →</a>
|
|
218
|
+
</p>
|
|
219
|
+
</td>
|
|
220
|
+
</tr>
|
|
221
|
+
{% else %}<tr><td colspan="{{ jobs|length + 1 }}" class="empty-note">No tasks in this comparison cohort.</td></tr>{% endfor %}
|
|
222
|
+
</tbody>
|
|
223
|
+
</table>
|
|
224
|
+
</div>
|
|
225
|
+
|
|
226
|
+
<script>
|
|
227
|
+
/* ---- drag a column header to reorder ----
|
|
228
|
+
Order is carried by qualified ?pick= params, so a drop navigates to the
|
|
229
|
+
reordered URL. Both tables and every detail row follow automatically, and
|
|
230
|
+
the arrangement stays shareable. The Group / Alternate buttons above the
|
|
231
|
+
table are the pointer-free path. */
|
|
232
|
+
(() => {
|
|
233
|
+
const ORDER = {{ picks|tojson }};
|
|
234
|
+
let dragged = null;
|
|
235
|
+
|
|
236
|
+
const clear = () => document.querySelectorAll('.draggable-col').forEach(th => {
|
|
237
|
+
th.classList.remove('dragging', 'drop-before', 'drop-after');
|
|
238
|
+
});
|
|
239
|
+
|
|
240
|
+
document.querySelectorAll('.draggable-col').forEach(th => {
|
|
241
|
+
th.addEventListener('dragstart', e => {
|
|
242
|
+
dragged = th.dataset.job;
|
|
243
|
+
th.classList.add('dragging');
|
|
244
|
+
e.dataTransfer.effectAllowed = 'move';
|
|
245
|
+
// Firefox needs data set for the drag to start at all.
|
|
246
|
+
e.dataTransfer.setData('text/plain', dragged);
|
|
247
|
+
});
|
|
248
|
+
th.addEventListener('dragend', clear);
|
|
249
|
+
|
|
250
|
+
th.addEventListener('dragover', e => {
|
|
251
|
+
if (!dragged || th.dataset.job === dragged) return;
|
|
252
|
+
e.preventDefault();
|
|
253
|
+
e.dataTransfer.dropEffect = 'move';
|
|
254
|
+
const box = th.getBoundingClientRect();
|
|
255
|
+
const after = e.clientX > box.left + box.width / 2;
|
|
256
|
+
th.classList.toggle('drop-before', !after);
|
|
257
|
+
th.classList.toggle('drop-after', after);
|
|
258
|
+
});
|
|
259
|
+
th.addEventListener('dragleave', () =>
|
|
260
|
+
th.classList.remove('drop-before', 'drop-after'));
|
|
261
|
+
|
|
262
|
+
th.addEventListener('drop', e => {
|
|
263
|
+
e.preventDefault();
|
|
264
|
+
const target = th.dataset.job;
|
|
265
|
+
if (!dragged || target === dragged) return clear();
|
|
266
|
+
const box = th.getBoundingClientRect();
|
|
267
|
+
const after = e.clientX > box.left + box.width / 2;
|
|
268
|
+
|
|
269
|
+
const next = ORDER.filter(id => id !== dragged);
|
|
270
|
+
const at = next.indexOf(target) + (after ? 1 : 0);
|
|
271
|
+
next.splice(at, 0, dragged);
|
|
272
|
+
|
|
273
|
+
const url = new URL('{{ url_for("compare") }}', location.origin);
|
|
274
|
+
next.forEach(id => url.searchParams.append('pick', id));
|
|
275
|
+
url.searchParams.set('cohort', {{ cohort|tojson }});
|
|
276
|
+
location.href = url;
|
|
277
|
+
});
|
|
278
|
+
});
|
|
279
|
+
})();
|
|
280
|
+
|
|
281
|
+
document.querySelectorAll('.disclose').forEach(btn => {
|
|
282
|
+
btn.addEventListener('click', () => {
|
|
283
|
+
const row = document.querySelector(`[data-detail="${btn.dataset.task}"]`);
|
|
284
|
+
const open = row.hidden;
|
|
285
|
+
row.hidden = !open;
|
|
286
|
+
btn.setAttribute('aria-expanded', String(open));
|
|
287
|
+
// the detail is built when it is first opened: every row's at once read every trajectory of both runs
|
|
288
|
+
const body = row.querySelector('[data-detail-url]');
|
|
289
|
+
if (open && body && !body.dataset.loaded) {
|
|
290
|
+
body.dataset.loaded = '1';
|
|
291
|
+
nulliusFetch(body.dataset.detailUrl).then(r => r.ok ? r.text() : Promise.reject(r.status))
|
|
292
|
+
.then(html => { body.innerHTML = html; })
|
|
293
|
+
.catch(() => { body.dataset.loaded = ''; body.innerHTML = '<p class="faint small">The detail could not be read; open the row again.</p>'; });
|
|
294
|
+
}
|
|
295
|
+
});
|
|
296
|
+
});
|
|
297
|
+
</script>
|
|
298
|
+
{% endblock %}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
{% extends "base.html" %}
|
|
2
|
+
{% block title %}Choose runs to compare — Nullius{% endblock %}
|
|
3
|
+
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='compare/comparison.css') }}">{% endblock %}
|
|
4
|
+
{% block crumb %}/ compare{% endblock %}
|
|
5
|
+
{% block body %}
|
|
6
|
+
<div class="compare-picker">
|
|
7
|
+
<div class="page-title"><h1>Compare runs</h1></div>
|
|
8
|
+
<p class="subtitle">Choose two runs from any saved workspace. Compare their full results or restrict the numbers to tasks both runs covered.</p>
|
|
9
|
+
<form class="panel" action="{{ url_for('compare_picker') }}" method="get">
|
|
10
|
+
<p class="compare-pick-step">01 / Choose workspaces</p>
|
|
11
|
+
<input type="hidden" name="cohort" value="{{ cohort }}">
|
|
12
|
+
<div class="compare-pick-grid">
|
|
13
|
+
{% for side in choices %}
|
|
14
|
+
<div class="compare-pick-side">
|
|
15
|
+
<label for="compare-workspace-{{ loop.index }}">{{ 'Left' if loop.first else 'Right' }} workspace</label>
|
|
16
|
+
<select id="compare-workspace-{{ loop.index }}" name="{{ 'left' if loop.first else 'right' }}">
|
|
17
|
+
{% for ws in workspace_options %}
|
|
18
|
+
<option value="{{ ws.name }}" {% if ws.name == side.name %}selected{% endif %}>{{ ws.label }}</option>
|
|
19
|
+
{% endfor %}
|
|
20
|
+
</select>
|
|
21
|
+
</div>
|
|
22
|
+
{% endfor %}
|
|
23
|
+
</div>
|
|
24
|
+
<div class="compare-pick-actions"><button class="btn ghost sm" type="submit">Show runs</button><span class="compare-pick-caption">Uses the runs already read by each workspace.</span></div>
|
|
25
|
+
</form>
|
|
26
|
+
<form class="panel" action="{{ url_for('compare') }}" method="get">
|
|
27
|
+
<p class="compare-pick-step">02 / Choose runs</p>
|
|
28
|
+
<div class="compare-pick-grid">
|
|
29
|
+
{% for side in choices %}
|
|
30
|
+
<div class="compare-pick-side">
|
|
31
|
+
<label for="compare-run-{{ loop.index }}">{{ 'Left' if loop.first else 'Right' }} run · {{ side.label }}</label>
|
|
32
|
+
<select id="compare-run-{{ loop.index }}" name="pick" required {% if not side.jobs %}disabled{% endif %}>
|
|
33
|
+
{% for run in side.jobs %}
|
|
34
|
+
{% set reference = side.name ~ ':' ~ run.job_id %}
|
|
35
|
+
<option value="{{ reference }}" {% if reference == side.selected %}selected{% endif %}>{{ run.agent_label or 'Unknown agent' }} · {{ run.job_id }} · {{ run.trials|length }} trials</option>
|
|
36
|
+
{% else %}<option value="">No runs available</option>{% endfor %}
|
|
37
|
+
</select>
|
|
38
|
+
<p class="compare-pick-caption">{{ side.jobs|length }} run{{ '' if side.jobs|length == 1 else 's' }} available{% if side.name %} · {{ side.name }}{% endif %}</p>
|
|
39
|
+
{% if not side.jobs %}<p class="compare-pick-caption">Open this workspace and refresh its runs, or <a href="{{ url_for('import_page') }}">import a run</a>.</p>{% endif %}
|
|
40
|
+
</div>
|
|
41
|
+
{% endfor %}
|
|
42
|
+
</div>
|
|
43
|
+
<div class="compare-pick-actions">
|
|
44
|
+
<label><input type="radio" name="cohort" value="whole" {% if cohort == 'whole' %}checked{% endif %}> Whole runs</label>
|
|
45
|
+
<label><input type="radio" name="cohort" value="matched" {% if cohort == 'matched' %}checked{% endif %}> Matched tasks</label>
|
|
46
|
+
<button class="btn" type="submit" {% if not choices[0].jobs or not choices[1].jobs %}disabled{% endif %}>Compare runs →</button>
|
|
47
|
+
</div>
|
|
48
|
+
</form>
|
|
49
|
+
<p class="compare-pick-caption">Matched tasks require the same recorded dataset and task IDs. Every attempt stays in the totals, and control runs remain unranked.</p>
|
|
50
|
+
</div>
|
|
51
|
+
{% endblock %}
|