nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,270 @@
|
|
|
1
|
+
{% extends "base.html" %}
|
|
2
|
+
{% from "catalogue/use_bundle.html" import use_bundle %}
|
|
3
|
+
{% block title %}{{ b.title }} — Nullius{% endblock %}
|
|
4
|
+
{% block crumb %}/ <a href="{{ url_for('benchmarks_page') }}">benchmarks</a> / {{ b.key }}{% endblock %}
|
|
5
|
+
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='catalogue/catalogue.css') }}">{% endblock %}
|
|
6
|
+
{% block body %}
|
|
7
|
+
{% set ds = cat.datasets_of(b) %}{% set tasks = cat.tasks_of(b) %}{% set files = cat.files(b.key) %}
|
|
8
|
+
{% set ways = (['the Nullius standard'] if 'standard' in b.harness else []) + (['Harbor'] if 'harbor' in b.harness else []) %}{% set runs_with = ways|join(' or ') if ways else 'no harness here' %}
|
|
9
|
+
<article class="bench-page" data-route-case="{{ b.route_case }}">
|
|
10
|
+
<header class="bench-head">
|
|
11
|
+
<div class="bench-head-copy">
|
|
12
|
+
<div class="bench-tags">{% if b.category %}<span class="cat-tag">{{ b.category }}</span>{% endif %}<span class="origin-tag {{ b.origin }}">{{ b.label }}</span></div>
|
|
13
|
+
<h1><svg class="bench-mark" aria-hidden="true"><use href="{{ url_for('static', filename='catalogue/marks.svg') }}#bm-{{ b.key }}"/></svg>{{ b.title }}</h1>
|
|
14
|
+
{% if b.domain %}<p class="bench-domain">{{ b.domain }}</p>{% endif %}
|
|
15
|
+
</div>
|
|
16
|
+
<div class="bench-actions">
|
|
17
|
+
{% if b.usable_routes %}<a class="btn" href="{{ url_for('new_run', standard=b.key) if 'standard' in b.harness else url_for('new_run', dataset=(b.datasets[0] if b.datasets else none)) }}">Build a run <span aria-hidden="true">→</span></a>{% endif %}
|
|
18
|
+
{% if b.citation %}<a class="btn ghost" href="#citation">Cite</a>{% endif %}
|
|
19
|
+
{% if editable %}<a class="btn ghost" href="{{ url_for('benchmark_edit', key=b.key) }}">Edit</a>{% endif %}
|
|
20
|
+
{#- deleting asks first, in place: the button opens the question, never a dialog #}
|
|
21
|
+
{% if restore_bundle %}{% if local %}{{ use_bundle('benchmarks', b.key, csrf) }}{% endif %}
|
|
22
|
+
{% elif editable and deletable %}<details class="bench-delete">
|
|
23
|
+
<summary class="btn ghost danger">Delete</summary>
|
|
24
|
+
<form method="post" action="{{ url_for('benchmark_delete', key=b.key) }}" class="bench-delete-ask">
|
|
25
|
+
<input type="hidden" name="csrf" value="{{ csrf }}">
|
|
26
|
+
<span>Delete <b>{{ b.key }}</b> and its folder? Runs made from it stay.</span>
|
|
27
|
+
<button type="submit" class="btn danger">Yes, delete</button>
|
|
28
|
+
<button type="button" class="btn ghost" onclick="this.closest('details').open = false">Keep it</button>
|
|
29
|
+
</form>
|
|
30
|
+
</details>{% endif %}
|
|
31
|
+
</div>
|
|
32
|
+
</header>
|
|
33
|
+
{% if b.blocker %}<p class="bench-blocked" role="note">No usable route — {{ b.blocker }}</p>{% endif %}
|
|
34
|
+
|
|
35
|
+
<div class="bench-body">
|
|
36
|
+
<div class="bench-main">
|
|
37
|
+
|
|
38
|
+
{% if std %}
|
|
39
|
+
{#- The standard's proofs of it, per dataset (standard/prove.py): its controls as chips, their source in the title - the
|
|
40
|
+
builder's dataset step draws the same - then its units and leak checks; and Prove it, cli prove as a run. -#}
|
|
41
|
+
{% macro proof(rec, failed, expect) -%}
|
|
42
|
+
{%- if rec %}<span class="std-proof std-proof-ok" title="proven on {{ rec.on }}{% if rec.limit %}, over the first {{ rec.limit }}{% endif %}{% if rec.shard %}, shard {{ rec.shard }}{% endif %}">✓ {{ rec.passed }}{% if rec.limit %} · first {{ rec.limit }}{% endif %}</span>
|
|
43
|
+
{%- elif failed and failed.value is defined %}{% set e = (failed.expect or expect or {'at_least': 1}) %}{% set k = e|list|first %}<span class="std-proof std-proof-no" title="not proven: {{ ([failed.passed, failed.on] + (failed.problems or []))|select|join(' · ') }}">✕ {{ failed.value }} {{ '>' if k == 'at_most' else '<' }} {{ e[k] }}</span>
|
|
44
|
+
{%- elif failed %}<span class="std-proof std-proof-no" title="{{ (failed.problems or [])|join(' · ') }}">✕ not proven</span>
|
|
45
|
+
{%- else %}<span class="std-proof std-proof-none">not run</span>{% endif -%}
|
|
46
|
+
{%- endmacro %}
|
|
47
|
+
{% macro chip(c) -%}
|
|
48
|
+
<span class="std-ctl" title="{{ {'nop': 'no answer at all: must score nothing', 'oracle': 'the reference answers: must score everything'}.get(c.key, '') }} - {{ {'official': 'from the authors', 'adapter': 'from Harbor\'s adapter', 'nullius': 'made by Nullius'}.get(c.source, c.source) }}: {{ c.from }}"><b>{{ c.key }}</b>{{ proof(c.proven, c.failed, c.expect) }}</span>
|
|
49
|
+
{%- endmacro %}
|
|
50
|
+
{% macro check(name, says, rec, failed) -%}
|
|
51
|
+
<span class="std-ctl std-checkchip"><b>{{ name }}</b>{% if rec %}<span class="std-proof std-proof-ok" title="{{ says }} · {{ rec.passed }} · on {{ rec.on }}">✓</span>{% else %}{{ proof(rec, failed) }}{% endif %}</span>
|
|
52
|
+
{%- endmacro %}
|
|
53
|
+
{%- if std.setups %}
|
|
54
|
+
<section class="bench-section std-setups" id="setups" aria-labelledby="setups-title">
|
|
55
|
+
<h2 id="setups-title">Setup</h2>
|
|
56
|
+
{% set x = std.setups[0] %}
|
|
57
|
+
<div class="std-setup-list"><div class="std-setup-row">
|
|
58
|
+
<h3>Original</h3>
|
|
59
|
+
<div class="std-setup-meta"><span><b>Model</b> Choose in the run builder</span></div>
|
|
60
|
+
</div></div>
|
|
61
|
+
<details class="std-setup-detail"><summary>How it runs</summary>
|
|
62
|
+
<div class="std-setup-description">
|
|
63
|
+
<p>{{ x.summary }}</p>
|
|
64
|
+
<small class="mono">{{ std.ref }}:original</small>
|
|
65
|
+
</div>
|
|
66
|
+
</details>
|
|
67
|
+
</section>
|
|
68
|
+
{%- endif %}
|
|
69
|
+
<section class="bench-section std-proofs" id="standard" aria-labelledby="standard-title">
|
|
70
|
+
<h2 id="standard-title">Proofs</h2>
|
|
71
|
+
{% if std.upstream.repository %}<p class="mono bench-runner-repo" title="The upstream it runs: the release tag, and the commit it was checked at">{{ std.upstream.repository|replace('https://github.com/', '') }} @ {% if std.upstream.tag %}{{ std.upstream.tag }} · {% endif %}{{ std.upstream.tested[:7] }}</p>{% endif %}
|
|
72
|
+
<p class="bench-ds-note">Null and reference controls check the scorer; stability and leak checks validate the tasks.</p>
|
|
73
|
+
|
|
74
|
+
<table class="std-prooftable">
|
|
75
|
+
<tbody>
|
|
76
|
+
{% for d in std.datasets %}
|
|
77
|
+
<tr>
|
|
78
|
+
<th scope="row">{{ d.label }}{% if d.units %}<small>{{ d.units }} {{ std.units.nouns }}</small>{% endif %}</th>
|
|
79
|
+
<td><div class="std-ctls">
|
|
80
|
+
{{ check('stable', 'the same ids twice, each unique', d.proofs.units, (d.failed or {}).units) }}
|
|
81
|
+
{% if std.controls|selectattr('key', 'equalto', 'oracle')|list %}{{ check('no leak', 'nothing of the reference shows before the run', d.proofs.leak, (d.failed or {}).leak) }}{% endif %}
|
|
82
|
+
{% for c in d.controls %}{{ chip(c) }}{% endfor %}
|
|
83
|
+
</div>
|
|
84
|
+
{%- for c in d.controls if c.proven and c.proven.job in std_jobs %}<a class="std-joblink" href="{{ url_for('job', job_id=c.proven.job) }}">{{ c.key }}'s job</a>{% endfor %}</td>
|
|
85
|
+
</tr>
|
|
86
|
+
{% endfor %}
|
|
87
|
+
</tbody>
|
|
88
|
+
</table>
|
|
89
|
+
{% if local %}
|
|
90
|
+
<details class="std-check">
|
|
91
|
+
<summary>Run control checks</summary>
|
|
92
|
+
<form method="post" action="{{ url_for('benchmark_prove', key=b.key) }}">
|
|
93
|
+
<input type="hidden" name="csrf" value="{{ csrf }}">
|
|
94
|
+
<p>Check dataset stability, answer leaks and control scores. Results update the recorded proofs.
|
|
95
|
+
{% if std.judged %}The judge needs a model key.{% else %}No model calls or key required.{% endif %}</p>
|
|
96
|
+
<div class="std-form">
|
|
97
|
+
{% if std.datasets|length == 1 %}
|
|
98
|
+
<input type="hidden" name="dataset" value="{{ std.datasets[0].id }}">
|
|
99
|
+
<div class="std-fixed-dataset"><span>Dataset</span><b>{{ std.datasets[0].label }}</b></div>
|
|
100
|
+
{% else %}
|
|
101
|
+
<label>Dataset<select name="dataset"><option value="">All datasets</option>{% for d in std.datasets %}<option value="{{ d.id }}">{{ d.label }}</option>{% endfor %}</select></label>
|
|
102
|
+
{% endif %}
|
|
103
|
+
<label>{{ std.units.nouns|capitalize }}<input name="limit" type="number" min="1" placeholder="all" autocomplete="off"></label>
|
|
104
|
+
{% if std.judged %}
|
|
105
|
+
{% if providers %}<label>Judge's provider<select name="provider">{% for p, label in providers %}<option value="{{ p }}">{{ label }}</option>{% endfor %}</select></label>{% endif %}
|
|
106
|
+
<label>Judge model<input name="judge_model" placeholder="its own" autocomplete="off" spellcheck="false"></label>
|
|
107
|
+
{% endif %}
|
|
108
|
+
</div>
|
|
109
|
+
{% if std.judged and not providers %}<p>The judge needs a model key first: set one in <a href="{{ url_for('settings_page') }}">Settings</a>.</p>
|
|
110
|
+
{% else %}<button class="btn" type="submit">Run checks</button>{% endif %}
|
|
111
|
+
</form>
|
|
112
|
+
</details>
|
|
113
|
+
{% endif %}
|
|
114
|
+
</section>
|
|
115
|
+
{% if std.options %}
|
|
116
|
+
{#- Its own options (a search, a mode): each value, a line on it, what it needs; the machine setup some need. -#}
|
|
117
|
+
<section class="bench-section std-benchopts" id="options" aria-labelledby="options-title">
|
|
118
|
+
<h2 id="options-title">Options</h2>
|
|
119
|
+
{% for n, o in std.options.items() %}
|
|
120
|
+
<h3 class="std-optname">{{ o.label }}{% if o.datasets %} <small>{{ o.datasets|join(', ') }}</small>{% endif %}</h3>
|
|
121
|
+
<p class="bench-ds-note">{{ o.summary }}</p>
|
|
122
|
+
{% if o.pattern %}
|
|
123
|
+
<div class="std-option-range"><span class="std-tag">{{ 'Provider default' if n == 'temperature' and o.default == 'default' else o.default }}</span>
|
|
124
|
+
{% if n == 'temperature' %}<span>Custom decimal, e.g. <code>0.5</code></span>{% else %}<code>{{ o.pattern }}</code>{% endif %}
|
|
125
|
+
</div>
|
|
126
|
+
{% else %}
|
|
127
|
+
<ul class="std-optvals{{ ' std-optvals-detailed' if o.about or o.needs else '' }}">
|
|
128
|
+
{% for v in o['values'] %}{% set need = o.needs.get(v) or {} %}
|
|
129
|
+
<li><b class="mono">{{ v }}</b>{% if v == o.default and v != 'default' %} <span class="std-tag">default</span>{% endif %}
|
|
130
|
+
{% if need.embeddings %}<span class="run-tag" title="its embedding calls go through the run's relay">OpenAI key · {{ need.embeddings }}</span>{% endif %}
|
|
131
|
+
{% for h in need.host or [] %}<a class="std-need" href="#setup-{{ h }}" title="{{ std.host[h].summary }}">needs {{ h }}</a>{% endfor %}
|
|
132
|
+
{% if o.about.get(v) %}<small>{{ o.about[v] }}</small>{% endif %}</li>
|
|
133
|
+
{% endfor %}
|
|
134
|
+
</ul>
|
|
135
|
+
{% endif %}
|
|
136
|
+
{% endfor %}
|
|
137
|
+
{% for h, r in std.host.items() %}
|
|
138
|
+
<div class="std-hostreq" id="setup-{{ h }}">
|
|
139
|
+
<h3 class="std-optname">Setup: {{ h }}</h3>
|
|
140
|
+
<p class="bench-ds-note">{{ r.summary }} Checked by {{ r.check|join(', ') }} on PATH, on each machine a run uses; the builder's Where step checks it{% if r.install %} and can install it{% else %}: a machine's own hardware, nothing installs it{% endif %}.</p>
|
|
141
|
+
{% if r.install %}<div class="std-code"><div class="std-code-head"><span>install, on each machine</span><button type="button" class="std-copy">Copy</button></div><pre><code>{{ r.install|join('\n') }}</code></pre></div>{% endif %}
|
|
142
|
+
</div>
|
|
143
|
+
{% endfor %}
|
|
144
|
+
</section>
|
|
145
|
+
{% endif %}
|
|
146
|
+
|
|
147
|
+
{% endif %}
|
|
148
|
+
|
|
149
|
+
{% if ds %}
|
|
150
|
+
<section class="bench-section" aria-labelledby="datasets">
|
|
151
|
+
<h2 id="datasets">{{ 'Datasets' if ds|length > 1 else 'Dataset' }}{% if ds|length > 1 %} <span class="bench-n">{{ ds|length }}</span>{% endif %}</h2>
|
|
152
|
+
{# Several datasets are tabs, #<dataset key> selecting one; without the
|
|
153
|
+
script they are stacked, each under its title. #}
|
|
154
|
+
{% if ds|length > 1 %}<div class="bench-tabs" role="tablist" aria-label="Datasets" hidden>{% for d in ds %}<button type="button" role="tab" id="tab-{{ d.key }}" aria-controls="{{ d.key }}" aria-selected="{{ 'true' if loop.first else 'false' }}">{{ d.title }}</button>{% endfor %}</div>{% endif %}
|
|
155
|
+
{% for d in ds %}
|
|
156
|
+
<section class="cat-dataset" id="{{ d.key }}" aria-label="Dataset: {{ d.title }}"{% if ds|length > 1 %} role="tabpanel" aria-labelledby="tab-{{ d.key }}"{% endif %}>
|
|
157
|
+
<h3 class="bench-ds-title">{{ d.title }}</h3>
|
|
158
|
+
<div class="bench-facts">
|
|
159
|
+
<div><span>Tasks</span><b>{{ '{:,}'.format(d.tasks) if d.tasks else '—' }}</b></div>
|
|
160
|
+
{% if not data[d.key] %}<div><span>Source</span><b>{{ d.source }}</b></div>{% endif %}
|
|
161
|
+
{% if d.official %}<div><span>Official</span><a href="{{ d.official }}">{{ d.official|replace('https://', '') }}</a></div>{% endif %}
|
|
162
|
+
</div>
|
|
163
|
+
{% if d.notes %}<p class="bench-ds-note">{{ d.notes }}</p>{% endif %}
|
|
164
|
+
<dl class="bench-rows">
|
|
165
|
+
{% if d.verifier %}<div><dt>Verifier</dt><dd>{{ d.verifier }}</dd></div>{% endif %}
|
|
166
|
+
{#- One row per way it runs: where that harness reads the data, at which pin. The harness
|
|
167
|
+
fetches it when a run starts; nobody downloads anything by hand. #}
|
|
168
|
+
{% if data[d.key] %}<div><dt>Data</dt><dd>
|
|
169
|
+
<ul class="bench-sources">
|
|
170
|
+
{% for r in data[d.key] %}
|
|
171
|
+
<li>
|
|
172
|
+
<span class="run-tag rt-{{ r.way }}" title="Runs with {{ r.label }}">{{ r.label }}</span>
|
|
173
|
+
<span class="src-from">{{ r.from }}</span>
|
|
174
|
+
{% if r.href %}<a class="mono" href="{{ r.href }}">{{ r.name }}</a>{% else %}<b class="mono">{{ r.name }}</b>{% endif %}
|
|
175
|
+
{% if r.pin %}<span class="src-pin mono">{{ r.pin }}</span>{% endif %}
|
|
176
|
+
<small class="src-by">{{ r.by }}{% if r.files %} · {% for f in r.files %}<span class="mono">{{ f.name }}</span>{% if f.size %} {{ f.size }}{% endif %}{% if not loop.last %}, {% endif %}{% endfor %}{% if r.files|selectattr('checked')|list|length == r.files|length %}, checked by sha256{% endif %}{% endif %}</small>
|
|
177
|
+
</li>
|
|
178
|
+
{% endfor %}
|
|
179
|
+
</ul>
|
|
180
|
+
</dd></div>{% endif %}
|
|
181
|
+
</dl>
|
|
182
|
+
</section>
|
|
183
|
+
{% endfor %}
|
|
184
|
+
</section>
|
|
185
|
+
{% endif %}
|
|
186
|
+
|
|
187
|
+
{% if b.good_to_know %}
|
|
188
|
+
<section class="bench-section" aria-labelledby="good-to-know">
|
|
189
|
+
<h2 id="good-to-know">Good to know</h2>
|
|
190
|
+
<ul class="bench-list">{% for note in b.good_to_know %}<li>{{ note }}</li>{% endfor %}</ul>
|
|
191
|
+
</section>
|
|
192
|
+
{% endif %}
|
|
193
|
+
{% if b.upstream_limitations or b.limitations %}
|
|
194
|
+
<div class="bench-cols one">
|
|
195
|
+
{% if b.upstream_limitations %}
|
|
196
|
+
<section class="bench-section" aria-labelledby="upstream-limitations">
|
|
197
|
+
<h2 id="upstream-limitations">Upstream limitations</h2>
|
|
198
|
+
<ul class="bench-list">{% for l in b.upstream_limitations %}<li>{{ l }}</li>{% endfor %}</ul>
|
|
199
|
+
</section>
|
|
200
|
+
{% endif %}
|
|
201
|
+
{% if b.limitations %}
|
|
202
|
+
<section class="bench-section" aria-labelledby="limitations">
|
|
203
|
+
<h2 id="limitations">Nullius limitations</h2>
|
|
204
|
+
<ul class="bench-list">{% for l in b.limitations %}<li>{{ l }}</li>{% endfor %}</ul>
|
|
205
|
+
</section>
|
|
206
|
+
{% endif %}
|
|
207
|
+
</div>
|
|
208
|
+
{% endif %}
|
|
209
|
+
|
|
210
|
+
{% if b.citation %}
|
|
211
|
+
<section class="bench-section cat-citation" id="citation" aria-labelledby="citation-title">
|
|
212
|
+
<div class="bench-section-head"><h2 id="citation-title">Citation</h2>{% if papers|length > 1 %}<button class="btn ghost sm" type="button" data-copy="{{ b.citation }}">Copy all {{ papers|length }} BibTeX</button>{% endif %}</div>
|
|
213
|
+
{%- if papers %}{% set p = papers[0] %}
|
|
214
|
+
{#- the venue seal: the venue, year and kind turning slowly round the year, beside the paper as a reader names it #}
|
|
215
|
+
<div class="cite-seal">
|
|
216
|
+
<div class="cite-emblem" role="img" aria-label="{{ p.venue_short }} {{ p.year }}, {{ p.kind_label|lower }}">
|
|
217
|
+
<svg viewBox="0 0 170 170" aria-hidden="true">
|
|
218
|
+
<circle class="ring" cx="85" cy="85" r="80" stroke-width="2"/><circle class="ring" cx="85" cy="85" r="48" stroke-width="1"/>
|
|
219
|
+
<text class="yr" x="85" y="98" text-anchor="middle" font-size="36">{{ p.year }}</text>
|
|
220
|
+
</svg>
|
|
221
|
+
<div class="spin" aria-hidden="true"><svg viewBox="0 0 170 170">
|
|
222
|
+
<defs><path id="cite-arc" d="M85,85 m-62,0 a62,62 0 1,1 124,0 a62,62 0 1,1 -124,0"/></defs>
|
|
223
|
+
{%- set ring = (p.venue_short ~ ' · ' ~ p.year ~ ' · ' ~ p.kind_label ~ ' · ')|upper %}
|
|
224
|
+
<text font-size="10.5"><textPath href="#cite-arc" textLength="386" lengthAdjust="spacing">{{ ring * (2 if ring|length < 28 else 1) }}</textPath></text>
|
|
225
|
+
</svg></div>
|
|
226
|
+
</div>
|
|
227
|
+
<div class="cite-seal-text">
|
|
228
|
+
<h3 class="cite-title">{{ p.title }}</h3>
|
|
229
|
+
<p class="cite-who">{{ p.authors_short }}</p>
|
|
230
|
+
<p class="cite-venue">{{ p.venue }}</p>
|
|
231
|
+
<div class="cite-actions"><button class="btn sm" type="button" data-copy="{{ p.bibtex }}">Copy BibTeX</button>{% if p.url %}<a class="btn ghost sm" href="{{ p.url }}" target="_blank" rel="noopener">Read the paper ↗</a>{% endif %}</div>
|
|
232
|
+
</div>
|
|
233
|
+
</div>
|
|
234
|
+
{% if papers|length > 1 %}
|
|
235
|
+
<ul class="cite-more">
|
|
236
|
+
{% for q in papers[1:] %}<li><span><b>{{ q.title }}</b> <span class="faint">{{ q.venue_short }} {{ q.year }}</span></span><button class="btn ghost sm" type="button" data-copy="{{ q.bibtex }}">Copy BibTeX</button></li>{% endfor %}
|
|
237
|
+
</ul>
|
|
238
|
+
{% endif %}
|
|
239
|
+
{%- else %}
|
|
240
|
+
<pre class="bench-bibtex">{{ b.citation }}</pre>
|
|
241
|
+
{%- endif %}
|
|
242
|
+
</section>
|
|
243
|
+
{% endif %}
|
|
244
|
+
{%- if b.published %}
|
|
245
|
+
{#- A Harbor benchmark runs as published through Harbor's own agent for it, on any model: the builder names it. -#}
|
|
246
|
+
<section class="bench-section std-setups" id="published" aria-labelledby="published-title">
|
|
247
|
+
<h2 id="published-title">As published</h2>
|
|
248
|
+
<p>Harbor's <span class="mono">{{ b.published.agent }}</span>: {{ b.published.says }}</p>
|
|
249
|
+
</section>
|
|
250
|
+
{%- endif %}
|
|
251
|
+
</div>
|
|
252
|
+
|
|
253
|
+
<aside class="bench-aside" aria-label="At a glance">
|
|
254
|
+
<dl class="bench-glance">
|
|
255
|
+
<div><dt>Tasks</dt><dd>{{ tasks[0] }}{{ ' each' if tasks[1] == 'tasks each' }}</dd></div>
|
|
256
|
+
<div><dt>{{ 'Datasets' if ds|length != 1 else 'Dataset' }}</dt><dd>{{ ds|length }}</dd></div>
|
|
257
|
+
{% if b.scoring %}<div><dt>Scored by</dt><dd>{{ b.scoring }}</dd></div>{% endif %}
|
|
258
|
+
<div><dt>Runs with</dt><dd class="run-tags">{% if 'standard' in b.harness %}<span class="run-tag rt-standard" title="Runs with the Nullius standard">Nullius</span>{% endif %}{% if 'harbor' in b.harness %}<span class="run-tag rt-harbor" title="Runs with Harbor">Harbor</span>{% endif %}</dd></div>
|
|
259
|
+
{% if b.citation_source %}<div><dt>Official</dt><dd><a href="{{ b.citation_source }}">{{ b.citation_source|replace('https://', '')|replace('github.com/', '') }}</a></dd></div>{% endif %}
|
|
260
|
+
</dl>
|
|
261
|
+
{% if files %}<div class="bench-tree-box">
|
|
262
|
+
<div class="bench-tree-label">Catalogue files</div>
|
|
263
|
+
<pre class="bench-tree">benchmarks/{{ b.key }}/{% for f in files %}
|
|
264
|
+
{{ '└─ ' if loop.last else '├─ ' }}{{ f }}{% endfor %}</pre>
|
|
265
|
+
</div>{% endif %}
|
|
266
|
+
</aside>
|
|
267
|
+
</div>
|
|
268
|
+
</article>
|
|
269
|
+
{% endblock %}
|
|
270
|
+
{% block scripts %}<script src="{{ url_for('static', filename='catalogue/catalogue.js') }}"></script>{% endblock %}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
{% extends "base.html" %}
|
|
2
|
+
{% block title %}Benchmarks — Nullius{% endblock %}
|
|
3
|
+
{% block crumb %}/ benchmarks{% endblock %}
|
|
4
|
+
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='catalogue/catalogue.css') }}">{% endblock %}
|
|
5
|
+
{% block body %}
|
|
6
|
+
{% set listed = cat.listed_benchmarks %}
|
|
7
|
+
{#- a route's name, the same on the filter chips and on the cards' bubbles #}
|
|
8
|
+
{% set run_labels = {'harbor': 'Harbor', 'standard': 'Nullius'} %}
|
|
9
|
+
{% set run_names = {'harbor': 'Harbor', 'standard': 'the Nullius standard'} %}
|
|
10
|
+
{% set count = namespace(datasets=0) %}{% for b in listed %}{% set count.datasets = count.datasets + b.datasets|length %}{% endfor %}
|
|
11
|
+
<div class="bench-index">
|
|
12
|
+
<header class="bench-index-head">
|
|
13
|
+
<div>
|
|
14
|
+
<h1>Benchmarks</h1>
|
|
15
|
+
<p class="bench-count">{{ listed|length }} benchmarks · {{ count.datasets }} datasets</p>
|
|
16
|
+
</div>
|
|
17
|
+
<div class="bench-head-tools">
|
|
18
|
+
<label class="bench-find">Find
|
|
19
|
+
<input type="search" id="bench-find" placeholder="Name, category, dataset" autocomplete="off">
|
|
20
|
+
</label>
|
|
21
|
+
<button type="button" class="btn" id="std-how-open" aria-haspopup="dialog">+ Add a benchmark</button>
|
|
22
|
+
</div>
|
|
23
|
+
</header>
|
|
24
|
+
|
|
25
|
+
{#- Two routes in: the Nullius standard (a paper's own scripts as its Original setup), or Harbor. The assistant asks
|
|
26
|
+
which (the import-benchmark skill); by hand, each route has its own way. #}
|
|
27
|
+
{% from "catalogue/add_dialog.html" import add_dialog %}
|
|
28
|
+
{% call add_dialog('a benchmark', prompts, 'benchmark') %}
|
|
29
|
+
<p>Two ways a benchmark runs here. Pick its route, then follow it.</p>
|
|
30
|
+
<ul class="std-routes">
|
|
31
|
+
<li><span class="run-tag rt-standard">Nullius</span><div><b>Through the Nullius standard</b>
|
|
32
|
+
<p>Any agent over ACP, the benchmark's own code as its driver: a <code>standard.json</code> and a <code>driver.py</code> in <code>{{ catalogue_dir }}/benchmarks/<key>/</code>, its controls proven with <code>cli prove</code>. A paper's own scripts run as its Original setup.</p>
|
|
33
|
+
<p>The steps: <code>docs/standard.md</code>, "Import by hand", "A benchmark".</p></div></li>
|
|
34
|
+
<li><span class="run-tag rt-harbor">Harbor</span><div><b>A dataset Harbor already runs</b>
|
|
35
|
+
<p>In Harbor's registry or Hub: name it, read it from Harbor, check the entry, save.</p>
|
|
36
|
+
<a class="btn" href="{{ url_for('benchmark_new') }}">Add a Harbor benchmark</a></div></li>
|
|
37
|
+
</ul>
|
|
38
|
+
{% endcall %}
|
|
39
|
+
|
|
40
|
+
<div class="bench-filters">
|
|
41
|
+
<div class="bench-cats" role="group" aria-label="Category" data-filter="category">
|
|
42
|
+
<button type="button" class="bench-cat on" data-value="" aria-pressed="true">All</button>
|
|
43
|
+
{% for c in cat.categories %}<button type="button" class="bench-cat" data-value="{{ c }}" aria-pressed="false">{{ c }}</button>
|
|
44
|
+
{% endfor %}
|
|
45
|
+
</div>
|
|
46
|
+
<div class="bench-cats" role="group" aria-label="Runs with" data-filter="runs">
|
|
47
|
+
<button type="button" class="bench-cat on" data-value="" aria-pressed="true">Any runner</button>
|
|
48
|
+
{% for r in ['standard', 'harbor'] %}<button type="button" class="bench-cat" data-value="{{ r }}" aria-pressed="false" title="Runs with {{ run_names[r] }}">{{ run_labels[r] }}</button>
|
|
49
|
+
{% endfor %}
|
|
50
|
+
</div>
|
|
51
|
+
<div class="bench-cats" role="group" aria-label="Origin" data-filter="origin">
|
|
52
|
+
<button type="button" class="bench-cat on" data-value="" aria-pressed="true">Any origin</button>
|
|
53
|
+
{% for o in origins %}<button type="button" class="bench-cat" data-value="{{ o }}" aria-pressed="false">{{ o|capitalize }}</button>
|
|
54
|
+
{% endfor %}
|
|
55
|
+
</div>
|
|
56
|
+
</div>
|
|
57
|
+
|
|
58
|
+
{# Two shelves: the official benchmarks, then the community benchmarks. A card opens the benchmark's own page. #}
|
|
59
|
+
{% for state, heading in [('official', 'Official'), ('community', 'Community')] %}
|
|
60
|
+
{% set shelf = listed|selectattr('origin', 'equalto', state)|list %}
|
|
61
|
+
{% if shelf %}
|
|
62
|
+
<section class="bench-shelf st-{{ state }}" data-shelf="{{ state }}" aria-labelledby="shelf-{{ state }}">
|
|
63
|
+
<h2 id="shelf-{{ state }}"><span class="shelf-dot" aria-hidden="true"></span>{{ heading }} <span class="shelf-n">{{ shelf|length }}</span></h2>
|
|
64
|
+
<div class="bench-grid">
|
|
65
|
+
{% for b in shelf %}{% set ds = cat.datasets_of(b) %}{% set tasks = cat.tasks_of(b) %}
|
|
66
|
+
<a class="bench-card" href="{{ url_for('benchmark_page', key=b.key) }}" data-category="{{ b.category }}" data-origin="{{ b.origin }}" data-runs="{{ ((['standard'] if 'standard' in b.harness else []) + (['harbor'] if 'harbor' in b.harness else []))|join(' ') }}"
|
|
67
|
+
data-keys="{{ ([b.key] + b.datasets)|join(' ') }}"
|
|
68
|
+
data-find="{{ ([b.title, b.category, b.key] + ds|map(attribute='title')|list + b.datasets)|join(' ')|lower }}">
|
|
69
|
+
{% set runs = (['harbor'] if 'harbor' in b.harness else []) + (['standard'] if 'standard' in b.harness else []) %}
|
|
70
|
+
<div class="bench-card-tags">{% if b.category %}<span class="cat-tag">{{ b.category }}</span>{% endif %}<span class="origin-tag {{ b.origin }}">{{ b.origin|capitalize }}</span></div>
|
|
71
|
+
<div class="bench-card-title"><svg class="bench-mark" aria-hidden="true"><use href="{{ url_for('static', filename='catalogue/marks.svg') }}#bm-{{ b.key }}"/></svg>{{ b.title }}</div>
|
|
72
|
+
{% if b.domain %}<div class="bench-card-domain">{{ b.domain }}</div>{% endif %}
|
|
73
|
+
<div class="bench-card-runs run-tags" aria-label="Runs with">{% for r in runs %}<span class="run-tag rt-{{ r }}" title="Runs with {{ run_names[r] }}">{{ run_labels[r] }}</span>{% endfor %}
|
|
74
|
+
</div>
|
|
75
|
+
{% if setup.get(b.key) %}<div class="bench-card-setup" title="Some of its options need {{ setup[b.key]|join(', ') }} installed on the machine it runs on: see its page"><svg viewBox="0 0 16 16" aria-hidden="true" fill="none" stroke="currentColor" stroke-width="1.5" stroke-linecap="round"><path d="M10.5 2.5a3 3 0 0 0-3.9 3.9L2.5 10.5v3h3l4.1-4.1a3 3 0 0 0 3.9-3.9l-2 2-2-.5-.5-2z"/></svg>needs {{ setup[b.key]|join(', ') }}</div>{% endif %}
|
|
76
|
+
<div class="bench-stats">
|
|
77
|
+
<div><b>{{ tasks[0] }}</b><span>{{ tasks[1] }}</span></div>
|
|
78
|
+
<div><b>{{ ds|length }}</b><span>{{ 'datasets' if ds|length != 1 else 'dataset' }}</span></div>
|
|
79
|
+
{% if b.scoring %}<div><b class="word">{{ b.scoring }}</b><span>scored by</span></div>{% endif %}
|
|
80
|
+
</div>
|
|
81
|
+
{% if b.blocker %}<div class="bench-card-blocker">{{ b.blocker[:1]|upper ~ b.blocker[1:] }}</div>{% endif %}
|
|
82
|
+
</a>
|
|
83
|
+
{% endfor %}
|
|
84
|
+
</div>
|
|
85
|
+
</section>
|
|
86
|
+
{% endif %}
|
|
87
|
+
{% endfor %}
|
|
88
|
+
<p class="bench-none" id="bench-none" hidden>No benchmark matches.</p>
|
|
89
|
+
</div>
|
|
90
|
+
{% endblock %}
|
|
91
|
+
{% block scripts %}<script src="{{ url_for('static', filename='catalogue/catalogue.js') }}"></script><script src="{{ url_for('static', filename='catalogue/add.js') }}"></script>{% endblock %}
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
{% extends "base.html" %}
|
|
2
|
+
{#- one form, two uses: `editing` is the key of the entry being changed, or none to add one #}
|
|
3
|
+
{% block title %}{{ 'Edit ' ~ values.get('title', editing) if editing else 'Add a Harbor benchmark' }} — Nullius{% endblock %}
|
|
4
|
+
{% block crumb %}/ <a href="{{ url_for('benchmarks_page') }}">benchmarks</a> / {% if editing %}<a href="{{ url_for('benchmark_page', key=editing) }}">{{ editing }}</a> / edit{% else %}add{% endif %}{% endblock %}
|
|
5
|
+
{% block head %}<link rel="stylesheet" href="{{ url_for('static', filename='catalogue/catalogue.css') }}">{% endblock %}
|
|
6
|
+
{% block body %}
|
|
7
|
+
{% set v = values %}
|
|
8
|
+
{% macro radio(name, value, title, sub='', disabled=false) %}
|
|
9
|
+
<label class="nb-opt{{ ' off' if disabled }}"><input type="radio" name="{{ name }}" value="{{ value }}"{{ ' checked' if v.get(name) == value }}{{ ' disabled' if disabled }}>
|
|
10
|
+
<span><b>{{ title }}</b>{% if sub %}<small>{{ sub }}</small>{% endif %}</span></label>
|
|
11
|
+
{% endmacro %}
|
|
12
|
+
<div class="bench-index bench-new">
|
|
13
|
+
<header class="bench-index-head">
|
|
14
|
+
<div>
|
|
15
|
+
{% if editing %}
|
|
16
|
+
<h1>Edit {{ values.get('title') or editing }}</h1>
|
|
17
|
+
<p class="bench-count">Saving writes <span class="mono">{{ catalogue_dir }}/benchmarks/{{ editing }}/benchmark.json</span>; the entry's other files and fields are kept.</p>
|
|
18
|
+
{% else %}
|
|
19
|
+
<h1>Add a Harbor benchmark</h1>
|
|
20
|
+
<p class="bench-count">Name the dataset, read it from Harbor, check the entry. Saving writes <span class="mono">{{ catalogue_dir }}/benchmarks/<key>/benchmark.json</span>.</p>
|
|
21
|
+
{% endif %}
|
|
22
|
+
</div>
|
|
23
|
+
</header>
|
|
24
|
+
|
|
25
|
+
{% if not writable %}<p class="nb-errors">Benchmarks are added from a browser on the application host only.</p>{% endif %}
|
|
26
|
+
{% if errors %}
|
|
27
|
+
<div class="nb-errors" role="alert"><b>Nothing was saved.</b>
|
|
28
|
+
<ul>{% for e in errors %}<li>{{ e }}</li>{% endfor %}</ul>
|
|
29
|
+
</div>
|
|
30
|
+
{% endif %}
|
|
31
|
+
|
|
32
|
+
<form method="post" action="{{ url_for('benchmark_edit_save', key=editing) if editing else url_for('benchmark_new_save') }}" enctype="multipart/form-data" id="nb-form" data-read="{{ url_for('benchmark_new_read') }}" data-keys="{{ cat.benchmarks|map(attribute='key')|list|tojson|forceescape }}">
|
|
33
|
+
<input type="hidden" name="csrf" value="{{ csrf }}">
|
|
34
|
+
|
|
35
|
+
<section class="nb-step">
|
|
36
|
+
<h2><span>1</span>Where Harbor gets it</h2>
|
|
37
|
+
<div class="nb-choices" role="radiogroup" aria-label="Source">
|
|
38
|
+
{{ radio('source', 'hub', 'Harbor hub', 'org/name, as on hub.harborframework.com') }}
|
|
39
|
+
{{ radio('source', 'registry', 'Harbor registry', 'name@version, as Harbor\'s registry lists it') }}
|
|
40
|
+
</div>
|
|
41
|
+
<div class="nb-read">
|
|
42
|
+
<input type="text" name="ref" id="nb-ref" value="{{ v.get('ref', '') }}" placeholder="org/name or name@version" spellcheck="false" autocomplete="off" required>
|
|
43
|
+
<button type="button" class="btn" id="nb-read">Read from Harbor</button>
|
|
44
|
+
</div>
|
|
45
|
+
<p class="nb-said" id="nb-said" aria-live="polite"></p>
|
|
46
|
+
</section>
|
|
47
|
+
|
|
48
|
+
<section class="nb-step">
|
|
49
|
+
<h2><span>2</span>Catalogue entry</h2>
|
|
50
|
+
{% if not editing and routable %}
|
|
51
|
+
{#- a benchmark already run on the Nullius standard gets this dataset as its Harbor route: one benchmark, one card #}
|
|
52
|
+
<label class="nb-route">Which benchmark is this?
|
|
53
|
+
<select name="route_for" id="nb-route-for">
|
|
54
|
+
<option value="">A new benchmark</option>
|
|
55
|
+
{% for key, title in routable %}<option value="{{ key }}"{{ ' selected' if v.get('route_for') == key }}>{{ title }}: run it with Harbor too</option>{% endfor %}
|
|
56
|
+
</select>
|
|
57
|
+
<span class="nb-msg" id="nb-route-msg" hidden>The dataset below is added to that benchmark beside its Nullius route; what the benchmark says stays as it is.</span></label>
|
|
58
|
+
{% endif %}
|
|
59
|
+
<div class="nb-grid nb-entry">
|
|
60
|
+
{#- labels carry no hints, so every box starts on the same line: a hint is the box's placeholder, and a
|
|
61
|
+
field says something below itself only when there is something to say (nb-msg) #}
|
|
62
|
+
<label>Key
|
|
63
|
+
<input type="text" name="key" id="nb-key" value="{{ v.get('key', '') }}" data-was="{{ editing or '' }}" pattern="[a-z0-9][a-z0-9-]*" required spellcheck="false" placeholder="lowercase-with-dashes">
|
|
64
|
+
<span class="nb-msg" id="nb-key-msg" hidden></span></label>
|
|
65
|
+
<label>Title
|
|
66
|
+
<input type="text" name="title" value="{{ v.get('title', '') }}" required></label>
|
|
67
|
+
<label>Category
|
|
68
|
+
<select name="category">
|
|
69
|
+
{% for c in cat.categories %}<option{{ ' selected' if v.get('category') == c }}>{{ c }}</option>{% endfor %}
|
|
70
|
+
</select></label>
|
|
71
|
+
<label>Or a new category
|
|
72
|
+
<input type="text" name="category_new" value="{{ v.get('category_new', '') }}" placeholder="leave empty to keep the one chosen"></label>
|
|
73
|
+
</div>
|
|
74
|
+
|
|
75
|
+
<fieldset class="nb-set nb-entry"><legend>Harness</legend>
|
|
76
|
+
<div class="nb-choices">
|
|
77
|
+
<label class="nb-opt"><input type="radio" name="harness" value="harbor" checked><span><b>Harbor</b><small>its tasks run as Harbor trials</small></span></label>
|
|
78
|
+
<label class="nb-opt off"><input type="radio" disabled><span><b>Nullius</b><small>its own scripts as its Original setup: the /import-standard skill</small></span></label>
|
|
79
|
+
</div>
|
|
80
|
+
</fieldset>
|
|
81
|
+
|
|
82
|
+
<fieldset class="nb-set nb-entry"><legend>Origin</legend>
|
|
83
|
+
<div class="nb-choices">
|
|
84
|
+
{% for o in origins %}{{ radio('origin', o, o|capitalize, 'the publisher\'s benchmark' if o == 'official' else 'adapted by the community') }}{% endfor %}
|
|
85
|
+
</div>
|
|
86
|
+
</fieldset>
|
|
87
|
+
|
|
88
|
+
<div class="nb-grid">
|
|
89
|
+
<label class="wide nb-entry">Domain
|
|
90
|
+
<input type="text" name="domain" value="{{ v.get('domain', '') }}" required placeholder="One line on the tasks"></label>
|
|
91
|
+
<label class="nb-entry">Scoring
|
|
92
|
+
<input type="text" name="scoring" value="{{ v.get('scoring', '') }}" required placeholder="e.g. Tests"></label>
|
|
93
|
+
<label>Tasks
|
|
94
|
+
<input type="number" name="tasks" min="1" value="{{ v.get('tasks', '') }}" required></label>
|
|
95
|
+
<label class="wide">Verifier
|
|
96
|
+
<input type="text" name="verifier" value="{{ v.get('verifier', '') }}" required
|
|
97
|
+
placeholder="e.g. The failing tests must pass and the passing ones keep passing."></label>
|
|
98
|
+
<label class="wide">Official page
|
|
99
|
+
<input type="url" name="official" value="{{ v.get('official', '') }}" placeholder="https:// the benchmark's own repository or site"></label>
|
|
100
|
+
</div>
|
|
101
|
+
|
|
102
|
+
<fieldset class="nb-set"><legend>Timeouts, seconds <small id="nb-from">as the dataset's tasks declare them: read from Harbor</small></legend>
|
|
103
|
+
<div class="nb-grid four">
|
|
104
|
+
{% for key, label in [('env_setup', 'Environment setup'), ('agent_setup', 'Agent setup'), ('agent_run', 'Agent'), ('verify', 'Verifier')] %}
|
|
105
|
+
<label>{{ label }}<input type="number" min="1" name="timeout_{{ key }}" value="{{ v.get('timeout_' ~ key, '') }}"></label>
|
|
106
|
+
{% endfor %}
|
|
107
|
+
</div>
|
|
108
|
+
</fieldset>
|
|
109
|
+
|
|
110
|
+
<div class="nb-grid nb-entry">
|
|
111
|
+
<label>Upstream limitations
|
|
112
|
+
<textarea name="upstream_limitations" rows="3" placeholder="One per line: what the benchmark as published gets wrong or leaves out">{{ v.get('upstream_limitations', '') }}</textarea></label>
|
|
113
|
+
<label>Nullius limitations
|
|
114
|
+
<textarea name="limitations" rows="3" placeholder="One per line: where running it here differs, or what is not offered yet">{{ v.get('limitations', '') }}</textarea></label>
|
|
115
|
+
<label class="wide">Citation
|
|
116
|
+
<textarea name="citation" rows="4" spellcheck="false" class="mono" placeholder="@article{...} BibTeX">{{ v.get('citation', '') }}</textarea></label>
|
|
117
|
+
<label>Citation from <input type="url" name="citation_source" value="{{ v.get('citation_source', '') }}" placeholder="https://"></label>
|
|
118
|
+
<label>Logo
|
|
119
|
+
<input type="file" name="logo" id="nb-logo" accept=".png,.jpg,.jpeg,.svg" data-max="200000">
|
|
120
|
+
<span class="nb-msg" id="nb-logo-msg"{% if not (v.get('logo_kept') or editing) %} hidden{% endif %}>
|
|
121
|
+
{%- if v.get('logo_kept') %}<b>{{ v.get('logo_kept_name') }}</b> is kept; choose another file to replace it
|
|
122
|
+
{%- elif editing %}Its current logo stays unless you choose a file{% endif %}</span></label>
|
|
123
|
+
{% if v.get('logo_kept') %}<input type="hidden" name="logo_kept" value="{{ v.get('logo_kept') }}"><input type="hidden" name="logo_kept_name" value="{{ v.get('logo_kept_name') }}">{% endif %}
|
|
124
|
+
</div>
|
|
125
|
+
|
|
126
|
+
<details class="nb-more"><summary>Dataset key and title</summary>
|
|
127
|
+
<div class="nb-grid">
|
|
128
|
+
<label>Dataset key<input type="text" name="dataset_key" value="{{ v.get('dataset_key', '') }}" spellcheck="false" placeholder="the benchmark's key"></label>
|
|
129
|
+
<label>Dataset title<input type="text" name="dataset_title" value="{{ v.get('dataset_title', '') }}" placeholder="the benchmark's title"></label>
|
|
130
|
+
</div>
|
|
131
|
+
</details>
|
|
132
|
+
</section>
|
|
133
|
+
|
|
134
|
+
<div class="nb-save">
|
|
135
|
+
<a class="btn ghost" href="{{ url_for('benchmark_page', key=editing) if editing else url_for('benchmarks_page') }}">Cancel</a>
|
|
136
|
+
<button type="submit" class="btn"{{ ' disabled' if not writable }}>{{ 'Save the changes' if editing else 'Save the entry' }}</button>
|
|
137
|
+
</div>
|
|
138
|
+
</form>
|
|
139
|
+
</div>
|
|
140
|
+
<script src="{{ url_for('static', filename='catalogue/new.js') }}"></script>
|
|
141
|
+
{% endblock %}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
{% macro use_bundle(group, key, csrf) -%}
|
|
2
|
+
<details class="catalogue-restore">
|
|
3
|
+
<summary class="btn ghost">Use bundled entry</summary>
|
|
4
|
+
<form method="post" action="{{ url_for('catalogue_use_bundled', group=group, key=key) }}" class="bench-delete-ask">
|
|
5
|
+
<input type="hidden" name="csrf" value="{{ csrf }}">
|
|
6
|
+
<span>Remove your custom files and local proof records for <b>{{ key }}</b>? Saved runs and results stay.</span>
|
|
7
|
+
<button type="submit" class="btn">Use bundled entry</button>
|
|
8
|
+
<button type="button" class="btn ghost" onclick="const summary = this.closest('details').querySelector('summary'); summary.click(); summary.focus()">Keep my copy</button>
|
|
9
|
+
</form>
|
|
10
|
+
</details>
|
|
11
|
+
{%- endmacro %}
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{#- One task's detail, the same metrics for each side so differences line up: asked for when its row opens (/compare/detail/<index>), never built for every row of the page. -#}
|
|
2
|
+
<table class="grid inner" style="--cols: {{ jobs|length }}">
|
|
3
|
+
<colgroup>
|
|
4
|
+
<col class="compare-label-col">
|
|
5
|
+
{% for j in jobs %}<col>{% endfor %}
|
|
6
|
+
</colgroup>
|
|
7
|
+
<tbody>
|
|
8
|
+
{% for r in d.rows %}
|
|
9
|
+
<tr>
|
|
10
|
+
<th class="rowhead" {% if r.hint %}title="{{ r.hint }}"{% endif %}>{{ r.label }}</th>
|
|
11
|
+
{% for v in r.cells %}
|
|
12
|
+
<td class="{{ r.classes[loop.index0] }}{% if r.cells|unique|list|length > 1 %} cell-differs{% endif %}">{{ v }}</td>
|
|
13
|
+
{% endfor %}
|
|
14
|
+
</tr>
|
|
15
|
+
{% endfor %}
|
|
16
|
+
<tr>
|
|
17
|
+
<th class="rowhead">Files edited</th>
|
|
18
|
+
{% for files in d.files %}
|
|
19
|
+
<td>
|
|
20
|
+
{% if files %}<ul class="files tight">{% for f in files %}<li class="mono">{{ f }}</li>{% endfor %}</ul>
|
|
21
|
+
{% else %}<span class="faint">—</span>{% endif %}
|
|
22
|
+
</td>
|
|
23
|
+
{% endfor %}
|
|
24
|
+
</tr>
|
|
25
|
+
{% if d.broke|select|list %}
|
|
26
|
+
<tr>
|
|
27
|
+
<th class="rowhead">Broke</th>
|
|
28
|
+
{% for names in d.broke %}
|
|
29
|
+
<td>
|
|
30
|
+
{% if names %}<ul class="files tight">{% for f in names %}<li class="mono bad">{{ f }}</li>{% endfor %}</ul>
|
|
31
|
+
{% else %}<span class="faint">—</span>{% endif %}
|
|
32
|
+
</td>
|
|
33
|
+
{% endfor %}
|
|
34
|
+
</tr>
|
|
35
|
+
{% endif %}
|
|
36
|
+
</tbody>
|
|
37
|
+
</table>
|