nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,370 @@
|
|
|
1
|
+
"""The benchmarks pages: the list, a card per benchmark, and one page per
|
|
2
|
+
benchmark folder - its harnesses, its datasets, its limitations, its citation."""
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import base64
|
|
6
|
+
import html
|
|
7
|
+
import json
|
|
8
|
+
import os
|
|
9
|
+
import re
|
|
10
|
+
import time
|
|
11
|
+
|
|
12
|
+
from flask import abort, flash, jsonify, redirect, render_template, request, url_for
|
|
13
|
+
from nullius import execution, paths, providers, secrets, settings
|
|
14
|
+
from nullius.catalogue import catalog, citation, harbor_entry, sources, storage
|
|
15
|
+
from nullius.standard import definitions as standard_definitions
|
|
16
|
+
from nullius.standard import howto, prove
|
|
17
|
+
from nullius.standard import recipe as standard_recipe
|
|
18
|
+
|
|
19
|
+
from .core import app
|
|
20
|
+
from . import execute
|
|
21
|
+
from . import settings as settings_web
|
|
22
|
+
from . import shared
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
@app.route("/benchmarks")
|
|
26
|
+
def benchmarks_page():
|
|
27
|
+
# the benchmarks of the standard some of whose options need a machine's setup (``host``): a bubble on their card
|
|
28
|
+
setup = {b["key"]: sorted(b["host"]) for b in standard_definitions.load("benchmark")[0] if b.get("host")}
|
|
29
|
+
return render_template("catalogue/benchmarks.html", cat=shared._catalog_cached(), origins=catalog.ORIGINS,
|
|
30
|
+
prompts=howto.prompts(), catalogue_dir=paths.catalogue_dir(), setup=setup)
|
|
31
|
+
|
|
32
|
+
|
|
33
|
+
@app.route("/benchmarks/<key>")
|
|
34
|
+
def benchmark_page(key):
|
|
35
|
+
"""One listed benchmark; an entry the list does not show has no page either."""
|
|
36
|
+
cat = shared._catalog_cached()
|
|
37
|
+
b = next((x for x in cat.listed_benchmarks if x.key == key), None)
|
|
38
|
+
if b is None:
|
|
39
|
+
abort(404)
|
|
40
|
+
entry = harbor_entry.entry_of(key)
|
|
41
|
+
# the standard's definition of it, when it has one: its proofs per dataset, and the form that runs cli prove
|
|
42
|
+
std = standard_recipe.page_of(key) if "standard" in b.harness else None
|
|
43
|
+
# where each dataset's data comes from, one row per way it runs - each fetched by its harness, never by hand
|
|
44
|
+
std_def = standard_definitions.find(standard_definitions.load("benchmark")[0], key) if std else None
|
|
45
|
+
ds = cat.datasets_of(b)
|
|
46
|
+
data = {d.key: sources.of(d, b.harness, std_def, only=len(ds) == 1) for d in ds}
|
|
47
|
+
jobs = {j.job_id for j in shared._jobs_for_request()} if std else set()
|
|
48
|
+
return render_template("catalogue/benchmark.html", cat=cat, b=b, csrf=shared._csrf_token(), papers=citation.parse(b.citation), data=data,
|
|
49
|
+
editable=entry is not None and not harbor_entry.not_editable(entry), std=std, std_jobs=jobs,
|
|
50
|
+
deletable=os.path.isfile(os.path.join(paths.catalogue_dir(), catalog.BENCHMARKS, key, catalog.ENTRY)),
|
|
51
|
+
restore_bundle=storage.can_restore("benchmarks", key, catalog.ROOT),
|
|
52
|
+
local=settings_web._local_client(),
|
|
53
|
+
providers=[(p, providers.label(p)) for p in providers.available(os.environ)])
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
@app.route("/catalogue/<group>/<key>/use-bundled", methods=["POST"])
|
|
57
|
+
def catalogue_use_bundled(group, key):
|
|
58
|
+
shared._require_csrf()
|
|
59
|
+
if not settings_web._local_client():
|
|
60
|
+
abort(403, description="Catalogue entries are changed from a browser on the application host only.")
|
|
61
|
+
endpoints = {"benchmarks": "benchmark_page", "agents": "agent_page", "capabilities": "feature_page"}
|
|
62
|
+
if group not in endpoints:
|
|
63
|
+
abort(404)
|
|
64
|
+
bundled = catalog.ROOT if group == "benchmarks" else standard_definitions.CATALOGUE
|
|
65
|
+
try:
|
|
66
|
+
storage.use_bundled(group, key, bundled)
|
|
67
|
+
except (ValueError, OSError) as error:
|
|
68
|
+
abort(409, description=str(error))
|
|
69
|
+
flash("Using the bundled entry. Saved runs and results are unchanged.", "ok")
|
|
70
|
+
return redirect(url_for(endpoints[group], key=key))
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
@app.route("/benchmarks/<key>/prove", methods=["POST"])
|
|
74
|
+
def benchmark_prove(key):
|
|
75
|
+
"""Prove a benchmark of the standard, here, as a run - its units, the leak
|
|
76
|
+
check and its controls (``cli prove``), as an agent's page proves its
|
|
77
|
+
ports; then the first control's job page, which follows the run. The
|
|
78
|
+
controls call no model, so an unjudged benchmark's run is handed no key
|
|
79
|
+
at all (``execution.Plan.keyless``)."""
|
|
80
|
+
shared._require_csrf()
|
|
81
|
+
if not settings_web._local_client():
|
|
82
|
+
abort(403, description="Proofs are started from a browser on the application host only.")
|
|
83
|
+
back = url_for("benchmark_page", key=key) + "#standard"
|
|
84
|
+
limit = request.form.get("limit", "").strip()
|
|
85
|
+
provider, endpoint = providers.split(request.form.get("provider", "openai"))
|
|
86
|
+
stamp = time.strftime(prove.STAMP)
|
|
87
|
+
got = standard_recipe.proving(key, dataset=request.form.get("dataset", ""), limit=int(limit) if limit.isdigit() else (limit or None),
|
|
88
|
+
provider=provider, judge_model=request.form.get("judge_model", "").strip(), stamp=stamp)
|
|
89
|
+
problem = " ".join(got["errors"]) or ("" if got["keyless"] else execute.provider_problem(provider, endpoint))
|
|
90
|
+
if problem:
|
|
91
|
+
flash(problem, "error")
|
|
92
|
+
return redirect(back)
|
|
93
|
+
plan = execution.Plan(name=f"prove-{key}-{time.strftime('%m%d-%H%M%S')}", script=got["script"], runner=standard_recipe.HARNESS,
|
|
94
|
+
**execute.credentials_for(provider, endpoint), keyless=got["keyless"], jobs=got["jobs"],
|
|
95
|
+
parts=[{"id": key, "label": got["label"] + "'s proofs", "script": got["script"]}])
|
|
96
|
+
problem = plan.check() or (execution.name_taken(plan.name) if os.path.exists(os.path.join(execution.runs_root(), plan.name)) else None)
|
|
97
|
+
if problem:
|
|
98
|
+
flash(problem, "error")
|
|
99
|
+
return redirect(back)
|
|
100
|
+
ex = execution.start(plan)
|
|
101
|
+
shared.adopt_roots([ex.jobs])
|
|
102
|
+
flash(f"Proving {got['label']}: its units and the leak check, then " + (f"{len(got['jobs'])} control runs." if got["jobs"] else "no control: it declares none."), "ok")
|
|
103
|
+
return redirect(url_for("job", job_id=got["jobs"][0]["name"]) if got["jobs"] else back)
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
@app.route("/benchmarks/<key>/licences", methods=["POST"])
|
|
107
|
+
def benchmark_licence(key):
|
|
108
|
+
"""Set, replace, remove or move one of a benchmark's credentials, kept in its
|
|
109
|
+
own store (``nullius.secrets``), managed in Settings: a key or a server's
|
|
110
|
+
address from the text field, a licence file from the upload. Only a name
|
|
111
|
+
the benchmark's environments declare, as the kind they declare it, never
|
|
112
|
+
a data provider's token (Settings → Data providers). A
|
|
113
|
+
browser on this machine only, with the form token, as Settings. *Move
|
|
114
|
+
from saved settings* takes a value saved settings still hold into the
|
|
115
|
+
store and out of the file - on this click, never by itself. Nothing is
|
|
116
|
+
ever sent back of a value."""
|
|
117
|
+
shared._require_csrf()
|
|
118
|
+
if not settings_web._local_client():
|
|
119
|
+
abort(403, description="Credentials are set from a browser on the application host only.")
|
|
120
|
+
back = settings_web._settings_return() + "#access-" + key
|
|
121
|
+
name, action = request.form.get("name", ""), request.form.get("action", "set")
|
|
122
|
+
declared = {s["name"]: s for e in standard_recipe.licences_of(key) for s in e["needs"]}
|
|
123
|
+
s = declared.get(name)
|
|
124
|
+
if s is None or s.get("provider"):
|
|
125
|
+
flash(f"{name} is set in Settings → Data providers." if s else f"{name or 'That'} is not a credential this benchmark needs.", "error")
|
|
126
|
+
return redirect(back)
|
|
127
|
+
try:
|
|
128
|
+
if action == "delete":
|
|
129
|
+
gone = secrets.delete(key, name)
|
|
130
|
+
flash(f"{name} removed." if gone else f"{name} was not set.", "ok" if gone else "error")
|
|
131
|
+
elif action == "move":
|
|
132
|
+
value = settings.in_file(shared.ENV_FILE).get(name)
|
|
133
|
+
if not value:
|
|
134
|
+
flash(f"Saved settings hold no {name}.", "error")
|
|
135
|
+
return redirect(back)
|
|
136
|
+
secrets.set(key, name, value, kind=s["kind"])
|
|
137
|
+
settings.take(shared.ENV_FILE, name)
|
|
138
|
+
flash(f"{name} moved out of saved settings.", "ok")
|
|
139
|
+
else:
|
|
140
|
+
expires = request.form.get("expires", "").strip()
|
|
141
|
+
if s["kind"] == "file":
|
|
142
|
+
f = request.files.get("file")
|
|
143
|
+
data = f.read(secrets.MAX_BYTES + 1) if f else b""
|
|
144
|
+
if not data:
|
|
145
|
+
flash(f"{name}: choose the licence file.", "error")
|
|
146
|
+
return redirect(back)
|
|
147
|
+
secrets.set(key, name, data, kind="file", filename=f.filename or "", expires=expires)
|
|
148
|
+
else:
|
|
149
|
+
secrets.set(key, name, request.form.get("value", ""), kind=s["kind"], expires=expires)
|
|
150
|
+
flash(f"{name} set.", "ok")
|
|
151
|
+
except ValueError as e:
|
|
152
|
+
flash(str(e), "error")
|
|
153
|
+
return redirect(back)
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
# --- adding a Harbor benchmark by hand ----------------------------------------------
|
|
157
|
+
|
|
158
|
+
MARKS = os.path.join(shared._APP_DIR, "static", "catalogue", "marks.svg")
|
|
159
|
+
LOGO_TYPES = {".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".svg": "image/svg+xml"}
|
|
160
|
+
LOGO_MAX = 200_000
|
|
161
|
+
# the mark of a benchmark added without a logo: a box of tasks, in the current colour
|
|
162
|
+
PLAIN_MARK = ('<symbol id="bm-{key}" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="1.8" '
|
|
163
|
+
'stroke-linecap="round" stroke-linejoin="round"><rect x="3" y="4" width="18" height="16" rx="2"/>'
|
|
164
|
+
'<path d="M7 9h10M7 13h10M7 17h6"/></symbol>')
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _guard():
|
|
168
|
+
"""Every write here: form token, and a browser on this machine - adding a
|
|
169
|
+
benchmark writes into the user's catalogue."""
|
|
170
|
+
shared._require_csrf()
|
|
171
|
+
if not settings_web._local_client():
|
|
172
|
+
abort(403, description="Benchmarks are added from a browser on the application host only.")
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
def _set_mark(key: str, logo) -> None:
|
|
176
|
+
"""Keep an uploaded logo beside the user entry; no logo keeps its mark."""
|
|
177
|
+
if logo is not None:
|
|
178
|
+
harbor_entry._write(os.path.join(catalog.writable_dir(key), "logo.json"), logo)
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
@app.route("/static/catalogue/marks.svg")
|
|
182
|
+
def benchmark_marks():
|
|
183
|
+
"""The bundled sprite with user logos and a fallback for custom entries.
|
|
184
|
+
|
|
185
|
+
The exact route precedes Flask's generic static-file route, so the
|
|
186
|
+
builder and every catalogue page keep their existing sprite URL.
|
|
187
|
+
"""
|
|
188
|
+
with open(MARKS, encoding="utf-8") as fh:
|
|
189
|
+
text = fh.read()
|
|
190
|
+
cat = shared._catalog_cached()
|
|
191
|
+
for b in cat.benchmarks:
|
|
192
|
+
symbol = None
|
|
193
|
+
try:
|
|
194
|
+
with open(os.path.join(cat.entry_dir(b.key), "logo.json"), encoding="utf-8") as fh:
|
|
195
|
+
logo = json.load(fh)
|
|
196
|
+
if (isinstance(logo, dict) and logo.get("kind") in LOGO_TYPES.values()
|
|
197
|
+
and isinstance(logo.get("data"), str) and len(logo["data"]) * 3 // 4 <= LOGO_MAX
|
|
198
|
+
and re.fullmatch(r"[A-Za-z0-9+/=]+", logo["data"])):
|
|
199
|
+
symbol = (f'<symbol id="bm-{html.escape(b.key, quote=True)}" viewBox="0 0 24 24">'
|
|
200
|
+
f'<image width="24" height="24" href="data:{logo["kind"]};base64,{logo["data"]}"/></symbol>')
|
|
201
|
+
except (OSError, ValueError):
|
|
202
|
+
pass
|
|
203
|
+
have = re.search(rf'<symbol id="bm-{re.escape(b.key)}"[\s\S]*?</symbol>', text)
|
|
204
|
+
if have:
|
|
205
|
+
if symbol:
|
|
206
|
+
text = text[:have.start()] + symbol + text[have.end():]
|
|
207
|
+
else:
|
|
208
|
+
at = text.rindex("</svg>")
|
|
209
|
+
text = text[:at] + " " + (symbol or PLAIN_MARK.format(key=html.escape(b.key, quote=True))) + "\n" + text[at:]
|
|
210
|
+
return app.response_class(text, mimetype="image/svg+xml", headers={"Cache-Control": "no-cache"})
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def _logo(values: dict):
|
|
214
|
+
"""``({kind, data, name} or None, problem or None)``: the file chosen now,
|
|
215
|
+
else the one a save that came back kept in the form (``logo_kept``)."""
|
|
216
|
+
up = request.files.get("logo")
|
|
217
|
+
if up is not None and up.filename:
|
|
218
|
+
kind = LOGO_TYPES.get(os.path.splitext(up.filename)[1].lower())
|
|
219
|
+
if kind is None:
|
|
220
|
+
return None, "logo: a PNG, JPEG or SVG file"
|
|
221
|
+
raw = up.read()
|
|
222
|
+
if len(raw) > LOGO_MAX:
|
|
223
|
+
return None, f"logo: {len(raw) // 1000} KB; keep it under {LOGO_MAX // 1000} KB (a 64 px image is plenty)"
|
|
224
|
+
return {"kind": kind, "data": base64.b64encode(raw).decode(), "name": os.path.basename(up.filename)}, None
|
|
225
|
+
kept = re.match(r"^data:(image/png|image/jpeg|image/svg\+xml);base64,([A-Za-z0-9+/=]+)$", values.get("logo_kept") or "")
|
|
226
|
+
if kept and len(kept.group(2)) * 3 // 4 <= LOGO_MAX:
|
|
227
|
+
return {"kind": kept.group(1), "data": kept.group(2), "name": values.get("logo_kept_name") or "the logo chosen"}, None
|
|
228
|
+
return None, None
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def _form(values, errors=(), status=200, editing=None):
|
|
232
|
+
"""The add form, or the same form editing ``editing`` (an entry's key)."""
|
|
233
|
+
cat = shared._catalog_cached()
|
|
234
|
+
return render_template("catalogue/new.html", cat=cat, values=values, errors=list(errors), editing=editing, routable=_routable(),
|
|
235
|
+
origins=catalog.ORIGINS, catalogue_dir=paths.catalogue_dir(),
|
|
236
|
+
writable=settings_web._local_client(), csrf=shared._csrf_token()), status
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _back(values, errors, logo, editing=None):
|
|
240
|
+
"""The form again, with what to fix and everything typed - the logo too,
|
|
241
|
+
which a browser never puts back in a file field."""
|
|
242
|
+
if logo is not None:
|
|
243
|
+
values = dict(values, logo_kept=f"data:{logo['kind']};base64,{logo['data']}", logo_kept_name=logo["name"])
|
|
244
|
+
return _form(values, errors, 400, editing)
|
|
245
|
+
|
|
246
|
+
|
|
247
|
+
def _opened(key: str):
|
|
248
|
+
listed = any(b.key == key for b in shared._catalog_cached().listed_benchmarks)
|
|
249
|
+
return redirect(url_for("benchmark_page", key=key) if listed else url_for("benchmarks_page"))
|
|
250
|
+
|
|
251
|
+
|
|
252
|
+
@app.route("/benchmarks/new", methods=["GET"])
|
|
253
|
+
def benchmark_new():
|
|
254
|
+
"""The form that adds a Harbor benchmark to the catalogue."""
|
|
255
|
+
return _form({"source": "hub", "origin": "official"})
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
@app.route("/benchmarks/new/read", methods=["POST"])
|
|
259
|
+
def benchmark_new_read():
|
|
260
|
+
"""What Harbor says of a dataset, to fill the form: its task count and
|
|
261
|
+
the limits its tasks declare. Asked on the button, never on a page load."""
|
|
262
|
+
_guard()
|
|
263
|
+
body = request.get_json(silent=True) or {}
|
|
264
|
+
try:
|
|
265
|
+
return jsonify(ok=True, **harbor_entry.read(str(body.get("source") or ""), str(body.get("ref") or "")))
|
|
266
|
+
except harbor_entry.ReadError as e:
|
|
267
|
+
return jsonify(ok=False, error=str(e)), 400
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
@app.route("/benchmarks/new", methods=["POST"])
|
|
271
|
+
def benchmark_new_save():
|
|
272
|
+
"""Write the entry, and its mark, then open its page. Nothing is written
|
|
273
|
+
while anything is wrong: the form comes back with what to fix."""
|
|
274
|
+
_guard()
|
|
275
|
+
values = request.form.to_dict()
|
|
276
|
+
route_for = (values.get("route_for") or "").strip()
|
|
277
|
+
if route_for:
|
|
278
|
+
return _add_route(route_for, values)
|
|
279
|
+
logo, bad_logo = _logo(values)
|
|
280
|
+
entry = harbor_entry.build(values)
|
|
281
|
+
errors = harbor_entry.problems(entry)
|
|
282
|
+
if entry["key"] == "new":
|
|
283
|
+
errors.append("key: new is the address of this form")
|
|
284
|
+
if bad_logo:
|
|
285
|
+
errors.append(bad_logo)
|
|
286
|
+
if errors:
|
|
287
|
+
return _back(values, errors, logo)
|
|
288
|
+
harbor_entry.save(entry)
|
|
289
|
+
_set_mark(entry["key"], logo)
|
|
290
|
+
return _opened(entry["key"])
|
|
291
|
+
|
|
292
|
+
|
|
293
|
+
def _routable() -> list:
|
|
294
|
+
"""The benchmarks a Harbor route can be added to: listed, and run so far only on the Nullius standard."""
|
|
295
|
+
out = []
|
|
296
|
+
for b in shared._catalog_cached().benchmarks:
|
|
297
|
+
if "harbor" not in b.harness:
|
|
298
|
+
out.append((b.key, b.title))
|
|
299
|
+
return out
|
|
300
|
+
|
|
301
|
+
|
|
302
|
+
def _add_route(key: str, values: dict):
|
|
303
|
+
"""The form's dataset as a Harbor route of an existing benchmark: one
|
|
304
|
+
entry, one card, run with Harbor or on the Nullius standard."""
|
|
305
|
+
old = harbor_entry.entry_of(key)
|
|
306
|
+
if old is None:
|
|
307
|
+
abort(404)
|
|
308
|
+
if "harbor" in (old.get("harness") or []):
|
|
309
|
+
return _form(values, [f"{old.get('title', key)} already runs with Harbor: edit it from its page"], 400)
|
|
310
|
+
entry = harbor_entry.harbor_route(old, values)
|
|
311
|
+
errors = harbor_entry.problems(entry, replacing=key, dataset=-1)
|
|
312
|
+
if errors:
|
|
313
|
+
return _form(values, errors, 400)
|
|
314
|
+
harbor_entry.update(entry, was=key, dataset=-1)
|
|
315
|
+
flash(f"{old.get('title', key)} now runs with Harbor too.", "ok")
|
|
316
|
+
return _opened(key)
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
def _editable(key: str) -> dict:
|
|
320
|
+
entry = harbor_entry.entry_of(key)
|
|
321
|
+
if entry is None:
|
|
322
|
+
abort(404)
|
|
323
|
+
why = harbor_entry.not_editable(entry)
|
|
324
|
+
if why:
|
|
325
|
+
abort(409, description=f"This entry is not edited here: {why}.")
|
|
326
|
+
return entry
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
@app.route("/benchmarks/<key>/edit", methods=["GET"])
|
|
330
|
+
def benchmark_edit(key):
|
|
331
|
+
"""The add form, filled with a Harbor entry, to change it."""
|
|
332
|
+
return _form(harbor_entry.values_of(_editable(key)), editing=key)
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
@app.route("/benchmarks/<key>/edit", methods=["POST"])
|
|
336
|
+
def benchmark_edit_save(key):
|
|
337
|
+
"""Write the changes over the entry, keeping all it holds that the form
|
|
338
|
+
does not show; a logo chosen replaces its mark. Nothing is written while
|
|
339
|
+
anything is wrong."""
|
|
340
|
+
_guard()
|
|
341
|
+
old = _editable(key)
|
|
342
|
+
values = request.form.to_dict()
|
|
343
|
+
logo, bad_logo = _logo(values)
|
|
344
|
+
entry = dict(harbor_entry.merged(old, harbor_entry.build(values)), key=values.get("key", "").strip() or key)
|
|
345
|
+
errors = harbor_entry.problems(entry, replacing=key)
|
|
346
|
+
if entry["key"] == "new":
|
|
347
|
+
errors.append("key: new is the address of the add form")
|
|
348
|
+
if bad_logo:
|
|
349
|
+
errors.append(bad_logo)
|
|
350
|
+
if errors:
|
|
351
|
+
return _back(values, errors, logo, editing=key)
|
|
352
|
+
harbor_entry.update(entry, was=key)
|
|
353
|
+
_set_mark(entry["key"], logo)
|
|
354
|
+
return _opened(entry["key"])
|
|
355
|
+
|
|
356
|
+
|
|
357
|
+
@app.route("/benchmarks/<key>/delete", methods=["POST"])
|
|
358
|
+
def benchmark_delete(key):
|
|
359
|
+
"""Remove a Harbor entry: its folder and its mark. The page asks first,
|
|
360
|
+
in place; runs already made from it are not touched."""
|
|
361
|
+
_guard()
|
|
362
|
+
entry = _editable(key)
|
|
363
|
+
try:
|
|
364
|
+
harbor_entry.delete(key)
|
|
365
|
+
except ValueError as e:
|
|
366
|
+
abort(409, description=str(e))
|
|
367
|
+
restored = harbor_entry.entry_of(key) is not None
|
|
368
|
+
flash(f"Deleted {entry.get('title') or key} from your catalogue; "
|
|
369
|
+
+ ("the bundled entry is shown again. " if restored else "") + "Runs made from it are untouched.", "ok")
|
|
370
|
+
return redirect(url_for("benchmarks_page"))
|