nullius-eval 0.1.0rc1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +54 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +16 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +744 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +994 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +453 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +25 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +34 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +354 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +87 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +387 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +145 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/providers.js +78 -0
- _nullius_app/static/settings/sandbox.js +83 -0
- _nullius_app/static/settings/settings.css +229 -0
- _nullius_app/static/style.css +1751 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +477 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +301 -0
- _nullius_app/templates/catalogue/benchmarks.html +92 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +446 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/settings.html +508 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +392 -0
- _nullius_app/web/catalogue.py +374 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +205 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +224 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +461 -0
- _nullius_app/web/settings.py +210 -0
- _nullius_app/web/shared.py +468 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +202 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +120 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +115 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +316 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +124 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +37 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +208 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +114 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +42 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +64 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +323 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +551 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +74 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +105 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +605 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +111 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +495 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +70 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +189 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +160 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +170 -0
- nullius/catalogue/catalog.py +413 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +85 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +83 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1624 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +339 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +416 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +172 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +112 -0
- nullius/paths.py +84 -0
- nullius/prerequisites.py +111 -0
- nullius/progress.py +294 -0
- nullius/providers.py +115 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +189 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +139 -0
- nullius/runs/live.py +705 -0
- nullius/runs/parse.py +991 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +146 -0
- nullius/secrets.py +192 -0
- nullius/settings.py +456 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +301 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +650 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1381 -0
- nullius/standard/definitions.py +1577 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +255 -0
- nullius/standard/forward.py +92 -0
- nullius/standard/hostreq.py +65 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +397 -0
- nullius/standard/job.py +541 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +209 -0
- nullius/standard/names.py +30 -0
- nullius/standard/payload.py +255 -0
- nullius/standard/ports.py +64 -0
- nullius/standard/prove.py +357 -0
- nullius/standard/recipe.py +663 -0
- nullius/standard/relay.py +629 -0
- nullius/standard/run.py +1367 -0
- nullius/standard/shard.py +71 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +409 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0rc1.dist-info/METADATA +93 -0
- nullius_eval-0.1.0rc1.dist-info/RECORD +268 -0
- nullius_eval-0.1.0rc1.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0rc1.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0rc1.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,663 @@
|
|
|
1
|
+
"""The standard in the run builder: its choices, its recipe, its compile.
|
|
2
|
+
|
|
3
|
+
The builder offers the standard as a runner beside Harbor and the runner
|
|
4
|
+
definitions. A recipe for it carries *choices* - a benchmark, one of its
|
|
5
|
+
datasets, an agent or a capability and any of its options (``options``, name ->
|
|
6
|
+
value; the rest keep the vendor's defaults), a provider and a model, a limit,
|
|
7
|
+
how many units at a time, and for an agent the explicit yes to running it
|
|
8
|
+
unsandboxed - and :func:`compile` turns them into one command, here, on the server:
|
|
9
|
+
the preview route and ``/run`` compile again from the definitions, never from a
|
|
10
|
+
script the page sent.
|
|
11
|
+
|
|
12
|
+
Every pair is shown, and the ones that do not fit are refused with
|
|
13
|
+
``match.fit``'s reasons, the same sentences the page greys a choice out with.
|
|
14
|
+
|
|
15
|
+
The command is ``nullius.standard.cli run``, and
|
|
16
|
+
``--from-run`` makes it read the key and endpoint the run is handed
|
|
17
|
+
(``execution``'s launcher exports them as ``OPENAI_API_KEY`` and
|
|
18
|
+
``OPENAI_BASE_URL``) instead of saved settings. It runs from
|
|
19
|
+
the installed ``nullius standard`` command, or the source/payload tree
|
|
20
|
+
named by ``--app`` or ``$NULLIUS_STANDARD``. Relative job paths stay relative
|
|
21
|
+
to the directory the script was started in.
|
|
22
|
+
|
|
23
|
+
*Where* (``where``) is this machine (``local``), a host from Settings, or a
|
|
24
|
+
list of them: one machine is one run, several split it - one shard per
|
|
25
|
+
machine (``shard.py``), each its own run with its own job
|
|
26
|
+
(``<name>-shard-<I>of<N>``), all under one launch id, the recipe's name, so
|
|
27
|
+
the runs page groups them and ``cli merge`` puts them together. A shard's
|
|
28
|
+
``limit`` is its own. ``backend``: ``local`` processes, or ``docker`` - each
|
|
29
|
+
unit's agent or capability in a container - which a host must have docker for
|
|
30
|
+
(``docker`` in the result: the run checks it before anything is sent).
|
|
31
|
+
"""
|
|
32
|
+
from __future__ import annotations
|
|
33
|
+
|
|
34
|
+
import re
|
|
35
|
+
import shlex
|
|
36
|
+
from typing import Callable, Dict, List, Optional
|
|
37
|
+
|
|
38
|
+
from nullius import hosts, providers
|
|
39
|
+
from nullius.providers import PROVIDER_LABEL, PROVIDERS, STEP
|
|
40
|
+
from nullius.standard import hostreq, definitions, match, ports, prove
|
|
41
|
+
|
|
42
|
+
HARNESS = "standard"
|
|
43
|
+
# The builder's runner card (command-builder-core.js, runners()): not a definition file, so builtin like Harbor's.
|
|
44
|
+
RUNNER = {"key": HARNESS, "label": "Nullius", "builtin": True, "standard": True,
|
|
45
|
+
"summary": "Any agent or capability that proved the ports a benchmark needs: ACP, MCP and the relay, on this machine."}
|
|
46
|
+
MAX_CONCURRENT = 32
|
|
47
|
+
BACKENDS = ("local", "docker")
|
|
48
|
+
_NAME = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,79}$")
|
|
49
|
+
_MODEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/@+-]{0,127}$")
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _script(command: str) -> str:
|
|
53
|
+
"""Run an installed standard or an explicit source/payload, without
|
|
54
|
+
changing where relative job paths resolve or reading saved credentials."""
|
|
55
|
+
args = command.removeprefix("nullius standard ")
|
|
56
|
+
# A local snapshot can be launched by the interpreter that already has
|
|
57
|
+
# Nullius installed. Keep exactly one backend provider: ignore cwd and
|
|
58
|
+
# ambient Python paths (-I), omit site-packages (-S), then add this tree.
|
|
59
|
+
# -I ignores PYTHONDONTWRITEBYTECODE, so -B keeps the snapshot immutable.
|
|
60
|
+
# PYTHONPATH remains available to the standard's subprocess helpers.
|
|
61
|
+
bootstrap = ('import runpy, sys; sys.path.insert(0, sys.argv.pop(1)); '
|
|
62
|
+
'runpy.run_module("nullius.standard.cli", run_name="__main__", alter_sys=True)')
|
|
63
|
+
return ("set +x\n"
|
|
64
|
+
"set -euo pipefail\n"
|
|
65
|
+
'nullius_app=${NULLIUS_STANDARD:-}\n'
|
|
66
|
+
'nullius_die() { printf \'Nullius: %s\\n\' "$*" >&2; exit 1; }\n'
|
|
67
|
+
'while (($#)); do\n'
|
|
68
|
+
' case "$1" in\n'
|
|
69
|
+
' --app) (($# >= 2)) && [[ -n "$2" ]] || nullius_die "--app requires a path"; nullius_app=$2; shift 2 ;;\n'
|
|
70
|
+
' --help|-h) printf \'%s\\n\' "Usage: bash recipe-run.sh [--app PATH]" '
|
|
71
|
+
'"Uses installed nullius, or --app PATH for a source checkout." '
|
|
72
|
+
'"Relative job paths resolve from the current directory."; exit 0 ;;\n'
|
|
73
|
+
' *) nullius_die "Unknown option; use --help" ;;\n'
|
|
74
|
+
' esac\n'
|
|
75
|
+
'done\n'
|
|
76
|
+
'if [[ -z "$nullius_app" && -f app/nullius/standard/cli.py ]]; then nullius_app=$PWD; fi\n'
|
|
77
|
+
'if [[ -n "$nullius_app" ]]; then\n'
|
|
78
|
+
' nullius_app_asked=$nullius_app\n'
|
|
79
|
+
' nullius_app=$(cd -- "$nullius_app_asked" 2>/dev/null && pwd -P) '
|
|
80
|
+
'|| nullius_die "Cannot open the application directory: $nullius_app_asked"\n'
|
|
81
|
+
' [[ -f "$nullius_app/app/nullius/standard/cli.py" ]] '
|
|
82
|
+
'|| nullius_die "Required file is missing: $nullius_app/app/nullius/standard/cli.py"\n'
|
|
83
|
+
' export PYTHONPATH="$nullius_app/app"\n'
|
|
84
|
+
f' exec "${{NULLIUS_PYTHON:-python3}}" -I -S -B -c {shlex.quote(bootstrap)} "$nullius_app/app" {args}\n'
|
|
85
|
+
'fi\n'
|
|
86
|
+
'command -v nullius >/dev/null 2>&1 || nullius_die "Install nullius or pass --app PATH"\n'
|
|
87
|
+
f'exec nullius standard {args}\n')
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def _all() -> Dict[str, List[dict]]:
|
|
91
|
+
return {k: definitions.load(k, every=True)[0] for k in ("benchmark", "agent", "capability")}
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _bench_page(b: dict) -> dict:
|
|
95
|
+
"""A benchmark as a page shows it. ``controls``: the ones it declares,
|
|
96
|
+
each with who wrote it and where it comes from - what the builder offers
|
|
97
|
+
beside the agents or capabilities. Each dataset carries the same controls,
|
|
98
|
+
each with its ``proven`` record on that dataset (None: not proven), and
|
|
99
|
+
``proofs``, the units and leak checks the same way (``prove.status``)."""
|
|
100
|
+
kinds = sorted({p.split(".", 1)[1].split("/")[0] for p in b.get("needs") or []}) if b["family"] == "capability" else []
|
|
101
|
+
status = prove.status(b)
|
|
102
|
+
controls = [{k: c[k] for k in ("key", "label", "source", "from", "expect")}
|
|
103
|
+
for c in (definitions.control(b, w) for w in definitions.CONTROLS) if c]
|
|
104
|
+
versioned = definitions.version_of(b) is not None
|
|
105
|
+
return {"key": b["key"], "label": b["label"], "summary": b["summary"], "family": b["family"], "kind": kinds[0] if kinds else "",
|
|
106
|
+
"ref": definitions.ref_of(b) if "_version" in b else b["key"],
|
|
107
|
+
"version": b["_version"] if versioned else "",
|
|
108
|
+
# its versions, each by the ref a recipe names it with (the bare key: the default)
|
|
109
|
+
"versions": [{"ref": b["key"] if v == b.get("_default") else f"{b['key']}@{v}", "version": v, "default": v == b.get("_default")}
|
|
110
|
+
for v in b.get("_versions") or []] if versioned else [],
|
|
111
|
+
"environments": {n: {"summary": e.get("summary", ""), "local_ok": bool(e.get("local_ok")),
|
|
112
|
+
"needs_secrets": [{"name": s["name"], "to": s["to"], "kind": s.get("kind") or "value",
|
|
113
|
+
"summary": s.get("summary", ""), **{k: s[k] for k in ("path", "env") if k in s}}
|
|
114
|
+
for s in e.get("needs_secrets") or []]}
|
|
115
|
+
for n, e in (b.get("environments") or {}).items()},
|
|
116
|
+
"needs": list(b.get("needs") or []), "units": b.get("units") or {"noun": "unit", "nouns": "units"},
|
|
117
|
+
"score": (b.get("score") or {}).get("says", ""), "judged": bool((b.get("score") or {}).get("judge")),
|
|
118
|
+
# the upstream it runs: its repository, the release tag when it has one, the commit that is the authority
|
|
119
|
+
"upstream": {k: str((b.get("upstream") or {}).get(k) or "") for k in ("repository", "tag", "tested")},
|
|
120
|
+
"controls": controls,
|
|
121
|
+
# One Original procedure, with separate reference-participant and paper-model metadata.
|
|
122
|
+
"setups": [{"key": k, "label": definitions.setup_label(b, k), "summary": x["summary"],
|
|
123
|
+
"reference_agent": x.get("reference_agent", ""),
|
|
124
|
+
"comparable": k == "original" and b["setups"]["original"].get("comparable", True), "units": (b["setups"]["original"].get("units") or b.get("units") or {}),
|
|
125
|
+
**({"models": x["models"], "default": x["default"], "providers": list(x["providers"]),
|
|
126
|
+
"sent": {p: {m: pre + (x.get("served_as") or {}).get(m, m) for m in x["models"]} for p, pre in x["providers"].items()}}
|
|
127
|
+
if "models" in x else {"any": True, "providers": list(x["providers"])})}
|
|
128
|
+
for k, x in (b.get("setups") or {}).items() if k == "original"],
|
|
129
|
+
# its own options (a search, a mode): each value with its line and what it needs; and the host requirements
|
|
130
|
+
"simulator": (b.get("simulator") or {}).get("model", ""), "judge": ((b.get("score") or {}).get("judge") or {}).get("model", ""),
|
|
131
|
+
# what the upstream's leaderboard fixes besides the ranked model (Like the leaderboard)
|
|
132
|
+
"leaderboard": b.get("leaderboard") or None,
|
|
133
|
+
"attempts": (b.get("limits") or {}).get("attempts", ""),
|
|
134
|
+
"options": {n: {"label": o.get("label") or n.replace("_", " ").capitalize(), "summary": o["summary"], "datasets": o.get("datasets") or [],
|
|
135
|
+
"setups": o.get("setups") or ["yours"],
|
|
136
|
+
**({"values": o["values"]} if "values" in o else {"pattern": o["pattern"]}), "default": o.get("default"), "about": o.get("about") or {}, "needs": o.get("needs") or {}, "when": o.get("when") or {}}
|
|
137
|
+
for n, o in (b.get("options") or {}).items()},
|
|
138
|
+
"host": {n: {"summary": r["summary"], "check": r["check"], "install": hostreq.install_lines(b, [n]), "needed_by": r["needed_by"]}
|
|
139
|
+
for n, r in hostreq.all_of(b).items()},
|
|
140
|
+
"datasets": [{"id": d["id"], "label": d.get("label") or d["id"], "summary": d.get("summary", ""), "units": d.get("units"),
|
|
141
|
+
"environment": d.get("environment") or "",
|
|
142
|
+
"controls": [dict(c, proven=status[d["id"]][c["key"]]["proven"], failed=status[d["id"]][c["key"]]["failed"])
|
|
143
|
+
for c in controls],
|
|
144
|
+
"proofs": {k: status[d["id"]][k]["proven"] for k in ("units", "leak")},
|
|
145
|
+
"failed": {k: status[d["id"]][k]["failed"] for k in ("units", "leak") if status[d["id"]][k]["failed"]}}
|
|
146
|
+
for d in b["datasets"]]}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def protocol(o: dict) -> str:
|
|
150
|
+
"""How Nullius talks to it, in a word or two for a page."""
|
|
151
|
+
if o["_kind"] == "control":
|
|
152
|
+
return "no agent"
|
|
153
|
+
if o["_kind"] == "capability":
|
|
154
|
+
return "JSON lines"
|
|
155
|
+
return "ACP" if "acp" in (o.get("start") or {}) else "a command"
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def skills_of(port: str, record: Optional[dict]) -> List[dict]:
|
|
159
|
+
"""The skills of ``port``'s kit as a page shows them under a proof:
|
|
160
|
+
{id, says, shown, passed} each, ``passed`` the count when the proof
|
|
161
|
+
recorded one. Empty without a proof, for a kit without skills, and for a
|
|
162
|
+
proof on an older kit from before skills were recorded - it says nothing
|
|
163
|
+
of them."""
|
|
164
|
+
have = definitions.port_skills(port)
|
|
165
|
+
if not record or not have or not (isinstance(record.get("skills"), dict) or definitions.on_current_kit(port, record)):
|
|
166
|
+
return []
|
|
167
|
+
shown, counts = definitions.skills_shown(port, record), record.get("skills") or {}
|
|
168
|
+
return [{"id": s, "says": says, "shown": s in shown, "passed": (counts.get(s) or {}).get("passed", "")}
|
|
169
|
+
for s, says in have.items()]
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _other_page(o: dict) -> dict:
|
|
173
|
+
return {"key": o["_ref"], "base": o["key"], "label": definitions.title(o), "summary": o["summary"], "kind": o["_kind"],
|
|
174
|
+
"featureKind": o.get("kind", ""), "protocol": protocol(o), "version": str((o.get("upstream") or {}).get("version") or ""),
|
|
175
|
+
"default": o.get("_version") == o.get("_default"),
|
|
176
|
+
"offers": list(o.get("offers") or []),
|
|
177
|
+
# each proof with the kit version it passed and, on an older kit than the current one, a note saying so
|
|
178
|
+
"proven": {p: {**{k: r.get(k) for k in ("passed", "on", "model")}, "kit": definitions.proof_kit(r),
|
|
179
|
+
"older": definitions.older_kit(p, r), "label": ports.PORTS[p]["label"], "skills": skills_of(p, r)}
|
|
180
|
+
for p, r in (o.get("proven") or {}).items() if ports.known(p)},
|
|
181
|
+
"options": {n: {"summary": x.get("summary", ""), "default": x.get("default"),
|
|
182
|
+
**({"values": list(x["values"])} if "values" in x else {"pattern": x.get("pattern")})}
|
|
183
|
+
for n, x in (o.get("options") or {}).items()},
|
|
184
|
+
**({"fidelity": {k: o["proven"]["fidelity"].get(k) for k in ("verdict", "on", "model")}}
|
|
185
|
+
if isinstance((o.get("proven") or {}).get("fidelity"), dict) else {})}
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def choices() -> dict:
|
|
189
|
+
"""What the page offers - choices and the reasons a pair does not fit,
|
|
190
|
+
never a command."""
|
|
191
|
+
d = _all()
|
|
192
|
+
# one entry per benchmark, its default version, which lists the others (``versions``); the fits cover every version
|
|
193
|
+
defaults = [b for b in d["benchmark"] if b.get("_version") == b.get("_default")]
|
|
194
|
+
return {"benchmarks": [_bench_page(b) for b in defaults],
|
|
195
|
+
# an example (the reference agent, the example memory) is there to read or copy, never to choose
|
|
196
|
+
"agents": [_other_page(o) for o in d["agent"] if not o.get("example")],
|
|
197
|
+
"capabilities": [_other_page(o) for o in d["capability"] if not o.get("example")],
|
|
198
|
+
"fits": match.table(d["benchmark"], d["agent"], d["capability"]),
|
|
199
|
+
"ports": {p: s["summary"] for p, s in ports.PORTS.items()},
|
|
200
|
+
"providers": list(PROVIDERS)}
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def licences_of(key: str) -> List[dict]:
|
|
204
|
+
"""The licences a benchmark's environments need, every version's, as its
|
|
205
|
+
page lists them: [{environment, summary, needs: [{name, kind, to,
|
|
206
|
+
summary, path?, env?, status}]}], ``status`` the store's
|
|
207
|
+
(``secrets.status``: dates, never a value) or {"set": False}."""
|
|
208
|
+
from nullius import secrets
|
|
209
|
+
have = secrets.status(key)
|
|
210
|
+
out: Dict[str, dict] = {}
|
|
211
|
+
for b in _all()["benchmark"]:
|
|
212
|
+
if b["key"] != key:
|
|
213
|
+
continue
|
|
214
|
+
for n, e in (b.get("environments") or {}).items():
|
|
215
|
+
env = out.setdefault(n, {"environment": n, "summary": e.get("summary", ""), "needs": []})
|
|
216
|
+
for s in definitions.secrets_of({"environments": {n: e}}):
|
|
217
|
+
if all(x["name"] != s["name"] for x in env["needs"]):
|
|
218
|
+
env["needs"].append(dict(s, status=have.get(s["name"]) or {"set": False}))
|
|
219
|
+
return [e for e in out.values() if e["needs"]]
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
def page_of(benchmark: str) -> Optional[dict]:
|
|
223
|
+
"""One benchmark of the standard as its page shows it (``_bench_page``), by
|
|
224
|
+
key or key@version; None when the standard has no such benchmark."""
|
|
225
|
+
b = definitions.find(_all()["benchmark"], benchmark or "")
|
|
226
|
+
return _bench_page(dict(b, label=definitions.title(b))) if b else None
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
def chosen_of(recipe: dict) -> List[str]:
|
|
230
|
+
"""The agents, capabilities and controls a recipe runs, each once, in its order:
|
|
231
|
+
``others``, or the one ``other`` of a recipe from before several could be chosen."""
|
|
232
|
+
picked = recipe.get("others")
|
|
233
|
+
if not isinstance(picked, list):
|
|
234
|
+
picked = [recipe.get("other")] if recipe.get("other") else []
|
|
235
|
+
return list(dict.fromkeys(str(x) for x in picked if isinstance(x, str) and x))
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def options_for(recipe: dict, ref: str, several: bool):
|
|
239
|
+
"""One choice's options: ``options`` is {ref: {name: word}}; a recipe from before, or one with a
|
|
240
|
+
single choice, may give that choice's {name: word} directly."""
|
|
241
|
+
given = recipe.get("options") or {}
|
|
242
|
+
if isinstance(given, dict) and ref in given and isinstance(given[ref], dict):
|
|
243
|
+
return given[ref]
|
|
244
|
+
if not several and isinstance(given, dict) and all(isinstance(v, str) for v in given.values()):
|
|
245
|
+
return given
|
|
246
|
+
return {}
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _int(v, low: int, high: Optional[int] = None) -> Optional[int]:
|
|
250
|
+
if isinstance(v, bool) or not isinstance(v, int) or v < low or (high is not None and v > high):
|
|
251
|
+
return None
|
|
252
|
+
return v
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def compile(recipe: dict, defs: Optional[Dict[str, List[dict]]] = None, *,
|
|
256
|
+
docker_unusable: Optional[Callable[[], Optional[str]]] = None) -> dict:
|
|
257
|
+
"""{errors, notes, review, jobs, confirm, script, parts, model}
|
|
258
|
+
for one recipe; errors name the step
|
|
259
|
+
that edits what is wrong. And ``shards``: one per machine of *Where*, each
|
|
260
|
+
{where, index, of, name, script, parts, jobs} - what one run on that
|
|
261
|
+
machine is given (``web/execute.py`` starts one per shard) - and
|
|
262
|
+
``docker``, whether the runs need docker where they run.
|
|
263
|
+
|
|
264
|
+
``docker_unusable``: asked, when the docker backend is chosen for this
|
|
265
|
+
machine, why docker cannot run here (``container.unusable``) - a reason
|
|
266
|
+
is the Where step's error. Not given, it is not asked: compiling stays a
|
|
267
|
+
function of the definitions alone, and the routes that preview or start
|
|
268
|
+
a run pass it."""
|
|
269
|
+
d = defs or _all()
|
|
270
|
+
errors, notes = [], []
|
|
271
|
+
add = lambda step, message: errors.append({"step": STEP[step], "message": message})
|
|
272
|
+
out = {"errors": errors, "notes": notes, "review": [], "jobs": [], "confirm": "", "script": "", "parts": [], "model": ""}
|
|
273
|
+
name = str(recipe.get("name") or "")
|
|
274
|
+
if not _NAME.match(name):
|
|
275
|
+
add("limits", "The recipe's name may hold letters, digits, dots, dashes and underscores.")
|
|
276
|
+
# the benchmark by key (its default version) or key@version
|
|
277
|
+
b = definitions.find(d["benchmark"], str(recipe.get("benchmark") or "")) if recipe.get("benchmark") else None
|
|
278
|
+
if b is None:
|
|
279
|
+
add("benchmark", "Choose a benchmark of the standard.")
|
|
280
|
+
return out
|
|
281
|
+
b = dict(b, label=definitions.title(b))
|
|
282
|
+
# one dataset or several (``datasets``; a recipe from before names one, ``dataset``): each its own jobs
|
|
283
|
+
wanted = recipe.get("datasets") if isinstance(recipe.get("datasets"), list) else ([recipe["dataset"]] if recipe.get("dataset") else [])
|
|
284
|
+
wanted = list(dict.fromkeys(str(x) for x in wanted if isinstance(x, str)))
|
|
285
|
+
dss = [x for x in b["datasets"] if x["id"] in wanted]
|
|
286
|
+
dss.sort(key=lambda x: wanted.index(x["id"]))
|
|
287
|
+
unknown = [x for x in wanted if x not in {y["id"] for y in b["datasets"]}]
|
|
288
|
+
if not dss or unknown:
|
|
289
|
+
add("dataset", f"Choose one or more of {b['label']}'s datasets" + (f" ({', '.join(unknown)} is not one)" if unknown else "") + ".")
|
|
290
|
+
side = "agent" if b["family"] == "agent" else "capability"
|
|
291
|
+
# A control the benchmark declares (nop, oracle) is a choice beside the agents or capabilities: it runs
|
|
292
|
+
# nothing as you and calls no model - a judged benchmark's judge only - so it asks for neither.
|
|
293
|
+
# Several may be chosen: each is its own job of the one run, one after the other.
|
|
294
|
+
backend = recipe.get("backend") or "local"
|
|
295
|
+
if backend not in BACKENDS:
|
|
296
|
+
add("where", "The backend is local processes or docker.")
|
|
297
|
+
backend = "local"
|
|
298
|
+
what = "an agent" if side == "agent" else f"a {_bench_page(b)['kind'] or 'capability'}"
|
|
299
|
+
refs = chosen_of(recipe)
|
|
300
|
+
if not refs:
|
|
301
|
+
add("agent", f"Choose how {b['label']} runs: " + ("as published, " if b.get("setups") else "") + f"with {what}, or a check.")
|
|
302
|
+
several = len(refs) > 1
|
|
303
|
+
picks = [] # [{o, control, given, opts}]
|
|
304
|
+
provider_ = recipe.get("provider") if recipe.get("provider") in PROVIDERS else ""
|
|
305
|
+
for ref in refs:
|
|
306
|
+
control = ref in definitions.CONTROLS
|
|
307
|
+
given, opts = {}, {}
|
|
308
|
+
if ref == "model":
|
|
309
|
+
add("agent", "Choose Original, then choose the model on the Model step.")
|
|
310
|
+
continue
|
|
311
|
+
if ref == "original":
|
|
312
|
+
orig = (b.get("setups") or {}).get("original") or {}
|
|
313
|
+
picked = str(recipe.get("model") or "").strip()
|
|
314
|
+
if not picked:
|
|
315
|
+
add("model", "Choose a model.")
|
|
316
|
+
continue
|
|
317
|
+
method = "model" if recipe.get("model_choice", "any") != "paper" and "model" in (b.get("setups") or {}) else "original"
|
|
318
|
+
o, why_not = definitions.setup(b, method, picked, provider_)
|
|
319
|
+
if o is None:
|
|
320
|
+
add("model" if orig else "agent", why_not[0].upper() + why_not[1:] + ".")
|
|
321
|
+
continue
|
|
322
|
+
picks.append({"o": o, "control": False, "setup": True, "given": {}, "opts": {}, "model": picked or o["asked"]})
|
|
323
|
+
continue
|
|
324
|
+
if control:
|
|
325
|
+
o = definitions.control(b, ref)
|
|
326
|
+
if o is None:
|
|
327
|
+
add("agent", f"{b['label']} declares no {ref} control.")
|
|
328
|
+
continue
|
|
329
|
+
else:
|
|
330
|
+
o = definitions.find(d[side], ref)
|
|
331
|
+
if o is None:
|
|
332
|
+
add("agent", f"Choose {what} for {b['label']}: {ref} is not one.")
|
|
333
|
+
continue
|
|
334
|
+
o = dict(o, label=definitions.title(o))
|
|
335
|
+
why = match.fit(b, o)
|
|
336
|
+
if why:
|
|
337
|
+
add("agent", f"{o['label']} cannot run {b['label']}: " + "; ".join(why) + ".")
|
|
338
|
+
given = options_for(recipe, ref, several)
|
|
339
|
+
if not isinstance(given, dict) or not all(isinstance(v, str) for v in given.values()):
|
|
340
|
+
add("agent", "Options are names with a word each.")
|
|
341
|
+
given = {}
|
|
342
|
+
opts, bad = definitions.options_of(o, given)
|
|
343
|
+
for why_not in bad:
|
|
344
|
+
add("agent", f"{o['label']}: " + why_not)
|
|
345
|
+
picks.append({"o": o, "control": control, "given": given, "opts": opts})
|
|
346
|
+
# the benchmark's own options (``bench_options``), each on the datasets it applies to; what the chosen
|
|
347
|
+
# values need: an embedding model (its calls through the run's relay, on the OpenAI provider), the machine's programs
|
|
348
|
+
chosen_opts = recipe.get("bench_options") or {}
|
|
349
|
+
if not isinstance(chosen_opts, dict) or not all(isinstance(v, str) for v in chosen_opts.values()):
|
|
350
|
+
add("dataset", "The benchmark's options are names with a word each.")
|
|
351
|
+
chosen_opts = {}
|
|
352
|
+
lb = (b.get("leaderboard") or {}) if recipe.get("model_choice") == "leaderboard" else {}
|
|
353
|
+
for k, v in (lb.get("bench_options") or {}).items():
|
|
354
|
+
if chosen_opts.get(k, v) != v:
|
|
355
|
+
add("dataset", f"Like the leaderboard sets {(b.get('options') or {})[k].get('label') or k.replace('_', ' ')} to {v}: choose Any of your choice for {chosen_opts[k]}.")
|
|
356
|
+
chosen_opts = {**(lb.get("bench_options") or {}), **chosen_opts}
|
|
357
|
+
per_ds, needs = {}, {"embeddings": [], "host": []}
|
|
358
|
+
for x in dss:
|
|
359
|
+
vals, bad = definitions.bench_options_of(b, x["id"], chosen_opts)
|
|
360
|
+
for why_not in bad:
|
|
361
|
+
add("dataset", why_not[0].upper() + why_not[1:] + ".")
|
|
362
|
+
per_ds[x["id"]] = vals
|
|
363
|
+
n = definitions.option_needs(b, vals)
|
|
364
|
+
needs["embeddings"] += [m for m in n["embeddings"] if m not in needs["embeddings"]]
|
|
365
|
+
needs["host"] += [h for h in n["host"] if h not in needs["host"]]
|
|
366
|
+
if needs["embeddings"] and recipe.get("provider") not in (None, "openai"):
|
|
367
|
+
add("model", f"{', '.join(needs['embeddings'])} is OpenAI's embedding model: choose the OpenAI provider for this search.")
|
|
368
|
+
agents_local = [p["o"]["label"] for p in picks if side == "agent" and not p["control"] and not p.get("setup") and backend == "local"]
|
|
369
|
+
if backend != "local" and any(p.get("setup") for p in picks):
|
|
370
|
+
add("where", "A setup runs the benchmark's own code on the machine, as published: choose local processes.")
|
|
371
|
+
if agents_local and recipe.get("unsandboxed") is not True:
|
|
372
|
+
add("where", f"{', '.join(agents_local)} would run as you, not sandboxed: tick Run it as me, or choose Docker.")
|
|
373
|
+
only_controls = bool(picks) and all(p["control"] for p in picks)
|
|
374
|
+
needs_model = any(not p["control"] for p in picks)
|
|
375
|
+
model_choice = recipe.get("model_choice", "any")
|
|
376
|
+
if model_choice not in ("paper", "leaderboard", "any"):
|
|
377
|
+
add("model", "Choose Like the paper, Like the leaderboard or Any of your choice.")
|
|
378
|
+
if model_choice == "leaderboard" and not b.get("leaderboard"):
|
|
379
|
+
add("model", f"{b['label']} declares no leaderboard; choose Like the paper or Any of your choice.")
|
|
380
|
+
orig = (b.get("setups") or {}).get("original") or {}
|
|
381
|
+
official = {"judge": ((b.get("score") or {}).get("judge") or {}).get("model", ""), "simulator": (b.get("simulator") or {}).get("model", "")}
|
|
382
|
+
fixed = {**official, **{k: lb[k] for k in ("judge", "simulator") if lb.get(k)}}
|
|
383
|
+
if model_choice == "paper" and needs_model and not orig.get("models") and not any(official.values()):
|
|
384
|
+
add("model", "No paper models are listed for this benchmark; choose Any of your choice.")
|
|
385
|
+
elif model_choice == "paper" and needs_model and orig.get("models"):
|
|
386
|
+
if recipe.get("model") not in orig["models"]:
|
|
387
|
+
add("model", "Choose one of the paper's listed models.")
|
|
388
|
+
elif provider_ not in orig.get("providers", {}):
|
|
389
|
+
add("model", "The paper's models are not listed for this provider.")
|
|
390
|
+
provider = recipe.get("provider")
|
|
391
|
+
if provider not in PROVIDERS:
|
|
392
|
+
add("model", "Choose a provider: OpenRouter, OpenAI or a local or self-hosted OpenAI-compatible endpoint.")
|
|
393
|
+
model = str(recipe.get("model") or "").strip()
|
|
394
|
+
if not _MODEL.match(model) and (needs_model or model):
|
|
395
|
+
add("model", "Choose a model or enter its ID.")
|
|
396
|
+
if model_choice == "paper" and model in orig.get("models", []) and provider_ in orig.get("providers", {}):
|
|
397
|
+
model = orig["providers"][provider_] + (orig.get("served_as") or {}).get(model, model)
|
|
398
|
+
# each unit tried K times (the upstream's --num-trials: pass^k); the user simulator's and the judge's models
|
|
399
|
+
repeat = _int(recipe.get("repeat") or lb.get("repeat") or 1, 1, 32)
|
|
400
|
+
if repeat is None:
|
|
401
|
+
add("limits", "Attempts per unit: a whole number from 1 to 32.")
|
|
402
|
+
repeat = 1
|
|
403
|
+
if lb and repeat < lb.get("repeat", 1):
|
|
404
|
+
add("limits", f"Like the leaderboard runs each {(b.get('units') or {}).get('noun', 'unit')} at least {lb['repeat']} times.")
|
|
405
|
+
sim_model = str(recipe.get("simulator_model") or "").strip()
|
|
406
|
+
judge_model = str(recipe.get("judge_model") or "").strip()
|
|
407
|
+
like = {"paper": "Like the paper", "leaderboard": "Like the leaderboard"}.get(model_choice)
|
|
408
|
+
for what, role, m in (("the user simulator's", "simulator", sim_model), ("the judge's", "judge", judge_model)):
|
|
409
|
+
if m and not _MODEL.match(m):
|
|
410
|
+
add("model", f"Enter {what} model as the provider lists it, or leave it empty for its official one.")
|
|
411
|
+
elif m and like and m not in (fixed[role], providers.served_as(fixed[role], provider_)):
|
|
412
|
+
add("model", f"{like} uses {what} model {fixed[role]}: choose Any of your choice for {m}.")
|
|
413
|
+
# the leaderboard's own models, as the run's provider spells them; the official ones stay the run's default
|
|
414
|
+
sim_model = providers.served_as(lb["simulator"], provider_) if lb.get("simulator") else sim_model
|
|
415
|
+
judge_model = providers.served_as(lb["judge"], provider_) if lb.get("judge") else judge_model
|
|
416
|
+
limit = recipe.get("limit")
|
|
417
|
+
if limit not in ("", None) and _int(limit, 1) is None:
|
|
418
|
+
add("limits", "The number of units is a whole number of at least 1, or empty for all of them.")
|
|
419
|
+
conc = _int(recipe.get("concurrency") or 1, 1, MAX_CONCURRENT) # empty: one at a time
|
|
420
|
+
if conc is None:
|
|
421
|
+
add("limits", f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.")
|
|
422
|
+
targets, why_not = where_of(recipe.get("where", "local"))
|
|
423
|
+
if why_not:
|
|
424
|
+
add("where", why_not)
|
|
425
|
+
elif backend == "docker" and "local" in targets and docker_unusable is not None:
|
|
426
|
+
why = docker_unusable()
|
|
427
|
+
if why:
|
|
428
|
+
add("where", f"The docker backend cannot run on this machine: {why}. Choose local processes, or a host with docker.")
|
|
429
|
+
if errors:
|
|
430
|
+
return out
|
|
431
|
+
nouns = (b.get("units") or {}).get("nouns", "units")
|
|
432
|
+
if picks and all(p.get("setup") for p in picks): # only setups: a unit is theirs (a conversation)
|
|
433
|
+
nouns = picks[0]["o"]["units"]["nouns"]
|
|
434
|
+
each = [None if picks and all(p.get("setup") for p in picks) else x.get("units") for x in dss] # a setup's count is not the dataset's
|
|
435
|
+
total = sum(each) if all(each) else None
|
|
436
|
+
n = sum(min(limit, t) for t in each) if isinstance(limit, int) and total else (limit or total)
|
|
437
|
+
of = len(targets)
|
|
438
|
+
many = len(dss) > 1
|
|
439
|
+
# what a run is made of: each dataset with each choice, a job each, in that order
|
|
440
|
+
pairs = [(x, p) for x in dss for p in picks]
|
|
441
|
+
|
|
442
|
+
def word_of(d: dict, p: dict) -> str:
|
|
443
|
+
return "-".join(([d["id"]] if many else []) + ([definitions.job_word(p["o"])] if several else []))
|
|
444
|
+
|
|
445
|
+
def job_of(d: dict, p: dict, index: int) -> str:
|
|
446
|
+
base = "-".join(filter(None, [name, word_of(d, p)]))
|
|
447
|
+
return f"{base}-shard-{index}of{of}" if of > 1 else base
|
|
448
|
+
|
|
449
|
+
def options_for_pick(d: dict, p: dict) -> dict:
|
|
450
|
+
"""The benchmark's options for this dataset and this pick: a setup's own, or those of the port."""
|
|
451
|
+
return definitions.bench_options_of(b, d["id"], chosen_opts, p["o"]["key"] if p.get("setup") else "yours")[0]
|
|
452
|
+
|
|
453
|
+
def command_for(d: dict, p: dict, index: int) -> str:
|
|
454
|
+
o, control, given = p["o"], p["control"], p["given"]
|
|
455
|
+
launch = "-".join(filter(None, [name, word_of(d, p)]))
|
|
456
|
+
m = p["model"] if p.get("setup") else model
|
|
457
|
+
words = ["nullius", "standard", "run", definitions.ref_of(b), o["_ref"], *(["--model", m] if m else []), "--provider", provider,
|
|
458
|
+
"--dataset", d["id"], *(["--limit", str(limit)] if isinstance(limit, int) else []), "-j", str(conc),
|
|
459
|
+
*[w for k, v in options_for_pick(d, p).items() for w in ("--bench-option", f"{k}={v}")],
|
|
460
|
+
*(["--repeat", str(repeat)] if repeat > 1 else []),
|
|
461
|
+
*(["--simulator-model", sim_model] if sim_model and not p["control"] else []),
|
|
462
|
+
*(["--judge-model", judge_model] if judge_model else []),
|
|
463
|
+
"--name", job_of(d, p, index), *[w for k, v in given.items() for w in ("--option", f"{k}={v}")],
|
|
464
|
+
*(["--backend", "docker"] if backend == "docker" else []),
|
|
465
|
+
*(["--unsandboxed"] if side == "agent" and not control and not p.get("setup") and backend == "local" else []),
|
|
466
|
+
*(["--shard", f"{index}/{of}", "--launch", launch] if of > 1 else []), "--from-run"]
|
|
467
|
+
return " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
|
|
468
|
+
|
|
469
|
+
def script_for(d: dict, p: dict, command: str, index: int) -> str:
|
|
470
|
+
return ("#!/usr/bin/env bash\n"
|
|
471
|
+
f"# {b['label']} ({d.get('label') or d['id']}) with {p['o']['label']}, through the Nullius standard (docs/standard.md)"
|
|
472
|
+
+ (f": shard {index} of {of}.\n" if of > 1 else ".\n") +
|
|
473
|
+
"# Uses installed nullius or --app PATH ($NULLIUS_STANDARD for an execution payload). The key and\n"
|
|
474
|
+
"# endpoint come from the run's environment: OPENAI_API_KEY, and OPENAI_BASE_URL for an endpoint\n"
|
|
475
|
+
"# other than the provider's own.\n"
|
|
476
|
+
+ _script(command))
|
|
477
|
+
|
|
478
|
+
out["shards"] = []
|
|
479
|
+
commands = []
|
|
480
|
+
for i, where in enumerate(targets, 1):
|
|
481
|
+
parts, jobs = [], []
|
|
482
|
+
for d, p in pairs:
|
|
483
|
+
command = command_for(d, p, i)
|
|
484
|
+
commands.append(command)
|
|
485
|
+
parts.append({"id": word_of(d, p) or definitions.job_word(p["o"]),
|
|
486
|
+
"label": " · ".join(([d.get("label") or d["id"]] if many else []) + [p["o"]["label"]]) + (f" · shard {i}/{of}" if of > 1 else ""),
|
|
487
|
+
"script": script_for(d, p, command, i)})
|
|
488
|
+
jobs.append({"name": job_of(d, p, i), "agent": p["o"]["_ref"], "dataset": definitions.ref_of(b)})
|
|
489
|
+
out["shards"].append({"where": where, "index": i, "of": of, "name": f"{name}-shard-{i}of{of}" if of > 1 else name,
|
|
490
|
+
"command": "\n".join(commands[-len(pairs):]), "script": parts[0]["script"], "parts": parts, "jobs": jobs})
|
|
491
|
+
out["script"] = "\n".join(pt["script"] for pt in out["shards"][0]["parts"]) if len(pairs) > 1 else out["shards"][0]["script"]
|
|
492
|
+
out["parts"] = [pt for sh in out["shards"] for pt in sh["parts"]]
|
|
493
|
+
out["jobs"] = [j for sh in out["shards"] for j in sh["jobs"]]
|
|
494
|
+
# docker where it runs: the docker backend, or a control run in each unit's own image, whatever the backend
|
|
495
|
+
own = any(definitions.own_image(definitions.environment_of(b, None, x)[1]) for x in dss)
|
|
496
|
+
out["docker"] = backend == "docker" or (own and any(p["control"] for p in picks))
|
|
497
|
+
out["model"] = model
|
|
498
|
+
# Nothing in the run asks a model: no key is handed to it, and none need be set.
|
|
499
|
+
out["keyless"] = only_controls and not (b.get("score") or {}).get("judge") and not needs["embeddings"]
|
|
500
|
+
# what the machines must have for the chosen options (``host``): the Where step and its test show it, Run checks it
|
|
501
|
+
reqs = hostreq.requirements(b, needs["host"])
|
|
502
|
+
out["host"] = {"benchmark": b["key"], "needs": [{"name": n, "summary": r["summary"], "check": r["check"]} for n, r in reqs.items()],
|
|
503
|
+
"check": [p for r in reqs.values() for p in r["check"]], "install": hostreq.install_lines(b, list(reqs))}
|
|
504
|
+
# The licences its environments may need, by name: what a run on a host is sent of the benchmark's store.
|
|
505
|
+
out["licences"] = {"benchmark": b["key"], "names": sorted({s["name"] for s in definitions.secrets_of(b)})}
|
|
506
|
+
|
|
507
|
+
|
|
508
|
+
def row(p: dict) -> list:
|
|
509
|
+
o = p["o"]
|
|
510
|
+
if p["control"]:
|
|
511
|
+
return [STEP["agent"], "Check", f"{o['label']} ({o['key']}) · {o['source']} · {o['from']}"]
|
|
512
|
+
if p.get("setup"):
|
|
513
|
+
original = (b.get("setups") or {}).get("original") or {}
|
|
514
|
+
label = original.get("reference_agent") or ("No agent · Original" if side == "agent" else "Original " + (_bench_page(b)["kind"] or "capability"))
|
|
515
|
+
return [STEP["agent"], "Participant", label]
|
|
516
|
+
proven = ", ".join(f"{q} {r.get('passed', '')}".strip() for q, r in (o.get("proven") or {}).items() if q in b["needs"])
|
|
517
|
+
return [STEP["agent"], "Agent" if side == "agent" else "Capability", f"{o['label']} · {protocol(o)} · proven {proven}"]
|
|
518
|
+
out["review"] = [[STEP["benchmark"], "Benchmark", f"{b['label']} · needs {', '.join(b['needs'])}"],
|
|
519
|
+
[STEP["runner"], "Runner", "Nullius"],
|
|
520
|
+
[STEP["dataset"], "Datasets" if many else "Dataset",
|
|
521
|
+
", ".join(f"{x.get('label') or x['id']}" + (f" · {x['units']} {nouns}" if x.get("units") else "") for x in dss)],
|
|
522
|
+
*[[STEP["dataset"], (b.get("options") or {})[k].get("label") or k.replace("_", " ").capitalize(),
|
|
523
|
+
f"{v}" + ("" if k in chosen_opts else " (its default)") + f" · {', '.join(x['id'] for x in dss if per_ds[x['id']].get(k) == v)}"]
|
|
524
|
+
for k, v in {k: v for vals in per_ds.values() for k, v in vals.items()}.items()],
|
|
525
|
+
*[row(p) for p in picks],
|
|
526
|
+
*[[STEP["agent"], "Options" + (f" of {p['o']['label']}" if several else ""),
|
|
527
|
+
", ".join(f"{k} {v}" + ("" if k in p["given"] else " (its default)") for k, v in p["opts"].items())]
|
|
528
|
+
for p in picks if p["opts"]],
|
|
529
|
+
[STEP["model"], "Model", "none: a control calls no model" if out["keyless"]
|
|
530
|
+
else f"{PROVIDER_LABEL[provider]} · " + (model if needs_model else ", ".join(p["o"]["sent"] for p in picks if p.get("setup")) or "the judge only")
|
|
531
|
+
+ (f" · user simulator {sim_model or (b.get('simulator') or {}).get('model')}" + (" (leaderboard)" if lb.get("simulator") else "" if sim_model else " (official)") if b.get("simulator") else "")
|
|
532
|
+
+ (f" · judge {judge_model or b['score']['judge']['model']}" + ("" if judge_model else " (official)") if (b.get("score") or {}).get("judge") else "")],
|
|
533
|
+
[STEP["limits"], "Scope", f"{'first ' + str(n) if isinstance(limit, int) else 'all ' + str(total or '')} {nouns}"
|
|
534
|
+
+ (" of each shard" if of > 1 and isinstance(limit, int) else "") + f" · {conc} at a time".replace(" ", " ")
|
|
535
|
+
+ (" of each dataset" if many and isinstance(limit, int) else "")
|
|
536
|
+
+ (f" · {len(pairs)} jobs, one after the other" if len(pairs) > 1 else "")
|
|
537
|
+
+ (f" · {repeat} attempts each (pass^1 to pass^{repeat})" if repeat > 1 else "")
|
|
538
|
+
+ (" · a setup counts " + next(p["o"]["units"]["nouns"] for p in picks if p.get("setup"))
|
|
539
|
+
if any(p.get("setup") for p in picks) and not all(p.get("setup") for p in picks) else "")],
|
|
540
|
+
[STEP["where"], "Where", (f"{of} shards, one per machine: " if of > 1 else "")
|
|
541
|
+
+ ", ".join(where_label(t) for t in targets) + f" · {backend} backend"]]
|
|
542
|
+
if agents_local:
|
|
543
|
+
notes.append(f"{', '.join(agents_local)} {'run' if len(agents_local) > 1 else 'runs'} as you on {'each machine' if of > 1 else where_label(targets[0])}, with its permission requests granted: "
|
|
544
|
+
"the local backend is no sandbox.")
|
|
545
|
+
if of > 1:
|
|
546
|
+
notes.append(f"Each machine runs the units whose shard is its own, by their ids' hash: {of} jobs under one launch, "
|
|
547
|
+
f"{name}, put together by cli merge.")
|
|
548
|
+
docker_only = [n for n, e in (b.get("environments") or {}).items() if not e.get("local_ok")]
|
|
549
|
+
if docker_only and not only_controls and backend == "local":
|
|
550
|
+
notes.append(f"Units in {b['label']}'s environment{'s' if len(docker_only) > 1 else ''} {', '.join(docker_only)} "
|
|
551
|
+
"need the docker backend: on local processes they are recorded as not run.")
|
|
552
|
+
if any(p["control"] for p in picks) and any(definitions.own_image(e) for e in (b.get("environments") or {}).values()):
|
|
553
|
+
notes.append(f"Each {(b.get('units') or {}).get('noun', 'unit')} of {b['label']} is its own image: the control "
|
|
554
|
+
"runs in it, so docker is needed where it runs, whatever the backend.")
|
|
555
|
+
if not isinstance(limit, int):
|
|
556
|
+
out["confirm"] = f"Run all {total or ''} {nouns} of {b['label']}{' (' + ', '.join(x.get('label') or x['id'] for x in dss) + ')' if many else ''} with {', '.join(p['o']['label'] for p in picks)}?".replace(" ", " ")
|
|
557
|
+
return out
|
|
558
|
+
|
|
559
|
+
|
|
560
|
+
def where_of(where) -> tuple:
|
|
561
|
+
"""*Where* as a list of machines - ``local`` or a host's entry - and why
|
|
562
|
+
it cannot be, or "": a word or a list, each machine once. Whether a host
|
|
563
|
+
is one of Settings' is the route's to check."""
|
|
564
|
+
targets = where if isinstance(where, list) else [where if where not in (None, "") else "local"]
|
|
565
|
+
if not targets:
|
|
566
|
+
return ["local"], "Choose where it runs: this machine, a host, or several."
|
|
567
|
+
out = []
|
|
568
|
+
for t in targets:
|
|
569
|
+
t = str(t).strip() if isinstance(t, str) else ""
|
|
570
|
+
if t != "local":
|
|
571
|
+
try:
|
|
572
|
+
t = hosts.parse(t).entry
|
|
573
|
+
except ValueError as e:
|
|
574
|
+
return ["local"], f"Where: {e}."
|
|
575
|
+
if t in out:
|
|
576
|
+
return ["local"], f"Where names {where_label(t)} twice: each machine runs one shard."
|
|
577
|
+
out.append(t)
|
|
578
|
+
return out, ""
|
|
579
|
+
|
|
580
|
+
|
|
581
|
+
def where_label(target: str) -> str:
|
|
582
|
+
return "this machine" if target == "local" else hosts.parse(target).label
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def conformance(kind: str, key: str, *, model: str, provider: str, unsandboxed: bool, stamp: str, backend: str = "local") -> dict:
|
|
586
|
+
"""The check an Agents or Capabilities page starts: every kit the agent or
|
|
587
|
+
capability can take, as one command (``cli conform``), the jobs it will
|
|
588
|
+
write named ahead (``conform.job_name``). {errors, script, jobs, label}."""
|
|
589
|
+
from nullius.standard import conform
|
|
590
|
+
o = definitions.get(kind, key) if kind in ("agent", "capability") else None
|
|
591
|
+
errors = []
|
|
592
|
+
if o is None:
|
|
593
|
+
return {"errors": [f"no {kind} {key!r}"], "script": "", "jobs": [], "label": key}
|
|
594
|
+
o = dict(o, label=definitions.title(o))
|
|
595
|
+
if provider not in PROVIDERS:
|
|
596
|
+
errors.append("Choose a provider: OpenRouter, OpenAI or a local or self-hosted OpenAI-compatible endpoint.")
|
|
597
|
+
if not _MODEL.match(model or ""):
|
|
598
|
+
errors.append("Choose a model or enter its ID.")
|
|
599
|
+
if backend not in ("local", "docker"):
|
|
600
|
+
errors.append("Choose local processes or Docker for the checks.")
|
|
601
|
+
if kind == "agent" and backend == "local" and unsandboxed is not True:
|
|
602
|
+
errors.append(f"{o['label']} runs on this machine as you while its kits run: the local backend is not a sandbox. Say yes to that to check it.")
|
|
603
|
+
if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""):
|
|
604
|
+
errors.append("a time stamp: YYYY-MM-DD__HH-MM-SS")
|
|
605
|
+
kits = [definitions.kit(p) for p in conform.candidates(o)]
|
|
606
|
+
kits = [k for k in kits if k]
|
|
607
|
+
if not kits:
|
|
608
|
+
errors.append(f"{o['label']} has no port with a kit to check.")
|
|
609
|
+
if errors:
|
|
610
|
+
return {"errors": errors, "script": "", "jobs": [], "label": o["label"]}
|
|
611
|
+
words = ["nullius", "standard", "conform", o["_ref"], "--model", model, "--provider", provider,
|
|
612
|
+
"--stamp", stamp, *(["--backend", "docker"] if backend == "docker" else []),
|
|
613
|
+
*(["--unsandboxed"] if kind == "agent" and backend == "local" else []), "--from-run"]
|
|
614
|
+
command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
|
|
615
|
+
script = ("#!/usr/bin/env bash\n"
|
|
616
|
+
f"# {o['label']}: the conformance kits of the ports it can offer (docs/standard.md, \"Ports\").\n"
|
|
617
|
+
+ _script(command))
|
|
618
|
+
return {"errors": [], "script": script, "label": o["label"],
|
|
619
|
+
"jobs": [{"name": conform.job_name(stamp, k["key"], definitions.job_word(o)), "agent": o["_ref"], "dataset": k["key"]}
|
|
620
|
+
for k in kits]}
|
|
621
|
+
|
|
622
|
+
|
|
623
|
+
def proving(benchmark: str, *, dataset: str = "", limit=None, provider: str = "openai", judge_model: str = "",
|
|
624
|
+
stamp: str, concurrency: int = 4) -> dict:
|
|
625
|
+
"""The proof a benchmark's page starts, as an Agents page starts a check
|
|
626
|
+
of ports (``conformance``): ``cli prove`` over one dataset or each - its
|
|
627
|
+
units, the leak check and every control it declares - as one command,
|
|
628
|
+
the control jobs it will write named ahead (``prove.job_name``).
|
|
629
|
+
``keyless``: the controls call no model, so the run is handed no key at
|
|
630
|
+
all - unless the benchmark is judged, whose judge still grades.
|
|
631
|
+
{errors, script, jobs, label, keyless}."""
|
|
632
|
+
errors: List[str] = []
|
|
633
|
+
b = definitions.find(_all()["benchmark"], benchmark or "")
|
|
634
|
+
if b is None:
|
|
635
|
+
return {"errors": [f"no benchmark {benchmark!r}"], "script": "", "jobs": [], "label": benchmark, "keyless": False}
|
|
636
|
+
b = dict(b, label=definitions.title(b))
|
|
637
|
+
judged = bool((b.get("score") or {}).get("judge"))
|
|
638
|
+
sets = [d for d in b["datasets"] if dataset in ("", d["id"])]
|
|
639
|
+
if not sets:
|
|
640
|
+
errors.append(f"Choose one of {b['label']}'s datasets, or each of them.")
|
|
641
|
+
if limit not in ("", None) and _int(limit, 1) is None:
|
|
642
|
+
errors.append("The number of units is a whole number of at least 1, or empty for all of them.")
|
|
643
|
+
if _int(concurrency, 1, MAX_CONCURRENT) is None:
|
|
644
|
+
errors.append(f"Units at a time: a whole number from 1 to {MAX_CONCURRENT}.")
|
|
645
|
+
if judged and provider not in PROVIDERS:
|
|
646
|
+
errors.append("Choose the judge's provider: OpenRouter, OpenAI or a local or self-hosted OpenAI-compatible endpoint.")
|
|
647
|
+
if judge_model and not _MODEL.match(judge_model):
|
|
648
|
+
errors.append("The judge model, as the provider lists it - or empty for the benchmark's own.")
|
|
649
|
+
if not re.match(r"^\d{4}-\d{2}-\d{2}__\d{2}-\d{2}-\d{2}$", stamp or ""):
|
|
650
|
+
errors.append("a time stamp: YYYY-MM-DD__HH-MM-SS")
|
|
651
|
+
if errors:
|
|
652
|
+
return {"errors": errors, "script": "", "jobs": [], "label": b["label"], "keyless": False}
|
|
653
|
+
words = ["nullius", "standard", "prove", definitions.ref_of(b), *(["--dataset", dataset] if dataset else []),
|
|
654
|
+
*(["--limit", str(limit)] if isinstance(limit, int) else []), *(["--provider", provider] if judged else []),
|
|
655
|
+
*(["--judge-model", judge_model] if judged and judge_model else []), "-j", str(concurrency), "--stamp", stamp, "--from-run"]
|
|
656
|
+
command = " ".join(shlex.quote(w) for w in words) + ' --jobs "${NULLIUS_JOBS:-jobs_standard}"'
|
|
657
|
+
script = ("#!/usr/bin/env bash\n"
|
|
658
|
+
f"# {b['label']}: its proofs - units, no leak, its controls (docs/standard.md, \"Controls and proofs\").\n"
|
|
659
|
+
+ _script(command))
|
|
660
|
+
declared = [c for c in definitions.CONTROLS if c in (b.get("controls") or {})]
|
|
661
|
+
return {"errors": [], "script": script, "label": b["label"], "keyless": not judged,
|
|
662
|
+
"jobs": [{"name": prove.job_name(stamp, b["key"], d["id"], c), "agent": c, "dataset": definitions.ref_of(b)}
|
|
663
|
+
for d in sets for c in declared]}
|