nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/standard/cli.py
ADDED
|
@@ -0,0 +1,703 @@
|
|
|
1
|
+
"""The standard from a shell.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.standard.cli list
|
|
4
|
+
PYTHONPATH=app python3 -m nullius.standard.cli check [PATH]
|
|
5
|
+
PYTHONPATH=app python3 -m nullius.standard.cli fit BENCHMARK[@VERSION]
|
|
6
|
+
PYTHONPATH=app python3 -m nullius.standard.cli pins BENCHMARK[@VERSION] [--tag TAG] [--limit N]
|
|
7
|
+
PYTHONPATH=app python3 -m nullius.standard.cli new benchmark|agent|capability KEY [--family F] [--port P]
|
|
8
|
+
PYTHONPATH=app python3 -m nullius.standard.cli new version KEY VERSION
|
|
9
|
+
PYTHONPATH=app python3 -m nullius.standard.cli conform AGENT|CAPABILITY --model M [--provider P] [--port PORT ...] [--repeat K] [--option NAME=VALUE ...] [--unsandboxed | --backend container]
|
|
10
|
+
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] AGENT|CAPABILITY --model M [--provider P] [--limit N] [-j N] [--option NAME=VALUE ...] [--bench-option NAME=VALUE ...] [--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H] ...] [--max-usd USD] [--max-usd-per-unit USD] [--unsandboxed | --backend container]
|
|
11
|
+
PYTHONPATH=app python3 -m nullius.standard.cli run BENCHMARK[@VERSION] nop|oracle [--dataset D] [--limit N]
|
|
12
|
+
PYTHONPATH=app python3 -m nullius.standard.cli prove BENCHMARK[@VERSION] [--dataset D] [--limit N]
|
|
13
|
+
PYTHONPATH=app python3 -m nullius.standard.cli run ... --shard I/N [--launch ID]
|
|
14
|
+
PYTHONPATH=app python3 -m nullius.standard.cli merge JOB_DIR... -o OUT [--force]
|
|
15
|
+
PYTHONPATH=app python3 -m nullius.standard.cli fidelity AGENT --model M [--provider P] [--option NAME=VALUE ...] [--passes N] --unsandboxed
|
|
16
|
+
|
|
17
|
+
An agent runs commands as you on the local backend, which is not a sandbox:
|
|
18
|
+
an agent's run or conformance check there is refused without ``--unsandboxed``.
|
|
19
|
+
``--backend container`` (``docker``, its earlier name) runs each unit's agent or capability in a container whose
|
|
20
|
+
only way out is the unit's relay, or a public web proxy when its environment
|
|
21
|
+
declares ``network`` (``standard/container.py``), and needs no such yes.
|
|
22
|
+
A benchmark's control (``nop``, ``oracle``: its ``controls``) runs no agent
|
|
23
|
+
and calls no model - only a judged benchmark's judge - so it asks for neither.
|
|
24
|
+
``prove`` runs a benchmark's controls and checks its units and that nothing
|
|
25
|
+
leaks, and writes what passed into its ``proven`` (``standard/prove.py``).
|
|
26
|
+
|
|
27
|
+
A benchmark, an agent or a capability is named by its key (its default
|
|
28
|
+
version) or ``key@version``. ``list`` shows every definition - each version - and which
|
|
29
|
+
pairs fit; ``check`` checks the
|
|
30
|
+
definitions (or one file); ``new`` writes a definition to fill in, or a new
|
|
31
|
+
version of an agent or a capability (``new version``: the default's pins, with
|
|
32
|
+
the version replaced, and nothing proven); ``fit`` says, for one benchmark, what can run it and
|
|
33
|
+
why the rest cannot; ``conform`` runs the kits and writes what passed into the
|
|
34
|
+
definition - the port, and each skill of its kit shown or not (``--repeat K``:
|
|
35
|
+
each check K times, held by a majority); ``run`` runs a benchmark into a job under ``--jobs`` (``jobs_standard/``),
|
|
36
|
+
which the application opens like any other; ``fidelity`` drives an agent over
|
|
37
|
+
ACP and through its own command, compares the requests its model was sent, and
|
|
38
|
+
writes the verdict into the definition (``standard/fidelity.py``). ``--shard I/N`` runs one
|
|
39
|
+
shard of the dataset's units (``standard/shard.py``: by the unit id's hash, then
|
|
40
|
+
``--limit``), the job recording its shard and its launch (``--launch``); ``merge``
|
|
41
|
+
puts a launch's shard jobs together into one job (``standard/merge.py``). ``--option``
|
|
42
|
+
sets one of the agent's or capability's options (``options`` in its definition);
|
|
43
|
+
the rest keep the vendor's defaults, and the job's config records them all.
|
|
44
|
+
A run says each stage before its first unit, and each unit's end, as one
|
|
45
|
+
``[nullius] stage=...`` line on stderr and in the job's ``progress.jsonl``
|
|
46
|
+
(``nullius/progress.py``): what the run's page shows while it goes.
|
|
47
|
+
|
|
48
|
+
Keys come from the process environment, then the saved user settings - the
|
|
49
|
+
file the Settings page edits. Nothing is printed of them. With ``--from-run`` (a run
|
|
50
|
+
the builder or a page started, ``standard/recipe.py``) saved settings are not
|
|
51
|
+
read: the key is the one the run was handed, ``OPENAI_API_KEY``, and the
|
|
52
|
+
endpoint ``OPENAI_BASE_URL`` when it names one (``run_env``). The licences a
|
|
53
|
+
benchmark's environments declare are never in either: the run reads them from
|
|
54
|
+
the benchmark's store (``nullius.secrets``) on this machine, or, on a host, from
|
|
55
|
+
the one line its launcher handed it (``licences``).
|
|
56
|
+
"""
|
|
57
|
+
from __future__ import annotations
|
|
58
|
+
|
|
59
|
+
import argparse
|
|
60
|
+
import json
|
|
61
|
+
import os
|
|
62
|
+
import shutil
|
|
63
|
+
import signal
|
|
64
|
+
import sys
|
|
65
|
+
import tempfile
|
|
66
|
+
|
|
67
|
+
from nullius import paths, prices as price_list, progress, secrets, settings
|
|
68
|
+
# The legacy module command needs saved path overrides before install.HOME
|
|
69
|
+
# and other runtime defaults are computed, just like the unified CLI does.
|
|
70
|
+
if __name__ == "__main__" and "--from-run" not in sys.argv[1:]:
|
|
71
|
+
settings.load(paths.settings_file())
|
|
72
|
+
|
|
73
|
+
from nullius.standard import conform, definitions, fidelity, job, match, merge, pins, ports, prove, run, shard
|
|
74
|
+
|
|
75
|
+
ENV_FILE = paths.settings_file()
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _all():
|
|
79
|
+
got, bad = {}, []
|
|
80
|
+
for kind in definitions.KINDS:
|
|
81
|
+
got[kind], b = definitions.load(kind, every=True)
|
|
82
|
+
bad += b
|
|
83
|
+
return got, bad
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def _other(key):
|
|
87
|
+
o = definitions.get("agent", key) or definitions.get("capability", key)
|
|
88
|
+
if not o:
|
|
89
|
+
sys.exit(f"no agent or capability {key!r}: {[d['_ref'] for k in ('agent', 'capability') for d in definitions.load(k, every=True)[0]]}")
|
|
90
|
+
return o
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def run_env(provider: str, env=os.environ) -> dict:
|
|
94
|
+
"""The environment a run started by Nullius reads its provider from:
|
|
95
|
+
``execution``'s launcher hands every run its one key as ``OPENAI_API_KEY``
|
|
96
|
+
and its endpoint as ``OPENAI_BASE_URL`` (empty for OpenAI's own), so they
|
|
97
|
+
are named here as the provider's own settings and nothing else is."""
|
|
98
|
+
p = run.PROVIDERS[provider]
|
|
99
|
+
out = {k: v for k, v in env.items() if k in run.KEEP}
|
|
100
|
+
if env.get("OPENAI_API_KEY"):
|
|
101
|
+
out[p["key"]] = env["OPENAI_API_KEY"]
|
|
102
|
+
if "base_var" in p and env.get("OPENAI_BASE_URL"):
|
|
103
|
+
out[p["base_var"]] = env["OPENAI_BASE_URL"]
|
|
104
|
+
return out
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def licences(bench: dict, env=os.environ):
|
|
108
|
+
"""(``run.Licences``, the folder to remove after) for a run on a host,
|
|
109
|
+
whose launcher handed it the benchmark's licences as one line
|
|
110
|
+
(``secrets.HANDED``, ``execution``): a licence file is written to
|
|
111
|
+
``NULLIUS_LICENCE_DIR`` - in memory, ``/dev/shm`` - and nowhere else.
|
|
112
|
+
The line leaves this process's environment once read. (None, None) on
|
|
113
|
+
this machine: the run reads the benchmark's own store."""
|
|
114
|
+
line = env.pop(secrets.HANDED, None)
|
|
115
|
+
if not line:
|
|
116
|
+
return None, None
|
|
117
|
+
shm = "/dev/shm" if os.path.isdir("/dev/shm") else tempfile.gettempdir()
|
|
118
|
+
spill = env.get("NULLIUS_LICENCE_DIR") or os.path.join(shm, f"nullius-{os.getpid()}")
|
|
119
|
+
return run.Licences.handed(bench, secrets.unpack(line), spill), spill
|
|
120
|
+
|
|
121
|
+
|
|
122
|
+
def older(d: dict, ports_=None) -> list:
|
|
123
|
+
"""``agent.tools/v1 proven on kit v1; kit v2 is current``, for each port
|
|
124
|
+
``d`` offers (of ``ports_``, when given) on a proof older than its kit."""
|
|
125
|
+
proven = d.get("proven") or {}
|
|
126
|
+
out = [(p, definitions.older_kit(p, proven.get(p))) for p in d.get("offers") or [] if ports_ is None or p in ports_]
|
|
127
|
+
return [f"{p} {note}" for p, note in out if note]
|
|
128
|
+
|
|
129
|
+
|
|
130
|
+
def skills(d: dict, ports_=None) -> list:
|
|
131
|
+
"""``agent.tools/v1 skills: decoys ✗ · chain ✓ · retry ✗``, for each port
|
|
132
|
+
``d`` offers (of ``ports_``, when given) whose kit has skills."""
|
|
133
|
+
from nullius.standard.recipe import skills_of
|
|
134
|
+
proven, out = d.get("proven") or {}, []
|
|
135
|
+
for p in d.get("offers") or []:
|
|
136
|
+
got = skills_of(p, proven.get(p)) if ports_ is None or p in ports_ else []
|
|
137
|
+
if got:
|
|
138
|
+
out.append(f"{p} skills: " + " · ".join(f"{s['id']} {'✓' if s['shown'] else '✗'}" for s in got))
|
|
139
|
+
return out
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def cmd_pins(a):
|
|
143
|
+
if a.limit is not None and a.limit < 1:
|
|
144
|
+
print("--limit must be a positive whole number", file=sys.stderr)
|
|
145
|
+
return 2
|
|
146
|
+
if a.tag is not None and not pins.TAG.fullmatch(a.tag):
|
|
147
|
+
print("invalid image tag", file=sys.stderr)
|
|
148
|
+
return 2
|
|
149
|
+
bench = definitions.get("benchmark", a.benchmark)
|
|
150
|
+
if bench is None:
|
|
151
|
+
print("no such benchmark", file=sys.stderr)
|
|
152
|
+
return 2
|
|
153
|
+
try:
|
|
154
|
+
refs = pins.references(bench["_here"])
|
|
155
|
+
except (OSError, ValueError, SyntaxError):
|
|
156
|
+
print("not checked: cannot read image pins from images.json or driver.py", file=sys.stderr)
|
|
157
|
+
return 2
|
|
158
|
+
if not refs:
|
|
159
|
+
print("not checked: no image pins found", file=sys.stderr)
|
|
160
|
+
return 2
|
|
161
|
+
code = 0
|
|
162
|
+
for name, ref in refs[:a.limit]:
|
|
163
|
+
state, detail = pins.check(ref, a.tag)
|
|
164
|
+
print(f"{name}: {state}: {detail}", flush=True)
|
|
165
|
+
code = max(code, 2 if state == "not checked" else 0 if state == "same" else 1)
|
|
166
|
+
return code
|
|
167
|
+
|
|
168
|
+
|
|
169
|
+
def cmd_list(a):
|
|
170
|
+
got, bad = _all()
|
|
171
|
+
for kind in definitions.KINDS:
|
|
172
|
+
print(definitions.KINDS[kind]["dir"] + ":")
|
|
173
|
+
for d in got[kind]:
|
|
174
|
+
side = d.get("needs") if kind == "benchmark" else d.get("offers")
|
|
175
|
+
print(f" {d.get('_ref') or d['key']:<18} {('needs ' if kind == 'benchmark' else 'offers ') + ', '.join(side or []) or 'nothing proven yet'}")
|
|
176
|
+
up = d.get("upstream") or {}
|
|
177
|
+
if up.get("repository"):
|
|
178
|
+
print(f" {'':<18} upstream {up['repository'].removeprefix('https://')} @ "
|
|
179
|
+
+ (f"{up['tag']} ({str(up.get('tested', ''))[:12]})" if up.get("tag") else str(up.get("tested", ""))[:12]))
|
|
180
|
+
if kind == "benchmark" and d.get("needs_skills"):
|
|
181
|
+
print(f" {'':<18} and the skills " + "; ".join(f"{', '.join(s)} of {p}" for p, s in d["needs_skills"].items()))
|
|
182
|
+
for note in ([] if kind == "benchmark" else older(d) + skills(d)):
|
|
183
|
+
print(f" {'':<18} {note}")
|
|
184
|
+
t = match.table(got["benchmark"], got["agent"], got["capability"])
|
|
185
|
+
print("pairs:")
|
|
186
|
+
for b, row in t.items():
|
|
187
|
+
print(f" {b}: " + ", ".join(f"{o} {'fits' if not why else 'no'}" for o, why in row.items()))
|
|
188
|
+
for b in bad:
|
|
189
|
+
print("problem:", b)
|
|
190
|
+
return 1 if bad else 0
|
|
191
|
+
|
|
192
|
+
|
|
193
|
+
def catalogue_problems(benches) -> list:
|
|
194
|
+
"""A standard benchmark is listed by the catalogue only through its entry:
|
|
195
|
+
benchmark.json beside it, naming the standard among its harnesses, and the
|
|
196
|
+
whole catalogue consistent (nullius.catalogue.catalog.validate)."""
|
|
197
|
+
from nullius.catalogue import catalog
|
|
198
|
+
bad = []
|
|
199
|
+
for b in benches:
|
|
200
|
+
entry = os.path.join(b["_here"], "benchmark.json")
|
|
201
|
+
if not os.path.isfile(entry):
|
|
202
|
+
bad.append(f"benchmark {b['key']}: no benchmark.json beside standard.json (the catalogue's entry)")
|
|
203
|
+
continue
|
|
204
|
+
with open(entry) as f:
|
|
205
|
+
if "standard" not in (json.load(f).get("harness") or []):
|
|
206
|
+
bad.append(f"benchmark {b['key']}: benchmark.json does not name 'standard' in harness")
|
|
207
|
+
# each benchmark's settings inventory, against every version of it (definitions.settings_problems)
|
|
208
|
+
every = definitions.load("benchmark", every=True)[0]
|
|
209
|
+
for key in dict.fromkeys(b["key"] for b in benches):
|
|
210
|
+
bad += [f"benchmark {key}: {p}" for p in definitions.settings_problems([v for v in every if v["key"] == key])]
|
|
211
|
+
bad += [f"catalogue: {p}" for p in catalog.validate(catalog.load())]
|
|
212
|
+
return bad
|
|
213
|
+
|
|
214
|
+
|
|
215
|
+
def cmd_check(a):
|
|
216
|
+
if a.path:
|
|
217
|
+
with open(a.path) as f:
|
|
218
|
+
d = definitions.names.definition(json.load(f))
|
|
219
|
+
kind = next((k for k, v in definitions.KINDS.items() if v["schema"] == d.get("schema")), None)
|
|
220
|
+
if not kind:
|
|
221
|
+
print(f"schema: one of {[v['schema'] for v in definitions.KINDS.values()]}")
|
|
222
|
+
return 1
|
|
223
|
+
bad = definitions.problems(dict(d, _here=os.path.dirname(os.path.abspath(a.path))), kind)
|
|
224
|
+
else:
|
|
225
|
+
got, bad = _all()
|
|
226
|
+
for k in definitions.kits():
|
|
227
|
+
bad += [f"kit {k['key']}: {p}" for p in definitions.problems(k, "benchmark")]
|
|
228
|
+
bad += catalogue_problems(got["benchmark"])
|
|
229
|
+
for b in bad:
|
|
230
|
+
print(b)
|
|
231
|
+
print("ok" if not bad else f"{len(bad)} problem(s)")
|
|
232
|
+
return 1 if bad else 0
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
def cmd_fit(a):
|
|
236
|
+
b = definitions.get("benchmark", a.benchmark)
|
|
237
|
+
if not b:
|
|
238
|
+
sys.exit(f"no benchmark {a.benchmark!r}")
|
|
239
|
+
print(f"{b['label']} ({b['family']} benchmark) needs: " + ", ".join(b["needs"]))
|
|
240
|
+
for p in b["needs"]:
|
|
241
|
+
print(f" {p}: {ports.PORTS[p]['summary']}")
|
|
242
|
+
have = definitions.port_skills(p)
|
|
243
|
+
for s in (b.get("needs_skills") or {}).get(p) or []:
|
|
244
|
+
print(f" and its {s} skill: {have.get(s, '')}")
|
|
245
|
+
others = definitions.load("agent" if b["family"] == "agent" else "capability", every=True)[0]
|
|
246
|
+
for o in others:
|
|
247
|
+
why = match.fit(b, o)
|
|
248
|
+
notes = [] if why else older(o, b["needs"]) + skills(o, b["needs"])
|
|
249
|
+
print(f" {'✓' if not why else '✗'} {o['_ref']}" + ("" if not why else ": " + "; ".join(why))
|
|
250
|
+
+ ("" if not notes else " (" + "; ".join(notes) + ")"))
|
|
251
|
+
return 0
|
|
252
|
+
|
|
253
|
+
|
|
254
|
+
TEMPLATES = {
|
|
255
|
+
"benchmark": {
|
|
256
|
+
"standard.json": {
|
|
257
|
+
"schema": "nullius.benchmark/v1", "key": "<KEY>", "label": "<KEY>", "summary": "SAY WHAT ONE UNIT IS AND HOW IT IS SCORED.",
|
|
258
|
+
"family": "{family}", "needs": ["{port}"],
|
|
259
|
+
"upstream": {"repository": "https://github.com/OWNER/REPO", "tested": "FULL 40-CHARACTER COMMIT", "license": "LICENSE"},
|
|
260
|
+
"driver": {"command": ["python3", "{here}/driver.py"]},
|
|
261
|
+
"datasets": [{"id": "main", "label": "main", "path": "PATH/IN/THE/UPSTREAM", "sha256": "SHA256 OF THE FILE", "units": 1}],
|
|
262
|
+
"units": {"noun": "task", "nouns": "tasks"},
|
|
263
|
+
"score": {"kind": "pass", "says": "WHAT A REWARD OF 1 MEANS"},
|
|
264
|
+
},
|
|
265
|
+
"driver.py": '''"""{key}'s driver for the Nullius standard: JSON lines on stdin and stdout.
|
|
266
|
+
|
|
267
|
+
$NULLIUS_DATASET is the dataset file (checked by its sha256), $NULLIUS_UPSTREAM
|
|
268
|
+
the upstream at its tested commit. Each request is one line, {{"op": ...}};
|
|
269
|
+
each reply one line, a JSON object, or {{"error": "..."}}.
|
|
270
|
+
"""
|
|
271
|
+
import json
|
|
272
|
+
import os
|
|
273
|
+
import sys
|
|
274
|
+
|
|
275
|
+
|
|
276
|
+
def units(dataset):
|
|
277
|
+
return {{"units": []}} # every unit's id, in order
|
|
278
|
+
|
|
279
|
+
|
|
280
|
+
def setup(unit, dir):
|
|
281
|
+
# fill dir, the unit's workspace; return the prompt (agent family) or
|
|
282
|
+
# {{"memory": {{"add": [...], "recall": [...]}}}} (a memory capability)
|
|
283
|
+
return {{"prompt": ""}}
|
|
284
|
+
{extra}
|
|
285
|
+
|
|
286
|
+
def score(unit, dir, transcript, answer):
|
|
287
|
+
return {{"reward": 0.0, "detail": {{}}}} # 1.0 is a pass; a score benchmark gives 0..1
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
HANDLERS = {{"units": units, "setup": setup, "score": score{handlers}}}
|
|
291
|
+
for line in sys.stdin:
|
|
292
|
+
try:
|
|
293
|
+
req = json.loads(line)
|
|
294
|
+
out = HANDLERS[req.pop("op")](**req)
|
|
295
|
+
except Exception as e:
|
|
296
|
+
out = {{"error": f"{{type(e).__name__}}: {{e}}"}}
|
|
297
|
+
print(json.dumps(out), flush=True)
|
|
298
|
+
''',
|
|
299
|
+
},
|
|
300
|
+
"agent": {
|
|
301
|
+
"agent.json": {
|
|
302
|
+
"schema": "nullius.agent/v1", "key": "<KEY>", "label": "<KEY>", "summary": "SAY WHAT THE AGENT IS AND HOW IT IS DRIVEN.",
|
|
303
|
+
"upstream": {"package": "PACKAGE", "version": "VERSION", "license": "LICENSE"},
|
|
304
|
+
"install": {"npm": ["PACKAGE@VERSION"]},
|
|
305
|
+
"start": {"acp": ["{install}/node_modules/.bin/PROGRAM"]},
|
|
306
|
+
"model": {"apis": ["chat"], "env": {"OPENAI_BASE_URL": "{base_url}", "OPENAI_API_KEY": "{key}", "MODEL": "{model}"}},
|
|
307
|
+
"offers": [], "proven": {},
|
|
308
|
+
},
|
|
309
|
+
},
|
|
310
|
+
"capability": {
|
|
311
|
+
"capability.json": {
|
|
312
|
+
"schema": "nullius.capability/v1", "key": "<KEY>", "label": "<KEY>", "summary": "SAY WHAT THE CAPABILITY KEEPS AND WHAT IT GIVES BACK.",
|
|
313
|
+
"kind": "memory", "start": {"command": ["python3", "{here}/capability.py"]}, "offers": [], "proven": {},
|
|
314
|
+
},
|
|
315
|
+
"capability.py": '''"""{key}: cap.memory/v1 in JSON lines.
|
|
316
|
+
|
|
317
|
+
add {{session}} keep a session: {{id, date, turns: [{{role, text}}], text}}
|
|
318
|
+
recall {{question}} -> {{"memory": "what was kept that bears on the question"}}
|
|
319
|
+
"""
|
|
320
|
+
import json
|
|
321
|
+
import sys
|
|
322
|
+
|
|
323
|
+
for line in sys.stdin:
|
|
324
|
+
req = json.loads(line)
|
|
325
|
+
try:
|
|
326
|
+
if req["op"] == "add":
|
|
327
|
+
out = {{}}
|
|
328
|
+
elif req["op"] == "recall":
|
|
329
|
+
out = {{"memory": ""}}
|
|
330
|
+
else:
|
|
331
|
+
out = {{"error": "unknown op " + req["op"]}}
|
|
332
|
+
except Exception as e:
|
|
333
|
+
out = {{"error": f"{{type(e).__name__}}: {{e}}"}}
|
|
334
|
+
print(json.dumps(out), flush=True)
|
|
335
|
+
''',
|
|
336
|
+
},
|
|
337
|
+
}
|
|
338
|
+
PRODUCE = '''
|
|
339
|
+
|
|
340
|
+
def produce(unit, dir, recalled, model, base_url, key):
|
|
341
|
+
# a capability benchmark's own step: answer from what the capability recalled,
|
|
342
|
+
# calling the model at base_url (the relay) with key (a placeholder)
|
|
343
|
+
return {{"answer": ""}}
|
|
344
|
+
'''
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def new_version(key: str, version: str) -> str:
|
|
348
|
+
"""Add ``version`` to an agent's or a capability's file: a copy of its default
|
|
349
|
+
version's block with the old version replaced in its pins, nothing proven.
|
|
350
|
+
A file without ``versions`` is turned into one first, its one version kept
|
|
351
|
+
as it was, proofs included. Returns the file's path."""
|
|
352
|
+
if not definitions._KEY.match(key):
|
|
353
|
+
sys.exit(f"no agent, capability or benchmark {key!r}")
|
|
354
|
+
kind = next((k for k in ("agent", "capability", "benchmark")
|
|
355
|
+
if (folder := definitions.entry_dir(k, key))
|
|
356
|
+
and os.path.isfile(definitions.definition_path(folder, k))), None)
|
|
357
|
+
if not kind:
|
|
358
|
+
sys.exit(f"no agent, capability or benchmark {key!r}")
|
|
359
|
+
if not definitions._VERSION.match(version):
|
|
360
|
+
sys.exit("a version is letters, digits and . + _ -")
|
|
361
|
+
path = definitions.definition_path(definitions.entry_dir(kind, key), kind)
|
|
362
|
+
with open(path) as f:
|
|
363
|
+
d = definitions.names.definition(json.load(f))
|
|
364
|
+
if "versions" not in d:
|
|
365
|
+
up = dict(d.get("upstream") or {})
|
|
366
|
+
old = str(up.pop("version", "") or "")
|
|
367
|
+
if not old:
|
|
368
|
+
sys.exit(f"{path}: upstream.version is not set, so its one version has no name")
|
|
369
|
+
# a benchmark's version holds only its proofs at first: what differs in the new one is the user's to say
|
|
370
|
+
block = {"proven": d.pop("proven", {})} if kind == "benchmark" else {"install": d.pop("install", {}), "proven": d.pop("proven", {})}
|
|
371
|
+
d.pop("offers", None)
|
|
372
|
+
d["upstream"] = up
|
|
373
|
+
d["default"], d["versions"] = old, {old: block}
|
|
374
|
+
if version in d["versions"]:
|
|
375
|
+
sys.exit(f"{key} has a version {version} already")
|
|
376
|
+
old = d["default"]
|
|
377
|
+
base = d["versions"][old]
|
|
378
|
+
copy = json.loads(json.dumps({k: v for k, v in base.items() if k != "proven"}).replace(old, version))
|
|
379
|
+
d["versions"] = {version: dict(copy, proven={}), **d["versions"]}
|
|
380
|
+
path = os.path.join(definitions.writable_dir(kind, key), definitions.KINDS[kind]["file"])
|
|
381
|
+
with open(path, "w") as f:
|
|
382
|
+
json.dump(d, f, indent=2)
|
|
383
|
+
f.write("\n")
|
|
384
|
+
print(f"wrote {path}: {key}@{version} beside {', '.join(v for v in d['versions'] if v != version)} (default still {old})")
|
|
385
|
+
if kind == "benchmark":
|
|
386
|
+
print(f"next: say what differs in it (upstream, datasets, driver, score, limits, controls, environments), then: "
|
|
387
|
+
f"cli check, and cli prove {key}@{version} --limit 3; make it the default by setting \"default\": \"{version}\"")
|
|
388
|
+
return path
|
|
389
|
+
print(f"next: check its pins, then: cli check, and cli conform {key}@{version} --model gpt-5-mini"
|
|
390
|
+
+ (" --unsandboxed" if kind == "agent" else "") + f"; make it the default by setting \"default\": \"{version}\"")
|
|
391
|
+
return path
|
|
392
|
+
|
|
393
|
+
|
|
394
|
+
def cmd_new(a):
|
|
395
|
+
"""A definition to fill in, in the catalogue folder it belongs in."""
|
|
396
|
+
if a.kind == "version":
|
|
397
|
+
if not a.version:
|
|
398
|
+
sys.exit("new version KEY VERSION")
|
|
399
|
+
new_version(a.key, a.version)
|
|
400
|
+
return 0
|
|
401
|
+
kind, key = a.kind, a.key
|
|
402
|
+
if not definitions._KEY.match(key):
|
|
403
|
+
sys.exit("a key is lower case letters, digits and dashes")
|
|
404
|
+
name = definitions.KINDS[kind]["file"]
|
|
405
|
+
selected_dir = definitions.entry_dir(kind, key)
|
|
406
|
+
selected = os.path.join(selected_dir, name) if selected_dir else None
|
|
407
|
+
if selected and os.path.exists(selected):
|
|
408
|
+
sys.exit(f"{selected} exists already")
|
|
409
|
+
folder = definitions.writable_dir(kind, key)
|
|
410
|
+
os.makedirs(folder, exist_ok=True)
|
|
411
|
+
family = a.family or "agent"
|
|
412
|
+
port = a.port or ("agent.task/v1" if family == "agent" else "cap.memory/v1")
|
|
413
|
+
for fname, body in TEMPLATES[kind].items():
|
|
414
|
+
if isinstance(body, dict):
|
|
415
|
+
text = json.dumps(body, indent=2).replace('"<KEY>"', json.dumps(key)).replace('"{family}"', json.dumps(family)) \
|
|
416
|
+
.replace('"{port}"', json.dumps(port))
|
|
417
|
+
else:
|
|
418
|
+
feat = family == "capability"
|
|
419
|
+
text = body.format(key=key, extra=PRODUCE.format() if feat else "", handlers=', "produce": produce' if feat else "")
|
|
420
|
+
path = os.path.join(folder, fname)
|
|
421
|
+
if os.path.exists(path):
|
|
422
|
+
print("kept", path)
|
|
423
|
+
continue
|
|
424
|
+
with open(path, "w") as f:
|
|
425
|
+
f.write(text + ("\n" if not text.endswith("\n") else ""))
|
|
426
|
+
print("wrote", path)
|
|
427
|
+
if kind == "benchmark":
|
|
428
|
+
then = f"cli run {key} reference --model gpt-5-mini --limit 2 --unsandboxed"
|
|
429
|
+
else:
|
|
430
|
+
then = f"cli conform {key} --model gpt-5-mini" + (" --unsandboxed" if kind == "agent" else "")
|
|
431
|
+
print(f"next: fill in the CAPITALS{', add benchmark.json beside it (the catalogue entry)' if kind == 'benchmark' else ''}, "
|
|
432
|
+
f"then: cli check, and {then}")
|
|
433
|
+
return 0
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def options(pairs) -> dict:
|
|
437
|
+
"""``--option name=value``, repeatable, as {name: value}."""
|
|
438
|
+
out = {}
|
|
439
|
+
for p in pairs or []:
|
|
440
|
+
name, eq, value = p.partition("=")
|
|
441
|
+
if not eq or not name:
|
|
442
|
+
sys.exit(f"--option {p!r}: NAME=VALUE")
|
|
443
|
+
out[name.strip()] = value.strip()
|
|
444
|
+
return out
|
|
445
|
+
|
|
446
|
+
|
|
447
|
+
def prices(given) -> dict:
|
|
448
|
+
"""``--price MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]``, repeatable: this run's own prices, over the user's list."""
|
|
449
|
+
out = {}
|
|
450
|
+
for p in given or []:
|
|
451
|
+
try:
|
|
452
|
+
name, row = price_list.parse(p)
|
|
453
|
+
except ValueError as e:
|
|
454
|
+
sys.exit(str(e))
|
|
455
|
+
out[name] = row
|
|
456
|
+
return out
|
|
457
|
+
|
|
458
|
+
|
|
459
|
+
def cmd_conform(a):
|
|
460
|
+
o = _other(a.other)
|
|
461
|
+
try:
|
|
462
|
+
got = conform.check(o, model=a.model, provider=a.provider, which=a.port or None, jobs_root=a.jobs, say=print,
|
|
463
|
+
unsandboxed=a.unsandboxed, stamp=a.stamp, backend=a.backend, env=run_env(a.provider) if a.from_run else None,
|
|
464
|
+
options=options(a.option), repeat=a.repeat)
|
|
465
|
+
except run.Refused as e:
|
|
466
|
+
print("refused:", e)
|
|
467
|
+
return 2
|
|
468
|
+
return 0 if all(r["ok"] for r in got.values()) else 1
|
|
469
|
+
|
|
470
|
+
|
|
471
|
+
def _bench(key):
|
|
472
|
+
"""A benchmark of the catalogue, or a conformance kit by its key."""
|
|
473
|
+
b = definitions.get("benchmark", key) or next((k for k in definitions.kits() if k["key"] == key), None)
|
|
474
|
+
if not b:
|
|
475
|
+
sys.exit(f"no benchmark {key!r}")
|
|
476
|
+
return b
|
|
477
|
+
|
|
478
|
+
|
|
479
|
+
def cmd_run(a):
|
|
480
|
+
b = _bench(a.benchmark)
|
|
481
|
+
if a.other in definitions.CONTROLS:
|
|
482
|
+
other = definitions.control(b, a.other)
|
|
483
|
+
if other is None:
|
|
484
|
+
sys.exit(f"{b['label']} declares no {a.other} control (controls in its standard.json)")
|
|
485
|
+
elif a.other in definitions.SETUPS:
|
|
486
|
+
other, why = definitions.setup(b, a.other, a.model or "", a.provider)
|
|
487
|
+
if other is None:
|
|
488
|
+
print("refused:", why)
|
|
489
|
+
return 2
|
|
490
|
+
elif not a.model:
|
|
491
|
+
sys.exit("--model: the model the agent or the capability is run with")
|
|
492
|
+
else:
|
|
493
|
+
other = _other(a.other)
|
|
494
|
+
held, spill = licences(b)
|
|
495
|
+
try:
|
|
496
|
+
job, failed = run.run_and_report(b, other, model=a.model or "", provider=a.provider, dataset=a.dataset, limit=a.limit,
|
|
497
|
+
concurrency=a.concurrency, jobs_root=a.jobs, name=a.name, unsandboxed=a.unsandboxed, judge_model=a.judge_model, backend=a.backend,
|
|
498
|
+
simulator_model=a.simulator_model, options=options(a.option), shard=_shard(a.shard), launch=a.launch,
|
|
499
|
+
licences=held, bench_options=options(a.bench_option), repeat=a.repeat, remove_images=a.remove_images,
|
|
500
|
+
prices=prices(a.price), max_usd=a.max_usd, max_usd_per_unit=a.max_usd_per_unit,
|
|
501
|
+
**({"env": run_env(a.provider)} if a.from_run else {}))
|
|
502
|
+
except run.Refused as e:
|
|
503
|
+
print("refused:", e)
|
|
504
|
+
return 2
|
|
505
|
+
finally:
|
|
506
|
+
if spill:
|
|
507
|
+
shutil.rmtree(spill, ignore_errors=True)
|
|
508
|
+
print(job)
|
|
509
|
+
# no unit reached a verdict (a refusing endpoint, every unit an error): the run failed, and says so
|
|
510
|
+
# by its exit status, so a launch of several agents pauses rather than running the next into the same error.
|
|
511
|
+
# A run its spending cap stopped did what was asked of it: no failure
|
|
512
|
+
return 3 if failed else 0
|
|
513
|
+
|
|
514
|
+
|
|
515
|
+
def _shard(spec):
|
|
516
|
+
if not spec:
|
|
517
|
+
return None
|
|
518
|
+
try:
|
|
519
|
+
return shard.parse(spec)
|
|
520
|
+
except ValueError as e:
|
|
521
|
+
sys.exit(str(e))
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def cmd_traces(a):
|
|
525
|
+
said = {"full": "with model calls and tool calls", "partial": "without tool times: the trial kept no ACP events",
|
|
526
|
+
"empty": "nothing recorded", "control": "controls, with nothing to show", "has_trace": "already have a trace, left as they are",
|
|
527
|
+
"not_standard": "not standard runs", "unreadable": "could not be read"}
|
|
528
|
+
counts = {}
|
|
529
|
+
for trial in job.trial_dirs(a.root):
|
|
530
|
+
try:
|
|
531
|
+
got = job.backfill_trace(trial, write=not a.dry_run)
|
|
532
|
+
except (OSError, ValueError, KeyError, TypeError, AttributeError) as e:
|
|
533
|
+
got = "unreadable"
|
|
534
|
+
print(f"{trial}: {e}", file=sys.stderr)
|
|
535
|
+
counts[got] = counts.get(got, 0) + 1
|
|
536
|
+
made = counts.get("full", 0) + counts.get("partial", 0)
|
|
537
|
+
print(f"{'would write' if a.dry_run else 'wrote'} {made} traces")
|
|
538
|
+
for name in ("full", "partial", "empty", "control", "has_trace", "not_standard", "unreadable"):
|
|
539
|
+
if counts.get(name):
|
|
540
|
+
print(f" {counts[name]:>5} {said[name]}")
|
|
541
|
+
return 1 if counts.get("unreadable") else 0
|
|
542
|
+
|
|
543
|
+
|
|
544
|
+
def cmd_merge(a):
|
|
545
|
+
try:
|
|
546
|
+
got = merge.merge(a.jobs, a.out, force=a.force)
|
|
547
|
+
except merge.Refused as e:
|
|
548
|
+
print("refused:", e)
|
|
549
|
+
return 2
|
|
550
|
+
print(f"{got['out']}: {got['trials']} trials from {len(got['merged_from'])} shards")
|
|
551
|
+
for line in got["notes"]:
|
|
552
|
+
print(" " + line)
|
|
553
|
+
return 0
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def cmd_fidelity(a):
|
|
557
|
+
o = definitions.get("agent", a.other)
|
|
558
|
+
if not o:
|
|
559
|
+
sys.exit(f"no agent {a.other!r}: {[d['_ref'] for d in definitions.load('agent', every=True)[0]]}")
|
|
560
|
+
try:
|
|
561
|
+
got = fidelity.check(o, model=a.model, provider=a.provider, jobs_root=a.jobs, passes=a.passes, say=print,
|
|
562
|
+
unsandboxed=a.unsandboxed, options=options(a.option),
|
|
563
|
+
env=run_env(a.provider) if a.from_run else None)
|
|
564
|
+
except run.Refused as e:
|
|
565
|
+
print("refused:", e)
|
|
566
|
+
return 2
|
|
567
|
+
return 0 if got["verdict"] == "same" else 1
|
|
568
|
+
|
|
569
|
+
|
|
570
|
+
def cmd_prove(a):
|
|
571
|
+
b = _bench(a.benchmark)
|
|
572
|
+
held, spill = licences(b)
|
|
573
|
+
try:
|
|
574
|
+
got = prove.prove(b, dataset=a.dataset, limit=a.limit, jobs_root=a.jobs, provider=a.provider, judge_model=a.judge_model,
|
|
575
|
+
stamp=a.stamp, concurrency=a.concurrency, say=print, shard=_shard(a.shard), launch=a.launch,
|
|
576
|
+
env=run_env(a.provider) if a.from_run else None, licences=held, remove_images=a.remove_images)
|
|
577
|
+
except run.Refused as e:
|
|
578
|
+
print("refused:", e)
|
|
579
|
+
return 2
|
|
580
|
+
finally:
|
|
581
|
+
if spill:
|
|
582
|
+
shutil.rmtree(spill, ignore_errors=True)
|
|
583
|
+
return 0 if all(r["ok"] for checks in got.values() for r in checks.values()) else 1
|
|
584
|
+
|
|
585
|
+
|
|
586
|
+
def _exit_on_term():
|
|
587
|
+
"""Stop sends SIGTERM, and its default ends the process where it stands:
|
|
588
|
+
Run.start's cleanup never ran, so a docker run's unit and relay
|
|
589
|
+
containers (the relay holding the key) kept running and the job had no
|
|
590
|
+
result.json. The first TERM is an exit, which runs every ``finally``; a
|
|
591
|
+
second one (Stop signals the group, then the process) is ignored so it
|
|
592
|
+
cannot cut that cleanup short."""
|
|
593
|
+
def stopped(signum, frame):
|
|
594
|
+
signal.signal(signal.SIGTERM, signal.SIG_IGN)
|
|
595
|
+
raise SystemExit(128 + signum)
|
|
596
|
+
signal.signal(signal.SIGTERM, stopped)
|
|
597
|
+
|
|
598
|
+
|
|
599
|
+
def main(argv=None):
|
|
600
|
+
ap = argparse.ArgumentParser(prog="nullius.standard.cli", description=__doc__.split("\n\n")[0])
|
|
601
|
+
sub = ap.add_subparsers(dest="cmd", required=True)
|
|
602
|
+
sub.add_parser("list", help="every definition, and which pairs fit")
|
|
603
|
+
c = sub.add_parser("check", help="check the definitions, or one file")
|
|
604
|
+
c.add_argument("path", nargs="?")
|
|
605
|
+
f = sub.add_parser("fit", help="what can run a benchmark, and why the rest cannot")
|
|
606
|
+
f.add_argument("benchmark")
|
|
607
|
+
p = sub.add_parser("pins", help="compare image pins with registry tags without changing them")
|
|
608
|
+
p.add_argument("benchmark")
|
|
609
|
+
p.add_argument("--tag", help="tag to compare (default: embedded tag, or latest)")
|
|
610
|
+
p.add_argument("--limit", type=int, help="check only the first N pins")
|
|
611
|
+
n = sub.add_parser("new", help="a definition to fill in, in its catalogue folder")
|
|
612
|
+
n.add_argument("kind", type=definitions.names.kind, choices=[*definitions.KINDS, "version"])
|
|
613
|
+
n.add_argument("key")
|
|
614
|
+
n.add_argument("version", nargs="?", help="new version: the version to add")
|
|
615
|
+
n.add_argument("--family", type=definitions.names.kind, choices=list(definitions.FAMILIES), help="a benchmark's family (agent by default)")
|
|
616
|
+
n.add_argument("--port", type=definitions.names.port, help="the port a new benchmark needs")
|
|
617
|
+
helps = {"conform": "run the kits, record what passed", "run": "run a benchmark into a job",
|
|
618
|
+
"fidelity": "drive an agent over ACP and through its own command: does its model get the same requests?"}
|
|
619
|
+
for name in ("conform", "run", "fidelity"):
|
|
620
|
+
p = sub.add_parser(name, help=helps[name])
|
|
621
|
+
if name == "run":
|
|
622
|
+
p.add_argument("benchmark")
|
|
623
|
+
p.add_argument("other", help="an agent or a capability: KEY (its default version) or KEY@VERSION"
|
|
624
|
+
+ ("; or nop, oracle: a control the benchmark declares; or original, model: one of its setups" if name == "run" else ""))
|
|
625
|
+
p.add_argument("--model", required=name == "conform",
|
|
626
|
+
help="the model (a control needs none; original takes one of the paper's, its first by default)" if name == "run" else None)
|
|
627
|
+
p.add_argument("--provider", default="openai", choices=sorted(run.PROVIDERS))
|
|
628
|
+
p.add_argument("--jobs", default="jobs_standard", help="where jobs are written (default jobs_standard/, gitignored)")
|
|
629
|
+
p.add_argument("--unsandboxed", action="store_true",
|
|
630
|
+
help="allow an agent, or a setup check without Docker, to execute code on this machine as you")
|
|
631
|
+
p.add_argument("--backend", default="local", type=definitions.names.backend, choices=list(run.BACKENDS),
|
|
632
|
+
help="local processes (the default), or container: each unit in a container, its relay beside it "
|
|
633
|
+
"(docker: its earlier name)")
|
|
634
|
+
p.add_argument("--from-run", action="store_true",
|
|
635
|
+
help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read")
|
|
636
|
+
p.add_argument("--option", action="append", metavar="NAME=VALUE",
|
|
637
|
+
help="one of the agent's or capability's options (repeatable); the rest keep their defaults")
|
|
638
|
+
if name == "fidelity":
|
|
639
|
+
p.add_argument("--passes", type=int, default=2, help="runs of each way (default 2: the second shows the noise)")
|
|
640
|
+
elif name == "conform":
|
|
641
|
+
p.add_argument("--port", type=definitions.names.port, action="append", help="only this port's kit (repeatable)")
|
|
642
|
+
p.add_argument("--stamp", help="the time stamp the kit jobs are named with (default: now)")
|
|
643
|
+
p.add_argument("--repeat", type=int, default=1, metavar="K",
|
|
644
|
+
help="run each check K times; a check passes when it passes in most of them (default 1)")
|
|
645
|
+
else:
|
|
646
|
+
p.add_argument("--dataset")
|
|
647
|
+
p.add_argument("--limit", type=int)
|
|
648
|
+
p.add_argument("-j", "--concurrency", type=int, default=4)
|
|
649
|
+
p.add_argument("--judge-model", help="a benchmark scored by a model: this judge instead of its official one")
|
|
650
|
+
p.add_argument("--simulator-model", help="a benchmark whose user a model plays: this model instead of its official one")
|
|
651
|
+
p.add_argument("--repeat", type=int, default=1, metavar="K",
|
|
652
|
+
help="run each unit K times, each an attempt of its own (the upstream's --num-trials, pass^k); default 1")
|
|
653
|
+
p.add_argument("--remove-images", action="store_true",
|
|
654
|
+
help="remove each image this run pulls or builds when its last unit ends (default: kept for the next run)")
|
|
655
|
+
p.add_argument("--name")
|
|
656
|
+
p.add_argument("--shard", metavar="I/N", help="only shard I of N of the units: by the unit id's hash, before --limit")
|
|
657
|
+
p.add_argument("--launch", help="the launch id every shard records (default: the day, the benchmark, the agent, the model)")
|
|
658
|
+
p.add_argument("--bench-option", action="append", metavar="NAME=VALUE",
|
|
659
|
+
help="one of the benchmark's own options (repeatable: a search, a mode); the rest keep their defaults")
|
|
660
|
+
p.add_argument("--price", action="append", metavar="MODEL=INPUT,OUTPUT[,CACHED,WRITE,WRITE_1H]",
|
|
661
|
+
help="this run's price for a model in USD per million tokens, over your price list (repeatable; nullius prices)")
|
|
662
|
+
p.add_argument("--max-usd", type=float, metavar="USD",
|
|
663
|
+
help="the run's spending cap: no call goes out once the spend plus the calls in flight would pass it")
|
|
664
|
+
p.add_argument("--max-usd-per-unit", type=float, metavar="USD",
|
|
665
|
+
help="each unit's own spending cap: a unit that reaches it is cut off there, and the run goes on")
|
|
666
|
+
v = sub.add_parser("prove", help="prove a benchmark: its controls, its units, no leak; record what passed")
|
|
667
|
+
v.add_argument("benchmark")
|
|
668
|
+
v.add_argument("--dataset", help="one dataset (default: each)")
|
|
669
|
+
v.add_argument("--limit", type=int, help="the first N units of each dataset (the record says so)")
|
|
670
|
+
v.add_argument("--provider", default="openai", choices=sorted(run.PROVIDERS), help="a judged benchmark's: the judge's provider")
|
|
671
|
+
v.add_argument("--judge-model", help="a judged benchmark: this judge instead of its official one")
|
|
672
|
+
v.add_argument("-j", "--concurrency", type=int, default=4)
|
|
673
|
+
v.add_argument("--jobs", default="jobs_standard", help="where the control jobs are written (default jobs_standard/)")
|
|
674
|
+
v.add_argument("--stamp", help="the time stamp the control jobs are named with (default: now)")
|
|
675
|
+
v.add_argument("--shard", metavar="I/N", help="the controls and the leak check on shard I of N only")
|
|
676
|
+
v.add_argument("--launch", help="the launch id the control jobs of every shard record")
|
|
677
|
+
v.add_argument("--remove-images", action="store_true",
|
|
678
|
+
help="remove each image this proof pulls or builds after its unit, so the oracle pulls it again (default: kept)")
|
|
679
|
+
g = sub.add_parser("merge", help="merge the shard jobs of one launch into one job")
|
|
680
|
+
g.add_argument("jobs", nargs="+", metavar="JOB_DIR")
|
|
681
|
+
g.add_argument("-o", "--out", required=True, help="the merged job's directory (its name is the job's)")
|
|
682
|
+
g.add_argument("--force", action="store_true", help="merge shards of different launches, versions, models or agents")
|
|
683
|
+
tr = sub.add_parser("traces", help="give past standard trials a trace, from the files they kept")
|
|
684
|
+
tr.add_argument("root", metavar="DIR", help="a job, or a folder of jobs")
|
|
685
|
+
tr.add_argument("--dry-run", action="store_true", help="say what it would write, and write nothing")
|
|
686
|
+
v.add_argument("--from-run", action="store_true",
|
|
687
|
+
help="started by Nullius: the key is OPENAI_API_KEY (and OPENAI_BASE_URL), saved settings are not read")
|
|
688
|
+
# Bootstrap decides whether settings may be read before these modules are
|
|
689
|
+
# imported. Keep --from-run exact here too, so --from-r cannot bypass it.
|
|
690
|
+
for parser in (ap, *sub.choices.values()):
|
|
691
|
+
parser.allow_abbrev = False
|
|
692
|
+
a = ap.parse_args(argv)
|
|
693
|
+
if a.cmd in ("run", "prove", "conform", "fidelity"):
|
|
694
|
+
_exit_on_term()
|
|
695
|
+
progress.ECHO = sys.stderr # each stage as a line (nullius/progress.py): stdout stays the job and its units
|
|
696
|
+
if not getattr(a, "from_run", False):
|
|
697
|
+
settings.load(ENV_FILE)
|
|
698
|
+
return {"list": cmd_list, "check": cmd_check, "fit": cmd_fit, "pins": cmd_pins, "new": cmd_new, "conform": cmd_conform, "run": cmd_run,
|
|
699
|
+
"fidelity": cmd_fidelity, "prove": cmd_prove, "merge": cmd_merge, "traces": cmd_traces}[a.cmd](a)
|
|
700
|
+
|
|
701
|
+
|
|
702
|
+
if __name__ == "__main__":
|
|
703
|
+
sys.exit(main())
|