nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,373 @@
|
|
|
1
|
+
"""A benchmark's proofs: its controls behave, its units are stable, nothing leaks.
|
|
2
|
+
|
|
3
|
+
An agent or a capability proves its ports by kits (``conform``); a benchmark
|
|
4
|
+
proves itself, per dataset, by four checks - what ``cli prove`` runs and
|
|
5
|
+
writes into the benchmark's ``proven``:
|
|
6
|
+
|
|
7
|
+
- ``nop``, the null control: no agent, an empty answer, the workspace as
|
|
8
|
+
setup left it. Nothing should pass - every score at the floor - up to the
|
|
9
|
+
bound the control declares (``controls.nop.expect.at_most``, 0 by default).
|
|
10
|
+
- ``oracle``, the reference control: the driver's ``oracle`` answer, and
|
|
11
|
+
what it writes into the workspace. Everything should pass - every score at
|
|
12
|
+
the ceiling - down to ``controls.oracle.expect.at_least`` (1 by default; a
|
|
13
|
+
judged benchmark may declare less, the judge being a model).
|
|
14
|
+
- ``units``: two ``units`` calls, each from a driver of its own, give the
|
|
15
|
+
same ids; the ids are unique, plain enough to name a trial, as many as the
|
|
16
|
+
dataset says it holds.
|
|
17
|
+
- ``leak``: after ``setup``, the reference is not already there to read. For
|
|
18
|
+
each unit the driver sets a workspace up, the oracle is run on a copy of
|
|
19
|
+
it, and what the oracle gives - its answer, and the lines it writes into
|
|
20
|
+
files - is looked for in the prompt (a capability benchmark's: its recall
|
|
21
|
+
questions, since the sessions it adds are meant to hold the evidence) and
|
|
22
|
+
in every file setup wrote. A text shorter than ``SHORT`` characters (a
|
|
23
|
+
number, a name) is not looked for - it would be found by chance - and the
|
|
24
|
+
record says how many were skipped. A benchmark whose questions may name
|
|
25
|
+
their answer among choices ("A or B?") says so (``controls.oracle.in_prompt``):
|
|
26
|
+
those units are counted and named in the record, not failed.
|
|
27
|
+
|
|
28
|
+
The controls are real runs, jobs like any other (named
|
|
29
|
+
``<stamp>__<benchmark>__<dataset>__prove-nop`` and ``…-oracle``, so the runs
|
|
30
|
+
pages pair them as a launch's controls); the units and leak checks talk to
|
|
31
|
+
the driver directly and leave nothing behind. A control calls no model, so
|
|
32
|
+
it needs no key - except a judged benchmark's, whose judge still grades.
|
|
33
|
+
|
|
34
|
+
A check that passes is recorded with its date, its counts and its job; one
|
|
35
|
+
that fails is taken back, as ``conform`` takes back a port - what it reached kept
|
|
36
|
+
under ``failed``, for a page to say why. A check run on
|
|
37
|
+
fewer units (``--limit``) does not replace one that covered more. A
|
|
38
|
+
benchmark whose units may not be redistributed (``units.redistribute``) is
|
|
39
|
+
recorded without what names them: no ``in_prompt.first``, no problems. Parity
|
|
40
|
+
with the authors' own scorer is not checked yet (``docs/standard.md``).
|
|
41
|
+
"""
|
|
42
|
+
from __future__ import annotations
|
|
43
|
+
|
|
44
|
+
import os
|
|
45
|
+
import re
|
|
46
|
+
import shutil
|
|
47
|
+
import tempfile
|
|
48
|
+
import time
|
|
49
|
+
from typing import Callable, Dict, List, Optional
|
|
50
|
+
|
|
51
|
+
from nullius.runs import parse
|
|
52
|
+
from nullius.standard import definitions, install, jsonl, run, shard as shards
|
|
53
|
+
|
|
54
|
+
STAMP = "%Y-%m-%d__%H-%M-%S"
|
|
55
|
+
SHORT = 12 # characters: a reference shorter than this is not looked for
|
|
56
|
+
CHECKS = ("units", "leak", "nop", "oracle")
|
|
57
|
+
_PLAIN_ID = re.compile(r"^[^\s\x00-\x1f]{1,1000}$") # whole in result.json; its trial folder takes the first 60 and the unit number
|
|
58
|
+
_MAX_FILE = 1 << 20 # a file setup wrote is read up to this size
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def job_name(stamp: str, bench: str, dataset: str, control: str, shard: Optional[tuple] = None) -> str:
|
|
62
|
+
return f"{stamp}__{bench}__{dataset}__prove-{control}" + (f"__shard-{shard[0]}of{shard[1]}" if shard else "")
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class _Driver:
|
|
66
|
+
"""The benchmark's driver, started as a run starts it, for the checks
|
|
67
|
+
that talk to it directly (units, leak)."""
|
|
68
|
+
|
|
69
|
+
def __init__(self, bench: dict, ds: dict, work: str, licences=None):
|
|
70
|
+
self.up = install.upstream(bench)
|
|
71
|
+
self.data = install.dataset(bench, ds, self.up)
|
|
72
|
+
install.driver_packages(bench) # its driver.install, once: start() then finds it
|
|
73
|
+
self.bench, self.ds, self.work = bench, ds, work
|
|
74
|
+
self.licences = licences if licences is not None else run.Licences(bench)
|
|
75
|
+
|
|
76
|
+
def start(self, n: int) -> jsonl.Proc:
|
|
77
|
+
home = os.path.join(self.work, f"driver-home-{n}")
|
|
78
|
+
os.makedirs(home, exist_ok=True)
|
|
79
|
+
cmd, extra = install.driver_argv(self.bench)
|
|
80
|
+
credentials = run.driver_secrets(self.bench, self.licences)
|
|
81
|
+
env = run.base_env(home, {**extra, **credentials, "NULLIUS_UPSTREAM": self.up or "", "NULLIUS_DATASET": self.data or "",
|
|
82
|
+
"NULLIUS_DATASET_ID": self.ds["id"], "PYTHONPATH": run.driver_path(), "PYTHONDONTWRITEBYTECODE": "1",
|
|
83
|
+
"NULLIUS_VERSION": str(self.bench.get("_version") or ""),
|
|
84
|
+
**run.option_env(definitions.bench_options_of(self.bench, self.ds["id"])[0])}) # the options at their defaults
|
|
85
|
+
return jsonl.Proc(cmd, self.bench["_here"], env, os.path.join(self.work, f"driver-{n}.log")).start()
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
def units_check(driver: _Driver, ds: dict) -> dict:
|
|
89
|
+
"""Two drivers, one ``units`` call each: the same ids, unique, plain."""
|
|
90
|
+
got = []
|
|
91
|
+
for n in (1, 2):
|
|
92
|
+
p = driver.start(n)
|
|
93
|
+
try:
|
|
94
|
+
got.append(p.request("units", dataset=ds["id"], timeout=300).get("units"))
|
|
95
|
+
except jsonl.Failed as e:
|
|
96
|
+
return {"ok": False, "units": 0, "problems": [str(e)], "ids": []}
|
|
97
|
+
finally:
|
|
98
|
+
p.close()
|
|
99
|
+
a, b = got
|
|
100
|
+
problems = []
|
|
101
|
+
if not isinstance(a, list) or not a:
|
|
102
|
+
return {"ok": False, "units": 0, "problems": ["units: no ids"], "ids": []}
|
|
103
|
+
if a != b:
|
|
104
|
+
problems.append("two calls gave different ids" + ("" if len(a) == len(b) else f" ({len(a)}, then {len(b)})"))
|
|
105
|
+
bad = [u for u in a if not isinstance(u, str) or not _PLAIN_ID.match(u)]
|
|
106
|
+
if bad:
|
|
107
|
+
problems.append(f"{len(bad)} id(s) cannot name a trial (a string, no spaces, at most 1000 characters): {bad[:3]!r}")
|
|
108
|
+
names = [str(u).replace("/", "-") for u in a] # what the pages read as a task's name (job.write_trial)
|
|
109
|
+
if len(set(names)) != len(names):
|
|
110
|
+
dup = sorted({x for x in names if names.count(x) > 1})
|
|
111
|
+
problems.append(f"{len(names) - len(set(names))} id(s) repeat: {dup[:3]!r}")
|
|
112
|
+
if isinstance(ds.get("units"), int) and ds["units"] != len(a):
|
|
113
|
+
problems.append(f"the dataset says {ds['units']} units, the driver gives {len(a)}")
|
|
114
|
+
return {"ok": not problems, "units": len(a), "problems": problems, "ids": [str(u) for u in a],
|
|
115
|
+
"passed": f"{len(a)} ids, the same twice, unique" if not problems else ""}
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _files(root: str) -> Dict[str, bytes]:
|
|
119
|
+
out = {}
|
|
120
|
+
for d, dirs, files in os.walk(root):
|
|
121
|
+
dirs[:] = [x for x in dirs if x != ".git"]
|
|
122
|
+
for f in files:
|
|
123
|
+
p = os.path.join(d, f)
|
|
124
|
+
try:
|
|
125
|
+
if os.path.islink(p) or os.path.getsize(p) > _MAX_FILE:
|
|
126
|
+
continue
|
|
127
|
+
with open(p, "rb") as fh:
|
|
128
|
+
out[os.path.relpath(p, root)] = fh.read()
|
|
129
|
+
except OSError:
|
|
130
|
+
continue
|
|
131
|
+
return out
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def _flat(text: str) -> str:
|
|
135
|
+
return " ".join(str(text).split())
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def leak_check(driver: _Driver, units: List[str], family: str, in_prompt: str = "") -> dict:
|
|
139
|
+
"""For each unit: set up, run the oracle on a copy, and look for what it
|
|
140
|
+
gave in what setup had already shown. ``in_prompt``: the benchmark's
|
|
141
|
+
reason a prompt may name its own answer (``controls.oracle.in_prompt``,
|
|
142
|
+
a choice question offering it among others); such a unit is counted and
|
|
143
|
+
named, not failed. Found in a file setup wrote, it is always a leak."""
|
|
144
|
+
p = driver.start(0)
|
|
145
|
+
looked, short, nothing, leaks, failed, bad, named = 0, 0, 0, [], [], set(), []
|
|
146
|
+
try:
|
|
147
|
+
for i, unit in enumerate(units):
|
|
148
|
+
a, b = os.path.join(driver.work, "leak", f"{i}", "a"), os.path.join(driver.work, "leak", f"{i}", "b")
|
|
149
|
+
os.makedirs(a)
|
|
150
|
+
try:
|
|
151
|
+
s = p.request("setup", unit=unit, dir=a, timeout=600)
|
|
152
|
+
before = _files(a)
|
|
153
|
+
shutil.copytree(a, b)
|
|
154
|
+
o = p.request("oracle", unit=unit, dir=b, timeout=600)
|
|
155
|
+
except jsonl.Failed as e:
|
|
156
|
+
failed.append(f"{unit}: {e}")
|
|
157
|
+
bad.add(unit)
|
|
158
|
+
continue
|
|
159
|
+
after = _files(b)
|
|
160
|
+
if family == "capability":
|
|
161
|
+
shown = " ".join(q.get("question", "") for q in (s.get("memory") or {}).get("recall") or [])
|
|
162
|
+
else:
|
|
163
|
+
shown = s.get("prompt") or ""
|
|
164
|
+
shown = _flat(shown)
|
|
165
|
+
files = _flat("\n".join(x.decode("utf-8", "replace") for x in before.values()))
|
|
166
|
+
needles = []
|
|
167
|
+
if str(o.get("answer") or "").strip():
|
|
168
|
+
needles.append(("the answer", str(o["answer"])))
|
|
169
|
+
for path, new in after.items():
|
|
170
|
+
old = before.get(path)
|
|
171
|
+
if old == new:
|
|
172
|
+
continue
|
|
173
|
+
kept = set(old.decode("utf-8", "replace").splitlines()) if old is not None else set()
|
|
174
|
+
added = "\n".join(l for l in new.decode("utf-8", "replace").splitlines() if l not in kept)
|
|
175
|
+
if added.strip():
|
|
176
|
+
needles.append((path, added))
|
|
177
|
+
if not needles:
|
|
178
|
+
nothing += 1
|
|
179
|
+
for what, text in needles:
|
|
180
|
+
flat = _flat(text)
|
|
181
|
+
if len(flat) < SHORT:
|
|
182
|
+
short += 1
|
|
183
|
+
continue
|
|
184
|
+
looked += 1
|
|
185
|
+
if flat in files or (flat in shown and not in_prompt):
|
|
186
|
+
leaks.append(f"{unit}: {what} is already there after setup, in {'a file' if flat in files else 'the prompt'}")
|
|
187
|
+
bad.add(unit)
|
|
188
|
+
elif flat in shown:
|
|
189
|
+
named.append(unit)
|
|
190
|
+
shutil.rmtree(os.path.dirname(a), ignore_errors=True)
|
|
191
|
+
finally:
|
|
192
|
+
p.close()
|
|
193
|
+
problems = leaks + [f"could not check {f}" for f in failed]
|
|
194
|
+
rec = {"ok": not problems, "units": len(units), "looked_for": looked, "short": short, "problems": problems,
|
|
195
|
+
"passed": f"{len(units) - len(bad)}/{len(units)}"}
|
|
196
|
+
notes = []
|
|
197
|
+
if short:
|
|
198
|
+
notes.append(f"{short} reference(s) under {SHORT} characters not looked for: too short to tell a leak from chance")
|
|
199
|
+
if nothing:
|
|
200
|
+
notes.append(f"{nothing} unit(s) whose oracle gave nothing to look for")
|
|
201
|
+
if named:
|
|
202
|
+
rec["in_prompt"] = {"units": len(named), "first": named[:5], "why": in_prompt}
|
|
203
|
+
notes.append(f"{len(named)} prompt(s) name their answer, as the benchmark says they may: {in_prompt}")
|
|
204
|
+
if notes:
|
|
205
|
+
rec["note"] = "; ".join(notes)
|
|
206
|
+
return rec
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def control_check(bench: dict, ds: dict, which: str, *, limit: Optional[int], jobs_root: str, stamp: str,
|
|
210
|
+
provider: str, model: str, judge_model: Optional[str], env, concurrency: int,
|
|
211
|
+
say: Callable[[str], None], shard: Optional[tuple] = None, launch: Optional[str] = None,
|
|
212
|
+
licences=None, remove_images: bool = False) -> dict:
|
|
213
|
+
"""Run one control over the dataset (or its first ``limit`` units, of
|
|
214
|
+
``shard`` when given) and hold it to its bound: the pass rate, or the mean score."""
|
|
215
|
+
other = definitions.control(bench, which)
|
|
216
|
+
job_dir = run.run(bench, other, model=model, provider=provider, dataset=ds["id"], limit=limit, concurrency=concurrency,
|
|
217
|
+
jobs_root=jobs_root, name=job_name(stamp, definitions.job_word(bench), ds["id"], which, shard), say=lambda s: say(" " + s),
|
|
218
|
+
env=env, judge_model=judge_model, shard=shard, launch=launch, licences=licences,
|
|
219
|
+
remove_images=remove_images)
|
|
220
|
+
j = parse.parse_job(job_dir)
|
|
221
|
+
n = len(j.trials)
|
|
222
|
+
errored = sum(1 for t in j.trials if t.exception_type)
|
|
223
|
+
if bench["score"]["kind"] == "pass":
|
|
224
|
+
top = sum(1 for t in j.trials if t.reward == 1.0)
|
|
225
|
+
value = top / n if n else 0.0
|
|
226
|
+
else:
|
|
227
|
+
scores = [t.score for t in j.trials if t.score is not None]
|
|
228
|
+
top = sum(1 for s in scores if s >= 1.0)
|
|
229
|
+
value = sum(scores) / len(scores) if scores else 0.0
|
|
230
|
+
(bound, expected), = other["expect"].items()
|
|
231
|
+
within = value <= expected if bound == "at_most" else value >= expected
|
|
232
|
+
problems = []
|
|
233
|
+
if not n:
|
|
234
|
+
problems.append("no unit ran")
|
|
235
|
+
if errored:
|
|
236
|
+
problems.append(f"{errored} unit(s) ended in an error before the score: read the job's trials")
|
|
237
|
+
if n and not within:
|
|
238
|
+
problems.append(f"{'pass rate' if bench['score']['kind'] == 'pass' else 'mean score'} {value:.4f}, expected {bound.replace('_', ' ')} {expected}")
|
|
239
|
+
rec = {"ok": not problems, "passed": f"{top}/{n}", "value": round(value, 4), "expect": {bound: expected},
|
|
240
|
+
"units": n, "job": os.path.basename(job_dir), "source": other["source"], "problems": problems}
|
|
241
|
+
if judge_model or (bench["score"].get("judge") or {}).get("model"):
|
|
242
|
+
rec["judge"] = judge_model or bench["score"]["judge"]["model"]
|
|
243
|
+
return rec
|
|
244
|
+
|
|
245
|
+
|
|
246
|
+
def prove(bench: dict, *, dataset: Optional[str] = None, limit: Optional[int] = None, jobs_root: str = "jobs",
|
|
247
|
+
provider: str = "openai", model: str = "", judge_model: Optional[str] = None, stamp: Optional[str] = None,
|
|
248
|
+
env=None, concurrency: int = 4, write: bool = True, say: Callable[[str], None] = print,
|
|
249
|
+
shard: Optional[tuple] = None, launch: Optional[str] = None, licences=None,
|
|
250
|
+
remove_images: bool = False) -> Dict[str, dict]:
|
|
251
|
+
"""Every check on every dataset (or ``dataset``); with ``write``, what
|
|
252
|
+
passed is recorded in the benchmark's ``proven``. ``shard`` (I, N): the
|
|
253
|
+
leak check and the controls cover that shard's units only (``shard``),
|
|
254
|
+
the units check all of them, and each record says which shard.
|
|
255
|
+
``remove_images``: each unit's own image is removed after it, so the
|
|
256
|
+
oracle pulls it again (by default they stay). Returns
|
|
257
|
+
{dataset: {check: record}}, each record with ``ok`` and its ``problems``."""
|
|
258
|
+
env = env if env is not None else os.environ
|
|
259
|
+
stamp = stamp or time.strftime(STAMP)
|
|
260
|
+
sets = [d for d in bench["datasets"] if dataset in (None, d["id"])]
|
|
261
|
+
if not sets:
|
|
262
|
+
raise run.Refused(f"{bench['label']} has no dataset {dataset!r}: {[d['id'] for d in bench['datasets']]}")
|
|
263
|
+
declared = [c for c in definitions.CONTROLS if c in (bench.get("controls") or {})]
|
|
264
|
+
if not declared:
|
|
265
|
+
say(f"{bench['label']} declares no controls: only its units and nothing else can be checked")
|
|
266
|
+
got: Dict[str, dict] = {}
|
|
267
|
+
for ds in sets:
|
|
268
|
+
say(f"{bench['key']}/{ds['id']}:")
|
|
269
|
+
work = tempfile.mkdtemp(prefix="prove-")
|
|
270
|
+
checks: Dict[str, dict] = {}
|
|
271
|
+
try:
|
|
272
|
+
try:
|
|
273
|
+
driver = _Driver(bench, ds, work, licences=licences)
|
|
274
|
+
except install.InstallFailed as e:
|
|
275
|
+
raise run.Refused(f"could not install what the proof needs: {e}")
|
|
276
|
+
checks["units"] = units_check(driver, ds)
|
|
277
|
+
ids = checks["units"].pop("ids")
|
|
278
|
+
if shard and shard[1] > 1:
|
|
279
|
+
ids = [u for _, u in shards.select(ids, *shard)]
|
|
280
|
+
ids = ids[:limit] if limit else ids
|
|
281
|
+
_say(say, "units", checks["units"])
|
|
282
|
+
if "oracle" in declared and ids:
|
|
283
|
+
checks["leak"] = leak_check(driver, ids, bench["family"], bench["controls"]["oracle"].get("in_prompt") or "")
|
|
284
|
+
_say(say, "leak", checks["leak"])
|
|
285
|
+
finally:
|
|
286
|
+
shutil.rmtree(work, ignore_errors=True)
|
|
287
|
+
for c in declared:
|
|
288
|
+
checks[c] = control_check(bench, ds, c, limit=limit, jobs_root=jobs_root, stamp=stamp, provider=provider,
|
|
289
|
+
model=model, judge_model=judge_model, env=env, concurrency=concurrency, say=say,
|
|
290
|
+
shard=shard if shard and shard[1] > 1 else None, launch=launch, licences=licences,
|
|
291
|
+
remove_images=remove_images)
|
|
292
|
+
_say(say, c, checks[c])
|
|
293
|
+
total = ds.get("units") if isinstance(ds.get("units"), int) else checks["units"]["units"]
|
|
294
|
+
for name, r in checks.items():
|
|
295
|
+
r["on"] = time.strftime("%Y-%m-%d")
|
|
296
|
+
if limit and name != "units" and limit < total:
|
|
297
|
+
r["limit"] = limit
|
|
298
|
+
if shard and shard[1] > 1 and name != "units":
|
|
299
|
+
r["shard"] = f"{shard[0]}/{shard[1]}"
|
|
300
|
+
got[ds["id"]] = checks
|
|
301
|
+
if write:
|
|
302
|
+
record(bench, got)
|
|
303
|
+
return got
|
|
304
|
+
|
|
305
|
+
|
|
306
|
+
def _say(say, name, r):
|
|
307
|
+
say(f" {name}: {'passed' if r['ok'] else 'FAILED'} {r.get('passed', '')}".rstrip()
|
|
308
|
+
+ "".join(f"\n {p}" for p in r.get("problems") or []) + (f"\n ({r['note']})" if r.get("note") else ""))
|
|
309
|
+
|
|
310
|
+
|
|
311
|
+
def record(bench: dict, got: Dict[str, dict]) -> None:
|
|
312
|
+
"""Write what passed into ``bench``'s ``proven`` and save it: a check that
|
|
313
|
+
passed replaces its record unless that one covered more units; one that
|
|
314
|
+
failed is taken back. The failure itself is kept under ``failed`` - what
|
|
315
|
+
it reached, against what bound, its job - so a page can say "0.9955 < 1"
|
|
316
|
+
rather than only "not proven"; the next pass of that check clears it.
|
|
317
|
+
Units that may not be redistributed are never named (:func:`_unnamed`)."""
|
|
318
|
+
if (bench.get("units") or {}).get("redistribute") is False:
|
|
319
|
+
got = {ds: {name: _unnamed(r) for name, r in checks.items()} for ds, checks in got.items()}
|
|
320
|
+
definitions.save(bench, lambda cur: cur.update(proven=_merged(cur.get("proven") or {}, got)))
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _unnamed(r: dict) -> dict:
|
|
324
|
+
"""A check's record without the units it names: the leak check's first
|
|
325
|
+
prompts that name their answer, and a failed check's problems."""
|
|
326
|
+
r = dict(r, problems=[])
|
|
327
|
+
if "in_prompt" in r:
|
|
328
|
+
r["in_prompt"] = {k: v for k, v in r["in_prompt"].items() if k != "first"}
|
|
329
|
+
return r
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
def _merged(proven: dict, got: Dict[str, dict]) -> dict:
|
|
333
|
+
proven = dict(proven)
|
|
334
|
+
for ds, checks in got.items():
|
|
335
|
+
cur = dict(proven.get(ds) or {})
|
|
336
|
+
failed = dict(cur.pop("failed", None) or {})
|
|
337
|
+
for name, r in checks.items():
|
|
338
|
+
if not r["ok"]:
|
|
339
|
+
cur.pop(name, None)
|
|
340
|
+
failed[name] = {k: r[k] for k in ("passed", "value", "expect", "units", "job", "on", "limit", "shard") if k in r}
|
|
341
|
+
failed[name]["problems"] = list(r.get("problems") or [])[:3]
|
|
342
|
+
continue
|
|
343
|
+
failed.pop(name, None)
|
|
344
|
+
old = cur.get(name) or {}
|
|
345
|
+
if old and old.get("units", 0) > r.get("units", 0):
|
|
346
|
+
continue
|
|
347
|
+
cur[name] = {k: v for k, v in r.items() if k not in ("ok", "problems")}
|
|
348
|
+
if cur or failed:
|
|
349
|
+
proven[ds] = {**{k: cur[k] for k in CHECKS if k in cur}, **({"failed": failed} if failed else {})}
|
|
350
|
+
else:
|
|
351
|
+
proven.pop(ds, None)
|
|
352
|
+
return proven
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def status(bench: dict) -> Dict[str, Dict[str, dict]]:
|
|
356
|
+
"""Per dataset, each control the benchmark declares - its source, where it
|
|
357
|
+
comes from, its bound - and whether it is proven on that dataset, with the
|
|
358
|
+
record; then the units and leak checks the same way. ``failed``: the last
|
|
359
|
+
attempt that did not pass, while none has since. What a page shows."""
|
|
360
|
+
out = {}
|
|
361
|
+
controls = bench.get("controls") or {}
|
|
362
|
+
for ds in bench["datasets"]:
|
|
363
|
+
rec = (bench.get("proven") or {}).get(ds["id"]) or {}
|
|
364
|
+
failed = rec.get("failed") if isinstance(rec.get("failed"), dict) else {}
|
|
365
|
+
row = {}
|
|
366
|
+
for c in definitions.CONTROLS:
|
|
367
|
+
if c in controls:
|
|
368
|
+
d = definitions.control(bench, c)
|
|
369
|
+
row[c] = {"source": d["source"], "from": d["from"], "expect": d["expect"], "proven": rec.get(c), "failed": failed.get(c)}
|
|
370
|
+
for c in ("units", "leak"):
|
|
371
|
+
row[c] = {"proven": rec.get(c), "failed": failed.get(c)}
|
|
372
|
+
out[ds["id"]] = row
|
|
373
|
+
return out
|