nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/cli.py
ADDED
|
@@ -0,0 +1,85 @@
|
|
|
1
|
+
"""One entry point for the application and its existing command lines.
|
|
2
|
+
|
|
3
|
+
Dispatch is lazy: reading runs or asking for help never imports the web stack.
|
|
4
|
+
The original ``python -m nullius.<capability>.cli`` commands remain available.
|
|
5
|
+
"""
|
|
6
|
+
from __future__ import annotations
|
|
7
|
+
|
|
8
|
+
import argparse
|
|
9
|
+
import importlib
|
|
10
|
+
import importlib.metadata
|
|
11
|
+
import importlib.util
|
|
12
|
+
from pathlib import Path
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
COMMANDS = {
|
|
16
|
+
"serve": "open the web application",
|
|
17
|
+
"runs": "inspect run output",
|
|
18
|
+
"compare": "compare runs",
|
|
19
|
+
"import": "import ATIF or OpenTelemetry output",
|
|
20
|
+
"standard": "inspect and run standard definitions",
|
|
21
|
+
"workspace": "manage workspaces",
|
|
22
|
+
"paths": "show persistent storage locations",
|
|
23
|
+
"prices": "show and update model prices",
|
|
24
|
+
"migrate": "copy legacy storage into the user profile",
|
|
25
|
+
}
|
|
26
|
+
MODULES = {
|
|
27
|
+
"runs": "nullius.runs.cli",
|
|
28
|
+
"compare": "nullius.compare.cli",
|
|
29
|
+
"import": "nullius.importers.cli",
|
|
30
|
+
"standard": "nullius.standard.cli",
|
|
31
|
+
"workspace": "nullius.workspace_cli",
|
|
32
|
+
"paths": "nullius.paths",
|
|
33
|
+
"prices": "nullius.prices",
|
|
34
|
+
"migrate": "nullius.migrate",
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
def _serve(argv):
|
|
39
|
+
"""Import the web entry point only when the server is requested.
|
|
40
|
+
|
|
41
|
+
The wheel keeps the web layer and assets in ``_nullius_app``; a source
|
|
42
|
+
checkout keeps them beside the backend in ``app/``. Neither layout adds
|
|
43
|
+
a second implementation of ``nullius`` to the interpreter's path.
|
|
44
|
+
"""
|
|
45
|
+
if importlib.util.find_spec("_nullius_app") is not None:
|
|
46
|
+
module = importlib.import_module("_nullius_app.app")
|
|
47
|
+
else:
|
|
48
|
+
source = Path(__file__).resolve().parents[1] / "app.py"
|
|
49
|
+
spec = importlib.util.spec_from_file_location("_nullius_source_app", source)
|
|
50
|
+
module = importlib.util.module_from_spec(spec)
|
|
51
|
+
spec.loader.exec_module(module)
|
|
52
|
+
return module.main(argv)
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def main(argv=None) -> int:
|
|
56
|
+
parser = argparse.ArgumentParser(
|
|
57
|
+
prog="nullius", description="Bring anything, run it, prove it.",
|
|
58
|
+
epilog="\n".join(f" {name:<10} {help}" for name, help in COMMANDS.items()),
|
|
59
|
+
formatter_class=argparse.RawDescriptionHelpFormatter,
|
|
60
|
+
)
|
|
61
|
+
parser.add_argument("--version", action="store_true", help="show the installed version")
|
|
62
|
+
parser.add_argument("command", nargs="?", choices=COMMANDS)
|
|
63
|
+
parser.add_argument("arguments", nargs=argparse.REMAINDER, help=argparse.SUPPRESS)
|
|
64
|
+
args = parser.parse_args(argv)
|
|
65
|
+
if args.version:
|
|
66
|
+
try:
|
|
67
|
+
version = importlib.metadata.version("nullius-eval")
|
|
68
|
+
except importlib.metadata.PackageNotFoundError:
|
|
69
|
+
version = "source checkout"
|
|
70
|
+
print(f"nullius {version}")
|
|
71
|
+
return 0
|
|
72
|
+
if not args.command:
|
|
73
|
+
parser.print_help()
|
|
74
|
+
return 0
|
|
75
|
+
# Some command modules compute their storage locations at import time.
|
|
76
|
+
from nullius import paths, settings
|
|
77
|
+
if not (args.command == "standard" and "--from-run" in args.arguments):
|
|
78
|
+
try:
|
|
79
|
+
settings.load(paths.settings_file())
|
|
80
|
+
except ValueError as e:
|
|
81
|
+
if args.command != "paths":
|
|
82
|
+
parser.exit(2, f"{e} Fix {paths.settings_file()}.\n")
|
|
83
|
+
if args.command == "serve":
|
|
84
|
+
return _serve(args.arguments) or 0
|
|
85
|
+
return importlib.import_module(MODULES[args.command]).main(args.arguments) or 0
|
|
File without changes
|
nullius/compare/cli.py
ADDED
|
@@ -0,0 +1,265 @@
|
|
|
1
|
+
"""Compare runs from a shell: the totals /compare renders, whole or matched.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.compare.cli <root> <job a> <job b> [<job c>...]
|
|
4
|
+
PYTHONPATH=app python3 -m nullius.compare.cli <root> <job a> <job b> --matched
|
|
5
|
+
PYTHONPATH=app python3 -m nullius.compare.cli <root> <job a> <job b> --json
|
|
6
|
+
|
|
7
|
+
``<root>`` is what the other CLIs take: a Harbor job directory, a parent of
|
|
8
|
+
several, or an imported tree; ``parse.load_all`` walks it. The jobs are
|
|
9
|
+
resolved and the cohort built by ``nullius.compare.comparison`` — the page's
|
|
10
|
+
own computation, not a second one — so ``--matched`` keeps exactly the
|
|
11
|
+
trials the page keeps (the intersection of exact dataset/task identifiers,
|
|
12
|
+
every recorded attempt), and the totals printed per column are the ``Job``
|
|
13
|
+
properties the page's summary table reads over that cohort: trials, the
|
|
14
|
+
mean score and its breakdown when the benchmark scores, solved, the two
|
|
15
|
+
solve rates, infra errors, regressions, the token block, the
|
|
16
|
+
estimated cost and, for whole runs only, the wall-clock interval. A matched
|
|
17
|
+
cohort has no truthful interval, so none is printed for it — as on the page.
|
|
18
|
+
|
|
19
|
+
The task rows below the totals are the page's: one per dataset/task, the
|
|
20
|
+
outcome recorded in each column with its score, the attempts each column made, the rows
|
|
21
|
+
that differ first. A development build (source or unknown runtime) is
|
|
22
|
+
compared when named, with a note on stderr; the page asks for a tick.
|
|
23
|
+
|
|
24
|
+
Reading only: nothing is written, anywhere, and no socket is opened. The
|
|
25
|
+
exit codes are the contract in ``docs/cli.md``, shared by the CLIs.
|
|
26
|
+
"""
|
|
27
|
+
from __future__ import annotations
|
|
28
|
+
|
|
29
|
+
import argparse
|
|
30
|
+
import json
|
|
31
|
+
import os
|
|
32
|
+
import sys
|
|
33
|
+
from datetime import datetime
|
|
34
|
+
|
|
35
|
+
from nullius.compare import comparison
|
|
36
|
+
from nullius.runs import parse
|
|
37
|
+
|
|
38
|
+
EXIT_OK = 0
|
|
39
|
+
EXIT_NOT_FOUND = 3
|
|
40
|
+
EXIT_FAILED = 4
|
|
41
|
+
|
|
42
|
+
EPILOG = """\
|
|
43
|
+
examples:
|
|
44
|
+
PYTHONPATH=app python3 -m nullius.compare.cli jobs 2026-09-12__22-53-55 2026-09-13__09-10-11
|
|
45
|
+
PYTHONPATH=app python3 -m nullius.compare.cli jobs 2026-09-12__22-53-55 2026-09-13__09-10-11 --matched
|
|
46
|
+
PYTHONPATH=app python3 -m nullius.compare.cli jobs 2026-09-12__22-53-55 2026-09-13__09-10-11 --json
|
|
47
|
+
|
|
48
|
+
exit codes:
|
|
49
|
+
0 done
|
|
50
|
+
1 unexpected error (a traceback: report it)
|
|
51
|
+
2 usage: a bad flag, a missing argument, fewer than two different job ids
|
|
52
|
+
3 input not found: no such root, no job under it, no job of that id
|
|
53
|
+
4 the operation could not be completed: a job id that occurs more than once under the root
|
|
54
|
+
With --json, stdout is exactly one JSON document; everything else goes to stderr.
|
|
55
|
+
"""
|
|
56
|
+
|
|
57
|
+
# The totals the page's summary table reads off each column's Job, in its
|
|
58
|
+
# order. Each is a Job property over the cohort's trials — comparison.build
|
|
59
|
+
# replaced the trial list, the arithmetic stayed Job's.
|
|
60
|
+
TOTALS = (
|
|
61
|
+
("n_trials", "Trials"),
|
|
62
|
+
("mean_score", "Mean score"),
|
|
63
|
+
("n_resolved", "Solved"),
|
|
64
|
+
("raw_solve_rate", "Raw rate"),
|
|
65
|
+
("adjusted_solve_rate", "Adjusted rate"),
|
|
66
|
+
("n_infra", "Infra errors"),
|
|
67
|
+
("n_regressions", "Broke tests"),
|
|
68
|
+
("total_input_tokens", "Prompt size"),
|
|
69
|
+
("fresh_input_tokens", "Fresh prompt"),
|
|
70
|
+
("total_cache_tokens", "Cached"),
|
|
71
|
+
("cache_ratio", "Cached ratio"),
|
|
72
|
+
("total_output_tokens", "Output tokens"),
|
|
73
|
+
("total_cost_usd", "Cost (estimated)"),
|
|
74
|
+
("duration_min", "Run wall time"),
|
|
75
|
+
)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
def _stamp(value):
|
|
79
|
+
return value.isoformat() if isinstance(value, datetime) else value
|
|
80
|
+
|
|
81
|
+
|
|
82
|
+
def _ratio(part, whole):
|
|
83
|
+
return (part / whole) if (part and whole) else None
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def totals_of(job: parse.Job, cohort: str) -> dict:
|
|
87
|
+
"""The column's totals as the page's summary rows read them."""
|
|
88
|
+
doc = {
|
|
89
|
+
"n_trials": len(job.trials),
|
|
90
|
+
# A benchmark that scores (an F1, partial credit): the mean over every
|
|
91
|
+
# part scored, and by the field it breaks the score down by. A run
|
|
92
|
+
# with a score and no pass or fail has nothing solved to count.
|
|
93
|
+
"mean_score": job.mean_score,
|
|
94
|
+
"score_by_group": job.score_by_group,
|
|
95
|
+
"parts_by": job.parts_by,
|
|
96
|
+
"scored_only": job.scored_only,
|
|
97
|
+
"n_resolved": job.n_resolved,
|
|
98
|
+
"raw_solve_rate": job.raw_solve_rate,
|
|
99
|
+
"adjusted_solve_rate": job.adjusted_solve_rate,
|
|
100
|
+
"n_infra": job.n_infra,
|
|
101
|
+
"n_regressions": job.n_regressions,
|
|
102
|
+
"total_input_tokens": job.total_input_tokens,
|
|
103
|
+
"fresh_input_tokens": job.total_input_tokens - job.total_cache_tokens,
|
|
104
|
+
"total_cache_tokens": job.total_cache_tokens,
|
|
105
|
+
"cache_ratio": _ratio(job.total_cache_tokens, job.total_input_tokens),
|
|
106
|
+
"total_output_tokens": job.total_output_tokens,
|
|
107
|
+
"total_cost_usd": job.total_cost_usd,
|
|
108
|
+
}
|
|
109
|
+
if cohort == "whole":
|
|
110
|
+
# No truthful wall-clock interval exists for a subset of parallel trials.
|
|
111
|
+
doc["duration_min"] = job.duration_min
|
|
112
|
+
return doc
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def column(job: parse.Job, cohort: str) -> dict:
|
|
116
|
+
return {
|
|
117
|
+
"job_id": job.job_id, "dir": job.dir, "dataset": job.dataset,
|
|
118
|
+
"model": job.model_name, "agent": job.agent_label, "kind": job.kind,
|
|
119
|
+
"is_control": job.is_control,
|
|
120
|
+
"counts": job.counts, "totals": totals_of(job, cohort),
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def document(view: comparison.Comparison) -> dict:
|
|
125
|
+
"""The comparison as the page renders it: the columns with their totals
|
|
126
|
+
over the cohort, and the task rows."""
|
|
127
|
+
return {
|
|
128
|
+
"cohort": view.cohort,
|
|
129
|
+
"runs": [column(job, view.cohort) for job in view.jobs],
|
|
130
|
+
"shared_tasks": view.shared_tasks,
|
|
131
|
+
"excluded_trials": list(view.excluded_trials),
|
|
132
|
+
"repeated_trials": list(view.repeated_trials),
|
|
133
|
+
"datasets_match": view.datasets_match,
|
|
134
|
+
"datasets_known": view.datasets_known,
|
|
135
|
+
"n_differing": view.n_differing,
|
|
136
|
+
"tasks": [{
|
|
137
|
+
"dataset": row.dataset, "task": row.task, "differs": row.differs,
|
|
138
|
+
"shared": row.shared, "attempts": list(row.attempts),
|
|
139
|
+
"trials": [t.trial_name if t else None for t in row.trials],
|
|
140
|
+
"outcomes": [t.outcome if t else None for t in row.trials],
|
|
141
|
+
"scores": [t.score if t else None for t in row.trials],
|
|
142
|
+
} for row in view.tasks],
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _cell(key: str, value) -> str:
|
|
147
|
+
if value is None:
|
|
148
|
+
return "—"
|
|
149
|
+
if key in ("raw_solve_rate", "adjusted_solve_rate", "cache_ratio"):
|
|
150
|
+
return f"{100 * value:.0f}%"
|
|
151
|
+
if key == "total_cost_usd":
|
|
152
|
+
return f"${value:.4f}"
|
|
153
|
+
if key == "mean_score":
|
|
154
|
+
return f"{value:.2f}"
|
|
155
|
+
if key == "duration_min":
|
|
156
|
+
return f"{value:.0f}m"
|
|
157
|
+
if key in ("n_infra", "n_regressions") and not value:
|
|
158
|
+
return "—"
|
|
159
|
+
return str(value)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
def _print(view: comparison.Comparison) -> None:
|
|
163
|
+
doc = document(view)
|
|
164
|
+
columns = doc["runs"]
|
|
165
|
+
width = max(12, *(len(c["job_id"]) for c in columns))
|
|
166
|
+
print(f"{view.cohort} runs: " + " vs ".join(c["job_id"] for c in columns))
|
|
167
|
+
if view.cohort == "matched":
|
|
168
|
+
print(f" shared tasks {view.shared_tasks} · excluded trials "
|
|
169
|
+
+ ", ".join(str(n) for n in view.excluded_trials)
|
|
170
|
+
+ (" · repeated attempts " + ", ".join(str(n) for n in view.repeated_trials)
|
|
171
|
+
if any(view.repeated_trials) else ""))
|
|
172
|
+
if not view.datasets_match:
|
|
173
|
+
print(" note: the runs record different datasets; task rows are kept apart by dataset")
|
|
174
|
+
if not view.datasets_known:
|
|
175
|
+
print(" note: a run records no dataset; its tasks cannot be matched by name alone")
|
|
176
|
+
rows = []
|
|
177
|
+
for key, label in TOTALS:
|
|
178
|
+
if key not in columns[0]["totals"]:
|
|
179
|
+
continue
|
|
180
|
+
if key == "mean_score":
|
|
181
|
+
if all(c["totals"]["mean_score"] is None for c in columns):
|
|
182
|
+
continue
|
|
183
|
+
rows.append((label, [_cell(key, c["totals"][key]) for c in columns]))
|
|
184
|
+
by = next((c["totals"]["parts_by"] for c in columns if c["totals"]["parts_by"]), "")
|
|
185
|
+
groups = dict.fromkeys(g["group"] for c in columns for g in c["totals"]["score_by_group"])
|
|
186
|
+
for g in groups:
|
|
187
|
+
means = [next((x["mean"] for x in c["totals"]["score_by_group"] if x["group"] == g), None) for c in columns]
|
|
188
|
+
rows.append((f"{by.capitalize()} {g}", [_cell(key, m) for m in means]))
|
|
189
|
+
continue
|
|
190
|
+
if key == "n_resolved":
|
|
191
|
+
rows.append((label, ["—" if c["totals"]["scored_only"] else _cell(key, c["totals"][key]) for c in columns]))
|
|
192
|
+
continue
|
|
193
|
+
rows.append((label, [_cell(key, c["totals"][key]) for c in columns]))
|
|
194
|
+
label_width = max(len(label) for label, _ in rows)
|
|
195
|
+
print(f" {'':<{label_width}} " + " ".join(f"{c['job_id']:>{width}}" for c in columns))
|
|
196
|
+
for label, cells in rows:
|
|
197
|
+
print(f" {label:<{label_width}} " + " ".join(f"{cell:>{width}}" for cell in cells))
|
|
198
|
+
if not doc["tasks"]:
|
|
199
|
+
print(" no tasks in this comparison cohort")
|
|
200
|
+
return
|
|
201
|
+
print(f" tasks: {len(doc['tasks'])} · differing {view.n_differing}")
|
|
202
|
+
task_width = max(len(row["task"]) for row in doc["tasks"])
|
|
203
|
+
for row in doc["tasks"]:
|
|
204
|
+
cells = []
|
|
205
|
+
for outcome, score, attempts in zip(row["outcomes"], row["scores"], row["attempts"]):
|
|
206
|
+
cell = outcome or "—"
|
|
207
|
+
if score is not None:
|
|
208
|
+
cell += f" {score:.2f}"
|
|
209
|
+
if attempts > 1:
|
|
210
|
+
cell += f" ×{attempts}"
|
|
211
|
+
cells.append(cell)
|
|
212
|
+
mark = "≠" if row["differs"] else " "
|
|
213
|
+
print(f" {mark} {row['task']:<{task_width}} " + " ".join(f"{c:>{width}}" for c in cells))
|
|
214
|
+
|
|
215
|
+
|
|
216
|
+
def _emit(doc) -> None:
|
|
217
|
+
json.dump(doc, sys.stdout, ensure_ascii=False, indent=1, default=_stamp)
|
|
218
|
+
sys.stdout.write("\n")
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def main(argv=None) -> int:
|
|
222
|
+
ap = argparse.ArgumentParser(
|
|
223
|
+
prog="python3 -m nullius.compare.cli", description=__doc__.splitlines()[0],
|
|
224
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
225
|
+
ap.add_argument("root", metavar="ROOT", help="the root the jobs are under")
|
|
226
|
+
ap.add_argument("job_ids", metavar="JOB_ID", nargs="+",
|
|
227
|
+
help="two or more job ids, as `runs.cli list` prints them; one column each")
|
|
228
|
+
ap.add_argument("--matched", action="store_true",
|
|
229
|
+
help="the matched cohort: only the tasks every run attempted, by exact "
|
|
230
|
+
"dataset and task id (the page's cohort=matched); whole runs otherwise")
|
|
231
|
+
ap.add_argument("--json", action="store_true",
|
|
232
|
+
help="one JSON document: the columns with their totals and the task rows")
|
|
233
|
+
args = ap.parse_args(argv)
|
|
234
|
+
ids = list(dict.fromkeys(args.job_ids))
|
|
235
|
+
if len(ids) < 2:
|
|
236
|
+
ap.error("two different job ids are needed to compare")
|
|
237
|
+
if not os.path.isdir(args.root):
|
|
238
|
+
print(f"no such directory: {args.root}", file=sys.stderr)
|
|
239
|
+
return EXIT_NOT_FOUND
|
|
240
|
+
jobs = parse.load_all([args.root])
|
|
241
|
+
if not jobs:
|
|
242
|
+
print(f"no job directory found under {args.root}", file=sys.stderr)
|
|
243
|
+
return EXIT_NOT_FOUND
|
|
244
|
+
known = ", ".join(j.job_id for j in jobs)
|
|
245
|
+
for job_id in ids:
|
|
246
|
+
n = sum(1 for j in jobs if j.job_id == job_id)
|
|
247
|
+
if n == 0:
|
|
248
|
+
print(f"no job named {job_id!r} under {args.root} (jobs there: {known})", file=sys.stderr)
|
|
249
|
+
return EXIT_NOT_FOUND
|
|
250
|
+
if n > 1:
|
|
251
|
+
print(f"job id {job_id!r} occurs {n} times under {args.root}; "
|
|
252
|
+
"name a root where it is unique", file=sys.stderr)
|
|
253
|
+
return EXIT_FAILED
|
|
254
|
+
selected = comparison.resolve([comparison.RunRef("", job_id).value for job_id in ids],
|
|
255
|
+
{"": jobs}, {"": args.root})
|
|
256
|
+
view = comparison.build(selected, "matched" if args.matched else "whole")
|
|
257
|
+
if args.json:
|
|
258
|
+
_emit(document(view))
|
|
259
|
+
else:
|
|
260
|
+
_print(view)
|
|
261
|
+
return EXIT_OK
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
if __name__ == "__main__": # pragma: no cover - entry point
|
|
265
|
+
raise SystemExit(main())
|
|
@@ -0,0 +1,184 @@
|
|
|
1
|
+
"""Qualified run selections and honest task cohorts, with no I/O or Flask.
|
|
2
|
+
|
|
3
|
+
Run ids are unique only inside a workspace. Task ids are unique only inside a
|
|
4
|
+
recorded dataset. Keeping both identities explicit prevents a comparison from
|
|
5
|
+
quietly joining two unrelated rows merely because their short names match.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
from collections import Counter
|
|
11
|
+
from dataclasses import dataclass, replace
|
|
12
|
+
from typing import Mapping, Sequence
|
|
13
|
+
|
|
14
|
+
from nullius.runs.parse import Job, Trial
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
@dataclass(frozen=True)
|
|
18
|
+
class RunRef:
|
|
19
|
+
workspace: str
|
|
20
|
+
job_id: str
|
|
21
|
+
|
|
22
|
+
@property
|
|
23
|
+
def value(self) -> str:
|
|
24
|
+
return f"{self.workspace}:{self.job_id}"
|
|
25
|
+
|
|
26
|
+
@classmethod
|
|
27
|
+
def parse(cls, value: str) -> "RunRef":
|
|
28
|
+
name, separator, job_id = value.partition(":")
|
|
29
|
+
if not separator or not job_id:
|
|
30
|
+
raise ValueError("A run selection must include its workspace and run id.")
|
|
31
|
+
# Empty means the active source, for legacy selections without a saved
|
|
32
|
+
# workspace. Named workspaces use the same slug alphabet as workspace.py.
|
|
33
|
+
if name and (len(name) > 48 or any(c not in "abcdefghijklmnopqrstuvwxyz0123456789-" for c in name)):
|
|
34
|
+
raise ValueError("Invalid workspace in the run selection.")
|
|
35
|
+
return cls(name, job_id)
|
|
36
|
+
|
|
37
|
+
|
|
38
|
+
@dataclass(frozen=True)
|
|
39
|
+
class SelectedRun:
|
|
40
|
+
ref: RunRef
|
|
41
|
+
job: Job
|
|
42
|
+
workspace_label: str
|
|
43
|
+
|
|
44
|
+
@property
|
|
45
|
+
def value(self) -> str:
|
|
46
|
+
return self.ref.value
|
|
47
|
+
|
|
48
|
+
@property
|
|
49
|
+
def workspace(self) -> str:
|
|
50
|
+
return self.ref.workspace
|
|
51
|
+
|
|
52
|
+
|
|
53
|
+
def resolve(values: Sequence[str], jobs_by_workspace: Mapping[str, Sequence[Job]],
|
|
54
|
+
labels: Mapping[str, str] | None = None) -> tuple[SelectedRun, ...]:
|
|
55
|
+
"""Resolve references in request order; never substitute another workspace.
|
|
56
|
+
|
|
57
|
+
A duplicate selection is one column. Duplicate ids *inside* a workspace are
|
|
58
|
+
ambiguous and refused, while the same id in two workspaces is valid.
|
|
59
|
+
"""
|
|
60
|
+
labels = labels or {}
|
|
61
|
+
selected, seen = [], set()
|
|
62
|
+
for value in values:
|
|
63
|
+
ref = RunRef.parse(value)
|
|
64
|
+
if ref in seen:
|
|
65
|
+
continue
|
|
66
|
+
seen.add(ref)
|
|
67
|
+
if ref.workspace not in jobs_by_workspace:
|
|
68
|
+
raise ValueError(f"Workspace {ref.workspace or 'active source'} is unavailable.")
|
|
69
|
+
matches = [job for job in jobs_by_workspace[ref.workspace] if job.job_id == ref.job_id]
|
|
70
|
+
if not matches:
|
|
71
|
+
raise ValueError(f"Run {ref.job_id} is unavailable in {ref.workspace or 'the active source'}.")
|
|
72
|
+
if len(matches) > 1:
|
|
73
|
+
raise ValueError(f"Run id {ref.job_id} occurs more than once in this workspace.")
|
|
74
|
+
selected.append(SelectedRun(ref, matches[0], labels.get(ref.workspace, ref.workspace or "Active source")))
|
|
75
|
+
return tuple(selected)
|
|
76
|
+
|
|
77
|
+
|
|
78
|
+
@dataclass(frozen=True)
|
|
79
|
+
class TaskRow:
|
|
80
|
+
dataset: str
|
|
81
|
+
task: str
|
|
82
|
+
trials: tuple[Trial | None, ...]
|
|
83
|
+
attempts: tuple[int, ...]
|
|
84
|
+
|
|
85
|
+
@property
|
|
86
|
+
def differs(self) -> bool:
|
|
87
|
+
# a scored task differs by its score too: two readers that both scored a
|
|
88
|
+
# conversation are told apart by how much
|
|
89
|
+
return len({(trial.outcome, None if trial.score is None else round(trial.score, 3)) for trial in self.trials if trial}) > 1
|
|
90
|
+
|
|
91
|
+
@property
|
|
92
|
+
def shared(self) -> bool:
|
|
93
|
+
return all(self.trials)
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
@dataclass(frozen=True)
|
|
97
|
+
class Comparison:
|
|
98
|
+
selected: tuple[SelectedRun, ...]
|
|
99
|
+
cohort: str
|
|
100
|
+
jobs: tuple[Job, ...]
|
|
101
|
+
tasks: tuple[TaskRow, ...]
|
|
102
|
+
shared_tasks: int
|
|
103
|
+
excluded_trials: tuple[int, ...]
|
|
104
|
+
repeated_trials: tuple[int, ...]
|
|
105
|
+
datasets_match: bool
|
|
106
|
+
datasets_known: bool
|
|
107
|
+
|
|
108
|
+
@property
|
|
109
|
+
def references(self) -> list[str]:
|
|
110
|
+
return [selection.value for selection in self.selected]
|
|
111
|
+
|
|
112
|
+
@property
|
|
113
|
+
def n_differing(self) -> int:
|
|
114
|
+
return sum(row.differs for row in self.tasks)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def build(selected: Sequence[SelectedRun], cohort: str = "whole") -> Comparison:
|
|
118
|
+
"""Build a whole-run or exact shared-task view without changing any Job.
|
|
119
|
+
|
|
120
|
+
Matching uses the dataset string as recorded, without alias/path guessing.
|
|
121
|
+
Every attempt on a shared task remains in the rates, costs and token sums;
|
|
122
|
+
task rows retain the last indexed attempt, as the existing table does.
|
|
123
|
+
Unknown outcomes remain in the adjusted-rate denominator because Job owns
|
|
124
|
+
the arithmetic. A cohort cannot manufacture a new wall-clock run interval.
|
|
125
|
+
"""
|
|
126
|
+
if cohort not in {"whole", "matched"}:
|
|
127
|
+
raise ValueError("Choose whole runs or matched tasks as the comparison cohort.")
|
|
128
|
+
selected = tuple(selected)
|
|
129
|
+
known = all(selection.job.dataset for selection in selected)
|
|
130
|
+
datasets_match = len({selection.job.dataset for selection in selected}) <= 1
|
|
131
|
+
|
|
132
|
+
def key(selection, trial):
|
|
133
|
+
# Missing dataset identities cannot prove that identical short names
|
|
134
|
+
# refer to the same task. A per-column sentinel keeps their rows apart.
|
|
135
|
+
dataset = selection.job.dataset or f"[unidentified:{selection.value}]"
|
|
136
|
+
return dataset, trial.task_base
|
|
137
|
+
|
|
138
|
+
sets = [{key(selection, trial) for trial in selection.job.trials} for selection in selected]
|
|
139
|
+
common = set.intersection(*sets) if sets else set()
|
|
140
|
+
jobs, excluded, repeated, by_task = [], [], [], {}
|
|
141
|
+
for column, selection in enumerate(selected):
|
|
142
|
+
original = selection.job
|
|
143
|
+
trials = [trial for trial in original.trials
|
|
144
|
+
if cohort == "whole" or key(selection, trial) in common]
|
|
145
|
+
# Aggregate properties derive from trials; copied clocks are suppressed
|
|
146
|
+
# for matched views because their interval still belongs to the full run.
|
|
147
|
+
view = replace(original, trials=trials,
|
|
148
|
+
**({"started_at": None, "finished_at": None} if cohort == "matched" else {}))
|
|
149
|
+
jobs.append(view)
|
|
150
|
+
excluded.append(len(original.trials) - len(trials))
|
|
151
|
+
counts = Counter(key(selection, trial) for trial in trials)
|
|
152
|
+
repeated.append(sum(count - 1 for count in counts.values()))
|
|
153
|
+
for trial in trials:
|
|
154
|
+
task_key = key(selection, trial)
|
|
155
|
+
row, attempts = by_task.setdefault(task_key, ([None] * len(selected), [0] * len(selected)))
|
|
156
|
+
row[column] = trial
|
|
157
|
+
attempts[column] += 1
|
|
158
|
+
|
|
159
|
+
rows = [TaskRow(dataset, task, tuple(trials), tuple(attempts))
|
|
160
|
+
for (dataset, task), (trials, attempts) in by_task.items()]
|
|
161
|
+
rows.sort(key=lambda row: (not row.differs, row.dataset, row.task))
|
|
162
|
+
return Comparison(selected, cohort, tuple(jobs), tuple(rows), len(common),
|
|
163
|
+
tuple(excluded), tuple(repeated), datasets_match, known)
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
# Two sides' scores closer than this are a tie. Half a rubric level: the
|
|
168
|
+
# judge's own spread on one state has been measured at 0.06, so anything
|
|
169
|
+
# under a level's half is within what a re-ask could flip. Change it here;
|
|
170
|
+
# nothing asks the model to compare.
|
|
171
|
+
TIE_MARGIN = 0.5
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def better_of(score_a: float | None, score_b: float | None, margin: float = TIE_MARGIN) -> str | None:
|
|
175
|
+
"""'A', 'B' or 'same' from two independent scores on one rubric, the
|
|
176
|
+
higher side winning by at least `margin`; None until both are scored.
|
|
177
|
+
The scores are the judge's expectations over the rubric's levels,
|
|
178
|
+
compared as given - never a magnitude read between two levels.
|
|
179
|
+
Symmetric by construction: swapping the sides mirrors the answer."""
|
|
180
|
+
if score_a is None or score_b is None:
|
|
181
|
+
return None
|
|
182
|
+
if abs(score_a - score_b) < margin:
|
|
183
|
+
return "same"
|
|
184
|
+
return "A" if score_a > score_b else "B"
|