nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/runs/cli.py
ADDED
|
@@ -0,0 +1,316 @@
|
|
|
1
|
+
"""Read the runs under a job root from a shell: what /runs and /job/<id> show.
|
|
2
|
+
|
|
3
|
+
PYTHONPATH=app python3 -m nullius.runs.cli list <root>... # job ids, one per line
|
|
4
|
+
PYTHONPATH=app python3 -m nullius.runs.cli list <root> --json # the runs table
|
|
5
|
+
PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> # the trials and their outcomes
|
|
6
|
+
PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> --json
|
|
7
|
+
PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> <trial> # one trial: what /trial/<job>/<trial> shows
|
|
8
|
+
PYTHONPATH=app python3 -m nullius.runs.cli show <root> <job id> <trial> --json
|
|
9
|
+
|
|
10
|
+
``<root>`` is anything ``app.py`` accepts as a job root: a Harbor job
|
|
11
|
+
directory, a parent of several, or a tree the import CLI has already
|
|
12
|
+
converted. The same reader the pages use (``parse.load_all``) walks it, so a
|
|
13
|
+
job id printed here is the one ``/job/<id>`` opens, and the outcomes are the
|
|
14
|
+
ones its table shows — from the files on disk, never from a workspace cache.
|
|
15
|
+
With a trial name, ``show`` reads that trial the way the trial page does —
|
|
16
|
+
the outcome and checks, the usage and cost, the exception, and the steps of
|
|
17
|
+
``agent/trajectory.json`` through ``traces.trajectory`` (already redacted on
|
|
18
|
+
the way in; the calls' arguments and observations, which can be the whole
|
|
19
|
+
patch, are left to the page's conversation browser).
|
|
20
|
+
|
|
21
|
+
Reading only: nothing is written, anywhere, and no socket is opened. The
|
|
22
|
+
exit codes are the contract in ``docs/cli.md``, shared by every CLI here.
|
|
23
|
+
"""
|
|
24
|
+
from __future__ import annotations
|
|
25
|
+
|
|
26
|
+
import argparse
|
|
27
|
+
import json
|
|
28
|
+
import os
|
|
29
|
+
import sys
|
|
30
|
+
from dataclasses import asdict
|
|
31
|
+
from datetime import datetime
|
|
32
|
+
|
|
33
|
+
from nullius.runs import parse
|
|
34
|
+
from nullius.traces import trajectory
|
|
35
|
+
|
|
36
|
+
# Exit codes — docs/cli.md is the contract; the other two CLIs use the same
|
|
37
|
+
# numbers. 1 is left to Python itself: an uncaught exception is a bug, and
|
|
38
|
+
# its traceback is the report.
|
|
39
|
+
EXIT_OK = 0
|
|
40
|
+
EXIT_NOT_FOUND = 3
|
|
41
|
+
|
|
42
|
+
EPILOG = """\
|
|
43
|
+
examples:
|
|
44
|
+
PYTHONPATH=app python3 -m nullius.runs.cli list jobs
|
|
45
|
+
PYTHONPATH=app python3 -m nullius.runs.cli list jobs jobs_imported --json
|
|
46
|
+
PYTHONPATH=app python3 -m nullius.runs.cli show jobs 2026-09-12__22-53-55 --json
|
|
47
|
+
PYTHONPATH=app python3 -m nullius.runs.cli show jobs 2026-09-12__22-53-55 astropy__astropy-8707__agWFDYq
|
|
48
|
+
|
|
49
|
+
exit codes:
|
|
50
|
+
0 done
|
|
51
|
+
1 unexpected error (a traceback: report it)
|
|
52
|
+
2 usage: a bad flag, a missing argument
|
|
53
|
+
3 input not found: no such root, no job under it, no job of that id, no trial of that name
|
|
54
|
+
4 the operation could not be completed
|
|
55
|
+
With --json, stdout is exactly one JSON document; everything else goes to stderr.
|
|
56
|
+
"""
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def _stamp(value):
|
|
60
|
+
return value.isoformat() if isinstance(value, datetime) else value
|
|
61
|
+
|
|
62
|
+
|
|
63
|
+
def job_row(job: parse.Job) -> dict:
|
|
64
|
+
"""One job as the runs table shows it: identity, state, the outcome mix
|
|
65
|
+
and the two solve rates. Derived properties are included by name because
|
|
66
|
+
``asdict`` cannot see them, and they are what the page renders."""
|
|
67
|
+
return {
|
|
68
|
+
"job_id": job.job_id,
|
|
69
|
+
"dir": job.dir,
|
|
70
|
+
"dataset": job.dataset,
|
|
71
|
+
"model": job.model_name,
|
|
72
|
+
"agent": job.agent_label,
|
|
73
|
+
"kind": job.kind,
|
|
74
|
+
"state": job.state,
|
|
75
|
+
"n_trials": len(job.trials),
|
|
76
|
+
"n_planned": job.n_planned,
|
|
77
|
+
"counts": job.counts,
|
|
78
|
+
"n_resolved": job.n_resolved,
|
|
79
|
+
"n_infra": job.n_infra,
|
|
80
|
+
"raw_solve_rate": job.raw_solve_rate,
|
|
81
|
+
"adjusted_solve_rate": job.adjusted_solve_rate,
|
|
82
|
+
"started_at": _stamp(job.started_at),
|
|
83
|
+
"finished_at": _stamp(job.finished_at),
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def job_detail(job: parse.Job) -> dict:
|
|
88
|
+
"""The whole parsed job — every field the workspace cache keeps, plus the
|
|
89
|
+
derived ones the job page renders, per trial too. ``exception_message``
|
|
90
|
+
was redacted on the way in (``parse.parse_trial``), so this is as safe to
|
|
91
|
+
print as the page is to render."""
|
|
92
|
+
data = asdict(job)
|
|
93
|
+
data.update(job_row(job))
|
|
94
|
+
data["harbor_label"] = job.harbor_label
|
|
95
|
+
for trial, raw in zip(job.trials, data["trials"]):
|
|
96
|
+
raw["outcome_label"] = trial.outcome_label
|
|
97
|
+
raw["is_infra"] = trial.is_infra
|
|
98
|
+
raw["is_regression"] = trial.is_regression
|
|
99
|
+
raw["has_trace"] = trial.has_trace
|
|
100
|
+
raw["started_at"] = _stamp(trial.started_at)
|
|
101
|
+
raw["finished_at"] = _stamp(trial.finished_at)
|
|
102
|
+
raw["phases"] = {k: [_stamp(a), _stamp(b)] for k, (a, b) in trial.phases.items()}
|
|
103
|
+
return data
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _trial_record(trial: parse.Trial) -> dict:
|
|
107
|
+
"""One trial as ``job_detail`` carries it: every parsed field plus the
|
|
108
|
+
derived ones the pages render, timestamps as ISO strings."""
|
|
109
|
+
raw = asdict(trial)
|
|
110
|
+
raw["outcome_label"] = trial.outcome_label
|
|
111
|
+
raw["is_infra"] = trial.is_infra
|
|
112
|
+
raw["is_regression"] = trial.is_regression
|
|
113
|
+
raw["has_trace"] = trial.has_trace
|
|
114
|
+
raw["started_at"] = _stamp(trial.started_at)
|
|
115
|
+
raw["finished_at"] = _stamp(trial.finished_at)
|
|
116
|
+
raw["phases"] = {k: [_stamp(a), _stamp(b)] for k, (a, b) in trial.phases.items()}
|
|
117
|
+
return raw
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def trial_detail(job: parse.Job, trial: parse.Trial) -> dict:
|
|
121
|
+
"""What the trial page shows: the trial's record, the check vocabulary
|
|
122
|
+
the dataset names its cards with, and the trajectory as ``trajectory.load``
|
|
123
|
+
reads it — steps with their tool calls by name, category, target and
|
|
124
|
+
failure, and the totals — or the reason it could not be read. Every
|
|
125
|
+
string came through ``redact`` in the reader; nothing is added here.
|
|
126
|
+
Arguments and observations are left out: they are the conversation
|
|
127
|
+
browser's, paged and bounded there, and can be the whole patch."""
|
|
128
|
+
vocab = trial.vocab
|
|
129
|
+
doc = {"job_id": job.job_id, "trial": _trial_record(trial),
|
|
130
|
+
"vocab": {"gained": vocab.gained, "held": vocab.held, "has_patch": vocab.has_patch},
|
|
131
|
+
"trajectory": None, "steps": []}
|
|
132
|
+
if not trial.has_trajectory:
|
|
133
|
+
return doc
|
|
134
|
+
traj = trajectory.load(trial.trajectory_path)
|
|
135
|
+
doc["trajectory"] = {
|
|
136
|
+
"error": traj.error, "n_steps": traj.n_steps, "n_tool_calls": traj.n_tool_calls,
|
|
137
|
+
"n_failed_calls": traj.n_failed_calls, "category_freq": dict(traj.category_freq),
|
|
138
|
+
"files_touched": list(traj.files_touched), "final_metrics": traj.final_metrics,
|
|
139
|
+
"has_step_clock": traj.has_step_clock, "last_agent_message": traj.last_agent_message,
|
|
140
|
+
"total_reasoning_tokens": traj.total_reasoning_tokens,
|
|
141
|
+
}
|
|
142
|
+
doc["steps"] = [{
|
|
143
|
+
"step_id": s.step_id, "source": s.source, "timestamp": s.timestamp, "dur": s.dur,
|
|
144
|
+
"message": s.message,
|
|
145
|
+
"tool_calls": [{"name": c["name"], "category": c["category"], "target": c["target"],
|
|
146
|
+
"failure": c["failure"], "obs_bytes": c["obs_bytes"]} for c in s.tool_calls],
|
|
147
|
+
} for s in traj.steps]
|
|
148
|
+
return doc
|
|
149
|
+
|
|
150
|
+
|
|
151
|
+
def _first_line(text: str, width: int = 96) -> str:
|
|
152
|
+
line = (text or "").strip().splitlines()[0] if (text or "").strip() else ""
|
|
153
|
+
return line if len(line) <= width else line[:width - 1] + "…"
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
def _secs(value) -> str:
|
|
157
|
+
return "—" if value is None else f"{value:.1f}s"
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def _print_trial(job: parse.Job, trial: parse.Trial) -> None:
|
|
161
|
+
doc = trial_detail(job, trial)
|
|
162
|
+
t, vocab = trial, doc["vocab"]
|
|
163
|
+
print(f"{job.job_id}/{t.trial_name}")
|
|
164
|
+
print(f" task {t.task_base} · {t.outcome_label} ({t.outcome}) · "
|
|
165
|
+
f"model {t.model_name or job.model_name or '—'} · reward "
|
|
166
|
+
f"{'—' if t.reward is None else t.reward}")
|
|
167
|
+
checks = []
|
|
168
|
+
if vocab["gained"]:
|
|
169
|
+
checks.append(f"{vocab['gained']} {t.f2p_pass} passed, {t.f2p_fail} failed")
|
|
170
|
+
if vocab["held"]:
|
|
171
|
+
checks.append(f"{vocab['held']} {t.p2p_pass} passed, {t.p2p_fail} failed")
|
|
172
|
+
if vocab["has_patch"]:
|
|
173
|
+
checks.append("patch " + ("—" if t.patch_exists is None else
|
|
174
|
+
("applied" if t.patch_applied else "not applied")))
|
|
175
|
+
if checks:
|
|
176
|
+
print(" checks: " + " · ".join(checks))
|
|
177
|
+
cost = "—" if t.cost_usd is None else f"${t.cost_usd:.4f}"
|
|
178
|
+
print(f" usage: input {t.n_input_tokens} · cached {t.n_cache_tokens} · "
|
|
179
|
+
f"output {t.n_output_tokens} · cost {cost} (estimated)")
|
|
180
|
+
print(f" time: total {_secs(t.dur_total)} · env {_secs(t.dur_env)} · "
|
|
181
|
+
f"setup {_secs(t.dur_agent_setup)} · agent {_secs(t.dur_agent)} · "
|
|
182
|
+
f"verifier {_secs(t.dur_verifier)}")
|
|
183
|
+
print(f" evidence: trajectory {'recorded' if t.has_trajectory else 'absent'} · "
|
|
184
|
+
f"trace {'recorded' if t.has_trace else 'absent'} · "
|
|
185
|
+
f"reward {'recorded' if t.reward is not None else 'absent'}")
|
|
186
|
+
if t.exception_type:
|
|
187
|
+
print(f" exception: {t.exception_type}: {_first_line(t.exception_message)}")
|
|
188
|
+
for name in t.p2p_failures:
|
|
189
|
+
print(f" broke: {name}")
|
|
190
|
+
traj = doc["trajectory"]
|
|
191
|
+
if traj is None:
|
|
192
|
+
return
|
|
193
|
+
if traj["error"]:
|
|
194
|
+
print(f" trajectory unreadable: {_first_line(traj['error'])}")
|
|
195
|
+
return
|
|
196
|
+
mix = " · ".join(f"{k} {v}" for k, v in sorted(traj["category_freq"].items()))
|
|
197
|
+
print(f" steps: {traj['n_steps']} · tool calls {traj['n_tool_calls']} · "
|
|
198
|
+
f"failed {traj['n_failed_calls']}" + (f" · {mix}" if mix else ""))
|
|
199
|
+
width = len(str(max((s["step_id"] for s in doc["steps"]), default=0)))
|
|
200
|
+
for s in doc["steps"]:
|
|
201
|
+
calls = ", ".join(c["name"] + (f" ✗ {c['failure']}" if c["failure"] else "")
|
|
202
|
+
for c in s["tool_calls"])
|
|
203
|
+
text = _first_line(s["message"]) if s["message"] and s["message"] != "(tool use)" else ""
|
|
204
|
+
body = " · ".join(x for x in (text, calls) if x)
|
|
205
|
+
dur = "" if s["dur"] is None else f" {s['dur']:.1f}s"
|
|
206
|
+
print(f" {s['step_id']:>{width}} {s['source']:<6} {body}{dur}")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def _emit(doc) -> None:
|
|
210
|
+
json.dump(doc, sys.stdout, ensure_ascii=False, indent=1, default=_stamp)
|
|
211
|
+
sys.stdout.write("\n")
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def _load(roots) -> list | None:
|
|
215
|
+
"""The jobs under *roots*, or None (said on stderr) when a root is missing."""
|
|
216
|
+
missing = [r for r in roots if not os.path.isdir(r)]
|
|
217
|
+
if missing:
|
|
218
|
+
for r in missing:
|
|
219
|
+
print(f"no such directory: {r}", file=sys.stderr)
|
|
220
|
+
return None
|
|
221
|
+
return parse.load_all(roots)
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def _list(args) -> int:
|
|
225
|
+
jobs = _load(args.roots)
|
|
226
|
+
if jobs is None:
|
|
227
|
+
return EXIT_NOT_FOUND
|
|
228
|
+
if not jobs:
|
|
229
|
+
print(f"no job directory found under {', '.join(args.roots)}", file=sys.stderr)
|
|
230
|
+
return EXIT_NOT_FOUND
|
|
231
|
+
if args.json:
|
|
232
|
+
_emit([job_row(j) for j in jobs])
|
|
233
|
+
else:
|
|
234
|
+
for j in jobs:
|
|
235
|
+
print(j.job_id)
|
|
236
|
+
return EXIT_OK
|
|
237
|
+
|
|
238
|
+
|
|
239
|
+
def _show(args) -> int:
|
|
240
|
+
jobs = _load([args.root])
|
|
241
|
+
if jobs is None:
|
|
242
|
+
return EXIT_NOT_FOUND
|
|
243
|
+
job = next((j for j in jobs if j.job_id == args.job_id), None)
|
|
244
|
+
if job is None:
|
|
245
|
+
known = ", ".join(j.job_id for j in jobs) or "none"
|
|
246
|
+
print(f"no job named {args.job_id!r} under {args.root} (jobs there: {known})",
|
|
247
|
+
file=sys.stderr)
|
|
248
|
+
return EXIT_NOT_FOUND
|
|
249
|
+
if args.trial_name:
|
|
250
|
+
trial = next((t for t in job.trials if t.trial_name == args.trial_name), None)
|
|
251
|
+
if trial is None:
|
|
252
|
+
known = ", ".join(t.trial_name for t in job.trials) or "none"
|
|
253
|
+
print(f"no trial named {args.trial_name!r} in {job.job_id} (trials there: {known})",
|
|
254
|
+
file=sys.stderr)
|
|
255
|
+
return EXIT_NOT_FOUND
|
|
256
|
+
if args.json:
|
|
257
|
+
_emit(trial_detail(job, trial))
|
|
258
|
+
else:
|
|
259
|
+
_print_trial(job, trial)
|
|
260
|
+
return EXIT_OK
|
|
261
|
+
if args.json:
|
|
262
|
+
_emit(job_detail(job))
|
|
263
|
+
return EXIT_OK
|
|
264
|
+
row = job_row(job)
|
|
265
|
+
rate = lambda v: "—" if v is None else f"{100 * v:.0f}%"
|
|
266
|
+
print(f"{job.job_id}")
|
|
267
|
+
print(f" dataset {row['dataset'] or '—'} · model {row['model'] or '—'} · "
|
|
268
|
+
f"agent {row['agent'] or '—'} · {row['state']}")
|
|
269
|
+
print(f" {row['n_trials']} trial(s) of {row['n_planned']} planned: "
|
|
270
|
+
f"resolved {row['n_resolved']} · infra {row['n_infra']} · "
|
|
271
|
+
f"raw {rate(row['raw_solve_rate'])} · adjusted {rate(row['adjusted_solve_rate'])}")
|
|
272
|
+
mix = " · ".join(f"{k} {v}" for k, v in row["counts"].items() if v)
|
|
273
|
+
if mix:
|
|
274
|
+
print(f" outcomes: {mix}")
|
|
275
|
+
width = max((len(t.trial_name) for t in job.trials), default=0)
|
|
276
|
+
for t in job.trials:
|
|
277
|
+
line = f" {t.trial_name:<{width}} {t.outcome}"
|
|
278
|
+
if t.exception_type:
|
|
279
|
+
line += f" {t.exception_type}"
|
|
280
|
+
print(line)
|
|
281
|
+
return EXIT_OK
|
|
282
|
+
|
|
283
|
+
|
|
284
|
+
def main(argv=None) -> int:
|
|
285
|
+
ap = argparse.ArgumentParser(
|
|
286
|
+
prog="python3 -m nullius.runs.cli", description=__doc__.splitlines()[0],
|
|
287
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
288
|
+
sub = ap.add_subparsers(dest="command", metavar="COMMAND")
|
|
289
|
+
ls = sub.add_parser("list", help="the job ids under one or more roots, newest first",
|
|
290
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
291
|
+
ls.add_argument("roots", nargs="+", metavar="ROOT",
|
|
292
|
+
help="a job directory, a parent of job directories, or an imported root")
|
|
293
|
+
ls.add_argument("--json", action="store_true",
|
|
294
|
+
help="one JSON array of job rows (what the runs table shows) instead of ids")
|
|
295
|
+
ls.set_defaults(run=_list)
|
|
296
|
+
sh = sub.add_parser("show", help="one job: its trials and their outcomes; or one trial",
|
|
297
|
+
epilog=EPILOG, formatter_class=argparse.RawDescriptionHelpFormatter)
|
|
298
|
+
sh.add_argument("root", metavar="ROOT", help="the root the job is under")
|
|
299
|
+
sh.add_argument("job_id", metavar="JOB_ID", help="a job id, as `list` prints it")
|
|
300
|
+
sh.add_argument("trial_name", metavar="TRIAL", nargs="?", default="",
|
|
301
|
+
help="a trial name, as `show` on the job prints it: that trial's outcome, "
|
|
302
|
+
"checks, usage, exception and steps, as /trial/<job>/<trial> shows them")
|
|
303
|
+
sh.add_argument("--json", action="store_true",
|
|
304
|
+
help="the whole parsed job (or the one trial, with its steps) as one JSON document")
|
|
305
|
+
sh.set_defaults(run=_show)
|
|
306
|
+
args = ap.parse_args(argv)
|
|
307
|
+
if not args.command:
|
|
308
|
+
ap.error("a command is required: list or show")
|
|
309
|
+
return args.run(args)
|
|
310
|
+
|
|
311
|
+
|
|
312
|
+
if __name__ == "__main__": # pragma: no cover - entry point
|
|
313
|
+
# Run as `PYTHONPATH=app python3 -m nullius.runs.cli`. A module, not a
|
|
314
|
+
# script, like its two siblings: without this guard `-m` imports the
|
|
315
|
+
# module, runs nothing and exits 0, which reads exactly like success.
|
|
316
|
+
raise SystemExit(main())
|