nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/runs/parse.py
ADDED
|
@@ -0,0 +1,1016 @@
|
|
|
1
|
+
"""Walk job directories and build the normalized data model.
|
|
2
|
+
|
|
3
|
+
A *job* is one Harbor run (a timestamped dir holding a ``result.json`` and one
|
|
4
|
+
subdir per *trial*). A *trial* is one task attempt. We read only what we need
|
|
5
|
+
and redact on the way in.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import os
|
|
11
|
+
import re
|
|
12
|
+
import shlex
|
|
13
|
+
from dataclasses import dataclass, field, replace
|
|
14
|
+
from datetime import datetime, timezone
|
|
15
|
+
|
|
16
|
+
from nullius.runs import verifier
|
|
17
|
+
from nullius.runs import classify
|
|
18
|
+
from nullius.redact import redact
|
|
19
|
+
|
|
20
|
+
# What a trace document in a trial's `agent/` may be called: the producer's own
|
|
21
|
+
# capture, or a filtered view saved from the trial page. A bare filename, so
|
|
22
|
+
# there is no path to traverse in the first place, and no leading dot to hide the
|
|
23
|
+
# file it writes.
|
|
24
|
+
#
|
|
25
|
+
# One definition, two readers: the trial page validates a name before writing it,
|
|
26
|
+
# and one arriving in a query string (?trace=) before reading it.
|
|
27
|
+
TRACE_NAME_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]{0,63}\.json$")
|
|
28
|
+
DEFAULT_TRACE_NAME = "trace.json"
|
|
29
|
+
|
|
30
|
+
# The shape this module produces. A workspace caches parsed Jobs keyed on it,
|
|
31
|
+
# so bump it when a Trial or Job field is added, renamed or means something
|
|
32
|
+
# new — otherwise a cache written by the old parse is read back as current.
|
|
33
|
+
SCHEMA = "15"
|
|
34
|
+
|
|
35
|
+
# What a rate-limit record in `agent/gateway.log` looks like. The agent's SDK
|
|
36
|
+
# retries a 429 below the harness (an agent's `max_retries` reaching its
|
|
37
|
+
# OpenAI client), so a limit the run survived leaves nothing in result.json —
|
|
38
|
+
# the outcome says `resolved` and the only evidence is the log. Three shapes
|
|
39
|
+
# are counted: the exception class or the API's error code (`RateLimitError`,
|
|
40
|
+
# `rate_limit_exceeded`, "Rate limit reached"), the HTTP reason phrase ("Too
|
|
41
|
+
# Many Requests"), and a bare 429 *as a status* — `status_code=429`, `HTTP 429`,
|
|
42
|
+
# `Error code: 429`. A bare 429 alone is not enough: event ids, hashes and
|
|
43
|
+
# token counts in the same log can carry those digits by chance.
|
|
44
|
+
# What is not counted: the SDK's "Retrying request …" line, which names no
|
|
45
|
+
# status, so a 5xx retry is not mistaken for a rate limit.
|
|
46
|
+
RATE_LIMIT_RE = re.compile(
|
|
47
|
+
rb"rate.?limit|too many requests"
|
|
48
|
+
rb"|(?:status(?:[_ -]?code)?|error code|code|http(?:/\d(?:\.\d)?)?)\W{0,4}429(?!\w|\.\d)",
|
|
49
|
+
re.IGNORECASE)
|
|
50
|
+
# How much of a gateway log the scan reads: the count is over the first
|
|
51
|
+
# GATEWAY_SCAN_BYTES only. Scanning is a bounded parse-time cost; for a log
|
|
52
|
+
# that outgrows the bound, the count is a floor.
|
|
53
|
+
GATEWAY_SCAN_BYTES = 8 * 1024 * 1024
|
|
54
|
+
|
|
55
|
+
|
|
56
|
+
def count_rate_limits(path: str) -> int | None:
|
|
57
|
+
"""Rate-limit records in a gateway log, or None when there is no log.
|
|
58
|
+
|
|
59
|
+
None and 0 are different answers: an agent that wrote no gateway log
|
|
60
|
+
(a control, an imported run, the direct transport) cannot be said to have
|
|
61
|
+
hit no limits, only that nothing recorded it. Lines are counted, bounded by
|
|
62
|
+
GATEWAY_SCAN_BYTES, and the file is never held whole.
|
|
63
|
+
"""
|
|
64
|
+
try:
|
|
65
|
+
handle = open(path, "rb")
|
|
66
|
+
except OSError:
|
|
67
|
+
return None
|
|
68
|
+
count, budget = 0, GATEWAY_SCAN_BYTES
|
|
69
|
+
with handle:
|
|
70
|
+
for line in handle:
|
|
71
|
+
if RATE_LIMIT_RE.search(line):
|
|
72
|
+
count += 1
|
|
73
|
+
budget -= len(line)
|
|
74
|
+
if budget <= 0:
|
|
75
|
+
break
|
|
76
|
+
return count
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def _iso(ts: str | None):
|
|
80
|
+
"""A stamp as an aware datetime. Harbor writes its stamps without a zone,
|
|
81
|
+
in UTC; the standard, OpenTelemetry and zoned ATIF imports write one. A
|
|
82
|
+
naive and an aware datetime cannot be compared or subtracted, so a page
|
|
83
|
+
that ordered or spanned both returned 500: every stamp is given UTC."""
|
|
84
|
+
if not ts or not isinstance(ts, str):
|
|
85
|
+
return None
|
|
86
|
+
try:
|
|
87
|
+
when = datetime.fromisoformat(ts.replace("Z", "+00:00"))
|
|
88
|
+
except ValueError:
|
|
89
|
+
return None
|
|
90
|
+
return when.replace(tzinfo=timezone.utc) if when.tzinfo is None else when
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def _dur(a: dict | None) -> float | None:
|
|
94
|
+
"""Seconds between started_at/finished_at of a phase block."""
|
|
95
|
+
if not a:
|
|
96
|
+
return None
|
|
97
|
+
s, f = _iso(a.get("started_at")), _iso(a.get("finished_at"))
|
|
98
|
+
if s and f:
|
|
99
|
+
return (f - s).total_seconds()
|
|
100
|
+
return None
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@dataclass
|
|
104
|
+
class Trial:
|
|
105
|
+
job_id: str
|
|
106
|
+
dir: str
|
|
107
|
+
task_name: str
|
|
108
|
+
trial_name: str
|
|
109
|
+
task_base: str # e.g. django__django-10554
|
|
110
|
+
source: str
|
|
111
|
+
model_name: str
|
|
112
|
+
outcome: str
|
|
113
|
+
reward: float | None
|
|
114
|
+
exception_type: str
|
|
115
|
+
exception_message: str
|
|
116
|
+
# verifier detail. The patch pair is `None` for a benchmark that has no
|
|
117
|
+
# patches to report on — templates show absence, not "No".
|
|
118
|
+
patch_exists: bool | None
|
|
119
|
+
patch_applied: bool | None
|
|
120
|
+
f2p_pass: int
|
|
121
|
+
f2p_fail: int
|
|
122
|
+
p2p_pass: int
|
|
123
|
+
p2p_fail: int
|
|
124
|
+
n_input_tokens: int
|
|
125
|
+
n_output_tokens: int
|
|
126
|
+
n_cache_tokens: int
|
|
127
|
+
cost_usd: float | None
|
|
128
|
+
# timings (seconds)
|
|
129
|
+
dur_total: float | None
|
|
130
|
+
dur_env: float | None
|
|
131
|
+
dur_agent_setup: float | None
|
|
132
|
+
dur_agent: float | None
|
|
133
|
+
dur_verifier: float | None
|
|
134
|
+
has_trajectory: bool = False
|
|
135
|
+
# The resolved agent version recorded in agent_info —
|
|
136
|
+
# what makes two runs of "the same" agent actually comparable.
|
|
137
|
+
agent_version: str = ""
|
|
138
|
+
# The agent the trial's own result.json names (agent_info.name). A job
|
|
139
|
+
# config written with defaults left out names no agent when the agent is
|
|
140
|
+
# Harbor's default (oracle); the trials still say who ran.
|
|
141
|
+
agent_info_name: str = ""
|
|
142
|
+
reasoning_effort: str = ""
|
|
143
|
+
# The environment the trial's own result.json records (`config.environment
|
|
144
|
+
# .type`, e.g. "docker") — the job-level config does not carry it.
|
|
145
|
+
env_type: str = ""
|
|
146
|
+
has_trace: bool = False
|
|
147
|
+
# Rate-limit records the agent's own log carries — 429s its SDK retried and
|
|
148
|
+
# the run survived, which no result.json field records (`count_rate_limits`).
|
|
149
|
+
# None when the trial has no `agent/gateway.log`: not measured, not zero.
|
|
150
|
+
n_rate_limited_retries: int | None = None
|
|
151
|
+
# A graded verdict (an F1, partial credit), beside the pass or fail the
|
|
152
|
+
# reward says: the runner records it when its benchmark scores that way.
|
|
153
|
+
# A benchmark that gives a score and no pass or fail leaves the reward
|
|
154
|
+
# out: the trial is then scored, not resolved or failed.
|
|
155
|
+
score: float | None = None
|
|
156
|
+
# How many parts that score is the mean of (the questions of a
|
|
157
|
+
# conversation), when the benchmark scores an item part by part; each
|
|
158
|
+
# part is in verifier/parts.json. None: the score is the item's own.
|
|
159
|
+
n_parts: int | None = None
|
|
160
|
+
# Those parts' scores by the field the benchmark breaks its score down by
|
|
161
|
+
# (a question's category): {value: [sum of scores, parts]}, a part with no
|
|
162
|
+
# score counting nothing, as in the trial's own mean; and that field's word.
|
|
163
|
+
part_groups: dict = field(default_factory=dict)
|
|
164
|
+
parts_by: str = ""
|
|
165
|
+
# Names of the tests that failed, kept so a regression can be shown, not
|
|
166
|
+
# just counted.
|
|
167
|
+
f2p_failures: list = field(default_factory=list)
|
|
168
|
+
p2p_failures: list = field(default_factory=list)
|
|
169
|
+
# Absolute clocks, for anything that places this trial on a timeline
|
|
170
|
+
# rather than measuring it — an exporter drawing spans needs where a phase
|
|
171
|
+
# *sat*, not just how long it took. `phases` maps a Harbor phase name
|
|
172
|
+
# (environment_setup, agent_setup, agent_execution, verifier) to its
|
|
173
|
+
# (started, finished) pair; a phase the run never recorded is absent, so a
|
|
174
|
+
# reader sees nothing rather than a zero-length span.
|
|
175
|
+
started_at: datetime | None = None
|
|
176
|
+
finished_at: datetime | None = None
|
|
177
|
+
phases: dict = field(default_factory=dict)
|
|
178
|
+
|
|
179
|
+
@property
|
|
180
|
+
def dur_phases(self) -> float:
|
|
181
|
+
"""Sum of the four phases Harbor breaks out."""
|
|
182
|
+
return sum(
|
|
183
|
+
v or 0
|
|
184
|
+
for v in (self.dur_env, self.dur_agent_setup, self.dur_agent, self.dur_verifier)
|
|
185
|
+
)
|
|
186
|
+
|
|
187
|
+
@property
|
|
188
|
+
def dur_other(self) -> float:
|
|
189
|
+
"""Trial time Harbor doesn't attribute to a phase.
|
|
190
|
+
|
|
191
|
+
Image pull, container start and teardown, artifact collection. Clamped
|
|
192
|
+
at 0: if the phases ever exceed the total — clock skew, or phases that
|
|
193
|
+
overlap — a negative value would corrupt the bar's geometry silently
|
|
194
|
+
rather than showing an error.
|
|
195
|
+
"""
|
|
196
|
+
return max(0.0, (self.dur_total or 0) - self.dur_phases)
|
|
197
|
+
|
|
198
|
+
@property
|
|
199
|
+
def is_regression(self) -> bool:
|
|
200
|
+
"""The patch broke tests that were passing before it was applied."""
|
|
201
|
+
return self.p2p_fail > 0
|
|
202
|
+
|
|
203
|
+
@property
|
|
204
|
+
def outcome_label(self) -> str:
|
|
205
|
+
return classify.LABELS.get(self.outcome, self.outcome)
|
|
206
|
+
|
|
207
|
+
@property
|
|
208
|
+
def vocab(self):
|
|
209
|
+
"""The dataset's own names for its checks — see `verifier.Vocabulary`."""
|
|
210
|
+
return verifier.vocabulary_for(self.source)
|
|
211
|
+
|
|
212
|
+
@property
|
|
213
|
+
def is_infra(self) -> bool:
|
|
214
|
+
return self.outcome in classify.INFRA_OUTCOMES
|
|
215
|
+
|
|
216
|
+
@property
|
|
217
|
+
def trajectory_path(self) -> str:
|
|
218
|
+
return os.path.join(self.dir, "agent", "trajectory.json")
|
|
219
|
+
|
|
220
|
+
@property
|
|
221
|
+
def trace_path(self) -> str:
|
|
222
|
+
"""The timing sidecar, written beside the trajectory when captured."""
|
|
223
|
+
return os.path.join(self.dir, "agent", DEFAULT_TRACE_NAME)
|
|
224
|
+
|
|
225
|
+
# Harbor's control agents. Neither calls a model: `oracle` applies the task's
|
|
226
|
+
# gold patch, `nop` does nothing. Their zero tokens and zero cost are true
|
|
227
|
+
# measurements, not missing ones — which is the whole reason to tell them apart.
|
|
228
|
+
CONTROL_AGENTS = {"oracle", "nop"}
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
@dataclass
|
|
232
|
+
class Job:
|
|
233
|
+
job_id: str
|
|
234
|
+
dir: str
|
|
235
|
+
model_name: str
|
|
236
|
+
dataset: str
|
|
237
|
+
n_tasks: int
|
|
238
|
+
started_at: datetime | None
|
|
239
|
+
finished_at: datetime | None
|
|
240
|
+
trials: list = field(default_factory=list) # list[Trial]
|
|
241
|
+
agent_name: str = "" # e.g. "claude-code", "opencode"
|
|
242
|
+
reasoning_effort: str = ""
|
|
243
|
+
# Harbor's own bookkeeping from the job-level result.json stats block.
|
|
244
|
+
# n_running/n_pending distinguish a finished run from one still going —
|
|
245
|
+
# without them an in-progress sweep reads as a run full of misses.
|
|
246
|
+
n_total_trials: int = 0
|
|
247
|
+
n_errored_trials: int = 0
|
|
248
|
+
n_running_trials: int = 0
|
|
249
|
+
n_pending_trials: int = 0
|
|
250
|
+
n_retries: int = 0
|
|
251
|
+
preparing: bool = False # launched, no result.json yet — see is_preparing_job_dir
|
|
252
|
+
# The dataset path exactly as the config recorded it (may be relative to
|
|
253
|
+
# wherever harbor ran, and empty for registry fetches). Kept because it's
|
|
254
|
+
# the only lead for finding the checkout — and its gold patches — locally.
|
|
255
|
+
dataset_path: str = ""
|
|
256
|
+
# The `harbor run` invocation reconstructed from this job's recorded
|
|
257
|
+
# config — what a copy-and-rerun needs. Empty when config.json is absent.
|
|
258
|
+
harbor_command: str = ""
|
|
259
|
+
#: the Harbor fork version the job recorded in its lock.json (`harbor.version`),
|
|
260
|
+
#: and whether that Harbor was an editable checkout — what actually ran,
|
|
261
|
+
#: as distinct from what a recipe intended; "" when the tree carries none
|
|
262
|
+
harbor_version: str = ""
|
|
263
|
+
harbor_editable: bool | None = None
|
|
264
|
+
#: a job whose config names `runner`: its runner and resolved upstream
|
|
265
|
+
#: commit, in place of Harbor metadata
|
|
266
|
+
runner: str = ""
|
|
267
|
+
runner_commit: str = ""
|
|
268
|
+
runner_judge: str = "" # the benchmark's own judge, as the run named it: "official gpt-4o · evaluate_qa.py"
|
|
269
|
+
questions_from: str = "" # a runner's recipe dataset: the items every arm answers, a control's with a file of its own included
|
|
270
|
+
runner_items: list = field(default_factory=list) # what the runner calls one item and many, from its definition: ["question", "questions"]
|
|
271
|
+
# A Nullius standard job that ran one of its benchmark's declared controls
|
|
272
|
+
# (config ``standard.control``): who wrote it - official, adapter, nullius.
|
|
273
|
+
# "" for every other job.
|
|
274
|
+
control_source: str = ""
|
|
275
|
+
# A Nullius standard job (config ``standard``): its trials are written as each unit ends, so the
|
|
276
|
+
# live board links one as soon as it has - unlike a runner's, imported at its end.
|
|
277
|
+
standard: bool = False
|
|
278
|
+
# The launch a Nullius standard job was one shard of (config
|
|
279
|
+
# ``standard.launch``): the shards of one launch, run on several machines,
|
|
280
|
+
# are grouped as one on the runs page (launches.group). "" for every other job.
|
|
281
|
+
launch_id: str = ""
|
|
282
|
+
# Source job names explicitly recorded by standard merge.
|
|
283
|
+
merged_from: list[str] = field(default_factory=list)
|
|
284
|
+
# How the job was launched, as its config.json recorded it with Harbor's
|
|
285
|
+
# defaults filled in (launch_of): what "New run like this" rebuilds a
|
|
286
|
+
# recipe from when the run kept none. Empty when config.json is absent.
|
|
287
|
+
launch: dict = field(default_factory=dict)
|
|
288
|
+
# A run with a spending cap (a Nullius standard job's ``result.json``, ``budget``): the cap and what was
|
|
289
|
+
# spent in USD, whether the cap stopped the run, the units it cut off and those never begun; a cap per unit
|
|
290
|
+
# (``unit_max_usd``) and the units that reached it (``over_unit``). {} without one.
|
|
291
|
+
budget: dict = field(default_factory=dict)
|
|
292
|
+
# A Harbor job run with several agents or models (`-m a -m b`, or a config
|
|
293
|
+
# listing several): each one's {name, model, effort}, which split_agents
|
|
294
|
+
# turns into one view per agent. [] for a job of one agent.
|
|
295
|
+
agents: list = field(default_factory=list)
|
|
296
|
+
|
|
297
|
+
@property
|
|
298
|
+
def agent_label(self) -> str:
|
|
299
|
+
"""Readable half of the agent id — the class, not the module path."""
|
|
300
|
+
return self.agent_name.split(":")[-1] if self.agent_name else ""
|
|
301
|
+
|
|
302
|
+
@property
|
|
303
|
+
def kind(self) -> str:
|
|
304
|
+
"""'control' for Harbor's oracle/nop baselines, else 'agent'. A
|
|
305
|
+
runner's arm is never Harbor's control, whatever it is called:
|
|
306
|
+
a runner's `oracle` arm is its own ceiling, not a reference solution.
|
|
307
|
+
A standard job is one when its config says it ran a declared control."""
|
|
308
|
+
named = self.agent_label.lower() in CONTROL_AGENTS
|
|
309
|
+
return "control" if named and (not self.runner or self.control_source) else "agent"
|
|
310
|
+
|
|
311
|
+
@property
|
|
312
|
+
def is_control(self) -> bool:
|
|
313
|
+
return self.kind == "control"
|
|
314
|
+
|
|
315
|
+
@property
|
|
316
|
+
def n_planned(self) -> int:
|
|
317
|
+
"""How many trials the run set out to do — Harbor's own count when it
|
|
318
|
+
recorded one, else the task count from the config. Comparing this with
|
|
319
|
+
the trials actually on disk exposes a partial or interrupted run."""
|
|
320
|
+
return self.n_total_trials or self.n_tasks
|
|
321
|
+
|
|
322
|
+
@property
|
|
323
|
+
def is_running(self) -> bool:
|
|
324
|
+
"""Trials still executing or queued — the run isn't over yet."""
|
|
325
|
+
return (self.n_running_trials + self.n_pending_trials) > 0
|
|
326
|
+
|
|
327
|
+
@property
|
|
328
|
+
def harbor_label(self) -> str:
|
|
329
|
+
"""The recorded Harbor: version, and `editable` when the lock says the
|
|
330
|
+
checkout was — a development Harbor, not a release; empty when unrecorded."""
|
|
331
|
+
if not self.harbor_version:
|
|
332
|
+
return ""
|
|
333
|
+
return self.harbor_version + (" · editable checkout" if self.harbor_editable else "")
|
|
334
|
+
|
|
335
|
+
@property
|
|
336
|
+
def is_preparing(self) -> bool:
|
|
337
|
+
"""Launched, not started: Harbor has written the job's config and
|
|
338
|
+
nothing else yet — no result record, no trial. The runs list shows the
|
|
339
|
+
row now rather than when the first event lands."""
|
|
340
|
+
return self.preparing
|
|
341
|
+
|
|
342
|
+
@property
|
|
343
|
+
def state(self) -> str:
|
|
344
|
+
"""``preparing`` → ``running`` → ``finished``, for a badge."""
|
|
345
|
+
return "preparing" if self.preparing else "running" if self.is_running else "finished"
|
|
346
|
+
|
|
347
|
+
@property
|
|
348
|
+
def agent_version(self) -> str:
|
|
349
|
+
"""The resolved agent version, taken from the first trial that has one."""
|
|
350
|
+
return next((t.agent_version for t in self.trials if t.agent_version), "")
|
|
351
|
+
|
|
352
|
+
@property
|
|
353
|
+
def vocab(self):
|
|
354
|
+
"""The check vocabulary of the dataset this job ran, from its trials.
|
|
355
|
+
|
|
356
|
+
Every job on disk runs one dataset, so the first trial that names its
|
|
357
|
+
checks speaks for the run. A job of only unknown datasets gets the
|
|
358
|
+
empty vocabulary, and the table drops the check columns rather than
|
|
359
|
+
labelling them with another benchmark's words.
|
|
360
|
+
"""
|
|
361
|
+
for t in self.trials:
|
|
362
|
+
v = t.vocab
|
|
363
|
+
if v.gained or v.held or v.has_patch:
|
|
364
|
+
return v
|
|
365
|
+
return verifier.GENERIC_VOCABULARY
|
|
366
|
+
|
|
367
|
+
@property
|
|
368
|
+
def duration_min(self) -> float | None:
|
|
369
|
+
if self.started_at and self.finished_at:
|
|
370
|
+
return (self.finished_at - self.started_at).total_seconds() / 60.0
|
|
371
|
+
return None
|
|
372
|
+
|
|
373
|
+
@property
|
|
374
|
+
def counts(self) -> dict:
|
|
375
|
+
c = {k: 0 for k in classify.ORDER}
|
|
376
|
+
for t in self.trials:
|
|
377
|
+
c[t.outcome] = c.get(t.outcome, 0) + 1
|
|
378
|
+
return c
|
|
379
|
+
|
|
380
|
+
@property
|
|
381
|
+
def total_input_tokens(self) -> int:
|
|
382
|
+
return sum(t.n_input_tokens or 0 for t in self.trials)
|
|
383
|
+
|
|
384
|
+
@property
|
|
385
|
+
def total_output_tokens(self) -> int:
|
|
386
|
+
return sum(t.n_output_tokens or 0 for t in self.trials)
|
|
387
|
+
|
|
388
|
+
@property
|
|
389
|
+
def total_cache_tokens(self) -> int:
|
|
390
|
+
return sum(t.n_cache_tokens or 0 for t in self.trials)
|
|
391
|
+
|
|
392
|
+
@property
|
|
393
|
+
def total_cost_usd(self):
|
|
394
|
+
"""None when no trial reported a cost — not every agent does."""
|
|
395
|
+
costs = [t.cost_usd for t in self.trials if t.cost_usd is not None]
|
|
396
|
+
return sum(costs) if costs else None
|
|
397
|
+
|
|
398
|
+
@property
|
|
399
|
+
def mean_score(self):
|
|
400
|
+
"""The mean of the trials' graded scores, None when none was graded.
|
|
401
|
+
A trial scored part by part weighs as many as it has: the mean is
|
|
402
|
+
then over every part - every question, not every conversation - which
|
|
403
|
+
is the number a paper that scores questions reports."""
|
|
404
|
+
scored = [(t.score, t.n_parts or 1) for t in self.trials if t.score is not None]
|
|
405
|
+
return sum(s * n for s, n in scored) / sum(n for _, n in scored) if scored else None
|
|
406
|
+
|
|
407
|
+
@property
|
|
408
|
+
def score_by_group(self) -> list:
|
|
409
|
+
"""The mean score by the field the benchmark breaks it down by - a
|
|
410
|
+
question's category - over every part of every trial: [{group, mean,
|
|
411
|
+
n}], in the order the values sort (numbers as numbers); [] when no
|
|
412
|
+
trial was scored that way."""
|
|
413
|
+
total = {}
|
|
414
|
+
for t in self.trials:
|
|
415
|
+
for g, (s, n) in (t.part_groups or {}).items():
|
|
416
|
+
acc = total.setdefault(g, [0.0, 0])
|
|
417
|
+
acc[0] += s
|
|
418
|
+
acc[1] += n
|
|
419
|
+
key = lambda g: (0, float(g), "") if g.replace(".", "", 1).lstrip("-").isdigit() else (1, 0.0, g)
|
|
420
|
+
return [{"group": g, "mean": s / n, "n": n} for g, (s, n) in sorted(total.items(), key=lambda kv: key(kv[0])) if n]
|
|
421
|
+
|
|
422
|
+
@property
|
|
423
|
+
def parts_by(self) -> str:
|
|
424
|
+
"""The word for what the score is broken down by, '' when it is not."""
|
|
425
|
+
return next((t.parts_by for t in self.trials if t.parts_by), "")
|
|
426
|
+
|
|
427
|
+
@property
|
|
428
|
+
def n_scored_parts(self):
|
|
429
|
+
"""How many parts the mean score is over, None when no trial has parts."""
|
|
430
|
+
counts = [t.n_parts for t in self.trials if t.score is not None and t.n_parts]
|
|
431
|
+
return sum(counts) if counts else None
|
|
432
|
+
|
|
433
|
+
@property
|
|
434
|
+
def scored_only(self) -> bool:
|
|
435
|
+
"""Scores and no pass or fail anywhere: the mean score is the
|
|
436
|
+
headline, and the solve rates stay absent rather than 0% - also when
|
|
437
|
+
a trial failed on an exception, which is a verdict of its own."""
|
|
438
|
+
return any(t.score is not None for t in self.trials) and not any(t.reward is not None for t in self.trials)
|
|
439
|
+
|
|
440
|
+
@property
|
|
441
|
+
def n_regressions(self) -> int:
|
|
442
|
+
"""Trials whose patch broke previously-passing tests."""
|
|
443
|
+
return sum(1 for t in self.trials if t.is_regression)
|
|
444
|
+
|
|
445
|
+
@property
|
|
446
|
+
def reports_regressions(self) -> bool:
|
|
447
|
+
"""Can this job's benchmark say whether anything broke?
|
|
448
|
+
|
|
449
|
+
`is_regression` reads `p2p_fail`, which a benchmark with no reader for
|
|
450
|
+
held checks leaves at 0 — indistinguishable from "nothing broke"
|
|
451
|
+
unless the question of whether it was asked is kept separate. A job
|
|
452
|
+
where some trial reported held checks at all can be counted; one where
|
|
453
|
+
none did shows absence, per the rule that a field a benchmark cannot
|
|
454
|
+
answer is never rendered as a plausible zero.
|
|
455
|
+
"""
|
|
456
|
+
return any(t.p2p_pass or t.p2p_fail for t in self.trials)
|
|
457
|
+
|
|
458
|
+
@property
|
|
459
|
+
def n_resolved(self) -> int:
|
|
460
|
+
return sum(1 for t in self.trials if t.outcome == classify.RESOLVED)
|
|
461
|
+
|
|
462
|
+
@property
|
|
463
|
+
def has_verdict(self) -> bool:
|
|
464
|
+
"""Did anything score this job?
|
|
465
|
+
|
|
466
|
+
A trial's outcome is `unknown` only when it carries no verdict at all
|
|
467
|
+
— no verifier report, no reward and no exception (`classify
|
|
468
|
+
.from_verdict`). A standalone ATIF trajectory or an OTel export is
|
|
469
|
+
nothing but such trials, and "nothing was scored"
|
|
470
|
+
is a different claim from "everything scored missed": the rates
|
|
471
|
+
below are `None` here, and the pages show absence rather than 0%.
|
|
472
|
+
One verified trial is enough — from then on the unknowns stay in the
|
|
473
|
+
denominator, the rule `.claude/rules/running.md` states.
|
|
474
|
+
"""
|
|
475
|
+
return any(t.outcome != classify.UNKNOWN for t in self.trials)
|
|
476
|
+
|
|
477
|
+
@property
|
|
478
|
+
def raw_solve_rate(self) -> float | None:
|
|
479
|
+
"""Resolved ÷ all trials; `None` when no trial carries a verdict."""
|
|
480
|
+
if not self.trials or not self.has_verdict or self.scored_only:
|
|
481
|
+
return None
|
|
482
|
+
return self.n_resolved / len(self.trials)
|
|
483
|
+
|
|
484
|
+
@property
|
|
485
|
+
def adjusted_solve_rate(self) -> float | None:
|
|
486
|
+
"""Solve rate over trials that actually ran (excludes infra failures);
|
|
487
|
+
`None` when no trial carries a verdict."""
|
|
488
|
+
if not self.has_verdict or self.scored_only:
|
|
489
|
+
return None
|
|
490
|
+
eligible = [t for t in self.trials if not t.is_infra]
|
|
491
|
+
return (self.n_resolved / len(eligible)) if eligible else None
|
|
492
|
+
|
|
493
|
+
@property
|
|
494
|
+
def n_infra(self) -> int:
|
|
495
|
+
return sum(1 for t in self.trials if t.is_infra)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def is_preparing_job_dir(path: str) -> bool:
|
|
499
|
+
"""A job directory Harbor has created but not started: its job-level
|
|
500
|
+
``config.json`` (the one naming ``agents`` or ``datasets`` — a trial's names
|
|
501
|
+
a ``task``) with no ``result.json`` yet. Everything between the launch and
|
|
502
|
+
``job.start`` looks like this on disk, and it is the state a person who
|
|
503
|
+
just pressed Run is waiting to see."""
|
|
504
|
+
if os.path.isfile(os.path.join(path, "result.json")):
|
|
505
|
+
return False
|
|
506
|
+
cfg = verifier._read_json(os.path.join(path, "config.json"))
|
|
507
|
+
return isinstance(cfg, dict) and ("agents" in cfg or "datasets" in cfg) and "task" not in cfg
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
def looks_like_job_dir(path: str) -> bool:
|
|
511
|
+
"""A ``result.json`` and either a trial that has one of its own or a live
|
|
512
|
+
feed — or a job that has been launched and not started yet. Harbor writes a
|
|
513
|
+
trial's ``result.json`` when the trial ends, so a job whose first trial is
|
|
514
|
+
still running has none, and that is exactly the job the live board exists
|
|
515
|
+
for: its ``events.jsonl`` is Harbor's own sign that the job is underway, so
|
|
516
|
+
it counts. Before even that, the job-level ``config.json`` alone means
|
|
517
|
+
*preparing* (see :func:`is_preparing_job_dir`). A bare ``result.json``
|
|
518
|
+
still does not count.
|
|
519
|
+
"""
|
|
520
|
+
if not os.path.isfile(os.path.join(path, "result.json")):
|
|
521
|
+
return is_preparing_job_dir(path)
|
|
522
|
+
if os.path.isfile(os.path.join(path, "events.jsonl")):
|
|
523
|
+
return True
|
|
524
|
+
return any(
|
|
525
|
+
os.path.isdir(os.path.join(path, d))
|
|
526
|
+
and os.path.isfile(os.path.join(path, d, "result.json"))
|
|
527
|
+
for d in os.listdir(path)
|
|
528
|
+
)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
def find_job_dirs(roots) -> list[str]:
|
|
532
|
+
"""Return every job directory found under the given root paths.
|
|
533
|
+
|
|
534
|
+
A root may itself be a job dir, or a parent containing job dirs (one level
|
|
535
|
+
down). Both are handled, and results are de-duplicated while preserving
|
|
536
|
+
order, so overlapping roots are safe.
|
|
537
|
+
"""
|
|
538
|
+
found: list[str] = []
|
|
539
|
+
seen: set[str] = set()
|
|
540
|
+
|
|
541
|
+
def add(path: str) -> None:
|
|
542
|
+
real = os.path.realpath(path)
|
|
543
|
+
if real not in seen:
|
|
544
|
+
seen.add(real)
|
|
545
|
+
found.append(path)
|
|
546
|
+
|
|
547
|
+
for root in roots:
|
|
548
|
+
if not os.path.isdir(root):
|
|
549
|
+
continue
|
|
550
|
+
if looks_like_job_dir(root):
|
|
551
|
+
add(root)
|
|
552
|
+
for name in sorted(os.listdir(root)):
|
|
553
|
+
p = os.path.join(root, name)
|
|
554
|
+
if os.path.isdir(p) and looks_like_job_dir(p):
|
|
555
|
+
add(p)
|
|
556
|
+
return found
|
|
557
|
+
|
|
558
|
+
|
|
559
|
+
def _token_totals(res: dict) -> dict:
|
|
560
|
+
"""Token/cost totals for a trial, whichever shape Harbor wrote.
|
|
561
|
+
|
|
562
|
+
Single-step trials carry one AgentContext on ``agent_result``; multi-step
|
|
563
|
+
trials leave that null and record one per entry of ``step_results``.
|
|
564
|
+
Mirrors Harbor's ``TrialResult.compute_token_cost_totals``: cost stays
|
|
565
|
+
None unless some step actually reported one — "unpriced" and "free" are
|
|
566
|
+
different claims.
|
|
567
|
+
"""
|
|
568
|
+
if res.get("agent_result"):
|
|
569
|
+
contexts = [res["agent_result"]]
|
|
570
|
+
else:
|
|
571
|
+
contexts = [
|
|
572
|
+
sr.get("agent_result")
|
|
573
|
+
for sr in res.get("step_results") or []
|
|
574
|
+
if isinstance(sr, dict) and sr.get("agent_result")
|
|
575
|
+
]
|
|
576
|
+
totals = {"n_input_tokens": 0, "n_output_tokens": 0, "n_cache_tokens": 0, "cost_usd": None}
|
|
577
|
+
for ctx in contexts:
|
|
578
|
+
for key in ("n_input_tokens", "n_output_tokens", "n_cache_tokens"):
|
|
579
|
+
totals[key] += ctx.get(key) or 0
|
|
580
|
+
if ctx.get("cost_usd") is not None:
|
|
581
|
+
totals["cost_usd"] = (totals["cost_usd"] or 0.0) + ctx["cost_usd"]
|
|
582
|
+
return totals
|
|
583
|
+
|
|
584
|
+
|
|
585
|
+
def parse_trial(job_id: str, trial_dir: str) -> Trial | None:
|
|
586
|
+
res = verifier._read_json(os.path.join(trial_dir, "result.json"))
|
|
587
|
+
if not res:
|
|
588
|
+
return None
|
|
589
|
+
|
|
590
|
+
task_name = res.get("task_name", os.path.basename(trial_dir))
|
|
591
|
+
task_base = task_name.split("/")[-1]
|
|
592
|
+
exc = res.get("exception_info") or {}
|
|
593
|
+
exc_type = exc.get("exception_type", "") or ""
|
|
594
|
+
|
|
595
|
+
agent_res = _token_totals(res)
|
|
596
|
+
ver_res = res.get("verifier_result") or {}
|
|
597
|
+
reward = None
|
|
598
|
+
if isinstance(ver_res, dict):
|
|
599
|
+
reward = (ver_res.get("rewards") or {}).get("reward")
|
|
600
|
+
|
|
601
|
+
# Older jobs record `source`; newer ones only carry it as the prefix of
|
|
602
|
+
# a "<dataset>/<task>" task name. It selects the verifier loader and
|
|
603
|
+
# reader, so it has to be resolved before the report is touched.
|
|
604
|
+
source = res.get("source") or (task_name.rsplit("/", 1)[0] if "/" in task_name else "")
|
|
605
|
+
|
|
606
|
+
report = verifier.load(source, trial_dir, task_base)
|
|
607
|
+
verdict = verifier.read(source, report, reward)
|
|
608
|
+
outcome = classify.from_verdict(exc_type, verdict)
|
|
609
|
+
rewards = (ver_res.get("rewards") or {}) if isinstance(ver_res, dict) else {}
|
|
610
|
+
score = rewards.get("score") if isinstance(rewards.get("score"), (int, float)) and not isinstance(rewards.get("score"), bool) else None
|
|
611
|
+
# the parts' scores by the field the benchmark breaks its score down by (verifier/parts.json)
|
|
612
|
+
part_groups, parts_by = {}, ""
|
|
613
|
+
pj = verifier._read_json(os.path.join(trial_dir, "verifier", "parts.json"))
|
|
614
|
+
if isinstance(pj, dict) and pj.get("by") and isinstance(pj.get("parts"), list):
|
|
615
|
+
parts_by = str(pj["by"])
|
|
616
|
+
for x in pj["parts"]:
|
|
617
|
+
if isinstance(x, dict) and "group" in x:
|
|
618
|
+
acc = part_groups.setdefault(str(x["group"]), [0.0, 0])
|
|
619
|
+
acc[0] += x["score"] if isinstance(x.get("score"), (int, float)) and not isinstance(x.get("score"), bool) else 0.0
|
|
620
|
+
acc[1] += 1
|
|
621
|
+
# a score and no pass or fail, no exception: the benchmark measured it and gave no verdict to classify
|
|
622
|
+
if outcome == classify.UNKNOWN and score is not None and reward is None and not exc_type:
|
|
623
|
+
outcome = classify.SCORED
|
|
624
|
+
|
|
625
|
+
traj_path = os.path.join(trial_dir, "agent", "trajectory.json")
|
|
626
|
+
trace_path = os.path.join(trial_dir, "agent", "trace.json")
|
|
627
|
+
|
|
628
|
+
phases = {}
|
|
629
|
+
for name in ("environment_setup", "agent_setup", "agent_execution", "verifier"):
|
|
630
|
+
block = res.get(name) or {}
|
|
631
|
+
started, finished = _iso(block.get("started_at")), _iso(block.get("finished_at"))
|
|
632
|
+
if started and finished:
|
|
633
|
+
phases[name] = (started, finished)
|
|
634
|
+
|
|
635
|
+
return Trial(
|
|
636
|
+
job_id=job_id,
|
|
637
|
+
dir=trial_dir,
|
|
638
|
+
task_name=task_name,
|
|
639
|
+
trial_name=res.get("trial_name", os.path.basename(trial_dir)),
|
|
640
|
+
task_base=task_base,
|
|
641
|
+
source=source,
|
|
642
|
+
model_name=(res.get("agent_info", {}).get("model_info", {}) or {}).get("name", ""),
|
|
643
|
+
agent_version=(res.get("agent_info") or {}).get("version") or "",
|
|
644
|
+
reasoning_effort=str((((res.get("config") or {}).get("agent") or {}).get("kwargs") or {}).get("reasoning_effort", "")),
|
|
645
|
+
agent_info_name=str((res.get("agent_info") or {}).get("name") or ""),
|
|
646
|
+
outcome=outcome,
|
|
647
|
+
reward=reward,
|
|
648
|
+
score=score,
|
|
649
|
+
part_groups=part_groups,
|
|
650
|
+
parts_by=parts_by,
|
|
651
|
+
n_parts=(ver_res.get("parts") or {}).get("n") if isinstance(ver_res, dict) and isinstance(ver_res.get("parts"), dict)
|
|
652
|
+
and isinstance((ver_res.get("parts") or {}).get("n"), int) and not isinstance((ver_res.get("parts") or {}).get("n"), bool) else None,
|
|
653
|
+
exception_type=exc_type,
|
|
654
|
+
exception_message=redact(exc.get("exception_message", "")),
|
|
655
|
+
patch_exists=verdict.produced,
|
|
656
|
+
patch_applied=verdict.usable,
|
|
657
|
+
f2p_pass=verdict.gained_pass,
|
|
658
|
+
f2p_fail=verdict.gained_fail,
|
|
659
|
+
p2p_pass=verdict.held_pass,
|
|
660
|
+
p2p_fail=verdict.held_fail,
|
|
661
|
+
f2p_failures=list(verdict.gained_failures),
|
|
662
|
+
p2p_failures=list(verdict.held_failures),
|
|
663
|
+
n_input_tokens=agent_res.get("n_input_tokens") or 0,
|
|
664
|
+
n_output_tokens=agent_res.get("n_output_tokens") or 0,
|
|
665
|
+
n_cache_tokens=agent_res.get("n_cache_tokens") or 0,
|
|
666
|
+
cost_usd=agent_res.get("cost_usd"),
|
|
667
|
+
dur_total=_dur(
|
|
668
|
+
{
|
|
669
|
+
"started_at": res.get("started_at"),
|
|
670
|
+
"finished_at": res.get("finished_at"),
|
|
671
|
+
}
|
|
672
|
+
),
|
|
673
|
+
dur_env=_dur(res.get("environment_setup")),
|
|
674
|
+
dur_agent_setup=_dur(res.get("agent_setup")),
|
|
675
|
+
dur_agent=_dur(res.get("agent_execution")),
|
|
676
|
+
dur_verifier=_dur(res.get("verifier")),
|
|
677
|
+
has_trajectory=os.path.isfile(traj_path),
|
|
678
|
+
has_trace=os.path.isfile(trace_path),
|
|
679
|
+
n_rate_limited_retries=count_rate_limits(os.path.join(trial_dir, "agent", "gateway.log")),
|
|
680
|
+
env_type=((res.get("config") or {}).get("environment") or {}).get("type", ""),
|
|
681
|
+
started_at=_iso(res.get("started_at")),
|
|
682
|
+
finished_at=_iso(res.get("finished_at")),
|
|
683
|
+
phases=phases,
|
|
684
|
+
)
|
|
685
|
+
|
|
686
|
+
|
|
687
|
+
def harbor_command(cfg: dict, env_type: str = "") -> str:
|
|
688
|
+
"""Reconstruct the `harbor run` invocation a job's config.json records.
|
|
689
|
+
|
|
690
|
+
Faithful to the file: only flags the harness actually recorded appear, so
|
|
691
|
+
a copied command reruns what ran rather than what a form would suggest —
|
|
692
|
+
including the `--ak` kwargs, which no other surface reproduces. The one
|
|
693
|
+
outside fact is `--env`, which the job-level config does not carry; the
|
|
694
|
+
caller passes it from a trial's recorded environment when it has one.
|
|
695
|
+
|
|
696
|
+
A job that another harness produced records its own invocation under
|
|
697
|
+
``command`` and it is returned verbatim: reconstructing a ``harbor run``
|
|
698
|
+
for a run Harbor never performed would print a line that cannot reproduce
|
|
699
|
+
it, which is the one thing this function exists not to do. No Harbor
|
|
700
|
+
config carries that key, so this cannot shadow a real one.
|
|
701
|
+
"""
|
|
702
|
+
recorded = cfg.get("command")
|
|
703
|
+
if isinstance(recorded, str) and recorded.strip():
|
|
704
|
+
return recorded.strip()
|
|
705
|
+
|
|
706
|
+
agents = cfg.get("agents") or [{}]
|
|
707
|
+
agent = agents[0]
|
|
708
|
+
name = agent.get("name", "")
|
|
709
|
+
if not name:
|
|
710
|
+
return ""
|
|
711
|
+
|
|
712
|
+
# No PYTHONPATH is reconstructed, whatever the recorded name looks like: the
|
|
713
|
+
# lane in service is Harbor's built-in agent, and a module:Class name in an
|
|
714
|
+
# old job is a fact about that job, not an instruction to rebuild the
|
|
715
|
+
# adapter checkout it once resolved in. The --agent value is what the job
|
|
716
|
+
# recorded, verbatim.
|
|
717
|
+
parts = ["harbor", "run"]
|
|
718
|
+
|
|
719
|
+
n_tasks = 0
|
|
720
|
+
datasets = cfg.get("datasets") or []
|
|
721
|
+
if datasets:
|
|
722
|
+
d0 = datasets[0]
|
|
723
|
+
if d0.get("path"):
|
|
724
|
+
parts += ["-p", shlex.quote(d0["path"])]
|
|
725
|
+
elif d0.get("name"):
|
|
726
|
+
ref = d0["name"] + (f"@{d0['version']}" if d0.get("version") else "")
|
|
727
|
+
parts += ["-d", shlex.quote(ref)]
|
|
728
|
+
n_tasks = d0.get("n_tasks") or 0
|
|
729
|
+
|
|
730
|
+
if env_type:
|
|
731
|
+
parts += ["--env", shlex.quote(env_type)]
|
|
732
|
+
if agent.get("model_name"):
|
|
733
|
+
parts += ["--model", shlex.quote(agent["model_name"])]
|
|
734
|
+
parts += ["--agent", shlex.quote(name)]
|
|
735
|
+
for key, value in (agent.get("kwargs") or {}).items():
|
|
736
|
+
parts += ["--ak", shlex.quote(f"{key}={value}")]
|
|
737
|
+
if n_tasks:
|
|
738
|
+
parts += ["--n-tasks", str(n_tasks)]
|
|
739
|
+
if cfg.get("n_concurrent_trials"):
|
|
740
|
+
parts += ["--n-concurrent", str(cfg["n_concurrent_trials"])]
|
|
741
|
+
if (cfg.get("retry") or {}).get("max_retries"):
|
|
742
|
+
parts += ["--max-retries", str(cfg["retry"]["max_retries"])]
|
|
743
|
+
return " ".join(parts)
|
|
744
|
+
|
|
745
|
+
|
|
746
|
+
# Harbor writes config.json with `exclude_defaults=True` (harbor/job.py,
|
|
747
|
+
# 0.22): a key it leaves out held Harbor's default, which is not the
|
|
748
|
+
# builder's (concurrency 4 and no retries here, 1 and 3 there). Read from
|
|
749
|
+
# harbor/models/job/config.py at 0.22; a launch says which ones it filled.
|
|
750
|
+
HARBOR_DEFAULTS = {"n_concurrent_trials": 4, "max_retries": 0, "n_attempts": 1, "timeout_multiplier": 1.0}
|
|
751
|
+
_PHASE_MULTIPLIERS = ("agent_timeout_multiplier", "agent_setup_timeout_multiplier",
|
|
752
|
+
"verifier_timeout_multiplier", "environment_build_timeout_multiplier")
|
|
753
|
+
# A launch carries only the name of a value filed under a name like these: it
|
|
754
|
+
# is a credential, and the record is embedded in the builder's page.
|
|
755
|
+
_SECRET_KEY = re.compile(r"(?:api[-_]?key|token|password|authorization|credential|secret)", re.I)
|
|
756
|
+
# What a recipe can say; any other key a config recorded is named, not dropped.
|
|
757
|
+
_LAUNCH_TOP = {"job_name", "jobs_dir", "quiet", "debug", "agents", "datasets", "tasks", "retry",
|
|
758
|
+
"n_concurrent_trials", "n_attempts", "timeout_multiplier", "environment", *_PHASE_MULTIPLIERS}
|
|
759
|
+
_LAUNCH_AGENT = {"name", "import_path", "model_name", "kwargs", "env"}
|
|
760
|
+
_LAUNCH_DATASET = {"path", "name", "version", "task_names", "exclude_task_names", "n_tasks"}
|
|
761
|
+
|
|
762
|
+
|
|
763
|
+
def _scrub(values, at: str = "") -> tuple:
|
|
764
|
+
"""(kept, secret names): string values redacted, credential-named ones left
|
|
765
|
+
out - at any depth, since a kwarg may nest them (``llm_kwargs.api_key``,
|
|
766
|
+
an MCP server's ``headers.Authorization``); a nested one is named by its path."""
|
|
767
|
+
kept, secret = {}, []
|
|
768
|
+
for key, value in (values if isinstance(values, dict) else {}).items():
|
|
769
|
+
name = f"{at}{key}"
|
|
770
|
+
if _SECRET_KEY.search(str(key)):
|
|
771
|
+
secret.append(name)
|
|
772
|
+
else:
|
|
773
|
+
kept[str(key)], inner = _scrub_value(value, name)
|
|
774
|
+
secret += inner
|
|
775
|
+
return kept, secret
|
|
776
|
+
|
|
777
|
+
|
|
778
|
+
def _scrub_value(value, at: str) -> tuple:
|
|
779
|
+
if isinstance(value, str):
|
|
780
|
+
return redact(value), []
|
|
781
|
+
if isinstance(value, dict):
|
|
782
|
+
return _scrub(value, at + ".")
|
|
783
|
+
if isinstance(value, list):
|
|
784
|
+
out, secret = [], []
|
|
785
|
+
for i, v in enumerate(value):
|
|
786
|
+
v, inner = _scrub_value(v, f"{at}[{i}]")
|
|
787
|
+
out.append(v); secret += inner
|
|
788
|
+
return out, secret
|
|
789
|
+
return value, []
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def launch_of(cfg: dict) -> dict:
|
|
793
|
+
"""How a job was launched, from its config.json: the agents with their
|
|
794
|
+
models, kwargs and environment, the first dataset and its task filters,
|
|
795
|
+
the limits and the timeout multipliers - Harbor's defaults filled in for
|
|
796
|
+
the keys it left out (``defaulted`` names them), credential-named values
|
|
797
|
+
left out (``secret`` names them per agent), and every recorded key a
|
|
798
|
+
builder recipe has no field for named in ``unexpressed``. Empty for a
|
|
799
|
+
config another harness wrote (``command``) and for no config at all."""
|
|
800
|
+
if not isinstance(cfg, dict) or not cfg or cfg.get("command"):
|
|
801
|
+
return {}
|
|
802
|
+
unexpressed = sorted(k for k in cfg if k not in _LAUNCH_TOP)
|
|
803
|
+
agents = []
|
|
804
|
+
for i, a in enumerate(cfg.get("agents") or []):
|
|
805
|
+
if not isinstance(a, dict):
|
|
806
|
+
continue
|
|
807
|
+
kwargs, secret = _scrub(a.get("kwargs"))
|
|
808
|
+
env, secret_env = _scrub(a.get("env"))
|
|
809
|
+
agents.append({"name": a.get("name") or a.get("import_path") or "", "model": a.get("model_name") or "",
|
|
810
|
+
"kwargs": kwargs, "env": env, "secret": secret + secret_env})
|
|
811
|
+
unexpressed += [f"agents[{i}].{k}" for k in sorted(a) if k not in _LAUNCH_AGENT]
|
|
812
|
+
datasets = [d for d in cfg.get("datasets") or [] if isinstance(d, dict)]
|
|
813
|
+
d0 = datasets[0] if datasets else {}
|
|
814
|
+
dataset = ({"path": redact(str(d0["path"]))} if d0.get("path") else
|
|
815
|
+
{"name": str(d0["name"]), "version": str(d0.get("version") or "")} if d0.get("name") else {})
|
|
816
|
+
unexpressed += [f"datasets[0].{k}" for k in sorted(d0) if k not in _LAUNCH_DATASET]
|
|
817
|
+
unexpressed += [f"datasets[{i}]" for i in range(1, len(datasets))]
|
|
818
|
+
retry = cfg.get("retry") if isinstance(cfg.get("retry"), dict) else {}
|
|
819
|
+
unexpressed += [f"retry.{k}" for k in sorted(retry) if k != "max_retries"]
|
|
820
|
+
environment = cfg.get("environment") if isinstance(cfg.get("environment"), dict) else {}
|
|
821
|
+
unexpressed += [f"environment.{k}" for k in sorted(environment) if k != "type"]
|
|
822
|
+
recorded = {"n_concurrent_trials": cfg.get("n_concurrent_trials"), "max_retries": retry.get("max_retries"),
|
|
823
|
+
"n_attempts": cfg.get("n_attempts"), "timeout_multiplier": cfg.get("timeout_multiplier")}
|
|
824
|
+
return {
|
|
825
|
+
"agents": agents,
|
|
826
|
+
"dataset": dataset,
|
|
827
|
+
"tasks": len(cfg.get("tasks") or []),
|
|
828
|
+
"task_names": [str(t) for t in d0.get("task_names") or []],
|
|
829
|
+
"exclude_task_names": [str(t) for t in d0.get("exclude_task_names") or []],
|
|
830
|
+
"n_tasks": d0.get("n_tasks") or None,
|
|
831
|
+
**{k: (HARBOR_DEFAULTS[k] if v is None else v) for k, v in recorded.items()},
|
|
832
|
+
**{k: cfg.get(k) for k in _PHASE_MULTIPLIERS},
|
|
833
|
+
"environment": str(environment.get("type") or ""),
|
|
834
|
+
"defaulted": [k for k, v in recorded.items() if v is None],
|
|
835
|
+
"unexpressed": unexpressed,
|
|
836
|
+
}
|
|
837
|
+
|
|
838
|
+
|
|
839
|
+
def _budget(b) -> dict:
|
|
840
|
+
"""A job's recorded spending cap, read leniently: {} unless it names a cap; an amount that is no number is
|
|
841
|
+
None (not reported), never zero."""
|
|
842
|
+
num = lambda v: v if isinstance(v, (int, float)) and not isinstance(v, bool) else None
|
|
843
|
+
if not isinstance(b, dict) or not (num(b.get("max_usd")) or num(b.get("unit_max_usd"))):
|
|
844
|
+
return {}
|
|
845
|
+
names = lambda v: [str(x) for x in v] if isinstance(v, list) else []
|
|
846
|
+
return {"max_usd": num(b.get("max_usd")), "spent_usd": num(b.get("spent_usd")), "unsettled_usd": num(b.get("unsettled_usd")),
|
|
847
|
+
# only the run's cap stops a run: without one there is no stop to say
|
|
848
|
+
"stopped": b.get("stopped") is True and num(b.get("max_usd")) is not None,
|
|
849
|
+
"cut_off": names(b.get("cut_off")), "not_run": names(b.get("not_run")),
|
|
850
|
+
"unit_max_usd": num(b.get("unit_max_usd")), "over_unit": names(b.get("over_unit")),
|
|
851
|
+
# a merged job: its shards, and how many stopped at their share
|
|
852
|
+
"shards": num(b.get("shards")) or 0, "shards_stopped": num(b.get("shards_stopped")) or 0}
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def parse_job(job_dir: str, reuse=None) -> Job:
|
|
856
|
+
"""One job directory. ``reuse(trial_dir)``: a Trial already parsed from
|
|
857
|
+
files that have not changed since (a workspace's cache, for a running
|
|
858
|
+
job), or None to parse it."""
|
|
859
|
+
res = verifier._read_json(os.path.join(job_dir, "result.json")) or {}
|
|
860
|
+
cfg = verifier._read_json(os.path.join(job_dir, "config.json")) or {}
|
|
861
|
+
stats = res.get("stats") or {}
|
|
862
|
+
|
|
863
|
+
agents = cfg.get("agents") or [{}]
|
|
864
|
+
model = agents[0].get("model_name", "")
|
|
865
|
+
|
|
866
|
+
# Three config shapes in the wild: `datasets: [{path, n_tasks}]` for a
|
|
867
|
+
# local checkout, `datasets: [{name, version, n_tasks}]` for a registry
|
|
868
|
+
# fetch (no `path` key at all), and `tasks: [{name: "<dataset>/<task>"}]`
|
|
869
|
+
# where the dataset is only a prefix.
|
|
870
|
+
datasets = cfg.get("datasets") or []
|
|
871
|
+
tasks = cfg.get("tasks") or []
|
|
872
|
+
dataset_path = ""
|
|
873
|
+
if datasets:
|
|
874
|
+
d0 = datasets[0]
|
|
875
|
+
dataset = dataset_path = d0.get("path", "")
|
|
876
|
+
if not dataset:
|
|
877
|
+
name, version = d0.get("name", ""), d0.get("version", "")
|
|
878
|
+
dataset = f"{name}@{version}" if name and version else name
|
|
879
|
+
n_tasks = d0.get("n_tasks") or 0
|
|
880
|
+
elif tasks:
|
|
881
|
+
prefixes = {
|
|
882
|
+
t["name"].rsplit("/", 1)[0]
|
|
883
|
+
for t in tasks
|
|
884
|
+
if isinstance(t.get("name"), str) and "/" in t["name"]
|
|
885
|
+
}
|
|
886
|
+
dataset = prefixes.pop() if len(prefixes) == 1 else ("mixed" if prefixes else "")
|
|
887
|
+
n_tasks = len(tasks)
|
|
888
|
+
else:
|
|
889
|
+
dataset, n_tasks = "", 0
|
|
890
|
+
|
|
891
|
+
job = Job(
|
|
892
|
+
job_id=os.path.basename(job_dir),
|
|
893
|
+
dir=job_dir,
|
|
894
|
+
model_name=model,
|
|
895
|
+
dataset=os.path.basename(dataset.rstrip("/")) if dataset else "",
|
|
896
|
+
n_tasks=n_tasks,
|
|
897
|
+
started_at=_iso(res.get("started_at")),
|
|
898
|
+
finished_at=_iso(res.get("finished_at")),
|
|
899
|
+
agent_name=agents[0].get("name", ""),
|
|
900
|
+
reasoning_effort=str((agents[0].get("kwargs") or {}).get("reasoning_effort", "")),
|
|
901
|
+
n_total_trials=res.get("n_total_trials") or 0,
|
|
902
|
+
n_errored_trials=stats.get("n_errored_trials") or 0,
|
|
903
|
+
n_running_trials=stats.get("n_running_trials") or 0,
|
|
904
|
+
n_pending_trials=stats.get("n_pending_trials") or 0,
|
|
905
|
+
n_retries=stats.get("n_retries") or 0,
|
|
906
|
+
preparing=is_preparing_job_dir(job_dir),
|
|
907
|
+
# what a job recorded about itself is shown and copied verbatim, so a
|
|
908
|
+
# key that leaked into a path or an importer's --command is scrubbed here
|
|
909
|
+
dataset_path=redact(dataset_path),
|
|
910
|
+
)
|
|
911
|
+
|
|
912
|
+
for name in sorted(os.listdir(job_dir)):
|
|
913
|
+
sub = os.path.join(job_dir, name)
|
|
914
|
+
if os.path.isdir(sub) and os.path.isfile(os.path.join(sub, "result.json")):
|
|
915
|
+
t = (reuse(sub) if reuse else None) or parse_trial(job.job_id, sub)
|
|
916
|
+
if t:
|
|
917
|
+
job.trials.append(t)
|
|
918
|
+
if not job.agent_name:
|
|
919
|
+
# a config written with defaults left out names no agent for Harbor's default one; the trials do
|
|
920
|
+
job.agent_name = next((t.agent_info_name for t in job.trials if t.agent_info_name), "")
|
|
921
|
+
job.runner = str(cfg.get("runner_label") or cfg.get("runner") or "") if isinstance(cfg.get("runner"), str) else ""
|
|
922
|
+
job.runner_commit = str(cfg.get("runner_commit") or "")[:12] if job.runner else ""
|
|
923
|
+
job.runner_judge = str(cfg.get("runner_judge") or "") if job.runner else ""
|
|
924
|
+
job.questions_from = str(cfg.get("questions_from") or "") if job.runner else ""
|
|
925
|
+
words = cfg.get("runner_items") if job.runner else None
|
|
926
|
+
job.runner_items = [str(w) for w in words[:2]] if isinstance(words, list) and len(words) >= 2 else []
|
|
927
|
+
std = cfg.get("standard") if isinstance(cfg.get("standard"), dict) else {}
|
|
928
|
+
ctl = std.get("control") if isinstance(std.get("control"), dict) else {}
|
|
929
|
+
job.control_source = str(ctl.get("source") or "control") if ctl.get("name") == job.agent_label.lower() else ""
|
|
930
|
+
job.launch_id = str(std.get("launch") or "")
|
|
931
|
+
merged = std.get("merged_from")
|
|
932
|
+
job.merged_from = [s["job"] for s in merged if isinstance(s, dict) and isinstance(s.get("job"), str) and s["job"]] if isinstance(merged, list) else []
|
|
933
|
+
job.budget = _budget(res.get("budget"))
|
|
934
|
+
job.standard = bool(std)
|
|
935
|
+
# a runner's job was never a `harbor run`: there is no command to rebuild
|
|
936
|
+
job.harbor_command = "" if job.runner else redact(harbor_command(
|
|
937
|
+
cfg, next((t.env_type for t in job.trials if t.env_type), "")))
|
|
938
|
+
job.launch = launch_of(cfg)
|
|
939
|
+
if len(agents) > 1 and not job.runner:
|
|
940
|
+
job.agents = [{"name": a.get("name") or a.get("import_path") or "", "model": a.get("model_name") or "",
|
|
941
|
+
"effort": str((a.get("kwargs") or {}).get("reasoning_effort", "")),
|
|
942
|
+
"command": redact(harbor_command(dict(cfg, agents=[a]),
|
|
943
|
+
next((t.env_type for t in job.trials if t.env_type), "")))}
|
|
944
|
+
for a in agents if isinstance(a, dict)]
|
|
945
|
+
lock = verifier._read_json(os.path.join(job_dir, "lock.json")) or {}
|
|
946
|
+
harbor = lock.get("harbor") if isinstance(lock.get("harbor"), dict) else {}
|
|
947
|
+
job.harbor_version = str(harbor.get("version") or "")
|
|
948
|
+
job.harbor_editable = harbor.get("is_editable") if isinstance(harbor.get("is_editable"), bool) else None
|
|
949
|
+
return job
|
|
950
|
+
|
|
951
|
+
|
|
952
|
+
def _agent_of(t, agents: list) -> int:
|
|
953
|
+
"""Which of a job's agents ran trial ``t``: by its model, then its name."""
|
|
954
|
+
def named(a):
|
|
955
|
+
return t.agent_info_name and t.agent_info_name in (a["name"], a["name"].split(":")[-1])
|
|
956
|
+
by_model = [i for i, a in enumerate(agents) if a["model"] == t.model_name]
|
|
957
|
+
if len(by_model) == 1:
|
|
958
|
+
return by_model[0]
|
|
959
|
+
pool = by_model or list(range(len(agents)))
|
|
960
|
+
named_pool = [i for i in pool if named(agents[i])]
|
|
961
|
+
pool = named_pool or pool
|
|
962
|
+
if len(pool) == 1:
|
|
963
|
+
return pool[0]
|
|
964
|
+
exact = [i for i in pool if agents[i]["effort"] == t.reasoning_effort]
|
|
965
|
+
return exact[0] if len(exact) == 1 else -1
|
|
966
|
+
|
|
967
|
+
|
|
968
|
+
def split_agents(jobs: list) -> list:
|
|
969
|
+
"""A Harbor job of several agents or models as one view per agent, in
|
|
970
|
+
their config's order: each its own trials, rate, control or not, and
|
|
971
|
+
command. Pooled under its first agent, model-b's misses were shown as
|
|
972
|
+
model-a's and an oracle listed first made the real agent a control. A
|
|
973
|
+
view's id is the job's with the agent after ``~``; its trials link to it.
|
|
974
|
+
Harbor's job-level counts are shared evenly; the rest is the view's own."""
|
|
975
|
+
out = []
|
|
976
|
+
for job in jobs:
|
|
977
|
+
if len(job.agents) < 2:
|
|
978
|
+
out.append(job)
|
|
979
|
+
continue
|
|
980
|
+
k = len(job.agents)
|
|
981
|
+
groups = {i: [] for i in range(k)}
|
|
982
|
+
for t in job.trials:
|
|
983
|
+
groups.setdefault(_agent_of(t, job.agents), []).append(t)
|
|
984
|
+
labels = set()
|
|
985
|
+
for i, a in enumerate(job.agents):
|
|
986
|
+
label = re.sub(r"[^A-Za-z0-9._-]+", "-", f"{a['name'].split(':')[-1]}-{a['model']}".strip("-")) or str(i + 1)
|
|
987
|
+
if label in labels:
|
|
988
|
+
label += f"-{i + 1}"
|
|
989
|
+
labels.add(label)
|
|
990
|
+
vid = f"{job.job_id}~{label}"
|
|
991
|
+
trials = [replace(t, job_id=vid) for t in groups[i]]
|
|
992
|
+
out.append(replace(
|
|
993
|
+
job, job_id=vid, model_name=a["model"], agent_name=a["name"], reasoning_effort=a["effort"],
|
|
994
|
+
trials=trials, harbor_command=a["command"], agents=[],
|
|
995
|
+
n_total_trials=job.n_total_trials // k if job.n_total_trials % k == 0 else len(trials),
|
|
996
|
+
n_errored_trials=sum(1 for t in trials if t.exception_type),
|
|
997
|
+
n_running_trials=job.n_running_trials // k + (i < job.n_running_trials % k),
|
|
998
|
+
n_pending_trials=job.n_pending_trials // k + (i < job.n_pending_trials % k),
|
|
999
|
+
n_retries=job.n_retries // k + (i < job.n_retries % k)))
|
|
1000
|
+
orphans = groups.get(-1)
|
|
1001
|
+
if orphans: # a trial of an agent the config does not list: kept, under the job's own id
|
|
1002
|
+
out.append(replace(job, trials=orphans, agents=[]))
|
|
1003
|
+
return out
|
|
1004
|
+
|
|
1005
|
+
|
|
1006
|
+
def load_all(roots) -> list[Job]:
|
|
1007
|
+
jobs = split_agents([parse_job(d) for d in find_job_dirs(roots)])
|
|
1008
|
+
# Newest first. Compared as epoch numbers, not datetimes: a job whose
|
|
1009
|
+
# result.json never got a started_at (a crashed run) yields None, and
|
|
1010
|
+
# datetime.min is naive where parsed stamps are aware — mixing the two
|
|
1011
|
+
# in one sort raises. -inf sorts the undated to the end instead.
|
|
1012
|
+
jobs.sort(
|
|
1013
|
+
key=lambda j: j.started_at.timestamp() if j.started_at else float("-inf"),
|
|
1014
|
+
reverse=True,
|
|
1015
|
+
)
|
|
1016
|
+
return jobs
|