nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
nullius/runs/live.py
ADDED
|
@@ -0,0 +1,715 @@
|
|
|
1
|
+
"""The live board's feed and fold: ``events.jsonl`` → the snapshot the job page renders.
|
|
2
|
+
|
|
3
|
+
A running job should look like something running. Every Harbor run from
|
|
4
|
+
here appends one ``events.jsonl`` per job from Harbor's own trial hooks
|
|
5
|
+
(``nullius.harbor.feed``, whatever the Harbor); this application reads that file
|
|
6
|
+
— never a socket, never an agent runtime — and folds it into a board: per-trial
|
|
7
|
+
phase, time in phase, counters, outcome; job counts; a rate; an ETA. The board is
|
|
8
|
+
the job page's Live view, beside Results; when the feed records ``job.end`` it
|
|
9
|
+
stops polling and Results becomes the view the page opens on.
|
|
10
|
+
|
|
11
|
+
The feed schema, the fold and the snapshot document are pinned by the
|
|
12
|
+
fixtures under ``app/tests/fixtures/live/``: each feed there folds to its
|
|
13
|
+
``expected.json``, byte for byte as JSON (``app/tests/test_live.py``).
|
|
14
|
+
|
|
15
|
+
Every line carries ``ts`` (ISO-8601 UTC), ``seq`` (int, strictly increasing
|
|
16
|
+
within the file), ``job``, ``trial`` (``None`` for job events) and ``event``,
|
|
17
|
+
one of :data:`EVENTS` — ``job.prepare`` is the one written at launch, before
|
|
18
|
+
the launcher has fetched a dataset or built an image, so a board shows the
|
|
19
|
+
preparation window instead of nothing (its ``stage`` names what is being
|
|
20
|
+
prepared; ``job.start`` keeps meaning "trials are about to be scheduled").
|
|
21
|
+
Payloads are read leniently — a missing counter is *absent*, never zero.
|
|
22
|
+
|
|
23
|
+
Four rules are load-bearing, and each has a test:
|
|
24
|
+
|
|
25
|
+
- **The last phase event wins**, per trial, and time in phase is ``now`` minus
|
|
26
|
+
that event's ``ts`` — the board never keeps its own clock.
|
|
27
|
+
- **Counters are the latest progress values, and an unreported one is
|
|
28
|
+
absent.** On the feed a writer *omits* a counter it does not have (a present
|
|
29
|
+
``null`` is read the same way, as not reported); in the snapshot every
|
|
30
|
+
counter key is present and ``null`` until something reported it — never
|
|
31
|
+
``0``. ``cost_usd`` is the case that matters: a zero there is a lie about
|
|
32
|
+
money, and the page renders ``null`` as "not reported".
|
|
33
|
+
- **The file is read as it grows.** A torn last line (the writer mid-append)
|
|
34
|
+
is dropped and noted, never raised; everything before it folds.
|
|
35
|
+
- **Free text is redacted on read**, with the same boundary-anchored patterns
|
|
36
|
+
every other reader here uses (``redact.redact``), so the page can render
|
|
37
|
+
``activity`` and ``line`` without a second scrubber.
|
|
38
|
+
|
|
39
|
+
A fifth rule covers a job that runs more than one agent (lanes and a winner
|
|
40
|
+
per task), and it too is folded from the same lines:
|
|
41
|
+
|
|
42
|
+
- **Lanes and winners are the fold's, not the page's.** The snapshot carries
|
|
43
|
+
one ``lane`` per agent — the agents ``job.start`` declared, in that order,
|
|
44
|
+
then any agent first seen on a trial the declaration did not name, then one
|
|
45
|
+
unnamed lane for trials whose agent never appears — each with its own
|
|
46
|
+
counts, mean reward, totals and win count over its trials. It carries one
|
|
47
|
+
``winners`` entry per task that more than one agent ran: the best reward
|
|
48
|
+
each agent reached on it so far, ``decided`` once every trial of that task
|
|
49
|
+
has an outcome, and ``winner`` only when decided and one agent's best is
|
|
50
|
+
strictly above every other's — a tie names nobody, and so does a task no
|
|
51
|
+
agent scored. A single-agent job has one lane and nothing to win.
|
|
52
|
+
"""
|
|
53
|
+
|
|
54
|
+
from __future__ import annotations
|
|
55
|
+
|
|
56
|
+
import json
|
|
57
|
+
import statistics
|
|
58
|
+
from dataclasses import dataclass, field
|
|
59
|
+
from datetime import datetime, timezone
|
|
60
|
+
from pathlib import Path
|
|
61
|
+
|
|
62
|
+
from nullius.redact import redact
|
|
63
|
+
|
|
64
|
+
EVENTS = ("job.prepare", "job.start", "trial.queued", "trial.phase", "trial.progress",
|
|
65
|
+
"trial.log", "trial.end", "job.end")
|
|
66
|
+
PHASES = ("env_setup", "agent_setup", "agent_run", "verify")
|
|
67
|
+
#: ``scored`` is a runner's: one whose
|
|
68
|
+
#: benchmark gives a trial a score and no pass or fail ends it so - neither
|
|
69
|
+
#: resolved nor unresolved - with ``score`` on the event and, when that is a
|
|
70
|
+
#: mean over the trial's parts, how many (``parts``). The snapshot says so only
|
|
71
|
+
#: where there is one: a ``scored`` count, and ``score`` / ``parts`` on that
|
|
72
|
+
#: trial. A feed without one folds to a snapshot with none of those keys.
|
|
73
|
+
#: ``budget`` is a run's spending cap: a unit its relay cut off there (the
|
|
74
|
+
#: standard's ``--max-usd``), outside the rates as ``infra_error`` is.
|
|
75
|
+
OUTCOMES = ("resolved", "unresolved", "scored", "agent_error", "infra_error",
|
|
76
|
+
"cancelled", "timeout", "budget")
|
|
77
|
+
ERROR_OUTCOMES = {"agent_error", "infra_error", "timeout", "budget"}
|
|
78
|
+
COUNTERS = ("step", "n_input_tokens", "n_cache_tokens", "n_output_tokens", "cost_usd")
|
|
79
|
+
TOKEN_FIELDS = ("n_input_tokens", "n_cache_tokens", "n_output_tokens")
|
|
80
|
+
|
|
81
|
+
#: The rate window and the tail the snapshot carries.
|
|
82
|
+
RATE_WINDOW_SEC = 30.0
|
|
83
|
+
TAIL_EVENTS = 50
|
|
84
|
+
ACTIVITY_MAX = 120
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
class FeedError(ValueError):
|
|
88
|
+
"""A line that is not a feed line. Raised for a complete, malformed line;
|
|
89
|
+
a torn last line is a notice instead (see :func:`read_events`)."""
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
@dataclass(frozen=True)
|
|
93
|
+
class Event:
|
|
94
|
+
ts: datetime
|
|
95
|
+
seq: int
|
|
96
|
+
job: str
|
|
97
|
+
trial: str | None
|
|
98
|
+
event: str
|
|
99
|
+
data: dict
|
|
100
|
+
|
|
101
|
+
def to_dict(self) -> dict:
|
|
102
|
+
return {"ts": _iso(self.ts), "seq": self.seq, "job": self.job,
|
|
103
|
+
"trial": self.trial, "event": self.event, **self.data}
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _iso(ts: datetime) -> str:
|
|
107
|
+
return ts.astimezone(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def parse_ts(value) -> datetime:
|
|
111
|
+
if not isinstance(value, str):
|
|
112
|
+
raise FeedError(f"ts must be an ISO-8601 string, got {value!r}")
|
|
113
|
+
text = value[:-1] + "+00:00" if value.endswith("Z") else value
|
|
114
|
+
try:
|
|
115
|
+
ts = datetime.fromisoformat(text)
|
|
116
|
+
except ValueError as e:
|
|
117
|
+
raise FeedError(f"ts is not ISO-8601: {value!r}") from e
|
|
118
|
+
if ts.tzinfo is None:
|
|
119
|
+
raise FeedError(f"ts must carry a UTC offset: {value!r}")
|
|
120
|
+
return ts.astimezone(timezone.utc)
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def parse_line(line: str) -> Event:
|
|
124
|
+
"""One feed line to one :class:`Event`; :class:`FeedError` when it is not one."""
|
|
125
|
+
try:
|
|
126
|
+
raw = json.loads(line)
|
|
127
|
+
except json.JSONDecodeError as e:
|
|
128
|
+
raise FeedError(f"not JSON: {e.msg}") from e
|
|
129
|
+
if not isinstance(raw, dict):
|
|
130
|
+
raise FeedError("a feed line is a JSON object")
|
|
131
|
+
for key in ("ts", "seq", "job", "event"):
|
|
132
|
+
if key not in raw:
|
|
133
|
+
raise FeedError(f"missing {key!r}")
|
|
134
|
+
if raw["event"] not in EVENTS:
|
|
135
|
+
raise FeedError(f"unknown event {raw['event']!r}")
|
|
136
|
+
if not isinstance(raw["seq"], int) or isinstance(raw["seq"], bool):
|
|
137
|
+
raise FeedError(f"seq must be an int, got {raw['seq']!r}")
|
|
138
|
+
trial = raw.get("trial")
|
|
139
|
+
if raw["event"].startswith("trial.") and not isinstance(trial, str):
|
|
140
|
+
raise FeedError(f"{raw['event']} needs a trial")
|
|
141
|
+
if raw["event"].startswith("job.") and trial is not None:
|
|
142
|
+
raise FeedError(f"{raw['event']} carries no trial")
|
|
143
|
+
data = {k: v for k, v in raw.items() if k not in ("ts", "seq", "job", "trial", "event")}
|
|
144
|
+
if raw["event"] == "trial.phase" and data.get("phase") not in PHASES:
|
|
145
|
+
raise FeedError(f"unknown phase {data.get('phase')!r}")
|
|
146
|
+
if raw["event"] == "trial.end" and data.get("outcome") not in OUTCOMES:
|
|
147
|
+
raise FeedError(f"unknown outcome {data.get('outcome')!r}")
|
|
148
|
+
for key in ("activity", "line"):
|
|
149
|
+
if isinstance(data.get(key), str):
|
|
150
|
+
data[key] = redact(data[key])[:ACTIVITY_MAX] if key == "activity" else redact(data[key])
|
|
151
|
+
return Event(parse_ts(raw["ts"]), raw["seq"], str(raw["job"]), trial, raw["event"], data)
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
@dataclass
|
|
155
|
+
class Feed:
|
|
156
|
+
events: list[Event]
|
|
157
|
+
notices: list[str] = field(default_factory=list)
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def read_events(path: Path | str) -> Feed:
|
|
161
|
+
"""Every complete line of the file, in ``seq`` order.
|
|
162
|
+
|
|
163
|
+
A last line with no trailing newline that fails to parse is the writer
|
|
164
|
+
mid-append: dropped, with a notice. Any other malformed line raises — a
|
|
165
|
+
feed with a bad line in the middle is a broken writer, not a race. A
|
|
166
|
+
``seq`` that does not strictly increase is a notice too, and the events
|
|
167
|
+
are folded in ``seq`` order regardless.
|
|
168
|
+
"""
|
|
169
|
+
text = Path(path).read_text(encoding="utf-8")
|
|
170
|
+
lines = text.split("\n")
|
|
171
|
+
torn = lines[-1] if lines and lines[-1] != "" else None
|
|
172
|
+
complete = lines[:-1] if lines else []
|
|
173
|
+
events: list[Event] = []
|
|
174
|
+
notices: list[str] = []
|
|
175
|
+
for n, line in enumerate(complete, 1):
|
|
176
|
+
if not line.strip():
|
|
177
|
+
continue
|
|
178
|
+
try:
|
|
179
|
+
events.append(parse_line(line))
|
|
180
|
+
except FeedError as e:
|
|
181
|
+
raise FeedError(f"line {n}: {e}") from e
|
|
182
|
+
if torn is not None:
|
|
183
|
+
try:
|
|
184
|
+
events.append(parse_line(torn))
|
|
185
|
+
except FeedError:
|
|
186
|
+
notices.append("last line torn (writer mid-append); dropped")
|
|
187
|
+
seqs = [e.seq for e in events]
|
|
188
|
+
if seqs != sorted(seqs) or len(set(seqs)) != len(seqs):
|
|
189
|
+
notices.append("seq not strictly increasing; folded in seq order")
|
|
190
|
+
events.sort(key=lambda e: e.seq)
|
|
191
|
+
return Feed(events, notices)
|
|
192
|
+
|
|
193
|
+
|
|
194
|
+
|
|
195
|
+
@dataclass
|
|
196
|
+
class TrialState:
|
|
197
|
+
trial: str
|
|
198
|
+
task: str | None = None
|
|
199
|
+
agent: str | None = None
|
|
200
|
+
model: str | None = None
|
|
201
|
+
position: int | None = None
|
|
202
|
+
phase: str | None = None #: current phase, None while queued
|
|
203
|
+
since: datetime | None = None #: ts of the winning phase event
|
|
204
|
+
timeout_sec: float | None = None
|
|
205
|
+
phases_done: list[str] = field(default_factory=list)
|
|
206
|
+
step: int | None = None
|
|
207
|
+
n_input_tokens: int | None = None
|
|
208
|
+
n_cache_tokens: int | None = None
|
|
209
|
+
n_output_tokens: int | None = None
|
|
210
|
+
cost_usd: float | None = None
|
|
211
|
+
activity: str | None = None
|
|
212
|
+
last_line: str | None = None
|
|
213
|
+
outcome: str | None = None
|
|
214
|
+
reward: float | None = None
|
|
215
|
+
score: float | None = None
|
|
216
|
+
parts: int | None = None
|
|
217
|
+
durations: dict = field(default_factory=dict)
|
|
218
|
+
started_at: datetime | None = None
|
|
219
|
+
attempt: int = 1 #: its attempt; above 1 when Harbor retried it under the same name
|
|
220
|
+
retried: list = field(default_factory=list) #: the outcomes its earlier attempts ended with
|
|
221
|
+
finished_at: datetime | None = None
|
|
222
|
+
#: (ts, {token field: value}) per report — only the keys that report
|
|
223
|
+
#: carried — for the rate window; see Board.tokens_per_sec.
|
|
224
|
+
_series: list = field(default_factory=list, repr=False)
|
|
225
|
+
|
|
226
|
+
@property
|
|
227
|
+
def state(self) -> str:
|
|
228
|
+
"""queued · running · done — the card's border colour."""
|
|
229
|
+
if self.outcome is not None:
|
|
230
|
+
return "done"
|
|
231
|
+
return "running" if self.phase else "queued"
|
|
232
|
+
|
|
233
|
+
def elapsed_in_phase(self, now: datetime) -> float | None:
|
|
234
|
+
if self.since is None or self.outcome is not None:
|
|
235
|
+
return None
|
|
236
|
+
return max(0.0, (now - self.since).total_seconds())
|
|
237
|
+
|
|
238
|
+
def total_duration(self) -> float | None:
|
|
239
|
+
if self.started_at is None:
|
|
240
|
+
return None
|
|
241
|
+
end = self.finished_at
|
|
242
|
+
return (end - self.started_at).total_seconds() if end else None
|
|
243
|
+
|
|
244
|
+
def snapshot(self, now: datetime) -> dict:
|
|
245
|
+
elapsed = self.elapsed_in_phase(now)
|
|
246
|
+
return {
|
|
247
|
+
"trial": self.trial, "task": self.task, "agent": self.agent, "model": self.model,
|
|
248
|
+
"position": self.position, "state": self.state,
|
|
249
|
+
"phase": self.phase, "since": _iso(self.since) if self.since else None,
|
|
250
|
+
"elapsed_sec": elapsed, "timeout_sec": self.timeout_sec,
|
|
251
|
+
"phases_done": list(self.phases_done),
|
|
252
|
+
"step": self.step, "n_input_tokens": self.n_input_tokens,
|
|
253
|
+
"n_cache_tokens": self.n_cache_tokens, "n_output_tokens": self.n_output_tokens,
|
|
254
|
+
"cost_usd": self.cost_usd, "activity": self.activity, "last_line": self.last_line,
|
|
255
|
+
"outcome": self.outcome, "reward": self.reward, "durations": dict(self.durations),
|
|
256
|
+
"total_sec": self.total_duration(),
|
|
257
|
+
**({"score": self.score, "parts": self.parts} if self.outcome == "scored" else {}),
|
|
258
|
+
**({"attempt": self.attempt, "retried": list(self.retried)} if self.attempt > 1 else {}),
|
|
259
|
+
}
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
@dataclass
|
|
263
|
+
class Board:
|
|
264
|
+
job: str | None = None
|
|
265
|
+
launched_at: datetime | None = None #: first job.prepare — when the user pressed Run
|
|
266
|
+
prepare_stage: str | None = None #: the last job.prepare's `stage`
|
|
267
|
+
prepare_since: datetime | None = None
|
|
268
|
+
started_at: datetime | None = None
|
|
269
|
+
finished_at: datetime | None = None
|
|
270
|
+
ended_by: str | None = None #: set when the runner, not the feed, ended the board
|
|
271
|
+
n_total: int | None = None
|
|
272
|
+
n_concurrent: int | None = None
|
|
273
|
+
wall_sec: float | None = None
|
|
274
|
+
budget: dict | None = None
|
|
275
|
+
agents: list = field(default_factory=list)
|
|
276
|
+
trials: list = field(default_factory=list) #: TrialState, in first-seen order
|
|
277
|
+
notices: list = field(default_factory=list)
|
|
278
|
+
tail: list = field(default_factory=list) #: the last TAIL_EVENTS events
|
|
279
|
+
now: datetime | None = None
|
|
280
|
+
|
|
281
|
+
def counts(self) -> dict:
|
|
282
|
+
return _ended(_counts(self.trials, self.n_total), self.finished_at)
|
|
283
|
+
|
|
284
|
+
def mean_reward(self) -> float | None:
|
|
285
|
+
return _mean_reward(self.trials)
|
|
286
|
+
|
|
287
|
+
def totals(self) -> dict:
|
|
288
|
+
return _totals(self.trials)
|
|
289
|
+
|
|
290
|
+
def lane_order(self) -> list:
|
|
291
|
+
"""Agents in lane order: as ``job.start`` declared them, then any agent
|
|
292
|
+
first seen on a trial the declaration did not name, then ``None`` for
|
|
293
|
+
trials whose agent never appears — present only when there are such
|
|
294
|
+
trials. Every trial falls in exactly one lane."""
|
|
295
|
+
order = list(self.agents)
|
|
296
|
+
for t in self.trials:
|
|
297
|
+
if t.agent is not None and t.agent not in order:
|
|
298
|
+
order.append(t.agent)
|
|
299
|
+
if any(t.agent is None for t in self.trials):
|
|
300
|
+
order.append(None)
|
|
301
|
+
return order
|
|
302
|
+
|
|
303
|
+
def lanes(self) -> list[dict]:
|
|
304
|
+
"""One lane per agent: its trials, and the header strip in miniature
|
|
305
|
+
over them — counts (``total`` is the trials seen so far, since the feed
|
|
306
|
+
does not say how many each agent will run), mean reward, totals, and
|
|
307
|
+
how many tasks it has won (see :meth:`winners`)."""
|
|
308
|
+
wins: dict = {}
|
|
309
|
+
for w in self.winners():
|
|
310
|
+
if w["winner"] is not None:
|
|
311
|
+
wins[w["winner"]] = wins.get(w["winner"], 0) + 1
|
|
312
|
+
out = []
|
|
313
|
+
for agent in self.lane_order():
|
|
314
|
+
members = [t for t in self.trials if t.agent == agent]
|
|
315
|
+
out.append({"agent": agent, "trials": [t.trial for t in members],
|
|
316
|
+
"counts": _ended(_counts(members, len(members)), self.finished_at),
|
|
317
|
+
"mean_reward": _mean_reward(members), "totals": _totals(members),
|
|
318
|
+
"wins": wins.get(agent, 0)})
|
|
319
|
+
return out
|
|
320
|
+
|
|
321
|
+
def winners(self) -> list[dict]:
|
|
322
|
+
"""One entry per task that more than one agent ran, in first-seen task
|
|
323
|
+
order: the agents that ran it (lane order), the best reward each has
|
|
324
|
+
reached on it so far (over its finished trials; ``None`` until it has
|
|
325
|
+
one), ``decided`` once every trial of the task has an outcome, and the
|
|
326
|
+
``winner`` — the one agent whose best is strictly above every other's,
|
|
327
|
+
named only when decided. A tie names nobody; so does a task nobody
|
|
328
|
+
scored. Trials with no task or no agent take part in no contest."""
|
|
329
|
+
by_task: dict = {}
|
|
330
|
+
for t in self.trials:
|
|
331
|
+
if t.task is not None and t.agent is not None:
|
|
332
|
+
by_task.setdefault(t.task, []).append(t)
|
|
333
|
+
order = [a for a in self.lane_order() if a is not None]
|
|
334
|
+
out = []
|
|
335
|
+
for task, members in by_task.items():
|
|
336
|
+
agents = [a for a in order if any(m.agent == a for m in members)]
|
|
337
|
+
if len(agents) < 2:
|
|
338
|
+
continue
|
|
339
|
+
rewards = {}
|
|
340
|
+
for a in agents:
|
|
341
|
+
scored = [m.reward for m in members
|
|
342
|
+
if m.agent == a and m.outcome is not None and m.reward is not None]
|
|
343
|
+
rewards[a] = max(scored) if scored else None
|
|
344
|
+
decided = all(m.outcome is not None for m in members)
|
|
345
|
+
winner = None
|
|
346
|
+
if decided:
|
|
347
|
+
best = [r for r in rewards.values() if r is not None]
|
|
348
|
+
if best:
|
|
349
|
+
top = [a for a, r in rewards.items() if r == max(best)]
|
|
350
|
+
winner = top[0] if len(top) == 1 else None
|
|
351
|
+
out.append({"task": task, "agents": agents, "rewards": rewards,
|
|
352
|
+
"decided": decided, "winner": winner})
|
|
353
|
+
return out
|
|
354
|
+
|
|
355
|
+
def tokens_per_sec(self, window: float = RATE_WINDOW_SEC) -> float | None:
|
|
356
|
+
"""Tokens reported over the last ``window`` seconds, per second — the
|
|
357
|
+
latest total per trial minus what it was at the window's start.
|
|
358
|
+
|
|
359
|
+
``None`` when no rate can be computed: a rate needs a trial with a
|
|
360
|
+
report inside the window *and* a baseline to measure from (a report at
|
|
361
|
+
or before the window's start, or a second report inside it). Totals
|
|
362
|
+
that arrive only on ``trial.end`` give one point per trial and no
|
|
363
|
+
rate; a real rate needs ``trial.progress``. A zero where nothing was
|
|
364
|
+
measured is a lie — the same rule as ``cost_usd`` — so a board renders
|
|
365
|
+
``None`` as "not reported".
|
|
366
|
+
|
|
367
|
+
The delta is taken **per counter key, over reports that carry that
|
|
368
|
+
key**. A writer may report ``n_input_tokens``/``n_output_tokens`` on
|
|
369
|
+
progress and add ``n_cache_tokens`` only on ``trial.end`` (Harbor's
|
|
370
|
+
does); summing whole totals would then read the cache figure's first
|
|
371
|
+
appearance as tokens produced in the window. A key needs its own
|
|
372
|
+
baseline to contribute to a rate."""
|
|
373
|
+
if self.now is None:
|
|
374
|
+
return None
|
|
375
|
+
start = self.now.timestamp() - window
|
|
376
|
+
delta = 0
|
|
377
|
+
measured = False
|
|
378
|
+
for t in self.trials:
|
|
379
|
+
for key in TOKEN_FIELDS:
|
|
380
|
+
series = [(ts.timestamp(), vals[key]) for ts, vals in t._series if key in vals]
|
|
381
|
+
before = [v for at, v in series if at <= start]
|
|
382
|
+
inside = [v for at, v in series if at > start]
|
|
383
|
+
if not inside or (not before and len(inside) < 2):
|
|
384
|
+
continue # no baseline for this key
|
|
385
|
+
# A trial's first report inside the window is the baseline when
|
|
386
|
+
# nothing precedes it: what it did before it started reporting
|
|
387
|
+
# is unknown, not "everything in the last 30 s".
|
|
388
|
+
base = before[-1] if before else inside[0]
|
|
389
|
+
delta += max(0, series[-1][1] - base)
|
|
390
|
+
measured = True
|
|
391
|
+
return delta / window if measured else None
|
|
392
|
+
|
|
393
|
+
def eta_sec(self) -> float | None:
|
|
394
|
+
"""The work left ÷ concurrency, the work counted in median finished
|
|
395
|
+
durations: one per pending trial, and for a running one what is left of
|
|
396
|
+
a median once its time so far is taken off - so the estimate counts down
|
|
397
|
+
between two ends instead of standing still, and is set again at each.
|
|
398
|
+
Hidden until two trials have finished, because one duration is not an
|
|
399
|
+
estimate."""
|
|
400
|
+
durations = [t.total_duration() for t in self.trials
|
|
401
|
+
if t.outcome is not None and t.total_duration() is not None]
|
|
402
|
+
if len(durations) < 2 or self.finished_at is not None:
|
|
403
|
+
return None
|
|
404
|
+
c = self.counts()
|
|
405
|
+
if c["running"] + c["pending"] <= 0:
|
|
406
|
+
return 0.0
|
|
407
|
+
median = statistics.median(durations)
|
|
408
|
+
now = self.now or datetime.now(timezone.utc)
|
|
409
|
+
running = [t for t in self.trials if t.state == "running"]
|
|
410
|
+
spent = [max(0.0, (now - t.started_at).total_seconds()) if t.started_at else 0.0 for t in running]
|
|
411
|
+
work = c["pending"] * median + sum(max(0.0, median - s) for s in spent)
|
|
412
|
+
return work / max(1, self.n_concurrent or 1)
|
|
413
|
+
|
|
414
|
+
@property
|
|
415
|
+
def state(self) -> str:
|
|
416
|
+
"""The header's one word, in order: `unknown` (no job event yet),
|
|
417
|
+
`preparing` (a job.prepare and no job.start - the launcher is fetching
|
|
418
|
+
the dataset, building images, whatever its `stage` says), `starting`
|
|
419
|
+
(job.start seen, no trial in a phase), `running`, `finished`."""
|
|
420
|
+
if self.finished_at is not None:
|
|
421
|
+
return "finished"
|
|
422
|
+
if any(t.state != "queued" for t in self.trials):
|
|
423
|
+
return "running"
|
|
424
|
+
if self.started_at is not None:
|
|
425
|
+
return "starting"
|
|
426
|
+
if self.trials:
|
|
427
|
+
return "running" # trial events with no job.start on record: something is running
|
|
428
|
+
if self.launched_at is not None:
|
|
429
|
+
return "preparing"
|
|
430
|
+
return "unknown"
|
|
431
|
+
|
|
432
|
+
def end_by_runner(self, at: datetime, how: str) -> None:
|
|
433
|
+
"""The feed has no ``job.end`` but the runner that started Harbor
|
|
434
|
+
knows the run ended (``how``: stopped, failed, done) at ``at``: the
|
|
435
|
+
board settles there. A trial in a phase is cancelled - nothing will
|
|
436
|
+
finish it - and the ones still queued never start; the notice says
|
|
437
|
+
so, since the feed alone would read as a job still going."""
|
|
438
|
+
if self.finished_at is not None:
|
|
439
|
+
return
|
|
440
|
+
self.finished_at = at
|
|
441
|
+
self.ended_by = how
|
|
442
|
+
cut = [t for t in self.trials if t.state == "running"]
|
|
443
|
+
for t in cut:
|
|
444
|
+
t.outcome, t.finished_at = "cancelled", at
|
|
445
|
+
never = self.counts().get("not_run", 0)
|
|
446
|
+
self.notices.append(
|
|
447
|
+
f"The run was {how} at {at.strftime('%H:%M:%S')}Z before Harbor wrote job.end: "
|
|
448
|
+
f"{len(cut)} trial{'s' if len(cut) != 1 else ''} cancelled"
|
|
449
|
+
+ (f", {never} never started" if never else "") + ".")
|
|
450
|
+
|
|
451
|
+
def elapsed_sec(self) -> float | None:
|
|
452
|
+
"""Since the launch when the feed recorded one, else since job.start:
|
|
453
|
+
the user's clock starts at Run, and a board that showed nothing for
|
|
454
|
+
the preparation window is what job.prepare exists to fix."""
|
|
455
|
+
origin = self.launched_at or self.started_at
|
|
456
|
+
if origin is None or self.now is None:
|
|
457
|
+
return None
|
|
458
|
+
end = self.finished_at or self.now
|
|
459
|
+
return (end - origin).total_seconds()
|
|
460
|
+
|
|
461
|
+
def preparing(self) -> dict | None:
|
|
462
|
+
"""The preparation window while it is open: its stage and how long."""
|
|
463
|
+
if self.state != "preparing" or self.now is None:
|
|
464
|
+
return None
|
|
465
|
+
since = self.prepare_since or self.launched_at
|
|
466
|
+
return {"stage": self.prepare_stage, "since": _iso(since),
|
|
467
|
+
"elapsed_sec": max(0.0, (self.now - since).total_seconds())}
|
|
468
|
+
|
|
469
|
+
def snapshot(self) -> dict:
|
|
470
|
+
"""The `/live` document: what a board renders, and nothing it computes
|
|
471
|
+
on its own — every number here is a function of the file and ``now``."""
|
|
472
|
+
now = self.now or datetime.now(timezone.utc)
|
|
473
|
+
return {
|
|
474
|
+
"job": self.job, "now": _iso(now), "state": self.state,
|
|
475
|
+
"launched_at": _iso(self.launched_at) if self.launched_at else None,
|
|
476
|
+
"preparing": self.preparing(),
|
|
477
|
+
"started_at": _iso(self.started_at) if self.started_at else None,
|
|
478
|
+
"finished_at": _iso(self.finished_at) if self.finished_at else None,
|
|
479
|
+
# Only when the runner ended the board: the document otherwise
|
|
480
|
+
# stays the fixtures' expected.json, byte for byte (test_live.Fixture).
|
|
481
|
+
**({"ended_by": self.ended_by} if self.ended_by else {}),
|
|
482
|
+
"running": self.finished_at is None and self.started_at is not None,
|
|
483
|
+
"n_concurrent": self.n_concurrent, "wall_sec": self.wall_sec, "budget": self.budget,
|
|
484
|
+
"agents": list(self.agents),
|
|
485
|
+
"counts": self.counts(), "mean_reward": self.mean_reward(),
|
|
486
|
+
"totals": self.totals(), "tokens_per_sec": self.tokens_per_sec(),
|
|
487
|
+
"elapsed_sec": self.elapsed_sec(), "eta_sec": self.eta_sec(),
|
|
488
|
+
"trials": [t.snapshot(now) for t in self.trials],
|
|
489
|
+
"lanes": self.lanes(), "winners": self.winners(),
|
|
490
|
+
"tail": [e.to_dict() for e in self.tail],
|
|
491
|
+
"notices": list(self.notices),
|
|
492
|
+
}
|
|
493
|
+
|
|
494
|
+
|
|
495
|
+
def _ended(c: dict, finished_at: datetime | None) -> dict:
|
|
496
|
+
"""Once the run has ended (its cap, a stop, its endpoint refusing), what never began never will: not run,
|
|
497
|
+
not pending - the number job.end and Results give. Only then, so a run that began every unit is unchanged."""
|
|
498
|
+
if finished_at is not None and c["pending"] > 0:
|
|
499
|
+
c["not_run"], c["pending"] = c["pending"], 0
|
|
500
|
+
return c
|
|
501
|
+
|
|
502
|
+
|
|
503
|
+
def _counts(trials: list, n_total: int | None) -> dict:
|
|
504
|
+
"""The count block over a list of trials. ``n_total`` is what the feed
|
|
505
|
+
declared for the job; a lane passes the trials it has seen."""
|
|
506
|
+
done = [t for t in trials if t.outcome is not None]
|
|
507
|
+
return {
|
|
508
|
+
# Unknown until job.prepare/job.start says, or a trial appears: a
|
|
509
|
+
# "0 tasks" header during preparation would be a lie. A lane passes
|
|
510
|
+
# the trials it has seen, so its total is always a number.
|
|
511
|
+
"total": n_total if n_total is not None else (len(trials) or None),
|
|
512
|
+
"running": sum(t.state == "running" for t in trials),
|
|
513
|
+
"pending": (n_total - len([t for t in trials if t.state != "queued"])
|
|
514
|
+
if n_total is not None else sum(t.state == "queued" for t in trials)),
|
|
515
|
+
"completed": len(done),
|
|
516
|
+
"resolved": sum(t.outcome == "resolved" for t in done),
|
|
517
|
+
"unresolved": sum(t.outcome == "unresolved" for t in done),
|
|
518
|
+
**({"scored": sum(t.outcome == "scored" for t in done)} if any(t.outcome == "scored" for t in done) else {}),
|
|
519
|
+
"errored": sum(t.outcome in ERROR_OUTCOMES for t in done),
|
|
520
|
+
"cancelled": sum(t.outcome == "cancelled" for t in done),
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _mean_reward(trials: list) -> float | None:
|
|
525
|
+
rewards = [t.reward for t in trials if t.outcome is not None and t.reward is not None]
|
|
526
|
+
return sum(rewards) / len(rewards) if rewards else None
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _totals(trials: list) -> dict:
|
|
530
|
+
out: dict = {f: None for f in TOKEN_FIELDS}
|
|
531
|
+
for f in TOKEN_FIELDS:
|
|
532
|
+
vals = [getattr(t, f) for t in trials if getattr(t, f) is not None]
|
|
533
|
+
out[f] = sum(vals) if vals else None
|
|
534
|
+
costs = [t.cost_usd for t in trials if t.cost_usd is not None]
|
|
535
|
+
out["cost_usd"] = sum(costs) if costs else None # absent, never zero
|
|
536
|
+
return out
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
def _num(value, kind=float):
|
|
540
|
+
"""A counter as reported, or None — never coerced from absence."""
|
|
541
|
+
if value is None or isinstance(value, bool):
|
|
542
|
+
return None
|
|
543
|
+
try:
|
|
544
|
+
return kind(value)
|
|
545
|
+
except (TypeError, ValueError):
|
|
546
|
+
return None
|
|
547
|
+
|
|
548
|
+
|
|
549
|
+
def fold(events: list[Event], now: datetime | None = None, notices: list[str] | None = None) -> Board:
|
|
550
|
+
"""Fold a feed into a :class:`Board` as of ``now`` (default: the last event's
|
|
551
|
+
``ts`` — replay of a finished file is exact; a live board passes the clock)."""
|
|
552
|
+
board = Board(notices=list(notices or []))
|
|
553
|
+
by_id: dict[str, TrialState] = {}
|
|
554
|
+
|
|
555
|
+
def trial_of(e: Event) -> TrialState:
|
|
556
|
+
t = by_id.get(e.trial)
|
|
557
|
+
if t is None:
|
|
558
|
+
t = by_id[e.trial] = TrialState(trial=e.trial)
|
|
559
|
+
board.trials.append(t)
|
|
560
|
+
return t
|
|
561
|
+
|
|
562
|
+
for e in events:
|
|
563
|
+
d = e.data
|
|
564
|
+
if board.job is None:
|
|
565
|
+
board.job = e.job
|
|
566
|
+
if e.trial is not None:
|
|
567
|
+
# The spec puts task/agent/model on trial.queued; Harbor's writer
|
|
568
|
+
# also stamps task and agent on every trial event. Either is read:
|
|
569
|
+
# the first value seen wins, and a later event never blanks it.
|
|
570
|
+
t = trial_of(e)
|
|
571
|
+
for key in ("task", "agent", "model"):
|
|
572
|
+
if getattr(t, key) is None and isinstance(d.get(key), str):
|
|
573
|
+
setattr(t, key, d[key])
|
|
574
|
+
if e.event == "job.prepare":
|
|
575
|
+
# Written at launch, before any preparation, and again as the
|
|
576
|
+
# stage changes (last one wins, like trial.phase). Carries what
|
|
577
|
+
# job.start will carry once the launcher knows it, so the header
|
|
578
|
+
# can already say "N tasks, M concurrent".
|
|
579
|
+
if board.launched_at is None:
|
|
580
|
+
board.launched_at = e.ts
|
|
581
|
+
board.prepare_stage = d.get("stage") if isinstance(d.get("stage"), str) else None
|
|
582
|
+
board.prepare_since = e.ts
|
|
583
|
+
if board.n_total is None:
|
|
584
|
+
board.n_total = _num(d.get("n_total"), int)
|
|
585
|
+
if board.n_concurrent is None:
|
|
586
|
+
board.n_concurrent = _num(d.get("n_concurrent"), int)
|
|
587
|
+
if not board.agents:
|
|
588
|
+
board.agents = list(d.get("agents") or [])
|
|
589
|
+
elif e.event == "job.start":
|
|
590
|
+
board.started_at = e.ts
|
|
591
|
+
board.n_total = _num(d.get("n_total"), int)
|
|
592
|
+
board.n_concurrent = _num(d.get("n_concurrent"), int)
|
|
593
|
+
board.wall_sec = _num(d.get("wall_sec"))
|
|
594
|
+
board.budget = d.get("budget") if isinstance(d.get("budget"), dict) else None
|
|
595
|
+
board.agents = list(d.get("agents") or [])
|
|
596
|
+
elif e.event == "job.end":
|
|
597
|
+
board.finished_at = e.ts
|
|
598
|
+
elif e.event == "trial.queued":
|
|
599
|
+
t = trial_of(e)
|
|
600
|
+
t.position = _num(d.get("position"), int)
|
|
601
|
+
elif e.event == "trial.phase":
|
|
602
|
+
t = trial_of(e)
|
|
603
|
+
if t.outcome is not None:
|
|
604
|
+
# A phase after the trial ended: Harbor retries a trial under its own name (max_retries), so this is
|
|
605
|
+
# its next attempt, running again; what the attempt before it ended with is kept beside it.
|
|
606
|
+
t.attempt += 1
|
|
607
|
+
t.retried.append(t.outcome)
|
|
608
|
+
t.outcome, t.reward, t.score, t.parts, t.finished_at = None, None, None, None, None
|
|
609
|
+
t.phases_done, t.durations, t.started_at = [], {}, e.ts
|
|
610
|
+
if t.phase and t.phase not in t.phases_done and t.phase != d["phase"]:
|
|
611
|
+
t.phases_done.append(t.phase)
|
|
612
|
+
t.phase, t.since = d["phase"], e.ts # the last phase event wins
|
|
613
|
+
t.timeout_sec = _num(d.get("timeout_sec"))
|
|
614
|
+
if t.started_at is None:
|
|
615
|
+
t.started_at = e.ts
|
|
616
|
+
elif e.event == "trial.progress":
|
|
617
|
+
t = trial_of(e)
|
|
618
|
+
for key in COUNTERS:
|
|
619
|
+
if key in d and d[key] is not None:
|
|
620
|
+
setattr(t, key, _num(d[key], int if key != "cost_usd" else float))
|
|
621
|
+
if d.get("activity") is not None:
|
|
622
|
+
t.activity = d["activity"]
|
|
623
|
+
reported = {k: _num(d[k], int) for k in TOKEN_FIELDS if d.get(k) is not None}
|
|
624
|
+
if reported:
|
|
625
|
+
t._series.append((e.ts, reported))
|
|
626
|
+
elif e.event == "trial.log":
|
|
627
|
+
t = trial_of(e)
|
|
628
|
+
if d.get("line") is not None:
|
|
629
|
+
t.last_line = d["line"]
|
|
630
|
+
elif e.event == "trial.end":
|
|
631
|
+
t = trial_of(e)
|
|
632
|
+
t.outcome = d["outcome"]
|
|
633
|
+
t.reward = _num(d.get("reward"))
|
|
634
|
+
t.score, t.parts = _num(d.get("score")), _num(d.get("parts"), int)
|
|
635
|
+
t.finished_at = e.ts
|
|
636
|
+
if t.phase and t.phase not in t.phases_done:
|
|
637
|
+
t.phases_done.append(t.phase)
|
|
638
|
+
t.phase, t.since = None, None
|
|
639
|
+
durations = d.get("durations")
|
|
640
|
+
t.durations = {k: _num(v) for k, v in durations.items()} if isinstance(durations, dict) else {}
|
|
641
|
+
for key in (*TOKEN_FIELDS, "cost_usd"):
|
|
642
|
+
if d.get(key) is not None:
|
|
643
|
+
setattr(t, key, _num(d[key], int if key != "cost_usd" else float))
|
|
644
|
+
reported = {k: _num(d[k], int) for k in TOKEN_FIELDS if d.get(k) is not None}
|
|
645
|
+
if reported:
|
|
646
|
+
t._series.append((e.ts, reported))
|
|
647
|
+
if t.started_at is None:
|
|
648
|
+
t.started_at = e.ts
|
|
649
|
+
|
|
650
|
+
board.now = now or (events[-1].ts if events else datetime.now(timezone.utc))
|
|
651
|
+
board.tail = events[-TAIL_EVENTS:]
|
|
652
|
+
return board
|
|
653
|
+
|
|
654
|
+
|
|
655
|
+
def load(path: Path | str, now: datetime | None = None, *, feed=None) -> Board:
|
|
656
|
+
"""Read and fold one job's feed - or the feed already read (``feed``):
|
|
657
|
+
the board every `/live` answer is made from (:func:`snapshot`)."""
|
|
658
|
+
feed = feed if feed is not None else read_events(path)
|
|
659
|
+
return fold(feed.events, now=now, notices=feed.notices)
|
|
660
|
+
|
|
661
|
+
|
|
662
|
+
|
|
663
|
+
FEED = "events.jsonl"
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
def feed_path(job_dir: str) -> Path:
|
|
667
|
+
return Path(job_dir) / FEED
|
|
668
|
+
|
|
669
|
+
|
|
670
|
+
def has_feed(job_dir: str) -> bool:
|
|
671
|
+
return feed_path(job_dir).is_file()
|
|
672
|
+
|
|
673
|
+
|
|
674
|
+
def timeline(events: list[Event]) -> dict:
|
|
675
|
+
"""When each trial entered each phase, and when it ended: ``{trial: {"starts":
|
|
676
|
+
{phase: ts}, "ended": ts | None}}``, the first ``trial.phase`` per phase and the
|
|
677
|
+
``trial.end``, as the feed recorded them.
|
|
678
|
+
|
|
679
|
+
Not part of the snapshot, which the fixtures pin and which stays byte for
|
|
680
|
+
byte what it folds (``test_live``); the page's route carries it beside the
|
|
681
|
+
snapshot, so a trial's phases can be drawn on its own clock. Only times the
|
|
682
|
+
feed wrote: nothing here is measured by this application."""
|
|
683
|
+
out: dict[str, dict] = {}
|
|
684
|
+
for e in events:
|
|
685
|
+
if e.trial is None:
|
|
686
|
+
continue
|
|
687
|
+
row = out.setdefault(e.trial, {"starts": {}, "ended": None})
|
|
688
|
+
if e.event == "trial.phase" and e.data.get("phase"):
|
|
689
|
+
row["starts"].setdefault(e.data["phase"], _iso(e.ts))
|
|
690
|
+
elif e.event == "trial.end" and row["ended"] is None:
|
|
691
|
+
row["ended"] = _iso(e.ts)
|
|
692
|
+
return out
|
|
693
|
+
|
|
694
|
+
|
|
695
|
+
def snapshot(job_dir: str, now: datetime | None = None, ended: tuple | None = None,
|
|
696
|
+
with_timeline: bool = False) -> dict:
|
|
697
|
+
"""Read and fold a job directory's feed. The one call the route makes.
|
|
698
|
+
|
|
699
|
+
Always ``now``-driven for a live board: `fold`'s default (the last
|
|
700
|
+
event's ``ts``) is exact for replaying a finished file, but a page that is
|
|
701
|
+
polling wants the timers to move between polls, so the route passes the
|
|
702
|
+
clock. ``ended`` is ``(how, at)`` from the runner's own record
|
|
703
|
+
(`execution.ended_for`) when it knows the run is over: a feed Harbor
|
|
704
|
+
never closed then settles instead of reading as a job still going.
|
|
705
|
+
``with_timeline`` adds :func:`timeline` under ``"timeline"`` for the page.
|
|
706
|
+
"""
|
|
707
|
+
path = feed_path(job_dir)
|
|
708
|
+
feed = read_events(path)
|
|
709
|
+
board = load(path, now=now or datetime.now(timezone.utc), feed=feed)
|
|
710
|
+
if ended is not None:
|
|
711
|
+
board.end_by_runner(ended[1], ended[0])
|
|
712
|
+
snap = board.snapshot()
|
|
713
|
+
if with_timeline:
|
|
714
|
+
snap["timeline"] = timeline(feed.events)
|
|
715
|
+
return snap
|