nullius-eval 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- _nullius_app/.env.example +55 -0
- _nullius_app/app.py +96 -0
- _nullius_app/requirements.txt +1 -0
- _nullius_app/static/brand/README.md +20 -0
- _nullius_app/static/brand/docker.svg +1 -0
- _nullius_app/static/brand/favicon.svg +1 -0
- _nullius_app/static/brand/huggingface.svg +37 -0
- _nullius_app/static/brand/mark-dark.svg +1 -0
- _nullius_app/static/brand/marks.svg +34 -0
- _nullius_app/static/brand/tokens.json +51 -0
- _nullius_app/static/brand/typesafe.png +0 -0
- _nullius_app/static/brand/wordmark-dark.svg +1 -0
- _nullius_app/static/builder/command-builder-core.js +763 -0
- _nullius_app/static/builder/command-builder.css +248 -0
- _nullius_app/static/builder/command-builder.js +997 -0
- _nullius_app/static/builder/model-picker.js +113 -0
- _nullius_app/static/catalogue/add.js +46 -0
- _nullius_app/static/catalogue/agents/README.md +19 -0
- _nullius_app/static/catalogue/agents/goose.svg +1 -0
- _nullius_app/static/catalogue/agents/jiuwenswarm.svg +33 -0
- _nullius_app/static/catalogue/agents/marks.json +8 -0
- _nullius_app/static/catalogue/agents/mini-swe-agent.svg +7 -0
- _nullius_app/static/catalogue/agents/opencode.svg +7 -0
- _nullius_app/static/catalogue/catalogue.css +436 -0
- _nullius_app/static/catalogue/catalogue.js +87 -0
- _nullius_app/static/catalogue/marks.svg +29 -0
- _nullius_app/static/catalogue/new.js +78 -0
- _nullius_app/static/catalogue/versions.js +27 -0
- _nullius_app/static/compare/comparison.css +117 -0
- _nullius_app/static/controls.js +117 -0
- _nullius_app/static/disclosures.js +70 -0
- _nullius_app/static/fonts/0569c822cab6c510.ttf +0 -0
- _nullius_app/static/fonts/DMSans-Medium.woff2 +0 -0
- _nullius_app/static/fonts/DMSans-Regular.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Medium.woff2 +0 -0
- _nullius_app/static/fonts/IBMPlexMono-Regular.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Italic.woff2 +0 -0
- _nullius_app/static/fonts/InstrumentSerif-Regular.woff2 +0 -0
- _nullius_app/static/fonts/README.md +11 -0
- _nullius_app/static/fonts/c1e7453ea26a1d1e.ttf +0 -0
- _nullius_app/static/fonts/dmsans-OFL.txt +93 -0
- _nullius_app/static/fonts/ibmplexmono-OFL.txt +93 -0
- _nullius_app/static/fonts/instrumentserif-OFL.txt +93 -0
- _nullius_app/static/fonts.css +49 -0
- _nullius_app/static/notifications.js +49 -0
- _nullius_app/static/onboarding.css +78 -0
- _nullius_app/static/onboarding.js +158 -0
- _nullius_app/static/runs/board.css +358 -0
- _nullius_app/static/runs/board.js +127 -0
- _nullius_app/static/runs/launches.css +88 -0
- _nullius_app/static/runs/live-board.css +186 -0
- _nullius_app/static/runs/live-board.js +394 -0
- _nullius_app/static/runs/path-suggestions.js +119 -0
- _nullius_app/static/runs/start.css +38 -0
- _nullius_app/static/settings/autosave.js +162 -0
- _nullius_app/static/settings/custom-keys.js +100 -0
- _nullius_app/static/settings/data-providers.js +88 -0
- _nullius_app/static/settings/judges.js +61 -0
- _nullius_app/static/settings/modal.css +32 -0
- _nullius_app/static/settings/modal.js +91 -0
- _nullius_app/static/settings/navigation.js +55 -0
- _nullius_app/static/settings/prices.js +68 -0
- _nullius_app/static/settings/providers.js +134 -0
- _nullius_app/static/settings/sandbox.js +20 -0
- _nullius_app/static/settings/settings.css +288 -0
- _nullius_app/static/style.css +1752 -0
- _nullius_app/static/traces/conversation.css +49 -0
- _nullius_app/static/traces/conversation.js +215 -0
- _nullius_app/templates/agents/entry.html +145 -0
- _nullius_app/templates/agents/kits.html +55 -0
- _nullius_app/templates/agents/standard.html +58 -0
- _nullius_app/templates/base.html +479 -0
- _nullius_app/templates/builder/new_run.html +55 -0
- _nullius_app/templates/catalogue/add_dialog.html +32 -0
- _nullius_app/templates/catalogue/benchmark.html +270 -0
- _nullius_app/templates/catalogue/benchmarks.html +91 -0
- _nullius_app/templates/catalogue/new.html +141 -0
- _nullius_app/templates/catalogue/use_bundle.html +11 -0
- _nullius_app/templates/compare/_detail.html +37 -0
- _nullius_app/templates/compare/compare.html +298 -0
- _nullius_app/templates/compare/compare_picker.html +51 -0
- _nullius_app/templates/compare/compare_task.html +329 -0
- _nullius_app/templates/runs/_execution.html +131 -0
- _nullius_app/templates/runs/_runtime.html +5 -0
- _nullius_app/templates/runs/ascii-mark.html +485 -0
- _nullius_app/templates/runs/feasibility.html +127 -0
- _nullius_app/templates/runs/index.html +609 -0
- _nullius_app/templates/runs/job.html +460 -0
- _nullius_app/templates/runs/preparing.html +14 -0
- _nullius_app/templates/runs/reconcile.html +124 -0
- _nullius_app/templates/runs/setup.html +272 -0
- _nullius_app/templates/runs/start.html +51 -0
- _nullius_app/templates/settings/custom_keys.html +82 -0
- _nullius_app/templates/settings/data_providers.html +27 -0
- _nullius_app/templates/settings/settings.html +564 -0
- _nullius_app/templates/traces/_charts.html +165 -0
- _nullius_app/templates/traces/_conversation.html +37 -0
- _nullius_app/templates/traces/_judgement.html +72 -0
- _nullius_app/templates/traces/artifact.html +15 -0
- _nullius_app/templates/traces/trial.html +838 -0
- _nullius_app/templates/workspaces/workspaces.html +54 -0
- _nullius_app/web/__init__.py +1 -0
- _nullius_app/web/agents.py +230 -0
- _nullius_app/web/builder.py +406 -0
- _nullius_app/web/catalogue.py +370 -0
- _nullius_app/web/compare.py +489 -0
- _nullius_app/web/core.py +17 -0
- _nullius_app/web/execute.py +223 -0
- _nullius_app/web/feasibility.py +60 -0
- _nullius_app/web/filters.py +231 -0
- _nullius_app/web/judge.py +623 -0
- _nullius_app/web/judge_tasks.py +33 -0
- _nullius_app/web/onboarding.py +20 -0
- _nullius_app/web/reconcile.py +47 -0
- _nullius_app/web/runs.py +464 -0
- _nullius_app/web/settings.py +324 -0
- _nullius_app/web/shared.py +507 -0
- _nullius_app/web/traces.py +306 -0
- _nullius_app/web/workspaces.py +84 -0
- nullius/__init__.py +32 -0
- nullius/__main__.py +5 -0
- nullius/catalogue/__init__.py +0 -0
- nullius/catalogue/agents/codex/agent.json +222 -0
- nullius/catalogue/agents/codex/config.toml +25 -0
- nullius/catalogue/agents/goose/agent.json +131 -0
- nullius/catalogue/agents/goose/config.yaml +20 -0
- nullius/catalogue/agents/jiuwenswarm/agent.json +197 -0
- nullius/catalogue/agents/jiuwenswarm/agent.py +355 -0
- nullius/catalogue/agents/mini-swe-agent/agent.json +68 -0
- nullius/catalogue/agents/mini-swe-agent/run.py +37 -0
- nullius/catalogue/agents/opencode/agent.json +135 -0
- nullius/catalogue/agents/opencode/opencode.json +32 -0
- nullius/catalogue/agents/reference/agent.json +89 -0
- nullius/catalogue/agents/reference/agent.py +209 -0
- nullius/catalogue/agents/terminus-2/agent.json +64 -0
- nullius/catalogue/agents/terminus-2/run.py +139 -0
- nullius/catalogue/benchmarks/gaia/benchmark.json +72 -0
- nullius/catalogue/benchmarks/gaia/data-source.json +29 -0
- nullius/catalogue/benchmarks/gaia/driver.py +188 -0
- nullius/catalogue/benchmarks/gaia/env/data.Dockerfile +1 -0
- nullius/catalogue/benchmarks/gaia/standard.json +278 -0
- nullius/catalogue/benchmarks/gaia/upstream-settings.json +275 -0
- nullius/catalogue/benchmarks/gsm8k/benchmark.json +34 -0
- nullius/catalogue/benchmarks/gsm8k/driver.py +101 -0
- nullius/catalogue/benchmarks/gsm8k/standard.json +124 -0
- nullius/catalogue/benchmarks/gsm8k/upstream-settings.json +86 -0
- nullius/catalogue/benchmarks/humaneval/benchmark.json +36 -0
- nullius/catalogue/benchmarks/humaneval/driver.py +243 -0
- nullius/catalogue/benchmarks/humaneval/standard.json +115 -0
- nullius/catalogue/benchmarks/humaneval/upstream-settings.json +72 -0
- nullius/catalogue/benchmarks/locomo/benchmark.json +43 -0
- nullius/catalogue/benchmarks/locomo/driver.py +270 -0
- nullius/catalogue/benchmarks/locomo/standard.json +265 -0
- nullius/catalogue/benchmarks/locomo/upstream-settings.json +112 -0
- nullius/catalogue/benchmarks/longmemeval/benchmark.json +65 -0
- nullius/catalogue/benchmarks/longmemeval/driver.py +379 -0
- nullius/catalogue/benchmarks/longmemeval/standard.json +733 -0
- nullius/catalogue/benchmarks/longmemeval/upstream-settings.json +76 -0
- nullius/catalogue/benchmarks/swe-bench-verified/benchmark.json +87 -0
- nullius/catalogue/benchmarks/swe-bench-verified/driver.py +116 -0
- nullius/catalogue/benchmarks/swe-bench-verified/images.json +502 -0
- nullius/catalogue/benchmarks/swe-bench-verified/standard.json +108 -0
- nullius/catalogue/benchmarks/swe-bench-verified/upstream-settings.json +32 -0
- nullius/catalogue/benchmarks/tau3-bench/benchmark.json +49 -0
- nullius/catalogue/benchmarks/tau3-bench/driver.py +608 -0
- nullius/catalogue/benchmarks/tau3-bench/mcp_server.py +114 -0
- nullius/catalogue/benchmarks/tau3-bench/search_relay.py +19 -0
- nullius/catalogue/benchmarks/tau3-bench/standard.json +547 -0
- nullius/catalogue/benchmarks/tau3-bench/upstream-settings.json +639 -0
- nullius/catalogue/benchmarks/terminal-bench/benchmark.json +69 -0
- nullius/catalogue/benchmarks/terminal-bench/driver.py +267 -0
- nullius/catalogue/benchmarks/terminal-bench/standard.json +166 -0
- nullius/catalogue/benchmarks/terminal-bench/upstream-settings.json +36 -0
- nullius/catalogue/capabilities/keyword-memory/capability.json +33 -0
- nullius/catalogue/capabilities/keyword-memory/capability.py +55 -0
- nullius/catalogue/capabilities/mem0/capability.json +135 -0
- nullius/catalogue/capabilities/mem0/capability.py +173 -0
- nullius/catalogue/catalog.py +426 -0
- nullius/catalogue/citation.py +116 -0
- nullius/catalogue/harbor_entry.py +367 -0
- nullius/catalogue/sources.py +107 -0
- nullius/catalogue/storage.py +125 -0
- nullius/cli.py +85 -0
- nullius/compare/__init__.py +0 -0
- nullius/compare/cli.py +265 -0
- nullius/compare/comparison.py +184 -0
- nullius/execution.py +1653 -0
- nullius/gitrefs.py +128 -0
- nullius/harbor/__init__.py +0 -0
- nullius/harbor/feed.py +228 -0
- nullius/harbor/source.py +582 -0
- nullius/hosts.py +368 -0
- nullius/importers/__init__.py +140 -0
- nullius/importers/atif.py +387 -0
- nullius/importers/cli.py +135 -0
- nullius/importers/otel.py +274 -0
- nullius/judge/__init__.py +14 -0
- nullius/judge/batch.py +122 -0
- nullius/judge/client.py +433 -0
- nullius/judge/floors.json +86 -0
- nullius/judge/lanes.py +89 -0
- nullius/judge/questions.py +309 -0
- nullius/judge/reconcile.py +176 -0
- nullius/judge/state.py +224 -0
- nullius/judge/store.py +167 -0
- nullius/migrate.py +268 -0
- nullius/onboarding.py +116 -0
- nullius/paths.py +89 -0
- nullius/prerequisites.py +129 -0
- nullius/prices.py +283 -0
- nullius/progress.py +294 -0
- nullius/providers.py +335 -0
- nullius/recipes/__init__.py +0 -0
- nullius/recipes/launch.py +78 -0
- nullius/redact.py +72 -0
- nullius/runs/__init__.py +0 -0
- nullius/runs/artifacts.py +67 -0
- nullius/runs/classify.py +216 -0
- nullius/runs/cli.py +316 -0
- nullius/runs/feasibility.py +442 -0
- nullius/runs/launches.py +149 -0
- nullius/runs/live.py +715 -0
- nullius/runs/parse.py +1016 -0
- nullius/runs/verifier.py +241 -0
- nullius/sandbox.py +109 -0
- nullius/secrets.py +218 -0
- nullius/settings.py +544 -0
- nullius/standard/__init__.py +15 -0
- nullius/standard/acp.py +305 -0
- nullius/standard/cancellation.py +19 -0
- nullius/standard/cli.py +703 -0
- nullius/standard/conform.py +149 -0
- nullius/standard/container.py +1870 -0
- nullius/standard/definitions.py +1651 -0
- nullius/standard/egress.py +192 -0
- nullius/standard/fidelity.py +256 -0
- nullius/standard/forward.py +139 -0
- nullius/standard/hostreq.py +105 -0
- nullius/standard/howto.py +220 -0
- nullius/standard/install.py +428 -0
- nullius/standard/job.py +551 -0
- nullius/standard/jsonl.py +129 -0
- nullius/standard/kits/agent-conversation-v1/driver.py +27 -0
- nullius/standard/kits/agent-conversation-v1/standard.json +26 -0
- nullius/standard/kits/agent-search-v1/driver.py +171 -0
- nullius/standard/kits/agent-search-v1/standard.json +99 -0
- nullius/standard/kits/agent-task-v1/driver.py +25 -0
- nullius/standard/kits/agent-task-v1/standard.json +100 -0
- nullius/standard/kits/agent-tools-v1/driver.py +206 -0
- nullius/standard/kits/agent-tools-v1/mcp_kit.py +209 -0
- nullius/standard/kits/agent-tools-v1/standard.json +119 -0
- nullius/standard/kits/agent-workspace-v1/driver.py +65 -0
- nullius/standard/kits/agent-workspace-v1/standard.json +95 -0
- nullius/standard/kits/cap-memory-v1/driver.py +39 -0
- nullius/standard/kits/cap-memory-v1/standard.json +91 -0
- nullius/standard/ledger.py +326 -0
- nullius/standard/match.py +66 -0
- nullius/standard/mcp_bridge.py +62 -0
- nullius/standard/mcp_machine.py +193 -0
- nullius/standard/merge.py +239 -0
- nullius/standard/names.py +37 -0
- nullius/standard/network_requirements.py +44 -0
- nullius/standard/pace.py +53 -0
- nullius/standard/payload.py +278 -0
- nullius/standard/pins.py +130 -0
- nullius/standard/ports.py +70 -0
- nullius/standard/prove.py +373 -0
- nullius/standard/recipe.py +746 -0
- nullius/standard/relay.py +785 -0
- nullius/standard/run.py +1781 -0
- nullius/standard/setup_container.py +208 -0
- nullius/standard/shard.py +71 -0
- nullius/standard/web_services.py +30 -0
- nullius/traces/__init__.py +0 -0
- nullius/traces/behavior.py +131 -0
- nullius/traces/charts.py +898 -0
- nullius/traces/conversation.py +159 -0
- nullius/traces/otel.py +608 -0
- nullius/traces/trace.py +1477 -0
- nullius/traces/trajectory.py +586 -0
- nullius/workspace.py +417 -0
- nullius/workspace_cli.py +119 -0
- nullius_eval-0.1.0.dist-info/METADATA +93 -0
- nullius_eval-0.1.0.dist-info/RECORD +291 -0
- nullius_eval-0.1.0.dist-info/WHEEL +5 -0
- nullius_eval-0.1.0.dist-info/entry_points.txt +2 -0
- nullius_eval-0.1.0.dist-info/licenses/LICENSE +202 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/dmsans-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/ibmplexmono-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/licenses/app/static/fonts/instrumentserif-OFL.txt +93 -0
- nullius_eval-0.1.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,442 @@
|
|
|
1
|
+
"""The feasibility sweep, read back from what the controls recorded.
|
|
2
|
+
|
|
3
|
+
A sweep runs Harbor's two control agents over a task set to ask one question
|
|
4
|
+
per task — *is the task feasible*: can an empty solution fail, and can a
|
|
5
|
+
reference solution pass? ``nop`` does nothing, so the verifier must reject
|
|
6
|
+
it; ``oracle`` applies the task's reference solution, so the verifier must
|
|
7
|
+
accept it. Environment setup, data loading and the verifier are all
|
|
8
|
+
exercised on the way. It is feasibility evidence and nothing else: not a
|
|
9
|
+
configuration or hyperparameter search, and no promise that an agent will
|
|
10
|
+
succeed on a feasible task.
|
|
11
|
+
|
|
12
|
+
This module builds the report from recorded jobs. It never runs anything.
|
|
13
|
+
|
|
14
|
+
Where the controls are on disk: a run started from the builder with several
|
|
15
|
+
agents writes one job per agent, ``<name>-nop``, ``<name>-oracle`` and
|
|
16
|
+
``<name>-<agent>``; the portable bundle's ``--bracket`` writes
|
|
17
|
+
``<name>-bracket-nop`` and ``<name>-bracket-oracle``; a sweep may also be
|
|
18
|
+
a control job on its own. ``related`` finds the controls that belong with a
|
|
19
|
+
job by those names, within the one workspace, and ``report`` reads them.
|
|
20
|
+
|
|
21
|
+
Per task the report keeps **both** control outcomes and classifies from the
|
|
22
|
+
pair, in this order — the first that applies wins, so a task is called
|
|
23
|
+
feasible only when both sides ran and said the right thing:
|
|
24
|
+
|
|
25
|
+
unexpected_reward nop was accepted: the verifier rejects nothing, and
|
|
26
|
+
nothing else about the task can be trusted
|
|
27
|
+
task_broken nop was rejected, but the verifier says the empty
|
|
28
|
+
solution failed tests that were passing before it —
|
|
29
|
+
held (PASS_TO_PASS) checks. Nothing was changed, so
|
|
30
|
+
the task was already broken as shipped, or its
|
|
31
|
+
verifier is unstable; whatever oracle said, the
|
|
32
|
+
task is not feasible
|
|
33
|
+
missing a side is absent (never ran, or not in this
|
|
34
|
+
workspace) or recorded no outcome
|
|
35
|
+
setup_failure a side failed before the verifier ran — an
|
|
36
|
+
environment, image, API or agent error — so the
|
|
37
|
+
task was not exercised
|
|
38
|
+
reference_rejected oracle ran and the verifier rejected the reference
|
|
39
|
+
solution: the task cannot be passed as shipped
|
|
40
|
+
feasible oracle accepted, nop rejected, both by the verifier
|
|
41
|
+
|
|
42
|
+
One side alone never validates a task: oracle-only evidence cannot show that
|
|
43
|
+
a verifier rejects an empty solution, nop-only evidence cannot show that a
|
|
44
|
+
solution can pass. The header says which controls actually executed, and
|
|
45
|
+
over what scope, so a report over one control reads as exactly that.
|
|
46
|
+
|
|
47
|
+
``task_broken`` is the same facet the trial page shows as *⚠ Broke tests*
|
|
48
|
+
(`Trial.is_regression`, from `p2p_fail`), read on the control instead of the
|
|
49
|
+
agent: on the agent's trial a held failure is a regression the patch caused;
|
|
50
|
+
on ``nop``, which applies nothing, it is a failure the task brought with it.
|
|
51
|
+
It is benchmark-agnostic — any reader that fills ``held`` gets it — and a
|
|
52
|
+
benchmark whose verifier reports no held checks simply never
|
|
53
|
+
produces the class, rather than a plausible "not broken".
|
|
54
|
+
"""
|
|
55
|
+
from __future__ import annotations
|
|
56
|
+
|
|
57
|
+
import re
|
|
58
|
+
from dataclasses import dataclass, field
|
|
59
|
+
|
|
60
|
+
from . import classify
|
|
61
|
+
from .parse import CONTROL_AGENTS
|
|
62
|
+
|
|
63
|
+
NOP, ORACLE = "nop", "oracle"
|
|
64
|
+
CONTROLS = (NOP, ORACLE)
|
|
65
|
+
|
|
66
|
+
FEASIBLE = "feasible"
|
|
67
|
+
UNEXPECTED_REWARD = "unexpected_reward"
|
|
68
|
+
TASK_BROKEN = "task_broken"
|
|
69
|
+
SETUP_FAILURE = "setup_failure"
|
|
70
|
+
REFERENCE_REJECTED = "reference_rejected"
|
|
71
|
+
MISSING = "missing"
|
|
72
|
+
ORDER = [FEASIBLE, UNEXPECTED_REWARD, TASK_BROKEN, REFERENCE_REJECTED, SETUP_FAILURE, MISSING]
|
|
73
|
+
LABELS = {
|
|
74
|
+
FEASIBLE: "Feasible",
|
|
75
|
+
UNEXPECTED_REWARD: "Unexpected reward",
|
|
76
|
+
TASK_BROKEN: "Task broken",
|
|
77
|
+
REFERENCE_REJECTED: "Reference rejected",
|
|
78
|
+
SETUP_FAILURE: "Setup/data failure",
|
|
79
|
+
MISSING: "Missing result",
|
|
80
|
+
}
|
|
81
|
+
DESCRIPTIONS = {
|
|
82
|
+
FEASIBLE: "The verifier rejected the empty solution and accepted the reference one. Both controls ran; the task is feasible.",
|
|
83
|
+
UNEXPECTED_REWARD: "The verifier accepted the empty solution. It rejects nothing, so a pass on this task means nothing — whatever the oracle did.",
|
|
84
|
+
TASK_BROKEN: "The empty solution failed tests that were passing before it. nop changes nothing, so the task was already broken as shipped, or its verifier is unstable — the failures are named. Not feasible, whatever the oracle did.",
|
|
85
|
+
REFERENCE_REJECTED: "Both controls reached the verifier, and it rejected the reference solution. The task cannot be passed as shipped.",
|
|
86
|
+
SETUP_FAILURE: "A control failed before the verifier ran — environment build, image, API or agent error — so the task was not exercised on that side.",
|
|
87
|
+
MISSING: "A control has no recorded outcome for this task: it did not run, ran elsewhere, or recorded nothing. Not validated.",
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
# Outcomes that mean a control reached the verifier and was rejected.
|
|
91
|
+
_REJECTED = {classify.FAILED_VERIFICATION, classify.PATCH_FAILED_TESTS, classify.PATCH_NOT_APPLIED, classify.NO_PATCH}
|
|
92
|
+
# Outcomes that mean the control failed before the verifier could say anything.
|
|
93
|
+
_FAILED_BEFORE_VERIFIER = classify.INFRA_OUTCOMES | {classify.AGENT_ERROR}
|
|
94
|
+
|
|
95
|
+
_SUFFIX = re.compile(r"-(?:bracket-)?(?:nop|oracle)$")
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@dataclass
|
|
99
|
+
class Side:
|
|
100
|
+
"""One control's outcome on one task. ``present`` is False when the
|
|
101
|
+
control never recorded this task — that is the missing case, kept apart
|
|
102
|
+
from a recorded outcome of ``unknown``."""
|
|
103
|
+
|
|
104
|
+
control: str
|
|
105
|
+
present: bool = False
|
|
106
|
+
job_id: str = ""
|
|
107
|
+
trial_name: str = ""
|
|
108
|
+
outcome: str = ""
|
|
109
|
+
reward: float | None = None
|
|
110
|
+
exception_type: str = ""
|
|
111
|
+
# Held (PASS_TO_PASS) checks the verifier saw fail on this control, as
|
|
112
|
+
# `Trial.p2p_fail` / `p2p_failures` record them — the same reading the
|
|
113
|
+
# trial page flags as *⚠ Broke tests*.
|
|
114
|
+
p2p_fail: int = 0
|
|
115
|
+
p2p_failures: list = field(default_factory=list)
|
|
116
|
+
|
|
117
|
+
@property
|
|
118
|
+
def label(self) -> str:
|
|
119
|
+
if not self.present:
|
|
120
|
+
return "not run"
|
|
121
|
+
return classify.LABELS.get(self.outcome, self.outcome)
|
|
122
|
+
|
|
123
|
+
@property
|
|
124
|
+
def accepted(self) -> bool:
|
|
125
|
+
return self.present and self.outcome == classify.RESOLVED
|
|
126
|
+
|
|
127
|
+
@property
|
|
128
|
+
def rejected(self) -> bool:
|
|
129
|
+
return self.present and self.outcome in _REJECTED
|
|
130
|
+
|
|
131
|
+
@property
|
|
132
|
+
def failed(self) -> bool:
|
|
133
|
+
return self.present and self.outcome in _FAILED_BEFORE_VERIFIER
|
|
134
|
+
|
|
135
|
+
@property
|
|
136
|
+
def unscored(self) -> bool:
|
|
137
|
+
return not self.present or self.outcome == classify.UNKNOWN
|
|
138
|
+
|
|
139
|
+
@property
|
|
140
|
+
def broke_tests(self) -> bool:
|
|
141
|
+
"""The verifier ran and saw previously-passing tests fail."""
|
|
142
|
+
return self.present and self.p2p_fail > 0
|
|
143
|
+
|
|
144
|
+
|
|
145
|
+
@dataclass
|
|
146
|
+
class TaskRow:
|
|
147
|
+
task_base: str
|
|
148
|
+
task_name: str
|
|
149
|
+
nop: Side
|
|
150
|
+
oracle: Side
|
|
151
|
+
verdict: str = MISSING
|
|
152
|
+
|
|
153
|
+
@property
|
|
154
|
+
def label(self) -> str:
|
|
155
|
+
return LABELS[self.verdict]
|
|
156
|
+
|
|
157
|
+
@property
|
|
158
|
+
def validated(self) -> bool:
|
|
159
|
+
"""True only for `feasible`: both sides ran and the verifier answered
|
|
160
|
+
both the right way. Never true with a side missing or failed."""
|
|
161
|
+
return self.verdict == FEASIBLE
|
|
162
|
+
|
|
163
|
+
@property
|
|
164
|
+
def why(self) -> str:
|
|
165
|
+
"""The row's reason in one phrase, naming the side that decided it."""
|
|
166
|
+
if self.verdict == UNEXPECTED_REWARD:
|
|
167
|
+
return "nop accepted"
|
|
168
|
+
if self.verdict == TASK_BROKEN:
|
|
169
|
+
n = self.nop.p2p_fail
|
|
170
|
+
return f"nop broke {n} passing test{'' if n == 1 else 's'}: " + ", ".join(self.nop.p2p_failures)
|
|
171
|
+
if self.verdict == MISSING:
|
|
172
|
+
sides = [s.control for s in (self.nop, self.oracle) if s.unscored]
|
|
173
|
+
return ", ".join(f"{c} {'not run' if not getattr(self, c).present else 'unscored'}" for c in sides)
|
|
174
|
+
if self.verdict == SETUP_FAILURE:
|
|
175
|
+
sides = [s for s in (self.nop, self.oracle) if s.failed]
|
|
176
|
+
return ", ".join(f"{s.control}: {s.exception_type or s.label}" for s in sides)
|
|
177
|
+
if self.verdict == REFERENCE_REJECTED:
|
|
178
|
+
return f"oracle {self.oracle.label.lower()}"
|
|
179
|
+
return "nop rejected, oracle accepted"
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
@dataclass
|
|
183
|
+
class ControlRun:
|
|
184
|
+
"""One control as it actually executed: the job it is, and how much of
|
|
185
|
+
the scope it covered."""
|
|
186
|
+
|
|
187
|
+
control: str
|
|
188
|
+
job_id: str
|
|
189
|
+
n_trials: int
|
|
190
|
+
n_planned: int
|
|
191
|
+
state: str # preparing | running | finished
|
|
192
|
+
|
|
193
|
+
@property
|
|
194
|
+
def short(self) -> str:
|
|
195
|
+
"""The control and its coverage, without the job id: what the page shows."""
|
|
196
|
+
planned = f" of {self.n_planned} planned" if self.n_planned and self.n_planned != self.n_trials else ""
|
|
197
|
+
return f"{self.control} · {self.n_trials} trial{'s' if self.n_trials != 1 else ''}{planned}{' · still ' + self.state if self.state != 'finished' else ''}"
|
|
198
|
+
|
|
199
|
+
@property
|
|
200
|
+
def summary(self) -> str:
|
|
201
|
+
"""The same with the job it is: the hover, and the log line."""
|
|
202
|
+
return self.short.replace(f"{self.control} · ", f"{self.control} · {self.job_id} · ", 1)
|
|
203
|
+
|
|
204
|
+
|
|
205
|
+
@dataclass
|
|
206
|
+
class ControlSummary:
|
|
207
|
+
"""One control job read as what it is: a check that behaved or did not,
|
|
208
|
+
never a score.
|
|
209
|
+
|
|
210
|
+
``nop`` behaves when the verifier rejects it, ``oracle`` when the
|
|
211
|
+
verifier accepts it; the other answer is the feasibility class for that
|
|
212
|
+
side — ``unexpected reward`` for a resolved nop, ``reference rejected``
|
|
213
|
+
for a failed oracle. A trial that failed before the verifier ran is a
|
|
214
|
+
setup failure, and one with no verdict is unscored; neither says the
|
|
215
|
+
task is feasible, so neither counts as behaved. The runs list shows
|
|
216
|
+
this in place of the control's Solved/Raw/Adjusted cells and keeps its
|
|
217
|
+
trials out of the agents' totals — a nop that failed every task is the
|
|
218
|
+
expected result, not "Failed verification".
|
|
219
|
+
"""
|
|
220
|
+
|
|
221
|
+
control: str
|
|
222
|
+
n: int
|
|
223
|
+
n_behaved: int
|
|
224
|
+
n_unexpected: int
|
|
225
|
+
n_failed: int
|
|
226
|
+
n_unscored: int
|
|
227
|
+
running: bool = False
|
|
228
|
+
|
|
229
|
+
@property
|
|
230
|
+
def expected(self) -> str:
|
|
231
|
+
"""What the verifier must say for this control to have behaved."""
|
|
232
|
+
return "rejected" if self.control == NOP else "passed"
|
|
233
|
+
|
|
234
|
+
@property
|
|
235
|
+
def problem(self) -> str:
|
|
236
|
+
"""The feasibility class of the side that did not behave, or ''."""
|
|
237
|
+
if self.n_unexpected:
|
|
238
|
+
return "unexpected reward" if self.control == NOP else "reference rejected"
|
|
239
|
+
if self.n_failed:
|
|
240
|
+
return "setup failure"
|
|
241
|
+
return ""
|
|
242
|
+
|
|
243
|
+
@property
|
|
244
|
+
def ok(self) -> bool:
|
|
245
|
+
return self.n > 0 and self.n_behaved == self.n
|
|
246
|
+
|
|
247
|
+
@property
|
|
248
|
+
def state(self) -> str:
|
|
249
|
+
"""`ok`, `warn` or `pending` — the tone of the line and the cells."""
|
|
250
|
+
if self.n_unexpected or self.n_failed:
|
|
251
|
+
return "warn"
|
|
252
|
+
if self.ok and not self.running:
|
|
253
|
+
return "ok"
|
|
254
|
+
return "pending"
|
|
255
|
+
|
|
256
|
+
@property
|
|
257
|
+
def label(self) -> str:
|
|
258
|
+
"""`5/5 rejected ✓`, `1 unexpected reward ⚠`, `3/5 passed · running`."""
|
|
259
|
+
if not self.n:
|
|
260
|
+
return "no trials yet" if self.running else "no trials"
|
|
261
|
+
if self.problem:
|
|
262
|
+
n = self.n_unexpected or self.n_failed
|
|
263
|
+
return f"{n} {self.problem} ⚠"
|
|
264
|
+
if self.n_unscored:
|
|
265
|
+
return f"{self.n_behaved}/{self.n} {self.expected} · {self.n_unscored} unscored"
|
|
266
|
+
if self.running:
|
|
267
|
+
return f"{self.n_behaved}/{self.n} {self.expected} · running"
|
|
268
|
+
return f"{self.n_behaved}/{self.n} {self.expected} ✓"
|
|
269
|
+
|
|
270
|
+
|
|
271
|
+
def summary_of(job) -> ControlSummary | None:
|
|
272
|
+
"""`job` read as a control, or None when it is an agent's run."""
|
|
273
|
+
control = control_of(job)
|
|
274
|
+
if not control:
|
|
275
|
+
return None
|
|
276
|
+
sides = [Side(control, True, job.job_id, t.trial_name, _outcome(t), t.reward, t.exception_type,
|
|
277
|
+
t.p2p_fail, list(t.p2p_failures)) for t in job.trials]
|
|
278
|
+
behaved = (lambda s: s.rejected) if control == NOP else (lambda s: s.accepted)
|
|
279
|
+
wrong = (lambda s: s.accepted) if control == NOP else (lambda s: s.rejected)
|
|
280
|
+
return ControlSummary(
|
|
281
|
+
control, len(sides),
|
|
282
|
+
sum(1 for s in sides if behaved(s)),
|
|
283
|
+
sum(1 for s in sides if wrong(s)),
|
|
284
|
+
sum(1 for s in sides if s.failed),
|
|
285
|
+
sum(1 for s in sides if s.unscored),
|
|
286
|
+
running=bool(job.is_running or job.is_preparing),
|
|
287
|
+
)
|
|
288
|
+
|
|
289
|
+
|
|
290
|
+
@dataclass
|
|
291
|
+
class Report:
|
|
292
|
+
job_id: str
|
|
293
|
+
dataset: str
|
|
294
|
+
n_tasks: int # the task limit the job recorded; 0 is the full dataset
|
|
295
|
+
executed: list = field(default_factory=list) # [ControlRun], in CONTROLS order
|
|
296
|
+
rows: list = field(default_factory=list) # [TaskRow], by task name
|
|
297
|
+
|
|
298
|
+
@property
|
|
299
|
+
def not_run(self) -> list:
|
|
300
|
+
ran = {c.control for c in self.executed}
|
|
301
|
+
return [c for c in CONTROLS if c not in ran]
|
|
302
|
+
|
|
303
|
+
@property
|
|
304
|
+
def scope(self) -> str:
|
|
305
|
+
"""The task scope the job recorded, as the recipe asked for it."""
|
|
306
|
+
ds = self.dataset or "the dataset"
|
|
307
|
+
if self.n_tasks:
|
|
308
|
+
return f"{self.n_tasks} task{'s' if self.n_tasks != 1 else ''} of {ds} (--n-tasks {self.n_tasks})"
|
|
309
|
+
return f"every task of {ds} (no task limit recorded)"
|
|
310
|
+
|
|
311
|
+
@property
|
|
312
|
+
def counts(self) -> dict:
|
|
313
|
+
c = {k: 0 for k in ORDER}
|
|
314
|
+
for r in self.rows:
|
|
315
|
+
c[r.verdict] += 1
|
|
316
|
+
return c
|
|
317
|
+
|
|
318
|
+
@property
|
|
319
|
+
def n_feasible(self) -> int:
|
|
320
|
+
return self.counts[FEASIBLE]
|
|
321
|
+
|
|
322
|
+
@property
|
|
323
|
+
def caveat(self) -> str:
|
|
324
|
+
"""What this evidence cannot say, when one control is absent."""
|
|
325
|
+
missing = self.not_run
|
|
326
|
+
if not missing:
|
|
327
|
+
return ""
|
|
328
|
+
if missing == [NOP]:
|
|
329
|
+
return "Only oracle ran: this cannot establish that the verifier rejects an empty solution. No task is validated."
|
|
330
|
+
if missing == [ORACLE]:
|
|
331
|
+
return "Only nop ran: this cannot establish that a solution can pass. No task is validated."
|
|
332
|
+
return "Neither control ran: nothing here is feasibility evidence."
|
|
333
|
+
|
|
334
|
+
|
|
335
|
+
def control_of(job) -> str:
|
|
336
|
+
"""`nop` or `oracle` when the job is one of Harbor's controls, or a
|
|
337
|
+
standard job that ran a control its benchmark declares, else ''. A
|
|
338
|
+
runner's arm is never one, whatever it is called (`Job.kind`)."""
|
|
339
|
+
name = job.agent_label.lower()
|
|
340
|
+
return name if name in CONTROL_AGENTS and job.is_control else ""
|
|
341
|
+
|
|
342
|
+
|
|
343
|
+
def _outcome(t) -> str:
|
|
344
|
+
"""A control's outcome on one task. A benchmark that scores and does not
|
|
345
|
+
pass or fail leaves *scored*: at the ceiling (1) that is the reference
|
|
346
|
+
accepted, at the floor (0) the answer rejected, in between neither."""
|
|
347
|
+
if t.outcome == classify.SCORED and t.score is not None:
|
|
348
|
+
if t.score >= 1.0:
|
|
349
|
+
return classify.RESOLVED
|
|
350
|
+
if t.score <= 0.0:
|
|
351
|
+
return classify.FAILED_VERIFICATION
|
|
352
|
+
return t.outcome
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
def base_name(job_id: str) -> str:
|
|
356
|
+
"""The recipe name a control job belongs to: the id without its control
|
|
357
|
+
suffix, in either spelling the builder writes."""
|
|
358
|
+
return _SUFFIX.sub("", job_id)
|
|
359
|
+
|
|
360
|
+
|
|
361
|
+
def related(job, jobs) -> dict:
|
|
362
|
+
"""The control jobs that belong with `job`, by control name.
|
|
363
|
+
|
|
364
|
+
Matched by the builder's naming: a job's own base name, and — for an
|
|
365
|
+
agent job named `<base>-<agent>` — the base with its last segment
|
|
366
|
+
dropped. A control job is always its own side. The dataset must agree
|
|
367
|
+
where both record one, so a control over another dataset that happens
|
|
368
|
+
to share a name is not read as this job's.
|
|
369
|
+
"""
|
|
370
|
+
out = {}
|
|
371
|
+
own = control_of(job)
|
|
372
|
+
if own:
|
|
373
|
+
out[own] = job
|
|
374
|
+
bases = [base_name(job.job_id)]
|
|
375
|
+
if not own and "-" in job.job_id:
|
|
376
|
+
bases.append(job.job_id.rsplit("-", 1)[0])
|
|
377
|
+
wanted = {f"{b}{infix}{c}": c for b in bases for infix in ("-", "-bracket-") for c in CONTROLS}
|
|
378
|
+
for other in jobs:
|
|
379
|
+
c = wanted.get(other.job_id)
|
|
380
|
+
if not c or c in out or control_of(other) != c:
|
|
381
|
+
continue
|
|
382
|
+
if job.dataset and other.dataset and job.dataset != other.dataset:
|
|
383
|
+
continue
|
|
384
|
+
out[c] = other
|
|
385
|
+
return out
|
|
386
|
+
|
|
387
|
+
|
|
388
|
+
def _last_by_task(job) -> dict:
|
|
389
|
+
"""The last recorded attempt per task, as the run pages show one."""
|
|
390
|
+
by_task = {}
|
|
391
|
+
for t in job.trials:
|
|
392
|
+
by_task[t.task_base] = t
|
|
393
|
+
return by_task
|
|
394
|
+
|
|
395
|
+
|
|
396
|
+
def _side(control, job, task_base) -> Side:
|
|
397
|
+
if job is None:
|
|
398
|
+
return Side(control)
|
|
399
|
+
t = _last_by_task(job).get(task_base)
|
|
400
|
+
if t is None:
|
|
401
|
+
return Side(control, job_id=job.job_id)
|
|
402
|
+
return Side(control, True, job.job_id, t.trial_name, _outcome(t), t.reward, t.exception_type,
|
|
403
|
+
t.p2p_fail, list(t.p2p_failures))
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def verdict(nop: Side, oracle: Side) -> str:
|
|
407
|
+
"""The class of a task from its two sides — see the module docstring for
|
|
408
|
+
the order and why a single side never validates."""
|
|
409
|
+
if nop.accepted:
|
|
410
|
+
return UNEXPECTED_REWARD
|
|
411
|
+
if nop.broke_tests:
|
|
412
|
+
return TASK_BROKEN
|
|
413
|
+
if nop.unscored or oracle.unscored:
|
|
414
|
+
return MISSING
|
|
415
|
+
if nop.failed or oracle.failed:
|
|
416
|
+
return SETUP_FAILURE
|
|
417
|
+
if not oracle.accepted:
|
|
418
|
+
return REFERENCE_REJECTED
|
|
419
|
+
return FEASIBLE
|
|
420
|
+
|
|
421
|
+
|
|
422
|
+
def report(job, jobs) -> Report:
|
|
423
|
+
"""The feasibility report for `job`: its controls, the scope they ran
|
|
424
|
+
over, and every task either control recorded with both outcomes."""
|
|
425
|
+
controls = related(job, jobs)
|
|
426
|
+
scoped = job if not control_of(job) else next((controls[c] for c in CONTROLS if c in controls), job)
|
|
427
|
+
rep = Report(job_id=job.job_id, dataset=scoped.dataset, n_tasks=scoped.n_tasks)
|
|
428
|
+
for c in CONTROLS:
|
|
429
|
+
cj = controls.get(c)
|
|
430
|
+
if cj is not None:
|
|
431
|
+
rep.executed.append(ControlRun(c, cj.job_id, len(cj.trials), cj.n_planned, cj.state))
|
|
432
|
+
# Every task any side recorded, and — for an agent job — every task the
|
|
433
|
+
# agent ran, so a task the bracket never covered shows as missing rather
|
|
434
|
+
# than disappearing from the report.
|
|
435
|
+
names = {}
|
|
436
|
+
for source in [job, *controls.values()]:
|
|
437
|
+
for t in source.trials:
|
|
438
|
+
names.setdefault(t.task_base, t.task_name)
|
|
439
|
+
for task_base in sorted(names):
|
|
440
|
+
nop, oracle = _side(NOP, controls.get(NOP), task_base), _side(ORACLE, controls.get(ORACLE), task_base)
|
|
441
|
+
rep.rows.append(TaskRow(task_base, names[task_base], nop, oracle, verdict(nop, oracle)))
|
|
442
|
+
return rep
|
nullius/runs/launches.py
ADDED
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
"""Runs grouped by the launch that made them.
|
|
2
|
+
|
|
3
|
+
Runs started together ran the same tasks: the agents a recipe named, and the
|
|
4
|
+
nop and oracle controls that check those tasks. The Runs page shows each launch
|
|
5
|
+
as one block, so it needs to know which runs belong together. Two sources, in
|
|
6
|
+
this order:
|
|
7
|
+
|
|
8
|
+
- **The run record.** A run started from the builder's Run writes
|
|
9
|
+
``runs/<name>/run.json``, and its plan names every job it launches
|
|
10
|
+
(``execution.Plan.jobs``). Those jobs are one launch, including the ones
|
|
11
|
+
Harbor has not written yet.
|
|
12
|
+
- **Inferred, for everything else** — an imported tree, a job run by hand, a run
|
|
13
|
+
from before the record existed: runs on the same dataset whose start is within
|
|
14
|
+
``WINDOW_SEC`` of the launch's first run are one launch. A run with no start
|
|
15
|
+
time at all is a launch of its own.
|
|
16
|
+
|
|
17
|
+
Either way, the shards of one launch of the Nullius standard - jobs whose
|
|
18
|
+
config names the same ``launch`` id (``Job.launch_id``), each run on a machine
|
|
19
|
+
of its own, perhaps by a run record of its own, perhaps hours apart - are one
|
|
20
|
+
launch: a shard joins the launch that holds another shard of its id before
|
|
21
|
+
any other rule is asked.
|
|
22
|
+
|
|
23
|
+
Nothing here reads a file: it takes the parsed jobs and the records' job lists,
|
|
24
|
+
and groups. Newest launch first.
|
|
25
|
+
"""
|
|
26
|
+
from __future__ import annotations
|
|
27
|
+
|
|
28
|
+
import os
|
|
29
|
+
from dataclasses import dataclass, field
|
|
30
|
+
from datetime import datetime, timezone
|
|
31
|
+
|
|
32
|
+
WINDOW_SEC = 30 * 60
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
@dataclass
|
|
36
|
+
class Launch:
|
|
37
|
+
key: str # "run:<execution id>" or "inferred:<first job id>"
|
|
38
|
+
source: str # "record" or "inferred"
|
|
39
|
+
dataset: str | None
|
|
40
|
+
started: datetime | None # the earliest start of its runs; none written yet, when it was launched
|
|
41
|
+
job_ids: list = field(default_factory=list) # written by Harbor, in start order
|
|
42
|
+
pending: list = field(default_factory=list) # named by the record, not written yet
|
|
43
|
+
name: str | None = None # the run record's name
|
|
44
|
+
launch_ids: set = field(default_factory=set) # the standard's launch ids among its runs (its shards)
|
|
45
|
+
|
|
46
|
+
@property
|
|
47
|
+
def title(self) -> str:
|
|
48
|
+
"""What the launch is called on the Runs page: the standard's launch id
|
|
49
|
+
when its runs are the shards of one launch - what every shard
|
|
50
|
+
records, and what ``cli merge`` puts together - else ``prefix``."""
|
|
51
|
+
return next(iter(self.launch_ids)) if len(self.launch_ids) == 1 else self.prefix
|
|
52
|
+
|
|
53
|
+
@property
|
|
54
|
+
def prefix(self) -> str:
|
|
55
|
+
"""What every run of the launch is called, up to the part that differs:
|
|
56
|
+
``tau-latest3-0919-132944-*`` over ``…-agent`` and ``…-nop``. The
|
|
57
|
+
common prefix of the ids, cut back to a separator; the id itself for
|
|
58
|
+
a launch of one run; the first id "and N more" when they share nothing."""
|
|
59
|
+
ids = self.job_ids + self.pending
|
|
60
|
+
if len(ids) == 1:
|
|
61
|
+
return ids[0]
|
|
62
|
+
common = os.path.commonprefix(ids)
|
|
63
|
+
cut = max(common.rfind("-"), common.rfind("_"))
|
|
64
|
+
if cut <= 0: # nothing shared worth a wildcard: name the first and count the rest
|
|
65
|
+
return f"{ids[0]} and {len(ids) - 1} more"
|
|
66
|
+
return common[:cut + 1] + "*"
|
|
67
|
+
|
|
68
|
+
|
|
69
|
+
def merged_sources(jobs) -> set[str]:
|
|
70
|
+
"""Source jobs represented by a finished merge in this workspace.
|
|
71
|
+
|
|
72
|
+
Use the recorded names, never a shared launch id: another agent or an
|
|
73
|
+
unmerged shard may belong to the same launch and must still count.
|
|
74
|
+
"""
|
|
75
|
+
return {source for j in jobs if not j.is_running and not j.is_preparing
|
|
76
|
+
for source in getattr(j, "merged_from", []) if source != j.job_id}
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def group(jobs, records: dict | None = None, window: float = WINDOW_SEC) -> list[Launch]:
|
|
80
|
+
"""``jobs``: the parsed jobs (``job_id``, ``dataset``, ``started_at``).
|
|
81
|
+
``records``: ``{execution id: (name, [job ids], launched at)}``, the builder's
|
|
82
|
+
run records, the time in seconds. Returns the launches, newest first, each job in exactly one."""
|
|
83
|
+
records = records or {}
|
|
84
|
+
by_id = {j.job_id: j for j in jobs}
|
|
85
|
+
placed: set[str] = set()
|
|
86
|
+
out: list[Launch] = []
|
|
87
|
+
for key, (name, ids, launched) in records.items():
|
|
88
|
+
mine = [i for i in ids if i in by_id and i not in placed]
|
|
89
|
+
pending = [i for i in ids if i not in by_id and i not in placed]
|
|
90
|
+
if not mine and not pending:
|
|
91
|
+
continue
|
|
92
|
+
placed.update(mine + pending)
|
|
93
|
+
written = sorted((by_id[i] for i in mine), key=_start_key)
|
|
94
|
+
ids = {_launch_id(j) for j in written} - {""}
|
|
95
|
+
home = next((l for l in out if l.launch_ids & ids), None)
|
|
96
|
+
if home is not None: # another shard of a launch a record placed already
|
|
97
|
+
_join(home, written, pending)
|
|
98
|
+
continue
|
|
99
|
+
out.append(Launch(f"run:{key}", "record", _dataset_of(written), _earliest(written) or _launched_at(launched),
|
|
100
|
+
[j.job_id for j in written], pending, name, ids))
|
|
101
|
+
rest = sorted((j for j in jobs if j.job_id not in placed), key=_start_key)
|
|
102
|
+
for j in rest:
|
|
103
|
+
dataset = getattr(j, "questions_from", "") or j.dataset
|
|
104
|
+
lid = _launch_id(j)
|
|
105
|
+
home = next((l for l in out if lid and lid in l.launch_ids), None)
|
|
106
|
+
if home is not None: # a shard of a launch placed already
|
|
107
|
+
_join(home, [j], [])
|
|
108
|
+
continue
|
|
109
|
+
home = next((l for l in out if l.source == "inferred" and l.dataset == dataset and l.started and j.started_at
|
|
110
|
+
and 0 <= (j.started_at - l.started).total_seconds() <= window), None)
|
|
111
|
+
if home is None:
|
|
112
|
+
out.append(Launch(f"inferred:{j.job_id}", "inferred", dataset, j.started_at, [j.job_id], launch_ids={lid} - {""}))
|
|
113
|
+
else:
|
|
114
|
+
home.job_ids.append(j.job_id)
|
|
115
|
+
home.launch_ids |= {lid} - {""}
|
|
116
|
+
return sorted(out, key=lambda l: (l.started is not None, l.started.timestamp() if l.started else 0.0), reverse=True)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def _launch_id(j) -> str:
|
|
120
|
+
return str(getattr(j, "launch_id", "") or "")
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
def _join(home: Launch, jobs, pending) -> None:
|
|
124
|
+
"""Shards of a launch placed already join it; its start is the earliest of them all."""
|
|
125
|
+
home.job_ids += [j.job_id for j in jobs]
|
|
126
|
+
home.pending += [p for p in pending if p not in home.pending]
|
|
127
|
+
starts = [s for s in [home.started] + [j.started_at for j in jobs] if s]
|
|
128
|
+
home.started = min(starts) if starts else None
|
|
129
|
+
home.launch_ids |= {_launch_id(j) for j in jobs} - {""}
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _start_key(j):
|
|
133
|
+
s = j.started_at
|
|
134
|
+
return (s is None, s.timestamp() if s else 0.0, j.job_id)
|
|
135
|
+
|
|
136
|
+
|
|
137
|
+
def _launched_at(seconds):
|
|
138
|
+
return datetime.fromtimestamp(seconds, timezone.utc) if seconds else None # aware, as a job's stamps are (parse._iso)
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
def _earliest(jobs):
|
|
142
|
+
starts = [j.started_at for j in jobs if j.started_at]
|
|
143
|
+
return min(starts) if starts else None
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _dataset_of(jobs):
|
|
147
|
+
# a runner's arms answer the same questions, the oracle's from a file of its own
|
|
148
|
+
names = {getattr(j, "questions_from", "") or j.dataset for j in jobs if j.dataset}
|
|
149
|
+
return names.pop() if len(names) == 1 else (None if not names else "mixed")
|