techtree 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- techtree/__init__.py +35 -0
- techtree/__main__.py +14 -0
- techtree/canonical.py +239 -0
- techtree/catalog/__init__.py +25 -0
- techtree/catalog/repository.py +400 -0
- techtree/catalog/service.py +419 -0
- techtree/cli/__init__.py +1 -0
- techtree/cli/app.py +416 -0
- techtree/cli/commands/__init__.py +1 -0
- techtree/cli/commands/climb.py +1223 -0
- techtree/cli/commands/doctor.py +147 -0
- techtree/cli/commands/engine.py +207 -0
- techtree/cli/commands/proof.py +556 -0
- techtree/cli/commands/publish.py +447 -0
- techtree/cli/commands/release.py +303 -0
- techtree/cli/commands/run.py +1067 -0
- techtree/cli/commands/setup.py +181 -0
- techtree/cli/commands/skill.py +221 -0
- techtree/cli/commands/uplift.py +698 -0
- techtree/cli/commands/withdraw.py +212 -0
- techtree/cli/confirm.py +47 -0
- techtree/cli/context.py +96 -0
- techtree/cli/invoke.py +220 -0
- techtree/cli/output.py +280 -0
- techtree/constants.py +138 -0
- techtree/crypto.py +128 -0
- techtree/doctor/__init__.py +1 -0
- techtree/doctor/checks.py +675 -0
- techtree/doctor/execution_checks.py +435 -0
- techtree/doctor/service.py +326 -0
- techtree/drafts/__init__.py +32 -0
- techtree/drafts/source.py +146 -0
- techtree/drafts/store.py +992 -0
- techtree/engines/__init__.py +1 -0
- techtree/engines/bundle.py +251 -0
- techtree/engines/installer.py +679 -0
- techtree/engines/registry.py +235 -0
- techtree/engines/runner.py +170 -0
- techtree/errors.py +262 -0
- techtree/fs.py +234 -0
- techtree/harness.py +108 -0
- techtree/identity/__init__.py +41 -0
- techtree/identity/models.py +113 -0
- techtree/identity/service.py +199 -0
- techtree/identity/store.py +263 -0
- techtree/ids.py +85 -0
- techtree/manifests/__init__.py +39 -0
- techtree/manifests/builder.py +433 -0
- techtree/manifests/compare.py +376 -0
- techtree/models/__init__.py +282 -0
- techtree/models/base.py +201 -0
- techtree/models/campaign.py +484 -0
- techtree/models/catalog.py +227 -0
- techtree/models/cli.py +151 -0
- techtree/models/climb.py +254 -0
- techtree/models/data_policy.py +130 -0
- techtree/models/engine.py +156 -0
- techtree/models/episode_receipt.py +130 -0
- techtree/models/evaluation_backend.py +113 -0
- techtree/models/experiment.py +154 -0
- techtree/models/run.py +214 -0
- techtree/models/skill.py +156 -0
- techtree/models/uplift_report.py +158 -0
- techtree/models/validation.py +299 -0
- techtree/paths.py +116 -0
- techtree/presentation/__init__.py +31 -0
- techtree/presentation/build.py +1242 -0
- techtree/presentation/compact.py +246 -0
- techtree/presentation/evidence.py +169 -0
- techtree/presentation/models.py +358 -0
- techtree/presentation/rich.py +312 -0
- techtree/presentation/sanitize.py +156 -0
- techtree/publication/__init__.py +44 -0
- techtree/publication/address.py +180 -0
- techtree/publication/coordinates.py +26 -0
- techtree/publication/journal.py +212 -0
- techtree/publication/keccak.py +183 -0
- techtree/publication/models.py +209 -0
- techtree/publication/offer.py +35 -0
- techtree/publication/service.py +618 -0
- techtree/publication/transport.py +296 -0
- techtree/publication/verify.py +242 -0
- techtree/publication/withdraw.py +156 -0
- techtree/py.typed +0 -0
- techtree/receipts/__init__.py +52 -0
- techtree/receipts/bundle.py +578 -0
- techtree/receipts/compare.py +1065 -0
- techtree/receipts/episode.py +672 -0
- techtree/receipts/execution.py +630 -0
- techtree/receipts/observed.py +474 -0
- techtree/receipts/set.py +336 -0
- techtree/receipts/uplift.py +655 -0
- techtree/receipts/verify.py +1055 -0
- techtree/release/__init__.py +9 -0
- techtree/release/bootstrap.py +509 -0
- techtree/release/checks.py +376 -0
- techtree/release/document.py +125 -0
- techtree/release/generate.py +221 -0
- techtree/release/models.py +293 -0
- techtree/release/provenance.py +109 -0
- techtree/resources/catalog/campaigns/hello-world-climb.json +1 -0
- techtree/resources/catalog/catalog.json +32 -0
- techtree/resources/catalog/climbs/hello-world-climb.json +1 -0
- techtree/resources/catalog/data-policies/hello-world-climb.json +1 -0
- techtree/resources/catalog/taskset-validations/hello-world-climb.json +1 -0
- techtree/resources/catalog/validation-evidence/hello-world-climb.json +1 -0
- techtree/resources/engines/default/engine.json +20 -0
- techtree/resources/engines/default/packages/procedure-transfer-v1/procedure_transfer_v1/__init__.py +7 -0
- techtree/resources/engines/default/packages/procedure-transfer-v1/procedure_transfer_v1/algorithm.py +136 -0
- techtree/resources/engines/default/packages/procedure-transfer-v1/procedure_transfer_v1/dataset.py +156 -0
- techtree/resources/engines/default/packages/procedure-transfer-v1/procedure_transfer_v1/env.py +48 -0
- techtree/resources/engines/default/packages/procedure-transfer-v1/procedure_transfer_v1/taskset.py +163 -0
- techtree/resources/engines/default/packages/procedure-transfer-v1/pyproject.toml +13 -0
- techtree/resources/engines/default/pyproject.toml +23 -0
- techtree/resources/engines/default/tools/inspect_taskset.py +124 -0
- techtree/resources/engines/default/tools/normalize_eval_output.py +470 -0
- techtree/resources/engines/default/tools/normalize_validation.py +222 -0
- techtree/resources/engines/default/uv.lock +1758 -0
- techtree/resources/harness/hermes-agent-0.19.0.json +69 -0
- techtree/resources/release/build-provenance.json +4 -0
- techtree/resources/release/release-core.json +24 -0
- techtree/runs/__init__.py +31 -0
- techtree/runs/artifacts.py +750 -0
- techtree/runs/child_registry.py +228 -0
- techtree/runs/events.py +478 -0
- techtree/runs/executor.py +140 -0
- techtree/runs/fake.py +741 -0
- techtree/runs/launcher.py +253 -0
- techtree/runs/machine.py +489 -0
- techtree/runs/real.py +789 -0
- techtree/runs/service.py +616 -0
- techtree/runs/store.py +555 -0
- techtree/runs/validation.py +259 -0
- techtree/runs/variants.py +684 -0
- techtree/settings.py +143 -0
- techtree/skills/__init__.py +14 -0
- techtree/skills/archive.py +282 -0
- techtree/skills/policy.py +62 -0
- techtree/skills/scanner.py +394 -0
- techtree/skills/service.py +752 -0
- techtree/skills/starter.py +434 -0
- techtree/tasksets/__init__.py +1 -0
- techtree/tasksets/membership.py +269 -0
- techtree/tasksets/provider.py +207 -0
- techtree/tasksets/resolver.py +311 -0
- techtree/tasksets/service.py +484 -0
- techtree/tasksets/verifiers_cli.py +538 -0
- techtree/uplift/__init__.py +20 -0
- techtree/uplift/context.py +544 -0
- techtree/uplift/derive.py +203 -0
- techtree/uplift/public_tasks.py +151 -0
- techtree/uplift/service.py +719 -0
- techtree/uplift/source.py +160 -0
- techtree/verifiers/__init__.py +31 -0
- techtree/verifiers/budget.py +219 -0
- techtree/verifiers/child.py +633 -0
- techtree/verifiers/compiler.py +432 -0
- techtree/verifiers/config.py +365 -0
- techtree/verifiers/credentials.py +321 -0
- techtree/verifiers/image.py +126 -0
- techtree/verifiers/models.py +527 -0
- techtree/verifiers/outputs.py +368 -0
- techtree/verifiers/progress.py +192 -0
- techtree/verifiers/supervisor.py +341 -0
- techtree/verifiers/verify.py +782 -0
- techtree/version.py +39 -0
- techtree/worker/__init__.py +18 -0
- techtree/worker/execute.py +487 -0
- techtree/worker/main.py +57 -0
- techtree-0.1.0.dist-info/METADATA +344 -0
- techtree-0.1.0.dist-info/RECORD +174 -0
- techtree-0.1.0.dist-info/WHEEL +4 -0
- techtree-0.1.0.dist-info/entry_points.txt +3 -0
- techtree-0.1.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,1067 @@
|
|
|
1
|
+
"""``techtree run status``, ``logs``, ``cancel``, and ``result``.
|
|
2
|
+
Spec §12.6 and PR8 §8.12-§8.15.
|
|
3
|
+
|
|
4
|
+
These four commands are the whole of what a caller can do with a run it has
|
|
5
|
+
already started, and they are written for a caller that may be a program.
|
|
6
|
+
|
|
7
|
+
*Status never lies about liveness.* It reports the projected phase, whether
|
|
8
|
+
the worker process still exists, and how old the heartbeat is, and it changes
|
|
9
|
+
nothing. A dead worker on a nonterminal run shows up as exactly that — not as
|
|
10
|
+
a failure the CLI invented on the reader's behalf.
|
|
11
|
+
|
|
12
|
+
*Logs are untrusted output.* A worker writes whatever it writes, including
|
|
13
|
+
whatever a library it called wrote, and what it wrote is what a reader gets:
|
|
14
|
+
decisions document 0036 removed the filter that used to rewrite a line on its
|
|
15
|
+
way out, so a line arrives exactly as the tool printed it. Two things are still
|
|
16
|
+
true of every reading, bounded and followed alike. What comes back is a window
|
|
17
|
+
rather than the file, and the response never contains the log's path: handing
|
|
18
|
+
an agent a filename is handing it an invitation to read something else.
|
|
19
|
+
|
|
20
|
+
*Cancelling is a mutation and is treated as one.* A person is asked; a program
|
|
21
|
+
must pass ``--confirm``. Possession of a run identifier is not intent to stop
|
|
22
|
+
the run.
|
|
23
|
+
|
|
24
|
+
*A fake result announces itself first.* Human output for ``result`` leads with
|
|
25
|
+
the development-only banner before a single number appears, and the machine
|
|
26
|
+
envelope carries the same caveat as a warning, because the caller most likely
|
|
27
|
+
to mistake an invented number for a measurement is the one reading JSON.
|
|
28
|
+
"""
|
|
29
|
+
|
|
30
|
+
from __future__ import annotations
|
|
31
|
+
|
|
32
|
+
import sys
|
|
33
|
+
import time
|
|
34
|
+
from collections.abc import Iterator
|
|
35
|
+
from enum import StrEnum
|
|
36
|
+
from typing import Annotated, Final, Literal
|
|
37
|
+
|
|
38
|
+
import typer
|
|
39
|
+
from rich.console import Console
|
|
40
|
+
from rich.table import Table
|
|
41
|
+
|
|
42
|
+
from techtree.cli.confirm import confirmed
|
|
43
|
+
from techtree.cli.context import CliContext, cli_context
|
|
44
|
+
from techtree.cli.invoke import CommandResult, invoke_command
|
|
45
|
+
from techtree.cli.output import human_console, render_pairs
|
|
46
|
+
from techtree.drafts.store import DraftStore
|
|
47
|
+
from techtree.errors import TechtreeError, UsageError, VerificationError
|
|
48
|
+
from techtree.identity.models import VerificationResult
|
|
49
|
+
from techtree.models.base import Digest, NonEmptyString, ProtocolModel, UtcDateTime
|
|
50
|
+
from techtree.models.cli import CliError, CliMessage, MessageLevel, NextAction
|
|
51
|
+
from techtree.models.experiment import ExperimentVariant
|
|
52
|
+
from techtree.models.run import RunPhase, RunProgress, VariantProgress
|
|
53
|
+
from techtree.models.uplift_report import UpliftReport
|
|
54
|
+
from techtree.presentation.build import build_uplift_presentation
|
|
55
|
+
from techtree.presentation.compact import render_uplift_markdown
|
|
56
|
+
from techtree.presentation.evidence import read_recorded_evidence
|
|
57
|
+
from techtree.presentation.models import TaskDisplay, UpliftPresentationPayload
|
|
58
|
+
from techtree.presentation.rich import render_uplift_console
|
|
59
|
+
from techtree.receipts.bundle import (
|
|
60
|
+
BUNDLE_DIRECTORY,
|
|
61
|
+
PROOF_BUNDLE_INVALID,
|
|
62
|
+
REPORT_FILENAME,
|
|
63
|
+
proof_bundle_dir,
|
|
64
|
+
)
|
|
65
|
+
from techtree.receipts.execution import (
|
|
66
|
+
ComparisonExecutionRecord,
|
|
67
|
+
read_execution_record,
|
|
68
|
+
)
|
|
69
|
+
from techtree.receipts.verify import LocalProofVerifier
|
|
70
|
+
from techtree.runs.artifacts import RunArtifactStore
|
|
71
|
+
from techtree.runs.launcher import (
|
|
72
|
+
WorkerLauncher,
|
|
73
|
+
default_worker_executable,
|
|
74
|
+
scrubbed_worker_environment,
|
|
75
|
+
)
|
|
76
|
+
from techtree.runs.machine import is_terminal
|
|
77
|
+
from techtree.runs.service import (
|
|
78
|
+
DEFAULT_LOG_TAIL,
|
|
79
|
+
MAXIMUM_LOG_TAIL,
|
|
80
|
+
MINIMUM_LOG_TAIL,
|
|
81
|
+
RunService,
|
|
82
|
+
)
|
|
83
|
+
from techtree.runs.store import RunStore
|
|
84
|
+
from techtree.verifiers.models import VariantName
|
|
85
|
+
|
|
86
|
+
__all__ = [
|
|
87
|
+
"CANCEL_COMMAND",
|
|
88
|
+
"CANCEL_CONFIRMATION_REQUIRED",
|
|
89
|
+
"DEVELOPMENT_ONLY_BANNER",
|
|
90
|
+
"DEVELOPMENT_ONLY_RESULT_NOTICE",
|
|
91
|
+
"LOGS_COMMAND",
|
|
92
|
+
"PROVISIONAL_SCORE",
|
|
93
|
+
"RESULT_COMMAND",
|
|
94
|
+
"RUN_FOLLOW_NOT_SUPPORTED_FOR_VARIANT",
|
|
95
|
+
"RUN_FOLLOW_NOT_SUPPORTED_IN_JSON",
|
|
96
|
+
"RUN_WATCH_NOT_SUPPORTED_IN_JSON",
|
|
97
|
+
"SCORE_PROVISIONAL_NOTICE",
|
|
98
|
+
"STATUS_COMMAND",
|
|
99
|
+
"ResultFormat",
|
|
100
|
+
"RunCancelPayload",
|
|
101
|
+
"RunLogsPayload",
|
|
102
|
+
"RunResultPayload",
|
|
103
|
+
"RunStatusPayload",
|
|
104
|
+
"build_run_service",
|
|
105
|
+
"cancel_run_command",
|
|
106
|
+
"development_only_result_notice",
|
|
107
|
+
"logs_run_command",
|
|
108
|
+
"result_run_command",
|
|
109
|
+
"status_run_command",
|
|
110
|
+
"watch_line",
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
STATUS_COMMAND: Final = "run status"
|
|
114
|
+
LOGS_COMMAND: Final = "run logs"
|
|
115
|
+
CANCEL_COMMAND: Final = "run cancel"
|
|
116
|
+
RESULT_COMMAND: Final = "run result"
|
|
117
|
+
|
|
118
|
+
#: Stable error codes this module reports. Spec PR8 §8.16 names the first;
|
|
119
|
+
#: the other two are the machine-mode rules the same section asks for.
|
|
120
|
+
RUN_FOLLOW_NOT_SUPPORTED_IN_JSON: Final = "run_follow_not_supported_in_json"
|
|
121
|
+
RUN_FOLLOW_NOT_SUPPORTED_FOR_VARIANT: Final = "run_follow_not_supported_for_variant"
|
|
122
|
+
RUN_WATCH_NOT_SUPPORTED_IN_JSON: Final = "run_watch_not_supported_in_json"
|
|
123
|
+
CANCEL_CONFIRMATION_REQUIRED: Final = "run_cancel_confirmation_required"
|
|
124
|
+
|
|
125
|
+
#: What every reader of a fake result is told before they read anything else.
|
|
126
|
+
DEVELOPMENT_ONLY_BANNER: Final = (
|
|
127
|
+
"DEVELOPMENT-ONLY FAKE RESULT\n"
|
|
128
|
+
"\n"
|
|
129
|
+
"No agent was evaluated.\n"
|
|
130
|
+
"No model was called.\n"
|
|
131
|
+
"This report is not publication eligible."
|
|
132
|
+
)
|
|
133
|
+
|
|
134
|
+
_FAKE_EXECUTOR_WARNING: Final = (
|
|
135
|
+
"This run was executed by the development fake executor. No agent was "
|
|
136
|
+
"evaluated, no model was called, and the numbers below are invented. The "
|
|
137
|
+
"report is not publication eligible."
|
|
138
|
+
)
|
|
139
|
+
|
|
140
|
+
#: Spec section 29, verbatim. A finished report is the one place a reader
|
|
141
|
+
#: decides what they may do with what they are holding, so it says both halves
|
|
142
|
+
#: at once: which part of the run was real, and whose rights still govern the
|
|
143
|
+
#: artifacts it produced. The DataPolicy digest is named rather than described,
|
|
144
|
+
#: because a digest is the only unambiguous way to point at one.
|
|
145
|
+
DEVELOPMENT_ONLY_RESULT_NOTICE: Final = (
|
|
146
|
+
"This is a development-only report.\n"
|
|
147
|
+
"\n"
|
|
148
|
+
"The taskset was validated through Prime Intellect Verifiers.\n"
|
|
149
|
+
"The baseline and candidate results were generated by the fake executor.\n"
|
|
150
|
+
"No agent was evaluated. The report is not publication eligible.\n"
|
|
151
|
+
"\n"
|
|
152
|
+
"The candidate and generated artifacts remain governed by DataPolicy:\n"
|
|
153
|
+
"{data_policy_digest}"
|
|
154
|
+
)
|
|
155
|
+
|
|
156
|
+
|
|
157
|
+
def development_only_result_notice(data_policy_digest: str) -> str:
|
|
158
|
+
"""Return the spec section 29 warning a development result must carry."""
|
|
159
|
+
return DEVELOPMENT_ONLY_RESULT_NOTICE.format(data_policy_digest=data_policy_digest)
|
|
160
|
+
|
|
161
|
+
|
|
162
|
+
#: What a variant's score column says while the comparison is still running.
|
|
163
|
+
#: Spec section 6.20 forbids showing a delta before both sides have finished
|
|
164
|
+
#: every task, and a blank cell would read as "nothing scored" rather than
|
|
165
|
+
#: "not yet answerable".
|
|
166
|
+
PROVISIONAL_SCORE: Final = "provisional only"
|
|
167
|
+
|
|
168
|
+
#: Spec section 6.20, the sentence under a progress table.
|
|
169
|
+
SCORE_PROVISIONAL_NOTICE: Final = (
|
|
170
|
+
"The score remains provisional until every task completes and Techtree "
|
|
171
|
+
"verifies that the observed configurations match."
|
|
172
|
+
)
|
|
173
|
+
|
|
174
|
+
#: How often human watch mode looks again.
|
|
175
|
+
_WATCH_INTERVAL_SECONDS: Final = 1.0
|
|
176
|
+
|
|
177
|
+
#: How long a follower waits before checking a quiet log again.
|
|
178
|
+
_FOLLOW_INTERVAL_SECONDS: Final = 0.25
|
|
179
|
+
|
|
180
|
+
|
|
181
|
+
class RunStatusPayload(ProtocolModel):
|
|
182
|
+
"""Everything a caller needs to decide what to do about a run next."""
|
|
183
|
+
|
|
184
|
+
run_id: NonEmptyString
|
|
185
|
+
phase: RunPhase
|
|
186
|
+
sequence: int
|
|
187
|
+
updated_at: UtcDateTime
|
|
188
|
+
progress: RunProgress | None
|
|
189
|
+
#: One entry per variant once a concurrent comparison is under way, keyed by
|
|
190
|
+
#: variant name. Spec section 6.20 exposes it here so a machine caller reads
|
|
191
|
+
#: both sides' episode counts without parsing anything meant for a person.
|
|
192
|
+
variant_progress: dict[str, VariantProgress]
|
|
193
|
+
worker_pid: int | None
|
|
194
|
+
worker_alive: bool
|
|
195
|
+
heartbeat_at: UtcDateTime | None
|
|
196
|
+
heartbeat_age_seconds: float | None
|
|
197
|
+
heartbeat_stale: bool
|
|
198
|
+
cancel_requested_at: UtcDateTime | None
|
|
199
|
+
terminal: bool
|
|
200
|
+
result_available: bool
|
|
201
|
+
result_digest: Digest | None
|
|
202
|
+
error: CliError | None
|
|
203
|
+
#: Whether this run used the fake executor, and so called no model at all.
|
|
204
|
+
#: It is not the Climb's proof grade: a Climb whose results may never be
|
|
205
|
+
#: published is still run for real, against a real model, at real cost.
|
|
206
|
+
fake_executor: bool
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
class RunLogsPayload(ProtocolModel):
|
|
210
|
+
"""A bounded window onto one run's worker log, line for line as written."""
|
|
211
|
+
|
|
212
|
+
run_id: NonEmptyString
|
|
213
|
+
lines: list[str]
|
|
214
|
+
truncated: bool
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
class RunCancelPayload(ProtocolModel):
|
|
218
|
+
"""What asking a run to stop achieved."""
|
|
219
|
+
|
|
220
|
+
run_id: NonEmptyString
|
|
221
|
+
outcome: Literal["requested", "already_requested", "already_terminal"]
|
|
222
|
+
phase: RunPhase
|
|
223
|
+
cancel_requested_at: UtcDateTime | None
|
|
224
|
+
worker_alive: bool
|
|
225
|
+
|
|
226
|
+
|
|
227
|
+
class ResultFormat(StrEnum):
|
|
228
|
+
"""How a person asked for a finished result to be drawn."""
|
|
229
|
+
|
|
230
|
+
RICH = "rich"
|
|
231
|
+
COMPACT = "compact"
|
|
232
|
+
PATH = "path"
|
|
233
|
+
|
|
234
|
+
|
|
235
|
+
class RunResultPayload(ProtocolModel):
|
|
236
|
+
"""A finished run's report, and the neutral payload every channel draws.
|
|
237
|
+
|
|
238
|
+
Both, always. The report is the evidence and the presentation is the view;
|
|
239
|
+
a machine caller that received only one of them would either have to render
|
|
240
|
+
the evidence itself — inventing the wording this product is careful about —
|
|
241
|
+
or trust a view it cannot check against anything.
|
|
242
|
+
|
|
243
|
+
``execution_record`` is the same argument applied to what the comparison
|
|
244
|
+
consumed. Decisions document 0007 R6 signs that separately from the
|
|
245
|
+
report, so the view's cost and timing are checkable against the record
|
|
246
|
+
they were read from rather than taken on the view's word. It is absent
|
|
247
|
+
for a run that produced none, which is a fact about the run and not about
|
|
248
|
+
the result.
|
|
249
|
+
"""
|
|
250
|
+
|
|
251
|
+
report: UpliftReport
|
|
252
|
+
presentation: UpliftPresentationPayload
|
|
253
|
+
execution_record: ComparisonExecutionRecord | None
|
|
254
|
+
format: ResultFormat
|
|
255
|
+
show_tasks: TaskDisplay
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def build_run_service(context: CliContext) -> RunService:
|
|
259
|
+
"""Construct the service every run command acts through."""
|
|
260
|
+
run_store = RunStore(context.paths)
|
|
261
|
+
return RunService(
|
|
262
|
+
paths=context.paths,
|
|
263
|
+
draft_store=DraftStore(context.paths),
|
|
264
|
+
run_store=run_store,
|
|
265
|
+
artifact_store=RunArtifactStore(context.paths),
|
|
266
|
+
launcher=WorkerLauncher(
|
|
267
|
+
worker_executable=default_worker_executable(),
|
|
268
|
+
run_store=run_store,
|
|
269
|
+
environment_builder=scrubbed_worker_environment(context.paths),
|
|
270
|
+
),
|
|
271
|
+
)
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
# ---------------------------------------------------------------------------
|
|
275
|
+
# status
|
|
276
|
+
# ---------------------------------------------------------------------------
|
|
277
|
+
|
|
278
|
+
|
|
279
|
+
def status_run_command(
|
|
280
|
+
ctx: typer.Context,
|
|
281
|
+
run_id: Annotated[
|
|
282
|
+
str,
|
|
283
|
+
typer.Argument(metavar="RUN_ID", help="The run to report on."),
|
|
284
|
+
],
|
|
285
|
+
watch: Annotated[
|
|
286
|
+
bool,
|
|
287
|
+
typer.Option(
|
|
288
|
+
"--watch",
|
|
289
|
+
help="Keep reporting until the run ends. Human output only.",
|
|
290
|
+
),
|
|
291
|
+
] = False,
|
|
292
|
+
) -> None:
|
|
293
|
+
"""Show how a run is progressing."""
|
|
294
|
+
context = cli_context(ctx)
|
|
295
|
+
|
|
296
|
+
def action() -> CommandResult[RunStatusPayload]:
|
|
297
|
+
if watch and context.json_output:
|
|
298
|
+
raise UsageError(
|
|
299
|
+
"--watch prints repeatedly and machine mode returns one "
|
|
300
|
+
"envelope; poll `techtree run status --json` instead",
|
|
301
|
+
code=RUN_WATCH_NOT_SUPPORTED_IN_JSON,
|
|
302
|
+
details={"run_id": run_id},
|
|
303
|
+
)
|
|
304
|
+
|
|
305
|
+
service = build_run_service(context)
|
|
306
|
+
payload = (
|
|
307
|
+
_watch_until_terminal(service, run_id, context)
|
|
308
|
+
if watch
|
|
309
|
+
else _status_payload(service, run_id)
|
|
310
|
+
)
|
|
311
|
+
return CommandResult(
|
|
312
|
+
data=payload,
|
|
313
|
+
warnings=_fake_executor_warnings(payload.fake_executor),
|
|
314
|
+
next_actions=_status_next_actions(payload),
|
|
315
|
+
)
|
|
316
|
+
|
|
317
|
+
invoke_command(context, STATUS_COMMAND, action, render_data=_render_status)
|
|
318
|
+
|
|
319
|
+
|
|
320
|
+
def _status_payload(service: RunService, run_id: str) -> RunStatusPayload:
|
|
321
|
+
status = service.status(run_id)
|
|
322
|
+
health = service.process_health(run_id)
|
|
323
|
+
state = status.state
|
|
324
|
+
return RunStatusPayload(
|
|
325
|
+
run_id=state.run_id,
|
|
326
|
+
phase=state.phase,
|
|
327
|
+
sequence=state.sequence,
|
|
328
|
+
updated_at=state.updated_at,
|
|
329
|
+
progress=state.progress,
|
|
330
|
+
variant_progress=dict(state.variant_progress),
|
|
331
|
+
worker_pid=state.worker_pid,
|
|
332
|
+
worker_alive=status.worker_alive,
|
|
333
|
+
heartbeat_at=state.heartbeat_at,
|
|
334
|
+
heartbeat_age_seconds=health.heartbeat_age_seconds,
|
|
335
|
+
heartbeat_stale=status.heartbeat_stale,
|
|
336
|
+
cancel_requested_at=state.cancel_requested_at,
|
|
337
|
+
terminal=is_terminal(state.phase),
|
|
338
|
+
result_available=status.result_available,
|
|
339
|
+
result_digest=state.result_digest,
|
|
340
|
+
error=state.error,
|
|
341
|
+
fake_executor=service.request(run_id).executor_kind == "fake",
|
|
342
|
+
)
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def _watch_until_terminal(
|
|
346
|
+
service: RunService, run_id: str, context: CliContext
|
|
347
|
+
) -> RunStatusPayload:
|
|
348
|
+
"""Report every second until the run ends or the reader stops watching.
|
|
349
|
+
|
|
350
|
+
A ``Ctrl-C`` here stops the watching, never the run. Anything else would
|
|
351
|
+
make looking at a run a way to lose one.
|
|
352
|
+
"""
|
|
353
|
+
console = human_console(no_color=context.no_color)
|
|
354
|
+
payload = _status_payload(service, run_id)
|
|
355
|
+
try:
|
|
356
|
+
while not payload.terminal:
|
|
357
|
+
console.print(watch_line(payload))
|
|
358
|
+
time.sleep(_WATCH_INTERVAL_SECONDS)
|
|
359
|
+
payload = _status_payload(service, run_id)
|
|
360
|
+
except KeyboardInterrupt:
|
|
361
|
+
console.print("Stopped watching. The run is still going.")
|
|
362
|
+
return payload
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
def watch_line(payload: RunStatusPayload) -> str:
|
|
366
|
+
"""Return one line describing where the run is right now.
|
|
367
|
+
|
|
368
|
+
While both variants are in flight the line carries both, because a single
|
|
369
|
+
number would have to be either one side's or a sum, and a watcher reading
|
|
370
|
+
"18/72" cannot tell a comparison that is running evenly from one whose
|
|
371
|
+
candidate has not started. No score appears: spec section 6.20 forbids a
|
|
372
|
+
delta before both sides have finished, and a watch line is the easiest
|
|
373
|
+
place for a partial one to be mistaken for the answer.
|
|
374
|
+
"""
|
|
375
|
+
if payload.variant_progress:
|
|
376
|
+
return " ".join(
|
|
377
|
+
[payload.phase.value, *_variant_watch_cells(payload.variant_progress)]
|
|
378
|
+
)
|
|
379
|
+
progress = payload.progress
|
|
380
|
+
if progress is None:
|
|
381
|
+
return payload.phase.value
|
|
382
|
+
return f"{payload.phase.value} {progress.current}/{progress.total}"
|
|
383
|
+
|
|
384
|
+
|
|
385
|
+
def _variant_watch_cells(progress: dict[str, VariantProgress]) -> list[str]:
|
|
386
|
+
"""Return one cell per side of the comparison, in comparison order."""
|
|
387
|
+
cells: list[str] = []
|
|
388
|
+
for variant in (VariantName.BASELINE, VariantName.CANDIDATE):
|
|
389
|
+
side = progress.get(variant.value)
|
|
390
|
+
counts = "not started" if side is None else f"{side.completed}/{side.total}"
|
|
391
|
+
cells.append(f"{variant.value} {counts}")
|
|
392
|
+
return cells
|
|
393
|
+
|
|
394
|
+
|
|
395
|
+
def _status_next_actions(payload: RunStatusPayload) -> list[NextAction]:
|
|
396
|
+
if payload.result_available:
|
|
397
|
+
return [_read_result(payload.run_id)]
|
|
398
|
+
if payload.terminal:
|
|
399
|
+
return [_read_logs(payload.run_id)]
|
|
400
|
+
return [_check_status(payload.run_id), _read_logs(payload.run_id)]
|
|
401
|
+
|
|
402
|
+
|
|
403
|
+
# ---------------------------------------------------------------------------
|
|
404
|
+
# logs
|
|
405
|
+
# ---------------------------------------------------------------------------
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
def logs_run_command(
|
|
409
|
+
ctx: typer.Context,
|
|
410
|
+
run_id: Annotated[
|
|
411
|
+
str,
|
|
412
|
+
typer.Argument(metavar="RUN_ID", help="The run to read the log of."),
|
|
413
|
+
],
|
|
414
|
+
tail: Annotated[
|
|
415
|
+
int,
|
|
416
|
+
typer.Option(
|
|
417
|
+
"--tail",
|
|
418
|
+
metavar="LINES",
|
|
419
|
+
help=(
|
|
420
|
+
f"How many trailing lines to show, from {MINIMUM_LOG_TAIL} to "
|
|
421
|
+
f"{MAXIMUM_LOG_TAIL}."
|
|
422
|
+
),
|
|
423
|
+
),
|
|
424
|
+
] = DEFAULT_LOG_TAIL,
|
|
425
|
+
follow: Annotated[
|
|
426
|
+
bool,
|
|
427
|
+
typer.Option(
|
|
428
|
+
"--follow",
|
|
429
|
+
help="Keep printing new lines as they arrive. Human output only.",
|
|
430
|
+
),
|
|
431
|
+
] = False,
|
|
432
|
+
variant: Annotated[
|
|
433
|
+
VariantName | None,
|
|
434
|
+
typer.Option(
|
|
435
|
+
"--variant",
|
|
436
|
+
help="Read one side of the comparison's evaluation log.",
|
|
437
|
+
),
|
|
438
|
+
] = None,
|
|
439
|
+
) -> None:
|
|
440
|
+
"""Show the log output of a run."""
|
|
441
|
+
context = cli_context(ctx)
|
|
442
|
+
|
|
443
|
+
def action() -> CommandResult[RunLogsPayload]:
|
|
444
|
+
# The narrower incompatibility is reported first. Following one side of
|
|
445
|
+
# a comparison is not supported in any output mode, so saying "not in
|
|
446
|
+
# machine mode" would send a caller off to try it in human mode.
|
|
447
|
+
if follow and variant is not None:
|
|
448
|
+
raise UsageError(
|
|
449
|
+
"--follow reads the worker's own log; a variant's evaluation "
|
|
450
|
+
"log is read a window at a time with --tail",
|
|
451
|
+
code=RUN_FOLLOW_NOT_SUPPORTED_FOR_VARIANT,
|
|
452
|
+
details={"run_id": run_id, "variant": variant.value},
|
|
453
|
+
)
|
|
454
|
+
if follow and context.json_output:
|
|
455
|
+
raise UsageError(
|
|
456
|
+
"--follow never ends and machine mode returns one envelope; "
|
|
457
|
+
"poll `techtree run logs --json` instead",
|
|
458
|
+
code=RUN_FOLLOW_NOT_SUPPORTED_IN_JSON,
|
|
459
|
+
details={"run_id": run_id},
|
|
460
|
+
)
|
|
461
|
+
|
|
462
|
+
service = build_run_service(context)
|
|
463
|
+
if variant is not None:
|
|
464
|
+
logs = service.variant_logs(run_id, variant, tail=tail)
|
|
465
|
+
return CommandResult(
|
|
466
|
+
data=RunLogsPayload(
|
|
467
|
+
run_id=logs.run_id,
|
|
468
|
+
lines=list(logs.lines),
|
|
469
|
+
truncated=logs.truncated,
|
|
470
|
+
),
|
|
471
|
+
next_actions=[_check_status(run_id)],
|
|
472
|
+
)
|
|
473
|
+
|
|
474
|
+
logs = service.logs(run_id, tail=tail)
|
|
475
|
+
if follow:
|
|
476
|
+
_follow_log(service, run_id, context, shown=logs.lines)
|
|
477
|
+
logs = service.logs(run_id, tail=tail)
|
|
478
|
+
|
|
479
|
+
return CommandResult(
|
|
480
|
+
data=RunLogsPayload(
|
|
481
|
+
run_id=logs.run_id,
|
|
482
|
+
lines=list(logs.lines),
|
|
483
|
+
truncated=logs.truncated,
|
|
484
|
+
),
|
|
485
|
+
next_actions=[_check_status(run_id)],
|
|
486
|
+
)
|
|
487
|
+
|
|
488
|
+
invoke_command(context, LOGS_COMMAND, action, render_data=_render_logs)
|
|
489
|
+
|
|
490
|
+
|
|
491
|
+
def _follow_log(
|
|
492
|
+
service: RunService,
|
|
493
|
+
run_id: str,
|
|
494
|
+
context: CliContext,
|
|
495
|
+
*,
|
|
496
|
+
shown: list[str],
|
|
497
|
+
) -> None:
|
|
498
|
+
"""Print new lines as they arrive, each exactly as the worker wrote it.
|
|
499
|
+
|
|
500
|
+
``Ctrl-C`` leaves the run alone: following a log is reading, and reading
|
|
501
|
+
has never been a way to cancel anything.
|
|
502
|
+
"""
|
|
503
|
+
console = human_console(no_color=context.no_color)
|
|
504
|
+
for line in shown:
|
|
505
|
+
console.print(line)
|
|
506
|
+
|
|
507
|
+
try:
|
|
508
|
+
for line in _new_log_lines(service, run_id):
|
|
509
|
+
console.print(line)
|
|
510
|
+
except KeyboardInterrupt:
|
|
511
|
+
console.print("Stopped following. The run is unaffected.")
|
|
512
|
+
|
|
513
|
+
|
|
514
|
+
def _new_log_lines(service: RunService, run_id: str) -> Iterator[str]:
|
|
515
|
+
"""Yield each new log line until the run ends."""
|
|
516
|
+
path = service.worker_log_path(run_id)
|
|
517
|
+
position = path.stat().st_size if path.exists() else 0
|
|
518
|
+
|
|
519
|
+
while True:
|
|
520
|
+
if path.exists():
|
|
521
|
+
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
522
|
+
handle.seek(position)
|
|
523
|
+
fresh = handle.read()
|
|
524
|
+
position = handle.tell()
|
|
525
|
+
yield from fresh.splitlines()
|
|
526
|
+
if is_terminal(service.status(run_id).state.phase):
|
|
527
|
+
return
|
|
528
|
+
time.sleep(_FOLLOW_INTERVAL_SECONDS)
|
|
529
|
+
|
|
530
|
+
|
|
531
|
+
# ---------------------------------------------------------------------------
|
|
532
|
+
# cancel
|
|
533
|
+
# ---------------------------------------------------------------------------
|
|
534
|
+
|
|
535
|
+
|
|
536
|
+
def cancel_run_command(
|
|
537
|
+
ctx: typer.Context,
|
|
538
|
+
run_id: Annotated[
|
|
539
|
+
str,
|
|
540
|
+
typer.Argument(metavar="RUN_ID", help="The run to stop."),
|
|
541
|
+
],
|
|
542
|
+
confirm: Annotated[
|
|
543
|
+
bool,
|
|
544
|
+
typer.Option(
|
|
545
|
+
"--confirm",
|
|
546
|
+
help="Confirm the cancellation without being asked.",
|
|
547
|
+
),
|
|
548
|
+
] = False,
|
|
549
|
+
) -> None:
|
|
550
|
+
"""Stop a run that is still in progress."""
|
|
551
|
+
context = cli_context(ctx)
|
|
552
|
+
|
|
553
|
+
def action() -> CommandResult[RunCancelPayload]:
|
|
554
|
+
service = build_run_service(context)
|
|
555
|
+
_require_cancel_confirmation(context, service, run_id, confirm=confirm)
|
|
556
|
+
|
|
557
|
+
cancellation = service.cancel(run_id)
|
|
558
|
+
state = cancellation.status.state
|
|
559
|
+
payload = RunCancelPayload(
|
|
560
|
+
run_id=state.run_id,
|
|
561
|
+
outcome=cancellation.outcome,
|
|
562
|
+
phase=state.phase,
|
|
563
|
+
cancel_requested_at=state.cancel_requested_at,
|
|
564
|
+
worker_alive=cancellation.status.worker_alive,
|
|
565
|
+
)
|
|
566
|
+
return CommandResult(
|
|
567
|
+
data=payload,
|
|
568
|
+
messages=[
|
|
569
|
+
CliMessage(
|
|
570
|
+
level=MessageLevel.INFO,
|
|
571
|
+
code=f"run_cancel_{cancellation.outcome}",
|
|
572
|
+
text=_cancel_message(payload),
|
|
573
|
+
)
|
|
574
|
+
],
|
|
575
|
+
warnings=_cancel_warnings(payload),
|
|
576
|
+
next_actions=[_check_status(run_id)],
|
|
577
|
+
)
|
|
578
|
+
|
|
579
|
+
invoke_command(context, CANCEL_COMMAND, action, render_data=_render_cancel)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _require_cancel_confirmation(
|
|
583
|
+
context: CliContext, service: RunService, run_id: str, *, confirm: bool
|
|
584
|
+
) -> None:
|
|
585
|
+
"""Require an explicit yes, from a person or from an option.
|
|
586
|
+
|
|
587
|
+
A confirmation is asked for because something is about to be stopped, so a
|
|
588
|
+
run that has already ended is not asked about at all. There is no work left
|
|
589
|
+
to lose, ``cancel`` will report that it changed nothing, and a question
|
|
590
|
+
whose only honest answer is "nothing would happen either way" is a question
|
|
591
|
+
that should not have been put to somebody.
|
|
592
|
+
|
|
593
|
+
Reading the run's state first is also how an identifier that names no run
|
|
594
|
+
stops being a question. Asking whether to stop a run that does not exist,
|
|
595
|
+
and only then saying it does not exist, wastes the reader's answer on a
|
|
596
|
+
decision there was never anything to decide.
|
|
597
|
+
"""
|
|
598
|
+
if confirm:
|
|
599
|
+
return
|
|
600
|
+
if is_terminal(service.status(run_id).state.phase):
|
|
601
|
+
return
|
|
602
|
+
if context.no_input:
|
|
603
|
+
raise UsageError(
|
|
604
|
+
"cancelling a run stops work that cannot be resumed, so it needs "
|
|
605
|
+
"--confirm when nothing can be asked",
|
|
606
|
+
code=CANCEL_CONFIRMATION_REQUIRED,
|
|
607
|
+
details={"run_id": run_id},
|
|
608
|
+
)
|
|
609
|
+
if not confirmed(f"Stop run {run_id}?"):
|
|
610
|
+
raise UsageError(
|
|
611
|
+
f"run {run_id} was left running",
|
|
612
|
+
code=CANCEL_CONFIRMATION_REQUIRED,
|
|
613
|
+
details={"run_id": run_id},
|
|
614
|
+
)
|
|
615
|
+
|
|
616
|
+
|
|
617
|
+
def _cancel_message(payload: RunCancelPayload) -> str:
|
|
618
|
+
if payload.outcome == "already_terminal":
|
|
619
|
+
return (
|
|
620
|
+
f"Run {payload.run_id} had already ended in {payload.phase.value}; "
|
|
621
|
+
"nothing was changed."
|
|
622
|
+
)
|
|
623
|
+
if payload.outcome == "already_requested":
|
|
624
|
+
return f"Run {payload.run_id} had already been asked to stop."
|
|
625
|
+
return f"Run {payload.run_id} has been asked to stop."
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _cancel_warnings(payload: RunCancelPayload) -> list[CliMessage]:
|
|
629
|
+
if payload.outcome == "already_terminal":
|
|
630
|
+
return [
|
|
631
|
+
CliMessage(
|
|
632
|
+
level=MessageLevel.WARNING,
|
|
633
|
+
code="run_already_terminal",
|
|
634
|
+
text=(
|
|
635
|
+
"A run that has ended cannot be cancelled, and its result "
|
|
636
|
+
"was left exactly as it was."
|
|
637
|
+
),
|
|
638
|
+
)
|
|
639
|
+
]
|
|
640
|
+
if payload.worker_alive:
|
|
641
|
+
return [
|
|
642
|
+
CliMessage(
|
|
643
|
+
level=MessageLevel.WARNING,
|
|
644
|
+
code="run_stopping",
|
|
645
|
+
text=(
|
|
646
|
+
"The worker was signalled and stops at its next safe point, "
|
|
647
|
+
"so the run may take a moment to report as cancelled."
|
|
648
|
+
),
|
|
649
|
+
)
|
|
650
|
+
]
|
|
651
|
+
return []
|
|
652
|
+
|
|
653
|
+
|
|
654
|
+
# ---------------------------------------------------------------------------
|
|
655
|
+
# result
|
|
656
|
+
# ---------------------------------------------------------------------------
|
|
657
|
+
|
|
658
|
+
|
|
659
|
+
def result_run_command(
|
|
660
|
+
ctx: typer.Context,
|
|
661
|
+
run_id: Annotated[
|
|
662
|
+
str,
|
|
663
|
+
typer.Argument(metavar="RUN_ID", help="The finished run to report on."),
|
|
664
|
+
],
|
|
665
|
+
result_format: Annotated[
|
|
666
|
+
ResultFormat | None,
|
|
667
|
+
typer.Option(
|
|
668
|
+
"--format",
|
|
669
|
+
help=(
|
|
670
|
+
"How to show the result. Defaults to rich in a terminal and "
|
|
671
|
+
"compact when the output is piped."
|
|
672
|
+
),
|
|
673
|
+
),
|
|
674
|
+
] = None,
|
|
675
|
+
show_tasks: Annotated[
|
|
676
|
+
TaskDisplay,
|
|
677
|
+
typer.Option("--show-tasks", help="Which per-task rows to print."),
|
|
678
|
+
] = TaskDisplay.CHANGED,
|
|
679
|
+
verify: Annotated[
|
|
680
|
+
bool,
|
|
681
|
+
typer.Option(
|
|
682
|
+
"--verify/--no-verify",
|
|
683
|
+
help="Check the run's local proof before showing the result.",
|
|
684
|
+
),
|
|
685
|
+
] = True,
|
|
686
|
+
) -> None:
|
|
687
|
+
"""Show the finished report for a run."""
|
|
688
|
+
context = cli_context(ctx)
|
|
689
|
+
|
|
690
|
+
def action() -> CommandResult[RunResultPayload]:
|
|
691
|
+
service = build_run_service(context)
|
|
692
|
+
try:
|
|
693
|
+
report = service.result(run_id)
|
|
694
|
+
except TechtreeError as error:
|
|
695
|
+
# "Not finished yet" is the ordinary answer here, and the caller
|
|
696
|
+
# should be told where to look rather than told to guess.
|
|
697
|
+
if not error.next_actions:
|
|
698
|
+
error.next_actions = [_check_status(run_id)]
|
|
699
|
+
raise
|
|
700
|
+
|
|
701
|
+
verification = _verify_proof(context, run_id, report) if verify else None
|
|
702
|
+
record = read_execution_record(proof_bundle_dir(context.paths.run_dir(run_id)))
|
|
703
|
+
payload = RunResultPayload(
|
|
704
|
+
report=report,
|
|
705
|
+
presentation=_presentation(context, run_id, report, verification, record),
|
|
706
|
+
execution_record=record,
|
|
707
|
+
format=result_format or _default_format(context),
|
|
708
|
+
show_tasks=show_tasks,
|
|
709
|
+
)
|
|
710
|
+
return CommandResult(
|
|
711
|
+
data=payload,
|
|
712
|
+
warnings=_result_warnings(report),
|
|
713
|
+
next_actions=[*payload.presentation.next_actions, _read_logs(run_id)],
|
|
714
|
+
# A result whose own proof does not check out is shown and
|
|
715
|
+
# reported as a failure: the caller still gets the report, and
|
|
716
|
+
# the exit code says not to believe it.
|
|
717
|
+
error=_unverified_error(run_id, verification),
|
|
718
|
+
)
|
|
719
|
+
|
|
720
|
+
invoke_command(context, RESULT_COMMAND, action, render_data=_render_result)
|
|
721
|
+
|
|
722
|
+
|
|
723
|
+
def _presentation(
|
|
724
|
+
context: CliContext,
|
|
725
|
+
run_id: str,
|
|
726
|
+
report: UpliftReport,
|
|
727
|
+
verification: VerificationResult | None,
|
|
728
|
+
execution_record: ComparisonExecutionRecord | None,
|
|
729
|
+
) -> UpliftPresentationPayload:
|
|
730
|
+
"""Build the channel-neutral payload every rendering of this result uses.
|
|
731
|
+
|
|
732
|
+
The Skills and the comparison's title come from the run's own staged
|
|
733
|
+
inputs, which the artifact store verifies against the run's immutable
|
|
734
|
+
request, so what is shown is what was executed rather than whatever a
|
|
735
|
+
draft or a catalog holds now.
|
|
736
|
+
|
|
737
|
+
``baseline_skill`` is the Skill the baseline variant carried, which is
|
|
738
|
+
``None`` for a Skill insertion and the Skill being revised for a Skill
|
|
739
|
+
replacement. It is read off the run rather than assumed, so the two
|
|
740
|
+
comparisons cannot be labelled as each other.
|
|
741
|
+
"""
|
|
742
|
+
artifacts = RunArtifactStore(context.paths)
|
|
743
|
+
inputs = artifacts.load_inputs(run_id, RunStore(context.paths).get_request(run_id))
|
|
744
|
+
baseline_skill = inputs.baseline_skill
|
|
745
|
+
return build_uplift_presentation(
|
|
746
|
+
report=report,
|
|
747
|
+
# The run's own signed Campaign: which model was measured, which is what
|
|
748
|
+
# a cost can be worked out from and what the weaker-claim warning names.
|
|
749
|
+
campaign=inputs.campaign,
|
|
750
|
+
baseline_receipts=artifacts.episode_receipts(
|
|
751
|
+
run_id, ExperimentVariant.BASELINE
|
|
752
|
+
),
|
|
753
|
+
candidate_receipts=artifacts.episode_receipts(
|
|
754
|
+
run_id, ExperimentVariant.CANDIDATE
|
|
755
|
+
),
|
|
756
|
+
campaign_title=inputs.source.title,
|
|
757
|
+
baseline_skill=None if baseline_skill is None else baseline_skill.artifact,
|
|
758
|
+
candidate_skill=inputs.candidate_skill.artifact,
|
|
759
|
+
verification=verification,
|
|
760
|
+
# Decisions 0007 R6's operational record, read from the same bundle
|
|
761
|
+
# the verification just checked. A run without one is shown as a run
|
|
762
|
+
# whose economics are unknown, never as one that cost nothing.
|
|
763
|
+
execution_record=execution_record,
|
|
764
|
+
# Turns and throttling, counted out of the run's own evaluation output
|
|
765
|
+
# and checked against the digests that record already holds. Nothing is
|
|
766
|
+
# written; a run whose output has been cleared away simply says less.
|
|
767
|
+
recorded_evidence=(
|
|
768
|
+
None
|
|
769
|
+
if execution_record is None
|
|
770
|
+
else read_recorded_evidence(context.paths.run_dir(run_id), execution_record)
|
|
771
|
+
),
|
|
772
|
+
)
|
|
773
|
+
|
|
774
|
+
|
|
775
|
+
def _verify_proof(
|
|
776
|
+
context: CliContext, run_id: str, report: UpliftReport
|
|
777
|
+
) -> VerificationResult | None:
|
|
778
|
+
"""Check the run's local proof, when it has one to check.
|
|
779
|
+
|
|
780
|
+
A development-only report has no proof bundle and never claimed one, so
|
|
781
|
+
there is nothing to verify and nothing is claimed about it. A graded report
|
|
782
|
+
always has one, and a missing bundle for a graded report is a failed
|
|
783
|
+
verification rather than a silent "not checked".
|
|
784
|
+
"""
|
|
785
|
+
if report.proof_grade == "development_only":
|
|
786
|
+
return None
|
|
787
|
+
return LocalProofVerifier().verify_bundle(
|
|
788
|
+
proof_bundle_dir(context.paths.run_dir(run_id))
|
|
789
|
+
)
|
|
790
|
+
|
|
791
|
+
|
|
792
|
+
def _unverified_error(
|
|
793
|
+
run_id: str, verification: VerificationResult | None
|
|
794
|
+
) -> VerificationError | None:
|
|
795
|
+
"""Return the failure a result carries when its proof did not verify."""
|
|
796
|
+
if verification is None or verification.verified:
|
|
797
|
+
return None
|
|
798
|
+
return VerificationError(
|
|
799
|
+
f"run {run_id} produced a report whose local proof does not verify",
|
|
800
|
+
code=PROOF_BUNDLE_INVALID,
|
|
801
|
+
details={
|
|
802
|
+
"run_id": run_id,
|
|
803
|
+
"failed_checks": [message.id for message in verification.failures],
|
|
804
|
+
},
|
|
805
|
+
)
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def _default_format(context: CliContext) -> ResultFormat:
|
|
809
|
+
"""Return the rendering a reader gets when they did not choose one.
|
|
810
|
+
|
|
811
|
+
Spec section 7.21: rich for a person at a terminal, compact when the output
|
|
812
|
+
is going somewhere else — a pipe, a file, or a gateway that will forward
|
|
813
|
+
it. Machine mode renders neither.
|
|
814
|
+
"""
|
|
815
|
+
return ResultFormat.RICH if sys.stdout.isatty() else ResultFormat.COMPACT
|
|
816
|
+
|
|
817
|
+
|
|
818
|
+
# ---------------------------------------------------------------------------
|
|
819
|
+
# Messages and next actions
|
|
820
|
+
# ---------------------------------------------------------------------------
|
|
821
|
+
|
|
822
|
+
|
|
823
|
+
def _fake_executor_warnings(fake_executor: bool) -> list[CliMessage]:
|
|
824
|
+
"""Return the caveat a run driven by the fake executor carries.
|
|
825
|
+
|
|
826
|
+
``fake_executor_run`` states one fact and only that fact: this run called
|
|
827
|
+
no model. It is the same code ``climb start`` fires off the same request
|
|
828
|
+
field, and it is deliberately not the code a development-only *grade*
|
|
829
|
+
fires, because the two are independent — a run against a real model at
|
|
830
|
+
real cost still produces a report that may not be published.
|
|
831
|
+
"""
|
|
832
|
+
if not fake_executor:
|
|
833
|
+
return []
|
|
834
|
+
return [
|
|
835
|
+
CliMessage(
|
|
836
|
+
level=MessageLevel.WARNING,
|
|
837
|
+
code="fake_executor_run",
|
|
838
|
+
text=_FAKE_EXECUTOR_WARNING,
|
|
839
|
+
)
|
|
840
|
+
]
|
|
841
|
+
|
|
842
|
+
|
|
843
|
+
def _result_warnings(report: UpliftReport) -> list[CliMessage]:
|
|
844
|
+
"""Return the caveat a finished report carries. Spec section 29.
|
|
845
|
+
|
|
846
|
+
``development_only_result`` states the report's proof grade and only that:
|
|
847
|
+
these numbers are not evidence and no verdict is stated. Which executor
|
|
848
|
+
produced them is a separate fact with its own code.
|
|
849
|
+
|
|
850
|
+
A report knows which DataPolicy its run executed under, so the warning on
|
|
851
|
+
a result can name it. Progress reporting cannot — a run in flight has no
|
|
852
|
+
report yet — which is why ``run status`` carries the shorter caveat.
|
|
853
|
+
"""
|
|
854
|
+
if report.proof_grade != "development_only":
|
|
855
|
+
return []
|
|
856
|
+
return [
|
|
857
|
+
CliMessage(
|
|
858
|
+
level=MessageLevel.WARNING,
|
|
859
|
+
code="development_only_result",
|
|
860
|
+
text=development_only_result_notice(report.data_policy_digest),
|
|
861
|
+
)
|
|
862
|
+
]
|
|
863
|
+
|
|
864
|
+
|
|
865
|
+
def _check_status(run_id: str) -> NextAction:
|
|
866
|
+
return NextAction(
|
|
867
|
+
id="run_status",
|
|
868
|
+
label="Check how the run is going",
|
|
869
|
+
reason="A run continues whether or not anything is watching it.",
|
|
870
|
+
cli=["techtree", "run", "status", run_id],
|
|
871
|
+
hermes_tool=None,
|
|
872
|
+
hermes_args=None,
|
|
873
|
+
requires_user_confirmation=False,
|
|
874
|
+
)
|
|
875
|
+
|
|
876
|
+
|
|
877
|
+
def _read_logs(run_id: str) -> NextAction:
|
|
878
|
+
return NextAction(
|
|
879
|
+
id="run_logs",
|
|
880
|
+
label="Read what the worker recorded",
|
|
881
|
+
reason="The log says what the run was doing when it got there.",
|
|
882
|
+
cli=["techtree", "run", "logs", run_id],
|
|
883
|
+
hermes_tool=None,
|
|
884
|
+
hermes_args=None,
|
|
885
|
+
requires_user_confirmation=False,
|
|
886
|
+
)
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
def _read_result(run_id: str) -> NextAction:
|
|
890
|
+
return NextAction(
|
|
891
|
+
id="run_result",
|
|
892
|
+
label="Read the finished report",
|
|
893
|
+
reason="The run has produced its result.",
|
|
894
|
+
cli=["techtree", "run", "result", run_id],
|
|
895
|
+
hermes_tool=None,
|
|
896
|
+
hermes_args=None,
|
|
897
|
+
requires_user_confirmation=False,
|
|
898
|
+
)
|
|
899
|
+
|
|
900
|
+
|
|
901
|
+
# ---------------------------------------------------------------------------
|
|
902
|
+
# Human rendering
|
|
903
|
+
# ---------------------------------------------------------------------------
|
|
904
|
+
|
|
905
|
+
|
|
906
|
+
def _render_status(data: object, console: Console) -> None:
|
|
907
|
+
"""Show where a run is, saying only what is true of where it is.
|
|
908
|
+
|
|
909
|
+
Progress measures work in flight through the phase the run is in now, and
|
|
910
|
+
a run that has ended has no work in flight. Its progress is not zero and it
|
|
911
|
+
is not complete; there is simply nothing left for the row to be about, and
|
|
912
|
+
``Phase`` above has already said how the run finished. So the row is left
|
|
913
|
+
out for a terminal run, the way ``Cancellation`` and ``Error`` are left out
|
|
914
|
+
of a run that has neither, rather than filled in with a number nothing
|
|
915
|
+
measured.
|
|
916
|
+
"""
|
|
917
|
+
if not isinstance(data, RunStatusPayload):
|
|
918
|
+
return
|
|
919
|
+
|
|
920
|
+
rows = [
|
|
921
|
+
("Run", data.run_id),
|
|
922
|
+
("Phase", data.phase.value),
|
|
923
|
+
]
|
|
924
|
+
if not data.terminal:
|
|
925
|
+
rows.append(("Progress", _progress_text(data.progress)))
|
|
926
|
+
rows += [
|
|
927
|
+
("Worker", _worker_text(data)),
|
|
928
|
+
("Heartbeat", _heartbeat_text(data)),
|
|
929
|
+
("Result", "ready" if data.result_available else "not yet"),
|
|
930
|
+
]
|
|
931
|
+
if data.cancel_requested_at is not None:
|
|
932
|
+
rows.append(("Cancellation", "requested"))
|
|
933
|
+
if data.error is not None:
|
|
934
|
+
rows.append(("Error", data.error.message))
|
|
935
|
+
render_pairs(rows, console)
|
|
936
|
+
|
|
937
|
+
if data.variant_progress:
|
|
938
|
+
console.print()
|
|
939
|
+
_print_variant_progress(console, data.variant_progress)
|
|
940
|
+
|
|
941
|
+
|
|
942
|
+
def _print_variant_progress(
|
|
943
|
+
console: Console, progress: dict[str, VariantProgress]
|
|
944
|
+
) -> None:
|
|
945
|
+
"""Show both sides of a comparison side by side. Spec section 6.20.
|
|
946
|
+
|
|
947
|
+
No score appears here, and no delta. Until both variants have finished
|
|
948
|
+
every task, any number would be a partial mean of a partial run, and a
|
|
949
|
+
reader who saw one would reasonably take it for the answer.
|
|
950
|
+
"""
|
|
951
|
+
baseline = progress.get(VariantName.BASELINE.value)
|
|
952
|
+
candidate = progress.get(VariantName.CANDIDATE.value)
|
|
953
|
+
|
|
954
|
+
table = Table(box=None, pad_edge=False, padding=(0, 2))
|
|
955
|
+
table.add_column("", no_wrap=True)
|
|
956
|
+
table.add_column("Baseline", no_wrap=True)
|
|
957
|
+
table.add_column("Skill candidate", no_wrap=True)
|
|
958
|
+
table.add_row("Episodes", _episode_text(baseline), _episode_text(candidate))
|
|
959
|
+
table.add_row("State", _variant_state(baseline), _variant_state(candidate))
|
|
960
|
+
table.add_row("Current score", PROVISIONAL_SCORE, PROVISIONAL_SCORE)
|
|
961
|
+
console.print(table)
|
|
962
|
+
console.print()
|
|
963
|
+
console.print(SCORE_PROVISIONAL_NOTICE)
|
|
964
|
+
|
|
965
|
+
|
|
966
|
+
def _episode_text(variant: VariantProgress | None) -> str:
|
|
967
|
+
if variant is None:
|
|
968
|
+
return "not started"
|
|
969
|
+
return f"{variant.completed} / {variant.total}"
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def _variant_state(variant: VariantProgress | None) -> str:
|
|
973
|
+
return "pending" if variant is None else variant.state
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _progress_text(progress: RunProgress | None) -> str:
|
|
977
|
+
"""Say how far through its current phase a live run has got.
|
|
978
|
+
|
|
979
|
+
A live run with nothing recorded yet has not started the work this phase
|
|
980
|
+
measures, which is what the words say. Only a live run is asked: a run that
|
|
981
|
+
has ended is not given this row at all, because "not started" would be an
|
|
982
|
+
answer about work that finished long ago.
|
|
983
|
+
"""
|
|
984
|
+
if progress is None:
|
|
985
|
+
return "not started"
|
|
986
|
+
return f"{progress.current} of {progress.total} {progress.label}"
|
|
987
|
+
|
|
988
|
+
|
|
989
|
+
def _worker_text(data: RunStatusPayload) -> str:
|
|
990
|
+
if data.worker_pid is None:
|
|
991
|
+
return "not started"
|
|
992
|
+
return f"{'running' if data.worker_alive else 'gone'} (pid {data.worker_pid})"
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def _heartbeat_text(data: RunStatusPayload) -> str:
|
|
996
|
+
if data.heartbeat_age_seconds is None:
|
|
997
|
+
return "none yet"
|
|
998
|
+
age = f"{data.heartbeat_age_seconds:.0f}s ago"
|
|
999
|
+
return f"{age} (stale)" if data.heartbeat_stale else age
|
|
1000
|
+
|
|
1001
|
+
|
|
1002
|
+
def _render_logs(data: object, console: Console) -> None:
|
|
1003
|
+
if not isinstance(data, RunLogsPayload):
|
|
1004
|
+
return
|
|
1005
|
+
if not data.lines:
|
|
1006
|
+
console.print("The worker has not written anything yet.")
|
|
1007
|
+
return
|
|
1008
|
+
if data.truncated:
|
|
1009
|
+
console.print(f"Showing the last {len(data.lines)} lines.")
|
|
1010
|
+
for line in data.lines:
|
|
1011
|
+
console.print(line)
|
|
1012
|
+
|
|
1013
|
+
|
|
1014
|
+
def _render_cancel(data: object, console: Console) -> None:
|
|
1015
|
+
if not isinstance(data, RunCancelPayload):
|
|
1016
|
+
return
|
|
1017
|
+
render_pairs(
|
|
1018
|
+
[
|
|
1019
|
+
("Run", data.run_id),
|
|
1020
|
+
("Phase", data.phase.value),
|
|
1021
|
+
("Worker", "running" if data.worker_alive else "stopped"),
|
|
1022
|
+
],
|
|
1023
|
+
console,
|
|
1024
|
+
)
|
|
1025
|
+
|
|
1026
|
+
|
|
1027
|
+
def _render_result(data: object, console: Console) -> None:
|
|
1028
|
+
"""Draw a finished result in the shape the reader asked for.
|
|
1029
|
+
|
|
1030
|
+
The development-only banner comes before everything in every format. A
|
|
1031
|
+
reader who stops at the first line of an invented result has still been
|
|
1032
|
+
told that it is invented.
|
|
1033
|
+
"""
|
|
1034
|
+
if not isinstance(data, RunResultPayload):
|
|
1035
|
+
return
|
|
1036
|
+
|
|
1037
|
+
if data.report.proof_grade == "development_only":
|
|
1038
|
+
console.print(DEVELOPMENT_ONLY_BANNER)
|
|
1039
|
+
console.print()
|
|
1040
|
+
|
|
1041
|
+
if data.format is ResultFormat.PATH:
|
|
1042
|
+
_render_result_paths(data, console)
|
|
1043
|
+
return
|
|
1044
|
+
if data.format is ResultFormat.COMPACT:
|
|
1045
|
+
console.print(
|
|
1046
|
+
render_uplift_markdown(data.presentation, show_tasks=data.show_tasks)
|
|
1047
|
+
)
|
|
1048
|
+
return
|
|
1049
|
+
render_uplift_console(data.presentation, console, show_tasks=data.show_tasks)
|
|
1050
|
+
|
|
1051
|
+
|
|
1052
|
+
def _render_result_paths(data: RunResultPayload, console: Console) -> None:
|
|
1053
|
+
"""Print where this run's result and proof live, and nothing else.
|
|
1054
|
+
|
|
1055
|
+
``--format path`` exists for a caller that is going to open the files
|
|
1056
|
+
itself, so it prints paths relative to the run directory rather than the
|
|
1057
|
+
absolute ones: the run identifier is what a caller already has.
|
|
1058
|
+
"""
|
|
1059
|
+
render_pairs(
|
|
1060
|
+
[
|
|
1061
|
+
("Run", data.report.run_id),
|
|
1062
|
+
("Report", "report/uplift.json"),
|
|
1063
|
+
("Proof bundle", f"{BUNDLE_DIRECTORY}/"),
|
|
1064
|
+
("Signed report", f"{BUNDLE_DIRECTORY}/{REPORT_FILENAME}"),
|
|
1065
|
+
],
|
|
1066
|
+
console,
|
|
1067
|
+
)
|