outerloop-science 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +18 -0
- outerloop/__main__.py +3 -0
- outerloop/appauth.py +213 -0
- outerloop/appmanifest.py +198 -0
- outerloop/attempt.py +3481 -0
- outerloop/brief.py +515 -0
- outerloop/cli.py +439 -0
- outerloop/climbboard.py +1145 -0
- outerloop/compute.py +482 -0
- outerloop/contract.py +483 -0
- outerloop/contract_cli.py +63 -0
- outerloop/disk.py +164 -0
- outerloop/dispatch.py +586 -0
- outerloop/followup.py +2143 -0
- outerloop/github.py +1486 -0
- outerloop/harness.py +1449 -0
- outerloop/housekeeping.py +167 -0
- outerloop/init.py +313 -0
- outerloop/intake.py +129 -0
- outerloop/limits.py +80 -0
- outerloop/markers.py +48 -0
- outerloop/measure.py +523 -0
- outerloop/orchestrator.py +1901 -0
- outerloop/panel.py +188 -0
- outerloop/paths.py +27 -0
- outerloop/posting.py +160 -0
- outerloop/progress.py +170 -0
- outerloop/py.typed +0 -0
- outerloop/review.py +611 -0
- outerloop/review_agent.py +263 -0
- outerloop/review_agent_cli.py +209 -0
- outerloop/review_post_cli.py +162 -0
- outerloop/review_summarize_cli.py +163 -0
- outerloop/role_runner.py +229 -0
- outerloop/roles.py +247 -0
- outerloop/rolespec.py +89 -0
- outerloop/runstate.py +385 -0
- outerloop/steward.py +852 -0
- outerloop/style.py +12 -0
- outerloop/syscall.py +977 -0
- outerloop/syscall_cli.py +531 -0
- outerloop/tick.py +3166 -0
- outerloop/verifier.py +403 -0
- outerloop/verify_agent.py +149 -0
- outerloop/verify_agent_cli.py +95 -0
- outerloop/verify_post_cli.py +116 -0
- outerloop_science-0.1.0.dev0.dist-info/METADATA +145 -0
- outerloop_science-0.1.0.dev0.dist-info/RECORD +52 -0
- outerloop_science-0.1.0.dev0.dist-info/WHEEL +4 -0
- outerloop_science-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- outerloop_science-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
- outerloop_science-0.1.0.dev0.dist-info/licenses/NOTICE +5 -0
outerloop/attempt.py
ADDED
|
@@ -0,0 +1,3481 @@
|
|
|
1
|
+
"""One live climb, end to end: clone → attempt_once → commit/push/PR → report.
|
|
2
|
+
|
|
3
|
+
This is the glue `orchestrator.attempt_once` deliberately does not own: the git
|
|
4
|
+
side (bot-auth clone, veto-checked commit, push, PR) and the run's durable
|
|
5
|
+
record. One invocation = one run = at most one PR.
|
|
6
|
+
|
|
7
|
+
Credential separation holds throughout: the bot PAT is read orchestrator-side
|
|
8
|
+
and used only by Workspace network calls and the PR client, after the session
|
|
9
|
+
has ended; the session sees only its own capped API key inside its container.
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
from __future__ import annotations
|
|
13
|
+
|
|
14
|
+
import argparse
|
|
15
|
+
import contextlib
|
|
16
|
+
import json
|
|
17
|
+
import logging
|
|
18
|
+
import os
|
|
19
|
+
import re
|
|
20
|
+
import shutil
|
|
21
|
+
from collections.abc import Callable, Iterable
|
|
22
|
+
from dataclasses import dataclass
|
|
23
|
+
from dataclasses import replace as dc_replace
|
|
24
|
+
from functools import partial
|
|
25
|
+
from pathlib import Path
|
|
26
|
+
from typing import Any, cast
|
|
27
|
+
|
|
28
|
+
from outerloop.appauth import resolve_bot_auth
|
|
29
|
+
from outerloop.brief import BudgetState, distill_lessons
|
|
30
|
+
from outerloop.compute import LocalCompute, local_mode
|
|
31
|
+
from outerloop.contract import Benchmark, Contract, contract_text_in_tree, load_contract
|
|
32
|
+
from outerloop.dispatch import (
|
|
33
|
+
Snapshot,
|
|
34
|
+
afterany_ids,
|
|
35
|
+
drop_snapshot,
|
|
36
|
+
should_dispatch,
|
|
37
|
+
snapshot_tree,
|
|
38
|
+
)
|
|
39
|
+
from outerloop.github import (
|
|
40
|
+
GitError,
|
|
41
|
+
GitHubClient,
|
|
42
|
+
TokenProvider,
|
|
43
|
+
Workspace,
|
|
44
|
+
contract_at,
|
|
45
|
+
ensure_regular_git_dir,
|
|
46
|
+
)
|
|
47
|
+
from outerloop.harness import Harness, SessionResult, redact
|
|
48
|
+
from outerloop.markers import has_marker
|
|
49
|
+
from outerloop.measure import DispatchedMeasurer, DispatchSettings
|
|
50
|
+
from outerloop.orchestrator import (
|
|
51
|
+
AttemptResult,
|
|
52
|
+
EvalError,
|
|
53
|
+
Measurer,
|
|
54
|
+
RunConfig,
|
|
55
|
+
RunParked,
|
|
56
|
+
_benchmark,
|
|
57
|
+
attempt_once,
|
|
58
|
+
pr_body,
|
|
59
|
+
resume_attempt,
|
|
60
|
+
)
|
|
61
|
+
from outerloop.panel import PanelLens, PanelVerdict, run_panel
|
|
62
|
+
from outerloop.paths import CONFIG_DIR
|
|
63
|
+
from outerloop.progress import (
|
|
64
|
+
PROGRESS_PATHS,
|
|
65
|
+
load_leader,
|
|
66
|
+
update_leader,
|
|
67
|
+
write_progress,
|
|
68
|
+
)
|
|
69
|
+
from outerloop.review import PullRequest
|
|
70
|
+
from outerloop.role_runner import build_harness, role_key
|
|
71
|
+
from outerloop.roles import author_spec
|
|
72
|
+
from outerloop.rolespec import RoleSpec
|
|
73
|
+
from outerloop.runstate import (
|
|
74
|
+
ABORTED,
|
|
75
|
+
BUDGET_EXHAUSTED,
|
|
76
|
+
ENDED,
|
|
77
|
+
IN_REVIEW,
|
|
78
|
+
NEGATIVE_RESULT,
|
|
79
|
+
STUCK,
|
|
80
|
+
WAITING,
|
|
81
|
+
RunRecord,
|
|
82
|
+
list_runs,
|
|
83
|
+
load_record,
|
|
84
|
+
save_record,
|
|
85
|
+
stamp_outage,
|
|
86
|
+
)
|
|
87
|
+
from outerloop.syscall import CHANNEL_DIR_NAMES, MAX_ARTIFACT_BYTES, SyscallRequest, channel_dir
|
|
88
|
+
from outerloop.syscall import ensure_excluded as syscall_excluded
|
|
89
|
+
from outerloop.syscall import install_tool as syscall_install_tool
|
|
90
|
+
from outerloop.syscall import write_budget as syscall_write_budget
|
|
91
|
+
from outerloop.syscall import write_siblings as syscall_write_siblings
|
|
92
|
+
from outerloop.verifier import MAX_CLAIM_CHARS
|
|
93
|
+
|
|
94
|
+
log = logging.getLogger(__name__)
|
|
95
|
+
|
|
96
|
+
# Where a climb job reads its keys unless the CLI flags say otherwise. The
|
|
97
|
+
# tick preflights the panel key (and compares it against the author key —
|
|
98
|
+
# role separation) before claiming/submitting.
|
|
99
|
+
PANEL_KEY_DEFAULT = str(CONFIG_DIR / "verifier_key")
|
|
100
|
+
HARNESS_KEY_DEFAULT = str(CONFIG_DIR / "harness_key") # the claude author key
|
|
101
|
+
CODEX_KEY_DEFAULT = str(CONFIG_DIR / "codex_key") # the codex author key
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
def resolve_author_key_file(backend: str, explicit: str = "") -> str:
|
|
105
|
+
"""The author key file for `backend`. Per-backend keys COEXIST (claude's and
|
|
106
|
+
codex's both on disk), selected by backend — so the author backend is a
|
|
107
|
+
config choice, not a key swap, and an in-flight run of either backend can
|
|
108
|
+
still be woken/serviced after a fleet flip. An explicit path always wins;
|
|
109
|
+
otherwise the per-backend env var, then the packaged default path. The result
|
|
110
|
+
is always ~-expanded, so every caller gets a real path (an env value like
|
|
111
|
+
"~/.config/..." must not reach the token provider verbatim)."""
|
|
112
|
+
if not explicit:
|
|
113
|
+
if backend == "codex":
|
|
114
|
+
explicit = os.environ.get("AUTORESEARCH_CODEX_KEY_FILE") or CODEX_KEY_DEFAULT
|
|
115
|
+
else:
|
|
116
|
+
explicit = os.environ.get("AUTORESEARCH_HARNESS_KEY_FILE") or HARNESS_KEY_DEFAULT
|
|
117
|
+
return os.path.expanduser(explicit)
|
|
118
|
+
|
|
119
|
+
|
|
120
|
+
def codex_author_config_error(backend: str, model: str, image: str) -> str:
|
|
121
|
+
"""Why a codex author would die at startup ("" when it won't). Validates the
|
|
122
|
+
EFFECTIVE (backend, model) — the fresh climb passes args; a wake/follow-up
|
|
123
|
+
passes the PARKED RUN's persisted pair — so backend and model are checked as
|
|
124
|
+
a unit and never a fleet backend against a run's model. codex writes+executes,
|
|
125
|
+
so it must be contained (--image) and needs a non-claude model."""
|
|
126
|
+
if backend not in ("claude", "codex"):
|
|
127
|
+
# a typo'd AUTORESEARCH_AUTHOR_BACKEND passes the env DEFAULT silently
|
|
128
|
+
# (argparse validates the flag, not its default) and the climb rejects it
|
|
129
|
+
# at build_harness — catch it on the tick host so a claimed intake
|
|
130
|
+
# issue never strands on it
|
|
131
|
+
return f"unknown author backend {backend!r} (expected 'claude' or 'codex')"
|
|
132
|
+
if backend == "claude":
|
|
133
|
+
# symmetric to the codex check: a claude harness 404s on a non-claude
|
|
134
|
+
# model (e.g. AUTORESEARCH_AUTHOR_MODEL left on a codex id while the
|
|
135
|
+
# backend is claude) — catch that misconfig before spend
|
|
136
|
+
if model and not model.startswith("claude"):
|
|
137
|
+
return f"author-backend claude needs a claude model (got {model!r})"
|
|
138
|
+
return ""
|
|
139
|
+
if not image:
|
|
140
|
+
return "author-backend codex requires --image (it runs contained)"
|
|
141
|
+
if not model or model.startswith("claude"):
|
|
142
|
+
return (
|
|
143
|
+
"author-backend codex needs a codex/openai model "
|
|
144
|
+
f"(e.g. gpt-5.6-terra), not the claude default (got {model!r})"
|
|
145
|
+
)
|
|
146
|
+
return ""
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def resume_author(record: object, fleet_model: str) -> tuple[str, str, str]:
|
|
150
|
+
"""The (backend, model, key_file) a wake/follow-up must reproduce for a parked
|
|
151
|
+
run — all from the RECORD, not the current fleet.
|
|
152
|
+
|
|
153
|
+
An empty backend is a legacy record (written before the field) and is
|
|
154
|
+
therefore CLAUDE, never the fleet default; the model pairs with that backend
|
|
155
|
+
(a claude backend falls back to the claude default, a codex backend to the
|
|
156
|
+
fleet model only as a last resort — codex records always carry their model);
|
|
157
|
+
the key file is the exact resolved path the run used (so an explicit
|
|
158
|
+
--key-file survives), falling back to the per-backend resolution for legacy
|
|
159
|
+
records that never recorded it."""
|
|
160
|
+
backend = getattr(record, "author_backend", "") or "claude"
|
|
161
|
+
model = getattr(record, "author_model", "") or (
|
|
162
|
+
"claude-opus-5" if backend == "claude" else fleet_model
|
|
163
|
+
)
|
|
164
|
+
key_file = getattr(record, "author_key_file", "") or resolve_author_key_file(backend)
|
|
165
|
+
return backend, model, key_file
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
class WorkspaceDrift(RuntimeError):
|
|
169
|
+
"""The tree changed between measurement and commit."""
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def _target_clone_url(target: str) -> str:
|
|
173
|
+
"""The canonical HTTPS clone URL for `owner/repo`. The one source of truth
|
|
174
|
+
for where a run's git pushes go — derived from the target, never read from
|
|
175
|
+
the session-writable `remote.origin.url`."""
|
|
176
|
+
return f"https://github.com/{target}.git"
|
|
177
|
+
|
|
178
|
+
|
|
179
|
+
def _blessed_head(ws: Workspace, result: Any, contract: Any) -> str:
|
|
180
|
+
"""The pushed PR head the tick may later self-merge — only when this
|
|
181
|
+
publish was under merge:auto with a CLEAN panel (#171's arming
|
|
182
|
+
condition); "" otherwise. Best-effort: an unreadable HEAD blesses
|
|
183
|
+
nothing (never arm on doubt)."""
|
|
184
|
+
if not (
|
|
185
|
+
result.panel_rounds > 0
|
|
186
|
+
and not (result.panel_blocking_open or result.panel_degraded)
|
|
187
|
+
and getattr(contract, "merge", "manual") == "auto"
|
|
188
|
+
):
|
|
189
|
+
return ""
|
|
190
|
+
try:
|
|
191
|
+
return ws.git("rev-parse", "HEAD").strip()
|
|
192
|
+
except Exception:
|
|
193
|
+
return ""
|
|
194
|
+
|
|
195
|
+
|
|
196
|
+
def _arm_unless_base_moved(
|
|
197
|
+
github: GitHubClient,
|
|
198
|
+
ws: Workspace,
|
|
199
|
+
target: str,
|
|
200
|
+
pr_number: str,
|
|
201
|
+
base_branch: str,
|
|
202
|
+
measured_base_sha: str,
|
|
203
|
+
secrets: tuple[str, ...],
|
|
204
|
+
merge_mode: str = "manual",
|
|
205
|
+
panel_ran: bool = False,
|
|
206
|
+
) -> None:
|
|
207
|
+
"""Arm auto-merge only while origin/<base_branch> still equals the base the
|
|
208
|
+
claim was measured against. A moved base still OPENS the PR — review owns
|
|
209
|
+
staleness — but never ARMS it: merging a tree whose gate/suite/panel read
|
|
210
|
+
is stale must be a human's deliberate act, not an armed automation. A
|
|
211
|
+
failed freshness fetch also declines to arm (fail-safe: un-armed is just a
|
|
212
|
+
normal PR). Best-effort throughout, like arming itself."""
|
|
213
|
+
|
|
214
|
+
def _check_and_arm() -> None:
|
|
215
|
+
ws.git_network("fetch", str(ws.url or ws.remote_url()), base_branch)
|
|
216
|
+
fresh = ws.git("rev-parse", "FETCH_HEAD").strip()
|
|
217
|
+
if fresh != measured_base_sha:
|
|
218
|
+
log.info(
|
|
219
|
+
"not arming auto-merge on %s#%s: %s moved since the claim was "
|
|
220
|
+
"measured (%s -> %s); a human merges this one",
|
|
221
|
+
target,
|
|
222
|
+
pr_number,
|
|
223
|
+
base_branch,
|
|
224
|
+
measured_base_sha[:12],
|
|
225
|
+
fresh[:12],
|
|
226
|
+
)
|
|
227
|
+
return
|
|
228
|
+
if merge_mode == "auto" and not panel_ran:
|
|
229
|
+
# the dial's own precondition: auto means GATE+PANEL clean, so a
|
|
230
|
+
# publish that ran no panel must not self-merge — fall back to
|
|
231
|
+
# the manual guard and say so (terra #171: a panel-less
|
|
232
|
+
# deployment could otherwise self-merge on the metric gate alone)
|
|
233
|
+
log.warning(
|
|
234
|
+
"merge mode auto on %s#%s but no panel ran this attempt; "
|
|
235
|
+
"arming manual-mode instead",
|
|
236
|
+
target,
|
|
237
|
+
pr_number,
|
|
238
|
+
)
|
|
239
|
+
if merge_mode == "auto" and panel_ran:
|
|
240
|
+
# the contract's autonomy dial: the owner opted this repo into
|
|
241
|
+
# self-merging gate-clean PRs — arm, or merge directly when
|
|
242
|
+
# nothing is pending to arm against
|
|
243
|
+
github.arm_auto_merge_auto_mode(target, int(pr_number))
|
|
244
|
+
else:
|
|
245
|
+
github.arm_auto_merge_when_review_required(target, int(pr_number))
|
|
246
|
+
|
|
247
|
+
_best_effort("auto-merge arming", _check_and_arm, secrets)
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _title_pair(a: float, b: float) -> str:
|
|
251
|
+
"""Compact but never ambiguous: widen precision until the two numbers
|
|
252
|
+
render differently (a title reading '10.00 -> 10.00' looks like no
|
|
253
|
+
change even when the improvement is real)."""
|
|
254
|
+
for precision in range(4, 12):
|
|
255
|
+
fa, fb = f"{a:.{precision}g}", f"{b:.{precision}g}"
|
|
256
|
+
if fa != fb:
|
|
257
|
+
return f"{fa} -> {fb}"
|
|
258
|
+
return f"{a} -> {b}"
|
|
259
|
+
|
|
260
|
+
|
|
261
|
+
RULER = (
|
|
262
|
+
"The metric is computed by the contract's eval command over a frozen "
|
|
263
|
+
"instance pool. Your claim is verified by the orchestrator re-running "
|
|
264
|
+
"that exact command on your tree — and again by CI after the PR opens. "
|
|
265
|
+
"Only changes inside the contract's allowed paths are ever measured."
|
|
266
|
+
)
|
|
267
|
+
|
|
268
|
+
_ENDINGS_BY_OUTCOME = {
|
|
269
|
+
"no-improvement": NEGATIVE_RESULT,
|
|
270
|
+
# the improvement was real but bought by regressing a sibling benchmark —
|
|
271
|
+
# an honest negative with a named cause, not a malfunction
|
|
272
|
+
"suite-regression": NEGATIVE_RESULT,
|
|
273
|
+
"session-error": ABORTED,
|
|
274
|
+
"session-budget": BUDGET_EXHAUSTED,
|
|
275
|
+
"session-outage": STUCK, # infrastructure failure, nothing about the run
|
|
276
|
+
"eval-error": ABORTED,
|
|
277
|
+
"scope-violation": ABORTED,
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
@dataclass(frozen=True)
|
|
282
|
+
class AttemptOutcome:
|
|
283
|
+
run_id: str
|
|
284
|
+
outcome: str
|
|
285
|
+
pr_url: str = ""
|
|
286
|
+
report_path: str = ""
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _best_effort(what: str, fn: Callable[[], object], secrets: tuple[str, ...] = ()) -> bool:
|
|
290
|
+
"""One ending step; a failure is logged, never raised.
|
|
291
|
+
|
|
292
|
+
The terminal sequence (record, report, issue post) must degrade
|
|
293
|
+
independently: a full disk must not block the GitHub post, and a network
|
|
294
|
+
failure must not block the record.
|
|
295
|
+
"""
|
|
296
|
+
try:
|
|
297
|
+
fn()
|
|
298
|
+
return True
|
|
299
|
+
except Exception as exc:
|
|
300
|
+
log.warning("%s failed: %s", what, redact(f"{type(exc).__name__}: {exc}", secrets))
|
|
301
|
+
return False
|
|
302
|
+
|
|
303
|
+
|
|
304
|
+
def _clear_stage(record: RunRecord) -> RunRecord:
|
|
305
|
+
"""Strip the WAITING-only bookkeeping from a record leaving `waiting` for a
|
|
306
|
+
terminal state. Otherwise a dispatched run's `stage`, `deadline`, and
|
|
307
|
+
especially `wake_attempts` ride into `in-review`, where in-review follow-up
|
|
308
|
+
servicing reuses `wake_attempts` as its OWN retry cap — so a run that woke
|
|
309
|
+
once would reach review with a shrunk follow-up budget."""
|
|
310
|
+
# the run's spend survives the wipe: terminal reporting (the climb
|
|
311
|
+
# board) reads it after the transition
|
|
312
|
+
kept = {k: record.stage[k] for k in ("gpu_hours_used",) if record.stage and k in record.stage}
|
|
313
|
+
return dc_replace(
|
|
314
|
+
record,
|
|
315
|
+
stage=kept,
|
|
316
|
+
experiment_job_id="",
|
|
317
|
+
deadline=0.0,
|
|
318
|
+
terminal_seen=0.0,
|
|
319
|
+
wake_attempts=0,
|
|
320
|
+
)
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def _post_issue_finished(
|
|
324
|
+
github: GitHubClient,
|
|
325
|
+
target: str,
|
|
326
|
+
issue_number: int,
|
|
327
|
+
run_id: str,
|
|
328
|
+
outcome_name: str,
|
|
329
|
+
pr_url: str,
|
|
330
|
+
summary: str,
|
|
331
|
+
secrets: tuple[str, ...],
|
|
332
|
+
) -> None:
|
|
333
|
+
"""Post a run's terminal result back to the issue that requested it. When
|
|
334
|
+
the run ends WITHOUT a PR (a negative or an error), include the
|
|
335
|
+
`RELEASE_MARKER` so `intake.pick_issue` can re-select the issue — a comment
|
|
336
|
+
alone does NOT un-claim it. An improved run KEEPS the claim: its PR
|
|
337
|
+
(`Addresses #N`) is the ongoing work, and `followup` releases the claim if
|
|
338
|
+
that PR later closes unmerged."""
|
|
339
|
+
if not issue_number:
|
|
340
|
+
return
|
|
341
|
+
from outerloop.intake import RELEASE_MARKER
|
|
342
|
+
|
|
343
|
+
link = f"\n\nPull request: {pr_url}" if pr_url else ""
|
|
344
|
+
# no PR opened -> nothing will ever resolve this issue, so free the claim
|
|
345
|
+
# (bounded by intake's per-issue attempt cap).
|
|
346
|
+
release = f"{RELEASE_MARKER}\n" if not pr_url else ""
|
|
347
|
+
_best_effort(
|
|
348
|
+
"issue report",
|
|
349
|
+
lambda: github.comment(
|
|
350
|
+
target,
|
|
351
|
+
issue_number,
|
|
352
|
+
f"{release}Run `{run_id}` finished ({outcome_name}).{link}\n\n{summary}",
|
|
353
|
+
),
|
|
354
|
+
secrets,
|
|
355
|
+
)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
# A parked run's deadline is the FLOOR beneath the afterany wake: submit +
|
|
359
|
+
# eval walltime + a generous queue/grace allowance. It must exceed the time a
|
|
360
|
+
# healthy eval can legitimately sit queued-then-running, or `tick._sweep_one`
|
|
361
|
+
# would cancel a still-queued job as "unschedulable".
|
|
362
|
+
PARK_QUEUE_SLACK_MIN = 12 * 60
|
|
363
|
+
# a jobless checkpoint sleep only needs to survive to the next sweep pass:
|
|
364
|
+
# one cadence + coalescing headroom, not queue slack
|
|
365
|
+
CHECKPOINT_SLEEP_SLACK_MIN = 45
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def _park_run(
|
|
369
|
+
run_root: Path,
|
|
370
|
+
record: RunRecord,
|
|
371
|
+
parked: RunParked,
|
|
372
|
+
candidate_ref: str,
|
|
373
|
+
eval_minutes: int | None,
|
|
374
|
+
now: float,
|
|
375
|
+
secrets: tuple[str, ...] = (),
|
|
376
|
+
keep_wake_attempts: bool = False,
|
|
377
|
+
base_branch: str = "main",
|
|
378
|
+
panel_reads: int = 0,
|
|
379
|
+
dispatch: DispatchSettings | None = None,
|
|
380
|
+
) -> None:
|
|
381
|
+
"""Persist a dispatched climb's re-entry point as a WAITING record: the
|
|
382
|
+
committed shas, drawn seeds, candidate snapshot ref, and afterany set a
|
|
383
|
+
fresh process reconstructs the measure-and-decide phase from. The caller
|
|
384
|
+
passes the EXACT `candidate_ref` it will keep alive (never re-derive it from
|
|
385
|
+
the commit — two snapshots can share a commit)."""
|
|
386
|
+
from outerloop.dispatch import effective_eval_minutes
|
|
387
|
+
|
|
388
|
+
job_ids = afterany_ids(parked.afterany)
|
|
389
|
+
stage: dict[str, object] = {
|
|
390
|
+
"phase": parked.phase,
|
|
391
|
+
"base_sha": parked.base_sha,
|
|
392
|
+
"candidate_sha": parked.candidate_sha,
|
|
393
|
+
"candidate_ref": candidate_ref,
|
|
394
|
+
"seed": parked.seed,
|
|
395
|
+
"suite_seed": parked.suite_seed,
|
|
396
|
+
"afterany": parked.afterany,
|
|
397
|
+
"launch_afterany": parked.launch_afterany,
|
|
398
|
+
# the branch the run targets, so a wake opens its PR against the SAME
|
|
399
|
+
# branch a non-default `--base-branch` selected — the wake CLI otherwise
|
|
400
|
+
# defaults to main and would mis-target.
|
|
401
|
+
"base_branch": base_branch,
|
|
402
|
+
# verification-panel reads so far — persisted so the next wake
|
|
403
|
+
# continues the count.
|
|
404
|
+
"panel_reads": panel_reads,
|
|
405
|
+
# the session's write-up + spend, saved so a candidate wake can build
|
|
406
|
+
# the PR body / panel claim and report the real cost WITHOUT re-running
|
|
407
|
+
# the session (its edits are already in candidate_sha). REDACTED before
|
|
408
|
+
# it lands in the durable record, like every other persisted final_text
|
|
409
|
+
# — a session that echoed a credential must not leave it in record.json.
|
|
410
|
+
# Empty/zero for a baseline park (the session has not run yet).
|
|
411
|
+
"report": redact(parked.session.final_text, secrets)[:MAX_CLAIM_CHARS]
|
|
412
|
+
if parked.session
|
|
413
|
+
else "",
|
|
414
|
+
"session_cost_usd": parked.session.cost_usd if parked.session else 0.0,
|
|
415
|
+
"session_turns": parked.session.num_turns if parked.session else 0,
|
|
416
|
+
}
|
|
417
|
+
if parked.submitted:
|
|
418
|
+
# a SUBMITTED candidate park (buildout Phase B): the wake delivers the
|
|
419
|
+
# gate + panel results back to the author instead of deciding by policy
|
|
420
|
+
stage["submitted"] = True
|
|
421
|
+
if parked.syscall is not None:
|
|
422
|
+
# A syscall park — an author-directed sleep (research-loop-buildout.md
|
|
423
|
+
# Phase A) or a submitted candidate carrying sibling launches: the wake
|
|
424
|
+
# gathers each launch's results by NAME from the run dir, delivers the
|
|
425
|
+
# declared artifacts, and resumes the SAME session — so the stage must
|
|
426
|
+
# carry the launch names/artifacts, the author's note, and the budget
|
|
427
|
+
# counts as of this park.
|
|
428
|
+
stage["syscall_launches"] = [
|
|
429
|
+
# minutes ride along so a RE-PARK's deadline floor still covers the
|
|
430
|
+
# longest launch — without them a rebuilt descriptor would
|
|
431
|
+
# undershoot the floor and the sweep could cancel a healthy
|
|
432
|
+
# queued sibling as "pending past deadline"
|
|
433
|
+
{
|
|
434
|
+
"name": launch.name,
|
|
435
|
+
"minutes": launch.minutes,
|
|
436
|
+
"artifacts": list(launch.artifacts),
|
|
437
|
+
**({"array": launch.array} if launch.array > 1 else {}),
|
|
438
|
+
}
|
|
439
|
+
for launch in parked.syscall.launches
|
|
440
|
+
]
|
|
441
|
+
stage["syscall_note"] = redact(parked.syscall.note, secrets)
|
|
442
|
+
# (the session id the wake resumes is the record's own
|
|
443
|
+
# resume_session_id, set below for every park — no stage duplicate)
|
|
444
|
+
stage["launches_used"] = parked.launches_used
|
|
445
|
+
stage["sleeps_used"] = parked.sleeps_used
|
|
446
|
+
stage["gpu_hours_used"] = parked.gpu_hours_used
|
|
447
|
+
if parked.judged is not None:
|
|
448
|
+
# the gate's last negative rides the park: a wake that ends on the
|
|
449
|
+
# same tree reuses it instead of measuring again
|
|
450
|
+
judged_sha, verdict = parked.judged
|
|
451
|
+
stage["judged"] = {
|
|
452
|
+
"sha": judged_sha,
|
|
453
|
+
"outcome": verdict.outcome,
|
|
454
|
+
"baseline": verdict.baseline,
|
|
455
|
+
"candidate": verdict.candidate,
|
|
456
|
+
"note": redact(verdict.note, secrets),
|
|
457
|
+
}
|
|
458
|
+
if parked.eval_minutes:
|
|
459
|
+
# the author's declared eval walltime rides the park: the wake's
|
|
460
|
+
# measurer and deadline floor must use it, not the contract's
|
|
461
|
+
stage["eval_minutes"] = parked.eval_minutes
|
|
462
|
+
# A single-job park records its one pollable id; a MULTI-job park records
|
|
463
|
+
# none — the sweep falls back to polling every id in the stage's `afterany`
|
|
464
|
+
# string and wakes only when ALL are done (tick._poll_targets).
|
|
465
|
+
experiment_job_id = job_ids[0] if len(job_ids) == 1 else ""
|
|
466
|
+
# The deadline is a FLOOR: park time (`now` here is the park moment, passed
|
|
467
|
+
# by the caller) + the eval walltime + a generous queue/grace slack, so a
|
|
468
|
+
# healthy queued-then-running eval never trips the sweep's cancel-on-pending.
|
|
469
|
+
floor_minutes = effective_eval_minutes(parked.eval_minutes or eval_minutes)
|
|
470
|
+
if parked.phase == "author-sleep" and parked.syscall is not None:
|
|
471
|
+
# an author launch's walltime is the LAUNCH's ask, not the benchmark's
|
|
472
|
+
# eval hint — the floor must sit past the LONGEST launch, or the sweep
|
|
473
|
+
# cancels still-queued author jobs (a benchmark can be in-job cheap,
|
|
474
|
+
# eval_minutes=None, while its author trains for hours). A checkpoint
|
|
475
|
+
# sleep has no jobs: floor 0 wakes it at the first deadline pass.
|
|
476
|
+
floor_minutes = max((la.minutes for la in parked.syscall.launches), default=0)
|
|
477
|
+
elif parked.syscall is not None:
|
|
478
|
+
# a submitted candidate with sibling launches waits on gate evals AND
|
|
479
|
+
# launches — the floor must sit past the longest of either
|
|
480
|
+
floor_minutes = max(floor_minutes, *(la.minutes for la in parked.syscall.launches), 0)
|
|
481
|
+
checkpoint_sleep = (
|
|
482
|
+
parked.phase == "author-sleep"
|
|
483
|
+
and parked.syscall is not None
|
|
484
|
+
and not parked.syscall.launches
|
|
485
|
+
)
|
|
486
|
+
if checkpoint_sleep:
|
|
487
|
+
# a CHECKPOINT SLEEP has nothing in any queue, so the 12h queue slack
|
|
488
|
+
# (sized to protect queued Slurm jobs from cancel-on-pending) does not
|
|
489
|
+
# apply — the deadline needs only to reach the sweep's next pass.
|
|
490
|
+
# Observed live (yolo heldout_probe, 2026-08-27): a jobless nap
|
|
491
|
+
# inherited the queue slack and became a 12h coma.
|
|
492
|
+
deadline = now + CHECKPOINT_SLEEP_SLACK_MIN * 60
|
|
493
|
+
else:
|
|
494
|
+
deadline = now + (floor_minutes + PARK_QUEUE_SLACK_MIN) * 60
|
|
495
|
+
waiting = RunRecord(
|
|
496
|
+
**{
|
|
497
|
+
**record.__dict__,
|
|
498
|
+
"state": WAITING,
|
|
499
|
+
"experiment_job_id": experiment_job_id,
|
|
500
|
+
"resume_session_id": parked.session.session_id if parked.session else "",
|
|
501
|
+
"deadline": deadline,
|
|
502
|
+
"stage": stage,
|
|
503
|
+
# wake_attempts = "wakes since the run last made progress"; the
|
|
504
|
+
# stuck cap ends a run that keeps waking without advancing. Reset on
|
|
505
|
+
# a PRODUCTIVE park (the IMPLEMENTING->park entry ran a session; a
|
|
506
|
+
# wake that resolved its measures and dispatched NEW ones). A
|
|
507
|
+
# no-progress re-park — results still pending, or a blind re-park
|
|
508
|
+
# (squeue unreachable, nothing new dispatched) — must KEEP the
|
|
509
|
+
# counter (`keep_wake_attempts`), or the loop never reaches the cap.
|
|
510
|
+
"wake_attempts": record.wake_attempts if keep_wake_attempts else 0,
|
|
511
|
+
"terminal_seen": 0.0,
|
|
512
|
+
}
|
|
513
|
+
)
|
|
514
|
+
save_record(run_root, waiting, now)
|
|
515
|
+
if dispatch is not None:
|
|
516
|
+
_arm_park_wake(run_root, record.run_id, now, dispatch)
|
|
517
|
+
|
|
518
|
+
|
|
519
|
+
def _arm_park_wake(run_root: Path, run_id: str, now: float, dispatch: DispatchSettings) -> str:
|
|
520
|
+
"""Submit the parked run's wake right away, depending on the jobs it
|
|
521
|
+
waits on (tick.arm_wake), when the tick has published its wake recipe.
|
|
522
|
+
Without the recipe — dispatched wakes not armed, or a local compute —
|
|
523
|
+
the sweep delivers as before."""
|
|
524
|
+
from outerloop.compute import SlurmCompute
|
|
525
|
+
from outerloop.tick import JobWakeDispatcher, arm_wake, dispatch_wake_armed, load_wake_spec
|
|
526
|
+
|
|
527
|
+
if not dispatch_wake_armed(run_root):
|
|
528
|
+
return "" # disarmed: a recipe the tick has not yet removed is not used
|
|
529
|
+
spec = load_wake_spec(run_root)
|
|
530
|
+
if spec is None or not isinstance(dispatch.compute, SlurmCompute):
|
|
531
|
+
return ""
|
|
532
|
+
try:
|
|
533
|
+
record = load_record(run_root, run_id)
|
|
534
|
+
dispatcher = JobWakeDispatcher(dispatch.compute, spec, now)
|
|
535
|
+
return arm_wake(
|
|
536
|
+
run_root, record, dispatcher, now, holder_job_id=os.environ.get("SLURM_JOB_ID", "")
|
|
537
|
+
)
|
|
538
|
+
except Exception as exc:
|
|
539
|
+
log.warning("park-time wake not armed for %s: %s: %s", run_id, type(exc).__name__, exc)
|
|
540
|
+
return ""
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def _lease_held_by_another_job(run_root: Path, run_id: str) -> str:
|
|
544
|
+
"""The job id of a wake that holds this run's lease and is not us, or "".
|
|
545
|
+
A resume with no job id of its own (a manual run) never counts as the
|
|
546
|
+
holder of a job-held lease."""
|
|
547
|
+
from outerloop.runstate import read_lease
|
|
548
|
+
|
|
549
|
+
lease = read_lease(run_root, run_id)
|
|
550
|
+
mine = os.environ.get("SLURM_JOB_ID", "")
|
|
551
|
+
if lease is not None and lease.holder_job_id and lease.holder_job_id != mine:
|
|
552
|
+
return lease.holder_job_id
|
|
553
|
+
return ""
|
|
554
|
+
|
|
555
|
+
|
|
556
|
+
def _release_own_lease(run_root: Path, run_id: str) -> None:
|
|
557
|
+
"""Release the run's lease only while this job still holds it: a park
|
|
558
|
+
hands the lease to the wake it arms, and that wake must keep it."""
|
|
559
|
+
from outerloop.runstate import release_lease
|
|
560
|
+
|
|
561
|
+
if _lease_held_by_another_job(run_root, run_id):
|
|
562
|
+
return
|
|
563
|
+
release_lease(run_root, run_id)
|
|
564
|
+
|
|
565
|
+
|
|
566
|
+
def _dispatch_settings(args: argparse.Namespace) -> DispatchSettings:
|
|
567
|
+
"""The cluster coordinates from the CLI, read in ONE place for both the
|
|
568
|
+
fresh climb and the wake (a second constructor drifted once — terra
|
|
569
|
+
#174: the wake dropped the GPU lane)."""
|
|
570
|
+
from outerloop.compute import compute_from_env
|
|
571
|
+
|
|
572
|
+
return DispatchSettings(
|
|
573
|
+
compute=compute_from_env(),
|
|
574
|
+
image=args.image,
|
|
575
|
+
account=args.account,
|
|
576
|
+
partition=args.partition,
|
|
577
|
+
gpu_partition=getattr(args, "gpu_partition", ""),
|
|
578
|
+
gpu_account=getattr(args, "gpu_account", ""),
|
|
579
|
+
)
|
|
580
|
+
|
|
581
|
+
|
|
582
|
+
def _make_launcher(
|
|
583
|
+
dispatch: DispatchSettings, run_dir: Path, workspace: Path, run_id: str, gpus: int = 0
|
|
584
|
+
):
|
|
585
|
+
"""The launch side of the author syscalls, shared by the first pass
|
|
586
|
+
(live_attempt) and the author-sleep wake: each launch becomes a jailed job on
|
|
587
|
+
the sealed snapshot (write_eval_job's copy-out handles artifacts), and a
|
|
588
|
+
partially-submitted batch is reaped rather than orphaned. `gpus` is the
|
|
589
|
+
benchmark's: an author's experiments run on the same lane as its evals."""
|
|
590
|
+
account, partition = dispatch.placement(gpus)
|
|
591
|
+
|
|
592
|
+
def launcher(sha: str, request: SyscallRequest) -> str:
|
|
593
|
+
from outerloop.dispatch import eval_job_spec, write_eval_job
|
|
594
|
+
from outerloop.syscall import launch_jobs
|
|
595
|
+
|
|
596
|
+
ids: list[str] = []
|
|
597
|
+
try:
|
|
598
|
+
for launch in request.launches:
|
|
599
|
+
# an array launch is N jobs of one command, each with its
|
|
600
|
+
# SWEEP_INDEX; one afterany wake covers them all
|
|
601
|
+
for job_name, extra_env in launch_jobs(launch):
|
|
602
|
+
script = write_eval_job(
|
|
603
|
+
run_dir,
|
|
604
|
+
f"launch-{job_name}",
|
|
605
|
+
repo_root=workspace,
|
|
606
|
+
snapshot_sha=sha,
|
|
607
|
+
command=launch.command,
|
|
608
|
+
image=dispatch.image,
|
|
609
|
+
extra_env=extra_env,
|
|
610
|
+
artifacts=launch.artifacts,
|
|
611
|
+
artifact_max_bytes=MAX_ARTIFACT_BYTES,
|
|
612
|
+
gpus=gpus,
|
|
613
|
+
)
|
|
614
|
+
ids.append(
|
|
615
|
+
dispatch.compute.submit(
|
|
616
|
+
eval_job_spec(
|
|
617
|
+
script,
|
|
618
|
+
job_name=f"{run_id}-launch-{job_name}",
|
|
619
|
+
account=account,
|
|
620
|
+
partition=partition,
|
|
621
|
+
eval_minutes=launch.minutes,
|
|
622
|
+
gpus=gpus,
|
|
623
|
+
)
|
|
624
|
+
)
|
|
625
|
+
)
|
|
626
|
+
except Exception:
|
|
627
|
+
# a partial batch must not orphan: no park record was written yet,
|
|
628
|
+
# so nothing would ever wake or cancel the jobs that DID submit —
|
|
629
|
+
# reap them here, then let the caller end the run as the error it
|
|
630
|
+
# is (same stance as the failed-_park_run cancel).
|
|
631
|
+
for job_id in ids:
|
|
632
|
+
with contextlib.suppress(Exception):
|
|
633
|
+
dispatch.compute.cancel(job_id)
|
|
634
|
+
raise
|
|
635
|
+
# a checkpoint sleep (no launches) parks with no dependency and wakes
|
|
636
|
+
# on the sweep's deadline floor — slow but correct; a fast requeue wake
|
|
637
|
+
# is a follow-up.
|
|
638
|
+
return "afterany:" + ":".join(ids) if ids else ""
|
|
639
|
+
|
|
640
|
+
return launcher
|
|
641
|
+
|
|
642
|
+
|
|
643
|
+
def _wake_author_sleep(
|
|
644
|
+
*,
|
|
645
|
+
run_root: Path,
|
|
646
|
+
run_id: str,
|
|
647
|
+
record: RunRecord,
|
|
648
|
+
ws: Workspace,
|
|
649
|
+
workspace: Path,
|
|
650
|
+
run_dir: Path,
|
|
651
|
+
dispatch: DispatchSettings,
|
|
652
|
+
github: GitHubClient,
|
|
653
|
+
now: float,
|
|
654
|
+
secrets: tuple[str, ...],
|
|
655
|
+
base_branch: str,
|
|
656
|
+
base_sha: str,
|
|
657
|
+
sleep_ref: str,
|
|
658
|
+
contract_text: str,
|
|
659
|
+
contract: Contract,
|
|
660
|
+
bench: Benchmark,
|
|
661
|
+
config: RunConfig,
|
|
662
|
+
measurer: Measurer,
|
|
663
|
+
harness: Harness | None,
|
|
664
|
+
spec: RoleSpec | None,
|
|
665
|
+
panel_lenses: tuple[PanelLens, ...],
|
|
666
|
+
issue_number: int,
|
|
667
|
+
eval_minutes: int | None,
|
|
668
|
+
extra_update: str = "",
|
|
669
|
+
judged: tuple[str, AttemptResult] | None = None,
|
|
670
|
+
) -> AttemptOutcome:
|
|
671
|
+
"""Wake a syscall park and resume the AUTHOR: deliver the launches' results
|
|
672
|
+
into the sandbox, resume the SAME session through the climb's resume-entry
|
|
673
|
+
with them (data-fenced), and let the climb run — it may sleep again
|
|
674
|
+
(re-park), submit, finish into the gate (whose dispatched measures park it
|
|
675
|
+
as a CANDIDATE), or end on a terminal. The session's workspace persisted on
|
|
676
|
+
disk exactly as the author left it (the launches ran on node-local
|
|
677
|
+
checkouts of the sealed sha), so the resumed session continues its own tree
|
|
678
|
+
— cumulative depth. `extra_update` leads the wake text — a SUBMITTED park's
|
|
679
|
+
gate result or panel verdict (buildout Phase B), delivered back to the
|
|
680
|
+
author to act on; `sleep_ref` is whichever snapshot ref this park holds
|
|
681
|
+
(the sleep seal, or the submitted candidate)."""
|
|
682
|
+
from outerloop.syscall import Launch as SyscallLaunch
|
|
683
|
+
from outerloop.syscall import (
|
|
684
|
+
annotate_launch_states,
|
|
685
|
+
gather_results,
|
|
686
|
+
render_wake,
|
|
687
|
+
write_budget,
|
|
688
|
+
)
|
|
689
|
+
|
|
690
|
+
def _end(result: AttemptResult, drop_refs: list[str]) -> AttemptOutcome:
|
|
691
|
+
# a terminal from the resumed climb: report, ending record, issue note —
|
|
692
|
+
# the same ending shape every other terminal takes. The line notebook
|
|
693
|
+
# records it first, while the tree is still the session's final tree.
|
|
694
|
+
_push_line_snapshot(
|
|
695
|
+
ws, _line_ref_for(bench, config.agent_id), run_id, result.outcome, secrets
|
|
696
|
+
)
|
|
697
|
+
for ref in drop_refs:
|
|
698
|
+
drop_snapshot(ws, Snapshot(commit="", tree="", ref=ref))
|
|
699
|
+
report_path = run_dir / "report.md"
|
|
700
|
+
_best_effort(
|
|
701
|
+
"run report",
|
|
702
|
+
lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
|
|
703
|
+
secrets,
|
|
704
|
+
)
|
|
705
|
+
ending = _ENDINGS_BY_OUTCOME.get(result.outcome, ABORTED)
|
|
706
|
+
final = _clear_stage(
|
|
707
|
+
RunRecord(
|
|
708
|
+
**{
|
|
709
|
+
**record.__dict__,
|
|
710
|
+
"state": ENDED,
|
|
711
|
+
"ending": ending,
|
|
712
|
+
"ending_note": redact(result.note, secrets),
|
|
713
|
+
}
|
|
714
|
+
)
|
|
715
|
+
)
|
|
716
|
+
_best_effort("final record", lambda: save_record(run_root, final, now), secrets)
|
|
717
|
+
_post_issue_finished(
|
|
718
|
+
github,
|
|
719
|
+
config.target,
|
|
720
|
+
issue_number,
|
|
721
|
+
run_id,
|
|
722
|
+
result.outcome,
|
|
723
|
+
"",
|
|
724
|
+
redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
|
|
725
|
+
secrets,
|
|
726
|
+
)
|
|
727
|
+
return AttemptOutcome(run_id=run_id, outcome=result.outcome, report_path=str(report_path))
|
|
728
|
+
|
|
729
|
+
# The wake NEEDS the author harness (it resumes the session). Fail as a
|
|
730
|
+
# named ending, not a crash: the run cannot proceed and re-waking will not
|
|
731
|
+
# help without the harness, so leaving it WAITING would just hit the stuck
|
|
732
|
+
# cap slowly.
|
|
733
|
+
if (
|
|
734
|
+
harness is None
|
|
735
|
+
or spec is None
|
|
736
|
+
or not record.resume_session_id
|
|
737
|
+
or not getattr(harness, "supports_resume", True)
|
|
738
|
+
):
|
|
739
|
+
return _end(
|
|
740
|
+
AttemptResult(
|
|
741
|
+
outcome="session-error",
|
|
742
|
+
note="author-sleep wake needs the author harness/spec and a resumable session",
|
|
743
|
+
),
|
|
744
|
+
drop_refs=[sleep_ref],
|
|
745
|
+
)
|
|
746
|
+
|
|
747
|
+
# Deliver: read each launch's job output, copy declared artifacts into the
|
|
748
|
+
# excluded channel, and render the data-fenced wake text. The stage stores
|
|
749
|
+
# only what the wake needs (names + artifacts); command/minutes placeholders
|
|
750
|
+
# never reach the author.
|
|
751
|
+
launches = tuple(
|
|
752
|
+
SyscallLaunch(
|
|
753
|
+
name=str(item.get("name", "")),
|
|
754
|
+
command="(ran)",
|
|
755
|
+
minutes=int(item.get("minutes") or 1),
|
|
756
|
+
artifacts=tuple(str(a) for a in item.get("artifacts", [])),
|
|
757
|
+
array=int(item.get("array") or 1),
|
|
758
|
+
)
|
|
759
|
+
for item in _stage_launches(record)
|
|
760
|
+
)
|
|
761
|
+
results = gather_results(run_dir, workspace, launches)
|
|
762
|
+
# A launch that left no exit code was SIGKILL'd before its wrapper ran (the
|
|
763
|
+
# cgroup OOM killer, a hard walltime kill, a node failure). The scheduler
|
|
764
|
+
# still knows which; surface it so the author reads "OOM"/"timeout" instead
|
|
765
|
+
# of a blank "job failure". The park's launch job ids align positionally
|
|
766
|
+
# with the results (same launch/array order). Best-effort — the wake never
|
|
767
|
+
# blocks on the scheduler query.
|
|
768
|
+
status_of = getattr(dispatch.compute, "status", None)
|
|
769
|
+
if status_of is not None:
|
|
770
|
+
results = annotate_launch_states(results, _stage_launch_job_ids(record), status_of)
|
|
771
|
+
launches_used = int(record.stage.get("launches_used", 0)) # type: ignore[call-overload]
|
|
772
|
+
sleeps_used = int(record.stage.get("sleeps_used", 0)) # type: ignore[call-overload]
|
|
773
|
+
gpu_hours_used = _reconcile_launch_hours(record, dispatch, bench.gpus, launches)
|
|
774
|
+
wake_text = render_wake(
|
|
775
|
+
results,
|
|
776
|
+
str(record.stage.get("syscall_note", "")),
|
|
777
|
+
launches_used=launches_used,
|
|
778
|
+
launch_budget=bench.depth_k,
|
|
779
|
+
sleeps_used=sleeps_used,
|
|
780
|
+
sleep_budget=bench.sleep_k,
|
|
781
|
+
gpu_hours_remaining=(
|
|
782
|
+
max(0.0, contract.budgets.gpu_hours_per_run - gpu_hours_used) if bench.gpus else None
|
|
783
|
+
),
|
|
784
|
+
gpus=bench.gpus,
|
|
785
|
+
)
|
|
786
|
+
if extra_update:
|
|
787
|
+
# a submitted park's gate/panel feedback leads; launch results follow
|
|
788
|
+
wake_text = f"{extra_update}\n\n{wake_text}"
|
|
789
|
+
_best_effort(
|
|
790
|
+
"budget refresh",
|
|
791
|
+
lambda: write_budget(
|
|
792
|
+
workspace,
|
|
793
|
+
launches_remaining=max(0, bench.depth_k - launches_used),
|
|
794
|
+
sleeps_remaining=max(0, bench.sleep_k - sleeps_used),
|
|
795
|
+
gpu_hours_remaining=(
|
|
796
|
+
max(0.0, contract.budgets.gpu_hours_per_run - gpu_hours_used)
|
|
797
|
+
if bench.gpus
|
|
798
|
+
else None
|
|
799
|
+
),
|
|
800
|
+
),
|
|
801
|
+
)
|
|
802
|
+
|
|
803
|
+
# The wake's climb IO: measures go through the DISPATCHED measurer (this is
|
|
804
|
+
# a wake job with bounded walltime — the gate's evals run as their own jobs
|
|
805
|
+
# and park the run as a CANDIDATE); snapshots parent on base (same as the
|
|
806
|
+
# first pass: the clone was at base).
|
|
807
|
+
snapshots: list[Snapshot] = []
|
|
808
|
+
wake_line = _line_ref_for(bench, config.agent_id)
|
|
809
|
+
|
|
810
|
+
def snapshot() -> str:
|
|
811
|
+
snap = snapshot_tree(ws, base_sha, exclude=LINE_MEMORY_PATHS if wake_line else ())
|
|
812
|
+
snapshots.append(snap)
|
|
813
|
+
return snap.commit
|
|
814
|
+
|
|
815
|
+
def changed_paths() -> list[str]:
|
|
816
|
+
return _paths_changed_from_base(
|
|
817
|
+
ws, f"refs/remotes/origin/{base_branch}", bool(wake_line), fallback=base_sha
|
|
818
|
+
)
|
|
819
|
+
|
|
820
|
+
panel_runner = (
|
|
821
|
+
build_panel_runner(
|
|
822
|
+
ws,
|
|
823
|
+
run_dir,
|
|
824
|
+
base_sha,
|
|
825
|
+
panel_lenses,
|
|
826
|
+
contract_text,
|
|
827
|
+
config.target,
|
|
828
|
+
config.benchmark,
|
|
829
|
+
config.bot_login,
|
|
830
|
+
_utc_date(now),
|
|
831
|
+
exclude=LINE_MEMORY_PATHS if wake_line else (),
|
|
832
|
+
)
|
|
833
|
+
if panel_lenses
|
|
834
|
+
else None
|
|
835
|
+
)
|
|
836
|
+
|
|
837
|
+
parked: RunParked | None = None
|
|
838
|
+
kept_ref = ""
|
|
839
|
+
try:
|
|
840
|
+
result = attempt_once(
|
|
841
|
+
config,
|
|
842
|
+
contract_text,
|
|
843
|
+
workspace,
|
|
844
|
+
harness,
|
|
845
|
+
measurer,
|
|
846
|
+
base_sha,
|
|
847
|
+
snapshot,
|
|
848
|
+
ruler=RULER,
|
|
849
|
+
changed_paths=changed_paths,
|
|
850
|
+
spec=spec,
|
|
851
|
+
panel_runner=panel_runner,
|
|
852
|
+
resume_session_id=record.resume_session_id,
|
|
853
|
+
improve_prompt=wake_text,
|
|
854
|
+
launcher=_make_launcher(dispatch, run_dir, workspace, run_id, gpus=bench.gpus),
|
|
855
|
+
tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(),
|
|
856
|
+
judged=judged or _stage_judged(record),
|
|
857
|
+
launches_used=launches_used,
|
|
858
|
+
sleeps_used=sleeps_used,
|
|
859
|
+
gpu_hours_used=gpu_hours_used,
|
|
860
|
+
)
|
|
861
|
+
except RunParked as p:
|
|
862
|
+
# slept again, or the gate dispatched its measures (a candidate park the
|
|
863
|
+
# existing wake path decides). Keep the NEW park's snapshot ref; the OLD
|
|
864
|
+
# sleep ref is superseded once the new park persists.
|
|
865
|
+
kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
|
|
866
|
+
import time
|
|
867
|
+
|
|
868
|
+
try:
|
|
869
|
+
_park_run(
|
|
870
|
+
run_root,
|
|
871
|
+
record,
|
|
872
|
+
p,
|
|
873
|
+
kept_ref,
|
|
874
|
+
eval_minutes,
|
|
875
|
+
time.time(),
|
|
876
|
+
secrets,
|
|
877
|
+
dispatch=dispatch,
|
|
878
|
+
base_branch=base_branch,
|
|
879
|
+
)
|
|
880
|
+
except Exception:
|
|
881
|
+
for job_id in afterany_ids(p.afterany):
|
|
882
|
+
dispatch.compute.cancel(job_id)
|
|
883
|
+
raise
|
|
884
|
+
parked = p
|
|
885
|
+
drop_snapshot(ws, Snapshot(commit="", tree="", ref=sleep_ref))
|
|
886
|
+
return AttemptOutcome(run_id=run_id, outcome="parked")
|
|
887
|
+
finally:
|
|
888
|
+
for snap in snapshots:
|
|
889
|
+
if parked and kept_ref and snap.ref == kept_ref:
|
|
890
|
+
continue
|
|
891
|
+
drop_snapshot(ws, snap)
|
|
892
|
+
|
|
893
|
+
# a terminal from the resumed session (session-error/-outage/-budget,
|
|
894
|
+
# scope-violation, eval-error; a dispatched gate never returns improved
|
|
895
|
+
# inline): end the run and release the sleep snapshot.
|
|
896
|
+
return _end(result, drop_refs=[sleep_ref])
|
|
897
|
+
|
|
898
|
+
|
|
899
|
+
def _stage_judged(record: RunRecord) -> tuple[str, AttemptResult] | None:
|
|
900
|
+
"""The gate verdict a park carried (written by `_park_run`), or None."""
|
|
901
|
+
j = (record.stage or {}).get("judged")
|
|
902
|
+
if not isinstance(j, dict) or not j.get("sha"):
|
|
903
|
+
return None
|
|
904
|
+
|
|
905
|
+
def num(v: object) -> float | None:
|
|
906
|
+
return float(v) if isinstance(v, int | float) and not isinstance(v, bool) else None
|
|
907
|
+
|
|
908
|
+
return (
|
|
909
|
+
str(j["sha"]),
|
|
910
|
+
AttemptResult(
|
|
911
|
+
outcome=str(j.get("outcome") or "no-improvement"),
|
|
912
|
+
baseline=num(j.get("baseline")),
|
|
913
|
+
candidate=num(j.get("candidate")),
|
|
914
|
+
note=str(j.get("note") or ""),
|
|
915
|
+
),
|
|
916
|
+
)
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def _stage_syscall_launches(record: RunRecord) -> tuple:
|
|
920
|
+
"""The park's launches as `Launch` values (command elided: they ran)."""
|
|
921
|
+
from outerloop.syscall import Launch
|
|
922
|
+
|
|
923
|
+
return tuple(
|
|
924
|
+
Launch(
|
|
925
|
+
name=str(item.get("name", "")),
|
|
926
|
+
command="(ran)",
|
|
927
|
+
minutes=int(item.get("minutes") or 1),
|
|
928
|
+
artifacts=tuple(str(a) for a in item.get("artifacts", [])),
|
|
929
|
+
array=int(item.get("array") or 1),
|
|
930
|
+
)
|
|
931
|
+
for item in _stage_launches(record)
|
|
932
|
+
)
|
|
933
|
+
|
|
934
|
+
|
|
935
|
+
def _stage_launch_job_ids(record: RunRecord) -> list[str]:
|
|
936
|
+
"""The park's launch jobs: `launch_afterany` when the park recorded it;
|
|
937
|
+
for an older author-sleep park every waited job was a launch; for an
|
|
938
|
+
older candidate park the gate's evals are mixed in, so none."""
|
|
939
|
+
stage = record.stage or {}
|
|
940
|
+
if "launch_afterany" in stage:
|
|
941
|
+
return afterany_ids(str(stage.get("launch_afterany") or ""))
|
|
942
|
+
if stage.get("phase") == "author-sleep":
|
|
943
|
+
return afterany_ids(str(stage.get("afterany") or ""))
|
|
944
|
+
return []
|
|
945
|
+
|
|
946
|
+
|
|
947
|
+
def _reconcile_launch_hours(
|
|
948
|
+
record: RunRecord, dispatch: DispatchSettings, gpus: int, launches: tuple
|
|
949
|
+
) -> float:
|
|
950
|
+
"""The run's GPU-hours after handing back the unused walltime of the
|
|
951
|
+
park's launch jobs — once: the stage remembers the refund, so a wake
|
|
952
|
+
that follows a gate decision on the same park does not refund twice.
|
|
953
|
+
Returns the (possibly corrected) `gpu_hours_used`."""
|
|
954
|
+
stage = record.stage or {}
|
|
955
|
+
used = float(stage.get("gpu_hours_used", 0.0)) # type: ignore[arg-type]
|
|
956
|
+
if not gpus or stage.get("launch_hours_refunded"):
|
|
957
|
+
return used
|
|
958
|
+
refund = _launch_refund(dispatch, launches, _stage_launch_job_ids(record), gpus)
|
|
959
|
+
if refund > 0:
|
|
960
|
+
log.info("%s: refunding %.2f GPU-hours of unused launch walltime", record.run_id, refund)
|
|
961
|
+
used = max(0.0, used - refund)
|
|
962
|
+
stage["gpu_hours_used"] = used
|
|
963
|
+
stage["launch_hours_refunded"] = True
|
|
964
|
+
return used
|
|
965
|
+
|
|
966
|
+
|
|
967
|
+
def _launch_refund(
|
|
968
|
+
dispatch: DispatchSettings, launches: tuple, job_ids: list[str], gpus: int
|
|
969
|
+
) -> float:
|
|
970
|
+
"""The unused walltime of a park's launch jobs, in GPU-hours, or 0 when
|
|
971
|
+
the compute cannot say how long they ran (nothing is refunded on a
|
|
972
|
+
guess)."""
|
|
973
|
+
from outerloop.syscall import launch_hours_refund
|
|
974
|
+
|
|
975
|
+
query = getattr(dispatch.compute, "elapsed_seconds", None)
|
|
976
|
+
if query is None or not job_ids:
|
|
977
|
+
return 0.0
|
|
978
|
+
try:
|
|
979
|
+
elapsed = [query(jid) for jid in job_ids]
|
|
980
|
+
except Exception as exc:
|
|
981
|
+
log.warning("launch walltime unknown (%s: %s); nothing refunded", type(exc).__name__, exc)
|
|
982
|
+
return 0.0
|
|
983
|
+
return launch_hours_refund(launches, elapsed, gpus=gpus)
|
|
984
|
+
|
|
985
|
+
|
|
986
|
+
RESEARCH_LOG_BRANCH = "research-log"
|
|
987
|
+
MAX_ARCHIVED_REPORTS = 30 # materialized for the session to read; newest first
|
|
988
|
+
MAX_ARCHIVED_REPORT_CHARS = 100_000 # per report; branch content is remote-controlled
|
|
989
|
+
|
|
990
|
+
|
|
991
|
+
def _fetch_research_reports(ws: Workspace, count: int) -> list[tuple[str, str]]:
|
|
992
|
+
"""The newest `count` reports from the target's research-log branch, as
|
|
993
|
+
(name, text), newest first — the shared memory of every attempt on this
|
|
994
|
+
target, wherever it ran. Fail-soft: a target with no research log yet
|
|
995
|
+
(or an unreachable remote) is an empty memory, never a dead attempt."""
|
|
996
|
+
try:
|
|
997
|
+
ws.fetch_branch(RESEARCH_LOG_BRANCH)
|
|
998
|
+
listing = ws.git("ls-tree", "-r", "--name-only", "FETCH_HEAD", "reports/")
|
|
999
|
+
# only direct children (the publisher's layout): a nested path would
|
|
1000
|
+
# flatten to a basename that overwrites another archived report
|
|
1001
|
+
names = [
|
|
1002
|
+
line.strip()
|
|
1003
|
+
for line in listing.splitlines()
|
|
1004
|
+
if line.strip().endswith(".md") and line.strip().count("/") == 1
|
|
1005
|
+
]
|
|
1006
|
+
# report files are dated (reports/<YYYY-MM-DD>-<run_id>.md): the name
|
|
1007
|
+
# sorts by day; same-day order is arbitrary and does not matter
|
|
1008
|
+
out: list[tuple[str, str]] = []
|
|
1009
|
+
for name in sorted(names, reverse=True):
|
|
1010
|
+
if len(out) >= count:
|
|
1011
|
+
break
|
|
1012
|
+
# size BEFORE content: `git show` would load the whole blob, and
|
|
1013
|
+
# the branch's content is remote-controlled
|
|
1014
|
+
if int(ws.git("cat-file", "-s", f"FETCH_HEAD:{name}").strip()) > (
|
|
1015
|
+
MAX_ARCHIVED_REPORT_CHARS
|
|
1016
|
+
):
|
|
1017
|
+
log.info("research report %s exceeds the size cap; skipped", name)
|
|
1018
|
+
continue
|
|
1019
|
+
out.append((Path(name).name, ws.git("show", f"FETCH_HEAD:{name}")))
|
|
1020
|
+
return out
|
|
1021
|
+
except Exception as exc:
|
|
1022
|
+
log.info("research log unavailable (%s: %s); starting without it", type(exc).__name__, exc)
|
|
1023
|
+
return []
|
|
1024
|
+
|
|
1025
|
+
|
|
1026
|
+
def _exclude_merge_artifacts(workspace: Path) -> None:
|
|
1027
|
+
"""Ignore *.orig and *.rej — git's merge/patch conflict backups, which
|
|
1028
|
+
should never be committed — via .git/info/exclude (repo-local, never a
|
|
1029
|
+
tracked edit). A line run merges main at start and the agent resolves
|
|
1030
|
+
conflicts as its first task; a leftover train.py.orig would otherwise
|
|
1031
|
+
read as an out-of-scope edit and abort the run at launch. Idempotent,
|
|
1032
|
+
and effective for the `git add -A` behind changed-paths and every seal."""
|
|
1033
|
+
exclude = workspace / ".git" / "info" / "exclude"
|
|
1034
|
+
wanted = ["*.orig", "*.rej"]
|
|
1035
|
+
try:
|
|
1036
|
+
existing = exclude.read_text()
|
|
1037
|
+
except OSError:
|
|
1038
|
+
existing = ""
|
|
1039
|
+
lines = existing.splitlines()
|
|
1040
|
+
missing = [p for p in wanted if p not in lines]
|
|
1041
|
+
if missing:
|
|
1042
|
+
exclude.parent.mkdir(parents=True, exist_ok=True)
|
|
1043
|
+
sep = "" if existing.endswith("\n") or not existing else "\n"
|
|
1044
|
+
exclude.write_text(existing + sep + "\n".join(missing) + "\n")
|
|
1045
|
+
|
|
1046
|
+
|
|
1047
|
+
def _reset_instruction_files(ws: Workspace, workspace: Path, base_ref: str) -> None:
|
|
1048
|
+
"""Make the checkout's instruction-bearing files EQUAL the base branch's
|
|
1049
|
+
reviewed versions (review_agent.INSTRUCTION_FILES owns the list): a line
|
|
1050
|
+
is an author-written tree, and must never instruct its own successor
|
|
1051
|
+
sessions — or a sibling's (docs/design/research-lines.md)."""
|
|
1052
|
+
from outerloop.review_agent import INSTRUCTION_FILES
|
|
1053
|
+
|
|
1054
|
+
# bottom-up so a removed directory doesn't orphan paths found beneath it
|
|
1055
|
+
for path in sorted(workspace.rglob("*"), key=lambda p: len(p.parts), reverse=True):
|
|
1056
|
+
if ".git" in path.parts or path.name not in INSTRUCTION_FILES:
|
|
1057
|
+
continue
|
|
1058
|
+
if path.is_dir() and not path.is_symlink():
|
|
1059
|
+
shutil.rmtree(path, ignore_errors=True)
|
|
1060
|
+
else:
|
|
1061
|
+
path.unlink(missing_ok=True)
|
|
1062
|
+
base_paths = [
|
|
1063
|
+
p
|
|
1064
|
+
for p in ws.git("ls-tree", "-r", "--name-only", base_ref).splitlines()
|
|
1065
|
+
if any(part in INSTRUCTION_FILES for part in Path(p).parts)
|
|
1066
|
+
]
|
|
1067
|
+
if base_paths:
|
|
1068
|
+
ws.git("checkout", base_ref, "--", *base_paths)
|
|
1069
|
+
|
|
1070
|
+
|
|
1071
|
+
# The agent's memory on its line (docs/design/research-lines.md): the bounded
|
|
1072
|
+
# index at the branch root plus the topic-file folder. They ride the NOTEBOOK
|
|
1073
|
+
# seal (the line branch is exactly where they live) and are excluded from
|
|
1074
|
+
# every MEASURABLE seal and from changed-path accounting — never a scope
|
|
1075
|
+
# violation, never claimable work, never part of a main-PR candidate.
|
|
1076
|
+
LINE_MEMORY_PATHS = ("AGENT_MEMORY.md", "agent_memory")
|
|
1077
|
+
|
|
1078
|
+
|
|
1079
|
+
def _is_line_memory(path: str) -> bool:
|
|
1080
|
+
return path in LINE_MEMORY_PATHS or path.startswith("agent_memory/")
|
|
1081
|
+
|
|
1082
|
+
|
|
1083
|
+
def _without_line_memory(paths: Iterable[str], line: str) -> list[str]:
|
|
1084
|
+
"""The run's OWN changes: with a line active, its memory paths are dropped.
|
|
1085
|
+
Every sealed tree excludes them by construction, and the run's base is the
|
|
1086
|
+
line tip that carries them — so a base..candidate diff lists them as
|
|
1087
|
+
deletions that are not the run's change (live: a dispatched wake on
|
|
1088
|
+
gpt-speedrun read eight memory files as out of scope and aborted a run
|
|
1089
|
+
whose paired eval had already finished)."""
|
|
1090
|
+
return [p for p in paths if not (line and _is_line_memory(p))]
|
|
1091
|
+
|
|
1092
|
+
|
|
1093
|
+
def _line_ref_for(bench: Benchmark | None, agent_id: str) -> str:
|
|
1094
|
+
"""The agent's line branch when the benchmark opted in, or the empty
|
|
1095
|
+
string when the feature is off. Wake paths recompute this from the
|
|
1096
|
+
record — a malformed agent id could never have created a line at run
|
|
1097
|
+
start, so empty (not an error) is right there too."""
|
|
1098
|
+
if bench is None or not bench.lines or not agent_id:
|
|
1099
|
+
return ""
|
|
1100
|
+
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", agent_id):
|
|
1101
|
+
return ""
|
|
1102
|
+
return f"agents/{agent_id}"
|
|
1103
|
+
|
|
1104
|
+
|
|
1105
|
+
def _push_line_snapshot(
|
|
1106
|
+
ws: Workspace, line_ref: str, run_id: str, outcome: str, secrets: tuple[str, ...] = ()
|
|
1107
|
+
) -> None:
|
|
1108
|
+
"""Publish the session's final tree to the agent's line as a sealed
|
|
1109
|
+
snapshot commit — every terminal path, any outcome
|
|
1110
|
+
(docs/design/research-lines.md). The seal parents on the LOCAL line ref
|
|
1111
|
+
and advances it, so sequential terminals within one run chain as
|
|
1112
|
+
fast-forwards; one slot never runs twice concurrently, so the remote
|
|
1113
|
+
cannot have moved under us. Best-effort throughout: the notebook never
|
|
1114
|
+
changes a run's outcome. An unchanged tree is not pushed (the run-start
|
|
1115
|
+
push already holds it)."""
|
|
1116
|
+
if not line_ref:
|
|
1117
|
+
return
|
|
1118
|
+
|
|
1119
|
+
def _seal_and_push() -> None:
|
|
1120
|
+
# raises if the ref is absent (e.g. a park that predates the line
|
|
1121
|
+
# feature) or the session altered .git (every ws.git call checks) —
|
|
1122
|
+
# _best_effort turns either into a logged skip
|
|
1123
|
+
local = ws.git("rev-parse", f"refs/heads/{line_ref}").strip()
|
|
1124
|
+
memory = tuple(p for p in LINE_MEMORY_PATHS if (Path(ws.root) / p).exists())
|
|
1125
|
+
last_exc: Exception | None = None
|
|
1126
|
+
# the line commit this workspace's untouched files currently match:
|
|
1127
|
+
# the local ref at first, then each remote head reconciled into it
|
|
1128
|
+
# (so a retry does not mistake copied-in files for this run's edits)
|
|
1129
|
+
fork = local
|
|
1130
|
+
for _ in range(3):
|
|
1131
|
+
parent = fork
|
|
1132
|
+
# A park frees the slot, so a newer run on the same line can end
|
|
1133
|
+
# (and push) while this one is parked: the remote line then sits
|
|
1134
|
+
# past our local ref, and a seal parented on the stale ref would be
|
|
1135
|
+
# refused as a non-fast-forward (gpt-speedrun, 2026-09-03: agent-01's
|
|
1136
|
+
# winning run left no snapshot this way, and its line fell behind
|
|
1137
|
+
# main). Parent on the remote head whenever our ref is an ancestor
|
|
1138
|
+
# of it, keeping the files that head added since; offline, seal on
|
|
1139
|
+
# the local ref as before.
|
|
1140
|
+
try:
|
|
1141
|
+
ws.fetch_origin()
|
|
1142
|
+
remote = ws.git("rev-parse", f"refs/remotes/origin/{line_ref}").strip()
|
|
1143
|
+
if remote != fork:
|
|
1144
|
+
ws.git("merge-base", "--is-ancestor", fork, remote) # raises when not
|
|
1145
|
+
_reconcile_with_remote(ws, fork, remote)
|
|
1146
|
+
fork = parent = remote
|
|
1147
|
+
except Exception as exc:
|
|
1148
|
+
log.info("line %s: sealing on the local ref (%s)", line_ref, type(exc).__name__)
|
|
1149
|
+
snap = snapshot_tree(ws, parent, force=memory)
|
|
1150
|
+
try:
|
|
1151
|
+
# seal only when the tree moved past the parent; the PUSH runs
|
|
1152
|
+
# either way — a session that COMMITTED its work advanced the
|
|
1153
|
+
# local ref without dirtying the tree, and that commit must
|
|
1154
|
+
# still reach the remote (an already-current ref push is a no-op)
|
|
1155
|
+
sealed = parent
|
|
1156
|
+
if snap.tree != ws.git("rev-parse", f"{parent}^{{tree}}").strip():
|
|
1157
|
+
sealed = ws.git(
|
|
1158
|
+
"-c",
|
|
1159
|
+
"user.name=autoresearch",
|
|
1160
|
+
"-c",
|
|
1161
|
+
"user.email=autoresearch@localhost",
|
|
1162
|
+
"commit-tree",
|
|
1163
|
+
snap.tree,
|
|
1164
|
+
"-p",
|
|
1165
|
+
parent,
|
|
1166
|
+
"-m",
|
|
1167
|
+
f"line snapshot: {run_id} ({outcome})",
|
|
1168
|
+
).strip()
|
|
1169
|
+
ws.git("update-ref", f"refs/heads/{line_ref}", sealed)
|
|
1170
|
+
try:
|
|
1171
|
+
ws.push(line_ref)
|
|
1172
|
+
return
|
|
1173
|
+
except Exception as exc:
|
|
1174
|
+
# another run pushed between our fetch and this push:
|
|
1175
|
+
# re-read the line and seal again on its new head
|
|
1176
|
+
last_exc = exc
|
|
1177
|
+
log.info(
|
|
1178
|
+
"line %s: push refused, re-sealing on the moved line (%s)",
|
|
1179
|
+
line_ref,
|
|
1180
|
+
type(exc).__name__,
|
|
1181
|
+
)
|
|
1182
|
+
finally:
|
|
1183
|
+
drop_snapshot(ws, snap)
|
|
1184
|
+
assert last_exc is not None
|
|
1185
|
+
raise last_exc
|
|
1186
|
+
|
|
1187
|
+
_best_effort(f"line push ({outcome})", _seal_and_push, secrets)
|
|
1188
|
+
|
|
1189
|
+
|
|
1190
|
+
def _reconcile_with_remote(ws: Workspace, old: str, new: str) -> None:
|
|
1191
|
+
"""The seal is built from THIS workspace, which forked the line at `old`;
|
|
1192
|
+
another run has since moved the line to `new`. For every path that run
|
|
1193
|
+
changed, take its state when this workspace left the path untouched
|
|
1194
|
+
since `old` (added: materialize, modified: update, deleted: remove); a
|
|
1195
|
+
path this run touched keeps this run's version, as the line always did.
|
|
1196
|
+
Without this, a seal would silently undo the other run's work on files
|
|
1197
|
+
this run never looked at."""
|
|
1198
|
+
entries = [
|
|
1199
|
+
e for e in ws.git("diff", "--name-status", "--no-renames", "-z", old, new).split("\0") if e
|
|
1200
|
+
]
|
|
1201
|
+
remote_changes = list(zip(entries[0::2], entries[1::2], strict=True))
|
|
1202
|
+
if not remote_changes:
|
|
1203
|
+
return
|
|
1204
|
+
touched = {
|
|
1205
|
+
p for p in ws.git("diff", "--name-only", "-z", old).split("\0") if p
|
|
1206
|
+
} # tracked paths this run changed or deleted
|
|
1207
|
+
touched |= {
|
|
1208
|
+
p for p in ws.git("ls-files", "--others", "--exclude-standard", "-z").split("\0") if p
|
|
1209
|
+
} # and the files it created
|
|
1210
|
+
for status, path in remote_changes:
|
|
1211
|
+
if path in touched:
|
|
1212
|
+
continue
|
|
1213
|
+
if status.startswith("D"):
|
|
1214
|
+
target = Path(ws.root) / path
|
|
1215
|
+
if target.is_file() or target.is_symlink():
|
|
1216
|
+
target.unlink()
|
|
1217
|
+
else:
|
|
1218
|
+
ws.git("checkout", new, "--", path)
|
|
1219
|
+
|
|
1220
|
+
|
|
1221
|
+
def _checkout_line(ws: Workspace, workspace: Path, agent_id: str, base_branch: str) -> str:
|
|
1222
|
+
"""Check out the agent's research line: the persistent branch
|
|
1223
|
+
`agents/<agent-id>`, created from the base branch when absent, with the
|
|
1224
|
+
base branch merged in when it exists — a conflicted merge is left in the
|
|
1225
|
+
tree as the session's first task. Instruction-bearing files are reset to
|
|
1226
|
+
the base branch's reviewed versions and the hygiene is committed (never
|
|
1227
|
+
left to masquerade as agent edits). Returns the line ref name; raises on
|
|
1228
|
+
anything unrecoverable (the caller falls back to the base branch)."""
|
|
1229
|
+
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", agent_id):
|
|
1230
|
+
raise ValueError(f"agent id {agent_id!r} cannot shape a line ref")
|
|
1231
|
+
line = f"agents/{agent_id}"
|
|
1232
|
+
base_ref = f"origin/{base_branch}"
|
|
1233
|
+
if not ws.git("branch", "--list", "-r", f"origin/{line}").strip():
|
|
1234
|
+
ws.git("checkout", "-q", "-B", line, base_ref)
|
|
1235
|
+
ws.push(line) # the line is durable from its first run
|
|
1236
|
+
return line
|
|
1237
|
+
ws.git("checkout", "-q", "-B", line, f"origin/{line}")
|
|
1238
|
+
conflicted = False
|
|
1239
|
+
try:
|
|
1240
|
+
ws.git(
|
|
1241
|
+
"-c",
|
|
1242
|
+
"user.name=autoresearch",
|
|
1243
|
+
"-c",
|
|
1244
|
+
"user.email=autoresearch@localhost",
|
|
1245
|
+
"merge",
|
|
1246
|
+
"--no-edit",
|
|
1247
|
+
base_ref,
|
|
1248
|
+
)
|
|
1249
|
+
except Exception:
|
|
1250
|
+
conflicted = True
|
|
1251
|
+
log.info(
|
|
1252
|
+
"line %s: merging %s conflicts; left as the session's first task", line, base_branch
|
|
1253
|
+
)
|
|
1254
|
+
_reset_instruction_files(ws, workspace, base_ref)
|
|
1255
|
+
if not conflicted:
|
|
1256
|
+
ws.git("add", "-A")
|
|
1257
|
+
if ws.git("status", "--porcelain").strip():
|
|
1258
|
+
ws.git(
|
|
1259
|
+
"-c",
|
|
1260
|
+
"user.name=autoresearch",
|
|
1261
|
+
"-c",
|
|
1262
|
+
"user.email=autoresearch@localhost",
|
|
1263
|
+
"commit",
|
|
1264
|
+
"-q",
|
|
1265
|
+
"-m",
|
|
1266
|
+
f"line hygiene: instruction files reset to {base_branch}",
|
|
1267
|
+
)
|
|
1268
|
+
# Run-START persistence: the branch exists on the remote from its
|
|
1269
|
+
# first run, and the merge-main + hygiene state survives a crashed
|
|
1270
|
+
# run. One slot never runs twice concurrently, so this is a fast-
|
|
1271
|
+
# forward. The run-END push of the sealed session tree is the next
|
|
1272
|
+
# phase PR (it requires all-terminal sealing; the push must publish
|
|
1273
|
+
# a sealed sha, never invent a commit). A conflicted merge is not
|
|
1274
|
+
# pushed: the conflict is session work, not line state.
|
|
1275
|
+
ws.push(line)
|
|
1276
|
+
return line
|
|
1277
|
+
|
|
1278
|
+
|
|
1279
|
+
def _paths_changed_from_base(
|
|
1280
|
+
ws: Workspace, base: str, exclude_memory: bool, fallback: str = "HEAD"
|
|
1281
|
+
) -> list[str]:
|
|
1282
|
+
"""The paths the session changed, measured against the BASE BRANCH head
|
|
1283
|
+
(`base`, a commit-ish such as refs/remotes/origin/main), not HEAD. On a
|
|
1284
|
+
research line HEAD can be behind main: a run starts by merging main in,
|
|
1285
|
+
and when that merge conflicts it stays uncommitted, so the files git
|
|
1286
|
+
auto-merged (the project's BENCHMARKS.md and results/leader.json) sit
|
|
1287
|
+
staged against the stale HEAD while being identical to main. Those are
|
|
1288
|
+
main's edits, not the agent's, and must not read as scope violations
|
|
1289
|
+
(gpt-speedrun, 2026-09-03: agent-01's first run after its own win ended
|
|
1290
|
+
scope-violation on exactly those two files). A path counts only when it
|
|
1291
|
+
moved against HEAD AND differs from the base; new and deleted files
|
|
1292
|
+
count. `fallback` is used when `base` does not resolve (no remote).
|
|
1293
|
+
`exclude_memory` drops the line's memory files whenever lines are active
|
|
1294
|
+
for the benchmark (a failed line checkout still keeps them out)."""
|
|
1295
|
+
try:
|
|
1296
|
+
base_commit = ws.git("rev-parse", "--verify", f"{base}^{{commit}}").strip()
|
|
1297
|
+
except Exception:
|
|
1298
|
+
base_commit = fallback
|
|
1299
|
+
ws.git("add", "-A")
|
|
1300
|
+
try:
|
|
1301
|
+
staged = ws.staged_paths()
|
|
1302
|
+
if not staged:
|
|
1303
|
+
return []
|
|
1304
|
+
# index vs base, restricted to what moved against HEAD: a path identical
|
|
1305
|
+
# to base drops out, a new or deleted file still counts
|
|
1306
|
+
differs = {
|
|
1307
|
+
entry
|
|
1308
|
+
for entry in ws.git(
|
|
1309
|
+
"diff", "--cached", "--name-only", "-z", base_commit, "--", *staged
|
|
1310
|
+
).split("\0")
|
|
1311
|
+
if entry
|
|
1312
|
+
}
|
|
1313
|
+
finally:
|
|
1314
|
+
ws.git("reset")
|
|
1315
|
+
kept = [p for p in staged if p in differs]
|
|
1316
|
+
return [p for p in kept if not (exclude_memory and _is_line_memory(p))]
|
|
1317
|
+
|
|
1318
|
+
|
|
1319
|
+
def _sibling_entries(ws: Workspace, self_agent: str) -> list[dict]:
|
|
1320
|
+
"""The other agents' live directions from the research-log's
|
|
1321
|
+
status.json (already fetched: the SAME FETCH_HEAD the reports came
|
|
1322
|
+
from). Size-checked BEFORE show like the report blobs, entries and
|
|
1323
|
+
fields bounded — the branch is bot-written but never trusted with
|
|
1324
|
+
unbounded memory. Any failure means no siblings known, never a crash."""
|
|
1325
|
+
try:
|
|
1326
|
+
blob = "FETCH_HEAD:climb/status.json"
|
|
1327
|
+
if int(ws.git("cat-file", "-s", blob).strip()) > 1_000_000:
|
|
1328
|
+
raise ValueError("status snapshot oversized; skipped")
|
|
1329
|
+
fleet = json.loads(ws.git("show", blob))
|
|
1330
|
+
return [
|
|
1331
|
+
{
|
|
1332
|
+
"agent": str(r.get("agent", ""))[:64],
|
|
1333
|
+
"state": str(r.get("state", ""))[:32],
|
|
1334
|
+
"phase": str(r.get("phase", ""))[:32],
|
|
1335
|
+
"direction": str(r.get("direction", ""))[:160],
|
|
1336
|
+
}
|
|
1337
|
+
for r in fleet.get("runs", [])[:64]
|
|
1338
|
+
if isinstance(r, dict) and r.get("agent") != self_agent
|
|
1339
|
+
]
|
|
1340
|
+
except Exception as exc:
|
|
1341
|
+
log.info("no sibling snapshot for this session (%s)", exc)
|
|
1342
|
+
return []
|
|
1343
|
+
|
|
1344
|
+
|
|
1345
|
+
def _install_report_archive(workspace: Path, reports: list[tuple[str, str]]) -> None:
|
|
1346
|
+
"""Materialize the fetched reports under the kernel-owned channel
|
|
1347
|
+
(`.outerloop/reports/`) so the session can read and search the full
|
|
1348
|
+
texts with its own tools; the brief inlines only the newest few."""
|
|
1349
|
+
dest = workspace / channel_dir(workspace) / "reports"
|
|
1350
|
+
dest.mkdir(parents=True, exist_ok=True)
|
|
1351
|
+
for name, text in reports:
|
|
1352
|
+
if Path(name).name != name: # branch content is remote-controlled
|
|
1353
|
+
continue
|
|
1354
|
+
(dest / name).write_text(text)
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def _stage_launches(record: RunRecord) -> list[dict]:
|
|
1358
|
+
"""The persisted launch descriptors of an author-sleep stage (name +
|
|
1359
|
+
artifacts), tolerating a malformed entry by skipping it (the job dirs are
|
|
1360
|
+
keyed by name; a nameless entry has nothing to gather)."""
|
|
1361
|
+
raw = record.stage.get("syscall_launches", [])
|
|
1362
|
+
if not isinstance(raw, list):
|
|
1363
|
+
return []
|
|
1364
|
+
return [item for item in raw if isinstance(item, dict) and item.get("name")]
|
|
1365
|
+
|
|
1366
|
+
|
|
1367
|
+
def _utc_date(now: float) -> str:
|
|
1368
|
+
from datetime import UTC, datetime
|
|
1369
|
+
|
|
1370
|
+
return datetime.fromtimestamp(now, UTC).strftime("%Y-%m-%d")
|
|
1371
|
+
|
|
1372
|
+
|
|
1373
|
+
def _is_git_tamper(exc: GitError) -> bool:
|
|
1374
|
+
"""True when a GitError means the workspace's object store or refs are
|
|
1375
|
+
damaged or session-altered — as opposed to an ordinary git failure (a push
|
|
1376
|
+
conflict, a fetch outage). The guard raises its own `_altered(...)` for the
|
|
1377
|
+
states it models; a raw object-store error ("could not parse object", "bad
|
|
1378
|
+
tree object", a corrupt or unreadable pack) is the same damage reaching a
|
|
1379
|
+
git command in the gap between the guard's check and the command. Either
|
|
1380
|
+
way the wake cannot be trusted and must END, not crash."""
|
|
1381
|
+
msg = str(exc).lower()
|
|
1382
|
+
return (
|
|
1383
|
+
"altered by the session" in msg # the guard's own _altered signal
|
|
1384
|
+
or "could not parse object" in msg
|
|
1385
|
+
or "bad tree object" in msg
|
|
1386
|
+
or "bad object" in msg
|
|
1387
|
+
or "unable to read tree" in msg
|
|
1388
|
+
or "data stream error" in msg # inflate: a corrupt pack
|
|
1389
|
+
or "is corrupt" in msg # "packed object ... is corrupt", loose too
|
|
1390
|
+
or ("object file" in msg and "empty" in msg)
|
|
1391
|
+
)
|
|
1392
|
+
|
|
1393
|
+
|
|
1394
|
+
def _end_refused_wake(
|
|
1395
|
+
run_root: Path, record: RunRecord, exc: Exception, now: float, secrets: tuple[str, ...]
|
|
1396
|
+
) -> AttemptOutcome:
|
|
1397
|
+
"""End a parked run whose workspace the wake refused (a session altered
|
|
1398
|
+
.git): ABORTED with the tampering as the note. The candidate snapshot's
|
|
1399
|
+
retaining ref lives inside that same, now untrusted, repository and is
|
|
1400
|
+
deliberately not touched: deleting it would mean writing through the
|
|
1401
|
+
very structure the guard refused (a symlinked refs dir carries the write
|
|
1402
|
+
elsewhere), and an ENDED workspace is inert — the ref keeps a commit
|
|
1403
|
+
alive only within a repository nothing will read again."""
|
|
1404
|
+
note = redact(str(exc), secrets)[:480]
|
|
1405
|
+
log.warning("wake refused for %s: %s", record.run_id, note)
|
|
1406
|
+
failed = _clear_stage(
|
|
1407
|
+
RunRecord(**{**record.__dict__, "state": ENDED, "ending": ABORTED, "ending_note": note})
|
|
1408
|
+
)
|
|
1409
|
+
_best_effort("ending record", lambda: save_record(run_root, failed, now), secrets)
|
|
1410
|
+
return AttemptOutcome(run_id=record.run_id, outcome="attempt-error")
|
|
1411
|
+
|
|
1412
|
+
|
|
1413
|
+
def resume_run(
|
|
1414
|
+
run_root: Path,
|
|
1415
|
+
run_id: str,
|
|
1416
|
+
*,
|
|
1417
|
+
dispatch: DispatchSettings,
|
|
1418
|
+
github: GitHubClient,
|
|
1419
|
+
bot_auth: TokenProvider,
|
|
1420
|
+
now: float,
|
|
1421
|
+
secrets: tuple[str, ...] = (),
|
|
1422
|
+
base_branch: str = "main",
|
|
1423
|
+
panel_lenses: tuple[PanelLens, ...] = (),
|
|
1424
|
+
harness: Harness | None = None,
|
|
1425
|
+
spec: RoleSpec | None = None,
|
|
1426
|
+
) -> AttemptOutcome:
|
|
1427
|
+
"""Wake a parked dispatched climb and re-enter its decision WITHOUT the
|
|
1428
|
+
session (`orchestrator.resume_attempt`), from the record `_park_run` wrote.
|
|
1429
|
+
The three exits:
|
|
1430
|
+
|
|
1431
|
+
* **re-park** — the wake dispatched a measure that is not done yet (the
|
|
1432
|
+
suite pairs an improving candidate fans out, "another round of
|
|
1433
|
+
experiments"): `resume_attempt` raises `RunParked`, and this re-persists
|
|
1434
|
+
the WAITING stage on the new afterany, keeping the same candidate
|
|
1435
|
+
snapshot;
|
|
1436
|
+
* **a negative terminal** (no-improvement / suite-regression / eval-error):
|
|
1437
|
+
drop the candidate snapshot and end the record;
|
|
1438
|
+
* **improved** — branch the SEALED `candidate_sha` (never the live tree,
|
|
1439
|
+
which may have drifted since the park; the diff was scope-checked so it
|
|
1440
|
+
carries only in-scope changes), layer the ledger update on top, push, and
|
|
1441
|
+
open the PR. A moved base is NOT merged and re-measured here
|
|
1442
|
+
(docs/design/research-loop.md): a stale PR is a re-wake, not an
|
|
1443
|
+
orchestrator auto-merge.
|
|
1444
|
+
"""
|
|
1445
|
+
run_dir = run_root / "runs" / run_id
|
|
1446
|
+
workspace = run_dir / "ws"
|
|
1447
|
+
record = load_record(run_root, run_id)
|
|
1448
|
+
stage = record.stage
|
|
1449
|
+
# Push to the CANONICAL target URL, never the workspace's remote.origin.url:
|
|
1450
|
+
# the session could have rewritten that config to exfil the bot token / code
|
|
1451
|
+
# to another remote. Passing `url` here means `Workspace.push` uses it
|
|
1452
|
+
# instead of reading `remote.origin.url`.
|
|
1453
|
+
ws = Workspace(root=workspace, auth=bot_auth, url=_target_clone_url(record.target))
|
|
1454
|
+
# A session reshaped .git (symlinked object store, gitdir file, FIFO) is
|
|
1455
|
+
# refused BEFORE anything writes through it: the exclude below opens
|
|
1456
|
+
# .git/info/exclude, and every ws.git call re-checks. The refusal ENDS
|
|
1457
|
+
# the parked run with the tampering as its note — the tree cannot be
|
|
1458
|
+
# trusted, and a record left waiting would only be re-woken into the
|
|
1459
|
+
# same refusal.
|
|
1460
|
+
try:
|
|
1461
|
+
ensure_regular_git_dir(workspace)
|
|
1462
|
+
except GitError as exc:
|
|
1463
|
+
return _end_refused_wake(run_root, record, exc, now, secrets)
|
|
1464
|
+
# Re-establish the merge-artifact exclude on the wake too: the workspace
|
|
1465
|
+
# persisted across the park, but a session could have removed the exclude,
|
|
1466
|
+
# and this wake's changed_paths / seal run `git add -A`. Idempotent.
|
|
1467
|
+
_exclude_merge_artifacts(workspace)
|
|
1468
|
+
# Refresh origin refs on EVERY wake: the clone's refs froze at run
|
|
1469
|
+
# start, and this is the one credential-free freshness point — the
|
|
1470
|
+
# kernel fetches (from the canonical URL, never the session-writable
|
|
1471
|
+
# remote config), the session only ever reads local refs. `sleep`
|
|
1472
|
+
# thereby doubles as the author's sync primitive. Best-effort: a fetch
|
|
1473
|
+
# outage must not cost the wake.
|
|
1474
|
+
try:
|
|
1475
|
+
ws.fetch_origin()
|
|
1476
|
+
except Exception as exc:
|
|
1477
|
+
log.warning("wake fetch failed for %s: %s", run_id, exc)
|
|
1478
|
+
|
|
1479
|
+
# Two park kinds reach the wake: a CANDIDATE park (the gate's measures were
|
|
1480
|
+
# dispatched) and an AUTHOR-SLEEP park (the author launched work and slept —
|
|
1481
|
+
# research-loop-buildout.md Phase A). Both carry a sealed sha. Anything else
|
|
1482
|
+
# is a stray record: guard rather than crash on `git diff base ""`.
|
|
1483
|
+
if stage.get("phase") not in ("candidate", "author-sleep") or not stage.get("candidate_sha"):
|
|
1484
|
+
raise EvalError(f"resume_run: run {run_id} is not a wakeable park (stage={stage!r})")
|
|
1485
|
+
|
|
1486
|
+
base_sha = str(stage["base_sha"])
|
|
1487
|
+
candidate_sha = str(stage["candidate_sha"])
|
|
1488
|
+
candidate_ref = str(stage["candidate_ref"])
|
|
1489
|
+
issue_number = record.issue_number
|
|
1490
|
+
# the run's target branch rides the stage, so a wake opens its PR against
|
|
1491
|
+
# the branch the ORIGINAL climb selected — not the CLI's default (the wake
|
|
1492
|
+
# job carries no --base-branch).
|
|
1493
|
+
base_branch = str(stage.get("base_branch") or base_branch)
|
|
1494
|
+
|
|
1495
|
+
# The contract gates scope and names the eval command, so read it from the
|
|
1496
|
+
# BASE commit (the tree the run started on), NOT the working tree the
|
|
1497
|
+
# session left dirty — a session that widened its own scope in
|
|
1498
|
+
# its contract must not have the wake gate on the doctored rules.
|
|
1499
|
+
contract_text = contract_at(ws, base_sha)
|
|
1500
|
+
contract = load_contract(contract_text, record.target)
|
|
1501
|
+
bench = _benchmark(contract, record.benchmark)
|
|
1502
|
+
config = RunConfig(target=record.target, benchmark=record.benchmark, agent_id=record.agent_id)
|
|
1503
|
+
eval_minutes = next(
|
|
1504
|
+
(b.eval_minutes for b in contract.benchmarks if b.name == record.benchmark), None
|
|
1505
|
+
)
|
|
1506
|
+
# a submitted park carries the author's declared eval walltime: the
|
|
1507
|
+
# wake's measurer (a re-dispatch) and deadline floor honor it
|
|
1508
|
+
declared = int(record.stage.get("eval_minutes", 0) or 0) # type: ignore[call-overload]
|
|
1509
|
+
if declared:
|
|
1510
|
+
eval_minutes = declared
|
|
1511
|
+
measurer = dispatch.measurer(
|
|
1512
|
+
run_dir, repo_root=workspace, eval_minutes=int(eval_minutes or 0), run_tag=run_id
|
|
1513
|
+
)
|
|
1514
|
+
# measured_paths from the COMMITTED base..candidate diff — the sealed
|
|
1515
|
+
# candidate, never `changed_paths()` on a live tree that may have drifted.
|
|
1516
|
+
# NUL-delimited (like Workspace.staged_paths) so a path with a space is one
|
|
1517
|
+
# entry, not two that could each slip past the scope check. The same
|
|
1518
|
+
# line-memory rule as the climb's changed_paths(): the base is the line
|
|
1519
|
+
# tip, the seal excluded the memory, the diff must not read it as a change.
|
|
1520
|
+
measured_paths = tuple(
|
|
1521
|
+
_without_line_memory(
|
|
1522
|
+
(
|
|
1523
|
+
p
|
|
1524
|
+
for p in ws.git("diff", "--name-only", "-z", base_sha, candidate_sha).split("\0")
|
|
1525
|
+
if p
|
|
1526
|
+
),
|
|
1527
|
+
_line_ref_for(bench, config.agent_id),
|
|
1528
|
+
)
|
|
1529
|
+
)
|
|
1530
|
+
seed = int(stage["seed"]) # type: ignore[call-overload]
|
|
1531
|
+
suite_seed = int(stage["suite_seed"]) # type: ignore[call-overload]
|
|
1532
|
+
panel_reads = int(stage.get("panel_reads", 0)) # type: ignore[call-overload]
|
|
1533
|
+
|
|
1534
|
+
if stage.get("phase") == "author-sleep":
|
|
1535
|
+
# The author slept on launches: deliver their results and RESUME the
|
|
1536
|
+
# same session through the climb's resume-entry. Every exit is a park
|
|
1537
|
+
# (slept again, or the gate dispatched its measures -> a candidate park
|
|
1538
|
+
# the NEXT wake decides through the path below) or a terminal ending —
|
|
1539
|
+
# never a publish, so the publish tail stays candidate-only.
|
|
1540
|
+
return _wake_author_sleep(
|
|
1541
|
+
run_root=run_root,
|
|
1542
|
+
run_id=run_id,
|
|
1543
|
+
record=record,
|
|
1544
|
+
ws=ws,
|
|
1545
|
+
workspace=workspace,
|
|
1546
|
+
run_dir=run_dir,
|
|
1547
|
+
dispatch=dispatch,
|
|
1548
|
+
github=github,
|
|
1549
|
+
now=now,
|
|
1550
|
+
secrets=secrets,
|
|
1551
|
+
base_branch=base_branch,
|
|
1552
|
+
base_sha=base_sha,
|
|
1553
|
+
sleep_ref=candidate_ref,
|
|
1554
|
+
contract_text=contract_text,
|
|
1555
|
+
contract=contract,
|
|
1556
|
+
bench=bench,
|
|
1557
|
+
config=config,
|
|
1558
|
+
measurer=measurer,
|
|
1559
|
+
harness=harness,
|
|
1560
|
+
spec=spec,
|
|
1561
|
+
panel_lenses=panel_lenses,
|
|
1562
|
+
issue_number=issue_number,
|
|
1563
|
+
eval_minutes=eval_minutes,
|
|
1564
|
+
)
|
|
1565
|
+
|
|
1566
|
+
# rebuild the session from what the park saved: the (redacted) write-up and
|
|
1567
|
+
# its real spend, so the report shows true cost/turns. It is never re-run.
|
|
1568
|
+
session = SessionResult(
|
|
1569
|
+
stop_reason="resumed",
|
|
1570
|
+
is_error=False,
|
|
1571
|
+
cost_usd=float(stage.get("session_cost_usd", 0.0)), # type: ignore[arg-type]
|
|
1572
|
+
num_turns=int(stage.get("session_turns", 0)), # type: ignore[call-overload]
|
|
1573
|
+
session_id=record.resume_session_id,
|
|
1574
|
+
final_text=str(stage.get("report", "")),
|
|
1575
|
+
transcript_path="",
|
|
1576
|
+
)
|
|
1577
|
+
try:
|
|
1578
|
+
result = resume_attempt(
|
|
1579
|
+
contract,
|
|
1580
|
+
bench,
|
|
1581
|
+
base_sha=base_sha,
|
|
1582
|
+
candidate_sha=candidate_sha,
|
|
1583
|
+
seed=seed,
|
|
1584
|
+
suite_seed=suite_seed,
|
|
1585
|
+
measured_paths=measured_paths,
|
|
1586
|
+
session=session,
|
|
1587
|
+
measurer=measurer,
|
|
1588
|
+
min_relative_improvement=config.min_relative_improvement,
|
|
1589
|
+
)
|
|
1590
|
+
except RunParked as parked:
|
|
1591
|
+
# another measure this wake dispatched is not done — re-park on the new
|
|
1592
|
+
# afterany, keeping the SAME candidate snapshot the next wake reads.
|
|
1593
|
+
# PROGRESS only if this wake dispatched a NEW job set (e.g. the candidate
|
|
1594
|
+
# resolved and the suite pairs fanned out); a blind re-park (empty
|
|
1595
|
+
# afterany) or the same jobs still pending is NO progress, so the stuck
|
|
1596
|
+
# cap must keep counting.
|
|
1597
|
+
if stage.get("submitted"):
|
|
1598
|
+
# No author was woken yet, so a SUBMITTED park's re-park (the suite
|
|
1599
|
+
# fanned out) still owes the author the gate+panel results and its
|
|
1600
|
+
# sibling launches' results — carry the submit context forward, or
|
|
1601
|
+
# the next wake drafts instead of waking the author and the launch
|
|
1602
|
+
# descriptors are lost. Commands/minutes are spent
|
|
1603
|
+
# history; the wake needs only names + artifacts (as persisted).
|
|
1604
|
+
from outerloop.syscall import Launch as _Launch
|
|
1605
|
+
from outerloop.syscall import SyscallRequest as _SyscallRequest
|
|
1606
|
+
|
|
1607
|
+
parked.submitted = True
|
|
1608
|
+
parked.launches_used = int(stage.get("launches_used", 0)) # type: ignore[call-overload]
|
|
1609
|
+
parked.sleeps_used = int(stage.get("sleeps_used", 0)) # type: ignore[call-overload]
|
|
1610
|
+
parked.gpu_hours_used = float(stage.get("gpu_hours_used", 0.0)) # type: ignore[arg-type]
|
|
1611
|
+
parked.eval_minutes = int(stage.get("eval_minutes", 0) or 0) or None # type: ignore[call-overload]
|
|
1612
|
+
parked.judged = parked.judged or _stage_judged(record)
|
|
1613
|
+
parked.launch_afterany = parked.launch_afterany or str(stage.get("launch_afterany", ""))
|
|
1614
|
+
if parked.syscall is None:
|
|
1615
|
+
parked.syscall = _SyscallRequest(
|
|
1616
|
+
launches=tuple(
|
|
1617
|
+
_Launch(
|
|
1618
|
+
name=str(item.get("name", "")),
|
|
1619
|
+
command="(ran)",
|
|
1620
|
+
# the persisted walltime, so the re-park's deadline
|
|
1621
|
+
# floor still covers the longest launch
|
|
1622
|
+
minutes=int(item.get("minutes") or 1),
|
|
1623
|
+
artifacts=tuple(str(a) for a in item.get("artifacts", [])),
|
|
1624
|
+
array=int(item.get("array") or 1),
|
|
1625
|
+
)
|
|
1626
|
+
for item in _stage_launches(record)
|
|
1627
|
+
),
|
|
1628
|
+
note=str(stage.get("syscall_note", "")),
|
|
1629
|
+
submit=True,
|
|
1630
|
+
)
|
|
1631
|
+
old_afterany = str(record.stage.get("afterany", ""))
|
|
1632
|
+
made_progress = bool(parked.afterany) and parked.afterany != old_afterany
|
|
1633
|
+
_park_run(
|
|
1634
|
+
run_root,
|
|
1635
|
+
record,
|
|
1636
|
+
parked,
|
|
1637
|
+
candidate_ref,
|
|
1638
|
+
eval_minutes,
|
|
1639
|
+
now,
|
|
1640
|
+
secrets,
|
|
1641
|
+
dispatch=dispatch,
|
|
1642
|
+
keep_wake_attempts=not made_progress,
|
|
1643
|
+
base_branch=base_branch,
|
|
1644
|
+
panel_reads=panel_reads,
|
|
1645
|
+
)
|
|
1646
|
+
return AttemptOutcome(run_id=run_id, outcome="parked")
|
|
1647
|
+
|
|
1648
|
+
# A SUBMITTED park (the author's `submit` syscall, buildout Phase B): gate
|
|
1649
|
+
# and panel results go back to the AUTHOR — it revises and resubmits, runs
|
|
1650
|
+
# more experiments, or concludes — instead of being decided by policy here.
|
|
1651
|
+
# Falls back to the plain-finish behavior (negative terminal / draft PR)
|
|
1652
|
+
# when the session cannot be resumed.
|
|
1653
|
+
submitted_park = bool(stage.get("submitted"))
|
|
1654
|
+
author_resumable = (
|
|
1655
|
+
harness is not None
|
|
1656
|
+
and spec is not None
|
|
1657
|
+
and bool(record.resume_session_id)
|
|
1658
|
+
and getattr(harness, "supports_resume", True)
|
|
1659
|
+
)
|
|
1660
|
+
|
|
1661
|
+
# the park's sibling launches are done too: settle their charge before
|
|
1662
|
+
# any path — publish or hand back to the author — reads the budget
|
|
1663
|
+
if _stage_launches(record):
|
|
1664
|
+
_reconcile_launch_hours(record, dispatch, bench.gpus, _stage_syscall_launches(record))
|
|
1665
|
+
|
|
1666
|
+
def _wake_author(
|
|
1667
|
+
extra_update: str, judged: tuple[str, AttemptResult] | None = None
|
|
1668
|
+
) -> AttemptOutcome:
|
|
1669
|
+
# resume the submitted park's author with the gate/panel feedback
|
|
1670
|
+
# leading its wake text; the candidate ref is this park's held snapshot
|
|
1671
|
+
return _wake_author_sleep(
|
|
1672
|
+
run_root=run_root,
|
|
1673
|
+
run_id=run_id,
|
|
1674
|
+
record=record,
|
|
1675
|
+
ws=ws,
|
|
1676
|
+
workspace=workspace,
|
|
1677
|
+
run_dir=run_dir,
|
|
1678
|
+
dispatch=dispatch,
|
|
1679
|
+
github=github,
|
|
1680
|
+
now=now,
|
|
1681
|
+
secrets=secrets,
|
|
1682
|
+
base_branch=base_branch,
|
|
1683
|
+
base_sha=base_sha,
|
|
1684
|
+
sleep_ref=candidate_ref,
|
|
1685
|
+
contract_text=contract_text,
|
|
1686
|
+
contract=contract,
|
|
1687
|
+
bench=bench,
|
|
1688
|
+
config=config,
|
|
1689
|
+
measurer=measurer,
|
|
1690
|
+
harness=harness,
|
|
1691
|
+
spec=spec,
|
|
1692
|
+
panel_lenses=panel_lenses,
|
|
1693
|
+
issue_number=issue_number,
|
|
1694
|
+
eval_minutes=eval_minutes,
|
|
1695
|
+
extra_update=extra_update,
|
|
1696
|
+
judged=judged,
|
|
1697
|
+
)
|
|
1698
|
+
|
|
1699
|
+
if (
|
|
1700
|
+
submitted_park
|
|
1701
|
+
and author_resumable
|
|
1702
|
+
and result.outcome in ("no-improvement", "suite-regression", "eval-error")
|
|
1703
|
+
):
|
|
1704
|
+
# the submitted candidate failed the gate — including an eval that
|
|
1705
|
+
# errored: feedback, never a silent terminal — the author decides
|
|
1706
|
+
# what happens next (rounds stay bounded by sleep_k)
|
|
1707
|
+
return _wake_author(
|
|
1708
|
+
"Your `submit` did NOT clear the gate: "
|
|
1709
|
+
f"{result.note or result.outcome} "
|
|
1710
|
+
f"(baseline {result.baseline}, candidate {result.candidate}). "
|
|
1711
|
+
"Revise and submit again, run more experiments, or finish with an "
|
|
1712
|
+
"honest negative report.",
|
|
1713
|
+
# the verdict rides the resume: the same tree, sealed again after
|
|
1714
|
+
# the author concludes, is not measured twice; only an explicit
|
|
1715
|
+
# resubmit runs an errored eval again
|
|
1716
|
+
judged=(candidate_sha, result),
|
|
1717
|
+
)
|
|
1718
|
+
|
|
1719
|
+
def _notebook(outcome: str) -> None:
|
|
1720
|
+
# Research lines: record the tree AS OF THIS DECIDED TERMINAL — never
|
|
1721
|
+
# earlier, because a blocking panel verdict can still resume the
|
|
1722
|
+
# author (a continuation, not a terminal).
|
|
1723
|
+
_push_line_snapshot(ws, _line_ref_for(bench, config.agent_id), run_id, outcome, secrets)
|
|
1724
|
+
|
|
1725
|
+
if result.outcome == "improved":
|
|
1726
|
+
# Publish: branch the SEALED candidate sha, fold in the ledger, push,
|
|
1727
|
+
# open the PR. No moved-base merge (research-loop.md) — a stale PR is a
|
|
1728
|
+
# re-wake, not an auto-merge.
|
|
1729
|
+
from datetime import UTC, datetime
|
|
1730
|
+
|
|
1731
|
+
assert result.baseline is not None and result.candidate is not None
|
|
1732
|
+
baseline, candidate = result.baseline, result.candidate
|
|
1733
|
+
# a zero-change "improvement" is metric noise, not progress — never a PR
|
|
1734
|
+
# (defense in depth; measure_and_decide already requires a real delta,
|
|
1735
|
+
# and an empty base..candidate diff implies baseline == candidate).
|
|
1736
|
+
if not measured_paths:
|
|
1737
|
+
result = dc_replace(
|
|
1738
|
+
result, outcome="no-improvement", note="no code change; metric noise"
|
|
1739
|
+
)
|
|
1740
|
+
drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
|
|
1741
|
+
final = _clear_stage(
|
|
1742
|
+
RunRecord(**{**record.__dict__, "state": ENDED, "ending": NEGATIVE_RESULT})
|
|
1743
|
+
)
|
|
1744
|
+
_best_effort("final record", lambda: save_record(run_root, final, now), secrets)
|
|
1745
|
+
_notebook("no-improvement")
|
|
1746
|
+
return AttemptOutcome(run_id=run_id, outcome="no-improvement")
|
|
1747
|
+
|
|
1748
|
+
from datetime import UTC as _UTC
|
|
1749
|
+
from datetime import datetime as _dt
|
|
1750
|
+
|
|
1751
|
+
# seal the notebook before any checkout mutates the persisted session
|
|
1752
|
+
# tree (the memory files are excluded from the sealed candidate and
|
|
1753
|
+
# would not survive the force-checkout + clean below)
|
|
1754
|
+
_notebook("improved")
|
|
1755
|
+
|
|
1756
|
+
branch = f"{config.branch_prefix}/{run_id}"
|
|
1757
|
+
|
|
1758
|
+
# IDEMPOTENCY: a prior wake may have opened the PR but died before
|
|
1759
|
+
# recording it (leaving the run WAITING). On re-entry, if a PR is already
|
|
1760
|
+
# open for this head->base, reconcile to it — do NOT re-push
|
|
1761
|
+
# (non-fast-forward) or open a duplicate. The reconcile does the FULL
|
|
1762
|
+
# terminal (branch checkout, arm, report, issue, in-review record); it
|
|
1763
|
+
# only SKIPS the push + create_pull the prior wake already did. A lookup
|
|
1764
|
+
# failure just falls through to the normal publish.
|
|
1765
|
+
existing: dict[str, object] | None = None
|
|
1766
|
+
try:
|
|
1767
|
+
existing = github.find_open_pull_for_head(config.target, branch, base_branch)
|
|
1768
|
+
except Exception as exc:
|
|
1769
|
+
log.warning(
|
|
1770
|
+
"idempotency PR lookup failed for %s: %s",
|
|
1771
|
+
run_id,
|
|
1772
|
+
redact(f"{type(exc).__name__}: {exc}", secrets),
|
|
1773
|
+
)
|
|
1774
|
+
if existing:
|
|
1775
|
+
pr_url = str(existing.get("html_url", ""))
|
|
1776
|
+
log.info("run %s: PR %s already open; reconciling the record", run_id, pr_url)
|
|
1777
|
+
# put the workspace on the branch (a later follow-up expects it) and
|
|
1778
|
+
# finish the steps the prior wake may have died before completing.
|
|
1779
|
+
_best_effort(
|
|
1780
|
+
"reconcile checkout", lambda: ws.git("checkout", "-f", "-B", branch, candidate_sha)
|
|
1781
|
+
)
|
|
1782
|
+
pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1]
|
|
1783
|
+
if pr_number.isdigit() and not existing.get("draft"):
|
|
1784
|
+
_arm_unless_base_moved(
|
|
1785
|
+
github,
|
|
1786
|
+
ws,
|
|
1787
|
+
config.target,
|
|
1788
|
+
pr_number,
|
|
1789
|
+
base_branch,
|
|
1790
|
+
base_sha,
|
|
1791
|
+
secrets,
|
|
1792
|
+
merge_mode=getattr(contract, "merge", "manual"),
|
|
1793
|
+
panel_ran=result.panel_rounds > 0,
|
|
1794
|
+
)
|
|
1795
|
+
report_path = run_dir / "report.md"
|
|
1796
|
+
_best_effort(
|
|
1797
|
+
"run report",
|
|
1798
|
+
lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
|
|
1799
|
+
secrets,
|
|
1800
|
+
)
|
|
1801
|
+
final = _clear_stage(
|
|
1802
|
+
RunRecord(
|
|
1803
|
+
**{
|
|
1804
|
+
**record.__dict__,
|
|
1805
|
+
"state": IN_REVIEW,
|
|
1806
|
+
"pr_url": pr_url,
|
|
1807
|
+
"auto_blessed_head": _blessed_head(ws, result, contract),
|
|
1808
|
+
"resume_session_id": result.session.session_id if result.session else "",
|
|
1809
|
+
"ending_note": pr_url,
|
|
1810
|
+
}
|
|
1811
|
+
)
|
|
1812
|
+
)
|
|
1813
|
+
if _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
|
|
1814
|
+
drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
|
|
1815
|
+
_post_issue_finished(
|
|
1816
|
+
github,
|
|
1817
|
+
config.target,
|
|
1818
|
+
issue_number,
|
|
1819
|
+
run_id,
|
|
1820
|
+
"improved",
|
|
1821
|
+
pr_url,
|
|
1822
|
+
redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
|
|
1823
|
+
secrets,
|
|
1824
|
+
)
|
|
1825
|
+
return AttemptOutcome(
|
|
1826
|
+
run_id=run_id, outcome="improved", pr_url=pr_url, report_path=str(report_path)
|
|
1827
|
+
)
|
|
1828
|
+
|
|
1829
|
+
try:
|
|
1830
|
+
# FORCE-checkout the sealed candidate: at wake the workspace still
|
|
1831
|
+
# holds the session's dirty tree (HEAD is pre_session_sha), so a
|
|
1832
|
+
# plain checkout could be blocked; the sha already captured exactly
|
|
1833
|
+
# the measured content.
|
|
1834
|
+
ws.git("checkout", "-f", "-B", branch, candidate_sha)
|
|
1835
|
+
# `checkout -f` does NOT remove untracked files, and the panel's
|
|
1836
|
+
# `git add -A` (in build_panel_runner) would sweep any post-snapshot
|
|
1837
|
+
# cruft into the tree it judges. Clean untracked (non-ignored) files
|
|
1838
|
+
# so the panel reads EXACTLY candidate_sha. The ledger commit stages
|
|
1839
|
+
# only PROGRESS_PATHS, so it was never affected.
|
|
1840
|
+
ws.git("clean", "-fd")
|
|
1841
|
+
|
|
1842
|
+
# Verification panel on the credited claim — the SAME gate the
|
|
1843
|
+
# inline path runs (docs/design/orchestrator-verify.md), so a
|
|
1844
|
+
# dispatched improvement is not published unverified. It reads the
|
|
1845
|
+
# workspace tree, now checked out to the SEALED candidate_sha (the
|
|
1846
|
+
# dispatched evals ran on node-local scratch, so the tree is exactly
|
|
1847
|
+
# what was measured), over base_sha. A blocking or degraded
|
|
1848
|
+
# verdict opens a DRAFT PR carrying the findings and never arms
|
|
1849
|
+
# auto-merge; a clean verdict (or no panel) arms.
|
|
1850
|
+
if panel_lenses:
|
|
1851
|
+
# A panel ERROR (a git op in build_panel_runner, not a finding)
|
|
1852
|
+
# must NOT abort the publish and drop the candidate snapshot —
|
|
1853
|
+
# the improvement is real and measured. Fail closed to DEGRADED:
|
|
1854
|
+
# open a DRAFT for a human, keep the candidate. (run_panel itself
|
|
1855
|
+
# already fails closed per-lens; this catches the git setup.)
|
|
1856
|
+
try:
|
|
1857
|
+
verdict = build_panel_runner(
|
|
1858
|
+
ws,
|
|
1859
|
+
run_dir,
|
|
1860
|
+
base_sha,
|
|
1861
|
+
panel_lenses,
|
|
1862
|
+
contract_text,
|
|
1863
|
+
config.target,
|
|
1864
|
+
config.benchmark,
|
|
1865
|
+
config.bot_login,
|
|
1866
|
+
_dt.fromtimestamp(now, _UTC).strftime("%Y-%m-%d"),
|
|
1867
|
+
exclude=(
|
|
1868
|
+
LINE_MEMORY_PATHS if _line_ref_for(bench, config.agent_id) else ()
|
|
1869
|
+
),
|
|
1870
|
+
)(baseline, candidate, str(stage.get("report", "")))
|
|
1871
|
+
except Exception as exc:
|
|
1872
|
+
if isinstance(exc, GitError) and _is_git_tamper(exc):
|
|
1873
|
+
# tamper during the panel is not a panel error to draft
|
|
1874
|
+
# around — end as a refused wake.
|
|
1875
|
+
return _end_refused_wake(run_root, record, exc, now, secrets)
|
|
1876
|
+
log.warning(
|
|
1877
|
+
"wake panel errored for %s (%s); opening a DRAFT",
|
|
1878
|
+
run_id,
|
|
1879
|
+
redact(f"{type(exc).__name__}: {exc}", secrets),
|
|
1880
|
+
)
|
|
1881
|
+
verdict = PanelVerdict(
|
|
1882
|
+
blocking=(),
|
|
1883
|
+
transcript="panel setup failed — NOT a clean read",
|
|
1884
|
+
wake_text="",
|
|
1885
|
+
degraded=True,
|
|
1886
|
+
)
|
|
1887
|
+
reads = panel_reads + 1
|
|
1888
|
+
# DEPTH AXIS (docs/design/research-loop.md): blocking findings
|
|
1889
|
+
# on a SUBMITTED claim go back to the AUTHOR (buildout Phase B)
|
|
1890
|
+
# — it revises and resubmits (a fresh seal + gate + panel), or
|
|
1891
|
+
# concludes. A plain finish (or an unresumable session) DRAFTs
|
|
1892
|
+
# the PR with the findings open for a human to triage.
|
|
1893
|
+
if bool(verdict.blocking) and submitted_park and author_resumable:
|
|
1894
|
+
return _wake_author(verdict.wake_text)
|
|
1895
|
+
result = dc_replace(
|
|
1896
|
+
result,
|
|
1897
|
+
panel_transcript=verdict.transcript,
|
|
1898
|
+
panel_rounds=reads,
|
|
1899
|
+
panel_blocking_open=bool(verdict.blocking),
|
|
1900
|
+
panel_degraded=verdict.degraded,
|
|
1901
|
+
)
|
|
1902
|
+
|
|
1903
|
+
entries = update_leader(
|
|
1904
|
+
load_leader(workspace),
|
|
1905
|
+
benchmark=bench.name,
|
|
1906
|
+
metric=bench.metric,
|
|
1907
|
+
direction=bench.direction,
|
|
1908
|
+
baseline=baseline,
|
|
1909
|
+
candidate=candidate,
|
|
1910
|
+
run_id=run_id,
|
|
1911
|
+
date=datetime.fromtimestamp(now, UTC).strftime("%Y-%m-%d"),
|
|
1912
|
+
run_seed=result.run_seed,
|
|
1913
|
+
)
|
|
1914
|
+
write_progress(
|
|
1915
|
+
workspace,
|
|
1916
|
+
entries,
|
|
1917
|
+
config.target,
|
|
1918
|
+
digits={b.name: b.display_digits for b in contract.benchmarks if b.display_digits},
|
|
1919
|
+
)
|
|
1920
|
+
# Stage ONLY the ledger files on top of the sealed candidate — never
|
|
1921
|
+
# `git add -A`, which would sweep in untracked cruft the session left
|
|
1922
|
+
# (eval caches) that was neither measured nor scope-checked. The
|
|
1923
|
+
# candidate content is already vetted (measure_and_decide's scope
|
|
1924
|
+
# check on measured_paths); assert nothing but the ledger is staged.
|
|
1925
|
+
ws.git("add", "--", *PROGRESS_PATHS)
|
|
1926
|
+
staged = ws.staged_paths()
|
|
1927
|
+
extra = [p for p in staged if p not in PROGRESS_PATHS]
|
|
1928
|
+
if extra:
|
|
1929
|
+
raise WorkspaceDrift(f"wake commit would stage non-ledger paths: {extra[:10]}")
|
|
1930
|
+
# Commit the ledger update on top of the sealed candidate ONLY when
|
|
1931
|
+
# it actually moved. When the candidate beat its baseline but not the
|
|
1932
|
+
# recorded best, update_leader is a no-op (the ledger's `best` does
|
|
1933
|
+
# not advance) — a valid composable win with no leaderboard change,
|
|
1934
|
+
# so push the candidate as-is rather than an empty commit.
|
|
1935
|
+
if staged:
|
|
1936
|
+
ws.git(
|
|
1937
|
+
"-c",
|
|
1938
|
+
f"user.name={config.bot_login}",
|
|
1939
|
+
"-c",
|
|
1940
|
+
f"user.email={config.bot_login}@users.noreply.github.com",
|
|
1941
|
+
"commit",
|
|
1942
|
+
"-m",
|
|
1943
|
+
f"agent: improve {config.benchmark} ({_title_pair(baseline, candidate)})"
|
|
1944
|
+
f"\n\nAgent: {config.agent_id}",
|
|
1945
|
+
)
|
|
1946
|
+
ws.push(branch)
|
|
1947
|
+
body = pr_body(
|
|
1948
|
+
result, config, redact_secrets=secrets, display_digits=bench.display_digits
|
|
1949
|
+
)
|
|
1950
|
+
if issue_number:
|
|
1951
|
+
body = f"Addresses #{issue_number}.\n\n{body}"
|
|
1952
|
+
# blocking findings still open at the panel, or a degraded final
|
|
1953
|
+
# read, mean a human must look: open a DRAFT and never arm. A clean
|
|
1954
|
+
# verdict (or no panel configured) opens non-draft and arms
|
|
1955
|
+
# auto-merge only where branch protection requires a review — same
|
|
1956
|
+
# policy as the inline path.
|
|
1957
|
+
draft = result.panel_blocking_open or result.panel_degraded
|
|
1958
|
+
pr_url = github.create_pull(
|
|
1959
|
+
config.target,
|
|
1960
|
+
title=f"[agent] {config.benchmark}: {_title_pair(baseline, candidate)}",
|
|
1961
|
+
head=branch,
|
|
1962
|
+
base=base_branch,
|
|
1963
|
+
body=body,
|
|
1964
|
+
draft=draft,
|
|
1965
|
+
)
|
|
1966
|
+
pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1]
|
|
1967
|
+
if pr_number.isdigit() and not draft:
|
|
1968
|
+
_arm_unless_base_moved(
|
|
1969
|
+
github,
|
|
1970
|
+
ws,
|
|
1971
|
+
config.target,
|
|
1972
|
+
pr_number,
|
|
1973
|
+
base_branch,
|
|
1974
|
+
base_sha,
|
|
1975
|
+
secrets,
|
|
1976
|
+
merge_mode=getattr(contract, "merge", "manual"),
|
|
1977
|
+
panel_ran=result.panel_rounds > 0,
|
|
1978
|
+
)
|
|
1979
|
+
except Exception as exc:
|
|
1980
|
+
if isinstance(exc, GitError) and _is_git_tamper(exc):
|
|
1981
|
+
# a concurrent object-store removal during the publish block is
|
|
1982
|
+
# tamper, not a publish failure: end as a refused wake (the
|
|
1983
|
+
# clean tamper terminal) rather than mask it as a publish-error.
|
|
1984
|
+
return _end_refused_wake(run_root, record, exc, now, secrets)
|
|
1985
|
+
# push / PR / commit failed — end as an error. Save the ENDED record
|
|
1986
|
+
# BEFORE dropping the snapshot (same ordering as the other terminals):
|
|
1987
|
+
# a failed save then leaves the run WAITING with its snapshot intact
|
|
1988
|
+
# (recoverable), never WAITING with the candidate already gone. On a
|
|
1989
|
+
# successful end, drop the snapshot — ENDED runs are never swept, so
|
|
1990
|
+
# keeping it would only leak the ref (a retry is a fresh climb, not a
|
|
1991
|
+
# re-wake). Never delete a remote branch (a push may have
|
|
1992
|
+
# half-succeeded).
|
|
1993
|
+
note = redact(f"{type(exc).__name__}: {exc}", secrets)[:480]
|
|
1994
|
+
log.warning("wake publish failed for %s: %s", run_id, note)
|
|
1995
|
+
failed = _clear_stage(
|
|
1996
|
+
RunRecord(
|
|
1997
|
+
**{**record.__dict__, "state": ENDED, "ending": ABORTED, "ending_note": note}
|
|
1998
|
+
)
|
|
1999
|
+
)
|
|
2000
|
+
if _best_effort("ending record", lambda: save_record(run_root, failed, now), secrets):
|
|
2001
|
+
drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
|
|
2002
|
+
_notebook("publish-error")
|
|
2003
|
+
return AttemptOutcome(run_id=run_id, outcome="publish-error")
|
|
2004
|
+
# PR opened. Record IN_REVIEW *before* dropping the snapshot: if the save
|
|
2005
|
+
# fails, the record stays `waiting` with the snapshot intact, so the run
|
|
2006
|
+
# is recoverable rather than an ABORTED record over a live PR.
|
|
2007
|
+
report_path = run_dir / "report.md"
|
|
2008
|
+
_best_effort(
|
|
2009
|
+
"run report",
|
|
2010
|
+
lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
|
|
2011
|
+
secrets,
|
|
2012
|
+
)
|
|
2013
|
+
final = _clear_stage(
|
|
2014
|
+
RunRecord(
|
|
2015
|
+
**{
|
|
2016
|
+
**record.__dict__,
|
|
2017
|
+
"state": IN_REVIEW,
|
|
2018
|
+
"pr_url": pr_url,
|
|
2019
|
+
"auto_blessed_head": _blessed_head(ws, result, contract),
|
|
2020
|
+
"resume_session_id": result.session.session_id if result.session else "",
|
|
2021
|
+
"ending_note": pr_url,
|
|
2022
|
+
}
|
|
2023
|
+
)
|
|
2024
|
+
)
|
|
2025
|
+
if _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
|
|
2026
|
+
drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
|
|
2027
|
+
else:
|
|
2028
|
+
log.warning(
|
|
2029
|
+
"run %s: PR %s opened but in-review record unsaved; snapshot kept", run_id, pr_url
|
|
2030
|
+
)
|
|
2031
|
+
_post_issue_finished(
|
|
2032
|
+
github,
|
|
2033
|
+
config.target,
|
|
2034
|
+
issue_number,
|
|
2035
|
+
run_id,
|
|
2036
|
+
"improved",
|
|
2037
|
+
pr_url,
|
|
2038
|
+
redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
|
|
2039
|
+
secrets,
|
|
2040
|
+
)
|
|
2041
|
+
return AttemptOutcome(
|
|
2042
|
+
run_id=run_id, outcome="improved", pr_url=pr_url, report_path=str(report_path)
|
|
2043
|
+
)
|
|
2044
|
+
|
|
2045
|
+
# a negative terminal: end the record, THEN release the snapshot. Save
|
|
2046
|
+
# BEFORE dropping (same ordering as the improved path): if the save fails,
|
|
2047
|
+
# the run stays WAITING with its snapshot intact, so a re-wake can still
|
|
2048
|
+
# reconstruct — never WAITING with the snapshot already gone.
|
|
2049
|
+
_notebook(result.outcome)
|
|
2050
|
+
report_path = run_dir / "report.md"
|
|
2051
|
+
_best_effort(
|
|
2052
|
+
"run report",
|
|
2053
|
+
lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
|
|
2054
|
+
secrets,
|
|
2055
|
+
)
|
|
2056
|
+
final = _clear_stage(
|
|
2057
|
+
RunRecord(
|
|
2058
|
+
**{
|
|
2059
|
+
**record.__dict__,
|
|
2060
|
+
"state": ENDED,
|
|
2061
|
+
"ending": _ENDINGS_BY_OUTCOME[result.outcome],
|
|
2062
|
+
"ending_note": redact(result.note, secrets),
|
|
2063
|
+
}
|
|
2064
|
+
)
|
|
2065
|
+
)
|
|
2066
|
+
if _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
|
|
2067
|
+
drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
|
|
2068
|
+
else:
|
|
2069
|
+
log.warning(
|
|
2070
|
+
"run %s: ended negative but record unsaved; snapshot kept for a re-wake", run_id
|
|
2071
|
+
)
|
|
2072
|
+
_post_issue_finished(
|
|
2073
|
+
github,
|
|
2074
|
+
config.target,
|
|
2075
|
+
issue_number,
|
|
2076
|
+
run_id,
|
|
2077
|
+
result.outcome,
|
|
2078
|
+
"",
|
|
2079
|
+
redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
|
|
2080
|
+
secrets,
|
|
2081
|
+
)
|
|
2082
|
+
return AttemptOutcome(run_id=run_id, outcome=result.outcome, report_path=str(report_path))
|
|
2083
|
+
|
|
2084
|
+
|
|
2085
|
+
def _judge_lens_key(
|
|
2086
|
+
*,
|
|
2087
|
+
backend: str,
|
|
2088
|
+
key_file_env: str,
|
|
2089
|
+
author_backend: str,
|
|
2090
|
+
claude_panel_path: Path,
|
|
2091
|
+
image: str,
|
|
2092
|
+
) -> str:
|
|
2093
|
+
"""Resolve a non-claude judge lens's OWN key file, enforcing the three
|
|
2094
|
+
separations every shelled judge shares (codex, hermes, any future
|
|
2095
|
+
backend): the image is required (a judge never runs uncontained next to
|
|
2096
|
+
key files), the key must be named explicitly, and it must differ from BOTH
|
|
2097
|
+
the author's key of the same provider AND the claude panel key (a
|
|
2098
|
+
cross-provider send would leak an anthropic credential to another login).
|
|
2099
|
+
Returns the redacted key (or "" under an ADC-covered deployment)."""
|
|
2100
|
+
if not image:
|
|
2101
|
+
raise ValueError(
|
|
2102
|
+
f"a {backend} panel lens requires --image (a shelled judge only "
|
|
2103
|
+
"ever runs inside the container)"
|
|
2104
|
+
)
|
|
2105
|
+
raw = os.environ.get(key_file_env, "").strip()
|
|
2106
|
+
if not raw:
|
|
2107
|
+
raise ValueError(
|
|
2108
|
+
f"a {backend} panel lens needs {key_file_env} "
|
|
2109
|
+
"(role separation: the judge's own key, never the author's)"
|
|
2110
|
+
)
|
|
2111
|
+
path = Path(raw).expanduser()
|
|
2112
|
+
author_path = Path(resolve_author_key_file(author_backend)).expanduser()
|
|
2113
|
+
if path.resolve() == author_path.resolve():
|
|
2114
|
+
raise ValueError(
|
|
2115
|
+
f"{backend} panel key file {path} is the {author_backend} author key "
|
|
2116
|
+
"(role separation: the judge needs its own key)"
|
|
2117
|
+
)
|
|
2118
|
+
if path.resolve() == claude_panel_path.resolve():
|
|
2119
|
+
raise ValueError(
|
|
2120
|
+
f"{backend} panel key file {path} is the claude panel key file "
|
|
2121
|
+
"(an anthropic key must never reach another provider's login)"
|
|
2122
|
+
)
|
|
2123
|
+
return role_key(raw, author_backend)
|
|
2124
|
+
|
|
2125
|
+
|
|
2126
|
+
def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str, ...]]:
|
|
2127
|
+
"""Build the verification-panel lenses from the CLI args (empty `--panel`
|
|
2128
|
+
disables it), returning `(lenses, panel_secrets)` — the ONE owner of
|
|
2129
|
+
panel credentials: each backend's judge key is read only when a lens uses
|
|
2130
|
+
it, role separation is enforced HERE (a manual climb gets the same rule
|
|
2131
|
+
as the tick preflight), and every key a judge holds joins the caller's
|
|
2132
|
+
redaction set via `panel_secrets`. Shared by the fresh-climb and the
|
|
2133
|
+
`--resume` wake paths so a dispatched improvement runs the SAME panel as
|
|
2134
|
+
an inline one. Raises ValueError on a bad panel/backend config — a
|
|
2135
|
+
configured gate must never silently vanish."""
|
|
2136
|
+
import os
|
|
2137
|
+
|
|
2138
|
+
if not args.panel.strip():
|
|
2139
|
+
return (), ()
|
|
2140
|
+
from outerloop.panel import parse_lenses
|
|
2141
|
+
from outerloop.roles import reviewer_spec
|
|
2142
|
+
|
|
2143
|
+
parsed = parse_lenses(args.panel)
|
|
2144
|
+
# the anthropic panel key is read only when a claude lens will use it —
|
|
2145
|
+
# a codex-only panel must not demand an unrelated credential
|
|
2146
|
+
panel_key = role_key(args.panel_key_file) if any(b == "claude" for _, b, _ in parsed) else ""
|
|
2147
|
+
lenses = []
|
|
2148
|
+
secrets: list[str] = [panel_key] if panel_key else []
|
|
2149
|
+
for kind, backend, model in parsed:
|
|
2150
|
+
hermes_repo_env = os.environ.get("REVIEW_HERMES_REPO", "").strip()
|
|
2151
|
+
# per-backend judge keys coexist — a codex lens is never handed the
|
|
2152
|
+
# anthropic panel key, and role separation forbids defaulting to the
|
|
2153
|
+
# AUTHOR's codex key: the judge key is its own, named explicitly
|
|
2154
|
+
claude_panel_path = Path(args.panel_key_file or PANEL_KEY_DEFAULT).expanduser()
|
|
2155
|
+
if backend == "codex":
|
|
2156
|
+
lens_key = _judge_lens_key(
|
|
2157
|
+
backend="codex",
|
|
2158
|
+
key_file_env="AUTORESEARCH_PANEL_CODEX_KEY_FILE",
|
|
2159
|
+
author_backend="codex",
|
|
2160
|
+
claude_panel_path=claude_panel_path,
|
|
2161
|
+
image=args.image,
|
|
2162
|
+
)
|
|
2163
|
+
if lens_key:
|
|
2164
|
+
secrets.append(lens_key)
|
|
2165
|
+
elif backend == "hermes":
|
|
2166
|
+
# hermes reads its key from its provider's env var, but the FILE
|
|
2167
|
+
# is resolved and separated exactly like codex's (the key still
|
|
2168
|
+
# lands next to the session). The author's OpenAI key coexists, so
|
|
2169
|
+
# separate against the codex author key.
|
|
2170
|
+
lens_key = _judge_lens_key(
|
|
2171
|
+
backend="hermes",
|
|
2172
|
+
key_file_env="AUTORESEARCH_PANEL_HERMES_KEY_FILE",
|
|
2173
|
+
author_backend="codex",
|
|
2174
|
+
claude_panel_path=claude_panel_path,
|
|
2175
|
+
image=args.image,
|
|
2176
|
+
)
|
|
2177
|
+
if lens_key:
|
|
2178
|
+
secrets.append(lens_key)
|
|
2179
|
+
else:
|
|
2180
|
+
lens_key = panel_key
|
|
2181
|
+
try:
|
|
2182
|
+
judge = build_harness(
|
|
2183
|
+
lens_key,
|
|
2184
|
+
reviewer_spec(),
|
|
2185
|
+
backend=backend,
|
|
2186
|
+
binary=args.claude_bin if backend == "claude" else args.codex_bin,
|
|
2187
|
+
model=model or None,
|
|
2188
|
+
# ALWAYS contained: the panel runs on the climb host next to key
|
|
2189
|
+
# files, and a judge now holds a shell (codex `danger-full-access`),
|
|
2190
|
+
# so it must run inside the image. `parse_lenses` gates panel
|
|
2191
|
+
# backends to those containable here (claude today); passing the
|
|
2192
|
+
# image unconditionally means codex is safe the moment it is
|
|
2193
|
+
# enabled, never accidentally uncontained.
|
|
2194
|
+
container_image=args.image,
|
|
2195
|
+
hermes_repo=Path(hermes_repo_env) if hermes_repo_env else None,
|
|
2196
|
+
hermes_provider=os.environ.get("REVIEW_HERMES_PROVIDER", "openrouter"),
|
|
2197
|
+
)
|
|
2198
|
+
except ValueError as exc:
|
|
2199
|
+
raise ValueError(f"panel entry {kind}:{backend}: {exc}") from exc
|
|
2200
|
+
lenses.append(PanelLens(kind=kind, harness=judge))
|
|
2201
|
+
return tuple(lenses), tuple(dict.fromkeys(secrets))
|
|
2202
|
+
|
|
2203
|
+
|
|
2204
|
+
def _panel_claim_body(
|
|
2205
|
+
benchmark: str, baseline: float, candidate: float, report: str, *, lines: bool
|
|
2206
|
+
) -> str:
|
|
2207
|
+
"""The synthetic claim the panel judges. On a research-lines target the
|
|
2208
|
+
one-contribution mandate is part of the claim itself: the panel is the
|
|
2209
|
+
backstop against a line's accumulated tweaks reaching main as one PR
|
|
2210
|
+
(docs/design/research-lines.md)."""
|
|
2211
|
+
mandate = (
|
|
2212
|
+
"\n\nThis target runs research lines: a PR to main must be ONE "
|
|
2213
|
+
"clean contribution, extracted onto the base branch. A diff that "
|
|
2214
|
+
"bundles unrelated or unablated changes is a BLOCKING finding — "
|
|
2215
|
+
"name the pieces that should be separated."
|
|
2216
|
+
if lines
|
|
2217
|
+
else ""
|
|
2218
|
+
)
|
|
2219
|
+
return (
|
|
2220
|
+
f"Automated improvement claim (pre-PR): {benchmark} "
|
|
2221
|
+
f"{baseline} -> {candidate}, measured by the orchestrator.{mandate}\n\n"
|
|
2222
|
+
f"## Research report\n\n*Session prose, written before "
|
|
2223
|
+
f"the orchestrator measured.*\n\n{report[:MAX_CLAIM_CHARS]}"
|
|
2224
|
+
)
|
|
2225
|
+
|
|
2226
|
+
|
|
2227
|
+
def build_panel_runner(
|
|
2228
|
+
ws: Workspace,
|
|
2229
|
+
run_dir: Path,
|
|
2230
|
+
base_sha: str,
|
|
2231
|
+
lenses: tuple[PanelLens, ...],
|
|
2232
|
+
contract_text: str,
|
|
2233
|
+
target: str,
|
|
2234
|
+
benchmark: str,
|
|
2235
|
+
bot_login: str,
|
|
2236
|
+
today: str,
|
|
2237
|
+
start_round: int = 0,
|
|
2238
|
+
exclude: tuple[str, ...] = (),
|
|
2239
|
+
claim_body: Callable[[float, float, str], str] | None = None,
|
|
2240
|
+
) -> Callable[[float, float, str], PanelVerdict]:
|
|
2241
|
+
"""The git half of the pre-PR panel: prepare the two read-only checkouts
|
|
2242
|
+
and the synthetic claim, then hand off to `run_panel` (which owns no git).
|
|
2243
|
+
|
|
2244
|
+
Each call snapshots the CURRENT workspace tree as a detached commit and
|
|
2245
|
+
checks it out as `pr-head/` (sanitized — the candidate is an untrusted
|
|
2246
|
+
tree), next to `base/` (the trusted pre-session commit: contract and
|
|
2247
|
+
ruler). Worktrees are removed after the read; a fresh pair is built per
|
|
2248
|
+
round because the tree changes with every revision.
|
|
2249
|
+
|
|
2250
|
+
`claim_body` renders the claim the panel judges from (baseline,
|
|
2251
|
+
candidate, report); the default is the pre-PR improvement claim, a
|
|
2252
|
+
follow-up re-read passes its own wording."""
|
|
2253
|
+
from outerloop.review_agent import sanitize_checkout
|
|
2254
|
+
|
|
2255
|
+
reads = {"n": start_round}
|
|
2256
|
+
render_claim = claim_body or (
|
|
2257
|
+
lambda baseline, candidate, report: _panel_claim_body(
|
|
2258
|
+
benchmark, baseline, candidate, report, lines=bool(exclude)
|
|
2259
|
+
)
|
|
2260
|
+
)
|
|
2261
|
+
|
|
2262
|
+
def runner(baseline: float, candidate: float, report: str) -> PanelVerdict:
|
|
2263
|
+
reads["n"] += 1
|
|
2264
|
+
panel_ws = run_dir / "panel"
|
|
2265
|
+
shutil.rmtree(panel_ws, ignore_errors=True)
|
|
2266
|
+
panel_ws.mkdir(parents=True, exist_ok=True)
|
|
2267
|
+
ws.git("add", "-A")
|
|
2268
|
+
if exclude:
|
|
2269
|
+
# the panel judges the CLAIM — the same tree the gate measured,
|
|
2270
|
+
# which excludes line memory (docs/design/research-lines.md)
|
|
2271
|
+
ws.git("rm", "--cached", "-r", "-q", "--ignore-unmatch", "--", *exclude)
|
|
2272
|
+
tree = ws.git("write-tree").strip()
|
|
2273
|
+
ws.git("reset")
|
|
2274
|
+
snapshot = ws.git(
|
|
2275
|
+
"-c",
|
|
2276
|
+
"user.name=panel",
|
|
2277
|
+
"-c",
|
|
2278
|
+
"user.email=panel@localhost",
|
|
2279
|
+
"commit-tree",
|
|
2280
|
+
tree,
|
|
2281
|
+
"-p",
|
|
2282
|
+
base_sha,
|
|
2283
|
+
"-m",
|
|
2284
|
+
"panel snapshot (never pushed)",
|
|
2285
|
+
).strip()
|
|
2286
|
+
try:
|
|
2287
|
+
ws.git("worktree", "add", "--detach", str(panel_ws / "base"), base_sha)
|
|
2288
|
+
ws.git("worktree", "add", "--detach", str(panel_ws / "pr-head"), snapshot)
|
|
2289
|
+
_renamed, failed = sanitize_checkout(panel_ws / "pr-head")
|
|
2290
|
+
if failed:
|
|
2291
|
+
# fail closed for the read, loudly in the transcript: an
|
|
2292
|
+
# unsanitizable tree is never judged, and never certified
|
|
2293
|
+
return PanelVerdict(
|
|
2294
|
+
blocking=(),
|
|
2295
|
+
transcript=(
|
|
2296
|
+
f"**Verification round {reads['n']}**\n- panel skipped: "
|
|
2297
|
+
f"the candidate tree could not be sanitized "
|
|
2298
|
+
f"({failed} instruction file(s) left) — NOT a clean read"
|
|
2299
|
+
),
|
|
2300
|
+
wake_text="",
|
|
2301
|
+
degraded=True,
|
|
2302
|
+
)
|
|
2303
|
+
claim = PullRequest(
|
|
2304
|
+
repo=target,
|
|
2305
|
+
number=0,
|
|
2306
|
+
title=f"[agent] {benchmark}: {_title_pair(baseline, candidate)}",
|
|
2307
|
+
body=render_claim(baseline, candidate, report),
|
|
2308
|
+
# base..snapshot, never base..worktree: the snapshot commit
|
|
2309
|
+
# includes newly ADDED files, which a working-tree diff omits.
|
|
2310
|
+
# Excluded (line-memory) paths are excluded from the diff too:
|
|
2311
|
+
# the snapshot dropped them, so against a line-tip base they
|
|
2312
|
+
# would read as deletions the author never made
|
|
2313
|
+
diff=ws.git(
|
|
2314
|
+
"diff",
|
|
2315
|
+
f"{base_sha}..{snapshot}",
|
|
2316
|
+
*(["--", ".", *(f":(exclude){p}" for p in exclude)] if exclude else []),
|
|
2317
|
+
),
|
|
2318
|
+
author=bot_login,
|
|
2319
|
+
)
|
|
2320
|
+
return run_panel(lenses, panel_ws, claim, contract_text, today, reads["n"])
|
|
2321
|
+
finally:
|
|
2322
|
+
for name in ("base", "pr-head"):
|
|
2323
|
+
_best_effort(
|
|
2324
|
+
"panel worktree cleanup",
|
|
2325
|
+
partial(ws.git, "worktree", "remove", "--force", str(panel_ws / name)),
|
|
2326
|
+
)
|
|
2327
|
+
_best_effort("panel dir removal", lambda: shutil.rmtree(panel_ws, ignore_errors=True))
|
|
2328
|
+
_best_effort("panel worktree prune", lambda: ws.git("worktree", "prune"))
|
|
2329
|
+
|
|
2330
|
+
return runner
|
|
2331
|
+
|
|
2332
|
+
|
|
2333
|
+
def live_attempt(
|
|
2334
|
+
config: RunConfig,
|
|
2335
|
+
run_root: Path,
|
|
2336
|
+
run_id: str,
|
|
2337
|
+
harness: Harness,
|
|
2338
|
+
github: GitHubClient,
|
|
2339
|
+
bot_auth: TokenProvider,
|
|
2340
|
+
now: float,
|
|
2341
|
+
created: str,
|
|
2342
|
+
secrets: tuple[str, ...] = (),
|
|
2343
|
+
base_branch: str = "main",
|
|
2344
|
+
issue_number: int = 0,
|
|
2345
|
+
author_backend: str = "claude",
|
|
2346
|
+
author_model: str = "",
|
|
2347
|
+
author_key_file: str = "",
|
|
2348
|
+
task_hypothesis: str = "",
|
|
2349
|
+
spec: RoleSpec | None = None,
|
|
2350
|
+
panel_lenses: tuple[PanelLens, ...] = (),
|
|
2351
|
+
dispatch: DispatchSettings | None = None,
|
|
2352
|
+
eval_image: str = "",
|
|
2353
|
+
) -> AttemptOutcome:
|
|
2354
|
+
"""Run one climb against the real target repo. With `panel_lenses`, the
|
|
2355
|
+
pre-PR verification panel gates the claim before any PR exists
|
|
2356
|
+
(docs/design/orchestrator-verify.md); blocking findings still open at
|
|
2357
|
+
the cap open a DRAFT PR carrying them."""
|
|
2358
|
+
run_dir = run_root / "runs" / run_id
|
|
2359
|
+
run_dir.mkdir(parents=True, exist_ok=True)
|
|
2360
|
+
workspace = run_dir / "ws"
|
|
2361
|
+
|
|
2362
|
+
# The record exists before any network or clone work: every crash from
|
|
2363
|
+
# here on has a record to end.
|
|
2364
|
+
import os as _os
|
|
2365
|
+
|
|
2366
|
+
record = RunRecord(
|
|
2367
|
+
run_id=run_id,
|
|
2368
|
+
target=config.target,
|
|
2369
|
+
task_title=f"improve {config.benchmark}",
|
|
2370
|
+
benchmark=config.benchmark,
|
|
2371
|
+
state="implementing",
|
|
2372
|
+
agent_id=config.agent_id,
|
|
2373
|
+
deadline=now + 24 * 3600,
|
|
2374
|
+
issue_number=issue_number,
|
|
2375
|
+
author_backend=author_backend,
|
|
2376
|
+
author_model=author_model,
|
|
2377
|
+
author_key_file=author_key_file,
|
|
2378
|
+
run_job_id=_os.environ.get("SLURM_JOB_ID", ""),
|
|
2379
|
+
)
|
|
2380
|
+
try:
|
|
2381
|
+
save_record(run_root, record, now)
|
|
2382
|
+
except Exception as exc:
|
|
2383
|
+
# No record could be written, so the run must not proceed invisibly:
|
|
2384
|
+
# nothing would ever end it. Submit-time evidence (the claim comment
|
|
2385
|
+
# or the pending marker) plus this post keep the failure visible.
|
|
2386
|
+
exc_name = type(exc).__name__
|
|
2387
|
+
log.warning(
|
|
2388
|
+
"could not create run record for %s: %s",
|
|
2389
|
+
run_id,
|
|
2390
|
+
redact(f"{exc_name}: {exc}", secrets),
|
|
2391
|
+
)
|
|
2392
|
+
if issue_number:
|
|
2393
|
+
_best_effort(
|
|
2394
|
+
"issue report",
|
|
2395
|
+
lambda: github.comment(
|
|
2396
|
+
config.target,
|
|
2397
|
+
issue_number,
|
|
2398
|
+
f"Run `{run_id}` could not start ({exc_name} while writing its run record).",
|
|
2399
|
+
),
|
|
2400
|
+
secrets,
|
|
2401
|
+
)
|
|
2402
|
+
return AttemptOutcome(run_id=run_id, outcome="attempt-error")
|
|
2403
|
+
|
|
2404
|
+
# what the attempt-error handler needs to salvage the line notebook: the
|
|
2405
|
+
# exception path cannot rely on names bound inside the try
|
|
2406
|
+
salvage: dict[str, object] = {}
|
|
2407
|
+
try:
|
|
2408
|
+
ws = Workspace.clone(_target_clone_url(config.target), workspace, auth=bot_auth)
|
|
2409
|
+
# Build ON the requested PR base: the clone checks out the remote
|
|
2410
|
+
# DEFAULT branch, which need not be `base_branch` — the session must
|
|
2411
|
+
# edit, and the gate must measure, the tree the PR will land on.
|
|
2412
|
+
# A missing base branch fails loudly as attempt-error.
|
|
2413
|
+
ws.git("checkout", "-q", "-B", base_branch, f"origin/{base_branch}")
|
|
2414
|
+
_exclude_merge_artifacts(workspace)
|
|
2415
|
+
contract_text = contract_text_in_tree(workspace)
|
|
2416
|
+
contract = load_contract(contract_text, config.target)
|
|
2417
|
+
# Load the brief budget from the contract and run state: callers do
|
|
2418
|
+
# not supply it (the dataclass default rendered "0.0 GPU-hours" and
|
|
2419
|
+
# honest agents refused to launch). Same weekly counting rule as the
|
|
2420
|
+
# tick's cap: records plus live pending markers, minus this run's own.
|
|
2421
|
+
from outerloop.tick import list_pendings
|
|
2422
|
+
|
|
2423
|
+
week_ago = now - 7 * 24 * 3600
|
|
2424
|
+
recent = [
|
|
2425
|
+
r for r in list_runs(run_root) if r.target == config.target and r.created >= week_ago
|
|
2426
|
+
]
|
|
2427
|
+
# a marker whose job already has a record (this run's included) is
|
|
2428
|
+
# the same attempt, not a second one — count each job once
|
|
2429
|
+
recorded_jobs = {r.run_job_id for r in recent if r.run_job_id} | {record.run_job_id}
|
|
2430
|
+
# every unrecorded week-fresh marker counts: the tick reaps dead
|
|
2431
|
+
# markers on its own cadence (with the squeue liveness reads a brief
|
|
2432
|
+
# must not make), so an unreaped marker is either a live queued run
|
|
2433
|
+
# the weekly cap WILL count, or dead for at most a sweep — the brief
|
|
2434
|
+
# stays on the cap's conservative side either way
|
|
2435
|
+
used_week = len(recent) + sum(
|
|
2436
|
+
1
|
|
2437
|
+
for _agent, marker in list_pendings(run_root, config.target)
|
|
2438
|
+
if float(marker.get("submitted_at", 0) or 0) >= week_ago
|
|
2439
|
+
and str(marker.get("job_id", "")) not in recorded_jobs
|
|
2440
|
+
)
|
|
2441
|
+
_budget_bench = next((b for b in contract.benchmarks if b.name == config.benchmark), None)
|
|
2442
|
+
config = dc_replace(
|
|
2443
|
+
config,
|
|
2444
|
+
budget=BudgetState(
|
|
2445
|
+
gpu_hours_remaining=(
|
|
2446
|
+
float(contract.budgets.gpu_hours_per_run or 0.0)
|
|
2447
|
+
if _budget_bench is not None and _budget_bench.gpus
|
|
2448
|
+
else 0.0
|
|
2449
|
+
),
|
|
2450
|
+
runs_remaining_this_week=max(0, int(contract.budgets.runs_per_week) - used_week),
|
|
2451
|
+
),
|
|
2452
|
+
)
|
|
2453
|
+
# Author syscalls (research-loop.md, "one syscall") are CONTRACT-DRIVEN:
|
|
2454
|
+
# armed whenever the deployment can deliver them — dispatch coords (the
|
|
2455
|
+
# launches and the gate run as Slurm jobs) and a resumable backend (the
|
|
2456
|
+
# wake resumes the SAME session) — and the benchmark has not opted out
|
|
2457
|
+
# (`depth_k: 0`). With the channel (`.outerloop/`) armed it never
|
|
2458
|
+
# enters diffs or scope — repo-local exclude. With the feature off, an
|
|
2459
|
+
# untracked `.outerloop/` file must be staged and judged like any
|
|
2460
|
+
# other agent edit, not silently hidden by a magic dir name (the off
|
|
2461
|
+
# state stays byte-identical).
|
|
2462
|
+
_bench = next((b for b in contract.benchmarks if b.name == config.benchmark), None)
|
|
2463
|
+
# Research lines: move HEAD to the agent's own branch BEFORE anything
|
|
2464
|
+
# reads the tree — the contract above came from the base branch (a
|
|
2465
|
+
# line must not shape its own budgets), and the syscall-channel check
|
|
2466
|
+
# below must see the line's tree. A failed checkout falls back to the
|
|
2467
|
+
# base branch: a run is never lost to its notebook.
|
|
2468
|
+
lines_active = _bench is not None and _bench.lines and bool(config.agent_id)
|
|
2469
|
+
line_ref = ""
|
|
2470
|
+
if lines_active:
|
|
2471
|
+
try:
|
|
2472
|
+
line_ref = _checkout_line(ws, workspace, config.agent_id, base_branch)
|
|
2473
|
+
except Exception as exc:
|
|
2474
|
+
log.warning(
|
|
2475
|
+
"line checkout failed (%s); running on %s",
|
|
2476
|
+
redact(f"{type(exc).__name__}: {exc}", secrets),
|
|
2477
|
+
base_branch,
|
|
2478
|
+
)
|
|
2479
|
+
_best_effort("line merge abort", lambda: ws.git("merge", "--abort"))
|
|
2480
|
+
ws.git("checkout", "-q", "-B", base_branch, f"origin/{base_branch}")
|
|
2481
|
+
line_memory = ""
|
|
2482
|
+
line_divergence = ""
|
|
2483
|
+
if line_ref:
|
|
2484
|
+
salvage.update(ws=ws, line_ref=line_ref)
|
|
2485
|
+
try:
|
|
2486
|
+
# the line's own memory index, rendered into the brief
|
|
2487
|
+
# (data-fenced there); topic files are read on demand from
|
|
2488
|
+
# the checkout, never rendered
|
|
2489
|
+
memory_path = workspace / "AGENT_MEMORY.md"
|
|
2490
|
+
if memory_path.is_file() and not memory_path.is_symlink():
|
|
2491
|
+
# byte-mode bounded read: never load an oversized file
|
|
2492
|
+
with memory_path.open("rb") as fh:
|
|
2493
|
+
line_memory = fh.read(65_536).decode("utf-8", errors="replace")
|
|
2494
|
+
except OSError as exc:
|
|
2495
|
+
log.warning("could not read AGENT_MEMORY.md: %s", exc)
|
|
2496
|
+
try:
|
|
2497
|
+
# divergence debt, made visible each session (a conflicted
|
|
2498
|
+
# merge skips it — the diff is not meaningful mid-merge)
|
|
2499
|
+
if not ws.git("diff", "--name-only", "--diff-filter=U").strip():
|
|
2500
|
+
line_divergence = ws.git(
|
|
2501
|
+
"diff", "--shortstat", f"origin/{base_branch}", "HEAD"
|
|
2502
|
+
).strip()
|
|
2503
|
+
except Exception:
|
|
2504
|
+
line_divergence = ""
|
|
2505
|
+
author_syscalls = (
|
|
2506
|
+
dispatch is not None
|
|
2507
|
+
and getattr(harness, "supports_resume", True)
|
|
2508
|
+
and _bench is not None
|
|
2509
|
+
and _bench.depth_k > 0
|
|
2510
|
+
)
|
|
2511
|
+
# The `.outerloop/` channel must be KERNEL-OWNED. In a fresh clone,
|
|
2512
|
+
# anything already at that path was committed by the TARGET — a symlink
|
|
2513
|
+
# (install would write through it to a host path with our permissions),
|
|
2514
|
+
# a tracked request (free cluster compute), or
|
|
2515
|
+
# any other booby trap. If the path pre-exists in ANY form, disable the
|
|
2516
|
+
# feature for the run, loudly; otherwise we create a dir we own.
|
|
2517
|
+
# A target must not ship EITHER channel name (both are booby-trap risks:
|
|
2518
|
+
# a symlink install writes through, a planted request steals compute).
|
|
2519
|
+
shipped = next(
|
|
2520
|
+
(
|
|
2521
|
+
n
|
|
2522
|
+
for n in CHANNEL_DIR_NAMES
|
|
2523
|
+
if (workspace / n).is_symlink() or (workspace / n).exists()
|
|
2524
|
+
),
|
|
2525
|
+
"",
|
|
2526
|
+
)
|
|
2527
|
+
if author_syscalls and shipped:
|
|
2528
|
+
log.warning(
|
|
2529
|
+
"target ships a %s path (symlink=%s); author syscalls disabled for this run",
|
|
2530
|
+
shipped,
|
|
2531
|
+
(workspace / shipped).is_symlink(),
|
|
2532
|
+
)
|
|
2533
|
+
author_syscalls = False
|
|
2534
|
+
reports = _fetch_research_reports(ws, MAX_ARCHIVED_REPORTS)
|
|
2535
|
+
if author_syscalls:
|
|
2536
|
+
assert _bench is not None
|
|
2537
|
+
syscall_excluded(workspace)
|
|
2538
|
+
# the author's interface is the TOOL (`python .outerloop/syscall
|
|
2539
|
+
# launch ... -- <cmd>`; `... sleep`), never the raw ABI file —
|
|
2540
|
+
# install it plus the informational budget its `status` shows.
|
|
2541
|
+
syscall_install_tool(workspace)
|
|
2542
|
+
syscall_write_budget(
|
|
2543
|
+
workspace,
|
|
2544
|
+
launches_remaining=_bench.depth_k,
|
|
2545
|
+
sleeps_remaining=_bench.sleep_k,
|
|
2546
|
+
gpu_hours_remaining=(
|
|
2547
|
+
float(contract.budgets.gpu_hours_per_run) if _bench.gpus else None
|
|
2548
|
+
),
|
|
2549
|
+
)
|
|
2550
|
+
# AFTER install_tool: installing the tool recreates the channel
|
|
2551
|
+
# dir it owns, which would delete an archive written earlier
|
|
2552
|
+
_install_report_archive(workspace, reports)
|
|
2553
|
+
# the fleet snapshot the `siblings` command shows — read from the
|
|
2554
|
+
# research-log's status.json (the SAME branch the reports came
|
|
2555
|
+
# from, so it works across clusters) and best-effort throughout:
|
|
2556
|
+
# a missing or malformed snapshot just means no siblings known
|
|
2557
|
+
syscall_write_siblings(workspace, _sibling_entries(ws, config.agent_id))
|
|
2558
|
+
|
|
2559
|
+
def changed_paths() -> list[str]:
|
|
2560
|
+
# against the base branch head, never the line tip a conflicted
|
|
2561
|
+
# merge can leave HEAD on (see _paths_changed_from_base)
|
|
2562
|
+
return _paths_changed_from_base(ws, f"refs/remotes/origin/{base_branch}", lines_active)
|
|
2563
|
+
|
|
2564
|
+
if issue_number:
|
|
2565
|
+
from outerloop.intake import CLAIM_MARKER
|
|
2566
|
+
|
|
2567
|
+
already = any(
|
|
2568
|
+
has_marker(str(c.get("body", "")), "claimed")
|
|
2569
|
+
for c in github.list_comments(config.target, issue_number)
|
|
2570
|
+
)
|
|
2571
|
+
if not already: # manual CLI runs claim here; tick runs claimed at submit
|
|
2572
|
+
github.comment(
|
|
2573
|
+
config.target,
|
|
2574
|
+
issue_number,
|
|
2575
|
+
f"{CLAIM_MARKER}\nPicked up as run `{run_id}` "
|
|
2576
|
+
f"(benchmark `{config.benchmark}`). A report will follow here.",
|
|
2577
|
+
)
|
|
2578
|
+
|
|
2579
|
+
# Expensive benchmarks measure as dispatched cluster jobs; cheap ones
|
|
2580
|
+
# (and any run with no cluster coordinates) measure inline. The choice
|
|
2581
|
+
# is the benchmark's eval-time hint against the in-job runway, decided
|
|
2582
|
+
# ONCE here so the baseline setup, the measurer, and the park deadline
|
|
2583
|
+
# all agree on it.
|
|
2584
|
+
eval_minutes = next(
|
|
2585
|
+
(b.eval_minutes for b in contract.benchmarks if b.name == config.benchmark), None
|
|
2586
|
+
)
|
|
2587
|
+
wants_dispatch = should_dispatch(eval_minutes)
|
|
2588
|
+
dispatched = dispatch is not None and wants_dispatch
|
|
2589
|
+
if wants_dispatch and dispatch is None:
|
|
2590
|
+
# a benchmark asked to be dispatched but no cluster coordinates
|
|
2591
|
+
# reached us — never silently: name it, then measure inline
|
|
2592
|
+
log.warning(
|
|
2593
|
+
"benchmark %s wants dispatched eval (eval_minutes=%s) but no cluster "
|
|
2594
|
+
"coordinates (image/account/partition) are set; measuring inline",
|
|
2595
|
+
config.benchmark,
|
|
2596
|
+
eval_minutes,
|
|
2597
|
+
)
|
|
2598
|
+
|
|
2599
|
+
# the panel's base is the PRE-SESSION commit — the exact tree the
|
|
2600
|
+
# baseline was measured on — never origin/<base_branch>, which can
|
|
2601
|
+
# name a different branch than the clone's checkout
|
|
2602
|
+
pre_session_sha = ws.git("rev-parse", "HEAD").strip()
|
|
2603
|
+
panel_runner = (
|
|
2604
|
+
build_panel_runner(
|
|
2605
|
+
ws,
|
|
2606
|
+
run_dir,
|
|
2607
|
+
pre_session_sha,
|
|
2608
|
+
panel_lenses,
|
|
2609
|
+
contract_text,
|
|
2610
|
+
config.target,
|
|
2611
|
+
config.benchmark,
|
|
2612
|
+
config.bot_login,
|
|
2613
|
+
created[:10],
|
|
2614
|
+
exclude=LINE_MEMORY_PATHS if lines_active else (),
|
|
2615
|
+
)
|
|
2616
|
+
if panel_lenses
|
|
2617
|
+
else None
|
|
2618
|
+
)
|
|
2619
|
+
# ONE measurer either way: every measure is a job that checks its
|
|
2620
|
+
# tree sha out fresh from this workspace's `refs/dispatch/*` and
|
|
2621
|
+
# writes its result to the run dir. DISPATCHED: the jobs go to the
|
|
2622
|
+
# cluster and a not-yet-done measure PARKS the climb. LOCAL: the SAME
|
|
2623
|
+
# jobs run synchronously in this allocation (LocalCompute), so every
|
|
2624
|
+
# measure is done when checked and nothing parks. `snapshot` commits
|
|
2625
|
+
# the workspace's current content to a candidate sha and we own the
|
|
2626
|
+
# ref lifecycle, keeping the one candidate ref a park needs and
|
|
2627
|
+
# dropping the rest when the climb ends.
|
|
2628
|
+
measurer: Measurer
|
|
2629
|
+
if dispatched:
|
|
2630
|
+
assert dispatch is not None and eval_minutes is not None # should_dispatch(None) False
|
|
2631
|
+
measurer = dispatch.measurer(
|
|
2632
|
+
run_dir, repo_root=workspace, eval_minutes=eval_minutes, run_tag=run_id
|
|
2633
|
+
)
|
|
2634
|
+
else:
|
|
2635
|
+
measurer = DispatchedMeasurer(
|
|
2636
|
+
compute=LocalCompute(),
|
|
2637
|
+
run_dir=run_dir,
|
|
2638
|
+
repo_root=workspace,
|
|
2639
|
+
# a configured image contains LOCAL evals too — the cluster
|
|
2640
|
+
# triple being incomplete must not silently drop the jail
|
|
2641
|
+
image=dispatch.image if dispatch is not None else eval_image,
|
|
2642
|
+
account="",
|
|
2643
|
+
partition="",
|
|
2644
|
+
eval_minutes=int(eval_minutes or 0),
|
|
2645
|
+
run_tag=run_id,
|
|
2646
|
+
# an inline gate shares the same target-wide baseline cache
|
|
2647
|
+
baseline_cache=run_dir.parent / "baselines",
|
|
2648
|
+
)
|
|
2649
|
+
snapshots: list[Snapshot] = []
|
|
2650
|
+
|
|
2651
|
+
def snapshot() -> str:
|
|
2652
|
+
snap = snapshot_tree(
|
|
2653
|
+
ws, pre_session_sha, exclude=LINE_MEMORY_PATHS if lines_active else ()
|
|
2654
|
+
)
|
|
2655
|
+
snapshots.append(snap)
|
|
2656
|
+
return snap.commit
|
|
2657
|
+
|
|
2658
|
+
# `author_syscalls` already folds every enablement condition — dispatch
|
|
2659
|
+
# coords, a resumable backend, the benchmark's opt-out, and the
|
|
2660
|
+
# channel-ownership guard above (a target-shipped `.autoresearch` —
|
|
2661
|
+
# symlink, tracked request, or any other pre-existing form — has
|
|
2662
|
+
# disabled the feature for this run).
|
|
2663
|
+
launcher = None
|
|
2664
|
+
if author_syscalls:
|
|
2665
|
+
assert dispatch is not None # folded into author_syscalls above
|
|
2666
|
+
launcher = _make_launcher(
|
|
2667
|
+
dispatch, run_dir, workspace, run_id, gpus=_bench.gpus if _bench else 0
|
|
2668
|
+
)
|
|
2669
|
+
|
|
2670
|
+
parked: RunParked | None = None
|
|
2671
|
+
kept_ref = "" # the ONE candidate snapshot ref that must outlive a park
|
|
2672
|
+
try:
|
|
2673
|
+
# the last-known score orients the brief only; the gate re-measures
|
|
2674
|
+
# both sides after the session, so None (a first run) is fine.
|
|
2675
|
+
prior_best = load_leader(workspace).get(config.benchmark)
|
|
2676
|
+
result = attempt_once(
|
|
2677
|
+
config,
|
|
2678
|
+
contract_text,
|
|
2679
|
+
workspace,
|
|
2680
|
+
harness,
|
|
2681
|
+
measurer,
|
|
2682
|
+
pre_session_sha,
|
|
2683
|
+
snapshot,
|
|
2684
|
+
ruler=RULER,
|
|
2685
|
+
changed_paths=changed_paths,
|
|
2686
|
+
created=created,
|
|
2687
|
+
task_hypothesis=task_hypothesis,
|
|
2688
|
+
recent_reports=tuple(text for _name, text in reports),
|
|
2689
|
+
lessons=distill_lessons(reports),
|
|
2690
|
+
report_archive=author_syscalls,
|
|
2691
|
+
spec=spec,
|
|
2692
|
+
panel_runner=panel_runner,
|
|
2693
|
+
brief_baseline=prior_best.best if prior_best else None,
|
|
2694
|
+
line_ref=line_ref,
|
|
2695
|
+
line_memory=line_memory,
|
|
2696
|
+
line_divergence=line_divergence,
|
|
2697
|
+
launcher=launcher,
|
|
2698
|
+
tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(),
|
|
2699
|
+
)
|
|
2700
|
+
except RunParked as p:
|
|
2701
|
+
# The climb dispatched its measures and hibernated. Persist the
|
|
2702
|
+
# re-entry stage as a WAITING record (not an error), keep the
|
|
2703
|
+
# candidate snapshot alive for the wake, and end. The wake re-enters
|
|
2704
|
+
# from the record. `parked` is set only
|
|
2705
|
+
# AFTER a successful write: if _park_run raises, it stays None so the
|
|
2706
|
+
# finally drops every snapshot (no leak) and the outer handler ends
|
|
2707
|
+
# the run as an error rather than a half-written hibernation.
|
|
2708
|
+
if p.phase in ("candidate", "author-sleep"):
|
|
2709
|
+
# keep exactly ONE snapshot for that sha (two can share a
|
|
2710
|
+
# commit); record and keep that same ref, drop the rest. An
|
|
2711
|
+
# author-sleep's snapshot is the tree the wake re-delivers.
|
|
2712
|
+
kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
|
|
2713
|
+
import time
|
|
2714
|
+
|
|
2715
|
+
# anchor the deadline to the PARK (when the evals were submitted),
|
|
2716
|
+
# not the run's start `now` — a session lasting hours would otherwise
|
|
2717
|
+
# eat the queue budget and let the sweep cancel a still-queued eval.
|
|
2718
|
+
try:
|
|
2719
|
+
_park_run(
|
|
2720
|
+
run_root,
|
|
2721
|
+
record,
|
|
2722
|
+
p,
|
|
2723
|
+
kept_ref,
|
|
2724
|
+
eval_minutes,
|
|
2725
|
+
time.time(),
|
|
2726
|
+
secrets,
|
|
2727
|
+
dispatch=dispatch,
|
|
2728
|
+
base_branch=base_branch,
|
|
2729
|
+
)
|
|
2730
|
+
except Exception:
|
|
2731
|
+
# The WAITING record did not persist, so nothing will ever wake
|
|
2732
|
+
# the eval jobs this park already submitted. Cancel them so they
|
|
2733
|
+
# don't sit in the queue as orphans (best-effort, self-logging),
|
|
2734
|
+
# then fall through to the error handler — `parked` stays None,
|
|
2735
|
+
# so the finally still drops every snapshot. A park only happens
|
|
2736
|
+
# on the dispatched path, so `dispatch` is set here.
|
|
2737
|
+
assert dispatch is not None
|
|
2738
|
+
for job_id in afterany_ids(p.afterany):
|
|
2739
|
+
dispatch.compute.cancel(job_id)
|
|
2740
|
+
raise
|
|
2741
|
+
parked = p
|
|
2742
|
+
return AttemptOutcome(run_id=run_id, outcome="parked")
|
|
2743
|
+
finally:
|
|
2744
|
+
for snap in snapshots:
|
|
2745
|
+
# a candidate park must OUTLIVE the wake — keep the ONE recorded
|
|
2746
|
+
# snapshot (matched by ref, not commit); drop every other one.
|
|
2747
|
+
if parked and kept_ref and snap.ref == kept_ref:
|
|
2748
|
+
continue
|
|
2749
|
+
drop_snapshot(ws, snap) # best-effort + self-logging; never raises
|
|
2750
|
+
except Exception as exc:
|
|
2751
|
+
exc_name = type(exc).__name__
|
|
2752
|
+
note = redact(f"{exc_name}: {exc}", secrets)[:500]
|
|
2753
|
+
log.warning("climb failed for %s: %s", run_id, note)
|
|
2754
|
+
if salvage:
|
|
2755
|
+
# a crashed attempt's tree is still notebook-worthy (best-effort)
|
|
2756
|
+
_push_line_snapshot(
|
|
2757
|
+
cast(Workspace, salvage["ws"]),
|
|
2758
|
+
str(salvage["line_ref"]),
|
|
2759
|
+
run_id,
|
|
2760
|
+
"attempt-error",
|
|
2761
|
+
secrets,
|
|
2762
|
+
)
|
|
2763
|
+
failed = RunRecord(
|
|
2764
|
+
**{
|
|
2765
|
+
**record.__dict__,
|
|
2766
|
+
"state": ENDED,
|
|
2767
|
+
"ending": ABORTED,
|
|
2768
|
+
"ending_note": note,
|
|
2769
|
+
}
|
|
2770
|
+
)
|
|
2771
|
+
report_path = run_dir / "report.md"
|
|
2772
|
+
_best_effort("ending record", lambda: save_record(run_root, failed, now), secrets)
|
|
2773
|
+
wrote = _best_effort(
|
|
2774
|
+
"error report",
|
|
2775
|
+
lambda: report_path.write_text(
|
|
2776
|
+
f"# Run report — {config.target} / {config.benchmark}\n"
|
|
2777
|
+
f"Outcome: **attempt-error**\n"
|
|
2778
|
+
f"Note: {note}\n"
|
|
2779
|
+
),
|
|
2780
|
+
secrets,
|
|
2781
|
+
)
|
|
2782
|
+
if issue_number:
|
|
2783
|
+
# Exception detail stays in the local record and report: redact()
|
|
2784
|
+
# only knows the secrets it was handed, and raw messages can carry
|
|
2785
|
+
# paths or tokens the tuple does not cover. The issue gets the
|
|
2786
|
+
# exception TYPE only.
|
|
2787
|
+
_best_effort(
|
|
2788
|
+
"issue report",
|
|
2789
|
+
lambda: github.comment(
|
|
2790
|
+
config.target,
|
|
2791
|
+
issue_number,
|
|
2792
|
+
f"Run `{run_id}` finished (attempt-error): {exc_name}. "
|
|
2793
|
+
f"Details are in the run's record and report on the orchestrator.",
|
|
2794
|
+
),
|
|
2795
|
+
secrets,
|
|
2796
|
+
)
|
|
2797
|
+
return AttemptOutcome(
|
|
2798
|
+
run_id=run_id,
|
|
2799
|
+
outcome="attempt-error",
|
|
2800
|
+
# an outcome must never point at a report that was not written
|
|
2801
|
+
report_path=str(report_path) if wrote else "",
|
|
2802
|
+
)
|
|
2803
|
+
|
|
2804
|
+
if result.outcome == "improved" and not result.measured_paths:
|
|
2805
|
+
# a zero-change "improvement" is metric noise, not progress — never a
|
|
2806
|
+
# PR (same rule as the wake publish)
|
|
2807
|
+
result = dc_replace(result, outcome="no-improvement", note="no code change; metric noise")
|
|
2808
|
+
|
|
2809
|
+
report = result.report(config, redact_secrets=secrets)
|
|
2810
|
+
report_path = run_dir / "report.md"
|
|
2811
|
+
wrote_report = _best_effort("run report", lambda: report_path.write_text(report), secrets)
|
|
2812
|
+
|
|
2813
|
+
# Research lines: seal the notebook NOW, while the tree is still the
|
|
2814
|
+
# session's final tree — the publish below force-checkouts the sealed
|
|
2815
|
+
# candidate and cleans untracked files, which would drop the agent's
|
|
2816
|
+
# memory (it is excluded from measurable seals by design). The label is
|
|
2817
|
+
# the GATE outcome, correct at this moment; a publish failure appends a
|
|
2818
|
+
# publish-error snapshot at the tail.
|
|
2819
|
+
_push_line_snapshot(ws, line_ref, run_id, result.outcome, secrets)
|
|
2820
|
+
|
|
2821
|
+
pr_url = ""
|
|
2822
|
+
outcome_name = result.outcome
|
|
2823
|
+
branch = ""
|
|
2824
|
+
pushed = False
|
|
2825
|
+
if result.outcome == "improved":
|
|
2826
|
+
try:
|
|
2827
|
+
# Publish the SEALED candidate sha — never the live tree, which
|
|
2828
|
+
# may have drifted since the snapshot (eval caches, stray writes);
|
|
2829
|
+
# the sha is exactly the measured, scope-checked content. A base
|
|
2830
|
+
# branch that moved during the climb is NOT merged and re-measured
|
|
2831
|
+
# here: a stale PR is review's to handle (research-loop.md).
|
|
2832
|
+
branch = f"{config.branch_prefix}/{run_id}"
|
|
2833
|
+
if result.baseline is None or result.candidate is None or not result.candidate_sha:
|
|
2834
|
+
raise EvalError("improved result missing measurements or the sealed sha")
|
|
2835
|
+
bench = next(b for b in contract.benchmarks if b.name == config.benchmark)
|
|
2836
|
+
baseline, candidate = result.baseline, result.candidate
|
|
2837
|
+
# FORCE-checkout: the workspace still holds the session's dirty
|
|
2838
|
+
# tree. The snapshot commit is anchored by the new branch (the
|
|
2839
|
+
# dropped dispatch ref left it unreferenced; nothing pruned it in
|
|
2840
|
+
# this process). clean -fd drops post-snapshot cruft so the
|
|
2841
|
+
# pushed tree is exactly candidate_sha plus the ledger commit.
|
|
2842
|
+
ws.git("checkout", "-f", "-B", branch, result.candidate_sha)
|
|
2843
|
+
ws.git("clean", "-fd")
|
|
2844
|
+
entries = update_leader(
|
|
2845
|
+
load_leader(workspace),
|
|
2846
|
+
benchmark=bench.name,
|
|
2847
|
+
metric=bench.metric,
|
|
2848
|
+
direction=bench.direction,
|
|
2849
|
+
baseline=baseline,
|
|
2850
|
+
candidate=candidate,
|
|
2851
|
+
run_id=run_id,
|
|
2852
|
+
date=created[:10],
|
|
2853
|
+
run_seed=result.run_seed,
|
|
2854
|
+
)
|
|
2855
|
+
write_progress(
|
|
2856
|
+
workspace,
|
|
2857
|
+
entries,
|
|
2858
|
+
config.target,
|
|
2859
|
+
digits={b.name: b.display_digits for b in contract.benchmarks if b.display_digits},
|
|
2860
|
+
)
|
|
2861
|
+
# Stage ONLY the ledger files on top of the sealed candidate —
|
|
2862
|
+
# never `git add -A`, which would sweep in anything a session or
|
|
2863
|
+
# eval left behind (same rule as the wake publish).
|
|
2864
|
+
ws.git("add", "--", *PROGRESS_PATHS)
|
|
2865
|
+
staged = ws.staged_paths()
|
|
2866
|
+
extra = [p for p in staged if p not in PROGRESS_PATHS]
|
|
2867
|
+
if extra:
|
|
2868
|
+
raise WorkspaceDrift(f"publish would stage non-ledger paths: {extra[:10]}")
|
|
2869
|
+
if staged:
|
|
2870
|
+
ws.git(
|
|
2871
|
+
"-c",
|
|
2872
|
+
f"user.name={config.bot_login}",
|
|
2873
|
+
"-c",
|
|
2874
|
+
f"user.email={config.bot_login}@users.noreply.github.com",
|
|
2875
|
+
"commit",
|
|
2876
|
+
"-m",
|
|
2877
|
+
f"agent: improve {config.benchmark} ({_title_pair(baseline, candidate)})"
|
|
2878
|
+
f"\n\nAgent: {config.agent_id}",
|
|
2879
|
+
)
|
|
2880
|
+
ws.push(branch)
|
|
2881
|
+
pushed = True
|
|
2882
|
+
body = pr_body(
|
|
2883
|
+
result, config, redact_secrets=secrets, display_digits=bench.display_digits
|
|
2884
|
+
)
|
|
2885
|
+
if issue_number:
|
|
2886
|
+
body = f"Addresses #{issue_number}.\n\n{body}"
|
|
2887
|
+
pr_url = github.create_pull(
|
|
2888
|
+
config.target,
|
|
2889
|
+
# short precision in the title; full precision lives in the
|
|
2890
|
+
# PR body table and the ledger
|
|
2891
|
+
title=f"[agent] {config.benchmark}: {_title_pair(baseline, candidate)}",
|
|
2892
|
+
head=branch,
|
|
2893
|
+
base=base_branch,
|
|
2894
|
+
body=body,
|
|
2895
|
+
# blocking findings open at the panel, or a degraded final
|
|
2896
|
+
# read: visible, plainly not merge-ready
|
|
2897
|
+
draft=result.panel_blocking_open or result.panel_degraded,
|
|
2898
|
+
)
|
|
2899
|
+
# Arm auto-merge, best-effort, and ONLY when branch protection
|
|
2900
|
+
# requires a human review — the guard keeps bot-never-merges
|
|
2901
|
+
# enforced in code, not in per-repo config. Never arm a draft,
|
|
2902
|
+
# and never arm a claim whose base has moved (_arm_unless_base_moved).
|
|
2903
|
+
pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1]
|
|
2904
|
+
if pr_number.isdigit() and not (result.panel_blocking_open or result.panel_degraded):
|
|
2905
|
+
_arm_unless_base_moved(
|
|
2906
|
+
github,
|
|
2907
|
+
ws,
|
|
2908
|
+
config.target,
|
|
2909
|
+
pr_number,
|
|
2910
|
+
base_branch,
|
|
2911
|
+
pre_session_sha,
|
|
2912
|
+
secrets,
|
|
2913
|
+
merge_mode=getattr(contract, "merge", "manual"),
|
|
2914
|
+
panel_ran=result.panel_rounds > 0,
|
|
2915
|
+
)
|
|
2916
|
+
final = RunRecord(
|
|
2917
|
+
**{
|
|
2918
|
+
**record.__dict__,
|
|
2919
|
+
"state": IN_REVIEW,
|
|
2920
|
+
"pr_url": pr_url,
|
|
2921
|
+
"auto_blessed_head": _blessed_head(ws, result, contract),
|
|
2922
|
+
"resume_session_id": result.session.session_id if result.session else "",
|
|
2923
|
+
"ending_note": pr_url,
|
|
2924
|
+
}
|
|
2925
|
+
)
|
|
2926
|
+
except Exception as exc:
|
|
2927
|
+
log.warning(
|
|
2928
|
+
"publish failed for %s: %s",
|
|
2929
|
+
run_id,
|
|
2930
|
+
redact(f"{type(exc).__name__}: {exc}", secrets),
|
|
2931
|
+
)
|
|
2932
|
+
# Never delete the remote branch: an exception from create_pull
|
|
2933
|
+
# does not prove no PR exists (a 422-already-exists or a timeout
|
|
2934
|
+
# after a successful POST both land here), and deleting the ref
|
|
2935
|
+
# would close such a PR and discard the only pushed copy. Leave
|
|
2936
|
+
# it and record it; a sweeper can reap confirmed orphans later.
|
|
2937
|
+
outcome_name = "publish-error"
|
|
2938
|
+
final = RunRecord(
|
|
2939
|
+
**{
|
|
2940
|
+
**record.__dict__,
|
|
2941
|
+
"state": ENDED,
|
|
2942
|
+
"ending": ABORTED,
|
|
2943
|
+
"ending_note": (
|
|
2944
|
+
(f"branch left on remote: {branch}; " if pushed else "")
|
|
2945
|
+
+ redact(f"{type(exc).__name__}: {exc}", secrets)[:480]
|
|
2946
|
+
),
|
|
2947
|
+
}
|
|
2948
|
+
)
|
|
2949
|
+
else:
|
|
2950
|
+
if result.outcome == "session-outage":
|
|
2951
|
+
_best_effort(
|
|
2952
|
+
"outage stamp",
|
|
2953
|
+
lambda: stamp_outage(run_root, redact(result.note, secrets)[:300], now),
|
|
2954
|
+
secrets,
|
|
2955
|
+
)
|
|
2956
|
+
final = RunRecord(
|
|
2957
|
+
**{
|
|
2958
|
+
**record.__dict__,
|
|
2959
|
+
"state": ENDED,
|
|
2960
|
+
"ending": _ENDINGS_BY_OUTCOME[result.outcome],
|
|
2961
|
+
"ending_note": redact(result.note, secrets),
|
|
2962
|
+
}
|
|
2963
|
+
)
|
|
2964
|
+
if not _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
|
|
2965
|
+
# The on-disk record still says `implementing`, so automated
|
|
2966
|
+
# follow-up servicing will not track this run — and if a PR was
|
|
2967
|
+
# opened, its humans are the only ones who can act. Say so WHERE
|
|
2968
|
+
# they are looking: GitHub is the one store still writable when the
|
|
2969
|
+
# local disk is gone.
|
|
2970
|
+
pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1] if pr_url else ""
|
|
2971
|
+
if pr_number.isdigit():
|
|
2972
|
+
_best_effort(
|
|
2973
|
+
"pr state warning",
|
|
2974
|
+
lambda: github.comment(
|
|
2975
|
+
config.target,
|
|
2976
|
+
int(pr_number),
|
|
2977
|
+
f"State record for run `{run_id}` could not be saved; "
|
|
2978
|
+
f"automated follow-up servicing is offline for this run. "
|
|
2979
|
+
f"A maintainer owns any follow-ups on this PR.",
|
|
2980
|
+
),
|
|
2981
|
+
secrets,
|
|
2982
|
+
)
|
|
2983
|
+
if issue_number:
|
|
2984
|
+
_post_issue_finished(
|
|
2985
|
+
github,
|
|
2986
|
+
config.target,
|
|
2987
|
+
issue_number,
|
|
2988
|
+
run_id,
|
|
2989
|
+
outcome_name,
|
|
2990
|
+
pr_url,
|
|
2991
|
+
redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
|
|
2992
|
+
secrets,
|
|
2993
|
+
)
|
|
2994
|
+
if outcome_name != result.outcome:
|
|
2995
|
+
# the publish failed after the gate credited the tree: the improved
|
|
2996
|
+
# snapshot above stands (the measurement was real); append the
|
|
2997
|
+
# publish-error marker so the notebook records how the run ended
|
|
2998
|
+
_push_line_snapshot(ws, line_ref, run_id, outcome_name, secrets)
|
|
2999
|
+
log.info("run %s: %s %s", run_id, outcome_name, pr_url)
|
|
3000
|
+
return AttemptOutcome(
|
|
3001
|
+
run_id=run_id,
|
|
3002
|
+
outcome=outcome_name,
|
|
3003
|
+
pr_url=pr_url,
|
|
3004
|
+
report_path=str(report_path) if wrote_report else "",
|
|
3005
|
+
)
|
|
3006
|
+
|
|
3007
|
+
|
|
3008
|
+
class Terminated(Exception):
|
|
3009
|
+
"""Slurm sent SIGTERM (walltime, preemption, scancel): raised into the
|
|
3010
|
+
main thread so the ordinary exception containment ends the run inside
|
|
3011
|
+
the KillWait grace window before SIGKILL arrives."""
|
|
3012
|
+
|
|
3013
|
+
|
|
3014
|
+
# Below this, arming is pointless: the alarm would fire during setup,
|
|
3015
|
+
# outside containment, and a job this short cannot finish a climb anyway.
|
|
3016
|
+
MIN_ARM_S = 180
|
|
3017
|
+
|
|
3018
|
+
|
|
3019
|
+
def arm_self_deadline(job_minutes: int, margin_s: float = 120.0) -> int:
|
|
3020
|
+
"""Arm our own end-of-walltime alarm; returns the armed seconds (0 = off).
|
|
3021
|
+
|
|
3022
|
+
Slurm delivers NO signal to our process on Torch before SIGKILL
|
|
3023
|
+
(scancel and walltime timeout both signal the
|
|
3024
|
+
batch shell only) — so the only way to end a run richly before the
|
|
3025
|
+
wall is our own clock. SIGALRM fires `margin_s` before the job's
|
|
3026
|
+
walltime and raises Terminated into the ordinary containment; the
|
|
3027
|
+
margin floor covers the containment's own tail (GitHub calls are 30s
|
|
3028
|
+
timeout x retries). The walltime clock starts at JOB start, not
|
|
3029
|
+
process start — SLURM_JOB_START_TIME anchors the deadline when
|
|
3030
|
+
present so startup latency erodes the runway, never the margin.
|
|
3031
|
+
"""
|
|
3032
|
+
if job_minutes <= 0:
|
|
3033
|
+
return 0
|
|
3034
|
+
import signal
|
|
3035
|
+
import time as _time
|
|
3036
|
+
|
|
3037
|
+
margin = max(60.0, margin_s)
|
|
3038
|
+
now = _time.time()
|
|
3039
|
+
start_raw = os.environ.get("SLURM_JOB_START_TIME", "")
|
|
3040
|
+
# Sanity-bounded: the env can carry a STALE value inherited from the
|
|
3041
|
+
# submitting job (tick jobs sbatch climb jobs). A start time outside
|
|
3042
|
+
# [now - walltime, now] is not this job's — fall back to the process
|
|
3043
|
+
# clock rather than silently disarm (past) or overshoot the wall
|
|
3044
|
+
# (future).
|
|
3045
|
+
if start_raw.isdigit() and now - job_minutes * 60 <= int(start_raw) <= now:
|
|
3046
|
+
remaining = int(int(start_raw) + job_minutes * 60 - margin - now)
|
|
3047
|
+
else:
|
|
3048
|
+
remaining = int(job_minutes * 60 - margin)
|
|
3049
|
+
if remaining < MIN_ARM_S:
|
|
3050
|
+
log.warning(
|
|
3051
|
+
"self-deadline NOT armed: %ds runway is below the %ds floor", remaining, MIN_ARM_S
|
|
3052
|
+
)
|
|
3053
|
+
return 0
|
|
3054
|
+
|
|
3055
|
+
def _on_alarm(signum: int, frame: object) -> None:
|
|
3056
|
+
raise Terminated(
|
|
3057
|
+
f"self-deadline: {margin:.0f}s before the job's {job_minutes}-minute walltime"
|
|
3058
|
+
)
|
|
3059
|
+
|
|
3060
|
+
signal.signal(signal.SIGALRM, _on_alarm)
|
|
3061
|
+
signal.alarm(remaining)
|
|
3062
|
+
return remaining
|
|
3063
|
+
|
|
3064
|
+
|
|
3065
|
+
def arm_sigterm_containment() -> None:
|
|
3066
|
+
"""Convert the FIRST SIGTERM into a Terminated exception, one-shot.
|
|
3067
|
+
|
|
3068
|
+
Repeats are absorbed by a flag rather than SIG_IGN: a second SIGTERM
|
|
3069
|
+
(repeated scancel, site KillWait re-sends) must not abort the very
|
|
3070
|
+
containment the first one enabled — and SIG_IGN would be inherited
|
|
3071
|
+
across exec by children spawned during containment, leaving them
|
|
3072
|
+
unkillable by TERM. A Python-level handler is reset on exec, so
|
|
3073
|
+
children keep default signal behavior.
|
|
3074
|
+
"""
|
|
3075
|
+
import signal
|
|
3076
|
+
|
|
3077
|
+
fired = {"done": False}
|
|
3078
|
+
|
|
3079
|
+
def _on_sigterm(signum: int, frame: object) -> None:
|
|
3080
|
+
if fired["done"]:
|
|
3081
|
+
return # containment already unwinding; absorb the repeat
|
|
3082
|
+
fired["done"] = True
|
|
3083
|
+
raise Terminated("SIGTERM from Slurm (walltime, preemption, or scancel)")
|
|
3084
|
+
|
|
3085
|
+
signal.signal(signal.SIGTERM, _on_sigterm)
|
|
3086
|
+
|
|
3087
|
+
|
|
3088
|
+
def main() -> int:
|
|
3089
|
+
import argparse
|
|
3090
|
+
import os
|
|
3091
|
+
import time
|
|
3092
|
+
from datetime import UTC, datetime
|
|
3093
|
+
|
|
3094
|
+
arm_sigterm_containment()
|
|
3095
|
+
|
|
3096
|
+
parser = argparse.ArgumentParser(description="One live attempt on one benchmark.")
|
|
3097
|
+
# --target/--benchmark drive a fresh climb; they are read from the record
|
|
3098
|
+
# on a --resume wake instead, so they are optional (validated below).
|
|
3099
|
+
parser.add_argument("--target", default="")
|
|
3100
|
+
parser.add_argument("--benchmark", default="")
|
|
3101
|
+
|
|
3102
|
+
# WIDTH: the tick assigns each concurrent slot its own agent identity;
|
|
3103
|
+
# branches, ledger rows, and reports key on it. Resumed runs inherit
|
|
3104
|
+
# the identity from their record instead.
|
|
3105
|
+
def _agent_id(value: str) -> str:
|
|
3106
|
+
# the id shapes refs (feat/auto/<id>/<run>, agents/<id>): slug only —
|
|
3107
|
+
# same rule the line checkout enforces
|
|
3108
|
+
if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", value):
|
|
3109
|
+
raise argparse.ArgumentTypeError(
|
|
3110
|
+
f"agent id {value!r} cannot shape a git ref (want [A-Za-z0-9][A-Za-z0-9_-]*)"
|
|
3111
|
+
)
|
|
3112
|
+
return value
|
|
3113
|
+
|
|
3114
|
+
parser.add_argument("--agent-id", default="agent-01", type=_agent_id)
|
|
3115
|
+
parser.add_argument("--run-root", required=True, type=Path)
|
|
3116
|
+
parser.add_argument(
|
|
3117
|
+
"--resume",
|
|
3118
|
+
default="",
|
|
3119
|
+
metavar="RUN_ID",
|
|
3120
|
+
help="wake a parked dispatched run instead of starting a fresh climb",
|
|
3121
|
+
)
|
|
3122
|
+
parser.add_argument("--base-branch", default="main")
|
|
3123
|
+
# All three default from the chain env the tick sets on the climb job, so
|
|
3124
|
+
# a contained run with AUTORESEARCH_{IMAGE,ACCOUNT,PARTITION} set selects
|
|
3125
|
+
# dispatched measurement without extra flags. The image also containers the
|
|
3126
|
+
# session + inline eval; absent any of the three, measurement stays inline
|
|
3127
|
+
# regardless of the benchmark's eval hint.
|
|
3128
|
+
parser.add_argument(
|
|
3129
|
+
"--image",
|
|
3130
|
+
default=os.environ.get("AUTORESEARCH_IMAGE", ""),
|
|
3131
|
+
help="apptainer image for session+eval",
|
|
3132
|
+
)
|
|
3133
|
+
parser.add_argument("--account", default=os.environ.get("AUTORESEARCH_ACCOUNT", ""))
|
|
3134
|
+
parser.add_argument("--partition", default=os.environ.get("AUTORESEARCH_PARTITION", ""))
|
|
3135
|
+
# the GPU lane for benchmarks with `gpus > 0` (evals + author launches);
|
|
3136
|
+
# empty = this deployment cannot place GPU jobs
|
|
3137
|
+
parser.add_argument("--gpu-partition", default=os.environ.get("AUTORESEARCH_GPU_PARTITION", ""))
|
|
3138
|
+
parser.add_argument("--gpu-account", default=os.environ.get("AUTORESEARCH_GPU_ACCOUNT", ""))
|
|
3139
|
+
parser.add_argument(
|
|
3140
|
+
"--uncontained",
|
|
3141
|
+
action="store_true",
|
|
3142
|
+
help="run WITHOUT a container (dev only: sessions can then read "
|
|
3143
|
+
"same-user files, including credential files)",
|
|
3144
|
+
)
|
|
3145
|
+
parser.add_argument("--claude-bin", default=os.path.expanduser("~/.local/bin/claude"))
|
|
3146
|
+
parser.add_argument(
|
|
3147
|
+
"--codex-bin",
|
|
3148
|
+
default=os.path.expanduser(
|
|
3149
|
+
os.environ.get("AUTORESEARCH_CODEX_BIN") or "~/.local/bin/codex"
|
|
3150
|
+
),
|
|
3151
|
+
help="host codex binary for the codex author; bind-mounted into apptainer "
|
|
3152
|
+
"(must be an absolute path).",
|
|
3153
|
+
)
|
|
3154
|
+
parser.add_argument(
|
|
3155
|
+
"--model", default=os.environ.get("AUTORESEARCH_AUTHOR_MODEL") or "claude-opus-5"
|
|
3156
|
+
)
|
|
3157
|
+
parser.add_argument(
|
|
3158
|
+
"--author-backend",
|
|
3159
|
+
choices=("claude", "codex"),
|
|
3160
|
+
default=os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude",
|
|
3161
|
+
help="agent backend for the author/editor role (config-driven: default "
|
|
3162
|
+
"from AUTORESEARCH_AUTHOR_BACKEND). codex runs contained (apptainer + "
|
|
3163
|
+
"--sandbox danger-full-access) and REQUIRES --image and a codex/openai "
|
|
3164
|
+
"--model (e.g. gpt-5.6-terra).",
|
|
3165
|
+
)
|
|
3166
|
+
parser.add_argument(
|
|
3167
|
+
"--codex-config",
|
|
3168
|
+
action="append",
|
|
3169
|
+
default=[],
|
|
3170
|
+
metavar="KEY=VALUE",
|
|
3171
|
+
help="codex `-c KEY=VALUE` config for the codex author (repeatable), "
|
|
3172
|
+
"e.g. --codex-config use_legacy_landlock=true for a host that needs it.",
|
|
3173
|
+
)
|
|
3174
|
+
parser.add_argument("--max-turns", type=int, default=60)
|
|
3175
|
+
parser.add_argument("--session-minutes", type=int, default=60)
|
|
3176
|
+
parser.add_argument(
|
|
3177
|
+
"--panel",
|
|
3178
|
+
default="",
|
|
3179
|
+
help=(
|
|
3180
|
+
"pre-PR verification lenses, comma-separated kind[:backend[:model]] "
|
|
3181
|
+
"entries (e.g. 'verify,review' or 'verify:claude:MODEL'); only the "
|
|
3182
|
+
"claude backend is contained on this host so far; empty disables "
|
|
3183
|
+
"the panel"
|
|
3184
|
+
),
|
|
3185
|
+
)
|
|
3186
|
+
parser.add_argument(
|
|
3187
|
+
"--panel-key-file",
|
|
3188
|
+
default=PANEL_KEY_DEFAULT,
|
|
3189
|
+
help="key file for panel judge sessions (the verifier's own key, never the author's)",
|
|
3190
|
+
)
|
|
3191
|
+
parser.add_argument(
|
|
3192
|
+
"--job-minutes",
|
|
3193
|
+
type=int,
|
|
3194
|
+
default=0,
|
|
3195
|
+
help="this job's Slurm walltime; arms the self-deadline (0 = off)",
|
|
3196
|
+
)
|
|
3197
|
+
parser.add_argument(
|
|
3198
|
+
"--deadline-margin-s",
|
|
3199
|
+
type=float,
|
|
3200
|
+
default=120.0,
|
|
3201
|
+
help="how long before the walltime the self-deadline fires (floor 60)",
|
|
3202
|
+
)
|
|
3203
|
+
parser.add_argument("--pat-file", default=str(CONFIG_DIR / "bot_pat"))
|
|
3204
|
+
parser.add_argument(
|
|
3205
|
+
"--github-app-file",
|
|
3206
|
+
default=os.environ.get("AUTORESEARCH_GITHUB_APP_FILE", ""),
|
|
3207
|
+
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
3208
|
+
"when set, installation tokens replace the PAT",
|
|
3209
|
+
)
|
|
3210
|
+
parser.add_argument(
|
|
3211
|
+
"--key-file",
|
|
3212
|
+
default="",
|
|
3213
|
+
help="author key file; default resolves per backend (config-driven): "
|
|
3214
|
+
"AUTORESEARCH_HARNESS_KEY_FILE for claude, AUTORESEARCH_CODEX_KEY_FILE for codex",
|
|
3215
|
+
)
|
|
3216
|
+
parser.add_argument("--issue", type=int, default=0)
|
|
3217
|
+
parser.add_argument(
|
|
3218
|
+
"--min-free-gb",
|
|
3219
|
+
type=float,
|
|
3220
|
+
default=10.0,
|
|
3221
|
+
help="refuse to start when the run root has less free space",
|
|
3222
|
+
)
|
|
3223
|
+
parser.add_argument(
|
|
3224
|
+
"--hypothesis-b64", default="", help="base64 task hypothesis (issue text, fenced)"
|
|
3225
|
+
)
|
|
3226
|
+
args = parser.parse_args()
|
|
3227
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
3228
|
+
if not args.image and not args.uncontained:
|
|
3229
|
+
parser.error("--image is required (or pass --uncontained explicitly, dev only)")
|
|
3230
|
+
# NOTE: the codex author is validated on the EFFECTIVE author per path — the
|
|
3231
|
+
# fresh climb on args (below), a wake on the parked run's persisted pair — not
|
|
3232
|
+
# here, where args.author_backend is the FLEET default and would misjudge a
|
|
3233
|
+
# resume after a fleet flip.
|
|
3234
|
+
# each --codex-config KEY=VALUE becomes a `-c KEY=VALUE` pair for codex
|
|
3235
|
+
codex_extra = tuple(a for c in args.codex_config for a in ("-c", c))
|
|
3236
|
+
|
|
3237
|
+
bot_auth = resolve_bot_auth(args.pat_file, args.github_app_file)
|
|
3238
|
+
|
|
3239
|
+
# --resume WAKES a parked dispatched run: rebuild the dispatched measurer
|
|
3240
|
+
# and re-enter the decision. The wake job the WakeDispatcher submits runs
|
|
3241
|
+
# exactly this.
|
|
3242
|
+
if args.resume:
|
|
3243
|
+
placed = bool(args.account and args.partition) or local_mode()
|
|
3244
|
+
if not (placed and args.image and Path(args.image).is_file()):
|
|
3245
|
+
parser.error(
|
|
3246
|
+
"--resume needs the cluster triple (--account/--partition/--image) "
|
|
3247
|
+
"to rebuild the dispatched measurer (local compute waives the "
|
|
3248
|
+
"account/partition pair, never the image)"
|
|
3249
|
+
)
|
|
3250
|
+
from outerloop.runstate import load_record
|
|
3251
|
+
|
|
3252
|
+
# a wake that is not the lease holder is a straggler (a replacement was
|
|
3253
|
+
# dispatched after it was cancelled, or it was armed and then lost):
|
|
3254
|
+
# it must not touch the run beside the holder
|
|
3255
|
+
other = _lease_held_by_another_job(args.run_root, args.resume)
|
|
3256
|
+
if other:
|
|
3257
|
+
print(f"run {args.resume}: wake job {other} holds the lease; this one exits")
|
|
3258
|
+
return 0
|
|
3259
|
+
|
|
3260
|
+
# Reproduce the PARKED run's author, not the current fleet default: the
|
|
3261
|
+
# (backend, model) PAIR is persisted on the record — a fleet flip must not
|
|
3262
|
+
# wake a codex run as claude (or with the new fleet's model). A legacy or
|
|
3263
|
+
# unreadable record is treated as claude (resume_author). The key then
|
|
3264
|
+
# resolves for THAT backend (keys coexist).
|
|
3265
|
+
try:
|
|
3266
|
+
_wake_record: object | None = load_record(args.run_root, args.resume)
|
|
3267
|
+
except Exception:
|
|
3268
|
+
# a wake must never crash on an unreadable/odd record — fall back to
|
|
3269
|
+
# the claude author (resume_author), same fail-safe as the sweep
|
|
3270
|
+
_wake_record = None
|
|
3271
|
+
wake_backend, wake_model, wake_key_file = resume_author(_wake_record, args.model)
|
|
3272
|
+
# an explicit --key-file still overrides (a manual re-run pinning a key)
|
|
3273
|
+
if args.key_file:
|
|
3274
|
+
wake_key_file = os.path.expanduser(args.key_file)
|
|
3275
|
+
_err = codex_author_config_error(wake_backend, wake_model, args.image)
|
|
3276
|
+
if _err:
|
|
3277
|
+
# this wake job HOLDS the run's lease (transferred on dispatch); release
|
|
3278
|
+
# it before exiting so a misconfig doesn't strand the run until the TTL
|
|
3279
|
+
# reap (the resume_run finally below only runs once we reach it)
|
|
3280
|
+
_release_own_lease(args.run_root, args.resume)
|
|
3281
|
+
parser.error(f"parked run {args.resume}: {_err}")
|
|
3282
|
+
# the wake runs the SAME verification panel as a fresh climb, so a
|
|
3283
|
+
# dispatched improvement is not published unverified.
|
|
3284
|
+
try:
|
|
3285
|
+
wake_lenses, wake_panel_secrets = _panel_lenses_from_args(args)
|
|
3286
|
+
except ValueError as exc:
|
|
3287
|
+
parser.error(str(exc))
|
|
3288
|
+
wake_api_key = ""
|
|
3289
|
+
wake_harness = None
|
|
3290
|
+
wake_spec = None
|
|
3291
|
+
# The editor harness is built when the wake may RESUME the session: a
|
|
3292
|
+
# panel is configured (a blocking finding wakes the author to revise),
|
|
3293
|
+
# or the park is an AUTHOR-SLEEP (that wake always resumes the session
|
|
3294
|
+
# with its launches' results). A panel-less candidate wake stays a pure
|
|
3295
|
+
# read-decide-publish job and must not require the author key.
|
|
3296
|
+
_wake_stage = getattr(_wake_record, "stage", None) or {}
|
|
3297
|
+
if (
|
|
3298
|
+
wake_lenses
|
|
3299
|
+
or _wake_stage.get("phase") == "author-sleep"
|
|
3300
|
+
or _wake_stage.get("submitted")
|
|
3301
|
+
):
|
|
3302
|
+
wake_api_key = role_key(wake_key_file, wake_backend)
|
|
3303
|
+
wake_spec = author_spec(max_turns=args.max_turns, walltime_s=args.session_minutes * 60)
|
|
3304
|
+
wake_harness = build_harness(
|
|
3305
|
+
wake_api_key,
|
|
3306
|
+
wake_spec,
|
|
3307
|
+
backend=wake_backend,
|
|
3308
|
+
binary=args.claude_bin if wake_backend == "claude" else args.codex_bin,
|
|
3309
|
+
model=wake_model,
|
|
3310
|
+
container_image=args.image,
|
|
3311
|
+
codex_extra_args=codex_extra,
|
|
3312
|
+
)
|
|
3313
|
+
wake_secrets = tuple(k for k in (bot_auth.token(), *wake_panel_secrets, wake_api_key) if k)
|
|
3314
|
+
try:
|
|
3315
|
+
resumed = resume_run(
|
|
3316
|
+
args.run_root,
|
|
3317
|
+
args.resume,
|
|
3318
|
+
dispatch=_dispatch_settings(args),
|
|
3319
|
+
github=GitHubClient(auth=bot_auth),
|
|
3320
|
+
bot_auth=bot_auth,
|
|
3321
|
+
now=time.time(),
|
|
3322
|
+
secrets=wake_secrets,
|
|
3323
|
+
base_branch=args.base_branch,
|
|
3324
|
+
panel_lenses=wake_lenses,
|
|
3325
|
+
harness=wake_harness,
|
|
3326
|
+
spec=wake_spec,
|
|
3327
|
+
)
|
|
3328
|
+
except GitError as exc:
|
|
3329
|
+
# A tamper-class GitError that escaped the wake body (a TOCTOU
|
|
3330
|
+
# object removal past the entry guard, or a git op that reached a
|
|
3331
|
+
# damaged object) ends the run cleanly as a refused wake — "a
|
|
3332
|
+
# refused wake ends the run" — instead of crashing the job into a
|
|
3333
|
+
# re-park that only wakes into the same failure. A non-tamper
|
|
3334
|
+
# GitError (push conflict, fetch outage) propagates unchanged.
|
|
3335
|
+
if not _is_git_tamper(exc):
|
|
3336
|
+
raise
|
|
3337
|
+
resumed = _end_refused_wake(
|
|
3338
|
+
args.run_root,
|
|
3339
|
+
load_record(args.run_root, args.resume),
|
|
3340
|
+
exc,
|
|
3341
|
+
time.time(),
|
|
3342
|
+
wake_secrets,
|
|
3343
|
+
)
|
|
3344
|
+
finally:
|
|
3345
|
+
# This wake job HOLDS the run's lease (the sweep transferred it on
|
|
3346
|
+
# dispatch); release it on every exit so a re-parked run is
|
|
3347
|
+
# immediately eligible for the next sweep instead of waiting out the
|
|
3348
|
+
# TTL reap. Idempotent (no-op if no lease file).
|
|
3349
|
+
_release_own_lease(args.run_root, args.resume)
|
|
3350
|
+
print(f"outcome={resumed.outcome} pr={resumed.pr_url or '-'} report={resumed.report_path}")
|
|
3351
|
+
return 0
|
|
3352
|
+
|
|
3353
|
+
if not (args.target and args.benchmark):
|
|
3354
|
+
parser.error("--target and --benchmark are required for a fresh climb")
|
|
3355
|
+
|
|
3356
|
+
# a fresh climb authors on the FLEET's configured backend; validate it (codex
|
|
3357
|
+
# writes+executes, so --image + a non-claude model) before any spend.
|
|
3358
|
+
_err = codex_author_config_error(args.author_backend, args.model, args.image)
|
|
3359
|
+
if _err:
|
|
3360
|
+
parser.error(_err)
|
|
3361
|
+
# config-driven: the author key defaults per backend (claude vs codex) so the
|
|
3362
|
+
# tick never threads it — see resolve_author_key_file (result is ~-expanded).
|
|
3363
|
+
args.key_file = resolve_author_key_file(args.author_backend, args.key_file)
|
|
3364
|
+
# same 0600 discipline as the PAT: this key spends real money. A missing
|
|
3365
|
+
# file is tolerated only when Vertex (ADC) covers the claude backend.
|
|
3366
|
+
api_key = role_key(args.key_file, args.author_backend)
|
|
3367
|
+
stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
|
|
3368
|
+
# the agent id keeps concurrent same-benchmark slots (the width dial's
|
|
3369
|
+
# portfolio case) from minting one run directory in the same second
|
|
3370
|
+
run_id = f"{args.benchmark}-{stamp}-{args.agent_id}"
|
|
3371
|
+
|
|
3372
|
+
# Disk preflight BEFORE any run state exists: a session started on a
|
|
3373
|
+
# full filesystem dies mid-flight in ways that lose its own evidence
|
|
3374
|
+
# (quota errors are invisible until a write fails on some clusters).
|
|
3375
|
+
from outerloop.disk import check_mount
|
|
3376
|
+
|
|
3377
|
+
health = check_mount(args.run_root, min_free_bytes=int(args.min_free_gb * 1024**3))
|
|
3378
|
+
if not health.ok():
|
|
3379
|
+
log.error("disk preflight failed: %s — refusing to start a run", health.describe())
|
|
3380
|
+
if args.issue:
|
|
3381
|
+
_best_effort(
|
|
3382
|
+
"issue report",
|
|
3383
|
+
lambda: GitHubClient(auth=bot_auth).comment(
|
|
3384
|
+
args.target,
|
|
3385
|
+
args.issue,
|
|
3386
|
+
"A run for this issue could not start: the orchestrator's "
|
|
3387
|
+
"storage failed its disk preflight. The claim on this issue "
|
|
3388
|
+
"stays until a maintainer removes the claim comment "
|
|
3389
|
+
"(automated claim release is on the roadmap).",
|
|
3390
|
+
),
|
|
3391
|
+
)
|
|
3392
|
+
return 3
|
|
3393
|
+
|
|
3394
|
+
# Armed LAST, immediately before the contained region — and DISARMED
|
|
3395
|
+
# right after it: a run finishing inside the margin must not have the
|
|
3396
|
+
# alarm fire during the uncontained epilogue (print/exit).
|
|
3397
|
+
import signal as _signal
|
|
3398
|
+
|
|
3399
|
+
armed = arm_self_deadline(args.job_minutes, args.deadline_margin_s)
|
|
3400
|
+
if armed:
|
|
3401
|
+
log.info("self-deadline armed: Terminated in %ds", armed)
|
|
3402
|
+
|
|
3403
|
+
# the manifest first, the harness from it: budget has one source (the args)
|
|
3404
|
+
spec = author_spec(max_turns=args.max_turns, walltime_s=args.session_minutes * 60)
|
|
3405
|
+
|
|
3406
|
+
# Pre-PR panel lenses: judge sessions on the verifier's own key (separate
|
|
3407
|
+
# identity from the author). kind[:backend[:model]]; claude by default.
|
|
3408
|
+
try:
|
|
3409
|
+
panel_lenses, panel_secrets = _panel_lenses_from_args(args)
|
|
3410
|
+
except ValueError as exc:
|
|
3411
|
+
parser.error(str(exc))
|
|
3412
|
+
|
|
3413
|
+
# Dispatched measurement needs the full cluster triple AND a real image
|
|
3414
|
+
# file to bind against; missing any, the climb measures inline (the tick
|
|
3415
|
+
# sets these on the climb job's env, a bare CLI run leaves them empty).
|
|
3416
|
+
dispatch: DispatchSettings | None = None
|
|
3417
|
+
if (bool(args.account and args.partition) or local_mode()) and (
|
|
3418
|
+
args.image and Path(args.image).is_file()
|
|
3419
|
+
):
|
|
3420
|
+
dispatch = _dispatch_settings(args)
|
|
3421
|
+
try:
|
|
3422
|
+
try:
|
|
3423
|
+
outcome = live_attempt(
|
|
3424
|
+
config=RunConfig(
|
|
3425
|
+
target=args.target, benchmark=args.benchmark, agent_id=args.agent_id
|
|
3426
|
+
),
|
|
3427
|
+
base_branch=args.base_branch,
|
|
3428
|
+
run_root=args.run_root,
|
|
3429
|
+
run_id=run_id,
|
|
3430
|
+
harness=build_harness(
|
|
3431
|
+
api_key,
|
|
3432
|
+
spec,
|
|
3433
|
+
backend=args.author_backend,
|
|
3434
|
+
binary=args.claude_bin if args.author_backend == "claude" else args.codex_bin,
|
|
3435
|
+
model=args.model,
|
|
3436
|
+
container_image=args.image,
|
|
3437
|
+
codex_extra_args=codex_extra,
|
|
3438
|
+
),
|
|
3439
|
+
spec=spec,
|
|
3440
|
+
panel_lenses=panel_lenses,
|
|
3441
|
+
dispatch=dispatch,
|
|
3442
|
+
eval_image=args.image,
|
|
3443
|
+
github=GitHubClient(auth=bot_auth),
|
|
3444
|
+
bot_auth=bot_auth,
|
|
3445
|
+
now=time.time(),
|
|
3446
|
+
created=datetime.now(UTC).isoformat(),
|
|
3447
|
+
# the panel key joins the redaction set: judge error text can
|
|
3448
|
+
# echo request material like any other model error
|
|
3449
|
+
secrets=tuple(
|
|
3450
|
+
k
|
|
3451
|
+
for k in (
|
|
3452
|
+
api_key,
|
|
3453
|
+
bot_auth.token(),
|
|
3454
|
+
*panel_secrets,
|
|
3455
|
+
)
|
|
3456
|
+
if k
|
|
3457
|
+
),
|
|
3458
|
+
issue_number=args.issue,
|
|
3459
|
+
author_backend=args.author_backend,
|
|
3460
|
+
author_model=args.model,
|
|
3461
|
+
author_key_file=args.key_file,
|
|
3462
|
+
task_hypothesis=(
|
|
3463
|
+
__import__("base64").b64decode(args.hypothesis_b64).decode()
|
|
3464
|
+
if args.hypothesis_b64
|
|
3465
|
+
else ""
|
|
3466
|
+
),
|
|
3467
|
+
)
|
|
3468
|
+
except Terminated as exc:
|
|
3469
|
+
# Fired in live_attempt's microseconds-wide pre-containment window:
|
|
3470
|
+
# any record it saved strands and the sweep ends it from Slurm
|
|
3471
|
+
# truth; here we only avoid dying as an unexplained traceback.
|
|
3472
|
+
log.error("self-deadline fired before containment: %s", exc)
|
|
3473
|
+
return 3
|
|
3474
|
+
finally:
|
|
3475
|
+
_signal.alarm(0)
|
|
3476
|
+
print(f"outcome={outcome.outcome} pr={outcome.pr_url or '-'} report={outcome.report_path}")
|
|
3477
|
+
return 0
|
|
3478
|
+
|
|
3479
|
+
|
|
3480
|
+
if __name__ == "__main__":
|
|
3481
|
+
raise SystemExit(main())
|