outerloop-science 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. outerloop/__init__.py +18 -0
  2. outerloop/__main__.py +3 -0
  3. outerloop/appauth.py +213 -0
  4. outerloop/appmanifest.py +198 -0
  5. outerloop/attempt.py +3481 -0
  6. outerloop/brief.py +515 -0
  7. outerloop/cli.py +439 -0
  8. outerloop/climbboard.py +1145 -0
  9. outerloop/compute.py +482 -0
  10. outerloop/contract.py +483 -0
  11. outerloop/contract_cli.py +63 -0
  12. outerloop/disk.py +164 -0
  13. outerloop/dispatch.py +586 -0
  14. outerloop/followup.py +2143 -0
  15. outerloop/github.py +1486 -0
  16. outerloop/harness.py +1449 -0
  17. outerloop/housekeeping.py +167 -0
  18. outerloop/init.py +313 -0
  19. outerloop/intake.py +129 -0
  20. outerloop/limits.py +80 -0
  21. outerloop/markers.py +48 -0
  22. outerloop/measure.py +523 -0
  23. outerloop/orchestrator.py +1901 -0
  24. outerloop/panel.py +188 -0
  25. outerloop/paths.py +27 -0
  26. outerloop/posting.py +160 -0
  27. outerloop/progress.py +170 -0
  28. outerloop/py.typed +0 -0
  29. outerloop/review.py +611 -0
  30. outerloop/review_agent.py +263 -0
  31. outerloop/review_agent_cli.py +209 -0
  32. outerloop/review_post_cli.py +162 -0
  33. outerloop/review_summarize_cli.py +163 -0
  34. outerloop/role_runner.py +229 -0
  35. outerloop/roles.py +247 -0
  36. outerloop/rolespec.py +89 -0
  37. outerloop/runstate.py +385 -0
  38. outerloop/steward.py +852 -0
  39. outerloop/style.py +12 -0
  40. outerloop/syscall.py +977 -0
  41. outerloop/syscall_cli.py +531 -0
  42. outerloop/tick.py +3166 -0
  43. outerloop/verifier.py +403 -0
  44. outerloop/verify_agent.py +149 -0
  45. outerloop/verify_agent_cli.py +95 -0
  46. outerloop/verify_post_cli.py +116 -0
  47. outerloop_science-0.1.0.dev0.dist-info/METADATA +145 -0
  48. outerloop_science-0.1.0.dev0.dist-info/RECORD +52 -0
  49. outerloop_science-0.1.0.dev0.dist-info/WHEEL +4 -0
  50. outerloop_science-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  51. outerloop_science-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
  52. outerloop_science-0.1.0.dev0.dist-info/licenses/NOTICE +5 -0
outerloop/attempt.py ADDED
@@ -0,0 +1,3481 @@
1
+ """One live climb, end to end: clone → attempt_once → commit/push/PR → report.
2
+
3
+ This is the glue `orchestrator.attempt_once` deliberately does not own: the git
4
+ side (bot-auth clone, veto-checked commit, push, PR) and the run's durable
5
+ record. One invocation = one run = at most one PR.
6
+
7
+ Credential separation holds throughout: the bot PAT is read orchestrator-side
8
+ and used only by Workspace network calls and the PR client, after the session
9
+ has ended; the session sees only its own capped API key inside its container.
10
+ """
11
+
12
+ from __future__ import annotations
13
+
14
+ import argparse
15
+ import contextlib
16
+ import json
17
+ import logging
18
+ import os
19
+ import re
20
+ import shutil
21
+ from collections.abc import Callable, Iterable
22
+ from dataclasses import dataclass
23
+ from dataclasses import replace as dc_replace
24
+ from functools import partial
25
+ from pathlib import Path
26
+ from typing import Any, cast
27
+
28
+ from outerloop.appauth import resolve_bot_auth
29
+ from outerloop.brief import BudgetState, distill_lessons
30
+ from outerloop.compute import LocalCompute, local_mode
31
+ from outerloop.contract import Benchmark, Contract, contract_text_in_tree, load_contract
32
+ from outerloop.dispatch import (
33
+ Snapshot,
34
+ afterany_ids,
35
+ drop_snapshot,
36
+ should_dispatch,
37
+ snapshot_tree,
38
+ )
39
+ from outerloop.github import (
40
+ GitError,
41
+ GitHubClient,
42
+ TokenProvider,
43
+ Workspace,
44
+ contract_at,
45
+ ensure_regular_git_dir,
46
+ )
47
+ from outerloop.harness import Harness, SessionResult, redact
48
+ from outerloop.markers import has_marker
49
+ from outerloop.measure import DispatchedMeasurer, DispatchSettings
50
+ from outerloop.orchestrator import (
51
+ AttemptResult,
52
+ EvalError,
53
+ Measurer,
54
+ RunConfig,
55
+ RunParked,
56
+ _benchmark,
57
+ attempt_once,
58
+ pr_body,
59
+ resume_attempt,
60
+ )
61
+ from outerloop.panel import PanelLens, PanelVerdict, run_panel
62
+ from outerloop.paths import CONFIG_DIR
63
+ from outerloop.progress import (
64
+ PROGRESS_PATHS,
65
+ load_leader,
66
+ update_leader,
67
+ write_progress,
68
+ )
69
+ from outerloop.review import PullRequest
70
+ from outerloop.role_runner import build_harness, role_key
71
+ from outerloop.roles import author_spec
72
+ from outerloop.rolespec import RoleSpec
73
+ from outerloop.runstate import (
74
+ ABORTED,
75
+ BUDGET_EXHAUSTED,
76
+ ENDED,
77
+ IN_REVIEW,
78
+ NEGATIVE_RESULT,
79
+ STUCK,
80
+ WAITING,
81
+ RunRecord,
82
+ list_runs,
83
+ load_record,
84
+ save_record,
85
+ stamp_outage,
86
+ )
87
+ from outerloop.syscall import CHANNEL_DIR_NAMES, MAX_ARTIFACT_BYTES, SyscallRequest, channel_dir
88
+ from outerloop.syscall import ensure_excluded as syscall_excluded
89
+ from outerloop.syscall import install_tool as syscall_install_tool
90
+ from outerloop.syscall import write_budget as syscall_write_budget
91
+ from outerloop.syscall import write_siblings as syscall_write_siblings
92
+ from outerloop.verifier import MAX_CLAIM_CHARS
93
+
94
+ log = logging.getLogger(__name__)
95
+
96
+ # Where a climb job reads its keys unless the CLI flags say otherwise. The
97
+ # tick preflights the panel key (and compares it against the author key —
98
+ # role separation) before claiming/submitting.
99
+ PANEL_KEY_DEFAULT = str(CONFIG_DIR / "verifier_key")
100
+ HARNESS_KEY_DEFAULT = str(CONFIG_DIR / "harness_key") # the claude author key
101
+ CODEX_KEY_DEFAULT = str(CONFIG_DIR / "codex_key") # the codex author key
102
+
103
+
104
+ def resolve_author_key_file(backend: str, explicit: str = "") -> str:
105
+ """The author key file for `backend`. Per-backend keys COEXIST (claude's and
106
+ codex's both on disk), selected by backend — so the author backend is a
107
+ config choice, not a key swap, and an in-flight run of either backend can
108
+ still be woken/serviced after a fleet flip. An explicit path always wins;
109
+ otherwise the per-backend env var, then the packaged default path. The result
110
+ is always ~-expanded, so every caller gets a real path (an env value like
111
+ "~/.config/..." must not reach the token provider verbatim)."""
112
+ if not explicit:
113
+ if backend == "codex":
114
+ explicit = os.environ.get("AUTORESEARCH_CODEX_KEY_FILE") or CODEX_KEY_DEFAULT
115
+ else:
116
+ explicit = os.environ.get("AUTORESEARCH_HARNESS_KEY_FILE") or HARNESS_KEY_DEFAULT
117
+ return os.path.expanduser(explicit)
118
+
119
+
120
+ def codex_author_config_error(backend: str, model: str, image: str) -> str:
121
+ """Why a codex author would die at startup ("" when it won't). Validates the
122
+ EFFECTIVE (backend, model) — the fresh climb passes args; a wake/follow-up
123
+ passes the PARKED RUN's persisted pair — so backend and model are checked as
124
+ a unit and never a fleet backend against a run's model. codex writes+executes,
125
+ so it must be contained (--image) and needs a non-claude model."""
126
+ if backend not in ("claude", "codex"):
127
+ # a typo'd AUTORESEARCH_AUTHOR_BACKEND passes the env DEFAULT silently
128
+ # (argparse validates the flag, not its default) and the climb rejects it
129
+ # at build_harness — catch it on the tick host so a claimed intake
130
+ # issue never strands on it
131
+ return f"unknown author backend {backend!r} (expected 'claude' or 'codex')"
132
+ if backend == "claude":
133
+ # symmetric to the codex check: a claude harness 404s on a non-claude
134
+ # model (e.g. AUTORESEARCH_AUTHOR_MODEL left on a codex id while the
135
+ # backend is claude) — catch that misconfig before spend
136
+ if model and not model.startswith("claude"):
137
+ return f"author-backend claude needs a claude model (got {model!r})"
138
+ return ""
139
+ if not image:
140
+ return "author-backend codex requires --image (it runs contained)"
141
+ if not model or model.startswith("claude"):
142
+ return (
143
+ "author-backend codex needs a codex/openai model "
144
+ f"(e.g. gpt-5.6-terra), not the claude default (got {model!r})"
145
+ )
146
+ return ""
147
+
148
+
149
+ def resume_author(record: object, fleet_model: str) -> tuple[str, str, str]:
150
+ """The (backend, model, key_file) a wake/follow-up must reproduce for a parked
151
+ run — all from the RECORD, not the current fleet.
152
+
153
+ An empty backend is a legacy record (written before the field) and is
154
+ therefore CLAUDE, never the fleet default; the model pairs with that backend
155
+ (a claude backend falls back to the claude default, a codex backend to the
156
+ fleet model only as a last resort — codex records always carry their model);
157
+ the key file is the exact resolved path the run used (so an explicit
158
+ --key-file survives), falling back to the per-backend resolution for legacy
159
+ records that never recorded it."""
160
+ backend = getattr(record, "author_backend", "") or "claude"
161
+ model = getattr(record, "author_model", "") or (
162
+ "claude-opus-5" if backend == "claude" else fleet_model
163
+ )
164
+ key_file = getattr(record, "author_key_file", "") or resolve_author_key_file(backend)
165
+ return backend, model, key_file
166
+
167
+
168
+ class WorkspaceDrift(RuntimeError):
169
+ """The tree changed between measurement and commit."""
170
+
171
+
172
+ def _target_clone_url(target: str) -> str:
173
+ """The canonical HTTPS clone URL for `owner/repo`. The one source of truth
174
+ for where a run's git pushes go — derived from the target, never read from
175
+ the session-writable `remote.origin.url`."""
176
+ return f"https://github.com/{target}.git"
177
+
178
+
179
+ def _blessed_head(ws: Workspace, result: Any, contract: Any) -> str:
180
+ """The pushed PR head the tick may later self-merge — only when this
181
+ publish was under merge:auto with a CLEAN panel (#171's arming
182
+ condition); "" otherwise. Best-effort: an unreadable HEAD blesses
183
+ nothing (never arm on doubt)."""
184
+ if not (
185
+ result.panel_rounds > 0
186
+ and not (result.panel_blocking_open or result.panel_degraded)
187
+ and getattr(contract, "merge", "manual") == "auto"
188
+ ):
189
+ return ""
190
+ try:
191
+ return ws.git("rev-parse", "HEAD").strip()
192
+ except Exception:
193
+ return ""
194
+
195
+
196
+ def _arm_unless_base_moved(
197
+ github: GitHubClient,
198
+ ws: Workspace,
199
+ target: str,
200
+ pr_number: str,
201
+ base_branch: str,
202
+ measured_base_sha: str,
203
+ secrets: tuple[str, ...],
204
+ merge_mode: str = "manual",
205
+ panel_ran: bool = False,
206
+ ) -> None:
207
+ """Arm auto-merge only while origin/<base_branch> still equals the base the
208
+ claim was measured against. A moved base still OPENS the PR — review owns
209
+ staleness — but never ARMS it: merging a tree whose gate/suite/panel read
210
+ is stale must be a human's deliberate act, not an armed automation. A
211
+ failed freshness fetch also declines to arm (fail-safe: un-armed is just a
212
+ normal PR). Best-effort throughout, like arming itself."""
213
+
214
+ def _check_and_arm() -> None:
215
+ ws.git_network("fetch", str(ws.url or ws.remote_url()), base_branch)
216
+ fresh = ws.git("rev-parse", "FETCH_HEAD").strip()
217
+ if fresh != measured_base_sha:
218
+ log.info(
219
+ "not arming auto-merge on %s#%s: %s moved since the claim was "
220
+ "measured (%s -> %s); a human merges this one",
221
+ target,
222
+ pr_number,
223
+ base_branch,
224
+ measured_base_sha[:12],
225
+ fresh[:12],
226
+ )
227
+ return
228
+ if merge_mode == "auto" and not panel_ran:
229
+ # the dial's own precondition: auto means GATE+PANEL clean, so a
230
+ # publish that ran no panel must not self-merge — fall back to
231
+ # the manual guard and say so (terra #171: a panel-less
232
+ # deployment could otherwise self-merge on the metric gate alone)
233
+ log.warning(
234
+ "merge mode auto on %s#%s but no panel ran this attempt; "
235
+ "arming manual-mode instead",
236
+ target,
237
+ pr_number,
238
+ )
239
+ if merge_mode == "auto" and panel_ran:
240
+ # the contract's autonomy dial: the owner opted this repo into
241
+ # self-merging gate-clean PRs — arm, or merge directly when
242
+ # nothing is pending to arm against
243
+ github.arm_auto_merge_auto_mode(target, int(pr_number))
244
+ else:
245
+ github.arm_auto_merge_when_review_required(target, int(pr_number))
246
+
247
+ _best_effort("auto-merge arming", _check_and_arm, secrets)
248
+
249
+
250
+ def _title_pair(a: float, b: float) -> str:
251
+ """Compact but never ambiguous: widen precision until the two numbers
252
+ render differently (a title reading '10.00 -> 10.00' looks like no
253
+ change even when the improvement is real)."""
254
+ for precision in range(4, 12):
255
+ fa, fb = f"{a:.{precision}g}", f"{b:.{precision}g}"
256
+ if fa != fb:
257
+ return f"{fa} -> {fb}"
258
+ return f"{a} -> {b}"
259
+
260
+
261
+ RULER = (
262
+ "The metric is computed by the contract's eval command over a frozen "
263
+ "instance pool. Your claim is verified by the orchestrator re-running "
264
+ "that exact command on your tree — and again by CI after the PR opens. "
265
+ "Only changes inside the contract's allowed paths are ever measured."
266
+ )
267
+
268
+ _ENDINGS_BY_OUTCOME = {
269
+ "no-improvement": NEGATIVE_RESULT,
270
+ # the improvement was real but bought by regressing a sibling benchmark —
271
+ # an honest negative with a named cause, not a malfunction
272
+ "suite-regression": NEGATIVE_RESULT,
273
+ "session-error": ABORTED,
274
+ "session-budget": BUDGET_EXHAUSTED,
275
+ "session-outage": STUCK, # infrastructure failure, nothing about the run
276
+ "eval-error": ABORTED,
277
+ "scope-violation": ABORTED,
278
+ }
279
+
280
+
281
+ @dataclass(frozen=True)
282
+ class AttemptOutcome:
283
+ run_id: str
284
+ outcome: str
285
+ pr_url: str = ""
286
+ report_path: str = ""
287
+
288
+
289
+ def _best_effort(what: str, fn: Callable[[], object], secrets: tuple[str, ...] = ()) -> bool:
290
+ """One ending step; a failure is logged, never raised.
291
+
292
+ The terminal sequence (record, report, issue post) must degrade
293
+ independently: a full disk must not block the GitHub post, and a network
294
+ failure must not block the record.
295
+ """
296
+ try:
297
+ fn()
298
+ return True
299
+ except Exception as exc:
300
+ log.warning("%s failed: %s", what, redact(f"{type(exc).__name__}: {exc}", secrets))
301
+ return False
302
+
303
+
304
+ def _clear_stage(record: RunRecord) -> RunRecord:
305
+ """Strip the WAITING-only bookkeeping from a record leaving `waiting` for a
306
+ terminal state. Otherwise a dispatched run's `stage`, `deadline`, and
307
+ especially `wake_attempts` ride into `in-review`, where in-review follow-up
308
+ servicing reuses `wake_attempts` as its OWN retry cap — so a run that woke
309
+ once would reach review with a shrunk follow-up budget."""
310
+ # the run's spend survives the wipe: terminal reporting (the climb
311
+ # board) reads it after the transition
312
+ kept = {k: record.stage[k] for k in ("gpu_hours_used",) if record.stage and k in record.stage}
313
+ return dc_replace(
314
+ record,
315
+ stage=kept,
316
+ experiment_job_id="",
317
+ deadline=0.0,
318
+ terminal_seen=0.0,
319
+ wake_attempts=0,
320
+ )
321
+
322
+
323
+ def _post_issue_finished(
324
+ github: GitHubClient,
325
+ target: str,
326
+ issue_number: int,
327
+ run_id: str,
328
+ outcome_name: str,
329
+ pr_url: str,
330
+ summary: str,
331
+ secrets: tuple[str, ...],
332
+ ) -> None:
333
+ """Post a run's terminal result back to the issue that requested it. When
334
+ the run ends WITHOUT a PR (a negative or an error), include the
335
+ `RELEASE_MARKER` so `intake.pick_issue` can re-select the issue — a comment
336
+ alone does NOT un-claim it. An improved run KEEPS the claim: its PR
337
+ (`Addresses #N`) is the ongoing work, and `followup` releases the claim if
338
+ that PR later closes unmerged."""
339
+ if not issue_number:
340
+ return
341
+ from outerloop.intake import RELEASE_MARKER
342
+
343
+ link = f"\n\nPull request: {pr_url}" if pr_url else ""
344
+ # no PR opened -> nothing will ever resolve this issue, so free the claim
345
+ # (bounded by intake's per-issue attempt cap).
346
+ release = f"{RELEASE_MARKER}\n" if not pr_url else ""
347
+ _best_effort(
348
+ "issue report",
349
+ lambda: github.comment(
350
+ target,
351
+ issue_number,
352
+ f"{release}Run `{run_id}` finished ({outcome_name}).{link}\n\n{summary}",
353
+ ),
354
+ secrets,
355
+ )
356
+
357
+
358
+ # A parked run's deadline is the FLOOR beneath the afterany wake: submit +
359
+ # eval walltime + a generous queue/grace allowance. It must exceed the time a
360
+ # healthy eval can legitimately sit queued-then-running, or `tick._sweep_one`
361
+ # would cancel a still-queued job as "unschedulable".
362
+ PARK_QUEUE_SLACK_MIN = 12 * 60
363
+ # a jobless checkpoint sleep only needs to survive to the next sweep pass:
364
+ # one cadence + coalescing headroom, not queue slack
365
+ CHECKPOINT_SLEEP_SLACK_MIN = 45
366
+
367
+
368
+ def _park_run(
369
+ run_root: Path,
370
+ record: RunRecord,
371
+ parked: RunParked,
372
+ candidate_ref: str,
373
+ eval_minutes: int | None,
374
+ now: float,
375
+ secrets: tuple[str, ...] = (),
376
+ keep_wake_attempts: bool = False,
377
+ base_branch: str = "main",
378
+ panel_reads: int = 0,
379
+ dispatch: DispatchSettings | None = None,
380
+ ) -> None:
381
+ """Persist a dispatched climb's re-entry point as a WAITING record: the
382
+ committed shas, drawn seeds, candidate snapshot ref, and afterany set a
383
+ fresh process reconstructs the measure-and-decide phase from. The caller
384
+ passes the EXACT `candidate_ref` it will keep alive (never re-derive it from
385
+ the commit — two snapshots can share a commit)."""
386
+ from outerloop.dispatch import effective_eval_minutes
387
+
388
+ job_ids = afterany_ids(parked.afterany)
389
+ stage: dict[str, object] = {
390
+ "phase": parked.phase,
391
+ "base_sha": parked.base_sha,
392
+ "candidate_sha": parked.candidate_sha,
393
+ "candidate_ref": candidate_ref,
394
+ "seed": parked.seed,
395
+ "suite_seed": parked.suite_seed,
396
+ "afterany": parked.afterany,
397
+ "launch_afterany": parked.launch_afterany,
398
+ # the branch the run targets, so a wake opens its PR against the SAME
399
+ # branch a non-default `--base-branch` selected — the wake CLI otherwise
400
+ # defaults to main and would mis-target.
401
+ "base_branch": base_branch,
402
+ # verification-panel reads so far — persisted so the next wake
403
+ # continues the count.
404
+ "panel_reads": panel_reads,
405
+ # the session's write-up + spend, saved so a candidate wake can build
406
+ # the PR body / panel claim and report the real cost WITHOUT re-running
407
+ # the session (its edits are already in candidate_sha). REDACTED before
408
+ # it lands in the durable record, like every other persisted final_text
409
+ # — a session that echoed a credential must not leave it in record.json.
410
+ # Empty/zero for a baseline park (the session has not run yet).
411
+ "report": redact(parked.session.final_text, secrets)[:MAX_CLAIM_CHARS]
412
+ if parked.session
413
+ else "",
414
+ "session_cost_usd": parked.session.cost_usd if parked.session else 0.0,
415
+ "session_turns": parked.session.num_turns if parked.session else 0,
416
+ }
417
+ if parked.submitted:
418
+ # a SUBMITTED candidate park (buildout Phase B): the wake delivers the
419
+ # gate + panel results back to the author instead of deciding by policy
420
+ stage["submitted"] = True
421
+ if parked.syscall is not None:
422
+ # A syscall park — an author-directed sleep (research-loop-buildout.md
423
+ # Phase A) or a submitted candidate carrying sibling launches: the wake
424
+ # gathers each launch's results by NAME from the run dir, delivers the
425
+ # declared artifacts, and resumes the SAME session — so the stage must
426
+ # carry the launch names/artifacts, the author's note, and the budget
427
+ # counts as of this park.
428
+ stage["syscall_launches"] = [
429
+ # minutes ride along so a RE-PARK's deadline floor still covers the
430
+ # longest launch — without them a rebuilt descriptor would
431
+ # undershoot the floor and the sweep could cancel a healthy
432
+ # queued sibling as "pending past deadline"
433
+ {
434
+ "name": launch.name,
435
+ "minutes": launch.minutes,
436
+ "artifacts": list(launch.artifacts),
437
+ **({"array": launch.array} if launch.array > 1 else {}),
438
+ }
439
+ for launch in parked.syscall.launches
440
+ ]
441
+ stage["syscall_note"] = redact(parked.syscall.note, secrets)
442
+ # (the session id the wake resumes is the record's own
443
+ # resume_session_id, set below for every park — no stage duplicate)
444
+ stage["launches_used"] = parked.launches_used
445
+ stage["sleeps_used"] = parked.sleeps_used
446
+ stage["gpu_hours_used"] = parked.gpu_hours_used
447
+ if parked.judged is not None:
448
+ # the gate's last negative rides the park: a wake that ends on the
449
+ # same tree reuses it instead of measuring again
450
+ judged_sha, verdict = parked.judged
451
+ stage["judged"] = {
452
+ "sha": judged_sha,
453
+ "outcome": verdict.outcome,
454
+ "baseline": verdict.baseline,
455
+ "candidate": verdict.candidate,
456
+ "note": redact(verdict.note, secrets),
457
+ }
458
+ if parked.eval_minutes:
459
+ # the author's declared eval walltime rides the park: the wake's
460
+ # measurer and deadline floor must use it, not the contract's
461
+ stage["eval_minutes"] = parked.eval_minutes
462
+ # A single-job park records its one pollable id; a MULTI-job park records
463
+ # none — the sweep falls back to polling every id in the stage's `afterany`
464
+ # string and wakes only when ALL are done (tick._poll_targets).
465
+ experiment_job_id = job_ids[0] if len(job_ids) == 1 else ""
466
+ # The deadline is a FLOOR: park time (`now` here is the park moment, passed
467
+ # by the caller) + the eval walltime + a generous queue/grace slack, so a
468
+ # healthy queued-then-running eval never trips the sweep's cancel-on-pending.
469
+ floor_minutes = effective_eval_minutes(parked.eval_minutes or eval_minutes)
470
+ if parked.phase == "author-sleep" and parked.syscall is not None:
471
+ # an author launch's walltime is the LAUNCH's ask, not the benchmark's
472
+ # eval hint — the floor must sit past the LONGEST launch, or the sweep
473
+ # cancels still-queued author jobs (a benchmark can be in-job cheap,
474
+ # eval_minutes=None, while its author trains for hours). A checkpoint
475
+ # sleep has no jobs: floor 0 wakes it at the first deadline pass.
476
+ floor_minutes = max((la.minutes for la in parked.syscall.launches), default=0)
477
+ elif parked.syscall is not None:
478
+ # a submitted candidate with sibling launches waits on gate evals AND
479
+ # launches — the floor must sit past the longest of either
480
+ floor_minutes = max(floor_minutes, *(la.minutes for la in parked.syscall.launches), 0)
481
+ checkpoint_sleep = (
482
+ parked.phase == "author-sleep"
483
+ and parked.syscall is not None
484
+ and not parked.syscall.launches
485
+ )
486
+ if checkpoint_sleep:
487
+ # a CHECKPOINT SLEEP has nothing in any queue, so the 12h queue slack
488
+ # (sized to protect queued Slurm jobs from cancel-on-pending) does not
489
+ # apply — the deadline needs only to reach the sweep's next pass.
490
+ # Observed live (yolo heldout_probe, 2026-08-27): a jobless nap
491
+ # inherited the queue slack and became a 12h coma.
492
+ deadline = now + CHECKPOINT_SLEEP_SLACK_MIN * 60
493
+ else:
494
+ deadline = now + (floor_minutes + PARK_QUEUE_SLACK_MIN) * 60
495
+ waiting = RunRecord(
496
+ **{
497
+ **record.__dict__,
498
+ "state": WAITING,
499
+ "experiment_job_id": experiment_job_id,
500
+ "resume_session_id": parked.session.session_id if parked.session else "",
501
+ "deadline": deadline,
502
+ "stage": stage,
503
+ # wake_attempts = "wakes since the run last made progress"; the
504
+ # stuck cap ends a run that keeps waking without advancing. Reset on
505
+ # a PRODUCTIVE park (the IMPLEMENTING->park entry ran a session; a
506
+ # wake that resolved its measures and dispatched NEW ones). A
507
+ # no-progress re-park — results still pending, or a blind re-park
508
+ # (squeue unreachable, nothing new dispatched) — must KEEP the
509
+ # counter (`keep_wake_attempts`), or the loop never reaches the cap.
510
+ "wake_attempts": record.wake_attempts if keep_wake_attempts else 0,
511
+ "terminal_seen": 0.0,
512
+ }
513
+ )
514
+ save_record(run_root, waiting, now)
515
+ if dispatch is not None:
516
+ _arm_park_wake(run_root, record.run_id, now, dispatch)
517
+
518
+
519
+ def _arm_park_wake(run_root: Path, run_id: str, now: float, dispatch: DispatchSettings) -> str:
520
+ """Submit the parked run's wake right away, depending on the jobs it
521
+ waits on (tick.arm_wake), when the tick has published its wake recipe.
522
+ Without the recipe — dispatched wakes not armed, or a local compute —
523
+ the sweep delivers as before."""
524
+ from outerloop.compute import SlurmCompute
525
+ from outerloop.tick import JobWakeDispatcher, arm_wake, dispatch_wake_armed, load_wake_spec
526
+
527
+ if not dispatch_wake_armed(run_root):
528
+ return "" # disarmed: a recipe the tick has not yet removed is not used
529
+ spec = load_wake_spec(run_root)
530
+ if spec is None or not isinstance(dispatch.compute, SlurmCompute):
531
+ return ""
532
+ try:
533
+ record = load_record(run_root, run_id)
534
+ dispatcher = JobWakeDispatcher(dispatch.compute, spec, now)
535
+ return arm_wake(
536
+ run_root, record, dispatcher, now, holder_job_id=os.environ.get("SLURM_JOB_ID", "")
537
+ )
538
+ except Exception as exc:
539
+ log.warning("park-time wake not armed for %s: %s: %s", run_id, type(exc).__name__, exc)
540
+ return ""
541
+
542
+
543
+ def _lease_held_by_another_job(run_root: Path, run_id: str) -> str:
544
+ """The job id of a wake that holds this run's lease and is not us, or "".
545
+ A resume with no job id of its own (a manual run) never counts as the
546
+ holder of a job-held lease."""
547
+ from outerloop.runstate import read_lease
548
+
549
+ lease = read_lease(run_root, run_id)
550
+ mine = os.environ.get("SLURM_JOB_ID", "")
551
+ if lease is not None and lease.holder_job_id and lease.holder_job_id != mine:
552
+ return lease.holder_job_id
553
+ return ""
554
+
555
+
556
+ def _release_own_lease(run_root: Path, run_id: str) -> None:
557
+ """Release the run's lease only while this job still holds it: a park
558
+ hands the lease to the wake it arms, and that wake must keep it."""
559
+ from outerloop.runstate import release_lease
560
+
561
+ if _lease_held_by_another_job(run_root, run_id):
562
+ return
563
+ release_lease(run_root, run_id)
564
+
565
+
566
+ def _dispatch_settings(args: argparse.Namespace) -> DispatchSettings:
567
+ """The cluster coordinates from the CLI, read in ONE place for both the
568
+ fresh climb and the wake (a second constructor drifted once — terra
569
+ #174: the wake dropped the GPU lane)."""
570
+ from outerloop.compute import compute_from_env
571
+
572
+ return DispatchSettings(
573
+ compute=compute_from_env(),
574
+ image=args.image,
575
+ account=args.account,
576
+ partition=args.partition,
577
+ gpu_partition=getattr(args, "gpu_partition", ""),
578
+ gpu_account=getattr(args, "gpu_account", ""),
579
+ )
580
+
581
+
582
+ def _make_launcher(
583
+ dispatch: DispatchSettings, run_dir: Path, workspace: Path, run_id: str, gpus: int = 0
584
+ ):
585
+ """The launch side of the author syscalls, shared by the first pass
586
+ (live_attempt) and the author-sleep wake: each launch becomes a jailed job on
587
+ the sealed snapshot (write_eval_job's copy-out handles artifacts), and a
588
+ partially-submitted batch is reaped rather than orphaned. `gpus` is the
589
+ benchmark's: an author's experiments run on the same lane as its evals."""
590
+ account, partition = dispatch.placement(gpus)
591
+
592
+ def launcher(sha: str, request: SyscallRequest) -> str:
593
+ from outerloop.dispatch import eval_job_spec, write_eval_job
594
+ from outerloop.syscall import launch_jobs
595
+
596
+ ids: list[str] = []
597
+ try:
598
+ for launch in request.launches:
599
+ # an array launch is N jobs of one command, each with its
600
+ # SWEEP_INDEX; one afterany wake covers them all
601
+ for job_name, extra_env in launch_jobs(launch):
602
+ script = write_eval_job(
603
+ run_dir,
604
+ f"launch-{job_name}",
605
+ repo_root=workspace,
606
+ snapshot_sha=sha,
607
+ command=launch.command,
608
+ image=dispatch.image,
609
+ extra_env=extra_env,
610
+ artifacts=launch.artifacts,
611
+ artifact_max_bytes=MAX_ARTIFACT_BYTES,
612
+ gpus=gpus,
613
+ )
614
+ ids.append(
615
+ dispatch.compute.submit(
616
+ eval_job_spec(
617
+ script,
618
+ job_name=f"{run_id}-launch-{job_name}",
619
+ account=account,
620
+ partition=partition,
621
+ eval_minutes=launch.minutes,
622
+ gpus=gpus,
623
+ )
624
+ )
625
+ )
626
+ except Exception:
627
+ # a partial batch must not orphan: no park record was written yet,
628
+ # so nothing would ever wake or cancel the jobs that DID submit —
629
+ # reap them here, then let the caller end the run as the error it
630
+ # is (same stance as the failed-_park_run cancel).
631
+ for job_id in ids:
632
+ with contextlib.suppress(Exception):
633
+ dispatch.compute.cancel(job_id)
634
+ raise
635
+ # a checkpoint sleep (no launches) parks with no dependency and wakes
636
+ # on the sweep's deadline floor — slow but correct; a fast requeue wake
637
+ # is a follow-up.
638
+ return "afterany:" + ":".join(ids) if ids else ""
639
+
640
+ return launcher
641
+
642
+
643
+ def _wake_author_sleep(
644
+ *,
645
+ run_root: Path,
646
+ run_id: str,
647
+ record: RunRecord,
648
+ ws: Workspace,
649
+ workspace: Path,
650
+ run_dir: Path,
651
+ dispatch: DispatchSettings,
652
+ github: GitHubClient,
653
+ now: float,
654
+ secrets: tuple[str, ...],
655
+ base_branch: str,
656
+ base_sha: str,
657
+ sleep_ref: str,
658
+ contract_text: str,
659
+ contract: Contract,
660
+ bench: Benchmark,
661
+ config: RunConfig,
662
+ measurer: Measurer,
663
+ harness: Harness | None,
664
+ spec: RoleSpec | None,
665
+ panel_lenses: tuple[PanelLens, ...],
666
+ issue_number: int,
667
+ eval_minutes: int | None,
668
+ extra_update: str = "",
669
+ judged: tuple[str, AttemptResult] | None = None,
670
+ ) -> AttemptOutcome:
671
+ """Wake a syscall park and resume the AUTHOR: deliver the launches' results
672
+ into the sandbox, resume the SAME session through the climb's resume-entry
673
+ with them (data-fenced), and let the climb run — it may sleep again
674
+ (re-park), submit, finish into the gate (whose dispatched measures park it
675
+ as a CANDIDATE), or end on a terminal. The session's workspace persisted on
676
+ disk exactly as the author left it (the launches ran on node-local
677
+ checkouts of the sealed sha), so the resumed session continues its own tree
678
+ — cumulative depth. `extra_update` leads the wake text — a SUBMITTED park's
679
+ gate result or panel verdict (buildout Phase B), delivered back to the
680
+ author to act on; `sleep_ref` is whichever snapshot ref this park holds
681
+ (the sleep seal, or the submitted candidate)."""
682
+ from outerloop.syscall import Launch as SyscallLaunch
683
+ from outerloop.syscall import (
684
+ annotate_launch_states,
685
+ gather_results,
686
+ render_wake,
687
+ write_budget,
688
+ )
689
+
690
+ def _end(result: AttemptResult, drop_refs: list[str]) -> AttemptOutcome:
691
+ # a terminal from the resumed climb: report, ending record, issue note —
692
+ # the same ending shape every other terminal takes. The line notebook
693
+ # records it first, while the tree is still the session's final tree.
694
+ _push_line_snapshot(
695
+ ws, _line_ref_for(bench, config.agent_id), run_id, result.outcome, secrets
696
+ )
697
+ for ref in drop_refs:
698
+ drop_snapshot(ws, Snapshot(commit="", tree="", ref=ref))
699
+ report_path = run_dir / "report.md"
700
+ _best_effort(
701
+ "run report",
702
+ lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
703
+ secrets,
704
+ )
705
+ ending = _ENDINGS_BY_OUTCOME.get(result.outcome, ABORTED)
706
+ final = _clear_stage(
707
+ RunRecord(
708
+ **{
709
+ **record.__dict__,
710
+ "state": ENDED,
711
+ "ending": ending,
712
+ "ending_note": redact(result.note, secrets),
713
+ }
714
+ )
715
+ )
716
+ _best_effort("final record", lambda: save_record(run_root, final, now), secrets)
717
+ _post_issue_finished(
718
+ github,
719
+ config.target,
720
+ issue_number,
721
+ run_id,
722
+ result.outcome,
723
+ "",
724
+ redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
725
+ secrets,
726
+ )
727
+ return AttemptOutcome(run_id=run_id, outcome=result.outcome, report_path=str(report_path))
728
+
729
+ # The wake NEEDS the author harness (it resumes the session). Fail as a
730
+ # named ending, not a crash: the run cannot proceed and re-waking will not
731
+ # help without the harness, so leaving it WAITING would just hit the stuck
732
+ # cap slowly.
733
+ if (
734
+ harness is None
735
+ or spec is None
736
+ or not record.resume_session_id
737
+ or not getattr(harness, "supports_resume", True)
738
+ ):
739
+ return _end(
740
+ AttemptResult(
741
+ outcome="session-error",
742
+ note="author-sleep wake needs the author harness/spec and a resumable session",
743
+ ),
744
+ drop_refs=[sleep_ref],
745
+ )
746
+
747
+ # Deliver: read each launch's job output, copy declared artifacts into the
748
+ # excluded channel, and render the data-fenced wake text. The stage stores
749
+ # only what the wake needs (names + artifacts); command/minutes placeholders
750
+ # never reach the author.
751
+ launches = tuple(
752
+ SyscallLaunch(
753
+ name=str(item.get("name", "")),
754
+ command="(ran)",
755
+ minutes=int(item.get("minutes") or 1),
756
+ artifacts=tuple(str(a) for a in item.get("artifacts", [])),
757
+ array=int(item.get("array") or 1),
758
+ )
759
+ for item in _stage_launches(record)
760
+ )
761
+ results = gather_results(run_dir, workspace, launches)
762
+ # A launch that left no exit code was SIGKILL'd before its wrapper ran (the
763
+ # cgroup OOM killer, a hard walltime kill, a node failure). The scheduler
764
+ # still knows which; surface it so the author reads "OOM"/"timeout" instead
765
+ # of a blank "job failure". The park's launch job ids align positionally
766
+ # with the results (same launch/array order). Best-effort — the wake never
767
+ # blocks on the scheduler query.
768
+ status_of = getattr(dispatch.compute, "status", None)
769
+ if status_of is not None:
770
+ results = annotate_launch_states(results, _stage_launch_job_ids(record), status_of)
771
+ launches_used = int(record.stage.get("launches_used", 0)) # type: ignore[call-overload]
772
+ sleeps_used = int(record.stage.get("sleeps_used", 0)) # type: ignore[call-overload]
773
+ gpu_hours_used = _reconcile_launch_hours(record, dispatch, bench.gpus, launches)
774
+ wake_text = render_wake(
775
+ results,
776
+ str(record.stage.get("syscall_note", "")),
777
+ launches_used=launches_used,
778
+ launch_budget=bench.depth_k,
779
+ sleeps_used=sleeps_used,
780
+ sleep_budget=bench.sleep_k,
781
+ gpu_hours_remaining=(
782
+ max(0.0, contract.budgets.gpu_hours_per_run - gpu_hours_used) if bench.gpus else None
783
+ ),
784
+ gpus=bench.gpus,
785
+ )
786
+ if extra_update:
787
+ # a submitted park's gate/panel feedback leads; launch results follow
788
+ wake_text = f"{extra_update}\n\n{wake_text}"
789
+ _best_effort(
790
+ "budget refresh",
791
+ lambda: write_budget(
792
+ workspace,
793
+ launches_remaining=max(0, bench.depth_k - launches_used),
794
+ sleeps_remaining=max(0, bench.sleep_k - sleeps_used),
795
+ gpu_hours_remaining=(
796
+ max(0.0, contract.budgets.gpu_hours_per_run - gpu_hours_used)
797
+ if bench.gpus
798
+ else None
799
+ ),
800
+ ),
801
+ )
802
+
803
+ # The wake's climb IO: measures go through the DISPATCHED measurer (this is
804
+ # a wake job with bounded walltime — the gate's evals run as their own jobs
805
+ # and park the run as a CANDIDATE); snapshots parent on base (same as the
806
+ # first pass: the clone was at base).
807
+ snapshots: list[Snapshot] = []
808
+ wake_line = _line_ref_for(bench, config.agent_id)
809
+
810
+ def snapshot() -> str:
811
+ snap = snapshot_tree(ws, base_sha, exclude=LINE_MEMORY_PATHS if wake_line else ())
812
+ snapshots.append(snap)
813
+ return snap.commit
814
+
815
+ def changed_paths() -> list[str]:
816
+ return _paths_changed_from_base(
817
+ ws, f"refs/remotes/origin/{base_branch}", bool(wake_line), fallback=base_sha
818
+ )
819
+
820
+ panel_runner = (
821
+ build_panel_runner(
822
+ ws,
823
+ run_dir,
824
+ base_sha,
825
+ panel_lenses,
826
+ contract_text,
827
+ config.target,
828
+ config.benchmark,
829
+ config.bot_login,
830
+ _utc_date(now),
831
+ exclude=LINE_MEMORY_PATHS if wake_line else (),
832
+ )
833
+ if panel_lenses
834
+ else None
835
+ )
836
+
837
+ parked: RunParked | None = None
838
+ kept_ref = ""
839
+ try:
840
+ result = attempt_once(
841
+ config,
842
+ contract_text,
843
+ workspace,
844
+ harness,
845
+ measurer,
846
+ base_sha,
847
+ snapshot,
848
+ ruler=RULER,
849
+ changed_paths=changed_paths,
850
+ spec=spec,
851
+ panel_runner=panel_runner,
852
+ resume_session_id=record.resume_session_id,
853
+ improve_prompt=wake_text,
854
+ launcher=_make_launcher(dispatch, run_dir, workspace, run_id, gpus=bench.gpus),
855
+ tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(),
856
+ judged=judged or _stage_judged(record),
857
+ launches_used=launches_used,
858
+ sleeps_used=sleeps_used,
859
+ gpu_hours_used=gpu_hours_used,
860
+ )
861
+ except RunParked as p:
862
+ # slept again, or the gate dispatched its measures (a candidate park the
863
+ # existing wake path decides). Keep the NEW park's snapshot ref; the OLD
864
+ # sleep ref is superseded once the new park persists.
865
+ kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
866
+ import time
867
+
868
+ try:
869
+ _park_run(
870
+ run_root,
871
+ record,
872
+ p,
873
+ kept_ref,
874
+ eval_minutes,
875
+ time.time(),
876
+ secrets,
877
+ dispatch=dispatch,
878
+ base_branch=base_branch,
879
+ )
880
+ except Exception:
881
+ for job_id in afterany_ids(p.afterany):
882
+ dispatch.compute.cancel(job_id)
883
+ raise
884
+ parked = p
885
+ drop_snapshot(ws, Snapshot(commit="", tree="", ref=sleep_ref))
886
+ return AttemptOutcome(run_id=run_id, outcome="parked")
887
+ finally:
888
+ for snap in snapshots:
889
+ if parked and kept_ref and snap.ref == kept_ref:
890
+ continue
891
+ drop_snapshot(ws, snap)
892
+
893
+ # a terminal from the resumed session (session-error/-outage/-budget,
894
+ # scope-violation, eval-error; a dispatched gate never returns improved
895
+ # inline): end the run and release the sleep snapshot.
896
+ return _end(result, drop_refs=[sleep_ref])
897
+
898
+
899
+ def _stage_judged(record: RunRecord) -> tuple[str, AttemptResult] | None:
900
+ """The gate verdict a park carried (written by `_park_run`), or None."""
901
+ j = (record.stage or {}).get("judged")
902
+ if not isinstance(j, dict) or not j.get("sha"):
903
+ return None
904
+
905
+ def num(v: object) -> float | None:
906
+ return float(v) if isinstance(v, int | float) and not isinstance(v, bool) else None
907
+
908
+ return (
909
+ str(j["sha"]),
910
+ AttemptResult(
911
+ outcome=str(j.get("outcome") or "no-improvement"),
912
+ baseline=num(j.get("baseline")),
913
+ candidate=num(j.get("candidate")),
914
+ note=str(j.get("note") or ""),
915
+ ),
916
+ )
917
+
918
+
919
+ def _stage_syscall_launches(record: RunRecord) -> tuple:
920
+ """The park's launches as `Launch` values (command elided: they ran)."""
921
+ from outerloop.syscall import Launch
922
+
923
+ return tuple(
924
+ Launch(
925
+ name=str(item.get("name", "")),
926
+ command="(ran)",
927
+ minutes=int(item.get("minutes") or 1),
928
+ artifacts=tuple(str(a) for a in item.get("artifacts", [])),
929
+ array=int(item.get("array") or 1),
930
+ )
931
+ for item in _stage_launches(record)
932
+ )
933
+
934
+
935
+ def _stage_launch_job_ids(record: RunRecord) -> list[str]:
936
+ """The park's launch jobs: `launch_afterany` when the park recorded it;
937
+ for an older author-sleep park every waited job was a launch; for an
938
+ older candidate park the gate's evals are mixed in, so none."""
939
+ stage = record.stage or {}
940
+ if "launch_afterany" in stage:
941
+ return afterany_ids(str(stage.get("launch_afterany") or ""))
942
+ if stage.get("phase") == "author-sleep":
943
+ return afterany_ids(str(stage.get("afterany") or ""))
944
+ return []
945
+
946
+
947
+ def _reconcile_launch_hours(
948
+ record: RunRecord, dispatch: DispatchSettings, gpus: int, launches: tuple
949
+ ) -> float:
950
+ """The run's GPU-hours after handing back the unused walltime of the
951
+ park's launch jobs — once: the stage remembers the refund, so a wake
952
+ that follows a gate decision on the same park does not refund twice.
953
+ Returns the (possibly corrected) `gpu_hours_used`."""
954
+ stage = record.stage or {}
955
+ used = float(stage.get("gpu_hours_used", 0.0)) # type: ignore[arg-type]
956
+ if not gpus or stage.get("launch_hours_refunded"):
957
+ return used
958
+ refund = _launch_refund(dispatch, launches, _stage_launch_job_ids(record), gpus)
959
+ if refund > 0:
960
+ log.info("%s: refunding %.2f GPU-hours of unused launch walltime", record.run_id, refund)
961
+ used = max(0.0, used - refund)
962
+ stage["gpu_hours_used"] = used
963
+ stage["launch_hours_refunded"] = True
964
+ return used
965
+
966
+
967
+ def _launch_refund(
968
+ dispatch: DispatchSettings, launches: tuple, job_ids: list[str], gpus: int
969
+ ) -> float:
970
+ """The unused walltime of a park's launch jobs, in GPU-hours, or 0 when
971
+ the compute cannot say how long they ran (nothing is refunded on a
972
+ guess)."""
973
+ from outerloop.syscall import launch_hours_refund
974
+
975
+ query = getattr(dispatch.compute, "elapsed_seconds", None)
976
+ if query is None or not job_ids:
977
+ return 0.0
978
+ try:
979
+ elapsed = [query(jid) for jid in job_ids]
980
+ except Exception as exc:
981
+ log.warning("launch walltime unknown (%s: %s); nothing refunded", type(exc).__name__, exc)
982
+ return 0.0
983
+ return launch_hours_refund(launches, elapsed, gpus=gpus)
984
+
985
+
986
+ RESEARCH_LOG_BRANCH = "research-log"
987
+ MAX_ARCHIVED_REPORTS = 30 # materialized for the session to read; newest first
988
+ MAX_ARCHIVED_REPORT_CHARS = 100_000 # per report; branch content is remote-controlled
989
+
990
+
991
+ def _fetch_research_reports(ws: Workspace, count: int) -> list[tuple[str, str]]:
992
+ """The newest `count` reports from the target's research-log branch, as
993
+ (name, text), newest first — the shared memory of every attempt on this
994
+ target, wherever it ran. Fail-soft: a target with no research log yet
995
+ (or an unreachable remote) is an empty memory, never a dead attempt."""
996
+ try:
997
+ ws.fetch_branch(RESEARCH_LOG_BRANCH)
998
+ listing = ws.git("ls-tree", "-r", "--name-only", "FETCH_HEAD", "reports/")
999
+ # only direct children (the publisher's layout): a nested path would
1000
+ # flatten to a basename that overwrites another archived report
1001
+ names = [
1002
+ line.strip()
1003
+ for line in listing.splitlines()
1004
+ if line.strip().endswith(".md") and line.strip().count("/") == 1
1005
+ ]
1006
+ # report files are dated (reports/<YYYY-MM-DD>-<run_id>.md): the name
1007
+ # sorts by day; same-day order is arbitrary and does not matter
1008
+ out: list[tuple[str, str]] = []
1009
+ for name in sorted(names, reverse=True):
1010
+ if len(out) >= count:
1011
+ break
1012
+ # size BEFORE content: `git show` would load the whole blob, and
1013
+ # the branch's content is remote-controlled
1014
+ if int(ws.git("cat-file", "-s", f"FETCH_HEAD:{name}").strip()) > (
1015
+ MAX_ARCHIVED_REPORT_CHARS
1016
+ ):
1017
+ log.info("research report %s exceeds the size cap; skipped", name)
1018
+ continue
1019
+ out.append((Path(name).name, ws.git("show", f"FETCH_HEAD:{name}")))
1020
+ return out
1021
+ except Exception as exc:
1022
+ log.info("research log unavailable (%s: %s); starting without it", type(exc).__name__, exc)
1023
+ return []
1024
+
1025
+
1026
+ def _exclude_merge_artifacts(workspace: Path) -> None:
1027
+ """Ignore *.orig and *.rej — git's merge/patch conflict backups, which
1028
+ should never be committed — via .git/info/exclude (repo-local, never a
1029
+ tracked edit). A line run merges main at start and the agent resolves
1030
+ conflicts as its first task; a leftover train.py.orig would otherwise
1031
+ read as an out-of-scope edit and abort the run at launch. Idempotent,
1032
+ and effective for the `git add -A` behind changed-paths and every seal."""
1033
+ exclude = workspace / ".git" / "info" / "exclude"
1034
+ wanted = ["*.orig", "*.rej"]
1035
+ try:
1036
+ existing = exclude.read_text()
1037
+ except OSError:
1038
+ existing = ""
1039
+ lines = existing.splitlines()
1040
+ missing = [p for p in wanted if p not in lines]
1041
+ if missing:
1042
+ exclude.parent.mkdir(parents=True, exist_ok=True)
1043
+ sep = "" if existing.endswith("\n") or not existing else "\n"
1044
+ exclude.write_text(existing + sep + "\n".join(missing) + "\n")
1045
+
1046
+
1047
+ def _reset_instruction_files(ws: Workspace, workspace: Path, base_ref: str) -> None:
1048
+ """Make the checkout's instruction-bearing files EQUAL the base branch's
1049
+ reviewed versions (review_agent.INSTRUCTION_FILES owns the list): a line
1050
+ is an author-written tree, and must never instruct its own successor
1051
+ sessions — or a sibling's (docs/design/research-lines.md)."""
1052
+ from outerloop.review_agent import INSTRUCTION_FILES
1053
+
1054
+ # bottom-up so a removed directory doesn't orphan paths found beneath it
1055
+ for path in sorted(workspace.rglob("*"), key=lambda p: len(p.parts), reverse=True):
1056
+ if ".git" in path.parts or path.name not in INSTRUCTION_FILES:
1057
+ continue
1058
+ if path.is_dir() and not path.is_symlink():
1059
+ shutil.rmtree(path, ignore_errors=True)
1060
+ else:
1061
+ path.unlink(missing_ok=True)
1062
+ base_paths = [
1063
+ p
1064
+ for p in ws.git("ls-tree", "-r", "--name-only", base_ref).splitlines()
1065
+ if any(part in INSTRUCTION_FILES for part in Path(p).parts)
1066
+ ]
1067
+ if base_paths:
1068
+ ws.git("checkout", base_ref, "--", *base_paths)
1069
+
1070
+
1071
+ # The agent's memory on its line (docs/design/research-lines.md): the bounded
1072
+ # index at the branch root plus the topic-file folder. They ride the NOTEBOOK
1073
+ # seal (the line branch is exactly where they live) and are excluded from
1074
+ # every MEASURABLE seal and from changed-path accounting — never a scope
1075
+ # violation, never claimable work, never part of a main-PR candidate.
1076
+ LINE_MEMORY_PATHS = ("AGENT_MEMORY.md", "agent_memory")
1077
+
1078
+
1079
+ def _is_line_memory(path: str) -> bool:
1080
+ return path in LINE_MEMORY_PATHS or path.startswith("agent_memory/")
1081
+
1082
+
1083
+ def _without_line_memory(paths: Iterable[str], line: str) -> list[str]:
1084
+ """The run's OWN changes: with a line active, its memory paths are dropped.
1085
+ Every sealed tree excludes them by construction, and the run's base is the
1086
+ line tip that carries them — so a base..candidate diff lists them as
1087
+ deletions that are not the run's change (live: a dispatched wake on
1088
+ gpt-speedrun read eight memory files as out of scope and aborted a run
1089
+ whose paired eval had already finished)."""
1090
+ return [p for p in paths if not (line and _is_line_memory(p))]
1091
+
1092
+
1093
+ def _line_ref_for(bench: Benchmark | None, agent_id: str) -> str:
1094
+ """The agent's line branch when the benchmark opted in, or the empty
1095
+ string when the feature is off. Wake paths recompute this from the
1096
+ record — a malformed agent id could never have created a line at run
1097
+ start, so empty (not an error) is right there too."""
1098
+ if bench is None or not bench.lines or not agent_id:
1099
+ return ""
1100
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", agent_id):
1101
+ return ""
1102
+ return f"agents/{agent_id}"
1103
+
1104
+
1105
+ def _push_line_snapshot(
1106
+ ws: Workspace, line_ref: str, run_id: str, outcome: str, secrets: tuple[str, ...] = ()
1107
+ ) -> None:
1108
+ """Publish the session's final tree to the agent's line as a sealed
1109
+ snapshot commit — every terminal path, any outcome
1110
+ (docs/design/research-lines.md). The seal parents on the LOCAL line ref
1111
+ and advances it, so sequential terminals within one run chain as
1112
+ fast-forwards; one slot never runs twice concurrently, so the remote
1113
+ cannot have moved under us. Best-effort throughout: the notebook never
1114
+ changes a run's outcome. An unchanged tree is not pushed (the run-start
1115
+ push already holds it)."""
1116
+ if not line_ref:
1117
+ return
1118
+
1119
+ def _seal_and_push() -> None:
1120
+ # raises if the ref is absent (e.g. a park that predates the line
1121
+ # feature) or the session altered .git (every ws.git call checks) —
1122
+ # _best_effort turns either into a logged skip
1123
+ local = ws.git("rev-parse", f"refs/heads/{line_ref}").strip()
1124
+ memory = tuple(p for p in LINE_MEMORY_PATHS if (Path(ws.root) / p).exists())
1125
+ last_exc: Exception | None = None
1126
+ # the line commit this workspace's untouched files currently match:
1127
+ # the local ref at first, then each remote head reconciled into it
1128
+ # (so a retry does not mistake copied-in files for this run's edits)
1129
+ fork = local
1130
+ for _ in range(3):
1131
+ parent = fork
1132
+ # A park frees the slot, so a newer run on the same line can end
1133
+ # (and push) while this one is parked: the remote line then sits
1134
+ # past our local ref, and a seal parented on the stale ref would be
1135
+ # refused as a non-fast-forward (gpt-speedrun, 2026-09-03: agent-01's
1136
+ # winning run left no snapshot this way, and its line fell behind
1137
+ # main). Parent on the remote head whenever our ref is an ancestor
1138
+ # of it, keeping the files that head added since; offline, seal on
1139
+ # the local ref as before.
1140
+ try:
1141
+ ws.fetch_origin()
1142
+ remote = ws.git("rev-parse", f"refs/remotes/origin/{line_ref}").strip()
1143
+ if remote != fork:
1144
+ ws.git("merge-base", "--is-ancestor", fork, remote) # raises when not
1145
+ _reconcile_with_remote(ws, fork, remote)
1146
+ fork = parent = remote
1147
+ except Exception as exc:
1148
+ log.info("line %s: sealing on the local ref (%s)", line_ref, type(exc).__name__)
1149
+ snap = snapshot_tree(ws, parent, force=memory)
1150
+ try:
1151
+ # seal only when the tree moved past the parent; the PUSH runs
1152
+ # either way — a session that COMMITTED its work advanced the
1153
+ # local ref without dirtying the tree, and that commit must
1154
+ # still reach the remote (an already-current ref push is a no-op)
1155
+ sealed = parent
1156
+ if snap.tree != ws.git("rev-parse", f"{parent}^{{tree}}").strip():
1157
+ sealed = ws.git(
1158
+ "-c",
1159
+ "user.name=autoresearch",
1160
+ "-c",
1161
+ "user.email=autoresearch@localhost",
1162
+ "commit-tree",
1163
+ snap.tree,
1164
+ "-p",
1165
+ parent,
1166
+ "-m",
1167
+ f"line snapshot: {run_id} ({outcome})",
1168
+ ).strip()
1169
+ ws.git("update-ref", f"refs/heads/{line_ref}", sealed)
1170
+ try:
1171
+ ws.push(line_ref)
1172
+ return
1173
+ except Exception as exc:
1174
+ # another run pushed between our fetch and this push:
1175
+ # re-read the line and seal again on its new head
1176
+ last_exc = exc
1177
+ log.info(
1178
+ "line %s: push refused, re-sealing on the moved line (%s)",
1179
+ line_ref,
1180
+ type(exc).__name__,
1181
+ )
1182
+ finally:
1183
+ drop_snapshot(ws, snap)
1184
+ assert last_exc is not None
1185
+ raise last_exc
1186
+
1187
+ _best_effort(f"line push ({outcome})", _seal_and_push, secrets)
1188
+
1189
+
1190
+ def _reconcile_with_remote(ws: Workspace, old: str, new: str) -> None:
1191
+ """The seal is built from THIS workspace, which forked the line at `old`;
1192
+ another run has since moved the line to `new`. For every path that run
1193
+ changed, take its state when this workspace left the path untouched
1194
+ since `old` (added: materialize, modified: update, deleted: remove); a
1195
+ path this run touched keeps this run's version, as the line always did.
1196
+ Without this, a seal would silently undo the other run's work on files
1197
+ this run never looked at."""
1198
+ entries = [
1199
+ e for e in ws.git("diff", "--name-status", "--no-renames", "-z", old, new).split("\0") if e
1200
+ ]
1201
+ remote_changes = list(zip(entries[0::2], entries[1::2], strict=True))
1202
+ if not remote_changes:
1203
+ return
1204
+ touched = {
1205
+ p for p in ws.git("diff", "--name-only", "-z", old).split("\0") if p
1206
+ } # tracked paths this run changed or deleted
1207
+ touched |= {
1208
+ p for p in ws.git("ls-files", "--others", "--exclude-standard", "-z").split("\0") if p
1209
+ } # and the files it created
1210
+ for status, path in remote_changes:
1211
+ if path in touched:
1212
+ continue
1213
+ if status.startswith("D"):
1214
+ target = Path(ws.root) / path
1215
+ if target.is_file() or target.is_symlink():
1216
+ target.unlink()
1217
+ else:
1218
+ ws.git("checkout", new, "--", path)
1219
+
1220
+
1221
+ def _checkout_line(ws: Workspace, workspace: Path, agent_id: str, base_branch: str) -> str:
1222
+ """Check out the agent's research line: the persistent branch
1223
+ `agents/<agent-id>`, created from the base branch when absent, with the
1224
+ base branch merged in when it exists — a conflicted merge is left in the
1225
+ tree as the session's first task. Instruction-bearing files are reset to
1226
+ the base branch's reviewed versions and the hygiene is committed (never
1227
+ left to masquerade as agent edits). Returns the line ref name; raises on
1228
+ anything unrecoverable (the caller falls back to the base branch)."""
1229
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", agent_id):
1230
+ raise ValueError(f"agent id {agent_id!r} cannot shape a line ref")
1231
+ line = f"agents/{agent_id}"
1232
+ base_ref = f"origin/{base_branch}"
1233
+ if not ws.git("branch", "--list", "-r", f"origin/{line}").strip():
1234
+ ws.git("checkout", "-q", "-B", line, base_ref)
1235
+ ws.push(line) # the line is durable from its first run
1236
+ return line
1237
+ ws.git("checkout", "-q", "-B", line, f"origin/{line}")
1238
+ conflicted = False
1239
+ try:
1240
+ ws.git(
1241
+ "-c",
1242
+ "user.name=autoresearch",
1243
+ "-c",
1244
+ "user.email=autoresearch@localhost",
1245
+ "merge",
1246
+ "--no-edit",
1247
+ base_ref,
1248
+ )
1249
+ except Exception:
1250
+ conflicted = True
1251
+ log.info(
1252
+ "line %s: merging %s conflicts; left as the session's first task", line, base_branch
1253
+ )
1254
+ _reset_instruction_files(ws, workspace, base_ref)
1255
+ if not conflicted:
1256
+ ws.git("add", "-A")
1257
+ if ws.git("status", "--porcelain").strip():
1258
+ ws.git(
1259
+ "-c",
1260
+ "user.name=autoresearch",
1261
+ "-c",
1262
+ "user.email=autoresearch@localhost",
1263
+ "commit",
1264
+ "-q",
1265
+ "-m",
1266
+ f"line hygiene: instruction files reset to {base_branch}",
1267
+ )
1268
+ # Run-START persistence: the branch exists on the remote from its
1269
+ # first run, and the merge-main + hygiene state survives a crashed
1270
+ # run. One slot never runs twice concurrently, so this is a fast-
1271
+ # forward. The run-END push of the sealed session tree is the next
1272
+ # phase PR (it requires all-terminal sealing; the push must publish
1273
+ # a sealed sha, never invent a commit). A conflicted merge is not
1274
+ # pushed: the conflict is session work, not line state.
1275
+ ws.push(line)
1276
+ return line
1277
+
1278
+
1279
+ def _paths_changed_from_base(
1280
+ ws: Workspace, base: str, exclude_memory: bool, fallback: str = "HEAD"
1281
+ ) -> list[str]:
1282
+ """The paths the session changed, measured against the BASE BRANCH head
1283
+ (`base`, a commit-ish such as refs/remotes/origin/main), not HEAD. On a
1284
+ research line HEAD can be behind main: a run starts by merging main in,
1285
+ and when that merge conflicts it stays uncommitted, so the files git
1286
+ auto-merged (the project's BENCHMARKS.md and results/leader.json) sit
1287
+ staged against the stale HEAD while being identical to main. Those are
1288
+ main's edits, not the agent's, and must not read as scope violations
1289
+ (gpt-speedrun, 2026-09-03: agent-01's first run after its own win ended
1290
+ scope-violation on exactly those two files). A path counts only when it
1291
+ moved against HEAD AND differs from the base; new and deleted files
1292
+ count. `fallback` is used when `base` does not resolve (no remote).
1293
+ `exclude_memory` drops the line's memory files whenever lines are active
1294
+ for the benchmark (a failed line checkout still keeps them out)."""
1295
+ try:
1296
+ base_commit = ws.git("rev-parse", "--verify", f"{base}^{{commit}}").strip()
1297
+ except Exception:
1298
+ base_commit = fallback
1299
+ ws.git("add", "-A")
1300
+ try:
1301
+ staged = ws.staged_paths()
1302
+ if not staged:
1303
+ return []
1304
+ # index vs base, restricted to what moved against HEAD: a path identical
1305
+ # to base drops out, a new or deleted file still counts
1306
+ differs = {
1307
+ entry
1308
+ for entry in ws.git(
1309
+ "diff", "--cached", "--name-only", "-z", base_commit, "--", *staged
1310
+ ).split("\0")
1311
+ if entry
1312
+ }
1313
+ finally:
1314
+ ws.git("reset")
1315
+ kept = [p for p in staged if p in differs]
1316
+ return [p for p in kept if not (exclude_memory and _is_line_memory(p))]
1317
+
1318
+
1319
+ def _sibling_entries(ws: Workspace, self_agent: str) -> list[dict]:
1320
+ """The other agents' live directions from the research-log's
1321
+ status.json (already fetched: the SAME FETCH_HEAD the reports came
1322
+ from). Size-checked BEFORE show like the report blobs, entries and
1323
+ fields bounded — the branch is bot-written but never trusted with
1324
+ unbounded memory. Any failure means no siblings known, never a crash."""
1325
+ try:
1326
+ blob = "FETCH_HEAD:climb/status.json"
1327
+ if int(ws.git("cat-file", "-s", blob).strip()) > 1_000_000:
1328
+ raise ValueError("status snapshot oversized; skipped")
1329
+ fleet = json.loads(ws.git("show", blob))
1330
+ return [
1331
+ {
1332
+ "agent": str(r.get("agent", ""))[:64],
1333
+ "state": str(r.get("state", ""))[:32],
1334
+ "phase": str(r.get("phase", ""))[:32],
1335
+ "direction": str(r.get("direction", ""))[:160],
1336
+ }
1337
+ for r in fleet.get("runs", [])[:64]
1338
+ if isinstance(r, dict) and r.get("agent") != self_agent
1339
+ ]
1340
+ except Exception as exc:
1341
+ log.info("no sibling snapshot for this session (%s)", exc)
1342
+ return []
1343
+
1344
+
1345
+ def _install_report_archive(workspace: Path, reports: list[tuple[str, str]]) -> None:
1346
+ """Materialize the fetched reports under the kernel-owned channel
1347
+ (`.outerloop/reports/`) so the session can read and search the full
1348
+ texts with its own tools; the brief inlines only the newest few."""
1349
+ dest = workspace / channel_dir(workspace) / "reports"
1350
+ dest.mkdir(parents=True, exist_ok=True)
1351
+ for name, text in reports:
1352
+ if Path(name).name != name: # branch content is remote-controlled
1353
+ continue
1354
+ (dest / name).write_text(text)
1355
+
1356
+
1357
+ def _stage_launches(record: RunRecord) -> list[dict]:
1358
+ """The persisted launch descriptors of an author-sleep stage (name +
1359
+ artifacts), tolerating a malformed entry by skipping it (the job dirs are
1360
+ keyed by name; a nameless entry has nothing to gather)."""
1361
+ raw = record.stage.get("syscall_launches", [])
1362
+ if not isinstance(raw, list):
1363
+ return []
1364
+ return [item for item in raw if isinstance(item, dict) and item.get("name")]
1365
+
1366
+
1367
+ def _utc_date(now: float) -> str:
1368
+ from datetime import UTC, datetime
1369
+
1370
+ return datetime.fromtimestamp(now, UTC).strftime("%Y-%m-%d")
1371
+
1372
+
1373
+ def _is_git_tamper(exc: GitError) -> bool:
1374
+ """True when a GitError means the workspace's object store or refs are
1375
+ damaged or session-altered — as opposed to an ordinary git failure (a push
1376
+ conflict, a fetch outage). The guard raises its own `_altered(...)` for the
1377
+ states it models; a raw object-store error ("could not parse object", "bad
1378
+ tree object", a corrupt or unreadable pack) is the same damage reaching a
1379
+ git command in the gap between the guard's check and the command. Either
1380
+ way the wake cannot be trusted and must END, not crash."""
1381
+ msg = str(exc).lower()
1382
+ return (
1383
+ "altered by the session" in msg # the guard's own _altered signal
1384
+ or "could not parse object" in msg
1385
+ or "bad tree object" in msg
1386
+ or "bad object" in msg
1387
+ or "unable to read tree" in msg
1388
+ or "data stream error" in msg # inflate: a corrupt pack
1389
+ or "is corrupt" in msg # "packed object ... is corrupt", loose too
1390
+ or ("object file" in msg and "empty" in msg)
1391
+ )
1392
+
1393
+
1394
+ def _end_refused_wake(
1395
+ run_root: Path, record: RunRecord, exc: Exception, now: float, secrets: tuple[str, ...]
1396
+ ) -> AttemptOutcome:
1397
+ """End a parked run whose workspace the wake refused (a session altered
1398
+ .git): ABORTED with the tampering as the note. The candidate snapshot's
1399
+ retaining ref lives inside that same, now untrusted, repository and is
1400
+ deliberately not touched: deleting it would mean writing through the
1401
+ very structure the guard refused (a symlinked refs dir carries the write
1402
+ elsewhere), and an ENDED workspace is inert — the ref keeps a commit
1403
+ alive only within a repository nothing will read again."""
1404
+ note = redact(str(exc), secrets)[:480]
1405
+ log.warning("wake refused for %s: %s", record.run_id, note)
1406
+ failed = _clear_stage(
1407
+ RunRecord(**{**record.__dict__, "state": ENDED, "ending": ABORTED, "ending_note": note})
1408
+ )
1409
+ _best_effort("ending record", lambda: save_record(run_root, failed, now), secrets)
1410
+ return AttemptOutcome(run_id=record.run_id, outcome="attempt-error")
1411
+
1412
+
1413
+ def resume_run(
1414
+ run_root: Path,
1415
+ run_id: str,
1416
+ *,
1417
+ dispatch: DispatchSettings,
1418
+ github: GitHubClient,
1419
+ bot_auth: TokenProvider,
1420
+ now: float,
1421
+ secrets: tuple[str, ...] = (),
1422
+ base_branch: str = "main",
1423
+ panel_lenses: tuple[PanelLens, ...] = (),
1424
+ harness: Harness | None = None,
1425
+ spec: RoleSpec | None = None,
1426
+ ) -> AttemptOutcome:
1427
+ """Wake a parked dispatched climb and re-enter its decision WITHOUT the
1428
+ session (`orchestrator.resume_attempt`), from the record `_park_run` wrote.
1429
+ The three exits:
1430
+
1431
+ * **re-park** — the wake dispatched a measure that is not done yet (the
1432
+ suite pairs an improving candidate fans out, "another round of
1433
+ experiments"): `resume_attempt` raises `RunParked`, and this re-persists
1434
+ the WAITING stage on the new afterany, keeping the same candidate
1435
+ snapshot;
1436
+ * **a negative terminal** (no-improvement / suite-regression / eval-error):
1437
+ drop the candidate snapshot and end the record;
1438
+ * **improved** — branch the SEALED `candidate_sha` (never the live tree,
1439
+ which may have drifted since the park; the diff was scope-checked so it
1440
+ carries only in-scope changes), layer the ledger update on top, push, and
1441
+ open the PR. A moved base is NOT merged and re-measured here
1442
+ (docs/design/research-loop.md): a stale PR is a re-wake, not an
1443
+ orchestrator auto-merge.
1444
+ """
1445
+ run_dir = run_root / "runs" / run_id
1446
+ workspace = run_dir / "ws"
1447
+ record = load_record(run_root, run_id)
1448
+ stage = record.stage
1449
+ # Push to the CANONICAL target URL, never the workspace's remote.origin.url:
1450
+ # the session could have rewritten that config to exfil the bot token / code
1451
+ # to another remote. Passing `url` here means `Workspace.push` uses it
1452
+ # instead of reading `remote.origin.url`.
1453
+ ws = Workspace(root=workspace, auth=bot_auth, url=_target_clone_url(record.target))
1454
+ # A session reshaped .git (symlinked object store, gitdir file, FIFO) is
1455
+ # refused BEFORE anything writes through it: the exclude below opens
1456
+ # .git/info/exclude, and every ws.git call re-checks. The refusal ENDS
1457
+ # the parked run with the tampering as its note — the tree cannot be
1458
+ # trusted, and a record left waiting would only be re-woken into the
1459
+ # same refusal.
1460
+ try:
1461
+ ensure_regular_git_dir(workspace)
1462
+ except GitError as exc:
1463
+ return _end_refused_wake(run_root, record, exc, now, secrets)
1464
+ # Re-establish the merge-artifact exclude on the wake too: the workspace
1465
+ # persisted across the park, but a session could have removed the exclude,
1466
+ # and this wake's changed_paths / seal run `git add -A`. Idempotent.
1467
+ _exclude_merge_artifacts(workspace)
1468
+ # Refresh origin refs on EVERY wake: the clone's refs froze at run
1469
+ # start, and this is the one credential-free freshness point — the
1470
+ # kernel fetches (from the canonical URL, never the session-writable
1471
+ # remote config), the session only ever reads local refs. `sleep`
1472
+ # thereby doubles as the author's sync primitive. Best-effort: a fetch
1473
+ # outage must not cost the wake.
1474
+ try:
1475
+ ws.fetch_origin()
1476
+ except Exception as exc:
1477
+ log.warning("wake fetch failed for %s: %s", run_id, exc)
1478
+
1479
+ # Two park kinds reach the wake: a CANDIDATE park (the gate's measures were
1480
+ # dispatched) and an AUTHOR-SLEEP park (the author launched work and slept —
1481
+ # research-loop-buildout.md Phase A). Both carry a sealed sha. Anything else
1482
+ # is a stray record: guard rather than crash on `git diff base ""`.
1483
+ if stage.get("phase") not in ("candidate", "author-sleep") or not stage.get("candidate_sha"):
1484
+ raise EvalError(f"resume_run: run {run_id} is not a wakeable park (stage={stage!r})")
1485
+
1486
+ base_sha = str(stage["base_sha"])
1487
+ candidate_sha = str(stage["candidate_sha"])
1488
+ candidate_ref = str(stage["candidate_ref"])
1489
+ issue_number = record.issue_number
1490
+ # the run's target branch rides the stage, so a wake opens its PR against
1491
+ # the branch the ORIGINAL climb selected — not the CLI's default (the wake
1492
+ # job carries no --base-branch).
1493
+ base_branch = str(stage.get("base_branch") or base_branch)
1494
+
1495
+ # The contract gates scope and names the eval command, so read it from the
1496
+ # BASE commit (the tree the run started on), NOT the working tree the
1497
+ # session left dirty — a session that widened its own scope in
1498
+ # its contract must not have the wake gate on the doctored rules.
1499
+ contract_text = contract_at(ws, base_sha)
1500
+ contract = load_contract(contract_text, record.target)
1501
+ bench = _benchmark(contract, record.benchmark)
1502
+ config = RunConfig(target=record.target, benchmark=record.benchmark, agent_id=record.agent_id)
1503
+ eval_minutes = next(
1504
+ (b.eval_minutes for b in contract.benchmarks if b.name == record.benchmark), None
1505
+ )
1506
+ # a submitted park carries the author's declared eval walltime: the
1507
+ # wake's measurer (a re-dispatch) and deadline floor honor it
1508
+ declared = int(record.stage.get("eval_minutes", 0) or 0) # type: ignore[call-overload]
1509
+ if declared:
1510
+ eval_minutes = declared
1511
+ measurer = dispatch.measurer(
1512
+ run_dir, repo_root=workspace, eval_minutes=int(eval_minutes or 0), run_tag=run_id
1513
+ )
1514
+ # measured_paths from the COMMITTED base..candidate diff — the sealed
1515
+ # candidate, never `changed_paths()` on a live tree that may have drifted.
1516
+ # NUL-delimited (like Workspace.staged_paths) so a path with a space is one
1517
+ # entry, not two that could each slip past the scope check. The same
1518
+ # line-memory rule as the climb's changed_paths(): the base is the line
1519
+ # tip, the seal excluded the memory, the diff must not read it as a change.
1520
+ measured_paths = tuple(
1521
+ _without_line_memory(
1522
+ (
1523
+ p
1524
+ for p in ws.git("diff", "--name-only", "-z", base_sha, candidate_sha).split("\0")
1525
+ if p
1526
+ ),
1527
+ _line_ref_for(bench, config.agent_id),
1528
+ )
1529
+ )
1530
+ seed = int(stage["seed"]) # type: ignore[call-overload]
1531
+ suite_seed = int(stage["suite_seed"]) # type: ignore[call-overload]
1532
+ panel_reads = int(stage.get("panel_reads", 0)) # type: ignore[call-overload]
1533
+
1534
+ if stage.get("phase") == "author-sleep":
1535
+ # The author slept on launches: deliver their results and RESUME the
1536
+ # same session through the climb's resume-entry. Every exit is a park
1537
+ # (slept again, or the gate dispatched its measures -> a candidate park
1538
+ # the NEXT wake decides through the path below) or a terminal ending —
1539
+ # never a publish, so the publish tail stays candidate-only.
1540
+ return _wake_author_sleep(
1541
+ run_root=run_root,
1542
+ run_id=run_id,
1543
+ record=record,
1544
+ ws=ws,
1545
+ workspace=workspace,
1546
+ run_dir=run_dir,
1547
+ dispatch=dispatch,
1548
+ github=github,
1549
+ now=now,
1550
+ secrets=secrets,
1551
+ base_branch=base_branch,
1552
+ base_sha=base_sha,
1553
+ sleep_ref=candidate_ref,
1554
+ contract_text=contract_text,
1555
+ contract=contract,
1556
+ bench=bench,
1557
+ config=config,
1558
+ measurer=measurer,
1559
+ harness=harness,
1560
+ spec=spec,
1561
+ panel_lenses=panel_lenses,
1562
+ issue_number=issue_number,
1563
+ eval_minutes=eval_minutes,
1564
+ )
1565
+
1566
+ # rebuild the session from what the park saved: the (redacted) write-up and
1567
+ # its real spend, so the report shows true cost/turns. It is never re-run.
1568
+ session = SessionResult(
1569
+ stop_reason="resumed",
1570
+ is_error=False,
1571
+ cost_usd=float(stage.get("session_cost_usd", 0.0)), # type: ignore[arg-type]
1572
+ num_turns=int(stage.get("session_turns", 0)), # type: ignore[call-overload]
1573
+ session_id=record.resume_session_id,
1574
+ final_text=str(stage.get("report", "")),
1575
+ transcript_path="",
1576
+ )
1577
+ try:
1578
+ result = resume_attempt(
1579
+ contract,
1580
+ bench,
1581
+ base_sha=base_sha,
1582
+ candidate_sha=candidate_sha,
1583
+ seed=seed,
1584
+ suite_seed=suite_seed,
1585
+ measured_paths=measured_paths,
1586
+ session=session,
1587
+ measurer=measurer,
1588
+ min_relative_improvement=config.min_relative_improvement,
1589
+ )
1590
+ except RunParked as parked:
1591
+ # another measure this wake dispatched is not done — re-park on the new
1592
+ # afterany, keeping the SAME candidate snapshot the next wake reads.
1593
+ # PROGRESS only if this wake dispatched a NEW job set (e.g. the candidate
1594
+ # resolved and the suite pairs fanned out); a blind re-park (empty
1595
+ # afterany) or the same jobs still pending is NO progress, so the stuck
1596
+ # cap must keep counting.
1597
+ if stage.get("submitted"):
1598
+ # No author was woken yet, so a SUBMITTED park's re-park (the suite
1599
+ # fanned out) still owes the author the gate+panel results and its
1600
+ # sibling launches' results — carry the submit context forward, or
1601
+ # the next wake drafts instead of waking the author and the launch
1602
+ # descriptors are lost. Commands/minutes are spent
1603
+ # history; the wake needs only names + artifacts (as persisted).
1604
+ from outerloop.syscall import Launch as _Launch
1605
+ from outerloop.syscall import SyscallRequest as _SyscallRequest
1606
+
1607
+ parked.submitted = True
1608
+ parked.launches_used = int(stage.get("launches_used", 0)) # type: ignore[call-overload]
1609
+ parked.sleeps_used = int(stage.get("sleeps_used", 0)) # type: ignore[call-overload]
1610
+ parked.gpu_hours_used = float(stage.get("gpu_hours_used", 0.0)) # type: ignore[arg-type]
1611
+ parked.eval_minutes = int(stage.get("eval_minutes", 0) or 0) or None # type: ignore[call-overload]
1612
+ parked.judged = parked.judged or _stage_judged(record)
1613
+ parked.launch_afterany = parked.launch_afterany or str(stage.get("launch_afterany", ""))
1614
+ if parked.syscall is None:
1615
+ parked.syscall = _SyscallRequest(
1616
+ launches=tuple(
1617
+ _Launch(
1618
+ name=str(item.get("name", "")),
1619
+ command="(ran)",
1620
+ # the persisted walltime, so the re-park's deadline
1621
+ # floor still covers the longest launch
1622
+ minutes=int(item.get("minutes") or 1),
1623
+ artifacts=tuple(str(a) for a in item.get("artifacts", [])),
1624
+ array=int(item.get("array") or 1),
1625
+ )
1626
+ for item in _stage_launches(record)
1627
+ ),
1628
+ note=str(stage.get("syscall_note", "")),
1629
+ submit=True,
1630
+ )
1631
+ old_afterany = str(record.stage.get("afterany", ""))
1632
+ made_progress = bool(parked.afterany) and parked.afterany != old_afterany
1633
+ _park_run(
1634
+ run_root,
1635
+ record,
1636
+ parked,
1637
+ candidate_ref,
1638
+ eval_minutes,
1639
+ now,
1640
+ secrets,
1641
+ dispatch=dispatch,
1642
+ keep_wake_attempts=not made_progress,
1643
+ base_branch=base_branch,
1644
+ panel_reads=panel_reads,
1645
+ )
1646
+ return AttemptOutcome(run_id=run_id, outcome="parked")
1647
+
1648
+ # A SUBMITTED park (the author's `submit` syscall, buildout Phase B): gate
1649
+ # and panel results go back to the AUTHOR — it revises and resubmits, runs
1650
+ # more experiments, or concludes — instead of being decided by policy here.
1651
+ # Falls back to the plain-finish behavior (negative terminal / draft PR)
1652
+ # when the session cannot be resumed.
1653
+ submitted_park = bool(stage.get("submitted"))
1654
+ author_resumable = (
1655
+ harness is not None
1656
+ and spec is not None
1657
+ and bool(record.resume_session_id)
1658
+ and getattr(harness, "supports_resume", True)
1659
+ )
1660
+
1661
+ # the park's sibling launches are done too: settle their charge before
1662
+ # any path — publish or hand back to the author — reads the budget
1663
+ if _stage_launches(record):
1664
+ _reconcile_launch_hours(record, dispatch, bench.gpus, _stage_syscall_launches(record))
1665
+
1666
+ def _wake_author(
1667
+ extra_update: str, judged: tuple[str, AttemptResult] | None = None
1668
+ ) -> AttemptOutcome:
1669
+ # resume the submitted park's author with the gate/panel feedback
1670
+ # leading its wake text; the candidate ref is this park's held snapshot
1671
+ return _wake_author_sleep(
1672
+ run_root=run_root,
1673
+ run_id=run_id,
1674
+ record=record,
1675
+ ws=ws,
1676
+ workspace=workspace,
1677
+ run_dir=run_dir,
1678
+ dispatch=dispatch,
1679
+ github=github,
1680
+ now=now,
1681
+ secrets=secrets,
1682
+ base_branch=base_branch,
1683
+ base_sha=base_sha,
1684
+ sleep_ref=candidate_ref,
1685
+ contract_text=contract_text,
1686
+ contract=contract,
1687
+ bench=bench,
1688
+ config=config,
1689
+ measurer=measurer,
1690
+ harness=harness,
1691
+ spec=spec,
1692
+ panel_lenses=panel_lenses,
1693
+ issue_number=issue_number,
1694
+ eval_minutes=eval_minutes,
1695
+ extra_update=extra_update,
1696
+ judged=judged,
1697
+ )
1698
+
1699
+ if (
1700
+ submitted_park
1701
+ and author_resumable
1702
+ and result.outcome in ("no-improvement", "suite-regression", "eval-error")
1703
+ ):
1704
+ # the submitted candidate failed the gate — including an eval that
1705
+ # errored: feedback, never a silent terminal — the author decides
1706
+ # what happens next (rounds stay bounded by sleep_k)
1707
+ return _wake_author(
1708
+ "Your `submit` did NOT clear the gate: "
1709
+ f"{result.note or result.outcome} "
1710
+ f"(baseline {result.baseline}, candidate {result.candidate}). "
1711
+ "Revise and submit again, run more experiments, or finish with an "
1712
+ "honest negative report.",
1713
+ # the verdict rides the resume: the same tree, sealed again after
1714
+ # the author concludes, is not measured twice; only an explicit
1715
+ # resubmit runs an errored eval again
1716
+ judged=(candidate_sha, result),
1717
+ )
1718
+
1719
+ def _notebook(outcome: str) -> None:
1720
+ # Research lines: record the tree AS OF THIS DECIDED TERMINAL — never
1721
+ # earlier, because a blocking panel verdict can still resume the
1722
+ # author (a continuation, not a terminal).
1723
+ _push_line_snapshot(ws, _line_ref_for(bench, config.agent_id), run_id, outcome, secrets)
1724
+
1725
+ if result.outcome == "improved":
1726
+ # Publish: branch the SEALED candidate sha, fold in the ledger, push,
1727
+ # open the PR. No moved-base merge (research-loop.md) — a stale PR is a
1728
+ # re-wake, not an auto-merge.
1729
+ from datetime import UTC, datetime
1730
+
1731
+ assert result.baseline is not None and result.candidate is not None
1732
+ baseline, candidate = result.baseline, result.candidate
1733
+ # a zero-change "improvement" is metric noise, not progress — never a PR
1734
+ # (defense in depth; measure_and_decide already requires a real delta,
1735
+ # and an empty base..candidate diff implies baseline == candidate).
1736
+ if not measured_paths:
1737
+ result = dc_replace(
1738
+ result, outcome="no-improvement", note="no code change; metric noise"
1739
+ )
1740
+ drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
1741
+ final = _clear_stage(
1742
+ RunRecord(**{**record.__dict__, "state": ENDED, "ending": NEGATIVE_RESULT})
1743
+ )
1744
+ _best_effort("final record", lambda: save_record(run_root, final, now), secrets)
1745
+ _notebook("no-improvement")
1746
+ return AttemptOutcome(run_id=run_id, outcome="no-improvement")
1747
+
1748
+ from datetime import UTC as _UTC
1749
+ from datetime import datetime as _dt
1750
+
1751
+ # seal the notebook before any checkout mutates the persisted session
1752
+ # tree (the memory files are excluded from the sealed candidate and
1753
+ # would not survive the force-checkout + clean below)
1754
+ _notebook("improved")
1755
+
1756
+ branch = f"{config.branch_prefix}/{run_id}"
1757
+
1758
+ # IDEMPOTENCY: a prior wake may have opened the PR but died before
1759
+ # recording it (leaving the run WAITING). On re-entry, if a PR is already
1760
+ # open for this head->base, reconcile to it — do NOT re-push
1761
+ # (non-fast-forward) or open a duplicate. The reconcile does the FULL
1762
+ # terminal (branch checkout, arm, report, issue, in-review record); it
1763
+ # only SKIPS the push + create_pull the prior wake already did. A lookup
1764
+ # failure just falls through to the normal publish.
1765
+ existing: dict[str, object] | None = None
1766
+ try:
1767
+ existing = github.find_open_pull_for_head(config.target, branch, base_branch)
1768
+ except Exception as exc:
1769
+ log.warning(
1770
+ "idempotency PR lookup failed for %s: %s",
1771
+ run_id,
1772
+ redact(f"{type(exc).__name__}: {exc}", secrets),
1773
+ )
1774
+ if existing:
1775
+ pr_url = str(existing.get("html_url", ""))
1776
+ log.info("run %s: PR %s already open; reconciling the record", run_id, pr_url)
1777
+ # put the workspace on the branch (a later follow-up expects it) and
1778
+ # finish the steps the prior wake may have died before completing.
1779
+ _best_effort(
1780
+ "reconcile checkout", lambda: ws.git("checkout", "-f", "-B", branch, candidate_sha)
1781
+ )
1782
+ pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1]
1783
+ if pr_number.isdigit() and not existing.get("draft"):
1784
+ _arm_unless_base_moved(
1785
+ github,
1786
+ ws,
1787
+ config.target,
1788
+ pr_number,
1789
+ base_branch,
1790
+ base_sha,
1791
+ secrets,
1792
+ merge_mode=getattr(contract, "merge", "manual"),
1793
+ panel_ran=result.panel_rounds > 0,
1794
+ )
1795
+ report_path = run_dir / "report.md"
1796
+ _best_effort(
1797
+ "run report",
1798
+ lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
1799
+ secrets,
1800
+ )
1801
+ final = _clear_stage(
1802
+ RunRecord(
1803
+ **{
1804
+ **record.__dict__,
1805
+ "state": IN_REVIEW,
1806
+ "pr_url": pr_url,
1807
+ "auto_blessed_head": _blessed_head(ws, result, contract),
1808
+ "resume_session_id": result.session.session_id if result.session else "",
1809
+ "ending_note": pr_url,
1810
+ }
1811
+ )
1812
+ )
1813
+ if _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
1814
+ drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
1815
+ _post_issue_finished(
1816
+ github,
1817
+ config.target,
1818
+ issue_number,
1819
+ run_id,
1820
+ "improved",
1821
+ pr_url,
1822
+ redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
1823
+ secrets,
1824
+ )
1825
+ return AttemptOutcome(
1826
+ run_id=run_id, outcome="improved", pr_url=pr_url, report_path=str(report_path)
1827
+ )
1828
+
1829
+ try:
1830
+ # FORCE-checkout the sealed candidate: at wake the workspace still
1831
+ # holds the session's dirty tree (HEAD is pre_session_sha), so a
1832
+ # plain checkout could be blocked; the sha already captured exactly
1833
+ # the measured content.
1834
+ ws.git("checkout", "-f", "-B", branch, candidate_sha)
1835
+ # `checkout -f` does NOT remove untracked files, and the panel's
1836
+ # `git add -A` (in build_panel_runner) would sweep any post-snapshot
1837
+ # cruft into the tree it judges. Clean untracked (non-ignored) files
1838
+ # so the panel reads EXACTLY candidate_sha. The ledger commit stages
1839
+ # only PROGRESS_PATHS, so it was never affected.
1840
+ ws.git("clean", "-fd")
1841
+
1842
+ # Verification panel on the credited claim — the SAME gate the
1843
+ # inline path runs (docs/design/orchestrator-verify.md), so a
1844
+ # dispatched improvement is not published unverified. It reads the
1845
+ # workspace tree, now checked out to the SEALED candidate_sha (the
1846
+ # dispatched evals ran on node-local scratch, so the tree is exactly
1847
+ # what was measured), over base_sha. A blocking or degraded
1848
+ # verdict opens a DRAFT PR carrying the findings and never arms
1849
+ # auto-merge; a clean verdict (or no panel) arms.
1850
+ if panel_lenses:
1851
+ # A panel ERROR (a git op in build_panel_runner, not a finding)
1852
+ # must NOT abort the publish and drop the candidate snapshot —
1853
+ # the improvement is real and measured. Fail closed to DEGRADED:
1854
+ # open a DRAFT for a human, keep the candidate. (run_panel itself
1855
+ # already fails closed per-lens; this catches the git setup.)
1856
+ try:
1857
+ verdict = build_panel_runner(
1858
+ ws,
1859
+ run_dir,
1860
+ base_sha,
1861
+ panel_lenses,
1862
+ contract_text,
1863
+ config.target,
1864
+ config.benchmark,
1865
+ config.bot_login,
1866
+ _dt.fromtimestamp(now, _UTC).strftime("%Y-%m-%d"),
1867
+ exclude=(
1868
+ LINE_MEMORY_PATHS if _line_ref_for(bench, config.agent_id) else ()
1869
+ ),
1870
+ )(baseline, candidate, str(stage.get("report", "")))
1871
+ except Exception as exc:
1872
+ if isinstance(exc, GitError) and _is_git_tamper(exc):
1873
+ # tamper during the panel is not a panel error to draft
1874
+ # around — end as a refused wake.
1875
+ return _end_refused_wake(run_root, record, exc, now, secrets)
1876
+ log.warning(
1877
+ "wake panel errored for %s (%s); opening a DRAFT",
1878
+ run_id,
1879
+ redact(f"{type(exc).__name__}: {exc}", secrets),
1880
+ )
1881
+ verdict = PanelVerdict(
1882
+ blocking=(),
1883
+ transcript="panel setup failed — NOT a clean read",
1884
+ wake_text="",
1885
+ degraded=True,
1886
+ )
1887
+ reads = panel_reads + 1
1888
+ # DEPTH AXIS (docs/design/research-loop.md): blocking findings
1889
+ # on a SUBMITTED claim go back to the AUTHOR (buildout Phase B)
1890
+ # — it revises and resubmits (a fresh seal + gate + panel), or
1891
+ # concludes. A plain finish (or an unresumable session) DRAFTs
1892
+ # the PR with the findings open for a human to triage.
1893
+ if bool(verdict.blocking) and submitted_park and author_resumable:
1894
+ return _wake_author(verdict.wake_text)
1895
+ result = dc_replace(
1896
+ result,
1897
+ panel_transcript=verdict.transcript,
1898
+ panel_rounds=reads,
1899
+ panel_blocking_open=bool(verdict.blocking),
1900
+ panel_degraded=verdict.degraded,
1901
+ )
1902
+
1903
+ entries = update_leader(
1904
+ load_leader(workspace),
1905
+ benchmark=bench.name,
1906
+ metric=bench.metric,
1907
+ direction=bench.direction,
1908
+ baseline=baseline,
1909
+ candidate=candidate,
1910
+ run_id=run_id,
1911
+ date=datetime.fromtimestamp(now, UTC).strftime("%Y-%m-%d"),
1912
+ run_seed=result.run_seed,
1913
+ )
1914
+ write_progress(
1915
+ workspace,
1916
+ entries,
1917
+ config.target,
1918
+ digits={b.name: b.display_digits for b in contract.benchmarks if b.display_digits},
1919
+ )
1920
+ # Stage ONLY the ledger files on top of the sealed candidate — never
1921
+ # `git add -A`, which would sweep in untracked cruft the session left
1922
+ # (eval caches) that was neither measured nor scope-checked. The
1923
+ # candidate content is already vetted (measure_and_decide's scope
1924
+ # check on measured_paths); assert nothing but the ledger is staged.
1925
+ ws.git("add", "--", *PROGRESS_PATHS)
1926
+ staged = ws.staged_paths()
1927
+ extra = [p for p in staged if p not in PROGRESS_PATHS]
1928
+ if extra:
1929
+ raise WorkspaceDrift(f"wake commit would stage non-ledger paths: {extra[:10]}")
1930
+ # Commit the ledger update on top of the sealed candidate ONLY when
1931
+ # it actually moved. When the candidate beat its baseline but not the
1932
+ # recorded best, update_leader is a no-op (the ledger's `best` does
1933
+ # not advance) — a valid composable win with no leaderboard change,
1934
+ # so push the candidate as-is rather than an empty commit.
1935
+ if staged:
1936
+ ws.git(
1937
+ "-c",
1938
+ f"user.name={config.bot_login}",
1939
+ "-c",
1940
+ f"user.email={config.bot_login}@users.noreply.github.com",
1941
+ "commit",
1942
+ "-m",
1943
+ f"agent: improve {config.benchmark} ({_title_pair(baseline, candidate)})"
1944
+ f"\n\nAgent: {config.agent_id}",
1945
+ )
1946
+ ws.push(branch)
1947
+ body = pr_body(
1948
+ result, config, redact_secrets=secrets, display_digits=bench.display_digits
1949
+ )
1950
+ if issue_number:
1951
+ body = f"Addresses #{issue_number}.\n\n{body}"
1952
+ # blocking findings still open at the panel, or a degraded final
1953
+ # read, mean a human must look: open a DRAFT and never arm. A clean
1954
+ # verdict (or no panel configured) opens non-draft and arms
1955
+ # auto-merge only where branch protection requires a review — same
1956
+ # policy as the inline path.
1957
+ draft = result.panel_blocking_open or result.panel_degraded
1958
+ pr_url = github.create_pull(
1959
+ config.target,
1960
+ title=f"[agent] {config.benchmark}: {_title_pair(baseline, candidate)}",
1961
+ head=branch,
1962
+ base=base_branch,
1963
+ body=body,
1964
+ draft=draft,
1965
+ )
1966
+ pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1]
1967
+ if pr_number.isdigit() and not draft:
1968
+ _arm_unless_base_moved(
1969
+ github,
1970
+ ws,
1971
+ config.target,
1972
+ pr_number,
1973
+ base_branch,
1974
+ base_sha,
1975
+ secrets,
1976
+ merge_mode=getattr(contract, "merge", "manual"),
1977
+ panel_ran=result.panel_rounds > 0,
1978
+ )
1979
+ except Exception as exc:
1980
+ if isinstance(exc, GitError) and _is_git_tamper(exc):
1981
+ # a concurrent object-store removal during the publish block is
1982
+ # tamper, not a publish failure: end as a refused wake (the
1983
+ # clean tamper terminal) rather than mask it as a publish-error.
1984
+ return _end_refused_wake(run_root, record, exc, now, secrets)
1985
+ # push / PR / commit failed — end as an error. Save the ENDED record
1986
+ # BEFORE dropping the snapshot (same ordering as the other terminals):
1987
+ # a failed save then leaves the run WAITING with its snapshot intact
1988
+ # (recoverable), never WAITING with the candidate already gone. On a
1989
+ # successful end, drop the snapshot — ENDED runs are never swept, so
1990
+ # keeping it would only leak the ref (a retry is a fresh climb, not a
1991
+ # re-wake). Never delete a remote branch (a push may have
1992
+ # half-succeeded).
1993
+ note = redact(f"{type(exc).__name__}: {exc}", secrets)[:480]
1994
+ log.warning("wake publish failed for %s: %s", run_id, note)
1995
+ failed = _clear_stage(
1996
+ RunRecord(
1997
+ **{**record.__dict__, "state": ENDED, "ending": ABORTED, "ending_note": note}
1998
+ )
1999
+ )
2000
+ if _best_effort("ending record", lambda: save_record(run_root, failed, now), secrets):
2001
+ drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
2002
+ _notebook("publish-error")
2003
+ return AttemptOutcome(run_id=run_id, outcome="publish-error")
2004
+ # PR opened. Record IN_REVIEW *before* dropping the snapshot: if the save
2005
+ # fails, the record stays `waiting` with the snapshot intact, so the run
2006
+ # is recoverable rather than an ABORTED record over a live PR.
2007
+ report_path = run_dir / "report.md"
2008
+ _best_effort(
2009
+ "run report",
2010
+ lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
2011
+ secrets,
2012
+ )
2013
+ final = _clear_stage(
2014
+ RunRecord(
2015
+ **{
2016
+ **record.__dict__,
2017
+ "state": IN_REVIEW,
2018
+ "pr_url": pr_url,
2019
+ "auto_blessed_head": _blessed_head(ws, result, contract),
2020
+ "resume_session_id": result.session.session_id if result.session else "",
2021
+ "ending_note": pr_url,
2022
+ }
2023
+ )
2024
+ )
2025
+ if _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
2026
+ drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
2027
+ else:
2028
+ log.warning(
2029
+ "run %s: PR %s opened but in-review record unsaved; snapshot kept", run_id, pr_url
2030
+ )
2031
+ _post_issue_finished(
2032
+ github,
2033
+ config.target,
2034
+ issue_number,
2035
+ run_id,
2036
+ "improved",
2037
+ pr_url,
2038
+ redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
2039
+ secrets,
2040
+ )
2041
+ return AttemptOutcome(
2042
+ run_id=run_id, outcome="improved", pr_url=pr_url, report_path=str(report_path)
2043
+ )
2044
+
2045
+ # a negative terminal: end the record, THEN release the snapshot. Save
2046
+ # BEFORE dropping (same ordering as the improved path): if the save fails,
2047
+ # the run stays WAITING with its snapshot intact, so a re-wake can still
2048
+ # reconstruct — never WAITING with the snapshot already gone.
2049
+ _notebook(result.outcome)
2050
+ report_path = run_dir / "report.md"
2051
+ _best_effort(
2052
+ "run report",
2053
+ lambda: report_path.write_text(result.report(config, redact_secrets=secrets)),
2054
+ secrets,
2055
+ )
2056
+ final = _clear_stage(
2057
+ RunRecord(
2058
+ **{
2059
+ **record.__dict__,
2060
+ "state": ENDED,
2061
+ "ending": _ENDINGS_BY_OUTCOME[result.outcome],
2062
+ "ending_note": redact(result.note, secrets),
2063
+ }
2064
+ )
2065
+ )
2066
+ if _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
2067
+ drop_snapshot(ws, Snapshot(commit=candidate_sha, tree="", ref=candidate_ref))
2068
+ else:
2069
+ log.warning(
2070
+ "run %s: ended negative but record unsaved; snapshot kept for a re-wake", run_id
2071
+ )
2072
+ _post_issue_finished(
2073
+ github,
2074
+ config.target,
2075
+ issue_number,
2076
+ run_id,
2077
+ result.outcome,
2078
+ "",
2079
+ redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
2080
+ secrets,
2081
+ )
2082
+ return AttemptOutcome(run_id=run_id, outcome=result.outcome, report_path=str(report_path))
2083
+
2084
+
2085
+ def _judge_lens_key(
2086
+ *,
2087
+ backend: str,
2088
+ key_file_env: str,
2089
+ author_backend: str,
2090
+ claude_panel_path: Path,
2091
+ image: str,
2092
+ ) -> str:
2093
+ """Resolve a non-claude judge lens's OWN key file, enforcing the three
2094
+ separations every shelled judge shares (codex, hermes, any future
2095
+ backend): the image is required (a judge never runs uncontained next to
2096
+ key files), the key must be named explicitly, and it must differ from BOTH
2097
+ the author's key of the same provider AND the claude panel key (a
2098
+ cross-provider send would leak an anthropic credential to another login).
2099
+ Returns the redacted key (or "" under an ADC-covered deployment)."""
2100
+ if not image:
2101
+ raise ValueError(
2102
+ f"a {backend} panel lens requires --image (a shelled judge only "
2103
+ "ever runs inside the container)"
2104
+ )
2105
+ raw = os.environ.get(key_file_env, "").strip()
2106
+ if not raw:
2107
+ raise ValueError(
2108
+ f"a {backend} panel lens needs {key_file_env} "
2109
+ "(role separation: the judge's own key, never the author's)"
2110
+ )
2111
+ path = Path(raw).expanduser()
2112
+ author_path = Path(resolve_author_key_file(author_backend)).expanduser()
2113
+ if path.resolve() == author_path.resolve():
2114
+ raise ValueError(
2115
+ f"{backend} panel key file {path} is the {author_backend} author key "
2116
+ "(role separation: the judge needs its own key)"
2117
+ )
2118
+ if path.resolve() == claude_panel_path.resolve():
2119
+ raise ValueError(
2120
+ f"{backend} panel key file {path} is the claude panel key file "
2121
+ "(an anthropic key must never reach another provider's login)"
2122
+ )
2123
+ return role_key(raw, author_backend)
2124
+
2125
+
2126
+ def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str, ...]]:
2127
+ """Build the verification-panel lenses from the CLI args (empty `--panel`
2128
+ disables it), returning `(lenses, panel_secrets)` — the ONE owner of
2129
+ panel credentials: each backend's judge key is read only when a lens uses
2130
+ it, role separation is enforced HERE (a manual climb gets the same rule
2131
+ as the tick preflight), and every key a judge holds joins the caller's
2132
+ redaction set via `panel_secrets`. Shared by the fresh-climb and the
2133
+ `--resume` wake paths so a dispatched improvement runs the SAME panel as
2134
+ an inline one. Raises ValueError on a bad panel/backend config — a
2135
+ configured gate must never silently vanish."""
2136
+ import os
2137
+
2138
+ if not args.panel.strip():
2139
+ return (), ()
2140
+ from outerloop.panel import parse_lenses
2141
+ from outerloop.roles import reviewer_spec
2142
+
2143
+ parsed = parse_lenses(args.panel)
2144
+ # the anthropic panel key is read only when a claude lens will use it —
2145
+ # a codex-only panel must not demand an unrelated credential
2146
+ panel_key = role_key(args.panel_key_file) if any(b == "claude" for _, b, _ in parsed) else ""
2147
+ lenses = []
2148
+ secrets: list[str] = [panel_key] if panel_key else []
2149
+ for kind, backend, model in parsed:
2150
+ hermes_repo_env = os.environ.get("REVIEW_HERMES_REPO", "").strip()
2151
+ # per-backend judge keys coexist — a codex lens is never handed the
2152
+ # anthropic panel key, and role separation forbids defaulting to the
2153
+ # AUTHOR's codex key: the judge key is its own, named explicitly
2154
+ claude_panel_path = Path(args.panel_key_file or PANEL_KEY_DEFAULT).expanduser()
2155
+ if backend == "codex":
2156
+ lens_key = _judge_lens_key(
2157
+ backend="codex",
2158
+ key_file_env="AUTORESEARCH_PANEL_CODEX_KEY_FILE",
2159
+ author_backend="codex",
2160
+ claude_panel_path=claude_panel_path,
2161
+ image=args.image,
2162
+ )
2163
+ if lens_key:
2164
+ secrets.append(lens_key)
2165
+ elif backend == "hermes":
2166
+ # hermes reads its key from its provider's env var, but the FILE
2167
+ # is resolved and separated exactly like codex's (the key still
2168
+ # lands next to the session). The author's OpenAI key coexists, so
2169
+ # separate against the codex author key.
2170
+ lens_key = _judge_lens_key(
2171
+ backend="hermes",
2172
+ key_file_env="AUTORESEARCH_PANEL_HERMES_KEY_FILE",
2173
+ author_backend="codex",
2174
+ claude_panel_path=claude_panel_path,
2175
+ image=args.image,
2176
+ )
2177
+ if lens_key:
2178
+ secrets.append(lens_key)
2179
+ else:
2180
+ lens_key = panel_key
2181
+ try:
2182
+ judge = build_harness(
2183
+ lens_key,
2184
+ reviewer_spec(),
2185
+ backend=backend,
2186
+ binary=args.claude_bin if backend == "claude" else args.codex_bin,
2187
+ model=model or None,
2188
+ # ALWAYS contained: the panel runs on the climb host next to key
2189
+ # files, and a judge now holds a shell (codex `danger-full-access`),
2190
+ # so it must run inside the image. `parse_lenses` gates panel
2191
+ # backends to those containable here (claude today); passing the
2192
+ # image unconditionally means codex is safe the moment it is
2193
+ # enabled, never accidentally uncontained.
2194
+ container_image=args.image,
2195
+ hermes_repo=Path(hermes_repo_env) if hermes_repo_env else None,
2196
+ hermes_provider=os.environ.get("REVIEW_HERMES_PROVIDER", "openrouter"),
2197
+ )
2198
+ except ValueError as exc:
2199
+ raise ValueError(f"panel entry {kind}:{backend}: {exc}") from exc
2200
+ lenses.append(PanelLens(kind=kind, harness=judge))
2201
+ return tuple(lenses), tuple(dict.fromkeys(secrets))
2202
+
2203
+
2204
+ def _panel_claim_body(
2205
+ benchmark: str, baseline: float, candidate: float, report: str, *, lines: bool
2206
+ ) -> str:
2207
+ """The synthetic claim the panel judges. On a research-lines target the
2208
+ one-contribution mandate is part of the claim itself: the panel is the
2209
+ backstop against a line's accumulated tweaks reaching main as one PR
2210
+ (docs/design/research-lines.md)."""
2211
+ mandate = (
2212
+ "\n\nThis target runs research lines: a PR to main must be ONE "
2213
+ "clean contribution, extracted onto the base branch. A diff that "
2214
+ "bundles unrelated or unablated changes is a BLOCKING finding — "
2215
+ "name the pieces that should be separated."
2216
+ if lines
2217
+ else ""
2218
+ )
2219
+ return (
2220
+ f"Automated improvement claim (pre-PR): {benchmark} "
2221
+ f"{baseline} -> {candidate}, measured by the orchestrator.{mandate}\n\n"
2222
+ f"## Research report\n\n*Session prose, written before "
2223
+ f"the orchestrator measured.*\n\n{report[:MAX_CLAIM_CHARS]}"
2224
+ )
2225
+
2226
+
2227
+ def build_panel_runner(
2228
+ ws: Workspace,
2229
+ run_dir: Path,
2230
+ base_sha: str,
2231
+ lenses: tuple[PanelLens, ...],
2232
+ contract_text: str,
2233
+ target: str,
2234
+ benchmark: str,
2235
+ bot_login: str,
2236
+ today: str,
2237
+ start_round: int = 0,
2238
+ exclude: tuple[str, ...] = (),
2239
+ claim_body: Callable[[float, float, str], str] | None = None,
2240
+ ) -> Callable[[float, float, str], PanelVerdict]:
2241
+ """The git half of the pre-PR panel: prepare the two read-only checkouts
2242
+ and the synthetic claim, then hand off to `run_panel` (which owns no git).
2243
+
2244
+ Each call snapshots the CURRENT workspace tree as a detached commit and
2245
+ checks it out as `pr-head/` (sanitized — the candidate is an untrusted
2246
+ tree), next to `base/` (the trusted pre-session commit: contract and
2247
+ ruler). Worktrees are removed after the read; a fresh pair is built per
2248
+ round because the tree changes with every revision.
2249
+
2250
+ `claim_body` renders the claim the panel judges from (baseline,
2251
+ candidate, report); the default is the pre-PR improvement claim, a
2252
+ follow-up re-read passes its own wording."""
2253
+ from outerloop.review_agent import sanitize_checkout
2254
+
2255
+ reads = {"n": start_round}
2256
+ render_claim = claim_body or (
2257
+ lambda baseline, candidate, report: _panel_claim_body(
2258
+ benchmark, baseline, candidate, report, lines=bool(exclude)
2259
+ )
2260
+ )
2261
+
2262
+ def runner(baseline: float, candidate: float, report: str) -> PanelVerdict:
2263
+ reads["n"] += 1
2264
+ panel_ws = run_dir / "panel"
2265
+ shutil.rmtree(panel_ws, ignore_errors=True)
2266
+ panel_ws.mkdir(parents=True, exist_ok=True)
2267
+ ws.git("add", "-A")
2268
+ if exclude:
2269
+ # the panel judges the CLAIM — the same tree the gate measured,
2270
+ # which excludes line memory (docs/design/research-lines.md)
2271
+ ws.git("rm", "--cached", "-r", "-q", "--ignore-unmatch", "--", *exclude)
2272
+ tree = ws.git("write-tree").strip()
2273
+ ws.git("reset")
2274
+ snapshot = ws.git(
2275
+ "-c",
2276
+ "user.name=panel",
2277
+ "-c",
2278
+ "user.email=panel@localhost",
2279
+ "commit-tree",
2280
+ tree,
2281
+ "-p",
2282
+ base_sha,
2283
+ "-m",
2284
+ "panel snapshot (never pushed)",
2285
+ ).strip()
2286
+ try:
2287
+ ws.git("worktree", "add", "--detach", str(panel_ws / "base"), base_sha)
2288
+ ws.git("worktree", "add", "--detach", str(panel_ws / "pr-head"), snapshot)
2289
+ _renamed, failed = sanitize_checkout(panel_ws / "pr-head")
2290
+ if failed:
2291
+ # fail closed for the read, loudly in the transcript: an
2292
+ # unsanitizable tree is never judged, and never certified
2293
+ return PanelVerdict(
2294
+ blocking=(),
2295
+ transcript=(
2296
+ f"**Verification round {reads['n']}**\n- panel skipped: "
2297
+ f"the candidate tree could not be sanitized "
2298
+ f"({failed} instruction file(s) left) — NOT a clean read"
2299
+ ),
2300
+ wake_text="",
2301
+ degraded=True,
2302
+ )
2303
+ claim = PullRequest(
2304
+ repo=target,
2305
+ number=0,
2306
+ title=f"[agent] {benchmark}: {_title_pair(baseline, candidate)}",
2307
+ body=render_claim(baseline, candidate, report),
2308
+ # base..snapshot, never base..worktree: the snapshot commit
2309
+ # includes newly ADDED files, which a working-tree diff omits.
2310
+ # Excluded (line-memory) paths are excluded from the diff too:
2311
+ # the snapshot dropped them, so against a line-tip base they
2312
+ # would read as deletions the author never made
2313
+ diff=ws.git(
2314
+ "diff",
2315
+ f"{base_sha}..{snapshot}",
2316
+ *(["--", ".", *(f":(exclude){p}" for p in exclude)] if exclude else []),
2317
+ ),
2318
+ author=bot_login,
2319
+ )
2320
+ return run_panel(lenses, panel_ws, claim, contract_text, today, reads["n"])
2321
+ finally:
2322
+ for name in ("base", "pr-head"):
2323
+ _best_effort(
2324
+ "panel worktree cleanup",
2325
+ partial(ws.git, "worktree", "remove", "--force", str(panel_ws / name)),
2326
+ )
2327
+ _best_effort("panel dir removal", lambda: shutil.rmtree(panel_ws, ignore_errors=True))
2328
+ _best_effort("panel worktree prune", lambda: ws.git("worktree", "prune"))
2329
+
2330
+ return runner
2331
+
2332
+
2333
+ def live_attempt(
2334
+ config: RunConfig,
2335
+ run_root: Path,
2336
+ run_id: str,
2337
+ harness: Harness,
2338
+ github: GitHubClient,
2339
+ bot_auth: TokenProvider,
2340
+ now: float,
2341
+ created: str,
2342
+ secrets: tuple[str, ...] = (),
2343
+ base_branch: str = "main",
2344
+ issue_number: int = 0,
2345
+ author_backend: str = "claude",
2346
+ author_model: str = "",
2347
+ author_key_file: str = "",
2348
+ task_hypothesis: str = "",
2349
+ spec: RoleSpec | None = None,
2350
+ panel_lenses: tuple[PanelLens, ...] = (),
2351
+ dispatch: DispatchSettings | None = None,
2352
+ eval_image: str = "",
2353
+ ) -> AttemptOutcome:
2354
+ """Run one climb against the real target repo. With `panel_lenses`, the
2355
+ pre-PR verification panel gates the claim before any PR exists
2356
+ (docs/design/orchestrator-verify.md); blocking findings still open at
2357
+ the cap open a DRAFT PR carrying them."""
2358
+ run_dir = run_root / "runs" / run_id
2359
+ run_dir.mkdir(parents=True, exist_ok=True)
2360
+ workspace = run_dir / "ws"
2361
+
2362
+ # The record exists before any network or clone work: every crash from
2363
+ # here on has a record to end.
2364
+ import os as _os
2365
+
2366
+ record = RunRecord(
2367
+ run_id=run_id,
2368
+ target=config.target,
2369
+ task_title=f"improve {config.benchmark}",
2370
+ benchmark=config.benchmark,
2371
+ state="implementing",
2372
+ agent_id=config.agent_id,
2373
+ deadline=now + 24 * 3600,
2374
+ issue_number=issue_number,
2375
+ author_backend=author_backend,
2376
+ author_model=author_model,
2377
+ author_key_file=author_key_file,
2378
+ run_job_id=_os.environ.get("SLURM_JOB_ID", ""),
2379
+ )
2380
+ try:
2381
+ save_record(run_root, record, now)
2382
+ except Exception as exc:
2383
+ # No record could be written, so the run must not proceed invisibly:
2384
+ # nothing would ever end it. Submit-time evidence (the claim comment
2385
+ # or the pending marker) plus this post keep the failure visible.
2386
+ exc_name = type(exc).__name__
2387
+ log.warning(
2388
+ "could not create run record for %s: %s",
2389
+ run_id,
2390
+ redact(f"{exc_name}: {exc}", secrets),
2391
+ )
2392
+ if issue_number:
2393
+ _best_effort(
2394
+ "issue report",
2395
+ lambda: github.comment(
2396
+ config.target,
2397
+ issue_number,
2398
+ f"Run `{run_id}` could not start ({exc_name} while writing its run record).",
2399
+ ),
2400
+ secrets,
2401
+ )
2402
+ return AttemptOutcome(run_id=run_id, outcome="attempt-error")
2403
+
2404
+ # what the attempt-error handler needs to salvage the line notebook: the
2405
+ # exception path cannot rely on names bound inside the try
2406
+ salvage: dict[str, object] = {}
2407
+ try:
2408
+ ws = Workspace.clone(_target_clone_url(config.target), workspace, auth=bot_auth)
2409
+ # Build ON the requested PR base: the clone checks out the remote
2410
+ # DEFAULT branch, which need not be `base_branch` — the session must
2411
+ # edit, and the gate must measure, the tree the PR will land on.
2412
+ # A missing base branch fails loudly as attempt-error.
2413
+ ws.git("checkout", "-q", "-B", base_branch, f"origin/{base_branch}")
2414
+ _exclude_merge_artifacts(workspace)
2415
+ contract_text = contract_text_in_tree(workspace)
2416
+ contract = load_contract(contract_text, config.target)
2417
+ # Load the brief budget from the contract and run state: callers do
2418
+ # not supply it (the dataclass default rendered "0.0 GPU-hours" and
2419
+ # honest agents refused to launch). Same weekly counting rule as the
2420
+ # tick's cap: records plus live pending markers, minus this run's own.
2421
+ from outerloop.tick import list_pendings
2422
+
2423
+ week_ago = now - 7 * 24 * 3600
2424
+ recent = [
2425
+ r for r in list_runs(run_root) if r.target == config.target and r.created >= week_ago
2426
+ ]
2427
+ # a marker whose job already has a record (this run's included) is
2428
+ # the same attempt, not a second one — count each job once
2429
+ recorded_jobs = {r.run_job_id for r in recent if r.run_job_id} | {record.run_job_id}
2430
+ # every unrecorded week-fresh marker counts: the tick reaps dead
2431
+ # markers on its own cadence (with the squeue liveness reads a brief
2432
+ # must not make), so an unreaped marker is either a live queued run
2433
+ # the weekly cap WILL count, or dead for at most a sweep — the brief
2434
+ # stays on the cap's conservative side either way
2435
+ used_week = len(recent) + sum(
2436
+ 1
2437
+ for _agent, marker in list_pendings(run_root, config.target)
2438
+ if float(marker.get("submitted_at", 0) or 0) >= week_ago
2439
+ and str(marker.get("job_id", "")) not in recorded_jobs
2440
+ )
2441
+ _budget_bench = next((b for b in contract.benchmarks if b.name == config.benchmark), None)
2442
+ config = dc_replace(
2443
+ config,
2444
+ budget=BudgetState(
2445
+ gpu_hours_remaining=(
2446
+ float(contract.budgets.gpu_hours_per_run or 0.0)
2447
+ if _budget_bench is not None and _budget_bench.gpus
2448
+ else 0.0
2449
+ ),
2450
+ runs_remaining_this_week=max(0, int(contract.budgets.runs_per_week) - used_week),
2451
+ ),
2452
+ )
2453
+ # Author syscalls (research-loop.md, "one syscall") are CONTRACT-DRIVEN:
2454
+ # armed whenever the deployment can deliver them — dispatch coords (the
2455
+ # launches and the gate run as Slurm jobs) and a resumable backend (the
2456
+ # wake resumes the SAME session) — and the benchmark has not opted out
2457
+ # (`depth_k: 0`). With the channel (`.outerloop/`) armed it never
2458
+ # enters diffs or scope — repo-local exclude. With the feature off, an
2459
+ # untracked `.outerloop/` file must be staged and judged like any
2460
+ # other agent edit, not silently hidden by a magic dir name (the off
2461
+ # state stays byte-identical).
2462
+ _bench = next((b for b in contract.benchmarks if b.name == config.benchmark), None)
2463
+ # Research lines: move HEAD to the agent's own branch BEFORE anything
2464
+ # reads the tree — the contract above came from the base branch (a
2465
+ # line must not shape its own budgets), and the syscall-channel check
2466
+ # below must see the line's tree. A failed checkout falls back to the
2467
+ # base branch: a run is never lost to its notebook.
2468
+ lines_active = _bench is not None and _bench.lines and bool(config.agent_id)
2469
+ line_ref = ""
2470
+ if lines_active:
2471
+ try:
2472
+ line_ref = _checkout_line(ws, workspace, config.agent_id, base_branch)
2473
+ except Exception as exc:
2474
+ log.warning(
2475
+ "line checkout failed (%s); running on %s",
2476
+ redact(f"{type(exc).__name__}: {exc}", secrets),
2477
+ base_branch,
2478
+ )
2479
+ _best_effort("line merge abort", lambda: ws.git("merge", "--abort"))
2480
+ ws.git("checkout", "-q", "-B", base_branch, f"origin/{base_branch}")
2481
+ line_memory = ""
2482
+ line_divergence = ""
2483
+ if line_ref:
2484
+ salvage.update(ws=ws, line_ref=line_ref)
2485
+ try:
2486
+ # the line's own memory index, rendered into the brief
2487
+ # (data-fenced there); topic files are read on demand from
2488
+ # the checkout, never rendered
2489
+ memory_path = workspace / "AGENT_MEMORY.md"
2490
+ if memory_path.is_file() and not memory_path.is_symlink():
2491
+ # byte-mode bounded read: never load an oversized file
2492
+ with memory_path.open("rb") as fh:
2493
+ line_memory = fh.read(65_536).decode("utf-8", errors="replace")
2494
+ except OSError as exc:
2495
+ log.warning("could not read AGENT_MEMORY.md: %s", exc)
2496
+ try:
2497
+ # divergence debt, made visible each session (a conflicted
2498
+ # merge skips it — the diff is not meaningful mid-merge)
2499
+ if not ws.git("diff", "--name-only", "--diff-filter=U").strip():
2500
+ line_divergence = ws.git(
2501
+ "diff", "--shortstat", f"origin/{base_branch}", "HEAD"
2502
+ ).strip()
2503
+ except Exception:
2504
+ line_divergence = ""
2505
+ author_syscalls = (
2506
+ dispatch is not None
2507
+ and getattr(harness, "supports_resume", True)
2508
+ and _bench is not None
2509
+ and _bench.depth_k > 0
2510
+ )
2511
+ # The `.outerloop/` channel must be KERNEL-OWNED. In a fresh clone,
2512
+ # anything already at that path was committed by the TARGET — a symlink
2513
+ # (install would write through it to a host path with our permissions),
2514
+ # a tracked request (free cluster compute), or
2515
+ # any other booby trap. If the path pre-exists in ANY form, disable the
2516
+ # feature for the run, loudly; otherwise we create a dir we own.
2517
+ # A target must not ship EITHER channel name (both are booby-trap risks:
2518
+ # a symlink install writes through, a planted request steals compute).
2519
+ shipped = next(
2520
+ (
2521
+ n
2522
+ for n in CHANNEL_DIR_NAMES
2523
+ if (workspace / n).is_symlink() or (workspace / n).exists()
2524
+ ),
2525
+ "",
2526
+ )
2527
+ if author_syscalls and shipped:
2528
+ log.warning(
2529
+ "target ships a %s path (symlink=%s); author syscalls disabled for this run",
2530
+ shipped,
2531
+ (workspace / shipped).is_symlink(),
2532
+ )
2533
+ author_syscalls = False
2534
+ reports = _fetch_research_reports(ws, MAX_ARCHIVED_REPORTS)
2535
+ if author_syscalls:
2536
+ assert _bench is not None
2537
+ syscall_excluded(workspace)
2538
+ # the author's interface is the TOOL (`python .outerloop/syscall
2539
+ # launch ... -- <cmd>`; `... sleep`), never the raw ABI file —
2540
+ # install it plus the informational budget its `status` shows.
2541
+ syscall_install_tool(workspace)
2542
+ syscall_write_budget(
2543
+ workspace,
2544
+ launches_remaining=_bench.depth_k,
2545
+ sleeps_remaining=_bench.sleep_k,
2546
+ gpu_hours_remaining=(
2547
+ float(contract.budgets.gpu_hours_per_run) if _bench.gpus else None
2548
+ ),
2549
+ )
2550
+ # AFTER install_tool: installing the tool recreates the channel
2551
+ # dir it owns, which would delete an archive written earlier
2552
+ _install_report_archive(workspace, reports)
2553
+ # the fleet snapshot the `siblings` command shows — read from the
2554
+ # research-log's status.json (the SAME branch the reports came
2555
+ # from, so it works across clusters) and best-effort throughout:
2556
+ # a missing or malformed snapshot just means no siblings known
2557
+ syscall_write_siblings(workspace, _sibling_entries(ws, config.agent_id))
2558
+
2559
+ def changed_paths() -> list[str]:
2560
+ # against the base branch head, never the line tip a conflicted
2561
+ # merge can leave HEAD on (see _paths_changed_from_base)
2562
+ return _paths_changed_from_base(ws, f"refs/remotes/origin/{base_branch}", lines_active)
2563
+
2564
+ if issue_number:
2565
+ from outerloop.intake import CLAIM_MARKER
2566
+
2567
+ already = any(
2568
+ has_marker(str(c.get("body", "")), "claimed")
2569
+ for c in github.list_comments(config.target, issue_number)
2570
+ )
2571
+ if not already: # manual CLI runs claim here; tick runs claimed at submit
2572
+ github.comment(
2573
+ config.target,
2574
+ issue_number,
2575
+ f"{CLAIM_MARKER}\nPicked up as run `{run_id}` "
2576
+ f"(benchmark `{config.benchmark}`). A report will follow here.",
2577
+ )
2578
+
2579
+ # Expensive benchmarks measure as dispatched cluster jobs; cheap ones
2580
+ # (and any run with no cluster coordinates) measure inline. The choice
2581
+ # is the benchmark's eval-time hint against the in-job runway, decided
2582
+ # ONCE here so the baseline setup, the measurer, and the park deadline
2583
+ # all agree on it.
2584
+ eval_minutes = next(
2585
+ (b.eval_minutes for b in contract.benchmarks if b.name == config.benchmark), None
2586
+ )
2587
+ wants_dispatch = should_dispatch(eval_minutes)
2588
+ dispatched = dispatch is not None and wants_dispatch
2589
+ if wants_dispatch and dispatch is None:
2590
+ # a benchmark asked to be dispatched but no cluster coordinates
2591
+ # reached us — never silently: name it, then measure inline
2592
+ log.warning(
2593
+ "benchmark %s wants dispatched eval (eval_minutes=%s) but no cluster "
2594
+ "coordinates (image/account/partition) are set; measuring inline",
2595
+ config.benchmark,
2596
+ eval_minutes,
2597
+ )
2598
+
2599
+ # the panel's base is the PRE-SESSION commit — the exact tree the
2600
+ # baseline was measured on — never origin/<base_branch>, which can
2601
+ # name a different branch than the clone's checkout
2602
+ pre_session_sha = ws.git("rev-parse", "HEAD").strip()
2603
+ panel_runner = (
2604
+ build_panel_runner(
2605
+ ws,
2606
+ run_dir,
2607
+ pre_session_sha,
2608
+ panel_lenses,
2609
+ contract_text,
2610
+ config.target,
2611
+ config.benchmark,
2612
+ config.bot_login,
2613
+ created[:10],
2614
+ exclude=LINE_MEMORY_PATHS if lines_active else (),
2615
+ )
2616
+ if panel_lenses
2617
+ else None
2618
+ )
2619
+ # ONE measurer either way: every measure is a job that checks its
2620
+ # tree sha out fresh from this workspace's `refs/dispatch/*` and
2621
+ # writes its result to the run dir. DISPATCHED: the jobs go to the
2622
+ # cluster and a not-yet-done measure PARKS the climb. LOCAL: the SAME
2623
+ # jobs run synchronously in this allocation (LocalCompute), so every
2624
+ # measure is done when checked and nothing parks. `snapshot` commits
2625
+ # the workspace's current content to a candidate sha and we own the
2626
+ # ref lifecycle, keeping the one candidate ref a park needs and
2627
+ # dropping the rest when the climb ends.
2628
+ measurer: Measurer
2629
+ if dispatched:
2630
+ assert dispatch is not None and eval_minutes is not None # should_dispatch(None) False
2631
+ measurer = dispatch.measurer(
2632
+ run_dir, repo_root=workspace, eval_minutes=eval_minutes, run_tag=run_id
2633
+ )
2634
+ else:
2635
+ measurer = DispatchedMeasurer(
2636
+ compute=LocalCompute(),
2637
+ run_dir=run_dir,
2638
+ repo_root=workspace,
2639
+ # a configured image contains LOCAL evals too — the cluster
2640
+ # triple being incomplete must not silently drop the jail
2641
+ image=dispatch.image if dispatch is not None else eval_image,
2642
+ account="",
2643
+ partition="",
2644
+ eval_minutes=int(eval_minutes or 0),
2645
+ run_tag=run_id,
2646
+ # an inline gate shares the same target-wide baseline cache
2647
+ baseline_cache=run_dir.parent / "baselines",
2648
+ )
2649
+ snapshots: list[Snapshot] = []
2650
+
2651
+ def snapshot() -> str:
2652
+ snap = snapshot_tree(
2653
+ ws, pre_session_sha, exclude=LINE_MEMORY_PATHS if lines_active else ()
2654
+ )
2655
+ snapshots.append(snap)
2656
+ return snap.commit
2657
+
2658
+ # `author_syscalls` already folds every enablement condition — dispatch
2659
+ # coords, a resumable backend, the benchmark's opt-out, and the
2660
+ # channel-ownership guard above (a target-shipped `.autoresearch` —
2661
+ # symlink, tracked request, or any other pre-existing form — has
2662
+ # disabled the feature for this run).
2663
+ launcher = None
2664
+ if author_syscalls:
2665
+ assert dispatch is not None # folded into author_syscalls above
2666
+ launcher = _make_launcher(
2667
+ dispatch, run_dir, workspace, run_id, gpus=_bench.gpus if _bench else 0
2668
+ )
2669
+
2670
+ parked: RunParked | None = None
2671
+ kept_ref = "" # the ONE candidate snapshot ref that must outlive a park
2672
+ try:
2673
+ # the last-known score orients the brief only; the gate re-measures
2674
+ # both sides after the session, so None (a first run) is fine.
2675
+ prior_best = load_leader(workspace).get(config.benchmark)
2676
+ result = attempt_once(
2677
+ config,
2678
+ contract_text,
2679
+ workspace,
2680
+ harness,
2681
+ measurer,
2682
+ pre_session_sha,
2683
+ snapshot,
2684
+ ruler=RULER,
2685
+ changed_paths=changed_paths,
2686
+ created=created,
2687
+ task_hypothesis=task_hypothesis,
2688
+ recent_reports=tuple(text for _name, text in reports),
2689
+ lessons=distill_lessons(reports),
2690
+ report_archive=author_syscalls,
2691
+ spec=spec,
2692
+ panel_runner=panel_runner,
2693
+ brief_baseline=prior_best.best if prior_best else None,
2694
+ line_ref=line_ref,
2695
+ line_memory=line_memory,
2696
+ line_divergence=line_divergence,
2697
+ launcher=launcher,
2698
+ tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(),
2699
+ )
2700
+ except RunParked as p:
2701
+ # The climb dispatched its measures and hibernated. Persist the
2702
+ # re-entry stage as a WAITING record (not an error), keep the
2703
+ # candidate snapshot alive for the wake, and end. The wake re-enters
2704
+ # from the record. `parked` is set only
2705
+ # AFTER a successful write: if _park_run raises, it stays None so the
2706
+ # finally drops every snapshot (no leak) and the outer handler ends
2707
+ # the run as an error rather than a half-written hibernation.
2708
+ if p.phase in ("candidate", "author-sleep"):
2709
+ # keep exactly ONE snapshot for that sha (two can share a
2710
+ # commit); record and keep that same ref, drop the rest. An
2711
+ # author-sleep's snapshot is the tree the wake re-delivers.
2712
+ kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
2713
+ import time
2714
+
2715
+ # anchor the deadline to the PARK (when the evals were submitted),
2716
+ # not the run's start `now` — a session lasting hours would otherwise
2717
+ # eat the queue budget and let the sweep cancel a still-queued eval.
2718
+ try:
2719
+ _park_run(
2720
+ run_root,
2721
+ record,
2722
+ p,
2723
+ kept_ref,
2724
+ eval_minutes,
2725
+ time.time(),
2726
+ secrets,
2727
+ dispatch=dispatch,
2728
+ base_branch=base_branch,
2729
+ )
2730
+ except Exception:
2731
+ # The WAITING record did not persist, so nothing will ever wake
2732
+ # the eval jobs this park already submitted. Cancel them so they
2733
+ # don't sit in the queue as orphans (best-effort, self-logging),
2734
+ # then fall through to the error handler — `parked` stays None,
2735
+ # so the finally still drops every snapshot. A park only happens
2736
+ # on the dispatched path, so `dispatch` is set here.
2737
+ assert dispatch is not None
2738
+ for job_id in afterany_ids(p.afterany):
2739
+ dispatch.compute.cancel(job_id)
2740
+ raise
2741
+ parked = p
2742
+ return AttemptOutcome(run_id=run_id, outcome="parked")
2743
+ finally:
2744
+ for snap in snapshots:
2745
+ # a candidate park must OUTLIVE the wake — keep the ONE recorded
2746
+ # snapshot (matched by ref, not commit); drop every other one.
2747
+ if parked and kept_ref and snap.ref == kept_ref:
2748
+ continue
2749
+ drop_snapshot(ws, snap) # best-effort + self-logging; never raises
2750
+ except Exception as exc:
2751
+ exc_name = type(exc).__name__
2752
+ note = redact(f"{exc_name}: {exc}", secrets)[:500]
2753
+ log.warning("climb failed for %s: %s", run_id, note)
2754
+ if salvage:
2755
+ # a crashed attempt's tree is still notebook-worthy (best-effort)
2756
+ _push_line_snapshot(
2757
+ cast(Workspace, salvage["ws"]),
2758
+ str(salvage["line_ref"]),
2759
+ run_id,
2760
+ "attempt-error",
2761
+ secrets,
2762
+ )
2763
+ failed = RunRecord(
2764
+ **{
2765
+ **record.__dict__,
2766
+ "state": ENDED,
2767
+ "ending": ABORTED,
2768
+ "ending_note": note,
2769
+ }
2770
+ )
2771
+ report_path = run_dir / "report.md"
2772
+ _best_effort("ending record", lambda: save_record(run_root, failed, now), secrets)
2773
+ wrote = _best_effort(
2774
+ "error report",
2775
+ lambda: report_path.write_text(
2776
+ f"# Run report — {config.target} / {config.benchmark}\n"
2777
+ f"Outcome: **attempt-error**\n"
2778
+ f"Note: {note}\n"
2779
+ ),
2780
+ secrets,
2781
+ )
2782
+ if issue_number:
2783
+ # Exception detail stays in the local record and report: redact()
2784
+ # only knows the secrets it was handed, and raw messages can carry
2785
+ # paths or tokens the tuple does not cover. The issue gets the
2786
+ # exception TYPE only.
2787
+ _best_effort(
2788
+ "issue report",
2789
+ lambda: github.comment(
2790
+ config.target,
2791
+ issue_number,
2792
+ f"Run `{run_id}` finished (attempt-error): {exc_name}. "
2793
+ f"Details are in the run's record and report on the orchestrator.",
2794
+ ),
2795
+ secrets,
2796
+ )
2797
+ return AttemptOutcome(
2798
+ run_id=run_id,
2799
+ outcome="attempt-error",
2800
+ # an outcome must never point at a report that was not written
2801
+ report_path=str(report_path) if wrote else "",
2802
+ )
2803
+
2804
+ if result.outcome == "improved" and not result.measured_paths:
2805
+ # a zero-change "improvement" is metric noise, not progress — never a
2806
+ # PR (same rule as the wake publish)
2807
+ result = dc_replace(result, outcome="no-improvement", note="no code change; metric noise")
2808
+
2809
+ report = result.report(config, redact_secrets=secrets)
2810
+ report_path = run_dir / "report.md"
2811
+ wrote_report = _best_effort("run report", lambda: report_path.write_text(report), secrets)
2812
+
2813
+ # Research lines: seal the notebook NOW, while the tree is still the
2814
+ # session's final tree — the publish below force-checkouts the sealed
2815
+ # candidate and cleans untracked files, which would drop the agent's
2816
+ # memory (it is excluded from measurable seals by design). The label is
2817
+ # the GATE outcome, correct at this moment; a publish failure appends a
2818
+ # publish-error snapshot at the tail.
2819
+ _push_line_snapshot(ws, line_ref, run_id, result.outcome, secrets)
2820
+
2821
+ pr_url = ""
2822
+ outcome_name = result.outcome
2823
+ branch = ""
2824
+ pushed = False
2825
+ if result.outcome == "improved":
2826
+ try:
2827
+ # Publish the SEALED candidate sha — never the live tree, which
2828
+ # may have drifted since the snapshot (eval caches, stray writes);
2829
+ # the sha is exactly the measured, scope-checked content. A base
2830
+ # branch that moved during the climb is NOT merged and re-measured
2831
+ # here: a stale PR is review's to handle (research-loop.md).
2832
+ branch = f"{config.branch_prefix}/{run_id}"
2833
+ if result.baseline is None or result.candidate is None or not result.candidate_sha:
2834
+ raise EvalError("improved result missing measurements or the sealed sha")
2835
+ bench = next(b for b in contract.benchmarks if b.name == config.benchmark)
2836
+ baseline, candidate = result.baseline, result.candidate
2837
+ # FORCE-checkout: the workspace still holds the session's dirty
2838
+ # tree. The snapshot commit is anchored by the new branch (the
2839
+ # dropped dispatch ref left it unreferenced; nothing pruned it in
2840
+ # this process). clean -fd drops post-snapshot cruft so the
2841
+ # pushed tree is exactly candidate_sha plus the ledger commit.
2842
+ ws.git("checkout", "-f", "-B", branch, result.candidate_sha)
2843
+ ws.git("clean", "-fd")
2844
+ entries = update_leader(
2845
+ load_leader(workspace),
2846
+ benchmark=bench.name,
2847
+ metric=bench.metric,
2848
+ direction=bench.direction,
2849
+ baseline=baseline,
2850
+ candidate=candidate,
2851
+ run_id=run_id,
2852
+ date=created[:10],
2853
+ run_seed=result.run_seed,
2854
+ )
2855
+ write_progress(
2856
+ workspace,
2857
+ entries,
2858
+ config.target,
2859
+ digits={b.name: b.display_digits for b in contract.benchmarks if b.display_digits},
2860
+ )
2861
+ # Stage ONLY the ledger files on top of the sealed candidate —
2862
+ # never `git add -A`, which would sweep in anything a session or
2863
+ # eval left behind (same rule as the wake publish).
2864
+ ws.git("add", "--", *PROGRESS_PATHS)
2865
+ staged = ws.staged_paths()
2866
+ extra = [p for p in staged if p not in PROGRESS_PATHS]
2867
+ if extra:
2868
+ raise WorkspaceDrift(f"publish would stage non-ledger paths: {extra[:10]}")
2869
+ if staged:
2870
+ ws.git(
2871
+ "-c",
2872
+ f"user.name={config.bot_login}",
2873
+ "-c",
2874
+ f"user.email={config.bot_login}@users.noreply.github.com",
2875
+ "commit",
2876
+ "-m",
2877
+ f"agent: improve {config.benchmark} ({_title_pair(baseline, candidate)})"
2878
+ f"\n\nAgent: {config.agent_id}",
2879
+ )
2880
+ ws.push(branch)
2881
+ pushed = True
2882
+ body = pr_body(
2883
+ result, config, redact_secrets=secrets, display_digits=bench.display_digits
2884
+ )
2885
+ if issue_number:
2886
+ body = f"Addresses #{issue_number}.\n\n{body}"
2887
+ pr_url = github.create_pull(
2888
+ config.target,
2889
+ # short precision in the title; full precision lives in the
2890
+ # PR body table and the ledger
2891
+ title=f"[agent] {config.benchmark}: {_title_pair(baseline, candidate)}",
2892
+ head=branch,
2893
+ base=base_branch,
2894
+ body=body,
2895
+ # blocking findings open at the panel, or a degraded final
2896
+ # read: visible, plainly not merge-ready
2897
+ draft=result.panel_blocking_open or result.panel_degraded,
2898
+ )
2899
+ # Arm auto-merge, best-effort, and ONLY when branch protection
2900
+ # requires a human review — the guard keeps bot-never-merges
2901
+ # enforced in code, not in per-repo config. Never arm a draft,
2902
+ # and never arm a claim whose base has moved (_arm_unless_base_moved).
2903
+ pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1]
2904
+ if pr_number.isdigit() and not (result.panel_blocking_open or result.panel_degraded):
2905
+ _arm_unless_base_moved(
2906
+ github,
2907
+ ws,
2908
+ config.target,
2909
+ pr_number,
2910
+ base_branch,
2911
+ pre_session_sha,
2912
+ secrets,
2913
+ merge_mode=getattr(contract, "merge", "manual"),
2914
+ panel_ran=result.panel_rounds > 0,
2915
+ )
2916
+ final = RunRecord(
2917
+ **{
2918
+ **record.__dict__,
2919
+ "state": IN_REVIEW,
2920
+ "pr_url": pr_url,
2921
+ "auto_blessed_head": _blessed_head(ws, result, contract),
2922
+ "resume_session_id": result.session.session_id if result.session else "",
2923
+ "ending_note": pr_url,
2924
+ }
2925
+ )
2926
+ except Exception as exc:
2927
+ log.warning(
2928
+ "publish failed for %s: %s",
2929
+ run_id,
2930
+ redact(f"{type(exc).__name__}: {exc}", secrets),
2931
+ )
2932
+ # Never delete the remote branch: an exception from create_pull
2933
+ # does not prove no PR exists (a 422-already-exists or a timeout
2934
+ # after a successful POST both land here), and deleting the ref
2935
+ # would close such a PR and discard the only pushed copy. Leave
2936
+ # it and record it; a sweeper can reap confirmed orphans later.
2937
+ outcome_name = "publish-error"
2938
+ final = RunRecord(
2939
+ **{
2940
+ **record.__dict__,
2941
+ "state": ENDED,
2942
+ "ending": ABORTED,
2943
+ "ending_note": (
2944
+ (f"branch left on remote: {branch}; " if pushed else "")
2945
+ + redact(f"{type(exc).__name__}: {exc}", secrets)[:480]
2946
+ ),
2947
+ }
2948
+ )
2949
+ else:
2950
+ if result.outcome == "session-outage":
2951
+ _best_effort(
2952
+ "outage stamp",
2953
+ lambda: stamp_outage(run_root, redact(result.note, secrets)[:300], now),
2954
+ secrets,
2955
+ )
2956
+ final = RunRecord(
2957
+ **{
2958
+ **record.__dict__,
2959
+ "state": ENDED,
2960
+ "ending": _ENDINGS_BY_OUTCOME[result.outcome],
2961
+ "ending_note": redact(result.note, secrets),
2962
+ }
2963
+ )
2964
+ if not _best_effort("final record", lambda: save_record(run_root, final, now), secrets):
2965
+ # The on-disk record still says `implementing`, so automated
2966
+ # follow-up servicing will not track this run — and if a PR was
2967
+ # opened, its humans are the only ones who can act. Say so WHERE
2968
+ # they are looking: GitHub is the one store still writable when the
2969
+ # local disk is gone.
2970
+ pr_number = pr_url.rstrip("/").rsplit("/", 1)[-1] if pr_url else ""
2971
+ if pr_number.isdigit():
2972
+ _best_effort(
2973
+ "pr state warning",
2974
+ lambda: github.comment(
2975
+ config.target,
2976
+ int(pr_number),
2977
+ f"State record for run `{run_id}` could not be saved; "
2978
+ f"automated follow-up servicing is offline for this run. "
2979
+ f"A maintainer owns any follow-ups on this PR.",
2980
+ ),
2981
+ secrets,
2982
+ )
2983
+ if issue_number:
2984
+ _post_issue_finished(
2985
+ github,
2986
+ config.target,
2987
+ issue_number,
2988
+ run_id,
2989
+ outcome_name,
2990
+ pr_url,
2991
+ redact(result.report(config, redact_secrets=secrets), secrets)[:8000],
2992
+ secrets,
2993
+ )
2994
+ if outcome_name != result.outcome:
2995
+ # the publish failed after the gate credited the tree: the improved
2996
+ # snapshot above stands (the measurement was real); append the
2997
+ # publish-error marker so the notebook records how the run ended
2998
+ _push_line_snapshot(ws, line_ref, run_id, outcome_name, secrets)
2999
+ log.info("run %s: %s %s", run_id, outcome_name, pr_url)
3000
+ return AttemptOutcome(
3001
+ run_id=run_id,
3002
+ outcome=outcome_name,
3003
+ pr_url=pr_url,
3004
+ report_path=str(report_path) if wrote_report else "",
3005
+ )
3006
+
3007
+
3008
+ class Terminated(Exception):
3009
+ """Slurm sent SIGTERM (walltime, preemption, scancel): raised into the
3010
+ main thread so the ordinary exception containment ends the run inside
3011
+ the KillWait grace window before SIGKILL arrives."""
3012
+
3013
+
3014
+ # Below this, arming is pointless: the alarm would fire during setup,
3015
+ # outside containment, and a job this short cannot finish a climb anyway.
3016
+ MIN_ARM_S = 180
3017
+
3018
+
3019
+ def arm_self_deadline(job_minutes: int, margin_s: float = 120.0) -> int:
3020
+ """Arm our own end-of-walltime alarm; returns the armed seconds (0 = off).
3021
+
3022
+ Slurm delivers NO signal to our process on Torch before SIGKILL
3023
+ (scancel and walltime timeout both signal the
3024
+ batch shell only) — so the only way to end a run richly before the
3025
+ wall is our own clock. SIGALRM fires `margin_s` before the job's
3026
+ walltime and raises Terminated into the ordinary containment; the
3027
+ margin floor covers the containment's own tail (GitHub calls are 30s
3028
+ timeout x retries). The walltime clock starts at JOB start, not
3029
+ process start — SLURM_JOB_START_TIME anchors the deadline when
3030
+ present so startup latency erodes the runway, never the margin.
3031
+ """
3032
+ if job_minutes <= 0:
3033
+ return 0
3034
+ import signal
3035
+ import time as _time
3036
+
3037
+ margin = max(60.0, margin_s)
3038
+ now = _time.time()
3039
+ start_raw = os.environ.get("SLURM_JOB_START_TIME", "")
3040
+ # Sanity-bounded: the env can carry a STALE value inherited from the
3041
+ # submitting job (tick jobs sbatch climb jobs). A start time outside
3042
+ # [now - walltime, now] is not this job's — fall back to the process
3043
+ # clock rather than silently disarm (past) or overshoot the wall
3044
+ # (future).
3045
+ if start_raw.isdigit() and now - job_minutes * 60 <= int(start_raw) <= now:
3046
+ remaining = int(int(start_raw) + job_minutes * 60 - margin - now)
3047
+ else:
3048
+ remaining = int(job_minutes * 60 - margin)
3049
+ if remaining < MIN_ARM_S:
3050
+ log.warning(
3051
+ "self-deadline NOT armed: %ds runway is below the %ds floor", remaining, MIN_ARM_S
3052
+ )
3053
+ return 0
3054
+
3055
+ def _on_alarm(signum: int, frame: object) -> None:
3056
+ raise Terminated(
3057
+ f"self-deadline: {margin:.0f}s before the job's {job_minutes}-minute walltime"
3058
+ )
3059
+
3060
+ signal.signal(signal.SIGALRM, _on_alarm)
3061
+ signal.alarm(remaining)
3062
+ return remaining
3063
+
3064
+
3065
+ def arm_sigterm_containment() -> None:
3066
+ """Convert the FIRST SIGTERM into a Terminated exception, one-shot.
3067
+
3068
+ Repeats are absorbed by a flag rather than SIG_IGN: a second SIGTERM
3069
+ (repeated scancel, site KillWait re-sends) must not abort the very
3070
+ containment the first one enabled — and SIG_IGN would be inherited
3071
+ across exec by children spawned during containment, leaving them
3072
+ unkillable by TERM. A Python-level handler is reset on exec, so
3073
+ children keep default signal behavior.
3074
+ """
3075
+ import signal
3076
+
3077
+ fired = {"done": False}
3078
+
3079
+ def _on_sigterm(signum: int, frame: object) -> None:
3080
+ if fired["done"]:
3081
+ return # containment already unwinding; absorb the repeat
3082
+ fired["done"] = True
3083
+ raise Terminated("SIGTERM from Slurm (walltime, preemption, or scancel)")
3084
+
3085
+ signal.signal(signal.SIGTERM, _on_sigterm)
3086
+
3087
+
3088
+ def main() -> int:
3089
+ import argparse
3090
+ import os
3091
+ import time
3092
+ from datetime import UTC, datetime
3093
+
3094
+ arm_sigterm_containment()
3095
+
3096
+ parser = argparse.ArgumentParser(description="One live attempt on one benchmark.")
3097
+ # --target/--benchmark drive a fresh climb; they are read from the record
3098
+ # on a --resume wake instead, so they are optional (validated below).
3099
+ parser.add_argument("--target", default="")
3100
+ parser.add_argument("--benchmark", default="")
3101
+
3102
+ # WIDTH: the tick assigns each concurrent slot its own agent identity;
3103
+ # branches, ledger rows, and reports key on it. Resumed runs inherit
3104
+ # the identity from their record instead.
3105
+ def _agent_id(value: str) -> str:
3106
+ # the id shapes refs (feat/auto/<id>/<run>, agents/<id>): slug only —
3107
+ # same rule the line checkout enforces
3108
+ if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_-]{0,63}", value):
3109
+ raise argparse.ArgumentTypeError(
3110
+ f"agent id {value!r} cannot shape a git ref (want [A-Za-z0-9][A-Za-z0-9_-]*)"
3111
+ )
3112
+ return value
3113
+
3114
+ parser.add_argument("--agent-id", default="agent-01", type=_agent_id)
3115
+ parser.add_argument("--run-root", required=True, type=Path)
3116
+ parser.add_argument(
3117
+ "--resume",
3118
+ default="",
3119
+ metavar="RUN_ID",
3120
+ help="wake a parked dispatched run instead of starting a fresh climb",
3121
+ )
3122
+ parser.add_argument("--base-branch", default="main")
3123
+ # All three default from the chain env the tick sets on the climb job, so
3124
+ # a contained run with AUTORESEARCH_{IMAGE,ACCOUNT,PARTITION} set selects
3125
+ # dispatched measurement without extra flags. The image also containers the
3126
+ # session + inline eval; absent any of the three, measurement stays inline
3127
+ # regardless of the benchmark's eval hint.
3128
+ parser.add_argument(
3129
+ "--image",
3130
+ default=os.environ.get("AUTORESEARCH_IMAGE", ""),
3131
+ help="apptainer image for session+eval",
3132
+ )
3133
+ parser.add_argument("--account", default=os.environ.get("AUTORESEARCH_ACCOUNT", ""))
3134
+ parser.add_argument("--partition", default=os.environ.get("AUTORESEARCH_PARTITION", ""))
3135
+ # the GPU lane for benchmarks with `gpus > 0` (evals + author launches);
3136
+ # empty = this deployment cannot place GPU jobs
3137
+ parser.add_argument("--gpu-partition", default=os.environ.get("AUTORESEARCH_GPU_PARTITION", ""))
3138
+ parser.add_argument("--gpu-account", default=os.environ.get("AUTORESEARCH_GPU_ACCOUNT", ""))
3139
+ parser.add_argument(
3140
+ "--uncontained",
3141
+ action="store_true",
3142
+ help="run WITHOUT a container (dev only: sessions can then read "
3143
+ "same-user files, including credential files)",
3144
+ )
3145
+ parser.add_argument("--claude-bin", default=os.path.expanduser("~/.local/bin/claude"))
3146
+ parser.add_argument(
3147
+ "--codex-bin",
3148
+ default=os.path.expanduser(
3149
+ os.environ.get("AUTORESEARCH_CODEX_BIN") or "~/.local/bin/codex"
3150
+ ),
3151
+ help="host codex binary for the codex author; bind-mounted into apptainer "
3152
+ "(must be an absolute path).",
3153
+ )
3154
+ parser.add_argument(
3155
+ "--model", default=os.environ.get("AUTORESEARCH_AUTHOR_MODEL") or "claude-opus-5"
3156
+ )
3157
+ parser.add_argument(
3158
+ "--author-backend",
3159
+ choices=("claude", "codex"),
3160
+ default=os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude",
3161
+ help="agent backend for the author/editor role (config-driven: default "
3162
+ "from AUTORESEARCH_AUTHOR_BACKEND). codex runs contained (apptainer + "
3163
+ "--sandbox danger-full-access) and REQUIRES --image and a codex/openai "
3164
+ "--model (e.g. gpt-5.6-terra).",
3165
+ )
3166
+ parser.add_argument(
3167
+ "--codex-config",
3168
+ action="append",
3169
+ default=[],
3170
+ metavar="KEY=VALUE",
3171
+ help="codex `-c KEY=VALUE` config for the codex author (repeatable), "
3172
+ "e.g. --codex-config use_legacy_landlock=true for a host that needs it.",
3173
+ )
3174
+ parser.add_argument("--max-turns", type=int, default=60)
3175
+ parser.add_argument("--session-minutes", type=int, default=60)
3176
+ parser.add_argument(
3177
+ "--panel",
3178
+ default="",
3179
+ help=(
3180
+ "pre-PR verification lenses, comma-separated kind[:backend[:model]] "
3181
+ "entries (e.g. 'verify,review' or 'verify:claude:MODEL'); only the "
3182
+ "claude backend is contained on this host so far; empty disables "
3183
+ "the panel"
3184
+ ),
3185
+ )
3186
+ parser.add_argument(
3187
+ "--panel-key-file",
3188
+ default=PANEL_KEY_DEFAULT,
3189
+ help="key file for panel judge sessions (the verifier's own key, never the author's)",
3190
+ )
3191
+ parser.add_argument(
3192
+ "--job-minutes",
3193
+ type=int,
3194
+ default=0,
3195
+ help="this job's Slurm walltime; arms the self-deadline (0 = off)",
3196
+ )
3197
+ parser.add_argument(
3198
+ "--deadline-margin-s",
3199
+ type=float,
3200
+ default=120.0,
3201
+ help="how long before the walltime the self-deadline fires (floor 60)",
3202
+ )
3203
+ parser.add_argument("--pat-file", default=str(CONFIG_DIR / "bot_pat"))
3204
+ parser.add_argument(
3205
+ "--github-app-file",
3206
+ default=os.environ.get("AUTORESEARCH_GITHUB_APP_FILE", ""),
3207
+ help="GitHub App config (JSON: app_id, installation_id, private_key); "
3208
+ "when set, installation tokens replace the PAT",
3209
+ )
3210
+ parser.add_argument(
3211
+ "--key-file",
3212
+ default="",
3213
+ help="author key file; default resolves per backend (config-driven): "
3214
+ "AUTORESEARCH_HARNESS_KEY_FILE for claude, AUTORESEARCH_CODEX_KEY_FILE for codex",
3215
+ )
3216
+ parser.add_argument("--issue", type=int, default=0)
3217
+ parser.add_argument(
3218
+ "--min-free-gb",
3219
+ type=float,
3220
+ default=10.0,
3221
+ help="refuse to start when the run root has less free space",
3222
+ )
3223
+ parser.add_argument(
3224
+ "--hypothesis-b64", default="", help="base64 task hypothesis (issue text, fenced)"
3225
+ )
3226
+ args = parser.parse_args()
3227
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
3228
+ if not args.image and not args.uncontained:
3229
+ parser.error("--image is required (or pass --uncontained explicitly, dev only)")
3230
+ # NOTE: the codex author is validated on the EFFECTIVE author per path — the
3231
+ # fresh climb on args (below), a wake on the parked run's persisted pair — not
3232
+ # here, where args.author_backend is the FLEET default and would misjudge a
3233
+ # resume after a fleet flip.
3234
+ # each --codex-config KEY=VALUE becomes a `-c KEY=VALUE` pair for codex
3235
+ codex_extra = tuple(a for c in args.codex_config for a in ("-c", c))
3236
+
3237
+ bot_auth = resolve_bot_auth(args.pat_file, args.github_app_file)
3238
+
3239
+ # --resume WAKES a parked dispatched run: rebuild the dispatched measurer
3240
+ # and re-enter the decision. The wake job the WakeDispatcher submits runs
3241
+ # exactly this.
3242
+ if args.resume:
3243
+ placed = bool(args.account and args.partition) or local_mode()
3244
+ if not (placed and args.image and Path(args.image).is_file()):
3245
+ parser.error(
3246
+ "--resume needs the cluster triple (--account/--partition/--image) "
3247
+ "to rebuild the dispatched measurer (local compute waives the "
3248
+ "account/partition pair, never the image)"
3249
+ )
3250
+ from outerloop.runstate import load_record
3251
+
3252
+ # a wake that is not the lease holder is a straggler (a replacement was
3253
+ # dispatched after it was cancelled, or it was armed and then lost):
3254
+ # it must not touch the run beside the holder
3255
+ other = _lease_held_by_another_job(args.run_root, args.resume)
3256
+ if other:
3257
+ print(f"run {args.resume}: wake job {other} holds the lease; this one exits")
3258
+ return 0
3259
+
3260
+ # Reproduce the PARKED run's author, not the current fleet default: the
3261
+ # (backend, model) PAIR is persisted on the record — a fleet flip must not
3262
+ # wake a codex run as claude (or with the new fleet's model). A legacy or
3263
+ # unreadable record is treated as claude (resume_author). The key then
3264
+ # resolves for THAT backend (keys coexist).
3265
+ try:
3266
+ _wake_record: object | None = load_record(args.run_root, args.resume)
3267
+ except Exception:
3268
+ # a wake must never crash on an unreadable/odd record — fall back to
3269
+ # the claude author (resume_author), same fail-safe as the sweep
3270
+ _wake_record = None
3271
+ wake_backend, wake_model, wake_key_file = resume_author(_wake_record, args.model)
3272
+ # an explicit --key-file still overrides (a manual re-run pinning a key)
3273
+ if args.key_file:
3274
+ wake_key_file = os.path.expanduser(args.key_file)
3275
+ _err = codex_author_config_error(wake_backend, wake_model, args.image)
3276
+ if _err:
3277
+ # this wake job HOLDS the run's lease (transferred on dispatch); release
3278
+ # it before exiting so a misconfig doesn't strand the run until the TTL
3279
+ # reap (the resume_run finally below only runs once we reach it)
3280
+ _release_own_lease(args.run_root, args.resume)
3281
+ parser.error(f"parked run {args.resume}: {_err}")
3282
+ # the wake runs the SAME verification panel as a fresh climb, so a
3283
+ # dispatched improvement is not published unverified.
3284
+ try:
3285
+ wake_lenses, wake_panel_secrets = _panel_lenses_from_args(args)
3286
+ except ValueError as exc:
3287
+ parser.error(str(exc))
3288
+ wake_api_key = ""
3289
+ wake_harness = None
3290
+ wake_spec = None
3291
+ # The editor harness is built when the wake may RESUME the session: a
3292
+ # panel is configured (a blocking finding wakes the author to revise),
3293
+ # or the park is an AUTHOR-SLEEP (that wake always resumes the session
3294
+ # with its launches' results). A panel-less candidate wake stays a pure
3295
+ # read-decide-publish job and must not require the author key.
3296
+ _wake_stage = getattr(_wake_record, "stage", None) or {}
3297
+ if (
3298
+ wake_lenses
3299
+ or _wake_stage.get("phase") == "author-sleep"
3300
+ or _wake_stage.get("submitted")
3301
+ ):
3302
+ wake_api_key = role_key(wake_key_file, wake_backend)
3303
+ wake_spec = author_spec(max_turns=args.max_turns, walltime_s=args.session_minutes * 60)
3304
+ wake_harness = build_harness(
3305
+ wake_api_key,
3306
+ wake_spec,
3307
+ backend=wake_backend,
3308
+ binary=args.claude_bin if wake_backend == "claude" else args.codex_bin,
3309
+ model=wake_model,
3310
+ container_image=args.image,
3311
+ codex_extra_args=codex_extra,
3312
+ )
3313
+ wake_secrets = tuple(k for k in (bot_auth.token(), *wake_panel_secrets, wake_api_key) if k)
3314
+ try:
3315
+ resumed = resume_run(
3316
+ args.run_root,
3317
+ args.resume,
3318
+ dispatch=_dispatch_settings(args),
3319
+ github=GitHubClient(auth=bot_auth),
3320
+ bot_auth=bot_auth,
3321
+ now=time.time(),
3322
+ secrets=wake_secrets,
3323
+ base_branch=args.base_branch,
3324
+ panel_lenses=wake_lenses,
3325
+ harness=wake_harness,
3326
+ spec=wake_spec,
3327
+ )
3328
+ except GitError as exc:
3329
+ # A tamper-class GitError that escaped the wake body (a TOCTOU
3330
+ # object removal past the entry guard, or a git op that reached a
3331
+ # damaged object) ends the run cleanly as a refused wake — "a
3332
+ # refused wake ends the run" — instead of crashing the job into a
3333
+ # re-park that only wakes into the same failure. A non-tamper
3334
+ # GitError (push conflict, fetch outage) propagates unchanged.
3335
+ if not _is_git_tamper(exc):
3336
+ raise
3337
+ resumed = _end_refused_wake(
3338
+ args.run_root,
3339
+ load_record(args.run_root, args.resume),
3340
+ exc,
3341
+ time.time(),
3342
+ wake_secrets,
3343
+ )
3344
+ finally:
3345
+ # This wake job HOLDS the run's lease (the sweep transferred it on
3346
+ # dispatch); release it on every exit so a re-parked run is
3347
+ # immediately eligible for the next sweep instead of waiting out the
3348
+ # TTL reap. Idempotent (no-op if no lease file).
3349
+ _release_own_lease(args.run_root, args.resume)
3350
+ print(f"outcome={resumed.outcome} pr={resumed.pr_url or '-'} report={resumed.report_path}")
3351
+ return 0
3352
+
3353
+ if not (args.target and args.benchmark):
3354
+ parser.error("--target and --benchmark are required for a fresh climb")
3355
+
3356
+ # a fresh climb authors on the FLEET's configured backend; validate it (codex
3357
+ # writes+executes, so --image + a non-claude model) before any spend.
3358
+ _err = codex_author_config_error(args.author_backend, args.model, args.image)
3359
+ if _err:
3360
+ parser.error(_err)
3361
+ # config-driven: the author key defaults per backend (claude vs codex) so the
3362
+ # tick never threads it — see resolve_author_key_file (result is ~-expanded).
3363
+ args.key_file = resolve_author_key_file(args.author_backend, args.key_file)
3364
+ # same 0600 discipline as the PAT: this key spends real money. A missing
3365
+ # file is tolerated only when Vertex (ADC) covers the claude backend.
3366
+ api_key = role_key(args.key_file, args.author_backend)
3367
+ stamp = datetime.now(UTC).strftime("%Y%m%d-%H%M%S")
3368
+ # the agent id keeps concurrent same-benchmark slots (the width dial's
3369
+ # portfolio case) from minting one run directory in the same second
3370
+ run_id = f"{args.benchmark}-{stamp}-{args.agent_id}"
3371
+
3372
+ # Disk preflight BEFORE any run state exists: a session started on a
3373
+ # full filesystem dies mid-flight in ways that lose its own evidence
3374
+ # (quota errors are invisible until a write fails on some clusters).
3375
+ from outerloop.disk import check_mount
3376
+
3377
+ health = check_mount(args.run_root, min_free_bytes=int(args.min_free_gb * 1024**3))
3378
+ if not health.ok():
3379
+ log.error("disk preflight failed: %s — refusing to start a run", health.describe())
3380
+ if args.issue:
3381
+ _best_effort(
3382
+ "issue report",
3383
+ lambda: GitHubClient(auth=bot_auth).comment(
3384
+ args.target,
3385
+ args.issue,
3386
+ "A run for this issue could not start: the orchestrator's "
3387
+ "storage failed its disk preflight. The claim on this issue "
3388
+ "stays until a maintainer removes the claim comment "
3389
+ "(automated claim release is on the roadmap).",
3390
+ ),
3391
+ )
3392
+ return 3
3393
+
3394
+ # Armed LAST, immediately before the contained region — and DISARMED
3395
+ # right after it: a run finishing inside the margin must not have the
3396
+ # alarm fire during the uncontained epilogue (print/exit).
3397
+ import signal as _signal
3398
+
3399
+ armed = arm_self_deadline(args.job_minutes, args.deadline_margin_s)
3400
+ if armed:
3401
+ log.info("self-deadline armed: Terminated in %ds", armed)
3402
+
3403
+ # the manifest first, the harness from it: budget has one source (the args)
3404
+ spec = author_spec(max_turns=args.max_turns, walltime_s=args.session_minutes * 60)
3405
+
3406
+ # Pre-PR panel lenses: judge sessions on the verifier's own key (separate
3407
+ # identity from the author). kind[:backend[:model]]; claude by default.
3408
+ try:
3409
+ panel_lenses, panel_secrets = _panel_lenses_from_args(args)
3410
+ except ValueError as exc:
3411
+ parser.error(str(exc))
3412
+
3413
+ # Dispatched measurement needs the full cluster triple AND a real image
3414
+ # file to bind against; missing any, the climb measures inline (the tick
3415
+ # sets these on the climb job's env, a bare CLI run leaves them empty).
3416
+ dispatch: DispatchSettings | None = None
3417
+ if (bool(args.account and args.partition) or local_mode()) and (
3418
+ args.image and Path(args.image).is_file()
3419
+ ):
3420
+ dispatch = _dispatch_settings(args)
3421
+ try:
3422
+ try:
3423
+ outcome = live_attempt(
3424
+ config=RunConfig(
3425
+ target=args.target, benchmark=args.benchmark, agent_id=args.agent_id
3426
+ ),
3427
+ base_branch=args.base_branch,
3428
+ run_root=args.run_root,
3429
+ run_id=run_id,
3430
+ harness=build_harness(
3431
+ api_key,
3432
+ spec,
3433
+ backend=args.author_backend,
3434
+ binary=args.claude_bin if args.author_backend == "claude" else args.codex_bin,
3435
+ model=args.model,
3436
+ container_image=args.image,
3437
+ codex_extra_args=codex_extra,
3438
+ ),
3439
+ spec=spec,
3440
+ panel_lenses=panel_lenses,
3441
+ dispatch=dispatch,
3442
+ eval_image=args.image,
3443
+ github=GitHubClient(auth=bot_auth),
3444
+ bot_auth=bot_auth,
3445
+ now=time.time(),
3446
+ created=datetime.now(UTC).isoformat(),
3447
+ # the panel key joins the redaction set: judge error text can
3448
+ # echo request material like any other model error
3449
+ secrets=tuple(
3450
+ k
3451
+ for k in (
3452
+ api_key,
3453
+ bot_auth.token(),
3454
+ *panel_secrets,
3455
+ )
3456
+ if k
3457
+ ),
3458
+ issue_number=args.issue,
3459
+ author_backend=args.author_backend,
3460
+ author_model=args.model,
3461
+ author_key_file=args.key_file,
3462
+ task_hypothesis=(
3463
+ __import__("base64").b64decode(args.hypothesis_b64).decode()
3464
+ if args.hypothesis_b64
3465
+ else ""
3466
+ ),
3467
+ )
3468
+ except Terminated as exc:
3469
+ # Fired in live_attempt's microseconds-wide pre-containment window:
3470
+ # any record it saved strands and the sweep ends it from Slurm
3471
+ # truth; here we only avoid dying as an unexplained traceback.
3472
+ log.error("self-deadline fired before containment: %s", exc)
3473
+ return 3
3474
+ finally:
3475
+ _signal.alarm(0)
3476
+ print(f"outcome={outcome.outcome} pr={outcome.pr_url or '-'} report={outcome.report_path}")
3477
+ return 0
3478
+
3479
+
3480
+ if __name__ == "__main__":
3481
+ raise SystemExit(main())