outerloop-science 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. outerloop/__init__.py +18 -0
  2. outerloop/__main__.py +3 -0
  3. outerloop/appauth.py +213 -0
  4. outerloop/appmanifest.py +198 -0
  5. outerloop/attempt.py +3481 -0
  6. outerloop/brief.py +515 -0
  7. outerloop/cli.py +439 -0
  8. outerloop/climbboard.py +1145 -0
  9. outerloop/compute.py +482 -0
  10. outerloop/contract.py +483 -0
  11. outerloop/contract_cli.py +63 -0
  12. outerloop/disk.py +164 -0
  13. outerloop/dispatch.py +586 -0
  14. outerloop/followup.py +2143 -0
  15. outerloop/github.py +1486 -0
  16. outerloop/harness.py +1449 -0
  17. outerloop/housekeeping.py +167 -0
  18. outerloop/init.py +313 -0
  19. outerloop/intake.py +129 -0
  20. outerloop/limits.py +80 -0
  21. outerloop/markers.py +48 -0
  22. outerloop/measure.py +523 -0
  23. outerloop/orchestrator.py +1901 -0
  24. outerloop/panel.py +188 -0
  25. outerloop/paths.py +27 -0
  26. outerloop/posting.py +160 -0
  27. outerloop/progress.py +170 -0
  28. outerloop/py.typed +0 -0
  29. outerloop/review.py +611 -0
  30. outerloop/review_agent.py +263 -0
  31. outerloop/review_agent_cli.py +209 -0
  32. outerloop/review_post_cli.py +162 -0
  33. outerloop/review_summarize_cli.py +163 -0
  34. outerloop/role_runner.py +229 -0
  35. outerloop/roles.py +247 -0
  36. outerloop/rolespec.py +89 -0
  37. outerloop/runstate.py +385 -0
  38. outerloop/steward.py +852 -0
  39. outerloop/style.py +12 -0
  40. outerloop/syscall.py +977 -0
  41. outerloop/syscall_cli.py +531 -0
  42. outerloop/tick.py +3166 -0
  43. outerloop/verifier.py +403 -0
  44. outerloop/verify_agent.py +149 -0
  45. outerloop/verify_agent_cli.py +95 -0
  46. outerloop/verify_post_cli.py +116 -0
  47. outerloop_science-0.1.0.dev0.dist-info/METADATA +145 -0
  48. outerloop_science-0.1.0.dev0.dist-info/RECORD +52 -0
  49. outerloop_science-0.1.0.dev0.dist-info/WHEEL +4 -0
  50. outerloop_science-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  51. outerloop_science-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
  52. outerloop_science-0.1.0.dev0.dist-info/licenses/NOTICE +5 -0
outerloop/tick.py ADDED
@@ -0,0 +1,3166 @@
1
+ """One tick of the loop: sentinel, heartbeat, and the fail-safe sweep.
2
+
3
+ The tick is stateless and bounded — everything durable lives in run-state
4
+ files (`runstate`) and Slurm. It implements the backup layers of the wake
5
+ design (docs/design/architecture.md, "Wake delivery and fail-safety"); the
6
+ primary layer (the afterany dependency job) is submitted by whoever launches
7
+ an experiment and needs no help from here.
8
+
9
+ Wake *delivery* is behind a seam (`WakeDispatcher`) so this module stays
10
+ testable and the actual session dispatch (harness + brief) can evolve
11
+ independently.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import contextlib
17
+ import json
18
+ import logging
19
+ import math
20
+ import os
21
+ import re
22
+ import socket
23
+ import subprocess
24
+ from collections.abc import Sequence
25
+ from dataclasses import asdict, dataclass, field, replace
26
+ from pathlib import Path
27
+ from typing import Any, Protocol
28
+ from uuid import uuid4
29
+
30
+ from outerloop.compute import (
31
+ GONE,
32
+ Compute,
33
+ JobSpec,
34
+ LocalCompute,
35
+ SlurmError,
36
+ SlurmQueryError,
37
+ compute_from_env,
38
+ is_pending,
39
+ is_terminal,
40
+ local_mode,
41
+ quote_command,
42
+ )
43
+ from outerloop.disk import DEFAULT_MIN_FREE_BYTES, check_disk
44
+ from outerloop.harness import DEFAULT_MAX_TURNS, redact
45
+ from outerloop.housekeeping import shed_ended_workspaces
46
+ from outerloop.limits import EffectiveLimits, effective_limits
47
+ from outerloop.markers import has_marker, marker
48
+ from outerloop.runstate import (
49
+ ABORTED,
50
+ ENDED,
51
+ IMPLEMENTING,
52
+ IN_REVIEW,
53
+ MAX_WAKE_ATTEMPTS,
54
+ STUCK,
55
+ WAITING,
56
+ Lease,
57
+ RunRecord,
58
+ acquire_lease,
59
+ lease_is_stale,
60
+ list_runs,
61
+ load_record,
62
+ outage_active,
63
+ read_lease,
64
+ reap_lease,
65
+ release_lease,
66
+ run_dir,
67
+ save_record,
68
+ update_lease_holder,
69
+ )
70
+
71
+ log = logging.getLogger(__name__)
72
+
73
+ PAUSE_SENTINEL = "PAUSE"
74
+ # Operator on-switch for dispatched-wake, mirroring PAUSE: a root-relative
75
+ # sentinel an operator arms/disarms with a touch/rm — no chain restart, no
76
+ # env-var surgery on a live tick. The AUTORESEARCH_DISPATCH_WAKE env var still
77
+ # works too (either arms it); the sentinel is the reversible, restart-free path.
78
+ DISPATCH_WAKE_SENTINEL = "DISPATCH_WAKE"
79
+ HEARTBEAT_NAME = "heartbeat.json"
80
+ # Written at a full tick's END (not its start) — the coalesce guard's signal, so
81
+ # a tick that crashes mid-work cannot suppress the next (recovery) tick.
82
+ WORK_MARKER_NAME = "last_worked.json"
83
+
84
+ # Grace between "experiment terminal" and the sweep stepping in: the afterany
85
+ # job gets this long to deliver before the backup assumes it lost.
86
+ DEFAULT_GRACE_S = 15 * 60
87
+ # a blind park (no job ids to poll) waits its eval walltime plus this queue
88
+ # slack before a follow-up is sent to look for the result
89
+ BLIND_PARK_SLACK_MIN = 12 * 60
90
+ # A held lease is stale after the session timeout plus slack.
91
+ DEFAULT_LEASE_TTL_S = 3600 + 15 * 60
92
+ # Coalesce guard: skip a tick's work if another ran within this window. Under
93
+ # partition congestion, queued ticks bunch up and become eligible together
94
+ # (serialized by the singleton dependency), so they would run back-to-back and
95
+ # redundantly re-sweep. The chain schedules ticks a full cadence apart by
96
+ # begin-time, so only late-bunched pile-ups fall inside this window; keep it
97
+ # well BELOW the cadence (default 30 min). 0 disables. Env: AUTORESEARCH_MIN_TICK_MINUTES.
98
+ DEFAULT_MIN_TICK_S = 10 * 60
99
+ # Ceiling for the coalesce window: a value above this is almost certainly a typo
100
+ # (a window near/over the cadence would coalesce every on-cadence tick and stall
101
+ # the loop). Clamp + warn rather than silently freeze. The operator is still
102
+ # responsible for keeping it below their configured cadence.
103
+ MAX_MIN_TICK_S = 60 * 60
104
+
105
+
106
+ class WakeDispatcher(Protocol):
107
+ """Delivers one wake, called with the lease already held.
108
+
109
+ Returns "" when delivery completed synchronously (the caller releases the
110
+ lease), or the Slurm job id of an asynchronous wake job that now owns the
111
+ lease (released by that job on completion; reaped by TTL if it dies).
112
+
113
+ Contract for real dispatchers: a wake that RESULTS IN PROGRESS
114
+ must either move the run out of `waiting` or reset `wake_attempts` —
115
+ the counter means "wakes since the run last made progress", and layer 5
116
+ ends the run as stuck when it reaches MAX_WAKE_ATTEMPTS."""
117
+
118
+ def dispatch(self, record: RunRecord, reason: str) -> str: ...
119
+
120
+
121
+ @dataclass
122
+ class RecordingDispatcher:
123
+ """Test/dry-run dispatcher: records what would have been woken."""
124
+
125
+ dispatched: list[tuple[str, str]] = field(default_factory=list)
126
+ holder_job_id: str = "" # set to simulate async dispatch
127
+
128
+ def dispatch(self, record: RunRecord, reason: str) -> str:
129
+ self.dispatched.append((record.run_id, reason))
130
+ return self.holder_job_id
131
+
132
+
133
+ @dataclass(frozen=True)
134
+ class TickReport:
135
+ paused: bool = False
136
+ coalesced: bool = False # skipped as a redundant pile-up (a tick ran too recently)
137
+ swept: int = 0
138
+ woken: tuple[tuple[str, str], ...] = () # (run_id, reason)
139
+ deferred: tuple[str, ...] = () # runs skipped on "Slurm unknown"
140
+ reaped_leases: tuple[str, ...] = ()
141
+ stuck: tuple[str, ...] = ()
142
+ implementing_ended: tuple[str, ...] = () # killed climbs the sweep closed out
143
+ review_ended: tuple[tuple[str, str], ...] = () # (run_id, ending)
144
+ followups_submitted: tuple[tuple[str, str], ...] = () # (run_id, job_id)
145
+ intake: tuple[str, str] = ("", "") # (issue tag, job_id) when one was claimed
146
+ self_initiated: tuple[str, str] = ("", "") # (benchmark, job_id) when one launched
147
+ steward: tuple[str, str] = ("", "") # (issue tag, job_id) when a stewardship launched
148
+ disk: tuple[str, ...] = () # preflight warnings (home entries are warn-only)
149
+ launch_blocked: bool = False # True when the preflight turned launch lanes off
150
+ shed: tuple[str, ...] = () # ended runs whose workspaces housekeeping removed
151
+
152
+
153
+ # The submitted walltime must never exceed the job partition's MaxTime —
154
+ # sbatch REJECTS a longer request outright, which would ground every climb.
155
+ # The DEFAULT matches cpu_short (6 h); an operator moving work jobs to a
156
+ # longer partition (AUTORESEARCH_JOB_PARTITION=cpu48) raises the cap with
157
+ # AUTORESEARCH_MAX_JOB_MINUTES. Code-side ceiling: the cap must stay under
158
+ # STRANDED_IMPLEMENTING_S or the picker declares live runs stranded — jobs
159
+ # longer than 10 h need that window made spec-aware first (named gap). The
160
+ # self-deadline arms at the CLAMPED value, so a job that wanted more time
161
+ # fails safe mid-panel instead of never starting.
162
+ MAX_ATTEMPT_JOB_MINUTES = 6 * 60
163
+ MAX_JOB_MINUTES_CEILING = 10 * 60
164
+
165
+
166
+ def _bot_login_default() -> str:
167
+ """FollowupSpec's login default, resolved at construction (the tick reads
168
+ the chain's env, jobs inherit it); github is imported here on purpose —
169
+ the tick module stays importable without it."""
170
+ from outerloop.github import bot_login_from_env
171
+
172
+ return bot_login_from_env()
173
+
174
+
175
+ @dataclass(frozen=True)
176
+ class FollowupSpec:
177
+ """How the tick launches follow-up jobs for in-review runs."""
178
+
179
+ account: str
180
+ partition: str
181
+ run_root: Path
182
+ image: str
183
+ home: Path # AUTORESEARCH_HOME: cwd for the submitted job
184
+ bot_login: str = field(default_factory=_bot_login_default)
185
+ time_minutes: int = 90 # min()'d with the contract's followup_job_minutes
186
+ max_turns: int = DEFAULT_MAX_TURNS # session turn budget for follow-up jobs
187
+ pat_file: str = "" # forwarded to the job; "" = the followup CLI default
188
+ # GitHub App config path; jobs inherit AUTORESEARCH_GITHUB_APP_FILE from
189
+ # the tick environment, so it is never threaded through argv
190
+ github_app_file: str = ""
191
+ target: str = "" # the repo the intake pass scans for requested-lane issues
192
+ # the STEWARD'S OWN key (role separation): the steward lane stays off
193
+ # until the operator provisions it
194
+ steward_key_file: str = ""
195
+ # Pre-PR verification panel for climb jobs (docs/design/orchestrator-verify.md).
196
+ # DEFAULT ON — the flip is code, the off-switch is AUTORESEARCH_PANEL="".
197
+ # The climb CLI fails LOUDLY on a bad panel config (a configured gate must
198
+ # never silently vanish); the tick preflights the same rules — lens
199
+ # grammar AND key file — before claiming or submitting so nothing is
200
+ # stranded. The panel's walltime is the orchestrator's own overhead: the
201
+ # tick ADDS a panel allowance to the contract-clamped job budget
202
+ # (_panel_job_minutes) rather than eating the author's time; a residual
203
+ # overrun still fails safe through the self-deadline.
204
+ panel: str = "verify,review"
205
+ panel_key_file: str = "" # "" = the climb CLI's default verifier-key path
206
+ # The GPU lane for benchmarks whose contract sets `gpus > 0` (their evals
207
+ # and author launches); empty = this deployment cannot place GPU jobs,
208
+ # and the launch lanes refuse such benchmarks (a queue that can never
209
+ # run is worse than a loud refusal). gpu_account "" = same as `account`.
210
+ gpu_partition: str = ""
211
+ gpu_account: str = ""
212
+ # Where submitted WORK jobs (climb/steward/followup) run; empty = same as
213
+ # `partition`. The tick chain itself always stays on `partition` — ticks
214
+ # are minutes, work jobs can be hours, and Slurm prices walltime into
215
+ # scheduling priority, so the two deserve independent placement.
216
+ job_partition: str = ""
217
+ # Partition MaxTime for work jobs — the panel-augmented walltime clamps
218
+ # here (see MAX_ATTEMPT_JOB_MINUTES). Raise together with job_partition.
219
+ max_job_minutes: int = MAX_ATTEMPT_JOB_MINUTES
220
+
221
+
222
+ # Generous vs the ~2 h job walltimes plus queue wait, tight enough that
223
+ # full trees + per-flight venvs cannot pile up for days; same-day
224
+ # forensics is the norm, and the disk preflight is the backstop.
225
+ FLIGHT_TTL_S = 24 * 3600
226
+
227
+
228
+ def flight_checkout(home: Path, name: str, now: float) -> Path:
229
+ """A detached git worktree of the checkout's HEAD commit, for one
230
+ submitted job to run from. The shared checkout is reset --hard at
231
+ every tick's deploy, so a queued job that cd's into it can have its
232
+ code swapped mid-flight; a flight pins the deployed commit, and the
233
+ tree survives for forensics after a crash. HEAD, deliberately:
234
+ uncommitted hand-edits in the shared checkout do not fly — only
235
+ deployed code does. Failures fall back to the shared checkout — a
236
+ snapshot must never ground the fleet."""
237
+ flights = home.parent / "flights"
238
+ try:
239
+ flights.mkdir(parents=True, exist_ok=True)
240
+ # same name in the same tick (e.g. two orders on one benchmark, or
241
+ # truncation collisions) must get its own tree, not a silent
242
+ # fallback: suffix until free, with a unique tail as the backstop
243
+ target = flights / f"{name}-{int(now)}"
244
+ for attempt in range(2, 6):
245
+ if not target.exists():
246
+ break
247
+ target = flights / f"{name}-{int(now)}-{attempt}"
248
+ if target.exists():
249
+ target = flights / f"{name}-{int(now)}-{uuid4().hex[:8]}"
250
+ subprocess.run(
251
+ ["git", "-C", str(home), "worktree", "add", "--detach", str(target)],
252
+ check=True,
253
+ capture_output=True,
254
+ text=True,
255
+ timeout=60,
256
+ )
257
+ return target
258
+ except (OSError, subprocess.SubprocessError) as exc:
259
+ log.warning("flight snapshot failed for %s (%s); using the shared checkout", name, exc)
260
+ return home
261
+
262
+
263
+ def _benchmark_gpus(contract: Any, benchmark: str) -> int:
264
+ bench = next((b for b in getattr(contract, "benchmarks", []) if b.name == benchmark), None)
265
+ return int(getattr(bench, "gpus", 0) or 0)
266
+
267
+
268
+ def _gpu_lane_error(contract: Any, benchmark: str, spec: FollowupSpec) -> str:
269
+ """Why an attempt on `benchmark` cannot launch here, or "": a contract
270
+ with GPU benchmarks needs this deployment to name a GPU lane — otherwise
271
+ evals would queue into jobs that can never run (the climb would then
272
+ park forever on a phantom eval). ANY GPU benchmark in the contract
273
+ counts, not just the climbed one: the suite gate measures siblings.
274
+ Local compute has no lanes — jobs run on whatever GPUs the machine
275
+ has — so the check is waived there."""
276
+ if spec.gpu_partition or local_mode():
277
+ return ""
278
+ gpu_benches = [
279
+ b.name for b in getattr(contract, "benchmarks", []) if int(getattr(b, "gpus", 0) or 0)
280
+ ]
281
+ if gpu_benches:
282
+ return (
283
+ f"contract has GPU benchmarks ({', '.join(gpu_benches)}) but no GPU lane is "
284
+ "configured (set AUTORESEARCH_GPU_PARTITION)"
285
+ )
286
+ return ""
287
+
288
+
289
+ def _flight_command(home: Path, job_name: str, now: float, argv: list[str]) -> str:
290
+ """The job's shell command, cd'ing into a fresh flight snapshot.
291
+
292
+ The flight is named FROM the job name (one truncation rule, here) so
293
+ the reaper's pending-job immunity — live job name prefixes flight
294
+ name — holds by construction at every submit site. argv must contain
295
+ absolute paths only; every spec path is absolute by construction, and
296
+ a relative path would resolve inside a tree that is reaped later."""
297
+ flight = flight_checkout(home, job_name[:40], now)
298
+ return f"cd {quote_command([str(flight)])} && {quote_command(argv)}"
299
+
300
+
301
+ def reap_flights(
302
+ home: Path,
303
+ now: float,
304
+ ttl_s: float = FLIGHT_TTL_S,
305
+ live_job_names: Sequence[str] = (),
306
+ ) -> int:
307
+ """Remove flight worktrees older than the TTL — age by directory
308
+ mtime, no name parsing — UNLESS a pending or running job's NAME
309
+ prefixes the flight's (flights are named after their job). Queue wait
310
+ is unbounded (GPU partitions can pend for days), so age alone must
311
+ never delete a tree a job will cd into; the TTL is purely the
312
+ forensics-retention window for flights whose job is gone. Name
313
+ matching is conservative: one live job name protects every flight it
314
+ prefixes. Best-effort: a stubborn flight is logged, not fatal."""
315
+ flights = home.parent / "flights"
316
+ if not flights.is_dir():
317
+ return 0
318
+ reaped = 0
319
+ for entry in flights.iterdir():
320
+ if any(name and entry.name.startswith(name[:40]) for name in live_job_names):
321
+ continue # a queued or running job still needs this tree
322
+ try:
323
+ age = now - entry.stat().st_mtime
324
+ except OSError:
325
+ continue
326
+ if age < ttl_s:
327
+ continue
328
+ try:
329
+ subprocess.run(
330
+ ["git", "-C", str(home), "worktree", "remove", "--force", str(entry)],
331
+ check=True,
332
+ capture_output=True,
333
+ text=True,
334
+ timeout=60,
335
+ )
336
+ reaped += 1
337
+ except (OSError, subprocess.SubprocessError):
338
+ # not a registered worktree (a half-created flight, or debris):
339
+ # remove the directory itself and prune the registry, or the
340
+ # entry warns forever without ever going away
341
+ import shutil
342
+
343
+ shutil.rmtree(entry, ignore_errors=True)
344
+ with contextlib.suppress(OSError, subprocess.SubprocessError):
345
+ subprocess.run(
346
+ ["git", "-C", str(home), "worktree", "prune"],
347
+ check=True,
348
+ capture_output=True,
349
+ text=True,
350
+ timeout=60,
351
+ )
352
+ if not entry.exists():
353
+ reaped += 1
354
+ else:
355
+ log.warning("could not reap flight %s", entry.name)
356
+ return reaped
357
+
358
+
359
+ CONTRACT_ALARM_MARKER = marker("contract-alarm")
360
+ CONTRACT_ALARM_AFTER = 3 # consecutive failing ticks (~1.5 h) before alarming
361
+
362
+
363
+ def contract_alarm(
364
+ root: Path,
365
+ github: Any,
366
+ target: str,
367
+ error: str | None,
368
+ now: float,
369
+ bot_login: str = "agentic-learning-bot",
370
+ ) -> None:
371
+ """Persistent contract failure must surface where humans look.
372
+
373
+ A rejected or unfetchable contract silently idles every launch lane.
374
+ After CONTRACT_ALARM_AFTER consecutive failing ticks this
375
+ opens ONE issue on the target repo; the next successful load closes
376
+ it and says so. Alarm plumbing is best-effort by construction: it
377
+ must never break the tick it reports for."""
378
+ state_path = root / "contract-alarm.json"
379
+ try:
380
+ state = json.loads(state_path.read_text())
381
+ except (OSError, ValueError):
382
+ state = {}
383
+ if error is None:
384
+ open_alarm = int(state.get("issue", 0))
385
+ if not open_alarm and state:
386
+ # creation may have landed without a recorded number (dry-run,
387
+ # odd response, lost state file): search so recovery can still
388
+ # close it. Only ticks that follow SOME failure signal pay the
389
+ # search; total state loss + instant recovery leaves the issue
390
+ # for the next alarm cycle's search to adopt and close.
391
+ with contextlib.suppress(Exception):
392
+ open_alarm = _find_alarm_issue(github, target, bot_login)
393
+ if open_alarm:
394
+ try:
395
+ github.close_issue(target, open_alarm)
396
+ except Exception as exc:
397
+ # keep the state so the NEXT healthy tick retries the close;
398
+ # unlinking here would orphan the open alarm forever — and
399
+ # no comment yet, or every retry would repeat it
400
+ log.warning("could not close contract alarm #%s: %s", open_alarm, exc)
401
+ return
402
+ with contextlib.suppress(Exception):
403
+ github.comment(
404
+ target, open_alarm, "The tick loads again cleanly; launch lanes resume."
405
+ )
406
+ if state:
407
+ with contextlib.suppress(OSError):
408
+ state_path.unlink()
409
+ return
410
+ count = int(state.get("count", 0)) + 1
411
+ state["count"] = count
412
+ # redacted (the client's own token is the one secret this process
413
+ # holds) and fenced with a run longer than any backtick run inside —
414
+ # transport errors can echo request material, loader errors can echo
415
+ # contract content, and both are untrusted for a public issue body
416
+ safe_error = redact(error, _client_secrets(github)).replace(str(Path.home()), "~")[:600]
417
+ # A recorded issue a human closed by hand stays closed: closing the
418
+ # alarm is the maintainer's "I know" — re-opening or re-creating it
419
+ # every threshold would be alarm spam, and recovery still clears state.
420
+ if count >= CONTRACT_ALARM_AFTER and not state.get("issue"):
421
+ # search open issues first: state loss must not spawn duplicates
422
+ try:
423
+ number = _find_alarm_issue(github, target, bot_login) or github.create_issue(
424
+ target,
425
+ "outerloop: launch lanes are paused",
426
+ f"{CONTRACT_ALARM_MARKER}\nThe orchestrator's launch lanes "
427
+ f"(intake, steward, self-initiated) have sat out {count} "
428
+ f"consecutive ticks. The error below names the cause — a "
429
+ f"contract that failed to load, or a panel config the climb "
430
+ f"would reject.\n\n"
431
+ f"{_fence(safe_error)}\n{safe_error}\n{_fence(safe_error)}\n\n"
432
+ f"This issue closes itself when a tick passes cleanly.",
433
+ )
434
+ if number:
435
+ state["issue"] = number
436
+ except Exception as exc:
437
+ log.warning("contract alarm could not post to %s: %s", target, exc)
438
+ with contextlib.suppress(OSError):
439
+ tmp = state_path.with_suffix(f".{os.getpid()}.tmp")
440
+ tmp.write_text(json.dumps(state))
441
+ os.replace(tmp, state_path)
442
+
443
+
444
+ def _client_secrets(github: Any) -> tuple[str, ...]:
445
+ try:
446
+ token = github.auth.token()
447
+ if token:
448
+ return (token,)
449
+ except Exception as exc:
450
+ # degraded redaction must not be silent: the error text goes to a
451
+ # public issue, and a renamed auth surface would no-op the redact
452
+ log.warning("alarm redaction has no client token (%s)", exc)
453
+ return ()
454
+
455
+
456
+ def _contract_text(github: Any, target: str, ref: str) -> str | None:
457
+ """The target's contract at `ref` — `.outerloop.yaml`, else the legacy
458
+ `.autoresearch.yaml` — or None when it has neither."""
459
+ from outerloop.contract import find_contract
460
+
461
+ found = find_contract(lambda name: github.get_file_content(target, name, ref))
462
+ return found[1] if found else None
463
+
464
+
465
+ def _fence(content: str) -> str:
466
+ longest = max((len(run) for run in re.findall(r"`+", content)), default=0)
467
+ return "`" * max(3, longest + 1)
468
+
469
+
470
+ def _find_alarm_issue(github: Any, target: str, bot_login: str) -> int:
471
+ """Only the BOT'S own marker'd issue counts: the marker is a public
472
+ string, and adopting a stranger's issue would let anyone suppress the
473
+ real alarm or get their issue closed by the bot."""
474
+ from outerloop.github import is_own_login
475
+
476
+ return next(
477
+ (
478
+ int(issue.get("number", 0))
479
+ for issue in github.list_open_issues(target, max_pages=10)
480
+ if has_marker(str(issue.get("body", "")), "contract-alarm")
481
+ and is_own_login(str((issue.get("user") or {}).get("login", "")), bot_login)
482
+ ),
483
+ 0,
484
+ )
485
+
486
+
487
+ def shape_followup_spec(spec: FollowupSpec, limits: EffectiveLimits, contract: Any) -> FollowupSpec:
488
+ """Clamp the operator's follow-up spec by the contract's effective
489
+ limits. Both knobs clamp only when the contract EXPLICITLY sets them:
490
+ a contract shapes spend downward, but an operator's deliberate config
491
+ is never silently reduced by defaults (raising budgets is operator
492
+ territory)."""
493
+ if contract is None:
494
+ return spec
495
+ # direct attribute access: Budgets is our typed model, and a rename
496
+ # must fail loudly here, not silently stop shaping spend downward
497
+ if contract.budgets.session_max_turns is not None:
498
+ spec = replace(spec, max_turns=min(spec.max_turns, limits.session_max_turns))
499
+ if contract.budgets.followup_job_minutes is not None:
500
+ spec = replace(spec, time_minutes=min(spec.time_minutes, limits.followup_job_minutes))
501
+ return spec
502
+
503
+
504
+ def _base_dial(
505
+ github: Any, target: str, pr: dict, main_contract: Any, main_target: str = ""
506
+ ) -> str:
507
+ """The merge dial that governs THIS PR: its own target's base-branch
508
+ contract. The tick's contract is read from ITS configured target's main;
509
+ it applies only to a main-based PR of that same target — any other
510
+ target or base is fetched from the PR's own coordinates, and unreadable
511
+ or unparsable means "manual" (never arm on doubt)."""
512
+ base_ref = str((pr.get("base") or {}).get("ref", "")) or "main"
513
+ if base_ref == "main" and target == main_target:
514
+ return str(getattr(main_contract, "merge", "manual"))
515
+ try:
516
+ from outerloop.contract import load_contract
517
+
518
+ raw = _contract_text(github, target, base_ref)
519
+ if raw is None:
520
+ return "manual"
521
+ return str(getattr(load_contract(raw, target), "merge", "manual"))
522
+ except Exception as exc:
523
+ log.warning("base-contract read failed for %s@%s: %s", target, base_ref, exc)
524
+ return "manual"
525
+
526
+
527
+ def service_in_review(
528
+ root: Path,
529
+ github: Any, # GitHubClient (Any keeps tick importable without github deps)
530
+ compute: Compute,
531
+ spec: FollowupSpec,
532
+ now: float,
533
+ dry_run: bool = False,
534
+ allow_submit: bool = True,
535
+ contract: Any = None,
536
+ ) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
537
+ """PR-state transitions + follow-up job submission for in-review runs.
538
+
539
+ allow_submit=False (disk preflight failed) keeps the cheap state
540
+ transitions — ending merged/closed runs still matters — but submits no
541
+ new session jobs.
542
+
543
+ The tick only READS GitHub here (cheap, every cycle); the session-running
544
+ work happens in a submitted job, which takes the run lease itself — a
545
+ duplicate submission no-ops on the lease, and `followup_job_id` keeps the
546
+ tick from queueing duplicates in the first place.
547
+ """
548
+ from outerloop.followup import (
549
+ _pr_number,
550
+ close_if_done,
551
+ conflict_wake_action,
552
+ has_new_comments,
553
+ panel_wake_pending,
554
+ )
555
+
556
+ ended: list[tuple[str, str]] = []
557
+ submitted: list[tuple[str, str]] = []
558
+ for record in list_runs(root):
559
+ if record.state != IN_REVIEW or not record.pr_url:
560
+ continue
561
+ try:
562
+ ending = close_if_done(root, record, github, now)
563
+ if ending:
564
+ ended.append((record.run_id, ending))
565
+ continue
566
+ # Steward records are serviced with the STEWARD'S key and the
567
+ # steward scope check (respond_once derives the mode from the
568
+ # record's agent id); without a provisioned steward key the
569
+ # lane stays human-answered.
570
+ is_steward = record.agent_id.startswith("steward")
571
+ if is_steward and not spec.steward_key_file:
572
+ continue
573
+ # per-ROLE outage latch: state transitions above still ran,
574
+ # only this record's session spawn sits the cooldown out
575
+ paused = outage_active(root, now, role="steward" if is_steward else "solver")
576
+ if paused:
577
+ log.info("follow-up for %s paused (api outage: %s)", record.run_id, paused)
578
+ continue
579
+ try:
580
+ pr = github.get_pull_request(record.target, _pr_number(record.pr_url))
581
+ except Exception:
582
+ continue # unreadable PR: nothing to decide this tick
583
+ # Idempotent auto-arm: once GitHub reports the PR CLEAN (green
584
+ # checks AND up-to-date with the CURRENT base — GitHub's own
585
+ # freshness proof), the kernel-read contract STILL says auto, and
586
+ # A dispatched re-measure in flight: nothing else is serviced (the
587
+ # sealed change lands first, so the next comment is answered on
588
+ # the tree it will actually see) and nothing is armed. Once every
589
+ # eval job is terminal, a follow-up is submitted to finish it.
590
+ measure_ready = False
591
+ if record.followup_stage:
592
+ raw_ids = record.followup_stage.get("job_ids")
593
+ job_ids = [str(j) for j in raw_ids] if isinstance(raw_ids, list) else []
594
+ if job_ids:
595
+ try:
596
+ states = [compute.status(j) for j in job_ids]
597
+ except SlurmQueryError:
598
+ continue # unknown: neither service nor arm
599
+ if not all(is_terminal(s) or s == GONE for s in states):
600
+ continue
601
+ measure_ready = True
602
+ else:
603
+ # a BLIND park (the measurer could not read the queue at
604
+ # dispatch): no ids to poll, so the eval walltime plus the
605
+ # climb's queue slack is the floor before a follow-up is
606
+ # sent to look — never one per tick (terra #241 r1)
607
+ from outerloop.dispatch import effective_eval_minutes
608
+
609
+ parked_at = float(record.followup_stage.get("parked_at", 0.0) or 0.0) # type: ignore[arg-type]
610
+ floor_min = int(record.followup_stage.get("eval_minutes", 0) or 0) # type: ignore[call-overload]
611
+ floor_s = (effective_eval_minutes(floor_min) + BLIND_PARK_SLACK_MIN) * 60
612
+ if now - parked_at < floor_s:
613
+ continue
614
+ measure_ready = True
615
+ wake_action = conflict_wake_action(record, pr)
616
+ if wake_action == "clear":
617
+ # the PR is clean again: re-arm the wake for this head — the
618
+ # base can move and conflict the SAME head a second time
619
+ try:
620
+ save_record(root, replace(record, dirty_wake_head=""), now)
621
+ except OSError as exc:
622
+ log.warning("conflict cursor clear failed for %s: %s", record.run_id, exc)
623
+ if (
624
+ not measure_ready
625
+ and not has_new_comments(record, github, spec.bot_login)
626
+ and wake_action != "wake"
627
+ and not panel_wake_pending(record, pr)
628
+ ):
629
+ # NOTHING awaits servicing — only a fully quiet PR may
630
+ # self-merge (pending reviewer feedback always wins over
631
+ # arming: a followup must service it first, and a pushed
632
+ # change would kill the blessing anyway). The RECORD says the
633
+ # publish was auto-eligible (published under merge:auto with
634
+ # a clean panel — #171's exact arming condition; a manual
635
+ # publish never consented, and contracts alone cannot prove
636
+ # either fact after a dial flip); GitHub's own CLEAN state is
637
+ # the freshness proof; the PR's base-branch contract is the
638
+ # governing dial. Running every tick survives any crash
639
+ # between a sync push and this step; the helper direct-merges
640
+ # when nothing is pending to arm against.
641
+ # ...and no follow-up job may be LIVE: a running responder can
642
+ # have pushed a code change whose record write (clearing the
643
+ # blessing) has not landed yet — arming on that head would
644
+ # merge code the panel never saw (terra #228 r7)
645
+ followup_live = False
646
+ if record.followup_job_id:
647
+ try:
648
+ state = compute.status(record.followup_job_id)
649
+ followup_live = not (is_terminal(state) or state == GONE)
650
+ except SlurmQueryError:
651
+ followup_live = True # unknown = assume live, never arm
652
+ if (
653
+ not dry_run
654
+ and not is_steward
655
+ and not followup_live
656
+ and record.auto_blessed_head
657
+ and str((pr.get("head") or {}).get("sha", "")) == record.auto_blessed_head
658
+ and contract is not None
659
+ and pr.get("state") == "open"
660
+ and not pr.get("merged")
661
+ and not pr.get("draft")
662
+ and pr.get("mergeable_state") == "clean"
663
+ and _base_dial(github, record.target, pr, contract, spec.target) == "auto"
664
+ ):
665
+ try:
666
+ # the mutation itself is bound to the blessed head: a
667
+ # push racing this check is refused by GitHub, not
668
+ # merged (terra #228 r9)
669
+ github.arm_auto_merge_auto_mode(
670
+ record.target,
671
+ _pr_number(record.pr_url),
672
+ expected_head=record.auto_blessed_head,
673
+ )
674
+ except Exception as exc:
675
+ log.warning("auto-arm failed for %s: %s", record.run_id, exc)
676
+ continue
677
+ if record.followup_job_id:
678
+ try:
679
+ state = compute.status(record.followup_job_id)
680
+ if not (is_terminal(state) or state == GONE):
681
+ continue # a follow-up job is already queued/running
682
+ except SlurmQueryError:
683
+ continue # unknown — do not stack another job
684
+ # the wake-attempt counter caps follow-up retries too: a responder
685
+ # that cannot advance its cursors must not burn a session per tick
686
+ if record.wake_attempts >= MAX_WAKE_ATTEMPTS:
687
+ log.warning(
688
+ "run %s: %d follow-up attempts without progress; not resubmitting",
689
+ record.run_id,
690
+ record.wake_attempts,
691
+ )
692
+ continue
693
+ if not allow_submit:
694
+ log.warning("run %s has new comments but disk preflight failed", record.run_id)
695
+ continue
696
+ if dry_run:
697
+ submitted.append((record.run_id, "dry-run"))
698
+ continue
699
+ # The author follow-up carries the climb's panel so a pushed code
700
+ # change is RE-READ before the tick may arm it (followup.py). The
701
+ # panel brings its own walltime, like the climb's allowance — the
702
+ # contract's followup budget caps the author, not the gate. A
703
+ # panel that would die at startup is left off: the reply still
704
+ # goes out, the PR simply stays human-merged (the tick's contract
705
+ # alarm already names the misconfig).
706
+ panel_argv: list[str] = []
707
+ panel_minutes = 0
708
+ if not is_steward and spec.panel.strip():
709
+ panel_error = _panel_preflight_error(spec)
710
+ if panel_error:
711
+ log.warning(
712
+ "follow-up for %s runs without the panel: %s", record.run_id, panel_error
713
+ )
714
+ else:
715
+ from outerloop.panel import panel_read_minutes
716
+
717
+ panel_argv = _climb_panel_argv(spec)
718
+ panel_minutes = panel_read_minutes(spec.panel)
719
+ # the author's budget first, the read on top, both under the
720
+ # partition cap — and the follow-up is told how many minutes the
721
+ # read actually got (--panel-minutes), so a cap that eats the
722
+ # allowance costs the READ (skipped, said so), never the author
723
+ author_minutes = min(spec.time_minutes, spec.max_job_minutes)
724
+ job_minutes = min(author_minutes + panel_minutes, spec.max_job_minutes)
725
+ if panel_argv:
726
+ panel_argv = [*panel_argv, "--panel-minutes", str(job_minutes - author_minutes)]
727
+ argv = [
728
+ "uv",
729
+ "run",
730
+ "python",
731
+ "-m",
732
+ "outerloop.followup",
733
+ "--run-root",
734
+ str(spec.run_root),
735
+ "--run-id",
736
+ record.run_id,
737
+ "--image",
738
+ spec.image,
739
+ "--bot-login",
740
+ spec.bot_login,
741
+ "--job-minutes",
742
+ # the SAME clamped value Slurm gets: a deadline armed past
743
+ # the real walltime is a Slurm kill before a clean ending
744
+ str(job_minutes),
745
+ "--max-turns",
746
+ str(spec.max_turns),
747
+ # the cluster coordinates the climb gets: a GPU benchmark's
748
+ # re-measure is dispatched to the GPU lane, never run here
749
+ "--account",
750
+ spec.account,
751
+ "--partition",
752
+ spec.partition,
753
+ "--gpu-partition",
754
+ spec.gpu_partition,
755
+ "--gpu-account",
756
+ spec.gpu_account,
757
+ *panel_argv,
758
+ ]
759
+ if spec.pat_file:
760
+ argv += ["--pat-file", spec.pat_file]
761
+ # config-driven author: the author follow-up resolves its key per the
762
+ # RUN's backend (from the record) inside followup.main — the tick does
763
+ # not thread it. The steward is a distinct role with its own key.
764
+ if is_steward and spec.steward_key_file:
765
+ argv += ["--key-file", spec.steward_key_file]
766
+ job_id = compute.submit(
767
+ JobSpec(
768
+ job_name=f"followup-{record.run_id}"[:60],
769
+ account=spec.account,
770
+ partition=spec.job_partition or spec.partition,
771
+ time_minutes=job_minutes,
772
+ command=_flight_command(spec.home, f"followup-{record.run_id}"[:60], now, argv),
773
+ cpus=4,
774
+ mem="8G",
775
+ )
776
+ )
777
+ # read-modify-write on the FRESH record: the submitted job may
778
+ # already be saving its own fields
779
+ latest = load_record(root, record.run_id)
780
+ save_record(
781
+ root,
782
+ replace(
783
+ latest,
784
+ followup_job_id=job_id,
785
+ wake_attempts=latest.wake_attempts + 1,
786
+ ),
787
+ now,
788
+ )
789
+ submitted.append((record.run_id, job_id))
790
+ except (SlurmError, Exception) as exc:
791
+ log.warning("in-review service failed for %s: %s", record.run_id, exc)
792
+ return ended, submitted
793
+
794
+
795
+ def _last_worked_ts(root: Path) -> float | None:
796
+ """The `ts` of the last tick that COMPLETED its work, or None if there is no
797
+ readable marker (first tick, or a corrupt/missing file). The coalesce guard
798
+ keys on this, NOT the heartbeat: the heartbeat is stamped at tick START (for
799
+ the watchdog), so a tick that crashes mid-work still leaves a fresh
800
+ heartbeat — coalescing on that would suppress the very recovery tick. The
801
+ work marker is written only at a full tick's END, so a failed tick never
802
+ hides behind it."""
803
+ # The marker is a best-effort optimization we write ourselves; ANY failure
804
+ # reading/parsing/converting a corrupt file (OSError, ValueError,
805
+ # OverflowError on a huge int, RecursionError on deep nesting, ...) must
806
+ # degrade to "no marker" so coalesce simply proceeds — it can never crash the
807
+ # tick before its heartbeat. bool is an int subclass, so exclude it; inf/nan
808
+ # are not usable elapsed anchors.
809
+ try:
810
+ payload = json.loads((root / WORK_MARKER_NAME).read_text())
811
+ ts = payload.get("ts") if isinstance(payload, dict) else None
812
+ if not isinstance(ts, int | float) or isinstance(ts, bool):
813
+ return None
814
+ val = float(ts)
815
+ return val if math.isfinite(val) else None
816
+ except Exception:
817
+ return None
818
+
819
+
820
+ def _mark_worked(root: Path, now: float) -> None:
821
+ """Record that a tick completed its work at `now` — the coalesce signal.
822
+ Best-effort: a marker that cannot be written must not fail the tick."""
823
+ try:
824
+ tmp = root / f".{WORK_MARKER_NAME}.tmp"
825
+ tmp.write_text(json.dumps({"ts": now}))
826
+ os.replace(tmp, root / WORK_MARKER_NAME)
827
+ except OSError as exc:
828
+ log.warning("work-marker write failed: %s", exc)
829
+
830
+
831
+ def mark_tick_complete(root: Path, report: TickReport, now: float) -> None:
832
+ """Stamp the coalesce marker iff the tick actually did work, at real
833
+ COMPLETION time (the caller passes time.time() AFTER tick() returns). A
834
+ paused/coalesced tick leaves it untouched; a tick that raised never reaches
835
+ here — so only a genuinely completed tick can coalesce the next one, and a
836
+ long tick's marker reflects when it finished, not when it started."""
837
+ if not report.paused and not report.coalesced:
838
+ _mark_worked(root, now)
839
+
840
+
841
+ def write_heartbeat(root: Path, now: float, disk: dict[str, object] | None = None) -> None:
842
+ """Best-effort: a heartbeat that cannot be written (full disk) must not
843
+ kill the tick — the tick can still end runs and post to GitHub."""
844
+ payload: dict[str, object] = {"ts": now, "host": socket.gethostname(), "pid": os.getpid()}
845
+ if disk is not None:
846
+ payload["disk"] = disk
847
+ try:
848
+ tmp = root / f".{HEARTBEAT_NAME}.tmp"
849
+ tmp.write_text(json.dumps(payload))
850
+ os.replace(tmp, root / HEARTBEAT_NAME)
851
+ except OSError as exc:
852
+ log.warning("heartbeat write failed: %s", exc)
853
+
854
+
855
+ def _holder_alive(compute: Compute, lease_job_id: str) -> bool | None:
856
+ """True/False when Slurm answered; None when it could not (an outage
857
+ must not look like a dead holder)."""
858
+ if not lease_job_id:
859
+ return None
860
+ try:
861
+ state = compute.status(lease_job_id)
862
+ except SlurmQueryError:
863
+ return None
864
+ return not (is_terminal(state) or state == GONE)
865
+
866
+
867
+ def _wake(
868
+ root: Path,
869
+ record: RunRecord,
870
+ reason: str,
871
+ dispatcher: WakeDispatcher,
872
+ now: float,
873
+ holder: str,
874
+ ) -> bool:
875
+ """Lease-guarded wake. True when this tick delivered (or handed off) it.
876
+
877
+ The attempt counter is bumped BEFORE dispatch, so a dispatcher that dies
878
+ mid-delivery still counts toward the stuck threshold.
879
+ """
880
+ if not acquire_lease(root, record.run_id, holder, holder_job_id="", now=now):
881
+ return False
882
+ bumped = replace(
883
+ record,
884
+ wake_attempts=record.wake_attempts + 1,
885
+ # repair legacy records as we touch them: save_record (rightly)
886
+ # refuses to write a waiting run without a deadline
887
+ deadline=record.deadline if record.deadline > 0 else now,
888
+ )
889
+ save_record(root, bumped, now)
890
+ try:
891
+ holder_job = dispatcher.dispatch(bumped, reason)
892
+ except Exception as exc:
893
+ log.warning("wake dispatch failed for %s: %s: %s", record.run_id, type(exc).__name__, exc)
894
+ release_lease(root, record.run_id)
895
+ return False
896
+ if holder_job and not local_mode():
897
+ # An async wake job now owns the lease; it releases on completion,
898
+ # and the TTL/holder-dead check reaps it if it dies.
899
+ update_lease_holder(root, record.run_id, f"wake-job:{holder_job}", holder_job, now)
900
+ else:
901
+ # No job to hand the lease to — or a LOCAL dispatch, which ran the
902
+ # whole wake synchronously: the attempt already finished and released
903
+ # its own lease, and recreating one under a terminal job id would
904
+ # make the next sweep reap a corpse instead of delivering.
905
+ release_lease(root, record.run_id)
906
+ return True
907
+
908
+
909
+ WAKE_SPEC_NAME = "wake-spec.json"
910
+
911
+
912
+ def dispatch_wake_armed(root: Path) -> bool:
913
+ """The operator's on-switch for dispatched wakes: the env var, or the
914
+ sentinel file (touch/rm, no chain restart). Read by the tick and by every
915
+ park, so a disarm takes effect at once."""
916
+ return (
917
+ bool(os.environ.get("AUTORESEARCH_DISPATCH_WAKE", "").strip())
918
+ or (root / DISPATCH_WAKE_SENTINEL).exists()
919
+ )
920
+
921
+
922
+ def write_wake_spec(root: Path, spec: FollowupSpec) -> None:
923
+ """Publish the tick's wake recipe for the jobs that park runs: a park
924
+ submits its own wake (`arm_wake`) with exactly the tick's settings, so
925
+ dispatched wakes stay one recipe with one owner."""
926
+ data = {k: (str(v) if isinstance(v, Path) else v) for k, v in asdict(spec).items()}
927
+ tmp = root / f".{WAKE_SPEC_NAME}.{os.getpid()}.tmp"
928
+ tmp.write_text(json.dumps(data))
929
+ os.replace(tmp, root / WAKE_SPEC_NAME)
930
+
931
+
932
+ def remove_wake_spec(root: Path) -> None:
933
+ with contextlib.suppress(FileNotFoundError):
934
+ (root / WAKE_SPEC_NAME).unlink()
935
+
936
+
937
+ def load_wake_spec(root: Path) -> FollowupSpec | None:
938
+ """The published wake recipe, or None when dispatched wakes are not armed
939
+ (or the file is unreadable — the sweep still delivers)."""
940
+ try:
941
+ data = json.loads((root / WAKE_SPEC_NAME).read_text())
942
+ except (OSError, ValueError):
943
+ return None
944
+ if not isinstance(data, dict):
945
+ return None
946
+ names = {f.name for f in FollowupSpec.__dataclass_fields__.values()}
947
+ kwargs: dict[str, Any] = {
948
+ k: (Path(v) if k in ("run_root", "home") else v) for k, v in data.items() if k in names
949
+ }
950
+ try:
951
+ return FollowupSpec(**kwargs)
952
+ except TypeError:
953
+ return None
954
+
955
+
956
+ def arm_wake(
957
+ root: Path, record: RunRecord, dispatcher: WakeDispatcher, now: float, *, holder_job_id: str
958
+ ) -> str:
959
+ """Submit a parked run's wake NOW, depending on the jobs it waits on, so
960
+ it fires the moment they finish instead of a sweep cadence (plus grace)
961
+ later. The same job and lease as a sweep-delivered wake: a wake job that
962
+ parks again hands its own lease to the wake it arms; any other holder (a
963
+ tick mid-delivery) keeps it and the sweep delivers as before. Arming is
964
+ not a redelivery, so it leaves `wake_attempts` — the sweep's stuck
965
+ counter — alone: an eval requeued by preemption fires the afterany
966
+ early, the wake re-parks, and that must not count toward STUCK. Returns
967
+ the wake job id, or "" when nothing was armed. A park with nothing to
968
+ depend on (a checkpoint sleep, a blind park) is not armed: it rides the
969
+ deadline floor, as before."""
970
+ if not _poll_targets(record):
971
+ return ""
972
+ lease = read_lease(root, record.run_id)
973
+ if lease is not None:
974
+ if not holder_job_id or lease.holder_job_id != holder_job_id:
975
+ return ""
976
+ elif not acquire_lease(
977
+ root, record.run_id, f"park:{holder_job_id or os.getpid()}", holder_job_id="", now=now
978
+ ):
979
+ return ""
980
+ if record.deadline <= 0: # a waiting record always carries a deadline
981
+ record = replace(record, deadline=now)
982
+ save_record(root, record, now)
983
+ try:
984
+ job = dispatcher.dispatch(record, "parked")
985
+ except Exception as exc:
986
+ log.warning("arming the wake failed for %s: %s: %s", record.run_id, type(exc).__name__, exc)
987
+ job = ""
988
+ if job:
989
+ update_lease_holder(root, record.run_id, f"wake-job:{job}", job, now)
990
+ elif lease is None:
991
+ release_lease(root, record.run_id)
992
+ return job
993
+
994
+
995
+ def _armed_wake_lost(
996
+ root: Path,
997
+ compute: Compute,
998
+ record: RunRecord,
999
+ lease: Lease,
1000
+ now: float,
1001
+ grace_s: float,
1002
+ dry_run: bool,
1003
+ ) -> bool:
1004
+ """An armed wake that is still PENDING on its dependency after every job
1005
+ it waits on has been terminal for a full grace window is not coming
1006
+ (Slurm reports the dependency as never satisfiable, or the afterany was
1007
+ lost). Cancel it so the sweep redelivers; the lease is then reaped.
1008
+
1009
+ A wake the SITE moved off the partition it was submitted to may be
1010
+ starving there — on Torch a pending job can be shifted to a lower-tier
1011
+ partition (2026-09-02, wake 16787511 sat on `all` for hours) — but
1012
+ relocation alone is routine (jobs move to `cs` while still waiting on
1013
+ their dependencies and start on time). So a relocated holder counts as
1014
+ lost only once every job it waited on is terminal AND the grace window
1015
+ has run out without it starting; then it is cancelled and redelivered
1016
+ onto the requested partition."""
1017
+ if not lease.holder_job_id:
1018
+ return False
1019
+ job_ids = _poll_targets(record)
1020
+ if not job_ids:
1021
+ return False
1022
+ try:
1023
+ holder_state = compute.status(lease.holder_job_id)
1024
+ if not is_pending(holder_state):
1025
+ return False
1026
+ reason = compute.pending_reason(lease.holder_job_id)
1027
+ states = [compute.status(jid) for jid in job_ids]
1028
+ except SlurmQueryError:
1029
+ return False
1030
+ moved = _moved_off_partition(root, compute, lease.holder_job_id)
1031
+ if reason == "DependencyNeverSatisfied":
1032
+ pass
1033
+ elif not all(is_terminal(s) for s in states):
1034
+ return False # its dependencies are still running: nothing to redeliver yet
1035
+ elif reason != "Dependency" and not moved:
1036
+ return False
1037
+ else:
1038
+ # Dependency-pending past its dependencies, or RELOCATED and eligible:
1039
+ # both get the grace window. Relocation alone is normal (the site
1040
+ # moves pending jobs routinely); a relocated wake counts as lost only
1041
+ # when every job it waited on is terminal and it still has not
1042
+ # started once the grace has run out — cancelling earlier would only
1043
+ # reset its queue age and burn a wake attempt.
1044
+ if moved:
1045
+ log.info(
1046
+ "armed wake %s for %s sits on partition %s (asked for %s) past its dependencies",
1047
+ lease.holder_job_id,
1048
+ record.run_id,
1049
+ moved[0],
1050
+ moved[1],
1051
+ )
1052
+ if record.terminal_seen <= 0:
1053
+ if not dry_run:
1054
+ save_record(root, replace(record, terminal_seen=now), now)
1055
+ return False
1056
+ if now - record.terminal_seen < grace_s:
1057
+ return False
1058
+ if dry_run:
1059
+ return True
1060
+ try:
1061
+ compute.cancel(lease.holder_job_id)
1062
+ # only a cancellation Slurm confirms lets the sweep redeliver: a
1063
+ # still-pending wake would otherwise run beside its replacement
1064
+ return not is_pending(compute.status(lease.holder_job_id))
1065
+ except Exception:
1066
+ return False
1067
+
1068
+
1069
+ def sweep(
1070
+ root: Path,
1071
+ compute: Compute,
1072
+ dispatcher: WakeDispatcher,
1073
+ now: float,
1074
+ grace_s: float = DEFAULT_GRACE_S,
1075
+ lease_ttl_s: float = DEFAULT_LEASE_TTL_S,
1076
+ dry_run: bool = False,
1077
+ ) -> TickReport:
1078
+ """The backup wake layers, applied to every waiting run.
1079
+
1080
+ dry_run reports what WOULD happen with zero writes — no leases, no
1081
+ attempt counters, no dispatch.
1082
+ """
1083
+ woken: list[tuple[str, str]] = []
1084
+ deferred: list[str] = []
1085
+ reaped: list[str] = []
1086
+ stuck: list[str] = []
1087
+ holder = f"tick:{socket.gethostname()}:{os.getpid()}"
1088
+ records = [r for r in list_runs(root) if r.state == WAITING]
1089
+
1090
+ def wake(record: RunRecord, reason: str, tag: str) -> None:
1091
+ if dry_run or _wake(root, record, reason, dispatcher, now, holder):
1092
+ woken.append((record.run_id, tag))
1093
+
1094
+ for record in records:
1095
+ try:
1096
+ _sweep_one(
1097
+ root,
1098
+ compute,
1099
+ dispatcher,
1100
+ now,
1101
+ grace_s,
1102
+ lease_ttl_s,
1103
+ dry_run,
1104
+ record,
1105
+ holder,
1106
+ wake,
1107
+ deferred,
1108
+ reaped,
1109
+ stuck,
1110
+ )
1111
+ except Exception as exc:
1112
+ log.warning("sweep failed on %s: %s: %s", record.run_id, type(exc).__name__, exc)
1113
+
1114
+ return TickReport(
1115
+ swept=len(records),
1116
+ woken=tuple(woken),
1117
+ deferred=tuple(deferred),
1118
+ reaped_leases=tuple(reaped),
1119
+ stuck=tuple(stuck),
1120
+ # NOT the global dry_run: that flag only dries WAKE delivery;
1121
+ # ending killed climbs' records dispatches nothing and must run
1122
+ # live even while wakes stay dry.
1123
+ implementing_ended=tuple(_sweep_implementing(root, compute, now, grace_s)),
1124
+ )
1125
+
1126
+
1127
+ RESEARCH_LOG_BRANCH = "research-log"
1128
+ RESEARCH_LOG_MARKER = marker("research-log")
1129
+ RESEARCH_LOG_PER_TICK = 3
1130
+
1131
+
1132
+ def _ledger_marker(root: Path, run_id: str) -> Path:
1133
+ return run_dir(root, run_id) / "ledger-published"
1134
+
1135
+
1136
+ def _ledger_since(root: Path, target: str) -> Path:
1137
+ return root / ("research-log-since-" + target.replace("/", "__"))
1138
+
1139
+
1140
+ def _ledger_issue_cache(root: Path, target: str) -> Path:
1141
+ return root / ("research-log-issue-" + target.replace("/", "__"))
1142
+
1143
+
1144
+ def service_research_log(root: Path, github: Any, spec: FollowupSpec, now: float) -> int:
1145
+ """STATE-driven ledger (terra #170 r1: wiring publisher calls at terminal
1146
+ sites missed four terminal paths — attempt error, zero-change resume,
1147
+ the direct live terminal, steward): any run of this target whose
1148
+ terminal report exists but carries no published marker gets archived on
1149
+ the `research-log` branch and a two-line pointer routed — to the run's
1150
+ claimed issue never (it already got the full finish post), else to an
1151
+ open order issue naming the benchmark, else to the rolling log issue
1152
+ whose number is CACHED in the state dir (no 300-issue pagination scan,
1153
+ no duplicate creation). Running in the single tick also removes the
1154
+ concurrent-first-archive races by construction. The pointer posts only
1155
+ AFTER a successful archive (no dead links); the marker is written only
1156
+ after full success, so a crashed publish retries next tick. Runs ended
1157
+ before the feature's first pass are marker-stamped silently (no
1158
+ backfill spam).
1159
+ """
1160
+ since_path = _ledger_since(root, spec.target)
1161
+ first_pass = not since_path.exists()
1162
+ if first_pass:
1163
+ with contextlib.suppress(OSError):
1164
+ since_path.write_text(str(now))
1165
+ published = 0
1166
+ for record in list_runs(root):
1167
+ if record.target != spec.target or record.state not in (ENDED, IN_REVIEW):
1168
+ continue
1169
+ marker = _ledger_marker(root, record.run_id)
1170
+ report_path = run_dir(root, record.run_id) / "report.md"
1171
+ state = ""
1172
+ with contextlib.suppress(OSError):
1173
+ state = marker.read_text()
1174
+ if state.startswith(("done", "adopted")) or not report_path.exists():
1175
+ continue
1176
+ if first_pass and (record.updated or record.created) < now:
1177
+ # adopt pre-feature history silently: browsable going forward,
1178
+ # no retroactive issue spam. Only records OLDER than the since
1179
+ # marker — a run that goes terminal during this very pass is new
1180
+ # work and publishes normally (terra #170 r2).
1181
+ with contextlib.suppress(OSError):
1182
+ marker.write_text("adopted-unpublished")
1183
+ continue
1184
+ if published >= RESEARCH_LOG_PER_TICK:
1185
+ break
1186
+ try:
1187
+ report = report_path.read_text()
1188
+ except OSError:
1189
+ continue
1190
+ outcome = record.ending or ("improved" if record.state == IN_REVIEW else "ended")
1191
+ if _publish_ledger_entry(github, spec.target, root, record, outcome, report, marker, state):
1192
+ published += 1
1193
+ return published
1194
+
1195
+
1196
+ def _publish_ledger_entry(
1197
+ github: Any,
1198
+ target: str,
1199
+ root: Path,
1200
+ record: RunRecord,
1201
+ outcome: str,
1202
+ report: str,
1203
+ marker: Path,
1204
+ state: str,
1205
+ ) -> bool:
1206
+ """Staged publish whose marker doubles as a WRITABILITY PROBE: stages
1207
+ are "archived" -> "pointer-pending" -> "done", and no pointer is ever
1208
+ posted in a pass where a marker write is failing (terra #170 r4: a
1209
+ persistently unwritable marker must stall the publish, not stream a
1210
+ duplicate pointer every tick). A pointer failure retries pointer-only;
1211
+ a marker lost after full success re-posts at most once; the residual
1212
+ crash-between-probe-and-post window costs at most one duplicate per
1213
+ incident, never an unbounded stream."""
1214
+ from datetime import UTC, datetime
1215
+
1216
+ date = datetime.fromtimestamp(record.updated or record.created, tz=UTC).strftime("%Y-%m-%d")
1217
+ path = f"reports/{date}-{record.run_id}.md"
1218
+ # an earlier pass may have archived under an earlier date (an in-review
1219
+ # archive whose record re-stamped `updated` at ENDED): the marker's own
1220
+ # second line is the authoritative path for retries and pointers
1221
+ prior = state.splitlines()
1222
+ if len(prior) > 1 and prior[1].startswith("reports/") and prior[1].endswith(".md"):
1223
+ path = prior[1]
1224
+
1225
+ def _mark(value: str) -> bool:
1226
+ try:
1227
+ # the path rides the marker so readers (the board) never have to
1228
+ # re-derive the date from a timestamp that may have moved on
1229
+ marker.write_text(value + "\n" + path)
1230
+ return True
1231
+ except OSError as exc:
1232
+ log.warning("ledger marker write failed for %s: %s", record.run_id, exc)
1233
+ return False
1234
+
1235
+ try:
1236
+ if not state.startswith(("archived", "pointer-pending")):
1237
+ if not github.ensure_branch(target, RESEARCH_LOG_BRANCH):
1238
+ return False
1239
+ if not github.put_file(
1240
+ target,
1241
+ path,
1242
+ report,
1243
+ RESEARCH_LOG_BRANCH,
1244
+ f"research log: {record.run_id} ({outcome})",
1245
+ ):
1246
+ return False # retry the whole publish next tick
1247
+ if not _mark("archived"):
1248
+ return False # unwritable state: stall BEFORE any pointer
1249
+ # the probe: a fresh successful write is the license to post
1250
+ if not _mark("pointer-pending"):
1251
+ return False
1252
+ url = f"https://github.com/{target}/blob/{RESEARCH_LOG_BRANCH}/{path}"
1253
+ line = f"**{outcome}** `{record.benchmark}` — [report]({url})"
1254
+ if record.pr_url:
1255
+ line += f" · {record.pr_url}"
1256
+ if record.issue_number:
1257
+ _mark("done") # the claimed issue already received the full finish
1258
+ return True
1259
+ bench = record.benchmark.casefold()
1260
+ posted = False
1261
+ if bench:
1262
+ for issue in github.list_open_issues(target):
1263
+ text = f"{issue.get('title', '')}\n{issue.get('body') or ''}"
1264
+ if has_marker(text, "research-log") or issue.get("pull_request"):
1265
+ continue
1266
+ if bench in text.casefold():
1267
+ github.comment(target, int(issue["number"]), line)
1268
+ posted = True
1269
+ break
1270
+ if not posted:
1271
+ cache = _ledger_issue_cache(root, target)
1272
+ log_issue = 0
1273
+ with contextlib.suppress(OSError, ValueError):
1274
+ log_issue = int(cache.read_text().strip())
1275
+ if not log_issue:
1276
+ # cache miss (first use, or a lost/failed cache write): find
1277
+ # the rolling issue by its marker BEFORE creating another —
1278
+ # the cache is a fast path, never the source of truth
1279
+ # (terra #170 r5: a failed cache write must not duplicate
1280
+ # the rolling issue)
1281
+ for issue in github.list_open_issues(target):
1282
+ if has_marker(str(issue.get("body") or ""), "research-log"):
1283
+ log_issue = int(issue.get("number", 0))
1284
+ break
1285
+ if not log_issue:
1286
+ log_issue = github.create_issue(
1287
+ target,
1288
+ "Research log",
1289
+ f"{RESEARCH_LOG_MARKER}\nOne two-line comment per finished "
1290
+ f"autoresearch run — full reports live on the [`{RESEARCH_LOG_BRANCH}`]"
1291
+ f"(https://github.com/{target}/tree/{RESEARCH_LOG_BRANCH}/reports) "
1292
+ "branch. Results relevant to an open order issue are posted "
1293
+ "there instead.",
1294
+ )
1295
+ if log_issue:
1296
+ with contextlib.suppress(OSError):
1297
+ cache.write_text(str(log_issue))
1298
+ try:
1299
+ github.comment(target, log_issue, line)
1300
+ except Exception:
1301
+ # a stale cached number (locked/deleted/transferred issue)
1302
+ # must not stall delivery forever: drop the cache so the
1303
+ # next pass re-discovers or re-creates (terra #170 r5)
1304
+ with contextlib.suppress(OSError):
1305
+ cache.unlink()
1306
+ raise
1307
+ _mark("done") # write just proved out via the probe; failure = freak
1308
+ return True
1309
+ except Exception as exc: # advisory ledger: never fail the tick
1310
+ log.warning("research-log publish failed for %s: %s", record.run_id, exc)
1311
+ return False
1312
+
1313
+
1314
+ def _kill_stamp(root: Path, run_id: str) -> Path:
1315
+ return run_dir(root, run_id) / "attempt-terminal-seen"
1316
+
1317
+
1318
+ def _sweep_implementing(root: Path, compute: Compute, now: float, grace_s: float) -> list[str]:
1319
+ """End `implementing` records whose climb job died without a verdict.
1320
+
1321
+ A climb that CRASHES contains its own ending (attempt.py); a climb that is
1322
+ KILLED — walltime, preemption, scancel after the SIGTERM grace, node
1323
+ death — leaves no exception to contain, so this pass records the ending
1324
+ (the picker's stranded guard only frees the lane). Slurm truth decides:
1325
+ job terminal or GONE, plus a grace so a just-finished healthy climb can
1326
+ write its own final state first. Outage never reads as dead. Legacy
1327
+ records without a job id age out on the stranded window instead.
1328
+ """
1329
+ ended: list[str] = []
1330
+ for record in list_runs(root):
1331
+ if record.state != IMPLEMENTING:
1332
+ continue
1333
+ try:
1334
+ if record.run_job_id:
1335
+ try:
1336
+ state = compute.status(record.run_job_id)
1337
+ except SlurmQueryError:
1338
+ continue # Slurm outage must not read as a dead job
1339
+ if not (is_terminal(state) or state == GONE):
1340
+ continue # alive; the climb owns its own record
1341
+ # Grace runs from FIRST OBSERVED terminal: during Slurm's
1342
+ # KillWait the job already reports terminal while the
1343
+ # SIGTERM containment is still writing its honest ending —
1344
+ # record age would be hours and protect nothing. The stamp
1345
+ # is a write-once SIDECAR file, never a record write: while
1346
+ # the climb may still be alive the sweep must not touch the
1347
+ # record at all (a load-modify-replace here could revert a
1348
+ # concurrently written ending), and the waiting sweep's
1349
+ # terminal_seen field stays reserved for the EXPERIMENT job.
1350
+ stamp = _kill_stamp(root, record.run_id)
1351
+ if not stamp.exists():
1352
+ try:
1353
+ fd = os.open(stamp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
1354
+ try:
1355
+ os.write(fd, f"{now}".encode())
1356
+ finally:
1357
+ os.close(fd)
1358
+ except FileExistsError:
1359
+ pass # a concurrent tick stamped it; its clock stands
1360
+ except OSError as exc:
1361
+ log.warning("kill-stamp write failed for %s: %s", record.run_id, exc)
1362
+ continue
1363
+ # An empty stamp (write failed after create — the disk-full
1364
+ # case — or a concurrent tick mid-write) must fall back to
1365
+ # mtime, NOT to epoch 0, which would skip the grace outright.
1366
+ try:
1367
+ raw = stamp.read_text().strip()
1368
+ seen = float(raw) if raw else stamp.stat().st_mtime
1369
+ except (OSError, ValueError):
1370
+ try:
1371
+ seen = stamp.stat().st_mtime
1372
+ except OSError:
1373
+ continue # stamp vanished mid-read; next tick decides
1374
+ if now - seen < grace_s:
1375
+ continue
1376
+ note = f"climb job {record.run_job_id} ended {state} without a verdict"
1377
+ else:
1378
+ # No Slurm evidence at all (legacy record, or a manual dev
1379
+ # invocation without SLURM_JOB_ID): only the run DEADLINE —
1380
+ # past which nothing legitimately lives — justifies a
1381
+ # terminal verdict; the shorter stranded window merely
1382
+ # frees the picker lane and must not author endings.
1383
+ deadline = record.deadline if record.deadline > 0 else (record.created + 24 * 3600)
1384
+ if now < deadline:
1385
+ continue
1386
+ note = "implementing with no recorded climb job, past its run deadline"
1387
+ fresh = load_record(root, record.run_id)
1388
+ if fresh.state != IMPLEMENTING:
1389
+ continue # the climb landed its own ending meanwhile
1390
+ for jid in _poll_targets(fresh):
1391
+ # defensive: no current path records an experiment while
1392
+ # still implementing, but an orphan GPU job burning budget
1393
+ # after its run is declared dead must never survive one
1394
+ with contextlib.suppress(Exception):
1395
+ compute.cancel(jid)
1396
+ save_record(
1397
+ root,
1398
+ replace(
1399
+ fresh,
1400
+ state=ENDED,
1401
+ ending=ABORTED,
1402
+ ending_note=(
1403
+ f"{note} — ended by the sweep (a killed climb "
1404
+ f"leaves no exception to contain)"
1405
+ ),
1406
+ ),
1407
+ now,
1408
+ )
1409
+ # every ending produces a report — but never clobber one the
1410
+ # climb already wrote before it was killed
1411
+ report_path = run_dir(root, record.run_id) / "report.md"
1412
+ if not report_path.exists():
1413
+ try:
1414
+ report_path.write_text(
1415
+ f"# Run report — {record.target} / {record.benchmark}\n"
1416
+ f"Outcome: **aborted** (climb job killed)\n"
1417
+ f"Note: {note}\n"
1418
+ )
1419
+ except OSError as exc:
1420
+ log.warning("sweep report write failed for %s: %s", record.run_id, exc)
1421
+ log.warning("sweep ended implementing run %s: %s", record.run_id, note)
1422
+ ended.append(record.run_id)
1423
+ except Exception as exc: # per-record isolation, like the waiting sweep
1424
+ log.warning("implementing-sweep failed on %s: %s", record.run_id, exc)
1425
+ return ended
1426
+
1427
+
1428
+ def _moved_off_partition(root: Path, compute: Compute, job_id: str) -> tuple[str, str] | None:
1429
+ """(actual, wanted) when a queued kernel job no longer sits in the
1430
+ partition the wake recipe asks for, else None. Unknown either way —
1431
+ no recipe, a compute without partitions, a failed query — is None:
1432
+ never cancel on doubt."""
1433
+ spec = load_wake_spec(root)
1434
+ if spec is None:
1435
+ return None
1436
+ wanted = spec.job_partition or spec.partition
1437
+ if not wanted:
1438
+ return None
1439
+ try:
1440
+ actual = compute.job_partition(job_id)
1441
+ except (SlurmQueryError, ValueError, AttributeError):
1442
+ return None
1443
+ # both sides may be comma-separated lists: moved only when the job holds
1444
+ # NONE of the partitions the recipe asked for; empty is unknown, not moved
1445
+ have = {p.strip() for p in actual.split(",") if p.strip()}
1446
+ want = {p.strip() for p in wanted.split(",") if p.strip()}
1447
+ if not have or have & want:
1448
+ return None
1449
+ return actual, wanted
1450
+
1451
+
1452
+ def _poll_targets(record: RunRecord) -> list[str]:
1453
+ """Every Slurm job this run waits on. The single `experiment_job_id` is
1454
+ the common case; a MULTI-job park (candidate + siblings, or several author
1455
+ launches) records no single id — its jobs live in the stage's `afterany`
1456
+ dependency string, the one source that always names them all. Without
1457
+ this fallback a multi-job park is blind and rides the deadline floor."""
1458
+ if record.experiment_job_id:
1459
+ return [record.experiment_job_id]
1460
+ afterany = str((record.stage or {}).get("afterany", ""))
1461
+ return [t for t in afterany.split(":")[1:] if t]
1462
+
1463
+
1464
+ def _sweep_one(
1465
+ root: Path,
1466
+ compute: Compute,
1467
+ dispatcher: WakeDispatcher,
1468
+ now: float,
1469
+ grace_s: float,
1470
+ lease_ttl_s: float,
1471
+ dry_run: bool,
1472
+ record: RunRecord,
1473
+ holder: str,
1474
+ wake,
1475
+ deferred: list[str],
1476
+ reaped: list[str],
1477
+ stuck: list[str],
1478
+ ) -> None:
1479
+ if True:
1480
+ # Leases first: a LIVE wake in flight owns this run — even the stuck
1481
+ # verdict must wait for it (its session may be the one that succeeds).
1482
+ lease = read_lease(root, record.run_id)
1483
+ if lease is not None:
1484
+ alive = _holder_alive(compute, lease.holder_job_id)
1485
+ if not lease_is_stale(lease, now, lease_ttl_s, alive):
1486
+ if not _armed_wake_lost(root, compute, record, lease, now, grace_s, dry_run):
1487
+ return
1488
+ record = load_record(root, record.run_id)
1489
+ if dry_run:
1490
+ reaped.append(record.run_id)
1491
+ return
1492
+ if not reap_lease(root, record.run_id, reaper=f"{os.getpid()}-{now}", expected=lease):
1493
+ return # a concurrent tick reaped it first; it owns redelivery
1494
+ reaped.append(record.run_id)
1495
+
1496
+ # Layer 5: too many failed attempts is a terminal, reported state.
1497
+ if record.wake_attempts >= MAX_WAKE_ATTEMPTS:
1498
+ if not dry_run:
1499
+ ended = replace(
1500
+ record,
1501
+ state=ENDED,
1502
+ ending=STUCK,
1503
+ ending_note=(
1504
+ f"{record.wake_attempts} wake attempts without the run leaving 'waiting'"
1505
+ ),
1506
+ )
1507
+ save_record(root, ended, now)
1508
+ stuck.append(record.run_id)
1509
+ return
1510
+
1511
+ job_ids = _poll_targets(record)
1512
+ if not job_ids:
1513
+ # No job ids to poll. A BLIND PARK (the measurer could not read Slurm,
1514
+ # so `MeasurementPending` carried no ids) still hibernated with a
1515
+ # deadline — the deadline floor is its ONLY wake, so fire on it. A
1516
+ # genuinely mid-write record has no deadline and is left alone.
1517
+ # (A jobless CHECKPOINT SLEEP arrives here too — its deadline is
1518
+ # near-term by construction, attempt.py sizes it to the next sweep
1519
+ # pass, not the 12h queue slack that protects queued jobs.)
1520
+ if record.deadline > 0 and now > record.deadline:
1521
+ wake(record, "blind park past deadline", "deadline")
1522
+ return
1523
+
1524
+ try:
1525
+ states = [compute.status(jid) for jid in job_ids]
1526
+ except SlurmQueryError:
1527
+ # Layer 4's rule: query failure is "Slurm unknown", never "gone".
1528
+ deferred.append(record.run_id)
1529
+ return
1530
+
1531
+ # deadline <= 0 cannot be written by save_record for waiting runs;
1532
+ # if one exists anyway (legacy/hand-edited), treat it as already past
1533
+ # for GONE — a vanished-experiment wake is safe — but never for
1534
+ # PENDING, where the consequence would be cancelling a healthy job.
1535
+ past_deadline = record.deadline <= 0 or now > record.deadline
1536
+
1537
+ if all(is_terminal(s) for s in states):
1538
+ state = ",".join(sorted(set(states)))
1539
+ # Layer 3, with real grace: time runs from when the sweep FIRST
1540
+ # saw every job terminal, not from submission — the afterany
1541
+ # job gets the full window to deliver before the backup steps in.
1542
+ # Local compute has no afterany jobs to wait for (jobs are
1543
+ # terminal at submit): the sweep IS the delivery, so grace would
1544
+ # only cost a whole extra loop iteration.
1545
+ if local_mode():
1546
+ wake(record, f"experiment {state}", state)
1547
+ return
1548
+ if record.terminal_seen <= 0:
1549
+ if dry_run:
1550
+ # no writes in dry-run: report the would-wake now so the
1551
+ # terminal path is visible to live plumbing checks
1552
+ wake(record, f"experiment {state}", state)
1553
+ else:
1554
+ save_record(
1555
+ root,
1556
+ replace(
1557
+ record,
1558
+ terminal_seen=now,
1559
+ # repair legacy records as we touch them (see _wake)
1560
+ deadline=record.deadline if record.deadline > 0 else now,
1561
+ ),
1562
+ now,
1563
+ )
1564
+ return
1565
+ if now - record.terminal_seen >= grace_s:
1566
+ wake(record, f"experiment {state}", state)
1567
+ elif any(is_pending(s) for s in states) and record.deadline > 0 and now > record.deadline:
1568
+ # Unschedulable in practice: cancel every non-terminal job
1569
+ # (best-effort — scancel trouble must not abort the sweep), then
1570
+ # wake with that fact.
1571
+ if not dry_run:
1572
+ for jid, s in zip(job_ids, states, strict=True):
1573
+ if is_terminal(s) or s == GONE:
1574
+ continue
1575
+ try:
1576
+ compute.cancel(jid)
1577
+ except Exception as exc: # scancel trouble is never fatal here
1578
+ log.warning("cancel %s failed: %s", jid, exc)
1579
+ wake(record, "experiment unschedulable (pending past deadline)", "unschedulable")
1580
+ elif all(is_terminal(s) or s == GONE for s in states):
1581
+ # done-or-vanished, at least one GONE (all-terminal handled above)
1582
+ if past_deadline:
1583
+ wake(record, "experiment vanished from Slurm", "vanished")
1584
+ # else: sacct lag right after submission is normal; wait.
1585
+ # something RUNNING (or recently pending): nothing to do yet.
1586
+
1587
+
1588
+ def tick(
1589
+ root: Path,
1590
+ compute: Compute,
1591
+ dispatcher: WakeDispatcher,
1592
+ now: float,
1593
+ grace_s: float = DEFAULT_GRACE_S,
1594
+ lease_ttl_s: float = DEFAULT_LEASE_TTL_S,
1595
+ dry_run: bool = False,
1596
+ github: Any = None,
1597
+ followup_spec: FollowupSpec | None = None,
1598
+ followup_dry_run: bool = False,
1599
+ min_free_bytes: int = DEFAULT_MIN_FREE_BYTES,
1600
+ min_tick_s: float = DEFAULT_MIN_TICK_S,
1601
+ ) -> TickReport:
1602
+ """One full tick. Pause sentinel wins over everything: a paused loop
1603
+ heartbeats (so the watchdog stays quiet) but touches nothing.
1604
+
1605
+ Disk preflight gates every lane that LAUNCHES new work (follow-up jobs,
1606
+ intake claims, self-initiated climbs): a session started on a full or
1607
+ nearly-full filesystem dies mid-flight in ways that lose data. The sweep
1608
+ still runs — its writes are small, per-record contained, and ending runs
1609
+ matters more when storage is failing, not less.
1610
+ """
1611
+ # Heartbeat FIRST, before any probe: check_disk touches $HOME (a
1612
+ # different filesystem), and a hung mount there must not starve the
1613
+ # watchdog signal. The disk-annotated heartbeat follows once known. The
1614
+ # coalesce guard reads the last COMPLETED tick's marker (not the heartbeat).
1615
+ prior_worked = _last_worked_ts(root)
1616
+ write_heartbeat(root, now)
1617
+ disk_health = check_disk(root, min_free_bytes=min_free_bytes)
1618
+ write_heartbeat(root, now, disk=disk_health.as_dict())
1619
+ for warning in disk_health.warnings():
1620
+ log.warning("disk: %s", warning)
1621
+ if (root / PAUSE_SENTINEL).exists():
1622
+ log.info("pause sentinel present; tick is a no-op")
1623
+ return TickReport(paused=True)
1624
+ # Coalesce a congestion pile-up: if a tick COMPLETED its work within
1625
+ # min_tick_s, this one is redundant (that recent tick already swept and
1626
+ # launched). Keyed on the work marker (stamped by the CALLER at real
1627
+ # completion time), not the heartbeat, so a tick that crashed mid-work does
1628
+ # not suppress this recovery tick. Heartbeat still written above, so the
1629
+ # watchdog stays fed and the chain stays alive.
1630
+ if min_tick_s > 0 and prior_worked is not None:
1631
+ elapsed = now - prior_worked
1632
+ if elapsed < 0:
1633
+ # marker dated in the future -> the clock jumped back; never coalesce
1634
+ # on it (that could stall the loop), and surface it rather than fail
1635
+ # silently.
1636
+ log.warning(
1637
+ "work marker is %.0fs in the future (clock skew?); not coalescing", -elapsed
1638
+ )
1639
+ elif elapsed < min_tick_s:
1640
+ log.info(
1641
+ "coalescing: last completed tick %.0fs ago (< %.0fs); tick is a no-op",
1642
+ elapsed,
1643
+ min_tick_s,
1644
+ )
1645
+ return TickReport(coalesced=True)
1646
+ report = sweep(root, compute, dispatcher, now, grace_s, lease_ttl_s, dry_run=dry_run)
1647
+ # Housekeeping: ended runs shed ws/ and ws-home/ after a grace period;
1648
+ # when the state filesystem's write probe failed, the grace is waived and
1649
+ # the sweep frees oldest-first until the probe passes, then the preflight
1650
+ # is taken again so launch lanes can come back this very tick.
1651
+ # Force-shed (waive the grace) only when the state root cannot be WRITTEN,
1652
+ # not merely when it is below the free-space threshold: a writable disk
1653
+ # that is just low keeps the 24 h grace so a post-mortem is not deleted
1654
+ # under someone. A dry-run tick sheds nothing (the destructive lane obeys
1655
+ # the zero-writes contract, like sweep()).
1656
+ shed: list[str] = []
1657
+ if not dry_run:
1658
+ cannot_write = not disk_health.state_root.writable
1659
+ # Bounded per tick so shedding (rm -rf of tens of thousands of files
1660
+ # per workspace on a networked FS) never blows the tick's own timeout:
1661
+ # a few runs on a healthy disk, more but still time-boxed when it is
1662
+ # failing. The backlog drains over several ticks.
1663
+ shed = shed_ended_workspaces(
1664
+ root,
1665
+ now,
1666
+ force=cannot_write,
1667
+ limit=25 if cannot_write else 3,
1668
+ time_budget_s=300.0 if cannot_write else 120.0,
1669
+ until_ok=(lambda: check_disk(root, min_free_bytes=min_free_bytes).state_root.writable)
1670
+ if cannot_write
1671
+ else None,
1672
+ )
1673
+ if shed and cannot_write:
1674
+ disk_health = check_disk(root, min_free_bytes=min_free_bytes)
1675
+ write_heartbeat(root, now, disk=disk_health.as_dict())
1676
+ if shed:
1677
+ from dataclasses import replace as _dc_replace
1678
+
1679
+ report = _dc_replace(report, shed=tuple(shed))
1680
+ launch_ok = disk_health.launch_ok()
1681
+ if not launch_ok:
1682
+ log.warning("disk preflight failed; launch lanes are OFF this tick")
1683
+ # Mid-leg sync is serviced regardless of follow-up/board servicing: it
1684
+ # only needs the workspace and the PAT (a git fetch, no GitHub REST and
1685
+ # no contract), and a live session waiting on `sync` must not depend on
1686
+ # whether github/contract loaded this tick.
1687
+ if followup_spec is not None:
1688
+ service_syncs(root, followup_spec, now)
1689
+ if github is not None and followup_spec is not None:
1690
+ # expired flight snapshots die with their TTL, not with a human.
1691
+ # One home suffices: every lane's spec derives from followup_spec
1692
+ # via replace(), so all flights share this checkout's flights/ dir.
1693
+ # Blind means delete nothing — but only QUERY failures count as
1694
+ # blindness; a compute backend missing the method is a programming
1695
+ # error and propagates.
1696
+ try:
1697
+ live_names = compute.active_job_names()
1698
+ except SlurmQueryError as exc:
1699
+ log.warning("cannot list live jobs (%s); reaping no flights this tick", exc)
1700
+ live_names = None
1701
+ if live_names is not None:
1702
+ with contextlib.suppress(Exception):
1703
+ reaped = reap_flights(followup_spec.home, now, live_job_names=live_names)
1704
+ if reaped:
1705
+ log.info("reaped %d expired flight snapshot(s)", reaped)
1706
+ # ONE contract fetch per tick feeds every lane: the requested and
1707
+ # self-initiated lanes need its benchmarks, and all three lanes now
1708
+ # take their session/job limits from its budgets — clamped by our
1709
+ # ceilings (limits.py), so a target shapes spend, never raises it.
1710
+ # A failed fetch leaves in-review servicing running on defaults;
1711
+ # the launch lanes need the contract and sit out this tick.
1712
+ contract = None
1713
+ if followup_spec.target:
1714
+ contract_error: str | None = "contract file missing on main"
1715
+ try:
1716
+ from outerloop.contract import load_contract
1717
+
1718
+ raw = _contract_text(github, followup_spec.target, "main")
1719
+ if raw is not None:
1720
+ contract = load_contract(raw, followup_spec.target)
1721
+ contract_error = None
1722
+ except Exception as exc:
1723
+ log.warning("contract fetch failed for %s: %s", followup_spec.target, exc)
1724
+ contract_error = f"{type(exc).__name__}: {exc}"
1725
+ if contract_error is None:
1726
+ # a bad panel config idles the same launch lanes a bad
1727
+ # contract does — same silent-idle class, so it rides the
1728
+ # same alarm
1729
+ panel_error = _panel_preflight_error(followup_spec)
1730
+ if panel_error:
1731
+ contract_error = f"panel preflight: {panel_error}"
1732
+ try:
1733
+ contract_alarm(
1734
+ root,
1735
+ github,
1736
+ followup_spec.target,
1737
+ contract_error,
1738
+ now,
1739
+ bot_login=followup_spec.bot_login,
1740
+ )
1741
+ except Exception as exc:
1742
+ log.warning("contract alarm failed: %s", exc)
1743
+ limits = effective_limits(contract.budgets if contract is not None else None)
1744
+ # The contract's followup walltime only overrides when EXPLICITLY
1745
+ # set — and only DOWNWARD from the operator's spec value: strictly-
1746
+ # downward shaping must hold against operator config too, not just
1747
+ # against the module defaults.
1748
+ spec = shape_followup_spec(followup_spec, limits, contract)
1749
+ ended, submitted = service_in_review(
1750
+ root,
1751
+ github,
1752
+ compute,
1753
+ spec,
1754
+ now,
1755
+ dry_run=followup_dry_run,
1756
+ allow_submit=launch_ok,
1757
+ contract=contract,
1758
+ )
1759
+ try:
1760
+ service_research_log(root, github, spec, now)
1761
+ except Exception as exc: # the ledger is advisory; the tick continues
1762
+ log.warning("research-log service failed: %s", exc)
1763
+ intake_job = (
1764
+ service_intake(
1765
+ root, github, compute, spec, now, contract, limits, dry_run=followup_dry_run
1766
+ )
1767
+ if launch_ok and contract is not None
1768
+ else None
1769
+ )
1770
+ steward_job = (
1771
+ service_steward(
1772
+ root, github, compute, spec, now, contract, limits, dry_run=followup_dry_run
1773
+ )
1774
+ if launch_ok and intake_job is None and contract is not None
1775
+ else None
1776
+ )
1777
+ self_job = None
1778
+ if launch_ok and intake_job is None and steward_job is None and contract is not None:
1779
+ try:
1780
+ self_job = service_self_initiated(
1781
+ root,
1782
+ compute,
1783
+ spec,
1784
+ contract,
1785
+ now,
1786
+ limits=limits,
1787
+ dry_run=followup_dry_run,
1788
+ )
1789
+ except Exception as exc:
1790
+ log.warning("self-initiated selection failed: %s", exc)
1791
+ # AFTER the launch block: a run started this tick is on the strip
1792
+ # this tick, not the next one
1793
+ service_boards(root, github, spec.target, contract, now)
1794
+ report = replace_report(
1795
+ report,
1796
+ ended,
1797
+ submitted,
1798
+ intake_job,
1799
+ self_job,
1800
+ disk_health.warnings(),
1801
+ not launch_ok,
1802
+ steward_job,
1803
+ )
1804
+ # The coalesce marker is stamped by the CALLER at real completion time (see
1805
+ # main / mark_tick_complete) — not here with the start-of-tick `now`, which
1806
+ # a tick longer than the window would leave stale.
1807
+ return report
1808
+
1809
+
1810
+ def service_syncs(root: Path, spec: Any, now: float) -> None:
1811
+ """Honor mid-leg sync requests: a LIVE session asked for fresh origin/*
1812
+ refs and is waiting inside its own clock. The fetch pins the canonical
1813
+ URL (never the workspace's mutable remote config) and only refs/remotes
1814
+ are written — safe next to the session's local git use. Best-effort per
1815
+ run; a failure leaves the request standing for the next cycle."""
1816
+ from outerloop.appauth import resolve_bot_auth
1817
+ from outerloop.attempt import _target_clone_url
1818
+ from outerloop.github import Workspace
1819
+ from outerloop.syscall import mark_synced, sync_requested
1820
+
1821
+ for record in list_runs(root):
1822
+ if record.state != IMPLEMENTING:
1823
+ continue
1824
+ workspace = run_dir(root, record.run_id) / "ws"
1825
+ if not workspace.is_dir():
1826
+ continue
1827
+ requested_at = sync_requested(workspace)
1828
+ if requested_at is None:
1829
+ continue
1830
+ try:
1831
+ ws = Workspace(
1832
+ root=workspace,
1833
+ auth=(
1834
+ resolve_bot_auth(spec.pat_file, spec.github_app_file)
1835
+ if (spec.pat_file or spec.github_app_file)
1836
+ else None
1837
+ ),
1838
+ url=_target_clone_url(record.target),
1839
+ )
1840
+ ws.fetch_origin()
1841
+ mark_synced(workspace, requested_at)
1842
+ log.info("synced origin refs for %s", record.run_id)
1843
+ except Exception as exc:
1844
+ log.warning("sync failed for %s: %s", record.run_id, exc)
1845
+
1846
+
1847
+ def service_boards(root: Path, github: Any, target: str, contract: Any, now: float) -> None:
1848
+ """The climb board and the live strip, together and advisory: the views
1849
+ publish from the first tick (before any run ends), and a failure never
1850
+ stops the tick."""
1851
+ try:
1852
+ from outerloop.climbboard import contract_directions, service_climb_board
1853
+
1854
+ service_climb_board(root, github, target, contract_directions(contract))
1855
+ except Exception as exc:
1856
+ log.warning("climb board service failed: %s", exc)
1857
+ try:
1858
+ from outerloop.climbboard import service_status
1859
+
1860
+ service_status(root, github, target, now, contract)
1861
+ except Exception as exc: # each is advisory ALONE: one failing never mutes the other
1862
+ log.warning("status strip service failed: %s", exc)
1863
+
1864
+
1865
+ def replace_report(
1866
+ report: TickReport,
1867
+ ended: list[tuple[str, str]],
1868
+ submitted: list[tuple[str, str]],
1869
+ intake_job: tuple[str, str] | None = None,
1870
+ self_job: tuple[str, str] | None = None,
1871
+ disk_warnings: list[str] | None = None,
1872
+ launch_blocked: bool = False,
1873
+ steward_job: tuple[str, str] | None = None,
1874
+ ) -> TickReport:
1875
+ from dataclasses import replace as dc_replace
1876
+
1877
+ return dc_replace(
1878
+ report,
1879
+ review_ended=tuple(ended),
1880
+ followups_submitted=tuple(submitted),
1881
+ intake=intake_job or ("", ""),
1882
+ self_initiated=self_job or ("", ""),
1883
+ disk=tuple(disk_warnings or ()),
1884
+ launch_blocked=launch_blocked,
1885
+ steward=steward_job or ("", ""),
1886
+ )
1887
+
1888
+
1889
+ MAX_ACTIVE_RUNS_PER_TARGET = 1
1890
+ SELF_INITIATED_COOLDOWN_S = 6 * 3600
1891
+ # the crash-loop floor: a launch that died pre-record backs off at least
1892
+ # this long regardless of the contract's cooldown dial
1893
+ DEAD_LAUNCH_BACKOFF_S = 30 * 60
1894
+ # An implementing run untouched for this long is a crashed climb job; it must
1895
+ # not block the lane forever, but the window must exceed the LONGEST honest
1896
+ # job — the 120-min contract ceiling plus the panel allowance the tick adds
1897
+ # (~4.5 h at the defaults) plus queue-start slack — or the picker declares a
1898
+ # live run stranded and starts a second one on the same target, breaking the
1899
+ # one-active-run serialization. Its cooldown entry still applies, so a
1900
+ # crashed benchmark isn't immediately retried.
1901
+ STRANDED_IMPLEMENTING_S = 12 * 3600
1902
+ # A pending marker older than this is dead even if squeue can't be read.
1903
+ PENDING_TTL_S = 4 * 3600
1904
+
1905
+
1906
+ def pick_self_initiated(
1907
+ records: list[RunRecord],
1908
+ contract: Any,
1909
+ target: str,
1910
+ now: float,
1911
+ dead_attempts: dict[str, float] | None = None,
1912
+ live_pendings: list[tuple[str, float]] | None = None,
1913
+ ) -> str | None:
1914
+ """The benchmark to climb next on `target`, or None.
1915
+
1916
+ Deliberately boring (the planning agent upgrades this later): serialize
1917
+ to one active run per target, respect the contract's weekly budget and a
1918
+ per-benchmark cooldown, then choose the benchmark least recently
1919
+ attempted — untouched ones first. Only this target's runs count toward
1920
+ any of it. `dead_attempts` maps benchmark -> submitted_at for launches
1921
+ that died BEFORE writing a run record (per-benchmark tombstones) — each
1922
+ counts toward cooldown with a crash-loop floor, so alternating
1923
+ pre-record failures can't ping-pong every tick (terra #172 r2/r3).
1924
+ """
1925
+ mine = [r for r in records if r.target == target]
1926
+
1927
+ def stranded(r: RunRecord) -> bool:
1928
+ return r.state == IMPLEMENTING and now - max(r.updated, r.created) > STRANDED_IMPLEMENTING_S
1929
+
1930
+ active = [r for r in mine if r.state != ENDED and not stranded(r)]
1931
+ if len(active) >= _attempt_width(contract):
1932
+ return None
1933
+ week_ago = now - 7 * 24 * 3600
1934
+ # queued slots count toward the weekly budget BEFORE their records
1935
+ # exist, or a width-N target with one run left could submit N (terra
1936
+ # #173 r2); when a marker lands, the service clears it before calling
1937
+ # here, so a run is never counted twice
1938
+ queued = sum(1 for _, submitted_at in live_pendings or [] if submitted_at >= week_ago)
1939
+ if sum(1 for r in mine if r.created >= week_ago) + queued >= contract.budgets.runs_per_week:
1940
+ return None
1941
+ last_attempt: dict[str, float] = {}
1942
+ for r in mine:
1943
+ if r.benchmark:
1944
+ last_attempt[r.benchmark] = max(last_attempt.get(r.benchmark, 0.0), r.created)
1945
+ for bench_name, submitted_at in (dead_attempts or {}).items():
1946
+ last_attempt[bench_name] = max(last_attempt.get(bench_name, 0.0), submitted_at)
1947
+ for bench_name, submitted_at in live_pendings or []:
1948
+ # a queued sibling starts its benchmark's cooldown clock too:
1949
+ # width spreads across benchmarks first, and re-picking the same
1950
+ # one needs the contract to have set cooldown to 0 (portfolio)
1951
+ if bench_name:
1952
+ last_attempt[bench_name] = max(last_attempt.get(bench_name, 0.0), submitted_at)
1953
+ cooldown_min = getattr(contract.budgets, "attempt_cooldown_minutes", None)
1954
+ cooldown_s = SELF_INITIATED_COOLDOWN_S if cooldown_min is None else cooldown_min * 60
1955
+ # A launch that died BEFORE writing a run record is invisible to the
1956
+ # runs_per_week cap, so its cooldown attribution is the ONLY crash-loop
1957
+ # guard — it keeps a floor even when the contract dials cooldown to 0
1958
+ # (terra #172: zero cooldown otherwise resubmits a crashing launch
1959
+ # every tick, uncapped).
1960
+ dead_benches = set(dead_attempts or ())
1961
+ candidates = sorted(
1962
+ contract.benchmarks,
1963
+ key=lambda b: (last_attempt.get(b.name, 0.0), b.name),
1964
+ )
1965
+ for bench in candidates:
1966
+ floor_s = cooldown_s
1967
+ if bench.name in dead_benches:
1968
+ floor_s = max(cooldown_s, DEAD_LAUNCH_BACKOFF_S)
1969
+ if now - last_attempt.get(bench.name, 0.0) >= floor_s:
1970
+ return str(bench.name)
1971
+ return None
1972
+
1973
+
1974
+ def _tombstone_path(root: Path, target: str, benchmark: str) -> Path:
1975
+ safe = f"{target.replace('/', '__')}__{benchmark}"
1976
+ return root / "pending-dead" / (safe + ".json")
1977
+
1978
+
1979
+ def write_tombstone(root: Path, target: str, benchmark: str, submitted_at: float) -> None:
1980
+ """Per-benchmark crash memory: a launch died before writing a record, so
1981
+ nothing else (runs_per_week, cooldown-by-records) can see it. The
1982
+ tombstone persists independently of the live pending marker — a second
1983
+ benchmark's launch must not erase it (terra #172 r3)."""
1984
+ path = _tombstone_path(root, target, benchmark)
1985
+ try:
1986
+ path.parent.mkdir(parents=True, exist_ok=True)
1987
+ path.write_text(json.dumps({"submitted_at": submitted_at}))
1988
+ except OSError as exc:
1989
+ log.warning("tombstone write failed for %s/%s: %s", target, benchmark, exc)
1990
+
1991
+
1992
+ def read_tombstones(root: Path, target: str, contract: Any, now: float) -> dict[str, float]:
1993
+ """benchmark -> submitted_at for unserved crash tombstones; entries past
1994
+ their window (the larger of the crash floor and the contract cooldown)
1995
+ are pruned on read."""
1996
+ cooldown_min = getattr(contract.budgets, "attempt_cooldown_minutes", None)
1997
+ cooldown_s = SELF_INITIATED_COOLDOWN_S if cooldown_min is None else cooldown_min * 60
1998
+ window = max(DEAD_LAUNCH_BACKOFF_S, cooldown_s)
1999
+ out: dict[str, float] = {}
2000
+ prefix = target.replace("/", "__") + "__"
2001
+ dead_dir = root / "pending-dead"
2002
+ if not dead_dir.is_dir():
2003
+ return out
2004
+ for path in dead_dir.glob(prefix + "*.json"):
2005
+ bench = path.stem[len(prefix) :]
2006
+ try:
2007
+ submitted_at = float(json.loads(path.read_text())["submitted_at"])
2008
+ except (OSError, ValueError, KeyError, TypeError):
2009
+ with contextlib.suppress(OSError):
2010
+ path.unlink()
2011
+ continue
2012
+ if now - submitted_at > window:
2013
+ with contextlib.suppress(OSError):
2014
+ path.unlink() # backoff served
2015
+ continue
2016
+ out[bench] = submitted_at
2017
+ return out
2018
+
2019
+
2020
+ # WIDTH slots are the only suffixed marker names; the pattern also fences
2021
+ # list_pendings against a longer target that shares this one's file-name
2022
+ # prefix (org/foo vs org/foobar — "/" encodes as "__", so glob alone is
2023
+ # ambiguous)
2024
+ _SLOT_AGENT_RE = re.compile(r"agent-\d+")
2025
+
2026
+
2027
+ def _pending_path(root: Path, target: str, agent: str = "") -> Path:
2028
+ # agent "" is the legacy single-slot name, still read for back-compat
2029
+ # with a marker written before the width dial deployed. The slot
2030
+ # separator is "@" because it CANNOT appear in a GitHub owner/repo
2031
+ # name — any character legal in repo names ("_", ".", "-") would make
2032
+ # org/pilot's slot file collide with some other target's legacy file
2033
+ # (org/pilot__agent-01 is a valid repo).
2034
+ suffix = f"@{agent}" if agent else ""
2035
+ return root / "pending" / (target.replace("/", "__") + suffix + ".json")
2036
+
2037
+
2038
+ def read_pending(root: Path, target: str, agent: str = "") -> dict[str, Any] | None:
2039
+ """The submit-time marker for a climb whose run record may not exist yet."""
2040
+ path = _pending_path(root, target, agent)
2041
+ try:
2042
+ data = json.loads(path.read_text())
2043
+ except (OSError, ValueError):
2044
+ return None
2045
+ return data if isinstance(data, dict) and "submitted_at" in data else None
2046
+
2047
+
2048
+ def list_pendings(root: Path, target: str) -> list[tuple[str, dict[str, Any]]]:
2049
+ """(agent, marker) for every live pending marker of `target` — one per
2050
+ WIDTH slot, plus the legacy un-suffixed marker from a pre-width deploy
2051
+ (attributed to agent-01)."""
2052
+ out: list[tuple[str, dict[str, Any]]] = []
2053
+ stem = target.replace("/", "__")
2054
+ pending_dir = root / "pending"
2055
+ if not pending_dir.is_dir():
2056
+ return out
2057
+ for path in sorted(pending_dir.glob(stem + "*.json")):
2058
+ name = path.stem
2059
+ if name == stem:
2060
+ agent = ""
2061
+ elif name.startswith(stem + "@") and _SLOT_AGENT_RE.fullmatch(name[len(stem) + 1 :]):
2062
+ agent = name[len(stem) + 1 :]
2063
+ else:
2064
+ continue # a longer target sharing this prefix (org/foo vs org/foobar)
2065
+ try:
2066
+ data = json.loads(path.read_text())
2067
+ except (OSError, ValueError):
2068
+ continue
2069
+ if isinstance(data, dict) and "submitted_at" in data:
2070
+ out.append((agent or str(data.get("agent_id") or "agent-01"), data))
2071
+ return out
2072
+
2073
+
2074
+ def write_pending(
2075
+ root: Path, target: str, benchmark: str, job_id: str, now: float, agent: str = ""
2076
+ ) -> None:
2077
+ path = _pending_path(root, target, agent)
2078
+ path.parent.mkdir(parents=True, exist_ok=True)
2079
+ tmp = path.with_suffix(".tmp")
2080
+ tmp.write_text(
2081
+ json.dumps(
2082
+ {"benchmark": benchmark, "job_id": job_id, "submitted_at": now, "agent_id": agent}
2083
+ )
2084
+ )
2085
+ os.replace(tmp, path)
2086
+
2087
+
2088
+ def clear_pending(root: Path, target: str, agent: str = "") -> None:
2089
+ _pending_path(root, target, agent).unlink(missing_ok=True)
2090
+
2091
+
2092
+ def _attempt_width(contract: Any) -> int:
2093
+ width = getattr(getattr(contract, "budgets", None), "max_active_attempts", None)
2094
+ return int(width) if width else MAX_ACTIVE_RUNS_PER_TARGET
2095
+
2096
+
2097
+ def _free_agent_slot(occupied: set[str], width: int) -> str | None:
2098
+ for i in range(1, width + 1):
2099
+ agent = f"agent-{i:02d}"
2100
+ if agent not in occupied:
2101
+ return agent
2102
+ return None
2103
+
2104
+
2105
+ def _climb_panel_argv(spec: FollowupSpec) -> list[str]:
2106
+ """Panel args for a climb job; empty when the operator disabled the panel."""
2107
+ if not spec.panel.strip():
2108
+ return []
2109
+ argv = ["--panel", spec.panel]
2110
+ if spec.panel_key_file:
2111
+ argv += ["--panel-key-file", spec.panel_key_file]
2112
+ return argv
2113
+
2114
+
2115
+ def _author_config_error(spec: FollowupSpec) -> str:
2116
+ """Why the config-driven author would die at the climb's startup ("" when it
2117
+ won't), checked on the tick host BEFORE a claim/submit so a codex misconfig
2118
+ (e.g. AUTORESEARCH_AUTHOR_BACKEND=codex with no non-claude model) never
2119
+ strands a claimed intake issue. Reads the fleet author config from env — the
2120
+ same source the climb defaults from — and the image the tick already knows."""
2121
+ from outerloop.attempt import codex_author_config_error
2122
+
2123
+ backend = os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude"
2124
+ model = os.environ.get("AUTORESEARCH_AUTHOR_MODEL") or "claude-opus-5"
2125
+ return codex_author_config_error(backend, model, spec.image)
2126
+
2127
+
2128
+ def _panel_preflight_error(spec: FollowupSpec) -> str:
2129
+ """Why the climb would die at startup on this panel config ("" when it
2130
+ won't): the lens spec, then the key file — each checked with the climb's
2131
+ OWN rules (parse_lenses for the grammar and claude-only backend;
2132
+ FileTokenProvider for exists/mode-600/non-empty), so preflight and climb
2133
+ cannot disagree.
2134
+
2135
+ Preflighted BEFORE claiming or submitting: the climb CLI fails loudly,
2136
+ but by then an intake issue is already claimed — and pick_issue never
2137
+ reclaims — so the strand must be caught on the tick host, which shares
2138
+ the home filesystem the climb will read."""
2139
+ if not spec.panel.strip():
2140
+ return ""
2141
+ try:
2142
+ from outerloop.attempt import PANEL_KEY_DEFAULT, resolve_author_key_file
2143
+ from outerloop.github import FileTokenProvider
2144
+ from outerloop.panel import parse_lenses
2145
+
2146
+ try:
2147
+ lenses = parse_lenses(spec.panel)
2148
+ except ValueError as exc:
2149
+ return str(exc)
2150
+ # non-claude (shelled) lenses: mirror the climb's rules exactly, per
2151
+ # backend — image required, the judge's OWN key (set + absolute +
2152
+ # neither the author's nor the claude panel key + readable), and for
2153
+ # hermes its pinned clone.
2154
+ shelled = {
2155
+ "codex": "AUTORESEARCH_PANEL_CODEX_KEY_FILE",
2156
+ "hermes": "AUTORESEARCH_PANEL_HERMES_KEY_FILE",
2157
+ }
2158
+ for lens_backend, key_env in shelled.items():
2159
+ if not any(backend == lens_backend for _, backend, _ in lenses):
2160
+ continue
2161
+ if not spec.image or not Path(spec.image).is_file():
2162
+ return (
2163
+ f"a {lens_backend} panel lens requires a real container image "
2164
+ f"(AUTORESEARCH_IMAGE={spec.image!r})"
2165
+ )
2166
+ key_raw = os.environ.get(key_env, "").strip()
2167
+ if not key_raw:
2168
+ return (
2169
+ f"a {lens_backend} panel lens needs {key_env} "
2170
+ "(role separation: the judge's own key, never the author's)"
2171
+ )
2172
+ key_path = Path(key_raw).expanduser()
2173
+ if not key_path.is_absolute():
2174
+ return (
2175
+ f"{lens_backend} panel key path {key_path} is relative; only absolute paths fly"
2176
+ )
2177
+ author = Path(resolve_author_key_file("codex")).expanduser()
2178
+ if key_path.resolve() == author.resolve():
2179
+ return (
2180
+ f"{lens_backend} panel key file {key_path} is the codex author "
2181
+ "key (role separation: the judge needs its own key)"
2182
+ )
2183
+ claude_panel = Path(spec.panel_key_file or PANEL_KEY_DEFAULT).expanduser()
2184
+ if key_path.resolve() == claude_panel.resolve():
2185
+ return (
2186
+ f"{lens_backend} panel key file {key_path} is the claude panel "
2187
+ "key file (an anthropic key must never reach another "
2188
+ "provider's login)"
2189
+ )
2190
+ FileTokenProvider(key_path).token()
2191
+ if lens_backend == "hermes":
2192
+ repo = os.environ.get("REVIEW_HERMES_REPO", "").strip()
2193
+ # a REAL clone, not merely a directory: the harness executes
2194
+ # run_agent.py from it with the panel key, so an arbitrary or
2195
+ # empty path must fail here, never after a run is claimed
2196
+ if not repo or not (Path(repo).expanduser() / "run_agent.py").is_file():
2197
+ return (
2198
+ f"a hermes panel lens needs REVIEW_HERMES_REPO pointing at "
2199
+ f"the pinned clone (run_agent.py not found under {repo!r})"
2200
+ )
2201
+ from outerloop.role_runner import _HERMES_PROVIDERS
2202
+
2203
+ provider = os.environ.get("REVIEW_HERMES_PROVIDER", "").lower() or "openrouter"
2204
+ if provider not in _HERMES_PROVIDERS:
2205
+ return (
2206
+ f"unknown REVIEW_HERMES_PROVIDER {provider!r} "
2207
+ f"(have: {sorted(_HERMES_PROVIDERS)})"
2208
+ )
2209
+ if not any(backend == "claude" for _, backend, _ in lenses):
2210
+ return "" # codex-only panel: the claude key checks below don't apply
2211
+ path = Path(spec.panel_key_file or PANEL_KEY_DEFAULT).expanduser()
2212
+ if not path.is_absolute():
2213
+ # the climb runs from a flight directory, not the tick's cwd — a
2214
+ # relative path that resolves here could still miss there
2215
+ return f"panel key path {path} is relative; only absolute paths fly"
2216
+ # the AUTHOR key the climb will actually use resolves per the fleet backend
2217
+ # (claude vs codex keys coexist), config-driven like the climb itself — so
2218
+ # the role-separation check compares the panel key against the RIGHT author
2219
+ # key, and a codex run is never judged by a stray Claude key.
2220
+ fleet_backend = os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude"
2221
+ author = Path(resolve_author_key_file(fleet_backend))
2222
+ if not author.is_absolute():
2223
+ # same rule as the panel key: the climb resolves paths from a
2224
+ # flight directory, so a relative author path both misconfigures
2225
+ # the author AND defeats the role-separation comparison below
2226
+ return f"author key path {author} is relative; only absolute paths fly"
2227
+ if path.resolve() == author.resolve():
2228
+ return (
2229
+ f"panel key file {path} is the author key file "
2230
+ "(role separation: the verifier needs its own key)"
2231
+ )
2232
+ # ADC-only deployments (Vertex covering the claude panel) hold no
2233
+ # Anthropic key at all — the same tolerance role_key applies at run
2234
+ # time, so the preflight and the climb agree.
2235
+ from outerloop.role_runner import role_key
2236
+
2237
+ role_key(path)
2238
+ return ""
2239
+ except Exception as exc:
2240
+ # never raises: an unexpected failure (partial deploy, ELOOP, unset
2241
+ # HOME) must fail closed WITH the alarm, not abort the tick that
2242
+ # would have written it
2243
+ return f"{type(exc).__name__}: {exc}"
2244
+
2245
+
2246
+ def _attempt_job_minutes(spec: FollowupSpec, limits: EffectiveLimits) -> int:
2247
+ """The submitted climb walltime: contract budget + panel allowance,
2248
+ clamped at the partition cap. Warns when the cap cuts below the session
2249
+ budget — the self-deadline would then fire before the author's own
2250
+ clock, and that must be a visible operator choice, never a silent
2251
+ surprise."""
2252
+ from outerloop.limits import ATTEMPT_OVERHEAD_MINUTES
2253
+
2254
+ wanted = limits.attempt_job_minutes + _panel_job_minutes(spec, limits)
2255
+ job = min(wanted, spec.max_job_minutes)
2256
+ if job < wanted:
2257
+ log.info(
2258
+ "climb job clamped to %d min by the partition cap (worst case "
2259
+ "wanted %d); slow panel rounds fail safe via the self-deadline",
2260
+ job,
2261
+ wanted,
2262
+ )
2263
+ if job < limits.session_minutes + ATTEMPT_OVERHEAD_MINUTES:
2264
+ log.warning(
2265
+ "work-job cap %d min leaves no runway around the %d-min session "
2266
+ "(the orchestrator needs ~%d min); sessions or endings will be "
2267
+ "cut short by the self-deadline",
2268
+ job,
2269
+ limits.session_minutes,
2270
+ ATTEMPT_OVERHEAD_MINUTES,
2271
+ )
2272
+ return job
2273
+
2274
+
2275
+ def _panel_job_minutes(spec: FollowupSpec, limits: EffectiveLimits) -> int:
2276
+ """Extra walltime the panel needs, ADDED to the contract-clamped job
2277
+ budget: the contract's knobs cap the AUTHOR's spend and their ceilings
2278
+ deliberately cannot raise ours (limits.py), so the panel — the
2279
+ orchestrator's own gate, flipped on by the tick — brings its own time.
2280
+ Worst case: three sequential reads of every lens (initial, post-revision,
2281
+ merged-tree) on the judge budget, plus one revision wake on the session
2282
+ budget. The revision's re-measure rides the margin the self-deadline
2283
+ already fails safe on."""
2284
+ lenses = [entry for entry in spec.panel.split(",") if entry.strip()]
2285
+ if not lenses:
2286
+ return 0
2287
+ from outerloop.roles import reviewer_spec, verifier_spec
2288
+
2289
+ judge_minutes = max(reviewer_spec().budget.walltime_s, verifier_spec().budget.walltime_s) // 60
2290
+ return 3 * len(lenses) * judge_minutes + limits.session_minutes
2291
+
2292
+
2293
+ def _climb_limit_argv(limits: EffectiveLimits, job_minutes: int) -> list[str]:
2294
+ """Climb-CLI flags carrying the tick-resolved limits: the job's ACTUAL
2295
+ walltime rides along (contract budget + any panel allowance, clamped at
2296
+ the partition cap — exactly what the JobSpec gets) so the climb arms its
2297
+ self-deadline against the real clock (Slurm delivers no signals to our
2298
+ processes on Torch). The session shrinks to fit a
2299
+ CAPPED job with the same rule limits.effective_limits applies to
2300
+ contract values — better a short session that ends cleanly than a full
2301
+ one the self-deadline kills mid-flight."""
2302
+ from outerloop.limits import ATTEMPT_OVERHEAD_MINUTES, SESSION_MINUTES_FLOOR
2303
+
2304
+ session = min(limits.session_minutes, job_minutes - ATTEMPT_OVERHEAD_MINUTES)
2305
+ return [
2306
+ "--max-turns",
2307
+ str(limits.session_max_turns),
2308
+ "--session-minutes",
2309
+ str(max(SESSION_MINUTES_FLOOR, session)),
2310
+ "--job-minutes",
2311
+ str(job_minutes),
2312
+ ]
2313
+
2314
+
2315
+ def service_self_initiated(
2316
+ root: Path,
2317
+ compute: Compute,
2318
+ spec: FollowupSpec,
2319
+ contract: Any,
2320
+ now: float,
2321
+ limits: EffectiveLimits | None = None,
2322
+ dry_run: bool = False,
2323
+ ) -> tuple[str, str] | None:
2324
+ """The default background mode: when nothing else needs doing, climb the
2325
+ least-recently-attempted benchmark.
2326
+
2327
+ A pending marker written at submit time bridges the gap between
2328
+ `compute.submit` and the climb job writing its run record — without it,
2329
+ every tick during Slurm queue latency would launch a duplicate climb.
2330
+ """
2331
+ limits = limits if limits is not None else effective_limits(getattr(contract, "budgets", None))
2332
+ paused = outage_active(root, now, role="solver")
2333
+ if paused:
2334
+ log.info("self-initiated lane paused (api outage: %s)", paused)
2335
+ return None
2336
+ try:
2337
+ records = list_runs(root)
2338
+ width = _attempt_width(contract)
2339
+ # WIDTH: every live pending marker occupies a slot; landed ones
2340
+ # clear; dead ones become per-benchmark tombstones and free theirs.
2341
+ occupied: set[str] = set()
2342
+ live_pendings: list[tuple[str, float]] = []
2343
+ nonslot_busy = False
2344
+ for agent, pending in list_pendings(root, spec.target):
2345
+ marker_agent = "" if not pending.get("agent_id") else agent
2346
+ submitted_at = float(pending["submitted_at"])
2347
+ # A slotted marker lands only when ITS OWN record appears —
2348
+ # matching on target+time alone would let a sibling slot's
2349
+ # record clear a still-live marker (terra #173). A legacy
2350
+ # marker names no slot, so it keeps the lax match.
2351
+ landed = any(
2352
+ r.target == spec.target
2353
+ and r.created >= submitted_at - 60
2354
+ and (not marker_agent or r.agent_id == marker_agent)
2355
+ for r in records
2356
+ )
2357
+ expired = now - submitted_at > PENDING_TTL_S
2358
+ if landed:
2359
+ clear_pending(root, spec.target, marker_agent)
2360
+ elif (alive := _holder_alive(compute, str(pending.get("job_id", "")))) is True or (
2361
+ not expired and alive is not False
2362
+ ):
2363
+ # climb queued or starting; its record isn't written yet. A
2364
+ # provably-alive job holds its SLOT regardless of the
2365
+ # marker's TTL — queue wait can exceed it — the TTL only
2366
+ # breaks ties when Slurm can't say.
2367
+ occupied.add(agent)
2368
+ live_pendings.append((str(pending.get("benchmark", "")), submitted_at))
2369
+ if not marker_agent:
2370
+ # a live un-slotted marker is another lane's submit
2371
+ # (steward/intake, or a pre-width deploy): serial
2372
+ nonslot_busy = True
2373
+ else:
2374
+ # Died before writing a record: persist the crash memory as a
2375
+ # PER-BENCHMARK tombstone (a sibling launch must not erase
2376
+ # this — terra #172 r3), then free the slot.
2377
+ write_tombstone(root, spec.target, str(pending.get("benchmark", "")), submitted_at)
2378
+ clear_pending(root, spec.target, marker_agent)
2379
+ stranded_cutoff = now - STRANDED_IMPLEMENTING_S
2380
+ for r in records:
2381
+ if r.target == spec.target and r.state != ENDED:
2382
+ if r.state == IMPLEMENTING and max(r.updated, r.created) <= stranded_cutoff:
2383
+ continue # stranded: pick ignores it, so must occupancy
2384
+ occupied.add(r.agent_id)
2385
+ if not _SLOT_AGENT_RE.fullmatch(r.agent_id):
2386
+ nonslot_busy = True
2387
+ if nonslot_busy:
2388
+ # steward and intake keep their pre-width one-run-per-target
2389
+ # exclusivity: width applies AMONG self-initiated slots, it
2390
+ # does not license launching beside another lane (terra #173)
2391
+ return None
2392
+ if len(occupied) >= width:
2393
+ return None
2394
+ slot_agent = _free_agent_slot(occupied, width)
2395
+ if slot_agent is None:
2396
+ return None
2397
+ dead_attempts = read_tombstones(root, spec.target, contract, now)
2398
+ benchmark = pick_self_initiated(
2399
+ records, contract, spec.target, now, dead_attempts, live_pendings
2400
+ )
2401
+ if benchmark is None:
2402
+ return None
2403
+ if getattr(contract, "merge", "manual") == "auto" and not spec.panel:
2404
+ # auto merge mode means gate+PANEL clean self-merges; a
2405
+ # deployment with no panel configured must not launch attempts
2406
+ # that would publish panel-less self-merging PRs (terra #171)
2407
+ log.error(
2408
+ "attempt on %s not launched: contract sets merge:auto but "
2409
+ "no panel is configured (set AUTORESEARCH_PANEL, or the "
2410
+ "contract back to merge:manual)",
2411
+ benchmark,
2412
+ )
2413
+ return None
2414
+ if lane_error := _gpu_lane_error(contract, benchmark, spec):
2415
+ log.error("attempt on %s not launched: %s", benchmark, lane_error)
2416
+ return None
2417
+ author_error = _author_config_error(spec)
2418
+ if author_error:
2419
+ log.error(
2420
+ "climb on %s not launched: author misconfigured — %s "
2421
+ "(fix AUTORESEARCH_AUTHOR_BACKEND/_MODEL)",
2422
+ benchmark,
2423
+ author_error,
2424
+ )
2425
+ return None
2426
+ panel_error = _panel_preflight_error(spec)
2427
+ if panel_error:
2428
+ log.error(
2429
+ "climb on %s not launched: panel misconfigured — %s "
2430
+ "(fix it, or set AUTORESEARCH_PANEL='' to disable the panel)",
2431
+ benchmark,
2432
+ panel_error,
2433
+ )
2434
+ return None
2435
+ if dry_run:
2436
+ return (benchmark, "dry-run")
2437
+ job_minutes = _attempt_job_minutes(spec, limits)
2438
+ argv = [
2439
+ "uv",
2440
+ "run",
2441
+ "python",
2442
+ "-m",
2443
+ "outerloop.attempt",
2444
+ "--target",
2445
+ spec.target,
2446
+ "--benchmark",
2447
+ benchmark,
2448
+ "--run-root",
2449
+ str(spec.run_root),
2450
+ "--image",
2451
+ spec.image,
2452
+ "--agent-id",
2453
+ slot_agent,
2454
+ *_climb_limit_argv(limits, job_minutes),
2455
+ *_climb_panel_argv(spec),
2456
+ ]
2457
+ if spec.pat_file:
2458
+ argv += ["--pat-file", spec.pat_file]
2459
+ # config-driven author: climb resolves the author backend/model/key from
2460
+ # AUTORESEARCH_AUTHOR_* env (inherited by the job), so the tick threads
2461
+ # neither the backend nor its key — a new backend needs zero tick change.
2462
+ job_id = compute.submit(
2463
+ JobSpec(
2464
+ job_name=f"climb-{benchmark}-{slot_agent}"[:60],
2465
+ account=spec.account,
2466
+ partition=spec.job_partition or spec.partition,
2467
+ time_minutes=job_minutes,
2468
+ command=_flight_command(
2469
+ spec.home, f"climb-{benchmark}-{slot_agent}"[:60], now, argv
2470
+ ),
2471
+ cpus=4,
2472
+ mem="8G",
2473
+ )
2474
+ )
2475
+ write_pending(root, spec.target, benchmark, job_id, now, agent=slot_agent)
2476
+ log.info("self-initiated climb on %s: job %s", benchmark, job_id)
2477
+ return (benchmark, job_id)
2478
+ except Exception as exc: # one bad pass must not break the tick
2479
+ log.warning("self-initiated pass failed: %s", exc)
2480
+ return None
2481
+
2482
+
2483
+ def service_steward(
2484
+ root: Path,
2485
+ github: Any,
2486
+ compute: Compute,
2487
+ spec: FollowupSpec,
2488
+ now: float,
2489
+ contract: Any,
2490
+ limits: EffectiveLimits,
2491
+ dry_run: bool = False,
2492
+ ) -> tuple[str, str] | None:
2493
+ """The steward lane: claim at most ONE labeled work-order issue per tick
2494
+ and submit a stewardship job. Off until the operator provisions the
2495
+ steward's own key (role separation) and the contract declares a steward
2496
+ scope."""
2497
+ from outerloop.steward import pick_steward_issue
2498
+
2499
+ target = spec.target
2500
+ if not target or not spec.steward_key_file:
2501
+ return None
2502
+ if getattr(contract, "steward", None) is None:
2503
+ return None
2504
+ try:
2505
+ from outerloop.steward import release_orphaned_claims
2506
+
2507
+ # ONE active run per target covers stewardships too: an env rewrite
2508
+ # must not fly alongside a solver climb or another stewardship.
2509
+ records = list_runs(root)
2510
+ # reconcile first: killed jobs never post their own release — and
2511
+ # BEFORE the outage pause below, because a claim orphaned by the
2512
+ # very session the outage killed must not stay held all cooldown
2513
+ # (reconciliation is model-free bookkeeping; only spawning pauses)
2514
+ release_orphaned_claims(github, target, records, now, bot_login=spec.bot_login)
2515
+ paused = outage_active(root, now, role="steward")
2516
+ if paused:
2517
+ log.info("steward lane paused (api outage: %s)", paused)
2518
+ return None
2519
+ if any(r.target == target and r.state != ENDED for r in records):
2520
+ return None
2521
+ # The queue window (submit -> job writes its record) is bridged by
2522
+ # the SAME per-target pending markers the self-initiated lane uses
2523
+ # — ALL of them, slotted included: a width slot queued without a
2524
+ # record yet must block a stewardship the same way an active run
2525
+ # does. Liveness first, TTL only breaks unknown ties (queue wait
2526
+ # can outlive the TTL).
2527
+ for slot, pending in list_pendings(root, target):
2528
+ marker_agent = "" if not pending.get("agent_id") else slot
2529
+ submitted_at = float(pending.get("submitted_at", 0.0))
2530
+ landed = any(
2531
+ r.target == target
2532
+ and r.created >= submitted_at - 60
2533
+ and (not marker_agent or r.agent_id == marker_agent)
2534
+ for r in records
2535
+ )
2536
+ expired = now - submitted_at > PENDING_TTL_S
2537
+ alive = _holder_alive(compute, str(pending.get("job_id", "")))
2538
+ if not landed and (alive is True or (not expired and alive is not False)):
2539
+ return None
2540
+ task = pick_steward_issue(github, target, contract, spec.bot_login)
2541
+ if task is None:
2542
+ return None
2543
+ if _benchmark_gpus(contract, task.benchmark) > 0:
2544
+ # the stewardship validates its rewrite IN-JOB (SubprocessEvaluator
2545
+ # inside the CPU work job — no GPUs, no --nv), so a GPU benchmark
2546
+ # cannot be stewarded yet; refuse rather than launch a validation
2547
+ # that can only fail (terra #174 r2)
2548
+ log.error(
2549
+ "stewardship on %s not launched: GPU benchmarks validate in-job "
2550
+ "and the steward job has no GPU allocation",
2551
+ task.benchmark,
2552
+ )
2553
+ return None
2554
+ if dry_run:
2555
+ return (f"steward-issue-{task.number}", "dry-run")
2556
+ from outerloop.intake import CLAIM_MARKER, issue_hypothesis
2557
+
2558
+ github.comment(
2559
+ target,
2560
+ task.number,
2561
+ f"{CLAIM_MARKER}\nClaimed by the steward for benchmark "
2562
+ f"`{task.benchmark}`; a run is queued and a report will follow here.",
2563
+ )
2564
+ import base64 as _b64
2565
+
2566
+ work_order_b64 = _b64.b64encode(issue_hypothesis(task).encode()).decode()
2567
+ argv = [
2568
+ "uv",
2569
+ "run",
2570
+ "python",
2571
+ "-m",
2572
+ "outerloop.steward",
2573
+ "--target",
2574
+ target,
2575
+ "--benchmark",
2576
+ task.benchmark,
2577
+ "--run-root",
2578
+ str(spec.run_root),
2579
+ "--image",
2580
+ spec.image,
2581
+ "--issue",
2582
+ str(task.number),
2583
+ "--work-order-b64",
2584
+ work_order_b64,
2585
+ "--key-file",
2586
+ spec.steward_key_file,
2587
+ # the SAME clamped walltime the JobSpec requests, so the
2588
+ # self-deadline arms against the real clock
2589
+ *_climb_limit_argv(limits, min(limits.attempt_job_minutes, spec.max_job_minutes)),
2590
+ ]
2591
+ if spec.pat_file:
2592
+ argv += ["--pat-file", spec.pat_file]
2593
+ try:
2594
+ job_id = compute.submit(
2595
+ JobSpec(
2596
+ job_name=f"steward-issue-{task.number}",
2597
+ account=spec.account,
2598
+ partition=spec.job_partition or spec.partition,
2599
+ time_minutes=min(limits.attempt_job_minutes, spec.max_job_minutes),
2600
+ command=_flight_command(spec.home, f"steward-issue-{task.number}", now, argv),
2601
+ cpus=4,
2602
+ mem="8G",
2603
+ )
2604
+ )
2605
+ except Exception:
2606
+ # release the claim: a claim with no job behind it would orphan
2607
+ # the work order forever (pick skips claimed issues)
2608
+ from outerloop.steward import RELEASE_MARKER
2609
+
2610
+ with contextlib.suppress(Exception):
2611
+ github.comment(
2612
+ target,
2613
+ task.number,
2614
+ f"{RELEASE_MARKER}\nSubmission failed; claim released — "
2615
+ f"a later tick will retry this work order.",
2616
+ )
2617
+ raise
2618
+ write_pending(root, target, f"steward:{task.benchmark}", job_id, now)
2619
+ log.info("steward issue #%s claimed for job %s", task.number, job_id)
2620
+ return (f"steward-issue-{task.number}", job_id)
2621
+ except Exception as exc: # the steward lane must not break the tick
2622
+ log.warning("steward pass failed: %s", exc)
2623
+ return None
2624
+
2625
+
2626
+ def service_intake(
2627
+ root: Path,
2628
+ github: Any,
2629
+ compute: Compute,
2630
+ spec: FollowupSpec,
2631
+ now: float,
2632
+ contract: Any = None,
2633
+ limits: EffectiveLimits | None = None,
2634
+ dry_run: bool = False,
2635
+ ) -> tuple[str, str] | None:
2636
+ """The requested lane: claim at most ONE qualifying issue per tick and
2637
+ submit a climb job for it. The claim comment (posted by the climb job
2638
+ before its session) marks an issue taken; one-per-tick keeps a burst of
2639
+ issues from bursting the budget. The contract arrives from the tick's
2640
+ single per-target fetch; None (fetch failed) sits the lane out."""
2641
+ from outerloop.contract import load_contract
2642
+ from outerloop.intake import issue_hypothesis, pick_issue
2643
+
2644
+ target = spec.target
2645
+ if not target:
2646
+ return None
2647
+ paused = outage_active(root, now, role="solver")
2648
+ if paused:
2649
+ log.info("intake lane paused (api outage: %s)", paused)
2650
+ return None
2651
+ try:
2652
+ if contract is None:
2653
+ contract_raw = _contract_text(github, target, "main")
2654
+ if contract_raw is None:
2655
+ return None
2656
+ contract = load_contract(contract_raw, target)
2657
+ limits = limits if limits is not None else effective_limits(contract.budgets)
2658
+ task = pick_issue(github, target, contract, spec.bot_login)
2659
+ if task is None:
2660
+ return None
2661
+ if getattr(contract, "merge", "manual") == "auto" and not spec.panel:
2662
+ # auto merge mode means gate+PANEL clean self-merges; a
2663
+ # deployment with no panel configured must not launch attempts
2664
+ # that would publish panel-less self-merging PRs (terra #171)
2665
+ log.error(
2666
+ "attempt on %s not launched: contract sets merge:auto but "
2667
+ "no panel is configured (set AUTORESEARCH_PANEL, or the "
2668
+ "contract back to merge:manual)",
2669
+ task.benchmark,
2670
+ )
2671
+ return None
2672
+ if lane_error := _gpu_lane_error(contract, task.benchmark, spec):
2673
+ log.error("attempt on %s not launched: %s", task.benchmark, lane_error)
2674
+ return None
2675
+ author_error = _author_config_error(spec)
2676
+ if author_error:
2677
+ log.error(
2678
+ "issue #%d not claimed: author misconfigured — %s "
2679
+ "(fix AUTORESEARCH_AUTHOR_BACKEND/_MODEL)",
2680
+ task.number,
2681
+ author_error,
2682
+ )
2683
+ return None
2684
+ panel_error = _panel_preflight_error(spec)
2685
+ if panel_error:
2686
+ log.error(
2687
+ "issue #%d not claimed: panel misconfigured — %s "
2688
+ "(fix it, or set AUTORESEARCH_PANEL='' to disable the panel)",
2689
+ task.number,
2690
+ panel_error,
2691
+ )
2692
+ return None
2693
+ if dry_run:
2694
+ return (f"issue-{task.number}", "dry-run")
2695
+ job_minutes = _attempt_job_minutes(spec, limits)
2696
+ # claim BEFORE submit: Slurm queueing can take minutes, and the next
2697
+ # tick must not re-claim the same issue in that window
2698
+ from outerloop.intake import CLAIM_MARKER, MAX_INTAKE_ATTEMPTS, RELEASE_MARKER
2699
+
2700
+ github.comment(
2701
+ target,
2702
+ task.number,
2703
+ f"{CLAIM_MARKER}\nClaimed for benchmark `{task.benchmark}`; a run "
2704
+ "is queued and a report will follow here.",
2705
+ )
2706
+ import base64 as _b64
2707
+
2708
+ hypothesis_b64 = _b64.b64encode(issue_hypothesis(task).encode()).decode()
2709
+ argv = [
2710
+ "uv",
2711
+ "run",
2712
+ "python",
2713
+ "-m",
2714
+ "outerloop.attempt",
2715
+ "--target",
2716
+ target,
2717
+ "--benchmark",
2718
+ task.benchmark,
2719
+ "--run-root",
2720
+ str(spec.run_root),
2721
+ "--image",
2722
+ spec.image,
2723
+ "--issue",
2724
+ str(task.number),
2725
+ "--hypothesis-b64",
2726
+ hypothesis_b64,
2727
+ *_climb_limit_argv(limits, job_minutes),
2728
+ *_climb_panel_argv(spec),
2729
+ ]
2730
+ if spec.pat_file:
2731
+ argv += ["--pat-file", spec.pat_file]
2732
+ # config-driven author: climb resolves the author key from the
2733
+ # AUTORESEARCH_AUTHOR_* env by backend; the tick does not thread it.
2734
+ try:
2735
+ job_id = compute.submit(
2736
+ JobSpec(
2737
+ job_name=f"climb-issue-{task.number}",
2738
+ account=spec.account,
2739
+ partition=spec.job_partition or spec.partition,
2740
+ time_minutes=job_minutes,
2741
+ command=_flight_command(spec.home, f"climb-issue-{task.number}", now, argv),
2742
+ cpus=4,
2743
+ mem="8G",
2744
+ )
2745
+ )
2746
+ except Exception:
2747
+ # the claim is already posted and pick_issue skips claimed
2748
+ # issues, so a failed submit must release it (same pattern as
2749
+ # the steward lane) or the issue is stranded forever
2750
+ with contextlib.suppress(Exception):
2751
+ github.comment(
2752
+ target,
2753
+ task.number,
2754
+ f"{RELEASE_MARKER}\nSubmission failed; claim released — "
2755
+ f"a later tick will retry this issue (intake gives up "
2756
+ f"after {MAX_INTAKE_ATTEMPTS} claim attempts and leaves "
2757
+ f"it for a human).",
2758
+ )
2759
+ raise
2760
+ log.info("issue #%s claimed for climb job %s", task.number, job_id)
2761
+ return (f"issue-{task.number}", job_id)
2762
+ except Exception as exc: # intake must not break the tick
2763
+ log.warning("intake pass failed: %s", exc)
2764
+ return None
2765
+
2766
+
2767
+ @dataclass
2768
+ class LoggingDispatcher:
2769
+ """Never dispatched in production: main() runs the sweep dry unless
2770
+ dispatched wake is armed, so no lease is taken and no attempt is
2771
+ counted. This exists for the seam."""
2772
+
2773
+ def dispatch(self, record: RunRecord, reason: str) -> str:
2774
+ log.info("WOULD WAKE %s (%s) — session dispatch lands in phase 5", record.run_id, reason)
2775
+ return ""
2776
+
2777
+
2778
+ @dataclass
2779
+ class JobWakeDispatcher:
2780
+ """Delivers a wake by submitting a Slurm job that runs the wake CLI
2781
+ (`climb --resume <run_id>`), depending on the run's eval jobs (the record's
2782
+ `afterany`) so it fires when they finish — or immediately if they already
2783
+ have. CPU-only and short: a wake reads cached results and opens a PR, it
2784
+ never holds a GPU. Returns the wake job id (async: it owns the lease until
2785
+ it completes)."""
2786
+
2787
+ compute: Compute
2788
+ spec: FollowupSpec
2789
+ now: float
2790
+ wake_minutes: int = 20
2791
+
2792
+ def dispatch(self, record: RunRecord, reason: str) -> str:
2793
+ argv = [
2794
+ "uv",
2795
+ "run",
2796
+ "python",
2797
+ "-m",
2798
+ "outerloop.attempt",
2799
+ "--resume",
2800
+ record.run_id,
2801
+ "--run-root",
2802
+ str(self.spec.run_root),
2803
+ "--image",
2804
+ self.spec.image,
2805
+ "--account",
2806
+ self.spec.account,
2807
+ "--partition",
2808
+ self.spec.partition,
2809
+ "--gpu-partition",
2810
+ self.spec.gpu_partition,
2811
+ "--gpu-account",
2812
+ self.spec.gpu_account,
2813
+ # the wake runs the SAME verification panel as the fresh climb, so a
2814
+ # dispatched improvement is verified before it is published.
2815
+ *_climb_panel_argv(self.spec),
2816
+ # session budget for the depth-axis REVISION (a blocking panel
2817
+ # finding wakes the author to revise).
2818
+ "--max-turns",
2819
+ str(self.spec.max_turns),
2820
+ ]
2821
+ # An AUTHOR-SLEEP wake resumes a FULL author session (not the short
2822
+ # read-decide a candidate wake runs), so the Slurm job must fit that
2823
+ # session or walltime kills the resumed session mid-run and the run just
2824
+ # waits for another wake. Size the job to the session
2825
+ # budget + overhead and pass --session-minutes so the in-job
2826
+ # self-deadline fires BEFORE Slurm's walltime. A candidate wake keeps
2827
+ # the short budget (read results + panel).
2828
+ from outerloop.limits import ATTEMPT_OVERHEAD_MINUTES
2829
+ from outerloop.roles import author_spec
2830
+
2831
+ if record.stage.get("phase") == "author-sleep":
2832
+ session_minutes = author_spec().budget.walltime_s // 60
2833
+ argv += ["--session-minutes", str(session_minutes)]
2834
+ job_minutes = min(session_minutes + ATTEMPT_OVERHEAD_MINUTES, self.spec.max_job_minutes)
2835
+ else:
2836
+ job_minutes = self.wake_minutes + _wake_panel_minutes(self.spec)
2837
+ if self.spec.pat_file:
2838
+ argv += ["--pat-file", self.spec.pat_file]
2839
+ # config-driven author: `climb --resume` resolves the author key from the
2840
+ # PARKED RUN's backend (persisted on its record) inside climb.main — the
2841
+ # tick does not thread the key, so a fleet flip picks the right one.
2842
+ name = f"wake-{record.run_id}"[:60]
2843
+ afterany = str(record.stage.get("afterany", ""))
2844
+ return self.compute.submit(
2845
+ JobSpec(
2846
+ job_name=name,
2847
+ account=self.spec.account,
2848
+ partition=self.spec.job_partition or self.spec.partition,
2849
+ time_minutes=job_minutes,
2850
+ command=_flight_command(self.spec.home, name, self.now, argv),
2851
+ dependency=afterany,
2852
+ cpus=2,
2853
+ mem="4G",
2854
+ )
2855
+ )
2856
+
2857
+
2858
+ def _wake_panel_minutes(spec: FollowupSpec) -> int:
2859
+ """Extra wake walltime for the verification panel it now runs — the base
2860
+ `wake_minutes` covers only reading results + opening the PR. Budgeted for
2861
+ the worst case a single wake reaches: one read per lens PLUS one revision
2862
+ author session (the depth-axis wake-to-revise). The revision's re-measure
2863
+ only DISPATCHES (then the job ends, parked), so it needs no extra time.
2864
+ Grounded in the same judge/author budgets the climb job uses."""
2865
+ from outerloop.panel import panel_read_minutes
2866
+ from outerloop.roles import author_spec
2867
+
2868
+ read_minutes = panel_read_minutes(spec.panel)
2869
+ if not read_minutes:
2870
+ return 0
2871
+ return read_minutes + author_spec().budget.walltime_s // 60
2872
+
2873
+
2874
+ def _wake_dispatcher_from_env(
2875
+ compute: Compute, followup_spec: FollowupSpec | None, now: float, root: Path
2876
+ ) -> tuple[WakeDispatcher, bool]:
2877
+ """The wake delivery for this tick, behind an EXPLICIT on-switch so the
2878
+ dispatched-wake path lands DARK. Returns `(dispatcher, live)`:
2879
+
2880
+ * armed (the `AUTORESEARCH_DISPATCH_WAKE` env var OR a `<root>/DISPATCH_WAKE`
2881
+ sentinel file) AND the chain env carries what a wake job needs -> the real
2882
+ `JobWakeDispatcher` and a LIVE sweep;
2883
+ * otherwise -> the `LoggingDispatcher` and a DRY sweep.
2884
+
2885
+ The sentinel mirrors PAUSE: an operator arms/disarms with a touch/rm, no
2886
+ chain restart. So dispatched climbing is turned on deliberately, and a
2887
+ half-configured environment fails safe to dry rather than to a wake job
2888
+ that cannot run."""
2889
+ if not dispatch_wake_armed(root):
2890
+ return LoggingDispatcher(), False
2891
+ if followup_spec is None:
2892
+ log.warning("dispatch-wake armed but the chain env is incomplete; wake stays dry")
2893
+ return LoggingDispatcher(), False
2894
+ log.info("dispatched-wake ON: the waiting-run sweep delivers real wakes this tick")
2895
+ return JobWakeDispatcher(compute, followup_spec, now), True
2896
+
2897
+
2898
+ def _max_job_minutes_from_env() -> int:
2899
+ """AUTORESEARCH_MAX_JOB_MINUTES, clamped into what the code can honor:
2900
+ at least the climb-job floor (an operator on a short-MaxTime partition
2901
+ must be able to LOWER the cap below cpu_short's 6h, or every submit is
2902
+ rejected), at most the ceiling the stranded window allows. A clamped
2903
+ value logs — a silently-changed cap would read as the partition
2904
+ rejecting jobs for no reason."""
2905
+ from outerloop.limits import ATTEMPT_JOB_MINUTES_FLOOR
2906
+
2907
+ raw = os.environ.get("AUTORESEARCH_MAX_JOB_MINUTES", "").strip()
2908
+ if not raw:
2909
+ return MAX_ATTEMPT_JOB_MINUTES
2910
+ try:
2911
+ value = int(raw)
2912
+ except ValueError:
2913
+ log.warning("AUTORESEARCH_MAX_JOB_MINUTES=%r is not an integer; using default", raw)
2914
+ return MAX_ATTEMPT_JOB_MINUTES
2915
+ clamped = max(ATTEMPT_JOB_MINUTES_FLOOR, min(value, MAX_JOB_MINUTES_CEILING))
2916
+ if clamped != value:
2917
+ log.warning("AUTORESEARCH_MAX_JOB_MINUTES=%d clamped to %d", value, clamped)
2918
+ return clamped
2919
+
2920
+
2921
+ def _cadence_s() -> float:
2922
+ """The chain's tick cadence in seconds (AUTORESEARCH_CADENCE_MIN, the same
2923
+ knob tick_chain.sbatch uses), defaulting to 30 min when unset/invalid."""
2924
+ raw = os.environ.get("AUTORESEARCH_CADENCE_MIN", "").strip()
2925
+ try:
2926
+ cadence_s = float(raw) * 60 if raw else 30 * 60
2927
+ except ValueError:
2928
+ cadence_s = 30 * 60
2929
+ return cadence_s if (math.isfinite(cadence_s) and cadence_s > 0) else 30 * 60
2930
+
2931
+
2932
+ def _coalesce_ceiling_s() -> float:
2933
+ """The largest SAFE coalesce window, bounding both the default and an
2934
+ explicit AUTORESEARCH_MIN_TICK_MINUTES: half the cadence (so an on-cadence
2935
+ tick is never coalesced even when the previous one ran a little late), and
2936
+ never above the absolute MAX_MIN_TICK_S. A window at/above the cadence would
2937
+ swallow every normal tick and stall the loop — this is what forbids it."""
2938
+ return min(float(MAX_MIN_TICK_S), _cadence_s() / 2)
2939
+
2940
+
2941
+ def _default_min_tick_s() -> float:
2942
+ """The coalesce window when none is set: the safe ceiling, further capped at
2943
+ the 10-min DEFAULT_MIN_TICK_S — small enough to only catch pile-ups, and
2944
+ cadence-aware so a short cadence scales it down instead of swallowing every
2945
+ tick."""
2946
+ return min(DEFAULT_MIN_TICK_S, _coalesce_ceiling_s())
2947
+
2948
+
2949
+ def _min_tick_s_from_env() -> float:
2950
+ """AUTORESEARCH_MIN_TICK_MINUTES -> the coalesce window in seconds. Unset
2951
+ derives a cadence-aware default; non-numeric/non-finite also fall back to it;
2952
+ negative clamps to 0 (coalesce disabled); a value at/above the safe ceiling
2953
+ (half the cadence, capped at MAX_MIN_TICK_S) clamps down so it cannot stall
2954
+ the loop."""
2955
+ raw = os.environ.get("AUTORESEARCH_MIN_TICK_MINUTES", "").strip()
2956
+ if not raw:
2957
+ return _default_min_tick_s()
2958
+ try:
2959
+ minutes = float(raw)
2960
+ except ValueError:
2961
+ log.warning("AUTORESEARCH_MIN_TICK_MINUTES=%r is not a number; using default", raw)
2962
+ return _default_min_tick_s()
2963
+ # reject inf/nan: an infinite window would coalesce every future tick and
2964
+ # freeze the loop (a finite elapsed time is always < inf)
2965
+ if not math.isfinite(minutes):
2966
+ log.warning("AUTORESEARCH_MIN_TICK_MINUTES=%r is not finite; using default", raw)
2967
+ return _default_min_tick_s()
2968
+ seconds = max(0.0, minutes * 60)
2969
+ ceiling = _coalesce_ceiling_s()
2970
+ if seconds > ceiling:
2971
+ log.warning(
2972
+ "AUTORESEARCH_MIN_TICK_MINUTES=%s exceeds the safe ceiling "
2973
+ "(%.0f min, ~half the cadence); clamping so normal ticks are not coalesced",
2974
+ raw,
2975
+ ceiling / 60,
2976
+ )
2977
+ return ceiling
2978
+ return seconds
2979
+
2980
+
2981
+ def _followup_spec_from_env(root: Path) -> tuple[Any, FollowupSpec | None]:
2982
+ """GitHub client + FollowupSpec from the chain environment, or Nones when
2983
+ the environment is incomplete (the tick then runs without in-review
2984
+ servicing, and logs what is absent)."""
2985
+ pat_file = os.environ.get("AUTORESEARCH_PAT_FILE", "")
2986
+ app_file = os.environ.get("AUTORESEARCH_GITHUB_APP_FILE", "")
2987
+ account = os.environ.get("AUTORESEARCH_ACCOUNT", "")
2988
+ partition = os.environ.get("AUTORESEARCH_PARTITION", "")
2989
+ image = os.environ.get(
2990
+ "AUTORESEARCH_IMAGE",
2991
+ os.path.expanduser("~/autoresearch-images/agent-py312.sif"),
2992
+ )
2993
+ home = os.environ.get("AUTORESEARCH_HOME", "")
2994
+ # Local compute runs jobs as subprocesses: Slurm placement is meaningless
2995
+ # there, so account/partition are only required when a cluster is in play.
2996
+ placed = bool(account and partition) or local_mode()
2997
+ if (pat_file or app_file) and placed and home and Path(image).is_file():
2998
+ from outerloop.appauth import resolve_bot_auth
2999
+ from outerloop.github import GitHubClient
3000
+
3001
+ try:
3002
+ github = GitHubClient(auth=resolve_bot_auth(pat_file, app_file))
3003
+ followup_spec = FollowupSpec(
3004
+ account=account,
3005
+ partition=partition,
3006
+ run_root=root,
3007
+ image=image,
3008
+ home=Path(home),
3009
+ pat_file=pat_file,
3010
+ github_app_file=app_file,
3011
+ target=os.environ.get(
3012
+ "AUTORESEARCH_TARGET", "agentic-learning-ai-lab/autoresearch-pilot"
3013
+ ),
3014
+ steward_key_file=os.environ.get("AUTORESEARCH_STEWARD_KEY_FILE", ""),
3015
+ panel=os.environ.get("AUTORESEARCH_PANEL", "verify,review"),
3016
+ panel_key_file=os.environ.get("AUTORESEARCH_PANEL_KEY_FILE", ""),
3017
+ job_partition=os.environ.get("AUTORESEARCH_JOB_PARTITION", ""),
3018
+ gpu_partition=os.environ.get("AUTORESEARCH_GPU_PARTITION", ""),
3019
+ gpu_account=os.environ.get("AUTORESEARCH_GPU_ACCOUNT", ""),
3020
+ max_job_minutes=_max_job_minutes_from_env(),
3021
+ )
3022
+ return github, followup_spec
3023
+ except Exception as exc:
3024
+ log.warning("in-review servicing disabled: %s", exc)
3025
+ return None, None
3026
+ absent = [
3027
+ name
3028
+ for name, value in [
3029
+ ("AUTORESEARCH_PAT_FILE or _GITHUB_APP_FILE", pat_file or app_file),
3030
+ ("AUTORESEARCH_ACCOUNT", account or ("-" if local_mode() else "")),
3031
+ ("AUTORESEARCH_PARTITION", partition or ("-" if local_mode() else "")),
3032
+ ("AUTORESEARCH_HOME", home),
3033
+ ]
3034
+ if not value
3035
+ ]
3036
+ if not Path(image).is_file():
3037
+ absent.append(f"image:{image}")
3038
+ log.info("in-review servicing disabled (missing: %s)", ", ".join(absent))
3039
+ return None, None
3040
+
3041
+
3042
+ def _loop_cadence_s(cadence_min: float) -> float:
3043
+ """The --loop sleep, clamped to [60s, 24h]: argparse accepts inf (which
3044
+ would OverflowError out of time.sleep) and sub-minute values would spin.
3045
+ A non-positive argument defers to AUTORESEARCH_CADENCE_MIN."""
3046
+ return min(24 * 3600.0, max(60.0, cadence_min * 60 if cadence_min > 0 else _cadence_s()))
3047
+
3048
+
3049
+ def main() -> int:
3050
+ import argparse
3051
+ import time
3052
+
3053
+ parser = argparse.ArgumentParser(description="One tick of the autoresearch loop.")
3054
+ parser.add_argument("--root", required=True, type=Path, help="state root on the shared FS")
3055
+ parser.add_argument("--grace-s", type=float, default=DEFAULT_GRACE_S)
3056
+ parser.add_argument("--lease-ttl-s", type=float, default=DEFAULT_LEASE_TTL_S)
3057
+ parser.add_argument(
3058
+ "--min-free-gb",
3059
+ type=float,
3060
+ default=DEFAULT_MIN_FREE_BYTES / 1024**3,
3061
+ help="skip launching new work when the state filesystem has less free",
3062
+ )
3063
+ parser.add_argument(
3064
+ "--loop",
3065
+ action="store_true",
3066
+ help="run a tick every cadence in the foreground — the local-mode "
3067
+ "chain (Slurm deployments use tick_chain.sbatch instead)",
3068
+ )
3069
+ parser.add_argument(
3070
+ "--cadence-min",
3071
+ type=float,
3072
+ default=0.0,
3073
+ help="minutes between --loop ticks; unset defers to "
3074
+ "AUTORESEARCH_CADENCE_MIN via the chain's own parser (default 30)",
3075
+ )
3076
+ args = parser.parse_args()
3077
+ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
3078
+
3079
+ args.root.mkdir(parents=True, exist_ok=True)
3080
+ # The tick's --root is the authority; children (and this process's own
3081
+ # LocalCompute) read AUTORESEARCH_ROOT, so a bare `tick --loop --root X`
3082
+ # must not split-brain them: local job states would land nowhere and
3083
+ # every finished job would read GONE until the park deadline.
3084
+ # RESOLVED: local jobs cd into flight checkouts, so a relative root
3085
+ # would scatter their state dirs across working directories
3086
+ if os.environ.get("AUTORESEARCH_ROOT", "") != str(args.root.resolve()):
3087
+ os.environ["AUTORESEARCH_ROOT"] = str(args.root.resolve())
3088
+ # In-review servicing is LIVE when credentials + image are available in the
3089
+ # chain environment. The waiting-run sweep delivers real wakes only when the
3090
+ # operator arms it — the AUTORESEARCH_DISPATCH_WAKE env var or a
3091
+ # <root>/DISPATCH_WAKE sentinel — and the env is complete; by default it
3092
+ # stays dry with the LoggingDispatcher — dispatched climbing lands DARK.
3093
+ # ONE compute for the process: LocalCompute remembers its jobs' states
3094
+ # in memory, so a --loop deployment must not discard them between ticks.
3095
+ compute = compute_from_env()
3096
+
3097
+ def run_once() -> None:
3098
+ github, followup_spec = _followup_spec_from_env(args.root)
3099
+ now = time.time()
3100
+ dispatcher, wake_live = _wake_dispatcher_from_env(compute, followup_spec, now, args.root)
3101
+ # parks arm their own wake from this recipe; without it the sweep delivers.
3102
+ # Local compute never arms: jobs are synchronous, so an afterany wake's
3103
+ # dependencies are terminal before submit returns — the next loop
3104
+ # iteration's sweep delivers every wake instead (wake latency = cadence).
3105
+ if wake_live and followup_spec is not None and not isinstance(compute, LocalCompute):
3106
+ write_wake_spec(args.root, followup_spec)
3107
+ else:
3108
+ remove_wake_spec(args.root)
3109
+
3110
+ report = tick(
3111
+ args.root,
3112
+ compute,
3113
+ dispatcher,
3114
+ now=now,
3115
+ grace_s=args.grace_s,
3116
+ lease_ttl_s=args.lease_ttl_s,
3117
+ dry_run=not wake_live,
3118
+ github=github,
3119
+ followup_spec=followup_spec,
3120
+ followup_dry_run=False,
3121
+ min_free_bytes=int(args.min_free_gb * 1024**3),
3122
+ min_tick_s=_min_tick_s_from_env(),
3123
+ )
3124
+ # Stamp the coalesce marker at REAL completion time (a fresh time.time(),
3125
+ # not the start-of-tick `now`), so a long tick does not leave a stale marker.
3126
+ mark_tick_complete(args.root, report, time.time())
3127
+ log.info(
3128
+ "tick done: paused=%s coalesced=%s swept=%d woken=%d deferred=%d reaped=%d stuck=%d "
3129
+ "impl_ended=%s review_ended=%s followups=%s intake=%s self_initiated=%s steward=%s "
3130
+ "disk=%s launch_blocked=%s shed=%d",
3131
+ report.paused,
3132
+ report.coalesced,
3133
+ report.swept,
3134
+ len(report.woken),
3135
+ len(report.deferred),
3136
+ len(report.reaped_leases),
3137
+ len(report.stuck),
3138
+ report.implementing_ended or "-",
3139
+ report.review_ended,
3140
+ report.followups_submitted,
3141
+ report.intake,
3142
+ report.self_initiated,
3143
+ report.steward,
3144
+ report.disk or "ok",
3145
+ report.launch_blocked,
3146
+ len(report.shed),
3147
+ )
3148
+
3149
+ if not args.loop:
3150
+ run_once()
3151
+ return 0
3152
+ # The local-mode chain: same stateless tick, a foreground loop instead of
3153
+ # sbatch successors. Records on disk carry all state, so killing and
3154
+ # restarting the loop resumes exactly like the Slurm chain would.
3155
+ cadence_s = _loop_cadence_s(args.cadence_min)
3156
+ while True:
3157
+ started = time.time()
3158
+ try:
3159
+ run_once()
3160
+ except Exception:
3161
+ log.exception("tick failed; the loop continues")
3162
+ time.sleep(max(0.0, cadence_s - (time.time() - started)))
3163
+
3164
+
3165
+ if __name__ == "__main__":
3166
+ raise SystemExit(main())