outerloop-science 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +18 -0
- outerloop/__main__.py +3 -0
- outerloop/appauth.py +213 -0
- outerloop/appmanifest.py +198 -0
- outerloop/attempt.py +3481 -0
- outerloop/brief.py +515 -0
- outerloop/cli.py +439 -0
- outerloop/climbboard.py +1145 -0
- outerloop/compute.py +482 -0
- outerloop/contract.py +483 -0
- outerloop/contract_cli.py +63 -0
- outerloop/disk.py +164 -0
- outerloop/dispatch.py +586 -0
- outerloop/followup.py +2143 -0
- outerloop/github.py +1486 -0
- outerloop/harness.py +1449 -0
- outerloop/housekeeping.py +167 -0
- outerloop/init.py +313 -0
- outerloop/intake.py +129 -0
- outerloop/limits.py +80 -0
- outerloop/markers.py +48 -0
- outerloop/measure.py +523 -0
- outerloop/orchestrator.py +1901 -0
- outerloop/panel.py +188 -0
- outerloop/paths.py +27 -0
- outerloop/posting.py +160 -0
- outerloop/progress.py +170 -0
- outerloop/py.typed +0 -0
- outerloop/review.py +611 -0
- outerloop/review_agent.py +263 -0
- outerloop/review_agent_cli.py +209 -0
- outerloop/review_post_cli.py +162 -0
- outerloop/review_summarize_cli.py +163 -0
- outerloop/role_runner.py +229 -0
- outerloop/roles.py +247 -0
- outerloop/rolespec.py +89 -0
- outerloop/runstate.py +385 -0
- outerloop/steward.py +852 -0
- outerloop/style.py +12 -0
- outerloop/syscall.py +977 -0
- outerloop/syscall_cli.py +531 -0
- outerloop/tick.py +3166 -0
- outerloop/verifier.py +403 -0
- outerloop/verify_agent.py +149 -0
- outerloop/verify_agent_cli.py +95 -0
- outerloop/verify_post_cli.py +116 -0
- outerloop_science-0.1.0.dev0.dist-info/METADATA +145 -0
- outerloop_science-0.1.0.dev0.dist-info/RECORD +52 -0
- outerloop_science-0.1.0.dev0.dist-info/WHEEL +4 -0
- outerloop_science-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- outerloop_science-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
- outerloop_science-0.1.0.dev0.dist-info/licenses/NOTICE +5 -0
outerloop/tick.py
ADDED
|
@@ -0,0 +1,3166 @@
|
|
|
1
|
+
"""One tick of the loop: sentinel, heartbeat, and the fail-safe sweep.
|
|
2
|
+
|
|
3
|
+
The tick is stateless and bounded — everything durable lives in run-state
|
|
4
|
+
files (`runstate`) and Slurm. It implements the backup layers of the wake
|
|
5
|
+
design (docs/design/architecture.md, "Wake delivery and fail-safety"); the
|
|
6
|
+
primary layer (the afterany dependency job) is submitted by whoever launches
|
|
7
|
+
an experiment and needs no help from here.
|
|
8
|
+
|
|
9
|
+
Wake *delivery* is behind a seam (`WakeDispatcher`) so this module stays
|
|
10
|
+
testable and the actual session dispatch (harness + brief) can evolve
|
|
11
|
+
independently.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import contextlib
|
|
17
|
+
import json
|
|
18
|
+
import logging
|
|
19
|
+
import math
|
|
20
|
+
import os
|
|
21
|
+
import re
|
|
22
|
+
import socket
|
|
23
|
+
import subprocess
|
|
24
|
+
from collections.abc import Sequence
|
|
25
|
+
from dataclasses import asdict, dataclass, field, replace
|
|
26
|
+
from pathlib import Path
|
|
27
|
+
from typing import Any, Protocol
|
|
28
|
+
from uuid import uuid4
|
|
29
|
+
|
|
30
|
+
from outerloop.compute import (
|
|
31
|
+
GONE,
|
|
32
|
+
Compute,
|
|
33
|
+
JobSpec,
|
|
34
|
+
LocalCompute,
|
|
35
|
+
SlurmError,
|
|
36
|
+
SlurmQueryError,
|
|
37
|
+
compute_from_env,
|
|
38
|
+
is_pending,
|
|
39
|
+
is_terminal,
|
|
40
|
+
local_mode,
|
|
41
|
+
quote_command,
|
|
42
|
+
)
|
|
43
|
+
from outerloop.disk import DEFAULT_MIN_FREE_BYTES, check_disk
|
|
44
|
+
from outerloop.harness import DEFAULT_MAX_TURNS, redact
|
|
45
|
+
from outerloop.housekeeping import shed_ended_workspaces
|
|
46
|
+
from outerloop.limits import EffectiveLimits, effective_limits
|
|
47
|
+
from outerloop.markers import has_marker, marker
|
|
48
|
+
from outerloop.runstate import (
|
|
49
|
+
ABORTED,
|
|
50
|
+
ENDED,
|
|
51
|
+
IMPLEMENTING,
|
|
52
|
+
IN_REVIEW,
|
|
53
|
+
MAX_WAKE_ATTEMPTS,
|
|
54
|
+
STUCK,
|
|
55
|
+
WAITING,
|
|
56
|
+
Lease,
|
|
57
|
+
RunRecord,
|
|
58
|
+
acquire_lease,
|
|
59
|
+
lease_is_stale,
|
|
60
|
+
list_runs,
|
|
61
|
+
load_record,
|
|
62
|
+
outage_active,
|
|
63
|
+
read_lease,
|
|
64
|
+
reap_lease,
|
|
65
|
+
release_lease,
|
|
66
|
+
run_dir,
|
|
67
|
+
save_record,
|
|
68
|
+
update_lease_holder,
|
|
69
|
+
)
|
|
70
|
+
|
|
71
|
+
log = logging.getLogger(__name__)
|
|
72
|
+
|
|
73
|
+
PAUSE_SENTINEL = "PAUSE"
|
|
74
|
+
# Operator on-switch for dispatched-wake, mirroring PAUSE: a root-relative
|
|
75
|
+
# sentinel an operator arms/disarms with a touch/rm — no chain restart, no
|
|
76
|
+
# env-var surgery on a live tick. The AUTORESEARCH_DISPATCH_WAKE env var still
|
|
77
|
+
# works too (either arms it); the sentinel is the reversible, restart-free path.
|
|
78
|
+
DISPATCH_WAKE_SENTINEL = "DISPATCH_WAKE"
|
|
79
|
+
HEARTBEAT_NAME = "heartbeat.json"
|
|
80
|
+
# Written at a full tick's END (not its start) — the coalesce guard's signal, so
|
|
81
|
+
# a tick that crashes mid-work cannot suppress the next (recovery) tick.
|
|
82
|
+
WORK_MARKER_NAME = "last_worked.json"
|
|
83
|
+
|
|
84
|
+
# Grace between "experiment terminal" and the sweep stepping in: the afterany
|
|
85
|
+
# job gets this long to deliver before the backup assumes it lost.
|
|
86
|
+
DEFAULT_GRACE_S = 15 * 60
|
|
87
|
+
# a blind park (no job ids to poll) waits its eval walltime plus this queue
|
|
88
|
+
# slack before a follow-up is sent to look for the result
|
|
89
|
+
BLIND_PARK_SLACK_MIN = 12 * 60
|
|
90
|
+
# A held lease is stale after the session timeout plus slack.
|
|
91
|
+
DEFAULT_LEASE_TTL_S = 3600 + 15 * 60
|
|
92
|
+
# Coalesce guard: skip a tick's work if another ran within this window. Under
|
|
93
|
+
# partition congestion, queued ticks bunch up and become eligible together
|
|
94
|
+
# (serialized by the singleton dependency), so they would run back-to-back and
|
|
95
|
+
# redundantly re-sweep. The chain schedules ticks a full cadence apart by
|
|
96
|
+
# begin-time, so only late-bunched pile-ups fall inside this window; keep it
|
|
97
|
+
# well BELOW the cadence (default 30 min). 0 disables. Env: AUTORESEARCH_MIN_TICK_MINUTES.
|
|
98
|
+
DEFAULT_MIN_TICK_S = 10 * 60
|
|
99
|
+
# Ceiling for the coalesce window: a value above this is almost certainly a typo
|
|
100
|
+
# (a window near/over the cadence would coalesce every on-cadence tick and stall
|
|
101
|
+
# the loop). Clamp + warn rather than silently freeze. The operator is still
|
|
102
|
+
# responsible for keeping it below their configured cadence.
|
|
103
|
+
MAX_MIN_TICK_S = 60 * 60
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
class WakeDispatcher(Protocol):
|
|
107
|
+
"""Delivers one wake, called with the lease already held.
|
|
108
|
+
|
|
109
|
+
Returns "" when delivery completed synchronously (the caller releases the
|
|
110
|
+
lease), or the Slurm job id of an asynchronous wake job that now owns the
|
|
111
|
+
lease (released by that job on completion; reaped by TTL if it dies).
|
|
112
|
+
|
|
113
|
+
Contract for real dispatchers: a wake that RESULTS IN PROGRESS
|
|
114
|
+
must either move the run out of `waiting` or reset `wake_attempts` —
|
|
115
|
+
the counter means "wakes since the run last made progress", and layer 5
|
|
116
|
+
ends the run as stuck when it reaches MAX_WAKE_ATTEMPTS."""
|
|
117
|
+
|
|
118
|
+
def dispatch(self, record: RunRecord, reason: str) -> str: ...
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
@dataclass
|
|
122
|
+
class RecordingDispatcher:
|
|
123
|
+
"""Test/dry-run dispatcher: records what would have been woken."""
|
|
124
|
+
|
|
125
|
+
dispatched: list[tuple[str, str]] = field(default_factory=list)
|
|
126
|
+
holder_job_id: str = "" # set to simulate async dispatch
|
|
127
|
+
|
|
128
|
+
def dispatch(self, record: RunRecord, reason: str) -> str:
|
|
129
|
+
self.dispatched.append((record.run_id, reason))
|
|
130
|
+
return self.holder_job_id
|
|
131
|
+
|
|
132
|
+
|
|
133
|
+
@dataclass(frozen=True)
|
|
134
|
+
class TickReport:
|
|
135
|
+
paused: bool = False
|
|
136
|
+
coalesced: bool = False # skipped as a redundant pile-up (a tick ran too recently)
|
|
137
|
+
swept: int = 0
|
|
138
|
+
woken: tuple[tuple[str, str], ...] = () # (run_id, reason)
|
|
139
|
+
deferred: tuple[str, ...] = () # runs skipped on "Slurm unknown"
|
|
140
|
+
reaped_leases: tuple[str, ...] = ()
|
|
141
|
+
stuck: tuple[str, ...] = ()
|
|
142
|
+
implementing_ended: tuple[str, ...] = () # killed climbs the sweep closed out
|
|
143
|
+
review_ended: tuple[tuple[str, str], ...] = () # (run_id, ending)
|
|
144
|
+
followups_submitted: tuple[tuple[str, str], ...] = () # (run_id, job_id)
|
|
145
|
+
intake: tuple[str, str] = ("", "") # (issue tag, job_id) when one was claimed
|
|
146
|
+
self_initiated: tuple[str, str] = ("", "") # (benchmark, job_id) when one launched
|
|
147
|
+
steward: tuple[str, str] = ("", "") # (issue tag, job_id) when a stewardship launched
|
|
148
|
+
disk: tuple[str, ...] = () # preflight warnings (home entries are warn-only)
|
|
149
|
+
launch_blocked: bool = False # True when the preflight turned launch lanes off
|
|
150
|
+
shed: tuple[str, ...] = () # ended runs whose workspaces housekeeping removed
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
# The submitted walltime must never exceed the job partition's MaxTime —
|
|
154
|
+
# sbatch REJECTS a longer request outright, which would ground every climb.
|
|
155
|
+
# The DEFAULT matches cpu_short (6 h); an operator moving work jobs to a
|
|
156
|
+
# longer partition (AUTORESEARCH_JOB_PARTITION=cpu48) raises the cap with
|
|
157
|
+
# AUTORESEARCH_MAX_JOB_MINUTES. Code-side ceiling: the cap must stay under
|
|
158
|
+
# STRANDED_IMPLEMENTING_S or the picker declares live runs stranded — jobs
|
|
159
|
+
# longer than 10 h need that window made spec-aware first (named gap). The
|
|
160
|
+
# self-deadline arms at the CLAMPED value, so a job that wanted more time
|
|
161
|
+
# fails safe mid-panel instead of never starting.
|
|
162
|
+
MAX_ATTEMPT_JOB_MINUTES = 6 * 60
|
|
163
|
+
MAX_JOB_MINUTES_CEILING = 10 * 60
|
|
164
|
+
|
|
165
|
+
|
|
166
|
+
def _bot_login_default() -> str:
|
|
167
|
+
"""FollowupSpec's login default, resolved at construction (the tick reads
|
|
168
|
+
the chain's env, jobs inherit it); github is imported here on purpose —
|
|
169
|
+
the tick module stays importable without it."""
|
|
170
|
+
from outerloop.github import bot_login_from_env
|
|
171
|
+
|
|
172
|
+
return bot_login_from_env()
|
|
173
|
+
|
|
174
|
+
|
|
175
|
+
@dataclass(frozen=True)
|
|
176
|
+
class FollowupSpec:
|
|
177
|
+
"""How the tick launches follow-up jobs for in-review runs."""
|
|
178
|
+
|
|
179
|
+
account: str
|
|
180
|
+
partition: str
|
|
181
|
+
run_root: Path
|
|
182
|
+
image: str
|
|
183
|
+
home: Path # AUTORESEARCH_HOME: cwd for the submitted job
|
|
184
|
+
bot_login: str = field(default_factory=_bot_login_default)
|
|
185
|
+
time_minutes: int = 90 # min()'d with the contract's followup_job_minutes
|
|
186
|
+
max_turns: int = DEFAULT_MAX_TURNS # session turn budget for follow-up jobs
|
|
187
|
+
pat_file: str = "" # forwarded to the job; "" = the followup CLI default
|
|
188
|
+
# GitHub App config path; jobs inherit AUTORESEARCH_GITHUB_APP_FILE from
|
|
189
|
+
# the tick environment, so it is never threaded through argv
|
|
190
|
+
github_app_file: str = ""
|
|
191
|
+
target: str = "" # the repo the intake pass scans for requested-lane issues
|
|
192
|
+
# the STEWARD'S OWN key (role separation): the steward lane stays off
|
|
193
|
+
# until the operator provisions it
|
|
194
|
+
steward_key_file: str = ""
|
|
195
|
+
# Pre-PR verification panel for climb jobs (docs/design/orchestrator-verify.md).
|
|
196
|
+
# DEFAULT ON — the flip is code, the off-switch is AUTORESEARCH_PANEL="".
|
|
197
|
+
# The climb CLI fails LOUDLY on a bad panel config (a configured gate must
|
|
198
|
+
# never silently vanish); the tick preflights the same rules — lens
|
|
199
|
+
# grammar AND key file — before claiming or submitting so nothing is
|
|
200
|
+
# stranded. The panel's walltime is the orchestrator's own overhead: the
|
|
201
|
+
# tick ADDS a panel allowance to the contract-clamped job budget
|
|
202
|
+
# (_panel_job_minutes) rather than eating the author's time; a residual
|
|
203
|
+
# overrun still fails safe through the self-deadline.
|
|
204
|
+
panel: str = "verify,review"
|
|
205
|
+
panel_key_file: str = "" # "" = the climb CLI's default verifier-key path
|
|
206
|
+
# The GPU lane for benchmarks whose contract sets `gpus > 0` (their evals
|
|
207
|
+
# and author launches); empty = this deployment cannot place GPU jobs,
|
|
208
|
+
# and the launch lanes refuse such benchmarks (a queue that can never
|
|
209
|
+
# run is worse than a loud refusal). gpu_account "" = same as `account`.
|
|
210
|
+
gpu_partition: str = ""
|
|
211
|
+
gpu_account: str = ""
|
|
212
|
+
# Where submitted WORK jobs (climb/steward/followup) run; empty = same as
|
|
213
|
+
# `partition`. The tick chain itself always stays on `partition` — ticks
|
|
214
|
+
# are minutes, work jobs can be hours, and Slurm prices walltime into
|
|
215
|
+
# scheduling priority, so the two deserve independent placement.
|
|
216
|
+
job_partition: str = ""
|
|
217
|
+
# Partition MaxTime for work jobs — the panel-augmented walltime clamps
|
|
218
|
+
# here (see MAX_ATTEMPT_JOB_MINUTES). Raise together with job_partition.
|
|
219
|
+
max_job_minutes: int = MAX_ATTEMPT_JOB_MINUTES
|
|
220
|
+
|
|
221
|
+
|
|
222
|
+
# Generous vs the ~2 h job walltimes plus queue wait, tight enough that
|
|
223
|
+
# full trees + per-flight venvs cannot pile up for days; same-day
|
|
224
|
+
# forensics is the norm, and the disk preflight is the backstop.
|
|
225
|
+
FLIGHT_TTL_S = 24 * 3600
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
def flight_checkout(home: Path, name: str, now: float) -> Path:
|
|
229
|
+
"""A detached git worktree of the checkout's HEAD commit, for one
|
|
230
|
+
submitted job to run from. The shared checkout is reset --hard at
|
|
231
|
+
every tick's deploy, so a queued job that cd's into it can have its
|
|
232
|
+
code swapped mid-flight; a flight pins the deployed commit, and the
|
|
233
|
+
tree survives for forensics after a crash. HEAD, deliberately:
|
|
234
|
+
uncommitted hand-edits in the shared checkout do not fly — only
|
|
235
|
+
deployed code does. Failures fall back to the shared checkout — a
|
|
236
|
+
snapshot must never ground the fleet."""
|
|
237
|
+
flights = home.parent / "flights"
|
|
238
|
+
try:
|
|
239
|
+
flights.mkdir(parents=True, exist_ok=True)
|
|
240
|
+
# same name in the same tick (e.g. two orders on one benchmark, or
|
|
241
|
+
# truncation collisions) must get its own tree, not a silent
|
|
242
|
+
# fallback: suffix until free, with a unique tail as the backstop
|
|
243
|
+
target = flights / f"{name}-{int(now)}"
|
|
244
|
+
for attempt in range(2, 6):
|
|
245
|
+
if not target.exists():
|
|
246
|
+
break
|
|
247
|
+
target = flights / f"{name}-{int(now)}-{attempt}"
|
|
248
|
+
if target.exists():
|
|
249
|
+
target = flights / f"{name}-{int(now)}-{uuid4().hex[:8]}"
|
|
250
|
+
subprocess.run(
|
|
251
|
+
["git", "-C", str(home), "worktree", "add", "--detach", str(target)],
|
|
252
|
+
check=True,
|
|
253
|
+
capture_output=True,
|
|
254
|
+
text=True,
|
|
255
|
+
timeout=60,
|
|
256
|
+
)
|
|
257
|
+
return target
|
|
258
|
+
except (OSError, subprocess.SubprocessError) as exc:
|
|
259
|
+
log.warning("flight snapshot failed for %s (%s); using the shared checkout", name, exc)
|
|
260
|
+
return home
|
|
261
|
+
|
|
262
|
+
|
|
263
|
+
def _benchmark_gpus(contract: Any, benchmark: str) -> int:
|
|
264
|
+
bench = next((b for b in getattr(contract, "benchmarks", []) if b.name == benchmark), None)
|
|
265
|
+
return int(getattr(bench, "gpus", 0) or 0)
|
|
266
|
+
|
|
267
|
+
|
|
268
|
+
def _gpu_lane_error(contract: Any, benchmark: str, spec: FollowupSpec) -> str:
|
|
269
|
+
"""Why an attempt on `benchmark` cannot launch here, or "": a contract
|
|
270
|
+
with GPU benchmarks needs this deployment to name a GPU lane — otherwise
|
|
271
|
+
evals would queue into jobs that can never run (the climb would then
|
|
272
|
+
park forever on a phantom eval). ANY GPU benchmark in the contract
|
|
273
|
+
counts, not just the climbed one: the suite gate measures siblings.
|
|
274
|
+
Local compute has no lanes — jobs run on whatever GPUs the machine
|
|
275
|
+
has — so the check is waived there."""
|
|
276
|
+
if spec.gpu_partition or local_mode():
|
|
277
|
+
return ""
|
|
278
|
+
gpu_benches = [
|
|
279
|
+
b.name for b in getattr(contract, "benchmarks", []) if int(getattr(b, "gpus", 0) or 0)
|
|
280
|
+
]
|
|
281
|
+
if gpu_benches:
|
|
282
|
+
return (
|
|
283
|
+
f"contract has GPU benchmarks ({', '.join(gpu_benches)}) but no GPU lane is "
|
|
284
|
+
"configured (set AUTORESEARCH_GPU_PARTITION)"
|
|
285
|
+
)
|
|
286
|
+
return ""
|
|
287
|
+
|
|
288
|
+
|
|
289
|
+
def _flight_command(home: Path, job_name: str, now: float, argv: list[str]) -> str:
|
|
290
|
+
"""The job's shell command, cd'ing into a fresh flight snapshot.
|
|
291
|
+
|
|
292
|
+
The flight is named FROM the job name (one truncation rule, here) so
|
|
293
|
+
the reaper's pending-job immunity — live job name prefixes flight
|
|
294
|
+
name — holds by construction at every submit site. argv must contain
|
|
295
|
+
absolute paths only; every spec path is absolute by construction, and
|
|
296
|
+
a relative path would resolve inside a tree that is reaped later."""
|
|
297
|
+
flight = flight_checkout(home, job_name[:40], now)
|
|
298
|
+
return f"cd {quote_command([str(flight)])} && {quote_command(argv)}"
|
|
299
|
+
|
|
300
|
+
|
|
301
|
+
def reap_flights(
|
|
302
|
+
home: Path,
|
|
303
|
+
now: float,
|
|
304
|
+
ttl_s: float = FLIGHT_TTL_S,
|
|
305
|
+
live_job_names: Sequence[str] = (),
|
|
306
|
+
) -> int:
|
|
307
|
+
"""Remove flight worktrees older than the TTL — age by directory
|
|
308
|
+
mtime, no name parsing — UNLESS a pending or running job's NAME
|
|
309
|
+
prefixes the flight's (flights are named after their job). Queue wait
|
|
310
|
+
is unbounded (GPU partitions can pend for days), so age alone must
|
|
311
|
+
never delete a tree a job will cd into; the TTL is purely the
|
|
312
|
+
forensics-retention window for flights whose job is gone. Name
|
|
313
|
+
matching is conservative: one live job name protects every flight it
|
|
314
|
+
prefixes. Best-effort: a stubborn flight is logged, not fatal."""
|
|
315
|
+
flights = home.parent / "flights"
|
|
316
|
+
if not flights.is_dir():
|
|
317
|
+
return 0
|
|
318
|
+
reaped = 0
|
|
319
|
+
for entry in flights.iterdir():
|
|
320
|
+
if any(name and entry.name.startswith(name[:40]) for name in live_job_names):
|
|
321
|
+
continue # a queued or running job still needs this tree
|
|
322
|
+
try:
|
|
323
|
+
age = now - entry.stat().st_mtime
|
|
324
|
+
except OSError:
|
|
325
|
+
continue
|
|
326
|
+
if age < ttl_s:
|
|
327
|
+
continue
|
|
328
|
+
try:
|
|
329
|
+
subprocess.run(
|
|
330
|
+
["git", "-C", str(home), "worktree", "remove", "--force", str(entry)],
|
|
331
|
+
check=True,
|
|
332
|
+
capture_output=True,
|
|
333
|
+
text=True,
|
|
334
|
+
timeout=60,
|
|
335
|
+
)
|
|
336
|
+
reaped += 1
|
|
337
|
+
except (OSError, subprocess.SubprocessError):
|
|
338
|
+
# not a registered worktree (a half-created flight, or debris):
|
|
339
|
+
# remove the directory itself and prune the registry, or the
|
|
340
|
+
# entry warns forever without ever going away
|
|
341
|
+
import shutil
|
|
342
|
+
|
|
343
|
+
shutil.rmtree(entry, ignore_errors=True)
|
|
344
|
+
with contextlib.suppress(OSError, subprocess.SubprocessError):
|
|
345
|
+
subprocess.run(
|
|
346
|
+
["git", "-C", str(home), "worktree", "prune"],
|
|
347
|
+
check=True,
|
|
348
|
+
capture_output=True,
|
|
349
|
+
text=True,
|
|
350
|
+
timeout=60,
|
|
351
|
+
)
|
|
352
|
+
if not entry.exists():
|
|
353
|
+
reaped += 1
|
|
354
|
+
else:
|
|
355
|
+
log.warning("could not reap flight %s", entry.name)
|
|
356
|
+
return reaped
|
|
357
|
+
|
|
358
|
+
|
|
359
|
+
CONTRACT_ALARM_MARKER = marker("contract-alarm")
|
|
360
|
+
CONTRACT_ALARM_AFTER = 3 # consecutive failing ticks (~1.5 h) before alarming
|
|
361
|
+
|
|
362
|
+
|
|
363
|
+
def contract_alarm(
|
|
364
|
+
root: Path,
|
|
365
|
+
github: Any,
|
|
366
|
+
target: str,
|
|
367
|
+
error: str | None,
|
|
368
|
+
now: float,
|
|
369
|
+
bot_login: str = "agentic-learning-bot",
|
|
370
|
+
) -> None:
|
|
371
|
+
"""Persistent contract failure must surface where humans look.
|
|
372
|
+
|
|
373
|
+
A rejected or unfetchable contract silently idles every launch lane.
|
|
374
|
+
After CONTRACT_ALARM_AFTER consecutive failing ticks this
|
|
375
|
+
opens ONE issue on the target repo; the next successful load closes
|
|
376
|
+
it and says so. Alarm plumbing is best-effort by construction: it
|
|
377
|
+
must never break the tick it reports for."""
|
|
378
|
+
state_path = root / "contract-alarm.json"
|
|
379
|
+
try:
|
|
380
|
+
state = json.loads(state_path.read_text())
|
|
381
|
+
except (OSError, ValueError):
|
|
382
|
+
state = {}
|
|
383
|
+
if error is None:
|
|
384
|
+
open_alarm = int(state.get("issue", 0))
|
|
385
|
+
if not open_alarm and state:
|
|
386
|
+
# creation may have landed without a recorded number (dry-run,
|
|
387
|
+
# odd response, lost state file): search so recovery can still
|
|
388
|
+
# close it. Only ticks that follow SOME failure signal pay the
|
|
389
|
+
# search; total state loss + instant recovery leaves the issue
|
|
390
|
+
# for the next alarm cycle's search to adopt and close.
|
|
391
|
+
with contextlib.suppress(Exception):
|
|
392
|
+
open_alarm = _find_alarm_issue(github, target, bot_login)
|
|
393
|
+
if open_alarm:
|
|
394
|
+
try:
|
|
395
|
+
github.close_issue(target, open_alarm)
|
|
396
|
+
except Exception as exc:
|
|
397
|
+
# keep the state so the NEXT healthy tick retries the close;
|
|
398
|
+
# unlinking here would orphan the open alarm forever — and
|
|
399
|
+
# no comment yet, or every retry would repeat it
|
|
400
|
+
log.warning("could not close contract alarm #%s: %s", open_alarm, exc)
|
|
401
|
+
return
|
|
402
|
+
with contextlib.suppress(Exception):
|
|
403
|
+
github.comment(
|
|
404
|
+
target, open_alarm, "The tick loads again cleanly; launch lanes resume."
|
|
405
|
+
)
|
|
406
|
+
if state:
|
|
407
|
+
with contextlib.suppress(OSError):
|
|
408
|
+
state_path.unlink()
|
|
409
|
+
return
|
|
410
|
+
count = int(state.get("count", 0)) + 1
|
|
411
|
+
state["count"] = count
|
|
412
|
+
# redacted (the client's own token is the one secret this process
|
|
413
|
+
# holds) and fenced with a run longer than any backtick run inside —
|
|
414
|
+
# transport errors can echo request material, loader errors can echo
|
|
415
|
+
# contract content, and both are untrusted for a public issue body
|
|
416
|
+
safe_error = redact(error, _client_secrets(github)).replace(str(Path.home()), "~")[:600]
|
|
417
|
+
# A recorded issue a human closed by hand stays closed: closing the
|
|
418
|
+
# alarm is the maintainer's "I know" — re-opening or re-creating it
|
|
419
|
+
# every threshold would be alarm spam, and recovery still clears state.
|
|
420
|
+
if count >= CONTRACT_ALARM_AFTER and not state.get("issue"):
|
|
421
|
+
# search open issues first: state loss must not spawn duplicates
|
|
422
|
+
try:
|
|
423
|
+
number = _find_alarm_issue(github, target, bot_login) or github.create_issue(
|
|
424
|
+
target,
|
|
425
|
+
"outerloop: launch lanes are paused",
|
|
426
|
+
f"{CONTRACT_ALARM_MARKER}\nThe orchestrator's launch lanes "
|
|
427
|
+
f"(intake, steward, self-initiated) have sat out {count} "
|
|
428
|
+
f"consecutive ticks. The error below names the cause — a "
|
|
429
|
+
f"contract that failed to load, or a panel config the climb "
|
|
430
|
+
f"would reject.\n\n"
|
|
431
|
+
f"{_fence(safe_error)}\n{safe_error}\n{_fence(safe_error)}\n\n"
|
|
432
|
+
f"This issue closes itself when a tick passes cleanly.",
|
|
433
|
+
)
|
|
434
|
+
if number:
|
|
435
|
+
state["issue"] = number
|
|
436
|
+
except Exception as exc:
|
|
437
|
+
log.warning("contract alarm could not post to %s: %s", target, exc)
|
|
438
|
+
with contextlib.suppress(OSError):
|
|
439
|
+
tmp = state_path.with_suffix(f".{os.getpid()}.tmp")
|
|
440
|
+
tmp.write_text(json.dumps(state))
|
|
441
|
+
os.replace(tmp, state_path)
|
|
442
|
+
|
|
443
|
+
|
|
444
|
+
def _client_secrets(github: Any) -> tuple[str, ...]:
|
|
445
|
+
try:
|
|
446
|
+
token = github.auth.token()
|
|
447
|
+
if token:
|
|
448
|
+
return (token,)
|
|
449
|
+
except Exception as exc:
|
|
450
|
+
# degraded redaction must not be silent: the error text goes to a
|
|
451
|
+
# public issue, and a renamed auth surface would no-op the redact
|
|
452
|
+
log.warning("alarm redaction has no client token (%s)", exc)
|
|
453
|
+
return ()
|
|
454
|
+
|
|
455
|
+
|
|
456
|
+
def _contract_text(github: Any, target: str, ref: str) -> str | None:
|
|
457
|
+
"""The target's contract at `ref` — `.outerloop.yaml`, else the legacy
|
|
458
|
+
`.autoresearch.yaml` — or None when it has neither."""
|
|
459
|
+
from outerloop.contract import find_contract
|
|
460
|
+
|
|
461
|
+
found = find_contract(lambda name: github.get_file_content(target, name, ref))
|
|
462
|
+
return found[1] if found else None
|
|
463
|
+
|
|
464
|
+
|
|
465
|
+
def _fence(content: str) -> str:
|
|
466
|
+
longest = max((len(run) for run in re.findall(r"`+", content)), default=0)
|
|
467
|
+
return "`" * max(3, longest + 1)
|
|
468
|
+
|
|
469
|
+
|
|
470
|
+
def _find_alarm_issue(github: Any, target: str, bot_login: str) -> int:
|
|
471
|
+
"""Only the BOT'S own marker'd issue counts: the marker is a public
|
|
472
|
+
string, and adopting a stranger's issue would let anyone suppress the
|
|
473
|
+
real alarm or get their issue closed by the bot."""
|
|
474
|
+
from outerloop.github import is_own_login
|
|
475
|
+
|
|
476
|
+
return next(
|
|
477
|
+
(
|
|
478
|
+
int(issue.get("number", 0))
|
|
479
|
+
for issue in github.list_open_issues(target, max_pages=10)
|
|
480
|
+
if has_marker(str(issue.get("body", "")), "contract-alarm")
|
|
481
|
+
and is_own_login(str((issue.get("user") or {}).get("login", "")), bot_login)
|
|
482
|
+
),
|
|
483
|
+
0,
|
|
484
|
+
)
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
def shape_followup_spec(spec: FollowupSpec, limits: EffectiveLimits, contract: Any) -> FollowupSpec:
|
|
488
|
+
"""Clamp the operator's follow-up spec by the contract's effective
|
|
489
|
+
limits. Both knobs clamp only when the contract EXPLICITLY sets them:
|
|
490
|
+
a contract shapes spend downward, but an operator's deliberate config
|
|
491
|
+
is never silently reduced by defaults (raising budgets is operator
|
|
492
|
+
territory)."""
|
|
493
|
+
if contract is None:
|
|
494
|
+
return spec
|
|
495
|
+
# direct attribute access: Budgets is our typed model, and a rename
|
|
496
|
+
# must fail loudly here, not silently stop shaping spend downward
|
|
497
|
+
if contract.budgets.session_max_turns is not None:
|
|
498
|
+
spec = replace(spec, max_turns=min(spec.max_turns, limits.session_max_turns))
|
|
499
|
+
if contract.budgets.followup_job_minutes is not None:
|
|
500
|
+
spec = replace(spec, time_minutes=min(spec.time_minutes, limits.followup_job_minutes))
|
|
501
|
+
return spec
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def _base_dial(
|
|
505
|
+
github: Any, target: str, pr: dict, main_contract: Any, main_target: str = ""
|
|
506
|
+
) -> str:
|
|
507
|
+
"""The merge dial that governs THIS PR: its own target's base-branch
|
|
508
|
+
contract. The tick's contract is read from ITS configured target's main;
|
|
509
|
+
it applies only to a main-based PR of that same target — any other
|
|
510
|
+
target or base is fetched from the PR's own coordinates, and unreadable
|
|
511
|
+
or unparsable means "manual" (never arm on doubt)."""
|
|
512
|
+
base_ref = str((pr.get("base") or {}).get("ref", "")) or "main"
|
|
513
|
+
if base_ref == "main" and target == main_target:
|
|
514
|
+
return str(getattr(main_contract, "merge", "manual"))
|
|
515
|
+
try:
|
|
516
|
+
from outerloop.contract import load_contract
|
|
517
|
+
|
|
518
|
+
raw = _contract_text(github, target, base_ref)
|
|
519
|
+
if raw is None:
|
|
520
|
+
return "manual"
|
|
521
|
+
return str(getattr(load_contract(raw, target), "merge", "manual"))
|
|
522
|
+
except Exception as exc:
|
|
523
|
+
log.warning("base-contract read failed for %s@%s: %s", target, base_ref, exc)
|
|
524
|
+
return "manual"
|
|
525
|
+
|
|
526
|
+
|
|
527
|
+
def service_in_review(
|
|
528
|
+
root: Path,
|
|
529
|
+
github: Any, # GitHubClient (Any keeps tick importable without github deps)
|
|
530
|
+
compute: Compute,
|
|
531
|
+
spec: FollowupSpec,
|
|
532
|
+
now: float,
|
|
533
|
+
dry_run: bool = False,
|
|
534
|
+
allow_submit: bool = True,
|
|
535
|
+
contract: Any = None,
|
|
536
|
+
) -> tuple[list[tuple[str, str]], list[tuple[str, str]]]:
|
|
537
|
+
"""PR-state transitions + follow-up job submission for in-review runs.
|
|
538
|
+
|
|
539
|
+
allow_submit=False (disk preflight failed) keeps the cheap state
|
|
540
|
+
transitions — ending merged/closed runs still matters — but submits no
|
|
541
|
+
new session jobs.
|
|
542
|
+
|
|
543
|
+
The tick only READS GitHub here (cheap, every cycle); the session-running
|
|
544
|
+
work happens in a submitted job, which takes the run lease itself — a
|
|
545
|
+
duplicate submission no-ops on the lease, and `followup_job_id` keeps the
|
|
546
|
+
tick from queueing duplicates in the first place.
|
|
547
|
+
"""
|
|
548
|
+
from outerloop.followup import (
|
|
549
|
+
_pr_number,
|
|
550
|
+
close_if_done,
|
|
551
|
+
conflict_wake_action,
|
|
552
|
+
has_new_comments,
|
|
553
|
+
panel_wake_pending,
|
|
554
|
+
)
|
|
555
|
+
|
|
556
|
+
ended: list[tuple[str, str]] = []
|
|
557
|
+
submitted: list[tuple[str, str]] = []
|
|
558
|
+
for record in list_runs(root):
|
|
559
|
+
if record.state != IN_REVIEW or not record.pr_url:
|
|
560
|
+
continue
|
|
561
|
+
try:
|
|
562
|
+
ending = close_if_done(root, record, github, now)
|
|
563
|
+
if ending:
|
|
564
|
+
ended.append((record.run_id, ending))
|
|
565
|
+
continue
|
|
566
|
+
# Steward records are serviced with the STEWARD'S key and the
|
|
567
|
+
# steward scope check (respond_once derives the mode from the
|
|
568
|
+
# record's agent id); without a provisioned steward key the
|
|
569
|
+
# lane stays human-answered.
|
|
570
|
+
is_steward = record.agent_id.startswith("steward")
|
|
571
|
+
if is_steward and not spec.steward_key_file:
|
|
572
|
+
continue
|
|
573
|
+
# per-ROLE outage latch: state transitions above still ran,
|
|
574
|
+
# only this record's session spawn sits the cooldown out
|
|
575
|
+
paused = outage_active(root, now, role="steward" if is_steward else "solver")
|
|
576
|
+
if paused:
|
|
577
|
+
log.info("follow-up for %s paused (api outage: %s)", record.run_id, paused)
|
|
578
|
+
continue
|
|
579
|
+
try:
|
|
580
|
+
pr = github.get_pull_request(record.target, _pr_number(record.pr_url))
|
|
581
|
+
except Exception:
|
|
582
|
+
continue # unreadable PR: nothing to decide this tick
|
|
583
|
+
# Idempotent auto-arm: once GitHub reports the PR CLEAN (green
|
|
584
|
+
# checks AND up-to-date with the CURRENT base — GitHub's own
|
|
585
|
+
# freshness proof), the kernel-read contract STILL says auto, and
|
|
586
|
+
# A dispatched re-measure in flight: nothing else is serviced (the
|
|
587
|
+
# sealed change lands first, so the next comment is answered on
|
|
588
|
+
# the tree it will actually see) and nothing is armed. Once every
|
|
589
|
+
# eval job is terminal, a follow-up is submitted to finish it.
|
|
590
|
+
measure_ready = False
|
|
591
|
+
if record.followup_stage:
|
|
592
|
+
raw_ids = record.followup_stage.get("job_ids")
|
|
593
|
+
job_ids = [str(j) for j in raw_ids] if isinstance(raw_ids, list) else []
|
|
594
|
+
if job_ids:
|
|
595
|
+
try:
|
|
596
|
+
states = [compute.status(j) for j in job_ids]
|
|
597
|
+
except SlurmQueryError:
|
|
598
|
+
continue # unknown: neither service nor arm
|
|
599
|
+
if not all(is_terminal(s) or s == GONE for s in states):
|
|
600
|
+
continue
|
|
601
|
+
measure_ready = True
|
|
602
|
+
else:
|
|
603
|
+
# a BLIND park (the measurer could not read the queue at
|
|
604
|
+
# dispatch): no ids to poll, so the eval walltime plus the
|
|
605
|
+
# climb's queue slack is the floor before a follow-up is
|
|
606
|
+
# sent to look — never one per tick (terra #241 r1)
|
|
607
|
+
from outerloop.dispatch import effective_eval_minutes
|
|
608
|
+
|
|
609
|
+
parked_at = float(record.followup_stage.get("parked_at", 0.0) or 0.0) # type: ignore[arg-type]
|
|
610
|
+
floor_min = int(record.followup_stage.get("eval_minutes", 0) or 0) # type: ignore[call-overload]
|
|
611
|
+
floor_s = (effective_eval_minutes(floor_min) + BLIND_PARK_SLACK_MIN) * 60
|
|
612
|
+
if now - parked_at < floor_s:
|
|
613
|
+
continue
|
|
614
|
+
measure_ready = True
|
|
615
|
+
wake_action = conflict_wake_action(record, pr)
|
|
616
|
+
if wake_action == "clear":
|
|
617
|
+
# the PR is clean again: re-arm the wake for this head — the
|
|
618
|
+
# base can move and conflict the SAME head a second time
|
|
619
|
+
try:
|
|
620
|
+
save_record(root, replace(record, dirty_wake_head=""), now)
|
|
621
|
+
except OSError as exc:
|
|
622
|
+
log.warning("conflict cursor clear failed for %s: %s", record.run_id, exc)
|
|
623
|
+
if (
|
|
624
|
+
not measure_ready
|
|
625
|
+
and not has_new_comments(record, github, spec.bot_login)
|
|
626
|
+
and wake_action != "wake"
|
|
627
|
+
and not panel_wake_pending(record, pr)
|
|
628
|
+
):
|
|
629
|
+
# NOTHING awaits servicing — only a fully quiet PR may
|
|
630
|
+
# self-merge (pending reviewer feedback always wins over
|
|
631
|
+
# arming: a followup must service it first, and a pushed
|
|
632
|
+
# change would kill the blessing anyway). The RECORD says the
|
|
633
|
+
# publish was auto-eligible (published under merge:auto with
|
|
634
|
+
# a clean panel — #171's exact arming condition; a manual
|
|
635
|
+
# publish never consented, and contracts alone cannot prove
|
|
636
|
+
# either fact after a dial flip); GitHub's own CLEAN state is
|
|
637
|
+
# the freshness proof; the PR's base-branch contract is the
|
|
638
|
+
# governing dial. Running every tick survives any crash
|
|
639
|
+
# between a sync push and this step; the helper direct-merges
|
|
640
|
+
# when nothing is pending to arm against.
|
|
641
|
+
# ...and no follow-up job may be LIVE: a running responder can
|
|
642
|
+
# have pushed a code change whose record write (clearing the
|
|
643
|
+
# blessing) has not landed yet — arming on that head would
|
|
644
|
+
# merge code the panel never saw (terra #228 r7)
|
|
645
|
+
followup_live = False
|
|
646
|
+
if record.followup_job_id:
|
|
647
|
+
try:
|
|
648
|
+
state = compute.status(record.followup_job_id)
|
|
649
|
+
followup_live = not (is_terminal(state) or state == GONE)
|
|
650
|
+
except SlurmQueryError:
|
|
651
|
+
followup_live = True # unknown = assume live, never arm
|
|
652
|
+
if (
|
|
653
|
+
not dry_run
|
|
654
|
+
and not is_steward
|
|
655
|
+
and not followup_live
|
|
656
|
+
and record.auto_blessed_head
|
|
657
|
+
and str((pr.get("head") or {}).get("sha", "")) == record.auto_blessed_head
|
|
658
|
+
and contract is not None
|
|
659
|
+
and pr.get("state") == "open"
|
|
660
|
+
and not pr.get("merged")
|
|
661
|
+
and not pr.get("draft")
|
|
662
|
+
and pr.get("mergeable_state") == "clean"
|
|
663
|
+
and _base_dial(github, record.target, pr, contract, spec.target) == "auto"
|
|
664
|
+
):
|
|
665
|
+
try:
|
|
666
|
+
# the mutation itself is bound to the blessed head: a
|
|
667
|
+
# push racing this check is refused by GitHub, not
|
|
668
|
+
# merged (terra #228 r9)
|
|
669
|
+
github.arm_auto_merge_auto_mode(
|
|
670
|
+
record.target,
|
|
671
|
+
_pr_number(record.pr_url),
|
|
672
|
+
expected_head=record.auto_blessed_head,
|
|
673
|
+
)
|
|
674
|
+
except Exception as exc:
|
|
675
|
+
log.warning("auto-arm failed for %s: %s", record.run_id, exc)
|
|
676
|
+
continue
|
|
677
|
+
if record.followup_job_id:
|
|
678
|
+
try:
|
|
679
|
+
state = compute.status(record.followup_job_id)
|
|
680
|
+
if not (is_terminal(state) or state == GONE):
|
|
681
|
+
continue # a follow-up job is already queued/running
|
|
682
|
+
except SlurmQueryError:
|
|
683
|
+
continue # unknown — do not stack another job
|
|
684
|
+
# the wake-attempt counter caps follow-up retries too: a responder
|
|
685
|
+
# that cannot advance its cursors must not burn a session per tick
|
|
686
|
+
if record.wake_attempts >= MAX_WAKE_ATTEMPTS:
|
|
687
|
+
log.warning(
|
|
688
|
+
"run %s: %d follow-up attempts without progress; not resubmitting",
|
|
689
|
+
record.run_id,
|
|
690
|
+
record.wake_attempts,
|
|
691
|
+
)
|
|
692
|
+
continue
|
|
693
|
+
if not allow_submit:
|
|
694
|
+
log.warning("run %s has new comments but disk preflight failed", record.run_id)
|
|
695
|
+
continue
|
|
696
|
+
if dry_run:
|
|
697
|
+
submitted.append((record.run_id, "dry-run"))
|
|
698
|
+
continue
|
|
699
|
+
# The author follow-up carries the climb's panel so a pushed code
|
|
700
|
+
# change is RE-READ before the tick may arm it (followup.py). The
|
|
701
|
+
# panel brings its own walltime, like the climb's allowance — the
|
|
702
|
+
# contract's followup budget caps the author, not the gate. A
|
|
703
|
+
# panel that would die at startup is left off: the reply still
|
|
704
|
+
# goes out, the PR simply stays human-merged (the tick's contract
|
|
705
|
+
# alarm already names the misconfig).
|
|
706
|
+
panel_argv: list[str] = []
|
|
707
|
+
panel_minutes = 0
|
|
708
|
+
if not is_steward and spec.panel.strip():
|
|
709
|
+
panel_error = _panel_preflight_error(spec)
|
|
710
|
+
if panel_error:
|
|
711
|
+
log.warning(
|
|
712
|
+
"follow-up for %s runs without the panel: %s", record.run_id, panel_error
|
|
713
|
+
)
|
|
714
|
+
else:
|
|
715
|
+
from outerloop.panel import panel_read_minutes
|
|
716
|
+
|
|
717
|
+
panel_argv = _climb_panel_argv(spec)
|
|
718
|
+
panel_minutes = panel_read_minutes(spec.panel)
|
|
719
|
+
# the author's budget first, the read on top, both under the
|
|
720
|
+
# partition cap — and the follow-up is told how many minutes the
|
|
721
|
+
# read actually got (--panel-minutes), so a cap that eats the
|
|
722
|
+
# allowance costs the READ (skipped, said so), never the author
|
|
723
|
+
author_minutes = min(spec.time_minutes, spec.max_job_minutes)
|
|
724
|
+
job_minutes = min(author_minutes + panel_minutes, spec.max_job_minutes)
|
|
725
|
+
if panel_argv:
|
|
726
|
+
panel_argv = [*panel_argv, "--panel-minutes", str(job_minutes - author_minutes)]
|
|
727
|
+
argv = [
|
|
728
|
+
"uv",
|
|
729
|
+
"run",
|
|
730
|
+
"python",
|
|
731
|
+
"-m",
|
|
732
|
+
"outerloop.followup",
|
|
733
|
+
"--run-root",
|
|
734
|
+
str(spec.run_root),
|
|
735
|
+
"--run-id",
|
|
736
|
+
record.run_id,
|
|
737
|
+
"--image",
|
|
738
|
+
spec.image,
|
|
739
|
+
"--bot-login",
|
|
740
|
+
spec.bot_login,
|
|
741
|
+
"--job-minutes",
|
|
742
|
+
# the SAME clamped value Slurm gets: a deadline armed past
|
|
743
|
+
# the real walltime is a Slurm kill before a clean ending
|
|
744
|
+
str(job_minutes),
|
|
745
|
+
"--max-turns",
|
|
746
|
+
str(spec.max_turns),
|
|
747
|
+
# the cluster coordinates the climb gets: a GPU benchmark's
|
|
748
|
+
# re-measure is dispatched to the GPU lane, never run here
|
|
749
|
+
"--account",
|
|
750
|
+
spec.account,
|
|
751
|
+
"--partition",
|
|
752
|
+
spec.partition,
|
|
753
|
+
"--gpu-partition",
|
|
754
|
+
spec.gpu_partition,
|
|
755
|
+
"--gpu-account",
|
|
756
|
+
spec.gpu_account,
|
|
757
|
+
*panel_argv,
|
|
758
|
+
]
|
|
759
|
+
if spec.pat_file:
|
|
760
|
+
argv += ["--pat-file", spec.pat_file]
|
|
761
|
+
# config-driven author: the author follow-up resolves its key per the
|
|
762
|
+
# RUN's backend (from the record) inside followup.main — the tick does
|
|
763
|
+
# not thread it. The steward is a distinct role with its own key.
|
|
764
|
+
if is_steward and spec.steward_key_file:
|
|
765
|
+
argv += ["--key-file", spec.steward_key_file]
|
|
766
|
+
job_id = compute.submit(
|
|
767
|
+
JobSpec(
|
|
768
|
+
job_name=f"followup-{record.run_id}"[:60],
|
|
769
|
+
account=spec.account,
|
|
770
|
+
partition=spec.job_partition or spec.partition,
|
|
771
|
+
time_minutes=job_minutes,
|
|
772
|
+
command=_flight_command(spec.home, f"followup-{record.run_id}"[:60], now, argv),
|
|
773
|
+
cpus=4,
|
|
774
|
+
mem="8G",
|
|
775
|
+
)
|
|
776
|
+
)
|
|
777
|
+
# read-modify-write on the FRESH record: the submitted job may
|
|
778
|
+
# already be saving its own fields
|
|
779
|
+
latest = load_record(root, record.run_id)
|
|
780
|
+
save_record(
|
|
781
|
+
root,
|
|
782
|
+
replace(
|
|
783
|
+
latest,
|
|
784
|
+
followup_job_id=job_id,
|
|
785
|
+
wake_attempts=latest.wake_attempts + 1,
|
|
786
|
+
),
|
|
787
|
+
now,
|
|
788
|
+
)
|
|
789
|
+
submitted.append((record.run_id, job_id))
|
|
790
|
+
except (SlurmError, Exception) as exc:
|
|
791
|
+
log.warning("in-review service failed for %s: %s", record.run_id, exc)
|
|
792
|
+
return ended, submitted
|
|
793
|
+
|
|
794
|
+
|
|
795
|
+
def _last_worked_ts(root: Path) -> float | None:
|
|
796
|
+
"""The `ts` of the last tick that COMPLETED its work, or None if there is no
|
|
797
|
+
readable marker (first tick, or a corrupt/missing file). The coalesce guard
|
|
798
|
+
keys on this, NOT the heartbeat: the heartbeat is stamped at tick START (for
|
|
799
|
+
the watchdog), so a tick that crashes mid-work still leaves a fresh
|
|
800
|
+
heartbeat — coalescing on that would suppress the very recovery tick. The
|
|
801
|
+
work marker is written only at a full tick's END, so a failed tick never
|
|
802
|
+
hides behind it."""
|
|
803
|
+
# The marker is a best-effort optimization we write ourselves; ANY failure
|
|
804
|
+
# reading/parsing/converting a corrupt file (OSError, ValueError,
|
|
805
|
+
# OverflowError on a huge int, RecursionError on deep nesting, ...) must
|
|
806
|
+
# degrade to "no marker" so coalesce simply proceeds — it can never crash the
|
|
807
|
+
# tick before its heartbeat. bool is an int subclass, so exclude it; inf/nan
|
|
808
|
+
# are not usable elapsed anchors.
|
|
809
|
+
try:
|
|
810
|
+
payload = json.loads((root / WORK_MARKER_NAME).read_text())
|
|
811
|
+
ts = payload.get("ts") if isinstance(payload, dict) else None
|
|
812
|
+
if not isinstance(ts, int | float) or isinstance(ts, bool):
|
|
813
|
+
return None
|
|
814
|
+
val = float(ts)
|
|
815
|
+
return val if math.isfinite(val) else None
|
|
816
|
+
except Exception:
|
|
817
|
+
return None
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def _mark_worked(root: Path, now: float) -> None:
|
|
821
|
+
"""Record that a tick completed its work at `now` — the coalesce signal.
|
|
822
|
+
Best-effort: a marker that cannot be written must not fail the tick."""
|
|
823
|
+
try:
|
|
824
|
+
tmp = root / f".{WORK_MARKER_NAME}.tmp"
|
|
825
|
+
tmp.write_text(json.dumps({"ts": now}))
|
|
826
|
+
os.replace(tmp, root / WORK_MARKER_NAME)
|
|
827
|
+
except OSError as exc:
|
|
828
|
+
log.warning("work-marker write failed: %s", exc)
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
def mark_tick_complete(root: Path, report: TickReport, now: float) -> None:
|
|
832
|
+
"""Stamp the coalesce marker iff the tick actually did work, at real
|
|
833
|
+
COMPLETION time (the caller passes time.time() AFTER tick() returns). A
|
|
834
|
+
paused/coalesced tick leaves it untouched; a tick that raised never reaches
|
|
835
|
+
here — so only a genuinely completed tick can coalesce the next one, and a
|
|
836
|
+
long tick's marker reflects when it finished, not when it started."""
|
|
837
|
+
if not report.paused and not report.coalesced:
|
|
838
|
+
_mark_worked(root, now)
|
|
839
|
+
|
|
840
|
+
|
|
841
|
+
def write_heartbeat(root: Path, now: float, disk: dict[str, object] | None = None) -> None:
|
|
842
|
+
"""Best-effort: a heartbeat that cannot be written (full disk) must not
|
|
843
|
+
kill the tick — the tick can still end runs and post to GitHub."""
|
|
844
|
+
payload: dict[str, object] = {"ts": now, "host": socket.gethostname(), "pid": os.getpid()}
|
|
845
|
+
if disk is not None:
|
|
846
|
+
payload["disk"] = disk
|
|
847
|
+
try:
|
|
848
|
+
tmp = root / f".{HEARTBEAT_NAME}.tmp"
|
|
849
|
+
tmp.write_text(json.dumps(payload))
|
|
850
|
+
os.replace(tmp, root / HEARTBEAT_NAME)
|
|
851
|
+
except OSError as exc:
|
|
852
|
+
log.warning("heartbeat write failed: %s", exc)
|
|
853
|
+
|
|
854
|
+
|
|
855
|
+
def _holder_alive(compute: Compute, lease_job_id: str) -> bool | None:
|
|
856
|
+
"""True/False when Slurm answered; None when it could not (an outage
|
|
857
|
+
must not look like a dead holder)."""
|
|
858
|
+
if not lease_job_id:
|
|
859
|
+
return None
|
|
860
|
+
try:
|
|
861
|
+
state = compute.status(lease_job_id)
|
|
862
|
+
except SlurmQueryError:
|
|
863
|
+
return None
|
|
864
|
+
return not (is_terminal(state) or state == GONE)
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def _wake(
|
|
868
|
+
root: Path,
|
|
869
|
+
record: RunRecord,
|
|
870
|
+
reason: str,
|
|
871
|
+
dispatcher: WakeDispatcher,
|
|
872
|
+
now: float,
|
|
873
|
+
holder: str,
|
|
874
|
+
) -> bool:
|
|
875
|
+
"""Lease-guarded wake. True when this tick delivered (or handed off) it.
|
|
876
|
+
|
|
877
|
+
The attempt counter is bumped BEFORE dispatch, so a dispatcher that dies
|
|
878
|
+
mid-delivery still counts toward the stuck threshold.
|
|
879
|
+
"""
|
|
880
|
+
if not acquire_lease(root, record.run_id, holder, holder_job_id="", now=now):
|
|
881
|
+
return False
|
|
882
|
+
bumped = replace(
|
|
883
|
+
record,
|
|
884
|
+
wake_attempts=record.wake_attempts + 1,
|
|
885
|
+
# repair legacy records as we touch them: save_record (rightly)
|
|
886
|
+
# refuses to write a waiting run without a deadline
|
|
887
|
+
deadline=record.deadline if record.deadline > 0 else now,
|
|
888
|
+
)
|
|
889
|
+
save_record(root, bumped, now)
|
|
890
|
+
try:
|
|
891
|
+
holder_job = dispatcher.dispatch(bumped, reason)
|
|
892
|
+
except Exception as exc:
|
|
893
|
+
log.warning("wake dispatch failed for %s: %s: %s", record.run_id, type(exc).__name__, exc)
|
|
894
|
+
release_lease(root, record.run_id)
|
|
895
|
+
return False
|
|
896
|
+
if holder_job and not local_mode():
|
|
897
|
+
# An async wake job now owns the lease; it releases on completion,
|
|
898
|
+
# and the TTL/holder-dead check reaps it if it dies.
|
|
899
|
+
update_lease_holder(root, record.run_id, f"wake-job:{holder_job}", holder_job, now)
|
|
900
|
+
else:
|
|
901
|
+
# No job to hand the lease to — or a LOCAL dispatch, which ran the
|
|
902
|
+
# whole wake synchronously: the attempt already finished and released
|
|
903
|
+
# its own lease, and recreating one under a terminal job id would
|
|
904
|
+
# make the next sweep reap a corpse instead of delivering.
|
|
905
|
+
release_lease(root, record.run_id)
|
|
906
|
+
return True
|
|
907
|
+
|
|
908
|
+
|
|
909
|
+
WAKE_SPEC_NAME = "wake-spec.json"
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def dispatch_wake_armed(root: Path) -> bool:
|
|
913
|
+
"""The operator's on-switch for dispatched wakes: the env var, or the
|
|
914
|
+
sentinel file (touch/rm, no chain restart). Read by the tick and by every
|
|
915
|
+
park, so a disarm takes effect at once."""
|
|
916
|
+
return (
|
|
917
|
+
bool(os.environ.get("AUTORESEARCH_DISPATCH_WAKE", "").strip())
|
|
918
|
+
or (root / DISPATCH_WAKE_SENTINEL).exists()
|
|
919
|
+
)
|
|
920
|
+
|
|
921
|
+
|
|
922
|
+
def write_wake_spec(root: Path, spec: FollowupSpec) -> None:
|
|
923
|
+
"""Publish the tick's wake recipe for the jobs that park runs: a park
|
|
924
|
+
submits its own wake (`arm_wake`) with exactly the tick's settings, so
|
|
925
|
+
dispatched wakes stay one recipe with one owner."""
|
|
926
|
+
data = {k: (str(v) if isinstance(v, Path) else v) for k, v in asdict(spec).items()}
|
|
927
|
+
tmp = root / f".{WAKE_SPEC_NAME}.{os.getpid()}.tmp"
|
|
928
|
+
tmp.write_text(json.dumps(data))
|
|
929
|
+
os.replace(tmp, root / WAKE_SPEC_NAME)
|
|
930
|
+
|
|
931
|
+
|
|
932
|
+
def remove_wake_spec(root: Path) -> None:
|
|
933
|
+
with contextlib.suppress(FileNotFoundError):
|
|
934
|
+
(root / WAKE_SPEC_NAME).unlink()
|
|
935
|
+
|
|
936
|
+
|
|
937
|
+
def load_wake_spec(root: Path) -> FollowupSpec | None:
|
|
938
|
+
"""The published wake recipe, or None when dispatched wakes are not armed
|
|
939
|
+
(or the file is unreadable — the sweep still delivers)."""
|
|
940
|
+
try:
|
|
941
|
+
data = json.loads((root / WAKE_SPEC_NAME).read_text())
|
|
942
|
+
except (OSError, ValueError):
|
|
943
|
+
return None
|
|
944
|
+
if not isinstance(data, dict):
|
|
945
|
+
return None
|
|
946
|
+
names = {f.name for f in FollowupSpec.__dataclass_fields__.values()}
|
|
947
|
+
kwargs: dict[str, Any] = {
|
|
948
|
+
k: (Path(v) if k in ("run_root", "home") else v) for k, v in data.items() if k in names
|
|
949
|
+
}
|
|
950
|
+
try:
|
|
951
|
+
return FollowupSpec(**kwargs)
|
|
952
|
+
except TypeError:
|
|
953
|
+
return None
|
|
954
|
+
|
|
955
|
+
|
|
956
|
+
def arm_wake(
|
|
957
|
+
root: Path, record: RunRecord, dispatcher: WakeDispatcher, now: float, *, holder_job_id: str
|
|
958
|
+
) -> str:
|
|
959
|
+
"""Submit a parked run's wake NOW, depending on the jobs it waits on, so
|
|
960
|
+
it fires the moment they finish instead of a sweep cadence (plus grace)
|
|
961
|
+
later. The same job and lease as a sweep-delivered wake: a wake job that
|
|
962
|
+
parks again hands its own lease to the wake it arms; any other holder (a
|
|
963
|
+
tick mid-delivery) keeps it and the sweep delivers as before. Arming is
|
|
964
|
+
not a redelivery, so it leaves `wake_attempts` — the sweep's stuck
|
|
965
|
+
counter — alone: an eval requeued by preemption fires the afterany
|
|
966
|
+
early, the wake re-parks, and that must not count toward STUCK. Returns
|
|
967
|
+
the wake job id, or "" when nothing was armed. A park with nothing to
|
|
968
|
+
depend on (a checkpoint sleep, a blind park) is not armed: it rides the
|
|
969
|
+
deadline floor, as before."""
|
|
970
|
+
if not _poll_targets(record):
|
|
971
|
+
return ""
|
|
972
|
+
lease = read_lease(root, record.run_id)
|
|
973
|
+
if lease is not None:
|
|
974
|
+
if not holder_job_id or lease.holder_job_id != holder_job_id:
|
|
975
|
+
return ""
|
|
976
|
+
elif not acquire_lease(
|
|
977
|
+
root, record.run_id, f"park:{holder_job_id or os.getpid()}", holder_job_id="", now=now
|
|
978
|
+
):
|
|
979
|
+
return ""
|
|
980
|
+
if record.deadline <= 0: # a waiting record always carries a deadline
|
|
981
|
+
record = replace(record, deadline=now)
|
|
982
|
+
save_record(root, record, now)
|
|
983
|
+
try:
|
|
984
|
+
job = dispatcher.dispatch(record, "parked")
|
|
985
|
+
except Exception as exc:
|
|
986
|
+
log.warning("arming the wake failed for %s: %s: %s", record.run_id, type(exc).__name__, exc)
|
|
987
|
+
job = ""
|
|
988
|
+
if job:
|
|
989
|
+
update_lease_holder(root, record.run_id, f"wake-job:{job}", job, now)
|
|
990
|
+
elif lease is None:
|
|
991
|
+
release_lease(root, record.run_id)
|
|
992
|
+
return job
|
|
993
|
+
|
|
994
|
+
|
|
995
|
+
def _armed_wake_lost(
|
|
996
|
+
root: Path,
|
|
997
|
+
compute: Compute,
|
|
998
|
+
record: RunRecord,
|
|
999
|
+
lease: Lease,
|
|
1000
|
+
now: float,
|
|
1001
|
+
grace_s: float,
|
|
1002
|
+
dry_run: bool,
|
|
1003
|
+
) -> bool:
|
|
1004
|
+
"""An armed wake that is still PENDING on its dependency after every job
|
|
1005
|
+
it waits on has been terminal for a full grace window is not coming
|
|
1006
|
+
(Slurm reports the dependency as never satisfiable, or the afterany was
|
|
1007
|
+
lost). Cancel it so the sweep redelivers; the lease is then reaped.
|
|
1008
|
+
|
|
1009
|
+
A wake the SITE moved off the partition it was submitted to may be
|
|
1010
|
+
starving there — on Torch a pending job can be shifted to a lower-tier
|
|
1011
|
+
partition (2026-09-02, wake 16787511 sat on `all` for hours) — but
|
|
1012
|
+
relocation alone is routine (jobs move to `cs` while still waiting on
|
|
1013
|
+
their dependencies and start on time). So a relocated holder counts as
|
|
1014
|
+
lost only once every job it waited on is terminal AND the grace window
|
|
1015
|
+
has run out without it starting; then it is cancelled and redelivered
|
|
1016
|
+
onto the requested partition."""
|
|
1017
|
+
if not lease.holder_job_id:
|
|
1018
|
+
return False
|
|
1019
|
+
job_ids = _poll_targets(record)
|
|
1020
|
+
if not job_ids:
|
|
1021
|
+
return False
|
|
1022
|
+
try:
|
|
1023
|
+
holder_state = compute.status(lease.holder_job_id)
|
|
1024
|
+
if not is_pending(holder_state):
|
|
1025
|
+
return False
|
|
1026
|
+
reason = compute.pending_reason(lease.holder_job_id)
|
|
1027
|
+
states = [compute.status(jid) for jid in job_ids]
|
|
1028
|
+
except SlurmQueryError:
|
|
1029
|
+
return False
|
|
1030
|
+
moved = _moved_off_partition(root, compute, lease.holder_job_id)
|
|
1031
|
+
if reason == "DependencyNeverSatisfied":
|
|
1032
|
+
pass
|
|
1033
|
+
elif not all(is_terminal(s) for s in states):
|
|
1034
|
+
return False # its dependencies are still running: nothing to redeliver yet
|
|
1035
|
+
elif reason != "Dependency" and not moved:
|
|
1036
|
+
return False
|
|
1037
|
+
else:
|
|
1038
|
+
# Dependency-pending past its dependencies, or RELOCATED and eligible:
|
|
1039
|
+
# both get the grace window. Relocation alone is normal (the site
|
|
1040
|
+
# moves pending jobs routinely); a relocated wake counts as lost only
|
|
1041
|
+
# when every job it waited on is terminal and it still has not
|
|
1042
|
+
# started once the grace has run out — cancelling earlier would only
|
|
1043
|
+
# reset its queue age and burn a wake attempt.
|
|
1044
|
+
if moved:
|
|
1045
|
+
log.info(
|
|
1046
|
+
"armed wake %s for %s sits on partition %s (asked for %s) past its dependencies",
|
|
1047
|
+
lease.holder_job_id,
|
|
1048
|
+
record.run_id,
|
|
1049
|
+
moved[0],
|
|
1050
|
+
moved[1],
|
|
1051
|
+
)
|
|
1052
|
+
if record.terminal_seen <= 0:
|
|
1053
|
+
if not dry_run:
|
|
1054
|
+
save_record(root, replace(record, terminal_seen=now), now)
|
|
1055
|
+
return False
|
|
1056
|
+
if now - record.terminal_seen < grace_s:
|
|
1057
|
+
return False
|
|
1058
|
+
if dry_run:
|
|
1059
|
+
return True
|
|
1060
|
+
try:
|
|
1061
|
+
compute.cancel(lease.holder_job_id)
|
|
1062
|
+
# only a cancellation Slurm confirms lets the sweep redeliver: a
|
|
1063
|
+
# still-pending wake would otherwise run beside its replacement
|
|
1064
|
+
return not is_pending(compute.status(lease.holder_job_id))
|
|
1065
|
+
except Exception:
|
|
1066
|
+
return False
|
|
1067
|
+
|
|
1068
|
+
|
|
1069
|
+
def sweep(
|
|
1070
|
+
root: Path,
|
|
1071
|
+
compute: Compute,
|
|
1072
|
+
dispatcher: WakeDispatcher,
|
|
1073
|
+
now: float,
|
|
1074
|
+
grace_s: float = DEFAULT_GRACE_S,
|
|
1075
|
+
lease_ttl_s: float = DEFAULT_LEASE_TTL_S,
|
|
1076
|
+
dry_run: bool = False,
|
|
1077
|
+
) -> TickReport:
|
|
1078
|
+
"""The backup wake layers, applied to every waiting run.
|
|
1079
|
+
|
|
1080
|
+
dry_run reports what WOULD happen with zero writes — no leases, no
|
|
1081
|
+
attempt counters, no dispatch.
|
|
1082
|
+
"""
|
|
1083
|
+
woken: list[tuple[str, str]] = []
|
|
1084
|
+
deferred: list[str] = []
|
|
1085
|
+
reaped: list[str] = []
|
|
1086
|
+
stuck: list[str] = []
|
|
1087
|
+
holder = f"tick:{socket.gethostname()}:{os.getpid()}"
|
|
1088
|
+
records = [r for r in list_runs(root) if r.state == WAITING]
|
|
1089
|
+
|
|
1090
|
+
def wake(record: RunRecord, reason: str, tag: str) -> None:
|
|
1091
|
+
if dry_run or _wake(root, record, reason, dispatcher, now, holder):
|
|
1092
|
+
woken.append((record.run_id, tag))
|
|
1093
|
+
|
|
1094
|
+
for record in records:
|
|
1095
|
+
try:
|
|
1096
|
+
_sweep_one(
|
|
1097
|
+
root,
|
|
1098
|
+
compute,
|
|
1099
|
+
dispatcher,
|
|
1100
|
+
now,
|
|
1101
|
+
grace_s,
|
|
1102
|
+
lease_ttl_s,
|
|
1103
|
+
dry_run,
|
|
1104
|
+
record,
|
|
1105
|
+
holder,
|
|
1106
|
+
wake,
|
|
1107
|
+
deferred,
|
|
1108
|
+
reaped,
|
|
1109
|
+
stuck,
|
|
1110
|
+
)
|
|
1111
|
+
except Exception as exc:
|
|
1112
|
+
log.warning("sweep failed on %s: %s: %s", record.run_id, type(exc).__name__, exc)
|
|
1113
|
+
|
|
1114
|
+
return TickReport(
|
|
1115
|
+
swept=len(records),
|
|
1116
|
+
woken=tuple(woken),
|
|
1117
|
+
deferred=tuple(deferred),
|
|
1118
|
+
reaped_leases=tuple(reaped),
|
|
1119
|
+
stuck=tuple(stuck),
|
|
1120
|
+
# NOT the global dry_run: that flag only dries WAKE delivery;
|
|
1121
|
+
# ending killed climbs' records dispatches nothing and must run
|
|
1122
|
+
# live even while wakes stay dry.
|
|
1123
|
+
implementing_ended=tuple(_sweep_implementing(root, compute, now, grace_s)),
|
|
1124
|
+
)
|
|
1125
|
+
|
|
1126
|
+
|
|
1127
|
+
RESEARCH_LOG_BRANCH = "research-log"
|
|
1128
|
+
RESEARCH_LOG_MARKER = marker("research-log")
|
|
1129
|
+
RESEARCH_LOG_PER_TICK = 3
|
|
1130
|
+
|
|
1131
|
+
|
|
1132
|
+
def _ledger_marker(root: Path, run_id: str) -> Path:
|
|
1133
|
+
return run_dir(root, run_id) / "ledger-published"
|
|
1134
|
+
|
|
1135
|
+
|
|
1136
|
+
def _ledger_since(root: Path, target: str) -> Path:
|
|
1137
|
+
return root / ("research-log-since-" + target.replace("/", "__"))
|
|
1138
|
+
|
|
1139
|
+
|
|
1140
|
+
def _ledger_issue_cache(root: Path, target: str) -> Path:
|
|
1141
|
+
return root / ("research-log-issue-" + target.replace("/", "__"))
|
|
1142
|
+
|
|
1143
|
+
|
|
1144
|
+
def service_research_log(root: Path, github: Any, spec: FollowupSpec, now: float) -> int:
|
|
1145
|
+
"""STATE-driven ledger (terra #170 r1: wiring publisher calls at terminal
|
|
1146
|
+
sites missed four terminal paths — attempt error, zero-change resume,
|
|
1147
|
+
the direct live terminal, steward): any run of this target whose
|
|
1148
|
+
terminal report exists but carries no published marker gets archived on
|
|
1149
|
+
the `research-log` branch and a two-line pointer routed — to the run's
|
|
1150
|
+
claimed issue never (it already got the full finish post), else to an
|
|
1151
|
+
open order issue naming the benchmark, else to the rolling log issue
|
|
1152
|
+
whose number is CACHED in the state dir (no 300-issue pagination scan,
|
|
1153
|
+
no duplicate creation). Running in the single tick also removes the
|
|
1154
|
+
concurrent-first-archive races by construction. The pointer posts only
|
|
1155
|
+
AFTER a successful archive (no dead links); the marker is written only
|
|
1156
|
+
after full success, so a crashed publish retries next tick. Runs ended
|
|
1157
|
+
before the feature's first pass are marker-stamped silently (no
|
|
1158
|
+
backfill spam).
|
|
1159
|
+
"""
|
|
1160
|
+
since_path = _ledger_since(root, spec.target)
|
|
1161
|
+
first_pass = not since_path.exists()
|
|
1162
|
+
if first_pass:
|
|
1163
|
+
with contextlib.suppress(OSError):
|
|
1164
|
+
since_path.write_text(str(now))
|
|
1165
|
+
published = 0
|
|
1166
|
+
for record in list_runs(root):
|
|
1167
|
+
if record.target != spec.target or record.state not in (ENDED, IN_REVIEW):
|
|
1168
|
+
continue
|
|
1169
|
+
marker = _ledger_marker(root, record.run_id)
|
|
1170
|
+
report_path = run_dir(root, record.run_id) / "report.md"
|
|
1171
|
+
state = ""
|
|
1172
|
+
with contextlib.suppress(OSError):
|
|
1173
|
+
state = marker.read_text()
|
|
1174
|
+
if state.startswith(("done", "adopted")) or not report_path.exists():
|
|
1175
|
+
continue
|
|
1176
|
+
if first_pass and (record.updated or record.created) < now:
|
|
1177
|
+
# adopt pre-feature history silently: browsable going forward,
|
|
1178
|
+
# no retroactive issue spam. Only records OLDER than the since
|
|
1179
|
+
# marker — a run that goes terminal during this very pass is new
|
|
1180
|
+
# work and publishes normally (terra #170 r2).
|
|
1181
|
+
with contextlib.suppress(OSError):
|
|
1182
|
+
marker.write_text("adopted-unpublished")
|
|
1183
|
+
continue
|
|
1184
|
+
if published >= RESEARCH_LOG_PER_TICK:
|
|
1185
|
+
break
|
|
1186
|
+
try:
|
|
1187
|
+
report = report_path.read_text()
|
|
1188
|
+
except OSError:
|
|
1189
|
+
continue
|
|
1190
|
+
outcome = record.ending or ("improved" if record.state == IN_REVIEW else "ended")
|
|
1191
|
+
if _publish_ledger_entry(github, spec.target, root, record, outcome, report, marker, state):
|
|
1192
|
+
published += 1
|
|
1193
|
+
return published
|
|
1194
|
+
|
|
1195
|
+
|
|
1196
|
+
def _publish_ledger_entry(
|
|
1197
|
+
github: Any,
|
|
1198
|
+
target: str,
|
|
1199
|
+
root: Path,
|
|
1200
|
+
record: RunRecord,
|
|
1201
|
+
outcome: str,
|
|
1202
|
+
report: str,
|
|
1203
|
+
marker: Path,
|
|
1204
|
+
state: str,
|
|
1205
|
+
) -> bool:
|
|
1206
|
+
"""Staged publish whose marker doubles as a WRITABILITY PROBE: stages
|
|
1207
|
+
are "archived" -> "pointer-pending" -> "done", and no pointer is ever
|
|
1208
|
+
posted in a pass where a marker write is failing (terra #170 r4: a
|
|
1209
|
+
persistently unwritable marker must stall the publish, not stream a
|
|
1210
|
+
duplicate pointer every tick). A pointer failure retries pointer-only;
|
|
1211
|
+
a marker lost after full success re-posts at most once; the residual
|
|
1212
|
+
crash-between-probe-and-post window costs at most one duplicate per
|
|
1213
|
+
incident, never an unbounded stream."""
|
|
1214
|
+
from datetime import UTC, datetime
|
|
1215
|
+
|
|
1216
|
+
date = datetime.fromtimestamp(record.updated or record.created, tz=UTC).strftime("%Y-%m-%d")
|
|
1217
|
+
path = f"reports/{date}-{record.run_id}.md"
|
|
1218
|
+
# an earlier pass may have archived under an earlier date (an in-review
|
|
1219
|
+
# archive whose record re-stamped `updated` at ENDED): the marker's own
|
|
1220
|
+
# second line is the authoritative path for retries and pointers
|
|
1221
|
+
prior = state.splitlines()
|
|
1222
|
+
if len(prior) > 1 and prior[1].startswith("reports/") and prior[1].endswith(".md"):
|
|
1223
|
+
path = prior[1]
|
|
1224
|
+
|
|
1225
|
+
def _mark(value: str) -> bool:
|
|
1226
|
+
try:
|
|
1227
|
+
# the path rides the marker so readers (the board) never have to
|
|
1228
|
+
# re-derive the date from a timestamp that may have moved on
|
|
1229
|
+
marker.write_text(value + "\n" + path)
|
|
1230
|
+
return True
|
|
1231
|
+
except OSError as exc:
|
|
1232
|
+
log.warning("ledger marker write failed for %s: %s", record.run_id, exc)
|
|
1233
|
+
return False
|
|
1234
|
+
|
|
1235
|
+
try:
|
|
1236
|
+
if not state.startswith(("archived", "pointer-pending")):
|
|
1237
|
+
if not github.ensure_branch(target, RESEARCH_LOG_BRANCH):
|
|
1238
|
+
return False
|
|
1239
|
+
if not github.put_file(
|
|
1240
|
+
target,
|
|
1241
|
+
path,
|
|
1242
|
+
report,
|
|
1243
|
+
RESEARCH_LOG_BRANCH,
|
|
1244
|
+
f"research log: {record.run_id} ({outcome})",
|
|
1245
|
+
):
|
|
1246
|
+
return False # retry the whole publish next tick
|
|
1247
|
+
if not _mark("archived"):
|
|
1248
|
+
return False # unwritable state: stall BEFORE any pointer
|
|
1249
|
+
# the probe: a fresh successful write is the license to post
|
|
1250
|
+
if not _mark("pointer-pending"):
|
|
1251
|
+
return False
|
|
1252
|
+
url = f"https://github.com/{target}/blob/{RESEARCH_LOG_BRANCH}/{path}"
|
|
1253
|
+
line = f"**{outcome}** `{record.benchmark}` — [report]({url})"
|
|
1254
|
+
if record.pr_url:
|
|
1255
|
+
line += f" · {record.pr_url}"
|
|
1256
|
+
if record.issue_number:
|
|
1257
|
+
_mark("done") # the claimed issue already received the full finish
|
|
1258
|
+
return True
|
|
1259
|
+
bench = record.benchmark.casefold()
|
|
1260
|
+
posted = False
|
|
1261
|
+
if bench:
|
|
1262
|
+
for issue in github.list_open_issues(target):
|
|
1263
|
+
text = f"{issue.get('title', '')}\n{issue.get('body') or ''}"
|
|
1264
|
+
if has_marker(text, "research-log") or issue.get("pull_request"):
|
|
1265
|
+
continue
|
|
1266
|
+
if bench in text.casefold():
|
|
1267
|
+
github.comment(target, int(issue["number"]), line)
|
|
1268
|
+
posted = True
|
|
1269
|
+
break
|
|
1270
|
+
if not posted:
|
|
1271
|
+
cache = _ledger_issue_cache(root, target)
|
|
1272
|
+
log_issue = 0
|
|
1273
|
+
with contextlib.suppress(OSError, ValueError):
|
|
1274
|
+
log_issue = int(cache.read_text().strip())
|
|
1275
|
+
if not log_issue:
|
|
1276
|
+
# cache miss (first use, or a lost/failed cache write): find
|
|
1277
|
+
# the rolling issue by its marker BEFORE creating another —
|
|
1278
|
+
# the cache is a fast path, never the source of truth
|
|
1279
|
+
# (terra #170 r5: a failed cache write must not duplicate
|
|
1280
|
+
# the rolling issue)
|
|
1281
|
+
for issue in github.list_open_issues(target):
|
|
1282
|
+
if has_marker(str(issue.get("body") or ""), "research-log"):
|
|
1283
|
+
log_issue = int(issue.get("number", 0))
|
|
1284
|
+
break
|
|
1285
|
+
if not log_issue:
|
|
1286
|
+
log_issue = github.create_issue(
|
|
1287
|
+
target,
|
|
1288
|
+
"Research log",
|
|
1289
|
+
f"{RESEARCH_LOG_MARKER}\nOne two-line comment per finished "
|
|
1290
|
+
f"autoresearch run — full reports live on the [`{RESEARCH_LOG_BRANCH}`]"
|
|
1291
|
+
f"(https://github.com/{target}/tree/{RESEARCH_LOG_BRANCH}/reports) "
|
|
1292
|
+
"branch. Results relevant to an open order issue are posted "
|
|
1293
|
+
"there instead.",
|
|
1294
|
+
)
|
|
1295
|
+
if log_issue:
|
|
1296
|
+
with contextlib.suppress(OSError):
|
|
1297
|
+
cache.write_text(str(log_issue))
|
|
1298
|
+
try:
|
|
1299
|
+
github.comment(target, log_issue, line)
|
|
1300
|
+
except Exception:
|
|
1301
|
+
# a stale cached number (locked/deleted/transferred issue)
|
|
1302
|
+
# must not stall delivery forever: drop the cache so the
|
|
1303
|
+
# next pass re-discovers or re-creates (terra #170 r5)
|
|
1304
|
+
with contextlib.suppress(OSError):
|
|
1305
|
+
cache.unlink()
|
|
1306
|
+
raise
|
|
1307
|
+
_mark("done") # write just proved out via the probe; failure = freak
|
|
1308
|
+
return True
|
|
1309
|
+
except Exception as exc: # advisory ledger: never fail the tick
|
|
1310
|
+
log.warning("research-log publish failed for %s: %s", record.run_id, exc)
|
|
1311
|
+
return False
|
|
1312
|
+
|
|
1313
|
+
|
|
1314
|
+
def _kill_stamp(root: Path, run_id: str) -> Path:
|
|
1315
|
+
return run_dir(root, run_id) / "attempt-terminal-seen"
|
|
1316
|
+
|
|
1317
|
+
|
|
1318
|
+
def _sweep_implementing(root: Path, compute: Compute, now: float, grace_s: float) -> list[str]:
|
|
1319
|
+
"""End `implementing` records whose climb job died without a verdict.
|
|
1320
|
+
|
|
1321
|
+
A climb that CRASHES contains its own ending (attempt.py); a climb that is
|
|
1322
|
+
KILLED — walltime, preemption, scancel after the SIGTERM grace, node
|
|
1323
|
+
death — leaves no exception to contain, so this pass records the ending
|
|
1324
|
+
(the picker's stranded guard only frees the lane). Slurm truth decides:
|
|
1325
|
+
job terminal or GONE, plus a grace so a just-finished healthy climb can
|
|
1326
|
+
write its own final state first. Outage never reads as dead. Legacy
|
|
1327
|
+
records without a job id age out on the stranded window instead.
|
|
1328
|
+
"""
|
|
1329
|
+
ended: list[str] = []
|
|
1330
|
+
for record in list_runs(root):
|
|
1331
|
+
if record.state != IMPLEMENTING:
|
|
1332
|
+
continue
|
|
1333
|
+
try:
|
|
1334
|
+
if record.run_job_id:
|
|
1335
|
+
try:
|
|
1336
|
+
state = compute.status(record.run_job_id)
|
|
1337
|
+
except SlurmQueryError:
|
|
1338
|
+
continue # Slurm outage must not read as a dead job
|
|
1339
|
+
if not (is_terminal(state) or state == GONE):
|
|
1340
|
+
continue # alive; the climb owns its own record
|
|
1341
|
+
# Grace runs from FIRST OBSERVED terminal: during Slurm's
|
|
1342
|
+
# KillWait the job already reports terminal while the
|
|
1343
|
+
# SIGTERM containment is still writing its honest ending —
|
|
1344
|
+
# record age would be hours and protect nothing. The stamp
|
|
1345
|
+
# is a write-once SIDECAR file, never a record write: while
|
|
1346
|
+
# the climb may still be alive the sweep must not touch the
|
|
1347
|
+
# record at all (a load-modify-replace here could revert a
|
|
1348
|
+
# concurrently written ending), and the waiting sweep's
|
|
1349
|
+
# terminal_seen field stays reserved for the EXPERIMENT job.
|
|
1350
|
+
stamp = _kill_stamp(root, record.run_id)
|
|
1351
|
+
if not stamp.exists():
|
|
1352
|
+
try:
|
|
1353
|
+
fd = os.open(stamp, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644)
|
|
1354
|
+
try:
|
|
1355
|
+
os.write(fd, f"{now}".encode())
|
|
1356
|
+
finally:
|
|
1357
|
+
os.close(fd)
|
|
1358
|
+
except FileExistsError:
|
|
1359
|
+
pass # a concurrent tick stamped it; its clock stands
|
|
1360
|
+
except OSError as exc:
|
|
1361
|
+
log.warning("kill-stamp write failed for %s: %s", record.run_id, exc)
|
|
1362
|
+
continue
|
|
1363
|
+
# An empty stamp (write failed after create — the disk-full
|
|
1364
|
+
# case — or a concurrent tick mid-write) must fall back to
|
|
1365
|
+
# mtime, NOT to epoch 0, which would skip the grace outright.
|
|
1366
|
+
try:
|
|
1367
|
+
raw = stamp.read_text().strip()
|
|
1368
|
+
seen = float(raw) if raw else stamp.stat().st_mtime
|
|
1369
|
+
except (OSError, ValueError):
|
|
1370
|
+
try:
|
|
1371
|
+
seen = stamp.stat().st_mtime
|
|
1372
|
+
except OSError:
|
|
1373
|
+
continue # stamp vanished mid-read; next tick decides
|
|
1374
|
+
if now - seen < grace_s:
|
|
1375
|
+
continue
|
|
1376
|
+
note = f"climb job {record.run_job_id} ended {state} without a verdict"
|
|
1377
|
+
else:
|
|
1378
|
+
# No Slurm evidence at all (legacy record, or a manual dev
|
|
1379
|
+
# invocation without SLURM_JOB_ID): only the run DEADLINE —
|
|
1380
|
+
# past which nothing legitimately lives — justifies a
|
|
1381
|
+
# terminal verdict; the shorter stranded window merely
|
|
1382
|
+
# frees the picker lane and must not author endings.
|
|
1383
|
+
deadline = record.deadline if record.deadline > 0 else (record.created + 24 * 3600)
|
|
1384
|
+
if now < deadline:
|
|
1385
|
+
continue
|
|
1386
|
+
note = "implementing with no recorded climb job, past its run deadline"
|
|
1387
|
+
fresh = load_record(root, record.run_id)
|
|
1388
|
+
if fresh.state != IMPLEMENTING:
|
|
1389
|
+
continue # the climb landed its own ending meanwhile
|
|
1390
|
+
for jid in _poll_targets(fresh):
|
|
1391
|
+
# defensive: no current path records an experiment while
|
|
1392
|
+
# still implementing, but an orphan GPU job burning budget
|
|
1393
|
+
# after its run is declared dead must never survive one
|
|
1394
|
+
with contextlib.suppress(Exception):
|
|
1395
|
+
compute.cancel(jid)
|
|
1396
|
+
save_record(
|
|
1397
|
+
root,
|
|
1398
|
+
replace(
|
|
1399
|
+
fresh,
|
|
1400
|
+
state=ENDED,
|
|
1401
|
+
ending=ABORTED,
|
|
1402
|
+
ending_note=(
|
|
1403
|
+
f"{note} — ended by the sweep (a killed climb "
|
|
1404
|
+
f"leaves no exception to contain)"
|
|
1405
|
+
),
|
|
1406
|
+
),
|
|
1407
|
+
now,
|
|
1408
|
+
)
|
|
1409
|
+
# every ending produces a report — but never clobber one the
|
|
1410
|
+
# climb already wrote before it was killed
|
|
1411
|
+
report_path = run_dir(root, record.run_id) / "report.md"
|
|
1412
|
+
if not report_path.exists():
|
|
1413
|
+
try:
|
|
1414
|
+
report_path.write_text(
|
|
1415
|
+
f"# Run report — {record.target} / {record.benchmark}\n"
|
|
1416
|
+
f"Outcome: **aborted** (climb job killed)\n"
|
|
1417
|
+
f"Note: {note}\n"
|
|
1418
|
+
)
|
|
1419
|
+
except OSError as exc:
|
|
1420
|
+
log.warning("sweep report write failed for %s: %s", record.run_id, exc)
|
|
1421
|
+
log.warning("sweep ended implementing run %s: %s", record.run_id, note)
|
|
1422
|
+
ended.append(record.run_id)
|
|
1423
|
+
except Exception as exc: # per-record isolation, like the waiting sweep
|
|
1424
|
+
log.warning("implementing-sweep failed on %s: %s", record.run_id, exc)
|
|
1425
|
+
return ended
|
|
1426
|
+
|
|
1427
|
+
|
|
1428
|
+
def _moved_off_partition(root: Path, compute: Compute, job_id: str) -> tuple[str, str] | None:
|
|
1429
|
+
"""(actual, wanted) when a queued kernel job no longer sits in the
|
|
1430
|
+
partition the wake recipe asks for, else None. Unknown either way —
|
|
1431
|
+
no recipe, a compute without partitions, a failed query — is None:
|
|
1432
|
+
never cancel on doubt."""
|
|
1433
|
+
spec = load_wake_spec(root)
|
|
1434
|
+
if spec is None:
|
|
1435
|
+
return None
|
|
1436
|
+
wanted = spec.job_partition or spec.partition
|
|
1437
|
+
if not wanted:
|
|
1438
|
+
return None
|
|
1439
|
+
try:
|
|
1440
|
+
actual = compute.job_partition(job_id)
|
|
1441
|
+
except (SlurmQueryError, ValueError, AttributeError):
|
|
1442
|
+
return None
|
|
1443
|
+
# both sides may be comma-separated lists: moved only when the job holds
|
|
1444
|
+
# NONE of the partitions the recipe asked for; empty is unknown, not moved
|
|
1445
|
+
have = {p.strip() for p in actual.split(",") if p.strip()}
|
|
1446
|
+
want = {p.strip() for p in wanted.split(",") if p.strip()}
|
|
1447
|
+
if not have or have & want:
|
|
1448
|
+
return None
|
|
1449
|
+
return actual, wanted
|
|
1450
|
+
|
|
1451
|
+
|
|
1452
|
+
def _poll_targets(record: RunRecord) -> list[str]:
|
|
1453
|
+
"""Every Slurm job this run waits on. The single `experiment_job_id` is
|
|
1454
|
+
the common case; a MULTI-job park (candidate + siblings, or several author
|
|
1455
|
+
launches) records no single id — its jobs live in the stage's `afterany`
|
|
1456
|
+
dependency string, the one source that always names them all. Without
|
|
1457
|
+
this fallback a multi-job park is blind and rides the deadline floor."""
|
|
1458
|
+
if record.experiment_job_id:
|
|
1459
|
+
return [record.experiment_job_id]
|
|
1460
|
+
afterany = str((record.stage or {}).get("afterany", ""))
|
|
1461
|
+
return [t for t in afterany.split(":")[1:] if t]
|
|
1462
|
+
|
|
1463
|
+
|
|
1464
|
+
def _sweep_one(
|
|
1465
|
+
root: Path,
|
|
1466
|
+
compute: Compute,
|
|
1467
|
+
dispatcher: WakeDispatcher,
|
|
1468
|
+
now: float,
|
|
1469
|
+
grace_s: float,
|
|
1470
|
+
lease_ttl_s: float,
|
|
1471
|
+
dry_run: bool,
|
|
1472
|
+
record: RunRecord,
|
|
1473
|
+
holder: str,
|
|
1474
|
+
wake,
|
|
1475
|
+
deferred: list[str],
|
|
1476
|
+
reaped: list[str],
|
|
1477
|
+
stuck: list[str],
|
|
1478
|
+
) -> None:
|
|
1479
|
+
if True:
|
|
1480
|
+
# Leases first: a LIVE wake in flight owns this run — even the stuck
|
|
1481
|
+
# verdict must wait for it (its session may be the one that succeeds).
|
|
1482
|
+
lease = read_lease(root, record.run_id)
|
|
1483
|
+
if lease is not None:
|
|
1484
|
+
alive = _holder_alive(compute, lease.holder_job_id)
|
|
1485
|
+
if not lease_is_stale(lease, now, lease_ttl_s, alive):
|
|
1486
|
+
if not _armed_wake_lost(root, compute, record, lease, now, grace_s, dry_run):
|
|
1487
|
+
return
|
|
1488
|
+
record = load_record(root, record.run_id)
|
|
1489
|
+
if dry_run:
|
|
1490
|
+
reaped.append(record.run_id)
|
|
1491
|
+
return
|
|
1492
|
+
if not reap_lease(root, record.run_id, reaper=f"{os.getpid()}-{now}", expected=lease):
|
|
1493
|
+
return # a concurrent tick reaped it first; it owns redelivery
|
|
1494
|
+
reaped.append(record.run_id)
|
|
1495
|
+
|
|
1496
|
+
# Layer 5: too many failed attempts is a terminal, reported state.
|
|
1497
|
+
if record.wake_attempts >= MAX_WAKE_ATTEMPTS:
|
|
1498
|
+
if not dry_run:
|
|
1499
|
+
ended = replace(
|
|
1500
|
+
record,
|
|
1501
|
+
state=ENDED,
|
|
1502
|
+
ending=STUCK,
|
|
1503
|
+
ending_note=(
|
|
1504
|
+
f"{record.wake_attempts} wake attempts without the run leaving 'waiting'"
|
|
1505
|
+
),
|
|
1506
|
+
)
|
|
1507
|
+
save_record(root, ended, now)
|
|
1508
|
+
stuck.append(record.run_id)
|
|
1509
|
+
return
|
|
1510
|
+
|
|
1511
|
+
job_ids = _poll_targets(record)
|
|
1512
|
+
if not job_ids:
|
|
1513
|
+
# No job ids to poll. A BLIND PARK (the measurer could not read Slurm,
|
|
1514
|
+
# so `MeasurementPending` carried no ids) still hibernated with a
|
|
1515
|
+
# deadline — the deadline floor is its ONLY wake, so fire on it. A
|
|
1516
|
+
# genuinely mid-write record has no deadline and is left alone.
|
|
1517
|
+
# (A jobless CHECKPOINT SLEEP arrives here too — its deadline is
|
|
1518
|
+
# near-term by construction, attempt.py sizes it to the next sweep
|
|
1519
|
+
# pass, not the 12h queue slack that protects queued jobs.)
|
|
1520
|
+
if record.deadline > 0 and now > record.deadline:
|
|
1521
|
+
wake(record, "blind park past deadline", "deadline")
|
|
1522
|
+
return
|
|
1523
|
+
|
|
1524
|
+
try:
|
|
1525
|
+
states = [compute.status(jid) for jid in job_ids]
|
|
1526
|
+
except SlurmQueryError:
|
|
1527
|
+
# Layer 4's rule: query failure is "Slurm unknown", never "gone".
|
|
1528
|
+
deferred.append(record.run_id)
|
|
1529
|
+
return
|
|
1530
|
+
|
|
1531
|
+
# deadline <= 0 cannot be written by save_record for waiting runs;
|
|
1532
|
+
# if one exists anyway (legacy/hand-edited), treat it as already past
|
|
1533
|
+
# for GONE — a vanished-experiment wake is safe — but never for
|
|
1534
|
+
# PENDING, where the consequence would be cancelling a healthy job.
|
|
1535
|
+
past_deadline = record.deadline <= 0 or now > record.deadline
|
|
1536
|
+
|
|
1537
|
+
if all(is_terminal(s) for s in states):
|
|
1538
|
+
state = ",".join(sorted(set(states)))
|
|
1539
|
+
# Layer 3, with real grace: time runs from when the sweep FIRST
|
|
1540
|
+
# saw every job terminal, not from submission — the afterany
|
|
1541
|
+
# job gets the full window to deliver before the backup steps in.
|
|
1542
|
+
# Local compute has no afterany jobs to wait for (jobs are
|
|
1543
|
+
# terminal at submit): the sweep IS the delivery, so grace would
|
|
1544
|
+
# only cost a whole extra loop iteration.
|
|
1545
|
+
if local_mode():
|
|
1546
|
+
wake(record, f"experiment {state}", state)
|
|
1547
|
+
return
|
|
1548
|
+
if record.terminal_seen <= 0:
|
|
1549
|
+
if dry_run:
|
|
1550
|
+
# no writes in dry-run: report the would-wake now so the
|
|
1551
|
+
# terminal path is visible to live plumbing checks
|
|
1552
|
+
wake(record, f"experiment {state}", state)
|
|
1553
|
+
else:
|
|
1554
|
+
save_record(
|
|
1555
|
+
root,
|
|
1556
|
+
replace(
|
|
1557
|
+
record,
|
|
1558
|
+
terminal_seen=now,
|
|
1559
|
+
# repair legacy records as we touch them (see _wake)
|
|
1560
|
+
deadline=record.deadline if record.deadline > 0 else now,
|
|
1561
|
+
),
|
|
1562
|
+
now,
|
|
1563
|
+
)
|
|
1564
|
+
return
|
|
1565
|
+
if now - record.terminal_seen >= grace_s:
|
|
1566
|
+
wake(record, f"experiment {state}", state)
|
|
1567
|
+
elif any(is_pending(s) for s in states) and record.deadline > 0 and now > record.deadline:
|
|
1568
|
+
# Unschedulable in practice: cancel every non-terminal job
|
|
1569
|
+
# (best-effort — scancel trouble must not abort the sweep), then
|
|
1570
|
+
# wake with that fact.
|
|
1571
|
+
if not dry_run:
|
|
1572
|
+
for jid, s in zip(job_ids, states, strict=True):
|
|
1573
|
+
if is_terminal(s) or s == GONE:
|
|
1574
|
+
continue
|
|
1575
|
+
try:
|
|
1576
|
+
compute.cancel(jid)
|
|
1577
|
+
except Exception as exc: # scancel trouble is never fatal here
|
|
1578
|
+
log.warning("cancel %s failed: %s", jid, exc)
|
|
1579
|
+
wake(record, "experiment unschedulable (pending past deadline)", "unschedulable")
|
|
1580
|
+
elif all(is_terminal(s) or s == GONE for s in states):
|
|
1581
|
+
# done-or-vanished, at least one GONE (all-terminal handled above)
|
|
1582
|
+
if past_deadline:
|
|
1583
|
+
wake(record, "experiment vanished from Slurm", "vanished")
|
|
1584
|
+
# else: sacct lag right after submission is normal; wait.
|
|
1585
|
+
# something RUNNING (or recently pending): nothing to do yet.
|
|
1586
|
+
|
|
1587
|
+
|
|
1588
|
+
def tick(
|
|
1589
|
+
root: Path,
|
|
1590
|
+
compute: Compute,
|
|
1591
|
+
dispatcher: WakeDispatcher,
|
|
1592
|
+
now: float,
|
|
1593
|
+
grace_s: float = DEFAULT_GRACE_S,
|
|
1594
|
+
lease_ttl_s: float = DEFAULT_LEASE_TTL_S,
|
|
1595
|
+
dry_run: bool = False,
|
|
1596
|
+
github: Any = None,
|
|
1597
|
+
followup_spec: FollowupSpec | None = None,
|
|
1598
|
+
followup_dry_run: bool = False,
|
|
1599
|
+
min_free_bytes: int = DEFAULT_MIN_FREE_BYTES,
|
|
1600
|
+
min_tick_s: float = DEFAULT_MIN_TICK_S,
|
|
1601
|
+
) -> TickReport:
|
|
1602
|
+
"""One full tick. Pause sentinel wins over everything: a paused loop
|
|
1603
|
+
heartbeats (so the watchdog stays quiet) but touches nothing.
|
|
1604
|
+
|
|
1605
|
+
Disk preflight gates every lane that LAUNCHES new work (follow-up jobs,
|
|
1606
|
+
intake claims, self-initiated climbs): a session started on a full or
|
|
1607
|
+
nearly-full filesystem dies mid-flight in ways that lose data. The sweep
|
|
1608
|
+
still runs — its writes are small, per-record contained, and ending runs
|
|
1609
|
+
matters more when storage is failing, not less.
|
|
1610
|
+
"""
|
|
1611
|
+
# Heartbeat FIRST, before any probe: check_disk touches $HOME (a
|
|
1612
|
+
# different filesystem), and a hung mount there must not starve the
|
|
1613
|
+
# watchdog signal. The disk-annotated heartbeat follows once known. The
|
|
1614
|
+
# coalesce guard reads the last COMPLETED tick's marker (not the heartbeat).
|
|
1615
|
+
prior_worked = _last_worked_ts(root)
|
|
1616
|
+
write_heartbeat(root, now)
|
|
1617
|
+
disk_health = check_disk(root, min_free_bytes=min_free_bytes)
|
|
1618
|
+
write_heartbeat(root, now, disk=disk_health.as_dict())
|
|
1619
|
+
for warning in disk_health.warnings():
|
|
1620
|
+
log.warning("disk: %s", warning)
|
|
1621
|
+
if (root / PAUSE_SENTINEL).exists():
|
|
1622
|
+
log.info("pause sentinel present; tick is a no-op")
|
|
1623
|
+
return TickReport(paused=True)
|
|
1624
|
+
# Coalesce a congestion pile-up: if a tick COMPLETED its work within
|
|
1625
|
+
# min_tick_s, this one is redundant (that recent tick already swept and
|
|
1626
|
+
# launched). Keyed on the work marker (stamped by the CALLER at real
|
|
1627
|
+
# completion time), not the heartbeat, so a tick that crashed mid-work does
|
|
1628
|
+
# not suppress this recovery tick. Heartbeat still written above, so the
|
|
1629
|
+
# watchdog stays fed and the chain stays alive.
|
|
1630
|
+
if min_tick_s > 0 and prior_worked is not None:
|
|
1631
|
+
elapsed = now - prior_worked
|
|
1632
|
+
if elapsed < 0:
|
|
1633
|
+
# marker dated in the future -> the clock jumped back; never coalesce
|
|
1634
|
+
# on it (that could stall the loop), and surface it rather than fail
|
|
1635
|
+
# silently.
|
|
1636
|
+
log.warning(
|
|
1637
|
+
"work marker is %.0fs in the future (clock skew?); not coalescing", -elapsed
|
|
1638
|
+
)
|
|
1639
|
+
elif elapsed < min_tick_s:
|
|
1640
|
+
log.info(
|
|
1641
|
+
"coalescing: last completed tick %.0fs ago (< %.0fs); tick is a no-op",
|
|
1642
|
+
elapsed,
|
|
1643
|
+
min_tick_s,
|
|
1644
|
+
)
|
|
1645
|
+
return TickReport(coalesced=True)
|
|
1646
|
+
report = sweep(root, compute, dispatcher, now, grace_s, lease_ttl_s, dry_run=dry_run)
|
|
1647
|
+
# Housekeeping: ended runs shed ws/ and ws-home/ after a grace period;
|
|
1648
|
+
# when the state filesystem's write probe failed, the grace is waived and
|
|
1649
|
+
# the sweep frees oldest-first until the probe passes, then the preflight
|
|
1650
|
+
# is taken again so launch lanes can come back this very tick.
|
|
1651
|
+
# Force-shed (waive the grace) only when the state root cannot be WRITTEN,
|
|
1652
|
+
# not merely when it is below the free-space threshold: a writable disk
|
|
1653
|
+
# that is just low keeps the 24 h grace so a post-mortem is not deleted
|
|
1654
|
+
# under someone. A dry-run tick sheds nothing (the destructive lane obeys
|
|
1655
|
+
# the zero-writes contract, like sweep()).
|
|
1656
|
+
shed: list[str] = []
|
|
1657
|
+
if not dry_run:
|
|
1658
|
+
cannot_write = not disk_health.state_root.writable
|
|
1659
|
+
# Bounded per tick so shedding (rm -rf of tens of thousands of files
|
|
1660
|
+
# per workspace on a networked FS) never blows the tick's own timeout:
|
|
1661
|
+
# a few runs on a healthy disk, more but still time-boxed when it is
|
|
1662
|
+
# failing. The backlog drains over several ticks.
|
|
1663
|
+
shed = shed_ended_workspaces(
|
|
1664
|
+
root,
|
|
1665
|
+
now,
|
|
1666
|
+
force=cannot_write,
|
|
1667
|
+
limit=25 if cannot_write else 3,
|
|
1668
|
+
time_budget_s=300.0 if cannot_write else 120.0,
|
|
1669
|
+
until_ok=(lambda: check_disk(root, min_free_bytes=min_free_bytes).state_root.writable)
|
|
1670
|
+
if cannot_write
|
|
1671
|
+
else None,
|
|
1672
|
+
)
|
|
1673
|
+
if shed and cannot_write:
|
|
1674
|
+
disk_health = check_disk(root, min_free_bytes=min_free_bytes)
|
|
1675
|
+
write_heartbeat(root, now, disk=disk_health.as_dict())
|
|
1676
|
+
if shed:
|
|
1677
|
+
from dataclasses import replace as _dc_replace
|
|
1678
|
+
|
|
1679
|
+
report = _dc_replace(report, shed=tuple(shed))
|
|
1680
|
+
launch_ok = disk_health.launch_ok()
|
|
1681
|
+
if not launch_ok:
|
|
1682
|
+
log.warning("disk preflight failed; launch lanes are OFF this tick")
|
|
1683
|
+
# Mid-leg sync is serviced regardless of follow-up/board servicing: it
|
|
1684
|
+
# only needs the workspace and the PAT (a git fetch, no GitHub REST and
|
|
1685
|
+
# no contract), and a live session waiting on `sync` must not depend on
|
|
1686
|
+
# whether github/contract loaded this tick.
|
|
1687
|
+
if followup_spec is not None:
|
|
1688
|
+
service_syncs(root, followup_spec, now)
|
|
1689
|
+
if github is not None and followup_spec is not None:
|
|
1690
|
+
# expired flight snapshots die with their TTL, not with a human.
|
|
1691
|
+
# One home suffices: every lane's spec derives from followup_spec
|
|
1692
|
+
# via replace(), so all flights share this checkout's flights/ dir.
|
|
1693
|
+
# Blind means delete nothing — but only QUERY failures count as
|
|
1694
|
+
# blindness; a compute backend missing the method is a programming
|
|
1695
|
+
# error and propagates.
|
|
1696
|
+
try:
|
|
1697
|
+
live_names = compute.active_job_names()
|
|
1698
|
+
except SlurmQueryError as exc:
|
|
1699
|
+
log.warning("cannot list live jobs (%s); reaping no flights this tick", exc)
|
|
1700
|
+
live_names = None
|
|
1701
|
+
if live_names is not None:
|
|
1702
|
+
with contextlib.suppress(Exception):
|
|
1703
|
+
reaped = reap_flights(followup_spec.home, now, live_job_names=live_names)
|
|
1704
|
+
if reaped:
|
|
1705
|
+
log.info("reaped %d expired flight snapshot(s)", reaped)
|
|
1706
|
+
# ONE contract fetch per tick feeds every lane: the requested and
|
|
1707
|
+
# self-initiated lanes need its benchmarks, and all three lanes now
|
|
1708
|
+
# take their session/job limits from its budgets — clamped by our
|
|
1709
|
+
# ceilings (limits.py), so a target shapes spend, never raises it.
|
|
1710
|
+
# A failed fetch leaves in-review servicing running on defaults;
|
|
1711
|
+
# the launch lanes need the contract and sit out this tick.
|
|
1712
|
+
contract = None
|
|
1713
|
+
if followup_spec.target:
|
|
1714
|
+
contract_error: str | None = "contract file missing on main"
|
|
1715
|
+
try:
|
|
1716
|
+
from outerloop.contract import load_contract
|
|
1717
|
+
|
|
1718
|
+
raw = _contract_text(github, followup_spec.target, "main")
|
|
1719
|
+
if raw is not None:
|
|
1720
|
+
contract = load_contract(raw, followup_spec.target)
|
|
1721
|
+
contract_error = None
|
|
1722
|
+
except Exception as exc:
|
|
1723
|
+
log.warning("contract fetch failed for %s: %s", followup_spec.target, exc)
|
|
1724
|
+
contract_error = f"{type(exc).__name__}: {exc}"
|
|
1725
|
+
if contract_error is None:
|
|
1726
|
+
# a bad panel config idles the same launch lanes a bad
|
|
1727
|
+
# contract does — same silent-idle class, so it rides the
|
|
1728
|
+
# same alarm
|
|
1729
|
+
panel_error = _panel_preflight_error(followup_spec)
|
|
1730
|
+
if panel_error:
|
|
1731
|
+
contract_error = f"panel preflight: {panel_error}"
|
|
1732
|
+
try:
|
|
1733
|
+
contract_alarm(
|
|
1734
|
+
root,
|
|
1735
|
+
github,
|
|
1736
|
+
followup_spec.target,
|
|
1737
|
+
contract_error,
|
|
1738
|
+
now,
|
|
1739
|
+
bot_login=followup_spec.bot_login,
|
|
1740
|
+
)
|
|
1741
|
+
except Exception as exc:
|
|
1742
|
+
log.warning("contract alarm failed: %s", exc)
|
|
1743
|
+
limits = effective_limits(contract.budgets if contract is not None else None)
|
|
1744
|
+
# The contract's followup walltime only overrides when EXPLICITLY
|
|
1745
|
+
# set — and only DOWNWARD from the operator's spec value: strictly-
|
|
1746
|
+
# downward shaping must hold against operator config too, not just
|
|
1747
|
+
# against the module defaults.
|
|
1748
|
+
spec = shape_followup_spec(followup_spec, limits, contract)
|
|
1749
|
+
ended, submitted = service_in_review(
|
|
1750
|
+
root,
|
|
1751
|
+
github,
|
|
1752
|
+
compute,
|
|
1753
|
+
spec,
|
|
1754
|
+
now,
|
|
1755
|
+
dry_run=followup_dry_run,
|
|
1756
|
+
allow_submit=launch_ok,
|
|
1757
|
+
contract=contract,
|
|
1758
|
+
)
|
|
1759
|
+
try:
|
|
1760
|
+
service_research_log(root, github, spec, now)
|
|
1761
|
+
except Exception as exc: # the ledger is advisory; the tick continues
|
|
1762
|
+
log.warning("research-log service failed: %s", exc)
|
|
1763
|
+
intake_job = (
|
|
1764
|
+
service_intake(
|
|
1765
|
+
root, github, compute, spec, now, contract, limits, dry_run=followup_dry_run
|
|
1766
|
+
)
|
|
1767
|
+
if launch_ok and contract is not None
|
|
1768
|
+
else None
|
|
1769
|
+
)
|
|
1770
|
+
steward_job = (
|
|
1771
|
+
service_steward(
|
|
1772
|
+
root, github, compute, spec, now, contract, limits, dry_run=followup_dry_run
|
|
1773
|
+
)
|
|
1774
|
+
if launch_ok and intake_job is None and contract is not None
|
|
1775
|
+
else None
|
|
1776
|
+
)
|
|
1777
|
+
self_job = None
|
|
1778
|
+
if launch_ok and intake_job is None and steward_job is None and contract is not None:
|
|
1779
|
+
try:
|
|
1780
|
+
self_job = service_self_initiated(
|
|
1781
|
+
root,
|
|
1782
|
+
compute,
|
|
1783
|
+
spec,
|
|
1784
|
+
contract,
|
|
1785
|
+
now,
|
|
1786
|
+
limits=limits,
|
|
1787
|
+
dry_run=followup_dry_run,
|
|
1788
|
+
)
|
|
1789
|
+
except Exception as exc:
|
|
1790
|
+
log.warning("self-initiated selection failed: %s", exc)
|
|
1791
|
+
# AFTER the launch block: a run started this tick is on the strip
|
|
1792
|
+
# this tick, not the next one
|
|
1793
|
+
service_boards(root, github, spec.target, contract, now)
|
|
1794
|
+
report = replace_report(
|
|
1795
|
+
report,
|
|
1796
|
+
ended,
|
|
1797
|
+
submitted,
|
|
1798
|
+
intake_job,
|
|
1799
|
+
self_job,
|
|
1800
|
+
disk_health.warnings(),
|
|
1801
|
+
not launch_ok,
|
|
1802
|
+
steward_job,
|
|
1803
|
+
)
|
|
1804
|
+
# The coalesce marker is stamped by the CALLER at real completion time (see
|
|
1805
|
+
# main / mark_tick_complete) — not here with the start-of-tick `now`, which
|
|
1806
|
+
# a tick longer than the window would leave stale.
|
|
1807
|
+
return report
|
|
1808
|
+
|
|
1809
|
+
|
|
1810
|
+
def service_syncs(root: Path, spec: Any, now: float) -> None:
|
|
1811
|
+
"""Honor mid-leg sync requests: a LIVE session asked for fresh origin/*
|
|
1812
|
+
refs and is waiting inside its own clock. The fetch pins the canonical
|
|
1813
|
+
URL (never the workspace's mutable remote config) and only refs/remotes
|
|
1814
|
+
are written — safe next to the session's local git use. Best-effort per
|
|
1815
|
+
run; a failure leaves the request standing for the next cycle."""
|
|
1816
|
+
from outerloop.appauth import resolve_bot_auth
|
|
1817
|
+
from outerloop.attempt import _target_clone_url
|
|
1818
|
+
from outerloop.github import Workspace
|
|
1819
|
+
from outerloop.syscall import mark_synced, sync_requested
|
|
1820
|
+
|
|
1821
|
+
for record in list_runs(root):
|
|
1822
|
+
if record.state != IMPLEMENTING:
|
|
1823
|
+
continue
|
|
1824
|
+
workspace = run_dir(root, record.run_id) / "ws"
|
|
1825
|
+
if not workspace.is_dir():
|
|
1826
|
+
continue
|
|
1827
|
+
requested_at = sync_requested(workspace)
|
|
1828
|
+
if requested_at is None:
|
|
1829
|
+
continue
|
|
1830
|
+
try:
|
|
1831
|
+
ws = Workspace(
|
|
1832
|
+
root=workspace,
|
|
1833
|
+
auth=(
|
|
1834
|
+
resolve_bot_auth(spec.pat_file, spec.github_app_file)
|
|
1835
|
+
if (spec.pat_file or spec.github_app_file)
|
|
1836
|
+
else None
|
|
1837
|
+
),
|
|
1838
|
+
url=_target_clone_url(record.target),
|
|
1839
|
+
)
|
|
1840
|
+
ws.fetch_origin()
|
|
1841
|
+
mark_synced(workspace, requested_at)
|
|
1842
|
+
log.info("synced origin refs for %s", record.run_id)
|
|
1843
|
+
except Exception as exc:
|
|
1844
|
+
log.warning("sync failed for %s: %s", record.run_id, exc)
|
|
1845
|
+
|
|
1846
|
+
|
|
1847
|
+
def service_boards(root: Path, github: Any, target: str, contract: Any, now: float) -> None:
|
|
1848
|
+
"""The climb board and the live strip, together and advisory: the views
|
|
1849
|
+
publish from the first tick (before any run ends), and a failure never
|
|
1850
|
+
stops the tick."""
|
|
1851
|
+
try:
|
|
1852
|
+
from outerloop.climbboard import contract_directions, service_climb_board
|
|
1853
|
+
|
|
1854
|
+
service_climb_board(root, github, target, contract_directions(contract))
|
|
1855
|
+
except Exception as exc:
|
|
1856
|
+
log.warning("climb board service failed: %s", exc)
|
|
1857
|
+
try:
|
|
1858
|
+
from outerloop.climbboard import service_status
|
|
1859
|
+
|
|
1860
|
+
service_status(root, github, target, now, contract)
|
|
1861
|
+
except Exception as exc: # each is advisory ALONE: one failing never mutes the other
|
|
1862
|
+
log.warning("status strip service failed: %s", exc)
|
|
1863
|
+
|
|
1864
|
+
|
|
1865
|
+
def replace_report(
|
|
1866
|
+
report: TickReport,
|
|
1867
|
+
ended: list[tuple[str, str]],
|
|
1868
|
+
submitted: list[tuple[str, str]],
|
|
1869
|
+
intake_job: tuple[str, str] | None = None,
|
|
1870
|
+
self_job: tuple[str, str] | None = None,
|
|
1871
|
+
disk_warnings: list[str] | None = None,
|
|
1872
|
+
launch_blocked: bool = False,
|
|
1873
|
+
steward_job: tuple[str, str] | None = None,
|
|
1874
|
+
) -> TickReport:
|
|
1875
|
+
from dataclasses import replace as dc_replace
|
|
1876
|
+
|
|
1877
|
+
return dc_replace(
|
|
1878
|
+
report,
|
|
1879
|
+
review_ended=tuple(ended),
|
|
1880
|
+
followups_submitted=tuple(submitted),
|
|
1881
|
+
intake=intake_job or ("", ""),
|
|
1882
|
+
self_initiated=self_job or ("", ""),
|
|
1883
|
+
disk=tuple(disk_warnings or ()),
|
|
1884
|
+
launch_blocked=launch_blocked,
|
|
1885
|
+
steward=steward_job or ("", ""),
|
|
1886
|
+
)
|
|
1887
|
+
|
|
1888
|
+
|
|
1889
|
+
MAX_ACTIVE_RUNS_PER_TARGET = 1
|
|
1890
|
+
SELF_INITIATED_COOLDOWN_S = 6 * 3600
|
|
1891
|
+
# the crash-loop floor: a launch that died pre-record backs off at least
|
|
1892
|
+
# this long regardless of the contract's cooldown dial
|
|
1893
|
+
DEAD_LAUNCH_BACKOFF_S = 30 * 60
|
|
1894
|
+
# An implementing run untouched for this long is a crashed climb job; it must
|
|
1895
|
+
# not block the lane forever, but the window must exceed the LONGEST honest
|
|
1896
|
+
# job — the 120-min contract ceiling plus the panel allowance the tick adds
|
|
1897
|
+
# (~4.5 h at the defaults) plus queue-start slack — or the picker declares a
|
|
1898
|
+
# live run stranded and starts a second one on the same target, breaking the
|
|
1899
|
+
# one-active-run serialization. Its cooldown entry still applies, so a
|
|
1900
|
+
# crashed benchmark isn't immediately retried.
|
|
1901
|
+
STRANDED_IMPLEMENTING_S = 12 * 3600
|
|
1902
|
+
# A pending marker older than this is dead even if squeue can't be read.
|
|
1903
|
+
PENDING_TTL_S = 4 * 3600
|
|
1904
|
+
|
|
1905
|
+
|
|
1906
|
+
def pick_self_initiated(
|
|
1907
|
+
records: list[RunRecord],
|
|
1908
|
+
contract: Any,
|
|
1909
|
+
target: str,
|
|
1910
|
+
now: float,
|
|
1911
|
+
dead_attempts: dict[str, float] | None = None,
|
|
1912
|
+
live_pendings: list[tuple[str, float]] | None = None,
|
|
1913
|
+
) -> str | None:
|
|
1914
|
+
"""The benchmark to climb next on `target`, or None.
|
|
1915
|
+
|
|
1916
|
+
Deliberately boring (the planning agent upgrades this later): serialize
|
|
1917
|
+
to one active run per target, respect the contract's weekly budget and a
|
|
1918
|
+
per-benchmark cooldown, then choose the benchmark least recently
|
|
1919
|
+
attempted — untouched ones first. Only this target's runs count toward
|
|
1920
|
+
any of it. `dead_attempts` maps benchmark -> submitted_at for launches
|
|
1921
|
+
that died BEFORE writing a run record (per-benchmark tombstones) — each
|
|
1922
|
+
counts toward cooldown with a crash-loop floor, so alternating
|
|
1923
|
+
pre-record failures can't ping-pong every tick (terra #172 r2/r3).
|
|
1924
|
+
"""
|
|
1925
|
+
mine = [r for r in records if r.target == target]
|
|
1926
|
+
|
|
1927
|
+
def stranded(r: RunRecord) -> bool:
|
|
1928
|
+
return r.state == IMPLEMENTING and now - max(r.updated, r.created) > STRANDED_IMPLEMENTING_S
|
|
1929
|
+
|
|
1930
|
+
active = [r for r in mine if r.state != ENDED and not stranded(r)]
|
|
1931
|
+
if len(active) >= _attempt_width(contract):
|
|
1932
|
+
return None
|
|
1933
|
+
week_ago = now - 7 * 24 * 3600
|
|
1934
|
+
# queued slots count toward the weekly budget BEFORE their records
|
|
1935
|
+
# exist, or a width-N target with one run left could submit N (terra
|
|
1936
|
+
# #173 r2); when a marker lands, the service clears it before calling
|
|
1937
|
+
# here, so a run is never counted twice
|
|
1938
|
+
queued = sum(1 for _, submitted_at in live_pendings or [] if submitted_at >= week_ago)
|
|
1939
|
+
if sum(1 for r in mine if r.created >= week_ago) + queued >= contract.budgets.runs_per_week:
|
|
1940
|
+
return None
|
|
1941
|
+
last_attempt: dict[str, float] = {}
|
|
1942
|
+
for r in mine:
|
|
1943
|
+
if r.benchmark:
|
|
1944
|
+
last_attempt[r.benchmark] = max(last_attempt.get(r.benchmark, 0.0), r.created)
|
|
1945
|
+
for bench_name, submitted_at in (dead_attempts or {}).items():
|
|
1946
|
+
last_attempt[bench_name] = max(last_attempt.get(bench_name, 0.0), submitted_at)
|
|
1947
|
+
for bench_name, submitted_at in live_pendings or []:
|
|
1948
|
+
# a queued sibling starts its benchmark's cooldown clock too:
|
|
1949
|
+
# width spreads across benchmarks first, and re-picking the same
|
|
1950
|
+
# one needs the contract to have set cooldown to 0 (portfolio)
|
|
1951
|
+
if bench_name:
|
|
1952
|
+
last_attempt[bench_name] = max(last_attempt.get(bench_name, 0.0), submitted_at)
|
|
1953
|
+
cooldown_min = getattr(contract.budgets, "attempt_cooldown_minutes", None)
|
|
1954
|
+
cooldown_s = SELF_INITIATED_COOLDOWN_S if cooldown_min is None else cooldown_min * 60
|
|
1955
|
+
# A launch that died BEFORE writing a run record is invisible to the
|
|
1956
|
+
# runs_per_week cap, so its cooldown attribution is the ONLY crash-loop
|
|
1957
|
+
# guard — it keeps a floor even when the contract dials cooldown to 0
|
|
1958
|
+
# (terra #172: zero cooldown otherwise resubmits a crashing launch
|
|
1959
|
+
# every tick, uncapped).
|
|
1960
|
+
dead_benches = set(dead_attempts or ())
|
|
1961
|
+
candidates = sorted(
|
|
1962
|
+
contract.benchmarks,
|
|
1963
|
+
key=lambda b: (last_attempt.get(b.name, 0.0), b.name),
|
|
1964
|
+
)
|
|
1965
|
+
for bench in candidates:
|
|
1966
|
+
floor_s = cooldown_s
|
|
1967
|
+
if bench.name in dead_benches:
|
|
1968
|
+
floor_s = max(cooldown_s, DEAD_LAUNCH_BACKOFF_S)
|
|
1969
|
+
if now - last_attempt.get(bench.name, 0.0) >= floor_s:
|
|
1970
|
+
return str(bench.name)
|
|
1971
|
+
return None
|
|
1972
|
+
|
|
1973
|
+
|
|
1974
|
+
def _tombstone_path(root: Path, target: str, benchmark: str) -> Path:
|
|
1975
|
+
safe = f"{target.replace('/', '__')}__{benchmark}"
|
|
1976
|
+
return root / "pending-dead" / (safe + ".json")
|
|
1977
|
+
|
|
1978
|
+
|
|
1979
|
+
def write_tombstone(root: Path, target: str, benchmark: str, submitted_at: float) -> None:
|
|
1980
|
+
"""Per-benchmark crash memory: a launch died before writing a record, so
|
|
1981
|
+
nothing else (runs_per_week, cooldown-by-records) can see it. The
|
|
1982
|
+
tombstone persists independently of the live pending marker — a second
|
|
1983
|
+
benchmark's launch must not erase it (terra #172 r3)."""
|
|
1984
|
+
path = _tombstone_path(root, target, benchmark)
|
|
1985
|
+
try:
|
|
1986
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
1987
|
+
path.write_text(json.dumps({"submitted_at": submitted_at}))
|
|
1988
|
+
except OSError as exc:
|
|
1989
|
+
log.warning("tombstone write failed for %s/%s: %s", target, benchmark, exc)
|
|
1990
|
+
|
|
1991
|
+
|
|
1992
|
+
def read_tombstones(root: Path, target: str, contract: Any, now: float) -> dict[str, float]:
|
|
1993
|
+
"""benchmark -> submitted_at for unserved crash tombstones; entries past
|
|
1994
|
+
their window (the larger of the crash floor and the contract cooldown)
|
|
1995
|
+
are pruned on read."""
|
|
1996
|
+
cooldown_min = getattr(contract.budgets, "attempt_cooldown_minutes", None)
|
|
1997
|
+
cooldown_s = SELF_INITIATED_COOLDOWN_S if cooldown_min is None else cooldown_min * 60
|
|
1998
|
+
window = max(DEAD_LAUNCH_BACKOFF_S, cooldown_s)
|
|
1999
|
+
out: dict[str, float] = {}
|
|
2000
|
+
prefix = target.replace("/", "__") + "__"
|
|
2001
|
+
dead_dir = root / "pending-dead"
|
|
2002
|
+
if not dead_dir.is_dir():
|
|
2003
|
+
return out
|
|
2004
|
+
for path in dead_dir.glob(prefix + "*.json"):
|
|
2005
|
+
bench = path.stem[len(prefix) :]
|
|
2006
|
+
try:
|
|
2007
|
+
submitted_at = float(json.loads(path.read_text())["submitted_at"])
|
|
2008
|
+
except (OSError, ValueError, KeyError, TypeError):
|
|
2009
|
+
with contextlib.suppress(OSError):
|
|
2010
|
+
path.unlink()
|
|
2011
|
+
continue
|
|
2012
|
+
if now - submitted_at > window:
|
|
2013
|
+
with contextlib.suppress(OSError):
|
|
2014
|
+
path.unlink() # backoff served
|
|
2015
|
+
continue
|
|
2016
|
+
out[bench] = submitted_at
|
|
2017
|
+
return out
|
|
2018
|
+
|
|
2019
|
+
|
|
2020
|
+
# WIDTH slots are the only suffixed marker names; the pattern also fences
|
|
2021
|
+
# list_pendings against a longer target that shares this one's file-name
|
|
2022
|
+
# prefix (org/foo vs org/foobar — "/" encodes as "__", so glob alone is
|
|
2023
|
+
# ambiguous)
|
|
2024
|
+
_SLOT_AGENT_RE = re.compile(r"agent-\d+")
|
|
2025
|
+
|
|
2026
|
+
|
|
2027
|
+
def _pending_path(root: Path, target: str, agent: str = "") -> Path:
|
|
2028
|
+
# agent "" is the legacy single-slot name, still read for back-compat
|
|
2029
|
+
# with a marker written before the width dial deployed. The slot
|
|
2030
|
+
# separator is "@" because it CANNOT appear in a GitHub owner/repo
|
|
2031
|
+
# name — any character legal in repo names ("_", ".", "-") would make
|
|
2032
|
+
# org/pilot's slot file collide with some other target's legacy file
|
|
2033
|
+
# (org/pilot__agent-01 is a valid repo).
|
|
2034
|
+
suffix = f"@{agent}" if agent else ""
|
|
2035
|
+
return root / "pending" / (target.replace("/", "__") + suffix + ".json")
|
|
2036
|
+
|
|
2037
|
+
|
|
2038
|
+
def read_pending(root: Path, target: str, agent: str = "") -> dict[str, Any] | None:
|
|
2039
|
+
"""The submit-time marker for a climb whose run record may not exist yet."""
|
|
2040
|
+
path = _pending_path(root, target, agent)
|
|
2041
|
+
try:
|
|
2042
|
+
data = json.loads(path.read_text())
|
|
2043
|
+
except (OSError, ValueError):
|
|
2044
|
+
return None
|
|
2045
|
+
return data if isinstance(data, dict) and "submitted_at" in data else None
|
|
2046
|
+
|
|
2047
|
+
|
|
2048
|
+
def list_pendings(root: Path, target: str) -> list[tuple[str, dict[str, Any]]]:
|
|
2049
|
+
"""(agent, marker) for every live pending marker of `target` — one per
|
|
2050
|
+
WIDTH slot, plus the legacy un-suffixed marker from a pre-width deploy
|
|
2051
|
+
(attributed to agent-01)."""
|
|
2052
|
+
out: list[tuple[str, dict[str, Any]]] = []
|
|
2053
|
+
stem = target.replace("/", "__")
|
|
2054
|
+
pending_dir = root / "pending"
|
|
2055
|
+
if not pending_dir.is_dir():
|
|
2056
|
+
return out
|
|
2057
|
+
for path in sorted(pending_dir.glob(stem + "*.json")):
|
|
2058
|
+
name = path.stem
|
|
2059
|
+
if name == stem:
|
|
2060
|
+
agent = ""
|
|
2061
|
+
elif name.startswith(stem + "@") and _SLOT_AGENT_RE.fullmatch(name[len(stem) + 1 :]):
|
|
2062
|
+
agent = name[len(stem) + 1 :]
|
|
2063
|
+
else:
|
|
2064
|
+
continue # a longer target sharing this prefix (org/foo vs org/foobar)
|
|
2065
|
+
try:
|
|
2066
|
+
data = json.loads(path.read_text())
|
|
2067
|
+
except (OSError, ValueError):
|
|
2068
|
+
continue
|
|
2069
|
+
if isinstance(data, dict) and "submitted_at" in data:
|
|
2070
|
+
out.append((agent or str(data.get("agent_id") or "agent-01"), data))
|
|
2071
|
+
return out
|
|
2072
|
+
|
|
2073
|
+
|
|
2074
|
+
def write_pending(
|
|
2075
|
+
root: Path, target: str, benchmark: str, job_id: str, now: float, agent: str = ""
|
|
2076
|
+
) -> None:
|
|
2077
|
+
path = _pending_path(root, target, agent)
|
|
2078
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
2079
|
+
tmp = path.with_suffix(".tmp")
|
|
2080
|
+
tmp.write_text(
|
|
2081
|
+
json.dumps(
|
|
2082
|
+
{"benchmark": benchmark, "job_id": job_id, "submitted_at": now, "agent_id": agent}
|
|
2083
|
+
)
|
|
2084
|
+
)
|
|
2085
|
+
os.replace(tmp, path)
|
|
2086
|
+
|
|
2087
|
+
|
|
2088
|
+
def clear_pending(root: Path, target: str, agent: str = "") -> None:
|
|
2089
|
+
_pending_path(root, target, agent).unlink(missing_ok=True)
|
|
2090
|
+
|
|
2091
|
+
|
|
2092
|
+
def _attempt_width(contract: Any) -> int:
|
|
2093
|
+
width = getattr(getattr(contract, "budgets", None), "max_active_attempts", None)
|
|
2094
|
+
return int(width) if width else MAX_ACTIVE_RUNS_PER_TARGET
|
|
2095
|
+
|
|
2096
|
+
|
|
2097
|
+
def _free_agent_slot(occupied: set[str], width: int) -> str | None:
|
|
2098
|
+
for i in range(1, width + 1):
|
|
2099
|
+
agent = f"agent-{i:02d}"
|
|
2100
|
+
if agent not in occupied:
|
|
2101
|
+
return agent
|
|
2102
|
+
return None
|
|
2103
|
+
|
|
2104
|
+
|
|
2105
|
+
def _climb_panel_argv(spec: FollowupSpec) -> list[str]:
|
|
2106
|
+
"""Panel args for a climb job; empty when the operator disabled the panel."""
|
|
2107
|
+
if not spec.panel.strip():
|
|
2108
|
+
return []
|
|
2109
|
+
argv = ["--panel", spec.panel]
|
|
2110
|
+
if spec.panel_key_file:
|
|
2111
|
+
argv += ["--panel-key-file", spec.panel_key_file]
|
|
2112
|
+
return argv
|
|
2113
|
+
|
|
2114
|
+
|
|
2115
|
+
def _author_config_error(spec: FollowupSpec) -> str:
|
|
2116
|
+
"""Why the config-driven author would die at the climb's startup ("" when it
|
|
2117
|
+
won't), checked on the tick host BEFORE a claim/submit so a codex misconfig
|
|
2118
|
+
(e.g. AUTORESEARCH_AUTHOR_BACKEND=codex with no non-claude model) never
|
|
2119
|
+
strands a claimed intake issue. Reads the fleet author config from env — the
|
|
2120
|
+
same source the climb defaults from — and the image the tick already knows."""
|
|
2121
|
+
from outerloop.attempt import codex_author_config_error
|
|
2122
|
+
|
|
2123
|
+
backend = os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude"
|
|
2124
|
+
model = os.environ.get("AUTORESEARCH_AUTHOR_MODEL") or "claude-opus-5"
|
|
2125
|
+
return codex_author_config_error(backend, model, spec.image)
|
|
2126
|
+
|
|
2127
|
+
|
|
2128
|
+
def _panel_preflight_error(spec: FollowupSpec) -> str:
|
|
2129
|
+
"""Why the climb would die at startup on this panel config ("" when it
|
|
2130
|
+
won't): the lens spec, then the key file — each checked with the climb's
|
|
2131
|
+
OWN rules (parse_lenses for the grammar and claude-only backend;
|
|
2132
|
+
FileTokenProvider for exists/mode-600/non-empty), so preflight and climb
|
|
2133
|
+
cannot disagree.
|
|
2134
|
+
|
|
2135
|
+
Preflighted BEFORE claiming or submitting: the climb CLI fails loudly,
|
|
2136
|
+
but by then an intake issue is already claimed — and pick_issue never
|
|
2137
|
+
reclaims — so the strand must be caught on the tick host, which shares
|
|
2138
|
+
the home filesystem the climb will read."""
|
|
2139
|
+
if not spec.panel.strip():
|
|
2140
|
+
return ""
|
|
2141
|
+
try:
|
|
2142
|
+
from outerloop.attempt import PANEL_KEY_DEFAULT, resolve_author_key_file
|
|
2143
|
+
from outerloop.github import FileTokenProvider
|
|
2144
|
+
from outerloop.panel import parse_lenses
|
|
2145
|
+
|
|
2146
|
+
try:
|
|
2147
|
+
lenses = parse_lenses(spec.panel)
|
|
2148
|
+
except ValueError as exc:
|
|
2149
|
+
return str(exc)
|
|
2150
|
+
# non-claude (shelled) lenses: mirror the climb's rules exactly, per
|
|
2151
|
+
# backend — image required, the judge's OWN key (set + absolute +
|
|
2152
|
+
# neither the author's nor the claude panel key + readable), and for
|
|
2153
|
+
# hermes its pinned clone.
|
|
2154
|
+
shelled = {
|
|
2155
|
+
"codex": "AUTORESEARCH_PANEL_CODEX_KEY_FILE",
|
|
2156
|
+
"hermes": "AUTORESEARCH_PANEL_HERMES_KEY_FILE",
|
|
2157
|
+
}
|
|
2158
|
+
for lens_backend, key_env in shelled.items():
|
|
2159
|
+
if not any(backend == lens_backend for _, backend, _ in lenses):
|
|
2160
|
+
continue
|
|
2161
|
+
if not spec.image or not Path(spec.image).is_file():
|
|
2162
|
+
return (
|
|
2163
|
+
f"a {lens_backend} panel lens requires a real container image "
|
|
2164
|
+
f"(AUTORESEARCH_IMAGE={spec.image!r})"
|
|
2165
|
+
)
|
|
2166
|
+
key_raw = os.environ.get(key_env, "").strip()
|
|
2167
|
+
if not key_raw:
|
|
2168
|
+
return (
|
|
2169
|
+
f"a {lens_backend} panel lens needs {key_env} "
|
|
2170
|
+
"(role separation: the judge's own key, never the author's)"
|
|
2171
|
+
)
|
|
2172
|
+
key_path = Path(key_raw).expanduser()
|
|
2173
|
+
if not key_path.is_absolute():
|
|
2174
|
+
return (
|
|
2175
|
+
f"{lens_backend} panel key path {key_path} is relative; only absolute paths fly"
|
|
2176
|
+
)
|
|
2177
|
+
author = Path(resolve_author_key_file("codex")).expanduser()
|
|
2178
|
+
if key_path.resolve() == author.resolve():
|
|
2179
|
+
return (
|
|
2180
|
+
f"{lens_backend} panel key file {key_path} is the codex author "
|
|
2181
|
+
"key (role separation: the judge needs its own key)"
|
|
2182
|
+
)
|
|
2183
|
+
claude_panel = Path(spec.panel_key_file or PANEL_KEY_DEFAULT).expanduser()
|
|
2184
|
+
if key_path.resolve() == claude_panel.resolve():
|
|
2185
|
+
return (
|
|
2186
|
+
f"{lens_backend} panel key file {key_path} is the claude panel "
|
|
2187
|
+
"key file (an anthropic key must never reach another "
|
|
2188
|
+
"provider's login)"
|
|
2189
|
+
)
|
|
2190
|
+
FileTokenProvider(key_path).token()
|
|
2191
|
+
if lens_backend == "hermes":
|
|
2192
|
+
repo = os.environ.get("REVIEW_HERMES_REPO", "").strip()
|
|
2193
|
+
# a REAL clone, not merely a directory: the harness executes
|
|
2194
|
+
# run_agent.py from it with the panel key, so an arbitrary or
|
|
2195
|
+
# empty path must fail here, never after a run is claimed
|
|
2196
|
+
if not repo or not (Path(repo).expanduser() / "run_agent.py").is_file():
|
|
2197
|
+
return (
|
|
2198
|
+
f"a hermes panel lens needs REVIEW_HERMES_REPO pointing at "
|
|
2199
|
+
f"the pinned clone (run_agent.py not found under {repo!r})"
|
|
2200
|
+
)
|
|
2201
|
+
from outerloop.role_runner import _HERMES_PROVIDERS
|
|
2202
|
+
|
|
2203
|
+
provider = os.environ.get("REVIEW_HERMES_PROVIDER", "").lower() or "openrouter"
|
|
2204
|
+
if provider not in _HERMES_PROVIDERS:
|
|
2205
|
+
return (
|
|
2206
|
+
f"unknown REVIEW_HERMES_PROVIDER {provider!r} "
|
|
2207
|
+
f"(have: {sorted(_HERMES_PROVIDERS)})"
|
|
2208
|
+
)
|
|
2209
|
+
if not any(backend == "claude" for _, backend, _ in lenses):
|
|
2210
|
+
return "" # codex-only panel: the claude key checks below don't apply
|
|
2211
|
+
path = Path(spec.panel_key_file or PANEL_KEY_DEFAULT).expanduser()
|
|
2212
|
+
if not path.is_absolute():
|
|
2213
|
+
# the climb runs from a flight directory, not the tick's cwd — a
|
|
2214
|
+
# relative path that resolves here could still miss there
|
|
2215
|
+
return f"panel key path {path} is relative; only absolute paths fly"
|
|
2216
|
+
# the AUTHOR key the climb will actually use resolves per the fleet backend
|
|
2217
|
+
# (claude vs codex keys coexist), config-driven like the climb itself — so
|
|
2218
|
+
# the role-separation check compares the panel key against the RIGHT author
|
|
2219
|
+
# key, and a codex run is never judged by a stray Claude key.
|
|
2220
|
+
fleet_backend = os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude"
|
|
2221
|
+
author = Path(resolve_author_key_file(fleet_backend))
|
|
2222
|
+
if not author.is_absolute():
|
|
2223
|
+
# same rule as the panel key: the climb resolves paths from a
|
|
2224
|
+
# flight directory, so a relative author path both misconfigures
|
|
2225
|
+
# the author AND defeats the role-separation comparison below
|
|
2226
|
+
return f"author key path {author} is relative; only absolute paths fly"
|
|
2227
|
+
if path.resolve() == author.resolve():
|
|
2228
|
+
return (
|
|
2229
|
+
f"panel key file {path} is the author key file "
|
|
2230
|
+
"(role separation: the verifier needs its own key)"
|
|
2231
|
+
)
|
|
2232
|
+
# ADC-only deployments (Vertex covering the claude panel) hold no
|
|
2233
|
+
# Anthropic key at all — the same tolerance role_key applies at run
|
|
2234
|
+
# time, so the preflight and the climb agree.
|
|
2235
|
+
from outerloop.role_runner import role_key
|
|
2236
|
+
|
|
2237
|
+
role_key(path)
|
|
2238
|
+
return ""
|
|
2239
|
+
except Exception as exc:
|
|
2240
|
+
# never raises: an unexpected failure (partial deploy, ELOOP, unset
|
|
2241
|
+
# HOME) must fail closed WITH the alarm, not abort the tick that
|
|
2242
|
+
# would have written it
|
|
2243
|
+
return f"{type(exc).__name__}: {exc}"
|
|
2244
|
+
|
|
2245
|
+
|
|
2246
|
+
def _attempt_job_minutes(spec: FollowupSpec, limits: EffectiveLimits) -> int:
|
|
2247
|
+
"""The submitted climb walltime: contract budget + panel allowance,
|
|
2248
|
+
clamped at the partition cap. Warns when the cap cuts below the session
|
|
2249
|
+
budget — the self-deadline would then fire before the author's own
|
|
2250
|
+
clock, and that must be a visible operator choice, never a silent
|
|
2251
|
+
surprise."""
|
|
2252
|
+
from outerloop.limits import ATTEMPT_OVERHEAD_MINUTES
|
|
2253
|
+
|
|
2254
|
+
wanted = limits.attempt_job_minutes + _panel_job_minutes(spec, limits)
|
|
2255
|
+
job = min(wanted, spec.max_job_minutes)
|
|
2256
|
+
if job < wanted:
|
|
2257
|
+
log.info(
|
|
2258
|
+
"climb job clamped to %d min by the partition cap (worst case "
|
|
2259
|
+
"wanted %d); slow panel rounds fail safe via the self-deadline",
|
|
2260
|
+
job,
|
|
2261
|
+
wanted,
|
|
2262
|
+
)
|
|
2263
|
+
if job < limits.session_minutes + ATTEMPT_OVERHEAD_MINUTES:
|
|
2264
|
+
log.warning(
|
|
2265
|
+
"work-job cap %d min leaves no runway around the %d-min session "
|
|
2266
|
+
"(the orchestrator needs ~%d min); sessions or endings will be "
|
|
2267
|
+
"cut short by the self-deadline",
|
|
2268
|
+
job,
|
|
2269
|
+
limits.session_minutes,
|
|
2270
|
+
ATTEMPT_OVERHEAD_MINUTES,
|
|
2271
|
+
)
|
|
2272
|
+
return job
|
|
2273
|
+
|
|
2274
|
+
|
|
2275
|
+
def _panel_job_minutes(spec: FollowupSpec, limits: EffectiveLimits) -> int:
|
|
2276
|
+
"""Extra walltime the panel needs, ADDED to the contract-clamped job
|
|
2277
|
+
budget: the contract's knobs cap the AUTHOR's spend and their ceilings
|
|
2278
|
+
deliberately cannot raise ours (limits.py), so the panel — the
|
|
2279
|
+
orchestrator's own gate, flipped on by the tick — brings its own time.
|
|
2280
|
+
Worst case: three sequential reads of every lens (initial, post-revision,
|
|
2281
|
+
merged-tree) on the judge budget, plus one revision wake on the session
|
|
2282
|
+
budget. The revision's re-measure rides the margin the self-deadline
|
|
2283
|
+
already fails safe on."""
|
|
2284
|
+
lenses = [entry for entry in spec.panel.split(",") if entry.strip()]
|
|
2285
|
+
if not lenses:
|
|
2286
|
+
return 0
|
|
2287
|
+
from outerloop.roles import reviewer_spec, verifier_spec
|
|
2288
|
+
|
|
2289
|
+
judge_minutes = max(reviewer_spec().budget.walltime_s, verifier_spec().budget.walltime_s) // 60
|
|
2290
|
+
return 3 * len(lenses) * judge_minutes + limits.session_minutes
|
|
2291
|
+
|
|
2292
|
+
|
|
2293
|
+
def _climb_limit_argv(limits: EffectiveLimits, job_minutes: int) -> list[str]:
|
|
2294
|
+
"""Climb-CLI flags carrying the tick-resolved limits: the job's ACTUAL
|
|
2295
|
+
walltime rides along (contract budget + any panel allowance, clamped at
|
|
2296
|
+
the partition cap — exactly what the JobSpec gets) so the climb arms its
|
|
2297
|
+
self-deadline against the real clock (Slurm delivers no signals to our
|
|
2298
|
+
processes on Torch). The session shrinks to fit a
|
|
2299
|
+
CAPPED job with the same rule limits.effective_limits applies to
|
|
2300
|
+
contract values — better a short session that ends cleanly than a full
|
|
2301
|
+
one the self-deadline kills mid-flight."""
|
|
2302
|
+
from outerloop.limits import ATTEMPT_OVERHEAD_MINUTES, SESSION_MINUTES_FLOOR
|
|
2303
|
+
|
|
2304
|
+
session = min(limits.session_minutes, job_minutes - ATTEMPT_OVERHEAD_MINUTES)
|
|
2305
|
+
return [
|
|
2306
|
+
"--max-turns",
|
|
2307
|
+
str(limits.session_max_turns),
|
|
2308
|
+
"--session-minutes",
|
|
2309
|
+
str(max(SESSION_MINUTES_FLOOR, session)),
|
|
2310
|
+
"--job-minutes",
|
|
2311
|
+
str(job_minutes),
|
|
2312
|
+
]
|
|
2313
|
+
|
|
2314
|
+
|
|
2315
|
+
def service_self_initiated(
|
|
2316
|
+
root: Path,
|
|
2317
|
+
compute: Compute,
|
|
2318
|
+
spec: FollowupSpec,
|
|
2319
|
+
contract: Any,
|
|
2320
|
+
now: float,
|
|
2321
|
+
limits: EffectiveLimits | None = None,
|
|
2322
|
+
dry_run: bool = False,
|
|
2323
|
+
) -> tuple[str, str] | None:
|
|
2324
|
+
"""The default background mode: when nothing else needs doing, climb the
|
|
2325
|
+
least-recently-attempted benchmark.
|
|
2326
|
+
|
|
2327
|
+
A pending marker written at submit time bridges the gap between
|
|
2328
|
+
`compute.submit` and the climb job writing its run record — without it,
|
|
2329
|
+
every tick during Slurm queue latency would launch a duplicate climb.
|
|
2330
|
+
"""
|
|
2331
|
+
limits = limits if limits is not None else effective_limits(getattr(contract, "budgets", None))
|
|
2332
|
+
paused = outage_active(root, now, role="solver")
|
|
2333
|
+
if paused:
|
|
2334
|
+
log.info("self-initiated lane paused (api outage: %s)", paused)
|
|
2335
|
+
return None
|
|
2336
|
+
try:
|
|
2337
|
+
records = list_runs(root)
|
|
2338
|
+
width = _attempt_width(contract)
|
|
2339
|
+
# WIDTH: every live pending marker occupies a slot; landed ones
|
|
2340
|
+
# clear; dead ones become per-benchmark tombstones and free theirs.
|
|
2341
|
+
occupied: set[str] = set()
|
|
2342
|
+
live_pendings: list[tuple[str, float]] = []
|
|
2343
|
+
nonslot_busy = False
|
|
2344
|
+
for agent, pending in list_pendings(root, spec.target):
|
|
2345
|
+
marker_agent = "" if not pending.get("agent_id") else agent
|
|
2346
|
+
submitted_at = float(pending["submitted_at"])
|
|
2347
|
+
# A slotted marker lands only when ITS OWN record appears —
|
|
2348
|
+
# matching on target+time alone would let a sibling slot's
|
|
2349
|
+
# record clear a still-live marker (terra #173). A legacy
|
|
2350
|
+
# marker names no slot, so it keeps the lax match.
|
|
2351
|
+
landed = any(
|
|
2352
|
+
r.target == spec.target
|
|
2353
|
+
and r.created >= submitted_at - 60
|
|
2354
|
+
and (not marker_agent or r.agent_id == marker_agent)
|
|
2355
|
+
for r in records
|
|
2356
|
+
)
|
|
2357
|
+
expired = now - submitted_at > PENDING_TTL_S
|
|
2358
|
+
if landed:
|
|
2359
|
+
clear_pending(root, spec.target, marker_agent)
|
|
2360
|
+
elif (alive := _holder_alive(compute, str(pending.get("job_id", "")))) is True or (
|
|
2361
|
+
not expired and alive is not False
|
|
2362
|
+
):
|
|
2363
|
+
# climb queued or starting; its record isn't written yet. A
|
|
2364
|
+
# provably-alive job holds its SLOT regardless of the
|
|
2365
|
+
# marker's TTL — queue wait can exceed it — the TTL only
|
|
2366
|
+
# breaks ties when Slurm can't say.
|
|
2367
|
+
occupied.add(agent)
|
|
2368
|
+
live_pendings.append((str(pending.get("benchmark", "")), submitted_at))
|
|
2369
|
+
if not marker_agent:
|
|
2370
|
+
# a live un-slotted marker is another lane's submit
|
|
2371
|
+
# (steward/intake, or a pre-width deploy): serial
|
|
2372
|
+
nonslot_busy = True
|
|
2373
|
+
else:
|
|
2374
|
+
# Died before writing a record: persist the crash memory as a
|
|
2375
|
+
# PER-BENCHMARK tombstone (a sibling launch must not erase
|
|
2376
|
+
# this — terra #172 r3), then free the slot.
|
|
2377
|
+
write_tombstone(root, spec.target, str(pending.get("benchmark", "")), submitted_at)
|
|
2378
|
+
clear_pending(root, spec.target, marker_agent)
|
|
2379
|
+
stranded_cutoff = now - STRANDED_IMPLEMENTING_S
|
|
2380
|
+
for r in records:
|
|
2381
|
+
if r.target == spec.target and r.state != ENDED:
|
|
2382
|
+
if r.state == IMPLEMENTING and max(r.updated, r.created) <= stranded_cutoff:
|
|
2383
|
+
continue # stranded: pick ignores it, so must occupancy
|
|
2384
|
+
occupied.add(r.agent_id)
|
|
2385
|
+
if not _SLOT_AGENT_RE.fullmatch(r.agent_id):
|
|
2386
|
+
nonslot_busy = True
|
|
2387
|
+
if nonslot_busy:
|
|
2388
|
+
# steward and intake keep their pre-width one-run-per-target
|
|
2389
|
+
# exclusivity: width applies AMONG self-initiated slots, it
|
|
2390
|
+
# does not license launching beside another lane (terra #173)
|
|
2391
|
+
return None
|
|
2392
|
+
if len(occupied) >= width:
|
|
2393
|
+
return None
|
|
2394
|
+
slot_agent = _free_agent_slot(occupied, width)
|
|
2395
|
+
if slot_agent is None:
|
|
2396
|
+
return None
|
|
2397
|
+
dead_attempts = read_tombstones(root, spec.target, contract, now)
|
|
2398
|
+
benchmark = pick_self_initiated(
|
|
2399
|
+
records, contract, spec.target, now, dead_attempts, live_pendings
|
|
2400
|
+
)
|
|
2401
|
+
if benchmark is None:
|
|
2402
|
+
return None
|
|
2403
|
+
if getattr(contract, "merge", "manual") == "auto" and not spec.panel:
|
|
2404
|
+
# auto merge mode means gate+PANEL clean self-merges; a
|
|
2405
|
+
# deployment with no panel configured must not launch attempts
|
|
2406
|
+
# that would publish panel-less self-merging PRs (terra #171)
|
|
2407
|
+
log.error(
|
|
2408
|
+
"attempt on %s not launched: contract sets merge:auto but "
|
|
2409
|
+
"no panel is configured (set AUTORESEARCH_PANEL, or the "
|
|
2410
|
+
"contract back to merge:manual)",
|
|
2411
|
+
benchmark,
|
|
2412
|
+
)
|
|
2413
|
+
return None
|
|
2414
|
+
if lane_error := _gpu_lane_error(contract, benchmark, spec):
|
|
2415
|
+
log.error("attempt on %s not launched: %s", benchmark, lane_error)
|
|
2416
|
+
return None
|
|
2417
|
+
author_error = _author_config_error(spec)
|
|
2418
|
+
if author_error:
|
|
2419
|
+
log.error(
|
|
2420
|
+
"climb on %s not launched: author misconfigured — %s "
|
|
2421
|
+
"(fix AUTORESEARCH_AUTHOR_BACKEND/_MODEL)",
|
|
2422
|
+
benchmark,
|
|
2423
|
+
author_error,
|
|
2424
|
+
)
|
|
2425
|
+
return None
|
|
2426
|
+
panel_error = _panel_preflight_error(spec)
|
|
2427
|
+
if panel_error:
|
|
2428
|
+
log.error(
|
|
2429
|
+
"climb on %s not launched: panel misconfigured — %s "
|
|
2430
|
+
"(fix it, or set AUTORESEARCH_PANEL='' to disable the panel)",
|
|
2431
|
+
benchmark,
|
|
2432
|
+
panel_error,
|
|
2433
|
+
)
|
|
2434
|
+
return None
|
|
2435
|
+
if dry_run:
|
|
2436
|
+
return (benchmark, "dry-run")
|
|
2437
|
+
job_minutes = _attempt_job_minutes(spec, limits)
|
|
2438
|
+
argv = [
|
|
2439
|
+
"uv",
|
|
2440
|
+
"run",
|
|
2441
|
+
"python",
|
|
2442
|
+
"-m",
|
|
2443
|
+
"outerloop.attempt",
|
|
2444
|
+
"--target",
|
|
2445
|
+
spec.target,
|
|
2446
|
+
"--benchmark",
|
|
2447
|
+
benchmark,
|
|
2448
|
+
"--run-root",
|
|
2449
|
+
str(spec.run_root),
|
|
2450
|
+
"--image",
|
|
2451
|
+
spec.image,
|
|
2452
|
+
"--agent-id",
|
|
2453
|
+
slot_agent,
|
|
2454
|
+
*_climb_limit_argv(limits, job_minutes),
|
|
2455
|
+
*_climb_panel_argv(spec),
|
|
2456
|
+
]
|
|
2457
|
+
if spec.pat_file:
|
|
2458
|
+
argv += ["--pat-file", spec.pat_file]
|
|
2459
|
+
# config-driven author: climb resolves the author backend/model/key from
|
|
2460
|
+
# AUTORESEARCH_AUTHOR_* env (inherited by the job), so the tick threads
|
|
2461
|
+
# neither the backend nor its key — a new backend needs zero tick change.
|
|
2462
|
+
job_id = compute.submit(
|
|
2463
|
+
JobSpec(
|
|
2464
|
+
job_name=f"climb-{benchmark}-{slot_agent}"[:60],
|
|
2465
|
+
account=spec.account,
|
|
2466
|
+
partition=spec.job_partition or spec.partition,
|
|
2467
|
+
time_minutes=job_minutes,
|
|
2468
|
+
command=_flight_command(
|
|
2469
|
+
spec.home, f"climb-{benchmark}-{slot_agent}"[:60], now, argv
|
|
2470
|
+
),
|
|
2471
|
+
cpus=4,
|
|
2472
|
+
mem="8G",
|
|
2473
|
+
)
|
|
2474
|
+
)
|
|
2475
|
+
write_pending(root, spec.target, benchmark, job_id, now, agent=slot_agent)
|
|
2476
|
+
log.info("self-initiated climb on %s: job %s", benchmark, job_id)
|
|
2477
|
+
return (benchmark, job_id)
|
|
2478
|
+
except Exception as exc: # one bad pass must not break the tick
|
|
2479
|
+
log.warning("self-initiated pass failed: %s", exc)
|
|
2480
|
+
return None
|
|
2481
|
+
|
|
2482
|
+
|
|
2483
|
+
def service_steward(
|
|
2484
|
+
root: Path,
|
|
2485
|
+
github: Any,
|
|
2486
|
+
compute: Compute,
|
|
2487
|
+
spec: FollowupSpec,
|
|
2488
|
+
now: float,
|
|
2489
|
+
contract: Any,
|
|
2490
|
+
limits: EffectiveLimits,
|
|
2491
|
+
dry_run: bool = False,
|
|
2492
|
+
) -> tuple[str, str] | None:
|
|
2493
|
+
"""The steward lane: claim at most ONE labeled work-order issue per tick
|
|
2494
|
+
and submit a stewardship job. Off until the operator provisions the
|
|
2495
|
+
steward's own key (role separation) and the contract declares a steward
|
|
2496
|
+
scope."""
|
|
2497
|
+
from outerloop.steward import pick_steward_issue
|
|
2498
|
+
|
|
2499
|
+
target = spec.target
|
|
2500
|
+
if not target or not spec.steward_key_file:
|
|
2501
|
+
return None
|
|
2502
|
+
if getattr(contract, "steward", None) is None:
|
|
2503
|
+
return None
|
|
2504
|
+
try:
|
|
2505
|
+
from outerloop.steward import release_orphaned_claims
|
|
2506
|
+
|
|
2507
|
+
# ONE active run per target covers stewardships too: an env rewrite
|
|
2508
|
+
# must not fly alongside a solver climb or another stewardship.
|
|
2509
|
+
records = list_runs(root)
|
|
2510
|
+
# reconcile first: killed jobs never post their own release — and
|
|
2511
|
+
# BEFORE the outage pause below, because a claim orphaned by the
|
|
2512
|
+
# very session the outage killed must not stay held all cooldown
|
|
2513
|
+
# (reconciliation is model-free bookkeeping; only spawning pauses)
|
|
2514
|
+
release_orphaned_claims(github, target, records, now, bot_login=spec.bot_login)
|
|
2515
|
+
paused = outage_active(root, now, role="steward")
|
|
2516
|
+
if paused:
|
|
2517
|
+
log.info("steward lane paused (api outage: %s)", paused)
|
|
2518
|
+
return None
|
|
2519
|
+
if any(r.target == target and r.state != ENDED for r in records):
|
|
2520
|
+
return None
|
|
2521
|
+
# The queue window (submit -> job writes its record) is bridged by
|
|
2522
|
+
# the SAME per-target pending markers the self-initiated lane uses
|
|
2523
|
+
# — ALL of them, slotted included: a width slot queued without a
|
|
2524
|
+
# record yet must block a stewardship the same way an active run
|
|
2525
|
+
# does. Liveness first, TTL only breaks unknown ties (queue wait
|
|
2526
|
+
# can outlive the TTL).
|
|
2527
|
+
for slot, pending in list_pendings(root, target):
|
|
2528
|
+
marker_agent = "" if not pending.get("agent_id") else slot
|
|
2529
|
+
submitted_at = float(pending.get("submitted_at", 0.0))
|
|
2530
|
+
landed = any(
|
|
2531
|
+
r.target == target
|
|
2532
|
+
and r.created >= submitted_at - 60
|
|
2533
|
+
and (not marker_agent or r.agent_id == marker_agent)
|
|
2534
|
+
for r in records
|
|
2535
|
+
)
|
|
2536
|
+
expired = now - submitted_at > PENDING_TTL_S
|
|
2537
|
+
alive = _holder_alive(compute, str(pending.get("job_id", "")))
|
|
2538
|
+
if not landed and (alive is True or (not expired and alive is not False)):
|
|
2539
|
+
return None
|
|
2540
|
+
task = pick_steward_issue(github, target, contract, spec.bot_login)
|
|
2541
|
+
if task is None:
|
|
2542
|
+
return None
|
|
2543
|
+
if _benchmark_gpus(contract, task.benchmark) > 0:
|
|
2544
|
+
# the stewardship validates its rewrite IN-JOB (SubprocessEvaluator
|
|
2545
|
+
# inside the CPU work job — no GPUs, no --nv), so a GPU benchmark
|
|
2546
|
+
# cannot be stewarded yet; refuse rather than launch a validation
|
|
2547
|
+
# that can only fail (terra #174 r2)
|
|
2548
|
+
log.error(
|
|
2549
|
+
"stewardship on %s not launched: GPU benchmarks validate in-job "
|
|
2550
|
+
"and the steward job has no GPU allocation",
|
|
2551
|
+
task.benchmark,
|
|
2552
|
+
)
|
|
2553
|
+
return None
|
|
2554
|
+
if dry_run:
|
|
2555
|
+
return (f"steward-issue-{task.number}", "dry-run")
|
|
2556
|
+
from outerloop.intake import CLAIM_MARKER, issue_hypothesis
|
|
2557
|
+
|
|
2558
|
+
github.comment(
|
|
2559
|
+
target,
|
|
2560
|
+
task.number,
|
|
2561
|
+
f"{CLAIM_MARKER}\nClaimed by the steward for benchmark "
|
|
2562
|
+
f"`{task.benchmark}`; a run is queued and a report will follow here.",
|
|
2563
|
+
)
|
|
2564
|
+
import base64 as _b64
|
|
2565
|
+
|
|
2566
|
+
work_order_b64 = _b64.b64encode(issue_hypothesis(task).encode()).decode()
|
|
2567
|
+
argv = [
|
|
2568
|
+
"uv",
|
|
2569
|
+
"run",
|
|
2570
|
+
"python",
|
|
2571
|
+
"-m",
|
|
2572
|
+
"outerloop.steward",
|
|
2573
|
+
"--target",
|
|
2574
|
+
target,
|
|
2575
|
+
"--benchmark",
|
|
2576
|
+
task.benchmark,
|
|
2577
|
+
"--run-root",
|
|
2578
|
+
str(spec.run_root),
|
|
2579
|
+
"--image",
|
|
2580
|
+
spec.image,
|
|
2581
|
+
"--issue",
|
|
2582
|
+
str(task.number),
|
|
2583
|
+
"--work-order-b64",
|
|
2584
|
+
work_order_b64,
|
|
2585
|
+
"--key-file",
|
|
2586
|
+
spec.steward_key_file,
|
|
2587
|
+
# the SAME clamped walltime the JobSpec requests, so the
|
|
2588
|
+
# self-deadline arms against the real clock
|
|
2589
|
+
*_climb_limit_argv(limits, min(limits.attempt_job_minutes, spec.max_job_minutes)),
|
|
2590
|
+
]
|
|
2591
|
+
if spec.pat_file:
|
|
2592
|
+
argv += ["--pat-file", spec.pat_file]
|
|
2593
|
+
try:
|
|
2594
|
+
job_id = compute.submit(
|
|
2595
|
+
JobSpec(
|
|
2596
|
+
job_name=f"steward-issue-{task.number}",
|
|
2597
|
+
account=spec.account,
|
|
2598
|
+
partition=spec.job_partition or spec.partition,
|
|
2599
|
+
time_minutes=min(limits.attempt_job_minutes, spec.max_job_minutes),
|
|
2600
|
+
command=_flight_command(spec.home, f"steward-issue-{task.number}", now, argv),
|
|
2601
|
+
cpus=4,
|
|
2602
|
+
mem="8G",
|
|
2603
|
+
)
|
|
2604
|
+
)
|
|
2605
|
+
except Exception:
|
|
2606
|
+
# release the claim: a claim with no job behind it would orphan
|
|
2607
|
+
# the work order forever (pick skips claimed issues)
|
|
2608
|
+
from outerloop.steward import RELEASE_MARKER
|
|
2609
|
+
|
|
2610
|
+
with contextlib.suppress(Exception):
|
|
2611
|
+
github.comment(
|
|
2612
|
+
target,
|
|
2613
|
+
task.number,
|
|
2614
|
+
f"{RELEASE_MARKER}\nSubmission failed; claim released — "
|
|
2615
|
+
f"a later tick will retry this work order.",
|
|
2616
|
+
)
|
|
2617
|
+
raise
|
|
2618
|
+
write_pending(root, target, f"steward:{task.benchmark}", job_id, now)
|
|
2619
|
+
log.info("steward issue #%s claimed for job %s", task.number, job_id)
|
|
2620
|
+
return (f"steward-issue-{task.number}", job_id)
|
|
2621
|
+
except Exception as exc: # the steward lane must not break the tick
|
|
2622
|
+
log.warning("steward pass failed: %s", exc)
|
|
2623
|
+
return None
|
|
2624
|
+
|
|
2625
|
+
|
|
2626
|
+
def service_intake(
|
|
2627
|
+
root: Path,
|
|
2628
|
+
github: Any,
|
|
2629
|
+
compute: Compute,
|
|
2630
|
+
spec: FollowupSpec,
|
|
2631
|
+
now: float,
|
|
2632
|
+
contract: Any = None,
|
|
2633
|
+
limits: EffectiveLimits | None = None,
|
|
2634
|
+
dry_run: bool = False,
|
|
2635
|
+
) -> tuple[str, str] | None:
|
|
2636
|
+
"""The requested lane: claim at most ONE qualifying issue per tick and
|
|
2637
|
+
submit a climb job for it. The claim comment (posted by the climb job
|
|
2638
|
+
before its session) marks an issue taken; one-per-tick keeps a burst of
|
|
2639
|
+
issues from bursting the budget. The contract arrives from the tick's
|
|
2640
|
+
single per-target fetch; None (fetch failed) sits the lane out."""
|
|
2641
|
+
from outerloop.contract import load_contract
|
|
2642
|
+
from outerloop.intake import issue_hypothesis, pick_issue
|
|
2643
|
+
|
|
2644
|
+
target = spec.target
|
|
2645
|
+
if not target:
|
|
2646
|
+
return None
|
|
2647
|
+
paused = outage_active(root, now, role="solver")
|
|
2648
|
+
if paused:
|
|
2649
|
+
log.info("intake lane paused (api outage: %s)", paused)
|
|
2650
|
+
return None
|
|
2651
|
+
try:
|
|
2652
|
+
if contract is None:
|
|
2653
|
+
contract_raw = _contract_text(github, target, "main")
|
|
2654
|
+
if contract_raw is None:
|
|
2655
|
+
return None
|
|
2656
|
+
contract = load_contract(contract_raw, target)
|
|
2657
|
+
limits = limits if limits is not None else effective_limits(contract.budgets)
|
|
2658
|
+
task = pick_issue(github, target, contract, spec.bot_login)
|
|
2659
|
+
if task is None:
|
|
2660
|
+
return None
|
|
2661
|
+
if getattr(contract, "merge", "manual") == "auto" and not spec.panel:
|
|
2662
|
+
# auto merge mode means gate+PANEL clean self-merges; a
|
|
2663
|
+
# deployment with no panel configured must not launch attempts
|
|
2664
|
+
# that would publish panel-less self-merging PRs (terra #171)
|
|
2665
|
+
log.error(
|
|
2666
|
+
"attempt on %s not launched: contract sets merge:auto but "
|
|
2667
|
+
"no panel is configured (set AUTORESEARCH_PANEL, or the "
|
|
2668
|
+
"contract back to merge:manual)",
|
|
2669
|
+
task.benchmark,
|
|
2670
|
+
)
|
|
2671
|
+
return None
|
|
2672
|
+
if lane_error := _gpu_lane_error(contract, task.benchmark, spec):
|
|
2673
|
+
log.error("attempt on %s not launched: %s", task.benchmark, lane_error)
|
|
2674
|
+
return None
|
|
2675
|
+
author_error = _author_config_error(spec)
|
|
2676
|
+
if author_error:
|
|
2677
|
+
log.error(
|
|
2678
|
+
"issue #%d not claimed: author misconfigured — %s "
|
|
2679
|
+
"(fix AUTORESEARCH_AUTHOR_BACKEND/_MODEL)",
|
|
2680
|
+
task.number,
|
|
2681
|
+
author_error,
|
|
2682
|
+
)
|
|
2683
|
+
return None
|
|
2684
|
+
panel_error = _panel_preflight_error(spec)
|
|
2685
|
+
if panel_error:
|
|
2686
|
+
log.error(
|
|
2687
|
+
"issue #%d not claimed: panel misconfigured — %s "
|
|
2688
|
+
"(fix it, or set AUTORESEARCH_PANEL='' to disable the panel)",
|
|
2689
|
+
task.number,
|
|
2690
|
+
panel_error,
|
|
2691
|
+
)
|
|
2692
|
+
return None
|
|
2693
|
+
if dry_run:
|
|
2694
|
+
return (f"issue-{task.number}", "dry-run")
|
|
2695
|
+
job_minutes = _attempt_job_minutes(spec, limits)
|
|
2696
|
+
# claim BEFORE submit: Slurm queueing can take minutes, and the next
|
|
2697
|
+
# tick must not re-claim the same issue in that window
|
|
2698
|
+
from outerloop.intake import CLAIM_MARKER, MAX_INTAKE_ATTEMPTS, RELEASE_MARKER
|
|
2699
|
+
|
|
2700
|
+
github.comment(
|
|
2701
|
+
target,
|
|
2702
|
+
task.number,
|
|
2703
|
+
f"{CLAIM_MARKER}\nClaimed for benchmark `{task.benchmark}`; a run "
|
|
2704
|
+
"is queued and a report will follow here.",
|
|
2705
|
+
)
|
|
2706
|
+
import base64 as _b64
|
|
2707
|
+
|
|
2708
|
+
hypothesis_b64 = _b64.b64encode(issue_hypothesis(task).encode()).decode()
|
|
2709
|
+
argv = [
|
|
2710
|
+
"uv",
|
|
2711
|
+
"run",
|
|
2712
|
+
"python",
|
|
2713
|
+
"-m",
|
|
2714
|
+
"outerloop.attempt",
|
|
2715
|
+
"--target",
|
|
2716
|
+
target,
|
|
2717
|
+
"--benchmark",
|
|
2718
|
+
task.benchmark,
|
|
2719
|
+
"--run-root",
|
|
2720
|
+
str(spec.run_root),
|
|
2721
|
+
"--image",
|
|
2722
|
+
spec.image,
|
|
2723
|
+
"--issue",
|
|
2724
|
+
str(task.number),
|
|
2725
|
+
"--hypothesis-b64",
|
|
2726
|
+
hypothesis_b64,
|
|
2727
|
+
*_climb_limit_argv(limits, job_minutes),
|
|
2728
|
+
*_climb_panel_argv(spec),
|
|
2729
|
+
]
|
|
2730
|
+
if spec.pat_file:
|
|
2731
|
+
argv += ["--pat-file", spec.pat_file]
|
|
2732
|
+
# config-driven author: climb resolves the author key from the
|
|
2733
|
+
# AUTORESEARCH_AUTHOR_* env by backend; the tick does not thread it.
|
|
2734
|
+
try:
|
|
2735
|
+
job_id = compute.submit(
|
|
2736
|
+
JobSpec(
|
|
2737
|
+
job_name=f"climb-issue-{task.number}",
|
|
2738
|
+
account=spec.account,
|
|
2739
|
+
partition=spec.job_partition or spec.partition,
|
|
2740
|
+
time_minutes=job_minutes,
|
|
2741
|
+
command=_flight_command(spec.home, f"climb-issue-{task.number}", now, argv),
|
|
2742
|
+
cpus=4,
|
|
2743
|
+
mem="8G",
|
|
2744
|
+
)
|
|
2745
|
+
)
|
|
2746
|
+
except Exception:
|
|
2747
|
+
# the claim is already posted and pick_issue skips claimed
|
|
2748
|
+
# issues, so a failed submit must release it (same pattern as
|
|
2749
|
+
# the steward lane) or the issue is stranded forever
|
|
2750
|
+
with contextlib.suppress(Exception):
|
|
2751
|
+
github.comment(
|
|
2752
|
+
target,
|
|
2753
|
+
task.number,
|
|
2754
|
+
f"{RELEASE_MARKER}\nSubmission failed; claim released — "
|
|
2755
|
+
f"a later tick will retry this issue (intake gives up "
|
|
2756
|
+
f"after {MAX_INTAKE_ATTEMPTS} claim attempts and leaves "
|
|
2757
|
+
f"it for a human).",
|
|
2758
|
+
)
|
|
2759
|
+
raise
|
|
2760
|
+
log.info("issue #%s claimed for climb job %s", task.number, job_id)
|
|
2761
|
+
return (f"issue-{task.number}", job_id)
|
|
2762
|
+
except Exception as exc: # intake must not break the tick
|
|
2763
|
+
log.warning("intake pass failed: %s", exc)
|
|
2764
|
+
return None
|
|
2765
|
+
|
|
2766
|
+
|
|
2767
|
+
@dataclass
|
|
2768
|
+
class LoggingDispatcher:
|
|
2769
|
+
"""Never dispatched in production: main() runs the sweep dry unless
|
|
2770
|
+
dispatched wake is armed, so no lease is taken and no attempt is
|
|
2771
|
+
counted. This exists for the seam."""
|
|
2772
|
+
|
|
2773
|
+
def dispatch(self, record: RunRecord, reason: str) -> str:
|
|
2774
|
+
log.info("WOULD WAKE %s (%s) — session dispatch lands in phase 5", record.run_id, reason)
|
|
2775
|
+
return ""
|
|
2776
|
+
|
|
2777
|
+
|
|
2778
|
+
@dataclass
|
|
2779
|
+
class JobWakeDispatcher:
|
|
2780
|
+
"""Delivers a wake by submitting a Slurm job that runs the wake CLI
|
|
2781
|
+
(`climb --resume <run_id>`), depending on the run's eval jobs (the record's
|
|
2782
|
+
`afterany`) so it fires when they finish — or immediately if they already
|
|
2783
|
+
have. CPU-only and short: a wake reads cached results and opens a PR, it
|
|
2784
|
+
never holds a GPU. Returns the wake job id (async: it owns the lease until
|
|
2785
|
+
it completes)."""
|
|
2786
|
+
|
|
2787
|
+
compute: Compute
|
|
2788
|
+
spec: FollowupSpec
|
|
2789
|
+
now: float
|
|
2790
|
+
wake_minutes: int = 20
|
|
2791
|
+
|
|
2792
|
+
def dispatch(self, record: RunRecord, reason: str) -> str:
|
|
2793
|
+
argv = [
|
|
2794
|
+
"uv",
|
|
2795
|
+
"run",
|
|
2796
|
+
"python",
|
|
2797
|
+
"-m",
|
|
2798
|
+
"outerloop.attempt",
|
|
2799
|
+
"--resume",
|
|
2800
|
+
record.run_id,
|
|
2801
|
+
"--run-root",
|
|
2802
|
+
str(self.spec.run_root),
|
|
2803
|
+
"--image",
|
|
2804
|
+
self.spec.image,
|
|
2805
|
+
"--account",
|
|
2806
|
+
self.spec.account,
|
|
2807
|
+
"--partition",
|
|
2808
|
+
self.spec.partition,
|
|
2809
|
+
"--gpu-partition",
|
|
2810
|
+
self.spec.gpu_partition,
|
|
2811
|
+
"--gpu-account",
|
|
2812
|
+
self.spec.gpu_account,
|
|
2813
|
+
# the wake runs the SAME verification panel as the fresh climb, so a
|
|
2814
|
+
# dispatched improvement is verified before it is published.
|
|
2815
|
+
*_climb_panel_argv(self.spec),
|
|
2816
|
+
# session budget for the depth-axis REVISION (a blocking panel
|
|
2817
|
+
# finding wakes the author to revise).
|
|
2818
|
+
"--max-turns",
|
|
2819
|
+
str(self.spec.max_turns),
|
|
2820
|
+
]
|
|
2821
|
+
# An AUTHOR-SLEEP wake resumes a FULL author session (not the short
|
|
2822
|
+
# read-decide a candidate wake runs), so the Slurm job must fit that
|
|
2823
|
+
# session or walltime kills the resumed session mid-run and the run just
|
|
2824
|
+
# waits for another wake. Size the job to the session
|
|
2825
|
+
# budget + overhead and pass --session-minutes so the in-job
|
|
2826
|
+
# self-deadline fires BEFORE Slurm's walltime. A candidate wake keeps
|
|
2827
|
+
# the short budget (read results + panel).
|
|
2828
|
+
from outerloop.limits import ATTEMPT_OVERHEAD_MINUTES
|
|
2829
|
+
from outerloop.roles import author_spec
|
|
2830
|
+
|
|
2831
|
+
if record.stage.get("phase") == "author-sleep":
|
|
2832
|
+
session_minutes = author_spec().budget.walltime_s // 60
|
|
2833
|
+
argv += ["--session-minutes", str(session_minutes)]
|
|
2834
|
+
job_minutes = min(session_minutes + ATTEMPT_OVERHEAD_MINUTES, self.spec.max_job_minutes)
|
|
2835
|
+
else:
|
|
2836
|
+
job_minutes = self.wake_minutes + _wake_panel_minutes(self.spec)
|
|
2837
|
+
if self.spec.pat_file:
|
|
2838
|
+
argv += ["--pat-file", self.spec.pat_file]
|
|
2839
|
+
# config-driven author: `climb --resume` resolves the author key from the
|
|
2840
|
+
# PARKED RUN's backend (persisted on its record) inside climb.main — the
|
|
2841
|
+
# tick does not thread the key, so a fleet flip picks the right one.
|
|
2842
|
+
name = f"wake-{record.run_id}"[:60]
|
|
2843
|
+
afterany = str(record.stage.get("afterany", ""))
|
|
2844
|
+
return self.compute.submit(
|
|
2845
|
+
JobSpec(
|
|
2846
|
+
job_name=name,
|
|
2847
|
+
account=self.spec.account,
|
|
2848
|
+
partition=self.spec.job_partition or self.spec.partition,
|
|
2849
|
+
time_minutes=job_minutes,
|
|
2850
|
+
command=_flight_command(self.spec.home, name, self.now, argv),
|
|
2851
|
+
dependency=afterany,
|
|
2852
|
+
cpus=2,
|
|
2853
|
+
mem="4G",
|
|
2854
|
+
)
|
|
2855
|
+
)
|
|
2856
|
+
|
|
2857
|
+
|
|
2858
|
+
def _wake_panel_minutes(spec: FollowupSpec) -> int:
|
|
2859
|
+
"""Extra wake walltime for the verification panel it now runs — the base
|
|
2860
|
+
`wake_minutes` covers only reading results + opening the PR. Budgeted for
|
|
2861
|
+
the worst case a single wake reaches: one read per lens PLUS one revision
|
|
2862
|
+
author session (the depth-axis wake-to-revise). The revision's re-measure
|
|
2863
|
+
only DISPATCHES (then the job ends, parked), so it needs no extra time.
|
|
2864
|
+
Grounded in the same judge/author budgets the climb job uses."""
|
|
2865
|
+
from outerloop.panel import panel_read_minutes
|
|
2866
|
+
from outerloop.roles import author_spec
|
|
2867
|
+
|
|
2868
|
+
read_minutes = panel_read_minutes(spec.panel)
|
|
2869
|
+
if not read_minutes:
|
|
2870
|
+
return 0
|
|
2871
|
+
return read_minutes + author_spec().budget.walltime_s // 60
|
|
2872
|
+
|
|
2873
|
+
|
|
2874
|
+
def _wake_dispatcher_from_env(
|
|
2875
|
+
compute: Compute, followup_spec: FollowupSpec | None, now: float, root: Path
|
|
2876
|
+
) -> tuple[WakeDispatcher, bool]:
|
|
2877
|
+
"""The wake delivery for this tick, behind an EXPLICIT on-switch so the
|
|
2878
|
+
dispatched-wake path lands DARK. Returns `(dispatcher, live)`:
|
|
2879
|
+
|
|
2880
|
+
* armed (the `AUTORESEARCH_DISPATCH_WAKE` env var OR a `<root>/DISPATCH_WAKE`
|
|
2881
|
+
sentinel file) AND the chain env carries what a wake job needs -> the real
|
|
2882
|
+
`JobWakeDispatcher` and a LIVE sweep;
|
|
2883
|
+
* otherwise -> the `LoggingDispatcher` and a DRY sweep.
|
|
2884
|
+
|
|
2885
|
+
The sentinel mirrors PAUSE: an operator arms/disarms with a touch/rm, no
|
|
2886
|
+
chain restart. So dispatched climbing is turned on deliberately, and a
|
|
2887
|
+
half-configured environment fails safe to dry rather than to a wake job
|
|
2888
|
+
that cannot run."""
|
|
2889
|
+
if not dispatch_wake_armed(root):
|
|
2890
|
+
return LoggingDispatcher(), False
|
|
2891
|
+
if followup_spec is None:
|
|
2892
|
+
log.warning("dispatch-wake armed but the chain env is incomplete; wake stays dry")
|
|
2893
|
+
return LoggingDispatcher(), False
|
|
2894
|
+
log.info("dispatched-wake ON: the waiting-run sweep delivers real wakes this tick")
|
|
2895
|
+
return JobWakeDispatcher(compute, followup_spec, now), True
|
|
2896
|
+
|
|
2897
|
+
|
|
2898
|
+
def _max_job_minutes_from_env() -> int:
|
|
2899
|
+
"""AUTORESEARCH_MAX_JOB_MINUTES, clamped into what the code can honor:
|
|
2900
|
+
at least the climb-job floor (an operator on a short-MaxTime partition
|
|
2901
|
+
must be able to LOWER the cap below cpu_short's 6h, or every submit is
|
|
2902
|
+
rejected), at most the ceiling the stranded window allows. A clamped
|
|
2903
|
+
value logs — a silently-changed cap would read as the partition
|
|
2904
|
+
rejecting jobs for no reason."""
|
|
2905
|
+
from outerloop.limits import ATTEMPT_JOB_MINUTES_FLOOR
|
|
2906
|
+
|
|
2907
|
+
raw = os.environ.get("AUTORESEARCH_MAX_JOB_MINUTES", "").strip()
|
|
2908
|
+
if not raw:
|
|
2909
|
+
return MAX_ATTEMPT_JOB_MINUTES
|
|
2910
|
+
try:
|
|
2911
|
+
value = int(raw)
|
|
2912
|
+
except ValueError:
|
|
2913
|
+
log.warning("AUTORESEARCH_MAX_JOB_MINUTES=%r is not an integer; using default", raw)
|
|
2914
|
+
return MAX_ATTEMPT_JOB_MINUTES
|
|
2915
|
+
clamped = max(ATTEMPT_JOB_MINUTES_FLOOR, min(value, MAX_JOB_MINUTES_CEILING))
|
|
2916
|
+
if clamped != value:
|
|
2917
|
+
log.warning("AUTORESEARCH_MAX_JOB_MINUTES=%d clamped to %d", value, clamped)
|
|
2918
|
+
return clamped
|
|
2919
|
+
|
|
2920
|
+
|
|
2921
|
+
def _cadence_s() -> float:
|
|
2922
|
+
"""The chain's tick cadence in seconds (AUTORESEARCH_CADENCE_MIN, the same
|
|
2923
|
+
knob tick_chain.sbatch uses), defaulting to 30 min when unset/invalid."""
|
|
2924
|
+
raw = os.environ.get("AUTORESEARCH_CADENCE_MIN", "").strip()
|
|
2925
|
+
try:
|
|
2926
|
+
cadence_s = float(raw) * 60 if raw else 30 * 60
|
|
2927
|
+
except ValueError:
|
|
2928
|
+
cadence_s = 30 * 60
|
|
2929
|
+
return cadence_s if (math.isfinite(cadence_s) and cadence_s > 0) else 30 * 60
|
|
2930
|
+
|
|
2931
|
+
|
|
2932
|
+
def _coalesce_ceiling_s() -> float:
|
|
2933
|
+
"""The largest SAFE coalesce window, bounding both the default and an
|
|
2934
|
+
explicit AUTORESEARCH_MIN_TICK_MINUTES: half the cadence (so an on-cadence
|
|
2935
|
+
tick is never coalesced even when the previous one ran a little late), and
|
|
2936
|
+
never above the absolute MAX_MIN_TICK_S. A window at/above the cadence would
|
|
2937
|
+
swallow every normal tick and stall the loop — this is what forbids it."""
|
|
2938
|
+
return min(float(MAX_MIN_TICK_S), _cadence_s() / 2)
|
|
2939
|
+
|
|
2940
|
+
|
|
2941
|
+
def _default_min_tick_s() -> float:
|
|
2942
|
+
"""The coalesce window when none is set: the safe ceiling, further capped at
|
|
2943
|
+
the 10-min DEFAULT_MIN_TICK_S — small enough to only catch pile-ups, and
|
|
2944
|
+
cadence-aware so a short cadence scales it down instead of swallowing every
|
|
2945
|
+
tick."""
|
|
2946
|
+
return min(DEFAULT_MIN_TICK_S, _coalesce_ceiling_s())
|
|
2947
|
+
|
|
2948
|
+
|
|
2949
|
+
def _min_tick_s_from_env() -> float:
|
|
2950
|
+
"""AUTORESEARCH_MIN_TICK_MINUTES -> the coalesce window in seconds. Unset
|
|
2951
|
+
derives a cadence-aware default; non-numeric/non-finite also fall back to it;
|
|
2952
|
+
negative clamps to 0 (coalesce disabled); a value at/above the safe ceiling
|
|
2953
|
+
(half the cadence, capped at MAX_MIN_TICK_S) clamps down so it cannot stall
|
|
2954
|
+
the loop."""
|
|
2955
|
+
raw = os.environ.get("AUTORESEARCH_MIN_TICK_MINUTES", "").strip()
|
|
2956
|
+
if not raw:
|
|
2957
|
+
return _default_min_tick_s()
|
|
2958
|
+
try:
|
|
2959
|
+
minutes = float(raw)
|
|
2960
|
+
except ValueError:
|
|
2961
|
+
log.warning("AUTORESEARCH_MIN_TICK_MINUTES=%r is not a number; using default", raw)
|
|
2962
|
+
return _default_min_tick_s()
|
|
2963
|
+
# reject inf/nan: an infinite window would coalesce every future tick and
|
|
2964
|
+
# freeze the loop (a finite elapsed time is always < inf)
|
|
2965
|
+
if not math.isfinite(minutes):
|
|
2966
|
+
log.warning("AUTORESEARCH_MIN_TICK_MINUTES=%r is not finite; using default", raw)
|
|
2967
|
+
return _default_min_tick_s()
|
|
2968
|
+
seconds = max(0.0, minutes * 60)
|
|
2969
|
+
ceiling = _coalesce_ceiling_s()
|
|
2970
|
+
if seconds > ceiling:
|
|
2971
|
+
log.warning(
|
|
2972
|
+
"AUTORESEARCH_MIN_TICK_MINUTES=%s exceeds the safe ceiling "
|
|
2973
|
+
"(%.0f min, ~half the cadence); clamping so normal ticks are not coalesced",
|
|
2974
|
+
raw,
|
|
2975
|
+
ceiling / 60,
|
|
2976
|
+
)
|
|
2977
|
+
return ceiling
|
|
2978
|
+
return seconds
|
|
2979
|
+
|
|
2980
|
+
|
|
2981
|
+
def _followup_spec_from_env(root: Path) -> tuple[Any, FollowupSpec | None]:
|
|
2982
|
+
"""GitHub client + FollowupSpec from the chain environment, or Nones when
|
|
2983
|
+
the environment is incomplete (the tick then runs without in-review
|
|
2984
|
+
servicing, and logs what is absent)."""
|
|
2985
|
+
pat_file = os.environ.get("AUTORESEARCH_PAT_FILE", "")
|
|
2986
|
+
app_file = os.environ.get("AUTORESEARCH_GITHUB_APP_FILE", "")
|
|
2987
|
+
account = os.environ.get("AUTORESEARCH_ACCOUNT", "")
|
|
2988
|
+
partition = os.environ.get("AUTORESEARCH_PARTITION", "")
|
|
2989
|
+
image = os.environ.get(
|
|
2990
|
+
"AUTORESEARCH_IMAGE",
|
|
2991
|
+
os.path.expanduser("~/autoresearch-images/agent-py312.sif"),
|
|
2992
|
+
)
|
|
2993
|
+
home = os.environ.get("AUTORESEARCH_HOME", "")
|
|
2994
|
+
# Local compute runs jobs as subprocesses: Slurm placement is meaningless
|
|
2995
|
+
# there, so account/partition are only required when a cluster is in play.
|
|
2996
|
+
placed = bool(account and partition) or local_mode()
|
|
2997
|
+
if (pat_file or app_file) and placed and home and Path(image).is_file():
|
|
2998
|
+
from outerloop.appauth import resolve_bot_auth
|
|
2999
|
+
from outerloop.github import GitHubClient
|
|
3000
|
+
|
|
3001
|
+
try:
|
|
3002
|
+
github = GitHubClient(auth=resolve_bot_auth(pat_file, app_file))
|
|
3003
|
+
followup_spec = FollowupSpec(
|
|
3004
|
+
account=account,
|
|
3005
|
+
partition=partition,
|
|
3006
|
+
run_root=root,
|
|
3007
|
+
image=image,
|
|
3008
|
+
home=Path(home),
|
|
3009
|
+
pat_file=pat_file,
|
|
3010
|
+
github_app_file=app_file,
|
|
3011
|
+
target=os.environ.get(
|
|
3012
|
+
"AUTORESEARCH_TARGET", "agentic-learning-ai-lab/autoresearch-pilot"
|
|
3013
|
+
),
|
|
3014
|
+
steward_key_file=os.environ.get("AUTORESEARCH_STEWARD_KEY_FILE", ""),
|
|
3015
|
+
panel=os.environ.get("AUTORESEARCH_PANEL", "verify,review"),
|
|
3016
|
+
panel_key_file=os.environ.get("AUTORESEARCH_PANEL_KEY_FILE", ""),
|
|
3017
|
+
job_partition=os.environ.get("AUTORESEARCH_JOB_PARTITION", ""),
|
|
3018
|
+
gpu_partition=os.environ.get("AUTORESEARCH_GPU_PARTITION", ""),
|
|
3019
|
+
gpu_account=os.environ.get("AUTORESEARCH_GPU_ACCOUNT", ""),
|
|
3020
|
+
max_job_minutes=_max_job_minutes_from_env(),
|
|
3021
|
+
)
|
|
3022
|
+
return github, followup_spec
|
|
3023
|
+
except Exception as exc:
|
|
3024
|
+
log.warning("in-review servicing disabled: %s", exc)
|
|
3025
|
+
return None, None
|
|
3026
|
+
absent = [
|
|
3027
|
+
name
|
|
3028
|
+
for name, value in [
|
|
3029
|
+
("AUTORESEARCH_PAT_FILE or _GITHUB_APP_FILE", pat_file or app_file),
|
|
3030
|
+
("AUTORESEARCH_ACCOUNT", account or ("-" if local_mode() else "")),
|
|
3031
|
+
("AUTORESEARCH_PARTITION", partition or ("-" if local_mode() else "")),
|
|
3032
|
+
("AUTORESEARCH_HOME", home),
|
|
3033
|
+
]
|
|
3034
|
+
if not value
|
|
3035
|
+
]
|
|
3036
|
+
if not Path(image).is_file():
|
|
3037
|
+
absent.append(f"image:{image}")
|
|
3038
|
+
log.info("in-review servicing disabled (missing: %s)", ", ".join(absent))
|
|
3039
|
+
return None, None
|
|
3040
|
+
|
|
3041
|
+
|
|
3042
|
+
def _loop_cadence_s(cadence_min: float) -> float:
|
|
3043
|
+
"""The --loop sleep, clamped to [60s, 24h]: argparse accepts inf (which
|
|
3044
|
+
would OverflowError out of time.sleep) and sub-minute values would spin.
|
|
3045
|
+
A non-positive argument defers to AUTORESEARCH_CADENCE_MIN."""
|
|
3046
|
+
return min(24 * 3600.0, max(60.0, cadence_min * 60 if cadence_min > 0 else _cadence_s()))
|
|
3047
|
+
|
|
3048
|
+
|
|
3049
|
+
def main() -> int:
|
|
3050
|
+
import argparse
|
|
3051
|
+
import time
|
|
3052
|
+
|
|
3053
|
+
parser = argparse.ArgumentParser(description="One tick of the autoresearch loop.")
|
|
3054
|
+
parser.add_argument("--root", required=True, type=Path, help="state root on the shared FS")
|
|
3055
|
+
parser.add_argument("--grace-s", type=float, default=DEFAULT_GRACE_S)
|
|
3056
|
+
parser.add_argument("--lease-ttl-s", type=float, default=DEFAULT_LEASE_TTL_S)
|
|
3057
|
+
parser.add_argument(
|
|
3058
|
+
"--min-free-gb",
|
|
3059
|
+
type=float,
|
|
3060
|
+
default=DEFAULT_MIN_FREE_BYTES / 1024**3,
|
|
3061
|
+
help="skip launching new work when the state filesystem has less free",
|
|
3062
|
+
)
|
|
3063
|
+
parser.add_argument(
|
|
3064
|
+
"--loop",
|
|
3065
|
+
action="store_true",
|
|
3066
|
+
help="run a tick every cadence in the foreground — the local-mode "
|
|
3067
|
+
"chain (Slurm deployments use tick_chain.sbatch instead)",
|
|
3068
|
+
)
|
|
3069
|
+
parser.add_argument(
|
|
3070
|
+
"--cadence-min",
|
|
3071
|
+
type=float,
|
|
3072
|
+
default=0.0,
|
|
3073
|
+
help="minutes between --loop ticks; unset defers to "
|
|
3074
|
+
"AUTORESEARCH_CADENCE_MIN via the chain's own parser (default 30)",
|
|
3075
|
+
)
|
|
3076
|
+
args = parser.parse_args()
|
|
3077
|
+
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
3078
|
+
|
|
3079
|
+
args.root.mkdir(parents=True, exist_ok=True)
|
|
3080
|
+
# The tick's --root is the authority; children (and this process's own
|
|
3081
|
+
# LocalCompute) read AUTORESEARCH_ROOT, so a bare `tick --loop --root X`
|
|
3082
|
+
# must not split-brain them: local job states would land nowhere and
|
|
3083
|
+
# every finished job would read GONE until the park deadline.
|
|
3084
|
+
# RESOLVED: local jobs cd into flight checkouts, so a relative root
|
|
3085
|
+
# would scatter their state dirs across working directories
|
|
3086
|
+
if os.environ.get("AUTORESEARCH_ROOT", "") != str(args.root.resolve()):
|
|
3087
|
+
os.environ["AUTORESEARCH_ROOT"] = str(args.root.resolve())
|
|
3088
|
+
# In-review servicing is LIVE when credentials + image are available in the
|
|
3089
|
+
# chain environment. The waiting-run sweep delivers real wakes only when the
|
|
3090
|
+
# operator arms it — the AUTORESEARCH_DISPATCH_WAKE env var or a
|
|
3091
|
+
# <root>/DISPATCH_WAKE sentinel — and the env is complete; by default it
|
|
3092
|
+
# stays dry with the LoggingDispatcher — dispatched climbing lands DARK.
|
|
3093
|
+
# ONE compute for the process: LocalCompute remembers its jobs' states
|
|
3094
|
+
# in memory, so a --loop deployment must not discard them between ticks.
|
|
3095
|
+
compute = compute_from_env()
|
|
3096
|
+
|
|
3097
|
+
def run_once() -> None:
|
|
3098
|
+
github, followup_spec = _followup_spec_from_env(args.root)
|
|
3099
|
+
now = time.time()
|
|
3100
|
+
dispatcher, wake_live = _wake_dispatcher_from_env(compute, followup_spec, now, args.root)
|
|
3101
|
+
# parks arm their own wake from this recipe; without it the sweep delivers.
|
|
3102
|
+
# Local compute never arms: jobs are synchronous, so an afterany wake's
|
|
3103
|
+
# dependencies are terminal before submit returns — the next loop
|
|
3104
|
+
# iteration's sweep delivers every wake instead (wake latency = cadence).
|
|
3105
|
+
if wake_live and followup_spec is not None and not isinstance(compute, LocalCompute):
|
|
3106
|
+
write_wake_spec(args.root, followup_spec)
|
|
3107
|
+
else:
|
|
3108
|
+
remove_wake_spec(args.root)
|
|
3109
|
+
|
|
3110
|
+
report = tick(
|
|
3111
|
+
args.root,
|
|
3112
|
+
compute,
|
|
3113
|
+
dispatcher,
|
|
3114
|
+
now=now,
|
|
3115
|
+
grace_s=args.grace_s,
|
|
3116
|
+
lease_ttl_s=args.lease_ttl_s,
|
|
3117
|
+
dry_run=not wake_live,
|
|
3118
|
+
github=github,
|
|
3119
|
+
followup_spec=followup_spec,
|
|
3120
|
+
followup_dry_run=False,
|
|
3121
|
+
min_free_bytes=int(args.min_free_gb * 1024**3),
|
|
3122
|
+
min_tick_s=_min_tick_s_from_env(),
|
|
3123
|
+
)
|
|
3124
|
+
# Stamp the coalesce marker at REAL completion time (a fresh time.time(),
|
|
3125
|
+
# not the start-of-tick `now`), so a long tick does not leave a stale marker.
|
|
3126
|
+
mark_tick_complete(args.root, report, time.time())
|
|
3127
|
+
log.info(
|
|
3128
|
+
"tick done: paused=%s coalesced=%s swept=%d woken=%d deferred=%d reaped=%d stuck=%d "
|
|
3129
|
+
"impl_ended=%s review_ended=%s followups=%s intake=%s self_initiated=%s steward=%s "
|
|
3130
|
+
"disk=%s launch_blocked=%s shed=%d",
|
|
3131
|
+
report.paused,
|
|
3132
|
+
report.coalesced,
|
|
3133
|
+
report.swept,
|
|
3134
|
+
len(report.woken),
|
|
3135
|
+
len(report.deferred),
|
|
3136
|
+
len(report.reaped_leases),
|
|
3137
|
+
len(report.stuck),
|
|
3138
|
+
report.implementing_ended or "-",
|
|
3139
|
+
report.review_ended,
|
|
3140
|
+
report.followups_submitted,
|
|
3141
|
+
report.intake,
|
|
3142
|
+
report.self_initiated,
|
|
3143
|
+
report.steward,
|
|
3144
|
+
report.disk or "ok",
|
|
3145
|
+
report.launch_blocked,
|
|
3146
|
+
len(report.shed),
|
|
3147
|
+
)
|
|
3148
|
+
|
|
3149
|
+
if not args.loop:
|
|
3150
|
+
run_once()
|
|
3151
|
+
return 0
|
|
3152
|
+
# The local-mode chain: same stateless tick, a foreground loop instead of
|
|
3153
|
+
# sbatch successors. Records on disk carry all state, so killing and
|
|
3154
|
+
# restarting the loop resumes exactly like the Slurm chain would.
|
|
3155
|
+
cadence_s = _loop_cadence_s(args.cadence_min)
|
|
3156
|
+
while True:
|
|
3157
|
+
started = time.time()
|
|
3158
|
+
try:
|
|
3159
|
+
run_once()
|
|
3160
|
+
except Exception:
|
|
3161
|
+
log.exception("tick failed; the loop continues")
|
|
3162
|
+
time.sleep(max(0.0, cadence_s - (time.time() - started)))
|
|
3163
|
+
|
|
3164
|
+
|
|
3165
|
+
if __name__ == "__main__":
|
|
3166
|
+
raise SystemExit(main())
|