outerloop-science 0.1.0.dev0__py3-none-any.whl → 0.1.0.dev2__py3-none-any.whl

outerloop/tick.py

@@ -21,6 +21,7 @@ import os
 import re
 import socket
 import subprocess
+import sys
 from collections.abc import Sequence
 from dataclasses import asdict, dataclass, field, replace
 from pathlib import Path
@@ -35,6 +36,7 @@ from outerloop.compute import (
     SlurmError,
     SlurmQueryError,
     compute_from_env,
+    gpus_in_gres,
     is_pending,
     is_terminal,
     local_mode,
@@ -73,7 +75,7 @@ log = logging.getLogger(__name__)
 PAUSE_SENTINEL = "PAUSE"
 # Operator on-switch for dispatched-wake, mirroring PAUSE: a root-relative
 # sentinel an operator arms/disarms with a touch/rm — no chain restart, no
-# env-var surgery on a live tick. The AUTORESEARCH_DISPATCH_WAKE env var still
+# env-var surgery on a live tick. The OUTERLOOP_DISPATCH_WAKE env var still
 # works too (either arms it); the sentinel is the reversible, restart-free path.
 DISPATCH_WAKE_SENTINEL = "DISPATCH_WAKE"
 HEARTBEAT_NAME = "heartbeat.json"
@@ -87,6 +89,34 @@ DEFAULT_GRACE_S = 15 * 60
 # a blind park (no job ids to poll) waits its eval walltime plus this queue
 # slack before a follow-up is sent to look for the result
 BLIND_PARK_SLACK_MIN = 12 * 60
+
+# Slurm pending reasons that mean "the job will run when its turn comes": the
+# queue is busy, a reservation or dependency is ahead of it, or the account's
+# OWN jobs hold a per-user/per-account/group cap that clears as they finish. A
+# job in this state keeps accruing priority age; cancelling and resubmitting
+# would put it at the back of the queue behind itself. Per-job limits
+# (`...PerJob...`), a dependency that can never be satisfied, an invalid
+# account/QOS or a held job are NOT waits: those never clear on their own.
+# JobHeldUser is the kernel's own launch admission (service_admission): a
+# held launch waits for a GPU slot, and an operator's hold is theirs to lift.
+QUEUE_WAIT_REASONS = frozenset(
+    {"Priority", "Resources", "Reservation", "Dependency", "ReqNodeNotAvail", "JobHeldUser"}
+)
+
+
+def is_queue_wait(reason: str) -> bool:
+    """Whether a squeue pending reason describes a healthy wait (see
+    QUEUE_WAIT_REASONS). Reasons arrive as `Name` or `Name, detail...`."""
+    head = reason.strip().split(",")[0].split(" ")[0].strip()
+    if not head:
+        return False
+    if head in QUEUE_WAIT_REASONS:
+        return True
+    if "PerJob" in head:
+        return False
+    return any(tag in head for tag in ("PerUser", "PerAccount", "Grp"))
+
+
 # A held lease is stale after the session timeout plus slack.
 DEFAULT_LEASE_TTL_S = 3600 + 15 * 60
 # Coalesce guard: skip a tick's work if another ran within this window. Under
@@ -94,7 +124,7 @@ DEFAULT_LEASE_TTL_S = 3600 + 15 * 60
 # (serialized by the singleton dependency), so they would run back-to-back and
 # redundantly re-sweep. The chain schedules ticks a full cadence apart by
 # begin-time, so only late-bunched pile-ups fall inside this window; keep it
-# well BELOW the cadence (default 30 min). 0 disables. Env: AUTORESEARCH_MIN_TICK_MINUTES.
+# well BELOW the cadence (default 30 min). 0 disables. Env: OUTERLOOP_MIN_TICK_MINUTES.
 DEFAULT_MIN_TICK_S = 10 * 60
 # Ceiling for the coalesce window: a value above this is almost certainly a typo
 # (a window near/over the cadence would coalesce every on-cadence tick and stall
@@ -153,8 +183,8 @@ class TickReport:
 # The submitted walltime must never exceed the job partition's MaxTime —
 # sbatch REJECTS a longer request outright, which would ground every climb.
 # The DEFAULT matches cpu_short (6 h); an operator moving work jobs to a
-# longer partition (AUTORESEARCH_JOB_PARTITION=cpu48) raises the cap with
-# AUTORESEARCH_MAX_JOB_MINUTES. Code-side ceiling: the cap must stay under
+# longer partition (OUTERLOOP_JOB_PARTITION=cpu48) raises the cap with
+# OUTERLOOP_MAX_JOB_MINUTES. Code-side ceiling: the cap must stay under
 # STRANDED_IMPLEMENTING_S or the picker declares live runs stranded — jobs
 # longer than 10 h need that window made spec-aware first (named gap). The
 # self-deadline arms at the CLAMPED value, so a job that wanted more time
@@ -180,12 +210,12 @@ class FollowupSpec:
     partition: str
     run_root: Path
     image: str
-    home: Path  # AUTORESEARCH_HOME: cwd for the submitted job
+    home: Path  # OUTERLOOP_HOME: cwd for the submitted job
     bot_login: str = field(default_factory=_bot_login_default)
     time_minutes: int = 90  # min()'d with the contract's followup_job_minutes
     max_turns: int = DEFAULT_MAX_TURNS  # session turn budget for follow-up jobs
     pat_file: str = ""  # forwarded to the job; "" = the followup CLI default
-    # GitHub App config path; jobs inherit AUTORESEARCH_GITHUB_APP_FILE from
+    # GitHub App config path; jobs inherit OUTERLOOP_GITHUB_APP_FILE from
     # the tick environment, so it is never threaded through argv
     github_app_file: str = ""
     target: str = ""  # the repo the intake pass scans for requested-lane issues
@@ -193,7 +223,7 @@ class FollowupSpec:
     # until the operator provisions it
     steward_key_file: str = ""
     # Pre-PR verification panel for climb jobs (docs/design/orchestrator-verify.md).
-    # DEFAULT ON — the flip is code, the off-switch is AUTORESEARCH_PANEL="".
+    # DEFAULT ON — the flip is code, the off-switch is OUTERLOOP_PANEL="".
     # The climb CLI fails LOUDLY on a bad panel config (a configured gate must
     # never silently vanish); the tick preflights the same rules — lens
     # grammar AND key file — before claiming or submitting so nothing is
@@ -214,6 +244,11 @@ class FollowupSpec:
     # are minutes, work jobs can be hours, and Slurm prices walltime into
     # scheduling priority, so the two deserve independent placement.
     job_partition: str = ""
+    # Launch admission: the most GPUs this user's jobs may hold at once
+    # (OUTERLOOP_MAX_LAUNCH_GPUS, e.g. the QOS's MaxTRESPerUser). 0 = off:
+    # launches queue as submitted. On, GPU launches are submitted held and
+    # service_admission releases them oldest-first under the cap.
+    max_launch_gpus: int = 0
     # Partition MaxTime for work jobs — the panel-augmented walltime clamps
     # here (see MAX_ATTEMPT_JOB_MINUTES). Raise together with job_partition.
     max_job_minutes: int = MAX_ATTEMPT_JOB_MINUTES
@@ -234,6 +269,10 @@ def flight_checkout(home: Path, name: str, now: float) -> Path:
     uncommitted hand-edits in the shared checkout do not fly — only
     deployed code does. Failures fall back to the shared checkout — a
     snapshot must never ground the fleet."""
+    if not (home / ".git").exists():
+        # not a checkout (the local loop on an installed package): nothing to
+        # pin, the job runs from the home directory itself
+        return home
     flights = home.parent / "flights"
     try:
         flights.mkdir(parents=True, exist_ok=True)
@@ -281,11 +320,30 @@ def _gpu_lane_error(contract: Any, benchmark: str, spec: FollowupSpec) -> str:
     if gpu_benches:
         return (
             f"contract has GPU benchmarks ({', '.join(gpu_benches)}) but no GPU lane is "
-            "configured (set AUTORESEARCH_GPU_PARTITION)"
+            "configured (set OUTERLOOP_GPU_PARTITION)"
         )
     return ""
 
 
+_UNCONTAINED_WARNED = False  # the local-mode warning is said once per process
+
+
+def _containment(image: str) -> list[str]:
+    """The job's containment flags: the image when there is one, else the
+    explicit uncontained flag every entry point requires in its absence."""
+    return ["--image", image] if image else ["--uncontained"]
+
+
+def _interpreter(home: Path) -> list[str]:
+    """How a job runs Python. From a source checkout, `uv run python` resolves
+    the project's own environment, as the cluster's flights do. From a plain
+    home directory (the local loop on an installed package) the job runs the
+    interpreter this tick runs under, which is where the package is."""
+    if (home / "pyproject.toml").is_file():
+        return ["uv", "run", "python"]
+    return [sys.executable]
+
+
 def _flight_command(home: Path, job_name: str, now: float, argv: list[str]) -> str:
     """The job's shell command, cd'ing into a fresh flight snapshot.
 
@@ -682,8 +740,29 @@ def service_in_review(
                 except SlurmQueryError:
                     continue  # unknown — do not stack another job
             # the wake-attempt counter caps follow-up retries too: a responder
-            # that cannot advance its cursors must not burn a session per tick
-            if record.wake_attempts >= MAX_WAKE_ATTEMPTS:
+            # that cannot advance its cursors must not burn a session per tick.
+            # A LANDED re-measure gets its own allowance: the sessions that
+            # synced the base and dispatched it were progress, and capping the
+            # follow-up that pushes the sealed number parks the result forever
+            # (speedrun agent-03, 2026-09-04: three syncs spent the cap, the
+            # measure completed, and the run idled for two days on this
+            # branch). The allowance is counted on the stage itself, so a
+            # finishing session that reverts and leaves the stage intact is
+            # retried MAX_WAKE_ATTEMPTS times, not once per tick; a stage that
+            # finishes is cleared, and with it the count.
+            finish_attempts = (
+                int(record.followup_stage.get("finish_attempts", 0) or 0)  # type: ignore[call-overload]
+                if measure_ready
+                else 0
+            )
+            if measure_ready and finish_attempts >= MAX_WAKE_ATTEMPTS:
+                log.warning(
+                    "run %s: %d follow-ups failed to finish the landed re-measure; parked",
+                    record.run_id,
+                    finish_attempts,
+                )
+                continue
+            if record.wake_attempts >= MAX_WAKE_ATTEMPTS and not measure_ready:
                 log.warning(
                     "run %s: %d follow-up attempts without progress; not resubmitting",
                     record.run_id,
@@ -725,17 +804,14 @@ def service_in_review(
             if panel_argv:
                 panel_argv = [*panel_argv, "--panel-minutes", str(job_minutes - author_minutes)]
             argv = [
-                "uv",
-                "run",
-                "python",
+                *_interpreter(spec.home),
                 "-m",
                 "outerloop.followup",
                 "--run-root",
                 str(spec.run_root),
                 "--run-id",
                 record.run_id,
-                "--image",
-                spec.image,
+                *_containment(spec.image),
                 "--bot-login",
                 spec.bot_login,
                 "--job-minutes",
@@ -777,11 +853,16 @@ def service_in_review(
             # read-modify-write on the FRESH record: the submitted job may
             # already be saving its own fields
             latest = load_record(root, record.run_id)
+            stage = latest.followup_stage
+            if measure_ready and stage:
+                # one finishing attempt billed against this landed measure
+                stage = {**stage, "finish_attempts": finish_attempts + 1}
             save_record(
                 root,
                 replace(
                     latest,
                     followup_job_id=job_id,
+                    followup_stage=stage,
                     wake_attempts=latest.wake_attempts + 1,
                 ),
                 now,
@@ -914,7 +995,7 @@ def dispatch_wake_armed(root: Path) -> bool:
     sentinel file (touch/rm, no chain restart). Read by the tick and by every
     park, so a disarm takes effect at once."""
     return (
-        bool(os.environ.get("AUTORESEARCH_DISPATCH_WAKE", "").strip())
+        bool(os.environ.get("OUTERLOOP_DISPATCH_WAKE", "").strip())
         or (root / DISPATCH_WAKE_SENTINEL).exists()
     )
 
@@ -1565,6 +1646,32 @@ def _sweep_one(
             if now - record.terminal_seen >= grace_s:
                 wake(record, f"experiment {state}", state)
         elif any(is_pending(s) for s in states) and record.deadline > 0 and now > record.deadline:
+            # Past the deadline with a job still queued. Ask Slurm WHY before
+            # calling it unschedulable: a busy queue or the account's own cap is
+            # a wait a scientist would sit out, so the deadline moves out by one
+            # slack window instead (Torch 2026-09-06: four launches pending on
+            # QOSMaxGRESPerUser were about to be cancelled and re-launched into
+            # the same cap). Local compute has no queue and no reasons.
+            pending = [j for j, s in zip(job_ids, states, strict=True) if is_pending(s)]
+            reason_of = getattr(compute, "pending_reason", None)
+            reasons: dict[str, str] = {}
+            if reason_of is not None:
+                try:
+                    reasons = {j: str(reason_of(j)) for j in pending}
+                except SlurmQueryError:
+                    deferred.append(record.run_id)  # unknown is never "cancel"
+                    return
+            if reasons and all(is_queue_wait(r) for r in reasons.values()):
+                extended = now + BLIND_PARK_SLACK_MIN * 60
+                if not dry_run:
+                    save_record(root, replace(record, deadline=extended), now)
+                log.info(
+                    "sweep: %s waits in the queue (%s); deadline extended by %d min",
+                    record.run_id,
+                    ", ".join(f"{j}={r}" for j, r in reasons.items()),
+                    BLIND_PARK_SLACK_MIN,
+                )
+                return
             # Unschedulable in practice: cancel every non-terminal job
             # (best-effort — scancel trouble must not abort the sweep), then
             # wake with that fact.
@@ -1756,6 +1863,12 @@ def tick(
             allow_submit=launch_ok,
             contract=contract,
         )
+        # ended runs are known now, so a held launch of an ended run is dropped
+        # and a GPU slot freed by a finished eval is handed to the oldest held one
+        try:
+            service_admission(root, compute, spec, dry_run=followup_dry_run)
+        except Exception as exc:  # admission is advisory to the tick
+            log.warning("admission service failed: %s", exc)
         try:
             service_research_log(root, github, spec, now)
         except Exception as exc:  # the ledger is advisory; the tick continues
@@ -1790,7 +1903,7 @@ def tick(
                 log.warning("self-initiated selection failed: %s", exc)
         # AFTER the launch block: a run started this tick is on the strip
         # this tick, not the next one
-        service_boards(root, github, spec.target, contract, now)
+        service_boards(root, github, spec.target, contract, now, compute)
         report = replace_report(
             report,
             ended,
@@ -1807,6 +1920,120 @@ def tick(
     return report
 
 
+LAUNCH_MARK = "-launch-"  # every author launch job is named <run_id>-launch-<name>
+
+
+@dataclass(frozen=True)
+class AdmissionReport:
+    released: tuple[str, ...] = ()  # job ids let into the queue this tick
+    dropped: tuple[str, ...] = ()  # held launches of runs that already ended
+    held: int = 0  # still waiting after this tick
+    gpus_in_use: int = 0  # GPUs held by this user's eligible jobs, after releases
+
+
+def _cap_reason(reason: str) -> bool:
+    head = reason.strip().split(",")[0].split(" ")[0]
+    return any(tag in head for tag in ("PerUser", "PerAccount", "Grp")) and "PerJob" not in head
+
+
+def service_admission(
+    root: Path, compute: Compute, spec: FollowupSpec, *, dry_run: bool = False
+) -> AdmissionReport:
+    """Launch admission (Torch 2026-09-06: six 8-GPU launches queued against a
+    16-GPU per-user cap, four of them pending for a day on QOSMaxGRESPerUser).
+    Author launches are submitted HELD when a cap is configured; every tick
+    this releases them oldest-first while the user's eligible GPU jobs (evals
+    and launches alike) fit under the cap, and cancels held launches whose run
+    has already ended. A released launch that Slurm then parks on a per-user
+    reason means the configured cap is too high: nothing more is released
+    that tick. Off (cap 0) and in local mode this does nothing. A failed
+    queue read releases nothing (blind is never "free")."""
+    if local_mode():
+        return AdmissionReport()  # local jobs are synchronous; nothing is ever held
+    try:
+        rows = compute.queue_snapshot()
+    except SlurmQueryError as exc:
+        log.warning("admission: cannot read the queue (%s); releasing nothing", exc)
+        return AdmissionReport()
+    # Only the kernel's own LAUNCH jobs are ever released or cancelled here,
+    # attributed by JOB ID through each record's launch ids (a candidate park's
+    # afterany also carries the gate's evals, which are never launches). Names
+    # are never parsed: they are cut to 60 characters at submission, and
+    # `squeue --me` lists every job of the Unix user, ours or not. A held job
+    # no record claims is left exactly as it is.
+    from outerloop.attempt import _stage_launch_job_ids
+
+    owner_state: dict[str, str] = {}
+    for record in list_runs(root):
+        for jid in _stage_launch_job_ids(record):
+            owner_state[jid] = record.state
+    held: list[dict[str, str]] = []
+    in_use = 0
+    at_cap = False
+    dropped: list[str] = []
+    for row in rows:
+        state, reason, jid = row.get("state", ""), row.get("reason", ""), row.get("id", "")
+        gpus = gpus_in_gres(row.get("gres", ""))
+        owner = owner_state.get(jid)
+        if is_pending(state) and reason.startswith("JobHeldUser"):
+            if owner is None:
+                continue  # not ours, or not a launch: never touched
+            if owner != WAITING:
+                # the run ended (PR landed, abandoned, stuck) before its
+                # launch ever ran: nothing will read the result
+                if not dry_run:
+                    with contextlib.suppress(Exception):
+                        compute.cancel(jid)
+                dropped.append(jid)
+                continue
+            held.append(row)
+            continue
+        # every GPU job that is not finished counts against the cap, including
+        # one still COMPLETING: Slurm holds its allocation until it exits
+        if gpus and not is_terminal(state) and state != GONE:
+            in_use += gpus
+            if is_pending(state) and owner is not None and _cap_reason(reason):
+                at_cap = True
+    held.sort(key=lambda r: int(r["id"]) if r["id"].isdigit() else 0)
+    released: list[str] = []
+    if spec.max_launch_gpus <= 0:
+        # admission switched off after these were submitted held: let every
+        # claimed launch go, or they would wait as JobHeldUser forever
+        for row in held:
+            if not dry_run:
+                compute.release(row["id"])
+            released.append(row["id"])
+    elif at_cap:
+        log.info(
+            "admission: a released launch waits on a per-user cap; %d held launch(es) stay held",
+            len(held),
+        )
+    else:
+        for row in held:
+            gpus = gpus_in_gres(row.get("gres", ""))
+            if in_use + gpus > spec.max_launch_gpus:
+                break
+            if not dry_run:
+                compute.release(row["id"])
+            released.append(row["id"])
+            in_use += gpus
+    if released or dropped:
+        log.info(
+            "admission: released %s; dropped %s; %d still held; %d/%d GPUs in use",
+            ",".join(released) or "-",
+            ",".join(dropped) or "-",
+            len(held) - len(released),
+            in_use,
+            spec.max_launch_gpus,
+        )
+    return AdmissionReport(
+        released=tuple(released),
+        dropped=tuple(dropped),
+        held=len(held) - len(released),
+        gpus_in_use=in_use,
+    )
+
+
 def service_syncs(root: Path, spec: Any, now: float) -> None:
     """Honor mid-leg sync requests: a LIVE session asked for fresh origin/*
     refs and is waiting inside its own clock. The fetch pins the canonical
@@ -1844,10 +2071,13 @@ def service_syncs(root: Path, spec: Any, now: float) -> None:
             log.warning("sync failed for %s: %s", record.run_id, exc)
 
 
-def service_boards(root: Path, github: Any, target: str, contract: Any, now: float) -> None:
+def service_boards(
+    root: Path, github: Any, target: str, contract: Any, now: float, compute: Any = None
+) -> None:
     """The climb board and the live strip, together and advisory: the views
     publish from the first tick (before any run ends), and a failure never
-    stops the tick."""
+    stops the tick. With a compute backend, the strip also carries the
+    kernel's queue (its own Slurm jobs, attributed to agents)."""
     try:
         from outerloop.climbboard import contract_directions, service_climb_board
 
@@ -1857,7 +2087,13 @@ def service_boards(root: Path, github: Any, target: str, contract: Any, now: flo
     try:
         from outerloop.climbboard import service_status
 
-        service_status(root, github, target, now, contract)
+        queue = None
+        if compute is not None:
+            try:
+                queue = compute.queue_snapshot()
+            except Exception as exc:  # blind this tick: the strip publishes without a queue
+                log.warning("queue snapshot failed: %s", exc)
+        service_status(root, github, target, now, contract, queue=queue)
     except Exception as exc:  # each is advisory ALONE: one failing never mutes the other
         log.warning("status strip service failed: %s", exc)
 
@@ -2115,13 +2351,13 @@ def _climb_panel_argv(spec: FollowupSpec) -> list[str]:
 def _author_config_error(spec: FollowupSpec) -> str:
     """Why the config-driven author would die at the climb's startup ("" when it
     won't), checked on the tick host BEFORE a claim/submit so a codex misconfig
-    (e.g. AUTORESEARCH_AUTHOR_BACKEND=codex with no non-claude model) never
+    (e.g. OUTERLOOP_AUTHOR_BACKEND=codex with no non-claude model) never
     strands a claimed intake issue. Reads the fleet author config from env — the
     same source the climb defaults from — and the image the tick already knows."""
     from outerloop.attempt import codex_author_config_error
 
-    backend = os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude"
-    model = os.environ.get("AUTORESEARCH_AUTHOR_MODEL") or "claude-opus-5"
+    backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude"
+    model = os.environ.get("OUTERLOOP_AUTHOR_MODEL") or "claude-opus-5"
     return codex_author_config_error(backend, model, spec.image)
 
 
@@ -2152,8 +2388,8 @@ def _panel_preflight_error(spec: FollowupSpec) -> str:
         # neither the author's nor the claude panel key + readable), and for
         # hermes its pinned clone.
         shelled = {
-            "codex": "AUTORESEARCH_PANEL_CODEX_KEY_FILE",
-            "hermes": "AUTORESEARCH_PANEL_HERMES_KEY_FILE",
+            "codex": "OUTERLOOP_PANEL_CODEX_KEY_FILE",
+            "hermes": "OUTERLOOP_PANEL_HERMES_KEY_FILE",
         }
         for lens_backend, key_env in shelled.items():
             if not any(backend == lens_backend for _, backend, _ in lenses):
@@ -2161,7 +2397,7 @@ def _panel_preflight_error(spec: FollowupSpec) -> str:
             if not spec.image or not Path(spec.image).is_file():
                 return (
                     f"a {lens_backend} panel lens requires a real container image "
-                    f"(AUTORESEARCH_IMAGE={spec.image!r})"
+                    f"(OUTERLOOP_IMAGE={spec.image!r})"
                 )
             key_raw = os.environ.get(key_env, "").strip()
             if not key_raw:
@@ -2217,7 +2453,7 @@ def _panel_preflight_error(spec: FollowupSpec) -> str:
         # (claude vs codex keys coexist), config-driven like the climb itself — so
         # the role-separation check compares the panel key against the RIGHT author
         # key, and a codex run is never judged by a stray Claude key.
-        fleet_backend = os.environ.get("AUTORESEARCH_AUTHOR_BACKEND") or "claude"
+        fleet_backend = os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude"
         author = Path(resolve_author_key_file(fleet_backend))
         if not author.is_absolute():
             # same rule as the panel key: the climb resolves paths from a
@@ -2406,7 +2642,7 @@ def service_self_initiated(
             # that would publish panel-less self-merging PRs (terra #171)
             log.error(
                 "attempt on %s not launched: contract sets merge:auto but "
-                "no panel is configured (set AUTORESEARCH_PANEL, or the "
+                "no panel is configured (set OUTERLOOP_PANEL, or the "
                 "contract back to merge:manual)",
                 benchmark,
             )
@@ -2418,7 +2654,7 @@ def service_self_initiated(
         if author_error:
             log.error(
                 "climb on %s not launched: author misconfigured — %s "
-                "(fix AUTORESEARCH_AUTHOR_BACKEND/_MODEL)",
+                "(fix OUTERLOOP_AUTHOR_BACKEND/_MODEL)",
                 benchmark,
                 author_error,
             )
@@ -2427,7 +2663,7 @@ def service_self_initiated(
         if panel_error:
             log.error(
                 "climb on %s not launched: panel misconfigured — %s "
-                "(fix it, or set AUTORESEARCH_PANEL='' to disable the panel)",
+                "(fix it, or set OUTERLOOP_PANEL='' to disable the panel)",
                 benchmark,
                 panel_error,
             )
@@ -2436,9 +2672,7 @@ def service_self_initiated(
             return (benchmark, "dry-run")
         job_minutes = _attempt_job_minutes(spec, limits)
         argv = [
-            "uv",
-            "run",
-            "python",
+            *_interpreter(spec.home),
             "-m",
             "outerloop.attempt",
             "--target",
@@ -2447,8 +2681,7 @@ def service_self_initiated(
             benchmark,
             "--run-root",
             str(spec.run_root),
-            "--image",
-            spec.image,
+            *_containment(spec.image),
             "--agent-id",
             slot_agent,
             *_climb_limit_argv(limits, job_minutes),
@@ -2457,7 +2690,7 @@ def service_self_initiated(
         if spec.pat_file:
             argv += ["--pat-file", spec.pat_file]
         # config-driven author: climb resolves the author backend/model/key from
-        # AUTORESEARCH_AUTHOR_* env (inherited by the job), so the tick threads
+        # OUTERLOOP_AUTHOR_* env (inherited by the job), so the tick threads
         # neither the backend nor its key — a new backend needs zero tick change.
         job_id = compute.submit(
             JobSpec(
@@ -2565,9 +2798,7 @@ def service_steward(
 
         work_order_b64 = _b64.b64encode(issue_hypothesis(task).encode()).decode()
         argv = [
-            "uv",
-            "run",
-            "python",
+            *_interpreter(spec.home),
             "-m",
             "outerloop.steward",
             "--target",
@@ -2576,8 +2807,7 @@ def service_steward(
             task.benchmark,
             "--run-root",
             str(spec.run_root),
-            "--image",
-            spec.image,
+            *_containment(spec.image),
             "--issue",
             str(task.number),
             "--work-order-b64",
@@ -2664,7 +2894,7 @@ def service_intake(
             # that would publish panel-less self-merging PRs (terra #171)
             log.error(
                 "attempt on %s not launched: contract sets merge:auto but "
-                "no panel is configured (set AUTORESEARCH_PANEL, or the "
+                "no panel is configured (set OUTERLOOP_PANEL, or the "
                 "contract back to merge:manual)",
                 task.benchmark,
             )
@@ -2676,7 +2906,7 @@ def service_intake(
         if author_error:
             log.error(
                 "issue #%d not claimed: author misconfigured — %s "
-                "(fix AUTORESEARCH_AUTHOR_BACKEND/_MODEL)",
+                "(fix OUTERLOOP_AUTHOR_BACKEND/_MODEL)",
                 task.number,
                 author_error,
             )
@@ -2685,7 +2915,7 @@ def service_intake(
         if panel_error:
             log.error(
                 "issue #%d not claimed: panel misconfigured — %s "
-                "(fix it, or set AUTORESEARCH_PANEL='' to disable the panel)",
+                "(fix it, or set OUTERLOOP_PANEL='' to disable the panel)",
                 task.number,
                 panel_error,
             )
@@ -2707,9 +2937,7 @@ def service_intake(
 
         hypothesis_b64 = _b64.b64encode(issue_hypothesis(task).encode()).decode()
         argv = [
-            "uv",
-            "run",
-            "python",
+            *_interpreter(spec.home),
             "-m",
             "outerloop.attempt",
             "--target",
@@ -2718,8 +2946,7 @@ def service_intake(
             task.benchmark,
             "--run-root",
             str(spec.run_root),
-            "--image",
-            spec.image,
+            *_containment(spec.image),
             "--issue",
             str(task.number),
             "--hypothesis-b64",
@@ -2730,7 +2957,7 @@ def service_intake(
         if spec.pat_file:
             argv += ["--pat-file", spec.pat_file]
         # config-driven author: climb resolves the author key from the
-        # AUTORESEARCH_AUTHOR_* env by backend; the tick does not thread it.
+        # OUTERLOOP_AUTHOR_* env by backend; the tick does not thread it.
         try:
             job_id = compute.submit(
                 JobSpec(
@@ -2791,17 +3018,14 @@ class JobWakeDispatcher:
 
     def dispatch(self, record: RunRecord, reason: str) -> str:
         argv = [
-            "uv",
-            "run",
-            "python",
+            *_interpreter(self.spec.home),
             "-m",
             "outerloop.attempt",
             "--resume",
             record.run_id,
             "--run-root",
             str(self.spec.run_root),
-            "--image",
-            self.spec.image,
+            *_containment(self.spec.image),
             "--account",
             self.spec.account,
             "--partition",
@@ -2877,7 +3101,7 @@ def _wake_dispatcher_from_env(
     """The wake delivery for this tick, behind an EXPLICIT on-switch so the
     dispatched-wake path lands DARK. Returns `(dispatcher, live)`:
 
-    * armed (the `AUTORESEARCH_DISPATCH_WAKE` env var OR a `<root>/DISPATCH_WAKE`
+    * armed (the `OUTERLOOP_DISPATCH_WAKE` env var OR a `<root>/DISPATCH_WAKE`
       sentinel file) AND the chain env carries what a wake job needs -> the real
       `JobWakeDispatcher` and a LIVE sweep;
     * otherwise -> the `LoggingDispatcher` and a DRY sweep.
@@ -2895,8 +3119,23 @@ def _wake_dispatcher_from_env(
     return JobWakeDispatcher(compute, followup_spec, now), True
 
 
+def max_launch_gpus_from_env() -> int:
+    """OUTERLOOP_MAX_LAUNCH_GPUS: the per-user GPU cap launch admission keeps
+    the fleet under; unset, empty or 0 = admission off. Read by the tick (to
+    release) and by the attempt (to submit held), so the two agree."""
+    raw = os.environ.get("OUTERLOOP_MAX_LAUNCH_GPUS", "").strip()
+    if not raw:
+        return 0
+    try:
+        value = int(raw)
+    except ValueError:
+        log.warning("OUTERLOOP_MAX_LAUNCH_GPUS=%r is not an integer; admission off", raw)
+        return 0
+    return max(value, 0)
+
+
 def _max_job_minutes_from_env() -> int:
-    """AUTORESEARCH_MAX_JOB_MINUTES, clamped into what the code can honor:
+    """OUTERLOOP_MAX_JOB_MINUTES, clamped into what the code can honor:
     at least the climb-job floor (an operator on a short-MaxTime partition
     must be able to LOWER the cap below cpu_short's 6h, or every submit is
     rejected), at most the ceiling the stranded window allows. A clamped
@@ -2904,24 +3143,24 @@ def _max_job_minutes_from_env() -> int:
     rejecting jobs for no reason."""
     from outerloop.limits import ATTEMPT_JOB_MINUTES_FLOOR
 
-    raw = os.environ.get("AUTORESEARCH_MAX_JOB_MINUTES", "").strip()
+    raw = os.environ.get("OUTERLOOP_MAX_JOB_MINUTES", "").strip()
     if not raw:
         return MAX_ATTEMPT_JOB_MINUTES
     try:
         value = int(raw)
     except ValueError:
-        log.warning("AUTORESEARCH_MAX_JOB_MINUTES=%r is not an integer; using default", raw)
+        log.warning("OUTERLOOP_MAX_JOB_MINUTES=%r is not an integer; using default", raw)
         return MAX_ATTEMPT_JOB_MINUTES
     clamped = max(ATTEMPT_JOB_MINUTES_FLOOR, min(value, MAX_JOB_MINUTES_CEILING))
     if clamped != value:
-        log.warning("AUTORESEARCH_MAX_JOB_MINUTES=%d clamped to %d", value, clamped)
+        log.warning("OUTERLOOP_MAX_JOB_MINUTES=%d clamped to %d", value, clamped)
     return clamped
 
 
 def _cadence_s() -> float:
-    """The chain's tick cadence in seconds (AUTORESEARCH_CADENCE_MIN, the same
+    """The chain's tick cadence in seconds (OUTERLOOP_CADENCE_MIN, the same
     knob tick_chain.sbatch uses), defaulting to 30 min when unset/invalid."""
-    raw = os.environ.get("AUTORESEARCH_CADENCE_MIN", "").strip()
+    raw = os.environ.get("OUTERLOOP_CADENCE_MIN", "").strip()
     try:
         cadence_s = float(raw) * 60 if raw else 30 * 60
     except ValueError:
@@ -2931,7 +3170,7 @@ def _cadence_s() -> float:
 
 def _coalesce_ceiling_s() -> float:
     """The largest SAFE coalesce window, bounding both the default and an
-    explicit AUTORESEARCH_MIN_TICK_MINUTES: half the cadence (so an on-cadence
+    explicit OUTERLOOP_MIN_TICK_MINUTES: half the cadence (so an on-cadence
     tick is never coalesced even when the previous one ran a little late), and
     never above the absolute MAX_MIN_TICK_S. A window at/above the cadence would
     swallow every normal tick and stall the loop — this is what forbids it."""
@@ -2947,29 +3186,29 @@ def _default_min_tick_s() -> float:
 
 
 def _min_tick_s_from_env() -> float:
-    """AUTORESEARCH_MIN_TICK_MINUTES -> the coalesce window in seconds. Unset
+    """OUTERLOOP_MIN_TICK_MINUTES -> the coalesce window in seconds. Unset
     derives a cadence-aware default; non-numeric/non-finite also fall back to it;
     negative clamps to 0 (coalesce disabled); a value at/above the safe ceiling
     (half the cadence, capped at MAX_MIN_TICK_S) clamps down so it cannot stall
     the loop."""
-    raw = os.environ.get("AUTORESEARCH_MIN_TICK_MINUTES", "").strip()
+    raw = os.environ.get("OUTERLOOP_MIN_TICK_MINUTES", "").strip()
     if not raw:
         return _default_min_tick_s()
     try:
         minutes = float(raw)
     except ValueError:
-        log.warning("AUTORESEARCH_MIN_TICK_MINUTES=%r is not a number; using default", raw)
+        log.warning("OUTERLOOP_MIN_TICK_MINUTES=%r is not a number; using default", raw)
         return _default_min_tick_s()
     # reject inf/nan: an infinite window would coalesce every future tick and
     # freeze the loop (a finite elapsed time is always < inf)
     if not math.isfinite(minutes):
-        log.warning("AUTORESEARCH_MIN_TICK_MINUTES=%r is not finite; using default", raw)
+        log.warning("OUTERLOOP_MIN_TICK_MINUTES=%r is not finite; using default", raw)
         return _default_min_tick_s()
     seconds = max(0.0, minutes * 60)
     ceiling = _coalesce_ceiling_s()
     if seconds > ceiling:
         log.warning(
-            "AUTORESEARCH_MIN_TICK_MINUTES=%s exceeds the safe ceiling "
+            "OUTERLOOP_MIN_TICK_MINUTES=%s exceeds the safe ceiling "
             "(%.0f min, ~half the cadence); clamping so normal ticks are not coalesced",
             raw,
             ceiling / 60,
@@ -2978,23 +3217,66 @@ def _min_tick_s_from_env() -> float:
     return seconds
 
 
+def _default_image() -> str:
+    """~/outerloop-images/agent-py312.sif, or the pre-rename ~/autoresearch-images
+    path when only that one exists. The fallback is dropped in the release after
+    0.1."""
+    new = os.path.expanduser("~/outerloop-images/agent-py312.sif")
+    old = os.path.expanduser("~/autoresearch-images/agent-py312.sif")
+    return old if (not os.path.isfile(new) and os.path.isfile(old)) else new
+
+
 def _followup_spec_from_env(root: Path) -> tuple[Any, FollowupSpec | None]:
     """GitHub client + FollowupSpec from the chain environment, or Nones when
     the environment is incomplete (the tick then runs without in-review
     servicing, and logs what is absent)."""
-    pat_file = os.environ.get("AUTORESEARCH_PAT_FILE", "")
-    app_file = os.environ.get("AUTORESEARCH_GITHUB_APP_FILE", "")
-    account = os.environ.get("AUTORESEARCH_ACCOUNT", "")
-    partition = os.environ.get("AUTORESEARCH_PARTITION", "")
-    image = os.environ.get(
-        "AUTORESEARCH_IMAGE",
-        os.path.expanduser("~/autoresearch-images/agent-py312.sif"),
-    )
-    home = os.environ.get("AUTORESEARCH_HOME", "")
-    # Local compute runs jobs as subprocesses: Slurm placement is meaningless
-    # there, so account/partition are only required when a cluster is in play.
-    placed = bool(account and partition) or local_mode()
-    if (pat_file or app_file) and placed and home and Path(image).is_file():
+    pat_file = os.environ.get("OUTERLOOP_PAT_FILE", "")
+    app_file = os.environ.get("OUTERLOOP_GITHUB_APP_FILE", "")
+    account = os.environ.get("OUTERLOOP_ACCOUNT", "")
+    partition = os.environ.get("OUTERLOOP_PARTITION", "")
+    image = os.environ.get("OUTERLOOP_IMAGE", _default_image())
+    home = os.environ.get("OUTERLOOP_HOME", "")
+    # Account and partition are optional on Slurm: empty ones leave the billing
+    # association and the partition to Slurm's defaults, as `start` already
+    # does for the resident. Local compute has no placement at all.
+    target = os.environ.get("OUTERLOOP_TARGET", "")
+    if not os.environ.get("OUTERLOOP_BOT_LOGIN", "").strip():
+        # the default is the lab's bot account; under any other credential
+        # the kernel would not recognize its own comments and PRs (#298)
+        from outerloop.github import DEFAULT_BOT_LOGIN
+
+        log.warning(
+            "OUTERLOOP_BOT_LOGIN is not set; posting as the default %r. Set it to the "
+            "login the kernel posts as (init records it) unless that is your account.",
+            DEFAULT_BOT_LOGIN,
+        )
+    image_ok = Path(image).is_file()
+    panel = os.environ.get("OUTERLOOP_PANEL", "verify,review")
+    if not image_ok and local_mode():
+        # The local loop on a machine with no container: sessions run under
+        # the harness's own sandbox and evaluations run bare, on the
+        # operator's own machine with the operator's own keys. The panel is
+        # off unless the operator opts in, because an uncontained judge holds
+        # a shell next to its own key file. Said once per process. Contained
+        # local mode needs apptainer and the image (docs/install.md).
+        global _UNCONTAINED_WARNED
+        if not _UNCONTAINED_WARNED:
+            _UNCONTAINED_WARNED = True
+            log.warning(
+                "local mode: no container image at %s; sessions run under the harness "
+                "sandbox and evaluations run bare on this machine; the panel is %s; a "
+                "codex author needs the image (docs/install.md, local mode)",
+                image,
+                "on by OUTERLOOP_PANEL_UNCONTAINED=1"
+                if os.environ.get("OUTERLOOP_PANEL_UNCONTAINED") == "1"
+                else "off (OUTERLOOP_PANEL_UNCONTAINED=1 turns it on)",
+            )
+        if os.environ.get("OUTERLOOP_PANEL_UNCONTAINED") != "1":
+            panel = ""
+        # the jobs must not inherit a path to an image that is not there
+        os.environ.pop("OUTERLOOP_IMAGE", None)
+        image, image_ok = "", True
+    if (pat_file or app_file) and home and target and image_ok:
         from outerloop.appauth import resolve_bot_auth
         from outerloop.github import GitHubClient
 
@@ -3008,16 +3290,15 @@ def _followup_spec_from_env(root: Path) -> tuple[Any, FollowupSpec | None]:
                 home=Path(home),
                 pat_file=pat_file,
                 github_app_file=app_file,
-                target=os.environ.get(
-                    "AUTORESEARCH_TARGET", "agentic-learning-ai-lab/autoresearch-pilot"
-                ),
-                steward_key_file=os.environ.get("AUTORESEARCH_STEWARD_KEY_FILE", ""),
-                panel=os.environ.get("AUTORESEARCH_PANEL", "verify,review"),
-                panel_key_file=os.environ.get("AUTORESEARCH_PANEL_KEY_FILE", ""),
-                job_partition=os.environ.get("AUTORESEARCH_JOB_PARTITION", ""),
-                gpu_partition=os.environ.get("AUTORESEARCH_GPU_PARTITION", ""),
-                gpu_account=os.environ.get("AUTORESEARCH_GPU_ACCOUNT", ""),
+                target=target,
+                steward_key_file=os.environ.get("OUTERLOOP_STEWARD_KEY_FILE", ""),
+                panel=panel,
+                panel_key_file=os.environ.get("OUTERLOOP_PANEL_KEY_FILE", ""),
+                job_partition=os.environ.get("OUTERLOOP_JOB_PARTITION", ""),
+                gpu_partition=os.environ.get("OUTERLOOP_GPU_PARTITION", ""),
+                gpu_account=os.environ.get("OUTERLOOP_GPU_ACCOUNT", ""),
                 max_job_minutes=_max_job_minutes_from_env(),
+                max_launch_gpus=max_launch_gpus_from_env(),
             )
             return github, followup_spec
         except Exception as exc:
@@ -3026,14 +3307,13 @@ def _followup_spec_from_env(root: Path) -> tuple[Any, FollowupSpec | None]:
     absent = [
         name
         for name, value in [
-            ("AUTORESEARCH_PAT_FILE or _GITHUB_APP_FILE", pat_file or app_file),
-            ("AUTORESEARCH_ACCOUNT", account or ("-" if local_mode() else "")),
-            ("AUTORESEARCH_PARTITION", partition or ("-" if local_mode() else "")),
-            ("AUTORESEARCH_HOME", home),
+            ("OUTERLOOP_PAT_FILE or _GITHUB_APP_FILE", pat_file or app_file),
+            ("OUTERLOOP_HOME", home),
+            ("OUTERLOOP_TARGET", target),
         ]
         if not value
     ]
-    if not Path(image).is_file():
+    if not image_ok:
         absent.append(f"image:{image}")
     log.info("in-review servicing disabled (missing: %s)", ", ".join(absent))
     return None, None
@@ -3042,7 +3322,7 @@ def _followup_spec_from_env(root: Path) -> tuple[Any, FollowupSpec | None]:
 def _loop_cadence_s(cadence_min: float) -> float:
     """The --loop sleep, clamped to [60s, 24h]: argparse accepts inf (which
     would OverflowError out of time.sleep) and sub-minute values would spin.
-    A non-positive argument defers to AUTORESEARCH_CADENCE_MIN."""
+    A non-positive argument defers to OUTERLOOP_CADENCE_MIN."""
     return min(24 * 3600.0, max(60.0, cadence_min * 60 if cadence_min > 0 else _cadence_s()))
 
 
@@ -3050,15 +3330,30 @@ def main() -> int:
     import argparse
     import time
 
-    parser = argparse.ArgumentParser(description="One tick of the autoresearch loop.")
+    parser = argparse.ArgumentParser(
+        prog="outerloop tick",
+        description="One tick of the loop: service the open runs, launch new work, wake "
+        "parked runs. The chain runs this every cadence; --loop does the same in the "
+        "foreground.",
+    )
     parser.add_argument("--root", required=True, type=Path, help="state root on the shared FS")
-    parser.add_argument("--grace-s", type=float, default=DEFAULT_GRACE_S)
-    parser.add_argument("--lease-ttl-s", type=float, default=DEFAULT_LEASE_TTL_S)
+    parser.add_argument(
+        "--grace-s",
+        type=float,
+        default=DEFAULT_GRACE_S,
+        help="seconds a finished experiment's delivery job gets before the sweep steps in",
+    )
+    parser.add_argument(
+        "--lease-ttl-s",
+        type=float,
+        default=DEFAULT_LEASE_TTL_S,
+        help="seconds after which a held run lease counts as stale",
+    )
     parser.add_argument(
         "--min-free-gb",
         type=float,
         default=DEFAULT_MIN_FREE_BYTES / 1024**3,
-        help="skip launching new work when the state filesystem has less free",
+        help="skip launching new work when the state filesystem has less than this many GB free",
     )
     parser.add_argument(
         "--loop",
@@ -3071,23 +3366,23 @@ def main() -> int:
         type=float,
         default=0.0,
         help="minutes between --loop ticks; unset defers to "
-        "AUTORESEARCH_CADENCE_MIN via the chain's own parser (default 30)",
+        "OUTERLOOP_CADENCE_MIN via the chain's own parser (default 30)",
     )
     args = parser.parse_args()
     logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
 
     args.root.mkdir(parents=True, exist_ok=True)
     # The tick's --root is the authority; children (and this process's own
-    # LocalCompute) read AUTORESEARCH_ROOT, so a bare `tick --loop --root X`
+    # LocalCompute) read OUTERLOOP_ROOT, so a bare `tick --loop --root X`
     # must not split-brain them: local job states would land nowhere and
     # every finished job would read GONE until the park deadline.
     # RESOLVED: local jobs cd into flight checkouts, so a relative root
     # would scatter their state dirs across working directories
-    if os.environ.get("AUTORESEARCH_ROOT", "") != str(args.root.resolve()):
-        os.environ["AUTORESEARCH_ROOT"] = str(args.root.resolve())
+    if os.environ.get("OUTERLOOP_ROOT", "") != str(args.root.resolve()):
+        os.environ["OUTERLOOP_ROOT"] = str(args.root.resolve())
     # In-review servicing is LIVE when credentials + image are available in the
     # chain environment. The waiting-run sweep delivers real wakes only when the
-    # operator arms it — the AUTORESEARCH_DISPATCH_WAKE env var or a
+    # operator arms it — the OUTERLOOP_DISPATCH_WAKE env var or a
     # <root>/DISPATCH_WAKE sentinel — and the env is complete; by default it
     # stays dry with the LoggingDispatcher — dispatched climbing lands DARK.
     # ONE compute for the process: LocalCompute remembers its jobs' states