outerloop-science 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl

outerloop/compute.py

@@ -18,6 +18,7 @@ from __future__ import annotations
 import contextlib
 import logging
 import os
+import re
 import shlex
 import signal
 import subprocess
@@ -67,7 +68,9 @@ Runner = Callable[[Sequence[str], int], CommandResult]
 
 
 def _subprocess_runner(argv: Sequence[str], timeout_s: int) -> CommandResult:
-    completed = subprocess.run(list(argv), capture_output=True, text=True, timeout=timeout_s)
+    completed = subprocess.run(
+        list(argv), capture_output=True, text=True, timeout=timeout_s, check=False
+    )
     return CommandResult(completed.returncode, completed.stdout, completed.stderr)
 
 
@@ -91,14 +94,16 @@ class JobSpec:
     mem: str = "2G"
     gpus: int = 0
     qos: str = ""
+    # a positive nice LOWERS priority (Slurm, like Unix): experiments yield to
+    # the kernel's own evals and re-measures when a slot frees
+    nice: int = 0
     output: str = "/dev/null"
     # Slurm scheduling controls
     dependency: str = ""  # e.g. "afterany:12345" or "singleton"
     begin: str = ""  # e.g. "now+30" or an absolute "YYYY-MM-DDTHH:MM:SS"
-    # submitted held (PENDING, reason JobHeldUser) until `release`: the
-    # tick's launch admission lets GPU launches into the queue in order,
-    # under the per-user cap, instead of queueing them all at once
-    hold: bool = False
+    # a job array: "0-15%4" runs tasks 0..15, at most 4 at a time; the queue
+    # holds one entry and squeue names its tasks `<id>_<k>`
+    array: str = ""
     extra: tuple[str, ...] = ()
 
     def to_argv(self) -> list[str]:
@@ -125,12 +130,14 @@ class JobSpec:
             argv.append(f"--gpus-per-node={self.gpus}")
         if self.qos:
             argv.append(f"--qos={self.qos}")
-        if self.hold:
-            argv.append("--hold")
+        if self.nice:
+            argv.append(f"--nice={self.nice}")
         if self.dependency:
             argv.append(f"--dependency={self.dependency}")
         if self.begin:
             argv.append(f"--begin={self.begin}")
+        if self.array:
+            argv.append(f"--array={self.array}")
         argv.extend(self.extra)
         if self.command:
             argv.append(f"--wrap={self.command}")
@@ -140,9 +147,19 @@ class JobSpec:
         return argv
 
 
-# `reason` and `gres` feed launch admission (why a job waits, how many GPUs it
-# asks for); the board reads the first six by key and ignores the rest
-QUEUE_FIELDS = ("id", "name", "state", "elapsed", "partition", "submitted", "reason", "gres")
+# `reason`, `gres` and `limit` feed the queue view: why a job waits, what it holds
+# waits, whether it holds GPUs); the board reads the first six by key
+QUEUE_FIELDS = (
+    "id",
+    "name",
+    "state",
+    "elapsed",
+    "partition",
+    "submitted",
+    "reason",
+    "gres",
+    "limit",
+)
 
 
 class Compute(Protocol):
@@ -155,9 +172,9 @@ class Compute(Protocol):
     def job_partition(self, job_id: str) -> str: ...
     def active_job_names(self) -> list[str]: ...
     def queue_snapshot(self) -> list[dict[str, str]]: ...
+    def lane_load(self, partition: str) -> dict[str, int]: ...
     def job_id_for_name(self, name: str) -> str: ...
-    def cancel(self, job_id: str) -> None: ...
-    def release(self, job_id: str) -> None: ...
+    def cancel(self, job_id: str) -> bool: ...
 
 
 def local_mode() -> bool:
@@ -174,6 +191,47 @@ def compute_from_env() -> SlurmCompute | LocalCompute:
     return LocalCompute() if local_mode() else SlurmCompute()
 
 
+_JOB_ID = re.compile(r"^\d+(_\d+)?$")  # a job, or one task of a job array (`<id>_<k>`)
+
+
+def _check_job_id(job_id: str) -> None:
+    if not _JOB_ID.match(job_id):
+        raise ValueError(f"not a job id: {job_id!r}")
+
+
+def array_indices(spec: str) -> list[int]:
+    """The task indices of an array spec: "0-15%4" -> 0..15 (the %K throttle
+    is the scheduler's concern); "" -> none."""
+    body = spec.split("%", 1)[0].strip()
+    if not body:
+        return []
+    lo, sep, hi = body.partition("-")
+    if not sep:
+        return [int(lo)] if lo.isdigit() else []
+    if not (lo.isdigit() and hi.isdigit()):
+        return []
+    return list(range(int(lo), int(hi) + 1))
+
+
+def combine_states(states: Sequence[str]) -> str:
+    """One state for a job array from its tasks' states: running while any
+    task runs, pending while any task waits, terminal only when every task
+    is — COMPLETED if all are, else the first other terminal state (FAILED,
+    TIMEOUT, CANCELLED...), so a sweep with one dead task reads as failed."""
+    if len(states) == 1:
+        return states[0]
+    for want in ("RUNNING", "COMPLETING"):
+        if any(s.startswith(want) for s in states):
+            return want
+    if any(is_pending(s) for s in states):
+        return "PENDING"
+    live = [s for s in states if not is_terminal(s)]
+    if live:
+        return live[0]
+    bad = [s for s in states if not s.startswith("COMPLETED")]
+    return bad[0] if bad else "COMPLETED"
+
+
 @dataclass
 class SlurmCompute:
     """The three verbs, plus afterany for wake jobs."""
@@ -195,8 +253,7 @@ class SlurmCompute:
     def status(self, job_id: str) -> str:
         """The job's Slurm state, or GONE when a *successful* query finds no
         record. Raises SlurmQueryError when the query itself fails."""
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
+        _check_job_id(job_id)
         try:
             result = self.runner(
                 ["sacct", "-j", job_id, "--parsable2", "--noheader", "-X", "-o", "State"],
@@ -206,14 +263,14 @@ class SlurmCompute:
             raise SlurmQueryError(f"sacct did not run: {exc}") from exc
         if result.returncode != 0:
             raise SlurmQueryError(f"sacct failed ({result.returncode}): {result.stderr.strip()}")
-        state = result.stdout.strip().splitlines()[0].strip() if result.stdout.strip() else ""
-        return state if state else GONE
+        # a job array answers one line per task; the array's state is theirs combined
+        states = [ln.strip() for ln in result.stdout.splitlines() if ln.strip()]
+        return combine_states(states) if states else GONE
 
     def elapsed_seconds(self, job_id: str) -> int | None:
         """How long the job actually ran (sacct Elapsed), or None when sacct
         has no record. Raises SlurmQueryError when the query itself fails."""
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
+        _check_job_id(job_id)
         try:
             result = self.runner(
                 ["sacct", "-j", job_id, "--parsable2", "--noheader", "-X", "-o", "Elapsed"],
@@ -230,8 +287,7 @@ class SlurmCompute:
         """Why a PENDING job is pending — Slurm's reason (`Dependency`,
         `DependencyNeverSatisfied`, `Priority`, ...), or "" when squeue no
         longer lists it. Raises SlurmQueryError when the query itself fails."""
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
+        _check_job_id(job_id)
         try:
             result = self.runner(["squeue", "-j", job_id, "-h", "-o", "%r"], self.command_timeout_s)
         except (OSError, subprocess.TimeoutExpired) as exc:
@@ -245,8 +301,7 @@ class SlurmCompute:
         them, or "" when squeue no longer lists it. A site can MOVE a pending
         job off the partition it was submitted to (Torch does, under
         congestion); callers compare this with what they asked for."""
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
+        _check_job_id(job_id)
         try:
             result = self.runner(["squeue", "-j", job_id, "-h", "-o", "%P"], self.command_timeout_s)
         except (OSError, subprocess.TimeoutExpired) as exc:
@@ -277,7 +332,7 @@ class SlurmCompute:
         on failure, like active_job_names."""
         try:
             result = self.runner(
-                ["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V|%r|%b"],
+                ["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V|%r|%b|%l"],
                 self.command_timeout_s,
             )
         except (OSError, subprocess.TimeoutExpired) as exc:
@@ -291,6 +346,30 @@ class SlurmCompute:
                 rows.append(dict(zip(QUEUE_FIELDS, parts, strict=True)))
         return rows
 
+    def lane_load(self, partition: str) -> dict[str, int]:
+        """Node counts by state on a lane (a partition or a comma-separated
+        list), from sinfo: {"idle": 3, "mixed": 20, "allocated": 11}. Context
+        for the queue view, nothing a caller acts on. Empty when no lane is
+        named; raises SlurmQueryError on a failed query."""
+        if not partition:
+            return {}
+        try:
+            result = self.runner(
+                ["sinfo", "--noheader", "-p", partition, "-o", "%T %D"], self.command_timeout_s
+            )
+        except (OSError, subprocess.TimeoutExpired) as exc:
+            raise SlurmQueryError(f"sinfo did not run: {exc}") from exc
+        if result.returncode != 0:
+            raise SlurmQueryError(f"sinfo failed ({result.returncode}): {result.stderr.strip()}")
+        load: dict[str, int] = {}
+        for line in result.stdout.splitlines():
+            parts = line.split()
+            if len(parts) != 2 or not parts[1].isdigit():
+                continue
+            state = parts[0].rstrip("*~#!%$@^-")  # sinfo's state flags (draining, no-respond...)
+            load[state] = load.get(state, 0) + int(parts[1])
+        return load
+
     def job_id_for_name(self, name: str) -> str:
         """The id of this user's PENDING/RUNNING job with exactly `name`, or
         "" if none. Authoritative for "is this still live" independent of any
@@ -309,22 +388,16 @@ class SlurmCompute:
         ids = [line.strip() for line in result.stdout.splitlines() if line.strip()]
         return ids[0] if ids else ""
 
-    def cancel(self, job_id: str) -> None:
-        """Cancel; idempotent (cancelling a finished job is not an error)."""
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
+    def cancel(self, job_id: str) -> bool:
+        """Cancel; idempotent (cancelling a finished job is not an error).
+        False when scancel itself failed, so a caller that must know (the
+        sweep's cancel-on-end) can try again; most callers are best-effort."""
+        _check_job_id(job_id)
         result = self.runner(["scancel", job_id], self.command_timeout_s)
         if result.returncode != 0:
             log.warning("scancel %s: %s", job_id, result.stderr.strip())
-
-    def release(self, job_id: str) -> None:
-        """Release a job submitted with `hold` so the scheduler may start it.
-        Releasing a job that is not held is not an error."""
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
-        result = self.runner(["scontrol", "release", job_id], self.command_timeout_s)
-        if result.returncode != 0:
-            log.warning("scontrol release %s: %s", job_id, result.stderr.strip())
+            return False
+        return True
 
 
 # Local job ids start far above any real Slurm id so the two can never be
@@ -389,6 +462,33 @@ class LocalCompute:
             if k in ("PATH", "HOME", "LANG", "TMPDIR", "SLURM_TMPDIR", "USER", "LOGNAME")
             or (k.startswith(("OUTERLOOP_", "REVIEW_HERMES_")) and not _secret_name(k))
         }
+        indices = array_indices(spec.array)
+        if indices:
+            # a job array runs its tasks in turn — there is no queue here to
+            # throttle; each task keeps its own state and output under
+            # `<id>_<k>`, and the array's own state is theirs combined
+            states = [
+                self._run_and_record(
+                    spec, argv, {**job_env, "SLURM_ARRAY_TASK_ID": str(i)}, f"{job_id}_{i}"
+                )
+                for i in indices
+            ]
+            state = combine_states(states)
+            self._record(spec, job_id, state, "")
+        else:
+            state = self._run_and_record(spec, argv, job_env, job_id)
+        state_dir = _local_state_dir()
+        where = (
+            f"; output in {state_dir / (job_id + '.out')}"
+            if state_dir and state != "COMPLETED"
+            else ""
+        )
+        log.info("ran %s locally as job %s: %s%s", spec.job_name, job_id, state, where)
+        return job_id
+
+    def _run_and_record(
+        self, spec: JobSpec, argv: list[str], job_env: dict[str, str], job_id: str
+    ) -> str:
         try:
             # the job runs in its OWN session (= process group), so the
             # walltime kill takes the whole tree — a job script waiting on
@@ -425,6 +525,10 @@ class LocalCompute:
                     "local job %s: an escaped child survived the walltime kill", spec.job_name
                 )
             state = "TIMEOUT"
+        self._record(spec, job_id, state, output)
+        return state
+
+    def _record(self, spec: JobSpec, job_id: str, state: str, output: str) -> None:
         state_dir = _local_state_dir()
         if state_dir is not None:
             try:
@@ -453,22 +557,14 @@ class LocalCompute:
                 log.warning("local job %s: state persist failed: %s", spec.job_name, exc)
         if spec.output and spec.output != "/dev/null":
             try:
-                with open(spec.output, "w") as fh:
+                with open(spec.output, "a" if "_" in job_id else "w") as fh:
                     fh.write(output)
             except OSError as exc:
                 log.warning("local job %s: output write failed: %s", spec.job_name, exc)
         self._states[job_id] = state
-        where = (
-            f"; output in {state_dir / (job_id + '.out')}"
-            if state_dir and state != "COMPLETED"
-            else ""
-        )
-        log.info("ran %s locally as job %s: %s%s", spec.job_name, job_id, state, where)
-        return job_id
 
     def status(self, job_id: str) -> str:
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
+        _check_job_id(job_id)
         state = self._states.get(job_id, "")
         if state:
             return state
@@ -494,16 +590,15 @@ class LocalCompute:
     def queue_snapshot(self) -> list[dict[str, str]]:
         return []
 
+    def lane_load(self, partition: str) -> dict[str, int]:
+        return {}  # no lanes in the monolith
+
     def job_id_for_name(self, name: str) -> str:
         return ""
 
-    def cancel(self, job_id: str) -> None:
-        if not job_id.isdigit():
-            raise ValueError(f"not a job id: {job_id!r}")
-        # already terminal; cancelling a finished job is not an error
-
-    def release(self, job_id: str) -> None:
-        """Local jobs run synchronously at submit; nothing is ever held."""
+    def cancel(self, job_id: str) -> bool:
+        _check_job_id(job_id)
+        return True  # already terminal; cancelling a finished job is not an error
 
 
 def parse_elapsed(text: str) -> int | None:

outerloop/contract.py

@@ -272,6 +272,14 @@ class Budgets(_StrictModel):
     # (agent-01..agent-0N), so branches, ledger rows, and reports stay
     # distinct. runs_per_week and gpu_hours_per_run remain the spend guards.
     max_active_attempts: int | None = Field(default=None, ge=1)
+    # The pace ceiling for an author's sweeps, in GPUs: one launch may hold at
+    # most this many at once, so a sweep of N tasks runs
+    # max_concurrent_gpus // gpus of them at a time (`--array=0-N%K`). Lenient
+    # by design — not the cap divided by the agent count (agents rarely launch
+    # at the same moment, and an idle share is wasted GPU); on a 16-GPU cap 12
+    # lets one sweep use most of the machine while a sibling's job still gets
+    # in. Unset: the author's own pace, the whole array by default.
+    max_concurrent_gpus: int | None = Field(default=None, ge=1)
 
     @model_validator(mode="before")
     @classmethod

outerloop/dispatch.py

@@ -36,8 +36,14 @@ from pathlib import Path
 from uuid import uuid4
 
 from outerloop.compute import JobSpec
-from outerloop.github import SAFE_GIT_FLAGS, GitError, Workspace, ensure_regular_git_dir
-from outerloop.orchestrator import EvalError, _metric_from_output, managed_eval_env
+from outerloop.github import (
+    SAFE_GIT_FLAGS,
+    GitError,
+    Workspace,
+    ensure_regular_git_dir,
+    git_identity,
+)
+from outerloop.orchestrator import EvalError, managed_eval_env, metric_from_output
 
 log = logging.getLogger(__name__)
 
@@ -94,7 +100,11 @@ class Snapshot:
 
 
 def snapshot_tree(
-    ws: Workspace, base_sha: str, exclude: tuple[str, ...] = (), force: tuple[str, ...] = ()
+    ws: Workspace,
+    base_sha: str,
+    exclude: tuple[str, ...] = (),
+    force: tuple[str, ...] = (),
+    author: str = "",
 ) -> Snapshot:
     """Snapshot the workspace's current CONTENT as a commit parented on
     `base_sha`, without touching the working index, and retain it under a
@@ -105,7 +115,8 @@ def snapshot_tree(
     keeps it (docs/design/research-lines.md). `force` adds those paths even
     when the target's ignore rules match them — the notebook seal uses it so
     a .gitignore entry cannot silently discard session memory; callers pass
-    only paths that exist.
+    only paths that exist. `author` is the bot login the seal commit is
+    made as (empty: OUTERLOOP_BOT_LOGIN).
     """
     # the snapshot writes an index, a tree, a commit, and a ref into this
     # repository: a session-reshaped .git is refused first, like every other
@@ -163,10 +174,7 @@ def snapshot_tree(
         commit = run(
             [
                 *git,
-                "-c",
-                "user.name=dispatch",
-                "-c",
-                "user.email=dispatch@localhost",
+                *git_identity(author),
                 "commit-tree",
                 tree,
                 "-p",
@@ -205,6 +213,7 @@ def _filter_neutral_env(base_git: list[str], env: dict[str, str]) -> dict[str, s
         capture_output=True,
         text=True,
         timeout=30,
+        check=False,
     )
     # -z: NUL-separated records, each "key\nvalue" — so a value containing a
     # newline can never masquerade as a second record.
@@ -281,9 +290,22 @@ def write_eval_job(
     artifacts: tuple[str, ...] = (),
     artifact_max_bytes: int = 0,
     gpus: int = 0,
+    array: int = 1,
+    seed_cache: Path | None = None,
 ) -> Path:
     """Write the orchestrator-authored job script for one dispatched eval.
 
+    `seed_cache` names the kernel-warmed seed for this target
+    (docs/design/eval-cache.md): the job copies its contents into its own
+    scratch cache before `uv` runs — a copy, never a bind or a hardlink, so the
+    job's cache is its own and the seed is never written; a missing seed or a
+    failed copy costs a download, never the eval.
+
+    `array` > 1 writes ONE script for a Slurm job array: each task derives its
+    own job dir `eval-<name>.<k>` from SLURM_ARRAY_TASK_ID (SWEEP_INDEX under a
+    local run) and sees the index as SWEEP_INDEX; the task dirs are created
+    and cleared here, the script lives under `eval-<name>/`.
+
     `gpus` > 0 adds `--nv` to the jail so the job's allocated GPUs (the
     JobSpec requests them) are visible inside the container; nothing else
     about the containment changes.
@@ -310,13 +332,16 @@ def write_eval_job(
     """
     ev = run_dir / f"eval-{name}"
     ev.mkdir(parents=True, exist_ok=True)
-    # a resubmitted eval must never be read as its predecessor: every prior
-    # artifact — including a leftover extracted tree — goes before submission
-    for stale in ("exit-code", "stdout", "stderr", "setup.log", "submitted", "artifacts.log"):
-        (ev / stale).unlink(missing_ok=True)
-    shutil.rmtree(ev / "tree", ignore_errors=True)
-    shutil.rmtree(ev / "artifacts", ignore_errors=True)
-    (ev / "command.txt").write_text(command)
+    task_dirs = [run_dir / f"eval-{name}.{k}" for k in range(array)] if array > 1 else [ev]
+    for task_dir in task_dirs:
+        task_dir.mkdir(parents=True, exist_ok=True)
+        # a resubmitted eval must never be read as its predecessor: every prior
+        # artifact — including a leftover extracted tree — goes before submission
+        for stale in ("exit-code", "stdout", "stderr", "setup.log", "submitted", "artifacts.log"):
+            (task_dir / stale).unlink(missing_ok=True)
+        shutil.rmtree(task_dir / "tree", ignore_errors=True)
+        shutil.rmtree(task_dir / "artifacts", ignore_errors=True)
+        (task_dir / "command.txt").write_text(command)
     # extra_env matches the in-job evaluator's contract: managed keys (HOME,
     # UV_*, PATH...) are DROPPED, never allowed to override the isolation, and
     # keys must be shell-identifier shaped (they are exported unquoted).
@@ -330,10 +355,20 @@ def write_eval_job(
     # as the snapshot, injected as GIT_CONFIG_* env (robust to '=' in a driver
     # name, unlike -c) so a smudge filter cannot execute during checkout
     neutral = _filter_neutral_env(["git", "-C", str(repo_root), *SAFE_GIT_FLAGS], {})
+    if array > 1:
+        # the task picks its own job dir; under Slurm the index is the array
+        # task id, under a local run the caller exports SWEEP_INDEX
+        ev_lines = [
+            'TASK="${SLURM_ARRAY_TASK_ID:-${SWEEP_INDEX:-0}}"',
+            f'EV={shlex.quote(str(ev))}."$TASK"',
+            'export SWEEP_INDEX="$TASK" APPTAINERENV_SWEEP_INDEX="$TASK"',
+        ]
+    else:
+        ev_lines = [f"EV={shlex.quote(str(ev))}"]
     lines = [
         "#!/bin/sh",
         "set -u",
-        f"EV={shlex.quote(str(ev))}",
+        *ev_lines,
         f"REPO={shlex.quote(str(repo_root))}",
         # the extracted tree lives on NODE-LOCAL scratch, not the shared run
         # dir: it dies with the job (nothing to reap on the shared FS), and
@@ -361,6 +396,11 @@ def write_eval_job(
     ]
     for k, v in neutral.items():
         lines.append(f"export {k}={shlex.quote(v)}")
+    if seed_cache is not None:
+        q = shlex.quote(str(seed_cache))
+        lines.append(
+            f'if [ -d {q} ]; then cp -a {q}/. "$SCRATCH/cache"/ 2>> "$EV/setup.log" || true; fi'
+        )
     lines += [
         # Materialize the snapshot by CHECKOUT, not `git archive`: a checkout
         # reproduces content faithfully — INCLUDING .gitattributes — and does
@@ -478,6 +518,8 @@ def eval_job_spec(
     cpus: int = 4,
     mem: str = "8G",
     gpus: int = 0,
+    nice: int = 0,
+    array: str = "",
 ) -> JobSpec:
     """The JobSpec for one dispatched eval: the hint CLAMPED to our ceiling
     plus setup slack — a contract value above EVAL_JOB_MINUTES_CEILING must
@@ -486,7 +528,8 @@ def eval_job_spec(
     the GPU lane (DispatchSettings.placement) when it is nonzero, and the
     job is sized for it: a GPU eval gets at least EVAL_CPUS_PER_GPU cores
     and EVAL_MEM_GB_PER_GPU GB per GPU (a training eval's data loading and
-    torch.compile workers do not fit the CPU eval's 4 cores / 8 GB)."""
+    torch.compile workers do not fit the CPU eval's 4 cores / 8 GB). `nice`
+    lowers the job's priority below the kernel's evals (the launcher sets it)."""
     if gpus > 0:
         cpus = max(cpus, EVAL_CPUS_PER_GPU * gpus)
         given = _mem_gb(mem)
@@ -503,6 +546,8 @@ def eval_job_spec(
         cpus=cpus,
         mem=mem,
         gpus=gpus,
+        nice=nice,
+        array=array,
     )
 
 
@@ -541,7 +586,7 @@ def read_eval_result(run_dir: Path, name: str, metric: str) -> float:
         with contextlib.suppress(OSError, ValueError):
             tail = (ev / "stderr").read_text(errors="replace")[-300:]
         raise EvalError(f"dispatched eval {name} failed ({code}): {tail}")
-    value = _metric_from_output(stdout, metric)
+    value = metric_from_output(stdout, metric)
     if value is None:
         raise EvalError(f"dispatched eval {name}: no readable {metric!r} in output")
     if not math.isfinite(value):