outerloop-science 0.1.0.dev1__py3-none-any.whl → 0.1.0.dev2__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
outerloop/cli.py CHANGED
@@ -2,7 +2,7 @@
2
2
 
3
3
  With `sbatch` on PATH it submits the resident tick
4
4
  (docs/design/resident-tick.md) and returns; without it, or with
5
- AUTORESEARCH_COMPUTE=local, it runs the local loop in the foreground.
5
+ OUTERLOOP_COMPUTE=local, it runs the local loop in the foreground.
6
6
  Settings come from flags, then the process environment, then
7
7
  ~/.config/outerloop/.env, read once here at launch. The running chain
8
8
  never takes identity or placement from that file (tick_deploy.sh reads an
@@ -18,49 +18,59 @@ import shutil
18
18
  import stat
19
19
  import subprocess
20
20
  import sys
21
+ from collections.abc import Mapping
21
22
  from dataclasses import dataclass
22
23
  from pathlib import Path
23
24
 
24
25
  from outerloop import paths
25
26
 
26
- RESIDENT_JOB_NAME = "autoresearch-resident"
27
+ RESIDENT_JOB_NAME = "outerloop-resident"
28
+ # A resident submitted before the rename. Slurm's singleton serializes jobs by
29
+ # NAME, so `start` must refuse while one of these is queued or running (two
30
+ # residents on one root is what the singleton prevents). Dropped in the
31
+ # release after 0.1.
32
+ LEGACY_RESIDENT_JOB_NAME = "autoresearch-resident"
27
33
  DEFAULT_RESIDENT_MINUTES = 360 # cpu_short's ceiling on Torch; the loop hands over to itself
28
- DEFAULT_LOCAL_ROOT = Path.home() / ".autoresearch"
34
+ DEFAULT_LOCAL_ROOT = Path.home() / ".outerloop"
35
+ LEGACY_LOCAL_ROOT_NAME = ".autoresearch" # pre-rename; honored until the release after 0.1
29
36
  ENV_FILE = paths.ENV_FILE # ~/.config/outerloop/.env, or the pre-rename dir (see paths.py)
30
37
 
31
38
  # What start itself decides from: mode, placement, root, cadence, walltime.
32
39
  START_KEYS = (
33
- "AUTORESEARCH_COMPUTE",
34
- "AUTORESEARCH_ROOT",
35
- "AUTORESEARCH_ACCOUNT",
36
- "AUTORESEARCH_PARTITION",
37
- "AUTORESEARCH_CADENCE_MIN",
38
- "AUTORESEARCH_RESIDENT_MINUTES",
39
- "AUTORESEARCH_PAT_FILE",
40
+ "OUTERLOOP_COMPUTE",
41
+ "OUTERLOOP_ROOT",
42
+ "OUTERLOOP_ACCOUNT",
43
+ "OUTERLOOP_PARTITION",
44
+ "OUTERLOOP_CADENCE_MIN",
45
+ "OUTERLOOP_RESIDENT_MINUTES",
46
+ "OUTERLOOP_PAT_FILE",
40
47
  )
41
48
  # The author knobs the chain's deploy step exports from .env every tick. The
42
49
  # local loop has no deploy step, so start exports them once at launch; a test
43
50
  # keeps this list identical to tick_deploy.sh's.
44
51
  TICK_ENV_KEYS = (
45
- "AUTORESEARCH_AUTHOR_BACKEND",
46
- "AUTORESEARCH_AUTHOR_MODEL",
47
- "AUTORESEARCH_CODEX_BIN",
48
- "AUTORESEARCH_CODEX_KEY_FILE",
49
- "AUTORESEARCH_CLAUDE_KEY_FILE",
50
- "AUTORESEARCH_HARNESS_KEY_FILE",
51
- "AUTORESEARCH_VERTEX_PROJECT",
52
- "AUTORESEARCH_VERTEX_REGION",
53
- "AUTORESEARCH_VERTEX_ADC",
54
- "AUTORESEARCH_TARGET",
55
- "AUTORESEARCH_GITHUB_APP_FILE",
56
- "AUTORESEARCH_BOT_LOGIN",
57
- "AUTORESEARCH_BOT_ALIASES",
58
- "AUTORESEARCH_GPU_PARTITION",
59
- "AUTORESEARCH_GPU_ACCOUNT",
60
- "AUTORESEARCH_PANEL",
61
- "AUTORESEARCH_PANEL_KEY_FILE",
62
- "AUTORESEARCH_PANEL_CODEX_KEY_FILE",
63
- "AUTORESEARCH_PANEL_HERMES_KEY_FILE",
52
+ "OUTERLOOP_AUTHOR_BACKEND",
53
+ "OUTERLOOP_AUTHOR_MODEL",
54
+ "OUTERLOOP_CLAUDE_BIN",
55
+ "OUTERLOOP_CODEX_BIN",
56
+ "OUTERLOOP_CODEX_KEY_FILE",
57
+ "OUTERLOOP_CLAUDE_KEY_FILE",
58
+ "OUTERLOOP_HARNESS_KEY_FILE",
59
+ "OUTERLOOP_VERTEX_PROJECT",
60
+ "OUTERLOOP_VERTEX_REGION",
61
+ "OUTERLOOP_VERTEX_ADC",
62
+ "OUTERLOOP_TARGET",
63
+ "OUTERLOOP_GITHUB_APP_FILE",
64
+ "OUTERLOOP_BOT_LOGIN",
65
+ "OUTERLOOP_BOT_ALIASES",
66
+ "OUTERLOOP_GPU_PARTITION",
67
+ "OUTERLOOP_GPU_ACCOUNT",
68
+ "OUTERLOOP_MAX_LAUNCH_GPUS",
69
+ "OUTERLOOP_IMAGE",
70
+ "OUTERLOOP_PANEL",
71
+ "OUTERLOOP_PANEL_KEY_FILE",
72
+ "OUTERLOOP_PANEL_CODEX_KEY_FILE",
73
+ "OUTERLOOP_PANEL_HERMES_KEY_FILE",
64
74
  "REVIEW_HERMES_REPO",
65
75
  "REVIEW_HERMES_PROVIDER",
66
76
  )
@@ -89,20 +99,26 @@ def env_file_values(path: Path = ENV_FILE, keys: tuple[str, ...] = START_KEYS) -
89
99
  except OSError as e:
90
100
  raise StartError(f"cannot read {path}: {e}") from None
91
101
  out: dict[str, str] = {}
102
+ canonical: set[str] = set() # keys set by their OUTERLOOP_ spelling
92
103
  for raw in text.splitlines():
93
104
  line = raw.rstrip("\r").strip()
94
105
  if not line or line.startswith("#") or "=" not in line:
95
106
  continue
96
107
  key, value = line.split("=", 1)
97
108
  key = key.strip()
98
- if key.startswith("OUTERLOOP_"): # the public name; `keys` are canonical
99
- key = "AUTORESEARCH_" + key[len("OUTERLOOP_") :]
109
+ legacy = key.startswith("AUTORESEARCH_") # the pre-rename name, one more release
110
+ if legacy:
111
+ key = "OUTERLOOP_" + key[len("AUTORESEARCH_") :]
100
112
  if key not in keys:
101
113
  continue
114
+ if legacy and key in canonical:
115
+ continue # the OUTERLOOP_ spelling wins whatever the file order
102
116
  value = value.strip()
103
117
  if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
104
118
  value = value[1:-1]
105
119
  out[key] = value
120
+ if not legacy:
121
+ canonical.add(key)
106
122
  return out
107
123
 
108
124
 
@@ -124,18 +140,19 @@ class StartPlan:
124
140
  reads as "whichever frees up first") without corrupting the export
125
141
  delimiter. start() merges these into the environment it hands sbatch."""
126
142
  env = {
127
- "AUTORESEARCH_RESIDENT": "1",
128
- "AUTORESEARCH_HOME": str(self.home),
129
- "AUTORESEARCH_ROOT": str(self.root),
130
- "AUTORESEARCH_ACCOUNT": self.account,
131
- "AUTORESEARCH_RESIDENT_MINUTES": str(self.resident_minutes), # successors reuse it
143
+ "OUTERLOOP_RESIDENT": "1",
144
+ "OUTERLOOP_HOME": str(self.home),
145
+ "OUTERLOOP_ROOT": str(self.root),
146
+ "OUTERLOOP_RESIDENT_MINUTES": str(self.resident_minutes), # successors reuse it
132
147
  }
148
+ if self.account:
149
+ env["OUTERLOOP_ACCOUNT"] = self.account
133
150
  if self.partition:
134
- env["AUTORESEARCH_PARTITION"] = self.partition
151
+ env["OUTERLOOP_PARTITION"] = self.partition
135
152
  if self.cadence_min:
136
- env["AUTORESEARCH_CADENCE_MIN"] = self.cadence_min
153
+ env["OUTERLOOP_CADENCE_MIN"] = self.cadence_min
137
154
  if self.pat_file:
138
- env["AUTORESEARCH_PAT_FILE"] = self.pat_file
155
+ env["OUTERLOOP_PAT_FILE"] = self.pat_file
139
156
  return env
140
157
 
141
158
  def command(self) -> list[str]:
@@ -147,8 +164,9 @@ class StartPlan:
147
164
  "--dependency=singleton", # two starts can both submit; only one ever runs
148
165
  f"--time={self.resident_minutes}",
149
166
  f"--job-name={RESIDENT_JOB_NAME}",
150
- f"--account={self.account}",
151
167
  ]
168
+ if self.account: # unset bills the caller's default Slurm association
169
+ argv.append(f"--account={self.account}")
152
170
  if self.partition: # unset lets Slurm choose its default partition
153
171
  argv.append(f"--partition={self.partition}")
154
172
  # --export=ALL carries export_env() from the inherited environment; a
@@ -168,7 +186,7 @@ def _setting(key: str, flag: str, environ: dict[str, str], from_file: dict[str,
168
186
 
169
187
  def _home(environ: dict[str, str], cwd: Path, *, local: bool, root: Path) -> Path:
170
188
  """The directory the loop runs from. On Slurm it must be a source
171
- checkout (AUTORESEARCH_HOME, else the current directory): the chain
189
+ checkout (OUTERLOOP_HOME, else the current directory): the chain
172
190
  deploys from it and every job runs from a flight snapshot of its HEAD.
173
191
  The local loop has no deploy step and runs the installed package, so it
174
192
  uses a checkout when one is at hand and otherwise a `home` directory
@@ -196,10 +214,10 @@ def plan_start(
196
214
  sbatch_on_path: bool,
197
215
  cwd: Path,
198
216
  ) -> StartPlan:
199
- compute = _setting("AUTORESEARCH_COMPUTE", "local" if local else "", environ, from_file)
217
+ compute = _setting("OUTERLOOP_COMPUTE", "local" if local else "", environ, from_file)
200
218
  mode = "local" if compute.strip().lower() == "local" or not sbatch_on_path else "slurm"
201
- root_s = _setting("AUTORESEARCH_ROOT", root, environ, from_file)
202
- cadence = _setting("AUTORESEARCH_CADENCE_MIN", "", environ, from_file)
219
+ root_s = _setting("OUTERLOOP_ROOT", root, environ, from_file)
220
+ cadence = _setting("OUTERLOOP_CADENCE_MIN", "", environ, from_file)
203
221
  if cadence:
204
222
  # the chain divides by it and the loop sleeps on it: a bad value would
205
223
  # only surface after the job started
@@ -209,13 +227,13 @@ def plan_start(
209
227
  cadence_ok = False
210
228
  if not cadence_ok:
211
229
  raise StartError(
212
- f"AUTORESEARCH_CADENCE_MIN must be a positive number of minutes, got {cadence!r}"
230
+ f"OUTERLOOP_CADENCE_MIN must be a positive number of minutes, got {cadence!r}"
213
231
  )
214
- pat = _setting("AUTORESEARCH_PAT_FILE", "", environ, from_file)
232
+ pat = _setting("OUTERLOOP_PAT_FILE", "", environ, from_file)
215
233
  # Slurm runs from a checkout; the local loop runs the installed package
216
234
  # and needs only a directory (the launch lanes and GitHub servicing
217
- # switch off without AUTORESEARCH_HOME, so one is always set)
218
- local_root = Path(root_s).expanduser() if root_s else DEFAULT_LOCAL_ROOT
235
+ # switch off without OUTERLOOP_HOME, so one is always set)
236
+ local_root = Path(root_s).expanduser() if root_s else default_local_root(environ)
219
237
  home = _home(environ, cwd, local=(mode == "local"), root=local_root)
220
238
  if mode == "local":
221
239
  return StartPlan(
@@ -228,23 +246,22 @@ def plan_start(
228
246
  if not root_s:
229
247
  raise StartError(
230
248
  "Slurm mode needs the state root on the shared filesystem: "
231
- "--root, AUTORESEARCH_ROOT, or AUTORESEARCH_ROOT= in ~/.config/outerloop/.env"
249
+ "--root, OUTERLOOP_ROOT in the environment, or OUTERLOOP_ROOT= in "
250
+ "~/.config/outerloop/.env"
232
251
  )
233
- acc = _setting("AUTORESEARCH_ACCOUNT", account, environ, from_file)
234
- part = _setting("AUTORESEARCH_PARTITION", partition, environ, from_file)
235
- # Partition is optional: left unset, Slurm places the job on its default
236
- # partition. Account stays required (clusters bill by it).
237
- if not acc:
238
- raise StartError("Slurm mode needs --account / AUTORESEARCH_ACCOUNT")
239
- minutes_s = _setting("AUTORESEARCH_RESIDENT_MINUTES", "", environ, from_file)
252
+ acc = _setting("OUTERLOOP_ACCOUNT", account, environ, from_file)
253
+ part = _setting("OUTERLOOP_PARTITION", partition, environ, from_file)
254
+ # Account and partition are both optional: left unset, Slurm bills the
255
+ # caller's default association and places the job on its default partition.
256
+ minutes_s = _setting("OUTERLOOP_RESIDENT_MINUTES", "", environ, from_file)
240
257
  try:
241
258
  minutes = int(minutes_s) if minutes_s else DEFAULT_RESIDENT_MINUTES
242
259
  except ValueError:
243
260
  raise StartError(
244
- f"AUTORESEARCH_RESIDENT_MINUTES must be a whole number of minutes, got {minutes_s!r}"
261
+ f"OUTERLOOP_RESIDENT_MINUTES must be a whole number of minutes, got {minutes_s!r}"
245
262
  ) from None
246
263
  if minutes <= 0:
247
- raise StartError("AUTORESEARCH_RESIDENT_MINUTES must be positive")
264
+ raise StartError("OUTERLOOP_RESIDENT_MINUTES must be positive")
248
265
  # These ride the inherited environment (sbatch --export=ALL), so a comma is
249
266
  # safe now (a multi-partition `a,b` is valid) — only a newline would corrupt
250
267
  # the environment or the sbatch argv.
@@ -270,16 +287,30 @@ def plan_start(
270
287
  )
271
288
 
272
289
 
290
+ def default_local_root(environ: Mapping[str, str]) -> Path:
291
+ """The local-mode state root when none is given: ~/.outerloop, or the
292
+ pre-rename ~/.autoresearch when only that one exists (state is never moved
293
+ behind the operator's back). The home comes from `environ` so a caller
294
+ with no HOME gets the plain default."""
295
+ home_s = environ.get("HOME", "")
296
+ if not home_s:
297
+ return DEFAULT_LOCAL_ROOT
298
+ home = Path(home_s)
299
+ new, old = home / DEFAULT_LOCAL_ROOT.name, home / LEGACY_LOCAL_ROOT_NAME
300
+ return old if (not new.exists() and old.is_dir()) else new
301
+
302
+
273
303
  def _resident_jobs() -> list[str] | None:
274
- """Ids of queued or running resident ticks, lowest first; None when the
275
- scheduler could not be asked (a failed lookup must never read as 'none')."""
304
+ """Ids of queued or running resident ticks under either name, lowest first;
305
+ None when the scheduler could not be asked (a failed lookup must never
306
+ read as 'none')."""
276
307
  try:
277
308
  proc = subprocess.run(
278
309
  [
279
310
  "squeue",
280
311
  "-u",
281
312
  os.environ.get("USER", ""),
282
- f"--name={RESIDENT_JOB_NAME}",
313
+ f"--name={RESIDENT_JOB_NAME},{LEGACY_RESIDENT_JOB_NAME}",
283
314
  "-h",
284
315
  "-o",
285
316
  "%i",
@@ -309,9 +340,37 @@ def _exec(cmd: list[str], env: dict[str, str]) -> int:
309
340
  return 1 # unreachable; keeps the signature honest for tests that stub this
310
341
 
311
342
 
343
+ HARNESS_BIN_KEYS = ("OUTERLOOP_CLAUDE_BIN", "OUTERLOOP_CODEX_BIN")
344
+
345
+
346
+ def _setting_of(key: str, values: Mapping[str, str], environ: Mapping[str, str]) -> str:
347
+ """The process environment wins over .env, including an explicit empty value
348
+ (the way to clear a recorded path from the shell)."""
349
+ return (environ[key] if key in environ else values.get(key, "")).strip()
350
+
351
+
352
+ def missing_harness_binary(values: Mapping[str, str], environ: Mapping[str, str]) -> str:
353
+ """Why start must not launch: the CONFIGURED author backend's recorded
354
+ binary is not there (init records the path; a moved or uninstalled CLI
355
+ would end every climb with spawn-error). The other backend's path is not
356
+ consulted: a stale entry for a backend the loop never spawns must not block
357
+ it. "" when the binary exists or none is recorded."""
358
+ backend = _setting_of("OUTERLOOP_AUTHOR_BACKEND", values, environ).lower() or "claude"
359
+ key = f"OUTERLOOP_{backend.upper()}_BIN"
360
+ if key not in HARNESS_BIN_KEYS:
361
+ return "" # an unknown backend is init's error to report, not this check's
362
+ path = _setting_of(key, values, environ)
363
+ if path and not Path(path).expanduser().is_file():
364
+ return f"{key}={path} is not a file; reinstall the CLI or run `outerloop init --force`"
365
+ return ""
366
+
367
+
312
368
  def start(args: argparse.Namespace) -> int:
313
369
  try:
314
370
  values = env_file_values(ENV_FILE, START_KEYS + TICK_ENV_KEYS) # one read for everything
371
+ problem = missing_harness_binary(values, os.environ)
372
+ if problem:
373
+ raise StartError(problem)
315
374
  from_file = {k: v for k, v in values.items() if k in START_KEYS}
316
375
  plan = plan_start(
317
376
  root=args.root or "",
@@ -337,14 +396,14 @@ def start(args: argparse.Namespace) -> int:
337
396
  for key, value in values.items():
338
397
  if key in TICK_ENV_KEYS:
339
398
  env.setdefault(key, value)
340
- env["AUTORESEARCH_COMPUTE"] = "local"
341
- env["AUTORESEARCH_ROOT"] = str(plan.root)
342
- env["AUTORESEARCH_HOME"] = str(plan.home)
399
+ env["OUTERLOOP_COMPUTE"] = "local"
400
+ env["OUTERLOOP_ROOT"] = str(plan.root)
401
+ env["OUTERLOOP_HOME"] = str(plan.home)
343
402
  plan.home.mkdir(parents=True, exist_ok=True) # <root>/home when there is no checkout
344
403
  if plan.cadence_min:
345
- env["AUTORESEARCH_CADENCE_MIN"] = plan.cadence_min
404
+ env["OUTERLOOP_CADENCE_MIN"] = plan.cadence_min
346
405
  if plan.pat_file:
347
- env["AUTORESEARCH_PAT_FILE"] = plan.pat_file
406
+ env["OUTERLOOP_PAT_FILE"] = plan.pat_file
348
407
  print(
349
408
  f"local loop: state in {plan.root}; Ctrl-C stops it, the records resume it",
350
409
  file=sys.stderr,
@@ -355,14 +414,14 @@ def start(args: argparse.Namespace) -> int:
355
414
  print(
356
415
  "outerloop start: could not ask the scheduler whether a resident tick "
357
416
  "exists (squeue failed); nothing submitted. Retry, or check "
358
- f"`squeue --name {RESIDENT_JOB_NAME}`.",
417
+ f"`squeue --name {RESIDENT_JOB_NAME},{LEGACY_RESIDENT_JOB_NAME}`.",
359
418
  file=sys.stderr,
360
419
  )
361
420
  return 1
362
421
  if existing:
363
422
  print(
364
423
  f"a resident tick is already queued or running (job {existing[0]}); nothing "
365
- f"submitted. Stop it with `scancel --name {RESIDENT_JOB_NAME}`, or pause it "
424
+ f"submitted. Stop it with `scancel {existing[0]}`, or pause it "
366
425
  f"with `touch {plan.root}/PAUSE`.",
367
426
  file=sys.stderr,
368
427
  )
@@ -406,23 +465,40 @@ def start(args: argparse.Namespace) -> int:
406
465
 
407
466
 
408
467
  def main(argv: list[str] | None = None) -> int:
468
+ from outerloop import __version__
469
+
409
470
  parser = argparse.ArgumentParser(
410
- prog="outerloop", description="autonomous research agents in an outer loop"
471
+ prog="outerloop",
472
+ description="autonomous research agents in an outer loop",
473
+ epilog="Run `outerloop init` once, then `outerloop start`. Settings live in "
474
+ "~/.config/outerloop/.env; the install guide is docs/install.md.",
411
475
  )
476
+ parser.add_argument("--version", action="version", version=f"outerloop {__version__}")
412
477
  sub = parser.add_subparsers(dest="command", required=True)
413
478
  p = sub.add_parser(
414
- "start", help="start the loop: the resident tick on Slurm, the local loop elsewhere"
479
+ "start",
480
+ help="start the loop: the resident tick on Slurm, the local loop elsewhere",
481
+ description="Submit the resident tick to Slurm, or run the local loop in the "
482
+ "foreground where there is no sbatch. Every flag defaults from "
483
+ "~/.config/outerloop/.env, written by `outerloop init`.",
484
+ )
485
+ p.add_argument(
486
+ "--root", help="state root (shared filesystem on Slurm; default ~/.outerloop locally)"
415
487
  )
416
488
  p.add_argument(
417
- "--root", help="state root (shared filesystem on Slurm; default ~/.autoresearch locally)"
489
+ "--account", help="Slurm account (optional; unset bills your default association)"
490
+ )
491
+ p.add_argument(
492
+ "--partition",
493
+ help="Slurm partition for the tick (optional; unset lets Slurm choose; a,b = list)",
418
494
  )
419
- p.add_argument("--account", help="Slurm account")
420
- p.add_argument("--partition", help="Slurm partition for the tick")
421
495
  p.add_argument(
422
496
  "--local", action="store_true", help="run the local loop even where sbatch exists"
423
497
  )
424
498
  p.add_argument("--dry-run", action="store_true", help="print the command and exit")
425
- sub.add_parser("tick", help="one tick, or --loop; the chain's own entry", add_help=False)
499
+ sub.add_parser(
500
+ "tick", help="run one tick now; --loop keeps ticking (the local loop)", add_help=False
501
+ )
426
502
  sub.add_parser(
427
503
  "init",
428
504
  help="guided setup: write ~/.config/outerloop/.env and the PAT file",
outerloop/compute.py CHANGED
@@ -95,6 +95,10 @@ class JobSpec:
95
95
  # Slurm scheduling controls
96
96
  dependency: str = "" # e.g. "afterany:12345" or "singleton"
97
97
  begin: str = "" # e.g. "now+30" or an absolute "YYYY-MM-DDTHH:MM:SS"
98
+ # submitted held (PENDING, reason JobHeldUser) until `release`: the
99
+ # tick's launch admission lets GPU launches into the queue in order,
100
+ # under the per-user cap, instead of queueing them all at once
101
+ hold: bool = False
98
102
  extra: tuple[str, ...] = ()
99
103
 
100
104
  def to_argv(self) -> list[str]:
@@ -104,12 +108,13 @@ class JobSpec:
104
108
  "sbatch",
105
109
  "--parsable",
106
110
  f"--job-name={self.job_name}",
107
- f"--account={self.account}",
108
111
  f"--time={self.time_minutes}",
109
112
  f"--cpus-per-task={self.cpus}",
110
113
  f"--mem={self.mem}",
111
114
  f"--output={self.output}",
112
115
  ]
116
+ if self.account: # unset bills the caller's default Slurm association
117
+ argv.append(f"--account={self.account}")
113
118
  if self.partition: # unset lets Slurm pick its default partition
114
119
  argv.append(f"--partition={self.partition}")
115
120
  if self.gpus:
@@ -120,6 +125,8 @@ class JobSpec:
120
125
  argv.append(f"--gpus-per-node={self.gpus}")
121
126
  if self.qos:
122
127
  argv.append(f"--qos={self.qos}")
128
+ if self.hold:
129
+ argv.append("--hold")
123
130
  if self.dependency:
124
131
  argv.append(f"--dependency={self.dependency}")
125
132
  if self.begin:
@@ -133,7 +140,9 @@ class JobSpec:
133
140
  return argv
134
141
 
135
142
 
136
- QUEUE_FIELDS = ("id", "name", "state", "elapsed", "partition", "submitted")
143
+ # `reason` and `gres` feed launch admission (why a job waits, how many GPUs it
144
+ # asks for); the board reads the first six by key and ignores the rest
145
+ QUEUE_FIELDS = ("id", "name", "state", "elapsed", "partition", "submitted", "reason", "gres")
137
146
 
138
147
 
139
148
  class Compute(Protocol):
@@ -148,15 +157,16 @@ class Compute(Protocol):
148
157
  def queue_snapshot(self) -> list[dict[str, str]]: ...
149
158
  def job_id_for_name(self, name: str) -> str: ...
150
159
  def cancel(self, job_id: str) -> None: ...
160
+ def release(self, job_id: str) -> None: ...
151
161
 
152
162
 
153
163
  def local_mode() -> bool:
154
- """AUTORESEARCH_COMPUTE=local selects the monolith: every job a
164
+ """OUTERLOOP_COMPUTE=local selects the monolith: every job a
155
165
  synchronous subprocess of the caller (docs/design/onboarding.md) — the
156
166
  zero-cluster on-ramp and the paper's serialized-baseline ablation. Any
157
167
  other value (or none) is Slurm. This helper is the only reader of the
158
168
  env var, so mode checks cannot drift."""
159
- return os.environ.get("AUTORESEARCH_COMPUTE", "").strip().lower() == "local"
169
+ return os.environ.get("OUTERLOOP_COMPUTE", "").strip().lower() == "local"
160
170
 
161
171
 
162
172
  def compute_from_env() -> SlurmCompute | LocalCompute:
@@ -267,7 +277,7 @@ class SlurmCompute:
267
277
  on failure, like active_job_names."""
268
278
  try:
269
279
  result = self.runner(
270
- ["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V"],
280
+ ["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V|%r|%b"],
271
281
  self.command_timeout_s,
272
282
  )
273
283
  except (OSError, subprocess.TimeoutExpired) as exc:
@@ -307,6 +317,15 @@ class SlurmCompute:
307
317
  if result.returncode != 0:
308
318
  log.warning("scancel %s: %s", job_id, result.stderr.strip())
309
319
 
320
+ def release(self, job_id: str) -> None:
321
+ """Release a job submitted with `hold` so the scheduler may start it.
322
+ Releasing a job that is not held is not an error."""
323
+ if not job_id.isdigit():
324
+ raise ValueError(f"not a job id: {job_id!r}")
325
+ result = self.runner(["scontrol", "release", job_id], self.command_timeout_s)
326
+ if result.returncode != 0:
327
+ log.warning("scontrol release %s: %s", job_id, result.stderr.strip())
328
+
310
329
 
311
330
  # Local job ids start far above any real Slurm id so the two can never be
312
331
  # confused in a record; they stay numeric because callers validate isdigit.
@@ -318,7 +337,7 @@ def _local_state_dir() -> Path | None:
318
337
  attempts it spawns each hold their own LocalCompute): under the state
319
338
  root when the deployment names one, else nowhere (memory-only — tests).
320
339
  Local jobs are synchronous, so only TERMINAL states ever need sharing."""
321
- root = os.environ.get("AUTORESEARCH_ROOT", "").strip()
340
+ root = os.environ.get("OUTERLOOP_ROOT", "").strip()
322
341
  return Path(root) / "local_jobs" if root else None
323
342
 
324
343
 
@@ -354,7 +373,7 @@ class LocalCompute:
354
373
  # the submitting process holds live keys (and any inherited
355
374
  # APPTAINERENV_* would cross --cleanenv into the container), so the
356
375
  # job script starts from a minimal environment and sets its own.
357
- # AUTORESEARCH_* / REVIEW_HERMES_* pass through as a PREFIX rule:
376
+ # OUTERLOOP_* / REVIEW_HERMES_* pass through as a PREFIX rule:
358
377
  # Slurm jobs inherit the tick's whole environment, and local jobs
359
378
  # need the same config surface (compute mode, author backend, panel,
360
379
  # key-file PATHS). Enumerating allowed names is how a mode flag dies
@@ -368,7 +387,7 @@ class LocalCompute:
368
387
  k: v
369
388
  for k, v in os.environ.items()
370
389
  if k in ("PATH", "HOME", "LANG", "TMPDIR", "SLURM_TMPDIR", "USER", "LOGNAME")
371
- or (k.startswith(("AUTORESEARCH_", "REVIEW_HERMES_")) and not _secret_name(k))
390
+ or (k.startswith(("OUTERLOOP_", "REVIEW_HERMES_")) and not _secret_name(k))
372
391
  }
373
392
  try:
374
393
  # the job runs in its OWN session (= process group), so the
@@ -413,6 +432,13 @@ class LocalCompute:
413
432
  tmp = state_dir / f".{job_id}.{os.getpid()}.tmp"
414
433
  tmp.write_text(state)
415
434
  os.replace(tmp, state_dir / job_id)
435
+ # the job's combined stdout/stderr beside its state: a failed local
436
+ # job otherwise leaves nothing to read (#295)
437
+ out_fd = os.open(
438
+ state_dir / f"{job_id}.out", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600
439
+ )
440
+ with os.fdopen(out_fd, "w") as fh:
441
+ fh.write(output)
416
442
  # opportunistic prune: one entry per job would leak forever
417
443
  # on a long-running loop; anything the sweep could still want
418
444
  # is far younger than a day
@@ -432,7 +458,12 @@ class LocalCompute:
432
458
  except OSError as exc:
433
459
  log.warning("local job %s: output write failed: %s", spec.job_name, exc)
434
460
  self._states[job_id] = state
435
- log.info("ran %s locally as job %s: %s", spec.job_name, job_id, state)
461
+ where = (
462
+ f"; output in {state_dir / (job_id + '.out')}"
463
+ if state_dir and state != "COMPLETED"
464
+ else ""
465
+ )
466
+ log.info("ran %s locally as job %s: %s%s", spec.job_name, job_id, state, where)
436
467
  return job_id
437
468
 
438
469
  def status(self, job_id: str) -> str:
@@ -471,6 +502,9 @@ class LocalCompute:
471
502
  raise ValueError(f"not a job id: {job_id!r}")
472
503
  # already terminal; cancelling a finished job is not an error
473
504
 
505
+ def release(self, job_id: str) -> None:
506
+ """Local jobs run synchronously at submit; nothing is ever held."""
507
+
474
508
 
475
509
  def parse_elapsed(text: str) -> int | None:
476
510
  """Seconds in a sacct Elapsed field: `MM:SS`, `HH:MM:SS` or `D-HH:MM:SS`.
@@ -504,6 +538,22 @@ def is_pending(state: str) -> bool:
504
538
  return state.startswith("PENDING")
505
539
 
506
540
 
541
+ def gpus_in_gres(gres: str) -> int:
542
+ """GPUs a queue row asks for, from squeue's %b (`gres/gpu:8`,
543
+ `gres/gpu:h200:2`, `gres:gpu:4`; `N/A` or anything else = 0)."""
544
+ total = 0
545
+ for part in gres.replace(";", ",").split(","):
546
+ fields = part.strip().split(":")
547
+ if (len(fields) >= 2 and fields[0] in ("gres/gpu", "gpu")) or (
548
+ len(fields) >= 3 and fields[0] == "gres" and fields[1] == "gpu"
549
+ ):
550
+ try:
551
+ total += int(fields[-1])
552
+ except ValueError:
553
+ continue
554
+ return total
555
+
556
+
507
557
  def quote_command(parts: Sequence[str]) -> str:
508
558
  """Shell-quote a command for JobSpec.command (--wrap takes a string)."""
509
559
  return " ".join(shlex.quote(p) for p in parts)
outerloop/contract.py CHANGED
@@ -26,6 +26,7 @@ SELF_REPO = "outerloop-science/outerloop"
26
26
  # `.autoresearch.yaml` (targets written before the rename) is still honored. Every
27
27
  # read goes through `find_contract`, which tries the new name first. Neither is
28
28
  # ever a writable path for the agent.
29
+ # The pre-rename name is dropped in the release after 0.1.
29
30
  CONTRACT_NAMES: tuple[str, ...] = (".outerloop.yaml", ".autoresearch.yaml")
30
31
  CONTRACT_NAME = CONTRACT_NAMES[0] # what the docs and new contracts use
31
32
  ALWAYS_FORBIDDEN: tuple[str, ...] = (".github", *CONTRACT_NAMES)
@@ -126,8 +127,8 @@ class Benchmark(_StrictModel):
126
127
  eval_minutes: int | None = Field(default=None, ge=1)
127
128
  # GPUs for every dispatched job of this benchmark — gate measures and
128
129
  # author launches alike. 0 (default) = CPU. A GPU benchmark needs the
129
- # deployment to name a GPU lane (AUTORESEARCH_GPU_PARTITION, optionally
130
- # AUTORESEARCH_GPU_ACCOUNT); without one the tick refuses to launch
130
+ # deployment to name a GPU lane (OUTERLOOP_GPU_PARTITION, optionally
131
+ # OUTERLOOP_GPU_ACCOUNT); without one the tick refuses to launch
131
132
  # attempts on it rather than queue evals that can never run. Bounded at
132
133
  # one node's worth: multi-node evals are not a shape the jail supports.
133
134
  gpus: int = Field(default=0, ge=0, le=8)