outerloop-science 0.1.0.dev1__py3-none-any.whl → 0.1.0.dev2__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +7 -7
- outerloop/appauth.py +3 -3
- outerloop/appmanifest.py +1 -1
- outerloop/attempt.py +55 -49
- outerloop/cli.py +150 -74
- outerloop/compute.py +59 -9
- outerloop/contract.py +3 -2
- outerloop/followup.py +14 -17
- outerloop/github.py +4 -4
- outerloop/harness.py +18 -7
- outerloop/image.py +372 -0
- outerloop/init.py +229 -14
- outerloop/limits.py +1 -1
- outerloop/measure.py +2 -2
- outerloop/orchestrator.py +1 -1
- outerloop/paths.py +2 -1
- outerloop/review.py +1 -1
- outerloop/steward.py +3 -3
- outerloop/syscall.py +1 -0
- outerloop/tick.py +302 -77
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/METADATA +2 -2
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/RECORD +26 -25
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev2.dist-info}/licenses/NOTICE +0 -0
outerloop/cli.py
CHANGED
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
With `sbatch` on PATH it submits the resident tick
|
|
4
4
|
(docs/design/resident-tick.md) and returns; without it, or with
|
|
5
|
-
|
|
5
|
+
OUTERLOOP_COMPUTE=local, it runs the local loop in the foreground.
|
|
6
6
|
Settings come from flags, then the process environment, then
|
|
7
7
|
~/.config/outerloop/.env, read once here at launch. The running chain
|
|
8
8
|
never takes identity or placement from that file (tick_deploy.sh reads an
|
|
@@ -18,49 +18,59 @@ import shutil
|
|
|
18
18
|
import stat
|
|
19
19
|
import subprocess
|
|
20
20
|
import sys
|
|
21
|
+
from collections.abc import Mapping
|
|
21
22
|
from dataclasses import dataclass
|
|
22
23
|
from pathlib import Path
|
|
23
24
|
|
|
24
25
|
from outerloop import paths
|
|
25
26
|
|
|
26
|
-
RESIDENT_JOB_NAME = "
|
|
27
|
+
RESIDENT_JOB_NAME = "outerloop-resident"
|
|
28
|
+
# A resident submitted before the rename. Slurm's singleton serializes jobs by
|
|
29
|
+
# NAME, so `start` must refuse while one of these is queued or running (two
|
|
30
|
+
# residents on one root is what the singleton prevents). Dropped in the
|
|
31
|
+
# release after 0.1.
|
|
32
|
+
LEGACY_RESIDENT_JOB_NAME = "autoresearch-resident"
|
|
27
33
|
DEFAULT_RESIDENT_MINUTES = 360 # cpu_short's ceiling on Torch; the loop hands over to itself
|
|
28
|
-
DEFAULT_LOCAL_ROOT = Path.home() / ".
|
|
34
|
+
DEFAULT_LOCAL_ROOT = Path.home() / ".outerloop"
|
|
35
|
+
LEGACY_LOCAL_ROOT_NAME = ".autoresearch" # pre-rename; honored until the release after 0.1
|
|
29
36
|
ENV_FILE = paths.ENV_FILE # ~/.config/outerloop/.env, or the pre-rename dir (see paths.py)
|
|
30
37
|
|
|
31
38
|
# What start itself decides from: mode, placement, root, cadence, walltime.
|
|
32
39
|
START_KEYS = (
|
|
33
|
-
"
|
|
34
|
-
"
|
|
35
|
-
"
|
|
36
|
-
"
|
|
37
|
-
"
|
|
38
|
-
"
|
|
39
|
-
"
|
|
40
|
+
"OUTERLOOP_COMPUTE",
|
|
41
|
+
"OUTERLOOP_ROOT",
|
|
42
|
+
"OUTERLOOP_ACCOUNT",
|
|
43
|
+
"OUTERLOOP_PARTITION",
|
|
44
|
+
"OUTERLOOP_CADENCE_MIN",
|
|
45
|
+
"OUTERLOOP_RESIDENT_MINUTES",
|
|
46
|
+
"OUTERLOOP_PAT_FILE",
|
|
40
47
|
)
|
|
41
48
|
# The author knobs the chain's deploy step exports from .env every tick. The
|
|
42
49
|
# local loop has no deploy step, so start exports them once at launch; a test
|
|
43
50
|
# keeps this list identical to tick_deploy.sh's.
|
|
44
51
|
TICK_ENV_KEYS = (
|
|
45
|
-
"
|
|
46
|
-
"
|
|
47
|
-
"
|
|
48
|
-
"
|
|
49
|
-
"
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
"
|
|
53
|
-
"
|
|
54
|
-
"
|
|
55
|
-
"
|
|
56
|
-
"
|
|
57
|
-
"
|
|
58
|
-
"
|
|
59
|
-
"
|
|
60
|
-
"
|
|
61
|
-
"
|
|
62
|
-
"
|
|
63
|
-
"
|
|
52
|
+
"OUTERLOOP_AUTHOR_BACKEND",
|
|
53
|
+
"OUTERLOOP_AUTHOR_MODEL",
|
|
54
|
+
"OUTERLOOP_CLAUDE_BIN",
|
|
55
|
+
"OUTERLOOP_CODEX_BIN",
|
|
56
|
+
"OUTERLOOP_CODEX_KEY_FILE",
|
|
57
|
+
"OUTERLOOP_CLAUDE_KEY_FILE",
|
|
58
|
+
"OUTERLOOP_HARNESS_KEY_FILE",
|
|
59
|
+
"OUTERLOOP_VERTEX_PROJECT",
|
|
60
|
+
"OUTERLOOP_VERTEX_REGION",
|
|
61
|
+
"OUTERLOOP_VERTEX_ADC",
|
|
62
|
+
"OUTERLOOP_TARGET",
|
|
63
|
+
"OUTERLOOP_GITHUB_APP_FILE",
|
|
64
|
+
"OUTERLOOP_BOT_LOGIN",
|
|
65
|
+
"OUTERLOOP_BOT_ALIASES",
|
|
66
|
+
"OUTERLOOP_GPU_PARTITION",
|
|
67
|
+
"OUTERLOOP_GPU_ACCOUNT",
|
|
68
|
+
"OUTERLOOP_MAX_LAUNCH_GPUS",
|
|
69
|
+
"OUTERLOOP_IMAGE",
|
|
70
|
+
"OUTERLOOP_PANEL",
|
|
71
|
+
"OUTERLOOP_PANEL_KEY_FILE",
|
|
72
|
+
"OUTERLOOP_PANEL_CODEX_KEY_FILE",
|
|
73
|
+
"OUTERLOOP_PANEL_HERMES_KEY_FILE",
|
|
64
74
|
"REVIEW_HERMES_REPO",
|
|
65
75
|
"REVIEW_HERMES_PROVIDER",
|
|
66
76
|
)
|
|
@@ -89,20 +99,26 @@ def env_file_values(path: Path = ENV_FILE, keys: tuple[str, ...] = START_KEYS) -
|
|
|
89
99
|
except OSError as e:
|
|
90
100
|
raise StartError(f"cannot read {path}: {e}") from None
|
|
91
101
|
out: dict[str, str] = {}
|
|
102
|
+
canonical: set[str] = set() # keys set by their OUTERLOOP_ spelling
|
|
92
103
|
for raw in text.splitlines():
|
|
93
104
|
line = raw.rstrip("\r").strip()
|
|
94
105
|
if not line or line.startswith("#") or "=" not in line:
|
|
95
106
|
continue
|
|
96
107
|
key, value = line.split("=", 1)
|
|
97
108
|
key = key.strip()
|
|
98
|
-
|
|
99
|
-
|
|
109
|
+
legacy = key.startswith("AUTORESEARCH_") # the pre-rename name, one more release
|
|
110
|
+
if legacy:
|
|
111
|
+
key = "OUTERLOOP_" + key[len("AUTORESEARCH_") :]
|
|
100
112
|
if key not in keys:
|
|
101
113
|
continue
|
|
114
|
+
if legacy and key in canonical:
|
|
115
|
+
continue # the OUTERLOOP_ spelling wins whatever the file order
|
|
102
116
|
value = value.strip()
|
|
103
117
|
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
|
104
118
|
value = value[1:-1]
|
|
105
119
|
out[key] = value
|
|
120
|
+
if not legacy:
|
|
121
|
+
canonical.add(key)
|
|
106
122
|
return out
|
|
107
123
|
|
|
108
124
|
|
|
@@ -124,18 +140,19 @@ class StartPlan:
|
|
|
124
140
|
reads as "whichever frees up first") without corrupting the export
|
|
125
141
|
delimiter. start() merges these into the environment it hands sbatch."""
|
|
126
142
|
env = {
|
|
127
|
-
"
|
|
128
|
-
"
|
|
129
|
-
"
|
|
130
|
-
"
|
|
131
|
-
"AUTORESEARCH_RESIDENT_MINUTES": str(self.resident_minutes), # successors reuse it
|
|
143
|
+
"OUTERLOOP_RESIDENT": "1",
|
|
144
|
+
"OUTERLOOP_HOME": str(self.home),
|
|
145
|
+
"OUTERLOOP_ROOT": str(self.root),
|
|
146
|
+
"OUTERLOOP_RESIDENT_MINUTES": str(self.resident_minutes), # successors reuse it
|
|
132
147
|
}
|
|
148
|
+
if self.account:
|
|
149
|
+
env["OUTERLOOP_ACCOUNT"] = self.account
|
|
133
150
|
if self.partition:
|
|
134
|
-
env["
|
|
151
|
+
env["OUTERLOOP_PARTITION"] = self.partition
|
|
135
152
|
if self.cadence_min:
|
|
136
|
-
env["
|
|
153
|
+
env["OUTERLOOP_CADENCE_MIN"] = self.cadence_min
|
|
137
154
|
if self.pat_file:
|
|
138
|
-
env["
|
|
155
|
+
env["OUTERLOOP_PAT_FILE"] = self.pat_file
|
|
139
156
|
return env
|
|
140
157
|
|
|
141
158
|
def command(self) -> list[str]:
|
|
@@ -147,8 +164,9 @@ class StartPlan:
|
|
|
147
164
|
"--dependency=singleton", # two starts can both submit; only one ever runs
|
|
148
165
|
f"--time={self.resident_minutes}",
|
|
149
166
|
f"--job-name={RESIDENT_JOB_NAME}",
|
|
150
|
-
f"--account={self.account}",
|
|
151
167
|
]
|
|
168
|
+
if self.account: # unset bills the caller's default Slurm association
|
|
169
|
+
argv.append(f"--account={self.account}")
|
|
152
170
|
if self.partition: # unset lets Slurm choose its default partition
|
|
153
171
|
argv.append(f"--partition={self.partition}")
|
|
154
172
|
# --export=ALL carries export_env() from the inherited environment; a
|
|
@@ -168,7 +186,7 @@ def _setting(key: str, flag: str, environ: dict[str, str], from_file: dict[str,
|
|
|
168
186
|
|
|
169
187
|
def _home(environ: dict[str, str], cwd: Path, *, local: bool, root: Path) -> Path:
|
|
170
188
|
"""The directory the loop runs from. On Slurm it must be a source
|
|
171
|
-
checkout (
|
|
189
|
+
checkout (OUTERLOOP_HOME, else the current directory): the chain
|
|
172
190
|
deploys from it and every job runs from a flight snapshot of its HEAD.
|
|
173
191
|
The local loop has no deploy step and runs the installed package, so it
|
|
174
192
|
uses a checkout when one is at hand and otherwise a `home` directory
|
|
@@ -196,10 +214,10 @@ def plan_start(
|
|
|
196
214
|
sbatch_on_path: bool,
|
|
197
215
|
cwd: Path,
|
|
198
216
|
) -> StartPlan:
|
|
199
|
-
compute = _setting("
|
|
217
|
+
compute = _setting("OUTERLOOP_COMPUTE", "local" if local else "", environ, from_file)
|
|
200
218
|
mode = "local" if compute.strip().lower() == "local" or not sbatch_on_path else "slurm"
|
|
201
|
-
root_s = _setting("
|
|
202
|
-
cadence = _setting("
|
|
219
|
+
root_s = _setting("OUTERLOOP_ROOT", root, environ, from_file)
|
|
220
|
+
cadence = _setting("OUTERLOOP_CADENCE_MIN", "", environ, from_file)
|
|
203
221
|
if cadence:
|
|
204
222
|
# the chain divides by it and the loop sleeps on it: a bad value would
|
|
205
223
|
# only surface after the job started
|
|
@@ -209,13 +227,13 @@ def plan_start(
|
|
|
209
227
|
cadence_ok = False
|
|
210
228
|
if not cadence_ok:
|
|
211
229
|
raise StartError(
|
|
212
|
-
f"
|
|
230
|
+
f"OUTERLOOP_CADENCE_MIN must be a positive number of minutes, got {cadence!r}"
|
|
213
231
|
)
|
|
214
|
-
pat = _setting("
|
|
232
|
+
pat = _setting("OUTERLOOP_PAT_FILE", "", environ, from_file)
|
|
215
233
|
# Slurm runs from a checkout; the local loop runs the installed package
|
|
216
234
|
# and needs only a directory (the launch lanes and GitHub servicing
|
|
217
|
-
# switch off without
|
|
218
|
-
local_root = Path(root_s).expanduser() if root_s else
|
|
235
|
+
# switch off without OUTERLOOP_HOME, so one is always set)
|
|
236
|
+
local_root = Path(root_s).expanduser() if root_s else default_local_root(environ)
|
|
219
237
|
home = _home(environ, cwd, local=(mode == "local"), root=local_root)
|
|
220
238
|
if mode == "local":
|
|
221
239
|
return StartPlan(
|
|
@@ -228,23 +246,22 @@ def plan_start(
|
|
|
228
246
|
if not root_s:
|
|
229
247
|
raise StartError(
|
|
230
248
|
"Slurm mode needs the state root on the shared filesystem: "
|
|
231
|
-
"--root,
|
|
249
|
+
"--root, OUTERLOOP_ROOT in the environment, or OUTERLOOP_ROOT= in "
|
|
250
|
+
"~/.config/outerloop/.env"
|
|
232
251
|
)
|
|
233
|
-
acc = _setting("
|
|
234
|
-
part = _setting("
|
|
235
|
-
#
|
|
236
|
-
#
|
|
237
|
-
|
|
238
|
-
raise StartError("Slurm mode needs --account / AUTORESEARCH_ACCOUNT")
|
|
239
|
-
minutes_s = _setting("AUTORESEARCH_RESIDENT_MINUTES", "", environ, from_file)
|
|
252
|
+
acc = _setting("OUTERLOOP_ACCOUNT", account, environ, from_file)
|
|
253
|
+
part = _setting("OUTERLOOP_PARTITION", partition, environ, from_file)
|
|
254
|
+
# Account and partition are both optional: left unset, Slurm bills the
|
|
255
|
+
# caller's default association and places the job on its default partition.
|
|
256
|
+
minutes_s = _setting("OUTERLOOP_RESIDENT_MINUTES", "", environ, from_file)
|
|
240
257
|
try:
|
|
241
258
|
minutes = int(minutes_s) if minutes_s else DEFAULT_RESIDENT_MINUTES
|
|
242
259
|
except ValueError:
|
|
243
260
|
raise StartError(
|
|
244
|
-
f"
|
|
261
|
+
f"OUTERLOOP_RESIDENT_MINUTES must be a whole number of minutes, got {minutes_s!r}"
|
|
245
262
|
) from None
|
|
246
263
|
if minutes <= 0:
|
|
247
|
-
raise StartError("
|
|
264
|
+
raise StartError("OUTERLOOP_RESIDENT_MINUTES must be positive")
|
|
248
265
|
# These ride the inherited environment (sbatch --export=ALL), so a comma is
|
|
249
266
|
# safe now (a multi-partition `a,b` is valid) — only a newline would corrupt
|
|
250
267
|
# the environment or the sbatch argv.
|
|
@@ -270,16 +287,30 @@ def plan_start(
|
|
|
270
287
|
)
|
|
271
288
|
|
|
272
289
|
|
|
290
|
+
def default_local_root(environ: Mapping[str, str]) -> Path:
|
|
291
|
+
"""The local-mode state root when none is given: ~/.outerloop, or the
|
|
292
|
+
pre-rename ~/.autoresearch when only that one exists (state is never moved
|
|
293
|
+
behind the operator's back). The home comes from `environ` so a caller
|
|
294
|
+
with no HOME gets the plain default."""
|
|
295
|
+
home_s = environ.get("HOME", "")
|
|
296
|
+
if not home_s:
|
|
297
|
+
return DEFAULT_LOCAL_ROOT
|
|
298
|
+
home = Path(home_s)
|
|
299
|
+
new, old = home / DEFAULT_LOCAL_ROOT.name, home / LEGACY_LOCAL_ROOT_NAME
|
|
300
|
+
return old if (not new.exists() and old.is_dir()) else new
|
|
301
|
+
|
|
302
|
+
|
|
273
303
|
def _resident_jobs() -> list[str] | None:
|
|
274
|
-
"""Ids of queued or running resident ticks, lowest first;
|
|
275
|
-
scheduler could not be asked (a failed lookup must never
|
|
304
|
+
"""Ids of queued or running resident ticks under either name, lowest first;
|
|
305
|
+
None when the scheduler could not be asked (a failed lookup must never
|
|
306
|
+
read as 'none')."""
|
|
276
307
|
try:
|
|
277
308
|
proc = subprocess.run(
|
|
278
309
|
[
|
|
279
310
|
"squeue",
|
|
280
311
|
"-u",
|
|
281
312
|
os.environ.get("USER", ""),
|
|
282
|
-
f"--name={RESIDENT_JOB_NAME}",
|
|
313
|
+
f"--name={RESIDENT_JOB_NAME},{LEGACY_RESIDENT_JOB_NAME}",
|
|
283
314
|
"-h",
|
|
284
315
|
"-o",
|
|
285
316
|
"%i",
|
|
@@ -309,9 +340,37 @@ def _exec(cmd: list[str], env: dict[str, str]) -> int:
|
|
|
309
340
|
return 1 # unreachable; keeps the signature honest for tests that stub this
|
|
310
341
|
|
|
311
342
|
|
|
343
|
+
HARNESS_BIN_KEYS = ("OUTERLOOP_CLAUDE_BIN", "OUTERLOOP_CODEX_BIN")
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _setting_of(key: str, values: Mapping[str, str], environ: Mapping[str, str]) -> str:
|
|
347
|
+
"""The process environment wins over .env, including an explicit empty value
|
|
348
|
+
(the way to clear a recorded path from the shell)."""
|
|
349
|
+
return (environ[key] if key in environ else values.get(key, "")).strip()
|
|
350
|
+
|
|
351
|
+
|
|
352
|
+
def missing_harness_binary(values: Mapping[str, str], environ: Mapping[str, str]) -> str:
|
|
353
|
+
"""Why start must not launch: the CONFIGURED author backend's recorded
|
|
354
|
+
binary is not there (init records the path; a moved or uninstalled CLI
|
|
355
|
+
would end every climb with spawn-error). The other backend's path is not
|
|
356
|
+
consulted: a stale entry for a backend the loop never spawns must not block
|
|
357
|
+
it. "" when the binary exists or none is recorded."""
|
|
358
|
+
backend = _setting_of("OUTERLOOP_AUTHOR_BACKEND", values, environ).lower() or "claude"
|
|
359
|
+
key = f"OUTERLOOP_{backend.upper()}_BIN"
|
|
360
|
+
if key not in HARNESS_BIN_KEYS:
|
|
361
|
+
return "" # an unknown backend is init's error to report, not this check's
|
|
362
|
+
path = _setting_of(key, values, environ)
|
|
363
|
+
if path and not Path(path).expanduser().is_file():
|
|
364
|
+
return f"{key}={path} is not a file; reinstall the CLI or run `outerloop init --force`"
|
|
365
|
+
return ""
|
|
366
|
+
|
|
367
|
+
|
|
312
368
|
def start(args: argparse.Namespace) -> int:
|
|
313
369
|
try:
|
|
314
370
|
values = env_file_values(ENV_FILE, START_KEYS + TICK_ENV_KEYS) # one read for everything
|
|
371
|
+
problem = missing_harness_binary(values, os.environ)
|
|
372
|
+
if problem:
|
|
373
|
+
raise StartError(problem)
|
|
315
374
|
from_file = {k: v for k, v in values.items() if k in START_KEYS}
|
|
316
375
|
plan = plan_start(
|
|
317
376
|
root=args.root or "",
|
|
@@ -337,14 +396,14 @@ def start(args: argparse.Namespace) -> int:
|
|
|
337
396
|
for key, value in values.items():
|
|
338
397
|
if key in TICK_ENV_KEYS:
|
|
339
398
|
env.setdefault(key, value)
|
|
340
|
-
env["
|
|
341
|
-
env["
|
|
342
|
-
env["
|
|
399
|
+
env["OUTERLOOP_COMPUTE"] = "local"
|
|
400
|
+
env["OUTERLOOP_ROOT"] = str(plan.root)
|
|
401
|
+
env["OUTERLOOP_HOME"] = str(plan.home)
|
|
343
402
|
plan.home.mkdir(parents=True, exist_ok=True) # <root>/home when there is no checkout
|
|
344
403
|
if plan.cadence_min:
|
|
345
|
-
env["
|
|
404
|
+
env["OUTERLOOP_CADENCE_MIN"] = plan.cadence_min
|
|
346
405
|
if plan.pat_file:
|
|
347
|
-
env["
|
|
406
|
+
env["OUTERLOOP_PAT_FILE"] = plan.pat_file
|
|
348
407
|
print(
|
|
349
408
|
f"local loop: state in {plan.root}; Ctrl-C stops it, the records resume it",
|
|
350
409
|
file=sys.stderr,
|
|
@@ -355,14 +414,14 @@ def start(args: argparse.Namespace) -> int:
|
|
|
355
414
|
print(
|
|
356
415
|
"outerloop start: could not ask the scheduler whether a resident tick "
|
|
357
416
|
"exists (squeue failed); nothing submitted. Retry, or check "
|
|
358
|
-
f"`squeue --name {RESIDENT_JOB_NAME}`.",
|
|
417
|
+
f"`squeue --name {RESIDENT_JOB_NAME},{LEGACY_RESIDENT_JOB_NAME}`.",
|
|
359
418
|
file=sys.stderr,
|
|
360
419
|
)
|
|
361
420
|
return 1
|
|
362
421
|
if existing:
|
|
363
422
|
print(
|
|
364
423
|
f"a resident tick is already queued or running (job {existing[0]}); nothing "
|
|
365
|
-
f"submitted. Stop it with `scancel
|
|
424
|
+
f"submitted. Stop it with `scancel {existing[0]}`, or pause it "
|
|
366
425
|
f"with `touch {plan.root}/PAUSE`.",
|
|
367
426
|
file=sys.stderr,
|
|
368
427
|
)
|
|
@@ -406,23 +465,40 @@ def start(args: argparse.Namespace) -> int:
|
|
|
406
465
|
|
|
407
466
|
|
|
408
467
|
def main(argv: list[str] | None = None) -> int:
|
|
468
|
+
from outerloop import __version__
|
|
469
|
+
|
|
409
470
|
parser = argparse.ArgumentParser(
|
|
410
|
-
prog="outerloop",
|
|
471
|
+
prog="outerloop",
|
|
472
|
+
description="autonomous research agents in an outer loop",
|
|
473
|
+
epilog="Run `outerloop init` once, then `outerloop start`. Settings live in "
|
|
474
|
+
"~/.config/outerloop/.env; the install guide is docs/install.md.",
|
|
411
475
|
)
|
|
476
|
+
parser.add_argument("--version", action="version", version=f"outerloop {__version__}")
|
|
412
477
|
sub = parser.add_subparsers(dest="command", required=True)
|
|
413
478
|
p = sub.add_parser(
|
|
414
|
-
"start",
|
|
479
|
+
"start",
|
|
480
|
+
help="start the loop: the resident tick on Slurm, the local loop elsewhere",
|
|
481
|
+
description="Submit the resident tick to Slurm, or run the local loop in the "
|
|
482
|
+
"foreground where there is no sbatch. Every flag defaults from "
|
|
483
|
+
"~/.config/outerloop/.env, written by `outerloop init`.",
|
|
484
|
+
)
|
|
485
|
+
p.add_argument(
|
|
486
|
+
"--root", help="state root (shared filesystem on Slurm; default ~/.outerloop locally)"
|
|
415
487
|
)
|
|
416
488
|
p.add_argument(
|
|
417
|
-
"--
|
|
489
|
+
"--account", help="Slurm account (optional; unset bills your default association)"
|
|
490
|
+
)
|
|
491
|
+
p.add_argument(
|
|
492
|
+
"--partition",
|
|
493
|
+
help="Slurm partition for the tick (optional; unset lets Slurm choose; a,b = list)",
|
|
418
494
|
)
|
|
419
|
-
p.add_argument("--account", help="Slurm account")
|
|
420
|
-
p.add_argument("--partition", help="Slurm partition for the tick")
|
|
421
495
|
p.add_argument(
|
|
422
496
|
"--local", action="store_true", help="run the local loop even where sbatch exists"
|
|
423
497
|
)
|
|
424
498
|
p.add_argument("--dry-run", action="store_true", help="print the command and exit")
|
|
425
|
-
sub.add_parser(
|
|
499
|
+
sub.add_parser(
|
|
500
|
+
"tick", help="run one tick now; --loop keeps ticking (the local loop)", add_help=False
|
|
501
|
+
)
|
|
426
502
|
sub.add_parser(
|
|
427
503
|
"init",
|
|
428
504
|
help="guided setup: write ~/.config/outerloop/.env and the PAT file",
|
outerloop/compute.py
CHANGED
|
@@ -95,6 +95,10 @@ class JobSpec:
|
|
|
95
95
|
# Slurm scheduling controls
|
|
96
96
|
dependency: str = "" # e.g. "afterany:12345" or "singleton"
|
|
97
97
|
begin: str = "" # e.g. "now+30" or an absolute "YYYY-MM-DDTHH:MM:SS"
|
|
98
|
+
# submitted held (PENDING, reason JobHeldUser) until `release`: the
|
|
99
|
+
# tick's launch admission lets GPU launches into the queue in order,
|
|
100
|
+
# under the per-user cap, instead of queueing them all at once
|
|
101
|
+
hold: bool = False
|
|
98
102
|
extra: tuple[str, ...] = ()
|
|
99
103
|
|
|
100
104
|
def to_argv(self) -> list[str]:
|
|
@@ -104,12 +108,13 @@ class JobSpec:
|
|
|
104
108
|
"sbatch",
|
|
105
109
|
"--parsable",
|
|
106
110
|
f"--job-name={self.job_name}",
|
|
107
|
-
f"--account={self.account}",
|
|
108
111
|
f"--time={self.time_minutes}",
|
|
109
112
|
f"--cpus-per-task={self.cpus}",
|
|
110
113
|
f"--mem={self.mem}",
|
|
111
114
|
f"--output={self.output}",
|
|
112
115
|
]
|
|
116
|
+
if self.account: # unset bills the caller's default Slurm association
|
|
117
|
+
argv.append(f"--account={self.account}")
|
|
113
118
|
if self.partition: # unset lets Slurm pick its default partition
|
|
114
119
|
argv.append(f"--partition={self.partition}")
|
|
115
120
|
if self.gpus:
|
|
@@ -120,6 +125,8 @@ class JobSpec:
|
|
|
120
125
|
argv.append(f"--gpus-per-node={self.gpus}")
|
|
121
126
|
if self.qos:
|
|
122
127
|
argv.append(f"--qos={self.qos}")
|
|
128
|
+
if self.hold:
|
|
129
|
+
argv.append("--hold")
|
|
123
130
|
if self.dependency:
|
|
124
131
|
argv.append(f"--dependency={self.dependency}")
|
|
125
132
|
if self.begin:
|
|
@@ -133,7 +140,9 @@ class JobSpec:
|
|
|
133
140
|
return argv
|
|
134
141
|
|
|
135
142
|
|
|
136
|
-
|
|
143
|
+
# `reason` and `gres` feed launch admission (why a job waits, how many GPUs it
|
|
144
|
+
# asks for); the board reads the first six by key and ignores the rest
|
|
145
|
+
QUEUE_FIELDS = ("id", "name", "state", "elapsed", "partition", "submitted", "reason", "gres")
|
|
137
146
|
|
|
138
147
|
|
|
139
148
|
class Compute(Protocol):
|
|
@@ -148,15 +157,16 @@ class Compute(Protocol):
|
|
|
148
157
|
def queue_snapshot(self) -> list[dict[str, str]]: ...
|
|
149
158
|
def job_id_for_name(self, name: str) -> str: ...
|
|
150
159
|
def cancel(self, job_id: str) -> None: ...
|
|
160
|
+
def release(self, job_id: str) -> None: ...
|
|
151
161
|
|
|
152
162
|
|
|
153
163
|
def local_mode() -> bool:
|
|
154
|
-
"""
|
|
164
|
+
"""OUTERLOOP_COMPUTE=local selects the monolith: every job a
|
|
155
165
|
synchronous subprocess of the caller (docs/design/onboarding.md) — the
|
|
156
166
|
zero-cluster on-ramp and the paper's serialized-baseline ablation. Any
|
|
157
167
|
other value (or none) is Slurm. This helper is the only reader of the
|
|
158
168
|
env var, so mode checks cannot drift."""
|
|
159
|
-
return os.environ.get("
|
|
169
|
+
return os.environ.get("OUTERLOOP_COMPUTE", "").strip().lower() == "local"
|
|
160
170
|
|
|
161
171
|
|
|
162
172
|
def compute_from_env() -> SlurmCompute | LocalCompute:
|
|
@@ -267,7 +277,7 @@ class SlurmCompute:
|
|
|
267
277
|
on failure, like active_job_names."""
|
|
268
278
|
try:
|
|
269
279
|
result = self.runner(
|
|
270
|
-
["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V"],
|
|
280
|
+
["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V|%r|%b"],
|
|
271
281
|
self.command_timeout_s,
|
|
272
282
|
)
|
|
273
283
|
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
@@ -307,6 +317,15 @@ class SlurmCompute:
|
|
|
307
317
|
if result.returncode != 0:
|
|
308
318
|
log.warning("scancel %s: %s", job_id, result.stderr.strip())
|
|
309
319
|
|
|
320
|
+
def release(self, job_id: str) -> None:
|
|
321
|
+
"""Release a job submitted with `hold` so the scheduler may start it.
|
|
322
|
+
Releasing a job that is not held is not an error."""
|
|
323
|
+
if not job_id.isdigit():
|
|
324
|
+
raise ValueError(f"not a job id: {job_id!r}")
|
|
325
|
+
result = self.runner(["scontrol", "release", job_id], self.command_timeout_s)
|
|
326
|
+
if result.returncode != 0:
|
|
327
|
+
log.warning("scontrol release %s: %s", job_id, result.stderr.strip())
|
|
328
|
+
|
|
310
329
|
|
|
311
330
|
# Local job ids start far above any real Slurm id so the two can never be
|
|
312
331
|
# confused in a record; they stay numeric because callers validate isdigit.
|
|
@@ -318,7 +337,7 @@ def _local_state_dir() -> Path | None:
|
|
|
318
337
|
attempts it spawns each hold their own LocalCompute): under the state
|
|
319
338
|
root when the deployment names one, else nowhere (memory-only — tests).
|
|
320
339
|
Local jobs are synchronous, so only TERMINAL states ever need sharing."""
|
|
321
|
-
root = os.environ.get("
|
|
340
|
+
root = os.environ.get("OUTERLOOP_ROOT", "").strip()
|
|
322
341
|
return Path(root) / "local_jobs" if root else None
|
|
323
342
|
|
|
324
343
|
|
|
@@ -354,7 +373,7 @@ class LocalCompute:
|
|
|
354
373
|
# the submitting process holds live keys (and any inherited
|
|
355
374
|
# APPTAINERENV_* would cross --cleanenv into the container), so the
|
|
356
375
|
# job script starts from a minimal environment and sets its own.
|
|
357
|
-
#
|
|
376
|
+
# OUTERLOOP_* / REVIEW_HERMES_* pass through as a PREFIX rule:
|
|
358
377
|
# Slurm jobs inherit the tick's whole environment, and local jobs
|
|
359
378
|
# need the same config surface (compute mode, author backend, panel,
|
|
360
379
|
# key-file PATHS). Enumerating allowed names is how a mode flag dies
|
|
@@ -368,7 +387,7 @@ class LocalCompute:
|
|
|
368
387
|
k: v
|
|
369
388
|
for k, v in os.environ.items()
|
|
370
389
|
if k in ("PATH", "HOME", "LANG", "TMPDIR", "SLURM_TMPDIR", "USER", "LOGNAME")
|
|
371
|
-
or (k.startswith(("
|
|
390
|
+
or (k.startswith(("OUTERLOOP_", "REVIEW_HERMES_")) and not _secret_name(k))
|
|
372
391
|
}
|
|
373
392
|
try:
|
|
374
393
|
# the job runs in its OWN session (= process group), so the
|
|
@@ -413,6 +432,13 @@ class LocalCompute:
|
|
|
413
432
|
tmp = state_dir / f".{job_id}.{os.getpid()}.tmp"
|
|
414
433
|
tmp.write_text(state)
|
|
415
434
|
os.replace(tmp, state_dir / job_id)
|
|
435
|
+
# the job's combined stdout/stderr beside its state: a failed local
|
|
436
|
+
# job otherwise leaves nothing to read (#295)
|
|
437
|
+
out_fd = os.open(
|
|
438
|
+
state_dir / f"{job_id}.out", os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600
|
|
439
|
+
)
|
|
440
|
+
with os.fdopen(out_fd, "w") as fh:
|
|
441
|
+
fh.write(output)
|
|
416
442
|
# opportunistic prune: one entry per job would leak forever
|
|
417
443
|
# on a long-running loop; anything the sweep could still want
|
|
418
444
|
# is far younger than a day
|
|
@@ -432,7 +458,12 @@ class LocalCompute:
|
|
|
432
458
|
except OSError as exc:
|
|
433
459
|
log.warning("local job %s: output write failed: %s", spec.job_name, exc)
|
|
434
460
|
self._states[job_id] = state
|
|
435
|
-
|
|
461
|
+
where = (
|
|
462
|
+
f"; output in {state_dir / (job_id + '.out')}"
|
|
463
|
+
if state_dir and state != "COMPLETED"
|
|
464
|
+
else ""
|
|
465
|
+
)
|
|
466
|
+
log.info("ran %s locally as job %s: %s%s", spec.job_name, job_id, state, where)
|
|
436
467
|
return job_id
|
|
437
468
|
|
|
438
469
|
def status(self, job_id: str) -> str:
|
|
@@ -471,6 +502,9 @@ class LocalCompute:
|
|
|
471
502
|
raise ValueError(f"not a job id: {job_id!r}")
|
|
472
503
|
# already terminal; cancelling a finished job is not an error
|
|
473
504
|
|
|
505
|
+
def release(self, job_id: str) -> None:
|
|
506
|
+
"""Local jobs run synchronously at submit; nothing is ever held."""
|
|
507
|
+
|
|
474
508
|
|
|
475
509
|
def parse_elapsed(text: str) -> int | None:
|
|
476
510
|
"""Seconds in a sacct Elapsed field: `MM:SS`, `HH:MM:SS` or `D-HH:MM:SS`.
|
|
@@ -504,6 +538,22 @@ def is_pending(state: str) -> bool:
|
|
|
504
538
|
return state.startswith("PENDING")
|
|
505
539
|
|
|
506
540
|
|
|
541
|
+
def gpus_in_gres(gres: str) -> int:
|
|
542
|
+
"""GPUs a queue row asks for, from squeue's %b (`gres/gpu:8`,
|
|
543
|
+
`gres/gpu:h200:2`, `gres:gpu:4`; `N/A` or anything else = 0)."""
|
|
544
|
+
total = 0
|
|
545
|
+
for part in gres.replace(";", ",").split(","):
|
|
546
|
+
fields = part.strip().split(":")
|
|
547
|
+
if (len(fields) >= 2 and fields[0] in ("gres/gpu", "gpu")) or (
|
|
548
|
+
len(fields) >= 3 and fields[0] == "gres" and fields[1] == "gpu"
|
|
549
|
+
):
|
|
550
|
+
try:
|
|
551
|
+
total += int(fields[-1])
|
|
552
|
+
except ValueError:
|
|
553
|
+
continue
|
|
554
|
+
return total
|
|
555
|
+
|
|
556
|
+
|
|
507
557
|
def quote_command(parts: Sequence[str]) -> str:
|
|
508
558
|
"""Shell-quote a command for JobSpec.command (--wrap takes a string)."""
|
|
509
559
|
return " ".join(shlex.quote(p) for p in parts)
|
outerloop/contract.py
CHANGED
|
@@ -26,6 +26,7 @@ SELF_REPO = "outerloop-science/outerloop"
|
|
|
26
26
|
# `.autoresearch.yaml` (targets written before the rename) is still honored. Every
|
|
27
27
|
# read goes through `find_contract`, which tries the new name first. Neither is
|
|
28
28
|
# ever a writable path for the agent.
|
|
29
|
+
# The pre-rename name is dropped in the release after 0.1.
|
|
29
30
|
CONTRACT_NAMES: tuple[str, ...] = (".outerloop.yaml", ".autoresearch.yaml")
|
|
30
31
|
CONTRACT_NAME = CONTRACT_NAMES[0] # what the docs and new contracts use
|
|
31
32
|
ALWAYS_FORBIDDEN: tuple[str, ...] = (".github", *CONTRACT_NAMES)
|
|
@@ -126,8 +127,8 @@ class Benchmark(_StrictModel):
|
|
|
126
127
|
eval_minutes: int | None = Field(default=None, ge=1)
|
|
127
128
|
# GPUs for every dispatched job of this benchmark — gate measures and
|
|
128
129
|
# author launches alike. 0 (default) = CPU. A GPU benchmark needs the
|
|
129
|
-
# deployment to name a GPU lane (
|
|
130
|
-
#
|
|
130
|
+
# deployment to name a GPU lane (OUTERLOOP_GPU_PARTITION, optionally
|
|
131
|
+
# OUTERLOOP_GPU_ACCOUNT); without one the tick refuses to launch
|
|
131
132
|
# attempts on it rather than queue evals that can never run. Bounded at
|
|
132
133
|
# one node's worth: multi-node evals are not a shape the jail supports.
|
|
133
134
|
gpus: int = Field(default=0, ge=0, le=8)
|