outerloop-science 0.1.0.dev2__py3-none-any.whl → 0.1.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +2 -2
- outerloop/attempt.py +310 -93
- outerloop/brief.py +38 -25
- outerloop/cli.py +40 -5
- outerloop/climbboard.py +3 -0
- outerloop/compute.py +148 -53
- outerloop/contract.py +8 -0
- outerloop/dispatch.py +63 -18
- outerloop/evalcache.py +147 -0
- outerloop/followup.py +38 -16
- outerloop/github.py +38 -13
- outerloop/harness.py +1 -18
- outerloop/housekeeping.py +1 -17
- outerloop/image.py +0 -4
- outerloop/init.py +19 -1
- outerloop/intake.py +4 -7
- outerloop/launchlog.py +239 -0
- outerloop/maintain.py +325 -0
- outerloop/maintain_agent_cli.py +81 -0
- outerloop/maintain_post_cli.py +140 -0
- outerloop/measure.py +6 -0
- outerloop/orchestrator.py +141 -31
- outerloop/panel.py +3 -3
- outerloop/review.py +4 -0
- outerloop/review_agent.py +7 -7
- outerloop/review_agent_cli.py +2 -2
- outerloop/review_post_cli.py +2 -2
- outerloop/review_summarize_cli.py +7 -5
- outerloop/roles.py +27 -0
- outerloop/rolespec.py +3 -1
- outerloop/steward.py +5 -5
- outerloop/syscall.py +261 -47
- outerloop/syscall_cli.py +243 -12
- outerloop/tick.py +90 -178
- outerloop/verify_agent.py +8 -6
- outerloop/verify_post_cli.py +2 -2
- outerloop/watcher.py +203 -0
- {outerloop_science-0.1.0.dev2.dist-info → outerloop_science-0.1.0.dev3.dist-info}/METADATA +4 -1
- outerloop_science-0.1.0.dev3.dist-info/RECORD +59 -0
- outerloop_science-0.1.0.dev2.dist-info/RECORD +0 -53
- {outerloop_science-0.1.0.dev2.dist-info → outerloop_science-0.1.0.dev3.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev2.dist-info → outerloop_science-0.1.0.dev3.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev2.dist-info → outerloop_science-0.1.0.dev3.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev2.dist-info → outerloop_science-0.1.0.dev3.dist-info}/licenses/NOTICE +0 -0
outerloop/dispatch.py
CHANGED
|
@@ -36,8 +36,14 @@ from pathlib import Path
|
|
|
36
36
|
from uuid import uuid4
|
|
37
37
|
|
|
38
38
|
from outerloop.compute import JobSpec
|
|
39
|
-
from outerloop.github import
|
|
40
|
-
|
|
39
|
+
from outerloop.github import (
|
|
40
|
+
SAFE_GIT_FLAGS,
|
|
41
|
+
GitError,
|
|
42
|
+
Workspace,
|
|
43
|
+
ensure_regular_git_dir,
|
|
44
|
+
git_identity,
|
|
45
|
+
)
|
|
46
|
+
from outerloop.orchestrator import EvalError, managed_eval_env, metric_from_output
|
|
41
47
|
|
|
42
48
|
log = logging.getLogger(__name__)
|
|
43
49
|
|
|
@@ -94,7 +100,11 @@ class Snapshot:
|
|
|
94
100
|
|
|
95
101
|
|
|
96
102
|
def snapshot_tree(
|
|
97
|
-
ws: Workspace,
|
|
103
|
+
ws: Workspace,
|
|
104
|
+
base_sha: str,
|
|
105
|
+
exclude: tuple[str, ...] = (),
|
|
106
|
+
force: tuple[str, ...] = (),
|
|
107
|
+
author: str = "",
|
|
98
108
|
) -> Snapshot:
|
|
99
109
|
"""Snapshot the workspace's current CONTENT as a commit parented on
|
|
100
110
|
`base_sha`, without touching the working index, and retain it under a
|
|
@@ -105,7 +115,8 @@ def snapshot_tree(
|
|
|
105
115
|
keeps it (docs/design/research-lines.md). `force` adds those paths even
|
|
106
116
|
when the target's ignore rules match them — the notebook seal uses it so
|
|
107
117
|
a .gitignore entry cannot silently discard session memory; callers pass
|
|
108
|
-
only paths that exist.
|
|
118
|
+
only paths that exist. `author` is the bot login the seal commit is
|
|
119
|
+
made as (empty: OUTERLOOP_BOT_LOGIN).
|
|
109
120
|
"""
|
|
110
121
|
# the snapshot writes an index, a tree, a commit, and a ref into this
|
|
111
122
|
# repository: a session-reshaped .git is refused first, like every other
|
|
@@ -163,10 +174,7 @@ def snapshot_tree(
|
|
|
163
174
|
commit = run(
|
|
164
175
|
[
|
|
165
176
|
*git,
|
|
166
|
-
|
|
167
|
-
"user.name=dispatch",
|
|
168
|
-
"-c",
|
|
169
|
-
"user.email=dispatch@localhost",
|
|
177
|
+
*git_identity(author),
|
|
170
178
|
"commit-tree",
|
|
171
179
|
tree,
|
|
172
180
|
"-p",
|
|
@@ -205,6 +213,7 @@ def _filter_neutral_env(base_git: list[str], env: dict[str, str]) -> dict[str, s
|
|
|
205
213
|
capture_output=True,
|
|
206
214
|
text=True,
|
|
207
215
|
timeout=30,
|
|
216
|
+
check=False,
|
|
208
217
|
)
|
|
209
218
|
# -z: NUL-separated records, each "key\nvalue" — so a value containing a
|
|
210
219
|
# newline can never masquerade as a second record.
|
|
@@ -281,9 +290,22 @@ def write_eval_job(
|
|
|
281
290
|
artifacts: tuple[str, ...] = (),
|
|
282
291
|
artifact_max_bytes: int = 0,
|
|
283
292
|
gpus: int = 0,
|
|
293
|
+
array: int = 1,
|
|
294
|
+
seed_cache: Path | None = None,
|
|
284
295
|
) -> Path:
|
|
285
296
|
"""Write the orchestrator-authored job script for one dispatched eval.
|
|
286
297
|
|
|
298
|
+
`seed_cache` names the kernel-warmed seed for this target
|
|
299
|
+
(docs/design/eval-cache.md): the job copies its contents into its own
|
|
300
|
+
scratch cache before `uv` runs — a copy, never a bind or a hardlink, so the
|
|
301
|
+
job's cache is its own and the seed is never written; a missing seed or a
|
|
302
|
+
failed copy costs a download, never the eval.
|
|
303
|
+
|
|
304
|
+
`array` > 1 writes ONE script for a Slurm job array: each task derives its
|
|
305
|
+
own job dir `eval-<name>.<k>` from SLURM_ARRAY_TASK_ID (SWEEP_INDEX under a
|
|
306
|
+
local run) and sees the index as SWEEP_INDEX; the task dirs are created
|
|
307
|
+
and cleared here, the script lives under `eval-<name>/`.
|
|
308
|
+
|
|
287
309
|
`gpus` > 0 adds `--nv` to the jail so the job's allocated GPUs (the
|
|
288
310
|
JobSpec requests them) are visible inside the container; nothing else
|
|
289
311
|
about the containment changes.
|
|
@@ -310,13 +332,16 @@ def write_eval_job(
|
|
|
310
332
|
"""
|
|
311
333
|
ev = run_dir / f"eval-{name}"
|
|
312
334
|
ev.mkdir(parents=True, exist_ok=True)
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
318
|
-
|
|
319
|
-
|
|
335
|
+
task_dirs = [run_dir / f"eval-{name}.{k}" for k in range(array)] if array > 1 else [ev]
|
|
336
|
+
for task_dir in task_dirs:
|
|
337
|
+
task_dir.mkdir(parents=True, exist_ok=True)
|
|
338
|
+
# a resubmitted eval must never be read as its predecessor: every prior
|
|
339
|
+
# artifact — including a leftover extracted tree — goes before submission
|
|
340
|
+
for stale in ("exit-code", "stdout", "stderr", "setup.log", "submitted", "artifacts.log"):
|
|
341
|
+
(task_dir / stale).unlink(missing_ok=True)
|
|
342
|
+
shutil.rmtree(task_dir / "tree", ignore_errors=True)
|
|
343
|
+
shutil.rmtree(task_dir / "artifacts", ignore_errors=True)
|
|
344
|
+
(task_dir / "command.txt").write_text(command)
|
|
320
345
|
# extra_env matches the in-job evaluator's contract: managed keys (HOME,
|
|
321
346
|
# UV_*, PATH...) are DROPPED, never allowed to override the isolation, and
|
|
322
347
|
# keys must be shell-identifier shaped (they are exported unquoted).
|
|
@@ -330,10 +355,20 @@ def write_eval_job(
|
|
|
330
355
|
# as the snapshot, injected as GIT_CONFIG_* env (robust to '=' in a driver
|
|
331
356
|
# name, unlike -c) so a smudge filter cannot execute during checkout
|
|
332
357
|
neutral = _filter_neutral_env(["git", "-C", str(repo_root), *SAFE_GIT_FLAGS], {})
|
|
358
|
+
if array > 1:
|
|
359
|
+
# the task picks its own job dir; under Slurm the index is the array
|
|
360
|
+
# task id, under a local run the caller exports SWEEP_INDEX
|
|
361
|
+
ev_lines = [
|
|
362
|
+
'TASK="${SLURM_ARRAY_TASK_ID:-${SWEEP_INDEX:-0}}"',
|
|
363
|
+
f'EV={shlex.quote(str(ev))}."$TASK"',
|
|
364
|
+
'export SWEEP_INDEX="$TASK" APPTAINERENV_SWEEP_INDEX="$TASK"',
|
|
365
|
+
]
|
|
366
|
+
else:
|
|
367
|
+
ev_lines = [f"EV={shlex.quote(str(ev))}"]
|
|
333
368
|
lines = [
|
|
334
369
|
"#!/bin/sh",
|
|
335
370
|
"set -u",
|
|
336
|
-
|
|
371
|
+
*ev_lines,
|
|
337
372
|
f"REPO={shlex.quote(str(repo_root))}",
|
|
338
373
|
# the extracted tree lives on NODE-LOCAL scratch, not the shared run
|
|
339
374
|
# dir: it dies with the job (nothing to reap on the shared FS), and
|
|
@@ -361,6 +396,11 @@ def write_eval_job(
|
|
|
361
396
|
]
|
|
362
397
|
for k, v in neutral.items():
|
|
363
398
|
lines.append(f"export {k}={shlex.quote(v)}")
|
|
399
|
+
if seed_cache is not None:
|
|
400
|
+
q = shlex.quote(str(seed_cache))
|
|
401
|
+
lines.append(
|
|
402
|
+
f'if [ -d {q} ]; then cp -a {q}/. "$SCRATCH/cache"/ 2>> "$EV/setup.log" || true; fi'
|
|
403
|
+
)
|
|
364
404
|
lines += [
|
|
365
405
|
# Materialize the snapshot by CHECKOUT, not `git archive`: a checkout
|
|
366
406
|
# reproduces content faithfully — INCLUDING .gitattributes — and does
|
|
@@ -478,6 +518,8 @@ def eval_job_spec(
|
|
|
478
518
|
cpus: int = 4,
|
|
479
519
|
mem: str = "8G",
|
|
480
520
|
gpus: int = 0,
|
|
521
|
+
nice: int = 0,
|
|
522
|
+
array: str = "",
|
|
481
523
|
) -> JobSpec:
|
|
482
524
|
"""The JobSpec for one dispatched eval: the hint CLAMPED to our ceiling
|
|
483
525
|
plus setup slack — a contract value above EVAL_JOB_MINUTES_CEILING must
|
|
@@ -486,7 +528,8 @@ def eval_job_spec(
|
|
|
486
528
|
the GPU lane (DispatchSettings.placement) when it is nonzero, and the
|
|
487
529
|
job is sized for it: a GPU eval gets at least EVAL_CPUS_PER_GPU cores
|
|
488
530
|
and EVAL_MEM_GB_PER_GPU GB per GPU (a training eval's data loading and
|
|
489
|
-
torch.compile workers do not fit the CPU eval's 4 cores / 8 GB).
|
|
531
|
+
torch.compile workers do not fit the CPU eval's 4 cores / 8 GB). `nice`
|
|
532
|
+
lowers the job's priority below the kernel's evals (the launcher sets it)."""
|
|
490
533
|
if gpus > 0:
|
|
491
534
|
cpus = max(cpus, EVAL_CPUS_PER_GPU * gpus)
|
|
492
535
|
given = _mem_gb(mem)
|
|
@@ -503,6 +546,8 @@ def eval_job_spec(
|
|
|
503
546
|
cpus=cpus,
|
|
504
547
|
mem=mem,
|
|
505
548
|
gpus=gpus,
|
|
549
|
+
nice=nice,
|
|
550
|
+
array=array,
|
|
506
551
|
)
|
|
507
552
|
|
|
508
553
|
|
|
@@ -541,7 +586,7 @@ def read_eval_result(run_dir: Path, name: str, metric: str) -> float:
|
|
|
541
586
|
with contextlib.suppress(OSError, ValueError):
|
|
542
587
|
tail = (ev / "stderr").read_text(errors="replace")[-300:]
|
|
543
588
|
raise EvalError(f"dispatched eval {name} failed ({code}): {tail}")
|
|
544
|
-
value =
|
|
589
|
+
value = metric_from_output(stdout, metric)
|
|
545
590
|
if value is None:
|
|
546
591
|
raise EvalError(f"dispatched eval {name}: no readable {metric!r} in output")
|
|
547
592
|
if not math.isfinite(value):
|
outerloop/evalcache.py
ADDED
|
@@ -0,0 +1,147 @@
|
|
|
1
|
+
"""The seed cache: one uv cache per target, warmed by the kernel on the tick
|
|
2
|
+
host and copied into each job's own scratch cache before `uv` runs there
|
|
3
|
+
(docs/design/eval-cache.md). Jobs never write shared state: the warmer
|
|
4
|
+
downloads and unpacks wheels only (`--no-build`, so no build backend of the
|
|
5
|
+
target's ever runs here), and a job's copy is its own from the first byte."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import shutil
|
|
13
|
+
import subprocess
|
|
14
|
+
import tempfile
|
|
15
|
+
from collections.abc import Callable
|
|
16
|
+
from pathlib import Path
|
|
17
|
+
from typing import Any
|
|
18
|
+
|
|
19
|
+
log = logging.getLogger(__name__)
|
|
20
|
+
|
|
21
|
+
SEED_DIR = "eval-cache"
|
|
22
|
+
LOCK_HASH = ".lockfile-sha256"
|
|
23
|
+
# what uv sync needs to resolve a locked project without its sources
|
|
24
|
+
WARM_FILES = ("pyproject.toml", "uv.lock")
|
|
25
|
+
# the interpreter the eval jobs run under (the agent image's); wheels are
|
|
26
|
+
# picked for it, and the tick host's uv fetches it when it has none
|
|
27
|
+
EVAL_PYTHON = "3.12"
|
|
28
|
+
WARM_TIMEOUT_S = 30 * 60
|
|
29
|
+
# what uv needs to download wheels and nothing more: the tick host's
|
|
30
|
+
# environment holds credentials (token providers, key paths) that an
|
|
31
|
+
# uncontained download process has no business seeing
|
|
32
|
+
WARM_ENV_KEYS = (
|
|
33
|
+
"PATH",
|
|
34
|
+
"HOME",
|
|
35
|
+
"LANG",
|
|
36
|
+
"LC_ALL",
|
|
37
|
+
"TMPDIR",
|
|
38
|
+
"HTTP_PROXY",
|
|
39
|
+
"HTTPS_PROXY",
|
|
40
|
+
"NO_PROXY",
|
|
41
|
+
"http_proxy",
|
|
42
|
+
"https_proxy",
|
|
43
|
+
"no_proxy",
|
|
44
|
+
"SSL_CERT_FILE",
|
|
45
|
+
"SSL_CERT_DIR",
|
|
46
|
+
"REQUESTS_CA_BUNDLE",
|
|
47
|
+
)
|
|
48
|
+
|
|
49
|
+
Runner = Callable[..., Any]
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def seed_dir(root: Path, target: str) -> Path:
|
|
53
|
+
"""Where a target's seed lives under the state root."""
|
|
54
|
+
return root / SEED_DIR / target.replace("/", "__")
|
|
55
|
+
|
|
56
|
+
|
|
57
|
+
def lockfile_hash(files: dict[str, str]) -> str:
|
|
58
|
+
h = hashlib.sha256()
|
|
59
|
+
for name in sorted(files):
|
|
60
|
+
h.update(name.encode())
|
|
61
|
+
h.update(b"\0")
|
|
62
|
+
h.update(files[name].encode())
|
|
63
|
+
h.update(b"\0")
|
|
64
|
+
return h.hexdigest()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def warm(
|
|
68
|
+
root: Path,
|
|
69
|
+
target: str,
|
|
70
|
+
github: Any,
|
|
71
|
+
ref: str,
|
|
72
|
+
*,
|
|
73
|
+
runner: Runner = subprocess.run,
|
|
74
|
+
uv: str | None = None,
|
|
75
|
+
python: str = EVAL_PYTHON,
|
|
76
|
+
) -> str:
|
|
77
|
+
"""Warm the target's seed from its lockfile at `ref`, when the lockfile
|
|
78
|
+
changed since the last warm. Fetches `pyproject.toml` and `uv.lock` only
|
|
79
|
+
(never the sources), and runs `uv sync --frozen --no-install-project
|
|
80
|
+
--no-build --all-extras` into a throwaway environment with the seed as
|
|
81
|
+
uv's cache. Returns one word on what happened, for the tick's report:
|
|
82
|
+
"warmed", "unchanged", or "skipped: ..." / "failed: ..." with the reason.
|
|
83
|
+
A failure leaves the seed as it was and the hash unrecorded, so the next
|
|
84
|
+
tick tries again."""
|
|
85
|
+
files: dict[str, str] = {}
|
|
86
|
+
for name in WARM_FILES:
|
|
87
|
+
text = github.get_file_content(target, name, ref)
|
|
88
|
+
if text is None:
|
|
89
|
+
return f"skipped: no {name} at {ref}"
|
|
90
|
+
files[name] = text
|
|
91
|
+
digest = lockfile_hash(files)
|
|
92
|
+
seed = seed_dir(root, target)
|
|
93
|
+
marker = seed / LOCK_HASH
|
|
94
|
+
try:
|
|
95
|
+
if marker.read_text().strip() == digest:
|
|
96
|
+
return "unchanged"
|
|
97
|
+
except OSError:
|
|
98
|
+
pass
|
|
99
|
+
uv = uv or shutil.which("uv")
|
|
100
|
+
if not uv:
|
|
101
|
+
return "skipped: uv is not on the tick host's PATH"
|
|
102
|
+
# warm into a fresh directory beside the seed and swap it in whole: the
|
|
103
|
+
# seed then holds exactly this lockfile's wheels (nothing accumulates
|
|
104
|
+
# across lockfile changes), and a job copying mid-swap fails its
|
|
105
|
+
# best-effort copy rather than seeing a half-written cache
|
|
106
|
+
seed.parent.mkdir(parents=True, exist_ok=True)
|
|
107
|
+
fresh = seed.parent / f".{seed.name}.warm-{digest[:12]}"
|
|
108
|
+
shutil.rmtree(fresh, ignore_errors=True)
|
|
109
|
+
fresh.mkdir()
|
|
110
|
+
with tempfile.TemporaryDirectory(prefix="outerloop-warm-") as tmp:
|
|
111
|
+
for name, text in files.items():
|
|
112
|
+
(Path(tmp) / name).write_text(text)
|
|
113
|
+
env = {k: os.environ[k] for k in WARM_ENV_KEYS if k in os.environ}
|
|
114
|
+
env.update(
|
|
115
|
+
UV_CACHE_DIR=str(fresh),
|
|
116
|
+
UV_PROJECT_ENVIRONMENT=str(Path(tmp) / ".venv"),
|
|
117
|
+
UV_LINK_MODE="copy",
|
|
118
|
+
)
|
|
119
|
+
argv = [
|
|
120
|
+
uv,
|
|
121
|
+
"sync",
|
|
122
|
+
"--frozen",
|
|
123
|
+
"--no-install-project",
|
|
124
|
+
"--no-build",
|
|
125
|
+
"--all-extras",
|
|
126
|
+
"--python",
|
|
127
|
+
python,
|
|
128
|
+
]
|
|
129
|
+
try:
|
|
130
|
+
proc = runner(
|
|
131
|
+
argv, cwd=tmp, env=env, capture_output=True, text=True, timeout=WARM_TIMEOUT_S
|
|
132
|
+
)
|
|
133
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
134
|
+
shutil.rmtree(fresh, ignore_errors=True)
|
|
135
|
+
return f"failed: {type(exc).__name__}: {exc}"
|
|
136
|
+
if proc.returncode != 0:
|
|
137
|
+
shutil.rmtree(fresh, ignore_errors=True)
|
|
138
|
+
tail = (proc.stderr or "").strip().splitlines()[-3:]
|
|
139
|
+
return f"failed: uv sync exited {proc.returncode}: {' | '.join(tail)}"
|
|
140
|
+
(fresh / LOCK_HASH).write_text(digest)
|
|
141
|
+
old = seed.parent / f".{seed.name}.old"
|
|
142
|
+
shutil.rmtree(old, ignore_errors=True)
|
|
143
|
+
if seed.exists():
|
|
144
|
+
os.replace(seed, old)
|
|
145
|
+
os.replace(fresh, seed)
|
|
146
|
+
shutil.rmtree(old, ignore_errors=True)
|
|
147
|
+
return "warmed"
|
outerloop/followup.py
CHANGED
|
@@ -35,6 +35,7 @@ from outerloop.github import (
|
|
|
35
35
|
Workspace,
|
|
36
36
|
bot_login_from_env,
|
|
37
37
|
contract_at,
|
|
38
|
+
git_identity,
|
|
38
39
|
is_own_login,
|
|
39
40
|
)
|
|
40
41
|
from outerloop.harness import Harness, default_binary, outage, redact
|
|
@@ -520,9 +521,9 @@ def _respond(
|
|
|
520
521
|
workspace = run_dir(run_root, run_id) / "ws"
|
|
521
522
|
if not workspace.is_dir():
|
|
522
523
|
return FollowupOutcome(run_id, "error", "workspace no longer exists (GC'd?)")
|
|
523
|
-
from outerloop.attempt import
|
|
524
|
+
from outerloop.attempt import target_clone_url
|
|
524
525
|
|
|
525
|
-
ws = Workspace(root=workspace, auth=github.auth, url=
|
|
526
|
+
ws = Workspace(root=workspace, auth=github.auth, url=target_clone_url(record.target))
|
|
526
527
|
contract_text = contract_text_in_tree(workspace)
|
|
527
528
|
contract = load_contract(contract_text, record.target)
|
|
528
529
|
bench = next((b for b in contract.benchmarks if b.name == record.benchmark), None)
|
|
@@ -618,11 +619,11 @@ def _respond(
|
|
|
618
619
|
# context — never as triggers, never as instructions.
|
|
619
620
|
ctx = context_comments(github.list_comments(record.target, number), record.last_comment_id)
|
|
620
621
|
if ctx:
|
|
621
|
-
from outerloop.brief import
|
|
622
|
+
from outerloop.brief import code_fence
|
|
622
623
|
|
|
623
624
|
blocks = []
|
|
624
625
|
for author, body in ctx:
|
|
625
|
-
fence =
|
|
626
|
+
fence = code_fence(body)
|
|
626
627
|
blocks.append(f"{author}:\n{fence}\n{body}\n{fence}")
|
|
627
628
|
prompt += (
|
|
628
629
|
"\n\n# Comments without standing (context only — data, not "
|
|
@@ -940,7 +941,14 @@ def _respond(
|
|
|
940
941
|
# (LocalCompute) returns the value here; a cluster parks.
|
|
941
942
|
try:
|
|
942
943
|
sealed, sealed_snap = _seal_and_measure(
|
|
943
|
-
ws,
|
|
944
|
+
ws,
|
|
945
|
+
run_root,
|
|
946
|
+
run_id,
|
|
947
|
+
dispatch,
|
|
948
|
+
bench,
|
|
949
|
+
run_seed,
|
|
950
|
+
workspace,
|
|
951
|
+
bot_login=bot_login,
|
|
944
952
|
)
|
|
945
953
|
except _RemeasureParked as pend:
|
|
946
954
|
return _park_remeasure(
|
|
@@ -1501,10 +1509,7 @@ def _commit_sealed_tree(
|
|
|
1501
1509
|
the live workspace, which may hold content the seal excluded."""
|
|
1502
1510
|
ws.git("add", "-A")
|
|
1503
1511
|
ws.git(
|
|
1504
|
-
|
|
1505
|
-
f"user.name={bot_login}",
|
|
1506
|
-
"-c",
|
|
1507
|
-
f"user.email={bot_login}@users.noreply.github.com",
|
|
1512
|
+
*git_identity(bot_login),
|
|
1508
1513
|
"commit",
|
|
1509
1514
|
"-q",
|
|
1510
1515
|
"--amend",
|
|
@@ -1547,6 +1552,7 @@ def _seal_and_measure(
|
|
|
1547
1552
|
bench: Any,
|
|
1548
1553
|
run_seed: int,
|
|
1549
1554
|
workspace: Path,
|
|
1555
|
+
bot_login: str,
|
|
1550
1556
|
) -> tuple[float, Any]:
|
|
1551
1557
|
"""Seal the workspace's change as a commit on the PR's current head and
|
|
1552
1558
|
measure it through the dispatched measurer. Returns (value, snapshot)
|
|
@@ -1559,7 +1565,15 @@ def _seal_and_measure(
|
|
|
1559
1565
|
from outerloop.measure import MeasurementPending
|
|
1560
1566
|
|
|
1561
1567
|
parent = ws.git("rev-parse", "HEAD").strip()
|
|
1562
|
-
|
|
1568
|
+
from outerloop.attempt import with_seed
|
|
1569
|
+
from outerloop.runstate import load_record
|
|
1570
|
+
|
|
1571
|
+
# the follow-up CLI carries the run id, not the target: the seed comes
|
|
1572
|
+
# from the record
|
|
1573
|
+
dispatch = with_seed(dispatch, run_root, load_record(run_root, run_id).target)
|
|
1574
|
+
snap = snapshot_tree(
|
|
1575
|
+
ws, parent, exclude=LINE_MEMORY_PATHS if bench.lines else (), author=bot_login
|
|
1576
|
+
)
|
|
1563
1577
|
measurer = dispatch.measurer(
|
|
1564
1578
|
run_dir(run_root, run_id),
|
|
1565
1579
|
repo_root=workspace,
|
|
@@ -1624,9 +1638,11 @@ def _park_remeasure(
|
|
|
1624
1638
|
"eval_minutes": int(bench.eval_minutes or 0),
|
|
1625
1639
|
}
|
|
1626
1640
|
note = (
|
|
1627
|
-
"\n\n_(
|
|
1628
|
-
f"({len(pend.job_ids)} job(s))
|
|
1629
|
-
"
|
|
1641
|
+
"\n\n_(Not pushed yet: the change above is local so far. The changed tree is "
|
|
1642
|
+
f"being re-measured on the GPU lane first ({len(pend.job_ids)} job(s) pending), "
|
|
1643
|
+
"and if the result can be applied the branch is updated with it and its number — "
|
|
1644
|
+
"until then GitHub still shows the previous head, including any conflict. "
|
|
1645
|
+
"Comments posted meanwhile are answered after that.)_"
|
|
1630
1646
|
)
|
|
1631
1647
|
stage["reply_note"] = note
|
|
1632
1648
|
stage["reply_posted"] = False
|
|
@@ -1692,9 +1708,9 @@ def _resume_measure(
|
|
|
1692
1708
|
"error",
|
|
1693
1709
|
"a dispatched re-measure is parked but no cluster coordinates were given",
|
|
1694
1710
|
)
|
|
1695
|
-
from outerloop.attempt import
|
|
1711
|
+
from outerloop.attempt import target_clone_url
|
|
1696
1712
|
|
|
1697
|
-
ws = Workspace(root=workspace, auth=github.auth, url=
|
|
1713
|
+
ws = Workspace(root=workspace, auth=github.auth, url=target_clone_url(record.target))
|
|
1698
1714
|
# the measurement's contract is the SEALED tree's — what was actually
|
|
1699
1715
|
# measured — never the live workspace file, which can change during the
|
|
1700
1716
|
# wait (terra #241 r1)
|
|
@@ -1986,7 +2002,11 @@ def main() -> int:
|
|
|
1986
2002
|
# the tick passes the effective limit explicitly; this fallback follows
|
|
1987
2003
|
# the harness ceiling so a bare CLI run is never silently starved
|
|
1988
2004
|
parser.add_argument("--max-turns", type=int, default=DEFAULT_MAX_TURNS)
|
|
1989
|
-
parser.add_argument(
|
|
2005
|
+
parser.add_argument(
|
|
2006
|
+
"--bot-login",
|
|
2007
|
+
default=bot_login_from_env(),
|
|
2008
|
+
help="the login the kernel posts as (OUTERLOOP_BOT_LOGIN); required",
|
|
2009
|
+
)
|
|
1990
2010
|
parser.add_argument(
|
|
1991
2011
|
"--job-minutes",
|
|
1992
2012
|
type=int,
|
|
@@ -2025,6 +2045,8 @@ def main() -> int:
|
|
|
2025
2045
|
"the read is skipped and said so; the author's budget is never the panel's)",
|
|
2026
2046
|
)
|
|
2027
2047
|
args = parser.parse_args()
|
|
2048
|
+
if not args.bot_login.strip():
|
|
2049
|
+
parser.error("--bot-login / OUTERLOOP_BOT_LOGIN is required (no default identity, #298)")
|
|
2028
2050
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
2029
2051
|
if not args.image and not args.uncontained:
|
|
2030
2052
|
parser.error("--image is required (or pass --uncontained explicitly, dev only)")
|
outerloop/github.py
CHANGED
|
@@ -37,18 +37,29 @@ from typing import Any, Protocol
|
|
|
37
37
|
from outerloop.contract import CONTRACT_NAMES, find_contract
|
|
38
38
|
from outerloop.markers import legacy_marker, marker
|
|
39
39
|
|
|
40
|
-
DEFAULT_BOT_LOGIN = "agentic-learning-bot"
|
|
41
40
|
|
|
42
|
-
|
|
43
|
-
def bot_login_from_env(default: str = DEFAULT_BOT_LOGIN) -> str:
|
|
41
|
+
def bot_login_from_env(default: str = "") -> str:
|
|
44
42
|
"""The login the kernel posts and pushes as — the bot ACCOUNT under a PAT,
|
|
45
43
|
the App's `<slug>[bot]` under App auth (docs/design/github-app-auth.md).
|
|
46
44
|
`OUTERLOOP_BOT_LOGIN` sets it for every role at once (the tick exports
|
|
47
45
|
it, jobs inherit it); every own-comment filter, alarm-issue lookup and
|
|
48
|
-
intake-claim scan keys on this string, so it must follow the credential.
|
|
46
|
+
intake-claim scan keys on this string, so it must follow the credential.
|
|
47
|
+
There is no built-in default (#298): a wrong identity hides the kernel's
|
|
48
|
+
own PRs from it, so an unset login fails closed — the tick refuses to
|
|
49
|
+
service a target without one, and `init` records it on both auth paths."""
|
|
49
50
|
return os.environ.get("OUTERLOOP_BOT_LOGIN", "").strip() or default
|
|
50
51
|
|
|
51
52
|
|
|
53
|
+
def git_identity(login: str = "") -> tuple[str, ...]:
|
|
54
|
+
"""`git -c` arguments that make a commit the kernel's: the bot login and
|
|
55
|
+
its GitHub noreply address, so GitHub attributes the commit to the App or
|
|
56
|
+
bot account like every other kernel write. One identity for every kernel
|
|
57
|
+
commit — seals, line merges, ledger commits alike. An empty login takes
|
|
58
|
+
OUTERLOOP_BOT_LOGIN."""
|
|
59
|
+
login = login or bot_login_from_env()
|
|
60
|
+
return ("-c", f"user.name={login}", "-c", f"user.email={login}@users.noreply.github.com")
|
|
61
|
+
|
|
62
|
+
|
|
52
63
|
def bot_aliases_from_env() -> tuple[str, ...]:
|
|
53
64
|
"""Former logins the kernel posted as (comma-separated
|
|
54
65
|
`OUTERLOOP_BOT_ALIASES`). Every issue, claim, alarm and PR created
|
|
@@ -271,9 +282,13 @@ class GitHubClient:
|
|
|
271
282
|
path = f"/repos/{urllib.parse.quote(repo)}/issues/{number}"
|
|
272
283
|
return self._expect_dict(self._request("GET", path), path)
|
|
273
284
|
|
|
274
|
-
def list_open_issues(
|
|
275
|
-
|
|
276
|
-
|
|
285
|
+
def list_open_issues(
|
|
286
|
+
self, repo: str, max_pages: int = 3, creator: str = ""
|
|
287
|
+
) -> list[dict[str, Any]]:
|
|
288
|
+
"""Open issues (PRs excluded — the issues API mixes them in); `creator`
|
|
289
|
+
narrows them to one author's, server-side."""
|
|
290
|
+
query = urllib.parse.urlencode({"creator": creator}) if creator else ""
|
|
291
|
+
items = self._paginate(f"/repos/{urllib.parse.quote(repo)}/issues", max_pages, query)
|
|
277
292
|
return [i for i in items if "pull_request" not in i]
|
|
278
293
|
|
|
279
294
|
def create_pull(
|
|
@@ -676,10 +691,11 @@ class GitHubClient:
|
|
|
676
691
|
f"/repos/{urllib.parse.quote(repo)}/pulls/{number}/comments", max_pages
|
|
677
692
|
)
|
|
678
693
|
|
|
679
|
-
def _paginate(self, base_path: str, max_pages: int) -> list[dict[str, Any]]:
|
|
694
|
+
def _paginate(self, base_path: str, max_pages: int, query: str = "") -> list[dict[str, Any]]:
|
|
680
695
|
items: list[dict[str, Any]] = []
|
|
696
|
+
extra = f"&{query}" if query else ""
|
|
681
697
|
for page in range(1, max_pages + 1):
|
|
682
|
-
data = self._request("GET", f"{base_path}?per_page=100&page={page}")
|
|
698
|
+
data = self._request("GET", f"{base_path}?per_page=100&page={page}{extra}")
|
|
683
699
|
if not isinstance(data, list) or not data:
|
|
684
700
|
break
|
|
685
701
|
items.extend(item for item in data if isinstance(item, dict))
|
|
@@ -752,6 +768,17 @@ class GitHubClient:
|
|
|
752
768
|
)
|
|
753
769
|
return int(data.get("number", 0)) if isinstance(data, dict) else 0
|
|
754
770
|
|
|
771
|
+
def update_issue(self, repo: str, number: int, body: str) -> None:
|
|
772
|
+
"""Replace an issue's body (the rolling maintenance digest)."""
|
|
773
|
+
if self.dry_run:
|
|
774
|
+
log.info("[dry-run] update issue %s#%s (%d chars)", repo, number, len(body))
|
|
775
|
+
return
|
|
776
|
+
self._request(
|
|
777
|
+
"PATCH",
|
|
778
|
+
f"/repos/{urllib.parse.quote(repo)}/issues/{number}",
|
|
779
|
+
{"body": body},
|
|
780
|
+
)
|
|
781
|
+
|
|
755
782
|
def close_issue(self, repo: str, number: int) -> None:
|
|
756
783
|
if self.dry_run:
|
|
757
784
|
log.info("[dry-run] close issue %s#%s", repo, number)
|
|
@@ -1282,6 +1309,7 @@ def _filter_override_pairs(root: Path | None) -> list[tuple[str, str]]:
|
|
|
1282
1309
|
},
|
|
1283
1310
|
capture_output=True,
|
|
1284
1311
|
timeout=30,
|
|
1312
|
+
check=False,
|
|
1285
1313
|
)
|
|
1286
1314
|
# BYTES + surrogateescape, never text=True: the config is session-written,
|
|
1287
1315
|
# so a non-UTF-8 byte in a value must not crash the git call — and the
|
|
@@ -1429,10 +1457,7 @@ class Workspace:
|
|
|
1429
1457
|
self.git("reset")
|
|
1430
1458
|
raise ForbiddenPathError(f"commit touches forbidden paths: {sorted(violations)}")
|
|
1431
1459
|
self.git(
|
|
1432
|
-
|
|
1433
|
-
f"user.name={author}",
|
|
1434
|
-
"-c",
|
|
1435
|
-
f"user.email={author}@users.noreply.github.com",
|
|
1460
|
+
*git_identity(author),
|
|
1436
1461
|
"commit",
|
|
1437
1462
|
"-m",
|
|
1438
1463
|
message,
|
outerloop/harness.py
CHANGED
|
@@ -952,6 +952,7 @@ class CodexHarness:
|
|
|
952
952
|
capture_output=True,
|
|
953
953
|
text=True,
|
|
954
954
|
timeout=60,
|
|
955
|
+
check=False,
|
|
955
956
|
)
|
|
956
957
|
except OSError as exc:
|
|
957
958
|
log.warning("could not spawn codex login: %s", exc)
|
|
@@ -1440,21 +1441,3 @@ class HermesHarness:
|
|
|
1440
1441
|
prior_turns.append({"role": "assistant", "text": redact(result.final_text, secrets)})
|
|
1441
1442
|
_save_resume_transcript(session_home, session_id, prior_turns)
|
|
1442
1443
|
return replace(result, session_id=session_id)
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
@dataclass
|
|
1446
|
-
class FakeHarness:
|
|
1447
|
-
"""Deterministic in-process harness for tests and dry runs."""
|
|
1448
|
-
|
|
1449
|
-
result: SessionResult
|
|
1450
|
-
script: Any = None # optional callable(brief_text, workspace) for side effects
|
|
1451
|
-
calls: list[tuple[str, str, str | None]] = field(default_factory=list)
|
|
1452
|
-
supports_resume: bool = True # a field so tests can exercise the no-resume path
|
|
1453
|
-
|
|
1454
|
-
def run(
|
|
1455
|
-
self, brief_text: str, workspace: Path, resume_session_id: str | None = None
|
|
1456
|
-
) -> SessionResult:
|
|
1457
|
-
self.calls.append((brief_text, str(workspace), resume_session_id))
|
|
1458
|
-
if self.script is not None:
|
|
1459
|
-
self.script(brief_text, workspace)
|
|
1460
|
-
return self.result
|
outerloop/housekeeping.py
CHANGED
|
@@ -30,7 +30,7 @@ import time
|
|
|
30
30
|
from dataclasses import replace
|
|
31
31
|
from pathlib import Path
|
|
32
32
|
|
|
33
|
-
from outerloop.runstate import ENDED, RunRecord,
|
|
33
|
+
from outerloop.runstate import ENDED, RunRecord, load_record, run_dir, save_record
|
|
34
34
|
|
|
35
35
|
log = logging.getLogger(__name__)
|
|
36
36
|
|
|
@@ -38,22 +38,6 @@ WORKSPACE_DIRS = ("ws", "ws-home")
|
|
|
38
38
|
DEFAULT_SHED_GRACE_S = 24 * 3600.0
|
|
39
39
|
|
|
40
40
|
|
|
41
|
-
def shed_candidates(root: Path, now: float, grace_s: float, force: bool) -> list[RunRecord]:
|
|
42
|
-
"""Ended runs whose workspaces may go now, oldest ending first. With
|
|
43
|
-
`force` (the disk is failing) the grace period does not apply."""
|
|
44
|
-
due: list[RunRecord] = []
|
|
45
|
-
for record in list_runs(root):
|
|
46
|
-
if record.state != ENDED or record.workspace_shed:
|
|
47
|
-
continue
|
|
48
|
-
if not force and now - record.updated < grace_s:
|
|
49
|
-
continue
|
|
50
|
-
if not any((run_dir(root, record.run_id) / d).exists() for d in WORKSPACE_DIRS):
|
|
51
|
-
continue
|
|
52
|
-
due.append(record)
|
|
53
|
-
due.sort(key=lambda r: r.updated)
|
|
54
|
-
return due
|
|
55
|
-
|
|
56
|
-
|
|
57
41
|
def shed_workspace(root: Path, record: RunRecord, now: float) -> bool:
|
|
58
42
|
"""Remove the run's `ws` and `ws-home` directories and stamp the record.
|
|
59
43
|
Returns False (and removes nothing) when either is a symlink."""
|
outerloop/image.py
CHANGED
|
@@ -190,10 +190,6 @@ def containment_check(*, runner: Callable[..., Any] = subprocess.run) -> str:
|
|
|
190
190
|
return ""
|
|
191
191
|
|
|
192
192
|
|
|
193
|
-
def containment_available() -> bool:
|
|
194
|
-
return containment_check() == ""
|
|
195
|
-
|
|
196
|
-
|
|
197
193
|
def _fmt_mb(n: int) -> str:
|
|
198
194
|
return f"{n / (1 << 20):,.0f} MB"
|
|
199
195
|
|