outerloop-science 0.1.0.dev3__py3-none-any.whl → 0.1.1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +1 -1
- outerloop/appauth.py +17 -0
- outerloop/attempt.py +100 -15
- outerloop/cli.py +64 -1
- outerloop/climbboard.py +64 -22
- outerloop/followup.py +2 -9
- outerloop/github.py +32 -12
- outerloop/harness.py +21 -29
- outerloop/init.py +26 -1
- outerloop/maintain.py +29 -1
- outerloop/maintain_post_cli.py +4 -4
- outerloop/review_summarize_cli.py +1 -1
- outerloop/steward.py +2 -9
- outerloop/tick.py +184 -135
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.1.dist-info}/METADATA +7 -2
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.1.dist-info}/RECORD +20 -20
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.1.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.1.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.1.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.1.dist-info}/licenses/NOTICE +0 -0
outerloop/__init__.py
CHANGED
outerloop/appauth.py
CHANGED
|
@@ -17,8 +17,10 @@ network; the production signer lives behind the `app-auth` extra.
|
|
|
17
17
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import argparse
|
|
20
21
|
import base64
|
|
21
22
|
import json
|
|
23
|
+
import os
|
|
22
24
|
import time
|
|
23
25
|
import urllib.error
|
|
24
26
|
import urllib.parse
|
|
@@ -29,6 +31,7 @@ from pathlib import Path
|
|
|
29
31
|
from typing import Any
|
|
30
32
|
|
|
31
33
|
from outerloop.github import AUTH_SAFE_OPENER, FileTokenProvider, TokenProvider
|
|
34
|
+
from outerloop.paths import CONFIG_DIR
|
|
32
35
|
|
|
33
36
|
# RS256-sign the JWT signing input, returning the raw signature bytes.
|
|
34
37
|
Signer = Callable[[bytes], bytes]
|
|
@@ -211,3 +214,17 @@ def resolve_bot_auth(pat_file: str | Path, app_file: str | Path = "") -> TokenPr
|
|
|
211
214
|
if str(app_file).strip():
|
|
212
215
|
return app_provider_from_file(Path(str(app_file)).expanduser())
|
|
213
216
|
return FileTokenProvider(Path(str(pat_file)).expanduser())
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def add_credential_args(parser: argparse.ArgumentParser) -> None:
|
|
220
|
+
"""Add the shared bot-auth options to a role parser: `--pat-file`, and
|
|
221
|
+
`--github-app-file`, which supplies installation tokens instead of the PAT
|
|
222
|
+
when set. `resolve_bot_auth` reads the pair. One owner so the defaults and
|
|
223
|
+
help cannot drift between attempt, followup, and steward."""
|
|
224
|
+
parser.add_argument("--pat-file", default=str(CONFIG_DIR / "bot_pat"))
|
|
225
|
+
parser.add_argument(
|
|
226
|
+
"--github-app-file",
|
|
227
|
+
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
228
|
+
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
229
|
+
"when set, installation tokens replace the PAT",
|
|
230
|
+
)
|
outerloop/attempt.py
CHANGED
|
@@ -19,6 +19,7 @@ import os
|
|
|
19
19
|
import re
|
|
20
20
|
import shutil
|
|
21
21
|
import time
|
|
22
|
+
import traceback
|
|
22
23
|
from collections.abc import Callable, Iterable
|
|
23
24
|
from dataclasses import dataclass
|
|
24
25
|
from dataclasses import replace as dc_replace
|
|
@@ -26,7 +27,7 @@ from functools import partial
|
|
|
26
27
|
from pathlib import Path
|
|
27
28
|
from typing import Any, cast
|
|
28
29
|
|
|
29
|
-
from outerloop.appauth import resolve_bot_auth
|
|
30
|
+
from outerloop.appauth import add_credential_args, resolve_bot_auth
|
|
30
31
|
from outerloop.brief import BudgetState, distill_lessons
|
|
31
32
|
from outerloop.compute import LocalCompute
|
|
32
33
|
from outerloop.contract import Benchmark, Contract, contract_text_in_tree, load_contract
|
|
@@ -319,18 +320,35 @@ class AttemptOutcome:
|
|
|
319
320
|
report_path: str = ""
|
|
320
321
|
|
|
321
322
|
|
|
323
|
+
def _attach_run_log(directory: Path) -> None:
|
|
324
|
+
"""Also write the kernel's log lines to `<run dir>/kernel.log`: the scheduler
|
|
325
|
+
discards a wake job's own stderr, so this is the only trace a swallowed
|
|
326
|
+
step leaves. Only for a run that exists — never a stray directory."""
|
|
327
|
+
if not directory.is_dir():
|
|
328
|
+
return
|
|
329
|
+
try:
|
|
330
|
+
handler = logging.FileHandler(directory / "kernel.log", encoding="utf-8")
|
|
331
|
+
except OSError as exc:
|
|
332
|
+
log.warning("run log unavailable: %s", exc)
|
|
333
|
+
return
|
|
334
|
+
handler.setFormatter(logging.Formatter("%(asctime)s %(message)s"))
|
|
335
|
+
logging.getLogger().addHandler(handler)
|
|
336
|
+
|
|
337
|
+
|
|
322
338
|
def _best_effort(what: str, fn: Callable[[], object], secrets: tuple[str, ...] = ()) -> bool:
|
|
323
|
-
"""One ending step; a failure is logged, never raised.
|
|
339
|
+
"""One ending step; a failure is logged with its traceback, never raised.
|
|
324
340
|
|
|
325
341
|
The terminal sequence (record, report, issue post) must degrade
|
|
326
342
|
independently: a full disk must not block the GitHub post, and a network
|
|
327
|
-
failure must not block the record.
|
|
343
|
+
failure must not block the record. The traceback is the only trace a
|
|
344
|
+
swallowed step leaves, so it is logged whole (redacted).
|
|
328
345
|
"""
|
|
329
346
|
try:
|
|
330
347
|
fn()
|
|
331
348
|
return True
|
|
332
349
|
except Exception as exc:
|
|
333
|
-
|
|
350
|
+
detail = "".join(traceback.format_exception(exc)).rstrip()
|
|
351
|
+
log.warning("%s failed: %s", what, redact(detail, secrets))
|
|
334
352
|
return False
|
|
335
353
|
|
|
336
354
|
|
|
@@ -928,6 +946,22 @@ def _wake_author_sleep(
|
|
|
928
946
|
if extra_update:
|
|
929
947
|
# a submitted park's gate/panel feedback leads; launch results follow
|
|
930
948
|
wake_text = f"{extra_update}\n\n{wake_text}"
|
|
949
|
+
# A research line whose base moved while it slept RE-PINS to the fresh base
|
|
950
|
+
# and is told to merge it: the agent does the merge (mirroring the in-review
|
|
951
|
+
# conflict wake, followup.py), the kernel only fetches and re-pins. Re-pinning
|
|
952
|
+
# base_sha to the fresh head is what makes the gate baseline and the scope
|
|
953
|
+
# base the CURRENT base (like followup's base_sha_at_fetch), so a sibling's
|
|
954
|
+
# merged work is never credited to this line and a forbidden conflict
|
|
955
|
+
# resolution (differing from the fresh base) is still scope-checked. Non-line
|
|
956
|
+
# runs and an unmoved base are untouched.
|
|
957
|
+
if _line_ref_for(bench, config.agent_id):
|
|
958
|
+
fresh_base = _line_base_advanced(ws, base_branch, base_sha)
|
|
959
|
+
if fresh_base:
|
|
960
|
+
digest = _reintegration_digest(ws, base_sha, fresh_base)
|
|
961
|
+
base_sha = fresh_base
|
|
962
|
+
wake_text = (
|
|
963
|
+
REINTEGRATE_PROMPT.format(base_branch=base_branch, digest=digest) + wake_text
|
|
964
|
+
)
|
|
931
965
|
_best_effort(
|
|
932
966
|
"budget refresh",
|
|
933
967
|
lambda: write_budget(
|
|
@@ -1009,8 +1043,6 @@ def _wake_author_sleep(
|
|
|
1009
1043
|
# existing wake path decides). Keep the NEW park's snapshot ref; the OLD
|
|
1010
1044
|
# sleep ref is superseded once the new park persists.
|
|
1011
1045
|
kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
|
|
1012
|
-
import time
|
|
1013
|
-
|
|
1014
1046
|
try:
|
|
1015
1047
|
_park_run(
|
|
1016
1048
|
run_root,
|
|
@@ -1296,6 +1328,54 @@ def _line_ref_for(bench: Benchmark | None, agent_id: str) -> str:
|
|
|
1296
1328
|
return f"agents/{agent_id}"
|
|
1297
1329
|
|
|
1298
1330
|
|
|
1331
|
+
# The base moved under a line while it slept: rather than the kernel doing a
|
|
1332
|
+
# git merge (which mishandles the agent's in-flight tree, its conflicts, and
|
|
1333
|
+
# the scope gate), the wake mirrors the in-review conflict wake — fetch the
|
|
1334
|
+
# fresh base into the workspace and TELL THE AGENT to merge it. The agent has
|
|
1335
|
+
# git and already resolves the run-start merge as its first task; only the
|
|
1336
|
+
# credential-bearing fetch/push are the kernel's. No commit text goes in the
|
|
1337
|
+
# prompt (no cross-agent prompt-injection surface); the agent reads what
|
|
1338
|
+
# landed from git itself.
|
|
1339
|
+
REINTEGRATE_PROMPT = (
|
|
1340
|
+
"# The base moved while you were asleep\n"
|
|
1341
|
+
"`origin/{base_branch}` advanced since your last run and is fetched into "
|
|
1342
|
+
"your workspace. What landed:\n{digest}\n"
|
|
1343
|
+
"Merge it into your line and resolve any conflicts honestly, then decide "
|
|
1344
|
+
"what to re-run given what landed — if a sibling took your direction "
|
|
1345
|
+
"further, pivot or say so plainly rather than pushing on. Your change is "
|
|
1346
|
+
"measured against the current base.\n\n"
|
|
1347
|
+
)
|
|
1348
|
+
|
|
1349
|
+
|
|
1350
|
+
def _reintegration_digest(ws: Workspace, base_sha: str, fresh_head: str) -> str:
|
|
1351
|
+
"""A short 'what landed' list: the subjects of the commits merged into the
|
|
1352
|
+
base since this line's base, newest first, capped. These are MERGED commits
|
|
1353
|
+
— vetted by the human-merge gate — so they are context, not untrusted input;
|
|
1354
|
+
the agent also has them in git to read in full."""
|
|
1355
|
+
try:
|
|
1356
|
+
out = ws.git("log", "--no-merges", "--format=%s", f"{base_sha}..{fresh_head}")
|
|
1357
|
+
except Exception:
|
|
1358
|
+
return " (recent changes on the base; see `git log`)"
|
|
1359
|
+
subjects = [ln.strip() for ln in out.splitlines() if ln.strip()][:12]
|
|
1360
|
+
return "\n".join(f" - {s}" for s in subjects) or " (a merge on the base; see `git log`)"
|
|
1361
|
+
|
|
1362
|
+
|
|
1363
|
+
def _line_base_advanced(ws: Workspace, base_branch: str, base_sha: str) -> str:
|
|
1364
|
+
"""Fetch `origin/<base_branch>` and return its head when it has advanced
|
|
1365
|
+
past the line's pinned base, else "". The fetch doubles as making the
|
|
1366
|
+
fresh base available for the agent to merge and refreshes the origin refs
|
|
1367
|
+
the scope/measure path pairs against. Best-effort: any git failure returns
|
|
1368
|
+
"" and the wake proceeds exactly as today."""
|
|
1369
|
+
try:
|
|
1370
|
+
ws.fetch_origin()
|
|
1371
|
+
new = ws.git("rev-parse", f"refs/remotes/origin/{base_branch}").strip()
|
|
1372
|
+
merge_base = ws.git("merge-base", new, base_sha).strip()
|
|
1373
|
+
return new if new and merge_base != new else ""
|
|
1374
|
+
except Exception as exc:
|
|
1375
|
+
log.warning("base-moved check failed (%s); wake proceeds unchanged", type(exc).__name__)
|
|
1376
|
+
return ""
|
|
1377
|
+
|
|
1378
|
+
|
|
1299
1379
|
def _push_line_snapshot(
|
|
1300
1380
|
ws: Workspace,
|
|
1301
1381
|
line_ref: str,
|
|
@@ -2953,8 +3033,6 @@ def live_attempt(
|
|
|
2953
3033
|
# commit); record and keep that same ref, drop the rest. An
|
|
2954
3034
|
# author-sleep's snapshot is the tree the wake re-delivers.
|
|
2955
3035
|
kept_ref = next((s.ref for s in snapshots if s.commit == p.candidate_sha), "")
|
|
2956
|
-
import time
|
|
2957
|
-
|
|
2958
3036
|
# anchor the deadline to the PARK (when the evals were submitted),
|
|
2959
3037
|
# not the run's start `now` — a session lasting hours would otherwise
|
|
2960
3038
|
# eat the queue budget and let the sweep cancel a still-queued eval.
|
|
@@ -3355,11 +3433,22 @@ def main() -> int:
|
|
|
3355
3433
|
)
|
|
3356
3434
|
return value
|
|
3357
3435
|
|
|
3436
|
+
def _run_id(value: str) -> str:
|
|
3437
|
+
# the id names the run directory under runs/: one path segment, so a
|
|
3438
|
+
# value cannot reach outside the run root before the record is read
|
|
3439
|
+
# ("" is the default: a fresh climb, not a resume)
|
|
3440
|
+
if value and not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.@-]{0,127}", value):
|
|
3441
|
+
raise argparse.ArgumentTypeError(
|
|
3442
|
+
f"run id {value!r} is not a run directory name (want [A-Za-z0-9][A-Za-z0-9_.@-]*)"
|
|
3443
|
+
)
|
|
3444
|
+
return value
|
|
3445
|
+
|
|
3358
3446
|
parser.add_argument("--agent-id", default="agent-01", type=_agent_id)
|
|
3359
3447
|
parser.add_argument("--run-root", required=True, type=Path)
|
|
3360
3448
|
parser.add_argument(
|
|
3361
3449
|
"--resume",
|
|
3362
3450
|
default="",
|
|
3451
|
+
type=_run_id,
|
|
3363
3452
|
metavar="RUN_ID",
|
|
3364
3453
|
help="wake a parked dispatched run instead of starting a fresh climb",
|
|
3365
3454
|
)
|
|
@@ -3447,13 +3536,7 @@ def main() -> int:
|
|
|
3447
3536
|
default=120.0,
|
|
3448
3537
|
help="how long before the walltime the self-deadline fires (floor 60)",
|
|
3449
3538
|
)
|
|
3450
|
-
parser
|
|
3451
|
-
parser.add_argument(
|
|
3452
|
-
"--github-app-file",
|
|
3453
|
-
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
3454
|
-
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
3455
|
-
"when set, installation tokens replace the PAT",
|
|
3456
|
-
)
|
|
3539
|
+
add_credential_args(parser)
|
|
3457
3540
|
parser.add_argument(
|
|
3458
3541
|
"--key-file",
|
|
3459
3542
|
default="",
|
|
@@ -3472,6 +3555,8 @@ def main() -> int:
|
|
|
3472
3555
|
)
|
|
3473
3556
|
args = parser.parse_args()
|
|
3474
3557
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(message)s")
|
|
3558
|
+
if args.resume:
|
|
3559
|
+
_attach_run_log(run_dir_of(args.run_root, args.resume))
|
|
3475
3560
|
if not args.image and not args.uncontained:
|
|
3476
3561
|
parser.error("--image is required (or pass --uncontained explicitly, dev only)")
|
|
3477
3562
|
# NOTE: the codex author is validated on the EFFECTIVE author per path — the
|
outerloop/cli.py
CHANGED
|
@@ -499,6 +499,55 @@ def start(args: argparse.Namespace) -> int:
|
|
|
499
499
|
return 0
|
|
500
500
|
|
|
501
501
|
|
|
502
|
+
DIST = "outerloop-science" # the PyPI distribution; imported as `outerloop`
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _installed_version(python: str) -> str:
|
|
506
|
+
"""The installed version of the distribution, read from a fresh interpreter
|
|
507
|
+
so it reflects what pip just wrote rather than this process's imported copy."""
|
|
508
|
+
proc = subprocess.run(
|
|
509
|
+
[python, "-c", f"import importlib.metadata as m; print(m.version({DIST!r}))"],
|
|
510
|
+
capture_output=True,
|
|
511
|
+
text=True,
|
|
512
|
+
check=False,
|
|
513
|
+
)
|
|
514
|
+
return proc.stdout.strip() or "unknown"
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def upgrade(args: argparse.Namespace) -> int:
|
|
518
|
+
"""Upgrade the installed package in place, the local adopter's one verb.
|
|
519
|
+
|
|
520
|
+
On Slurm the resident tick self-updates through OUTERLOOP_AUTO_UPDATE; a local
|
|
521
|
+
install has no such loop, so this is the equivalent: `pip install --upgrade`,
|
|
522
|
+
then you restart the loop to pick the new code up. Pip already picks the newest
|
|
523
|
+
release and only falls back to a pre-release when that is all that is published;
|
|
524
|
+
--pre forces pre-releases even once a stable exists."""
|
|
525
|
+
cmd = [sys.executable, "-m", "pip", "install", "--upgrade", DIST]
|
|
526
|
+
if args.pre:
|
|
527
|
+
cmd.append("--pre")
|
|
528
|
+
if args.dry_run:
|
|
529
|
+
print(shlex.join(cmd))
|
|
530
|
+
return 0
|
|
531
|
+
before = _installed_version(sys.executable)
|
|
532
|
+
proc = subprocess.run(cmd, check=False)
|
|
533
|
+
if proc.returncode != 0:
|
|
534
|
+
print(
|
|
535
|
+
f"upgrade failed (pip exited {proc.returncode}). If this environment has no "
|
|
536
|
+
f"pip, upgrade through its installer instead, e.g. `uv pip install --upgrade {DIST}`.",
|
|
537
|
+
file=sys.stderr,
|
|
538
|
+
)
|
|
539
|
+
return proc.returncode
|
|
540
|
+
after = _installed_version(sys.executable)
|
|
541
|
+
if before == after:
|
|
542
|
+
print(f"already up to date: outerloop {after}.")
|
|
543
|
+
else:
|
|
544
|
+
print(
|
|
545
|
+
f"upgraded outerloop {before} -> {after}. Restart the loop to pick it up: "
|
|
546
|
+
f"stop the running tick, then `outerloop start`."
|
|
547
|
+
)
|
|
548
|
+
return 0
|
|
549
|
+
|
|
550
|
+
|
|
502
551
|
def main(argv: list[str] | None = None) -> int:
|
|
503
552
|
from outerloop import __version__
|
|
504
553
|
|
|
@@ -539,6 +588,17 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
539
588
|
help="guided setup: write ~/.config/outerloop/.env and the PAT file",
|
|
540
589
|
add_help=False,
|
|
541
590
|
)
|
|
591
|
+
up = sub.add_parser(
|
|
592
|
+
"upgrade",
|
|
593
|
+
help="upgrade the installed package, then restart the loop to pick it up",
|
|
594
|
+
description="Upgrade outerloop-science in this environment with pip. On Slurm the "
|
|
595
|
+
"resident tick self-updates through OUTERLOOP_AUTO_UPDATE; this is the equivalent "
|
|
596
|
+
"for a local install. Restart the loop afterwards to run the new code.",
|
|
597
|
+
)
|
|
598
|
+
up.add_argument(
|
|
599
|
+
"--pre", action="store_true", help="include pre-releases even once a stable exists"
|
|
600
|
+
)
|
|
601
|
+
up.add_argument("--dry-run", action="store_true", help="print the command and exit")
|
|
542
602
|
argv = sys.argv[1:] if argv is None else list(argv)
|
|
543
603
|
if argv[:1] == ["tick"]:
|
|
544
604
|
# the tick entry owns its own parser; hand it the rest untouched
|
|
@@ -551,7 +611,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
551
611
|
from outerloop import init
|
|
552
612
|
|
|
553
613
|
return init.main(argv[1:])
|
|
554
|
-
|
|
614
|
+
args = parser.parse_args(argv)
|
|
615
|
+
if args.command == "upgrade":
|
|
616
|
+
return upgrade(args)
|
|
617
|
+
return start(args)
|
|
555
618
|
|
|
556
619
|
|
|
557
620
|
if __name__ == "__main__":
|
outerloop/climbboard.py
CHANGED
|
@@ -29,7 +29,7 @@ from pathlib import Path
|
|
|
29
29
|
from typing import Any
|
|
30
30
|
|
|
31
31
|
from outerloop.markers import marker
|
|
32
|
-
from outerloop.runstate import ENDED, list_runs, run_dir
|
|
32
|
+
from outerloop.runstate import ENDED, RunRecord, list_runs, run_dir
|
|
33
33
|
|
|
34
34
|
log = logging.getLogger("outerloop.climbboard")
|
|
35
35
|
|
|
@@ -169,12 +169,19 @@ def _curve_from_eval(run_directory: Path) -> list[list[float]]:
|
|
|
169
169
|
return best
|
|
170
170
|
|
|
171
171
|
|
|
172
|
-
def collect_rows(
|
|
173
|
-
|
|
172
|
+
def collect_rows(
|
|
173
|
+
root: Path, target: str, records: list[RunRecord] | None = None
|
|
174
|
+
) -> dict[str, list[ClimbRow]]:
|
|
175
|
+
"""Terminal attempts of `target` with a report, grouped by benchmark.
|
|
176
|
+
|
|
177
|
+
`records` is one tick-wide snapshot the board threads to every collector;
|
|
178
|
+
absent, the rows are read fresh (direct callers, tests)."""
|
|
174
179
|
from datetime import UTC, datetime
|
|
175
180
|
|
|
181
|
+
if records is None:
|
|
182
|
+
records = list_runs(root)
|
|
176
183
|
out: dict[str, list[ClimbRow]] = {}
|
|
177
|
-
for record in
|
|
184
|
+
for record in records:
|
|
178
185
|
# only ENDED runs: an in-review run's outcome is not known yet (its
|
|
179
186
|
# PR may be rejected), and a published row is never rewritten
|
|
180
187
|
if record.target != target or record.state != ENDED:
|
|
@@ -872,13 +879,19 @@ def is_fixed_kernel_job(name: str) -> bool:
|
|
|
872
879
|
return any(p.match(name) for p in FIXED_JOB_PATTERNS)
|
|
873
880
|
|
|
874
881
|
|
|
875
|
-
def run_job_names(
|
|
882
|
+
def run_job_names(
|
|
883
|
+
root: Path, target: str, records: list[RunRecord] | None = None
|
|
884
|
+
) -> dict[str, tuple[str, str, bool]]:
|
|
876
885
|
"""Expected per-run job names for `target`: name -> (run_id, agent,
|
|
877
886
|
is_prefix). Exact for wake and followup; a prefix for launches, whose
|
|
878
887
|
names end in the experiment's own label. Every run of the target counts,
|
|
879
|
-
ended ones too: a finishing job can outlive its record's state.
|
|
888
|
+
ended ones too: a finishing job can outlive its record's state.
|
|
889
|
+
|
|
890
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
891
|
+
if records is None:
|
|
892
|
+
records = list_runs(root)
|
|
880
893
|
out: dict[str, tuple[str, str, bool]] = {}
|
|
881
|
-
for record in
|
|
894
|
+
for record in records:
|
|
882
895
|
if record.target != target:
|
|
883
896
|
continue
|
|
884
897
|
rid, agent = record.run_id, record.agent_id
|
|
@@ -897,13 +910,19 @@ def run_job_names(root: Path, target: str) -> dict[str, tuple[str, str, bool]]:
|
|
|
897
910
|
return out
|
|
898
911
|
|
|
899
912
|
|
|
900
|
-
def eval_job_owners(
|
|
913
|
+
def eval_job_owners(
|
|
914
|
+
root: Path, target: str, records: list[RunRecord] | None = None
|
|
915
|
+
) -> dict[str, tuple[str, str]]:
|
|
901
916
|
"""Slurm job id -> (run_id, agent) for every eval a live run of `target`
|
|
902
917
|
has dispatched: the measurer leaves the id in eval-*/submitted. Eval job
|
|
903
918
|
names are liveness hashes with no agent in them, so this is how the queue
|
|
904
|
-
view knows whose eval is whose.
|
|
919
|
+
view knows whose eval is whose.
|
|
920
|
+
|
|
921
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
922
|
+
if records is None:
|
|
923
|
+
records = list_runs(root)
|
|
905
924
|
out: dict[str, tuple[str, str]] = {}
|
|
906
|
-
for record in
|
|
925
|
+
for record in records:
|
|
907
926
|
if record.target != target or record.state == ENDED:
|
|
908
927
|
continue
|
|
909
928
|
for submitted in run_dir(root, record.run_id).glob("eval-*/submitted"):
|
|
@@ -916,12 +935,20 @@ def eval_job_owners(root: Path, target: str) -> dict[str, tuple[str, str]]:
|
|
|
916
935
|
return out
|
|
917
936
|
|
|
918
937
|
|
|
919
|
-
def queue_rows(
|
|
938
|
+
def queue_rows(
|
|
939
|
+
root: Path,
|
|
940
|
+
target: str,
|
|
941
|
+
snapshot: list[dict[str, str]],
|
|
942
|
+
records: list[RunRecord] | None = None,
|
|
943
|
+
) -> list[dict[str, str]]:
|
|
920
944
|
"""The published queue: kernel jobs from a `Compute.queue_snapshot()`,
|
|
921
945
|
each attributed to an agent (by eval marker, else by the agent id every
|
|
922
|
-
other kernel job carries in its name).
|
|
923
|
-
|
|
924
|
-
|
|
946
|
+
other kernel job carries in its name).
|
|
947
|
+
|
|
948
|
+
`records` shares the board's one snapshot across both owner maps; absent,
|
|
949
|
+
each reads fresh (watcher, tests)."""
|
|
950
|
+
owners = eval_job_owners(root, target, records)
|
|
951
|
+
expected = run_job_names(root, target, records)
|
|
925
952
|
rows: list[dict[str, str]] = []
|
|
926
953
|
for job in snapshot:
|
|
927
954
|
name = str(job.get("name", ""))
|
|
@@ -1001,19 +1028,25 @@ def collect_status(
|
|
|
1001
1028
|
now: float,
|
|
1002
1029
|
contract: Any = None,
|
|
1003
1030
|
queue: list[dict[str, str]] | None = None,
|
|
1031
|
+
records: list[RunRecord] | None = None,
|
|
1004
1032
|
) -> dict[str, Any]:
|
|
1005
1033
|
"""The fleet's live picture for `target`: one entry per non-terminal run.
|
|
1006
1034
|
Timestamps, not durations — the page computes elapsed time client-side,
|
|
1007
|
-
so the strip feels live between pushes.
|
|
1035
|
+
so the strip feels live between pushes.
|
|
1036
|
+
|
|
1037
|
+
`records` shares the board's one snapshot (the runs list and the queue's
|
|
1038
|
+
owner maps read the same records); absent, it reads fresh."""
|
|
1008
1039
|
from outerloop.dispatch import effective_eval_minutes
|
|
1009
1040
|
|
|
1041
|
+
if records is None:
|
|
1042
|
+
records = list_runs(root)
|
|
1010
1043
|
budgets = {
|
|
1011
1044
|
b.name: (b.depth_k, b.sleep_k, getattr(b, "eval_minutes", 0) or 0)
|
|
1012
1045
|
for b in getattr(contract, "benchmarks", ())
|
|
1013
1046
|
}
|
|
1014
1047
|
gpu_budget = getattr(getattr(contract, "budgets", None), "gpu_hours_per_run", None)
|
|
1015
1048
|
runs = []
|
|
1016
|
-
for record in
|
|
1049
|
+
for record in records:
|
|
1017
1050
|
if record.target != target or record.state not in _LIVE_STATES:
|
|
1018
1051
|
continue
|
|
1019
1052
|
stage = record.stage or {}
|
|
@@ -1058,7 +1091,7 @@ def collect_status(
|
|
|
1058
1091
|
runs.sort(key=lambda r: str(r.get("run_id")))
|
|
1059
1092
|
status: dict[str, Any] = {"target": target, "published": now, "runs": runs}
|
|
1060
1093
|
if queue is not None: # a snapshot was taken (Slurm); the local loop has no queue
|
|
1061
|
-
status["queue"] = queue_rows(root, target, queue)
|
|
1094
|
+
status["queue"] = queue_rows(root, target, queue, records)
|
|
1062
1095
|
return status
|
|
1063
1096
|
|
|
1064
1097
|
|
|
@@ -1069,12 +1102,15 @@ def service_status(
|
|
|
1069
1102
|
now: float,
|
|
1070
1103
|
contract: Any = None,
|
|
1071
1104
|
queue: list[dict[str, str]] | None = None,
|
|
1105
|
+
records: list[RunRecord] | None = None,
|
|
1072
1106
|
) -> bool:
|
|
1073
1107
|
"""Publish the strip when the fleet's SHAPE changed — a run appearing,
|
|
1074
1108
|
leaving, or changing state/phase — never on every tick: the page shows
|
|
1075
1109
|
elapsed time client-side, so timestamp-only drift is not worth a commit.
|
|
1076
|
-
Advisory like the board; True when a write happened.
|
|
1077
|
-
|
|
1110
|
+
Advisory like the board; True when a write happened.
|
|
1111
|
+
|
|
1112
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
1113
|
+
status = collect_status(root, target, now, contract, queue, records)
|
|
1078
1114
|
try:
|
|
1079
1115
|
existing_raw: str | None = github.get_file(target, STATUS_PATH, BOARD_BRANCH)
|
|
1080
1116
|
except Exception as exc:
|
|
@@ -1247,7 +1283,11 @@ def _read_index(github: Any, target: str) -> dict[str, str] | None:
|
|
|
1247
1283
|
|
|
1248
1284
|
|
|
1249
1285
|
def service_climb_board(
|
|
1250
|
-
root: Path,
|
|
1286
|
+
root: Path,
|
|
1287
|
+
github: Any,
|
|
1288
|
+
target: str,
|
|
1289
|
+
directions: dict[str, str] | None = None,
|
|
1290
|
+
records: list[RunRecord] | None = None,
|
|
1251
1291
|
) -> int:
|
|
1252
1292
|
"""Publish the board for `target`. Returns how many files changed.
|
|
1253
1293
|
|
|
@@ -1255,8 +1295,10 @@ def service_climb_board(
|
|
|
1255
1295
|
as ONE commit (`put_files`) — data, curves, and the views are atomic,
|
|
1256
1296
|
so the page can never point at data that is not on the branch, and a
|
|
1257
1297
|
board pass costs at most one commit of research-log history. A failed
|
|
1258
|
-
batch changes nothing; the whole pass retries next tick.
|
|
1259
|
-
|
|
1298
|
+
batch changes nothing; the whole pass retries next tick.
|
|
1299
|
+
|
|
1300
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
1301
|
+
local = collect_rows(root, target, records)
|
|
1260
1302
|
# snapshot BEFORE any branch read: put_files refuses if the head moves
|
|
1261
1303
|
# mid-pass, so a concurrent write is never buried under stale content.
|
|
1262
1304
|
# "" = branch missing (nothing to protect); None = outage — writing
|
outerloop/followup.py
CHANGED
|
@@ -49,7 +49,6 @@ from outerloop.orchestrator import (
|
|
|
49
49
|
steward_out_of_scope,
|
|
50
50
|
)
|
|
51
51
|
from outerloop.orchestrator import improved as orch_improved
|
|
52
|
-
from outerloop.paths import CONFIG_DIR
|
|
53
52
|
from outerloop.progress import (
|
|
54
53
|
PROGRESS_PATHS,
|
|
55
54
|
fmt_metric,
|
|
@@ -1970,7 +1969,7 @@ def main() -> int:
|
|
|
1970
1969
|
import os
|
|
1971
1970
|
import time
|
|
1972
1971
|
|
|
1973
|
-
from outerloop.appauth import resolve_bot_auth
|
|
1972
|
+
from outerloop.appauth import add_credential_args, resolve_bot_auth
|
|
1974
1973
|
from outerloop.harness import DEFAULT_MAX_TURNS
|
|
1975
1974
|
from outerloop.orchestrator import SubprocessEvaluator
|
|
1976
1975
|
|
|
@@ -2013,13 +2012,7 @@ def main() -> int:
|
|
|
2013
2012
|
default=0,
|
|
2014
2013
|
help="this job's Slurm walltime; arms the self-deadline (0 = off)",
|
|
2015
2014
|
)
|
|
2016
|
-
parser
|
|
2017
|
-
parser.add_argument(
|
|
2018
|
-
"--github-app-file",
|
|
2019
|
-
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
2020
|
-
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
2021
|
-
"when set, installation tokens replace the PAT",
|
|
2022
|
-
)
|
|
2015
|
+
add_credential_args(parser)
|
|
2023
2016
|
parser.add_argument(
|
|
2024
2017
|
"--key-file",
|
|
2025
2018
|
default="",
|
outerloop/github.py
CHANGED
|
@@ -503,10 +503,20 @@ class GitHubClient:
|
|
|
503
503
|
return True
|
|
504
504
|
|
|
505
505
|
def disable_auto_merge(self, repo: str, number: int) -> bool:
|
|
506
|
-
"""
|
|
507
|
-
|
|
508
|
-
merge the NEW head on green
|
|
509
|
-
(terra #171).
|
|
506
|
+
"""Ensure GitHub auto-merge will not fire on this PR, and report whether
|
|
507
|
+
that is confirmed. The follow-up lane calls this before pushing new
|
|
508
|
+
commits to an auto-mode PR: an armed PR would merge the NEW head on green
|
|
509
|
+
CI without a fresh gate/suite/panel (terra #171).
|
|
510
|
+
|
|
511
|
+
The property to confirm is "auto-merge is not armed". We always run the
|
|
512
|
+
disarm mutation — never skipping it on a prior read, which would let
|
|
513
|
+
another actor arm auto-merge between the read and the push. When nothing
|
|
514
|
+
is armed the mutation errors, and its text varies by the repo's "Allow
|
|
515
|
+
auto-merge" setting and the token type (App vs PAT), so we confirm by
|
|
516
|
+
RE-READING the PR's `auto_merge` state rather than matching a string: if
|
|
517
|
+
auto-merge is off afterward, the safety property holds and pushing is
|
|
518
|
+
safe. Returns False only when auto-merge is still armed and could not be
|
|
519
|
+
disarmed."""
|
|
510
520
|
if self.dry_run:
|
|
511
521
|
log.info("[dry-run] disarm auto-merge on %s#%d", repo, number)
|
|
512
522
|
return True
|
|
@@ -519,12 +529,18 @@ class GitHubClient:
|
|
|
519
529
|
" disablePullRequestAutoMerge(input: {pullRequestId: $pr})"
|
|
520
530
|
" { pullRequest { number } } }"
|
|
521
531
|
)
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
except GitHubError as exc:
|
|
525
|
-
if "not enabled" in str(exc).casefold():
|
|
526
|
-
# nothing was armed — the state we wanted; pushing is safe
|
|
532
|
+
try:
|
|
533
|
+
self._graphql(mutation, {"pr": str(node_id)})
|
|
527
534
|
return True
|
|
535
|
+
except GitHubError as exc:
|
|
536
|
+
# Disarming a PR with nothing armed errors; the message is not a
|
|
537
|
+
# reliable signal (merge:manual repo, "Allow auto-merge" off, or
|
|
538
|
+
# App-token phrasing), so confirm by state: off now = safe.
|
|
539
|
+
if not self.get_pull_request(repo, number).get("auto_merge"):
|
|
540
|
+
return True
|
|
541
|
+
log.warning("auto-merge disarm on %s#%s failed: %s", repo, number, exc)
|
|
542
|
+
return False
|
|
543
|
+
except GitHubError as exc:
|
|
528
544
|
log.warning("auto-merge disarm on %s#%s failed: %s", repo, number, exc)
|
|
529
545
|
return False
|
|
530
546
|
|
|
@@ -768,15 +784,19 @@ class GitHubClient:
|
|
|
768
784
|
)
|
|
769
785
|
return int(data.get("number", 0)) if isinstance(data, dict) else 0
|
|
770
786
|
|
|
771
|
-
def update_issue(self, repo: str, number: int, body: str) -> None:
|
|
772
|
-
"""Replace an issue's body (the rolling maintenance digest)
|
|
787
|
+
def update_issue(self, repo: str, number: int, body: str, *, title: str | None = None) -> None:
|
|
788
|
+
"""Replace an issue's body (the rolling maintenance digest), and its
|
|
789
|
+
title when one is given (the digest carries the scan date there)."""
|
|
773
790
|
if self.dry_run:
|
|
774
791
|
log.info("[dry-run] update issue %s#%s (%d chars)", repo, number, len(body))
|
|
775
792
|
return
|
|
793
|
+
payload: dict[str, str] = {"body": body}
|
|
794
|
+
if title is not None:
|
|
795
|
+
payload["title"] = title
|
|
776
796
|
self._request(
|
|
777
797
|
"PATCH",
|
|
778
798
|
f"/repos/{urllib.parse.quote(repo)}/issues/{number}",
|
|
779
|
-
|
|
799
|
+
payload,
|
|
780
800
|
)
|
|
781
801
|
|
|
782
802
|
def close_issue(self, repo: str, number: int) -> None:
|