outerloop-science 0.1.0.dev3__py3-none-any.whl → 0.1.0.dev4__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +1 -1
- outerloop/appauth.py +17 -0
- outerloop/attempt.py +66 -8
- outerloop/cli.py +64 -1
- outerloop/climbboard.py +64 -22
- outerloop/followup.py +2 -9
- outerloop/github.py +32 -12
- outerloop/harness.py +21 -29
- outerloop/init.py +26 -1
- outerloop/maintain.py +29 -1
- outerloop/maintain_post_cli.py +4 -4
- outerloop/review_summarize_cli.py +1 -1
- outerloop/steward.py +2 -9
- outerloop/tick.py +184 -135
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.0.dev4.dist-info}/METADATA +1 -1
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.0.dev4.dist-info}/RECORD +20 -20
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.0.dev4.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.0.dev4.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.0.dev4.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev3.dist-info → outerloop_science-0.1.0.dev4.dist-info}/licenses/NOTICE +0 -0
outerloop/__init__.py
CHANGED
outerloop/appauth.py
CHANGED
|
@@ -17,8 +17,10 @@ network; the production signer lives behind the `app-auth` extra.
|
|
|
17
17
|
|
|
18
18
|
from __future__ import annotations
|
|
19
19
|
|
|
20
|
+
import argparse
|
|
20
21
|
import base64
|
|
21
22
|
import json
|
|
23
|
+
import os
|
|
22
24
|
import time
|
|
23
25
|
import urllib.error
|
|
24
26
|
import urllib.parse
|
|
@@ -29,6 +31,7 @@ from pathlib import Path
|
|
|
29
31
|
from typing import Any
|
|
30
32
|
|
|
31
33
|
from outerloop.github import AUTH_SAFE_OPENER, FileTokenProvider, TokenProvider
|
|
34
|
+
from outerloop.paths import CONFIG_DIR
|
|
32
35
|
|
|
33
36
|
# RS256-sign the JWT signing input, returning the raw signature bytes.
|
|
34
37
|
Signer = Callable[[bytes], bytes]
|
|
@@ -211,3 +214,17 @@ def resolve_bot_auth(pat_file: str | Path, app_file: str | Path = "") -> TokenPr
|
|
|
211
214
|
if str(app_file).strip():
|
|
212
215
|
return app_provider_from_file(Path(str(app_file)).expanduser())
|
|
213
216
|
return FileTokenProvider(Path(str(pat_file)).expanduser())
|
|
217
|
+
|
|
218
|
+
|
|
219
|
+
def add_credential_args(parser: argparse.ArgumentParser) -> None:
|
|
220
|
+
"""Add the shared bot-auth options to a role parser: `--pat-file`, and
|
|
221
|
+
`--github-app-file`, which supplies installation tokens instead of the PAT
|
|
222
|
+
when set. `resolve_bot_auth` reads the pair. One owner so the defaults and
|
|
223
|
+
help cannot drift between attempt, followup, and steward."""
|
|
224
|
+
parser.add_argument("--pat-file", default=str(CONFIG_DIR / "bot_pat"))
|
|
225
|
+
parser.add_argument(
|
|
226
|
+
"--github-app-file",
|
|
227
|
+
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
228
|
+
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
229
|
+
"when set, installation tokens replace the PAT",
|
|
230
|
+
)
|
outerloop/attempt.py
CHANGED
|
@@ -26,7 +26,7 @@ from functools import partial
|
|
|
26
26
|
from pathlib import Path
|
|
27
27
|
from typing import Any, cast
|
|
28
28
|
|
|
29
|
-
from outerloop.appauth import resolve_bot_auth
|
|
29
|
+
from outerloop.appauth import add_credential_args, resolve_bot_auth
|
|
30
30
|
from outerloop.brief import BudgetState, distill_lessons
|
|
31
31
|
from outerloop.compute import LocalCompute
|
|
32
32
|
from outerloop.contract import Benchmark, Contract, contract_text_in_tree, load_contract
|
|
@@ -928,6 +928,22 @@ def _wake_author_sleep(
|
|
|
928
928
|
if extra_update:
|
|
929
929
|
# a submitted park's gate/panel feedback leads; launch results follow
|
|
930
930
|
wake_text = f"{extra_update}\n\n{wake_text}"
|
|
931
|
+
# A research line whose base moved while it slept RE-PINS to the fresh base
|
|
932
|
+
# and is told to merge it: the agent does the merge (mirroring the in-review
|
|
933
|
+
# conflict wake, followup.py), the kernel only fetches and re-pins. Re-pinning
|
|
934
|
+
# base_sha to the fresh head is what makes the gate baseline and the scope
|
|
935
|
+
# base the CURRENT base (like followup's base_sha_at_fetch), so a sibling's
|
|
936
|
+
# merged work is never credited to this line and a forbidden conflict
|
|
937
|
+
# resolution (differing from the fresh base) is still scope-checked. Non-line
|
|
938
|
+
# runs and an unmoved base are untouched.
|
|
939
|
+
if _line_ref_for(bench, config.agent_id):
|
|
940
|
+
fresh_base = _line_base_advanced(ws, base_branch, base_sha)
|
|
941
|
+
if fresh_base:
|
|
942
|
+
digest = _reintegration_digest(ws, base_sha, fresh_base)
|
|
943
|
+
base_sha = fresh_base
|
|
944
|
+
wake_text = (
|
|
945
|
+
REINTEGRATE_PROMPT.format(base_branch=base_branch, digest=digest) + wake_text
|
|
946
|
+
)
|
|
931
947
|
_best_effort(
|
|
932
948
|
"budget refresh",
|
|
933
949
|
lambda: write_budget(
|
|
@@ -1296,6 +1312,54 @@ def _line_ref_for(bench: Benchmark | None, agent_id: str) -> str:
|
|
|
1296
1312
|
return f"agents/{agent_id}"
|
|
1297
1313
|
|
|
1298
1314
|
|
|
1315
|
+
# The base moved under a line while it slept: rather than the kernel doing a
|
|
1316
|
+
# git merge (which mishandles the agent's in-flight tree, its conflicts, and
|
|
1317
|
+
# the scope gate), the wake mirrors the in-review conflict wake — fetch the
|
|
1318
|
+
# fresh base into the workspace and TELL THE AGENT to merge it. The agent has
|
|
1319
|
+
# git and already resolves the run-start merge as its first task; only the
|
|
1320
|
+
# credential-bearing fetch/push are the kernel's. No commit text goes in the
|
|
1321
|
+
# prompt (no cross-agent prompt-injection surface); the agent reads what
|
|
1322
|
+
# landed from git itself.
|
|
1323
|
+
REINTEGRATE_PROMPT = (
|
|
1324
|
+
"# The base moved while you were asleep\n"
|
|
1325
|
+
"`origin/{base_branch}` advanced since your last run and is fetched into "
|
|
1326
|
+
"your workspace. What landed:\n{digest}\n"
|
|
1327
|
+
"Merge it into your line and resolve any conflicts honestly, then decide "
|
|
1328
|
+
"what to re-run given what landed — if a sibling took your direction "
|
|
1329
|
+
"further, pivot or say so plainly rather than pushing on. Your change is "
|
|
1330
|
+
"measured against the current base.\n\n"
|
|
1331
|
+
)
|
|
1332
|
+
|
|
1333
|
+
|
|
1334
|
+
def _reintegration_digest(ws: Workspace, base_sha: str, fresh_head: str) -> str:
|
|
1335
|
+
"""A short 'what landed' list: the subjects of the commits merged into the
|
|
1336
|
+
base since this line's base, newest first, capped. These are MERGED commits
|
|
1337
|
+
— vetted by the human-merge gate — so they are context, not untrusted input;
|
|
1338
|
+
the agent also has them in git to read in full."""
|
|
1339
|
+
try:
|
|
1340
|
+
out = ws.git("log", "--no-merges", "--format=%s", f"{base_sha}..{fresh_head}")
|
|
1341
|
+
except Exception:
|
|
1342
|
+
return " (recent changes on the base; see `git log`)"
|
|
1343
|
+
subjects = [ln.strip() for ln in out.splitlines() if ln.strip()][:12]
|
|
1344
|
+
return "\n".join(f" - {s}" for s in subjects) or " (a merge on the base; see `git log`)"
|
|
1345
|
+
|
|
1346
|
+
|
|
1347
|
+
def _line_base_advanced(ws: Workspace, base_branch: str, base_sha: str) -> str:
|
|
1348
|
+
"""Fetch `origin/<base_branch>` and return its head when it has advanced
|
|
1349
|
+
past the line's pinned base, else "". The fetch doubles as making the
|
|
1350
|
+
fresh base available for the agent to merge and refreshes the origin refs
|
|
1351
|
+
the scope/measure path pairs against. Best-effort: any git failure returns
|
|
1352
|
+
"" and the wake proceeds exactly as today."""
|
|
1353
|
+
try:
|
|
1354
|
+
ws.fetch_origin()
|
|
1355
|
+
new = ws.git("rev-parse", f"refs/remotes/origin/{base_branch}").strip()
|
|
1356
|
+
merge_base = ws.git("merge-base", new, base_sha).strip()
|
|
1357
|
+
return new if new and merge_base != new else ""
|
|
1358
|
+
except Exception as exc:
|
|
1359
|
+
log.warning("base-moved check failed (%s); wake proceeds unchanged", type(exc).__name__)
|
|
1360
|
+
return ""
|
|
1361
|
+
|
|
1362
|
+
|
|
1299
1363
|
def _push_line_snapshot(
|
|
1300
1364
|
ws: Workspace,
|
|
1301
1365
|
line_ref: str,
|
|
@@ -3447,13 +3511,7 @@ def main() -> int:
|
|
|
3447
3511
|
default=120.0,
|
|
3448
3512
|
help="how long before the walltime the self-deadline fires (floor 60)",
|
|
3449
3513
|
)
|
|
3450
|
-
parser
|
|
3451
|
-
parser.add_argument(
|
|
3452
|
-
"--github-app-file",
|
|
3453
|
-
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
3454
|
-
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
3455
|
-
"when set, installation tokens replace the PAT",
|
|
3456
|
-
)
|
|
3514
|
+
add_credential_args(parser)
|
|
3457
3515
|
parser.add_argument(
|
|
3458
3516
|
"--key-file",
|
|
3459
3517
|
default="",
|
outerloop/cli.py
CHANGED
|
@@ -499,6 +499,55 @@ def start(args: argparse.Namespace) -> int:
|
|
|
499
499
|
return 0
|
|
500
500
|
|
|
501
501
|
|
|
502
|
+
DIST = "outerloop-science" # the PyPI distribution; imported as `outerloop`
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def _installed_version(python: str) -> str:
|
|
506
|
+
"""The installed version of the distribution, read from a fresh interpreter
|
|
507
|
+
so it reflects what pip just wrote rather than this process's imported copy."""
|
|
508
|
+
proc = subprocess.run(
|
|
509
|
+
[python, "-c", f"import importlib.metadata as m; print(m.version({DIST!r}))"],
|
|
510
|
+
capture_output=True,
|
|
511
|
+
text=True,
|
|
512
|
+
check=False,
|
|
513
|
+
)
|
|
514
|
+
return proc.stdout.strip() or "unknown"
|
|
515
|
+
|
|
516
|
+
|
|
517
|
+
def upgrade(args: argparse.Namespace) -> int:
|
|
518
|
+
"""Upgrade the installed package in place, the local adopter's one verb.
|
|
519
|
+
|
|
520
|
+
On Slurm the resident tick self-updates through OUTERLOOP_AUTO_UPDATE; a local
|
|
521
|
+
install has no such loop, so this is the equivalent: `pip install --upgrade`,
|
|
522
|
+
then you restart the loop to pick the new code up. Pip already picks the newest
|
|
523
|
+
release and only falls back to a pre-release when that is all that is published;
|
|
524
|
+
--pre forces pre-releases even once a stable exists."""
|
|
525
|
+
cmd = [sys.executable, "-m", "pip", "install", "--upgrade", DIST]
|
|
526
|
+
if args.pre:
|
|
527
|
+
cmd.append("--pre")
|
|
528
|
+
if args.dry_run:
|
|
529
|
+
print(shlex.join(cmd))
|
|
530
|
+
return 0
|
|
531
|
+
before = _installed_version(sys.executable)
|
|
532
|
+
proc = subprocess.run(cmd, check=False)
|
|
533
|
+
if proc.returncode != 0:
|
|
534
|
+
print(
|
|
535
|
+
f"upgrade failed (pip exited {proc.returncode}). If this environment has no "
|
|
536
|
+
f"pip, upgrade through its installer instead, e.g. `uv pip install --upgrade {DIST}`.",
|
|
537
|
+
file=sys.stderr,
|
|
538
|
+
)
|
|
539
|
+
return proc.returncode
|
|
540
|
+
after = _installed_version(sys.executable)
|
|
541
|
+
if before == after:
|
|
542
|
+
print(f"already up to date: outerloop {after}.")
|
|
543
|
+
else:
|
|
544
|
+
print(
|
|
545
|
+
f"upgraded outerloop {before} -> {after}. Restart the loop to pick it up: "
|
|
546
|
+
f"stop the running tick, then `outerloop start`."
|
|
547
|
+
)
|
|
548
|
+
return 0
|
|
549
|
+
|
|
550
|
+
|
|
502
551
|
def main(argv: list[str] | None = None) -> int:
|
|
503
552
|
from outerloop import __version__
|
|
504
553
|
|
|
@@ -539,6 +588,17 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
539
588
|
help="guided setup: write ~/.config/outerloop/.env and the PAT file",
|
|
540
589
|
add_help=False,
|
|
541
590
|
)
|
|
591
|
+
up = sub.add_parser(
|
|
592
|
+
"upgrade",
|
|
593
|
+
help="upgrade the installed package, then restart the loop to pick it up",
|
|
594
|
+
description="Upgrade outerloop-science in this environment with pip. On Slurm the "
|
|
595
|
+
"resident tick self-updates through OUTERLOOP_AUTO_UPDATE; this is the equivalent "
|
|
596
|
+
"for a local install. Restart the loop afterwards to run the new code.",
|
|
597
|
+
)
|
|
598
|
+
up.add_argument(
|
|
599
|
+
"--pre", action="store_true", help="include pre-releases even once a stable exists"
|
|
600
|
+
)
|
|
601
|
+
up.add_argument("--dry-run", action="store_true", help="print the command and exit")
|
|
542
602
|
argv = sys.argv[1:] if argv is None else list(argv)
|
|
543
603
|
if argv[:1] == ["tick"]:
|
|
544
604
|
# the tick entry owns its own parser; hand it the rest untouched
|
|
@@ -551,7 +611,10 @@ def main(argv: list[str] | None = None) -> int:
|
|
|
551
611
|
from outerloop import init
|
|
552
612
|
|
|
553
613
|
return init.main(argv[1:])
|
|
554
|
-
|
|
614
|
+
args = parser.parse_args(argv)
|
|
615
|
+
if args.command == "upgrade":
|
|
616
|
+
return upgrade(args)
|
|
617
|
+
return start(args)
|
|
555
618
|
|
|
556
619
|
|
|
557
620
|
if __name__ == "__main__":
|
outerloop/climbboard.py
CHANGED
|
@@ -29,7 +29,7 @@ from pathlib import Path
|
|
|
29
29
|
from typing import Any
|
|
30
30
|
|
|
31
31
|
from outerloop.markers import marker
|
|
32
|
-
from outerloop.runstate import ENDED, list_runs, run_dir
|
|
32
|
+
from outerloop.runstate import ENDED, RunRecord, list_runs, run_dir
|
|
33
33
|
|
|
34
34
|
log = logging.getLogger("outerloop.climbboard")
|
|
35
35
|
|
|
@@ -169,12 +169,19 @@ def _curve_from_eval(run_directory: Path) -> list[list[float]]:
|
|
|
169
169
|
return best
|
|
170
170
|
|
|
171
171
|
|
|
172
|
-
def collect_rows(
|
|
173
|
-
|
|
172
|
+
def collect_rows(
|
|
173
|
+
root: Path, target: str, records: list[RunRecord] | None = None
|
|
174
|
+
) -> dict[str, list[ClimbRow]]:
|
|
175
|
+
"""Terminal attempts of `target` with a report, grouped by benchmark.
|
|
176
|
+
|
|
177
|
+
`records` is one tick-wide snapshot the board threads to every collector;
|
|
178
|
+
absent, the rows are read fresh (direct callers, tests)."""
|
|
174
179
|
from datetime import UTC, datetime
|
|
175
180
|
|
|
181
|
+
if records is None:
|
|
182
|
+
records = list_runs(root)
|
|
176
183
|
out: dict[str, list[ClimbRow]] = {}
|
|
177
|
-
for record in
|
|
184
|
+
for record in records:
|
|
178
185
|
# only ENDED runs: an in-review run's outcome is not known yet (its
|
|
179
186
|
# PR may be rejected), and a published row is never rewritten
|
|
180
187
|
if record.target != target or record.state != ENDED:
|
|
@@ -872,13 +879,19 @@ def is_fixed_kernel_job(name: str) -> bool:
|
|
|
872
879
|
return any(p.match(name) for p in FIXED_JOB_PATTERNS)
|
|
873
880
|
|
|
874
881
|
|
|
875
|
-
def run_job_names(
|
|
882
|
+
def run_job_names(
|
|
883
|
+
root: Path, target: str, records: list[RunRecord] | None = None
|
|
884
|
+
) -> dict[str, tuple[str, str, bool]]:
|
|
876
885
|
"""Expected per-run job names for `target`: name -> (run_id, agent,
|
|
877
886
|
is_prefix). Exact for wake and followup; a prefix for launches, whose
|
|
878
887
|
names end in the experiment's own label. Every run of the target counts,
|
|
879
|
-
ended ones too: a finishing job can outlive its record's state.
|
|
888
|
+
ended ones too: a finishing job can outlive its record's state.
|
|
889
|
+
|
|
890
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
891
|
+
if records is None:
|
|
892
|
+
records = list_runs(root)
|
|
880
893
|
out: dict[str, tuple[str, str, bool]] = {}
|
|
881
|
-
for record in
|
|
894
|
+
for record in records:
|
|
882
895
|
if record.target != target:
|
|
883
896
|
continue
|
|
884
897
|
rid, agent = record.run_id, record.agent_id
|
|
@@ -897,13 +910,19 @@ def run_job_names(root: Path, target: str) -> dict[str, tuple[str, str, bool]]:
|
|
|
897
910
|
return out
|
|
898
911
|
|
|
899
912
|
|
|
900
|
-
def eval_job_owners(
|
|
913
|
+
def eval_job_owners(
|
|
914
|
+
root: Path, target: str, records: list[RunRecord] | None = None
|
|
915
|
+
) -> dict[str, tuple[str, str]]:
|
|
901
916
|
"""Slurm job id -> (run_id, agent) for every eval a live run of `target`
|
|
902
917
|
has dispatched: the measurer leaves the id in eval-*/submitted. Eval job
|
|
903
918
|
names are liveness hashes with no agent in them, so this is how the queue
|
|
904
|
-
view knows whose eval is whose.
|
|
919
|
+
view knows whose eval is whose.
|
|
920
|
+
|
|
921
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
922
|
+
if records is None:
|
|
923
|
+
records = list_runs(root)
|
|
905
924
|
out: dict[str, tuple[str, str]] = {}
|
|
906
|
-
for record in
|
|
925
|
+
for record in records:
|
|
907
926
|
if record.target != target or record.state == ENDED:
|
|
908
927
|
continue
|
|
909
928
|
for submitted in run_dir(root, record.run_id).glob("eval-*/submitted"):
|
|
@@ -916,12 +935,20 @@ def eval_job_owners(root: Path, target: str) -> dict[str, tuple[str, str]]:
|
|
|
916
935
|
return out
|
|
917
936
|
|
|
918
937
|
|
|
919
|
-
def queue_rows(
|
|
938
|
+
def queue_rows(
|
|
939
|
+
root: Path,
|
|
940
|
+
target: str,
|
|
941
|
+
snapshot: list[dict[str, str]],
|
|
942
|
+
records: list[RunRecord] | None = None,
|
|
943
|
+
) -> list[dict[str, str]]:
|
|
920
944
|
"""The published queue: kernel jobs from a `Compute.queue_snapshot()`,
|
|
921
945
|
each attributed to an agent (by eval marker, else by the agent id every
|
|
922
|
-
other kernel job carries in its name).
|
|
923
|
-
|
|
924
|
-
|
|
946
|
+
other kernel job carries in its name).
|
|
947
|
+
|
|
948
|
+
`records` shares the board's one snapshot across both owner maps; absent,
|
|
949
|
+
each reads fresh (watcher, tests)."""
|
|
950
|
+
owners = eval_job_owners(root, target, records)
|
|
951
|
+
expected = run_job_names(root, target, records)
|
|
925
952
|
rows: list[dict[str, str]] = []
|
|
926
953
|
for job in snapshot:
|
|
927
954
|
name = str(job.get("name", ""))
|
|
@@ -1001,19 +1028,25 @@ def collect_status(
|
|
|
1001
1028
|
now: float,
|
|
1002
1029
|
contract: Any = None,
|
|
1003
1030
|
queue: list[dict[str, str]] | None = None,
|
|
1031
|
+
records: list[RunRecord] | None = None,
|
|
1004
1032
|
) -> dict[str, Any]:
|
|
1005
1033
|
"""The fleet's live picture for `target`: one entry per non-terminal run.
|
|
1006
1034
|
Timestamps, not durations — the page computes elapsed time client-side,
|
|
1007
|
-
so the strip feels live between pushes.
|
|
1035
|
+
so the strip feels live between pushes.
|
|
1036
|
+
|
|
1037
|
+
`records` shares the board's one snapshot (the runs list and the queue's
|
|
1038
|
+
owner maps read the same records); absent, it reads fresh."""
|
|
1008
1039
|
from outerloop.dispatch import effective_eval_minutes
|
|
1009
1040
|
|
|
1041
|
+
if records is None:
|
|
1042
|
+
records = list_runs(root)
|
|
1010
1043
|
budgets = {
|
|
1011
1044
|
b.name: (b.depth_k, b.sleep_k, getattr(b, "eval_minutes", 0) or 0)
|
|
1012
1045
|
for b in getattr(contract, "benchmarks", ())
|
|
1013
1046
|
}
|
|
1014
1047
|
gpu_budget = getattr(getattr(contract, "budgets", None), "gpu_hours_per_run", None)
|
|
1015
1048
|
runs = []
|
|
1016
|
-
for record in
|
|
1049
|
+
for record in records:
|
|
1017
1050
|
if record.target != target or record.state not in _LIVE_STATES:
|
|
1018
1051
|
continue
|
|
1019
1052
|
stage = record.stage or {}
|
|
@@ -1058,7 +1091,7 @@ def collect_status(
|
|
|
1058
1091
|
runs.sort(key=lambda r: str(r.get("run_id")))
|
|
1059
1092
|
status: dict[str, Any] = {"target": target, "published": now, "runs": runs}
|
|
1060
1093
|
if queue is not None: # a snapshot was taken (Slurm); the local loop has no queue
|
|
1061
|
-
status["queue"] = queue_rows(root, target, queue)
|
|
1094
|
+
status["queue"] = queue_rows(root, target, queue, records)
|
|
1062
1095
|
return status
|
|
1063
1096
|
|
|
1064
1097
|
|
|
@@ -1069,12 +1102,15 @@ def service_status(
|
|
|
1069
1102
|
now: float,
|
|
1070
1103
|
contract: Any = None,
|
|
1071
1104
|
queue: list[dict[str, str]] | None = None,
|
|
1105
|
+
records: list[RunRecord] | None = None,
|
|
1072
1106
|
) -> bool:
|
|
1073
1107
|
"""Publish the strip when the fleet's SHAPE changed — a run appearing,
|
|
1074
1108
|
leaving, or changing state/phase — never on every tick: the page shows
|
|
1075
1109
|
elapsed time client-side, so timestamp-only drift is not worth a commit.
|
|
1076
|
-
Advisory like the board; True when a write happened.
|
|
1077
|
-
|
|
1110
|
+
Advisory like the board; True when a write happened.
|
|
1111
|
+
|
|
1112
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
1113
|
+
status = collect_status(root, target, now, contract, queue, records)
|
|
1078
1114
|
try:
|
|
1079
1115
|
existing_raw: str | None = github.get_file(target, STATUS_PATH, BOARD_BRANCH)
|
|
1080
1116
|
except Exception as exc:
|
|
@@ -1247,7 +1283,11 @@ def _read_index(github: Any, target: str) -> dict[str, str] | None:
|
|
|
1247
1283
|
|
|
1248
1284
|
|
|
1249
1285
|
def service_climb_board(
|
|
1250
|
-
root: Path,
|
|
1286
|
+
root: Path,
|
|
1287
|
+
github: Any,
|
|
1288
|
+
target: str,
|
|
1289
|
+
directions: dict[str, str] | None = None,
|
|
1290
|
+
records: list[RunRecord] | None = None,
|
|
1251
1291
|
) -> int:
|
|
1252
1292
|
"""Publish the board for `target`. Returns how many files changed.
|
|
1253
1293
|
|
|
@@ -1255,8 +1295,10 @@ def service_climb_board(
|
|
|
1255
1295
|
as ONE commit (`put_files`) — data, curves, and the views are atomic,
|
|
1256
1296
|
so the page can never point at data that is not on the branch, and a
|
|
1257
1297
|
board pass costs at most one commit of research-log history. A failed
|
|
1258
|
-
batch changes nothing; the whole pass retries next tick.
|
|
1259
|
-
|
|
1298
|
+
batch changes nothing; the whole pass retries next tick.
|
|
1299
|
+
|
|
1300
|
+
`records` shares the board's one snapshot; absent, it reads fresh."""
|
|
1301
|
+
local = collect_rows(root, target, records)
|
|
1260
1302
|
# snapshot BEFORE any branch read: put_files refuses if the head moves
|
|
1261
1303
|
# mid-pass, so a concurrent write is never buried under stale content.
|
|
1262
1304
|
# "" = branch missing (nothing to protect); None = outage — writing
|
outerloop/followup.py
CHANGED
|
@@ -49,7 +49,6 @@ from outerloop.orchestrator import (
|
|
|
49
49
|
steward_out_of_scope,
|
|
50
50
|
)
|
|
51
51
|
from outerloop.orchestrator import improved as orch_improved
|
|
52
|
-
from outerloop.paths import CONFIG_DIR
|
|
53
52
|
from outerloop.progress import (
|
|
54
53
|
PROGRESS_PATHS,
|
|
55
54
|
fmt_metric,
|
|
@@ -1970,7 +1969,7 @@ def main() -> int:
|
|
|
1970
1969
|
import os
|
|
1971
1970
|
import time
|
|
1972
1971
|
|
|
1973
|
-
from outerloop.appauth import resolve_bot_auth
|
|
1972
|
+
from outerloop.appauth import add_credential_args, resolve_bot_auth
|
|
1974
1973
|
from outerloop.harness import DEFAULT_MAX_TURNS
|
|
1975
1974
|
from outerloop.orchestrator import SubprocessEvaluator
|
|
1976
1975
|
|
|
@@ -2013,13 +2012,7 @@ def main() -> int:
|
|
|
2013
2012
|
default=0,
|
|
2014
2013
|
help="this job's Slurm walltime; arms the self-deadline (0 = off)",
|
|
2015
2014
|
)
|
|
2016
|
-
parser
|
|
2017
|
-
parser.add_argument(
|
|
2018
|
-
"--github-app-file",
|
|
2019
|
-
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
2020
|
-
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
2021
|
-
"when set, installation tokens replace the PAT",
|
|
2022
|
-
)
|
|
2015
|
+
add_credential_args(parser)
|
|
2023
2016
|
parser.add_argument(
|
|
2024
2017
|
"--key-file",
|
|
2025
2018
|
default="",
|
outerloop/github.py
CHANGED
|
@@ -503,10 +503,20 @@ class GitHubClient:
|
|
|
503
503
|
return True
|
|
504
504
|
|
|
505
505
|
def disable_auto_merge(self, repo: str, number: int) -> bool:
|
|
506
|
-
"""
|
|
507
|
-
|
|
508
|
-
merge the NEW head on green
|
|
509
|
-
(terra #171).
|
|
506
|
+
"""Ensure GitHub auto-merge will not fire on this PR, and report whether
|
|
507
|
+
that is confirmed. The follow-up lane calls this before pushing new
|
|
508
|
+
commits to an auto-mode PR: an armed PR would merge the NEW head on green
|
|
509
|
+
CI without a fresh gate/suite/panel (terra #171).
|
|
510
|
+
|
|
511
|
+
The property to confirm is "auto-merge is not armed". We always run the
|
|
512
|
+
disarm mutation — never skipping it on a prior read, which would let
|
|
513
|
+
another actor arm auto-merge between the read and the push. When nothing
|
|
514
|
+
is armed the mutation errors, and its text varies by the repo's "Allow
|
|
515
|
+
auto-merge" setting and the token type (App vs PAT), so we confirm by
|
|
516
|
+
RE-READING the PR's `auto_merge` state rather than matching a string: if
|
|
517
|
+
auto-merge is off afterward, the safety property holds and pushing is
|
|
518
|
+
safe. Returns False only when auto-merge is still armed and could not be
|
|
519
|
+
disarmed."""
|
|
510
520
|
if self.dry_run:
|
|
511
521
|
log.info("[dry-run] disarm auto-merge on %s#%d", repo, number)
|
|
512
522
|
return True
|
|
@@ -519,12 +529,18 @@ class GitHubClient:
|
|
|
519
529
|
" disablePullRequestAutoMerge(input: {pullRequestId: $pr})"
|
|
520
530
|
" { pullRequest { number } } }"
|
|
521
531
|
)
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
except GitHubError as exc:
|
|
525
|
-
if "not enabled" in str(exc).casefold():
|
|
526
|
-
# nothing was armed — the state we wanted; pushing is safe
|
|
532
|
+
try:
|
|
533
|
+
self._graphql(mutation, {"pr": str(node_id)})
|
|
527
534
|
return True
|
|
535
|
+
except GitHubError as exc:
|
|
536
|
+
# Disarming a PR with nothing armed errors; the message is not a
|
|
537
|
+
# reliable signal (merge:manual repo, "Allow auto-merge" off, or
|
|
538
|
+
# App-token phrasing), so confirm by state: off now = safe.
|
|
539
|
+
if not self.get_pull_request(repo, number).get("auto_merge"):
|
|
540
|
+
return True
|
|
541
|
+
log.warning("auto-merge disarm on %s#%s failed: %s", repo, number, exc)
|
|
542
|
+
return False
|
|
543
|
+
except GitHubError as exc:
|
|
528
544
|
log.warning("auto-merge disarm on %s#%s failed: %s", repo, number, exc)
|
|
529
545
|
return False
|
|
530
546
|
|
|
@@ -768,15 +784,19 @@ class GitHubClient:
|
|
|
768
784
|
)
|
|
769
785
|
return int(data.get("number", 0)) if isinstance(data, dict) else 0
|
|
770
786
|
|
|
771
|
-
def update_issue(self, repo: str, number: int, body: str) -> None:
|
|
772
|
-
"""Replace an issue's body (the rolling maintenance digest)
|
|
787
|
+
def update_issue(self, repo: str, number: int, body: str, *, title: str | None = None) -> None:
|
|
788
|
+
"""Replace an issue's body (the rolling maintenance digest), and its
|
|
789
|
+
title when one is given (the digest carries the scan date there)."""
|
|
773
790
|
if self.dry_run:
|
|
774
791
|
log.info("[dry-run] update issue %s#%s (%d chars)", repo, number, len(body))
|
|
775
792
|
return
|
|
793
|
+
payload: dict[str, str] = {"body": body}
|
|
794
|
+
if title is not None:
|
|
795
|
+
payload["title"] = title
|
|
776
796
|
self._request(
|
|
777
797
|
"PATCH",
|
|
778
798
|
f"/repos/{urllib.parse.quote(repo)}/issues/{number}",
|
|
779
|
-
|
|
799
|
+
payload,
|
|
780
800
|
)
|
|
781
801
|
|
|
782
802
|
def close_issue(self, repo: str, number: int) -> None:
|
outerloop/harness.py
CHANGED
|
@@ -460,6 +460,24 @@ def _int(value: Any, default: int = 0) -> int:
|
|
|
460
460
|
return default
|
|
461
461
|
|
|
462
462
|
|
|
463
|
+
def _kill_and_drain(process: subprocess.Popen[str]) -> str:
|
|
464
|
+
"""Kill a timed-out session's whole process group, then drain its pipe within
|
|
465
|
+
a bound so `run()` always returns — a descendant that left the group (setsid)
|
|
466
|
+
can hold the pipe open past the kill. Returns whatever stdout drained (may be
|
|
467
|
+
empty). Shared by every backend's timeout path; keep the sequence in one place
|
|
468
|
+
so the safety-sensitive termination cannot drift between them."""
|
|
469
|
+
with contextlib.suppress(ProcessLookupError, PermissionError):
|
|
470
|
+
os.killpg(process.pid, signal.SIGKILL)
|
|
471
|
+
try:
|
|
472
|
+
stdout, _ = process.communicate(timeout=10)
|
|
473
|
+
except subprocess.TimeoutExpired:
|
|
474
|
+
process.kill()
|
|
475
|
+
stdout = ""
|
|
476
|
+
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
477
|
+
stdout, _ = process.communicate(timeout=5)
|
|
478
|
+
return stdout or ""
|
|
479
|
+
|
|
480
|
+
|
|
463
481
|
@dataclass
|
|
464
482
|
class ClaudeCodeHarness:
|
|
465
483
|
"""Headless Claude Code (`claude -p`): the JSON output carries cost,
|
|
@@ -625,17 +643,7 @@ class ClaudeCodeHarness:
|
|
|
625
643
|
try:
|
|
626
644
|
stdout, stderr = process.communicate(input=brief_text, timeout=self.timeout_s)
|
|
627
645
|
except subprocess.TimeoutExpired:
|
|
628
|
-
|
|
629
|
-
os.killpg(process.pid, signal.SIGKILL)
|
|
630
|
-
# Bounded drain: a descendant that left the process group (setsid)
|
|
631
|
-
# can hold the pipe open past the kill; run() must still return.
|
|
632
|
-
try:
|
|
633
|
-
stdout, _ = process.communicate(timeout=10)
|
|
634
|
-
except subprocess.TimeoutExpired:
|
|
635
|
-
process.kill()
|
|
636
|
-
stdout = ""
|
|
637
|
-
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
638
|
-
stdout, _ = process.communicate(timeout=5)
|
|
646
|
+
stdout = _kill_and_drain(process)
|
|
639
647
|
path = _write_private(
|
|
640
648
|
workspace.parent, transcript_stem, ".json", redact(stdout or "", (self.api_key,))
|
|
641
649
|
)
|
|
@@ -1077,15 +1085,7 @@ class CodexHarness:
|
|
|
1077
1085
|
try:
|
|
1078
1086
|
stdout, stderr = process.communicate(input=brief_text, timeout=self.timeout_s)
|
|
1079
1087
|
except subprocess.TimeoutExpired:
|
|
1080
|
-
|
|
1081
|
-
os.killpg(process.pid, signal.SIGKILL)
|
|
1082
|
-
try:
|
|
1083
|
-
stdout, _ = process.communicate(timeout=10)
|
|
1084
|
-
except subprocess.TimeoutExpired:
|
|
1085
|
-
process.kill()
|
|
1086
|
-
stdout = ""
|
|
1087
|
-
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1088
|
-
stdout, _ = process.communicate(timeout=5)
|
|
1088
|
+
stdout = _kill_and_drain(process)
|
|
1089
1089
|
path = _write_private(
|
|
1090
1090
|
workspace.parent, transcript_stem, ".jsonl", redact(stdout or "", (self.api_key,))
|
|
1091
1091
|
)
|
|
@@ -1402,15 +1402,7 @@ class HermesHarness:
|
|
|
1402
1402
|
try:
|
|
1403
1403
|
stdout, _ = process.communicate(timeout=self.timeout_s)
|
|
1404
1404
|
except subprocess.TimeoutExpired:
|
|
1405
|
-
|
|
1406
|
-
os.killpg(process.pid, signal.SIGKILL)
|
|
1407
|
-
try:
|
|
1408
|
-
stdout, _ = process.communicate(timeout=10)
|
|
1409
|
-
except subprocess.TimeoutExpired:
|
|
1410
|
-
process.kill()
|
|
1411
|
-
stdout = ""
|
|
1412
|
-
with contextlib.suppress(subprocess.TimeoutExpired):
|
|
1413
|
-
stdout, _ = process.communicate(timeout=5)
|
|
1405
|
+
stdout = _kill_and_drain(process)
|
|
1414
1406
|
path = _write_private(
|
|
1415
1407
|
workspace.parent, transcript_stem, ".log", redact(stdout or "", (self.api_key,))
|
|
1416
1408
|
)
|
outerloop/init.py
CHANGED
|
@@ -367,6 +367,26 @@ def _owner_type(owner: str) -> str:
|
|
|
367
367
|
return ""
|
|
368
368
|
|
|
369
369
|
|
|
370
|
+
def _app_owner_mismatch_note(conversion: dict[str, Any], expected_owner: str, target: str) -> str:
|
|
371
|
+
"""When the App landed under a different account than intended — GitHub's
|
|
372
|
+
fallback when someone who is not an org owner tries to create an org-owned
|
|
373
|
+
App — explain the path that still works, since a personal App does not
|
|
374
|
+
install on an org repo by default. Empty when the App landed as intended."""
|
|
375
|
+
actual = str((conversion.get("owner") or {}).get("login", "")).strip()
|
|
376
|
+
if not actual or actual.casefold() == expected_owner.casefold():
|
|
377
|
+
return ""
|
|
378
|
+
org = target.split("/")[0]
|
|
379
|
+
return (
|
|
380
|
+
f" note: the App was created under '{actual}', not '{expected_owner}'. Creating an\n"
|
|
381
|
+
f" App owned by an organization needs org-owner rights, so GitHub made a personal\n"
|
|
382
|
+
f" one — which will not install on {target} by default. To use it there anyway:\n"
|
|
383
|
+
f" make the App public (its Settings > 'Make public'), request its installation on\n"
|
|
384
|
+
f" {target}, and have an owner of {org} approve it; then rerun\n"
|
|
385
|
+
f" `outerloop init --force --github-app`. For lab ownership, you can later transfer\n"
|
|
386
|
+
f" the App to {org} from its Advanced settings — the key keeps working."
|
|
387
|
+
)
|
|
388
|
+
|
|
389
|
+
|
|
370
390
|
def _app_failure(answers: InitAnswers, slug: str, problem: str) -> int:
|
|
371
391
|
"""The App cannot write the target: say so, keep the credentials, and point
|
|
372
392
|
at the fix and the re-check. Never `start` — the check just failed."""
|
|
@@ -376,7 +396,9 @@ def _app_failure(answers: InitAnswers, slug: str, problem: str) -> int:
|
|
|
376
396
|
f" https://github.com/apps/{slug}/installations/new (or the App's page under\n"
|
|
377
397
|
" Settings > Installations > Configure) install it on that repository and accept\n"
|
|
378
398
|
" contents, issues and pull-request write access, then run\n"
|
|
379
|
-
" `outerloop init --force --github-app` to re-check with these credentials
|
|
399
|
+
" `outerloop init --force --github-app` to re-check with these credentials.\n"
|
|
400
|
+
" If the App is under your personal account and the repository belongs to an\n"
|
|
401
|
+
" organization, make the App public first and have an org owner approve the install.",
|
|
380
402
|
file=sys.stderr,
|
|
381
403
|
)
|
|
382
404
|
return 1
|
|
@@ -449,6 +471,9 @@ def _github_app_setup(
|
|
|
449
471
|
pem_path, app_json = appmanifest.save_app_creds(conversion, CONFIG_DIR)
|
|
450
472
|
repo = answers.target.split("/", 1)[-1]
|
|
451
473
|
print(f" created App '{conversion['slug']}'; credentials in {app_json} and {pem_path} (0600)")
|
|
474
|
+
mismatch = _app_owner_mismatch_note(conversion, owner, answers.target)
|
|
475
|
+
if mismatch:
|
|
476
|
+
print(mismatch, file=sys.stderr)
|
|
452
477
|
print()
|
|
453
478
|
print(f"Step 2 of 3: install the App on {answers.target}.")
|
|
454
479
|
print(f" Open {appmanifest.install_url(conversion)}")
|