outerloop-science 0.1.0.dev1__py3-none-any.whl → 0.1.0.dev3__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +8 -8
- outerloop/appauth.py +3 -3
- outerloop/appmanifest.py +1 -1
- outerloop/attempt.py +350 -127
- outerloop/brief.py +38 -25
- outerloop/cli.py +189 -78
- outerloop/climbboard.py +3 -0
- outerloop/compute.py +178 -33
- outerloop/contract.py +11 -2
- outerloop/dispatch.py +63 -18
- outerloop/evalcache.py +147 -0
- outerloop/followup.py +52 -33
- outerloop/github.py +42 -17
- outerloop/harness.py +19 -25
- outerloop/housekeeping.py +1 -17
- outerloop/image.py +368 -0
- outerloop/init.py +248 -15
- outerloop/intake.py +4 -7
- outerloop/launchlog.py +239 -0
- outerloop/limits.py +1 -1
- outerloop/maintain.py +325 -0
- outerloop/maintain_agent_cli.py +81 -0
- outerloop/maintain_post_cli.py +140 -0
- outerloop/measure.py +8 -2
- outerloop/orchestrator.py +142 -32
- outerloop/panel.py +3 -3
- outerloop/paths.py +2 -1
- outerloop/review.py +5 -1
- outerloop/review_agent.py +7 -7
- outerloop/review_agent_cli.py +2 -2
- outerloop/review_post_cli.py +2 -2
- outerloop/review_summarize_cli.py +7 -5
- outerloop/roles.py +27 -0
- outerloop/rolespec.py +3 -1
- outerloop/steward.py +8 -8
- outerloop/syscall.py +262 -47
- outerloop/syscall_cli.py +243 -12
- outerloop/tick.py +240 -103
- outerloop/verify_agent.py +8 -6
- outerloop/verify_post_cli.py +2 -2
- outerloop/watcher.py +203 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev3.dist-info}/METADATA +5 -2
- outerloop_science-0.1.0.dev3.dist-info/RECORD +59 -0
- outerloop_science-0.1.0.dev1.dist-info/RECORD +0 -52
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev3.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev3.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev3.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev1.dist-info → outerloop_science-0.1.0.dev3.dist-info}/licenses/NOTICE +0 -0
outerloop/attempt.py
CHANGED
|
@@ -18,6 +18,7 @@ import logging
|
|
|
18
18
|
import os
|
|
19
19
|
import re
|
|
20
20
|
import shutil
|
|
21
|
+
import time
|
|
21
22
|
from collections.abc import Callable, Iterable
|
|
22
23
|
from dataclasses import dataclass
|
|
23
24
|
from dataclasses import replace as dc_replace
|
|
@@ -27,7 +28,7 @@ from typing import Any, cast
|
|
|
27
28
|
|
|
28
29
|
from outerloop.appauth import resolve_bot_auth
|
|
29
30
|
from outerloop.brief import BudgetState, distill_lessons
|
|
30
|
-
from outerloop.compute import LocalCompute
|
|
31
|
+
from outerloop.compute import LocalCompute
|
|
31
32
|
from outerloop.contract import Benchmark, Contract, contract_text_in_tree, load_contract
|
|
32
33
|
from outerloop.dispatch import (
|
|
33
34
|
Snapshot,
|
|
@@ -36,6 +37,7 @@ from outerloop.dispatch import (
|
|
|
36
37
|
should_dispatch,
|
|
37
38
|
snapshot_tree,
|
|
38
39
|
)
|
|
40
|
+
from outerloop.evalcache import seed_dir
|
|
39
41
|
from outerloop.github import (
|
|
40
42
|
GitError,
|
|
41
43
|
GitHubClient,
|
|
@@ -43,8 +45,10 @@ from outerloop.github import (
|
|
|
43
45
|
Workspace,
|
|
44
46
|
contract_at,
|
|
45
47
|
ensure_regular_git_dir,
|
|
48
|
+
git_identity,
|
|
46
49
|
)
|
|
47
|
-
from outerloop.harness import Harness, SessionResult, redact
|
|
50
|
+
from outerloop.harness import Harness, SessionResult, default_binary, redact
|
|
51
|
+
from outerloop.launchlog import append_ended, append_submitted, experiments_rows
|
|
48
52
|
from outerloop.markers import has_marker
|
|
49
53
|
from outerloop.measure import DispatchedMeasurer, DispatchSettings
|
|
50
54
|
from outerloop.orchestrator import (
|
|
@@ -84,9 +88,20 @@ from outerloop.runstate import (
|
|
|
84
88
|
save_record,
|
|
85
89
|
stamp_outage,
|
|
86
90
|
)
|
|
87
|
-
from outerloop.
|
|
91
|
+
from outerloop.runstate import (
|
|
92
|
+
run_dir as run_dir_of,
|
|
93
|
+
)
|
|
94
|
+
from outerloop.syscall import (
|
|
95
|
+
CHANNEL_DIR_NAMES,
|
|
96
|
+
MAX_ARTIFACT_BYTES,
|
|
97
|
+
SyscallRequest,
|
|
98
|
+
channel_dir,
|
|
99
|
+
launch_task_ids,
|
|
100
|
+
tool_update_note,
|
|
101
|
+
)
|
|
88
102
|
from outerloop.syscall import ensure_excluded as syscall_excluded
|
|
89
103
|
from outerloop.syscall import install_tool as syscall_install_tool
|
|
104
|
+
from outerloop.syscall import refresh_tool as syscall_refresh_tool
|
|
90
105
|
from outerloop.syscall import write_budget as syscall_write_budget
|
|
91
106
|
from outerloop.syscall import write_siblings as syscall_write_siblings
|
|
92
107
|
from outerloop.verifier import MAX_CLAIM_CHARS
|
|
@@ -110,18 +125,19 @@ def resolve_author_key_file(backend: str, explicit: str = "") -> str:
|
|
|
110
125
|
config choice, not a key swap, and an in-flight run of either backend can
|
|
111
126
|
still be woken/serviced after a fleet flip. An explicit path always wins;
|
|
112
127
|
otherwise the per-backend env var, then the default path. For claude the
|
|
113
|
-
legacy spellings (`AUTORESEARCH_HARNESS_KEY_FILE`,
|
|
128
|
+
legacy spellings (`AUTORESEARCH_HARNESS_KEY_FILE`, read through its bridged
|
|
129
|
+
`OUTERLOOP_` twin, and `harness_key`) are still
|
|
114
130
|
honored, so a machine set up before the rename keeps working; the new name
|
|
115
131
|
wins when both exist. The result is always ~-expanded, so every caller gets
|
|
116
132
|
a real path (an env value like "~/.config/..." must not reach the token
|
|
117
133
|
provider verbatim)."""
|
|
118
134
|
if not explicit:
|
|
119
135
|
if backend == "codex":
|
|
120
|
-
explicit = os.environ.get("
|
|
136
|
+
explicit = os.environ.get("OUTERLOOP_CODEX_KEY_FILE") or CODEX_KEY_DEFAULT
|
|
121
137
|
else:
|
|
122
138
|
explicit = (
|
|
123
|
-
os.environ.get("
|
|
124
|
-
or os.environ.get("
|
|
139
|
+
os.environ.get("OUTERLOOP_CLAUDE_KEY_FILE")
|
|
140
|
+
or os.environ.get("OUTERLOOP_HARNESS_KEY_FILE")
|
|
125
141
|
or ""
|
|
126
142
|
)
|
|
127
143
|
if not explicit:
|
|
@@ -140,14 +156,14 @@ def codex_author_config_error(backend: str, model: str, image: str) -> str:
|
|
|
140
156
|
a unit and never a fleet backend against a run's model. codex writes+executes,
|
|
141
157
|
so it must be contained (--image) and needs a non-claude model."""
|
|
142
158
|
if backend not in ("claude", "codex"):
|
|
143
|
-
# a typo'd
|
|
159
|
+
# a typo'd OUTERLOOP_AUTHOR_BACKEND passes the env DEFAULT silently
|
|
144
160
|
# (argparse validates the flag, not its default) and the climb rejects it
|
|
145
161
|
# at build_harness — catch it on the tick host so a claimed intake
|
|
146
162
|
# issue never strands on it
|
|
147
163
|
return f"unknown author backend {backend!r} (expected 'claude' or 'codex')"
|
|
148
164
|
if backend == "claude":
|
|
149
165
|
# symmetric to the codex check: a claude harness 404s on a non-claude
|
|
150
|
-
# model (e.g.
|
|
166
|
+
# model (e.g. OUTERLOOP_AUTHOR_MODEL left on a codex id while the
|
|
151
167
|
# backend is claude) — catch that misconfig before spend
|
|
152
168
|
if model and not model.startswith("claude"):
|
|
153
169
|
return f"author-backend claude needs a claude model (got {model!r})"
|
|
@@ -185,7 +201,7 @@ class WorkspaceDrift(RuntimeError):
|
|
|
185
201
|
"""The tree changed between measurement and commit."""
|
|
186
202
|
|
|
187
203
|
|
|
188
|
-
def
|
|
204
|
+
def target_clone_url(target: str) -> str:
|
|
189
205
|
"""The canonical HTTPS clone URL for `owner/repo`. The one source of truth
|
|
190
206
|
for where a run's git pushes go — derived from the target, never read from
|
|
191
207
|
the session-writable `remote.origin.url`."""
|
|
@@ -205,7 +221,8 @@ def _blessed_head(ws: Workspace, result: Any, contract: Any) -> str:
|
|
|
205
221
|
return ""
|
|
206
222
|
try:
|
|
207
223
|
return ws.git("rev-parse", "HEAD").strip()
|
|
208
|
-
except Exception:
|
|
224
|
+
except Exception as exc:
|
|
225
|
+
log.warning("could not read HEAD; not arming self-merge: %s", exc)
|
|
209
226
|
return ""
|
|
210
227
|
|
|
211
228
|
|
|
@@ -424,9 +441,14 @@ def _park_run(
|
|
|
424
441
|
# it lands in the durable record, like every other persisted final_text
|
|
425
442
|
# — a session that echoed a credential must not leave it in record.json.
|
|
426
443
|
# Empty/zero for a baseline park (the session has not run yet).
|
|
427
|
-
|
|
428
|
-
|
|
429
|
-
|
|
444
|
+
# the author's report at submit, else the session's last words (a park
|
|
445
|
+
# with no submit); either way what the wake's panel and the PR body read
|
|
446
|
+
"report": redact(
|
|
447
|
+
parked.syscall.report
|
|
448
|
+
if parked.syscall is not None and parked.syscall.report
|
|
449
|
+
else (parked.session.final_text if parked.session else ""),
|
|
450
|
+
secrets,
|
|
451
|
+
)[:MAX_CLAIM_CHARS],
|
|
430
452
|
"session_cost_usd": parked.session.cost_usd if parked.session else 0.0,
|
|
431
453
|
"session_turns": parked.session.num_turns if parked.session else 0,
|
|
432
454
|
}
|
|
@@ -451,10 +473,35 @@ def _park_run(
|
|
|
451
473
|
"minutes": launch.minutes,
|
|
452
474
|
"artifacts": list(launch.artifacts),
|
|
453
475
|
**({"array": launch.array} if launch.array > 1 else {}),
|
|
476
|
+
**({"why": redact(launch.why, secrets)} if launch.why else {}),
|
|
477
|
+
**({"concurrency": launch.concurrency} if launch.concurrency else {}),
|
|
454
478
|
}
|
|
455
479
|
for launch in parked.syscall.launches
|
|
456
480
|
]
|
|
457
481
|
stage["syscall_note"] = redact(parked.syscall.note, secrets)
|
|
482
|
+
if parked.syscall.launches:
|
|
483
|
+
# the run's launch ledger (`history`, and the queue view's labels):
|
|
484
|
+
# ids align with launch_jobs order, as stage_launch_job_ids reads them
|
|
485
|
+
if parked.launch_afterany:
|
|
486
|
+
launch_ids = afterany_ids(parked.launch_afterany)
|
|
487
|
+
elif parked.phase == "author-sleep":
|
|
488
|
+
launch_ids = list(job_ids)
|
|
489
|
+
else:
|
|
490
|
+
launch_ids = []
|
|
491
|
+
ledger_launches = tuple(
|
|
492
|
+
dc_replace(launch, why=redact(launch.why, secrets))
|
|
493
|
+
for launch in parked.syscall.launches
|
|
494
|
+
)
|
|
495
|
+
_best_effort(
|
|
496
|
+
"launch ledger",
|
|
497
|
+
lambda: append_submitted(
|
|
498
|
+
run_dir_of(run_root, record.run_id),
|
|
499
|
+
sleep=parked.sleeps_used,
|
|
500
|
+
launches=ledger_launches,
|
|
501
|
+
job_ids=launch_ids,
|
|
502
|
+
at=now,
|
|
503
|
+
),
|
|
504
|
+
)
|
|
458
505
|
# (the session id the wake resumes is the record's own
|
|
459
506
|
# resume_session_id, set below for every park — no stage duplicate)
|
|
460
507
|
stage["launches_used"] = parked.launches_used
|
|
@@ -585,6 +632,7 @@ def _dispatch_settings(args: argparse.Namespace) -> DispatchSettings:
|
|
|
585
632
|
#174: the wake dropped the GPU lane)."""
|
|
586
633
|
from outerloop.compute import compute_from_env
|
|
587
634
|
|
|
635
|
+
target = getattr(args, "target", "") or ""
|
|
588
636
|
return DispatchSettings(
|
|
589
637
|
compute=compute_from_env(),
|
|
590
638
|
image=args.image,
|
|
@@ -592,9 +640,34 @@ def _dispatch_settings(args: argparse.Namespace) -> DispatchSettings:
|
|
|
592
640
|
partition=args.partition,
|
|
593
641
|
gpu_partition=getattr(args, "gpu_partition", ""),
|
|
594
642
|
gpu_account=getattr(args, "gpu_account", ""),
|
|
643
|
+
seed_cache=seed_dir(Path(args.run_root), target) if target else None,
|
|
595
644
|
)
|
|
596
645
|
|
|
597
646
|
|
|
647
|
+
# Experiments yield to verification. Every kernel job is one Slurm user, so
|
|
648
|
+
# among the kernel's own pending jobs the priority order is ours: launches
|
|
649
|
+
# carry this nice so a gate eval or a follow-up re-measure (nice 0) starts
|
|
650
|
+
# first when the cap frees a slot. Sized above the factors that differ between
|
|
651
|
+
# our jobs on Torch — age tops out at 1000 after a week, job size at 1000, the
|
|
652
|
+
# per-GPU TRES share stays in the hundreds — so the order holds however long a
|
|
653
|
+
# launch has waited. Other users' jobs and the group cap are untouched.
|
|
654
|
+
LAUNCH_NICE = 5000
|
|
655
|
+
|
|
656
|
+
|
|
657
|
+
def with_seed(dispatch: DispatchSettings, run_root: Path, target: str) -> DispatchSettings:
|
|
658
|
+
"""These settings with the target's seed cache filled in from the record's
|
|
659
|
+
target when the CLI gave none (wake and follow-up jobs carry the run id,
|
|
660
|
+
not the target)."""
|
|
661
|
+
# tolerant of any settings object: a backend that knows no seed (or a
|
|
662
|
+
# test double) is left exactly as it is
|
|
663
|
+
if not target or getattr(dispatch, "seed_cache", "unknown") is not None:
|
|
664
|
+
return dispatch
|
|
665
|
+
try:
|
|
666
|
+
return dc_replace(dispatch, seed_cache=seed_dir(run_root, target))
|
|
667
|
+
except TypeError:
|
|
668
|
+
return dispatch
|
|
669
|
+
|
|
670
|
+
|
|
598
671
|
def _make_launcher(
|
|
599
672
|
dispatch: DispatchSettings, run_dir: Path, workspace: Path, run_id: str, gpus: int = 0
|
|
600
673
|
):
|
|
@@ -607,38 +680,39 @@ def _make_launcher(
|
|
|
607
680
|
|
|
608
681
|
def launcher(sha: str, request: SyscallRequest) -> str:
|
|
609
682
|
from outerloop.dispatch import eval_job_spec, write_eval_job
|
|
610
|
-
from outerloop.syscall import
|
|
683
|
+
from outerloop.syscall import array_spec
|
|
611
684
|
|
|
612
685
|
ids: list[str] = []
|
|
613
686
|
try:
|
|
614
687
|
for launch in request.launches:
|
|
615
|
-
#
|
|
616
|
-
#
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
688
|
+
# a sweep is ONE Slurm job array (`--array=0-N%K`): the queue
|
|
689
|
+
# holds one entry, Slurm runs at most K tasks at once, each task
|
|
690
|
+
# derives its job dir and SWEEP_INDEX from its array index, and
|
|
691
|
+
# one afterany on the array id covers every task
|
|
692
|
+
script = write_eval_job(
|
|
693
|
+
run_dir,
|
|
694
|
+
f"launch-{launch.name}",
|
|
695
|
+
repo_root=workspace,
|
|
696
|
+
snapshot_sha=sha,
|
|
697
|
+
command=launch.command,
|
|
698
|
+
image=dispatch.image,
|
|
699
|
+
artifacts=launch.artifacts,
|
|
700
|
+
artifact_max_bytes=MAX_ARTIFACT_BYTES,
|
|
701
|
+
gpus=gpus,
|
|
702
|
+
array=launch.array,
|
|
703
|
+
seed_cache=dispatch.seed_cache,
|
|
704
|
+
)
|
|
705
|
+
spec = eval_job_spec(
|
|
706
|
+
script,
|
|
707
|
+
job_name=f"{run_id}-launch-{launch.name}",
|
|
708
|
+
account=account,
|
|
709
|
+
partition=partition,
|
|
710
|
+
eval_minutes=launch.minutes,
|
|
711
|
+
gpus=gpus,
|
|
712
|
+
nice=LAUNCH_NICE,
|
|
713
|
+
array=array_spec(launch),
|
|
714
|
+
)
|
|
715
|
+
ids.append(dispatch.compute.submit(spec))
|
|
642
716
|
except Exception:
|
|
643
717
|
# a partial batch must not orphan: no park record was written yet,
|
|
644
718
|
# so nothing would ever wake or cancel the jobs that DID submit —
|
|
@@ -656,6 +730,26 @@ def _make_launcher(
|
|
|
656
730
|
return launcher
|
|
657
731
|
|
|
658
732
|
|
|
733
|
+
def _make_watcher(
|
|
734
|
+
dispatch: DispatchSettings, run_root: Path, run_id: str, workspace: Path, config: RunConfig
|
|
735
|
+
) -> Callable[[], Any]:
|
|
736
|
+
"""The session watcher for one run: a thread beside the harness that
|
|
737
|
+
answers `queue` and `history` from the channel (docs/design/session-watcher.md).
|
|
738
|
+
Shared by the first pass and every wake leg."""
|
|
739
|
+
from outerloop.watcher import SessionWatcher, WatcherContext
|
|
740
|
+
|
|
741
|
+
ctx = WatcherContext(
|
|
742
|
+
workspace=workspace,
|
|
743
|
+
run_root=run_root,
|
|
744
|
+
run_id=run_id,
|
|
745
|
+
target=config.target,
|
|
746
|
+
agent_id=config.agent_id,
|
|
747
|
+
compute=dispatch.compute,
|
|
748
|
+
gpu_partition=dispatch.gpu_partition,
|
|
749
|
+
)
|
|
750
|
+
return lambda: SessionWatcher(ctx)
|
|
751
|
+
|
|
752
|
+
|
|
659
753
|
def _wake_author_sleep(
|
|
660
754
|
*,
|
|
661
755
|
run_root: Path,
|
|
@@ -708,7 +802,12 @@ def _wake_author_sleep(
|
|
|
708
802
|
# the same ending shape every other terminal takes. The line notebook
|
|
709
803
|
# records it first, while the tree is still the session's final tree.
|
|
710
804
|
_push_line_snapshot(
|
|
711
|
-
ws,
|
|
805
|
+
ws,
|
|
806
|
+
_line_ref_for(bench, config.agent_id),
|
|
807
|
+
run_id,
|
|
808
|
+
result.outcome,
|
|
809
|
+
secrets,
|
|
810
|
+
bot_login=config.bot_login,
|
|
712
811
|
)
|
|
713
812
|
for ref in drop_refs:
|
|
714
813
|
drop_snapshot(ws, Snapshot(commit="", tree="", ref=ref))
|
|
@@ -771,6 +870,8 @@ def _wake_author_sleep(
|
|
|
771
870
|
minutes=int(item.get("minutes") or 1),
|
|
772
871
|
artifacts=tuple(str(a) for a in item.get("artifacts", [])),
|
|
773
872
|
array=int(item.get("array") or 1),
|
|
873
|
+
why=str(item.get("why") or ""),
|
|
874
|
+
concurrency=int(item.get("concurrency") or 0),
|
|
774
875
|
)
|
|
775
876
|
for item in _stage_launches(record)
|
|
776
877
|
)
|
|
@@ -781,12 +882,28 @@ def _wake_author_sleep(
|
|
|
781
882
|
# of a blank "job failure". The park's launch job ids align positionally
|
|
782
883
|
# with the results (same launch/array order). Best-effort — the wake never
|
|
783
884
|
# blocks on the scheduler query.
|
|
885
|
+
task_ids = launch_task_ids(launches, stage_launch_job_ids(record))
|
|
784
886
|
status_of = getattr(dispatch.compute, "status", None)
|
|
785
887
|
if status_of is not None:
|
|
786
|
-
results = annotate_launch_states(results,
|
|
888
|
+
results = annotate_launch_states(results, task_ids, status_of)
|
|
787
889
|
launches_used = int(record.stage.get("launches_used", 0)) # type: ignore[call-overload]
|
|
788
890
|
sleeps_used = int(record.stage.get("sleeps_used", 0)) # type: ignore[call-overload]
|
|
789
|
-
|
|
891
|
+
elapsed = _launch_elapsed(dispatch, task_ids) if task_ids else None
|
|
892
|
+
_best_effort(
|
|
893
|
+
"launch ledger",
|
|
894
|
+
lambda: append_ended(
|
|
895
|
+
run_dir, sleep=sleeps_used, results=results, at=time.time(), elapsed_seconds=elapsed
|
|
896
|
+
),
|
|
897
|
+
)
|
|
898
|
+
gpu_hours_used = _reconcile_launch_hours(record, dispatch, bench.gpus, launches, elapsed)
|
|
899
|
+
# the tool the session invokes comes from THIS kernel: a session that
|
|
900
|
+
# started under an older one gets today's verbs and flags at its wake, and
|
|
901
|
+
# is told what is new
|
|
902
|
+
tool_changed = False
|
|
903
|
+
try:
|
|
904
|
+
tool_changed = syscall_refresh_tool(workspace)
|
|
905
|
+
except Exception as exc:
|
|
906
|
+
log.warning("tool refresh failed: %s", redact(f"{type(exc).__name__}: {exc}", secrets))
|
|
790
907
|
wake_text = render_wake(
|
|
791
908
|
results,
|
|
792
909
|
str(record.stage.get("syscall_note", "")),
|
|
@@ -799,6 +916,15 @@ def _wake_author_sleep(
|
|
|
799
916
|
),
|
|
800
917
|
gpus=bench.gpus,
|
|
801
918
|
)
|
|
919
|
+
pacing = [
|
|
920
|
+
f"sweep `{la.name}`: {la.array} tasks, at most {la.concurrency or la.array} at a time"
|
|
921
|
+
for la in launches
|
|
922
|
+
if la.array > 1
|
|
923
|
+
]
|
|
924
|
+
if pacing:
|
|
925
|
+
wake_text = f"{wake_text}\n\n" + "\n".join(pacing) + " (the contract's ceiling applies)."
|
|
926
|
+
if tool_changed:
|
|
927
|
+
wake_text = f"{wake_text}\n\n{tool_update_note(channel_dir(workspace))}"
|
|
802
928
|
if extra_update:
|
|
803
929
|
# a submitted park's gate/panel feedback leads; launch results follow
|
|
804
930
|
wake_text = f"{extra_update}\n\n{wake_text}"
|
|
@@ -824,7 +950,9 @@ def _wake_author_sleep(
|
|
|
824
950
|
wake_line = _line_ref_for(bench, config.agent_id)
|
|
825
951
|
|
|
826
952
|
def snapshot() -> str:
|
|
827
|
-
snap = snapshot_tree(
|
|
953
|
+
snap = snapshot_tree(
|
|
954
|
+
ws, base_sha, exclude=LINE_MEMORY_PATHS if wake_line else (), author=config.bot_login
|
|
955
|
+
)
|
|
828
956
|
snapshots.append(snap)
|
|
829
957
|
return snap.commit
|
|
830
958
|
|
|
@@ -845,6 +973,7 @@ def _wake_author_sleep(
|
|
|
845
973
|
config.bot_login,
|
|
846
974
|
_utc_date(now),
|
|
847
975
|
exclude=LINE_MEMORY_PATHS if wake_line else (),
|
|
976
|
+
secrets=secrets,
|
|
848
977
|
)
|
|
849
978
|
if panel_lenses
|
|
850
979
|
else None
|
|
@@ -868,6 +997,7 @@ def _wake_author_sleep(
|
|
|
868
997
|
resume_session_id=record.resume_session_id,
|
|
869
998
|
improve_prompt=wake_text,
|
|
870
999
|
launcher=_make_launcher(dispatch, run_dir, workspace, run_id, gpus=bench.gpus),
|
|
1000
|
+
watcher=_make_watcher(dispatch, run_root, run_id, workspace, config),
|
|
871
1001
|
tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(),
|
|
872
1002
|
judged=judged or _stage_judged(record),
|
|
873
1003
|
launches_used=launches_used,
|
|
@@ -932,6 +1062,32 @@ def _stage_judged(record: RunRecord) -> tuple[str, AttemptResult] | None:
|
|
|
932
1062
|
)
|
|
933
1063
|
|
|
934
1064
|
|
|
1065
|
+
def _ledger_ended(
|
|
1066
|
+
run_dir: Path,
|
|
1067
|
+
record: RunRecord,
|
|
1068
|
+
launches: tuple,
|
|
1069
|
+
task_ids: list[str],
|
|
1070
|
+
dispatch: DispatchSettings,
|
|
1071
|
+
elapsed: list[int | None] | None,
|
|
1072
|
+
) -> None:
|
|
1073
|
+
"""Record a park's finished launches in the ledger from the run dir alone
|
|
1074
|
+
(no delivery into a workspace), with the scheduler's state for jobs that
|
|
1075
|
+
left no exit code."""
|
|
1076
|
+
from outerloop.syscall import annotate_launch_states, read_results
|
|
1077
|
+
|
|
1078
|
+
results = read_results(run_dir, launches)
|
|
1079
|
+
status_of = getattr(dispatch.compute, "status", None)
|
|
1080
|
+
if status_of is not None:
|
|
1081
|
+
results = annotate_launch_states(results, task_ids, status_of)
|
|
1082
|
+
append_ended(
|
|
1083
|
+
run_dir,
|
|
1084
|
+
sleep=int(record.stage.get("sleeps_used", 0)), # type: ignore[call-overload]
|
|
1085
|
+
results=results,
|
|
1086
|
+
at=time.time(),
|
|
1087
|
+
elapsed_seconds=elapsed,
|
|
1088
|
+
)
|
|
1089
|
+
|
|
1090
|
+
|
|
935
1091
|
def _stage_syscall_launches(record: RunRecord) -> tuple:
|
|
936
1092
|
"""The park's launches as `Launch` values (command elided: they ran)."""
|
|
937
1093
|
from outerloop.syscall import Launch
|
|
@@ -943,12 +1099,14 @@ def _stage_syscall_launches(record: RunRecord) -> tuple:
|
|
|
943
1099
|
minutes=int(item.get("minutes") or 1),
|
|
944
1100
|
artifacts=tuple(str(a) for a in item.get("artifacts", [])),
|
|
945
1101
|
array=int(item.get("array") or 1),
|
|
1102
|
+
why=str(item.get("why") or ""),
|
|
1103
|
+
concurrency=int(item.get("concurrency") or 0),
|
|
946
1104
|
)
|
|
947
1105
|
for item in _stage_launches(record)
|
|
948
1106
|
)
|
|
949
1107
|
|
|
950
1108
|
|
|
951
|
-
def
|
|
1109
|
+
def stage_launch_job_ids(record: RunRecord) -> list[str]:
|
|
952
1110
|
"""The park's launch jobs: `launch_afterany` when the park recorded it;
|
|
953
1111
|
for an older author-sleep park every waited job was a launch; for an
|
|
954
1112
|
older candidate park the gate's evals are mixed in, so none."""
|
|
@@ -961,7 +1119,11 @@ def _stage_launch_job_ids(record: RunRecord) -> list[str]:
|
|
|
961
1119
|
|
|
962
1120
|
|
|
963
1121
|
def _reconcile_launch_hours(
|
|
964
|
-
record: RunRecord,
|
|
1122
|
+
record: RunRecord,
|
|
1123
|
+
dispatch: DispatchSettings,
|
|
1124
|
+
gpus: int,
|
|
1125
|
+
launches: tuple,
|
|
1126
|
+
elapsed: list[int | None] | None = None,
|
|
965
1127
|
) -> float:
|
|
966
1128
|
"""The run's GPU-hours after handing back the unused walltime of the
|
|
967
1129
|
park's launch jobs — once: the stage remembers the refund, so a wake
|
|
@@ -971,7 +1133,9 @@ def _reconcile_launch_hours(
|
|
|
971
1133
|
used = float(stage.get("gpu_hours_used", 0.0)) # type: ignore[arg-type]
|
|
972
1134
|
if not gpus or stage.get("launch_hours_refunded"):
|
|
973
1135
|
return used
|
|
974
|
-
refund = _launch_refund(
|
|
1136
|
+
refund = _launch_refund(
|
|
1137
|
+
dispatch, launches, launch_task_ids(launches, stage_launch_job_ids(record)), gpus, elapsed
|
|
1138
|
+
)
|
|
975
1139
|
if refund > 0:
|
|
976
1140
|
log.info("%s: refunding %.2f GPU-hours of unused launch walltime", record.run_id, refund)
|
|
977
1141
|
used = max(0.0, used - refund)
|
|
@@ -980,21 +1144,35 @@ def _reconcile_launch_hours(
|
|
|
980
1144
|
return used
|
|
981
1145
|
|
|
982
1146
|
|
|
1147
|
+
def _launch_elapsed(dispatch: DispatchSettings, job_ids: list[str]) -> list[int | None] | None:
|
|
1148
|
+
"""How long each launch job ran, from the compute, aligned with `job_ids`;
|
|
1149
|
+
None when the compute cannot say (nothing is refunded or recorded on a
|
|
1150
|
+
guess)."""
|
|
1151
|
+
query = getattr(dispatch.compute, "elapsed_seconds", None)
|
|
1152
|
+
if query is None or not job_ids:
|
|
1153
|
+
return None
|
|
1154
|
+
try:
|
|
1155
|
+
return [query(jid) for jid in job_ids]
|
|
1156
|
+
except Exception as exc:
|
|
1157
|
+
log.warning("launch walltime unknown (%s: %s)", type(exc).__name__, exc)
|
|
1158
|
+
return None
|
|
1159
|
+
|
|
1160
|
+
|
|
983
1161
|
def _launch_refund(
|
|
984
|
-
dispatch: DispatchSettings,
|
|
1162
|
+
dispatch: DispatchSettings,
|
|
1163
|
+
launches: tuple,
|
|
1164
|
+
job_ids: list[str],
|
|
1165
|
+
gpus: int,
|
|
1166
|
+
elapsed: list[int | None] | None = None,
|
|
985
1167
|
) -> float:
|
|
986
1168
|
"""The unused walltime of a park's launch jobs, in GPU-hours, or 0 when
|
|
987
1169
|
the compute cannot say how long they ran (nothing is refunded on a
|
|
988
|
-
guess)."""
|
|
1170
|
+
guess). `elapsed` may be handed in when the caller already asked."""
|
|
989
1171
|
from outerloop.syscall import launch_hours_refund
|
|
990
1172
|
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
try:
|
|
995
|
-
elapsed = [query(jid) for jid in job_ids]
|
|
996
|
-
except Exception as exc:
|
|
997
|
-
log.warning("launch walltime unknown (%s: %s); nothing refunded", type(exc).__name__, exc)
|
|
1173
|
+
if elapsed is None:
|
|
1174
|
+
elapsed = _launch_elapsed(dispatch, job_ids)
|
|
1175
|
+
if elapsed is None:
|
|
998
1176
|
return 0.0
|
|
999
1177
|
return launch_hours_refund(launches, elapsed, gpus=gpus)
|
|
1000
1178
|
|
|
@@ -1119,7 +1297,12 @@ def _line_ref_for(bench: Benchmark | None, agent_id: str) -> str:
|
|
|
1119
1297
|
|
|
1120
1298
|
|
|
1121
1299
|
def _push_line_snapshot(
|
|
1122
|
-
ws: Workspace,
|
|
1300
|
+
ws: Workspace,
|
|
1301
|
+
line_ref: str,
|
|
1302
|
+
run_id: str,
|
|
1303
|
+
outcome: str,
|
|
1304
|
+
secrets: tuple[str, ...] = (),
|
|
1305
|
+
bot_login: str = "",
|
|
1123
1306
|
) -> None:
|
|
1124
1307
|
"""Publish the session's final tree to the agent's line as a sealed
|
|
1125
1308
|
snapshot commit — every terminal path, any outcome
|
|
@@ -1162,7 +1345,7 @@ def _push_line_snapshot(
|
|
|
1162
1345
|
fork = parent = remote
|
|
1163
1346
|
except Exception as exc:
|
|
1164
1347
|
log.info("line %s: sealing on the local ref (%s)", line_ref, type(exc).__name__)
|
|
1165
|
-
snap = snapshot_tree(ws, parent, force=memory)
|
|
1348
|
+
snap = snapshot_tree(ws, parent, force=memory, author=bot_login)
|
|
1166
1349
|
try:
|
|
1167
1350
|
# seal only when the tree moved past the parent; the PUSH runs
|
|
1168
1351
|
# either way — a session that COMMITTED its work advanced the
|
|
@@ -1171,10 +1354,7 @@ def _push_line_snapshot(
|
|
|
1171
1354
|
sealed = parent
|
|
1172
1355
|
if snap.tree != ws.git("rev-parse", f"{parent}^{{tree}}").strip():
|
|
1173
1356
|
sealed = ws.git(
|
|
1174
|
-
|
|
1175
|
-
"user.name=autoresearch",
|
|
1176
|
-
"-c",
|
|
1177
|
-
"user.email=autoresearch@localhost",
|
|
1357
|
+
*git_identity(bot_login),
|
|
1178
1358
|
"commit-tree",
|
|
1179
1359
|
snap.tree,
|
|
1180
1360
|
"-p",
|
|
@@ -1234,7 +1414,9 @@ def _reconcile_with_remote(ws: Workspace, old: str, new: str) -> None:
|
|
|
1234
1414
|
ws.git("checkout", new, "--", path)
|
|
1235
1415
|
|
|
1236
1416
|
|
|
1237
|
-
def _checkout_line(
|
|
1417
|
+
def _checkout_line(
|
|
1418
|
+
ws: Workspace, workspace: Path, agent_id: str, base_branch: str, bot_login: str = ""
|
|
1419
|
+
) -> str:
|
|
1238
1420
|
"""Check out the agent's research line: the persistent branch
|
|
1239
1421
|
`agents/<agent-id>`, created from the base branch when absent, with the
|
|
1240
1422
|
base branch merged in when it exists — a conflicted merge is left in the
|
|
@@ -1254,10 +1436,7 @@ def _checkout_line(ws: Workspace, workspace: Path, agent_id: str, base_branch: s
|
|
|
1254
1436
|
conflicted = False
|
|
1255
1437
|
try:
|
|
1256
1438
|
ws.git(
|
|
1257
|
-
|
|
1258
|
-
"user.name=autoresearch",
|
|
1259
|
-
"-c",
|
|
1260
|
-
"user.email=autoresearch@localhost",
|
|
1439
|
+
*git_identity(bot_login),
|
|
1261
1440
|
"merge",
|
|
1262
1441
|
"--no-edit",
|
|
1263
1442
|
base_ref,
|
|
@@ -1272,10 +1451,7 @@ def _checkout_line(ws: Workspace, workspace: Path, agent_id: str, base_branch: s
|
|
|
1272
1451
|
ws.git("add", "-A")
|
|
1273
1452
|
if ws.git("status", "--porcelain").strip():
|
|
1274
1453
|
ws.git(
|
|
1275
|
-
|
|
1276
|
-
"user.name=autoresearch",
|
|
1277
|
-
"-c",
|
|
1278
|
-
"user.email=autoresearch@localhost",
|
|
1454
|
+
*git_identity(bot_login),
|
|
1279
1455
|
"commit",
|
|
1280
1456
|
"-q",
|
|
1281
1457
|
"-m",
|
|
@@ -1461,12 +1637,13 @@ def resume_run(
|
|
|
1461
1637
|
run_dir = run_root / "runs" / run_id
|
|
1462
1638
|
workspace = run_dir / "ws"
|
|
1463
1639
|
record = load_record(run_root, run_id)
|
|
1640
|
+
dispatch = with_seed(dispatch, run_root, record.target)
|
|
1464
1641
|
stage = record.stage
|
|
1465
1642
|
# Push to the CANONICAL target URL, never the workspace's remote.origin.url:
|
|
1466
1643
|
# the session could have rewritten that config to exfil the bot token / code
|
|
1467
1644
|
# to another remote. Passing `url` here means `Workspace.push` uses it
|
|
1468
1645
|
# instead of reading `remote.origin.url`.
|
|
1469
|
-
ws = Workspace(root=workspace, auth=bot_auth, url=
|
|
1646
|
+
ws = Workspace(root=workspace, auth=bot_auth, url=target_clone_url(record.target))
|
|
1470
1647
|
# A session reshaped .git (symlinked object store, gitdir file, FIFO) is
|
|
1471
1648
|
# refused BEFORE anything writes through it: the exclude below opens
|
|
1472
1649
|
# .git/info/exclude, and every ws.git call re-checks. The refusal ENDS
|
|
@@ -1638,11 +1815,16 @@ def resume_run(
|
|
|
1638
1815
|
minutes=int(item.get("minutes") or 1),
|
|
1639
1816
|
artifacts=tuple(str(a) for a in item.get("artifacts", [])),
|
|
1640
1817
|
array=int(item.get("array") or 1),
|
|
1818
|
+
why=str(item.get("why") or ""),
|
|
1819
|
+
concurrency=int(item.get("concurrency") or 0),
|
|
1641
1820
|
)
|
|
1642
1821
|
for item in _stage_launches(record)
|
|
1643
1822
|
),
|
|
1644
1823
|
note=str(stage.get("syscall_note", "")),
|
|
1645
1824
|
submit=True,
|
|
1825
|
+
# the author's report rides every re-park: a suite fan-out
|
|
1826
|
+
# must not drop what the panel and the PR read
|
|
1827
|
+
report=str(stage.get("report", "")),
|
|
1646
1828
|
)
|
|
1647
1829
|
old_afterany = str(record.stage.get("afterany", ""))
|
|
1648
1830
|
made_progress = bool(parked.afterany) and parked.afterany != old_afterany
|
|
@@ -1677,7 +1859,19 @@ def resume_run(
|
|
|
1677
1859
|
# the park's sibling launches are done too: settle their charge before
|
|
1678
1860
|
# any path — publish or hand back to the author — reads the budget
|
|
1679
1861
|
if _stage_launches(record):
|
|
1680
|
-
|
|
1862
|
+
sibling_launches = _stage_syscall_launches(record)
|
|
1863
|
+
sibling_ids = launch_task_ids(sibling_launches, stage_launch_job_ids(record))
|
|
1864
|
+
sibling_elapsed = _launch_elapsed(dispatch, sibling_ids) if sibling_ids else None
|
|
1865
|
+
_reconcile_launch_hours(record, dispatch, bench.gpus, sibling_launches, sibling_elapsed)
|
|
1866
|
+
# the ledger's ended records for the sibling launches, whether or not
|
|
1867
|
+
# the author is woken: the PR's experiments table reads them (an
|
|
1868
|
+
# author wake that follows records nothing twice)
|
|
1869
|
+
_best_effort(
|
|
1870
|
+
"launch ledger",
|
|
1871
|
+
lambda: _ledger_ended(
|
|
1872
|
+
run_dir, record, sibling_launches, sibling_ids, dispatch, sibling_elapsed
|
|
1873
|
+
),
|
|
1874
|
+
)
|
|
1681
1875
|
|
|
1682
1876
|
def _wake_author(
|
|
1683
1877
|
extra_update: str, judged: tuple[str, AttemptResult] | None = None
|
|
@@ -1736,7 +1930,14 @@ def resume_run(
|
|
|
1736
1930
|
# Research lines: record the tree AS OF THIS DECIDED TERMINAL — never
|
|
1737
1931
|
# earlier, because a blocking panel verdict can still resume the
|
|
1738
1932
|
# author (a continuation, not a terminal).
|
|
1739
|
-
_push_line_snapshot(
|
|
1933
|
+
_push_line_snapshot(
|
|
1934
|
+
ws,
|
|
1935
|
+
_line_ref_for(bench, config.agent_id),
|
|
1936
|
+
run_id,
|
|
1937
|
+
outcome,
|
|
1938
|
+
secrets,
|
|
1939
|
+
bot_login=config.bot_login,
|
|
1940
|
+
)
|
|
1740
1941
|
|
|
1741
1942
|
if result.outcome == "improved":
|
|
1742
1943
|
# Publish: branch the SEALED candidate sha, fold in the ledger, push,
|
|
@@ -1883,6 +2084,7 @@ def resume_run(
|
|
|
1883
2084
|
exclude=(
|
|
1884
2085
|
LINE_MEMORY_PATHS if _line_ref_for(bench, config.agent_id) else ()
|
|
1885
2086
|
),
|
|
2087
|
+
secrets=secrets,
|
|
1886
2088
|
)(baseline, candidate, str(stage.get("report", "")))
|
|
1887
2089
|
except Exception as exc:
|
|
1888
2090
|
if isinstance(exc, GitError) and _is_git_tamper(exc):
|
|
@@ -1950,18 +2152,23 @@ def resume_run(
|
|
|
1950
2152
|
# so push the candidate as-is rather than an empty commit.
|
|
1951
2153
|
if staged:
|
|
1952
2154
|
ws.git(
|
|
1953
|
-
|
|
1954
|
-
f"user.name={config.bot_login}",
|
|
1955
|
-
"-c",
|
|
1956
|
-
f"user.email={config.bot_login}@users.noreply.github.com",
|
|
2155
|
+
*git_identity(config.bot_login),
|
|
1957
2156
|
"commit",
|
|
1958
2157
|
"-m",
|
|
1959
2158
|
f"agent: improve {config.benchmark} ({_title_pair(baseline, candidate)})"
|
|
1960
2159
|
f"\n\nAgent: {config.agent_id}",
|
|
1961
2160
|
)
|
|
1962
2161
|
ws.push(branch)
|
|
2162
|
+
if record.stage.get("submitted"):
|
|
2163
|
+
# the author's report at submit rides the stage: the PR shows it
|
|
2164
|
+
# as the research report, over the ledger's experiments
|
|
2165
|
+
result = dc_replace(result, submit_report=str(record.stage.get("report") or ""))
|
|
1963
2166
|
body = pr_body(
|
|
1964
|
-
result,
|
|
2167
|
+
result,
|
|
2168
|
+
config,
|
|
2169
|
+
redact_secrets=secrets,
|
|
2170
|
+
display_digits=bench.display_digits,
|
|
2171
|
+
experiments=experiments_rows(run_dir),
|
|
1965
2172
|
)
|
|
1966
2173
|
if issue_number:
|
|
1967
2174
|
body = f"Addresses #{issue_number}.\n\n{body}"
|
|
@@ -2171,7 +2378,7 @@ def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str
|
|
|
2171
2378
|
if backend == "codex":
|
|
2172
2379
|
lens_key = _judge_lens_key(
|
|
2173
2380
|
backend="codex",
|
|
2174
|
-
key_file_env="
|
|
2381
|
+
key_file_env="OUTERLOOP_PANEL_CODEX_KEY_FILE",
|
|
2175
2382
|
author_backend="codex",
|
|
2176
2383
|
claude_panel_path=claude_panel_path,
|
|
2177
2384
|
image=args.image,
|
|
@@ -2185,7 +2392,7 @@ def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str
|
|
|
2185
2392
|
# separate against the codex author key.
|
|
2186
2393
|
lens_key = _judge_lens_key(
|
|
2187
2394
|
backend="hermes",
|
|
2188
|
-
key_file_env="
|
|
2395
|
+
key_file_env="OUTERLOOP_PANEL_HERMES_KEY_FILE",
|
|
2189
2396
|
author_backend="codex",
|
|
2190
2397
|
claude_panel_path=claude_panel_path,
|
|
2191
2398
|
image=args.image,
|
|
@@ -2259,6 +2466,7 @@ def build_panel_runner(
|
|
|
2259
2466
|
start_round: int = 0,
|
|
2260
2467
|
exclude: tuple[str, ...] = (),
|
|
2261
2468
|
claim_body: Callable[[float, float, str], str] | None = None,
|
|
2469
|
+
secrets: tuple[str, ...] = (),
|
|
2262
2470
|
) -> Callable[[float, float, str], PanelVerdict]:
|
|
2263
2471
|
"""The git half of the pre-PR panel: prepare the two read-only checkouts
|
|
2264
2472
|
and the synthetic claim, then hand off to `run_panel` (which owns no git).
|
|
@@ -2282,6 +2490,10 @@ def build_panel_runner(
|
|
|
2282
2490
|
)
|
|
2283
2491
|
|
|
2284
2492
|
def runner(baseline: float, candidate: float, report: str) -> PanelVerdict:
|
|
2493
|
+
# the claim is author text (the report at submit, or the session's
|
|
2494
|
+
# last words): redacted before any lens sees it, like the record and
|
|
2495
|
+
# the PR body
|
|
2496
|
+
report = redact(report, secrets)
|
|
2285
2497
|
reads["n"] += 1
|
|
2286
2498
|
panel_ws = run_dir / "panel"
|
|
2287
2499
|
shutil.rmtree(panel_ws, ignore_errors=True)
|
|
@@ -2294,10 +2506,7 @@ def build_panel_runner(
|
|
|
2294
2506
|
tree = ws.git("write-tree").strip()
|
|
2295
2507
|
ws.git("reset")
|
|
2296
2508
|
snapshot = ws.git(
|
|
2297
|
-
|
|
2298
|
-
"user.name=panel",
|
|
2299
|
-
"-c",
|
|
2300
|
-
"user.email=panel@localhost",
|
|
2509
|
+
*git_identity(bot_login),
|
|
2301
2510
|
"commit-tree",
|
|
2302
2511
|
tree,
|
|
2303
2512
|
"-p",
|
|
@@ -2427,7 +2636,7 @@ def live_attempt(
|
|
|
2427
2636
|
# exception path cannot rely on names bound inside the try
|
|
2428
2637
|
salvage: dict[str, object] = {}
|
|
2429
2638
|
try:
|
|
2430
|
-
ws = Workspace.clone(
|
|
2639
|
+
ws = Workspace.clone(target_clone_url(config.target), workspace, auth=bot_auth)
|
|
2431
2640
|
# Build ON the requested PR base: the clone checks out the remote
|
|
2432
2641
|
# DEFAULT branch, which need not be `base_branch` — the session must
|
|
2433
2642
|
# edit, and the gate must measure, the tree the PR will land on.
|
|
@@ -2491,7 +2700,9 @@ def live_attempt(
|
|
|
2491
2700
|
line_ref = ""
|
|
2492
2701
|
if lines_active:
|
|
2493
2702
|
try:
|
|
2494
|
-
line_ref = _checkout_line(
|
|
2703
|
+
line_ref = _checkout_line(
|
|
2704
|
+
ws, workspace, config.agent_id, base_branch, config.bot_login
|
|
2705
|
+
)
|
|
2495
2706
|
except Exception as exc:
|
|
2496
2707
|
log.warning(
|
|
2497
2708
|
"line checkout failed (%s); running on %s",
|
|
@@ -2634,6 +2845,7 @@ def live_attempt(
|
|
|
2634
2845
|
config.bot_login,
|
|
2635
2846
|
created[:10],
|
|
2636
2847
|
exclude=LINE_MEMORY_PATHS if lines_active else (),
|
|
2848
|
+
secrets=secrets,
|
|
2637
2849
|
)
|
|
2638
2850
|
if panel_lenses
|
|
2639
2851
|
else None
|
|
@@ -2667,12 +2879,16 @@ def live_attempt(
|
|
|
2667
2879
|
run_tag=run_id,
|
|
2668
2880
|
# an inline gate shares the same target-wide baseline cache
|
|
2669
2881
|
baseline_cache=run_dir.parent / "baselines",
|
|
2882
|
+
seed_cache=dispatch.seed_cache if dispatch is not None else None,
|
|
2670
2883
|
)
|
|
2671
2884
|
snapshots: list[Snapshot] = []
|
|
2672
2885
|
|
|
2673
2886
|
def snapshot() -> str:
|
|
2674
2887
|
snap = snapshot_tree(
|
|
2675
|
-
ws,
|
|
2888
|
+
ws,
|
|
2889
|
+
pre_session_sha,
|
|
2890
|
+
exclude=LINE_MEMORY_PATHS if lines_active else (),
|
|
2891
|
+
author=config.bot_login,
|
|
2676
2892
|
)
|
|
2677
2893
|
snapshots.append(snap)
|
|
2678
2894
|
return snap.commit
|
|
@@ -2717,6 +2933,11 @@ def live_attempt(
|
|
|
2717
2933
|
line_memory=line_memory,
|
|
2718
2934
|
line_divergence=line_divergence,
|
|
2719
2935
|
launcher=launcher,
|
|
2936
|
+
watcher=(
|
|
2937
|
+
_make_watcher(dispatch, run_root, run_id, workspace, config)
|
|
2938
|
+
if dispatch is not None
|
|
2939
|
+
else None
|
|
2940
|
+
),
|
|
2720
2941
|
tree_of=lambda sha: ws.git("rev-parse", f"{sha}^{{tree}}").strip(),
|
|
2721
2942
|
)
|
|
2722
2943
|
except RunParked as p:
|
|
@@ -2781,6 +3002,7 @@ def live_attempt(
|
|
|
2781
3002
|
run_id,
|
|
2782
3003
|
"attempt-error",
|
|
2783
3004
|
secrets,
|
|
3005
|
+
bot_login=config.bot_login,
|
|
2784
3006
|
)
|
|
2785
3007
|
failed = RunRecord(
|
|
2786
3008
|
**{
|
|
@@ -2838,7 +3060,7 @@ def live_attempt(
|
|
|
2838
3060
|
# memory (it is excluded from measurable seals by design). The label is
|
|
2839
3061
|
# the GATE outcome, correct at this moment; a publish failure appends a
|
|
2840
3062
|
# publish-error snapshot at the tail.
|
|
2841
|
-
_push_line_snapshot(ws, line_ref, run_id, result.outcome, secrets)
|
|
3063
|
+
_push_line_snapshot(ws, line_ref, run_id, result.outcome, secrets, bot_login=config.bot_login)
|
|
2842
3064
|
|
|
2843
3065
|
pr_url = ""
|
|
2844
3066
|
outcome_name = result.outcome
|
|
@@ -2890,10 +3112,7 @@ def live_attempt(
|
|
|
2890
3112
|
raise WorkspaceDrift(f"publish would stage non-ledger paths: {extra[:10]}")
|
|
2891
3113
|
if staged:
|
|
2892
3114
|
ws.git(
|
|
2893
|
-
|
|
2894
|
-
f"user.name={config.bot_login}",
|
|
2895
|
-
"-c",
|
|
2896
|
-
f"user.email={config.bot_login}@users.noreply.github.com",
|
|
3115
|
+
*git_identity(config.bot_login),
|
|
2897
3116
|
"commit",
|
|
2898
3117
|
"-m",
|
|
2899
3118
|
f"agent: improve {config.benchmark} ({_title_pair(baseline, candidate)})"
|
|
@@ -2902,7 +3121,11 @@ def live_attempt(
|
|
|
2902
3121
|
ws.push(branch)
|
|
2903
3122
|
pushed = True
|
|
2904
3123
|
body = pr_body(
|
|
2905
|
-
result,
|
|
3124
|
+
result,
|
|
3125
|
+
config,
|
|
3126
|
+
redact_secrets=secrets,
|
|
3127
|
+
display_digits=bench.display_digits,
|
|
3128
|
+
experiments=experiments_rows(run_dir),
|
|
2906
3129
|
)
|
|
2907
3130
|
if issue_number:
|
|
2908
3131
|
body = f"Addresses #{issue_number}.\n\n{body}"
|
|
@@ -3017,7 +3240,7 @@ def live_attempt(
|
|
|
3017
3240
|
# the publish failed after the gate credited the tree: the improved
|
|
3018
3241
|
# snapshot above stands (the measurement was real); append the
|
|
3019
3242
|
# publish-error marker so the notebook records how the run ended
|
|
3020
|
-
_push_line_snapshot(ws, line_ref, run_id, outcome_name, secrets)
|
|
3243
|
+
_push_line_snapshot(ws, line_ref, run_id, outcome_name, secrets, bot_login=config.bot_login)
|
|
3021
3244
|
log.info("run %s: %s %s", run_id, outcome_name, pr_url)
|
|
3022
3245
|
return AttemptOutcome(
|
|
3023
3246
|
run_id=run_id,
|
|
@@ -3110,7 +3333,6 @@ def arm_sigterm_containment() -> None:
|
|
|
3110
3333
|
def main() -> int:
|
|
3111
3334
|
import argparse
|
|
3112
3335
|
import os
|
|
3113
|
-
import time
|
|
3114
3336
|
from datetime import UTC, datetime
|
|
3115
3337
|
|
|
3116
3338
|
arm_sigterm_containment()
|
|
@@ -3143,45 +3365,48 @@ def main() -> int:
|
|
|
3143
3365
|
)
|
|
3144
3366
|
parser.add_argument("--base-branch", default="main")
|
|
3145
3367
|
# All three default from the chain env the tick sets on the climb job, so
|
|
3146
|
-
# a contained run with
|
|
3147
|
-
# dispatched measurement without extra flags.
|
|
3148
|
-
# session + inline eval;
|
|
3149
|
-
# regardless of the benchmark's eval hint.
|
|
3368
|
+
# a contained run with OUTERLOOP_{IMAGE,ACCOUNT,PARTITION} set selects
|
|
3369
|
+
# dispatched measurement without extra flags. Only the image is required
|
|
3370
|
+
# for dispatch (it also contains the session + inline eval); without it,
|
|
3371
|
+
# measurement stays inline regardless of the benchmark's eval hint. Empty
|
|
3372
|
+
# account/partition use Slurm's defaults.
|
|
3150
3373
|
parser.add_argument(
|
|
3151
3374
|
"--image",
|
|
3152
|
-
default=os.environ.get("
|
|
3375
|
+
default=os.environ.get("OUTERLOOP_IMAGE", ""),
|
|
3153
3376
|
help="apptainer image for session+eval",
|
|
3154
3377
|
)
|
|
3155
|
-
parser.add_argument("--account", default=os.environ.get("
|
|
3156
|
-
parser.add_argument("--partition", default=os.environ.get("
|
|
3378
|
+
parser.add_argument("--account", default=os.environ.get("OUTERLOOP_ACCOUNT", ""))
|
|
3379
|
+
parser.add_argument("--partition", default=os.environ.get("OUTERLOOP_PARTITION", ""))
|
|
3157
3380
|
# the GPU lane for benchmarks with `gpus > 0` (evals + author launches);
|
|
3158
3381
|
# empty = this deployment cannot place GPU jobs
|
|
3159
|
-
parser.add_argument("--gpu-partition", default=os.environ.get("
|
|
3160
|
-
parser.add_argument("--gpu-account", default=os.environ.get("
|
|
3382
|
+
parser.add_argument("--gpu-partition", default=os.environ.get("OUTERLOOP_GPU_PARTITION", ""))
|
|
3383
|
+
parser.add_argument("--gpu-account", default=os.environ.get("OUTERLOOP_GPU_ACCOUNT", ""))
|
|
3161
3384
|
parser.add_argument(
|
|
3162
3385
|
"--uncontained",
|
|
3163
3386
|
action="store_true",
|
|
3164
3387
|
help="run WITHOUT a container (dev only: sessions can then read "
|
|
3165
3388
|
"same-user files, including credential files)",
|
|
3166
3389
|
)
|
|
3167
|
-
parser.add_argument(
|
|
3390
|
+
parser.add_argument(
|
|
3391
|
+
"--claude-bin",
|
|
3392
|
+
default=default_binary("claude"),
|
|
3393
|
+
help="host claude binary for the claude author (OUTERLOOP_CLAUDE_BIN, which init records)",
|
|
3394
|
+
)
|
|
3168
3395
|
parser.add_argument(
|
|
3169
3396
|
"--codex-bin",
|
|
3170
|
-
default=
|
|
3171
|
-
os.environ.get("AUTORESEARCH_CODEX_BIN") or "~/.local/bin/codex"
|
|
3172
|
-
),
|
|
3397
|
+
default=default_binary("codex"),
|
|
3173
3398
|
help="host codex binary for the codex author; bind-mounted into apptainer "
|
|
3174
3399
|
"(must be an absolute path).",
|
|
3175
3400
|
)
|
|
3176
3401
|
parser.add_argument(
|
|
3177
|
-
"--model", default=os.environ.get("
|
|
3402
|
+
"--model", default=os.environ.get("OUTERLOOP_AUTHOR_MODEL") or "claude-opus-5"
|
|
3178
3403
|
)
|
|
3179
3404
|
parser.add_argument(
|
|
3180
3405
|
"--author-backend",
|
|
3181
3406
|
choices=("claude", "codex"),
|
|
3182
|
-
default=os.environ.get("
|
|
3407
|
+
default=os.environ.get("OUTERLOOP_AUTHOR_BACKEND") or "claude",
|
|
3183
3408
|
help="agent backend for the author/editor role (config-driven: default "
|
|
3184
|
-
"from
|
|
3409
|
+
"from OUTERLOOP_AUTHOR_BACKEND). codex runs contained (apptainer + "
|
|
3185
3410
|
"--sandbox danger-full-access) and REQUIRES --image and a codex/openai "
|
|
3186
3411
|
"--model (e.g. gpt-5.6-terra).",
|
|
3187
3412
|
)
|
|
@@ -3225,7 +3450,7 @@ def main() -> int:
|
|
|
3225
3450
|
parser.add_argument("--pat-file", default=str(CONFIG_DIR / "bot_pat"))
|
|
3226
3451
|
parser.add_argument(
|
|
3227
3452
|
"--github-app-file",
|
|
3228
|
-
default=os.environ.get("
|
|
3453
|
+
default=os.environ.get("OUTERLOOP_GITHUB_APP_FILE", ""),
|
|
3229
3454
|
help="GitHub App config (JSON: app_id, installation_id, private_key); "
|
|
3230
3455
|
"when set, installation tokens replace the PAT",
|
|
3231
3456
|
)
|
|
@@ -3233,7 +3458,7 @@ def main() -> int:
|
|
|
3233
3458
|
"--key-file",
|
|
3234
3459
|
default="",
|
|
3235
3460
|
help="author key file; default resolves per backend (config-driven): "
|
|
3236
|
-
"
|
|
3461
|
+
"OUTERLOOP_CLAUDE_KEY_FILE for claude, OUTERLOOP_CODEX_KEY_FILE for codex",
|
|
3237
3462
|
)
|
|
3238
3463
|
parser.add_argument("--issue", type=int, default=0)
|
|
3239
3464
|
parser.add_argument(
|
|
@@ -3262,12 +3487,11 @@ def main() -> int:
|
|
|
3262
3487
|
# and re-enter the decision. The wake job the WakeDispatcher submits runs
|
|
3263
3488
|
# exactly this.
|
|
3264
3489
|
if args.resume:
|
|
3265
|
-
|
|
3266
|
-
if not (placed and args.image and Path(args.image).is_file()):
|
|
3490
|
+
if not (args.image and Path(args.image).is_file()):
|
|
3267
3491
|
parser.error(
|
|
3268
|
-
"--resume needs the
|
|
3269
|
-
"
|
|
3270
|
-
"
|
|
3492
|
+
"--resume needs --image to rebuild the dispatched measurer "
|
|
3493
|
+
"(account and partition are optional: empty ones use Slurm's "
|
|
3494
|
+
"defaults, and local compute has no placement)"
|
|
3271
3495
|
)
|
|
3272
3496
|
from outerloop.runstate import load_record
|
|
3273
3497
|
|
|
@@ -3432,13 +3656,12 @@ def main() -> int:
|
|
|
3432
3656
|
except ValueError as exc:
|
|
3433
3657
|
parser.error(str(exc))
|
|
3434
3658
|
|
|
3435
|
-
# Dispatched measurement needs
|
|
3436
|
-
#
|
|
3437
|
-
# sets
|
|
3659
|
+
# Dispatched measurement needs a real image file to bind against; without
|
|
3660
|
+
# one the climb measures inline. Account and partition are optional: empty
|
|
3661
|
+
# ones use Slurm's defaults (the tick sets them on the climb job's env from
|
|
3662
|
+
# the chain's), and local compute has no placement.
|
|
3438
3663
|
dispatch: DispatchSettings | None = None
|
|
3439
|
-
if
|
|
3440
|
-
args.image and Path(args.image).is_file()
|
|
3441
|
-
):
|
|
3664
|
+
if args.image and Path(args.image).is_file():
|
|
3442
3665
|
dispatch = _dispatch_settings(args)
|
|
3443
3666
|
try:
|
|
3444
3667
|
try:
|