outerloop-science 0.1.0.dev0__py3-none-any.whl → 0.1.0.dev1__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +1 -1
- outerloop/appmanifest.py +15 -10
- outerloop/attempt.py +29 -7
- outerloop/cli.py +24 -16
- outerloop/climbboard.py +209 -4
- outerloop/compute.py +27 -0
- outerloop/followup.py +24 -7
- outerloop/init.py +192 -19
- outerloop/paths.py +12 -0
- outerloop/tick.py +107 -37
- {outerloop_science-0.1.0.dev0.dist-info → outerloop_science-0.1.0.dev1.dist-info}/METADATA +13 -9
- {outerloop_science-0.1.0.dev0.dist-info → outerloop_science-0.1.0.dev1.dist-info}/RECORD +16 -16
- {outerloop_science-0.1.0.dev0.dist-info → outerloop_science-0.1.0.dev1.dist-info}/WHEEL +0 -0
- {outerloop_science-0.1.0.dev0.dist-info → outerloop_science-0.1.0.dev1.dist-info}/entry_points.txt +0 -0
- {outerloop_science-0.1.0.dev0.dist-info → outerloop_science-0.1.0.dev1.dist-info}/licenses/LICENSE +0 -0
- {outerloop_science-0.1.0.dev0.dist-info → outerloop_science-0.1.0.dev1.dist-info}/licenses/NOTICE +0 -0
outerloop/__init__.py
CHANGED
outerloop/appmanifest.py
CHANGED
|
@@ -25,12 +25,14 @@ import os
|
|
|
25
25
|
import secrets
|
|
26
26
|
import time
|
|
27
27
|
import urllib.error
|
|
28
|
+
import urllib.parse
|
|
28
29
|
import urllib.request
|
|
29
30
|
from collections.abc import Callable
|
|
30
31
|
from pathlib import Path
|
|
31
32
|
from typing import Any
|
|
32
33
|
|
|
33
34
|
from outerloop.appauth import API, build_app_jwt, signer_from_private_key
|
|
35
|
+
from outerloop.paths import write_private
|
|
34
36
|
|
|
35
37
|
# The hosted helper page: it auto-POSTs the manifest to GitHub, then displays the
|
|
36
38
|
# returned code. It is also the manifest's redirect_url, so GitHub sends the code
|
|
@@ -90,10 +92,15 @@ def request_manifest_code(
|
|
|
90
92
|
state = secrets.token_urlsafe(16)
|
|
91
93
|
manifest = build_manifest(name, homepage_url)
|
|
92
94
|
url = build_setup_url(manifest, state, org)
|
|
93
|
-
|
|
95
|
+
where = f"the {org} organization" if org else "your account"
|
|
96
|
+
print_fn("Step 1 of 3: create the GitHub App.")
|
|
97
|
+
print_fn(" Open this link in a browser where you are signed in to GitHub (any machine):")
|
|
94
98
|
print_fn(f"\n {url}\n")
|
|
95
|
-
print_fn("
|
|
96
|
-
|
|
99
|
+
print_fn(f" GitHub shows a page titled 'Create GitHub App' under {where}. Click the")
|
|
100
|
+
host = urllib.parse.urlsplit(SETUP_URL).netloc or SETUP_URL
|
|
101
|
+
print_fn(" green 'Create GitHub App' button. GitHub then sends you to")
|
|
102
|
+
print_fn(f" {host}, which shows a one-time code.")
|
|
103
|
+
return input_fn("Paste that code here: ").strip()
|
|
97
104
|
|
|
98
105
|
|
|
99
106
|
def _default_transport(request: urllib.request.Request) -> Any:
|
|
@@ -136,10 +143,10 @@ def save_app_creds(
|
|
|
136
143
|
config_dir.mkdir(parents=True, exist_ok=True)
|
|
137
144
|
slug = str(conversion["slug"])
|
|
138
145
|
pem_path = config_dir / f"{slug}-app.pem"
|
|
139
|
-
pem_path
|
|
140
|
-
pem_path.chmod(0o600)
|
|
146
|
+
write_private(pem_path, str(conversion["pem"]))
|
|
141
147
|
app_json = config_dir / f"github_app.{slug}.json"
|
|
142
|
-
|
|
148
|
+
write_private(
|
|
149
|
+
app_json,
|
|
143
150
|
json.dumps(
|
|
144
151
|
{
|
|
145
152
|
"app_id": int(conversion["id"]),
|
|
@@ -148,9 +155,8 @@ def save_app_creds(
|
|
|
148
155
|
},
|
|
149
156
|
indent=2,
|
|
150
157
|
)
|
|
151
|
-
+ "\n"
|
|
158
|
+
+ "\n",
|
|
152
159
|
)
|
|
153
|
-
app_json.chmod(0o600)
|
|
154
160
|
return pem_path, app_json
|
|
155
161
|
|
|
156
162
|
|
|
@@ -194,5 +200,4 @@ def set_installation_id(app_json: Path, installation_id: int) -> None:
|
|
|
194
200
|
"""Fill the installation id into an already-written github_app.<slug>.json."""
|
|
195
201
|
data = json.loads(app_json.read_text())
|
|
196
202
|
data["installation_id"] = int(installation_id)
|
|
197
|
-
app_json
|
|
198
|
-
app_json.chmod(0o600)
|
|
203
|
+
write_private(app_json, json.dumps(data, indent=2) + "\n")
|
outerloop/attempt.py
CHANGED
|
@@ -97,8 +97,11 @@ log = logging.getLogger(__name__)
|
|
|
97
97
|
# tick preflights the panel key (and compares it against the author key —
|
|
98
98
|
# role separation) before claiming/submitting.
|
|
99
99
|
PANEL_KEY_DEFAULT = str(CONFIG_DIR / "verifier_key")
|
|
100
|
-
|
|
101
|
-
|
|
100
|
+
# One rule for author keys: ~/.config/outerloop/<backend>_key. `harness_key` is
|
|
101
|
+
# the pre-rename name of the claude key; it is still read, never written.
|
|
102
|
+
CLAUDE_KEY_DEFAULT = str(CONFIG_DIR / "claude_key")
|
|
103
|
+
HARNESS_KEY_DEFAULT = str(CONFIG_DIR / "harness_key") # legacy claude key
|
|
104
|
+
CODEX_KEY_DEFAULT = str(CONFIG_DIR / "codex_key")
|
|
102
105
|
|
|
103
106
|
|
|
104
107
|
def resolve_author_key_file(backend: str, explicit: str = "") -> str:
|
|
@@ -106,14 +109,27 @@ def resolve_author_key_file(backend: str, explicit: str = "") -> str:
|
|
|
106
109
|
codex's both on disk), selected by backend — so the author backend is a
|
|
107
110
|
config choice, not a key swap, and an in-flight run of either backend can
|
|
108
111
|
still be woken/serviced after a fleet flip. An explicit path always wins;
|
|
109
|
-
otherwise the per-backend env var, then the
|
|
110
|
-
|
|
111
|
-
|
|
112
|
+
otherwise the per-backend env var, then the default path. For claude the
|
|
113
|
+
legacy spellings (`AUTORESEARCH_HARNESS_KEY_FILE`, `harness_key`) are still
|
|
114
|
+
honored, so a machine set up before the rename keeps working; the new name
|
|
115
|
+
wins when both exist. The result is always ~-expanded, so every caller gets
|
|
116
|
+
a real path (an env value like "~/.config/..." must not reach the token
|
|
117
|
+
provider verbatim)."""
|
|
112
118
|
if not explicit:
|
|
113
119
|
if backend == "codex":
|
|
114
120
|
explicit = os.environ.get("AUTORESEARCH_CODEX_KEY_FILE") or CODEX_KEY_DEFAULT
|
|
115
121
|
else:
|
|
116
|
-
explicit =
|
|
122
|
+
explicit = (
|
|
123
|
+
os.environ.get("AUTORESEARCH_CLAUDE_KEY_FILE")
|
|
124
|
+
or os.environ.get("AUTORESEARCH_HARNESS_KEY_FILE")
|
|
125
|
+
or ""
|
|
126
|
+
)
|
|
127
|
+
if not explicit:
|
|
128
|
+
explicit = CLAUDE_KEY_DEFAULT
|
|
129
|
+
if not os.path.exists(os.path.expanduser(explicit)) and os.path.exists(
|
|
130
|
+
os.path.expanduser(HARNESS_KEY_DEFAULT)
|
|
131
|
+
):
|
|
132
|
+
explicit = HARNESS_KEY_DEFAULT
|
|
117
133
|
return os.path.expanduser(explicit)
|
|
118
134
|
|
|
119
135
|
|
|
@@ -2179,6 +2195,12 @@ def _panel_lenses_from_args(args: Any) -> tuple[tuple[PanelLens, ...], tuple[str
|
|
|
2179
2195
|
else:
|
|
2180
2196
|
lens_key = panel_key
|
|
2181
2197
|
try:
|
|
2198
|
+
if not args.image:
|
|
2199
|
+
log.warning(
|
|
2200
|
+
"panel lens %s runs uncontained (local mode, no image): the judge "
|
|
2201
|
+
"shares this machine with the operator's keys",
|
|
2202
|
+
backend,
|
|
2203
|
+
)
|
|
2182
2204
|
judge = build_harness(
|
|
2183
2205
|
lens_key,
|
|
2184
2206
|
reviewer_spec(),
|
|
@@ -3211,7 +3233,7 @@ def main() -> int:
|
|
|
3211
3233
|
"--key-file",
|
|
3212
3234
|
default="",
|
|
3213
3235
|
help="author key file; default resolves per backend (config-driven): "
|
|
3214
|
-
"
|
|
3236
|
+
"AUTORESEARCH_CLAUDE_KEY_FILE for claude, AUTORESEARCH_CODEX_KEY_FILE for codex",
|
|
3215
3237
|
)
|
|
3216
3238
|
parser.add_argument("--issue", type=int, default=0)
|
|
3217
3239
|
parser.add_argument(
|
outerloop/cli.py
CHANGED
|
@@ -46,6 +46,7 @@ TICK_ENV_KEYS = (
|
|
|
46
46
|
"AUTORESEARCH_AUTHOR_MODEL",
|
|
47
47
|
"AUTORESEARCH_CODEX_BIN",
|
|
48
48
|
"AUTORESEARCH_CODEX_KEY_FILE",
|
|
49
|
+
"AUTORESEARCH_CLAUDE_KEY_FILE",
|
|
49
50
|
"AUTORESEARCH_HARNESS_KEY_FILE",
|
|
50
51
|
"AUTORESEARCH_VERTEX_PROJECT",
|
|
51
52
|
"AUTORESEARCH_VERTEX_REGION",
|
|
@@ -165,19 +166,23 @@ def _setting(key: str, flag: str, environ: dict[str, str], from_file: dict[str,
|
|
|
165
166
|
return from_file.get(key, "")
|
|
166
167
|
|
|
167
168
|
|
|
168
|
-
def _home(environ: dict[str, str], cwd: Path) -> Path:
|
|
169
|
-
"""The
|
|
170
|
-
|
|
171
|
-
and
|
|
172
|
-
|
|
173
|
-
|
|
169
|
+
def _home(environ: dict[str, str], cwd: Path, *, local: bool, root: Path) -> Path:
|
|
170
|
+
"""The directory the loop runs from. On Slurm it must be a source
|
|
171
|
+
checkout (AUTORESEARCH_HOME, else the current directory): the chain
|
|
172
|
+
deploys from it and every job runs from a flight snapshot of its HEAD.
|
|
173
|
+
The local loop has no deploy step and runs the installed package, so it
|
|
174
|
+
uses a checkout when one is at hand and otherwise a `home` directory
|
|
175
|
+
under the state root, where flights and logs land."""
|
|
176
|
+
named = environ.get("OUTERLOOP_HOME") or environ.get("AUTORESEARCH_HOME")
|
|
177
|
+
home = Path(named).expanduser() if named else cwd
|
|
178
|
+
if (home / "scripts" / "tick_chain.sbatch").is_file():
|
|
179
|
+
return home
|
|
180
|
+
if local and not named:
|
|
181
|
+
return root / "home"
|
|
182
|
+
raise StartError(
|
|
183
|
+
f"{home} is not a source checkout (no scripts/tick_chain.sbatch); "
|
|
184
|
+
"run start from a clone of the kernel, or set OUTERLOOP_HOME to one"
|
|
174
185
|
)
|
|
175
|
-
if not (home / "scripts" / "tick_chain.sbatch").is_file():
|
|
176
|
-
raise StartError(
|
|
177
|
-
f"{home} is not an autoresearch checkout (no scripts/tick_chain.sbatch); "
|
|
178
|
-
"run start from the checkout the chain should deploy from, or set AUTORESEARCH_HOME"
|
|
179
|
-
)
|
|
180
|
-
return home
|
|
181
186
|
|
|
182
187
|
|
|
183
188
|
def plan_start(
|
|
@@ -207,13 +212,15 @@ def plan_start(
|
|
|
207
212
|
f"AUTORESEARCH_CADENCE_MIN must be a positive number of minutes, got {cadence!r}"
|
|
208
213
|
)
|
|
209
214
|
pat = _setting("AUTORESEARCH_PAT_FILE", "", environ, from_file)
|
|
210
|
-
#
|
|
211
|
-
#
|
|
212
|
-
|
|
215
|
+
# Slurm runs from a checkout; the local loop runs the installed package
|
|
216
|
+
# and needs only a directory (the launch lanes and GitHub servicing
|
|
217
|
+
# switch off without AUTORESEARCH_HOME, so one is always set)
|
|
218
|
+
local_root = Path(root_s).expanduser() if root_s else DEFAULT_LOCAL_ROOT
|
|
219
|
+
home = _home(environ, cwd, local=(mode == "local"), root=local_root)
|
|
213
220
|
if mode == "local":
|
|
214
221
|
return StartPlan(
|
|
215
222
|
mode="local",
|
|
216
|
-
root=
|
|
223
|
+
root=local_root,
|
|
217
224
|
home=home,
|
|
218
225
|
cadence_min=cadence,
|
|
219
226
|
pat_file=pat,
|
|
@@ -333,6 +340,7 @@ def start(args: argparse.Namespace) -> int:
|
|
|
333
340
|
env["AUTORESEARCH_COMPUTE"] = "local"
|
|
334
341
|
env["AUTORESEARCH_ROOT"] = str(plan.root)
|
|
335
342
|
env["AUTORESEARCH_HOME"] = str(plan.home)
|
|
343
|
+
plan.home.mkdir(parents=True, exist_ok=True) # <root>/home when there is no checkout
|
|
336
344
|
if plan.cadence_min:
|
|
337
345
|
env["AUTORESEARCH_CADENCE_MIN"] = plan.cadence_min
|
|
338
346
|
if plan.pat_file:
|
outerloop/climbboard.py
CHANGED
|
@@ -391,6 +391,22 @@ def render_html(
|
|
|
391
391
|
"border-radius:2px;display:inline-block;box-sizing:border-box}\n"
|
|
392
392
|
".pill{border-radius:.6rem;padding:.05rem .55rem;font-size:.72rem;\n"
|
|
393
393
|
"font-weight:600;color:#fff}\n"
|
|
394
|
+
".run.queue{width:100%;padding-bottom:.55rem}\n"
|
|
395
|
+
".run.queue table{width:100%;border-collapse:collapse;margin-top:.35rem;\n"
|
|
396
|
+
" font-size:.72rem;font-variant-numeric:tabular-nums}\n"
|
|
397
|
+
".run.queue th{text-align:left;font-weight:500;color:var(--muted);\n"
|
|
398
|
+
" padding:.1rem .6rem .2rem 0;border-bottom:1px solid var(--line)}\n"
|
|
399
|
+
".run.queue td{padding:.17rem .6rem .17rem 0;border-bottom:1px solid var(--line);\n"
|
|
400
|
+
" white-space:nowrap;vertical-align:middle;color:var(--ink)}\n"
|
|
401
|
+
".run.queue tr:last-child td{border-bottom:0}\n"
|
|
402
|
+
".run.queue .mono{font-family:ui-monospace,SFMono-Regular,Menlo,Consolas,monospace}\n"
|
|
403
|
+
".run.queue td.name{max-width:24rem;overflow:hidden;text-overflow:ellipsis}\n"
|
|
404
|
+
".run.queue td.part,.run.queue td.time{color:var(--muted)}\n"
|
|
405
|
+
".run.queue .st{display:inline-block;min-width:1.5rem;text-align:center;\n"
|
|
406
|
+
" border-radius:.4rem;padding:0 .35rem;font-size:.66rem;font-weight:600}\n"
|
|
407
|
+
".run.queue .st-R{background:var(--win);color:#fff}\n"
|
|
408
|
+
".run.queue .st-PD{background:var(--line);color:var(--muted)}\n"
|
|
409
|
+
".run.queue .st-X{background:var(--near);color:#fff}\n"
|
|
394
410
|
".card{background:var(--card);border:1px solid var(--line);border-radius:.5rem;\n"
|
|
395
411
|
"padding:1rem}\n"
|
|
396
412
|
".card label{font-size:.8rem;color:var(--muted);margin-right:1rem}\n"
|
|
@@ -773,6 +789,55 @@ def render_html(
|
|
|
773
789
|
" now.append(card);\n"
|
|
774
790
|
" }\n"
|
|
775
791
|
" if (!strip.runs.length) now.textContent = 'no active runs';\n"
|
|
792
|
+
" // the kernel's own Slurm jobs, as squeue would list them (when\n"
|
|
793
|
+
" // published): one full-width card in the strip, running jobs first\n"
|
|
794
|
+
" if (Array.isArray(strip.queue) && strip.queue.length) {\n"
|
|
795
|
+
" const ST = {RUNNING: 'R', PENDING: 'PD', COMPLETING: 'CG', CONFIGURING: 'CF'};\n"
|
|
796
|
+
" // squeue's reading order: running, finishing, starting, waiting, other\n"
|
|
797
|
+
" const RANK = {RUNNING: 0, COMPLETING: 1, CONFIGURING: 2, PENDING: 3};\n"
|
|
798
|
+
" const rank = j => RANK[j.state] ?? 4;\n"
|
|
799
|
+
" const jobs = [...strip.queue].sort((a, b) =>\n"
|
|
800
|
+
" rank(a) - rank(b) || String(a.id).localeCompare(String(b.id)));\n"
|
|
801
|
+
" const card = document.createElement('span'); card.className = 'run queue';\n"
|
|
802
|
+
" const top = document.createElement('span'); top.className = 'top';\n"
|
|
803
|
+
" const who = document.createElement('b'); who.textContent = 'squeue';\n"
|
|
804
|
+
" const meta = document.createElement('span');\n"
|
|
805
|
+
" const running = jobs.filter(j => j.state === 'RUNNING').length;\n"
|
|
806
|
+
" meta.textContent = jobs.length + ' jobs · ' + running + ' running';\n"
|
|
807
|
+
' meta.title = "the kernel\'s own Slurm jobs, as squeue --me lists them";\n'
|
|
808
|
+
" top.append(who, meta); card.append(top);\n"
|
|
809
|
+
" const table = document.createElement('table');\n"
|
|
810
|
+
" const thead = table.createTHead().insertRow();\n"
|
|
811
|
+
" for (const h of ['JOBID', 'NAME', 'ST', 'TIME', 'PARTITION', 'AGENT']) {\n"
|
|
812
|
+
" const th = document.createElement('th'); th.textContent = h; thead.append(th);\n"
|
|
813
|
+
" }\n"
|
|
814
|
+
" const body = table.createTBody();\n"
|
|
815
|
+
" const td = (tr, text, cls) => {\n"
|
|
816
|
+
" const c = tr.insertCell(); c.textContent = text ?? '';\n"
|
|
817
|
+
" if (cls) c.className = cls; return c;\n"
|
|
818
|
+
" };\n"
|
|
819
|
+
" for (const j of jobs) {\n"
|
|
820
|
+
" const tr = body.insertRow();\n"
|
|
821
|
+
" td(tr, j.id, 'mono');\n"
|
|
822
|
+
" td(tr, j.name, 'mono name').title = j.name;\n"
|
|
823
|
+
" const st = document.createElement('span');\n"
|
|
824
|
+
" const code = ST[j.state] || j.state;\n"
|
|
825
|
+
" st.className = 'st st-' + (code === 'R' || code === 'PD' ? code : 'X');\n"
|
|
826
|
+
" st.textContent = code; st.title = j.state;\n"
|
|
827
|
+
" tr.insertCell().append(st);\n"
|
|
828
|
+
" td(tr, j.elapsed, 'mono time');\n"
|
|
829
|
+
" // a job eligible for several partitions lists them comma-joined;\n"
|
|
830
|
+
" // show the first and count the rest, full list on hover\n"
|
|
831
|
+
" const parts = String(j.partition || '').split(',').filter(Boolean);\n"
|
|
832
|
+
" const pc = td(tr, parts.length > 1 ? parts[0] + ' +' + (parts.length - 1)\n"
|
|
833
|
+
" : parts[0] || '', 'part');\n"
|
|
834
|
+
" pc.title = parts.join(', ');\n"
|
|
835
|
+
" const a = td(tr, j.agent || '—', 'agent');\n"
|
|
836
|
+
" if (j.agent) { a.style.color = agentColor(j.agent); a.style.fontWeight = '600'; }\n"
|
|
837
|
+
" }\n"
|
|
838
|
+
" card.append(table);\n"
|
|
839
|
+
" now.append(card);\n"
|
|
840
|
+
" }\n"
|
|
776
841
|
"};\n"
|
|
777
842
|
"refresh();\n"
|
|
778
843
|
"// the strip re-FETCHES every few minutes (new/left/changed runs) and\n"
|
|
@@ -784,6 +849,110 @@ def render_html(
|
|
|
784
849
|
|
|
785
850
|
|
|
786
851
|
STATUS_PATH = "climb/status.json"
|
|
852
|
+
# The queue view shows the kernel's own jobs only. Fixed-name jobs (the tick
|
|
853
|
+
# chain, issue sessions, climb sessions) are matched by shape; per-run jobs
|
|
854
|
+
# (wake, followup, launch) are matched against the names the kernel itself
|
|
855
|
+
# derives from this target's run ids, with the same 60-character cut Slurm
|
|
856
|
+
# forces on them; an eval's name is a liveness hash and is claimed only
|
|
857
|
+
# through its run's marker. Anything else on the account is the operator's
|
|
858
|
+
# and never leaves the cluster.
|
|
859
|
+
JOB_NAME_LIMIT = 60
|
|
860
|
+
FIXED_JOB_PATTERNS = tuple(
|
|
861
|
+
re.compile(p)
|
|
862
|
+
for p in (
|
|
863
|
+
r"^(autoresearch|outerloop)-(resident|tick)$",
|
|
864
|
+
r"^climb-[\w.-]+-agent-\d+$",
|
|
865
|
+
r"^(steward|climb)-issue-\d+$",
|
|
866
|
+
)
|
|
867
|
+
)
|
|
868
|
+
_AGENT_RE = re.compile(r"agent-\d+")
|
|
869
|
+
|
|
870
|
+
|
|
871
|
+
def is_fixed_kernel_job(name: str) -> bool:
|
|
872
|
+
return any(p.match(name) for p in FIXED_JOB_PATTERNS)
|
|
873
|
+
|
|
874
|
+
|
|
875
|
+
def run_job_names(root: Path, target: str) -> dict[str, tuple[str, str, bool]]:
|
|
876
|
+
"""Expected per-run job names for `target`: name -> (run_id, agent,
|
|
877
|
+
is_prefix). Exact for wake and followup; a prefix for launches, whose
|
|
878
|
+
names end in the experiment's own label. Every run of the target counts,
|
|
879
|
+
ended ones too: a finishing job can outlive its record's state."""
|
|
880
|
+
out: dict[str, tuple[str, str, bool]] = {}
|
|
881
|
+
for record in list_runs(root):
|
|
882
|
+
if record.target != target:
|
|
883
|
+
continue
|
|
884
|
+
rid, agent = record.run_id, record.agent_id
|
|
885
|
+
for name, prefix in (
|
|
886
|
+
(f"wake-{rid}", False),
|
|
887
|
+
(f"followup-{rid}", False),
|
|
888
|
+
(f"{rid}-launch-", True),
|
|
889
|
+
):
|
|
890
|
+
key = name[:JOB_NAME_LIMIT]
|
|
891
|
+
if key in out and out[key][0] != rid:
|
|
892
|
+
# two runs whose names collide under the cut: the job is still
|
|
893
|
+
# the kernel's, but nobody can say whose, so it is claimed by none
|
|
894
|
+
out[key] = ("", "", prefix)
|
|
895
|
+
else:
|
|
896
|
+
out[key] = (rid, agent, prefix)
|
|
897
|
+
return out
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def eval_job_owners(root: Path, target: str) -> dict[str, tuple[str, str]]:
|
|
901
|
+
"""Slurm job id -> (run_id, agent) for every eval a live run of `target`
|
|
902
|
+
has dispatched: the measurer leaves the id in eval-*/submitted. Eval job
|
|
903
|
+
names are liveness hashes with no agent in them, so this is how the queue
|
|
904
|
+
view knows whose eval is whose."""
|
|
905
|
+
out: dict[str, tuple[str, str]] = {}
|
|
906
|
+
for record in list_runs(root):
|
|
907
|
+
if record.target != target or record.state == ENDED:
|
|
908
|
+
continue
|
|
909
|
+
for submitted in run_dir(root, record.run_id).glob("eval-*/submitted"):
|
|
910
|
+
try:
|
|
911
|
+
job_id = submitted.read_text().strip()
|
|
912
|
+
except OSError:
|
|
913
|
+
continue
|
|
914
|
+
if job_id:
|
|
915
|
+
out[job_id] = (record.run_id, record.agent_id)
|
|
916
|
+
return out
|
|
917
|
+
|
|
918
|
+
|
|
919
|
+
def queue_rows(root: Path, target: str, snapshot: list[dict[str, str]]) -> list[dict[str, str]]:
|
|
920
|
+
"""The published queue: kernel jobs from a `Compute.queue_snapshot()`,
|
|
921
|
+
each attributed to an agent (by eval marker, else by the agent id every
|
|
922
|
+
other kernel job carries in its name)."""
|
|
923
|
+
owners = eval_job_owners(root, target)
|
|
924
|
+
expected = run_job_names(root, target)
|
|
925
|
+
rows: list[dict[str, str]] = []
|
|
926
|
+
for job in snapshot:
|
|
927
|
+
name = str(job.get("name", ""))
|
|
928
|
+
job_id = str(job.get("id", ""))
|
|
929
|
+
run_id = agent = ""
|
|
930
|
+
if job_id in owners: # an eval of one of this target's live runs
|
|
931
|
+
run_id, agent = owners[job_id]
|
|
932
|
+
elif name in expected and not expected[name][2]: # wake / followup, exact
|
|
933
|
+
run_id, agent = expected[name][:2]
|
|
934
|
+
elif any(name.startswith(k) for k, v in expected.items() if v[2]): # a launch
|
|
935
|
+
run_id, agent = next(v[:2] for k, v in expected.items() if v[2] and name.startswith(k))
|
|
936
|
+
elif is_fixed_kernel_job(name):
|
|
937
|
+
found = _AGENT_RE.search(name)
|
|
938
|
+
agent = found.group(0) if found else ""
|
|
939
|
+
else:
|
|
940
|
+
continue # an eval whose marker is not written yet, or the operator's own job
|
|
941
|
+
rows.append(
|
|
942
|
+
{
|
|
943
|
+
"id": str(job.get("id", "")),
|
|
944
|
+
"name": name,
|
|
945
|
+
"state": str(job.get("state", "")),
|
|
946
|
+
"elapsed": str(job.get("elapsed", "")),
|
|
947
|
+
"partition": str(job.get("partition", "")),
|
|
948
|
+
"submitted": str(job.get("submitted", "")),
|
|
949
|
+
"agent": agent,
|
|
950
|
+
"run_id": run_id,
|
|
951
|
+
}
|
|
952
|
+
)
|
|
953
|
+
return rows
|
|
954
|
+
|
|
955
|
+
|
|
787
956
|
_LIVE_STATES = ("implementing", "waiting", "in-review", "concluding")
|
|
788
957
|
|
|
789
958
|
|
|
@@ -823,7 +992,13 @@ def _experiment_progress(root: Path, record: Any) -> tuple[int, int, int]:
|
|
|
823
992
|
return (done, len(names), minutes)
|
|
824
993
|
|
|
825
994
|
|
|
826
|
-
def collect_status(
|
|
995
|
+
def collect_status(
|
|
996
|
+
root: Path,
|
|
997
|
+
target: str,
|
|
998
|
+
now: float,
|
|
999
|
+
contract: Any = None,
|
|
1000
|
+
queue: list[dict[str, str]] | None = None,
|
|
1001
|
+
) -> dict[str, Any]:
|
|
827
1002
|
"""The fleet's live picture for `target`: one entry per non-terminal run.
|
|
828
1003
|
Timestamps, not durations — the page computes elapsed time client-side,
|
|
829
1004
|
so the strip feels live between pushes."""
|
|
@@ -878,15 +1053,25 @@ def collect_status(root: Path, target: str, now: float, contract: Any = None) ->
|
|
|
878
1053
|
}
|
|
879
1054
|
)
|
|
880
1055
|
runs.sort(key=lambda r: str(r.get("run_id")))
|
|
881
|
-
|
|
1056
|
+
status: dict[str, Any] = {"target": target, "published": now, "runs": runs}
|
|
1057
|
+
if queue is not None: # a snapshot was taken (Slurm); the local loop has no queue
|
|
1058
|
+
status["queue"] = queue_rows(root, target, queue)
|
|
1059
|
+
return status
|
|
882
1060
|
|
|
883
1061
|
|
|
884
|
-
def service_status(
|
|
1062
|
+
def service_status(
|
|
1063
|
+
root: Path,
|
|
1064
|
+
github: Any,
|
|
1065
|
+
target: str,
|
|
1066
|
+
now: float,
|
|
1067
|
+
contract: Any = None,
|
|
1068
|
+
queue: list[dict[str, str]] | None = None,
|
|
1069
|
+
) -> bool:
|
|
885
1070
|
"""Publish the strip when the fleet's SHAPE changed — a run appearing,
|
|
886
1071
|
leaving, or changing state/phase — never on every tick: the page shows
|
|
887
1072
|
elapsed time client-side, so timestamp-only drift is not worth a commit.
|
|
888
1073
|
Advisory like the board; True when a write happened."""
|
|
889
|
-
status = collect_status(root, target, now, contract)
|
|
1074
|
+
status = collect_status(root, target, now, contract, queue)
|
|
890
1075
|
try:
|
|
891
1076
|
existing_raw: str | None = github.get_file(target, STATUS_PATH, BOARD_BRANCH)
|
|
892
1077
|
except Exception as exc:
|
|
@@ -920,10 +1105,30 @@ def service_status(root: Path, github: Any, target: str, now: float, contract: A
|
|
|
920
1105
|
"sleep_k",
|
|
921
1106
|
)
|
|
922
1107
|
shape = lambda runs: [{k: r.get(k) for k in keys} for r in runs]
|
|
1108
|
+
# the queue's shape is which jobs exist, their state, partition and
|
|
1109
|
+
# attribution (an eval claimed by its marker one tick later must
|
|
1110
|
+
# republish); elapsed time drifts every tick and is left out
|
|
1111
|
+
# an absent queue (no snapshot) and an empty one (nothing running)
|
|
1112
|
+
# are different shapes: recovering from a blind tick must publish
|
|
1113
|
+
qshape = lambda q: (
|
|
1114
|
+
None
|
|
1115
|
+
if q is None
|
|
1116
|
+
else [
|
|
1117
|
+
(
|
|
1118
|
+
j.get("id"),
|
|
1119
|
+
j.get("state"),
|
|
1120
|
+
j.get("partition"),
|
|
1121
|
+
j.get("agent"),
|
|
1122
|
+
j.get("run_id"),
|
|
1123
|
+
)
|
|
1124
|
+
for j in q
|
|
1125
|
+
]
|
|
1126
|
+
)
|
|
923
1127
|
if (
|
|
924
1128
|
isinstance(existing, dict)
|
|
925
1129
|
and isinstance(existing.get("runs"), list)
|
|
926
1130
|
and shape(existing["runs"]) == shape(status["runs"])
|
|
1131
|
+
and qshape(existing.get("queue")) == qshape(status.get("queue"))
|
|
927
1132
|
):
|
|
928
1133
|
return False
|
|
929
1134
|
except (ValueError, TypeError, AttributeError):
|
outerloop/compute.py
CHANGED
|
@@ -133,6 +133,9 @@ class JobSpec:
|
|
|
133
133
|
return argv
|
|
134
134
|
|
|
135
135
|
|
|
136
|
+
QUEUE_FIELDS = ("id", "name", "state", "elapsed", "partition", "submitted")
|
|
137
|
+
|
|
138
|
+
|
|
136
139
|
class Compute(Protocol):
|
|
137
140
|
"""The verbs every compute backend implements. Callers (the measurer, the
|
|
138
141
|
launcher, the wake dispatcher) depend on this, never on a backend."""
|
|
@@ -142,6 +145,7 @@ class Compute(Protocol):
|
|
|
142
145
|
def pending_reason(self, job_id: str) -> str: ...
|
|
143
146
|
def job_partition(self, job_id: str) -> str: ...
|
|
144
147
|
def active_job_names(self) -> list[str]: ...
|
|
148
|
+
def queue_snapshot(self) -> list[dict[str, str]]: ...
|
|
145
149
|
def job_id_for_name(self, name: str) -> str: ...
|
|
146
150
|
def cancel(self, job_id: str) -> None: ...
|
|
147
151
|
|
|
@@ -257,6 +261,26 @@ class SlurmCompute:
|
|
|
257
261
|
raise SlurmQueryError(f"squeue failed ({result.returncode}): {result.stderr.strip()}")
|
|
258
262
|
return [line.strip() for line in result.stdout.splitlines() if line.strip()]
|
|
259
263
|
|
|
264
|
+
def queue_snapshot(self) -> list[dict[str, str]]:
|
|
265
|
+
"""This user's PENDING and RUNNING jobs as rows of QUEUE_FIELDS — what
|
|
266
|
+
a queue view needs, nothing a caller acts on. Raises SlurmQueryError
|
|
267
|
+
on failure, like active_job_names."""
|
|
268
|
+
try:
|
|
269
|
+
result = self.runner(
|
|
270
|
+
["squeue", "--me", "--noheader", "-o", "%i|%j|%T|%M|%P|%V"],
|
|
271
|
+
self.command_timeout_s,
|
|
272
|
+
)
|
|
273
|
+
except (OSError, subprocess.TimeoutExpired) as exc:
|
|
274
|
+
raise SlurmQueryError(f"squeue did not run: {exc}") from exc
|
|
275
|
+
if result.returncode != 0:
|
|
276
|
+
raise SlurmQueryError(f"squeue failed ({result.returncode}): {result.stderr.strip()}")
|
|
277
|
+
rows: list[dict[str, str]] = []
|
|
278
|
+
for line in result.stdout.splitlines():
|
|
279
|
+
parts = line.strip().split("|")
|
|
280
|
+
if len(parts) == len(QUEUE_FIELDS) and parts[0]:
|
|
281
|
+
rows.append(dict(zip(QUEUE_FIELDS, parts, strict=True)))
|
|
282
|
+
return rows
|
|
283
|
+
|
|
260
284
|
def job_id_for_name(self, name: str) -> str:
|
|
261
285
|
"""The id of this user's PENDING/RUNNING job with exactly `name`, or
|
|
262
286
|
"" if none. Authoritative for "is this still live" independent of any
|
|
@@ -436,6 +460,9 @@ class LocalCompute:
|
|
|
436
460
|
def active_job_names(self) -> list[str]:
|
|
437
461
|
return [] # synchronous: nothing is ever pending or running
|
|
438
462
|
|
|
463
|
+
def queue_snapshot(self) -> list[dict[str, str]]:
|
|
464
|
+
return []
|
|
465
|
+
|
|
439
466
|
def job_id_for_name(self, name: str) -> str:
|
|
440
467
|
return ""
|
|
441
468
|
|
outerloop/followup.py
CHANGED
|
@@ -958,6 +958,7 @@ def _respond(
|
|
|
958
958
|
changed=changed,
|
|
959
959
|
conflict_head=conflict_head if conflict_wake else "",
|
|
960
960
|
base_synced=base_synced,
|
|
961
|
+
pr_head=str((pr.get("head") or {}).get("sha", "")),
|
|
961
962
|
panel_wake=panel_wake,
|
|
962
963
|
now=now,
|
|
963
964
|
secrets=secrets,
|
|
@@ -1593,6 +1594,7 @@ def _park_remeasure(
|
|
|
1593
1594
|
changed: list[str],
|
|
1594
1595
|
conflict_head: str,
|
|
1595
1596
|
base_synced: bool,
|
|
1597
|
+
pr_head: str,
|
|
1596
1598
|
panel_wake: bool,
|
|
1597
1599
|
now: float,
|
|
1598
1600
|
secrets: tuple[str, ...],
|
|
@@ -1606,6 +1608,11 @@ def _park_remeasure(
|
|
|
1606
1608
|
"candidate_sha": snap.commit,
|
|
1607
1609
|
"candidate_ref": snap.ref,
|
|
1608
1610
|
"parent": ws.git("rev-parse", "HEAD").strip(),
|
|
1611
|
+
# the PR's head on GitHub at park. After a base sync `parent` is the
|
|
1612
|
+
# session's local merge, which GitHub never saw: the resume compares
|
|
1613
|
+
# the live head against THIS, not against parent (speedrun agent-03,
|
|
1614
|
+
# 2026-09-06: every base-synced re-measure was abandoned as "moved")
|
|
1615
|
+
"pr_head": pr_head,
|
|
1609
1616
|
"job_ids": list(pend.job_ids),
|
|
1610
1617
|
"afterany": pend.afterany(),
|
|
1611
1618
|
"seed": run_seed,
|
|
@@ -1729,7 +1736,13 @@ def _resume_measure(
|
|
|
1729
1736
|
ws.git("clean", "-fdq")
|
|
1730
1737
|
drop_snapshot(ws, snapshot)
|
|
1731
1738
|
github.comment(record.target, number, f"{REPLY_MARKER}\n{note}")
|
|
1732
|
-
|
|
1739
|
+
# the thread was told and invited to ask again: that next ask needs a
|
|
1740
|
+
# follow-up, so the landed measure counts as progress for the cap
|
|
1741
|
+
save_record(
|
|
1742
|
+
run_root,
|
|
1743
|
+
replace(load_record(run_root, run_id), followup_stage={}, wake_attempts=0),
|
|
1744
|
+
now,
|
|
1745
|
+
)
|
|
1733
1746
|
return FollowupOutcome(run_id, "replied", "dispatched re-measure abandoned")
|
|
1734
1747
|
|
|
1735
1748
|
try:
|
|
@@ -1764,12 +1777,16 @@ def _resume_measure(
|
|
|
1764
1777
|
)
|
|
1765
1778
|
return FollowupOutcome(run_id, "replied", "dispatched re-measure already landed")
|
|
1766
1779
|
|
|
1767
|
-
# the sealed commit
|
|
1768
|
-
#
|
|
1769
|
-
#
|
|
1770
|
-
|
|
1780
|
+
# the sealed commit descends from the head the PR had at park (directly,
|
|
1781
|
+
# or through the session's local base-sync merge): a push since (a
|
|
1782
|
+
# maintainer's) makes it unpushable AND measured on a tree that is no
|
|
1783
|
+
# longer the PR's — abandon honestly rather than force or rebuild. Stages
|
|
1784
|
+
# parked before `pr_head` was recorded fall back to conflict_head, then
|
|
1785
|
+
# parent.
|
|
1786
|
+
parked_head = str(stage.get("pr_head") or stage.get("conflict_head") or parent)
|
|
1787
|
+
if head_now and parked_head and head_now != parked_head:
|
|
1771
1788
|
return _abandon(
|
|
1772
|
-
f"_(The PR's head moved while the re-measure ran (`{
|
|
1789
|
+
f"_(The PR's head moved while the re-measure ran (`{parked_head[:12]}` → "
|
|
1773
1790
|
f"`{head_now[:12]}`), so the measured change no longer applies to this "
|
|
1774
1791
|
"branch and was not pushed. Ask again and it will be redone on the new head.)_"
|
|
1775
1792
|
)
|
|
@@ -1989,7 +2006,7 @@ def main() -> int:
|
|
|
1989
2006
|
"--key-file",
|
|
1990
2007
|
default="",
|
|
1991
2008
|
help="author key file; default resolves per backend (config-driven): "
|
|
1992
|
-
"
|
|
2009
|
+
"AUTORESEARCH_CLAUDE_KEY_FILE for claude, AUTORESEARCH_CODEX_KEY_FILE for codex",
|
|
1993
2010
|
)
|
|
1994
2011
|
parser.add_argument(
|
|
1995
2012
|
"--panel",
|