outerloop-science 0.1.0.dev0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. outerloop/__init__.py +18 -0
  2. outerloop/__main__.py +3 -0
  3. outerloop/appauth.py +213 -0
  4. outerloop/appmanifest.py +198 -0
  5. outerloop/attempt.py +3481 -0
  6. outerloop/brief.py +515 -0
  7. outerloop/cli.py +439 -0
  8. outerloop/climbboard.py +1145 -0
  9. outerloop/compute.py +482 -0
  10. outerloop/contract.py +483 -0
  11. outerloop/contract_cli.py +63 -0
  12. outerloop/disk.py +164 -0
  13. outerloop/dispatch.py +586 -0
  14. outerloop/followup.py +2143 -0
  15. outerloop/github.py +1486 -0
  16. outerloop/harness.py +1449 -0
  17. outerloop/housekeeping.py +167 -0
  18. outerloop/init.py +313 -0
  19. outerloop/intake.py +129 -0
  20. outerloop/limits.py +80 -0
  21. outerloop/markers.py +48 -0
  22. outerloop/measure.py +523 -0
  23. outerloop/orchestrator.py +1901 -0
  24. outerloop/panel.py +188 -0
  25. outerloop/paths.py +27 -0
  26. outerloop/posting.py +160 -0
  27. outerloop/progress.py +170 -0
  28. outerloop/py.typed +0 -0
  29. outerloop/review.py +611 -0
  30. outerloop/review_agent.py +263 -0
  31. outerloop/review_agent_cli.py +209 -0
  32. outerloop/review_post_cli.py +162 -0
  33. outerloop/review_summarize_cli.py +163 -0
  34. outerloop/role_runner.py +229 -0
  35. outerloop/roles.py +247 -0
  36. outerloop/rolespec.py +89 -0
  37. outerloop/runstate.py +385 -0
  38. outerloop/steward.py +852 -0
  39. outerloop/style.py +12 -0
  40. outerloop/syscall.py +977 -0
  41. outerloop/syscall_cli.py +531 -0
  42. outerloop/tick.py +3166 -0
  43. outerloop/verifier.py +403 -0
  44. outerloop/verify_agent.py +149 -0
  45. outerloop/verify_agent_cli.py +95 -0
  46. outerloop/verify_post_cli.py +116 -0
  47. outerloop_science-0.1.0.dev0.dist-info/METADATA +145 -0
  48. outerloop_science-0.1.0.dev0.dist-info/RECORD +52 -0
  49. outerloop_science-0.1.0.dev0.dist-info/WHEEL +4 -0
  50. outerloop_science-0.1.0.dev0.dist-info/entry_points.txt +2 -0
  51. outerloop_science-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
  52. outerloop_science-0.1.0.dev0.dist-info/licenses/NOTICE +5 -0
@@ -0,0 +1,1145 @@
1
+ """The climb board: what every attempt on a benchmark tried and what came
2
+ of it, published to the target's `research-log` branch as data plus two
3
+ views whenever runs end.
4
+
5
+ - `climb/data/<benchmark>.json` — one row per terminal attempt, dedup by run
6
+ id (its own directory: benchmark names are contract-controlled, and a
7
+ benchmark named `index` must not collide with `climb/index.json`);
8
+ the contract everything else derives from (and what an external dashboard
9
+ reads after the public flip).
10
+ - `CLIMB.md` — the numbers and the attempts table, rendered by GitHub.
11
+ - `index.html` — a self-contained page that charts the JSON next to it
12
+ (open it from a clone; a Pages site can serve it as-is later).
13
+
14
+ Publishing is idempotent by construction: rows merge by run id and files
15
+ are written only when their content changes, so the tick can call this
16
+ every pass without commit spam. Like the research-log ledger, the board is
17
+ advisory — a failure never stops the tick.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import contextlib
23
+ import json
24
+ import logging
25
+ import math
26
+ import re
27
+ from dataclasses import asdict, dataclass
28
+ from pathlib import Path
29
+ from typing import Any
30
+
31
+ from outerloop.markers import marker
32
+ from outerloop.runstate import ENDED, list_runs, run_dir
33
+
34
+ log = logging.getLogger("outerloop.climbboard")
35
+
36
+ BOARD_BRANCH = "research-log"
37
+ MAX_HYPOTHESIS_CHARS = 160
38
+ MAX_SUMMARY_CHARS = 90 # what the table shows; the full line stays in the row
39
+ MAX_CURVE_POINTS = 160
40
+ MAX_CURVE_RUNS_PER_AGENT = 5 # at most this many curves per agent, so one
41
+ # busy agent cannot crowd the panel AND every active agent stays represented
42
+ # (this per-agent cap is the whole bound — no global total ceiling, which
43
+ # would drop the oldest agent once agents * 5 exceeded it)
44
+ MAX_ROWS_PER_BENCHMARK = 2000
45
+
46
+
47
+ @dataclass(frozen=True)
48
+ class ClimbRow:
49
+ run_id: str
50
+ agent: str
51
+ ended: str # ISO date
52
+ outcome: str
53
+ baseline: float | None
54
+ candidate: float | None
55
+ gpu_hours: float
56
+ hypothesis: str
57
+ pr_url: str
58
+ report: str = "" # reports/<file>.md when the ledger has archived it
59
+ note: str = "" # the gate's own verdict sentence, when it recorded one
60
+
61
+
62
+ _NUM = re.compile(r"^(Baseline|Candidate): ([-+0-9.e]+)", re.M)
63
+ _HYP = re.compile(r"Hypothesis[:*\s]+(.+)", re.I)
64
+
65
+
66
+ def _report_fields(text: str) -> tuple[float | None, float | None, str]:
67
+ """(baseline, candidate, hypothesis one-liner) out of a run report."""
68
+ baseline = candidate = None
69
+ for key, raw in _NUM.findall(text):
70
+ try:
71
+ value = float(raw)
72
+ except ValueError:
73
+ continue
74
+ if key == "Baseline":
75
+ baseline = value
76
+ else:
77
+ candidate = value
78
+ hyp = ""
79
+ m = _HYP.search(text)
80
+ if m:
81
+ hyp = re.sub(r"[`*_]|\s+", lambda g: " " if g.group().isspace() else "", m.group(1))
82
+ hyp = hyp.strip().rstrip("-").strip()[:MAX_HYPOTHESIS_CHARS]
83
+ return baseline, candidate, hyp
84
+
85
+
86
+ def summarize(text: str, cap: int = MAX_SUMMARY_CHARS) -> str:
87
+ """The first sentence, capped — a table cell, not a paragraph."""
88
+ text = text.strip()
89
+ for stop in (". ", "; "):
90
+ i = text.find(stop)
91
+ if 0 < i < cap:
92
+ return text[: i + 1]
93
+ return text if len(text) <= cap else text[: cap - 1].rsplit(" ", 1)[0] + "…"
94
+
95
+
96
+ _CURVE_LINE = re.compile(
97
+ # digit bounds in the pattern: int() never sees more digits than fit a
98
+ # JS-safe integer, float() never sees a 400-nines mantissa (a longer
99
+ # number simply fails the match and the line sits out)
100
+ r"^step (\d{1,15}) val loss (\d{1,10}(?:\.\d{1,12})?(?:[eE][+-]?\d{1,3})?)(?=\s|$)",
101
+ re.M,
102
+ )
103
+ # a verbose eval must not exhaust the tick: stdout is scanned line by
104
+ # line and abandoned past this many bytes (curves are diagnostics)
105
+ MAX_CURVE_STDOUT_BYTES = 32 * 1024 * 1024
106
+
107
+
108
+ def _parse_curve_stdout(stdout_path: Path) -> list[list[float]]:
109
+ """(step, val loss) points parsed from an eval's stdout, downsampled.
110
+ steps.jsonl dies with the job's scratch; stdout is what survives."""
111
+ try:
112
+ with stdout_path.open("rb") as fh:
113
+ # a byte-mode bounded read caps memory whatever the content — a
114
+ # text-mode read counts characters and 4-byte UTF-8 overshoots 4x
115
+ raw = fh.read(MAX_CURVE_STDOUT_BYTES + 1)
116
+ except OSError:
117
+ return []
118
+ if len(raw) > MAX_CURVE_STDOUT_BYTES:
119
+ # truncated: drop the partial tail line so the EOF-tolerant pattern
120
+ # can never publish a number the cap cut in half
121
+ raw = raw[:MAX_CURVE_STDOUT_BYTES].rsplit(b"\n", 1)[0]
122
+ text = raw.decode("utf-8", errors="replace")
123
+ points = []
124
+ for m in _CURVE_LINE.finditer(text):
125
+ val = float(m.group(2))
126
+ if not math.isfinite(val): # e-notation can still overflow (1e999)
127
+ continue
128
+ points.append([int(m.group(1)), val])
129
+ if len(points) > MAX_CURVE_POINTS:
130
+ stride = len(points) / (MAX_CURVE_POINTS - 1)
131
+ points = [points[int(i * stride)] for i in range(MAX_CURVE_POINTS - 1)] + [points[-1]]
132
+ return points
133
+
134
+
135
+ def _curve_from_eval(run_directory: Path) -> list[list[float]]:
136
+ """The training curve behind a run's row: the newest CANDIDATE eval when
137
+ the run submitted, else the run's best LAUNCH experiment (lowest final
138
+ val loss). Without the launch fallback a run that tested experiments
139
+ but did not submit would show no curve, and its agent would vanish from
140
+ the panel."""
141
+
142
+ def mtime(d: Path) -> float:
143
+ try:
144
+ return d.stat().st_mtime
145
+ except OSError:
146
+ return 0.0 # vanished between glob and sort: sorts oldest, still readable-guarded
147
+
148
+ candidates = sorted(
149
+ (d for d in run_directory.glob("eval-candidate-*") if (d / "stdout").is_file()),
150
+ key=mtime,
151
+ )
152
+ if candidates:
153
+ points = _parse_curve_stdout(candidates[-1] / "stdout")
154
+ if points:
155
+ return points
156
+ # a candidate whose stdout has no parsable points falls through to
157
+ # the launch fallback rather than leaving the run curveless
158
+ # launch fallback: the experiment the agent did best on (lowest final val
159
+ # loss — a display heuristic for the min-oriented speedrun curve, not a
160
+ # credited measurement)
161
+ best: list[list[float]] = []
162
+ best_final: float | None = None
163
+ for d in run_directory.glob("eval-launch-*"):
164
+ if not (d / "stdout").is_file():
165
+ continue
166
+ points = _parse_curve_stdout(d / "stdout")
167
+ if points and (best_final is None or points[-1][1] < best_final):
168
+ best_final, best = points[-1][1], points
169
+ return best
170
+
171
+
172
+ def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]:
173
+ """Terminal attempts of `target` with a report, grouped by benchmark."""
174
+ from datetime import UTC, datetime
175
+
176
+ out: dict[str, list[ClimbRow]] = {}
177
+ for record in list_runs(root):
178
+ # only ENDED runs: an in-review run's outcome is not known yet (its
179
+ # PR may be rejected), and a published row is never rewritten
180
+ if record.target != target or record.state != ENDED:
181
+ continue
182
+ try:
183
+ report = (run_dir(root, record.run_id) / "report.md").read_text()
184
+ except OSError:
185
+ continue
186
+ baseline, candidate, hyp = _report_fields(report)
187
+ stage = record.stage or {}
188
+ ended = datetime.fromtimestamp(record.updated or record.created, tz=UTC)
189
+ outcome = record.ending or "ended"
190
+ # link the report only when the ledger's own marker says it is on the
191
+ # branch (adopted-unpublished history and not-yet-archived runs would
192
+ # otherwise render dead links)
193
+ report = ""
194
+ try:
195
+ marker = (run_dir(root, record.run_id) / "ledger-published").read_text()
196
+ lines = marker.splitlines()
197
+ if lines and lines[0].startswith(("archived", "pointer-pending", "done")):
198
+ if len(lines) > 1 and lines[1].startswith("reports/") and lines[1].endswith(".md"):
199
+ # the ledger's own path: an in-review archive keeps its
200
+ # date even after the ENDED transition re-stamps updated
201
+ report = lines[1]
202
+ else: # legacy marker without a path line
203
+ report = f"reports/{ended.strftime('%Y-%m-%d')}-{record.run_id}.md"
204
+ except OSError:
205
+ pass
206
+ out.setdefault(record.benchmark or "benchmark", []).append(
207
+ ClimbRow(
208
+ run_id=record.run_id,
209
+ agent=record.agent_id,
210
+ ended=ended.strftime("%Y-%m-%d %H:%M:%S"),
211
+ outcome=outcome,
212
+ baseline=baseline,
213
+ candidate=candidate,
214
+ gpu_hours=round(float(stage.get("gpu_hours_used") or 0.0), 2), # type: ignore[arg-type]
215
+ hypothesis=hyp,
216
+ pr_url=record.pr_url,
217
+ report=report,
218
+ note=summarize(record.ending_note or "", 120),
219
+ )
220
+ )
221
+ return out
222
+
223
+
224
+ def merge_rows(existing_json: str | None, fresh: list[ClimbRow]) -> list[dict[str, Any]]:
225
+ """Existing board rows plus any new ones, one per run id, oldest first.
226
+ A run already on the board keeps its published row (reports are final at
227
+ terminal state; the board never rewrites history)."""
228
+ rows: list[dict[str, Any]] = []
229
+ seen: set[str] = set()
230
+ if existing_json:
231
+ try:
232
+ for item in json.loads(existing_json):
233
+ if isinstance(item, dict) and item.get("run_id") not in seen:
234
+ rows.append(item)
235
+ seen.add(str(item.get("run_id")))
236
+ except ValueError:
237
+ log.warning("unreadable board JSON; rebuilding from local records")
238
+ for row in fresh:
239
+ if row.run_id not in seen:
240
+ rows.append(asdict(row))
241
+ seen.add(row.run_id)
242
+ rows.sort(key=lambda r: str(r.get("ended", "")))
243
+ # the board is a bounded VIEW (the contents API caps file sizes); the
244
+ # full history stays in reports/ on this same branch, and the trim is
245
+ # said out loud in CLIMB.md, never silent
246
+ return rows[-MAX_ROWS_PER_BENCHMARK:]
247
+
248
+
249
+ def _fmt(value: Any) -> str:
250
+ if value is None or value == "":
251
+ return "—"
252
+ if isinstance(value, float) and value == int(value):
253
+ return str(int(value))
254
+ return str(value)
255
+
256
+
257
+ def render_md(
258
+ target: str,
259
+ boards: dict[str, list[dict[str, Any]]],
260
+ directions: dict[str, str],
261
+ starts: dict[str, float] | None = None,
262
+ ) -> str:
263
+ """CLIMB.md: per benchmark, the headline numbers and the attempts table
264
+ (newest first). Plain markdown; the chart lives in index.html."""
265
+ lines = [
266
+ marker("climb-board"),
267
+ f"# Climb — {target}",
268
+ "",
269
+ "Written by the kernel when runs end. Data: `climb/data/<benchmark>.json`;",
270
+ "chart: open `index.html` from a clone of this branch.",
271
+ ]
272
+ for benchmark in sorted(boards):
273
+ rows = boards[benchmark]
274
+ direction = directions.get(benchmark, "min")
275
+ pick = max if direction == "max" else min
276
+ measured = [r for r in rows if isinstance(r.get("candidate"), int | float)]
277
+ improved = [r for r in rows if r.get("outcome") in ("merged", "improved")]
278
+ best = pick((r["candidate"] for r in measured), default=None)
279
+ gpu = sum(float(r.get("gpu_hours") or 0.0) for r in rows)
280
+ # baseline = the campaign's STARTING POSITION, one fixed ledger
281
+ # number (owner decision: per-run declared bases confused more than
282
+ # they informed as a headline)
283
+ start = (starts or {}).get(benchmark)
284
+ start_chip = f" · baseline (start): **{_fmt(start)}**" if start is not None else ""
285
+ lines += [
286
+ "",
287
+ f"## {benchmark}",
288
+ "",
289
+ f"Attempts: **{len(rows)}** ({len(improved)} improved) · best candidate: "
290
+ f"**{_fmt(best)}** ({direction}){start_chip} · GPU-hours: **{gpu:.1f}**",
291
+ ]
292
+ if len(rows) >= MAX_ROWS_PER_BENCHMARK:
293
+ lines += [
294
+ "",
295
+ f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts are on the board; "
296
+ "archived reports stay in `reports/` on this branch.",
297
+ ]
298
+ lines += [
299
+ "",
300
+ "| ended (UTC) | agent | hypothesis | outcome | candidate | GPU-h | full |",
301
+ "| --- | --- | --- | --- | --- | --- | --- |",
302
+ ]
303
+ for r in reversed(rows):
304
+ outcome = str(r.get("outcome", ""))
305
+ if r.get("pr_url"):
306
+ outcome = f"[{outcome}]({r['pr_url']})"
307
+ if r.get("note"):
308
+ # the gate's reason makes a near miss legible ("real
309
+ # movement, not creditable" reads differently from a DNF)
310
+ # flattened: a newline inside a cell would split the table row
311
+ flat = " ".join(str(r["note"]).split())
312
+ outcome += " — " + summarize(flat, 80).replace("|", "\\|")
313
+ hyp = summarize(str(r.get("hypothesis") or "")).replace("|", "\\|")
314
+ ended = str(r.get("ended", ""))
315
+ report = f"[report]({r['report']})" if r.get("report") else ""
316
+ lines.append(
317
+ f"| {ended} | {r.get('agent', '')} | {hyp} | {outcome} "
318
+ f"| {_fmt(r.get('candidate'))} | {_fmt(r.get('gpu_hours'))} | {report} |"
319
+ )
320
+ return "\n".join(lines) + "\n"
321
+
322
+
323
+ def render_html(
324
+ target: str,
325
+ boards: dict[str, list[dict[str, Any]]],
326
+ directions: dict[str, str],
327
+ curves: dict[str, dict[str, list[list[float]]]] | None = None,
328
+ ) -> str:
329
+ """One self-contained page: the data is EMBEDDED (a browser blocks
330
+ fetch() from a file:// page, and the direct-from-clone view must work),
331
+ only chart.js arrives from its CDN — nothing bulky is committed to the
332
+ target's branch. Light and dark follow the viewer's system theme."""
333
+ # "<" is escaped INSIDE the JSON (still valid JSON): a hypothesis line is
334
+ # agent-written text, and a literal </script> in it would close the inline
335
+ # script and run whatever follows in the published page
336
+ payload = json.dumps(
337
+ {"boards": boards, "directions": directions, "curves": curves or {}}
338
+ ).replace("<", "\\u003c")
339
+ return (
340
+ "<!doctype html>\n<html><head><meta charset='utf-8'>\n"
341
+ "<meta name='viewport' content='width=device-width, initial-scale=1'>\n"
342
+ f"<title>Climb — {target}</title>\n"
343
+ "<script src='https://cdn.jsdelivr.net/npm/chart.js@4'></script>\n"
344
+ "<style>\n"
345
+ ":root{--bg:#f6f7f4;--card:#ffffff;--ink:#1c2025;--muted:#5d6570;\n"
346
+ "--line:#e1e3de;--accent:#2d54ae;--win:#157a4b;--lose:#a7acb0;\n"
347
+ "--base:#b8433a;--near:#ac7714}\n"
348
+ "@media (prefers-color-scheme: dark){:root:not([data-theme=light]){\n"
349
+ "--bg:#121418;--card:#1a1d23;--ink:#e6e8eb;--muted:#99a1ad;\n"
350
+ "--line:#2a2e36;--accent:#7398e6;--win:#3cbd83;--lose:#656c76;\n"
351
+ "--base:#de7263;--near:#cf9c33}}\n"
352
+ ":root[data-theme=dark]{--bg:#121418;--card:#1a1d23;--ink:#e6e8eb;\n"
353
+ "--muted:#99a1ad;--line:#2a2e36;--accent:#7398e6;--win:#3cbd83;\n"
354
+ "--lose:#656c76;--base:#de7263;--near:#cf9c33}\n"
355
+ "a{color:var(--accent)}\n"
356
+ "header{display:flex;align-items:center;justify-content:space-between}\n"
357
+ "#theme{background:var(--card);border:1px solid var(--line);\n"
358
+ "color:var(--muted);border-radius:.4rem;padding:.2rem .6rem;\n"
359
+ "font-size:.75rem;cursor:pointer}\n"
360
+ "body{font-family:system-ui,-apple-system,sans-serif;background:var(--bg);\n"
361
+ "color:var(--ink);margin:0;padding:2.5rem 1.5rem;line-height:1.5}\n"
362
+ "main{max-width:920px;margin:0 auto}\n"
363
+ "header p{color:var(--muted);margin:.25rem 0 0;font-size:.9rem}\n"
364
+ "h1{font-size:1.5rem;margin:0}\n"
365
+ "h1 span{color:var(--muted);font-weight:400}\n"
366
+ "h2{font-size:1.1rem;margin:2.5rem 0 .75rem}\n"
367
+ ".chips{display:flex;flex-wrap:wrap;gap:.5rem;margin-bottom:1rem}\n"
368
+ ".chip{background:var(--card);border:1px solid var(--line);border-radius:.4rem;\n"
369
+ "padding:.35rem .7rem;font-size:.85rem;color:var(--muted)}\n"
370
+ ".chip b{color:var(--ink);font-variant-numeric:tabular-nums;font-weight:600}\n"
371
+ ".run{background:var(--card);border:1px solid var(--line);border-radius:.4rem;\n"
372
+ "padding:.45rem .7rem;font-size:.85rem;color:var(--muted);\n"
373
+ "width:16.5rem;box-sizing:border-box}\n"
374
+ ".run .top{display:flex;align-items:center;gap:.5rem}\n"
375
+ ".run b{color:var(--ink)}\n"
376
+ ".run .dir{margin-top:.15rem;font-size:.8rem}\n"
377
+ ".run .lm{display:grid;grid-template-columns:auto 1fr auto;\n"
378
+ " gap:.18rem .4rem;align-items:center;margin-top:.3rem;\n"
379
+ " font-size:.65rem;color:var(--muted)}\n"
380
+ ".run .bar{position:relative;height:3px;border-radius:2px;\n"
381
+ " background:var(--line);overflow:hidden}\n"
382
+ ".run .bar span{position:absolute;left:0;top:0;height:100%}\n"
383
+ ".run .meter{display:flex;gap:2px}\n"
384
+ ".run .meter i{flex:1;height:3px;border-radius:1px;background:var(--line)}\n"
385
+ ".clegend{display:flex;flex-wrap:wrap;gap:.4rem 1.4rem;\n"
386
+ "margin:.5rem 0 .25rem;font-size:.75rem;color:var(--muted)}\n"
387
+ ".clegend .lgroup{display:flex;align-items:center;gap:.55rem}\n"
388
+ ".clegend b{cursor:pointer;font-weight:600}\n"
389
+ ".clegend .litem{display:flex;align-items:center;gap:.3rem;cursor:pointer}\n"
390
+ ".clegend .litem i{width:14px;height:8px;border:2px solid;\n"
391
+ "border-radius:2px;display:inline-block;box-sizing:border-box}\n"
392
+ ".pill{border-radius:.6rem;padding:.05rem .55rem;font-size:.72rem;\n"
393
+ "font-weight:600;color:#fff}\n"
394
+ ".card{background:var(--card);border:1px solid var(--line);border-radius:.5rem;\n"
395
+ "padding:1rem}\n"
396
+ ".card label{font-size:.8rem;color:var(--muted);margin-right:1rem}\n"
397
+ "</style></head><body><main>\n"
398
+ f"<header><h1>{target} <span>· climb</span></h1>\n"
399
+ "<button id='theme' title='theme: auto / light / dark'>auto</button>\n"
400
+ "</header>\n"
401
+ "<div id='now' class='chips'></div>\n"
402
+ "<div id='charts'></div>\n<script>\n"
403
+ f"const data = {payload};\n"
404
+ "const css = n => getComputedStyle(document.body).getPropertyValue(n);\n"
405
+ "// theme: auto follows the OS; the button cycles auto/light/dark and\n"
406
+ "// the charts redraw so their computed colors follow\n"
407
+ "const themeBtn = document.getElementById('theme');\n"
408
+ "let onTheme = () => {};\n"
409
+ "const applyTheme = t => {\n"
410
+ " if (t === 'light' || t === 'dark')\n"
411
+ " document.documentElement.dataset.theme = t;\n"
412
+ " else delete document.documentElement.dataset.theme;\n"
413
+ " themeBtn.textContent = t;\n"
414
+ "};\n"
415
+ "let theme = 'auto';\n"
416
+ "try { theme = localStorage.getItem('climb-theme') || 'auto'; }\n"
417
+ "catch (e) {}\n"
418
+ "applyTheme(theme);\n"
419
+ "themeBtn.onclick = () => {\n"
420
+ " theme = theme === 'auto' ? 'light' : theme === 'light' ? 'dark' : 'auto';\n"
421
+ " try { localStorage.setItem('climb-theme', theme); } catch (e) {}\n"
422
+ " applyTheme(theme);\n"
423
+ " redraws.forEach(f => f()); onTheme();\n"
424
+ "};\n"
425
+ "// some hosts stamp data-theme on the root after load; without\n"
426
+ "// this the charts keep first-load colors on the flipped ground\n"
427
+ "new MutationObserver(() => { redraws.forEach(f => f()); onTheme(); })\n"
428
+ " .observe(document.documentElement,\n"
429
+ " {attributes: true, attributeFilter: ['data-theme']});\n"
430
+ "const mq = matchMedia('(prefers-color-scheme: dark)');\n"
431
+ "const onMq = () => { redraws.forEach(f => f()); onTheme(); };\n"
432
+ "// older iOS Safari has addListener only\n"
433
+ "if (mq.addEventListener) mq.addEventListener('change', onMq);\n"
434
+ "else if (mq.addListener) mq.addListener(onMq);\n"
435
+ "// legend: solid box = shown, hollow box = hidden (no strikethrough)\n"
436
+ "const boxLegend = {labels: {generateLabels: (chart) =>\n"
437
+ " chart.data.datasets.map((ds, i) => {\n"
438
+ " const shown = chart.isDatasetVisible(i);\n"
439
+ " const color = ds.borderColor || ds.pointBackgroundColor;\n"
440
+ " return {text: ds.label, datasetIndex: i, hidden: false,\n"
441
+ " fillStyle: shown ? color : 'rgba(0,0,0,0)',\n"
442
+ " strokeStyle: color, lineWidth: 2,\n"
443
+ " fontColor: css('--muted')};\n"
444
+ " })}};\n"
445
+ "const axis = () => ({ticks: {color: css('--muted')}, grid: {color: css('--line')}});\n"
446
+ "// tooltip copy stays a glance: first clause, strip-sized\n"
447
+ "const phrase = s => {\n"
448
+ " if (!s) return '';\n"
449
+ " for (const sep of ['. ', '; ']) {\n"
450
+ " const i = s.indexOf(sep);\n"
451
+ " if (i > 0 && i < 200) { s = s.slice(0, i + 1); break; }\n"
452
+ " }\n"
453
+ " for (const sep of [': ', ' \u2014 ', ' -- ']) {\n"
454
+ " const i = s.indexOf(sep);\n"
455
+ " if (i > 0) s = s.slice(0, i);\n"
456
+ " }\n"
457
+ " if (s.length > 64) s = s.slice(0, 63).trimEnd() + '\u2026';\n"
458
+ " return s;\n"
459
+ "};\n"
460
+ "const hues = [212, 152, 22, 282, 342, 62, 122, 242];\n"
461
+ "// identity color from the agent id itself, so the tooltip, the\n"
462
+ "// curves chart, and the strip agree even across benchmarks\n"
463
+ "const agentHue = a => {\n"
464
+ " const m = /(\\d+)$/.exec(a || '');\n"
465
+ " const i = m ? parseInt(m[1], 10) - 1\n"
466
+ " : [...String(a || '')].reduce((h, c) => h + c.charCodeAt(0), 0);\n"
467
+ " return hues[((i % 8) + 8) % 8];\n"
468
+ "};\n"
469
+ "const isDark = () => document.documentElement.dataset.theme === 'dark'\n"
470
+ " || (document.documentElement.dataset.theme !== 'light'\n"
471
+ " && matchMedia('(prefers-color-scheme: dark)').matches);\n"
472
+ "// t in [0,1]: 1 = the agent's identity color (its newest curve);\n"
473
+ "// smaller t fades toward the background, so earlier runs recede\n"
474
+ "// (lighter on paper, dimmer in the dark theme)\n"
475
+ "const agentShade = (a, t) => isDark()\n"
476
+ " ? `hsl(${agentHue(a)} ${40 + 25 * t}% ${30 + 22 * t}%)`\n"
477
+ " : `hsl(${agentHue(a)} ${45 + 15 * t}% ${72 - 24 * t}%)`;\n"
478
+ "const agentColor = a => agentShade(a, 1);\n"
479
+ "const redraws = [];\n"
480
+ "for (const b of Object.keys(data.boards).sort()) {\n"
481
+ " const rows = data.boards[b];\n"
482
+ " const dir = data.directions[b] === 'max' ? 'max' : 'min';\n"
483
+ " const pick = dir === 'max' ? Math.max : Math.min;\n"
484
+ " const won = new Set(['merged', 'improved']);\n"
485
+ " const beats = r => typeof r.baseline === 'number' && (dir === 'max'\n"
486
+ " ? r.candidate > r.baseline : r.candidate < r.baseline);\n"
487
+ " const measured = rows.filter(r => typeof r.candidate === 'number');\n"
488
+ " const baselines = measured.map(r => r.baseline).filter(v => typeof v === 'number');\n"
489
+ " // off-scale attempts (a DNF scored as the whole budget) squash the\n"
490
+ " // axis: hidden unless asked for. Off-scale = worse than 1.5x the\n"
491
+ " // worst baseline, whichever way this benchmark points.\n"
492
+ " const worstBase = baselines.length ? (dir === 'max'\n"
493
+ " ? Math.min(...baselines) : Math.max(...baselines)) : null;\n"
494
+ " const offScale = v => worstBase !== null && (dir === 'max'\n"
495
+ " ? v < worstBase / 1.5 : v > worstBase * 1.5);\n"
496
+ " let best;\n"
497
+ " const bestSoFar = measured.map(r =>\n"
498
+ " best = best === undefined ? r.candidate : pick(best, r.candidate));\n"
499
+ " const wins = rows.filter(r => won.has(r.outcome)).length;\n"
500
+ " const gpu = rows.reduce((a, r) => a + (r.gpu_hours || 0), 0);\n"
501
+ " const el = document.getElementById('charts');\n"
502
+ " const h = document.createElement('h2'); h.textContent = b; el.append(h);\n"
503
+ " const chips = document.createElement('div'); chips.className = 'chips';\n"
504
+ " const chip = (label, value) => {\n"
505
+ " const c = document.createElement('span'); c.className = 'chip';\n"
506
+ " const strong = document.createElement('b'); strong.textContent = value;\n"
507
+ " c.append(label + ' ', strong); chips.append(c); };\n"
508
+ " chip('attempts', rows.length); chip('improved', wins);\n"
509
+ " chip('best (' + dir + ')', measured.length ? bestSoFar[bestSoFar.length - 1] : '—');\n"
510
+ " chip('GPU-hours (recorded)', gpu.toFixed(1)); el.append(chips);\n"
511
+ " const card = document.createElement('div'); card.className = 'card';\n"
512
+ " const c = document.createElement('canvas'); card.append(c); el.append(card);\n"
513
+ " const logBox = document.createElement('label');\n"
514
+ " const cb = document.createElement('input'); cb.type = 'checkbox';\n"
515
+ " logBox.append(cb, ' log scale');\n"
516
+ " const outBox = document.createElement('label');\n"
517
+ " const ob = document.createElement('input'); ob.type = 'checkbox';\n"
518
+ " outBox.append(ob, ' show off-scale attempts');\n"
519
+ " card.append(logBox, outBox);\n"
520
+ " let chart;\n"
521
+ " const draw = () => {\n"
522
+ " const view = measured.filter(r => ob.checked || !offScale(r.candidate));\n"
523
+ " let vb;\n"
524
+ " const viewBest = view.map(r => vb = vb === undefined ? r.candidate\n"
525
+ " : pick(vb, r.candidate));\n"
526
+ " if (chart) chart.destroy();\n"
527
+ " chart = new Chart(c, {type: 'line', data: {labels: view.map(r => r.ended),\n"
528
+ " datasets: [\n"
529
+ " {label: 'candidate', data: view.map(r => r.candidate),\n"
530
+ " showLine: false, pointRadius: 4,\n"
531
+ " borderColor: css('--lose'),\n"
532
+ " pointBackgroundColor: view.map(r =>\n"
533
+ " won.has(r.outcome) ? css('--win')\n"
534
+ " : beats(r) ? css('--near') : css('--lose'))},\n"
535
+ " {label: 'best so far', data: viewBest, stepped: true,\n"
536
+ " borderColor: css('--accent'), borderWidth: 2, pointRadius: 0},\n"
537
+ " // markers, never a connected line: run base is per-run\n"
538
+ " // pairing context — a line reads as a trend, and a stale-base\n"
539
+ " // run landing after a merge can draw an upward step\n"
540
+ " // (borderColor feeds the legend swatch; no line is drawn)\n"
541
+ " {label: 'run base', data: view.map(r => r.baseline),\n"
542
+ " showLine: false, pointStyle: 'crossRot', pointRadius: 4,\n"
543
+ " borderColor: css('--base'),\n"
544
+ " pointBorderColor: css('--base'), pointBorderWidth: 1.5}]},\n"
545
+ " options: {color: css('--muted'),\n"
546
+ " scales: {x: {...axis(), ticks: {color: css('--muted'), maxTicksLimit: 8}},\n"
547
+ " y: {...axis(), type: cb.checked ? 'logarithmic' : 'linear'}},\n"
548
+ " plugins: {legend: boxLegend,\n"
549
+ " tooltip: {\n"
550
+ " // best-so-far and baseline pass exactly through candidate\n"
551
+ " // points; without the filter every coincident item repeats\n"
552
+ " // the same row in the pop-up\n"
553
+ " filter: i => i.datasetIndex === 0,\n"
554
+ " callbacks: {\n"
555
+ " title: its => { const r0 = view[its[0]?.dataIndex];\n"
556
+ " return r0 ? r0.agent + ' ' + (r0.ended || '').slice(0, 16) : ''; },\n"
557
+ " labelColor: i => ({\n"
558
+ " borderColor: agentColor(view[i.dataIndex].agent),\n"
559
+ " backgroundColor: agentColor(view[i.dataIndex].agent)}),\n"
560
+ " afterLabel: (i) =>\n"
561
+ " [phrase(view[i.dataIndex].hypothesis), view[i.dataIndex].note]\n"
562
+ " .filter(Boolean).join('\\n')}}}}});\n"
563
+ " };\n"
564
+ " cb.onchange = draw; ob.onchange = draw; draw(); redraws.push(draw);\n"
565
+ " // the training curves behind the numbers: newest attempts overlaid\n"
566
+ " const bcurves = data.curves[b] || {};\n"
567
+ " // newest curves, capped per agent so every agent stays on the panel\n"
568
+ " // (rows are oldest-first; walk newest-first, keep <=5 per agent)\n"
569
+ " const perAgent = {}; const picked = [];\n"
570
+ " for (let i = rows.length - 1; i >= 0; i--) {\n"
571
+ " const r = rows[i]; if (!bcurves[r.run_id]) continue;\n"
572
+ " perAgent[r.agent] = (perAgent[r.agent] || 0) + 1;\n"
573
+ " if (perAgent[r.agent] <= 5) picked.push(r);\n"
574
+ " }\n"
575
+ " const withCurve = picked.reverse();\n"
576
+ " if (withCurve.length) {\n"
577
+ " const h3 = document.createElement('h2');\n"
578
+ " h3.textContent = b + ' — training curves (newest attempts)';\n"
579
+ " const card2 = document.createElement('div'); card2.className = 'card';\n"
580
+ " const c2 = document.createElement('canvas'); card2.append(c2);\n"
581
+ " const lx = document.createElement('label');\n"
582
+ " const lxb = document.createElement('input'); lxb.type = 'checkbox';\n"
583
+ " lx.append(lxb, ' log x');\n"
584
+ " const ly = document.createElement('label');\n"
585
+ " const lyb = document.createElement('input'); lyb.type = 'checkbox';\n"
586
+ " ly.append(lyb, ' log y');\n"
587
+ " // custom legend: every run stays its own item, grouped under\n"
588
+ " // its agent — the agent name toggles the whole group, a run's\n"
589
+ " // box toggles that line (solid = visible, hollow = hidden)\n"
590
+ " const legendEl = document.createElement('div');\n"
591
+ " legendEl.className = 'clegend';\n"
592
+ " card2.append(legendEl, lx, ly);\n"
593
+ " el.append(h3, card2);\n"
594
+ " let chart2;\n"
595
+ " const buildLegend = () => {\n"
596
+ " legendEl.textContent = '';\n"
597
+ " const groups = [];\n"
598
+ " withCurve.forEach((r, i) => {\n"
599
+ " let g = groups.find(x => x.agent === r.agent);\n"
600
+ " if (!g) { g = {agent: r.agent, items: []}; groups.push(g); }\n"
601
+ " g.items.push(i);\n"
602
+ " });\n"
603
+ " for (const g of groups) {\n"
604
+ " const wrap = document.createElement('span');\n"
605
+ " wrap.className = 'lgroup';\n"
606
+ " const name = document.createElement('b');\n"
607
+ " name.textContent = g.agent;\n"
608
+ " name.style.color = agentColor(g.agent);\n"
609
+ " name.title = 'toggle all ' + g.agent + ' runs';\n"
610
+ " name.onclick = () => {\n"
611
+ " const anyOn = g.items.some(i => chart2.isDatasetVisible(i));\n"
612
+ " g.items.forEach(i => chart2.setDatasetVisibility(i, !anyOn));\n"
613
+ " chart2.update(); buildLegend();\n"
614
+ " };\n"
615
+ " wrap.append(name);\n"
616
+ " for (const i of g.items) {\n"
617
+ " const it = document.createElement('span');\n"
618
+ " it.className = 'litem';\n"
619
+ " const box = document.createElement('i');\n"
620
+ " const color = chart2.data.datasets[i].borderColor;\n"
621
+ " box.style.borderColor = color;\n"
622
+ " if (chart2.isDatasetVisible(i)) box.style.background = color;\n"
623
+ " const lbl = document.createElement('span');\n"
624
+ " lbl.textContent = (withCurve[i].ended || '').slice(5, 16);\n"
625
+ " it.title = 'toggle this run';\n"
626
+ " it.append(box, lbl);\n"
627
+ " it.onclick = () => {\n"
628
+ " chart2.setDatasetVisibility(i, !chart2.isDatasetVisible(i));\n"
629
+ " chart2.update(); buildLegend();\n"
630
+ " };\n"
631
+ " wrap.append(it);\n"
632
+ " }\n"
633
+ " legendEl.append(wrap);\n"
634
+ " }\n"
635
+ " };\n"
636
+ " const draw2 = () => {\n"
637
+ " const hidden = chart2\n"
638
+ " ? withCurve.map((_, i) => !chart2.isDatasetVisible(i)) : [];\n"
639
+ " if (chart2) chart2.destroy();\n"
640
+ " const order = {};\n"
641
+ " withCurve.forEach((r, i) => {\n"
642
+ " (order[r.agent] = order[r.agent] || []).push(i);\n"
643
+ " });\n"
644
+ " const recency = (r, i) => {\n"
645
+ " const sibs = order[r.agent];\n"
646
+ " return sibs.length > 1 ? sibs.indexOf(i) / (sibs.length - 1) : 1;\n"
647
+ " };\n"
648
+ " chart2 = new Chart(c2, {type: 'line', data: {datasets:\n"
649
+ " withCurve.map((r, i) => ({\n"
650
+ " label: r.agent + ' ' + (r.ended || '').slice(5, 16),\n"
651
+ " data: bcurves[r.run_id].map(p => ({x: p[0], y: p[1]})),\n"
652
+ " borderColor: agentShade(r.agent, recency(r, i)),\n"
653
+ " hidden: hidden[i] || false,\n"
654
+ " borderWidth: 1.5, pointRadius: 0}))},\n"
655
+ " options: {color: css('--muted'), parsing: false,\n"
656
+ " scales: {x: {...axis(), type: lxb.checked ? 'logarithmic' : 'linear'},\n"
657
+ " y: {...axis(), type: lyb.checked ? 'logarithmic' : 'linear'}},\n"
658
+ " plugins: {legend: {display: false}}}});\n"
659
+ " buildLegend();\n"
660
+ " };\n"
661
+ " lxb.onchange = draw2; lyb.onchange = draw2; draw2();\n"
662
+ " redraws.push(draw2);\n"
663
+ " }\n"
664
+ "}\n"
665
+ "const now = document.getElementById('now');\n"
666
+ "let strip = null;\n"
667
+ "const refresh = () =>\n"
668
+ " fetch('climb/status.json', {cache: 'no-cache'})\n"
669
+ " .then(r => r.ok ? r.json() : null)\n"
670
+ " .then(s => { if (s && Array.isArray(s.runs)) { strip = s; render(); } })\n"
671
+ " .catch(() => {});\n"
672
+ "const stateHue = r => r.state === 'in-review' ? '150 55% 38%'\n"
673
+ " : r.state === 'implementing' ? '262 45% 52%'\n"
674
+ " : r.phase === 'author-sleep' ? '212 55% 46%' : '38 65% 42%';\n"
675
+ "// kernel phase names, translated for the page: 'in gate' = the\n"
676
+ "// kernel is measuring a submitted candidate; 'experiments' = the\n"
677
+ "// author launched its own jobs and sleeps until they finish\n"
678
+ "const stateName = r => r.state !== 'waiting' ? r.state.replace('-', ' ')\n"
679
+ " : r.phase === 'author-sleep' ? 'experiments'\n"
680
+ " : r.phase === 'candidate' ? 'in gate' : 'waiting';\n"
681
+ "const render = () => {\n"
682
+ " if (!strip) return;\n"
683
+ " now.textContent = '';\n"
684
+ " for (const r of strip.runs) {\n"
685
+ " const card = document.createElement('span'); card.className = 'run';\n"
686
+ " const top = document.createElement('span'); top.className = 'top';\n"
687
+ " const pill = document.createElement('span'); pill.className = 'pill';\n"
688
+ " pill.style.background = `hsl(${stateHue(r)})`;\n"
689
+ " pill.textContent = stateName(r);\n"
690
+ " const who = document.createElement('b'); who.textContent = r.agent;\n"
691
+ " who.style.color = agentColor(r.agent);\n"
692
+ " const mins = Math.max(0, (Date.now() / 1000 - r.since) / 60);\n"
693
+ " const t = mins >= 90 ? (mins / 60).toFixed(1) + ' h' : Math.round(mins) + ' min';\n"
694
+ " const meta = document.createElement('span');\n"
695
+ " meta.title = 'time in this state '\n"
696
+ " + '(since the last transition: park, submit, wake)';\n"
697
+ " meta.textContent = t;\n"
698
+ " top.append(who, pill, meta);\n"
699
+ " if (r.pr_url) {\n"
700
+ " const a = document.createElement('a'); a.href = r.pr_url;\n"
701
+ " a.textContent = 'PR'; top.append(a);\n"
702
+ " }\n"
703
+ " card.append(top);\n"
704
+ " if (r.direction) {\n"
705
+ " const d = document.createElement('span'); d.className = 'dir';\n"
706
+ " d.textContent = r.direction; d.style.display = 'block';\n"
707
+ " card.append(d);\n"
708
+ " }\n"
709
+ " // the agent's life panel: 'exps'/'gate' is the current wait\n"
710
+ " // (bar = progressing), 'depth' and 'GPU-h' are budgets being\n"
711
+ " // consumed (dash meters = life spent)\n"
712
+ " const lm = document.createElement('span'); lm.className = 'lm';\n"
713
+ " const cell = (cls) => {\n"
714
+ " const s = document.createElement('span');\n"
715
+ " if (cls) s.className = cls; return s;\n"
716
+ " };\n"
717
+ " const addRow = (label, mid, value, title) => {\n"
718
+ " const l = cell(); l.textContent = label;\n"
719
+ " const v = cell(); v.textContent = value;\n"
720
+ " l.title = mid.title = v.title = title;\n"
721
+ " lm.append(l, mid, v);\n"
722
+ " return v;\n"
723
+ " };\n"
724
+ " const timeFrac = m =>\n"
725
+ " Math.min(1, (Date.now() / 1000 - r.since) / (m * 60));\n"
726
+ " const layer = (w, color) => {\n"
727
+ " const f = document.createElement('span');\n"
728
+ " f.style.width = Math.round(w * 100) + '%';\n"
729
+ " f.style.background = color; return f;\n"
730
+ " };\n"
731
+ " const dashes = (filled, segs) => {\n"
732
+ " const m = cell('meter');\n"
733
+ " for (let i = 0; i < segs; i++) {\n"
734
+ " const seg = document.createElement('i');\n"
735
+ " if (i < filled) seg.style.background = agentColor(r.agent);\n"
736
+ " m.append(seg);\n"
737
+ " }\n"
738
+ " return m;\n"
739
+ " };\n"
740
+ " if (r.exp_total) {\n"
741
+ " const bar = cell('bar');\n"
742
+ " if (r.exp_minutes)\n"
743
+ " bar.append(layer(timeFrac(r.exp_minutes),\n"
744
+ " agentColor(r.agent).slice(0, -1) + ' / .3)'));\n"
745
+ " bar.append(layer(r.exp_done / r.exp_total, agentColor(r.agent)));\n"
746
+ " addRow('exps', bar, r.exp_done + '/' + r.exp_total + ' done',\n"
747
+ " 'experiment jobs: solid = finished, '\n"
748
+ " + 'faint = elapsed vs the longest walltime cap');\n"
749
+ " } else if (r.phase === 'candidate' && r.eval_minutes) {\n"
750
+ " const bar = cell('bar');\n"
751
+ " // the fill stays the agent's color; overdue is flagged by\n"
752
+ " // the number turning the baseline red\n"
753
+ " bar.append(layer(timeFrac(r.eval_minutes), agentColor(r.agent)));\n"
754
+ " const v = addRow('gate', bar,\n"
755
+ " ((Date.now() / 1000 - r.since) / 3600).toFixed(1)\n"
756
+ " + '/' + (r.eval_minutes / 60).toFixed(1) + ' h',\n"
757
+ " 'gate eval: time since submission vs its walltime cap. Queue'\n"
758
+ " + ' time counts, so past the cap the job is either still'\n"
759
+ " + ' queued or timed out awaiting the next sweep.');\n"
760
+ " if (timeFrac(r.eval_minutes) >= 1) v.style.color = 'var(--base)';\n"
761
+ " }\n"
762
+ " if (r.depth_k && r.launches_used != null)\n"
763
+ " addRow('depth', dashes(r.launches_used, r.depth_k),\n"
764
+ " r.launches_used + '/' + r.depth_k,\n"
765
+ " 'experiment launches used, of the contract depth_k');\n"
766
+ " if (r.gpu_hours_budget && r.gpu_hours_used != null)\n"
767
+ " addRow('GPU-h',\n"
768
+ " dashes(Math.round(20 * r.gpu_hours_used / r.gpu_hours_budget), 20),\n"
769
+ " Number(r.gpu_hours_used).toFixed(0)\n"
770
+ " + '/' + Number(r.gpu_hours_budget),\n"
771
+ " 'GPU-hours charged to this run, of the contract budget');\n"
772
+ " if (lm.childNodes.length) card.append(lm);\n"
773
+ " now.append(card);\n"
774
+ " }\n"
775
+ " if (!strip.runs.length) now.textContent = 'no active runs';\n"
776
+ "};\n"
777
+ "refresh();\n"
778
+ "// the strip re-FETCHES every few minutes (new/left/changed runs) and\n"
779
+ "// re-renders the elapsed time locally between fetches\n"
780
+ "onTheme = render;\n"
781
+ "setInterval(refresh, 180000); setInterval(render, 30000);\n"
782
+ "</script></main></body></html>\n"
783
+ )
784
+
785
+
786
+ STATUS_PATH = "climb/status.json"
787
+ _LIVE_STATES = ("implementing", "waiting", "in-review", "concluding")
788
+
789
+
790
+ def _phrase(text: str, cap: int = 64) -> str:
791
+ """A strip-sized phrase: the first clause of the first sentence."""
792
+ # markdown bold reads as literal asterisks on the strip (underscores
793
+ # stay: __init__.py is a filename far more often than __bold__)
794
+ first = summarize(text.replace("**", ""), 200)
795
+ for sep in (": ", " — ", " -- "):
796
+ first = first.split(sep, 1)[0]
797
+ return summarize(first, cap)
798
+
799
+
800
+ def _experiment_progress(root: Path, record: Any) -> tuple[int, int, int]:
801
+ """(finished, launched, longest walltime minutes) across the current
802
+ park's experiment jobs, counted by the exit-code files the job wrappers
803
+ leave — filesystem only, no Slurm calls on the board path."""
804
+ stage = record.stage or {}
805
+ raw = stage.get("syscall_launches", [])
806
+ if not isinstance(raw, list):
807
+ return (0, 0, 0)
808
+ names: list[str] = []
809
+ minutes = 0
810
+ for item in raw:
811
+ if not (isinstance(item, dict) and item.get("name")):
812
+ continue
813
+ try:
814
+ array = int(item.get("array") or 1)
815
+ except (TypeError, ValueError):
816
+ array = 1
817
+ with contextlib.suppress(TypeError, ValueError):
818
+ minutes = max(minutes, int(item.get("minutes") or 0))
819
+ name = str(item["name"])
820
+ names += [f"{name}.{i}" for i in range(array)] if array > 1 else [name]
821
+ rd = run_dir(root, record.run_id)
822
+ done = sum(1 for n in names if (rd / f"eval-launch-{n}" / "exit-code").exists())
823
+ return (done, len(names), minutes)
824
+
825
+
826
+ def collect_status(root: Path, target: str, now: float, contract: Any = None) -> dict[str, Any]:
827
+ """The fleet's live picture for `target`: one entry per non-terminal run.
828
+ Timestamps, not durations — the page computes elapsed time client-side,
829
+ so the strip feels live between pushes."""
830
+ from outerloop.dispatch import effective_eval_minutes
831
+
832
+ budgets = {
833
+ b.name: (b.depth_k, b.sleep_k, getattr(b, "eval_minutes", 0) or 0)
834
+ for b in getattr(contract, "benchmarks", ())
835
+ }
836
+ gpu_budget = getattr(getattr(contract, "budgets", None), "gpu_hours_per_run", None)
837
+ runs = []
838
+ for record in list_runs(root):
839
+ if record.target != target or record.state not in _LIVE_STATES:
840
+ continue
841
+ stage = record.stage or {}
842
+ note = str(stage.get("syscall_note") or stage.get("report") or "")
843
+ _b, _c, hyp = _report_fields(note)
844
+ exp_done, exp_total, exp_minutes = _experiment_progress(root, record)
845
+ depth_k, sleep_k, bench_minutes = budgets.get(record.benchmark, (None, None, 0))
846
+ runs.append(
847
+ {
848
+ "run_id": record.run_id,
849
+ "agent": record.agent_id,
850
+ "benchmark": record.benchmark,
851
+ "state": record.state,
852
+ "phase": stage.get("phase", ""),
853
+ # the agent's own headline: what it says it is working on
854
+ "direction": _phrase(hyp or note.replace("\n", " ")),
855
+ "since": record.updated or record.created,
856
+ # a run that never launched HAS used zero — absent keys must
857
+ # not blank the card's meters (a plain submit writes none)
858
+ "launches_used": int(stage.get("launches_used") or 0), # type: ignore[call-overload]
859
+ "sleeps_used": int(stage.get("sleeps_used") or 0), # type: ignore[call-overload]
860
+ "depth_k": depth_k,
861
+ "sleep_k": sleep_k,
862
+ # experiment fan-out of the current park, and the gate eval's
863
+ # walltime cap — the page turns these into progress bars
864
+ "exp_done": exp_done,
865
+ "exp_total": exp_total,
866
+ "exp_minutes": exp_minutes,
867
+ # submit without --minutes stores nothing: fall back to the
868
+ # contract cap, CLAMPED like the dispatched job itself is —
869
+ # the meter must show the limit the gate actually runs under
870
+ "eval_minutes": (
871
+ effective_eval_minutes(m)
872
+ if (m := int(stage.get("eval_minutes", 0) or 0) or bench_minutes) # type: ignore[call-overload]
873
+ else 0
874
+ ),
875
+ "gpu_hours_used": float(stage.get("gpu_hours_used") or 0.0), # type: ignore[arg-type]
876
+ "gpu_hours_budget": gpu_budget,
877
+ "pr_url": record.pr_url,
878
+ }
879
+ )
880
+ runs.sort(key=lambda r: str(r.get("run_id")))
881
+ return {"target": target, "published": now, "runs": runs}
882
+
883
+
884
+ def service_status(root: Path, github: Any, target: str, now: float, contract: Any = None) -> bool:
885
+ """Publish the strip when the fleet's SHAPE changed — a run appearing,
886
+ leaving, or changing state/phase — never on every tick: the page shows
887
+ elapsed time client-side, so timestamp-only drift is not worth a commit.
888
+ Advisory like the board; True when a write happened."""
889
+ status = collect_status(root, target, now, contract)
890
+ try:
891
+ existing_raw: str | None = github.get_file(target, STATUS_PATH, BOARD_BRANCH)
892
+ except Exception as exc:
893
+ if getattr(exc, "status", None) != 404:
894
+ # an outage must not look like a missing file: a rewrite here
895
+ # would commit a new timestamp on every affected tick
896
+ log.warning("status unreadable (%s); not rewritten", exc)
897
+ return False
898
+ existing_raw = None
899
+ if existing_raw:
900
+ try:
901
+ existing = json.loads(existing_raw)
902
+ # spend belongs in the shape: a same-phase re-park after new
903
+ # launches moves gpu_hours_used and the strip must not go stale
904
+ # exp_done/launches_used move when an experiment finishes or a
905
+ # re-park launches more; the contract caps move when the owner
906
+ # edits the contract — all real transitions the strip must show
907
+ keys = (
908
+ "run_id",
909
+ "state",
910
+ "phase",
911
+ "gpu_hours_used",
912
+ "gpu_hours_budget",
913
+ "direction",
914
+ "exp_done",
915
+ "exp_total",
916
+ "exp_minutes",
917
+ "eval_minutes",
918
+ "launches_used",
919
+ "depth_k",
920
+ "sleep_k",
921
+ )
922
+ shape = lambda runs: [{k: r.get(k) for k in keys} for r in runs]
923
+ if (
924
+ isinstance(existing, dict)
925
+ and isinstance(existing.get("runs"), list)
926
+ and shape(existing["runs"]) == shape(status["runs"])
927
+ ):
928
+ return False
929
+ except (ValueError, TypeError, AttributeError):
930
+ pass # malformed: the strip is derived, not history — rewrite it
931
+ if not github.ensure_branch(target, BOARD_BRANCH):
932
+ return False
933
+ return bool(
934
+ github.put_file(
935
+ target, STATUS_PATH, json.dumps(status, indent=1) + "\n", BOARD_BRANCH, "fleet status"
936
+ )
937
+ )
938
+
939
+
940
+ def _valid_curve(curve: Any) -> bool:
941
+ """A publishable curve: [step, value] pairs, numbers only — one null
942
+ point in a published file would throw in the page's chart code."""
943
+
944
+ def finite(x: Any) -> bool:
945
+ try:
946
+ return math.isfinite(x)
947
+ except OverflowError: # an int too large for float is not a chart point
948
+ return False
949
+
950
+ return isinstance(curve, list) and all(
951
+ isinstance(p, list)
952
+ and len(p) == 2
953
+ and all(isinstance(x, int | float) and not isinstance(x, bool) and finite(x) for x in p)
954
+ for p in curve
955
+ )
956
+
957
+
958
+ def _merge_curves(
959
+ github: Any,
960
+ target: str,
961
+ benchmark: str,
962
+ rows: list[dict[str, Any]],
963
+ fresh_for: Any,
964
+ ) -> dict[str, Any] | None:
965
+ """Published curves plus new ones, kept only for the newest rows (curves
966
+ are heavy; the cap is by recency of the attempt, and published curves
967
+ are never rewritten). `fresh_for(run_id)` parses a run's eval stdout ON
968
+ DEMAND — a run with a published curve costs no I/O at all. None when
969
+ the published file cannot be read — the same sit-the-pass-out stance
970
+ as everything else on the branch."""
971
+ path = f"climb/curves/{benchmark}.json"
972
+ try:
973
+ raw: str | None = github.get_file(target, path, BOARD_BRANCH)
974
+ except Exception as exc:
975
+ if getattr(exc, "status", None) != 404:
976
+ log.warning("board curves unreadable for %s (%s); skipped", benchmark, exc)
977
+ return None
978
+ raw = None
979
+ published: dict[str, list[list[float]]] = {}
980
+ if raw:
981
+ try:
982
+ data = json.loads(raw)
983
+ except ValueError:
984
+ data = None
985
+ if not isinstance(data, dict) or not all(_valid_curve(v) for v in data.values()):
986
+ # PARTLY malformed is malformed: dropping the bad values and
987
+ # rewriting would lose published curves
988
+ log.warning("board curves malformed for %s; skipped", benchmark)
989
+ return None
990
+ published = {str(k): v for k, v in data.items()}
991
+ # keep the newest curves, at most MAX_CURVE_RUNS_PER_AGENT per agent so
992
+ # one busy agent cannot crowd out the others. Walk ALL rows newest-first
993
+ # (no global pre-slice — that could bury a quiet agent whose rows all sit
994
+ # past the cut), and count the quota only for rows that ACTUALLY have a
995
+ # curve (a curveless attempt must not spend an agent's five slots).
996
+ per_agent: dict[str, int] = {}
997
+ merged: dict[str, list[list[float]]] = {}
998
+ for r in reversed(rows):
999
+ agent = str(r.get("agent") or "")
1000
+ if per_agent.get(agent, 0) >= MAX_CURVE_RUNS_PER_AGENT:
1001
+ continue
1002
+ run_id = str(r.get("run_id"))
1003
+ curve = published.get(run_id) or fresh_for(run_id)
1004
+ if not curve:
1005
+ continue
1006
+ per_agent[agent] = per_agent.get(agent, 0) + 1
1007
+ merged[run_id] = curve
1008
+ return {"data": merged, "changed": merged != published}
1009
+
1010
+
1011
+ def contract_directions(contract: Any) -> dict[str, str]:
1012
+ """{benchmark: direction} out of a loaded contract (None -> {})."""
1013
+ if contract is None:
1014
+ return {}
1015
+ return {b.name: b.direction for b in contract.benchmarks}
1016
+
1017
+
1018
+ def _read_index(github: Any, target: str) -> dict[str, str] | None:
1019
+ """climb/index.json: {benchmark: direction} for every benchmark ever
1020
+ published — the board's own memory, so a benchmark whose local records
1021
+ were cleaned up (or that left the contract) keeps its place in the
1022
+ views. {} means the board has no index yet (a 404); None means the read
1023
+ FAILED — the caller must not rewrite the index it could not see, or a
1024
+ transient outage would shrink it."""
1025
+ try:
1026
+ raw = github.get_file(target, "climb/index.json", BOARD_BRANCH)
1027
+ except Exception as exc:
1028
+ status = getattr(exc, "status", None)
1029
+ if status == 404:
1030
+ return {}
1031
+ log.warning("climb index unreadable (%s); not rewriting it", exc)
1032
+ return None
1033
+ try:
1034
+ data = json.loads(raw)
1035
+ except ValueError:
1036
+ log.warning("climb index malformed; not rewriting it")
1037
+ return None # same stance as an outage: never rewrite what we cannot see
1038
+ return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else None
1039
+
1040
+
1041
+ def service_climb_board(
1042
+ root: Path, github: Any, target: str, directions: dict[str, str] | None = None
1043
+ ) -> int:
1044
+ """Publish the board for `target`. Returns how many files changed.
1045
+
1046
+ Every file is compared against the branch, and all changed files land
1047
+ as ONE commit (`put_files`) — data, curves, and the views are atomic,
1048
+ so the page can never point at data that is not on the branch, and a
1049
+ board pass costs at most one commit of research-log history. A failed
1050
+ batch changes nothing; the whole pass retries next tick."""
1051
+ local = collect_rows(root, target)
1052
+ # snapshot BEFORE any branch read: put_files refuses if the head moves
1053
+ # mid-pass, so a concurrent write is never buried under stale content.
1054
+ # "" = branch missing (nothing to protect); None = outage — writing
1055
+ # unguarded could bury a mid-pass write, so the pass sits out
1056
+ head = github.branch_head(target, BOARD_BRANCH)
1057
+ if head is None:
1058
+ log.warning("board head unreadable for %s; pass sits out", target)
1059
+ return 0
1060
+ index = _read_index(github, target)
1061
+ names = set(local) | set(index or {})
1062
+ directions = {**(index or {}), **(directions or {})}
1063
+ pending: dict[str, str] = {}
1064
+ boards: dict[str, list[dict[str, Any]]] = {}
1065
+ curves: dict[str, dict[str, list[list[float]]]] = {}
1066
+ curves_ok = True
1067
+ for benchmark in sorted(names):
1068
+ path = f"climb/data/{benchmark}.json"
1069
+ try:
1070
+ existing: str | None = github.get_file(target, path, BOARD_BRANCH)
1071
+ if existing is not None:
1072
+ parsed = json.loads(existing)
1073
+ if not isinstance(parsed, list):
1074
+ raise ValueError("board data is not a list")
1075
+ except (ValueError, TypeError) as exc:
1076
+ # readable but not a row list: same stance as an outage — never
1077
+ # let a fresh merge overwrite what we cannot interpret
1078
+ log.warning("board JSON malformed for %s (%s); skipped", benchmark, exc)
1079
+ continue
1080
+ except Exception as exc:
1081
+ if getattr(exc, "status", None) != 404:
1082
+ # an outage is not an empty board: overwriting would replace
1083
+ # this benchmark's published history — sit the pass out (its
1084
+ # index entry survives)
1085
+ log.warning("board JSON unreadable for %s (%s); skipped", benchmark, exc)
1086
+ continue
1087
+ existing = None
1088
+ rows = merge_rows(existing, local.get(benchmark, []))
1089
+ if not rows:
1090
+ continue
1091
+ boards[benchmark] = rows
1092
+ text = json.dumps(rows, indent=1) + "\n"
1093
+ if text != existing:
1094
+ pending[path] = text
1095
+ merged_curves = _merge_curves(
1096
+ github, target, benchmark, rows, lambda rid: _curve_from_eval(run_dir(root, rid))
1097
+ )
1098
+ if merged_curves is None:
1099
+ # an unreadable curve file must not republish the page without
1100
+ # its curves — the html rewrite sits this pass out
1101
+ curves_ok = False
1102
+ continue
1103
+ curves[benchmark] = merged_curves["data"]
1104
+ if merged_curves["changed"]:
1105
+ pending[f"climb/curves/{benchmark}.json"] = (
1106
+ json.dumps(merged_curves["data"], indent=1) + "\n"
1107
+ )
1108
+ if not boards and names:
1109
+ return 0 # every benchmark sat the pass out: leave the views alone
1110
+ # the index keeps every benchmark it knows — a transient failed read of
1111
+ # one benchmark's JSON must not orphan its history; the views simply
1112
+ # render without it until a later pass reads it again. An index that
1113
+ # could not be READ at all (None) is never rewritten this pass.
1114
+ wanted = {b: directions.get(b, "min") for b in sorted(set(boards) | set(index or {}))}
1115
+ # the starting positions come from the target's ledger on its default
1116
+ # branch — best-effort: a missing or unreadable ledger just drops the chip
1117
+ starts: dict[str, float] = {}
1118
+ try:
1119
+ # ref HEAD = the repo's default branch, whatever it is named
1120
+ raw_leader = github.get_file_content(target, "results/leader.json", "HEAD")
1121
+ if raw_leader:
1122
+ for name, entry in json.loads(raw_leader).items():
1123
+ value = entry.get("baseline") if isinstance(entry, dict) else None
1124
+ # json.loads admits NaN/Infinity, which _fmt cannot render
1125
+ if isinstance(value, int | float) and math.isfinite(value):
1126
+ starts[str(name)] = float(value)
1127
+ except Exception:
1128
+ starts = {}
1129
+ views: list[tuple[str, str]] = [("CLIMB.md", render_md(target, boards, wanted, starts))]
1130
+ if curves_ok:
1131
+ views.append(("index.html", render_html(target, boards, wanted, curves)))
1132
+ if index is not None:
1133
+ views.insert(0, ("climb/index.json", json.dumps(wanted, indent=1) + "\n"))
1134
+ for path, content in views:
1135
+ if content != github.get_file_content(target, path, BOARD_BRANCH):
1136
+ pending[path] = content
1137
+ if not pending:
1138
+ return 0
1139
+ if not github.ensure_branch(target, BOARD_BRANCH):
1140
+ return 0
1141
+ if not github.put_files(
1142
+ target, pending, BOARD_BRANCH, "climb board", expected_head=head or None
1143
+ ):
1144
+ return 0
1145
+ return len(pending)