outerloop-science 0.1.0.dev0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- outerloop/__init__.py +18 -0
- outerloop/__main__.py +3 -0
- outerloop/appauth.py +213 -0
- outerloop/appmanifest.py +198 -0
- outerloop/attempt.py +3481 -0
- outerloop/brief.py +515 -0
- outerloop/cli.py +439 -0
- outerloop/climbboard.py +1145 -0
- outerloop/compute.py +482 -0
- outerloop/contract.py +483 -0
- outerloop/contract_cli.py +63 -0
- outerloop/disk.py +164 -0
- outerloop/dispatch.py +586 -0
- outerloop/followup.py +2143 -0
- outerloop/github.py +1486 -0
- outerloop/harness.py +1449 -0
- outerloop/housekeeping.py +167 -0
- outerloop/init.py +313 -0
- outerloop/intake.py +129 -0
- outerloop/limits.py +80 -0
- outerloop/markers.py +48 -0
- outerloop/measure.py +523 -0
- outerloop/orchestrator.py +1901 -0
- outerloop/panel.py +188 -0
- outerloop/paths.py +27 -0
- outerloop/posting.py +160 -0
- outerloop/progress.py +170 -0
- outerloop/py.typed +0 -0
- outerloop/review.py +611 -0
- outerloop/review_agent.py +263 -0
- outerloop/review_agent_cli.py +209 -0
- outerloop/review_post_cli.py +162 -0
- outerloop/review_summarize_cli.py +163 -0
- outerloop/role_runner.py +229 -0
- outerloop/roles.py +247 -0
- outerloop/rolespec.py +89 -0
- outerloop/runstate.py +385 -0
- outerloop/steward.py +852 -0
- outerloop/style.py +12 -0
- outerloop/syscall.py +977 -0
- outerloop/syscall_cli.py +531 -0
- outerloop/tick.py +3166 -0
- outerloop/verifier.py +403 -0
- outerloop/verify_agent.py +149 -0
- outerloop/verify_agent_cli.py +95 -0
- outerloop/verify_post_cli.py +116 -0
- outerloop_science-0.1.0.dev0.dist-info/METADATA +145 -0
- outerloop_science-0.1.0.dev0.dist-info/RECORD +52 -0
- outerloop_science-0.1.0.dev0.dist-info/WHEEL +4 -0
- outerloop_science-0.1.0.dev0.dist-info/entry_points.txt +2 -0
- outerloop_science-0.1.0.dev0.dist-info/licenses/LICENSE +202 -0
- outerloop_science-0.1.0.dev0.dist-info/licenses/NOTICE +5 -0
outerloop/climbboard.py
ADDED
|
@@ -0,0 +1,1145 @@
|
|
|
1
|
+
"""The climb board: what every attempt on a benchmark tried and what came
|
|
2
|
+
of it, published to the target's `research-log` branch as data plus two
|
|
3
|
+
views whenever runs end.
|
|
4
|
+
|
|
5
|
+
- `climb/data/<benchmark>.json` — one row per terminal attempt, dedup by run
|
|
6
|
+
id (its own directory: benchmark names are contract-controlled, and a
|
|
7
|
+
benchmark named `index` must not collide with `climb/index.json`);
|
|
8
|
+
the contract everything else derives from (and what an external dashboard
|
|
9
|
+
reads after the public flip).
|
|
10
|
+
- `CLIMB.md` — the numbers and the attempts table, rendered by GitHub.
|
|
11
|
+
- `index.html` — a self-contained page that charts the JSON next to it
|
|
12
|
+
(open it from a clone; a Pages site can serve it as-is later).
|
|
13
|
+
|
|
14
|
+
Publishing is idempotent by construction: rows merge by run id and files
|
|
15
|
+
are written only when their content changes, so the tick can call this
|
|
16
|
+
every pass without commit spam. Like the research-log ledger, the board is
|
|
17
|
+
advisory — a failure never stops the tick.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import contextlib
|
|
23
|
+
import json
|
|
24
|
+
import logging
|
|
25
|
+
import math
|
|
26
|
+
import re
|
|
27
|
+
from dataclasses import asdict, dataclass
|
|
28
|
+
from pathlib import Path
|
|
29
|
+
from typing import Any
|
|
30
|
+
|
|
31
|
+
from outerloop.markers import marker
|
|
32
|
+
from outerloop.runstate import ENDED, list_runs, run_dir
|
|
33
|
+
|
|
34
|
+
log = logging.getLogger("outerloop.climbboard")
|
|
35
|
+
|
|
36
|
+
BOARD_BRANCH = "research-log"
|
|
37
|
+
MAX_HYPOTHESIS_CHARS = 160
|
|
38
|
+
MAX_SUMMARY_CHARS = 90 # what the table shows; the full line stays in the row
|
|
39
|
+
MAX_CURVE_POINTS = 160
|
|
40
|
+
MAX_CURVE_RUNS_PER_AGENT = 5 # at most this many curves per agent, so one
|
|
41
|
+
# busy agent cannot crowd the panel AND every active agent stays represented
|
|
42
|
+
# (this per-agent cap is the whole bound — no global total ceiling, which
|
|
43
|
+
# would drop the oldest agent once agents * 5 exceeded it)
|
|
44
|
+
MAX_ROWS_PER_BENCHMARK = 2000
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
@dataclass(frozen=True)
|
|
48
|
+
class ClimbRow:
|
|
49
|
+
run_id: str
|
|
50
|
+
agent: str
|
|
51
|
+
ended: str # ISO date
|
|
52
|
+
outcome: str
|
|
53
|
+
baseline: float | None
|
|
54
|
+
candidate: float | None
|
|
55
|
+
gpu_hours: float
|
|
56
|
+
hypothesis: str
|
|
57
|
+
pr_url: str
|
|
58
|
+
report: str = "" # reports/<file>.md when the ledger has archived it
|
|
59
|
+
note: str = "" # the gate's own verdict sentence, when it recorded one
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
_NUM = re.compile(r"^(Baseline|Candidate): ([-+0-9.e]+)", re.M)
|
|
63
|
+
_HYP = re.compile(r"Hypothesis[:*\s]+(.+)", re.I)
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def _report_fields(text: str) -> tuple[float | None, float | None, str]:
|
|
67
|
+
"""(baseline, candidate, hypothesis one-liner) out of a run report."""
|
|
68
|
+
baseline = candidate = None
|
|
69
|
+
for key, raw in _NUM.findall(text):
|
|
70
|
+
try:
|
|
71
|
+
value = float(raw)
|
|
72
|
+
except ValueError:
|
|
73
|
+
continue
|
|
74
|
+
if key == "Baseline":
|
|
75
|
+
baseline = value
|
|
76
|
+
else:
|
|
77
|
+
candidate = value
|
|
78
|
+
hyp = ""
|
|
79
|
+
m = _HYP.search(text)
|
|
80
|
+
if m:
|
|
81
|
+
hyp = re.sub(r"[`*_]|\s+", lambda g: " " if g.group().isspace() else "", m.group(1))
|
|
82
|
+
hyp = hyp.strip().rstrip("-").strip()[:MAX_HYPOTHESIS_CHARS]
|
|
83
|
+
return baseline, candidate, hyp
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def summarize(text: str, cap: int = MAX_SUMMARY_CHARS) -> str:
|
|
87
|
+
"""The first sentence, capped — a table cell, not a paragraph."""
|
|
88
|
+
text = text.strip()
|
|
89
|
+
for stop in (". ", "; "):
|
|
90
|
+
i = text.find(stop)
|
|
91
|
+
if 0 < i < cap:
|
|
92
|
+
return text[: i + 1]
|
|
93
|
+
return text if len(text) <= cap else text[: cap - 1].rsplit(" ", 1)[0] + "…"
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
_CURVE_LINE = re.compile(
|
|
97
|
+
# digit bounds in the pattern: int() never sees more digits than fit a
|
|
98
|
+
# JS-safe integer, float() never sees a 400-nines mantissa (a longer
|
|
99
|
+
# number simply fails the match and the line sits out)
|
|
100
|
+
r"^step (\d{1,15}) val loss (\d{1,10}(?:\.\d{1,12})?(?:[eE][+-]?\d{1,3})?)(?=\s|$)",
|
|
101
|
+
re.M,
|
|
102
|
+
)
|
|
103
|
+
# a verbose eval must not exhaust the tick: stdout is scanned line by
|
|
104
|
+
# line and abandoned past this many bytes (curves are diagnostics)
|
|
105
|
+
MAX_CURVE_STDOUT_BYTES = 32 * 1024 * 1024
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def _parse_curve_stdout(stdout_path: Path) -> list[list[float]]:
|
|
109
|
+
"""(step, val loss) points parsed from an eval's stdout, downsampled.
|
|
110
|
+
steps.jsonl dies with the job's scratch; stdout is what survives."""
|
|
111
|
+
try:
|
|
112
|
+
with stdout_path.open("rb") as fh:
|
|
113
|
+
# a byte-mode bounded read caps memory whatever the content — a
|
|
114
|
+
# text-mode read counts characters and 4-byte UTF-8 overshoots 4x
|
|
115
|
+
raw = fh.read(MAX_CURVE_STDOUT_BYTES + 1)
|
|
116
|
+
except OSError:
|
|
117
|
+
return []
|
|
118
|
+
if len(raw) > MAX_CURVE_STDOUT_BYTES:
|
|
119
|
+
# truncated: drop the partial tail line so the EOF-tolerant pattern
|
|
120
|
+
# can never publish a number the cap cut in half
|
|
121
|
+
raw = raw[:MAX_CURVE_STDOUT_BYTES].rsplit(b"\n", 1)[0]
|
|
122
|
+
text = raw.decode("utf-8", errors="replace")
|
|
123
|
+
points = []
|
|
124
|
+
for m in _CURVE_LINE.finditer(text):
|
|
125
|
+
val = float(m.group(2))
|
|
126
|
+
if not math.isfinite(val): # e-notation can still overflow (1e999)
|
|
127
|
+
continue
|
|
128
|
+
points.append([int(m.group(1)), val])
|
|
129
|
+
if len(points) > MAX_CURVE_POINTS:
|
|
130
|
+
stride = len(points) / (MAX_CURVE_POINTS - 1)
|
|
131
|
+
points = [points[int(i * stride)] for i in range(MAX_CURVE_POINTS - 1)] + [points[-1]]
|
|
132
|
+
return points
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def _curve_from_eval(run_directory: Path) -> list[list[float]]:
|
|
136
|
+
"""The training curve behind a run's row: the newest CANDIDATE eval when
|
|
137
|
+
the run submitted, else the run's best LAUNCH experiment (lowest final
|
|
138
|
+
val loss). Without the launch fallback a run that tested experiments
|
|
139
|
+
but did not submit would show no curve, and its agent would vanish from
|
|
140
|
+
the panel."""
|
|
141
|
+
|
|
142
|
+
def mtime(d: Path) -> float:
|
|
143
|
+
try:
|
|
144
|
+
return d.stat().st_mtime
|
|
145
|
+
except OSError:
|
|
146
|
+
return 0.0 # vanished between glob and sort: sorts oldest, still readable-guarded
|
|
147
|
+
|
|
148
|
+
candidates = sorted(
|
|
149
|
+
(d for d in run_directory.glob("eval-candidate-*") if (d / "stdout").is_file()),
|
|
150
|
+
key=mtime,
|
|
151
|
+
)
|
|
152
|
+
if candidates:
|
|
153
|
+
points = _parse_curve_stdout(candidates[-1] / "stdout")
|
|
154
|
+
if points:
|
|
155
|
+
return points
|
|
156
|
+
# a candidate whose stdout has no parsable points falls through to
|
|
157
|
+
# the launch fallback rather than leaving the run curveless
|
|
158
|
+
# launch fallback: the experiment the agent did best on (lowest final val
|
|
159
|
+
# loss — a display heuristic for the min-oriented speedrun curve, not a
|
|
160
|
+
# credited measurement)
|
|
161
|
+
best: list[list[float]] = []
|
|
162
|
+
best_final: float | None = None
|
|
163
|
+
for d in run_directory.glob("eval-launch-*"):
|
|
164
|
+
if not (d / "stdout").is_file():
|
|
165
|
+
continue
|
|
166
|
+
points = _parse_curve_stdout(d / "stdout")
|
|
167
|
+
if points and (best_final is None or points[-1][1] < best_final):
|
|
168
|
+
best_final, best = points[-1][1], points
|
|
169
|
+
return best
|
|
170
|
+
|
|
171
|
+
|
|
172
|
+
def collect_rows(root: Path, target: str) -> dict[str, list[ClimbRow]]:
|
|
173
|
+
"""Terminal attempts of `target` with a report, grouped by benchmark."""
|
|
174
|
+
from datetime import UTC, datetime
|
|
175
|
+
|
|
176
|
+
out: dict[str, list[ClimbRow]] = {}
|
|
177
|
+
for record in list_runs(root):
|
|
178
|
+
# only ENDED runs: an in-review run's outcome is not known yet (its
|
|
179
|
+
# PR may be rejected), and a published row is never rewritten
|
|
180
|
+
if record.target != target or record.state != ENDED:
|
|
181
|
+
continue
|
|
182
|
+
try:
|
|
183
|
+
report = (run_dir(root, record.run_id) / "report.md").read_text()
|
|
184
|
+
except OSError:
|
|
185
|
+
continue
|
|
186
|
+
baseline, candidate, hyp = _report_fields(report)
|
|
187
|
+
stage = record.stage or {}
|
|
188
|
+
ended = datetime.fromtimestamp(record.updated or record.created, tz=UTC)
|
|
189
|
+
outcome = record.ending or "ended"
|
|
190
|
+
# link the report only when the ledger's own marker says it is on the
|
|
191
|
+
# branch (adopted-unpublished history and not-yet-archived runs would
|
|
192
|
+
# otherwise render dead links)
|
|
193
|
+
report = ""
|
|
194
|
+
try:
|
|
195
|
+
marker = (run_dir(root, record.run_id) / "ledger-published").read_text()
|
|
196
|
+
lines = marker.splitlines()
|
|
197
|
+
if lines and lines[0].startswith(("archived", "pointer-pending", "done")):
|
|
198
|
+
if len(lines) > 1 and lines[1].startswith("reports/") and lines[1].endswith(".md"):
|
|
199
|
+
# the ledger's own path: an in-review archive keeps its
|
|
200
|
+
# date even after the ENDED transition re-stamps updated
|
|
201
|
+
report = lines[1]
|
|
202
|
+
else: # legacy marker without a path line
|
|
203
|
+
report = f"reports/{ended.strftime('%Y-%m-%d')}-{record.run_id}.md"
|
|
204
|
+
except OSError:
|
|
205
|
+
pass
|
|
206
|
+
out.setdefault(record.benchmark or "benchmark", []).append(
|
|
207
|
+
ClimbRow(
|
|
208
|
+
run_id=record.run_id,
|
|
209
|
+
agent=record.agent_id,
|
|
210
|
+
ended=ended.strftime("%Y-%m-%d %H:%M:%S"),
|
|
211
|
+
outcome=outcome,
|
|
212
|
+
baseline=baseline,
|
|
213
|
+
candidate=candidate,
|
|
214
|
+
gpu_hours=round(float(stage.get("gpu_hours_used") or 0.0), 2), # type: ignore[arg-type]
|
|
215
|
+
hypothesis=hyp,
|
|
216
|
+
pr_url=record.pr_url,
|
|
217
|
+
report=report,
|
|
218
|
+
note=summarize(record.ending_note or "", 120),
|
|
219
|
+
)
|
|
220
|
+
)
|
|
221
|
+
return out
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def merge_rows(existing_json: str | None, fresh: list[ClimbRow]) -> list[dict[str, Any]]:
|
|
225
|
+
"""Existing board rows plus any new ones, one per run id, oldest first.
|
|
226
|
+
A run already on the board keeps its published row (reports are final at
|
|
227
|
+
terminal state; the board never rewrites history)."""
|
|
228
|
+
rows: list[dict[str, Any]] = []
|
|
229
|
+
seen: set[str] = set()
|
|
230
|
+
if existing_json:
|
|
231
|
+
try:
|
|
232
|
+
for item in json.loads(existing_json):
|
|
233
|
+
if isinstance(item, dict) and item.get("run_id") not in seen:
|
|
234
|
+
rows.append(item)
|
|
235
|
+
seen.add(str(item.get("run_id")))
|
|
236
|
+
except ValueError:
|
|
237
|
+
log.warning("unreadable board JSON; rebuilding from local records")
|
|
238
|
+
for row in fresh:
|
|
239
|
+
if row.run_id not in seen:
|
|
240
|
+
rows.append(asdict(row))
|
|
241
|
+
seen.add(row.run_id)
|
|
242
|
+
rows.sort(key=lambda r: str(r.get("ended", "")))
|
|
243
|
+
# the board is a bounded VIEW (the contents API caps file sizes); the
|
|
244
|
+
# full history stays in reports/ on this same branch, and the trim is
|
|
245
|
+
# said out loud in CLIMB.md, never silent
|
|
246
|
+
return rows[-MAX_ROWS_PER_BENCHMARK:]
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
def _fmt(value: Any) -> str:
|
|
250
|
+
if value is None or value == "":
|
|
251
|
+
return "—"
|
|
252
|
+
if isinstance(value, float) and value == int(value):
|
|
253
|
+
return str(int(value))
|
|
254
|
+
return str(value)
|
|
255
|
+
|
|
256
|
+
|
|
257
|
+
def render_md(
|
|
258
|
+
target: str,
|
|
259
|
+
boards: dict[str, list[dict[str, Any]]],
|
|
260
|
+
directions: dict[str, str],
|
|
261
|
+
starts: dict[str, float] | None = None,
|
|
262
|
+
) -> str:
|
|
263
|
+
"""CLIMB.md: per benchmark, the headline numbers and the attempts table
|
|
264
|
+
(newest first). Plain markdown; the chart lives in index.html."""
|
|
265
|
+
lines = [
|
|
266
|
+
marker("climb-board"),
|
|
267
|
+
f"# Climb — {target}",
|
|
268
|
+
"",
|
|
269
|
+
"Written by the kernel when runs end. Data: `climb/data/<benchmark>.json`;",
|
|
270
|
+
"chart: open `index.html` from a clone of this branch.",
|
|
271
|
+
]
|
|
272
|
+
for benchmark in sorted(boards):
|
|
273
|
+
rows = boards[benchmark]
|
|
274
|
+
direction = directions.get(benchmark, "min")
|
|
275
|
+
pick = max if direction == "max" else min
|
|
276
|
+
measured = [r for r in rows if isinstance(r.get("candidate"), int | float)]
|
|
277
|
+
improved = [r for r in rows if r.get("outcome") in ("merged", "improved")]
|
|
278
|
+
best = pick((r["candidate"] for r in measured), default=None)
|
|
279
|
+
gpu = sum(float(r.get("gpu_hours") or 0.0) for r in rows)
|
|
280
|
+
# baseline = the campaign's STARTING POSITION, one fixed ledger
|
|
281
|
+
# number (owner decision: per-run declared bases confused more than
|
|
282
|
+
# they informed as a headline)
|
|
283
|
+
start = (starts or {}).get(benchmark)
|
|
284
|
+
start_chip = f" · baseline (start): **{_fmt(start)}**" if start is not None else ""
|
|
285
|
+
lines += [
|
|
286
|
+
"",
|
|
287
|
+
f"## {benchmark}",
|
|
288
|
+
"",
|
|
289
|
+
f"Attempts: **{len(rows)}** ({len(improved)} improved) · best candidate: "
|
|
290
|
+
f"**{_fmt(best)}** ({direction}){start_chip} · GPU-hours: **{gpu:.1f}**",
|
|
291
|
+
]
|
|
292
|
+
if len(rows) >= MAX_ROWS_PER_BENCHMARK:
|
|
293
|
+
lines += [
|
|
294
|
+
"",
|
|
295
|
+
f"Only the newest {MAX_ROWS_PER_BENCHMARK} attempts are on the board; "
|
|
296
|
+
"archived reports stay in `reports/` on this branch.",
|
|
297
|
+
]
|
|
298
|
+
lines += [
|
|
299
|
+
"",
|
|
300
|
+
"| ended (UTC) | agent | hypothesis | outcome | candidate | GPU-h | full |",
|
|
301
|
+
"| --- | --- | --- | --- | --- | --- | --- |",
|
|
302
|
+
]
|
|
303
|
+
for r in reversed(rows):
|
|
304
|
+
outcome = str(r.get("outcome", ""))
|
|
305
|
+
if r.get("pr_url"):
|
|
306
|
+
outcome = f"[{outcome}]({r['pr_url']})"
|
|
307
|
+
if r.get("note"):
|
|
308
|
+
# the gate's reason makes a near miss legible ("real
|
|
309
|
+
# movement, not creditable" reads differently from a DNF)
|
|
310
|
+
# flattened: a newline inside a cell would split the table row
|
|
311
|
+
flat = " ".join(str(r["note"]).split())
|
|
312
|
+
outcome += " — " + summarize(flat, 80).replace("|", "\\|")
|
|
313
|
+
hyp = summarize(str(r.get("hypothesis") or "")).replace("|", "\\|")
|
|
314
|
+
ended = str(r.get("ended", ""))
|
|
315
|
+
report = f"[report]({r['report']})" if r.get("report") else ""
|
|
316
|
+
lines.append(
|
|
317
|
+
f"| {ended} | {r.get('agent', '')} | {hyp} | {outcome} "
|
|
318
|
+
f"| {_fmt(r.get('candidate'))} | {_fmt(r.get('gpu_hours'))} | {report} |"
|
|
319
|
+
)
|
|
320
|
+
return "\n".join(lines) + "\n"
|
|
321
|
+
|
|
322
|
+
|
|
323
|
+
def render_html(
|
|
324
|
+
target: str,
|
|
325
|
+
boards: dict[str, list[dict[str, Any]]],
|
|
326
|
+
directions: dict[str, str],
|
|
327
|
+
curves: dict[str, dict[str, list[list[float]]]] | None = None,
|
|
328
|
+
) -> str:
|
|
329
|
+
"""One self-contained page: the data is EMBEDDED (a browser blocks
|
|
330
|
+
fetch() from a file:// page, and the direct-from-clone view must work),
|
|
331
|
+
only chart.js arrives from its CDN — nothing bulky is committed to the
|
|
332
|
+
target's branch. Light and dark follow the viewer's system theme."""
|
|
333
|
+
# "<" is escaped INSIDE the JSON (still valid JSON): a hypothesis line is
|
|
334
|
+
# agent-written text, and a literal </script> in it would close the inline
|
|
335
|
+
# script and run whatever follows in the published page
|
|
336
|
+
payload = json.dumps(
|
|
337
|
+
{"boards": boards, "directions": directions, "curves": curves or {}}
|
|
338
|
+
).replace("<", "\\u003c")
|
|
339
|
+
return (
|
|
340
|
+
"<!doctype html>\n<html><head><meta charset='utf-8'>\n"
|
|
341
|
+
"<meta name='viewport' content='width=device-width, initial-scale=1'>\n"
|
|
342
|
+
f"<title>Climb — {target}</title>\n"
|
|
343
|
+
"<script src='https://cdn.jsdelivr.net/npm/chart.js@4'></script>\n"
|
|
344
|
+
"<style>\n"
|
|
345
|
+
":root{--bg:#f6f7f4;--card:#ffffff;--ink:#1c2025;--muted:#5d6570;\n"
|
|
346
|
+
"--line:#e1e3de;--accent:#2d54ae;--win:#157a4b;--lose:#a7acb0;\n"
|
|
347
|
+
"--base:#b8433a;--near:#ac7714}\n"
|
|
348
|
+
"@media (prefers-color-scheme: dark){:root:not([data-theme=light]){\n"
|
|
349
|
+
"--bg:#121418;--card:#1a1d23;--ink:#e6e8eb;--muted:#99a1ad;\n"
|
|
350
|
+
"--line:#2a2e36;--accent:#7398e6;--win:#3cbd83;--lose:#656c76;\n"
|
|
351
|
+
"--base:#de7263;--near:#cf9c33}}\n"
|
|
352
|
+
":root[data-theme=dark]{--bg:#121418;--card:#1a1d23;--ink:#e6e8eb;\n"
|
|
353
|
+
"--muted:#99a1ad;--line:#2a2e36;--accent:#7398e6;--win:#3cbd83;\n"
|
|
354
|
+
"--lose:#656c76;--base:#de7263;--near:#cf9c33}\n"
|
|
355
|
+
"a{color:var(--accent)}\n"
|
|
356
|
+
"header{display:flex;align-items:center;justify-content:space-between}\n"
|
|
357
|
+
"#theme{background:var(--card);border:1px solid var(--line);\n"
|
|
358
|
+
"color:var(--muted);border-radius:.4rem;padding:.2rem .6rem;\n"
|
|
359
|
+
"font-size:.75rem;cursor:pointer}\n"
|
|
360
|
+
"body{font-family:system-ui,-apple-system,sans-serif;background:var(--bg);\n"
|
|
361
|
+
"color:var(--ink);margin:0;padding:2.5rem 1.5rem;line-height:1.5}\n"
|
|
362
|
+
"main{max-width:920px;margin:0 auto}\n"
|
|
363
|
+
"header p{color:var(--muted);margin:.25rem 0 0;font-size:.9rem}\n"
|
|
364
|
+
"h1{font-size:1.5rem;margin:0}\n"
|
|
365
|
+
"h1 span{color:var(--muted);font-weight:400}\n"
|
|
366
|
+
"h2{font-size:1.1rem;margin:2.5rem 0 .75rem}\n"
|
|
367
|
+
".chips{display:flex;flex-wrap:wrap;gap:.5rem;margin-bottom:1rem}\n"
|
|
368
|
+
".chip{background:var(--card);border:1px solid var(--line);border-radius:.4rem;\n"
|
|
369
|
+
"padding:.35rem .7rem;font-size:.85rem;color:var(--muted)}\n"
|
|
370
|
+
".chip b{color:var(--ink);font-variant-numeric:tabular-nums;font-weight:600}\n"
|
|
371
|
+
".run{background:var(--card);border:1px solid var(--line);border-radius:.4rem;\n"
|
|
372
|
+
"padding:.45rem .7rem;font-size:.85rem;color:var(--muted);\n"
|
|
373
|
+
"width:16.5rem;box-sizing:border-box}\n"
|
|
374
|
+
".run .top{display:flex;align-items:center;gap:.5rem}\n"
|
|
375
|
+
".run b{color:var(--ink)}\n"
|
|
376
|
+
".run .dir{margin-top:.15rem;font-size:.8rem}\n"
|
|
377
|
+
".run .lm{display:grid;grid-template-columns:auto 1fr auto;\n"
|
|
378
|
+
" gap:.18rem .4rem;align-items:center;margin-top:.3rem;\n"
|
|
379
|
+
" font-size:.65rem;color:var(--muted)}\n"
|
|
380
|
+
".run .bar{position:relative;height:3px;border-radius:2px;\n"
|
|
381
|
+
" background:var(--line);overflow:hidden}\n"
|
|
382
|
+
".run .bar span{position:absolute;left:0;top:0;height:100%}\n"
|
|
383
|
+
".run .meter{display:flex;gap:2px}\n"
|
|
384
|
+
".run .meter i{flex:1;height:3px;border-radius:1px;background:var(--line)}\n"
|
|
385
|
+
".clegend{display:flex;flex-wrap:wrap;gap:.4rem 1.4rem;\n"
|
|
386
|
+
"margin:.5rem 0 .25rem;font-size:.75rem;color:var(--muted)}\n"
|
|
387
|
+
".clegend .lgroup{display:flex;align-items:center;gap:.55rem}\n"
|
|
388
|
+
".clegend b{cursor:pointer;font-weight:600}\n"
|
|
389
|
+
".clegend .litem{display:flex;align-items:center;gap:.3rem;cursor:pointer}\n"
|
|
390
|
+
".clegend .litem i{width:14px;height:8px;border:2px solid;\n"
|
|
391
|
+
"border-radius:2px;display:inline-block;box-sizing:border-box}\n"
|
|
392
|
+
".pill{border-radius:.6rem;padding:.05rem .55rem;font-size:.72rem;\n"
|
|
393
|
+
"font-weight:600;color:#fff}\n"
|
|
394
|
+
".card{background:var(--card);border:1px solid var(--line);border-radius:.5rem;\n"
|
|
395
|
+
"padding:1rem}\n"
|
|
396
|
+
".card label{font-size:.8rem;color:var(--muted);margin-right:1rem}\n"
|
|
397
|
+
"</style></head><body><main>\n"
|
|
398
|
+
f"<header><h1>{target} <span>· climb</span></h1>\n"
|
|
399
|
+
"<button id='theme' title='theme: auto / light / dark'>auto</button>\n"
|
|
400
|
+
"</header>\n"
|
|
401
|
+
"<div id='now' class='chips'></div>\n"
|
|
402
|
+
"<div id='charts'></div>\n<script>\n"
|
|
403
|
+
f"const data = {payload};\n"
|
|
404
|
+
"const css = n => getComputedStyle(document.body).getPropertyValue(n);\n"
|
|
405
|
+
"// theme: auto follows the OS; the button cycles auto/light/dark and\n"
|
|
406
|
+
"// the charts redraw so their computed colors follow\n"
|
|
407
|
+
"const themeBtn = document.getElementById('theme');\n"
|
|
408
|
+
"let onTheme = () => {};\n"
|
|
409
|
+
"const applyTheme = t => {\n"
|
|
410
|
+
" if (t === 'light' || t === 'dark')\n"
|
|
411
|
+
" document.documentElement.dataset.theme = t;\n"
|
|
412
|
+
" else delete document.documentElement.dataset.theme;\n"
|
|
413
|
+
" themeBtn.textContent = t;\n"
|
|
414
|
+
"};\n"
|
|
415
|
+
"let theme = 'auto';\n"
|
|
416
|
+
"try { theme = localStorage.getItem('climb-theme') || 'auto'; }\n"
|
|
417
|
+
"catch (e) {}\n"
|
|
418
|
+
"applyTheme(theme);\n"
|
|
419
|
+
"themeBtn.onclick = () => {\n"
|
|
420
|
+
" theme = theme === 'auto' ? 'light' : theme === 'light' ? 'dark' : 'auto';\n"
|
|
421
|
+
" try { localStorage.setItem('climb-theme', theme); } catch (e) {}\n"
|
|
422
|
+
" applyTheme(theme);\n"
|
|
423
|
+
" redraws.forEach(f => f()); onTheme();\n"
|
|
424
|
+
"};\n"
|
|
425
|
+
"// some hosts stamp data-theme on the root after load; without\n"
|
|
426
|
+
"// this the charts keep first-load colors on the flipped ground\n"
|
|
427
|
+
"new MutationObserver(() => { redraws.forEach(f => f()); onTheme(); })\n"
|
|
428
|
+
" .observe(document.documentElement,\n"
|
|
429
|
+
" {attributes: true, attributeFilter: ['data-theme']});\n"
|
|
430
|
+
"const mq = matchMedia('(prefers-color-scheme: dark)');\n"
|
|
431
|
+
"const onMq = () => { redraws.forEach(f => f()); onTheme(); };\n"
|
|
432
|
+
"// older iOS Safari has addListener only\n"
|
|
433
|
+
"if (mq.addEventListener) mq.addEventListener('change', onMq);\n"
|
|
434
|
+
"else if (mq.addListener) mq.addListener(onMq);\n"
|
|
435
|
+
"// legend: solid box = shown, hollow box = hidden (no strikethrough)\n"
|
|
436
|
+
"const boxLegend = {labels: {generateLabels: (chart) =>\n"
|
|
437
|
+
" chart.data.datasets.map((ds, i) => {\n"
|
|
438
|
+
" const shown = chart.isDatasetVisible(i);\n"
|
|
439
|
+
" const color = ds.borderColor || ds.pointBackgroundColor;\n"
|
|
440
|
+
" return {text: ds.label, datasetIndex: i, hidden: false,\n"
|
|
441
|
+
" fillStyle: shown ? color : 'rgba(0,0,0,0)',\n"
|
|
442
|
+
" strokeStyle: color, lineWidth: 2,\n"
|
|
443
|
+
" fontColor: css('--muted')};\n"
|
|
444
|
+
" })}};\n"
|
|
445
|
+
"const axis = () => ({ticks: {color: css('--muted')}, grid: {color: css('--line')}});\n"
|
|
446
|
+
"// tooltip copy stays a glance: first clause, strip-sized\n"
|
|
447
|
+
"const phrase = s => {\n"
|
|
448
|
+
" if (!s) return '';\n"
|
|
449
|
+
" for (const sep of ['. ', '; ']) {\n"
|
|
450
|
+
" const i = s.indexOf(sep);\n"
|
|
451
|
+
" if (i > 0 && i < 200) { s = s.slice(0, i + 1); break; }\n"
|
|
452
|
+
" }\n"
|
|
453
|
+
" for (const sep of [': ', ' \u2014 ', ' -- ']) {\n"
|
|
454
|
+
" const i = s.indexOf(sep);\n"
|
|
455
|
+
" if (i > 0) s = s.slice(0, i);\n"
|
|
456
|
+
" }\n"
|
|
457
|
+
" if (s.length > 64) s = s.slice(0, 63).trimEnd() + '\u2026';\n"
|
|
458
|
+
" return s;\n"
|
|
459
|
+
"};\n"
|
|
460
|
+
"const hues = [212, 152, 22, 282, 342, 62, 122, 242];\n"
|
|
461
|
+
"// identity color from the agent id itself, so the tooltip, the\n"
|
|
462
|
+
"// curves chart, and the strip agree even across benchmarks\n"
|
|
463
|
+
"const agentHue = a => {\n"
|
|
464
|
+
" const m = /(\\d+)$/.exec(a || '');\n"
|
|
465
|
+
" const i = m ? parseInt(m[1], 10) - 1\n"
|
|
466
|
+
" : [...String(a || '')].reduce((h, c) => h + c.charCodeAt(0), 0);\n"
|
|
467
|
+
" return hues[((i % 8) + 8) % 8];\n"
|
|
468
|
+
"};\n"
|
|
469
|
+
"const isDark = () => document.documentElement.dataset.theme === 'dark'\n"
|
|
470
|
+
" || (document.documentElement.dataset.theme !== 'light'\n"
|
|
471
|
+
" && matchMedia('(prefers-color-scheme: dark)').matches);\n"
|
|
472
|
+
"// t in [0,1]: 1 = the agent's identity color (its newest curve);\n"
|
|
473
|
+
"// smaller t fades toward the background, so earlier runs recede\n"
|
|
474
|
+
"// (lighter on paper, dimmer in the dark theme)\n"
|
|
475
|
+
"const agentShade = (a, t) => isDark()\n"
|
|
476
|
+
" ? `hsl(${agentHue(a)} ${40 + 25 * t}% ${30 + 22 * t}%)`\n"
|
|
477
|
+
" : `hsl(${agentHue(a)} ${45 + 15 * t}% ${72 - 24 * t}%)`;\n"
|
|
478
|
+
"const agentColor = a => agentShade(a, 1);\n"
|
|
479
|
+
"const redraws = [];\n"
|
|
480
|
+
"for (const b of Object.keys(data.boards).sort()) {\n"
|
|
481
|
+
" const rows = data.boards[b];\n"
|
|
482
|
+
" const dir = data.directions[b] === 'max' ? 'max' : 'min';\n"
|
|
483
|
+
" const pick = dir === 'max' ? Math.max : Math.min;\n"
|
|
484
|
+
" const won = new Set(['merged', 'improved']);\n"
|
|
485
|
+
" const beats = r => typeof r.baseline === 'number' && (dir === 'max'\n"
|
|
486
|
+
" ? r.candidate > r.baseline : r.candidate < r.baseline);\n"
|
|
487
|
+
" const measured = rows.filter(r => typeof r.candidate === 'number');\n"
|
|
488
|
+
" const baselines = measured.map(r => r.baseline).filter(v => typeof v === 'number');\n"
|
|
489
|
+
" // off-scale attempts (a DNF scored as the whole budget) squash the\n"
|
|
490
|
+
" // axis: hidden unless asked for. Off-scale = worse than 1.5x the\n"
|
|
491
|
+
" // worst baseline, whichever way this benchmark points.\n"
|
|
492
|
+
" const worstBase = baselines.length ? (dir === 'max'\n"
|
|
493
|
+
" ? Math.min(...baselines) : Math.max(...baselines)) : null;\n"
|
|
494
|
+
" const offScale = v => worstBase !== null && (dir === 'max'\n"
|
|
495
|
+
" ? v < worstBase / 1.5 : v > worstBase * 1.5);\n"
|
|
496
|
+
" let best;\n"
|
|
497
|
+
" const bestSoFar = measured.map(r =>\n"
|
|
498
|
+
" best = best === undefined ? r.candidate : pick(best, r.candidate));\n"
|
|
499
|
+
" const wins = rows.filter(r => won.has(r.outcome)).length;\n"
|
|
500
|
+
" const gpu = rows.reduce((a, r) => a + (r.gpu_hours || 0), 0);\n"
|
|
501
|
+
" const el = document.getElementById('charts');\n"
|
|
502
|
+
" const h = document.createElement('h2'); h.textContent = b; el.append(h);\n"
|
|
503
|
+
" const chips = document.createElement('div'); chips.className = 'chips';\n"
|
|
504
|
+
" const chip = (label, value) => {\n"
|
|
505
|
+
" const c = document.createElement('span'); c.className = 'chip';\n"
|
|
506
|
+
" const strong = document.createElement('b'); strong.textContent = value;\n"
|
|
507
|
+
" c.append(label + ' ', strong); chips.append(c); };\n"
|
|
508
|
+
" chip('attempts', rows.length); chip('improved', wins);\n"
|
|
509
|
+
" chip('best (' + dir + ')', measured.length ? bestSoFar[bestSoFar.length - 1] : '—');\n"
|
|
510
|
+
" chip('GPU-hours (recorded)', gpu.toFixed(1)); el.append(chips);\n"
|
|
511
|
+
" const card = document.createElement('div'); card.className = 'card';\n"
|
|
512
|
+
" const c = document.createElement('canvas'); card.append(c); el.append(card);\n"
|
|
513
|
+
" const logBox = document.createElement('label');\n"
|
|
514
|
+
" const cb = document.createElement('input'); cb.type = 'checkbox';\n"
|
|
515
|
+
" logBox.append(cb, ' log scale');\n"
|
|
516
|
+
" const outBox = document.createElement('label');\n"
|
|
517
|
+
" const ob = document.createElement('input'); ob.type = 'checkbox';\n"
|
|
518
|
+
" outBox.append(ob, ' show off-scale attempts');\n"
|
|
519
|
+
" card.append(logBox, outBox);\n"
|
|
520
|
+
" let chart;\n"
|
|
521
|
+
" const draw = () => {\n"
|
|
522
|
+
" const view = measured.filter(r => ob.checked || !offScale(r.candidate));\n"
|
|
523
|
+
" let vb;\n"
|
|
524
|
+
" const viewBest = view.map(r => vb = vb === undefined ? r.candidate\n"
|
|
525
|
+
" : pick(vb, r.candidate));\n"
|
|
526
|
+
" if (chart) chart.destroy();\n"
|
|
527
|
+
" chart = new Chart(c, {type: 'line', data: {labels: view.map(r => r.ended),\n"
|
|
528
|
+
" datasets: [\n"
|
|
529
|
+
" {label: 'candidate', data: view.map(r => r.candidate),\n"
|
|
530
|
+
" showLine: false, pointRadius: 4,\n"
|
|
531
|
+
" borderColor: css('--lose'),\n"
|
|
532
|
+
" pointBackgroundColor: view.map(r =>\n"
|
|
533
|
+
" won.has(r.outcome) ? css('--win')\n"
|
|
534
|
+
" : beats(r) ? css('--near') : css('--lose'))},\n"
|
|
535
|
+
" {label: 'best so far', data: viewBest, stepped: true,\n"
|
|
536
|
+
" borderColor: css('--accent'), borderWidth: 2, pointRadius: 0},\n"
|
|
537
|
+
" // markers, never a connected line: run base is per-run\n"
|
|
538
|
+
" // pairing context — a line reads as a trend, and a stale-base\n"
|
|
539
|
+
" // run landing after a merge can draw an upward step\n"
|
|
540
|
+
" // (borderColor feeds the legend swatch; no line is drawn)\n"
|
|
541
|
+
" {label: 'run base', data: view.map(r => r.baseline),\n"
|
|
542
|
+
" showLine: false, pointStyle: 'crossRot', pointRadius: 4,\n"
|
|
543
|
+
" borderColor: css('--base'),\n"
|
|
544
|
+
" pointBorderColor: css('--base'), pointBorderWidth: 1.5}]},\n"
|
|
545
|
+
" options: {color: css('--muted'),\n"
|
|
546
|
+
" scales: {x: {...axis(), ticks: {color: css('--muted'), maxTicksLimit: 8}},\n"
|
|
547
|
+
" y: {...axis(), type: cb.checked ? 'logarithmic' : 'linear'}},\n"
|
|
548
|
+
" plugins: {legend: boxLegend,\n"
|
|
549
|
+
" tooltip: {\n"
|
|
550
|
+
" // best-so-far and baseline pass exactly through candidate\n"
|
|
551
|
+
" // points; without the filter every coincident item repeats\n"
|
|
552
|
+
" // the same row in the pop-up\n"
|
|
553
|
+
" filter: i => i.datasetIndex === 0,\n"
|
|
554
|
+
" callbacks: {\n"
|
|
555
|
+
" title: its => { const r0 = view[its[0]?.dataIndex];\n"
|
|
556
|
+
" return r0 ? r0.agent + ' ' + (r0.ended || '').slice(0, 16) : ''; },\n"
|
|
557
|
+
" labelColor: i => ({\n"
|
|
558
|
+
" borderColor: agentColor(view[i.dataIndex].agent),\n"
|
|
559
|
+
" backgroundColor: agentColor(view[i.dataIndex].agent)}),\n"
|
|
560
|
+
" afterLabel: (i) =>\n"
|
|
561
|
+
" [phrase(view[i.dataIndex].hypothesis), view[i.dataIndex].note]\n"
|
|
562
|
+
" .filter(Boolean).join('\\n')}}}}});\n"
|
|
563
|
+
" };\n"
|
|
564
|
+
" cb.onchange = draw; ob.onchange = draw; draw(); redraws.push(draw);\n"
|
|
565
|
+
" // the training curves behind the numbers: newest attempts overlaid\n"
|
|
566
|
+
" const bcurves = data.curves[b] || {};\n"
|
|
567
|
+
" // newest curves, capped per agent so every agent stays on the panel\n"
|
|
568
|
+
" // (rows are oldest-first; walk newest-first, keep <=5 per agent)\n"
|
|
569
|
+
" const perAgent = {}; const picked = [];\n"
|
|
570
|
+
" for (let i = rows.length - 1; i >= 0; i--) {\n"
|
|
571
|
+
" const r = rows[i]; if (!bcurves[r.run_id]) continue;\n"
|
|
572
|
+
" perAgent[r.agent] = (perAgent[r.agent] || 0) + 1;\n"
|
|
573
|
+
" if (perAgent[r.agent] <= 5) picked.push(r);\n"
|
|
574
|
+
" }\n"
|
|
575
|
+
" const withCurve = picked.reverse();\n"
|
|
576
|
+
" if (withCurve.length) {\n"
|
|
577
|
+
" const h3 = document.createElement('h2');\n"
|
|
578
|
+
" h3.textContent = b + ' — training curves (newest attempts)';\n"
|
|
579
|
+
" const card2 = document.createElement('div'); card2.className = 'card';\n"
|
|
580
|
+
" const c2 = document.createElement('canvas'); card2.append(c2);\n"
|
|
581
|
+
" const lx = document.createElement('label');\n"
|
|
582
|
+
" const lxb = document.createElement('input'); lxb.type = 'checkbox';\n"
|
|
583
|
+
" lx.append(lxb, ' log x');\n"
|
|
584
|
+
" const ly = document.createElement('label');\n"
|
|
585
|
+
" const lyb = document.createElement('input'); lyb.type = 'checkbox';\n"
|
|
586
|
+
" ly.append(lyb, ' log y');\n"
|
|
587
|
+
" // custom legend: every run stays its own item, grouped under\n"
|
|
588
|
+
" // its agent — the agent name toggles the whole group, a run's\n"
|
|
589
|
+
" // box toggles that line (solid = visible, hollow = hidden)\n"
|
|
590
|
+
" const legendEl = document.createElement('div');\n"
|
|
591
|
+
" legendEl.className = 'clegend';\n"
|
|
592
|
+
" card2.append(legendEl, lx, ly);\n"
|
|
593
|
+
" el.append(h3, card2);\n"
|
|
594
|
+
" let chart2;\n"
|
|
595
|
+
" const buildLegend = () => {\n"
|
|
596
|
+
" legendEl.textContent = '';\n"
|
|
597
|
+
" const groups = [];\n"
|
|
598
|
+
" withCurve.forEach((r, i) => {\n"
|
|
599
|
+
" let g = groups.find(x => x.agent === r.agent);\n"
|
|
600
|
+
" if (!g) { g = {agent: r.agent, items: []}; groups.push(g); }\n"
|
|
601
|
+
" g.items.push(i);\n"
|
|
602
|
+
" });\n"
|
|
603
|
+
" for (const g of groups) {\n"
|
|
604
|
+
" const wrap = document.createElement('span');\n"
|
|
605
|
+
" wrap.className = 'lgroup';\n"
|
|
606
|
+
" const name = document.createElement('b');\n"
|
|
607
|
+
" name.textContent = g.agent;\n"
|
|
608
|
+
" name.style.color = agentColor(g.agent);\n"
|
|
609
|
+
" name.title = 'toggle all ' + g.agent + ' runs';\n"
|
|
610
|
+
" name.onclick = () => {\n"
|
|
611
|
+
" const anyOn = g.items.some(i => chart2.isDatasetVisible(i));\n"
|
|
612
|
+
" g.items.forEach(i => chart2.setDatasetVisibility(i, !anyOn));\n"
|
|
613
|
+
" chart2.update(); buildLegend();\n"
|
|
614
|
+
" };\n"
|
|
615
|
+
" wrap.append(name);\n"
|
|
616
|
+
" for (const i of g.items) {\n"
|
|
617
|
+
" const it = document.createElement('span');\n"
|
|
618
|
+
" it.className = 'litem';\n"
|
|
619
|
+
" const box = document.createElement('i');\n"
|
|
620
|
+
" const color = chart2.data.datasets[i].borderColor;\n"
|
|
621
|
+
" box.style.borderColor = color;\n"
|
|
622
|
+
" if (chart2.isDatasetVisible(i)) box.style.background = color;\n"
|
|
623
|
+
" const lbl = document.createElement('span');\n"
|
|
624
|
+
" lbl.textContent = (withCurve[i].ended || '').slice(5, 16);\n"
|
|
625
|
+
" it.title = 'toggle this run';\n"
|
|
626
|
+
" it.append(box, lbl);\n"
|
|
627
|
+
" it.onclick = () => {\n"
|
|
628
|
+
" chart2.setDatasetVisibility(i, !chart2.isDatasetVisible(i));\n"
|
|
629
|
+
" chart2.update(); buildLegend();\n"
|
|
630
|
+
" };\n"
|
|
631
|
+
" wrap.append(it);\n"
|
|
632
|
+
" }\n"
|
|
633
|
+
" legendEl.append(wrap);\n"
|
|
634
|
+
" }\n"
|
|
635
|
+
" };\n"
|
|
636
|
+
" const draw2 = () => {\n"
|
|
637
|
+
" const hidden = chart2\n"
|
|
638
|
+
" ? withCurve.map((_, i) => !chart2.isDatasetVisible(i)) : [];\n"
|
|
639
|
+
" if (chart2) chart2.destroy();\n"
|
|
640
|
+
" const order = {};\n"
|
|
641
|
+
" withCurve.forEach((r, i) => {\n"
|
|
642
|
+
" (order[r.agent] = order[r.agent] || []).push(i);\n"
|
|
643
|
+
" });\n"
|
|
644
|
+
" const recency = (r, i) => {\n"
|
|
645
|
+
" const sibs = order[r.agent];\n"
|
|
646
|
+
" return sibs.length > 1 ? sibs.indexOf(i) / (sibs.length - 1) : 1;\n"
|
|
647
|
+
" };\n"
|
|
648
|
+
" chart2 = new Chart(c2, {type: 'line', data: {datasets:\n"
|
|
649
|
+
" withCurve.map((r, i) => ({\n"
|
|
650
|
+
" label: r.agent + ' ' + (r.ended || '').slice(5, 16),\n"
|
|
651
|
+
" data: bcurves[r.run_id].map(p => ({x: p[0], y: p[1]})),\n"
|
|
652
|
+
" borderColor: agentShade(r.agent, recency(r, i)),\n"
|
|
653
|
+
" hidden: hidden[i] || false,\n"
|
|
654
|
+
" borderWidth: 1.5, pointRadius: 0}))},\n"
|
|
655
|
+
" options: {color: css('--muted'), parsing: false,\n"
|
|
656
|
+
" scales: {x: {...axis(), type: lxb.checked ? 'logarithmic' : 'linear'},\n"
|
|
657
|
+
" y: {...axis(), type: lyb.checked ? 'logarithmic' : 'linear'}},\n"
|
|
658
|
+
" plugins: {legend: {display: false}}}});\n"
|
|
659
|
+
" buildLegend();\n"
|
|
660
|
+
" };\n"
|
|
661
|
+
" lxb.onchange = draw2; lyb.onchange = draw2; draw2();\n"
|
|
662
|
+
" redraws.push(draw2);\n"
|
|
663
|
+
" }\n"
|
|
664
|
+
"}\n"
|
|
665
|
+
"const now = document.getElementById('now');\n"
|
|
666
|
+
"let strip = null;\n"
|
|
667
|
+
"const refresh = () =>\n"
|
|
668
|
+
" fetch('climb/status.json', {cache: 'no-cache'})\n"
|
|
669
|
+
" .then(r => r.ok ? r.json() : null)\n"
|
|
670
|
+
" .then(s => { if (s && Array.isArray(s.runs)) { strip = s; render(); } })\n"
|
|
671
|
+
" .catch(() => {});\n"
|
|
672
|
+
"const stateHue = r => r.state === 'in-review' ? '150 55% 38%'\n"
|
|
673
|
+
" : r.state === 'implementing' ? '262 45% 52%'\n"
|
|
674
|
+
" : r.phase === 'author-sleep' ? '212 55% 46%' : '38 65% 42%';\n"
|
|
675
|
+
"// kernel phase names, translated for the page: 'in gate' = the\n"
|
|
676
|
+
"// kernel is measuring a submitted candidate; 'experiments' = the\n"
|
|
677
|
+
"// author launched its own jobs and sleeps until they finish\n"
|
|
678
|
+
"const stateName = r => r.state !== 'waiting' ? r.state.replace('-', ' ')\n"
|
|
679
|
+
" : r.phase === 'author-sleep' ? 'experiments'\n"
|
|
680
|
+
" : r.phase === 'candidate' ? 'in gate' : 'waiting';\n"
|
|
681
|
+
"const render = () => {\n"
|
|
682
|
+
" if (!strip) return;\n"
|
|
683
|
+
" now.textContent = '';\n"
|
|
684
|
+
" for (const r of strip.runs) {\n"
|
|
685
|
+
" const card = document.createElement('span'); card.className = 'run';\n"
|
|
686
|
+
" const top = document.createElement('span'); top.className = 'top';\n"
|
|
687
|
+
" const pill = document.createElement('span'); pill.className = 'pill';\n"
|
|
688
|
+
" pill.style.background = `hsl(${stateHue(r)})`;\n"
|
|
689
|
+
" pill.textContent = stateName(r);\n"
|
|
690
|
+
" const who = document.createElement('b'); who.textContent = r.agent;\n"
|
|
691
|
+
" who.style.color = agentColor(r.agent);\n"
|
|
692
|
+
" const mins = Math.max(0, (Date.now() / 1000 - r.since) / 60);\n"
|
|
693
|
+
" const t = mins >= 90 ? (mins / 60).toFixed(1) + ' h' : Math.round(mins) + ' min';\n"
|
|
694
|
+
" const meta = document.createElement('span');\n"
|
|
695
|
+
" meta.title = 'time in this state '\n"
|
|
696
|
+
" + '(since the last transition: park, submit, wake)';\n"
|
|
697
|
+
" meta.textContent = t;\n"
|
|
698
|
+
" top.append(who, pill, meta);\n"
|
|
699
|
+
" if (r.pr_url) {\n"
|
|
700
|
+
" const a = document.createElement('a'); a.href = r.pr_url;\n"
|
|
701
|
+
" a.textContent = 'PR'; top.append(a);\n"
|
|
702
|
+
" }\n"
|
|
703
|
+
" card.append(top);\n"
|
|
704
|
+
" if (r.direction) {\n"
|
|
705
|
+
" const d = document.createElement('span'); d.className = 'dir';\n"
|
|
706
|
+
" d.textContent = r.direction; d.style.display = 'block';\n"
|
|
707
|
+
" card.append(d);\n"
|
|
708
|
+
" }\n"
|
|
709
|
+
" // the agent's life panel: 'exps'/'gate' is the current wait\n"
|
|
710
|
+
" // (bar = progressing), 'depth' and 'GPU-h' are budgets being\n"
|
|
711
|
+
" // consumed (dash meters = life spent)\n"
|
|
712
|
+
" const lm = document.createElement('span'); lm.className = 'lm';\n"
|
|
713
|
+
" const cell = (cls) => {\n"
|
|
714
|
+
" const s = document.createElement('span');\n"
|
|
715
|
+
" if (cls) s.className = cls; return s;\n"
|
|
716
|
+
" };\n"
|
|
717
|
+
" const addRow = (label, mid, value, title) => {\n"
|
|
718
|
+
" const l = cell(); l.textContent = label;\n"
|
|
719
|
+
" const v = cell(); v.textContent = value;\n"
|
|
720
|
+
" l.title = mid.title = v.title = title;\n"
|
|
721
|
+
" lm.append(l, mid, v);\n"
|
|
722
|
+
" return v;\n"
|
|
723
|
+
" };\n"
|
|
724
|
+
" const timeFrac = m =>\n"
|
|
725
|
+
" Math.min(1, (Date.now() / 1000 - r.since) / (m * 60));\n"
|
|
726
|
+
" const layer = (w, color) => {\n"
|
|
727
|
+
" const f = document.createElement('span');\n"
|
|
728
|
+
" f.style.width = Math.round(w * 100) + '%';\n"
|
|
729
|
+
" f.style.background = color; return f;\n"
|
|
730
|
+
" };\n"
|
|
731
|
+
" const dashes = (filled, segs) => {\n"
|
|
732
|
+
" const m = cell('meter');\n"
|
|
733
|
+
" for (let i = 0; i < segs; i++) {\n"
|
|
734
|
+
" const seg = document.createElement('i');\n"
|
|
735
|
+
" if (i < filled) seg.style.background = agentColor(r.agent);\n"
|
|
736
|
+
" m.append(seg);\n"
|
|
737
|
+
" }\n"
|
|
738
|
+
" return m;\n"
|
|
739
|
+
" };\n"
|
|
740
|
+
" if (r.exp_total) {\n"
|
|
741
|
+
" const bar = cell('bar');\n"
|
|
742
|
+
" if (r.exp_minutes)\n"
|
|
743
|
+
" bar.append(layer(timeFrac(r.exp_minutes),\n"
|
|
744
|
+
" agentColor(r.agent).slice(0, -1) + ' / .3)'));\n"
|
|
745
|
+
" bar.append(layer(r.exp_done / r.exp_total, agentColor(r.agent)));\n"
|
|
746
|
+
" addRow('exps', bar, r.exp_done + '/' + r.exp_total + ' done',\n"
|
|
747
|
+
" 'experiment jobs: solid = finished, '\n"
|
|
748
|
+
" + 'faint = elapsed vs the longest walltime cap');\n"
|
|
749
|
+
" } else if (r.phase === 'candidate' && r.eval_minutes) {\n"
|
|
750
|
+
" const bar = cell('bar');\n"
|
|
751
|
+
" // the fill stays the agent's color; overdue is flagged by\n"
|
|
752
|
+
" // the number turning the baseline red\n"
|
|
753
|
+
" bar.append(layer(timeFrac(r.eval_minutes), agentColor(r.agent)));\n"
|
|
754
|
+
" const v = addRow('gate', bar,\n"
|
|
755
|
+
" ((Date.now() / 1000 - r.since) / 3600).toFixed(1)\n"
|
|
756
|
+
" + '/' + (r.eval_minutes / 60).toFixed(1) + ' h',\n"
|
|
757
|
+
" 'gate eval: time since submission vs its walltime cap. Queue'\n"
|
|
758
|
+
" + ' time counts, so past the cap the job is either still'\n"
|
|
759
|
+
" + ' queued or timed out awaiting the next sweep.');\n"
|
|
760
|
+
" if (timeFrac(r.eval_minutes) >= 1) v.style.color = 'var(--base)';\n"
|
|
761
|
+
" }\n"
|
|
762
|
+
" if (r.depth_k && r.launches_used != null)\n"
|
|
763
|
+
" addRow('depth', dashes(r.launches_used, r.depth_k),\n"
|
|
764
|
+
" r.launches_used + '/' + r.depth_k,\n"
|
|
765
|
+
" 'experiment launches used, of the contract depth_k');\n"
|
|
766
|
+
" if (r.gpu_hours_budget && r.gpu_hours_used != null)\n"
|
|
767
|
+
" addRow('GPU-h',\n"
|
|
768
|
+
" dashes(Math.round(20 * r.gpu_hours_used / r.gpu_hours_budget), 20),\n"
|
|
769
|
+
" Number(r.gpu_hours_used).toFixed(0)\n"
|
|
770
|
+
" + '/' + Number(r.gpu_hours_budget),\n"
|
|
771
|
+
" 'GPU-hours charged to this run, of the contract budget');\n"
|
|
772
|
+
" if (lm.childNodes.length) card.append(lm);\n"
|
|
773
|
+
" now.append(card);\n"
|
|
774
|
+
" }\n"
|
|
775
|
+
" if (!strip.runs.length) now.textContent = 'no active runs';\n"
|
|
776
|
+
"};\n"
|
|
777
|
+
"refresh();\n"
|
|
778
|
+
"// the strip re-FETCHES every few minutes (new/left/changed runs) and\n"
|
|
779
|
+
"// re-renders the elapsed time locally between fetches\n"
|
|
780
|
+
"onTheme = render;\n"
|
|
781
|
+
"setInterval(refresh, 180000); setInterval(render, 30000);\n"
|
|
782
|
+
"</script></main></body></html>\n"
|
|
783
|
+
)
|
|
784
|
+
|
|
785
|
+
|
|
786
|
+
STATUS_PATH = "climb/status.json"
|
|
787
|
+
_LIVE_STATES = ("implementing", "waiting", "in-review", "concluding")
|
|
788
|
+
|
|
789
|
+
|
|
790
|
+
def _phrase(text: str, cap: int = 64) -> str:
|
|
791
|
+
"""A strip-sized phrase: the first clause of the first sentence."""
|
|
792
|
+
# markdown bold reads as literal asterisks on the strip (underscores
|
|
793
|
+
# stay: __init__.py is a filename far more often than __bold__)
|
|
794
|
+
first = summarize(text.replace("**", ""), 200)
|
|
795
|
+
for sep in (": ", " — ", " -- "):
|
|
796
|
+
first = first.split(sep, 1)[0]
|
|
797
|
+
return summarize(first, cap)
|
|
798
|
+
|
|
799
|
+
|
|
800
|
+
def _experiment_progress(root: Path, record: Any) -> tuple[int, int, int]:
|
|
801
|
+
"""(finished, launched, longest walltime minutes) across the current
|
|
802
|
+
park's experiment jobs, counted by the exit-code files the job wrappers
|
|
803
|
+
leave — filesystem only, no Slurm calls on the board path."""
|
|
804
|
+
stage = record.stage or {}
|
|
805
|
+
raw = stage.get("syscall_launches", [])
|
|
806
|
+
if not isinstance(raw, list):
|
|
807
|
+
return (0, 0, 0)
|
|
808
|
+
names: list[str] = []
|
|
809
|
+
minutes = 0
|
|
810
|
+
for item in raw:
|
|
811
|
+
if not (isinstance(item, dict) and item.get("name")):
|
|
812
|
+
continue
|
|
813
|
+
try:
|
|
814
|
+
array = int(item.get("array") or 1)
|
|
815
|
+
except (TypeError, ValueError):
|
|
816
|
+
array = 1
|
|
817
|
+
with contextlib.suppress(TypeError, ValueError):
|
|
818
|
+
minutes = max(minutes, int(item.get("minutes") or 0))
|
|
819
|
+
name = str(item["name"])
|
|
820
|
+
names += [f"{name}.{i}" for i in range(array)] if array > 1 else [name]
|
|
821
|
+
rd = run_dir(root, record.run_id)
|
|
822
|
+
done = sum(1 for n in names if (rd / f"eval-launch-{n}" / "exit-code").exists())
|
|
823
|
+
return (done, len(names), minutes)
|
|
824
|
+
|
|
825
|
+
|
|
826
|
+
def collect_status(root: Path, target: str, now: float, contract: Any = None) -> dict[str, Any]:
|
|
827
|
+
"""The fleet's live picture for `target`: one entry per non-terminal run.
|
|
828
|
+
Timestamps, not durations — the page computes elapsed time client-side,
|
|
829
|
+
so the strip feels live between pushes."""
|
|
830
|
+
from outerloop.dispatch import effective_eval_minutes
|
|
831
|
+
|
|
832
|
+
budgets = {
|
|
833
|
+
b.name: (b.depth_k, b.sleep_k, getattr(b, "eval_minutes", 0) or 0)
|
|
834
|
+
for b in getattr(contract, "benchmarks", ())
|
|
835
|
+
}
|
|
836
|
+
gpu_budget = getattr(getattr(contract, "budgets", None), "gpu_hours_per_run", None)
|
|
837
|
+
runs = []
|
|
838
|
+
for record in list_runs(root):
|
|
839
|
+
if record.target != target or record.state not in _LIVE_STATES:
|
|
840
|
+
continue
|
|
841
|
+
stage = record.stage or {}
|
|
842
|
+
note = str(stage.get("syscall_note") or stage.get("report") or "")
|
|
843
|
+
_b, _c, hyp = _report_fields(note)
|
|
844
|
+
exp_done, exp_total, exp_minutes = _experiment_progress(root, record)
|
|
845
|
+
depth_k, sleep_k, bench_minutes = budgets.get(record.benchmark, (None, None, 0))
|
|
846
|
+
runs.append(
|
|
847
|
+
{
|
|
848
|
+
"run_id": record.run_id,
|
|
849
|
+
"agent": record.agent_id,
|
|
850
|
+
"benchmark": record.benchmark,
|
|
851
|
+
"state": record.state,
|
|
852
|
+
"phase": stage.get("phase", ""),
|
|
853
|
+
# the agent's own headline: what it says it is working on
|
|
854
|
+
"direction": _phrase(hyp or note.replace("\n", " ")),
|
|
855
|
+
"since": record.updated or record.created,
|
|
856
|
+
# a run that never launched HAS used zero — absent keys must
|
|
857
|
+
# not blank the card's meters (a plain submit writes none)
|
|
858
|
+
"launches_used": int(stage.get("launches_used") or 0), # type: ignore[call-overload]
|
|
859
|
+
"sleeps_used": int(stage.get("sleeps_used") or 0), # type: ignore[call-overload]
|
|
860
|
+
"depth_k": depth_k,
|
|
861
|
+
"sleep_k": sleep_k,
|
|
862
|
+
# experiment fan-out of the current park, and the gate eval's
|
|
863
|
+
# walltime cap — the page turns these into progress bars
|
|
864
|
+
"exp_done": exp_done,
|
|
865
|
+
"exp_total": exp_total,
|
|
866
|
+
"exp_minutes": exp_minutes,
|
|
867
|
+
# submit without --minutes stores nothing: fall back to the
|
|
868
|
+
# contract cap, CLAMPED like the dispatched job itself is —
|
|
869
|
+
# the meter must show the limit the gate actually runs under
|
|
870
|
+
"eval_minutes": (
|
|
871
|
+
effective_eval_minutes(m)
|
|
872
|
+
if (m := int(stage.get("eval_minutes", 0) or 0) or bench_minutes) # type: ignore[call-overload]
|
|
873
|
+
else 0
|
|
874
|
+
),
|
|
875
|
+
"gpu_hours_used": float(stage.get("gpu_hours_used") or 0.0), # type: ignore[arg-type]
|
|
876
|
+
"gpu_hours_budget": gpu_budget,
|
|
877
|
+
"pr_url": record.pr_url,
|
|
878
|
+
}
|
|
879
|
+
)
|
|
880
|
+
runs.sort(key=lambda r: str(r.get("run_id")))
|
|
881
|
+
return {"target": target, "published": now, "runs": runs}
|
|
882
|
+
|
|
883
|
+
|
|
884
|
+
def service_status(root: Path, github: Any, target: str, now: float, contract: Any = None) -> bool:
|
|
885
|
+
"""Publish the strip when the fleet's SHAPE changed — a run appearing,
|
|
886
|
+
leaving, or changing state/phase — never on every tick: the page shows
|
|
887
|
+
elapsed time client-side, so timestamp-only drift is not worth a commit.
|
|
888
|
+
Advisory like the board; True when a write happened."""
|
|
889
|
+
status = collect_status(root, target, now, contract)
|
|
890
|
+
try:
|
|
891
|
+
existing_raw: str | None = github.get_file(target, STATUS_PATH, BOARD_BRANCH)
|
|
892
|
+
except Exception as exc:
|
|
893
|
+
if getattr(exc, "status", None) != 404:
|
|
894
|
+
# an outage must not look like a missing file: a rewrite here
|
|
895
|
+
# would commit a new timestamp on every affected tick
|
|
896
|
+
log.warning("status unreadable (%s); not rewritten", exc)
|
|
897
|
+
return False
|
|
898
|
+
existing_raw = None
|
|
899
|
+
if existing_raw:
|
|
900
|
+
try:
|
|
901
|
+
existing = json.loads(existing_raw)
|
|
902
|
+
# spend belongs in the shape: a same-phase re-park after new
|
|
903
|
+
# launches moves gpu_hours_used and the strip must not go stale
|
|
904
|
+
# exp_done/launches_used move when an experiment finishes or a
|
|
905
|
+
# re-park launches more; the contract caps move when the owner
|
|
906
|
+
# edits the contract — all real transitions the strip must show
|
|
907
|
+
keys = (
|
|
908
|
+
"run_id",
|
|
909
|
+
"state",
|
|
910
|
+
"phase",
|
|
911
|
+
"gpu_hours_used",
|
|
912
|
+
"gpu_hours_budget",
|
|
913
|
+
"direction",
|
|
914
|
+
"exp_done",
|
|
915
|
+
"exp_total",
|
|
916
|
+
"exp_minutes",
|
|
917
|
+
"eval_minutes",
|
|
918
|
+
"launches_used",
|
|
919
|
+
"depth_k",
|
|
920
|
+
"sleep_k",
|
|
921
|
+
)
|
|
922
|
+
shape = lambda runs: [{k: r.get(k) for k in keys} for r in runs]
|
|
923
|
+
if (
|
|
924
|
+
isinstance(existing, dict)
|
|
925
|
+
and isinstance(existing.get("runs"), list)
|
|
926
|
+
and shape(existing["runs"]) == shape(status["runs"])
|
|
927
|
+
):
|
|
928
|
+
return False
|
|
929
|
+
except (ValueError, TypeError, AttributeError):
|
|
930
|
+
pass # malformed: the strip is derived, not history — rewrite it
|
|
931
|
+
if not github.ensure_branch(target, BOARD_BRANCH):
|
|
932
|
+
return False
|
|
933
|
+
return bool(
|
|
934
|
+
github.put_file(
|
|
935
|
+
target, STATUS_PATH, json.dumps(status, indent=1) + "\n", BOARD_BRANCH, "fleet status"
|
|
936
|
+
)
|
|
937
|
+
)
|
|
938
|
+
|
|
939
|
+
|
|
940
|
+
def _valid_curve(curve: Any) -> bool:
|
|
941
|
+
"""A publishable curve: [step, value] pairs, numbers only — one null
|
|
942
|
+
point in a published file would throw in the page's chart code."""
|
|
943
|
+
|
|
944
|
+
def finite(x: Any) -> bool:
|
|
945
|
+
try:
|
|
946
|
+
return math.isfinite(x)
|
|
947
|
+
except OverflowError: # an int too large for float is not a chart point
|
|
948
|
+
return False
|
|
949
|
+
|
|
950
|
+
return isinstance(curve, list) and all(
|
|
951
|
+
isinstance(p, list)
|
|
952
|
+
and len(p) == 2
|
|
953
|
+
and all(isinstance(x, int | float) and not isinstance(x, bool) and finite(x) for x in p)
|
|
954
|
+
for p in curve
|
|
955
|
+
)
|
|
956
|
+
|
|
957
|
+
|
|
958
|
+
def _merge_curves(
|
|
959
|
+
github: Any,
|
|
960
|
+
target: str,
|
|
961
|
+
benchmark: str,
|
|
962
|
+
rows: list[dict[str, Any]],
|
|
963
|
+
fresh_for: Any,
|
|
964
|
+
) -> dict[str, Any] | None:
|
|
965
|
+
"""Published curves plus new ones, kept only for the newest rows (curves
|
|
966
|
+
are heavy; the cap is by recency of the attempt, and published curves
|
|
967
|
+
are never rewritten). `fresh_for(run_id)` parses a run's eval stdout ON
|
|
968
|
+
DEMAND — a run with a published curve costs no I/O at all. None when
|
|
969
|
+
the published file cannot be read — the same sit-the-pass-out stance
|
|
970
|
+
as everything else on the branch."""
|
|
971
|
+
path = f"climb/curves/{benchmark}.json"
|
|
972
|
+
try:
|
|
973
|
+
raw: str | None = github.get_file(target, path, BOARD_BRANCH)
|
|
974
|
+
except Exception as exc:
|
|
975
|
+
if getattr(exc, "status", None) != 404:
|
|
976
|
+
log.warning("board curves unreadable for %s (%s); skipped", benchmark, exc)
|
|
977
|
+
return None
|
|
978
|
+
raw = None
|
|
979
|
+
published: dict[str, list[list[float]]] = {}
|
|
980
|
+
if raw:
|
|
981
|
+
try:
|
|
982
|
+
data = json.loads(raw)
|
|
983
|
+
except ValueError:
|
|
984
|
+
data = None
|
|
985
|
+
if not isinstance(data, dict) or not all(_valid_curve(v) for v in data.values()):
|
|
986
|
+
# PARTLY malformed is malformed: dropping the bad values and
|
|
987
|
+
# rewriting would lose published curves
|
|
988
|
+
log.warning("board curves malformed for %s; skipped", benchmark)
|
|
989
|
+
return None
|
|
990
|
+
published = {str(k): v for k, v in data.items()}
|
|
991
|
+
# keep the newest curves, at most MAX_CURVE_RUNS_PER_AGENT per agent so
|
|
992
|
+
# one busy agent cannot crowd out the others. Walk ALL rows newest-first
|
|
993
|
+
# (no global pre-slice — that could bury a quiet agent whose rows all sit
|
|
994
|
+
# past the cut), and count the quota only for rows that ACTUALLY have a
|
|
995
|
+
# curve (a curveless attempt must not spend an agent's five slots).
|
|
996
|
+
per_agent: dict[str, int] = {}
|
|
997
|
+
merged: dict[str, list[list[float]]] = {}
|
|
998
|
+
for r in reversed(rows):
|
|
999
|
+
agent = str(r.get("agent") or "")
|
|
1000
|
+
if per_agent.get(agent, 0) >= MAX_CURVE_RUNS_PER_AGENT:
|
|
1001
|
+
continue
|
|
1002
|
+
run_id = str(r.get("run_id"))
|
|
1003
|
+
curve = published.get(run_id) or fresh_for(run_id)
|
|
1004
|
+
if not curve:
|
|
1005
|
+
continue
|
|
1006
|
+
per_agent[agent] = per_agent.get(agent, 0) + 1
|
|
1007
|
+
merged[run_id] = curve
|
|
1008
|
+
return {"data": merged, "changed": merged != published}
|
|
1009
|
+
|
|
1010
|
+
|
|
1011
|
+
def contract_directions(contract: Any) -> dict[str, str]:
|
|
1012
|
+
"""{benchmark: direction} out of a loaded contract (None -> {})."""
|
|
1013
|
+
if contract is None:
|
|
1014
|
+
return {}
|
|
1015
|
+
return {b.name: b.direction for b in contract.benchmarks}
|
|
1016
|
+
|
|
1017
|
+
|
|
1018
|
+
def _read_index(github: Any, target: str) -> dict[str, str] | None:
|
|
1019
|
+
"""climb/index.json: {benchmark: direction} for every benchmark ever
|
|
1020
|
+
published — the board's own memory, so a benchmark whose local records
|
|
1021
|
+
were cleaned up (or that left the contract) keeps its place in the
|
|
1022
|
+
views. {} means the board has no index yet (a 404); None means the read
|
|
1023
|
+
FAILED — the caller must not rewrite the index it could not see, or a
|
|
1024
|
+
transient outage would shrink it."""
|
|
1025
|
+
try:
|
|
1026
|
+
raw = github.get_file(target, "climb/index.json", BOARD_BRANCH)
|
|
1027
|
+
except Exception as exc:
|
|
1028
|
+
status = getattr(exc, "status", None)
|
|
1029
|
+
if status == 404:
|
|
1030
|
+
return {}
|
|
1031
|
+
log.warning("climb index unreadable (%s); not rewriting it", exc)
|
|
1032
|
+
return None
|
|
1033
|
+
try:
|
|
1034
|
+
data = json.loads(raw)
|
|
1035
|
+
except ValueError:
|
|
1036
|
+
log.warning("climb index malformed; not rewriting it")
|
|
1037
|
+
return None # same stance as an outage: never rewrite what we cannot see
|
|
1038
|
+
return {str(k): str(v) for k, v in data.items()} if isinstance(data, dict) else None
|
|
1039
|
+
|
|
1040
|
+
|
|
1041
|
+
def service_climb_board(
|
|
1042
|
+
root: Path, github: Any, target: str, directions: dict[str, str] | None = None
|
|
1043
|
+
) -> int:
|
|
1044
|
+
"""Publish the board for `target`. Returns how many files changed.
|
|
1045
|
+
|
|
1046
|
+
Every file is compared against the branch, and all changed files land
|
|
1047
|
+
as ONE commit (`put_files`) — data, curves, and the views are atomic,
|
|
1048
|
+
so the page can never point at data that is not on the branch, and a
|
|
1049
|
+
board pass costs at most one commit of research-log history. A failed
|
|
1050
|
+
batch changes nothing; the whole pass retries next tick."""
|
|
1051
|
+
local = collect_rows(root, target)
|
|
1052
|
+
# snapshot BEFORE any branch read: put_files refuses if the head moves
|
|
1053
|
+
# mid-pass, so a concurrent write is never buried under stale content.
|
|
1054
|
+
# "" = branch missing (nothing to protect); None = outage — writing
|
|
1055
|
+
# unguarded could bury a mid-pass write, so the pass sits out
|
|
1056
|
+
head = github.branch_head(target, BOARD_BRANCH)
|
|
1057
|
+
if head is None:
|
|
1058
|
+
log.warning("board head unreadable for %s; pass sits out", target)
|
|
1059
|
+
return 0
|
|
1060
|
+
index = _read_index(github, target)
|
|
1061
|
+
names = set(local) | set(index or {})
|
|
1062
|
+
directions = {**(index or {}), **(directions or {})}
|
|
1063
|
+
pending: dict[str, str] = {}
|
|
1064
|
+
boards: dict[str, list[dict[str, Any]]] = {}
|
|
1065
|
+
curves: dict[str, dict[str, list[list[float]]]] = {}
|
|
1066
|
+
curves_ok = True
|
|
1067
|
+
for benchmark in sorted(names):
|
|
1068
|
+
path = f"climb/data/{benchmark}.json"
|
|
1069
|
+
try:
|
|
1070
|
+
existing: str | None = github.get_file(target, path, BOARD_BRANCH)
|
|
1071
|
+
if existing is not None:
|
|
1072
|
+
parsed = json.loads(existing)
|
|
1073
|
+
if not isinstance(parsed, list):
|
|
1074
|
+
raise ValueError("board data is not a list")
|
|
1075
|
+
except (ValueError, TypeError) as exc:
|
|
1076
|
+
# readable but not a row list: same stance as an outage — never
|
|
1077
|
+
# let a fresh merge overwrite what we cannot interpret
|
|
1078
|
+
log.warning("board JSON malformed for %s (%s); skipped", benchmark, exc)
|
|
1079
|
+
continue
|
|
1080
|
+
except Exception as exc:
|
|
1081
|
+
if getattr(exc, "status", None) != 404:
|
|
1082
|
+
# an outage is not an empty board: overwriting would replace
|
|
1083
|
+
# this benchmark's published history — sit the pass out (its
|
|
1084
|
+
# index entry survives)
|
|
1085
|
+
log.warning("board JSON unreadable for %s (%s); skipped", benchmark, exc)
|
|
1086
|
+
continue
|
|
1087
|
+
existing = None
|
|
1088
|
+
rows = merge_rows(existing, local.get(benchmark, []))
|
|
1089
|
+
if not rows:
|
|
1090
|
+
continue
|
|
1091
|
+
boards[benchmark] = rows
|
|
1092
|
+
text = json.dumps(rows, indent=1) + "\n"
|
|
1093
|
+
if text != existing:
|
|
1094
|
+
pending[path] = text
|
|
1095
|
+
merged_curves = _merge_curves(
|
|
1096
|
+
github, target, benchmark, rows, lambda rid: _curve_from_eval(run_dir(root, rid))
|
|
1097
|
+
)
|
|
1098
|
+
if merged_curves is None:
|
|
1099
|
+
# an unreadable curve file must not republish the page without
|
|
1100
|
+
# its curves — the html rewrite sits this pass out
|
|
1101
|
+
curves_ok = False
|
|
1102
|
+
continue
|
|
1103
|
+
curves[benchmark] = merged_curves["data"]
|
|
1104
|
+
if merged_curves["changed"]:
|
|
1105
|
+
pending[f"climb/curves/{benchmark}.json"] = (
|
|
1106
|
+
json.dumps(merged_curves["data"], indent=1) + "\n"
|
|
1107
|
+
)
|
|
1108
|
+
if not boards and names:
|
|
1109
|
+
return 0 # every benchmark sat the pass out: leave the views alone
|
|
1110
|
+
# the index keeps every benchmark it knows — a transient failed read of
|
|
1111
|
+
# one benchmark's JSON must not orphan its history; the views simply
|
|
1112
|
+
# render without it until a later pass reads it again. An index that
|
|
1113
|
+
# could not be READ at all (None) is never rewritten this pass.
|
|
1114
|
+
wanted = {b: directions.get(b, "min") for b in sorted(set(boards) | set(index or {}))}
|
|
1115
|
+
# the starting positions come from the target's ledger on its default
|
|
1116
|
+
# branch — best-effort: a missing or unreadable ledger just drops the chip
|
|
1117
|
+
starts: dict[str, float] = {}
|
|
1118
|
+
try:
|
|
1119
|
+
# ref HEAD = the repo's default branch, whatever it is named
|
|
1120
|
+
raw_leader = github.get_file_content(target, "results/leader.json", "HEAD")
|
|
1121
|
+
if raw_leader:
|
|
1122
|
+
for name, entry in json.loads(raw_leader).items():
|
|
1123
|
+
value = entry.get("baseline") if isinstance(entry, dict) else None
|
|
1124
|
+
# json.loads admits NaN/Infinity, which _fmt cannot render
|
|
1125
|
+
if isinstance(value, int | float) and math.isfinite(value):
|
|
1126
|
+
starts[str(name)] = float(value)
|
|
1127
|
+
except Exception:
|
|
1128
|
+
starts = {}
|
|
1129
|
+
views: list[tuple[str, str]] = [("CLIMB.md", render_md(target, boards, wanted, starts))]
|
|
1130
|
+
if curves_ok:
|
|
1131
|
+
views.append(("index.html", render_html(target, boards, wanted, curves)))
|
|
1132
|
+
if index is not None:
|
|
1133
|
+
views.insert(0, ("climb/index.json", json.dumps(wanted, indent=1) + "\n"))
|
|
1134
|
+
for path, content in views:
|
|
1135
|
+
if content != github.get_file_content(target, path, BOARD_BRANCH):
|
|
1136
|
+
pending[path] = content
|
|
1137
|
+
if not pending:
|
|
1138
|
+
return 0
|
|
1139
|
+
if not github.ensure_branch(target, BOARD_BRANCH):
|
|
1140
|
+
return 0
|
|
1141
|
+
if not github.put_files(
|
|
1142
|
+
target, pending, BOARD_BRANCH, "climb board", expected_head=head or None
|
|
1143
|
+
):
|
|
1144
|
+
return 0
|
|
1145
|
+
return len(pending)
|