flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. flashml_workloads/__init__.py +7 -0
  2. flashml_workloads/fedavg_driver.py +569 -0
  3. flashml_workloads/fedavg_weights.py +223 -0
  4. flashml_workloads/fedavg_worker.py +166 -0
  5. flashml_workloads/kmeans_driver.py +134 -0
  6. flashml_workloads/kmeans_shard.py +69 -0
  7. flashml_workloads/sgd_trainer.py +127 -0
  8. flashml_workloads/sharded_kmeans.py +323 -0
  9. flashml_workloads/sklearn_trial.py +89 -0
  10. flashruntime/__init__.py +125 -0
  11. flashruntime/artifacts/__init__.py +25 -0
  12. flashruntime/artifacts/store.py +228 -0
  13. flashruntime/backends/__init__.py +26 -0
  14. flashruntime/backends/base.py +63 -0
  15. flashruntime/backends/kuberay.py +465 -0
  16. flashruntime/checkpoint/__init__.py +20 -0
  17. flashruntime/checkpoint/catalog.py +198 -0
  18. flashruntime/checkpoint/local.py +109 -0
  19. flashruntime/checkpoint/store.py +86 -0
  20. flashruntime/integrations/__init__.py +5 -0
  21. flashruntime/integrations/huggingface.py +59 -0
  22. flashruntime/integrations/pytorch.py +52 -0
  23. flashruntime/integrations/sklearn.py +42 -0
  24. flashruntime/launchers/__init__.py +130 -0
  25. flashruntime/launchers/local.py +126 -0
  26. flashruntime/leases/__init__.py +27 -0
  27. flashruntime/leases/manager.py +365 -0
  28. flashruntime/leases/sqlite_store.py +169 -0
  29. flashruntime/leases/store.py +103 -0
  30. flashruntime/monitor/__init__.py +7 -0
  31. flashruntime/monitor/sampler.py +232 -0
  32. flashruntime/planner/__init__.py +56 -0
  33. flashruntime/planner/candidates.py +597 -0
  34. flashruntime/planner/catalog.py +129 -0
  35. flashruntime/planner/comm.py +95 -0
  36. flashruntime/planner/explain.py +109 -0
  37. flashruntime/planner/memory.py +166 -0
  38. flashruntime/planner/resolve.py +120 -0
  39. flashruntime/planner/selector.py +169 -0
  40. flashruntime/planner/timecost.py +81 -0
  41. flashruntime/profiling/__init__.py +113 -0
  42. flashruntime/protocol/__init__.py +18 -0
  43. flashruntime/protocol/plan_v1alpha1.py +320 -0
  44. flashruntime/protocol/v1alpha1.py +465 -0
  45. flashruntime/providers/__init__.py +138 -0
  46. flashruntime/py.typed +0 -0
  47. flashruntime/recipes/__init__.py +135 -0
  48. flashruntime/recipes/command.py +166 -0
  49. flashruntime/recovery/__init__.py +21 -0
  50. flashruntime/recovery/policy.py +170 -0
  51. flashruntime/recovery/signals.py +135 -0
  52. flashruntime/recovery/taxonomy.py +91 -0
  53. flashruntime/scheduler/__init__.py +170 -0
  54. flashruntime/sdk.py +402 -0
  55. flashruntime/service/__init__.py +3 -0
  56. flashruntime/service/app.py +391 -0
  57. flashruntime/service/auth.py +180 -0
  58. flashruntime/service/checkpoints.py +90 -0
  59. flashruntime/service/cli.py +167 -0
  60. flashruntime/service/dashboard.py +193 -0
  61. flashruntime/service/ledger.py +101 -0
  62. flashruntime/service/modea.py +821 -0
  63. flashruntime/strategies/__init__.py +156 -0
  64. flashruntime/strategies/command.py +56 -0
  65. flashruntime/torch/__init__.py +274 -0
  66. flashruntime/viewer/__init__.py +20 -0
  67. flashruntime/viewer/_docs/benchmarks.html +771 -0
  68. flashruntime/viewer/_docs/concepts/architecture.html +302 -0
  69. flashruntime/viewer/_docs/get-started.html +263 -0
  70. flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
  71. flashruntime/viewer/_docs/guides/huggingface.html +223 -0
  72. flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
  73. flashruntime/viewer/_docs/guides/pytorch.html +313 -0
  74. flashruntime/viewer/_docs/guides/sklearn.html +232 -0
  75. flashruntime/viewer/_docs/index.html +251 -0
  76. flashruntime/viewer/_docs/reference/cli.html +254 -0
  77. flashruntime/viewer/_docs/reference/integrations.html +240 -0
  78. flashruntime/viewer/_docs/reference/sdk.html +341 -0
  79. flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
  80. flashruntime/viewer/_docs/search-index.json +1 -0
  81. flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
  82. flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
  83. flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
  84. flashruntime/viewer/flowmap.py +307 -0
  85. flashruntime/viewer/page.py +594 -0
  86. flashruntime/viewer/server.py +134 -0
  87. flashruntime/viewer/state.py +250 -0
  88. flashruntime/workloads/__init__.py +6 -0
  89. flashruntime/workloads/command.py +127 -0
  90. flashruntime-0.3.0.dist-info/METADATA +365 -0
  91. flashruntime-0.3.0.dist-info/RECORD +95 -0
  92. flashruntime-0.3.0.dist-info/WHEEL +5 -0
  93. flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
  94. flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
  95. flashruntime-0.3.0.dist-info/top_level.txt +2 -0
flashruntime/viewer/page.py ADDED
@@ -0,0 +1,594 @@
1
+ """render() — the live run page, one self-contained HTML document.
2
+
3
+ This is the single sanctioned large file of the viewer: a `flash.submit(
4
+ watch=True)` opens it, and it polls `/api/state` (the `state.collect()`
5
+ snapshot) every couple of seconds to draw one run's story — its process map,
6
+ its loss curve, its checkpoints, and its recovery decisions — with ZERO
7
+ external assets. No CDN, no web font fetch, no remote image: the page must
8
+ render with the network cut, because the only server it may ever talk to is
9
+ the loopback `RunViewerServer` that served it. Everything (CSS, JS) is
10
+ inline; the only same-origin links are `/api/state` and `/docs/`.
11
+
12
+ DESIGN CONTRACT (the house style, shared with `service/dashboard.py`):
13
+ the palette lives in `TOKENS` at the top of this module as the SINGLE SOURCE
14
+ OF TRUTH — a dark `#0d1117` field, `#161b22` panels, `#21262d` borders,
15
+ `#c9d1d9`/`#8b949e` text, and five oklch accents (cyan running, green
16
+ succeeded+verified, amber leased+recovering, red failed, violet
17
+ checkpoints). `dashboard.py` imports these same values so the two surfaces
18
+ read as one product. Sections, top to bottom (layout A, spec 2026-07-23):
19
+ header → KPI strip → process flow map (machine → workers → ranks; click a
20
+ node for the slide-in detail panel) → loss + resource charts → checkpoint
21
+ timeline → events feed → collapsible logs. The flow-map/KPI component
22
+ itself lives in `viewer/flowmap.py` — shared with the coordinator
23
+ dashboard.
24
+
25
+ The drawing is plain, commented JS organized by section (`renderFlowmap`,
26
+ `drawLoss`, `drawResources`, `renderCheckpoints`, …) — no framework, no
27
+ build step, no minification. Same readability bar as the Python (spec §2b).
28
+ """
29
+
30
+ from __future__ import annotations
31
+
32
+ import json
33
+
34
+ from flashruntime.viewer.flowmap import FLOWMAP_CSS, FLOWMAP_JS
35
+
36
+ # --------------------------------------------------------------------------
37
+ # Visual tokens — the single source of truth for BOTH this page and the
38
+ # coordinator dashboard (`service/dashboard.py` imports these values). Accents
39
+ # are oklch so they stay perceptually even across hues; the neutrals are the
40
+ # established GitHub-dark family the house style already used.
41
+ # --------------------------------------------------------------------------
42
+ TOKENS: dict[str, str] = {
43
+ "bg": "#0d1117", # page field
44
+ "bg_inset": "#010409", # deepest wells (canvas backdrops, code)
45
+ "panel": "#161b22", # raised panels / hover
46
+ "border": "#21262d", # hairlines
47
+ "text": "#c9d1d9", # body text
48
+ "text_bright": "#e6edf3", # headings
49
+ "muted": "#8b949e", # secondary text / section labels
50
+ "running": "oklch(0.80 0.16 200)", # cyan — RUNNING
51
+ "ok": "oklch(0.76 0.18 145)", # green — SUCCEEDED / hash-verified
52
+ "warn": "oklch(0.80 0.18 60)", # amber — LEASED / RECOVERING / classified
53
+ "fail": "oklch(0.70 0.20 20)", # red — FAILED / invalid
54
+ "ckpt": "oklch(0.65 0.20 290)", # violet — checkpoints
55
+ "font": "ui-monospace, SFMono-Regular, Menlo, Consolas, monospace",
56
+ }
57
+
58
+ # The subset of tokens the JS drawing code needs (canvas fillStyles). Injected
59
+ # as a JS object literal so the colors are never duplicated between CSS and JS.
60
+ _JS_TOKEN_KEYS = ("bg", "bg_inset", "panel", "border", "text", "muted", "running", "ok", "warn", "fail", "ckpt")
61
+
62
+
63
+ # The document itself. CSS uses `%%token%%` placeholders (replaced in render());
64
+ # the JS reads colors from the injected `T` object. Nothing here reaches
65
+ # off-host — every href is same-origin (`/docs/`), and there are no src= assets.
66
+ _TEMPLATE = r"""<!doctype html>
67
+ <html lang="en">
68
+ <head>
69
+ <meta charset="utf-8">
70
+ <meta name="viewport" content="width=device-width, initial-scale=1">
71
+ <title>flashruntime — live run</title>
72
+ <style>
73
+ :root { color-scheme: dark; }
74
+ * { box-sizing: border-box; margin: 0; }
75
+ body { font: 13px/1.5 %%font%%; background: %%bg%%; color: %%text%%; padding: 20px; }
76
+ a { color: %%running%%; text-decoration: none; }
77
+ a:hover { text-decoration: underline; }
78
+
79
+ /* header ------------------------------------------------------------- */
80
+ header { display: flex; align-items: baseline; gap: 16px; flex-wrap: wrap;
81
+ border-bottom: 1px solid %%border%%; padding-bottom: 14px; }
82
+ header .title { font-size: 15px; color: %%text_bright%%; }
83
+ header .cmd { color: %%muted%%; word-break: break-all; }
84
+ header .spacer { flex: 1 1 auto; }
85
+ .meta { display: flex; gap: 18px; align-items: center; margin-top: 4px;
86
+ flex-wrap: wrap; color: %%muted%%; }
87
+ .meta b { color: %%text%%; font-weight: 600; }
88
+ .badge { display: inline-block; padding: 2px 10px; border-radius: 999px;
89
+ border: 1px solid %%border%%; font-size: 12px; }
90
+
91
+ /* section scaffolding ------------------------------------------------ */
92
+ section { margin-top: 22px; }
93
+ h2 { font-size: 11px; text-transform: uppercase; letter-spacing: .1em;
94
+ color: %%muted%%; margin-bottom: 8px; font-weight: 600; }
95
+ .panel { background: %%panel%%; border: 1px solid %%border%%; border-radius: 8px;
96
+ padding: 12px; }
97
+ canvas { display: block; width: 100%; background: %%bg_inset%%; border-radius: 6px; }
98
+ #loss { height: 200px; }
99
+ #resources { height: 160px; }
100
+ .charts { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
101
+ @media (max-width: 900px) { .charts { grid-template-columns: 1fr; } }
102
+ %%FLOWMAP_CSS%%
103
+
104
+ /* checkpoint timeline ------------------------------------------------ */
105
+ #checkpoints { display: flex; gap: 10px; overflow-x: auto; padding: 6px 2px; }
106
+ #checkpoints .empty { color: %%muted%%; }
107
+ .ckpt { flex: 0 0 auto; border: 1px solid %%border%%; border-left: 3px solid %%ckpt%%;
108
+ border-radius: 6px; padding: 8px 10px; background: %%bg%%; min-width: 96px; }
109
+ .ckpt .step { color: %%ckpt%%; font-weight: 600; }
110
+ .ckpt .b { display: inline-block; margin-top: 4px; font-size: 11px;
111
+ padding: 1px 6px; border-radius: 4px; }
112
+ .ckpt .verified { color: %%ok%%; border: 1px solid %%ok%%; }
113
+ .ckpt .invalid { color: %%fail%%; border: 1px solid %%fail%%; }
114
+ .ckpt .latest { color: %%text_bright%%; }
115
+ .ckpt .sub { color: %%muted%%; font-size: 11px; margin-top: 2px; }
116
+
117
+ /* events feed -------------------------------------------------------- */
118
+ #events { max-height: 300px; overflow-y: auto; }
119
+ #events .empty { color: %%muted%%; }
120
+ .ev { display: flex; gap: 8px; padding: 3px 0; border-bottom: 1px solid %%border%%; }
121
+ .ev:last-child { border-bottom: 0; }
122
+ .ev .t { color: %%muted%%; white-space: nowrap; }
123
+ .ev .ty { white-space: nowrap; }
124
+ .ev .msg { color: %%text%%; word-break: break-word; }
125
+ .ty-classified { color: %%warn%%; }
126
+ .ty-recovery { color: %%running%%; }
127
+ .ty-other { color: %%muted%%; }
128
+
129
+ /* logs --------------------------------------------------------------- */
130
+ details summary { cursor: pointer; color: %%muted%%; }
131
+ pre#logbody { margin-top: 8px; padding: 10px; background: %%bg_inset%%;
132
+ border-radius: 6px; max-height: 320px; overflow: auto;
133
+ color: %%text%%; white-space: pre-wrap; word-break: break-word; }
134
+
135
+ #err { display: none; margin-top: 12px; padding: 10px; border-radius: 6px;
136
+ border: 1px solid %%fail%%; color: %%fail%%; }
137
+ </style>
138
+ </head>
139
+ <body>
140
+
141
+ <header>
142
+ <div>
143
+ <div class="title">flashruntime <span id="hstate" class="badge">…</span></div>
144
+ <div class="cmd" id="hcmd">connecting…</div>
145
+ <div class="meta">
146
+ <span>mode <b id="hmode">—</b></span>
147
+ <span>restarts <b id="hrestarts">—</b></span>
148
+ <span>attempts <b id="hattempts">—</b></span>
149
+ </div>
150
+ </div>
151
+ <div class="spacer"></div>
152
+ <div><a href="/docs/">Docs ↗</a></div>
153
+ </header>
154
+
155
+ <div id="err"></div>
156
+
157
+ <section>
158
+ <h2>Run</h2>
159
+ <div id="kpis"></div>
160
+ </section>
161
+
162
+ <section>
163
+ <h2>Process map</h2>
164
+ <div class="panel" style="position: relative;">
165
+ <div id="flowmap"></div>
166
+ <div id="detail" hidden></div>
167
+ </div>
168
+ </section>
169
+
170
+ <section class="charts">
171
+ <div>
172
+ <h2>Loss</h2>
173
+ <div class="panel"><canvas id="loss"></canvas></div>
174
+ </div>
175
+ <div>
176
+ <h2>Resources</h2>
177
+ <div class="panel"><canvas id="resources"></canvas></div>
178
+ </div>
179
+ </section>
180
+
181
+ <section>
182
+ <h2>Checkpoints</h2>
183
+ <div class="panel"><div id="checkpoints"><span class="empty">no checkpoints yet</span></div></div>
184
+ </section>
185
+
186
+ <section>
187
+ <h2>Events</h2>
188
+ <div class="panel"><div id="events"><span class="empty">no events yet</span></div></div>
189
+ </section>
190
+
191
+ <section>
192
+ <details>
193
+ <summary>Logs (tail)</summary>
194
+ <pre id="logbody">—</pre>
195
+ </details>
196
+ </section>
197
+
198
+ <script>
199
+ // The color tokens, shared with the CSS above and with the dashboard. Kept in
200
+ // one object so a canvas fillStyle and a CSS rule never drift apart.
201
+ const T = %%tokens_json%%;
202
+ const POLL_MS = 2000; // /api/state cadence — see poll()
203
+ let STATE = null; // latest snapshot; every draw reads from this
204
+
205
+ const $ = (id) => document.getElementById(id);
206
+ const esc = (s) => String(s == null ? "" : s).replace(/[&<>"]/g, (c) =>
207
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
208
+ const clockTime = (ts) => (typeof ts === "number" ? new Date(ts * 1000).toLocaleTimeString() : "");
209
+
210
+ %%FLOWMAP_JS%%
211
+
212
+ // Size a canvas for the device pixel ratio so lines are crisp on retina and
213
+ // the drawing coordinate system stays in CSS pixels. Returns {ctx, w, h}.
214
+ function fitCanvas(canvas) {
215
+ const dpr = window.devicePixelRatio || 1;
216
+ const rect = canvas.getBoundingClientRect();
217
+ const w = Math.max(1, Math.floor(rect.width));
218
+ const h = Math.max(1, Math.floor(rect.height));
219
+ if (canvas.width !== w * dpr || canvas.height !== h * dpr) {
220
+ canvas.width = w * dpr;
221
+ canvas.height = h * dpr;
222
+ }
223
+ const ctx = canvas.getContext("2d");
224
+ ctx.setTransform(dpr, 0, 0, dpr, 0, 0); // 1 unit == 1 CSS pixel
225
+ return { ctx, w, h };
226
+ }
227
+
228
+ // ---- header --------------------------------------------------------------
229
+ function renderHeader(s) {
230
+ const cmd = (s.workload && s.workload.command) || [];
231
+ $("hcmd").textContent = Array.isArray(cmd) ? cmd.join(" ") : String(cmd);
232
+ $("hmode").textContent = (s.workload && s.workload.mode) || "—";
233
+ const badge = $("hstate");
234
+ badge.textContent = s.state || "—";
235
+ badge.style.color = fmStateColor(s.state);
236
+ badge.style.borderColor = fmStateColor(s.state);
237
+ const attempts = s.attempts || [];
238
+ // "restarts used" = attempts whose id carries an -rN recovery suffix.
239
+ const used = attempts.filter((a) => /-r\d+$/.test(a.attempt_id || "")).length;
240
+ $("hrestarts").textContent = used + " / " + (s.max_restarts == null ? "?" : s.max_restarts);
241
+ $("hattempts").textContent = String(attempts.length);
242
+ }
243
+
244
+ function fontFamily() {
245
+ return "ui-monospace, SFMono-Regular, Menlo, monospace";
246
+ }
247
+
248
+ // ---- loss canvas ---------------------------------------------------------
249
+ // Pick the loss series out of the metrics tail (prefer a "loss" key, else the
250
+ // first numeric field that isn't a step counter), merge across attempts, and
251
+ // draw an autoscaled curve with the last value labeled.
252
+ function lossKey(rec) {
253
+ if (typeof rec.loss === "number") return "loss";
254
+ for (const k of Object.keys(rec)) {
255
+ if (k === "step" || k === "epoch" || k === "index") continue;
256
+ if (typeof rec[k] === "number" && isFinite(rec[k])) return k;
257
+ }
258
+ return null;
259
+ }
260
+
261
+ function collectLoss(s) {
262
+ const pts = [];
263
+ let key = "loss";
264
+ (s.attempts || []).forEach((a) => {
265
+ (a.metrics || []).forEach((m) => {
266
+ const k = lossKey(m);
267
+ if (k == null) return;
268
+ key = k;
269
+ const y = m[k];
270
+ if (typeof y !== "number" || !isFinite(y)) return;
271
+ const x = typeof m.step === "number" ? m.step : pts.length;
272
+ pts.push({ x, y });
273
+ });
274
+ });
275
+ pts.sort((p, q) => p.x - q.x);
276
+ return { pts, key };
277
+ }
278
+
279
+ function drawLoss() {
280
+ const canvas = $("loss");
281
+ const { ctx, w, h } = fitCanvas(canvas);
282
+ ctx.clearRect(0, 0, w, h);
283
+ if (!STATE || STATE.error) return;
284
+
285
+ const { pts, key } = collectLoss(STATE);
286
+ if (pts.length === 0) {
287
+ ctx.fillStyle = T.muted;
288
+ ctx.font = "11px " + fontFamily();
289
+ ctx.textAlign = "center";
290
+ ctx.fillText("no metrics yet", w / 2, h / 2);
291
+ return;
292
+ }
293
+
294
+ const padL = 8, padR = 64, padT = 14, padB = 18;
295
+ const plotW = w - padL - padR, plotH = h - padT - padB;
296
+ let minX = pts[0].x, maxX = pts[0].x, minY = pts[0].y, maxY = pts[0].y;
297
+ for (const p of pts) {
298
+ minX = Math.min(minX, p.x); maxX = Math.max(maxX, p.x);
299
+ minY = Math.min(minY, p.y); maxY = Math.max(maxY, p.y);
300
+ }
301
+ // pad the y-bounds so the curve never touches the frame; guard a flat series
302
+ const span = maxY - minY || Math.abs(maxY) || 1;
303
+ minY -= span * 0.08; maxY += span * 0.08;
304
+ const xRange = maxX - minX || 1;
305
+ const yRange = maxY - minY || 1;
306
+ const sx = (x) => padL + ((x - minX) / xRange) * plotW;
307
+ const sy = (y) => padT + (1 - (y - minY) / yRange) * plotH;
308
+
309
+ // baseline
310
+ ctx.strokeStyle = T.border;
311
+ ctx.lineWidth = 1;
312
+ ctx.beginPath();
313
+ ctx.moveTo(padL, padT + plotH);
314
+ ctx.lineTo(padL + plotW, padT + plotH);
315
+ ctx.stroke();
316
+
317
+ // the curve
318
+ ctx.strokeStyle = T.running;
319
+ ctx.lineWidth = 1.5;
320
+ ctx.beginPath();
321
+ pts.forEach((p, i) => (i ? ctx.lineTo(sx(p.x), sy(p.y)) : ctx.moveTo(sx(p.x), sy(p.y))));
322
+ ctx.stroke();
323
+
324
+ // last value marker + label
325
+ const last = pts[pts.length - 1];
326
+ ctx.fillStyle = T.running;
327
+ ctx.beginPath();
328
+ ctx.arc(sx(last.x), sy(last.y), 3, 0, 2 * Math.PI);
329
+ ctx.fill();
330
+ ctx.fillStyle = T.text;
331
+ ctx.font = "11px " + fontFamily();
332
+ ctx.textAlign = "left";
333
+ const label = key + " " + (Math.abs(last.y) < 1e-3 ? last.y.toExponential(2) : last.y.toFixed(4));
334
+ ctx.fillText(label, Math.min(sx(last.x) + 6, w - padR), sy(last.y) - 4);
335
+ }
336
+
337
+ // ---- checkpoint timeline -------------------------------------------------
338
+ // Each manifest is a violet marker: its step, a hash-verified / invalid badge
339
+ // (the RE-VERIFIED state from state.collect, not the manifest's own claim),
340
+ // and a note when it is the one recovery would restore (latest_valid).
341
+ function renderCheckpoints(s) {
342
+ const el = $("checkpoints");
343
+ const cks = s.checkpoints || [];
344
+ if (cks.length === 0) {
345
+ el.innerHTML = '<span class="empty">no checkpoints yet</span>';
346
+ return;
347
+ }
348
+ el.innerHTML = cks
349
+ .map((c) => {
350
+ const verified = c.validation === "hash_verified";
351
+ const badge = verified
352
+ ? '<span class="b verified">hash-verified</span>'
353
+ : '<span class="b invalid">invalid</span>';
354
+ const latest = c.latest_valid ? ' <span class="latest">★ latest</span>' : "";
355
+ const age = typeof c.age_s === "number" ? Math.round(c.age_s) + "s ago" : "";
356
+ return (
357
+ '<div class="ckpt"><div class="step">step ' + esc(c.step) + "</div>" +
358
+ badge + latest +
359
+ '<div class="sub">' + esc(c.parts) + " parts · " + esc(age) + "</div></div>"
360
+ );
361
+ })
362
+ .join("");
363
+ }
364
+
365
+ // ---- events feed ---------------------------------------------------------
366
+ // Newest first. Recovery decisions are highlighted per the contract:
367
+ // FAILURE_CLASSIFIED amber, RECOVERY_ACTION_SELECTED cyan — the message
368
+ // already carries the failure class and the policy's human reason.
369
+ function eventClass(type) {
370
+ if (type === "FAILURE_CLASSIFIED") return "ty-classified";
371
+ if (type === "RECOVERY_ACTION_SELECTED") return "ty-recovery";
372
+ return "ty-other";
373
+ }
374
+
375
+ function renderEvents(s) {
376
+ const el = $("events");
377
+ const evs = s.events || [];
378
+ if (evs.length === 0) {
379
+ el.innerHTML = '<span class="empty">no events yet</span>';
380
+ return;
381
+ }
382
+ el.innerHTML = evs
383
+ .slice()
384
+ .reverse()
385
+ .map((e) =>
386
+ '<div class="ev"><span class="t">' + esc(clockTime(e.ts)) + "</span>" +
387
+ '<span class="ty ' + eventClass(e.type) + '">' + esc(e.type) + "</span>" +
388
+ '<span class="msg">' + esc(e.message) + "</span></div>"
389
+ )
390
+ .join("");
391
+ }
392
+
393
+ // ---- logs tail -----------------------------------------------------------
394
+ function renderLogs(s) {
395
+ const attempts = s.attempts || [];
396
+ const chunks = attempts
397
+ .filter((a) => a.log_tail)
398
+ .map((a) => "--- " + (a.attempt_id || "") + " (" + (a.state || "") + ") ---\n" + a.log_tail);
399
+ $("logbody").textContent = chunks.length ? chunks.join("\n\n") : "—";
400
+ }
401
+
402
+ // ---- KPI tiles + steps/s derivation ---------------------------------------
403
+ // steps/s comes from consecutive rank heartbeats (ts + step) observed across
404
+ // polls — the only honest rate source; without heartbeats the tile shows "—".
405
+ let prevBeat = null; // {ts, step} from the previous poll
406
+ let stepsPerSec = null; // last derived rate
407
+
408
+ function fmtDur(sec) {
409
+ if (typeof sec !== "number" || !isFinite(sec) || sec < 0) return "—";
410
+ const m = Math.floor(sec / 60), r = Math.floor(sec % 60), h = Math.floor(m / 60);
411
+ return h ? h + "h " + (m % 60) + "m" : m + ":" + String(r).padStart(2, "0");
412
+ }
413
+
414
+ function newestBeat(s) {
415
+ let best = null;
416
+ (s.attempts || []).forEach((a) => (a.ranks || []).forEach((r) => {
417
+ if (typeof r.step === "number" && typeof r.ts === "number" &&
418
+ (best == null || r.ts > best.ts)) best = r;
419
+ }));
420
+ return best;
421
+ }
422
+
423
+ function updateRate(s) {
424
+ const beat = newestBeat(s);
425
+ if (!beat) return;
426
+ if (prevBeat && beat.ts > prevBeat.ts && beat.step >= prevBeat.step) {
427
+ const rate = (beat.step - prevBeat.step) / (beat.ts - prevBeat.ts);
428
+ if (isFinite(rate) && rate > 0) stepsPerSec = rate;
429
+ }
430
+ prevBeat = { ts: beat.ts, step: beat.step };
431
+ }
432
+
433
+ function buildKpiTiles(s) {
434
+ const m = (s.monitor && s.monitor.machine) || null;
435
+ const beat = newestBeat(s);
436
+ const attempts = s.attempts || [];
437
+ const used = attempts.filter((a) => /-r\d+$/.test(a.attempt_id || "")).length;
438
+ const verified = (s.checkpoints || []).filter((c) => c.validation === "hash_verified").length;
439
+ const end = s.finished_at || Date.now() / 1000;
440
+ const hint = m && m.limited ? "flashruntime[monitor]" : undefined;
441
+ // A derived rate is only honest while it is still current: once the run
442
+ // stalls or finishes, the last-computed stepsPerSec is a fabricated
443
+ // "current" number if we keep showing it. "Live" = RUNNING and the
444
+ // newest heartbeat is recent (~3 poll periods, POLL_MS=2000).
445
+ const rateLive = s.state === "RUNNING" && beat && (Date.now() / 1000 - beat.ts) < 6;
446
+ const tiles = [
447
+ { label: "state", value: s.state || "—", color: fmStateColor(s.state) },
448
+ { label: "elapsed", value: fmtDur(end - s.started_at) },
449
+ { label: "step", value: beat && beat.step != null ? String(beat.step) : "—" },
450
+ { label: "steps/s", value: rateLive && stepsPerSec != null ? stepsPerSec.toFixed(2) : "—" },
451
+ { label: "step latency", value: rateLive && stepsPerSec ? Math.round(1000 / stepsPerSec) + " ms" : "—" },
452
+ { label: "cpu", value: m && m.cpu_percent != null ? Math.round(m.cpu_percent) + "%" : "—", hint: hint },
453
+ { label: "memory", value: m && m.mem_used != null ? fmFmtBytes(m.mem_used) : "—", hint: hint },
454
+ { label: "restarts", value: used + " / " + (s.max_restarts == null ? "?" : s.max_restarts) },
455
+ { label: "ckpts ✓", value: String(verified) },
456
+ ];
457
+ if (m && m.gpus && m.gpus.length) {
458
+ tiles.splice(7, 0, { label: "gpu", value: Math.round(m.gpus[0].util_percent) + "%" });
459
+ }
460
+ return tiles;
461
+ }
462
+
463
+ // ---- resource chart ---------------------------------------------------------
464
+ // cpu% (cyan) + memory% (violet) over the telemetry tail, 0–100 fixed axis.
465
+ function collectResources(s) {
466
+ const samples = [];
467
+ (s.attempts || []).forEach((a) => (a.telemetry || []).forEach((t) => {
468
+ if (t && t.machine && typeof t.ts === "number") samples.push(t);
469
+ }));
470
+ samples.sort((p, q) => p.ts - q.ts);
471
+ return samples.slice(-150);
472
+ }
473
+
474
+ function drawResources() {
475
+ const canvas = $("resources");
476
+ const { ctx, w, h } = fitCanvas(canvas);
477
+ ctx.clearRect(0, 0, w, h);
478
+ if (!STATE || STATE.error) return;
479
+ const samples = collectResources(STATE);
480
+ const usable = samples.filter((t) => t.machine.cpu_percent != null);
481
+ if (usable.length < 2) {
482
+ ctx.fillStyle = T.muted;
483
+ ctx.font = "11px " + fontFamily();
484
+ ctx.textAlign = "center";
485
+ const msg = samples.length
486
+ ? 'cpu/mem need psutil — pip install "flashruntime[monitor]"'
487
+ : "no telemetry yet";
488
+ ctx.fillText(msg, w / 2, h / 2);
489
+ return;
490
+ }
491
+ const padL = 8, padR = 56, padT = 10, padB = 14;
492
+ const plotW = w - padL - padR, plotH = h - padT - padB;
493
+ const t0 = usable[0].ts, t1 = usable[usable.length - 1].ts;
494
+ const sx = (ts) => padL + ((ts - t0) / (t1 - t0 || 1)) * plotW;
495
+ const sy = (pct) => padT + (1 - Math.min(100, Math.max(0, pct)) / 100) * plotH;
496
+ ctx.strokeStyle = T.border;
497
+ ctx.lineWidth = 1;
498
+ ctx.beginPath();
499
+ ctx.moveTo(padL, padT + plotH);
500
+ ctx.lineTo(padL + plotW, padT + plotH);
501
+ ctx.stroke();
502
+ const series = (pick, color) => {
503
+ ctx.strokeStyle = color;
504
+ ctx.lineWidth = 1.5;
505
+ ctx.beginPath();
506
+ let started = false;
507
+ usable.forEach((t) => {
508
+ const v = pick(t);
509
+ if (typeof v !== "number") return;
510
+ started ? ctx.lineTo(sx(t.ts), sy(v)) : ctx.moveTo(sx(t.ts), sy(v));
511
+ started = true;
512
+ });
513
+ ctx.stroke();
514
+ };
515
+ const memPct = (t) => t.machine.mem_total
516
+ ? 100 * t.machine.mem_used / t.machine.mem_total : null;
517
+ series((t) => t.machine.cpu_percent, T.running);
518
+ series(memPct, T.ckpt);
519
+ const last = usable[usable.length - 1];
520
+ ctx.font = "11px " + fontFamily();
521
+ ctx.textAlign = "left";
522
+ ctx.fillStyle = T.running;
523
+ ctx.fillText("cpu " + Math.round(last.machine.cpu_percent) + "%",
524
+ padL + plotW + 4, sy(last.machine.cpu_percent));
525
+ const mp = memPct(last);
526
+ if (mp != null) {
527
+ ctx.fillStyle = T.ckpt;
528
+ ctx.fillText("mem " + Math.round(mp) + "%", padL + plotW + 4, sy(mp));
529
+ }
530
+ }
531
+
532
+ // ---- everything that redraws on a poll ------------------------------------
533
+ function renderStatic() {
534
+ const s = STATE;
535
+ const err = $("err");
536
+ if (!s) return;
537
+ if (s.error) {
538
+ err.style.display = "block";
539
+ err.textContent = "snapshot error: " + s.error;
540
+ return;
541
+ }
542
+ err.style.display = "none";
543
+ renderHeader(s);
544
+ updateRate(s);
545
+ renderKpiTiles(buildKpiTiles(s));
546
+ renderFlowmap(s);
547
+ renderDetail(s);
548
+ drawLoss();
549
+ drawResources();
550
+ renderCheckpoints(s);
551
+ renderEvents(s);
552
+ renderLogs(s);
553
+ }
554
+
555
+ // ---- polling -------------------------------------------------------------
556
+ async function poll() {
557
+ // Pause polling when the tab is hidden: this page can sit open for a whole
558
+ // training run, and there is no reason to spend a laptop's battery fetching
559
+ // a snapshot nobody is looking at. A visibilitychange handler resumes it.
560
+ if (document.hidden) return;
561
+ try {
562
+ const r = await fetch("/api/state", { cache: "no-store" });
563
+ STATE = await r.json();
564
+ } catch (e) {
565
+ STATE = { error: String(e) };
566
+ }
567
+ renderStatic();
568
+ }
569
+
570
+ window.addEventListener("resize", () => renderStatic());
571
+ document.addEventListener("visibilitychange", () => { if (!document.hidden) poll(); });
572
+
573
+ poll();
574
+ setInterval(poll, POLL_MS);
575
+ </script>
576
+ </body>
577
+ </html>
578
+ """
579
+
580
+
581
+ def render() -> str:
582
+ """Return the live run page as one self-contained HTML string.
583
+
584
+ The flow-map component (viewer/flowmap.py) is spliced in first, THEN the
585
+ `%%token%%` placeholders are substituted from `TOKENS` — so the component's
586
+ CSS resolves against the same single-source-of-truth palette. No other
587
+ templating — the page is otherwise a static document.
588
+ """
589
+ html = _TEMPLATE.replace("%%FLOWMAP_CSS%%", FLOWMAP_CSS).replace("%%FLOWMAP_JS%%", FLOWMAP_JS)
590
+ subs = {f"%%{name}%%": value for name, value in TOKENS.items()}
591
+ subs["%%tokens_json%%"] = json.dumps({k: TOKENS[k] for k in _JS_TOKEN_KEYS})
592
+ for placeholder, value in subs.items():
593
+ html = html.replace(placeholder, value)
594
+ return html