flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. flashml_workloads/__init__.py +7 -0
  2. flashml_workloads/fedavg_driver.py +569 -0
  3. flashml_workloads/fedavg_weights.py +223 -0
  4. flashml_workloads/fedavg_worker.py +166 -0
  5. flashml_workloads/kmeans_driver.py +134 -0
  6. flashml_workloads/kmeans_shard.py +69 -0
  7. flashml_workloads/sgd_trainer.py +127 -0
  8. flashml_workloads/sharded_kmeans.py +323 -0
  9. flashml_workloads/sklearn_trial.py +89 -0
  10. flashruntime/__init__.py +125 -0
  11. flashruntime/artifacts/__init__.py +25 -0
  12. flashruntime/artifacts/store.py +228 -0
  13. flashruntime/backends/__init__.py +26 -0
  14. flashruntime/backends/base.py +63 -0
  15. flashruntime/backends/kuberay.py +465 -0
  16. flashruntime/checkpoint/__init__.py +20 -0
  17. flashruntime/checkpoint/catalog.py +198 -0
  18. flashruntime/checkpoint/local.py +109 -0
  19. flashruntime/checkpoint/store.py +86 -0
  20. flashruntime/integrations/__init__.py +5 -0
  21. flashruntime/integrations/huggingface.py +59 -0
  22. flashruntime/integrations/pytorch.py +52 -0
  23. flashruntime/integrations/sklearn.py +42 -0
  24. flashruntime/launchers/__init__.py +130 -0
  25. flashruntime/launchers/local.py +126 -0
  26. flashruntime/leases/__init__.py +27 -0
  27. flashruntime/leases/manager.py +365 -0
  28. flashruntime/leases/sqlite_store.py +169 -0
  29. flashruntime/leases/store.py +103 -0
  30. flashruntime/monitor/__init__.py +7 -0
  31. flashruntime/monitor/sampler.py +232 -0
  32. flashruntime/planner/__init__.py +56 -0
  33. flashruntime/planner/candidates.py +597 -0
  34. flashruntime/planner/catalog.py +129 -0
  35. flashruntime/planner/comm.py +95 -0
  36. flashruntime/planner/explain.py +109 -0
  37. flashruntime/planner/memory.py +166 -0
  38. flashruntime/planner/resolve.py +120 -0
  39. flashruntime/planner/selector.py +169 -0
  40. flashruntime/planner/timecost.py +81 -0
  41. flashruntime/profiling/__init__.py +113 -0
  42. flashruntime/protocol/__init__.py +18 -0
  43. flashruntime/protocol/plan_v1alpha1.py +320 -0
  44. flashruntime/protocol/v1alpha1.py +465 -0
  45. flashruntime/providers/__init__.py +138 -0
  46. flashruntime/py.typed +0 -0
  47. flashruntime/recipes/__init__.py +135 -0
  48. flashruntime/recipes/command.py +166 -0
  49. flashruntime/recovery/__init__.py +21 -0
  50. flashruntime/recovery/policy.py +170 -0
  51. flashruntime/recovery/signals.py +135 -0
  52. flashruntime/recovery/taxonomy.py +91 -0
  53. flashruntime/scheduler/__init__.py +170 -0
  54. flashruntime/sdk.py +402 -0
  55. flashruntime/service/__init__.py +3 -0
  56. flashruntime/service/app.py +391 -0
  57. flashruntime/service/auth.py +180 -0
  58. flashruntime/service/checkpoints.py +90 -0
  59. flashruntime/service/cli.py +167 -0
  60. flashruntime/service/dashboard.py +193 -0
  61. flashruntime/service/ledger.py +101 -0
  62. flashruntime/service/modea.py +821 -0
  63. flashruntime/strategies/__init__.py +156 -0
  64. flashruntime/strategies/command.py +56 -0
  65. flashruntime/torch/__init__.py +274 -0
  66. flashruntime/viewer/__init__.py +20 -0
  67. flashruntime/viewer/_docs/benchmarks.html +771 -0
  68. flashruntime/viewer/_docs/concepts/architecture.html +302 -0
  69. flashruntime/viewer/_docs/get-started.html +263 -0
  70. flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
  71. flashruntime/viewer/_docs/guides/huggingface.html +223 -0
  72. flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
  73. flashruntime/viewer/_docs/guides/pytorch.html +313 -0
  74. flashruntime/viewer/_docs/guides/sklearn.html +232 -0
  75. flashruntime/viewer/_docs/index.html +251 -0
  76. flashruntime/viewer/_docs/reference/cli.html +254 -0
  77. flashruntime/viewer/_docs/reference/integrations.html +240 -0
  78. flashruntime/viewer/_docs/reference/sdk.html +341 -0
  79. flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
  80. flashruntime/viewer/_docs/search-index.json +1 -0
  81. flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
  82. flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
  83. flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
  84. flashruntime/viewer/flowmap.py +307 -0
  85. flashruntime/viewer/page.py +594 -0
  86. flashruntime/viewer/server.py +134 -0
  87. flashruntime/viewer/state.py +250 -0
  88. flashruntime/workloads/__init__.py +6 -0
  89. flashruntime/workloads/command.py +127 -0
  90. flashruntime-0.3.0.dist-info/METADATA +365 -0
  91. flashruntime-0.3.0.dist-info/RECORD +95 -0
  92. flashruntime-0.3.0.dist-info/WHEEL +5 -0
  93. flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
  94. flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
  95. flashruntime-0.3.0.dist-info/top_level.txt +2 -0
flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html ADDED
@@ -0,0 +1,278 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>Tutorial: parallel scikit-learn sweeps — flashruntime docs</title>
7
+ <style>
8
+ :root { color-scheme: dark; }
9
+ * { box-sizing: border-box; margin: 0; }
10
+ body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
11
+ a { color: oklch(0.80 0.16 200); text-decoration: none; }
12
+ a:hover { text-decoration: underline; }
13
+ .layout { display: flex; align-items: flex-start; min-height: 100vh; }
14
+
15
+ /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
16
+ .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
17
+ border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
18
+ .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
19
+ .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
20
+ text-transform: uppercase; letter-spacing: .14em; }
21
+ .search { position: relative; margin: 16px 0; }
22
+ #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
23
+ color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
24
+ #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
25
+ #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
26
+ background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
27
+ #results.open { display: block; }
28
+ #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
29
+ #results a:last-child { border-bottom: 0; }
30
+ #results a:hover { background: #010409; text-decoration: none; }
31
+ #results a b { color: #e6edf3; font-weight: 600; }
32
+ #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
33
+ #results .nohit { padding: 7px 9px; color: #8b949e; }
34
+ .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
35
+ text-transform: uppercase; letter-spacing: .14em; }
36
+ .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
37
+ .nav-link:hover { background: #161b22; text-decoration: none; }
38
+ .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
39
+
40
+ /* content */
41
+ .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
42
+ .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
43
+ .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
44
+ padding-top: 10px; border-top: 1px solid #21262d; }
45
+ .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
46
+ .content p, .content li { color: #c9d1d9; }
47
+ .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
48
+ .content li { margin: 4px 0; }
49
+ .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
50
+ background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
51
+ .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
52
+ border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
53
+ .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
54
+ .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
55
+ .content th { color: #e6edf3; background: #161b22; }
56
+
57
+ /* code blocks + copy button */
58
+ .codeblock { position: relative; margin: 14px 0; }
59
+ .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
60
+ padding: 14px 16px; overflow-x: auto; }
61
+ .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
62
+ .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
63
+ background: #161b22; border: 1px solid #21262d; border-radius: 5px;
64
+ color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
65
+ .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
66
+
67
+ @media (max-width: 800px) {
68
+ .layout { flex-direction: column; }
69
+ .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
70
+ border-right: 0; border-bottom: 1px solid #21262d; }
71
+ .content { padding: 24px 18px 60px; }
72
+ }
73
+ </style>
74
+ </head>
75
+ <body>
76
+ <div class="layout">
77
+ <aside class="sidebar">
78
+ <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
79
+ <div class="search">
80
+ <input id="q" type="search" placeholder="Search docs (press /)" autocomplete="off" spellcheck="false">
81
+ <div id="results"></div>
82
+ </div>
83
+ <nav><div class="nav-section">Getting started</div>
84
+ <a class="nav-link" href="../index.html">FlashRuntime</a>
85
+ <a class="nav-link" href="../get-started.html">Get started</a>
86
+ <div class="nav-section">Tutorials</div>
87
+ <a class="nav-link" href="convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
88
+ <a class="nav-link active" href="sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
89
+ <a class="nav-link" href="fault-tolerance.html">Tutorial: automatic recovery, explained</a>
90
+ <div class="nav-section">Guides</div>
91
+ <a class="nav-link" href="../guides/pytorch.html">PyTorch guide</a>
92
+ <a class="nav-link" href="../guides/sklearn.html">scikit-learn guide</a>
93
+ <a class="nav-link" href="../guides/huggingface.html">Hugging Face guide</a>
94
+ <a class="nav-link" href="../guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
95
+ <a class="nav-link" href="../guides/federated-averaging.html">Federated averaging</a>
96
+ <div class="nav-section">Concepts</div>
97
+ <a class="nav-link" href="../concepts/architecture.html">Architecture</a>
98
+ <div class="nav-section">Reference</div>
99
+ <a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
100
+ <a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
101
+ <a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
102
+ <a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
103
+ <div class="nav-section">Benchmarks</div>
104
+ <a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
105
+ </aside>
106
+ <main class="content"><h1 id="tutorial-parallel-scikit-learn-sweeps">Tutorial: parallel scikit-learn sweeps</h1>
107
+ <p>scikit-learn work is <em>embarrassingly parallel across runs</em> — a grid of
108
+ independent fits, never a single <code>.fit()</code> you split internally. FlashRuntime
109
+ fans a grid out into one independent task per trial, runs them, and ranks the
110
+ results, while your script stays plain sklearn with no FlashRuntime import.</p>
111
+ <p>This tutorial builds a sweep from an ordinary script. You need
112
+ <code>pip install "flashruntime[sklearn]"</code> (numpy + scikit-learn) — see
113
+ <a href="../get-started.html">Get started</a>.</p>
114
+ <hr />
115
+ <h2 id="1-a-plain-sklearn-script-flags-in-metricsjson-out">1. A plain sklearn script (flags in, <code>metrics.json</code> out)</h2>
116
+ <p>The only contract FlashRuntime asks: read hyperparameters from CLI flags, write
117
+ a flat <code>metrics.json</code>. No FlashRuntime import anywhere.</p>
118
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import argparse
119
+ import json
120
+
121
+
122
+ def main():
123
+ parser = argparse.ArgumentParser()
124
+ parser.add_argument(&quot;--model&quot;, default=&quot;logreg&quot;)
125
+ parser.add_argument(&quot;--C&quot;, type=float, default=1.0)
126
+ parser.add_argument(&quot;--n_estimators&quot;, type=int, default=50)
127
+ args = parser.parse_args()
128
+
129
+ from sklearn.datasets import make_classification
130
+ from sklearn.ensemble import RandomForestClassifier
131
+ from sklearn.linear_model import LogisticRegression
132
+ from sklearn.model_selection import cross_val_score
133
+
134
+ X, y = make_classification(n_samples=600, n_features=12, random_state=0)
135
+ if args.model == &quot;logreg&quot;:
136
+ estimator = LogisticRegression(C=args.C, max_iter=500, random_state=0)
137
+ elif args.model == &quot;rf&quot;:
138
+ estimator = RandomForestClassifier(n_estimators=args.n_estimators, random_state=0)
139
+ else:
140
+ raise SystemExit(f&quot;unknown model {args.model!r} (logreg|rf)&quot;)
141
+
142
+ scores = cross_val_score(estimator, X, y, cv=3)
143
+ metrics = {
144
+ &quot;model&quot;: args.model,
145
+ &quot;C&quot;: args.C,
146
+ &quot;n_estimators&quot;: args.n_estimators,
147
+ &quot;accuracy_mean&quot;: round(float(scores.mean()), 4),
148
+ }
149
+ with open(&quot;metrics.json&quot;, &quot;w&quot;) as f:
150
+ json.dump(metrics, f, indent=2)
151
+ print(metrics)
152
+
153
+
154
+ if __name__ == &quot;__main__&quot;:
155
+ main()
156
+ </code></pre></div>
157
+ <p>Run one trial by hand to confirm it works:</p>
158
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-bash">python train.py --model rf --n_estimators 100
159
+ </code></pre></div>
160
+ <hr />
161
+ <h2 id="2-fan-a-grid-out-with-fr_sklearnhpo">2. Fan a grid out with <code>fr_sklearn.hpo(...)</code></h2>
162
+ <p>The <code>integrations.sklearn</code> adapter builds the workload from that script. Give
163
+ it a grid; it expands the Cartesian product into one task per trial:</p>
164
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
165
+ from flashruntime.integrations import sklearn as fr_sklearn
166
+
167
+ run = flash.submit(fr_sklearn.hpo(
168
+ &quot;train.py&quot;,
169
+ {&quot;model&quot;: [&quot;logreg&quot;, &quot;rf&quot;], &quot;C&quot;: [0.1, 1.0], &quot;n_estimators&quot;: [50]},
170
+ source=&quot;.&quot;,
171
+ ))
172
+ print(f&quot;state={run.state.value} trials={len(run.trials)}&quot;)
173
+ print(&quot;best:&quot;, run.best_trial())
174
+ </code></pre></div>
175
+ <p><code>{"model": ["logreg", "rf"], "C": [0.1, 1.0], "n_estimators": [50]}</code> expands to
176
+ <code>2 × 2 × 1 = 4</code> trials. Each <code>{placeholder}</code> in the built command is filled
177
+ from the trial's params, so one task receives <code>--model rf --C 1.0
178
+ --n_estimators 50</code> and so on. Every trial's <code>metrics.json</code> is collected and
179
+ recorded on <code>run.trials</code>, with its params merged in.</p>
180
+ <hr />
181
+ <h2 id="3-read-the-winner">3. Read the winner</h2>
182
+ <p>Because <code>hpo</code> (via <code>sweep</code>) sets <code>outputs.primary_metric="accuracy_mean"</code>,
183
+ <code>run.best_trial()</code> needs no arguments — it returns the trial with the highest
184
+ <code>accuracy_mean</code>:</p>
185
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">best = run.best_trial() # ranks by accuracy_mean (maximize=True)
186
+ worst = run.best_trial(maximize=False) # or flip it
187
+ by_other = run.best_trial(metric=&quot;C&quot;) # or rank by any reported key
188
+ </code></pre></div>
189
+ <p><code>best_trial(metric=None, maximize=None)</code> falls back to the <code>OutputSpec</code>
190
+ defaults the adapter set; pass <code>metric=</code> / <code>maximize=</code> to override. It returns
191
+ <code>None</code> if no trial reported the metric.</p>
192
+ <hr />
193
+ <h2 id="how-the-fan-out-stays-correct">How the fan-out stays correct</h2>
194
+ <ul>
195
+ <li><strong>Sequential and isolated.</strong> <code>flash.submit()</code> runs one trial at a time and
196
+ copies each trial's <code>metrics.json</code> out <strong>before</strong> the next trial can
197
+ overwrite it — so a trial's outputs are always its own.</li>
198
+ <li><strong>Independent checkpoint trees.</strong> Each trial gets its own job-scoped
199
+ checkpoint tree, so trials never cross-contaminate. (This matters for
200
+ checkpointed workloads; a pure sklearn fit has none.)</li>
201
+ <li><strong>Two API shapes.</strong> <code>hpo(script, grid, **kwargs)</code> is grid sugar over
202
+ <code>sweep(script, task_params, *, source=".", metric="accuracy_mean",
203
+ maximize=True, python="python")</code>. Use <code>sweep</code> directly to pass an explicit
204
+ list of param dicts (e.g. a hand-picked, non-Cartesian set).</li>
205
+ </ul>
206
+ <p>Add a restart budget the same way as any run: <code>flash.submit(..., max_restarts=1)</code>
207
+ retries a <em>transient</em> trial failure and fails fast on a deterministic one — see
208
+ the <a href="fault-tolerance.html">fault-tolerance tutorial</a>.</p>
209
+ <hr />
210
+ <h2 id="where-to-go-next">Where to go next</h2>
211
+ <ul>
212
+ <li><strong><a href="../guides/sklearn.html">scikit-learn guide</a></strong> — the adapter reference and
213
+ the "distribute across runs, never inside <code>.fit()</code>" rule.</li>
214
+ <li><strong><a href="convnet.html">ConvNet tutorial</a></strong> — the PyTorch DDP + checkpoint-resume
215
+ story.</li>
216
+ <li><strong><a href="../reference/sdk.html">SDK reference</a></strong> — every <code>Run</code> attribute and
217
+ <code>submit()</code> argument.</li>
218
+ </ul></main>
219
+ </div>
220
+
221
+ <script>
222
+ // ---- client-side search: fetch the builder's index, filter as you type -----
223
+ // (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
224
+ let INDEX = [];
225
+ // Path from THIS page back to the site root. The index stores root-relative
226
+ // URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
227
+ // against ROOT so a page in a subdir resolves them too (see build_docs.py).
228
+ const ROOT = "../";
229
+ const q = document.getElementById("q");
230
+ const results = document.getElementById("results");
231
+ const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
232
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
233
+ fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
234
+
235
+ // A short context window around the first match, so a hit shows WHY it matched.
236
+ function snippet(text, needle) {
237
+ const i = text.toLowerCase().indexOf(needle);
238
+ if (i < 0) return "";
239
+ const start = Math.max(0, i - 32);
240
+ return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
241
+ }
242
+ function runSearch() {
243
+ const needle = q.value.trim().toLowerCase();
244
+ if (!needle) { results.className = ""; results.innerHTML = ""; return; }
245
+ const hits = INDEX.map((p) => {
246
+ const inTitle = p.title.toLowerCase().includes(needle);
247
+ const inText = p.text.toLowerCase().includes(needle);
248
+ if (!inTitle && !inText) return null;
249
+ return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
250
+ }).filter(Boolean).slice(0, 20);
251
+ results.className = "open";
252
+ results.innerHTML = hits.length
253
+ ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
254
+ (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
255
+ : '<div class="nohit">no matches</div>';
256
+ }
257
+ q.addEventListener("input", runSearch);
258
+ q.addEventListener("focus", runSearch);
259
+ document.addEventListener("click", (e) => {
260
+ if (!e.target.closest(".search")) { results.className = ""; } // dismiss on outside click
261
+ });
262
+ document.addEventListener("keydown", (e) => { // "/" focuses search, like PyTorch docs
263
+ if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
264
+ });
265
+
266
+ // ---- copy buttons: read the <pre>'s own text into the clipboard ------------
267
+ document.querySelectorAll(".copybtn").forEach((btn) => {
268
+ btn.addEventListener("click", () => {
269
+ const code = btn.parentElement.querySelector("pre").innerText;
270
+ navigator.clipboard.writeText(code).then(() => {
271
+ const was = btn.textContent; btn.textContent = "Copied";
272
+ setTimeout(() => { btn.textContent = was; }, 1200);
273
+ }).catch(() => {});
274
+ });
275
+ });
276
+ </script>
277
+ </body>
278
+ </html>
flashruntime/viewer/flowmap.py ADDED
@@ -0,0 +1,307 @@
1
+ """The flow-map + KPI strip component, shared between the run viewer page
2
+ (`viewer/page.py`) and — in a later phase — the coordinator dashboard
3
+ (`service/dashboard.py`). Divergence between the two surfaces is a bug, so
4
+ the CSS and JS live here once, as strings the host page embeds inline.
5
+
6
+ Contract with the host page:
7
+ * CSS colors are `%%token%%` placeholders resolved by the host's render()
8
+ from `viewer.page.TOKENS` (the single source of truth).
9
+ * The JS expects a global `T` (the injected token object) and three
10
+ elements: `#kpis`, `#flowmap`, `#detail`.
11
+ * The host polls its own data and calls `renderKpiTiles(tiles)`,
12
+ `renderFlowmap(snapshot)`, `renderDetail(snapshot)` on each tick. The
13
+ component never fetches — data acquisition is the host's business.
14
+
15
+ The map draws machine → workers (attempts) → ranks as DOM nodes (free click
16
+ targets + text layout) with one SVG layer behind them for the edges. It is
17
+ rebuilt idempotently from each snapshot; `fmSelected` (the clicked node)
18
+ survives rebuilds and feeds the slide-in detail panel with live values.
19
+
20
+ Single-machine today, on purpose: the machine column renders exactly one
21
+ node — the local viewer's reality, since `flash.submit(watch=True)` only
22
+ ever observes the machine it is running on. Growing that to N machines for
23
+ the coordinator dashboard (phase 3) is a change to THIS component — extend
24
+ the machine column here so both surfaces gain it together, never fork a
25
+ per-host copy of the flow map. Before that adoption lands, the coordinator
26
+ dashboard also needs two host-page prerequisites it does not have yet: a
27
+ `T` token global (the injected color-token object this JS reads, see
28
+ `viewer/page.py`'s `%%tokens_json%%`) and a `%%token%%` substitution pass
29
+ over its CSS — `service/dashboard.py`'s current `_recolor` hex pipeline is a
30
+ different mechanism and must be reworked onto the same placeholder scheme
31
+ before it can host this component.
32
+ """
33
+
34
+ from __future__ import annotations
35
+
36
+ FLOWMAP_CSS = r"""
37
+ /* KPI strip -------------------------------------------------------------- */
38
+ #kpis { display: grid; grid-template-columns: repeat(auto-fit, minmax(110px, 1fr)); gap: 8px; }
39
+ .kpi { background: %%panel%%; border: 1px solid %%border%%; border-radius: 8px; padding: 8px 12px; }
40
+ .kpi-label { color: %%muted%%; font-size: 11px; text-transform: uppercase; letter-spacing: .06em; }
41
+ .kpi-value { color: %%text_bright%%; font-size: 18px; font-weight: 600; margin-top: 2px; }
42
+ .kpi-hint { color: %%muted%%; font-size: 10px; margin-top: 2px; }
43
+
44
+ /* flow map ---------------------------------------------------------------- */
45
+ #flowmap { position: relative; display: flex; gap: 48px; padding: 14px;
46
+ min-height: 180px; background: %%bg_inset%%; border-radius: 6px;
47
+ overflow-x: auto; }
48
+ #fm-edges { position: absolute; inset: 0; pointer-events: none; }
49
+ .fm-col { display: flex; flex-direction: column; gap: 10px; justify-content: center;
50
+ position: relative; z-index: 1; min-width: 180px; }
51
+ .fm-node { background: %%panel%%; border: 1px solid %%border%%; border-radius: 8px;
52
+ padding: 8px 10px; cursor: pointer; }
53
+ .fm-node:hover { filter: brightness(1.25); }
54
+ .fm-selected { outline: 2px solid %%running%%; outline-offset: 1px; }
55
+ .fm-machine { border-color: %%running%%; }
56
+ .fm-done { opacity: .55; }
57
+ .fm-live .fm-title { animation: fm-pulse 2s ease-in-out infinite; }
58
+ @keyframes fm-pulse { 50% { opacity: .55; } }
59
+ .fm-title { color: %%text_bright%%; }
60
+ .fm-sub { color: %%muted%%; font-size: 11px; margin-top: 2px; }
61
+ .fm-badge { color: %%warn%%; border: 1px solid %%warn%%; border-radius: 4px;
62
+ font-size: 10px; padding: 0 4px; }
63
+ .fm-more { color: %%muted%%; font-size: 11px; }
64
+
65
+ /* detail panel (slides in over the map's right edge) ---------------------- */
66
+ #detail { position: absolute; top: 8px; right: 8px; bottom: 8px; width: min(360px, 60%);
67
+ background: %%panel%%; border: 1px solid %%running%%; border-radius: 8px;
68
+ padding: 14px; overflow-y: auto; z-index: 2; }
69
+ #detail h3 { color: %%text_bright%%; font-size: 13px; margin-bottom: 10px;
70
+ text-transform: none; letter-spacing: 0; }
71
+ .fm-kv { display: flex; justify-content: space-between; gap: 12px; padding: 3px 0;
72
+ border-bottom: 1px solid %%border%%; }
73
+ .fm-kv span { color: %%muted%%; }
74
+ .fm-kv b { color: %%text%%; font-weight: 600; text-align: right; word-break: break-all; }
75
+ .fm-close { position: absolute; top: 8px; right: 8px; background: none;
76
+ border: 1px solid %%border%%; color: %%muted%%; border-radius: 6px;
77
+ cursor: pointer; padding: 2px 8px; font: inherit; }
78
+ .fm-close:hover { color: %%text_bright%%; }
79
+ .fm-log { margin-top: 10px; padding: 8px; background: %%bg_inset%%; border-radius: 6px;
80
+ white-space: pre-wrap; word-break: break-word; color: %%text%%;
81
+ font-size: 11px; max-height: 200px; overflow-y: auto; }
82
+ .fm-hint { margin-top: 10px; color: %%warn%%; font-size: 11px; }
83
+ """
84
+
85
+ FLOWMAP_JS = r"""
86
+ // ==== flow map + KPI components (viewer/flowmap.py — shared surface) =======
87
+ // Requires: global `T` (color tokens) and elements #kpis, #flowmap, #detail.
88
+ // The host page owns polling; these functions only render a given snapshot.
89
+
90
+ const fmEsc = (s) => String(s == null ? "" : s).replace(/[&<>"]/g, (c) =>
91
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
92
+
93
+ // Lifecycle state → accent. The shared vocabulary both surfaces use.
94
+ function fmStateColor(s) {
95
+ switch (s) {
96
+ case "RUNNING": return T.running; // cyan
97
+ case "LEASED": case "RECOVERING": return T.warn; // amber
98
+ case "SUCCEEDED": case "COMPLETED": return T.ok; // green
99
+ case "FAILED": return T.fail; // red
100
+ default: return T.muted; // PENDING / CANCELLED
101
+ }
102
+ }
103
+
104
+ function fmFmtBytes(n) {
105
+ if (typeof n !== "number" || !isFinite(n)) return "—";
106
+ const gb = n / (1024 ** 3);
107
+ return gb >= 1 ? gb.toFixed(1) + " GB" : Math.round(n / (1024 ** 2)) + " MB";
108
+ }
109
+
110
+ // ---- KPI strip -------------------------------------------------------------
111
+ // tiles: [{label, value, hint?, color?}]. A tile with no data source shows
112
+ // "—" (the host builds tiles that way) — the strip never invents a number.
113
+ function renderKpiTiles(tiles) {
114
+ const el = document.getElementById("kpis");
115
+ el.innerHTML = tiles.map((t) =>
116
+ '<div class="kpi"><div class="kpi-label">' + fmEsc(t.label) + "</div>" +
117
+ '<div class="kpi-value"' + (t.color ? ' style="color:' + fmEsc(t.color) + '"' : "") + ">" +
118
+ fmEsc(t.value) + "</div>" +
119
+ (t.hint ? '<div class="kpi-hint">' + fmEsc(t.hint) + "</div>" : "") +
120
+ "</div>"
121
+ ).join("");
122
+ }
123
+
124
+ // ---- flow map ---------------------------------------------------------------
125
+ // Selection survives re-renders: nodes are keyed {kind, id} and the map is
126
+ // rebuilt from scratch on every snapshot (idempotent — no DOM bookkeeping).
127
+ let fmSelected = null; // {kind: "machine"|"worker"|"rank", id: string} | null
128
+
129
+ // newest telemetry sample carried by one attempt row (or null)
130
+ function fmLastSample(a) {
131
+ const tel = a.telemetry || [];
132
+ return tel.length ? tel[tel.length - 1] : null;
133
+ }
134
+
135
+ function renderFlowmap(s) {
136
+ const el = document.getElementById("flowmap");
137
+ const attempts = s.attempts || [];
138
+ const machine = (s.monitor && s.monitor.machine) || null;
139
+
140
+ // machine column — one node (multi-machine arrives with the coordinator)
141
+ const mSel = fmSelected && fmSelected.kind === "machine";
142
+ const host = machine ? machine.hostname : "localhost";
143
+ const mSub = machine
144
+ ? (machine.cpu_count || "?") + " cores" +
145
+ (machine.cpu_percent != null ? " · cpu " + Math.round(machine.cpu_percent) + "%" : "") +
146
+ (machine.gpus && machine.gpus.length ? " · " + machine.gpus.length + " gpu" : "")
147
+ : "no telemetry yet";
148
+ let html = '<div class="fm-col">' +
149
+ '<div class="fm-node fm-machine' + (mSel ? " fm-selected" : "") +
150
+ '" data-kind="machine" data-id="machine">' +
151
+ '<div class="fm-title">▣ ' + fmEsc(host) + "</div>" +
152
+ '<div class="fm-sub">' + fmEsc(mSub) + "</div></div></div>";
153
+
154
+ // worker column — one node per attempt, newest kept (cap so a 100-trial
155
+ // sweep stays readable; the count of hidden ones is stated, not silent)
156
+ const shown = attempts.slice(-16);
157
+ html += '<div class="fm-col">' + (attempts.length > shown.length
158
+ ? '<div class="fm-more">… ' + (attempts.length - shown.length) + " earlier attempts</div>" : "");
159
+ for (const a of shown) {
160
+ const sel = fmSelected && fmSelected.kind === "worker" && fmSelected.id === a.attempt_id;
161
+ const color = fmStateColor(a.state);
162
+ const respawn = /-r\d+$/.test(a.attempt_id || "");
163
+ const mark = a.state === "RUNNING" ? "●" : a.state === "SUCCEEDED" ? "✓"
164
+ : a.state === "FAILED" ? "✗" : "○";
165
+ html += '<div class="fm-node fm-worker' + (sel ? " fm-selected" : "") +
166
+ (a.state === "RUNNING" ? " fm-live" : " fm-done") + '"' +
167
+ ' data-kind="worker" data-id="' + fmEsc(a.attempt_id) + '" style="border-color:' + color + '">' +
168
+ '<div class="fm-title" style="color:' + color + '">' + mark + " " + fmEsc(a.attempt_id) +
169
+ (respawn ? ' <span class="fm-badge">⟳ respawn</span>' : "") + "</div>" +
170
+ '<div class="fm-sub">pid ' + fmEsc(a.pid) + " · " + fmEsc(a.state) + "</div></div>";
171
+ }
172
+ html += "</div>";
173
+
174
+ // rank column — heartbeat ranks per attempt; for a live uninstrumented
175
+ // attempt, fall back to its process tree's children (pid-only nodes)
176
+ html += '<div class="fm-col">';
177
+ for (const a of shown) {
178
+ const ranks = a.ranks || [];
179
+ const sample = fmLastSample(a);
180
+ const procs = (sample && sample.processes) || [];
181
+ const byPid = {};
182
+ for (const p of procs) byPid[p.pid] = p;
183
+ if (ranks.length) {
184
+ for (const r of ranks) {
185
+ const rid = a.attempt_id + "/" + r.rank;
186
+ const sel = fmSelected && fmSelected.kind === "rank" && fmSelected.id === rid;
187
+ const proc = byPid[r.pid];
188
+ html += '<div class="fm-node fm-rank' + (sel ? " fm-selected" : "") +
189
+ (a.state === "RUNNING" ? " fm-live" : " fm-done") + '"' +
190
+ ' data-kind="rank" data-id="' + fmEsc(rid) + '" data-worker="' + fmEsc(a.attempt_id) + '">' +
191
+ '<div class="fm-title">rank ' + fmEsc(r.rank) + " · pid " + fmEsc(r.pid) + "</div>" +
192
+ '<div class="fm-sub">' + fmEsc(r.device || "?") +
193
+ (r.step != null ? " · step " + fmEsc(r.step) : "") +
194
+ (proc && proc.cpu_percent != null ? " · cpu " + Math.round(proc.cpu_percent) + "%" : "") +
195
+ "</div></div>";
196
+ }
197
+ } else if (a.state === "RUNNING" && procs.length > 1) {
198
+ for (const p of procs.slice(1, 9)) { // [0] is the launched root itself
199
+ const rid = a.attempt_id + "/pid-" + p.pid;
200
+ const sel = fmSelected && fmSelected.kind === "rank" && fmSelected.id === rid;
201
+ html += '<div class="fm-node fm-rank fm-live' + (sel ? " fm-selected" : "") + '"' +
202
+ ' data-kind="rank" data-id="' + fmEsc(rid) + '" data-worker="' + fmEsc(a.attempt_id) + '">' +
203
+ '<div class="fm-title">pid ' + fmEsc(p.pid) + "</div>" +
204
+ '<div class="fm-sub">' + fmEsc(p.cmd || "") +
205
+ (p.cpu_percent != null ? " · cpu " + Math.round(p.cpu_percent) + "%" : "") +
206
+ "</div></div>";
207
+ }
208
+ }
209
+ }
210
+ html += "</div>";
211
+
212
+ el.innerHTML = html + '<svg id="fm-edges"></svg>';
213
+
214
+ el.querySelectorAll(".fm-node").forEach((node) => {
215
+ node.addEventListener("click", () => {
216
+ fmSelected = { kind: node.dataset.kind, id: node.dataset.id };
217
+ renderFlowmap(s); // re-render for the selection outline
218
+ renderDetail(s);
219
+ });
220
+ });
221
+
222
+ fmDrawEdges(el);
223
+ }
224
+
225
+ // Bézier connectors machine→worker and worker→its ranks, drawn into the SVG
226
+ // layer AFTER the DOM has laid the nodes out (positions read back from
227
+ // getBoundingClientRect, so the lines are correct at any width).
228
+ function fmDrawEdges(el) {
229
+ const svg = el.querySelector("#fm-edges");
230
+ const box = el.getBoundingClientRect();
231
+ svg.setAttribute("width", box.width);
232
+ svg.setAttribute("height", box.height);
233
+ const anchor = (n, side) => {
234
+ const r = n.getBoundingClientRect();
235
+ return { x: (side === "r" ? r.right : r.left) - box.left,
236
+ y: r.top + r.height / 2 - box.top };
237
+ };
238
+ const curve = (a, b) =>
239
+ '<path d="M' + a.x + " " + a.y +
240
+ " C" + ((a.x + b.x) / 2) + " " + a.y + "," +
241
+ ((a.x + b.x) / 2) + " " + b.y + "," + b.x + " " + b.y +
242
+ '" stroke="' + T.border + '" fill="none" stroke-width="1"/>';
243
+ const machine = el.querySelector(".fm-machine");
244
+ let lines = "";
245
+ el.querySelectorAll(".fm-worker").forEach((w) => {
246
+ if (machine) lines += curve(anchor(machine, "r"), anchor(w, "l"));
247
+ el.querySelectorAll('.fm-rank[data-worker="' + CSS.escape(w.dataset.id) + '"]')
248
+ .forEach((r) => { lines += curve(anchor(w, "r"), anchor(r, "l")); });
249
+ });
250
+ svg.innerHTML = lines;
251
+ }
252
+
253
+ // ---- detail panel -----------------------------------------------------------
254
+ // Re-rendered on every snapshot while open, so its numbers stay live.
255
+ function renderDetail(s) {
256
+ const el = document.getElementById("detail");
257
+ if (!fmSelected) { el.hidden = true; return; }
258
+ const kv = (k, v) => '<div class="fm-kv"><span>' + fmEsc(k) + "</span><b>" + fmEsc(v) + "</b></div>";
259
+ let body = "";
260
+ if (fmSelected.kind === "machine") {
261
+ const m = (s.monitor && s.monitor.machine) || {};
262
+ body = "<h3>machine · " + fmEsc(m.hostname || "localhost") + "</h3>" +
263
+ kv("cores", m.cpu_count != null ? m.cpu_count : "—") +
264
+ kv("cpu", m.cpu_percent != null ? Math.round(m.cpu_percent) + "%" : "—") +
265
+ kv("memory", m.mem_used != null ? fmFmtBytes(m.mem_used) + " / " + fmFmtBytes(m.mem_total) : "—") +
266
+ kv("load avg", m.load_avg ? m.load_avg.map((x) => x.toFixed(2)).join(" ") : "—") +
267
+ ((m.gpus || []).map((g, i) => kv("gpu " + i, g.name + " · " + Math.round(g.util_percent) +
268
+ "% · " + Math.round(g.mem_used_mb) + "/" + Math.round(g.mem_total_mb) + " MB")).join("")) +
269
+ (m.limited ? '<div class="fm-hint">full stats: pip install "flashruntime[monitor]"</div>' : "");
270
+ } else {
271
+ const attemptId = fmSelected.kind === "worker" ? fmSelected.id : fmSelected.id.split("/")[0];
272
+ const a = (s.attempts || []).find((x) => x.attempt_id === attemptId);
273
+ if (!a) { el.hidden = true; return; }
274
+ if (fmSelected.kind === "worker") {
275
+ const dur = a.finished_at
276
+ ? (a.finished_at - a.started_at).toFixed(1) + "s"
277
+ : ((Date.now() / 1000) - a.started_at).toFixed(0) + "s so far";
278
+ body = "<h3>" + fmEsc(a.attempt_id) + "</h3>" +
279
+ kv("state", a.state) + kv("pid", a.pid) + kv("job", a.job_id) + kv("runtime", dur) +
280
+ '<div class="fm-log">' +
281
+ fmEsc((a.log_tail || "").split("\n").slice(-12).join("\n") || "no log yet") + "</div>";
282
+ } else {
283
+ const key = fmSelected.id.split("/")[1];
284
+ const sample = fmLastSample(a);
285
+ const procs = (sample && sample.processes) || [];
286
+ const isPid = key.startsWith("pid-");
287
+ const r = isPid ? null : (a.ranks || []).find((x) => String(x.rank) === key);
288
+ const pid = isPid ? Number(key.slice(4)) : r && r.pid;
289
+ const proc = procs.find((p) => p.pid === pid);
290
+ body = "<h3>" + (r ? "rank " + fmEsc(r.rank) : "pid " + fmEsc(pid)) + "</h3>" +
291
+ (r ? kv("device", r.device || "—") + kv("backend", r.backend || "—") +
292
+ kv("world size", r.world_size) + kv("step", r.step != null ? r.step : "—") : "") +
293
+ kv("pid", pid != null ? pid : "—") +
294
+ kv("cpu", proc && proc.cpu_percent != null ? Math.round(proc.cpu_percent) + "%" : "—") +
295
+ kv("memory", proc ? fmFmtBytes(proc.rss_bytes) : "—") +
296
+ kv("status", proc ? (proc.status || "—") : a.state);
297
+ }
298
+ }
299
+ el.innerHTML = '<button id="fm-close" class="fm-close">✕</button>' + body;
300
+ el.hidden = false;
301
+ el.querySelector("#fm-close").addEventListener("click", () => {
302
+ fmSelected = null;
303
+ el.hidden = true;
304
+ renderFlowmap(s);
305
+ });
306
+ }
307
+ """