flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. flashml_workloads/__init__.py +7 -0
  2. flashml_workloads/fedavg_driver.py +569 -0
  3. flashml_workloads/fedavg_weights.py +223 -0
  4. flashml_workloads/fedavg_worker.py +166 -0
  5. flashml_workloads/kmeans_driver.py +134 -0
  6. flashml_workloads/kmeans_shard.py +69 -0
  7. flashml_workloads/sgd_trainer.py +127 -0
  8. flashml_workloads/sharded_kmeans.py +323 -0
  9. flashml_workloads/sklearn_trial.py +89 -0
  10. flashruntime/__init__.py +125 -0
  11. flashruntime/artifacts/__init__.py +25 -0
  12. flashruntime/artifacts/store.py +228 -0
  13. flashruntime/backends/__init__.py +26 -0
  14. flashruntime/backends/base.py +63 -0
  15. flashruntime/backends/kuberay.py +465 -0
  16. flashruntime/checkpoint/__init__.py +20 -0
  17. flashruntime/checkpoint/catalog.py +198 -0
  18. flashruntime/checkpoint/local.py +109 -0
  19. flashruntime/checkpoint/store.py +86 -0
  20. flashruntime/integrations/__init__.py +5 -0
  21. flashruntime/integrations/huggingface.py +59 -0
  22. flashruntime/integrations/pytorch.py +52 -0
  23. flashruntime/integrations/sklearn.py +42 -0
  24. flashruntime/launchers/__init__.py +130 -0
  25. flashruntime/launchers/local.py +126 -0
  26. flashruntime/leases/__init__.py +27 -0
  27. flashruntime/leases/manager.py +365 -0
  28. flashruntime/leases/sqlite_store.py +169 -0
  29. flashruntime/leases/store.py +103 -0
  30. flashruntime/monitor/__init__.py +7 -0
  31. flashruntime/monitor/sampler.py +232 -0
  32. flashruntime/planner/__init__.py +56 -0
  33. flashruntime/planner/candidates.py +597 -0
  34. flashruntime/planner/catalog.py +129 -0
  35. flashruntime/planner/comm.py +95 -0
  36. flashruntime/planner/explain.py +109 -0
  37. flashruntime/planner/memory.py +166 -0
  38. flashruntime/planner/resolve.py +120 -0
  39. flashruntime/planner/selector.py +169 -0
  40. flashruntime/planner/timecost.py +81 -0
  41. flashruntime/profiling/__init__.py +113 -0
  42. flashruntime/protocol/__init__.py +18 -0
  43. flashruntime/protocol/plan_v1alpha1.py +320 -0
  44. flashruntime/protocol/v1alpha1.py +465 -0
  45. flashruntime/providers/__init__.py +138 -0
  46. flashruntime/py.typed +0 -0
  47. flashruntime/recipes/__init__.py +135 -0
  48. flashruntime/recipes/command.py +166 -0
  49. flashruntime/recovery/__init__.py +21 -0
  50. flashruntime/recovery/policy.py +170 -0
  51. flashruntime/recovery/signals.py +135 -0
  52. flashruntime/recovery/taxonomy.py +91 -0
  53. flashruntime/scheduler/__init__.py +170 -0
  54. flashruntime/sdk.py +402 -0
  55. flashruntime/service/__init__.py +3 -0
  56. flashruntime/service/app.py +391 -0
  57. flashruntime/service/auth.py +180 -0
  58. flashruntime/service/checkpoints.py +90 -0
  59. flashruntime/service/cli.py +167 -0
  60. flashruntime/service/dashboard.py +193 -0
  61. flashruntime/service/ledger.py +101 -0
  62. flashruntime/service/modea.py +821 -0
  63. flashruntime/strategies/__init__.py +156 -0
  64. flashruntime/strategies/command.py +56 -0
  65. flashruntime/torch/__init__.py +274 -0
  66. flashruntime/viewer/__init__.py +20 -0
  67. flashruntime/viewer/_docs/benchmarks.html +771 -0
  68. flashruntime/viewer/_docs/concepts/architecture.html +302 -0
  69. flashruntime/viewer/_docs/get-started.html +263 -0
  70. flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
  71. flashruntime/viewer/_docs/guides/huggingface.html +223 -0
  72. flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
  73. flashruntime/viewer/_docs/guides/pytorch.html +313 -0
  74. flashruntime/viewer/_docs/guides/sklearn.html +232 -0
  75. flashruntime/viewer/_docs/index.html +251 -0
  76. flashruntime/viewer/_docs/reference/cli.html +254 -0
  77. flashruntime/viewer/_docs/reference/integrations.html +240 -0
  78. flashruntime/viewer/_docs/reference/sdk.html +341 -0
  79. flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
  80. flashruntime/viewer/_docs/search-index.json +1 -0
  81. flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
  82. flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
  83. flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
  84. flashruntime/viewer/flowmap.py +307 -0
  85. flashruntime/viewer/page.py +594 -0
  86. flashruntime/viewer/server.py +134 -0
  87. flashruntime/viewer/state.py +250 -0
  88. flashruntime/workloads/__init__.py +6 -0
  89. flashruntime/workloads/command.py +127 -0
  90. flashruntime-0.3.0.dist-info/METADATA +365 -0
  91. flashruntime-0.3.0.dist-info/RECORD +95 -0
  92. flashruntime-0.3.0.dist-info/WHEEL +5 -0
  93. flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
  94. flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
  95. flashruntime-0.3.0.dist-info/top_level.txt +2 -0
flashruntime/viewer/_docs/tutorials/fault-tolerance.html ADDED
@@ -0,0 +1,375 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>Tutorial: automatic recovery, explained — flashruntime docs</title>
7
+ <style>
8
+ :root { color-scheme: dark; }
9
+ * { box-sizing: border-box; margin: 0; }
10
+ body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
11
+ a { color: oklch(0.80 0.16 200); text-decoration: none; }
12
+ a:hover { text-decoration: underline; }
13
+ .layout { display: flex; align-items: flex-start; min-height: 100vh; }
14
+
15
+ /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
16
+ .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
17
+ border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
18
+ .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
19
+ .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
20
+ text-transform: uppercase; letter-spacing: .14em; }
21
+ .search { position: relative; margin: 16px 0; }
22
+ #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
23
+ color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
24
+ #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
25
+ #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
26
+ background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
27
+ #results.open { display: block; }
28
+ #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
29
+ #results a:last-child { border-bottom: 0; }
30
+ #results a:hover { background: #010409; text-decoration: none; }
31
+ #results a b { color: #e6edf3; font-weight: 600; }
32
+ #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
33
+ #results .nohit { padding: 7px 9px; color: #8b949e; }
34
+ .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
35
+ text-transform: uppercase; letter-spacing: .14em; }
36
+ .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
37
+ .nav-link:hover { background: #161b22; text-decoration: none; }
38
+ .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
39
+
40
+ /* content */
41
+ .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
42
+ .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
43
+ .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
44
+ padding-top: 10px; border-top: 1px solid #21262d; }
45
+ .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
46
+ .content p, .content li { color: #c9d1d9; }
47
+ .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
48
+ .content li { margin: 4px 0; }
49
+ .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
50
+ background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
51
+ .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
52
+ border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
53
+ .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
54
+ .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
55
+ .content th { color: #e6edf3; background: #161b22; }
56
+
57
+ /* code blocks + copy button */
58
+ .codeblock { position: relative; margin: 14px 0; }
59
+ .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
60
+ padding: 14px 16px; overflow-x: auto; }
61
+ .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
62
+ .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
63
+ background: #161b22; border: 1px solid #21262d; border-radius: 5px;
64
+ color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
65
+ .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
66
+
67
+ @media (max-width: 800px) {
68
+ .layout { flex-direction: column; }
69
+ .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
70
+ border-right: 0; border-bottom: 1px solid #21262d; }
71
+ .content { padding: 24px 18px 60px; }
72
+ }
73
+ </style>
74
+ </head>
75
+ <body>
76
+ <div class="layout">
77
+ <aside class="sidebar">
78
+ <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
79
+ <div class="search">
80
+ <input id="q" type="search" placeholder="Search docs (press /)" autocomplete="off" spellcheck="false">
81
+ <div id="results"></div>
82
+ </div>
83
+ <nav><div class="nav-section">Getting started</div>
84
+ <a class="nav-link" href="../index.html">FlashRuntime</a>
85
+ <a class="nav-link" href="../get-started.html">Get started</a>
86
+ <div class="nav-section">Tutorials</div>
87
+ <a class="nav-link" href="convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
88
+ <a class="nav-link" href="sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
89
+ <a class="nav-link active" href="fault-tolerance.html">Tutorial: automatic recovery, explained</a>
90
+ <div class="nav-section">Guides</div>
91
+ <a class="nav-link" href="../guides/pytorch.html">PyTorch guide</a>
92
+ <a class="nav-link" href="../guides/sklearn.html">scikit-learn guide</a>
93
+ <a class="nav-link" href="../guides/huggingface.html">Hugging Face guide</a>
94
+ <a class="nav-link" href="../guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
95
+ <a class="nav-link" href="../guides/federated-averaging.html">Federated averaging</a>
96
+ <div class="nav-section">Concepts</div>
97
+ <a class="nav-link" href="../concepts/architecture.html">Architecture</a>
98
+ <div class="nav-section">Reference</div>
99
+ <a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
100
+ <a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
101
+ <a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
102
+ <a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
103
+ <div class="nav-section">Benchmarks</div>
104
+ <a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
105
+ </aside>
106
+ <main class="content"><h1 id="tutorial-automatic-recovery-explained">Tutorial: automatic recovery, explained</h1>
107
+ <p>When a run fails, FlashRuntime does not guess what to do. It turns the failure
108
+ into typed <strong>signals</strong>, <strong>classifies</strong> them into one failure class, and looks
109
+ that class up in a <strong>versioned, deterministic policy table</strong>. Same failure +
110
+ same policy version ⇒ same action, every time. No LLM, no scoring, no learned
111
+ model in the loop.</p>
112
+ <p>This tutorial follows one crash all the way through that pipeline, using the
113
+ local <code>flash.submit()</code> path. It assumes you have done the
114
+ <a href="convnet.html">ConvNet tutorial</a>, whose kill-and-resume run is the worked
115
+ example here.</p>
116
+ <hr />
117
+ <h2 id="the-budget-max_restarts">The budget: <code>max_restarts</code></h2>
118
+ <p>Recovery on the local path is opt-in through one argument:</p>
119
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
120
+ from flashruntime.integrations import pytorch as fr_torch
121
+
122
+ run = flash.submit(
123
+ fr_torch.ddp(&quot;train.py&quot;, source=&quot;.&quot;, nproc_per_node=1,
124
+ script_args=&quot;--steps 24 --checkpoint-every 8 --kill-at-step 8&quot;),
125
+ output_dir=&quot;out/convnet&quot;,
126
+ max_restarts=1, # the automatic fault-tolerance budget (default 0 = no retry)
127
+ )
128
+ </code></pre></div>
129
+ <p><code>max_restarts=0</code> (the default) means "run once, no retry" — the original
130
+ behavior. Any higher number is the recovery budget: on a FAILED attempt,
131
+ FlashRuntime consults the policy and, unless the failure is a deterministic
132
+ application error, relaunches the <strong>same spec</strong> from the job-scoped checkpoint,
133
+ up to that many times.</p>
134
+ <hr />
135
+ <h2 id="step-1-signals-what-a-local-process-can-actually-tell-you">Step 1 — signals: what a local process can actually tell you</h2>
136
+ <p>The first thing FlashRuntime does with a dead attempt is translate it into
137
+ <code>FailureSignals</code>. On the local path the translator is
138
+ <code>recovery.signals.from_local_launch(exit_code, log_tail)</code>, and it is
139
+ <strong>deliberately narrow</strong>: a single local process can only evidence two things —
140
+ its OS exit code and the tail of its captured stdout+stderr. It never
141
+ fabricates node, accelerator, communication, or storage signals it cannot
142
+ observe; those classes belong to the distributed coordinator, not a local
143
+ launcher.</p>
144
+ <p>It is a transparent lookup table, not an inference engine. The rules, checked
145
+ top to bottom, first match wins:</p>
146
+ <table>
147
+ <thead>
148
+ <tr>
149
+ <th>#</th>
150
+ <th>Evidence</th>
151
+ <th>Signals produced</th>
152
+ </tr>
153
+ </thead>
154
+ <tbody>
155
+ <tr>
156
+ <td>1</td>
157
+ <td>exit 0 / never started</td>
158
+ <td>neutral (nothing broke — a guard that keeps the function total)</td>
159
+ </tr>
160
+ <tr>
161
+ <td>2</td>
162
+ <td>a named deterministic exception on a traceback's <strong>terminal line</strong> (<code>ModuleNotFoundError:</code>, <code>NameError:</code>, <code>SyntaxError:</code>, <code>ImportError:</code>, <code>IndentationError:</code>, <code>AttributeError:</code>)</td>
163
+ <td><code>exit_deterministic=True</code></td>
164
+ </tr>
165
+ <tr>
166
+ <td>3</td>
167
+ <td>a bare <code>Traceback</code> that is <strong>not</strong> the <code>torchrun</code> wrapper</td>
168
+ <td><code>exit_deterministic=True</code></td>
169
+ </tr>
170
+ <tr>
171
+ <td>4</td>
172
+ <td>everything else (signal death / OOM / bare <code>SystemExit</code> / <code>torchrun</code> <code>ChildFailedError</code>)</td>
173
+ <td>plain crash (transient)</td>
174
+ </tr>
175
+ </tbody>
176
+ </table>
177
+ <p>Two subtleties worth knowing, because they are where naive implementations get
178
+ it wrong:</p>
179
+ <ul>
180
+ <li><strong>Rule 2 is anchored to the start of the traceback's terminal line, not a
181
+ bare substring.</strong> CPython prints the failing exception type at the <em>start</em> of
182
+ the final traceback line, while prose only ever <em>mentions</em> a name mid-line.
183
+ The classic false positive is the startup log <code>... WARNING ImportError:
184
+ flash_attn not available, falling back to eager</code> — a <em>transient</em> run that a
185
+ substring scan would wrongly fail fast. Line-anchoring ignores the name in
186
+ prose and fires only on a real terminal.</li>
187
+ <li><strong><code>torchrun</code> wraps every worker death — transient ones included — in a
188
+ <code>ChildFailedError</code> and prints its own traceback.</strong> That would trip rule 3, so
189
+ rule 3 is disqualified whenever <code>ChildFailedError</code> is present, letting the
190
+ death fall through to the transient default (rule 4). This is exactly why the
191
+ ConvNet kill-and-resume run recovers instead of failing fast.</li>
192
+ </ul>
193
+ <hr />
194
+ <h2 id="step-2-classify-signals-to-one-failure-class">Step 2 — classify: signals to one failure class</h2>
195
+ <p><code>recovery.classify(signals)</code> maps the signals to exactly one
196
+ <code>FailureClass</code>, precedence-ordered so <strong>systemic evidence beats node evidence
197
+ beats process evidence beats application evidence</strong> — a worker crash <em>during</em> a
198
+ correlated incident is the incident, not the worker. The classes
199
+ <code>classify()</code> can produce locally (the full <code>FailureClass</code> enum has two more —
200
+ <code>data_error</code>, <code>network_degradation</code> — reachable only with coordinator-side
201
+ signals):</p>
202
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>correlated_incident systemic: many failures in the window — stop acting
203
+ control_plane_failure coordinator unreachable
204
+ preemption spot capacity reclaimed
205
+ accelerator_failure GPU / driver / XID fault
206
+ node_loss heartbeat lost / node unreachable
207
+ communication_error NCCL / RCCL / rendezvous
208
+ storage_timeout object-store errors
209
+ artifact_corruption hash mismatch / failed validation
210
+ application_error deterministic bug (exit_deterministic) ── fail fast
211
+ worker_crash non-deterministic process death ── one fresh attempt
212
+ unknown nothing matched
213
+ </code></pre></div>
214
+ <p>On the local path only the last three are reachable, because those are the only
215
+ classes a single process can evidence. A crashed <code>torchrun</code> worker with no
216
+ deterministic terminal is a <strong><code>worker_crash</code></strong>.</p>
217
+ <hr />
218
+ <h2 id="step-3-decide-class-mode-to-a-typed-action">Step 3 — decide: class × mode to a typed action</h2>
219
+ <p><code>recovery.decide(failure_class, mode)</code> is a pure table lookup returning a typed
220
+ <code>RecoveryDecision</code> stamped with <code>POLICY_VERSION</code>. The <strong>mode</strong> matters because
221
+ the same failure has a different blast radius:</p>
222
+ <ul>
223
+ <li><code>independent_tasks</code> (a fan-out sweep) — one task retries; the others are
224
+ untouched.</li>
225
+ <li><code>coordinated_training</code> (a <code>torchrun</code> group) — a lost rank stops the whole
226
+ group, so recovery is a whole-group restart from the latest valid checkpoint
227
+ (NCCL collective state is not repairable in place).</li>
228
+ </ul>
229
+ <p><code>flash.submit()</code> picks the mode from the workload: a fan-out is
230
+ <code>independent_tasks</code>, anything else (including a <code>torchrun</code> DDP run) is
231
+ <code>coordinated_training</code>. A few rows of the table:</p>
232
+ <table>
233
+ <thead>
234
+ <tr>
235
+ <th>Failure class</th>
236
+ <th>independent_tasks</th>
237
+ <th>coordinated_training</th>
238
+ </tr>
239
+ </thead>
240
+ <tbody>
241
+ <tr>
242
+ <td><code>application_error</code></td>
243
+ <td><strong>fail_job</strong> (retrying burns money on a bug)</td>
244
+ <td><strong>fail_job</strong></td>
245
+ </tr>
246
+ <tr>
247
+ <td><code>worker_crash</code></td>
248
+ <td>retry_task</td>
249
+ <td><strong>restart_group</strong> (from latest valid checkpoint)</td>
250
+ </tr>
251
+ <tr>
252
+ <td><code>node_loss</code></td>
253
+ <td>retry_task (cordon + requeue)</td>
254
+ <td>replace_node</td>
255
+ </tr>
256
+ <tr>
257
+ <td><code>storage_timeout</code></td>
258
+ <td>pause_job (don't burn compute on dead storage)</td>
259
+ <td>pause_job</td>
260
+ </tr>
261
+ <tr>
262
+ <td><code>correlated_incident</code></td>
263
+ <td><strong>freeze_automation</strong> (no retry storms)</td>
264
+ <td>freeze_automation</td>
265
+ </tr>
266
+ </tbody>
267
+ </table>
268
+ <p>The most important action in the table is <code>freeze_automation</code>: during a
269
+ correlated incident, the policy's job is to know when to <strong>stop</strong> acting.</p>
270
+ <hr />
271
+ <h2 id="putting-it-together-the-convnet-crash">Putting it together: the ConvNet crash</h2>
272
+ <p>The kill-and-resume run from the ConvNet tutorial produces exactly this trail
273
+ in <code>run.events</code> (<code>FAILURE_CLASSIFIED</code> is amber and <code>RECOVERY_ACTION_SELECTED</code>
274
+ is cyan on the live page):</p>
275
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>LAUNCH_STARTED task-000 launched (pid ...)
276
+ FAILURE_CLASSIFIED task-000: worker_crash (exit 1)
277
+ RECOVERY_ACTION_SELECTED task-000: restart_group — a lost rank stops the group —
278
+ restart all workers from latest valid checkpoint
279
+ LAUNCH_STARTED task-000-r1 launched (pid ...)
280
+ </code></pre></div>
281
+ <p>Reading it back through the pipeline: the <code>torchrun</code> child raised
282
+ <code>SystemExit(3)</code>; <code>torchrun</code> re-raised a <code>ChildFailedError</code> (so rules 2 and 3
283
+ did <strong>not</strong> fire) and exited non-zero → signals say "plain crash" → <code>classify</code>
284
+ returns <code>worker_crash</code> → <code>decide(worker_crash, "coordinated_training")</code> returns
285
+ <code>restart_group</code>. The retry (<code>task-000-r1</code>) keeps the <strong>same</strong> job id, so its
286
+ <code>ft.prepare()</code> finds the predecessor's newest valid manifest and resumes —
287
+ <code>resumed_from: 8</code>, not a restart from zero.</p>
288
+ <p>And the fail-fast case: introduce a <code>NameError</code> in the script and the terminal
289
+ traceback trips rule 2 → <code>application_error</code> → <code>fail_job</code>. FlashRuntime does
290
+ <strong>not</strong> spend your restart budget re-hitting a deterministic bug; it fails
291
+ immediately and tells you.</p>
292
+ <hr />
293
+ <h2 id="honest-scope">Honest scope</h2>
294
+ <ul>
295
+ <li><strong>This is the local signal surface.</strong> <code>from_local_launch</code> sees an exit code
296
+ and a log tail — nothing more. The richer classes (<code>node_loss</code>,
297
+ <code>accelerator_failure</code>, <code>communication_error</code>, <code>correlated_incident</code>) are real
298
+ and fully policy-covered, but they are evidenced by the distributed
299
+ <strong>coordinator</strong> (leases, heartbeats, health signals), not by one local
300
+ process. The taxonomy and policy table are the same in both places.</li>
301
+ <li><strong>Recovery never changes the math.</strong> A resumed run restores a verified,
302
+ topology-compatible checkpoint and continues; determinism (fixed seeds, the
303
+ seed-0 <code>DistributedSampler</code>) is what makes a resumed run reproduce the
304
+ uninterrupted one.</li>
305
+ </ul>
306
+ <hr />
307
+ <h2 id="where-to-go-next">Where to go next</h2>
308
+ <ul>
309
+ <li><strong><a href="../concepts/architecture.html">Architecture</a></strong> — recovery as one of the four
310
+ axes, with the leases and manifests it relies on.</li>
311
+ <li><strong><a href="../reference/sdk.html">SDK reference</a></strong> — <code>submit(..., max_restarts=N)</code> and
312
+ the <code>Run</code> event log.</li>
313
+ <li><strong><a href="../guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a></strong> — how the
314
+ same recovery machinery runs on a coordinator, under leases.</li>
315
+ </ul></main>
316
+ </div>
317
+
318
+ <script>
319
+ // ---- client-side search: fetch the builder's index, filter as you type -----
320
+ // (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
321
+ let INDEX = [];
322
+ // Path from THIS page back to the site root. The index stores root-relative
323
+ // URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
324
+ // against ROOT so a page in a subdir resolves them too (see build_docs.py).
325
+ const ROOT = "../";
326
+ const q = document.getElementById("q");
327
+ const results = document.getElementById("results");
328
+ const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
329
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
330
+ fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
331
+
332
+ // A short context window around the first match, so a hit shows WHY it matched.
333
+ function snippet(text, needle) {
334
+ const i = text.toLowerCase().indexOf(needle);
335
+ if (i < 0) return "";
336
+ const start = Math.max(0, i - 32);
337
+ return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
338
+ }
339
+ function runSearch() {
340
+ const needle = q.value.trim().toLowerCase();
341
+ if (!needle) { results.className = ""; results.innerHTML = ""; return; }
342
+ const hits = INDEX.map((p) => {
343
+ const inTitle = p.title.toLowerCase().includes(needle);
344
+ const inText = p.text.toLowerCase().includes(needle);
345
+ if (!inTitle && !inText) return null;
346
+ return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
347
+ }).filter(Boolean).slice(0, 20);
348
+ results.className = "open";
349
+ results.innerHTML = hits.length
350
+ ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
351
+ (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
352
+ : '<div class="nohit">no matches</div>';
353
+ }
354
+ q.addEventListener("input", runSearch);
355
+ q.addEventListener("focus", runSearch);
356
+ document.addEventListener("click", (e) => {
357
+ if (!e.target.closest(".search")) { results.className = ""; } // dismiss on outside click
358
+ });
359
+ document.addEventListener("keydown", (e) => { // "/" focuses search, like PyTorch docs
360
+ if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
361
+ });
362
+
363
+ // ---- copy buttons: read the <pre>'s own text into the clipboard ------------
364
+ document.querySelectorAll(".copybtn").forEach((btn) => {
365
+ btn.addEventListener("click", () => {
366
+ const code = btn.parentElement.querySelector("pre").innerText;
367
+ navigator.clipboard.writeText(code).then(() => {
368
+ const was = btn.textContent; btn.textContent = "Copied";
369
+ setTimeout(() => { btn.textContent = was; }, 1200);
370
+ }).catch(() => {});
371
+ });
372
+ });
373
+ </script>
374
+ </body>
375
+ </html>