PyPI - flashruntime - Versions diffs - 0.3.0__py3-none-any.whl - Mend

flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (95) hide show

flashml_workloads/__init__.py +7 -0
flashml_workloads/fedavg_driver.py +569 -0
flashml_workloads/fedavg_weights.py +223 -0
flashml_workloads/fedavg_worker.py +166 -0
flashml_workloads/kmeans_driver.py +134 -0
flashml_workloads/kmeans_shard.py +69 -0
flashml_workloads/sgd_trainer.py +127 -0
flashml_workloads/sharded_kmeans.py +323 -0
flashml_workloads/sklearn_trial.py +89 -0
flashruntime/__init__.py +125 -0
flashruntime/artifacts/__init__.py +25 -0
flashruntime/artifacts/store.py +228 -0
flashruntime/backends/__init__.py +26 -0
flashruntime/backends/base.py +63 -0
flashruntime/backends/kuberay.py +465 -0
flashruntime/checkpoint/__init__.py +20 -0
flashruntime/checkpoint/catalog.py +198 -0
flashruntime/checkpoint/local.py +109 -0
flashruntime/checkpoint/store.py +86 -0
flashruntime/integrations/__init__.py +5 -0
flashruntime/integrations/huggingface.py +59 -0
flashruntime/integrations/pytorch.py +52 -0
flashruntime/integrations/sklearn.py +42 -0
flashruntime/launchers/__init__.py +130 -0
flashruntime/launchers/local.py +126 -0
flashruntime/leases/__init__.py +27 -0
flashruntime/leases/manager.py +365 -0
flashruntime/leases/sqlite_store.py +169 -0
flashruntime/leases/store.py +103 -0
flashruntime/monitor/__init__.py +7 -0
flashruntime/monitor/sampler.py +232 -0
flashruntime/planner/__init__.py +56 -0
flashruntime/planner/candidates.py +597 -0
flashruntime/planner/catalog.py +129 -0
flashruntime/planner/comm.py +95 -0
flashruntime/planner/explain.py +109 -0
flashruntime/planner/memory.py +166 -0
flashruntime/planner/resolve.py +120 -0
flashruntime/planner/selector.py +169 -0
flashruntime/planner/timecost.py +81 -0
flashruntime/profiling/__init__.py +113 -0
flashruntime/protocol/__init__.py +18 -0
flashruntime/protocol/plan_v1alpha1.py +320 -0
flashruntime/protocol/v1alpha1.py +465 -0
flashruntime/providers/__init__.py +138 -0
flashruntime/py.typed +0 -0
flashruntime/recipes/__init__.py +135 -0
flashruntime/recipes/command.py +166 -0
flashruntime/recovery/__init__.py +21 -0
flashruntime/recovery/policy.py +170 -0
flashruntime/recovery/signals.py +135 -0
flashruntime/recovery/taxonomy.py +91 -0
flashruntime/scheduler/__init__.py +170 -0
flashruntime/sdk.py +402 -0
flashruntime/service/__init__.py +3 -0
flashruntime/service/app.py +391 -0
flashruntime/service/auth.py +180 -0
flashruntime/service/checkpoints.py +90 -0
flashruntime/service/cli.py +167 -0
flashruntime/service/dashboard.py +193 -0
flashruntime/service/ledger.py +101 -0
flashruntime/service/modea.py +821 -0
flashruntime/strategies/__init__.py +156 -0
flashruntime/strategies/command.py +56 -0
flashruntime/torch/__init__.py +274 -0
flashruntime/viewer/__init__.py +20 -0
flashruntime/viewer/_docs/benchmarks.html +771 -0
flashruntime/viewer/_docs/concepts/architecture.html +302 -0
flashruntime/viewer/_docs/get-started.html +263 -0
flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
flashruntime/viewer/_docs/guides/huggingface.html +223 -0
flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
flashruntime/viewer/_docs/guides/pytorch.html +313 -0
flashruntime/viewer/_docs/guides/sklearn.html +232 -0
flashruntime/viewer/_docs/index.html +251 -0
flashruntime/viewer/_docs/reference/cli.html +254 -0
flashruntime/viewer/_docs/reference/integrations.html +240 -0
flashruntime/viewer/_docs/reference/sdk.html +341 -0
flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
flashruntime/viewer/_docs/search-index.json +1 -0
flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
flashruntime/viewer/flowmap.py +307 -0
flashruntime/viewer/page.py +594 -0
flashruntime/viewer/server.py +134 -0
flashruntime/viewer/state.py +250 -0
flashruntime/workloads/__init__.py +6 -0
flashruntime/workloads/command.py +127 -0
flashruntime-0.3.0.dist-info/METADATA +365 -0
flashruntime-0.3.0.dist-info/RECORD +95 -0
flashruntime-0.3.0.dist-info/WHEEL +5 -0
flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
flashruntime-0.3.0.dist-info/top_level.txt +2 -0

flashruntime/viewer/_docs/tutorials/convnet.html ADDED Viewed

@@ -0,0 +1,571 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Tutorial: make a ConvNet fault-tolerant — flashruntime docs</title>
+<style>
+  :root { color-scheme: dark; }
+  * { box-sizing: border-box; margin: 0; }
+  body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
+  a { color: oklch(0.80 0.16 200); text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  .layout { display: flex; align-items: flex-start; min-height: 100vh; }
+  /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
+  .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
+             border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
+  .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
+  .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .search { position: relative; margin: 16px 0; }
+  #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
+       color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
+  #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
+  #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
+             background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
+  #results.open { display: block; }
+  #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
+  #results a:last-child { border-bottom: 0; }
+  #results a:hover { background: #010409; text-decoration: none; }
+  #results a b { color: #e6edf3; font-weight: 600; }
+  #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
+  #results .nohit { padding: 7px 9px; color: #8b949e; }
+  .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
+  .nav-link:hover { background: #161b22; text-decoration: none; }
+  .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
+  /* content */
+  .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
+  .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
+  .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
+                padding-top: 10px; border-top: 1px solid #21262d; }
+  .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
+  .content p, .content li { color: #c9d1d9; }
+  .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
+  .content li { margin: 4px 0; }
+  .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
+                        background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
+  .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
+                              border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
+  .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
+  .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
+  .content th { color: #e6edf3; background: #161b22; }
+  /* code blocks + copy button */
+  .codeblock { position: relative; margin: 14px 0; }
+  .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
+                   padding: 14px 16px; overflow-x: auto; }
+  .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
+  .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
+             background: #161b22; border: 1px solid #21262d; border-radius: 5px;
+             color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+  .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
+  @media (max-width: 800px) {
+    .layout { flex-direction: column; }
+    .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
+               border-right: 0; border-bottom: 1px solid #21262d; }
+    .content { padding: 24px 18px 60px; }
+  }
+</style>
+</head>
+<body>
+<div class="layout">
+  <aside class="sidebar">
+    <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
+    <div class="search">
+      <input id="q" type="search" placeholder="Search docs  (press /)" autocomplete="off" spellcheck="false">
+      <div id="results"></div>
+    </div>
+    <nav><div class="nav-section">Getting started</div>
+<a class="nav-link" href="../index.html">FlashRuntime</a>
+<a class="nav-link" href="../get-started.html">Get started</a>
+<div class="nav-section">Tutorials</div>
+<a class="nav-link active" href="convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
+<a class="nav-link" href="sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
+<a class="nav-link" href="fault-tolerance.html">Tutorial: automatic recovery, explained</a>
+<div class="nav-section">Guides</div>
+<a class="nav-link" href="../guides/pytorch.html">PyTorch guide</a>
+<a class="nav-link" href="../guides/sklearn.html">scikit-learn guide</a>
+<a class="nav-link" href="../guides/huggingface.html">Hugging Face guide</a>
+<a class="nav-link" href="../guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
+<a class="nav-link" href="../guides/federated-averaging.html">Federated averaging</a>
+<div class="nav-section">Concepts</div>
+<a class="nav-link" href="../concepts/architecture.html">Architecture</a>
+<div class="nav-section">Reference</div>
+<a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
+<a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
+<a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
+<a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
+<div class="nav-section">Benchmarks</div>
+<a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
+  </aside>
+  <main class="content"><h1 id="tutorial-make-a-convnet-fault-tolerant">Tutorial: make a ConvNet fault-tolerant</h1>
+<p>This is the flagship walkthrough. You start with an ordinary PyTorch ConvNet
+and an ordinary training loop, and end with the <em>same</em> code — unchanged in its
+math — running as a 2-process DDP job that survives a crash and resumes from
+its last verified checkpoint, with a live page you can watch.</p>
+<p>The promise throughout: <strong>FlashRuntime operates your code; it never rewrites
+it.</strong> The model below is used verbatim, including one unusual thing it does,
+because "your code, unmodified" is the whole point.</p>
+<p>If you have not installed FlashRuntime and PyTorch yet, do the
+<a href="../get-started.html">Get started</a> page first — you need <code>pip install
+flashruntime</code> and a CPU build of <code>torch</code>. No GPU, no cluster.</p>
+<hr />
+<h2 id="1-the-plain-script-no-flashruntime">1. The plain script (no FlashRuntime)</h2>
+<p>Here is the model and a plain single-process training loop. It reads its
+hyperparameters from <code>argparse</code> and writes a <code>metrics.json</code> at the end — the
+one convention FlashRuntime asks of any script. There is <strong>no FlashRuntime
+import</strong> yet.</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import argparse
+import json
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, TensorDataset
+class OurConvNet(torch.nn.Module):
+    def __init__(self, num_outputs=20):
+        super(OurConvNet, self).__init__()
+        self.Conv1 = nn.Conv2d(3, 64, 5, 1, 2)
+        self.Sigma = nn.Sigmoid()
+        self.Avg = nn.AvgPool2d(2, stride=2, padding=0)
+        self.Conv2 = nn.Conv2d(64, 128, 5, 1)
+        self.Fl = nn.Flatten()
+        self.Linear = nn.Linear(128 * 5 * 5, num_outputs)
+    def forward(self, x):
+        out = self.Sigma(x)   # NOTE: the Sigmoid is applied to the INPUT first.
+        out = self.Conv1(out)
+        out = self.Avg(out)
+        out = self.Conv2(out)
+        out = self.Avg(out)
+        out = self.Fl(out)
+        out = self.Linear(out)
+        return out
+def make_data(n=256, num_outputs=20, seed=0):
+    # Synthetic data with a fixed seed keeps the run deterministic on CPU.
+    g = torch.Generator().manual_seed(seed)
+    x = torch.randn(n, 3, 28, 28, generator=g)
+    y = torch.randint(0, num_outputs, (n,), generator=g)
+    return TensorDataset(x, y)
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(&quot;--steps&quot;, type=int, default=40)
+    parser.add_argument(&quot;--lr&quot;, type=float, default=0.05)
+    args = parser.parse_args()
+    torch.manual_seed(0)
+    model = OurConvNet(num_outputs=20)
+    optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
+    loader = DataLoader(make_data(), batch_size=32, shuffle=False)
+    step = 0
+    loss = torch.tensor(0.0)
+    while step &lt; args.steps:
+        for x, y in loader:
+            if step &gt;= args.steps:
+                break
+            loss = torch.nn.functional.cross_entropy(model(x), y)
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            step += 1
+    with open(&quot;metrics.json&quot;, &quot;w&quot;) as f:
+        json.dump({&quot;steps&quot;: step, &quot;final_loss&quot;: round(loss.item(), 6)}, f)
+    print(&quot;done&quot;, step, loss.item())
+if __name__ == &quot;__main__&quot;:
+    main()
+</code></pre></div>
+<p>Run it like any script:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-bash">python train.py --steps 30
+</code></pre></div>
+<h3 id="the-shape-math-so-128-5-5-is-not-a-magic-number">The shape math (so <code>128 * 5 * 5</code> is not a magic number)</h3>
+<p>The input is <code>3 × 28 × 28</code>. Follow one image through <code>forward</code>, and every
+<code>nn.Linear(128 * 5 * 5, ...)</code> factor falls out of the convolution arithmetic:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code> 28  --Conv1(k5, s1, p2)--&gt;  28   (padding 2 keeps the size)
+     --Avg(2, 2)----------&gt;  14
+     --Conv2(k5, s1, p0)--&gt;  10   (no padding: 14 - 5 + 1 = 10)
+     --Avg(2, 2)----------&gt;   5
+  =&gt;  Flatten = 128 channels · 5 · 5 = 3200 = nn.Linear(128 * 5 * 5, num_outputs)
+</code></pre></div>
+<p>So <code>3 × 28 × 28</code> is the input size that makes the flattened feature map land
+<strong>exactly</strong> on <code>128 * 5 * 5</code>. Feed a different size and the <code>Flatten → Linear</code>
+handoff mismatches; keep 28 and it fits.</p>
+<h3 id="one-honest-note-about-this-model">One honest note about this model</h3>
+<p><code>forward</code> applies <code>self.Sigma</code> (a <code>Sigmoid</code>) to the <strong>raw input</strong> before the
+first convolution — an unusual ordering (a sigmoid is normally an activation
+<em>between</em> layers, not a preprocessing step on the pixels). We keep it exactly
+as written. FlashRuntime's promise is to operate <em>your</em> code, so we do not
+"fix" the model to match convention — we run what you wrote.</p>
+<hr />
+<h2 id="2-make-it-resumable-import-flashruntimetorch-as-ft">2. Make it resumable: <code>import flashruntime.torch as ft</code></h2>
+<p>The plain script has no checkpoints, so a crash starts over from step 0. One
+import — <code>flashruntime.torch</code> (aliased <code>ft</code>) — gives you launch-anywhere DDP
+and fault-tolerant checkpointing without rebuilding any framework machinery.
+Here is the whole change, as a diff:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-diff"> import argparse
+ import json
+ import torch
+ import torch.nn as nn
+ from torch.utils.data import DataLoader, TensorDataset
++import flashruntime.torch as ft
++
+ def main():
+     parser = argparse.ArgumentParser()
+     parser.add_argument(&quot;--steps&quot;, type=int, default=40)
+     parser.add_argument(&quot;--lr&quot;, type=float, default=0.05)
++    parser.add_argument(&quot;--checkpoint-every&quot;, type=int, default=8)
++    parser.add_argument(&quot;--kill-at-step&quot;, type=int, default=None,
++                        help=&quot;simulate a crash (fresh runs only; a resumed retry finishes)&quot;)
+     args = parser.parse_args()
+     torch.manual_seed(0)
+     model = OurConvNet(num_outputs=20)
+     optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
+     loader = DataLoader(make_data(), batch_size=32, shuffle=False)
+-    step = 0
++    model, optimizer, loader = ft.prepare(model, optimizer, loader)
++    start = ft.start_step()
++
++    step = start
+     loss = torch.tensor(0.0)
+     while step &lt; args.steps:
+         for x, y in loader:
+             if step &gt;= args.steps:
+                 break
+             loss = torch.nn.functional.cross_entropy(model(x), y)
+             optimizer.zero_grad()
+             loss.backward()
+             optimizer.step()
+             step += 1
++            ft.checkpoint(model, optimizer, step=step, every=args.checkpoint_every)
++            ft.log_metrics({&quot;step&quot;: step, &quot;loss&quot;: round(loss.item(), 6)})
++            if args.kill_at_step and start == 0 and step &gt;= args.kill_at_step:
++                raise SystemExit(3)  # fresh run only — the retry resumes past this
+-    with open(&quot;metrics.json&quot;, &quot;w&quot;) as f:
+-        json.dump({&quot;steps&quot;: step, &quot;final_loss&quot;: round(loss.item(), 6)}, f)
+-    print(&quot;done&quot;, step, loss.item())
++    ft.checkpoint(model, optimizer, step=step)  # final checkpoint
++    if ft.is_main():
++        metrics = {&quot;steps&quot;: step, &quot;resumed_from&quot;: start, &quot;final_loss&quot;: round(loss.item(), 6)}
++        with open(&quot;metrics.json&quot;, &quot;w&quot;) as f:
++            json.dump(metrics, f)
++        print(metrics)
+</code></pre></div>
+<p>What each added call does — and <em>nothing more</em> (this is the whole surface):</p>
+<ul>
+<li><strong><code>ft.prepare(model, optimizer, loader)</code></strong> — launched distributed
+  (<code>WORLD_SIZE &gt; 1</code>), it initializes torch's own process group (<code>gloo</code> on CPU,
+  <code>nccl</code> on GPU), wraps the model in <code>DistributedDataParallel</code>, and swaps the
+  DataLoader's sampler for a seed-0 <code>DistributedSampler</code> so each rank sees a
+  disjoint shard. It then restores the newest <strong>valid</strong> checkpoint if one
+  exists. Launched as plain <code>python train.py</code>, it is a no-op passthrough.</li>
+<li><strong><code>ft.start_step()</code></strong> — <code>0</code> on a fresh run, <code>&gt;0</code> after a resume (it is the
+  step <code>prepare</code> restored to). The loop starts from it.</li>
+<li><strong><code>ft.checkpoint(model, optimizer, step=step, every=8)</code></strong> — rank 0 writes a
+  checkpoint under the parts-first / manifest-last contract; the manifest is
+  written <strong>last</strong>, so a half-written checkpoint is never treated as valid.</li>
+<li><strong><code>ft.log_metrics({...})</code></strong> — rank 0 appends one JSON line to
+  <code>metrics.jsonl</code>; this is the streaming series the live page draws as a loss
+  curve. It never raises.</li>
+</ul>
+<p>The full surface is three verbs plus read-only launch-fact accessors — see
+the <a href="../reference/torch-helper.html">torch helper reference</a>. There are no FSDP
+policies, no autocast, no DeepSpeed config here: that is deliberate (ADR-0003 —
+we do not rebuild Accelerate). A script that wants those uses the real
+framework features; the launcher still launches it correctly.</p>
+<p>The finished, copy-paste-runnable version of this script is in
+<a href="#5-the-final-script-copy-paste-runnable">section 5</a> below.</p>
+<hr />
+<h2 id="3-operate-it-flashsubmitfr_torchddp">3. Operate it: <code>flash.submit(fr_torch.ddp(...))</code></h2>
+<p>Now hand the script to FlashRuntime. The <code>integrations.pytorch</code> adapter builds
+the <code>torchrun</code> command that starts N worker processes; <code>flash.submit()</code> runs
+it, waits, collects the artifacts, and hands back a <code>Run</code>:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+from flashruntime.integrations import pytorch as fr_torch
+run = flash.submit(fr_torch.ddp(
+    &quot;train.py&quot;,
+    source=&quot;.&quot;,              # directory holding train.py
+    nproc_per_node=2,        # two worker processes on this host — gloo/CPU, no GPU
+    script_args=&quot;--steps 16&quot;,
+))
+print(run.state.value, run.trials)
+# SUCCEEDED [{'steps': 16, 'resumed_from': 0, 'final_loss': 2.98...}]
+</code></pre></div>
+<p><code>ddp(script, *, source=".", nproc_per_node=2, nnodes=1, script_args="",
+env=None)</code> emits <code>torchrun --nproc-per-node=2 --nnodes=1 --standalone
+--local-addr=127.0.0.1 train.py --steps 16</code>. Two processes rendezvous on
+loopback; each gets its <code>RANK</code> / <code>WORLD_SIZE</code> / <code>LOCAL_RANK</code>, and
+<code>ft.prepare()</code> wires DDP from there. <code>nnodes &gt; 1</code> raises <code>NotImplementedError</code>
+today — multi-node rendezvous is a later slice.</p>
+<blockquote>
+<p><strong><code>torchrun</code> must be on your <code>PATH</code>.</strong> It ships with <code>torch</code>; if you
+installed torch into a virtualenv, run from that environment so its
+<code>torchrun</code> is found.</p>
+</blockquote>
+<hr />
+<h2 id="4-crash-it-and-let-it-recover">4. Crash it, and let it recover</h2>
+<p>Now the payoff. Point a run at a fixed <code>output_dir</code> so its checkpoint tree
+persists, tell the script to crash mid-way with <code>--kill-at-step</code>, and give
+<code>submit()</code> a restart budget with <code>max_restarts=1</code>. That is the <strong>only</strong> change
+— one keyword argument — and recovery is automatic:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+from flashruntime.integrations import pytorch as fr_torch
+run = flash.submit(
+    fr_torch.ddp(
+        &quot;train.py&quot;,
+        source=&quot;.&quot;,
+        nproc_per_node=1,
+        script_args=&quot;--steps 24 --checkpoint-every 8 --kill-at-step 8&quot;,
+    ),
+    output_dir=&quot;out/convnet&quot;,   # a FIXED dir — the checkpoint tree lives here
+    max_restarts=1,             # one automatic recovery attempt
+)
+print(run.state.value, run.trials)
+# SUCCEEDED [{'steps': 24, 'resumed_from': 8, 'final_loss': 2.99...}]
+</code></pre></div>
+<p><code>resumed_from: 8</code> is the proof: the first attempt crashed at step 8, and the
+retry <strong>resumed from the step-8 checkpoint</strong> rather than restarting at zero.
+The run's event log tells the whole story:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>LAUNCH_STARTED          task-000 launched (pid ...)
+FAILURE_CLASSIFIED      task-000: worker_crash (exit 1)
+RECOVERY_ACTION_SELECTED task-000: restart_group — a lost rank stops the group —
+                        restart all workers from latest valid checkpoint
+LAUNCH_STARTED          task-000-r1 launched (pid ...)
+</code></pre></div>
+<p>That decision is not a guess. FlashRuntime turned the crash into failure
+signals, <code>classify()</code>'d them (a <code>torchrun</code>-wrapped worker death is a
+non-deterministic <code>worker_crash</code>, not a code bug), and looked the class up in a
+versioned, deterministic policy table. Same failure + same policy version ⇒
+same action, every time — no LLM in the loop. The mechanics are the subject of
+the <a href="fault-tolerance.html">fault-tolerance tutorial</a>.</p>
+<p>One honest constraint on bit-exact resume: on resume the <code>for</code> loop restarts
+the dataloader at batch 0, so the resumed step must land on an <strong>epoch
+boundary</strong>. With 256 samples / batch 32 = 8 batches per epoch single-process,
+keep <code>--checkpoint-every</code> (and <code>--kill-at-step</code>) multiples of that. Off a
+boundary, the resumed run is still correct training — it just will not match an
+uninterrupted run byte-for-byte.</p>
+<hr />
+<h2 id="watch-it-live">Watch it live</h2>
+<p>Pass <code>watch=True</code> (or just run at an interactive terminal, where it is the
+default) and <code>flash.submit()</code> opens a live run page in your browser and records
+its URL on <code>run.viewer_url</code>:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+from flashruntime.integrations import pytorch as fr_torch
+run = flash.submit(
+    fr_torch.ddp(&quot;train.py&quot;, source=&quot;.&quot;, nproc_per_node=2, script_args=&quot;--steps 16&quot;),
+    output_dir=&quot;out/convnet&quot;,
+    watch=True,
+)
+print(run.viewer_url)   # http://127.0.0.1:&lt;port&gt;
+</code></pre></div>
+<p>The page polls a loopback server every couple of seconds and draws the run with
+<strong>zero external assets</strong> (no CDN, no web font, no remote image) — it renders
+with the network cut. Top to bottom, the panels are:</p>
+<ul>
+<li><strong>Header</strong> — a colored state badge (RUNNING cyan, SUCCEEDED green, FAILED
+  red), the exact command, the execution <code>mode</code>, <code>restarts</code> used out of your
+  budget, and the total <code>attempts</code> count.</li>
+<li><strong>Topology</strong> — a single machine box labeled <code>127.0.0.1 · localhost</code> with one
+  node per attempt/rank, colored by state; a RUNNING node softly pulses. A
+  2-process DDP run shows two nodes.</li>
+<li><strong>Loss</strong> — an autoscaled curve built from the <code>metrics.jsonl</code> your
+  <code>ft.log_metrics(...)</code> calls stream, with the latest value labeled. (No
+  <code>log_metrics</code> calls ⇒ no curve — this panel is fed by that stream.)</li>
+<li><strong>Checkpoints</strong> — one violet marker per manifest: its step, a
+  <strong>hash-verified</strong> or <strong>invalid</strong> badge (re-verified at read time, not taken
+  on trust), a ★ <strong>latest</strong> tag on the one recovery would restore, and the part
+  count plus age.</li>
+<li><strong>Events</strong> — newest first, the same log shown above;
+  <code>FAILURE_CLASSIFIED</code> is amber and <code>RECOVERY_ACTION_SELECTED</code> is cyan, each
+  carrying the failure class and the policy's human-readable reason.</li>
+<li><strong>Logs</strong> — a collapsible tail of each attempt's captured stdout+stderr.</li>
+</ul>
+<p>These docs are served from that same viewer at <code>/docs</code>, so the page you watch
+and the page you are reading are one product.</p>
+<hr />
+<h2 id="5-the-final-script-copy-paste-runnable">5. The final script (copy-paste runnable)</h2>
+<p>The complete <code>train.py</code> from sections 2–4. It runs three ways from one file —
+<code>python train.py</code>, <code>torchrun ... train.py</code>, or <code>flash.submit(fr_torch.ddp(
+...))</code> — and needs only a CPU:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">&quot;&quot;&quot;ConvNet tutorial — your model, unmodified, made fault-tolerant.
+    python train.py --steps 40                          # single process
+    torchrun --nproc-per-node=2 --standalone train.py   # DDP by hand
+    flash.submit(fr_torch.ddp(&quot;train.py&quot;, ...))         # operated by FlashRuntime
+Shapes: input 3x28x28 -&gt; Conv1(k5,p2) 28 -&gt; Avg 14 -&gt; Conv2(k5) 10 -&gt; Avg 5,
+so Flatten = 128 * 5 * 5 = 3200 = nn.Linear(128*5*5, num_outputs).
+&quot;&quot;&quot;
+import argparse
+import json
+import torch
+import torch.nn as nn
+from torch.utils.data import DataLoader, TensorDataset
+import flashruntime.torch as ft
+class OurConvNet(torch.nn.Module):
+    def __init__(self, num_outputs=20):
+        super(OurConvNet, self).__init__()
+        self.Conv1 = nn.Conv2d(3, 64, 5, 1, 2)
+        self.Sigma = nn.Sigmoid()
+        self.Avg = nn.AvgPool2d(2, stride=2, padding=0)
+        self.Conv2 = nn.Conv2d(64, 128, 5, 1)
+        self.Fl = nn.Flatten()
+        self.Linear = nn.Linear(128 * 5 * 5, num_outputs)
+    def forward(self, x):
+        out = self.Sigma(x)   # NOTE: Sigmoid applied to the INPUT first — kept verbatim.
+        out = self.Conv1(out)
+        out = self.Avg(out)
+        out = self.Conv2(out)
+        out = self.Avg(out)
+        out = self.Fl(out)
+        out = self.Linear(out)
+        return out
+def make_data(n=256, num_outputs=20, seed=0):
+    g = torch.Generator().manual_seed(seed)
+    x = torch.randn(n, 3, 28, 28, generator=g)
+    y = torch.randint(0, num_outputs, (n,), generator=g)
+    return TensorDataset(x, y)
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument(&quot;--steps&quot;, type=int, default=40)
+    parser.add_argument(&quot;--lr&quot;, type=float, default=0.05)
+    parser.add_argument(&quot;--checkpoint-every&quot;, type=int, default=8)
+    parser.add_argument(&quot;--kill-at-step&quot;, type=int, default=None,
+                        help=&quot;simulate a crash (fresh runs only; a resumed retry finishes)&quot;)
+    args = parser.parse_args()
+    torch.manual_seed(0)
+    model = OurConvNet(num_outputs=20)
+    optimizer = torch.optim.SGD(model.parameters(), lr=args.lr)
+    loader = DataLoader(make_data(), batch_size=32, shuffle=False)
+    model, optimizer, loader = ft.prepare(model, optimizer, loader)
+    start = ft.start_step()
+    step = start
+    loss = torch.tensor(0.0)
+    while step &lt; args.steps:
+        for x, y in loader:
+            if step &gt;= args.steps:
+                break
+            loss = torch.nn.functional.cross_entropy(model(x), y)
+            optimizer.zero_grad()
+            loss.backward()
+            optimizer.step()
+            step += 1
+            ft.checkpoint(model, optimizer, step=step, every=args.checkpoint_every)
+            ft.log_metrics({&quot;step&quot;: step, &quot;loss&quot;: round(loss.item(), 6)})
+            if args.kill_at_step and start == 0 and step &gt;= args.kill_at_step:
+                raise SystemExit(3)  # fresh run only — the retry resumes past this
+    ft.checkpoint(model, optimizer, step=step)  # final checkpoint
+    if ft.is_main():
+        metrics = {&quot;steps&quot;: step, &quot;resumed_from&quot;: start, &quot;final_loss&quot;: round(loss.item(), 6)}
+        with open(&quot;metrics.json&quot;, &quot;w&quot;) as f:
+            json.dump(metrics, f)
+        print(metrics)
+if __name__ == &quot;__main__&quot;:
+    main()
+</code></pre></div>
+<hr />
+<h2 id="where-to-go-next">Where to go next</h2>
+<ul>
+<li><strong><a href="fault-tolerance.html">Fault tolerance, in depth</a></strong> — how a crash becomes
+  signals, a class, and a typed recovery action.</li>
+<li><strong><a href="../guides/pytorch.html">PyTorch guide</a></strong> — the two launch paths (already-DDP
+  scripts vs. the <code>ft.</code> helper) and every caveat.</li>
+<li><strong><a href="../concepts/architecture.html">Architecture</a></strong> — the four axes, leases,
+  manifests, and recovery that make the above work.</li>
+</ul></main>
+</div>
+<script>
+// ---- client-side search: fetch the builder's index, filter as you type -----
+// (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
+let INDEX = [];
+// Path from THIS page back to the site root. The index stores root-relative
+// URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
+// against ROOT so a page in a subdir resolves them too (see build_docs.py).
+const ROOT = "../";
+const q = document.getElementById("q");
+const results = document.getElementById("results");
+const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
+  ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
+fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
+// A short context window around the first match, so a hit shows WHY it matched.
+function snippet(text, needle) {
+  const i = text.toLowerCase().indexOf(needle);
+  if (i < 0) return "";
+  const start = Math.max(0, i - 32);
+  return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
+}
+function runSearch() {
+  const needle = q.value.trim().toLowerCase();
+  if (!needle) { results.className = ""; results.innerHTML = ""; return; }
+  const hits = INDEX.map((p) => {
+    const inTitle = p.title.toLowerCase().includes(needle);
+    const inText = p.text.toLowerCase().includes(needle);
+    if (!inTitle && !inText) return null;
+    return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
+  }).filter(Boolean).slice(0, 20);
+  results.className = "open";
+  results.innerHTML = hits.length
+    ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
+        (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
+    : '<div class="nohit">no matches</div>';
+}
+q.addEventListener("input", runSearch);
+q.addEventListener("focus", runSearch);
+document.addEventListener("click", (e) => {
+  if (!e.target.closest(".search")) { results.className = ""; }  // dismiss on outside click
+});
+document.addEventListener("keydown", (e) => {  // "/" focuses search, like PyTorch docs
+  if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
+});
+// ---- copy buttons: read the <pre>'s own text into the clipboard ------------
+document.querySelectorAll(".copybtn").forEach((btn) => {
+  btn.addEventListener("click", () => {
+    const code = btn.parentElement.querySelector("pre").innerText;
+    navigator.clipboard.writeText(code).then(() => {
+      const was = btn.textContent; btn.textContent = "Copied";
+      setTimeout(() => { btn.textContent = was; }, 1200);
+    }).catch(() => {});
+  });
+});
+</script>
+</body>
+</html>