PyPI - flashruntime - Versions diffs - 0.3.0__py3-none-any.whl - Mend

flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (95) hide show

flashml_workloads/__init__.py +7 -0
flashml_workloads/fedavg_driver.py +569 -0
flashml_workloads/fedavg_weights.py +223 -0
flashml_workloads/fedavg_worker.py +166 -0
flashml_workloads/kmeans_driver.py +134 -0
flashml_workloads/kmeans_shard.py +69 -0
flashml_workloads/sgd_trainer.py +127 -0
flashml_workloads/sharded_kmeans.py +323 -0
flashml_workloads/sklearn_trial.py +89 -0
flashruntime/__init__.py +125 -0
flashruntime/artifacts/__init__.py +25 -0
flashruntime/artifacts/store.py +228 -0
flashruntime/backends/__init__.py +26 -0
flashruntime/backends/base.py +63 -0
flashruntime/backends/kuberay.py +465 -0
flashruntime/checkpoint/__init__.py +20 -0
flashruntime/checkpoint/catalog.py +198 -0
flashruntime/checkpoint/local.py +109 -0
flashruntime/checkpoint/store.py +86 -0
flashruntime/integrations/__init__.py +5 -0
flashruntime/integrations/huggingface.py +59 -0
flashruntime/integrations/pytorch.py +52 -0
flashruntime/integrations/sklearn.py +42 -0
flashruntime/launchers/__init__.py +130 -0
flashruntime/launchers/local.py +126 -0
flashruntime/leases/__init__.py +27 -0
flashruntime/leases/manager.py +365 -0
flashruntime/leases/sqlite_store.py +169 -0
flashruntime/leases/store.py +103 -0
flashruntime/monitor/__init__.py +7 -0
flashruntime/monitor/sampler.py +232 -0
flashruntime/planner/__init__.py +56 -0
flashruntime/planner/candidates.py +597 -0
flashruntime/planner/catalog.py +129 -0
flashruntime/planner/comm.py +95 -0
flashruntime/planner/explain.py +109 -0
flashruntime/planner/memory.py +166 -0
flashruntime/planner/resolve.py +120 -0
flashruntime/planner/selector.py +169 -0
flashruntime/planner/timecost.py +81 -0
flashruntime/profiling/__init__.py +113 -0
flashruntime/protocol/__init__.py +18 -0
flashruntime/protocol/plan_v1alpha1.py +320 -0
flashruntime/protocol/v1alpha1.py +465 -0
flashruntime/providers/__init__.py +138 -0
flashruntime/py.typed +0 -0
flashruntime/recipes/__init__.py +135 -0
flashruntime/recipes/command.py +166 -0
flashruntime/recovery/__init__.py +21 -0
flashruntime/recovery/policy.py +170 -0
flashruntime/recovery/signals.py +135 -0
flashruntime/recovery/taxonomy.py +91 -0
flashruntime/scheduler/__init__.py +170 -0
flashruntime/sdk.py +402 -0
flashruntime/service/__init__.py +3 -0
flashruntime/service/app.py +391 -0
flashruntime/service/auth.py +180 -0
flashruntime/service/checkpoints.py +90 -0
flashruntime/service/cli.py +167 -0
flashruntime/service/dashboard.py +193 -0
flashruntime/service/ledger.py +101 -0
flashruntime/service/modea.py +821 -0
flashruntime/strategies/__init__.py +156 -0
flashruntime/strategies/command.py +56 -0
flashruntime/torch/__init__.py +274 -0
flashruntime/viewer/__init__.py +20 -0
flashruntime/viewer/_docs/benchmarks.html +771 -0
flashruntime/viewer/_docs/concepts/architecture.html +302 -0
flashruntime/viewer/_docs/get-started.html +263 -0
flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
flashruntime/viewer/_docs/guides/huggingface.html +223 -0
flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
flashruntime/viewer/_docs/guides/pytorch.html +313 -0
flashruntime/viewer/_docs/guides/sklearn.html +232 -0
flashruntime/viewer/_docs/index.html +251 -0
flashruntime/viewer/_docs/reference/cli.html +254 -0
flashruntime/viewer/_docs/reference/integrations.html +240 -0
flashruntime/viewer/_docs/reference/sdk.html +341 -0
flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
flashruntime/viewer/_docs/search-index.json +1 -0
flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
flashruntime/viewer/flowmap.py +307 -0
flashruntime/viewer/page.py +594 -0
flashruntime/viewer/server.py +134 -0
flashruntime/viewer/state.py +250 -0
flashruntime/workloads/__init__.py +6 -0
flashruntime/workloads/command.py +127 -0
flashruntime-0.3.0.dist-info/METADATA +365 -0
flashruntime-0.3.0.dist-info/RECORD +95 -0
flashruntime-0.3.0.dist-info/WHEEL +5 -0
flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
flashruntime-0.3.0.dist-info/top_level.txt +2 -0

flashruntime/viewer/_docs/guides/jobspec-and-isolation.html ADDED Viewed

@@ -0,0 +1,271 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>JobSpec &amp; isolation guide — flashruntime docs</title>
+<style>
+  :root { color-scheme: dark; }
+  * { box-sizing: border-box; margin: 0; }
+  body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
+  a { color: oklch(0.80 0.16 200); text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  .layout { display: flex; align-items: flex-start; min-height: 100vh; }
+  /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
+  .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
+             border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
+  .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
+  .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .search { position: relative; margin: 16px 0; }
+  #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
+       color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
+  #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
+  #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
+             background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
+  #results.open { display: block; }
+  #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
+  #results a:last-child { border-bottom: 0; }
+  #results a:hover { background: #010409; text-decoration: none; }
+  #results a b { color: #e6edf3; font-weight: 600; }
+  #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
+  #results .nohit { padding: 7px 9px; color: #8b949e; }
+  .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
+  .nav-link:hover { background: #161b22; text-decoration: none; }
+  .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
+  /* content */
+  .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
+  .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
+  .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
+                padding-top: 10px; border-top: 1px solid #21262d; }
+  .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
+  .content p, .content li { color: #c9d1d9; }
+  .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
+  .content li { margin: 4px 0; }
+  .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
+                        background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
+  .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
+                              border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
+  .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
+  .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
+  .content th { color: #e6edf3; background: #161b22; }
+  /* code blocks + copy button */
+  .codeblock { position: relative; margin: 14px 0; }
+  .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
+                   padding: 14px 16px; overflow-x: auto; }
+  .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
+  .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
+             background: #161b22; border: 1px solid #21262d; border-radius: 5px;
+             color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+  .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
+  @media (max-width: 800px) {
+    .layout { flex-direction: column; }
+    .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
+               border-right: 0; border-bottom: 1px solid #21262d; }
+    .content { padding: 24px 18px 60px; }
+  }
+</style>
+</head>
+<body>
+<div class="layout">
+  <aside class="sidebar">
+    <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
+    <div class="search">
+      <input id="q" type="search" placeholder="Search docs  (press /)" autocomplete="off" spellcheck="false">
+      <div id="results"></div>
+    </div>
+    <nav><div class="nav-section">Getting started</div>
+<a class="nav-link" href="../index.html">FlashRuntime</a>
+<a class="nav-link" href="../get-started.html">Get started</a>
+<div class="nav-section">Tutorials</div>
+<a class="nav-link" href="../tutorials/convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
+<a class="nav-link" href="../tutorials/sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
+<a class="nav-link" href="../tutorials/fault-tolerance.html">Tutorial: automatic recovery, explained</a>
+<div class="nav-section">Guides</div>
+<a class="nav-link" href="pytorch.html">PyTorch guide</a>
+<a class="nav-link" href="sklearn.html">scikit-learn guide</a>
+<a class="nav-link" href="huggingface.html">Hugging Face guide</a>
+<a class="nav-link active" href="jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
+<a class="nav-link" href="federated-averaging.html">Federated averaging</a>
+<div class="nav-section">Concepts</div>
+<a class="nav-link" href="../concepts/architecture.html">Architecture</a>
+<div class="nav-section">Reference</div>
+<a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
+<a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
+<a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
+<a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
+<div class="nav-section">Benchmarks</div>
+<a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
+  </aside>
+  <main class="content"><h1 id="jobspec-isolation-guide">JobSpec &amp; isolation guide</h1>
+<p><code>flash.submit()</code> runs your workload on the <strong>local</strong> machine. To hand a command
+workload to a FlashRuntime <strong>coordinator</strong> — so nodes pull and run it under
+leases, heartbeats, and recovery — you compile it to the versioned wire form (a
+<code>JobSpec</code>) and POST it.</p>
+<p>This guide covers that compile step and the <strong>isolation tier</strong> that decides
+which machines a task is allowed to land on. For the local path, see the
+<a href="pytorch.html">PyTorch</a> and <a href="sklearn.html">scikit-learn</a> guides.</p>
+<hr />
+<h2 id="compile-to-a-jobspec">Compile to a JobSpec</h2>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">from flashruntime.workloads.command import to_jobspec
+from flashruntime.protocol.v1alpha1 import ImageSpec
+jobspec = to_jobspec(
+    workload,                                    # a CommandWorkload
+    name=&quot;my-sweep&quot;,
+    image=ImageSpec(repository=&quot;myrepo/trainer&quot;, tag=&quot;2026.07-a1b2c3&quot;),
+)
+# POST jobspec.model_dump() to  POST /v1alpha1/jobs
+# (or from the CLI:  flashruntime submit-spec spec.json)
+</code></pre></div>
+<p><code>to_jobspec(workload, name, image=None)</code> produces a
+<code>JobSpec{execution.backend: "leases", workload.type: "command"}</code>. <strong>A pinned
+image is required</strong> — remote runs must be reproducible, and the schema already
+rejects the tag <code>latest</code>. On the coordinator the <code>command</code> recipe expands the
+job into one <code>TaskSpec</code> per <code>task_params</code> entry (or a single task), each
+carrying an <code>argv</code> payload, its env, its <code>artifact://</code> inputs, and its
+isolation requirement.</p>
+<hr />
+<h2 id="isolation-tiers-fail-closed">Isolation tiers (fail closed)</h2>
+<p>Every command task carries an isolation tier from <code>workload.isolation</code>:</p>
+<table>
+<thead>
+<tr>
+<th>Tier</th>
+<th>Where it runs</th>
+<th>Meaning</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>standard</code> (default)</td>
+<td>your own machines, RunPod, trusted pools</td>
+<td>ordinary placement — runs anywhere</td>
+</tr>
+<tr>
+<td><code>sandboxed</code></td>
+<td>community / untrusted machines</td>
+<td>may only be leased to a node advertising <code>sandbox_capable is True</code></td>
+</tr>
+</tbody>
+</table>
+<p>The placement gate (<code>scheduler.IsolationAwarePlacement</code>) is <strong>fail-closed</strong> on
+the security-relevant field, per the schema-security rule:</p>
+<ul>
+<li>A node counts as capable <strong>only</strong> when <code>sandbox_capable is True</code> — a truthy
+  stand-in (the string <code>"false"</code>, <code>1</code>, <code>"yes"</code>) does <strong>not</strong> count.</li>
+<li>Any tier that is not <code>None</code> / <code>""</code> / <code>"standard"</code> (including a mistyped
+  <code>"Sandboxed"</code>) is treated as requiring capability — no silent downgrade.</li>
+<li>A <code>sandboxed</code> task never falls back to an uncapable node unless the workload
+  explicitly sets <code>isolation.allowFallback = True</code>.</li>
+</ul>
+<p>So a <code>sandboxed</code> task will sit unclaimed rather than land on a node that cannot
+isolate it. That is the intended behavior: unsafe placement fails closed.</p>
+<hr />
+<h2 id="what-runs-where-today">What runs where today</h2>
+<blockquote>
+<p><strong>Local SDK path — works now.</strong> <code>flash.submit()</code> runs sklearn sweeps,
+2-process CPU DDP (via <code>gloo</code>), and kill-and-resume from checkpoints on this
+machine. All three are proven by the example e2e tests.</p>
+<p><strong>Service-side command jobs — expansion, placement, and execution all
+work.</strong> POSTing a <code>to_jobspec()</code> workload expands it into leased tasks,
+places them fail-closed by isolation tier, and — with a FlashNode agent
+running <code>--runner argv</code> — executes the <code>argv</code> payload inside a hardened,
+network-isolated container and commits the result. <code>sandboxed</code> tasks are
+only ever placed on a node that advertises both <code>sandbox_capable</code> and
+<code>argv_capable</code>; see the repo's <code>docs/guides/donate-a-machine.md</code> for exactly
+what that container confines (and does not).</p>
+<p><strong>Later slices.</strong> Multi-node DDP (<code>nnodes &gt; 1</code> rendezvous — not available on
+volunteer nodes even later, since <code>--network none</code> rules out rendezvous),
+result verification for untrusted volunteer nodes, remote providers
+(RunPod) with source packaging (<code>git_revision</code>), and
+<code>flash.run(StrategyPlan)</code> wiring are open follow-ups.</p>
+</blockquote>
+<p>For how a leased task recovers when a node disappears, see the
+<a href="../tutorials/fault-tolerance.html">fault-tolerance tutorial</a> and the
+<a href="../concepts/architecture.html">architecture</a> page — the same failure taxonomy
+and policy table drive both the local path and the coordinator.</p>
+<hr />
+<h2 id="adding-another-framework">Adding another framework</h2>
+<p>Isolation and JobSpec compilation are <strong>framework-neutral</strong>: <code>to_jobspec</code>
+serializes any <code>CommandWorkload</code>, whatever built it (<code>fr_torch.ddp</code>,
+<code>fr_sklearn.sweep</code>, <code>fr_hf.trainer</code>, or one you hand-construct). So a new
+framework adapter (see the <a href="pytorch.html#adding-another-framework">PyTorch adapter</a>)
+gets coordinator submission and isolation-aware placement for free — it only
+has to return a <code>CommandWorkload</code>.</p>
+<hr />
+<h2 id="built-in-task-modules">Built-in task modules</h2>
+<p>Besides <code>command</code> workloads, the coordinator ships three allowlisted task
+modules under <code>flashml_workloads/</code> — <code>sklearn_trial</code> (hyperparameter
+trials), <code>kmeans_shard</code>/<code>kmeans_driver</code> (sharded K-means), and
+<code>sgd_trainer</code> (checkpointable SGD with bit-identical resume). They are
+reference workloads for the lease protocol, not a required path: they
+predate <code>command</code> workloads and remain the workspace e2e's proof fixtures.
+Their contract is documented in each module's docstring and in the repo's
+<code>AGENTS.md</code>.</p></main>
+</div>
+<script>
+// ---- client-side search: fetch the builder's index, filter as you type -----
+// (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
+let INDEX = [];
+// Path from THIS page back to the site root. The index stores root-relative
+// URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
+// against ROOT so a page in a subdir resolves them too (see build_docs.py).
+const ROOT = "../";
+const q = document.getElementById("q");
+const results = document.getElementById("results");
+const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
+  ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
+fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
+// A short context window around the first match, so a hit shows WHY it matched.
+function snippet(text, needle) {
+  const i = text.toLowerCase().indexOf(needle);
+  if (i < 0) return "";
+  const start = Math.max(0, i - 32);
+  return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
+}
+function runSearch() {
+  const needle = q.value.trim().toLowerCase();
+  if (!needle) { results.className = ""; results.innerHTML = ""; return; }
+  const hits = INDEX.map((p) => {
+    const inTitle = p.title.toLowerCase().includes(needle);
+    const inText = p.text.toLowerCase().includes(needle);
+    if (!inTitle && !inText) return null;
+    return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
+  }).filter(Boolean).slice(0, 20);
+  results.className = "open";
+  results.innerHTML = hits.length
+    ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
+        (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
+    : '<div class="nohit">no matches</div>';
+}
+q.addEventListener("input", runSearch);
+q.addEventListener("focus", runSearch);
+document.addEventListener("click", (e) => {
+  if (!e.target.closest(".search")) { results.className = ""; }  // dismiss on outside click
+});
+document.addEventListener("keydown", (e) => {  // "/" focuses search, like PyTorch docs
+  if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
+});
+// ---- copy buttons: read the <pre>'s own text into the clipboard ------------
+document.querySelectorAll(".copybtn").forEach((btn) => {
+  btn.addEventListener("click", () => {
+    const code = btn.parentElement.querySelector("pre").innerText;
+    navigator.clipboard.writeText(code).then(() => {
+      const was = btn.textContent; btn.textContent = "Copied";
+      setTimeout(() => { btn.textContent = was; }, 1200);
+    }).catch(() => {});
+  });
+});
+</script>
+</body>
+</html>

flashruntime/viewer/_docs/guides/pytorch.html ADDED Viewed

@@ -0,0 +1,313 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>PyTorch guide — flashruntime docs</title>
+<style>
+  :root { color-scheme: dark; }
+  * { box-sizing: border-box; margin: 0; }
+  body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
+  a { color: oklch(0.80 0.16 200); text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  .layout { display: flex; align-items: flex-start; min-height: 100vh; }
+  /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
+  .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
+             border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
+  .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
+  .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .search { position: relative; margin: 16px 0; }
+  #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
+       color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
+  #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
+  #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
+             background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
+  #results.open { display: block; }
+  #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
+  #results a:last-child { border-bottom: 0; }
+  #results a:hover { background: #010409; text-decoration: none; }
+  #results a b { color: #e6edf3; font-weight: 600; }
+  #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
+  #results .nohit { padding: 7px 9px; color: #8b949e; }
+  .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
+  .nav-link:hover { background: #161b22; text-decoration: none; }
+  .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
+  /* content */
+  .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
+  .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
+  .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
+                padding-top: 10px; border-top: 1px solid #21262d; }
+  .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
+  .content p, .content li { color: #c9d1d9; }
+  .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
+  .content li { margin: 4px 0; }
+  .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
+                        background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
+  .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
+                              border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
+  .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
+  .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
+  .content th { color: #e6edf3; background: #161b22; }
+  /* code blocks + copy button */
+  .codeblock { position: relative; margin: 14px 0; }
+  .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
+                   padding: 14px 16px; overflow-x: auto; }
+  .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
+  .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
+             background: #161b22; border: 1px solid #21262d; border-radius: 5px;
+             color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+  .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
+  @media (max-width: 800px) {
+    .layout { flex-direction: column; }
+    .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
+               border-right: 0; border-bottom: 1px solid #21262d; }
+    .content { padding: 24px 18px 60px; }
+  }
+</style>
+</head>
+<body>
+<div class="layout">
+  <aside class="sidebar">
+    <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
+    <div class="search">
+      <input id="q" type="search" placeholder="Search docs  (press /)" autocomplete="off" spellcheck="false">
+      <div id="results"></div>
+    </div>
+    <nav><div class="nav-section">Getting started</div>
+<a class="nav-link" href="../index.html">FlashRuntime</a>
+<a class="nav-link" href="../get-started.html">Get started</a>
+<div class="nav-section">Tutorials</div>
+<a class="nav-link" href="../tutorials/convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
+<a class="nav-link" href="../tutorials/sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
+<a class="nav-link" href="../tutorials/fault-tolerance.html">Tutorial: automatic recovery, explained</a>
+<div class="nav-section">Guides</div>
+<a class="nav-link active" href="pytorch.html">PyTorch guide</a>
+<a class="nav-link" href="sklearn.html">scikit-learn guide</a>
+<a class="nav-link" href="huggingface.html">Hugging Face guide</a>
+<a class="nav-link" href="jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
+<a class="nav-link" href="federated-averaging.html">Federated averaging</a>
+<div class="nav-section">Concepts</div>
+<a class="nav-link" href="../concepts/architecture.html">Architecture</a>
+<div class="nav-section">Reference</div>
+<a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
+<a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
+<a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
+<a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
+<div class="nav-section">Benchmarks</div>
+<a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
+  </aside>
+  <main class="content"><h1 id="pytorch-guide">PyTorch guide</h1>
+<p>FlashRuntime <strong>operates</strong> your PyTorch job — it never rewrites your model. You
+keep the framework, the model, the loop, and the loss you already have;
+FlashRuntime launches your command, injects the environment it promises, tracks
+metrics, validates checkpoints, retries on failure, and collects artifacts.</p>
+<p>This is ADR-0003's fourth axis in practice: <strong>recipes integrate user code</strong>.
+FlashRuntime plans, launches, observes, and recovers; the distributed math is
+always done by PyTorch (<code>torchrun</code>, DDP). There are two paths, and both operate
+<em>unmodified</em> torch code.</p>
+<p>For a full worked walkthrough, do the <a href="../tutorials/convnet.html">ConvNet tutorial</a>.</p>
+<hr />
+<h2 id="path-1-a-script-that-is-already-ddp-ready">Path 1 — a script that is already DDP-ready</h2>
+<p>If your script already calls <code>dist.init_process_group()</code> and wraps its model in
+<code>DistributedDataParallel</code> itself, there are <strong>zero code changes</strong>. The adapter
+just builds the <code>torchrun</code> command:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+from flashruntime.integrations import pytorch as fr_torch
+run = flash.submit(fr_torch.ddp(
+    &quot;train.py&quot;,
+    source=&quot;examples/user_pytorch_vanilla&quot;,
+    nproc_per_node=2,           # 2 processes on this host — gloo/CPU works, no GPU
+    script_args=&quot;--steps 100&quot;,
+))
+print(run.state.value, run.trials)
+</code></pre></div>
+<p><code>ddp(script, *, source=".", nproc_per_node=2, nnodes=1, script_args="",
+env=None)</code> emits <code>torchrun --nproc-per-node=N --nnodes=1 --standalone
+--local-addr=127.0.0.1 &lt;script&gt; &lt;args&gt;</code>. <code>--local-addr=127.0.0.1</code> pins the
+advertised rendezvous address to loopback (otherwise torchrun advertises
+<code>socket.getfqdn()</code>, which on some macOS DNS setups is unresolvable and the run
+hangs before spawning a process).</p>
+<p><code>nnodes &gt; 1</code> raises <code>NotImplementedError</code> today — multi-node rendezvous is a
+launcher concern for a later slice. <code>--standalone</code> is single-node by
+definition.</p>
+<hr />
+<h2 id="path-2-flashruntimetorch-the-optional-in-script-helper">Path 2 — <code>flashruntime.torch</code>, the optional in-script helper</h2>
+<p>For a script you <em>are</em> willing to touch, one import makes it both
+launch-anywhere and fault-tolerant, without rebuilding any framework machinery.
+The surface is three verbs plus read-only launch-fact accessors:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime.torch as ft
+model, optimizer, loader = ft.prepare(model, optimizer, loader)
+start = ft.start_step()                        # 0 fresh, &gt;0 after a resume
+...
+ft.checkpoint(model, optimizer, step=step, every=100)
+ft.log_metrics({&quot;step&quot;: step, &quot;loss&quot;: float(loss)})
+if ft.is_main():                               # ft.rank(), ft.world_size() too
+    ...
+</code></pre></div>
+<ul>
+<li><strong><code>prepare(model, optimizer=None, dataloader=None)</code></strong> — launched distributed
+  (<code>WORLD_SIZE &gt; 1</code>) it initializes torch's <em>own</em> process group (<code>nccl</code> on GPU,
+  <code>gloo</code> on CPU), wraps the model in <code>DistributedDataParallel</code>, and <strong>swaps the
+  DataLoader's sampler for a seed-0 <code>DistributedSampler</code></strong> so each rank sees a
+  disjoint, deterministically-shuffled shard. It then restores the newest
+  <em>valid</em> checkpoint manifest if one exists, setting the resume step. Launched
+  as plain <code>python train.py</code>, it is a no-op passthrough.</li>
+</ul>
+<p><strong>CUDA device placement is wired.</strong> <code>prepare</code> selects <code>nccl</code> when CUDA is
+  present, moves your model onto this rank's GPU <em>before</em> the DDP wrap, and
+  binds DDP with <code>device_ids</code> / <code>output_device</code> — so a single-GPU box "just
+  works" and you no longer call <code>model.to(device)</code> yourself. The everyday e2e
+  tests exercise CPU / <code>gloo</code>; real-GPU validation is tracked in the workspace
+  progress log. <strong>Multi-node DDP (<code>nnodes &gt; 1</code>) is a later slice.</strong></p>
+<p>One caveat: <code>prepare</code> rebuilds the DataLoader carrying over <code>batch_size</code>,
+  <code>collate_fn</code>, <code>num_workers</code>, and <code>drop_last</code> — <strong><code>shuffle</code> and <code>pin_memory</code>
+  are not carried over</strong> (the <code>DistributedSampler</code> owns shuffling, at seed 0).</p>
+<ul>
+<li>
+<p><strong><code>checkpoint(model, optimizer=None, *, step, every=None)</code></strong> — rank 0 writes
+  a checkpoint under the parts-first / manifest-last contract (the manifest is
+  written last, so a half-written checkpoint is never <code>latest_valid</code>). <code>every=N</code>
+  no-ops except on multiples of N. Every rank synchronizes on a barrier so no
+  one races past a partial write.</p>
+</li>
+<li>
+<p><strong><code>log_metrics(dict)</code></strong> — rank 0 appends one JSON record per call to
+  <code>metrics.jsonl</code> (streaming history; the live page's loss curve reads it). It
+  never raises — metrics must never kill training. This is <em>separate</em> from the
+  final <code>metrics.json</code> your script writes for <code>run.trials</code>.</p>
+</li>
+<li>
+<p><strong><code>start_step()</code> / <code>rank()</code> / <code>world_size()</code> / <code>is_main()</code> / <code>device()</code> /
+  <code>backend()</code></strong> — the small read-only helpers (<code>device()</code>/<code>backend()</code> let a
+  script report where it actually trained, e.g. into <code>metrics.json</code> — how
+  the GPU e2e proves the CUDA/nccl path). Full signatures in the
+  <a href="../reference/torch-helper.html">torch helper reference</a>.</p>
+</li>
+</ul>
+<p>The same file, three ways:</p>
+<table>
+<thead>
+<tr>
+<th>Command</th>
+<th>What runs</th>
+</tr>
+</thead>
+<tbody>
+<tr>
+<td><code>python train.py --steps 200</code></td>
+<td>single process, <code>prepare</code> is a passthrough</td>
+</tr>
+<tr>
+<td><code>torchrun --nproc-per-node=2 --standalone train.py</code></td>
+<td>DDP by hand</td>
+</tr>
+<tr>
+<td><code>flash.submit(fr_torch.ddp("train.py", ...))</code></td>
+<td>operated by FlashRuntime</td>
+</tr>
+</tbody>
+</table>
+<h3 id="determinism-bit-exact-resume">Determinism / bit-exact resume</h3>
+<p>Keep the script deterministic on CPU (fixed seeds; the seed-0
+<code>DistributedSampler</code> repeats its order every epoch) and a killed-and-resumed
+run reproduces the uninterrupted result — recovery must not change the math.
+There is one alignment constraint: on resume the <code>for</code> loop restarts the
+dataloader at batch 0, so the resumed step must land on an <strong>epoch boundary</strong> —
+a multiple of batches-per-rank-per-epoch. Keep <code>--checkpoint-every</code> a multiple
+of that.</p>
+<blockquote>
+<p><strong>One <code>output_dir</code> is one workload.</strong> Resume works by reusing the job-scoped
+checkpoint tree under <code>output_dir</code>. Point a <em>different</em> workload at an
+<code>output_dir</code> that already holds another workload's checkpoints and
+<code>prepare()</code> will happily restore those foreign weights — silent wrong
+results, not an error. Use a <strong>fresh <code>output_dir</code> per workload</strong>; reusing one
+for the <em>same</em> workload is exactly how kill-and-resume is meant to work.
+(Fan-out sweeps are safe automatically: each trial gets its own checkpoint
+tree.)</p>
+</blockquote>
+<h3 id="guardrail-adr-0003-do-not-rebuild-accelerate">Guardrail (ADR-0003 — do not rebuild Accelerate)</h3>
+<p><code>flashruntime.torch</code> wraps torch's <em>own</em> DDP and stops. There are no FSDP
+policies, no autocast, no DeepSpeed config in this surface. Users who want those
+use the real framework features directly — the launcher still launches such a
+script correctly, because launching is orthogonal to the strategy your code
+chooses.</p>
+<hr />
+<h2 id="adding-another-framework">Adding another framework</h2>
+<p>The PyTorch adapter is ~50 lines: it builds a <code>torchrun</code> command string and
+returns a <code>CommandWorkload</code>. That is the whole extensibility pattern — a new
+framework adapter is a small function under <code>flashruntime/integrations/</code> that
+returns a <code>CommandWorkload</code>, reusing the same launch/collect/recover machinery.
+<code>integrations.huggingface</code> is literally a thin wrapper over this
+<code>ddp()</code> — see the <a href="huggingface.html">Hugging Face guide</a> — and
+<code>integrations.sklearn</code> is the same shape for the fan-out case. No core change is
+needed to teach FlashRuntime a new framework; you describe <em>what to run</em>, and
+the four axes handle the rest.</p></main>
+</div>
+<script>
+// ---- client-side search: fetch the builder's index, filter as you type -----
+// (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
+let INDEX = [];
+// Path from THIS page back to the site root. The index stores root-relative
+// URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
+// against ROOT so a page in a subdir resolves them too (see build_docs.py).
+const ROOT = "../";
+const q = document.getElementById("q");
+const results = document.getElementById("results");
+const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
+  ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
+fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
+// A short context window around the first match, so a hit shows WHY it matched.
+function snippet(text, needle) {
+  const i = text.toLowerCase().indexOf(needle);
+  if (i < 0) return "";
+  const start = Math.max(0, i - 32);
+  return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
+}
+function runSearch() {
+  const needle = q.value.trim().toLowerCase();
+  if (!needle) { results.className = ""; results.innerHTML = ""; return; }
+  const hits = INDEX.map((p) => {
+    const inTitle = p.title.toLowerCase().includes(needle);
+    const inText = p.text.toLowerCase().includes(needle);
+    if (!inTitle && !inText) return null;
+    return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
+  }).filter(Boolean).slice(0, 20);
+  results.className = "open";
+  results.innerHTML = hits.length
+    ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
+        (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
+    : '<div class="nohit">no matches</div>';
+}
+q.addEventListener("input", runSearch);
+q.addEventListener("focus", runSearch);
+document.addEventListener("click", (e) => {
+  if (!e.target.closest(".search")) { results.className = ""; }  // dismiss on outside click
+});
+document.addEventListener("keydown", (e) => {  // "/" focuses search, like PyTorch docs
+  if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
+});
+// ---- copy buttons: read the <pre>'s own text into the clipboard ------------
+document.querySelectorAll(".copybtn").forEach((btn) => {
+  btn.addEventListener("click", () => {
+    const code = btn.parentElement.querySelector("pre").innerText;
+    navigator.clipboard.writeText(code).then(() => {
+      const was = btn.textContent; btn.textContent = "Copied";
+      setTimeout(() => { btn.textContent = was; }, 1200);
+    }).catch(() => {});
+  });
+});
+</script>
+</body>
+</html>