PyPI - flashruntime - Versions diffs - 0.3.0__py3-none-any.whl - Mend

flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (95) hide show

flashml_workloads/__init__.py +7 -0
flashml_workloads/fedavg_driver.py +569 -0
flashml_workloads/fedavg_weights.py +223 -0
flashml_workloads/fedavg_worker.py +166 -0
flashml_workloads/kmeans_driver.py +134 -0
flashml_workloads/kmeans_shard.py +69 -0
flashml_workloads/sgd_trainer.py +127 -0
flashml_workloads/sharded_kmeans.py +323 -0
flashml_workloads/sklearn_trial.py +89 -0
flashruntime/__init__.py +125 -0
flashruntime/artifacts/__init__.py +25 -0
flashruntime/artifacts/store.py +228 -0
flashruntime/backends/__init__.py +26 -0
flashruntime/backends/base.py +63 -0
flashruntime/backends/kuberay.py +465 -0
flashruntime/checkpoint/__init__.py +20 -0
flashruntime/checkpoint/catalog.py +198 -0
flashruntime/checkpoint/local.py +109 -0
flashruntime/checkpoint/store.py +86 -0
flashruntime/integrations/__init__.py +5 -0
flashruntime/integrations/huggingface.py +59 -0
flashruntime/integrations/pytorch.py +52 -0
flashruntime/integrations/sklearn.py +42 -0
flashruntime/launchers/__init__.py +130 -0
flashruntime/launchers/local.py +126 -0
flashruntime/leases/__init__.py +27 -0
flashruntime/leases/manager.py +365 -0
flashruntime/leases/sqlite_store.py +169 -0
flashruntime/leases/store.py +103 -0
flashruntime/monitor/__init__.py +7 -0
flashruntime/monitor/sampler.py +232 -0
flashruntime/planner/__init__.py +56 -0
flashruntime/planner/candidates.py +597 -0
flashruntime/planner/catalog.py +129 -0
flashruntime/planner/comm.py +95 -0
flashruntime/planner/explain.py +109 -0
flashruntime/planner/memory.py +166 -0
flashruntime/planner/resolve.py +120 -0
flashruntime/planner/selector.py +169 -0
flashruntime/planner/timecost.py +81 -0
flashruntime/profiling/__init__.py +113 -0
flashruntime/protocol/__init__.py +18 -0
flashruntime/protocol/plan_v1alpha1.py +320 -0
flashruntime/protocol/v1alpha1.py +465 -0
flashruntime/providers/__init__.py +138 -0
flashruntime/py.typed +0 -0
flashruntime/recipes/__init__.py +135 -0
flashruntime/recipes/command.py +166 -0
flashruntime/recovery/__init__.py +21 -0
flashruntime/recovery/policy.py +170 -0
flashruntime/recovery/signals.py +135 -0
flashruntime/recovery/taxonomy.py +91 -0
flashruntime/scheduler/__init__.py +170 -0
flashruntime/sdk.py +402 -0
flashruntime/service/__init__.py +3 -0
flashruntime/service/app.py +391 -0
flashruntime/service/auth.py +180 -0
flashruntime/service/checkpoints.py +90 -0
flashruntime/service/cli.py +167 -0
flashruntime/service/dashboard.py +193 -0
flashruntime/service/ledger.py +101 -0
flashruntime/service/modea.py +821 -0
flashruntime/strategies/__init__.py +156 -0
flashruntime/strategies/command.py +56 -0
flashruntime/torch/__init__.py +274 -0
flashruntime/viewer/__init__.py +20 -0
flashruntime/viewer/_docs/benchmarks.html +771 -0
flashruntime/viewer/_docs/concepts/architecture.html +302 -0
flashruntime/viewer/_docs/get-started.html +263 -0
flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
flashruntime/viewer/_docs/guides/huggingface.html +223 -0
flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
flashruntime/viewer/_docs/guides/pytorch.html +313 -0
flashruntime/viewer/_docs/guides/sklearn.html +232 -0
flashruntime/viewer/_docs/index.html +251 -0
flashruntime/viewer/_docs/reference/cli.html +254 -0
flashruntime/viewer/_docs/reference/integrations.html +240 -0
flashruntime/viewer/_docs/reference/sdk.html +341 -0
flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
flashruntime/viewer/_docs/search-index.json +1 -0
flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
flashruntime/viewer/flowmap.py +307 -0
flashruntime/viewer/page.py +594 -0
flashruntime/viewer/server.py +134 -0
flashruntime/viewer/state.py +250 -0
flashruntime/workloads/__init__.py +6 -0
flashruntime/workloads/command.py +127 -0
flashruntime-0.3.0.dist-info/METADATA +365 -0
flashruntime-0.3.0.dist-info/RECORD +95 -0
flashruntime-0.3.0.dist-info/WHEEL +5 -0
flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
flashruntime-0.3.0.dist-info/top_level.txt +2 -0

flashruntime/viewer/_docs/concepts/architecture.html ADDED Viewed

@@ -0,0 +1,302 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Architecture — flashruntime docs</title>
+<style>
+  :root { color-scheme: dark; }
+  * { box-sizing: border-box; margin: 0; }
+  body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
+  a { color: oklch(0.80 0.16 200); text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  .layout { display: flex; align-items: flex-start; min-height: 100vh; }
+  /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
+  .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
+             border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
+  .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
+  .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .search { position: relative; margin: 16px 0; }
+  #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
+       color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
+  #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
+  #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
+             background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
+  #results.open { display: block; }
+  #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
+  #results a:last-child { border-bottom: 0; }
+  #results a:hover { background: #010409; text-decoration: none; }
+  #results a b { color: #e6edf3; font-weight: 600; }
+  #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
+  #results .nohit { padding: 7px 9px; color: #8b949e; }
+  .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
+  .nav-link:hover { background: #161b22; text-decoration: none; }
+  .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
+  /* content */
+  .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
+  .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
+  .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
+                padding-top: 10px; border-top: 1px solid #21262d; }
+  .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
+  .content p, .content li { color: #c9d1d9; }
+  .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
+  .content li { margin: 4px 0; }
+  .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
+                        background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
+  .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
+                              border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
+  .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
+  .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
+  .content th { color: #e6edf3; background: #161b22; }
+  /* code blocks + copy button */
+  .codeblock { position: relative; margin: 14px 0; }
+  .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
+                   padding: 14px 16px; overflow-x: auto; }
+  .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
+  .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
+             background: #161b22; border: 1px solid #21262d; border-radius: 5px;
+             color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+  .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
+  @media (max-width: 800px) {
+    .layout { flex-direction: column; }
+    .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
+               border-right: 0; border-bottom: 1px solid #21262d; }
+    .content { padding: 24px 18px 60px; }
+  }
+</style>
+</head>
+<body>
+<div class="layout">
+  <aside class="sidebar">
+    <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
+    <div class="search">
+      <input id="q" type="search" placeholder="Search docs  (press /)" autocomplete="off" spellcheck="false">
+      <div id="results"></div>
+    </div>
+    <nav><div class="nav-section">Getting started</div>
+<a class="nav-link" href="../index.html">FlashRuntime</a>
+<a class="nav-link" href="../get-started.html">Get started</a>
+<div class="nav-section">Tutorials</div>
+<a class="nav-link" href="../tutorials/convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
+<a class="nav-link" href="../tutorials/sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
+<a class="nav-link" href="../tutorials/fault-tolerance.html">Tutorial: automatic recovery, explained</a>
+<div class="nav-section">Guides</div>
+<a class="nav-link" href="../guides/pytorch.html">PyTorch guide</a>
+<a class="nav-link" href="../guides/sklearn.html">scikit-learn guide</a>
+<a class="nav-link" href="../guides/huggingface.html">Hugging Face guide</a>
+<a class="nav-link" href="../guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
+<a class="nav-link" href="../guides/federated-averaging.html">Federated averaging</a>
+<div class="nav-section">Concepts</div>
+<a class="nav-link active" href="architecture.html">Architecture</a>
+<div class="nav-section">Reference</div>
+<a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
+<a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
+<a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
+<a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
+<div class="nav-section">Benchmarks</div>
+<a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
+  </aside>
+  <main class="content"><h1 id="architecture">Architecture</h1>
+<p>FlashRuntime plans, launches, observes, and recovers a distributed ML job. It
+<strong>never</strong> reimplements the distributed math — that always belongs to your
+framework (PyTorch DDP/FSDP, <code>torchrun</code>, Ray, Hugging Face). This page explains
+the shape of the system: the four axes it is built from, and the three
+mechanisms — leases, manifests, recovery — that make "runs to <em>verifiably</em>
+completed on unreliable machines" a real guarantee rather than a slogan.</p>
+<p>The design decisions here are recorded in ADR-0003 (<em>Reliability runtime first;
+planner as an explainable feasibility filter</em>).</p>
+<hr />
+<h2 id="the-four-orthogonal-axes">The four orthogonal axes</h2>
+<p>The central idea: <strong>getting machines, starting processes, configuring
+execution, and integrating user code are four independent concerns.</strong> Keeping
+them orthogonal is what lets the same job run on your laptop, on RunPod, or on a
+community pool without rewriting anything.</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>   ┌──────────────────────────────────────────────────────────────┐
+   │                     your training job                          │
+   └──────────────────────────────────────────────────────────────┘
+        │             │              │                  │
+        ▼             ▼              ▼                  ▼
+   ┌─────────┐   ┌──────────┐   ┌────────────┐   ┌──────────────┐
+   │providers│   │launchers │   │ strategies │   │   recipes    │
+   │─────────│   │──────────│   │────────────│   │──────────────│
+   │  get    │   │  start   │   │ configure  │   │  integrate   │
+   │machines │   │processes │   │ execution  │   │  user code   │
+   └─────────┘   └──────────┘   └────────────┘   └──────────────┘
+   RunPod,       torchrun,      DDP / FSDP2 /    PyTorch, sklearn,
+   local,        local proc,    zero3 offload,   Hugging Face
+   K8s pools     leased node    single-GPU       (this is the
+                                                  workload layer)
+</code></pre></div>
+<p>The rule that keeps them honest: <strong>Hugging Face / PyTorch code lives in the
+recipe (workload) layer — it is never a backend.</strong> The planner emits a
+backend-neutral <code>StrategyPlan</code> and never imports framework code (no <code>import
+transformers</code> / <code>torch.distributed</code> / <code>ray</code> inside the planner). Launching is
+orthogonal to the strategy your code chooses, which is why FlashRuntime can
+launch an FSDP or DeepSpeed script correctly without knowing anything about
+FSDP or DeepSpeed.</p>
+<p>The <code>integrations.*</code> adapters you use from the SDK (<code>fr_torch.ddp</code>,
+<code>fr_sklearn.sweep</code>, <code>fr_hf.trainer</code>) are the recipe axis in practice: each
+returns a <code>CommandWorkload</code> describing <em>what to run</em>, and the other three axes
+handle the rest.</p>
+<hr />
+<h2 id="leases-the-mode-a-reliability-core">Leases — the Mode A reliability core</h2>
+<p>A <strong>lease</strong> is a time-bounded right to run one task. It is the layer no
+existing distributed-ML library provides, so FlashRuntime builds it first
+(Mode A) before coordinated training (Mode B). A node <em>claims</em> a task, sends
+<em>heartbeats</em> to keep the lease alive, and <em>commits</em> the result idempotently; if
+the heartbeats stop, the lease <em>expires</em> and the task <em>requeues</em> — automatic
+recovery with no central decision required.</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>   task: PENDING
+      │  claim  (a node takes a time-bounded lease)
+      ▼
+   LEASED ──heartbeat──► LEASED ──heartbeat──► LEASED
+      │                                           │
+      │ no heartbeat within TTL                   │ commit (validated, idempotent)
+      ▼                                           ▼
+   EXPIRED ──requeue──► PENDING              COMPLETED
+</code></pre></div>
+<p>Key properties:</p>
+<ul>
+<li><strong>Status is derived from an append-only ledger of events</strong>, never a
+  hand-mutated field. "What is this job's state?" is always answered by
+  replaying events, so the answer is reproducible and auditable.</li>
+<li><strong>Commit is idempotent and validated.</strong> A result is accepted only if it
+  matches the task's expected commit key (a sha256), so a duplicate or corrupt
+  commit cannot poison the job.</li>
+<li><strong>Lease state is durable.</strong> In-flight leases survive a coordinator restart
+  (a SQLite-backed store); agents re-register on their own.</li>
+</ul>
+<hr />
+<h2 id="manifests-checkpoint-validity-by-construction">Manifests — checkpoint validity by construction</h2>
+<p>A checkpoint is only useful if you can trust it after a crash. FlashRuntime
+makes validity <strong>structural</strong> with a parts-first / manifest-last commit: the
+checkpoint's parts are written first, then — only after their hashes verify —
+the <strong>manifest</strong> is written last. No manifest, no checkpoint.</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>   write step-000123/
+     ├─ model.pt         (part)   ─┐
+     ├─ optimizer.pt     (part)    │  written FIRST
+     └─ ...                        ─┘
+                                    │  hashes verified
+     └─ manifest         ──────────┘  written LAST  ✔ now &quot;latest_valid&quot;
+   crash between parts and manifest  ⇒  no manifest  ⇒  never selected
+</code></pre></div>
+<p>So a half-written checkpoint — the exact thing a crash tends to produce — can
+never look valid. Recovery restores only a <strong>verified, topology-compatible</strong>
+manifest (the newest whose world size and framework match), which is why a
+resumed run continues correctly instead of loading garbage. <code>ft.checkpoint(...)</code>
+in the <a href="../reference/torch-helper.html">torch helper</a> writes under this contract;
+the Hugging Face callback commits Trainer checkpoints the same way.</p>
+<hr />
+<h2 id="recovery-typed-deterministic-logged">Recovery — typed, deterministic, logged</h2>
+<p>When something fails, recovery is a <strong>pure function of the evidence</strong>, not a
+judgment call. Raw signals are classified into one failure class, and the class
+(with the execution mode) is looked up in a versioned policy table that returns
+a typed action. There is no LLM, no scoring, no learned model — same failure +
+same policy version ⇒ same action, always.</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>   failure evidence            classify()              decide(class, mode)
+   ────────────────      ─────────────────────      ──────────────────────────
+   exit code,            precedence-ordered:         table lookup, versioned:
+   log tail,       ───►  systemic &gt; node &gt;     ───►  worker_crash + coordinated
+   heartbeat loss,       process &gt; app               → restart_group (from ckpt)
+   health signals        → one FailureClass          app_error → fail_job (fast)
+                                                      correlated → freeze_automation
+</code></pre></div>
+<p>The design commitments:</p>
+<ul>
+<li><strong>Deterministic application errors are never retried</strong> — fail fast and tell
+  the user. Burning capacity re-hitting a bug is the most expensive kind of
+  "recovery".</li>
+<li><strong>Blast radius depends on mode.</strong> A <code>worker_crash</code> costs one task retry in
+  <code>independent_tasks</code> mode but a whole-group restart in <code>coordinated_training</code>
+  (NCCL collective state is not repairable in place).</li>
+<li><strong>Correlated incidents freeze automation.</strong> The policy's most important
+  action is knowing when to <em>stop</em> acting — retry storms during a systemic
+  incident are how orchestrators destroy trust.</li>
+<li><strong>Every decision is logged</strong> with its failure class and human-readable
+  reason, and emitted as <code>FAILURE_CLASSIFIED</code> / <code>RECOVERY_ACTION_SELECTED</code>
+  events the live page and ledger both show.</li>
+</ul>
+<p>The <a href="../tutorials/fault-tolerance.html">fault-tolerance tutorial</a> walks one real
+crash through this pipeline end to end.</p>
+<hr />
+<h2 id="how-it-fits-together">How it fits together</h2>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>   plan  ──►  launch  ──►  observe  ──►  recover
+   (StrategyPlan,    (providers +   (leases +        (classify + decide,
+    explained)        launchers)     heartbeats +     typed actions from
+                                     manifests)       the policy table)
+</code></pre></div>
+<p>The runtime is the spine; the planner is an explainable feasibility filter that
+sits in front of it (<code>flash.plan()</code>), and the runtime's ledger is the planner's
+dataset. Everything above is usable <strong>without the cloud</strong> — a self-hosted local
+coordinator is a first-class mode, not a demo shim.</p>
+<p>See also: the <a href="../reference/sdk.html">SDK reference</a> for the entry points, and the
+<a href="../guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a> for the
+coordinator wire form.</p></main>
+</div>
+<script>
+// ---- client-side search: fetch the builder's index, filter as you type -----
+// (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
+let INDEX = [];
+// Path from THIS page back to the site root. The index stores root-relative
+// URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
+// against ROOT so a page in a subdir resolves them too (see build_docs.py).
+const ROOT = "../";
+const q = document.getElementById("q");
+const results = document.getElementById("results");
+const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
+  ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
+fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
+// A short context window around the first match, so a hit shows WHY it matched.
+function snippet(text, needle) {
+  const i = text.toLowerCase().indexOf(needle);
+  if (i < 0) return "";
+  const start = Math.max(0, i - 32);
+  return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
+}
+function runSearch() {
+  const needle = q.value.trim().toLowerCase();
+  if (!needle) { results.className = ""; results.innerHTML = ""; return; }
+  const hits = INDEX.map((p) => {
+    const inTitle = p.title.toLowerCase().includes(needle);
+    const inText = p.text.toLowerCase().includes(needle);
+    if (!inTitle && !inText) return null;
+    return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
+  }).filter(Boolean).slice(0, 20);
+  results.className = "open";
+  results.innerHTML = hits.length
+    ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
+        (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
+    : '<div class="nohit">no matches</div>';
+}
+q.addEventListener("input", runSearch);
+q.addEventListener("focus", runSearch);
+document.addEventListener("click", (e) => {
+  if (!e.target.closest(".search")) { results.className = ""; }  // dismiss on outside click
+});
+document.addEventListener("keydown", (e) => {  // "/" focuses search, like PyTorch docs
+  if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
+});
+// ---- copy buttons: read the <pre>'s own text into the clipboard ------------
+document.querySelectorAll(".copybtn").forEach((btn) => {
+  btn.addEventListener("click", () => {
+    const code = btn.parentElement.querySelector("pre").innerText;
+    navigator.clipboard.writeText(code).then(() => {
+      const was = btn.textContent; btn.textContent = "Copied";
+      setTimeout(() => { btn.textContent = was; }, 1200);
+    }).catch(() => {});
+  });
+});
+</script>
+</body>
+</html>

flashruntime/viewer/_docs/get-started.html ADDED Viewed

@@ -0,0 +1,263 @@
+<!doctype html>
+<html lang="en">
+<head>
+<meta charset="utf-8">
+<meta name="viewport" content="width=device-width, initial-scale=1">
+<title>Get started — flashruntime docs</title>
+<style>
+  :root { color-scheme: dark; }
+  * { box-sizing: border-box; margin: 0; }
+  body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
+  a { color: oklch(0.80 0.16 200); text-decoration: none; }
+  a:hover { text-decoration: underline; }
+  .layout { display: flex; align-items: flex-start; min-height: 100vh; }
+  /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
+  .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
+             border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
+  .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
+  .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .search { position: relative; margin: 16px 0; }
+  #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
+       color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
+  #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
+  #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
+             background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
+  #results.open { display: block; }
+  #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
+  #results a:last-child { border-bottom: 0; }
+  #results a:hover { background: #010409; text-decoration: none; }
+  #results a b { color: #e6edf3; font-weight: 600; }
+  #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
+  #results .nohit { padding: 7px 9px; color: #8b949e; }
+  .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
+                 text-transform: uppercase; letter-spacing: .14em; }
+  .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
+  .nav-link:hover { background: #161b22; text-decoration: none; }
+  .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
+  /* content */
+  .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
+  .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
+  .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
+                padding-top: 10px; border-top: 1px solid #21262d; }
+  .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
+  .content p, .content li { color: #c9d1d9; }
+  .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
+  .content li { margin: 4px 0; }
+  .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
+                        background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
+  .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
+                              border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
+  .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
+  .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
+  .content th { color: #e6edf3; background: #161b22; }
+  /* code blocks + copy button */
+  .codeblock { position: relative; margin: 14px 0; }
+  .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
+                   padding: 14px 16px; overflow-x: auto; }
+  .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
+  .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
+             background: #161b22; border: 1px solid #21262d; border-radius: 5px;
+             color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
+  .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
+  @media (max-width: 800px) {
+    .layout { flex-direction: column; }
+    .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
+               border-right: 0; border-bottom: 1px solid #21262d; }
+    .content { padding: 24px 18px 60px; }
+  }
+</style>
+</head>
+<body>
+<div class="layout">
+  <aside class="sidebar">
+    <a class="brand" href="index.html">flashruntime<small>documentation</small></a>
+    <div class="search">
+      <input id="q" type="search" placeholder="Search docs  (press /)" autocomplete="off" spellcheck="false">
+      <div id="results"></div>
+    </div>
+    <nav><div class="nav-section">Getting started</div>
+<a class="nav-link" href="index.html">FlashRuntime</a>
+<a class="nav-link active" href="get-started.html">Get started</a>
+<div class="nav-section">Tutorials</div>
+<a class="nav-link" href="tutorials/convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
+<a class="nav-link" href="tutorials/sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
+<a class="nav-link" href="tutorials/fault-tolerance.html">Tutorial: automatic recovery, explained</a>
+<div class="nav-section">Guides</div>
+<a class="nav-link" href="guides/pytorch.html">PyTorch guide</a>
+<a class="nav-link" href="guides/sklearn.html">scikit-learn guide</a>
+<a class="nav-link" href="guides/huggingface.html">Hugging Face guide</a>
+<a class="nav-link" href="guides/jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
+<a class="nav-link" href="guides/federated-averaging.html">Federated averaging</a>
+<div class="nav-section">Concepts</div>
+<a class="nav-link" href="concepts/architecture.html">Architecture</a>
+<div class="nav-section">Reference</div>
+<a class="nav-link" href="reference/sdk.html">Reference: SDK (`flashruntime`)</a>
+<a class="nav-link" href="reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
+<a class="nav-link" href="reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
+<a class="nav-link" href="reference/cli.html">Reference: CLI (`flashruntime`)</a>
+<div class="nav-section">Benchmarks</div>
+<a class="nav-link" href="benchmarks.html">Benchmarks</a></nav>
+  </aside>
+  <main class="content"><h1 id="get-started">Get started</h1>
+<p>This page takes you from an empty environment to a fault-tolerant job and your
+first 2-process DDP run — on CPU, with no cluster and no GPU. For the what and
+why, see the <strong><a href="index.html">overview</a></strong>.</p>
+<h2 id="install">Install</h2>
+<p>The core is deliberately tiny — <code>pip install flashruntime</code> brings <strong>only
+pydantic</strong>, and every core module (planner, leases, checkpoints, recovery, the
+<code>flash.submit()</code> SDK) works with zero infrastructure:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-bash">pip install flashruntime
+</code></pre></div>
+<p>Infrastructure integrations are opt-in extras, never core imports:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-bash">pip install &quot;flashruntime[service]&quot;   # the FastAPI coordinator + CLI
+pip install &quot;flashruntime[sklearn]&quot;   # numpy + scikit-learn for the sweep examples
+</code></pre></div>
+<p><strong>Torch is not a dependency.</strong> FlashRuntime <em>launches</em> PyTorch; it never
+imports it (the four-axes rule — launching is orthogonal to your framework). To
+run the DDP example below, install PyTorch yourself. A CPU-only build is enough
+— DDP works over the <code>gloo</code> backend with no GPU:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-bash">pip install torch   # CPU build is fine; gloo needs no CUDA
+</code></pre></div>
+<h2 id="your-first-run">Your first run</h2>
+<p><code>flash.submit()</code> operates any command. The one convention your script owes
+FlashRuntime is to write a <code>metrics.json</code> (a flat JSON object) into its working
+directory; FlashRuntime collects it and records it as a trial:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+run = flash.submit(flash.CommandWorkload(
+    command=&quot;python -c \&quot;import json; json.dump({'accuracy': 0.91}, open('metrics.json','w'))\&quot;&quot;,
+    source=flash.Source(path=&quot;.&quot;),
+))
+print(run.state.value)   # &quot;SUCCEEDED&quot;
+print(run.trials)        # [{'accuracy': 0.91}]
+print(run.artifacts)     # [PosixPath('.../metrics.json')]
+</code></pre></div>
+<p>Rerunning <code>flash.submit(workload, output_dir=...)</code> against the <strong>same</strong>
+<code>output_dir</code> reuses the job id, so a checkpointed script resumes instead of
+restarting. Pair that with <code>max_restarts=N</code> and a crash mid-run is recovered
+automatically from the last valid checkpoint.</p>
+<h2 id="your-first-ddp-run">Your first DDP run</h2>
+<p>The <code>integrations.pytorch</code> adapter builds the <code>torchrun</code> command for you. If
+your script already calls <code>torch.distributed.init_process_group()</code> and wraps
+its model in <code>DistributedDataParallel</code>, there are <strong>zero code changes</strong> — the
+adapter just launches it:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+from flashruntime.integrations import pytorch as fr_torch
+run = flash.submit(fr_torch.ddp(
+    &quot;train.py&quot;,
+    source=&quot;examples/user_pytorch_vanilla&quot;,
+    nproc_per_node=2,          # 2 processes on this host — gloo/CPU works
+    script_args=&quot;--steps 100&quot;,
+))
+print(run.state.value, run.trials)
+</code></pre></div>
+<p><code>ddp(script, *, source=".", nproc_per_node=2, nnodes=1, script_args="",
+env=None)</code> emits <code>torchrun --nproc-per-node=N --nnodes=1 --standalone
+--local-addr=127.0.0.1 &lt;script&gt; &lt;args&gt;</code>. The <code>--local-addr=127.0.0.1</code> pins the
+advertised rendezvous address to loopback (otherwise torchrun advertises
+<code>socket.getfqdn()</code>, which on some macOS DNS setups is unresolvable and the run
+hangs before spawning a process). <code>nproc_per_node=2</code> starts two worker
+processes that rendezvous on loopback and hand each rank its <code>RANK</code> /
+<code>WORLD_SIZE</code> / <code>LOCAL_RANK</code> — a real distributed run on a single machine, no
+GPU required.</p>
+<p><code>nnodes &gt; 1</code> raises <code>NotImplementedError</code> today: multi-node rendezvous is a
+launcher concern for a later slice. <code>--standalone</code> is single-node by
+definition.</p>
+<blockquote>
+<p>Want fault-tolerant checkpointing inside a script you <em>are</em> willing to touch?
+<code>import flashruntime.torch as ft</code> gives you <code>ft.prepare(...)</code>,
+<code>ft.checkpoint(...)</code>, and <code>ft.log_metrics(...)</code> — torch's own DDP wrapped
+under the parts-first / manifest-last checkpoint contract, so a killed run
+resumes with its final loss matching an uninterrupted run to 1e-6 (the
+e2e's assertion). It is optional sugar on the same launch-only
+contract, never required.</p>
+</blockquote>
+<p>The same <code>flashruntime.torch</code> path runs unchanged on GPUs — <code>ft.prepare</code>
+places each rank's model on its <code>cuda:N</code>, initializes the <code>nccl</code> backend, and
+restores checkpoints across the CUDA↔CPU boundary. <strong>Validated on real GPUs
+(2×RTX 4090, nccl) — 2026-07-23</strong> (torch 2.7.1+cu128, CUDA 12.8), covering the
+2-process nccl DDP run and a GPU kill-and-resume; see <code>tests/test_gpu_e2e.py</code>.</p>
+<h2 id="watch-it-run">Watch it run</h2>
+<p>Pass <code>watch=True</code> (or just run at an interactive terminal, where it is the
+default) and <code>flash.submit()</code> opens a live run page in your browser and prints
+its URL:</p>
+<div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
+run = flash.submit(
+    flash.CommandWorkload(command=&quot;python train.py&quot;, source=flash.Source(path=&quot;.&quot;)),
+    watch=True,
+)
+print(run.viewer_url)   # http://127.0.0.1:&lt;port&gt;
+</code></pre></div>
+<p>The page draws the run's topology, loss curve, verified checkpoints, and every
+recovery decision, refreshing every couple of seconds — served entirely from a
+loopback server with <strong>no external assets</strong>, so it renders with the network
+cut. These docs are served from that same viewer at <code>/docs</code>.</p></main>
+</div>
+<script>
+// ---- client-side search: fetch the builder's index, filter as you type -----
+// (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
+let INDEX = [];
+// Path from THIS page back to the site root. The index stores root-relative
+// URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
+// against ROOT so a page in a subdir resolves them too (see build_docs.py).
+const ROOT = "";
+const q = document.getElementById("q");
+const results = document.getElementById("results");
+const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
+  ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
+fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
+// A short context window around the first match, so a hit shows WHY it matched.
+function snippet(text, needle) {
+  const i = text.toLowerCase().indexOf(needle);
+  if (i < 0) return "";
+  const start = Math.max(0, i - 32);
+  return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
+}
+function runSearch() {
+  const needle = q.value.trim().toLowerCase();
+  if (!needle) { results.className = ""; results.innerHTML = ""; return; }
+  const hits = INDEX.map((p) => {
+    const inTitle = p.title.toLowerCase().includes(needle);
+    const inText = p.text.toLowerCase().includes(needle);
+    if (!inTitle && !inText) return null;
+    return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
+  }).filter(Boolean).slice(0, 20);
+  results.className = "open";
+  results.innerHTML = hits.length
+    ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
+        (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
+    : '<div class="nohit">no matches</div>';
+}
+q.addEventListener("input", runSearch);
+q.addEventListener("focus", runSearch);
+document.addEventListener("click", (e) => {
+  if (!e.target.closest(".search")) { results.className = ""; }  // dismiss on outside click
+});
+document.addEventListener("keydown", (e) => {  // "/" focuses search, like PyTorch docs
+  if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
+});
+// ---- copy buttons: read the <pre>'s own text into the clipboard ------------
+document.querySelectorAll(".copybtn").forEach((btn) => {
+  btn.addEventListener("click", () => {
+    const code = btn.parentElement.querySelector("pre").innerText;
+    navigator.clipboard.writeText(code).then(() => {
+      const was = btn.textContent; btn.textContent = "Copied";
+      setTimeout(() => { btn.textContent = was; }, 1200);
+    }).catch(() => {});
+  });
+});
+</script>
+</body>
+</html>