flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. flashml_workloads/__init__.py +7 -0
  2. flashml_workloads/fedavg_driver.py +569 -0
  3. flashml_workloads/fedavg_weights.py +223 -0
  4. flashml_workloads/fedavg_worker.py +166 -0
  5. flashml_workloads/kmeans_driver.py +134 -0
  6. flashml_workloads/kmeans_shard.py +69 -0
  7. flashml_workloads/sgd_trainer.py +127 -0
  8. flashml_workloads/sharded_kmeans.py +323 -0
  9. flashml_workloads/sklearn_trial.py +89 -0
  10. flashruntime/__init__.py +125 -0
  11. flashruntime/artifacts/__init__.py +25 -0
  12. flashruntime/artifacts/store.py +228 -0
  13. flashruntime/backends/__init__.py +26 -0
  14. flashruntime/backends/base.py +63 -0
  15. flashruntime/backends/kuberay.py +465 -0
  16. flashruntime/checkpoint/__init__.py +20 -0
  17. flashruntime/checkpoint/catalog.py +198 -0
  18. flashruntime/checkpoint/local.py +109 -0
  19. flashruntime/checkpoint/store.py +86 -0
  20. flashruntime/integrations/__init__.py +5 -0
  21. flashruntime/integrations/huggingface.py +59 -0
  22. flashruntime/integrations/pytorch.py +52 -0
  23. flashruntime/integrations/sklearn.py +42 -0
  24. flashruntime/launchers/__init__.py +130 -0
  25. flashruntime/launchers/local.py +126 -0
  26. flashruntime/leases/__init__.py +27 -0
  27. flashruntime/leases/manager.py +365 -0
  28. flashruntime/leases/sqlite_store.py +169 -0
  29. flashruntime/leases/store.py +103 -0
  30. flashruntime/monitor/__init__.py +7 -0
  31. flashruntime/monitor/sampler.py +232 -0
  32. flashruntime/planner/__init__.py +56 -0
  33. flashruntime/planner/candidates.py +597 -0
  34. flashruntime/planner/catalog.py +129 -0
  35. flashruntime/planner/comm.py +95 -0
  36. flashruntime/planner/explain.py +109 -0
  37. flashruntime/planner/memory.py +166 -0
  38. flashruntime/planner/resolve.py +120 -0
  39. flashruntime/planner/selector.py +169 -0
  40. flashruntime/planner/timecost.py +81 -0
  41. flashruntime/profiling/__init__.py +113 -0
  42. flashruntime/protocol/__init__.py +18 -0
  43. flashruntime/protocol/plan_v1alpha1.py +320 -0
  44. flashruntime/protocol/v1alpha1.py +465 -0
  45. flashruntime/providers/__init__.py +138 -0
  46. flashruntime/py.typed +0 -0
  47. flashruntime/recipes/__init__.py +135 -0
  48. flashruntime/recipes/command.py +166 -0
  49. flashruntime/recovery/__init__.py +21 -0
  50. flashruntime/recovery/policy.py +170 -0
  51. flashruntime/recovery/signals.py +135 -0
  52. flashruntime/recovery/taxonomy.py +91 -0
  53. flashruntime/scheduler/__init__.py +170 -0
  54. flashruntime/sdk.py +402 -0
  55. flashruntime/service/__init__.py +3 -0
  56. flashruntime/service/app.py +391 -0
  57. flashruntime/service/auth.py +180 -0
  58. flashruntime/service/checkpoints.py +90 -0
  59. flashruntime/service/cli.py +167 -0
  60. flashruntime/service/dashboard.py +193 -0
  61. flashruntime/service/ledger.py +101 -0
  62. flashruntime/service/modea.py +821 -0
  63. flashruntime/strategies/__init__.py +156 -0
  64. flashruntime/strategies/command.py +56 -0
  65. flashruntime/torch/__init__.py +274 -0
  66. flashruntime/viewer/__init__.py +20 -0
  67. flashruntime/viewer/_docs/benchmarks.html +771 -0
  68. flashruntime/viewer/_docs/concepts/architecture.html +302 -0
  69. flashruntime/viewer/_docs/get-started.html +263 -0
  70. flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
  71. flashruntime/viewer/_docs/guides/huggingface.html +223 -0
  72. flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
  73. flashruntime/viewer/_docs/guides/pytorch.html +313 -0
  74. flashruntime/viewer/_docs/guides/sklearn.html +232 -0
  75. flashruntime/viewer/_docs/index.html +251 -0
  76. flashruntime/viewer/_docs/reference/cli.html +254 -0
  77. flashruntime/viewer/_docs/reference/integrations.html +240 -0
  78. flashruntime/viewer/_docs/reference/sdk.html +341 -0
  79. flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
  80. flashruntime/viewer/_docs/search-index.json +1 -0
  81. flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
  82. flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
  83. flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
  84. flashruntime/viewer/flowmap.py +307 -0
  85. flashruntime/viewer/page.py +594 -0
  86. flashruntime/viewer/server.py +134 -0
  87. flashruntime/viewer/state.py +250 -0
  88. flashruntime/workloads/__init__.py +6 -0
  89. flashruntime/workloads/command.py +127 -0
  90. flashruntime-0.3.0.dist-info/METADATA +365 -0
  91. flashruntime-0.3.0.dist-info/RECORD +95 -0
  92. flashruntime-0.3.0.dist-info/WHEEL +5 -0
  93. flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
  94. flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
  95. flashruntime-0.3.0.dist-info/top_level.txt +2 -0
flashruntime/viewer/_docs/guides/federated-averaging.html ADDED
@@ -0,0 +1,363 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>Federated averaging — flashruntime docs</title>
7
+ <style>
8
+ :root { color-scheme: dark; }
9
+ * { box-sizing: border-box; margin: 0; }
10
+ body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
11
+ a { color: oklch(0.80 0.16 200); text-decoration: none; }
12
+ a:hover { text-decoration: underline; }
13
+ .layout { display: flex; align-items: flex-start; min-height: 100vh; }
14
+
15
+ /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
16
+ .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
17
+ border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
18
+ .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
19
+ .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
20
+ text-transform: uppercase; letter-spacing: .14em; }
21
+ .search { position: relative; margin: 16px 0; }
22
+ #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
23
+ color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
24
+ #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
25
+ #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
26
+ background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
27
+ #results.open { display: block; }
28
+ #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
29
+ #results a:last-child { border-bottom: 0; }
30
+ #results a:hover { background: #010409; text-decoration: none; }
31
+ #results a b { color: #e6edf3; font-weight: 600; }
32
+ #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
33
+ #results .nohit { padding: 7px 9px; color: #8b949e; }
34
+ .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
35
+ text-transform: uppercase; letter-spacing: .14em; }
36
+ .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
37
+ .nav-link:hover { background: #161b22; text-decoration: none; }
38
+ .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
39
+
40
+ /* content */
41
+ .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
42
+ .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
43
+ .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
44
+ padding-top: 10px; border-top: 1px solid #21262d; }
45
+ .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
46
+ .content p, .content li { color: #c9d1d9; }
47
+ .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
48
+ .content li { margin: 4px 0; }
49
+ .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
50
+ background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
51
+ .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
52
+ border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
53
+ .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
54
+ .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
55
+ .content th { color: #e6edf3; background: #161b22; }
56
+
57
+ /* code blocks + copy button */
58
+ .codeblock { position: relative; margin: 14px 0; }
59
+ .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
60
+ padding: 14px 16px; overflow-x: auto; }
61
+ .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
62
+ .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
63
+ background: #161b22; border: 1px solid #21262d; border-radius: 5px;
64
+ color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
65
+ .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
66
+
67
+ @media (max-width: 800px) {
68
+ .layout { flex-direction: column; }
69
+ .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
70
+ border-right: 0; border-bottom: 1px solid #21262d; }
71
+ .content { padding: 24px 18px 60px; }
72
+ }
73
+ </style>
74
+ </head>
75
+ <body>
76
+ <div class="layout">
77
+ <aside class="sidebar">
78
+ <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
79
+ <div class="search">
80
+ <input id="q" type="search" placeholder="Search docs (press /)" autocomplete="off" spellcheck="false">
81
+ <div id="results"></div>
82
+ </div>
83
+ <nav><div class="nav-section">Getting started</div>
84
+ <a class="nav-link" href="../index.html">FlashRuntime</a>
85
+ <a class="nav-link" href="../get-started.html">Get started</a>
86
+ <div class="nav-section">Tutorials</div>
87
+ <a class="nav-link" href="../tutorials/convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
88
+ <a class="nav-link" href="../tutorials/sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
89
+ <a class="nav-link" href="../tutorials/fault-tolerance.html">Tutorial: automatic recovery, explained</a>
90
+ <div class="nav-section">Guides</div>
91
+ <a class="nav-link" href="pytorch.html">PyTorch guide</a>
92
+ <a class="nav-link" href="sklearn.html">scikit-learn guide</a>
93
+ <a class="nav-link" href="huggingface.html">Hugging Face guide</a>
94
+ <a class="nav-link" href="jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
95
+ <a class="nav-link active" href="federated-averaging.html">Federated averaging</a>
96
+ <div class="nav-section">Concepts</div>
97
+ <a class="nav-link" href="../concepts/architecture.html">Architecture</a>
98
+ <div class="nav-section">Reference</div>
99
+ <a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
100
+ <a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
101
+ <a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
102
+ <a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
103
+ <div class="nav-section">Benchmarks</div>
104
+ <a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
105
+ </aside>
106
+ <main class="content"><h1 id="federated-averaging">Federated averaging</h1>
107
+ <p>Federated averaging (FedAvg) is FlashRuntime's answer to a question the
108
+ <a href="pytorch.html">PyTorch guide</a> and the
109
+ <a href="jobspec-and-isolation.html">JobSpec &amp; isolation guide</a> both run into: what do
110
+ you do when the machines that want to help train a model <strong>cannot talk to
111
+ each other</strong>? Volunteer nodes run their task containers with <code>--network
112
+ none</code> (see the repo's <code>docs/guides/donate-a-machine.md</code>) — no LAN, no
113
+ internet, no way for one container to find another. Coordinated
114
+ multi-process training (DDP, FSDP) needs the opposite: every rank must
115
+ rendezvous with every other rank over a process group before the first
116
+ <code>all_reduce</code>. On a volunteer pool those two requirements are irreconcilable,
117
+ so FlashRuntime does not attempt coordinated training there at all.</p>
118
+ <p>FedAvg sidesteps the rendezvous problem by never requiring it. Instead of
119
+ ranks synchronizing gradients mid-step, <strong>rounds</strong> synchronize whole models
120
+ between steps:</p>
121
+ <ol>
122
+ <li>The driver broadcasts the current weights as a plain <code>artifact://</code> blob.</li>
123
+ <li>Each participating node downloads the weights, trains <strong>independently</strong>
124
+ for a fixed number of local steps on its own data shard, and uploads a
125
+ weight <strong>delta</strong> (not the new weights — see below).</li>
126
+ <li>Once enough deltas have committed, the driver averages them, applies the
127
+ result to the broadcast weights, and starts the next round.</li>
128
+ </ol>
129
+ <p>No node ever needs to see another node's IP address, let alone open a
130
+ connection to it. Every cross-node interaction is a <code>PUT</code>/<code>GET</code> against the
131
+ coordinator's artifact store, which is exactly the same shape of traffic a
132
+ volunteer node already does to pull its task inputs and push its results.
133
+ That is why this is a <strong>round loop implemented as a driver chaining
134
+ ordinary lease jobs</strong> (<code>flashml_workloads/fedavg_driver.py</code>,
135
+ <code>flashml_workloads/fedavg_worker.py</code>) — the same "pipelines are jobs chained
136
+ by a driver, not a new execution mode" pattern as the sharded-k-means POC —
137
+ rather than a new backend.</p>
138
+ <h2 id="why-a-delta-not-the-new-weights">Why a delta, not the new weights</h2>
139
+ <p>Each worker's task uploads <code>delta.json</code> (the change it made to the weights
140
+ it started from) alongside <code>metrics.json</code>. The driver averages <strong>deltas</strong>,
141
+ not raw weight snapshots, because a delta is a direction that stays
142
+ meaningful even if the weights it was computed against are no longer the
143
+ newest ones — the exact situation a straggling volunteer produces when it
144
+ finally reports in after the round has moved on. Averaging final weights
145
+ directly would require every worker to have started from the <em>same</em>
146
+ snapshot; averaging deltas only requires knowing what each worker started
147
+ from, which the driver already does.</p>
148
+ <h2 id="the-quorum-rule-and-why-late-deltas-are-discarded">The quorum rule, and why late deltas are discarded</h2>
149
+ <p><code>kmeans_driver</code> (the other job-chaining driver in this codebase) requires
150
+ <strong>every</strong> dispatched shard to report before it aggregates. FedAvg
151
+ deliberately does not: <code>run_fedavg(..., min_participants=N)</code> aggregates as
152
+ soon as <code>N</code> of the round's shards have committed, not when all of them have.</p>
153
+ <p>This is not a shortcut — it is the correct policy for volunteer compute.
154
+ Machines that donate spare cycles are unequal and unreliable by
155
+ construction: laptops close, Wi-Fi drops, a slow machine might still be on
156
+ local step 3 when a fast one has already finished. Requiring all of them
157
+ before a round can proceed would let a single closed laptop stall every
158
+ other participant's contribution indefinitely. Quorum aggregation lets the
159
+ round move on as soon as it has a statistically meaningful sample.</p>
160
+ <p>The corollary is what makes quorum aggregation <em>safe</em> rather than merely
161
+ convenient: once the driver has read the quorum's deltas and applied them,
162
+ <strong>any delta that commits afterward for that round is discarded</strong>, never
163
+ folded into a later round. <code>run_fedavg</code> freezes the participant set at the
164
+ moment quorum is reached and never re-reads that job's artifacts again
165
+ (<code>fedavg_driver.py</code>, <code>run_fedavg</code>). A late delta was computed against
166
+ weights that no longer exist by the time it arrives — the model has already
167
+ moved past them — and applying it on top of a newer round's weights would
168
+ not be "one more contribution," it would silently corrupt the average with
169
+ a step that was never actually taken from the current state. Discarding is
170
+ the honest behavior; a driver that tried to be more "inclusive" here would
171
+ be quietly wrong instead.</p>
172
+ <p><code>tests/test_fedavg_convergence.py::test_round_completes_on_quorum_when_a_node_never_reports</code>
173
+ pins exactly this: three shards are dispatched but the test's agent pool is
174
+ capped to exactly two successful claims and then stops claiming, so the
175
+ third shard is never bound to any node and sits PENDING for the life of the
176
+ test. The round still aggregates on the two that committed — with an exact
177
+ <code>participants == 2</code> assertion — rather than hanging until the deadline
178
+ waiting for the shard nobody was ever going to serve. (The cap on claims,
179
+ not the node count, is what makes the third shard genuinely abandoned:
180
+ either registered node can claim either shard, so without the cap both
181
+ nodes could sequentially serve all three before the driver's poll notices
182
+ quorum.)</p>
183
+ <h2 id="what-counts-as-a-participant-and-what-the-driver-refuses">What counts as a participant, and what the driver refuses</h2>
184
+ <p>Everything a volunteer node produces — the delta, the sample count, the
185
+ metrics file, the filenames — is attacker-controlled input. Result
186
+ <em>verification</em> (catching a node that lies about a delta it honestly
187
+ computed) is a later milestone, but input validation and containment are
188
+ not deferred:</p>
189
+ <ul>
190
+ <li><strong>A participant is an accepted commit, not an uploaded file.</strong> The driver
191
+ counts only keys that exactly match the round's dispatched task set
192
+ (<code>jobs/{job_id}/shard-{i:03d}/metrics.json</code> for <code>i &lt; num_shards</code>), and
193
+ cross-checks them against the tasks the coordinator reports <code>COMPLETED</code>
194
+ (<code>GET /v1alpha1/jobs/{id}/tasks</code>). Both halves are load-bearing: the agent
195
+ uploads a task's output tree recursively, so a nested <code>out/a/metrics.json</code>
196
+ would otherwise mint a second participant from one lease; and uploads
197
+ happen <em>before</em> the commit is offered, so an attempt the coordinator
198
+ rejected (lost lease, sha256 mismatch) would otherwise still be averaged
199
+ in.</li>
200
+ <li><strong>Sample counts must be positive.</strong> Validating only the total is not
201
+ enough — <code>(delta=-999, n=-999)</code> plus <code>(delta=1.0, n=1000)</code> totals a
202
+ healthy 1 sample but yields a weight of <code>999001.0</code> where the honest step
203
+ is <code>1.0</code>. A sample-weighted mean is only a convex combination when every
204
+ count is positive.</li>
205
+ <li><strong>NaN and Inf are rejected, not averaged.</strong> Python's <code>json</code> both emits and
206
+ parses <code>NaN</code>/<code>Infinity</code>, and NaN is absorbing: one non-finite value turns
207
+ every weight NaN, and every later round then trains from NaN while the run
208
+ still reports success. This one needs no attacker — a learning rate that
209
+ diverges on one shard does it. <code>fedavg_weights</code> fails closed on any
210
+ non-finite value entering the reduce or leaving <code>apply_delta</code>/<code>subtract</code>,
211
+ naming the parameter and index.</li>
212
+ <li><strong><code>lease_seconds</code> is bounded</strong> (<code>modea.MAX_LEASE_SECONDS</code>, one hour). A
213
+ lease deadline is the only thing that returns an abandoned task to the
214
+ queue, so <code>1e9</code> would pin a shard to a closed laptop for ~31 years and
215
+ <code>inf</code> overflows <code>timedelta</code> inside the coordinator's claim path.</li>
216
+ </ul>
217
+ <p>Artifact <code>PUT</code> is now authenticated and lease-scoped when the coordinator
218
+ sets <code>FLASHML_NODE_TOKENS</code> (the per-machine-token slice): a node token can
219
+ only write under <code>jobs/{job}/{task}/</code> for a task it currently holds a live
220
+ lease on. The round-weights key
221
+ (<code>jobs/{job_id}/round-{round:03d}/weights.json</code>) belongs to no task and no
222
+ node's lease, so a plain node token cannot write it — the driver instead
223
+ authenticates with an <strong>operator token</strong> (<code>FLASHML_OPERATOR_TOKENS</code>), which
224
+ is attributable but deliberately not lease-scoped, exactly because drivers
225
+ are legitimate writers outside any lease (see
226
+ <code>docs/guides/donate-a-machine.md</code>). Result verification is still a separate,
227
+ unbuilt concern: this scoping stops an unrelated node from <em>overwriting</em> the
228
+ round weights, not from a participant lying about the delta it honestly
229
+ computed.</p>
230
+ <h2 id="the-flashmlyaml-shape">The <code>flashml.yaml</code> shape</h2>
231
+ <p>A federated-averaging round is submitted as an ordinary lease-mode job:</p>
232
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-yaml">apiVersion: flashml.dev/v1alpha1
233
+ kind: Job
234
+ metadata:
235
+ name: fedavg-r000
236
+ spec:
237
+ execution:
238
+ backend: leases
239
+ image:
240
+ repository: local/tier1
241
+ tag: dev
242
+ workload:
243
+ type: federated_averaging
244
+ parameters:
245
+ round: 0
246
+ num_shards: 2
247
+ local_steps: 20
248
+ lr: 0.1
249
+ batch_size: 16
250
+ seed: 0
251
+ in_dim: 8
252
+ hidden: 16
253
+ out_dim: 2
254
+ dataset_size: 256
255
+ # weights: artifact://jobs/&lt;prev-job&gt;/round-000/weights.json
256
+ # (omitted on round 0 — each worker seeds its own model from `seed`)
257
+ </code></pre></div>
258
+ <p><code>isolation.tier</code> is left at its default, <code>"standard"</code>, deliberately: unlike
259
+ the <code>argv</code> runner tier for arbitrary bring-your-code jobs, a
260
+ <code>federated_averaging</code> task's payload is a fixed, trusted <code>module</code> execution
261
+ (<code>flashml_workloads.fedavg_worker</code>), so it does not need the sandboxed argv
262
+ path and its <code>argv_capable</code> gate. A node only needs <code>module_capable</code>
263
+ (fail-open — absent counts as capable) to be eligible. <code>run_fedavg</code> builds
264
+ this JobSpec once per round and submits it as a new job
265
+ (<code>flashml_workloads/fedavg_driver.py:_round_body</code>) — the round number is
266
+ the only thing that changes between the driver's own resume points. The
267
+ image and isolation tier are <code>run_fedavg</code> parameters
268
+ (<code>image=</code>, <code>isolation_tier=</code>); the defaults above are this repo's e2e
269
+ fixture image, which only works because <code>SubprocessRunner</code> ignores <code>image</code>
270
+ entirely — a docker-tier volunteer needs a real, pullable reference.</p>
271
+ <h2 id="what-this-proves-and-what-it-does-not">What this proves — and what it does not</h2>
272
+ <p><code>tests/test_fedavg_convergence.py</code> runs this loop against a <strong>real</strong>
273
+ coordinator over real HTTP: real job expansion, real leases, real local
274
+ artifact storage (<code>FLASHML_LOCAL_ARTIFACTS_DIR</code>), and real commit-time
275
+ sha256 validation on every uploaded artifact. Two independent worker
276
+ "agents" (a few lines of <code>urllib</code>, standing in for <code>flashnode work</code> — see
277
+ the test file's docstring for why an in-repo test cannot import <code>flashnode</code>
278
+ directly) pull leases, train, and commit without ever talking to each
279
+ other. The measured per-round mean loss across four rounds with two
280
+ participating nodes:</p>
281
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code>round 0 participants 2/2 mean_loss 0.5361
282
+ round 1 participants 2/2 mean_loss 0.3781
283
+ round 2 participants 2/2 mean_loss 0.2548
284
+ round 3 participants 2/2 mean_loss 0.1757
285
+ converged: 0.5361 -&gt; 0.1757 over 4 rounds
286
+ </code></pre></div>
287
+ <p>(<code>scripts/fedavg_local_demo.py</code> reproduces this and exits non-zero if the
288
+ final round's loss is not below the first — a demo that prints numbers
289
+ nobody checks is not evidence.)</p>
290
+ <p>Read that number correctly: <strong>this proves collaborative training, not
291
+ faster training.</strong> Two nodes did not finish training in half the wall-clock
292
+ time of one — they trained <em>sequentially</em> through four rounds, each doing
293
+ its own local steps, and the loss came down because their independently
294
+ computed updates were combined. Nothing here claims a throughput or
295
+ speed-up result; DDP/FSDP make that claim, on a coordinated pool that can
296
+ rendezvous, and that claim is out of scope for volunteer nodes entirely (the
297
+ repo's <code>docs/guides/donate-a-machine.md</code> has the full list of things the
298
+ volunteer pool does not attempt, including "no coordinated multi-process
299
+ training"). What FedAvg proves is that machines which cannot see or trust
300
+ each other — and
301
+ in the volunteer case, cannot even reach each other over the network — can
302
+ still jointly move one model's loss in the right direction, coordinated
303
+ entirely through the coordinator's leases and artifact store.</p></main>
304
+ </div>
305
+
306
+ <script>
307
+ // ---- client-side search: fetch the builder's index, filter as you type -----
308
+ // (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
309
+ let INDEX = [];
310
+ // Path from THIS page back to the site root. The index stores root-relative
311
+ // URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
312
+ // against ROOT so a page in a subdir resolves them too (see build_docs.py).
313
+ const ROOT = "../";
314
+ const q = document.getElementById("q");
315
+ const results = document.getElementById("results");
316
+ const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
317
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
318
+ fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
319
+
320
+ // A short context window around the first match, so a hit shows WHY it matched.
321
+ function snippet(text, needle) {
322
+ const i = text.toLowerCase().indexOf(needle);
323
+ if (i < 0) return "";
324
+ const start = Math.max(0, i - 32);
325
+ return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
326
+ }
327
+ function runSearch() {
328
+ const needle = q.value.trim().toLowerCase();
329
+ if (!needle) { results.className = ""; results.innerHTML = ""; return; }
330
+ const hits = INDEX.map((p) => {
331
+ const inTitle = p.title.toLowerCase().includes(needle);
332
+ const inText = p.text.toLowerCase().includes(needle);
333
+ if (!inTitle && !inText) return null;
334
+ return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
335
+ }).filter(Boolean).slice(0, 20);
336
+ results.className = "open";
337
+ results.innerHTML = hits.length
338
+ ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
339
+ (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
340
+ : '<div class="nohit">no matches</div>';
341
+ }
342
+ q.addEventListener("input", runSearch);
343
+ q.addEventListener("focus", runSearch);
344
+ document.addEventListener("click", (e) => {
345
+ if (!e.target.closest(".search")) { results.className = ""; } // dismiss on outside click
346
+ });
347
+ document.addEventListener("keydown", (e) => { // "/" focuses search, like PyTorch docs
348
+ if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
349
+ });
350
+
351
+ // ---- copy buttons: read the <pre>'s own text into the clipboard ------------
352
+ document.querySelectorAll(".copybtn").forEach((btn) => {
353
+ btn.addEventListener("click", () => {
354
+ const code = btn.parentElement.querySelector("pre").innerText;
355
+ navigator.clipboard.writeText(code).then(() => {
356
+ const was = btn.textContent; btn.textContent = "Copied";
357
+ setTimeout(() => { btn.textContent = was; }, 1200);
358
+ }).catch(() => {});
359
+ });
360
+ });
361
+ </script>
362
+ </body>
363
+ </html>
flashruntime/viewer/_docs/guides/huggingface.html ADDED
@@ -0,0 +1,223 @@
1
+ <!doctype html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="utf-8">
5
+ <meta name="viewport" content="width=device-width, initial-scale=1">
6
+ <title>Hugging Face guide — flashruntime docs</title>
7
+ <style>
8
+ :root { color-scheme: dark; }
9
+ * { box-sizing: border-box; margin: 0; }
10
+ body { font: 14px/1.65 ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; background: #0d1117; color: #c9d1d9; }
11
+ a { color: oklch(0.80 0.16 200); text-decoration: none; }
12
+ a:hover { text-decoration: underline; }
13
+ .layout { display: flex; align-items: flex-start; min-height: 100vh; }
14
+
15
+ /* sidebar — PyTorch-docs-like: sections from _nav.yml, current page active */
16
+ .sidebar { flex: 0 0 264px; position: sticky; top: 0; height: 100vh; overflow-y: auto;
17
+ border-right: 1px solid #21262d; background: #0d1117; padding: 20px 16px; }
18
+ .brand { color: #e6edf3; font-size: 15px; letter-spacing: .04em; }
19
+ .brand small { display: block; margin-top: 2px; color: #8b949e; font-size: 10px;
20
+ text-transform: uppercase; letter-spacing: .14em; }
21
+ .search { position: relative; margin: 16px 0; }
22
+ #q { width: 100%; background: #010409; border: 1px solid #21262d; border-radius: 6px;
23
+ color: #c9d1d9; font: 12px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; padding: 7px 9px; }
24
+ #q:focus { outline: 1px solid oklch(0.80 0.16 200); }
25
+ #results { position: absolute; left: 0; right: 0; top: 112%; z-index: 5; display: none;
26
+ background: #161b22; border: 1px solid #21262d; border-radius: 6px; overflow: hidden; }
27
+ #results.open { display: block; }
28
+ #results a { display: block; padding: 7px 9px; border-bottom: 1px solid #21262d; color: #c9d1d9; }
29
+ #results a:last-child { border-bottom: 0; }
30
+ #results a:hover { background: #010409; text-decoration: none; }
31
+ #results a b { color: #e6edf3; font-weight: 600; }
32
+ #results a span { display: block; margin-top: 2px; color: #8b949e; font-size: 11px; }
33
+ #results .nohit { padding: 7px 9px; color: #8b949e; }
34
+ .nav-section { margin: 16px 0 6px; color: #8b949e; font-size: 10px; font-weight: 600;
35
+ text-transform: uppercase; letter-spacing: .14em; }
36
+ .nav-link { display: block; padding: 4px 8px; border-radius: 5px; color: #c9d1d9; font-size: 13px; }
37
+ .nav-link:hover { background: #161b22; text-decoration: none; }
38
+ .nav-link.active { background: #161b22; color: #e6edf3; box-shadow: inset 2px 0 0 oklch(0.80 0.16 200); }
39
+
40
+ /* content */
41
+ .content { flex: 1 1 auto; max-width: 840px; min-width: 0; padding: 34px 40px 80px; }
42
+ .content h1 { color: #e6edf3; font-size: 26px; margin: 0 0 16px; }
43
+ .content h2 { color: #e6edf3; font-size: 18px; margin: 30px 0 10px;
44
+ padding-top: 10px; border-top: 1px solid #21262d; }
45
+ .content h3 { color: #e6edf3; font-size: 15px; margin: 22px 0 8px; }
46
+ .content p, .content li { color: #c9d1d9; }
47
+ .content ul, .content ol { padding-left: 22px; margin: 10px 0; }
48
+ .content li { margin: 4px 0; }
49
+ .content blockquote { margin: 14px 0; padding: 2px 14px; border-left: 3px solid oklch(0.80 0.18 60);
50
+ background: #161b22; border-radius: 0 6px 6px 0; color: #8b949e; }
51
+ .content :not(pre) > code { background: #010409; border: 1px solid #21262d;
52
+ border-radius: 4px; padding: 1px 5px; font-size: 12.5px; }
53
+ .content table { border-collapse: collapse; margin: 14px 0; display: block; overflow-x: auto; }
54
+ .content th, .content td { border: 1px solid #21262d; padding: 6px 10px; text-align: left; }
55
+ .content th { color: #e6edf3; background: #161b22; }
56
+
57
+ /* code blocks + copy button */
58
+ .codeblock { position: relative; margin: 14px 0; }
59
+ .codeblock pre { background: #010409; border: 1px solid #21262d; border-radius: 8px;
60
+ padding: 14px 16px; overflow-x: auto; }
61
+ .codeblock pre code { padding: 0; border: 0; background: none; font-size: 12.5px; color: #c9d1d9; }
62
+ .copybtn { position: absolute; top: 8px; right: 8px; padding: 3px 8px; cursor: pointer;
63
+ background: #161b22; border: 1px solid #21262d; border-radius: 5px;
64
+ color: #8b949e; font: 11px ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
65
+ .copybtn:hover { color: #e6edf3; border-color: oklch(0.80 0.16 200); }
66
+
67
+ @media (max-width: 800px) {
68
+ .layout { flex-direction: column; }
69
+ .sidebar { position: static; height: auto; width: 100%; flex-basis: auto;
70
+ border-right: 0; border-bottom: 1px solid #21262d; }
71
+ .content { padding: 24px 18px 60px; }
72
+ }
73
+ </style>
74
+ </head>
75
+ <body>
76
+ <div class="layout">
77
+ <aside class="sidebar">
78
+ <a class="brand" href="../index.html">flashruntime<small>documentation</small></a>
79
+ <div class="search">
80
+ <input id="q" type="search" placeholder="Search docs (press /)" autocomplete="off" spellcheck="false">
81
+ <div id="results"></div>
82
+ </div>
83
+ <nav><div class="nav-section">Getting started</div>
84
+ <a class="nav-link" href="../index.html">FlashRuntime</a>
85
+ <a class="nav-link" href="../get-started.html">Get started</a>
86
+ <div class="nav-section">Tutorials</div>
87
+ <a class="nav-link" href="../tutorials/convnet.html">Tutorial: make a ConvNet fault-tolerant</a>
88
+ <a class="nav-link" href="../tutorials/sklearn-sweeps.html">Tutorial: parallel scikit-learn sweeps</a>
89
+ <a class="nav-link" href="../tutorials/fault-tolerance.html">Tutorial: automatic recovery, explained</a>
90
+ <div class="nav-section">Guides</div>
91
+ <a class="nav-link" href="pytorch.html">PyTorch guide</a>
92
+ <a class="nav-link" href="sklearn.html">scikit-learn guide</a>
93
+ <a class="nav-link active" href="huggingface.html">Hugging Face guide</a>
94
+ <a class="nav-link" href="jobspec-and-isolation.html">JobSpec &amp; isolation guide</a>
95
+ <a class="nav-link" href="federated-averaging.html">Federated averaging</a>
96
+ <div class="nav-section">Concepts</div>
97
+ <a class="nav-link" href="../concepts/architecture.html">Architecture</a>
98
+ <div class="nav-section">Reference</div>
99
+ <a class="nav-link" href="../reference/sdk.html">Reference: SDK (`flashruntime`)</a>
100
+ <a class="nav-link" href="../reference/integrations.html">Reference: integrations (`flashruntime.integrations`)</a>
101
+ <a class="nav-link" href="../reference/torch-helper.html">Reference: torch helper (`flashruntime.torch`)</a>
102
+ <a class="nav-link" href="../reference/cli.html">Reference: CLI (`flashruntime`)</a>
103
+ <div class="nav-section">Benchmarks</div>
104
+ <a class="nav-link" href="../benchmarks.html">Benchmarks</a></nav>
105
+ </aside>
106
+ <main class="content"><h1 id="hugging-face-guide">Hugging Face guide</h1>
107
+ <p>FlashRuntime <strong>operates</strong> your Hugging Face job — it never rewrites your
108
+ <code>Trainer</code>. HF Trainer already wraps DDP/FSDP internally when it is launched by
109
+ <code>torchrun</code>, so <em>launching</em> an HF job is just the <a href="pytorch.html">PyTorch path</a>.
110
+ What <code>integrations.huggingface</code> adds is the <strong>callback seam</strong> that commits
111
+ Trainer checkpoints as verified manifests and relays Trainer metrics.</p>
112
+ <p><code>transformers</code> is imported only inside your training process — never in
113
+ FlashRuntime's core.</p>
114
+ <hr />
115
+ <h2 id="launching">Launching</h2>
116
+ <p><code>trainer(script, *, source=".", nproc_per_node=1, script_args="")</code> is a thin
117
+ wrapper over the PyTorch <code>ddp()</code> adapter, so everything in the
118
+ <a href="pytorch.html">PyTorch guide</a> about launching and multi-process applies
119
+ unchanged:</p>
120
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">import flashruntime as flash
121
+ from flashruntime.integrations import huggingface as fr_hf
122
+
123
+ run = flash.submit(fr_hf.trainer(
124
+ &quot;train_hf.py&quot;,
125
+ source=&quot;~/hf-project&quot;,
126
+ nproc_per_node=1,
127
+ script_args=&quot;--model_name_or_path bert-base-uncased&quot;,
128
+ ))
129
+ </code></pre></div>
130
+ <hr />
131
+ <h2 id="the-callback-seam">The callback seam</h2>
132
+ <p>Inside your training script, wire the callback and the resume in the usual HF
133
+ way:</p>
134
+ <div class="codeblock"><button class="copybtn" type="button" aria-label="Copy code to clipboard">Copy</button><pre><code class="language-python">from flashruntime.integrations import huggingface as fr_hf
135
+
136
+ trainer.add_callback(fr_hf.flashruntime_callback()) # on_save -&gt; manifest, on_log -&gt; metrics
137
+
138
+ resume = fr_hf.latest_checkpoint(training_args.output_dir) # newest VALID checkpoint dir, or None
139
+ trainer.train(resume_from_checkpoint=resume)
140
+ </code></pre></div>
141
+ <ul>
142
+ <li><strong><code>flashruntime_callback()</code></strong> builds a <code>TrainerCallback</code> whose <code>on_save</code>
143
+ writes a verified manifest for <code>checkpoint-&lt;step&gt;/</code> (rank 0 only) and whose
144
+ <code>on_log</code> relays metrics through <code>flashruntime.torch.log_metrics</code>. The
145
+ <code>transformers</code> import is paid inside this factory, in your process.</li>
146
+ <li><strong><code>latest_checkpoint(output_dir)</code></strong> returns the storage prefix of the newest
147
+ checkpoint dir with a <em>valid</em> manifest (<code>None</code> means fresh start) — pass it
148
+ straight to <code>resume_from_checkpoint</code>.</li>
149
+ </ul>
150
+ <p>So a Trainer run gets the same verified, parts-first / manifest-last checkpoint
151
+ guarantee as a hand-written loop: a half-written checkpoint is never selected
152
+ for resume, because the manifest is written last.</p>
153
+ <hr />
154
+ <h2 id="adding-another-framework">Adding another framework</h2>
155
+ <p><code>integrations.huggingface</code> is the smallest possible adapter: <code>trainer(...)</code>
156
+ delegates straight to <code>ddp(...)</code>, and the only HF-specific code is the callback
157
+ that maps <code>on_save</code>/<code>on_log</code> onto FlashRuntime's manifest and metric contracts.
158
+ That is the extensibility pattern — teach FlashRuntime a new framework by
159
+ writing a small adapter under <code>flashruntime/integrations/</code> that (a) returns a
160
+ <code>CommandWorkload</code> describing what to launch, and (b), if the framework has its
161
+ own callback/hook system, maps those hooks onto <code>write_manifest</code> /
162
+ <code>log_metrics</code>. No core change is needed. See the
163
+ <a href="pytorch.html#adding-another-framework">PyTorch adapter</a> for the base case.</p></main>
164
+ </div>
165
+
166
+ <script>
167
+ // ---- client-side search: fetch the builder's index, filter as you type -----
168
+ // (<=60 lines, vanilla JS, no external anything — the index is a sibling file.)
169
+ let INDEX = [];
170
+ // Path from THIS page back to the site root. The index stores root-relative
171
+ // URLs (e.g. "tutorials/deep.html"); we fetch it and build result links
172
+ // against ROOT so a page in a subdir resolves them too (see build_docs.py).
173
+ const ROOT = "../";
174
+ const q = document.getElementById("q");
175
+ const results = document.getElementById("results");
176
+ const esc = (s) => String(s).replace(/[&<>"]/g, (c) =>
177
+ ({ "&": "&amp;", "<": "&lt;", ">": "&gt;", '"': "&quot;" }[c]));
178
+ fetch(ROOT + "search-index.json").then((r) => r.json()).then((d) => { INDEX = d; }).catch(() => {});
179
+
180
+ // A short context window around the first match, so a hit shows WHY it matched.
181
+ function snippet(text, needle) {
182
+ const i = text.toLowerCase().indexOf(needle);
183
+ if (i < 0) return "";
184
+ const start = Math.max(0, i - 32);
185
+ return (start > 0 ? "…" : "") + text.slice(start, i + needle.length + 44).trim() + "…";
186
+ }
187
+ function runSearch() {
188
+ const needle = q.value.trim().toLowerCase();
189
+ if (!needle) { results.className = ""; results.innerHTML = ""; return; }
190
+ const hits = INDEX.map((p) => {
191
+ const inTitle = p.title.toLowerCase().includes(needle);
192
+ const inText = p.text.toLowerCase().includes(needle);
193
+ if (!inTitle && !inText) return null;
194
+ return { url: p.url, title: p.title, snip: inText ? snippet(p.text, needle) : "" };
195
+ }).filter(Boolean).slice(0, 20);
196
+ results.className = "open";
197
+ results.innerHTML = hits.length
198
+ ? hits.map((h) => '<a href="' + ROOT + h.url + '"><b>' + esc(h.title) + "</b>" +
199
+ (h.snip ? "<span>" + esc(h.snip) + "</span>" : "") + "</a>").join("")
200
+ : '<div class="nohit">no matches</div>';
201
+ }
202
+ q.addEventListener("input", runSearch);
203
+ q.addEventListener("focus", runSearch);
204
+ document.addEventListener("click", (e) => {
205
+ if (!e.target.closest(".search")) { results.className = ""; } // dismiss on outside click
206
+ });
207
+ document.addEventListener("keydown", (e) => { // "/" focuses search, like PyTorch docs
208
+ if (e.key === "/" && document.activeElement !== q) { e.preventDefault(); q.focus(); }
209
+ });
210
+
211
+ // ---- copy buttons: read the <pre>'s own text into the clipboard ------------
212
+ document.querySelectorAll(".copybtn").forEach((btn) => {
213
+ btn.addEventListener("click", () => {
214
+ const code = btn.parentElement.querySelector("pre").innerText;
215
+ navigator.clipboard.writeText(code).then(() => {
216
+ const was = btn.textContent; btn.textContent = "Copied";
217
+ setTimeout(() => { btn.textContent = was; }, 1200);
218
+ }).catch(() => {});
219
+ });
220
+ });
221
+ </script>
222
+ </body>
223
+ </html>