flashruntime 0.3.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (95) hide show
  1. flashml_workloads/__init__.py +7 -0
  2. flashml_workloads/fedavg_driver.py +569 -0
  3. flashml_workloads/fedavg_weights.py +223 -0
  4. flashml_workloads/fedavg_worker.py +166 -0
  5. flashml_workloads/kmeans_driver.py +134 -0
  6. flashml_workloads/kmeans_shard.py +69 -0
  7. flashml_workloads/sgd_trainer.py +127 -0
  8. flashml_workloads/sharded_kmeans.py +323 -0
  9. flashml_workloads/sklearn_trial.py +89 -0
  10. flashruntime/__init__.py +125 -0
  11. flashruntime/artifacts/__init__.py +25 -0
  12. flashruntime/artifacts/store.py +228 -0
  13. flashruntime/backends/__init__.py +26 -0
  14. flashruntime/backends/base.py +63 -0
  15. flashruntime/backends/kuberay.py +465 -0
  16. flashruntime/checkpoint/__init__.py +20 -0
  17. flashruntime/checkpoint/catalog.py +198 -0
  18. flashruntime/checkpoint/local.py +109 -0
  19. flashruntime/checkpoint/store.py +86 -0
  20. flashruntime/integrations/__init__.py +5 -0
  21. flashruntime/integrations/huggingface.py +59 -0
  22. flashruntime/integrations/pytorch.py +52 -0
  23. flashruntime/integrations/sklearn.py +42 -0
  24. flashruntime/launchers/__init__.py +130 -0
  25. flashruntime/launchers/local.py +126 -0
  26. flashruntime/leases/__init__.py +27 -0
  27. flashruntime/leases/manager.py +365 -0
  28. flashruntime/leases/sqlite_store.py +169 -0
  29. flashruntime/leases/store.py +103 -0
  30. flashruntime/monitor/__init__.py +7 -0
  31. flashruntime/monitor/sampler.py +232 -0
  32. flashruntime/planner/__init__.py +56 -0
  33. flashruntime/planner/candidates.py +597 -0
  34. flashruntime/planner/catalog.py +129 -0
  35. flashruntime/planner/comm.py +95 -0
  36. flashruntime/planner/explain.py +109 -0
  37. flashruntime/planner/memory.py +166 -0
  38. flashruntime/planner/resolve.py +120 -0
  39. flashruntime/planner/selector.py +169 -0
  40. flashruntime/planner/timecost.py +81 -0
  41. flashruntime/profiling/__init__.py +113 -0
  42. flashruntime/protocol/__init__.py +18 -0
  43. flashruntime/protocol/plan_v1alpha1.py +320 -0
  44. flashruntime/protocol/v1alpha1.py +465 -0
  45. flashruntime/providers/__init__.py +138 -0
  46. flashruntime/py.typed +0 -0
  47. flashruntime/recipes/__init__.py +135 -0
  48. flashruntime/recipes/command.py +166 -0
  49. flashruntime/recovery/__init__.py +21 -0
  50. flashruntime/recovery/policy.py +170 -0
  51. flashruntime/recovery/signals.py +135 -0
  52. flashruntime/recovery/taxonomy.py +91 -0
  53. flashruntime/scheduler/__init__.py +170 -0
  54. flashruntime/sdk.py +402 -0
  55. flashruntime/service/__init__.py +3 -0
  56. flashruntime/service/app.py +391 -0
  57. flashruntime/service/auth.py +180 -0
  58. flashruntime/service/checkpoints.py +90 -0
  59. flashruntime/service/cli.py +167 -0
  60. flashruntime/service/dashboard.py +193 -0
  61. flashruntime/service/ledger.py +101 -0
  62. flashruntime/service/modea.py +821 -0
  63. flashruntime/strategies/__init__.py +156 -0
  64. flashruntime/strategies/command.py +56 -0
  65. flashruntime/torch/__init__.py +274 -0
  66. flashruntime/viewer/__init__.py +20 -0
  67. flashruntime/viewer/_docs/benchmarks.html +771 -0
  68. flashruntime/viewer/_docs/concepts/architecture.html +302 -0
  69. flashruntime/viewer/_docs/get-started.html +263 -0
  70. flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
  71. flashruntime/viewer/_docs/guides/huggingface.html +223 -0
  72. flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
  73. flashruntime/viewer/_docs/guides/pytorch.html +313 -0
  74. flashruntime/viewer/_docs/guides/sklearn.html +232 -0
  75. flashruntime/viewer/_docs/index.html +251 -0
  76. flashruntime/viewer/_docs/reference/cli.html +254 -0
  77. flashruntime/viewer/_docs/reference/integrations.html +240 -0
  78. flashruntime/viewer/_docs/reference/sdk.html +341 -0
  79. flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
  80. flashruntime/viewer/_docs/search-index.json +1 -0
  81. flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
  82. flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
  83. flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
  84. flashruntime/viewer/flowmap.py +307 -0
  85. flashruntime/viewer/page.py +594 -0
  86. flashruntime/viewer/server.py +134 -0
  87. flashruntime/viewer/state.py +250 -0
  88. flashruntime/workloads/__init__.py +6 -0
  89. flashruntime/workloads/command.py +127 -0
  90. flashruntime-0.3.0.dist-info/METADATA +365 -0
  91. flashruntime-0.3.0.dist-info/RECORD +95 -0
  92. flashruntime-0.3.0.dist-info/WHEEL +5 -0
  93. flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
  94. flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
  95. flashruntime-0.3.0.dist-info/top_level.txt +2 -0
@@ -0,0 +1,365 @@
1
+ Metadata-Version: 2.4
2
+ Name: flashruntime
3
+ Version: 0.3.0
4
+ Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
5
+ Author: Zolli Labs
6
+ License-Expression: Apache-2.0
7
+ Project-URL: Homepage, https://github.com/Zolli-Labs/flashruntime
8
+ Project-URL: Documentation, https://zolli-labs.github.io/flashruntime/
9
+ Project-URL: Repository, https://github.com/Zolli-Labs/flashruntime
10
+ Project-URL: Issues, https://github.com/Zolli-Labs/flashruntime/issues
11
+ Project-URL: Changelog, https://github.com/Zolli-Labs/flashruntime/blob/main/CHANGELOG.md
12
+ Keywords: machine-learning,distributed-training,fault-tolerance,checkpointing,pytorch,ray,mlops,reliability,hyperparameter-search
13
+ Classifier: Development Status :: 4 - Beta
14
+ Classifier: Intended Audience :: Developers
15
+ Classifier: Intended Audience :: Science/Research
16
+ Classifier: Operating System :: OS Independent
17
+ Classifier: Programming Language :: Python :: 3
18
+ Classifier: Programming Language :: Python :: 3.10
19
+ Classifier: Programming Language :: Python :: 3.11
20
+ Classifier: Programming Language :: Python :: 3.12
21
+ Classifier: Programming Language :: Python :: 3.13
22
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
23
+ Classifier: Topic :: System :: Distributed Computing
24
+ Classifier: Typing :: Typed
25
+ Requires-Python: >=3.10
26
+ Description-Content-Type: text/markdown
27
+ License-File: LICENSE
28
+ Requires-Dist: pydantic>=2.7
29
+ Provides-Extra: sklearn
30
+ Requires-Dist: numpy>=1.26; extra == "sklearn"
31
+ Requires-Dist: scikit-learn>=1.3; extra == "sklearn"
32
+ Provides-Extra: k8s
33
+ Requires-Dist: kubernetes>=29; extra == "k8s"
34
+ Provides-Extra: artifacts
35
+ Requires-Dist: minio>=7.2; extra == "artifacts"
36
+ Provides-Extra: oss
37
+ Requires-Dist: oss2>=2.18; extra == "oss"
38
+ Provides-Extra: monitor
39
+ Requires-Dist: psutil>=5.9; extra == "monitor"
40
+ Provides-Extra: service
41
+ Requires-Dist: fastapi>=0.111; extra == "service"
42
+ Requires-Dist: uvicorn>=0.30; extra == "service"
43
+ Requires-Dist: httpx>=0.27; extra == "service"
44
+ Requires-Dist: kubernetes>=29; extra == "service"
45
+ Requires-Dist: minio>=7.2; extra == "service"
46
+ Requires-Dist: pyyaml>=6; extra == "service"
47
+ Provides-Extra: dev
48
+ Requires-Dist: pytest; extra == "dev"
49
+ Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
50
+ Requires-Dist: numpy>=1.26; extra == "dev"
51
+ Requires-Dist: pyyaml>=6; extra == "dev"
52
+ Requires-Dist: markdown>=3.6; extra == "dev"
53
+ Dynamic: license-file
54
+
55
+ # FlashRuntime
56
+
57
+ > **The open fault-tolerant distributed ML runtime.** FlashRuntime *operates*
58
+ > your training job — it never rewrites it. You keep the model, the training
59
+ > loop, the loss, the data, and the framework you already have. FlashRuntime
60
+ > wraps the reliability layer around them: it launches your command, injects
61
+ > the environment it promised, tracks your metrics, validates your checkpoints,
62
+ > **recovers from crashes in one call**, and collects your artifacts. The
63
+ > distributed computation itself is always done by established libraries
64
+ > (PyTorch DDP/FSDP2/torchrun, Ray, Hugging Face, scikit-learn).
65
+
66
+ FlashRuntime is one of three components in the FlashML system by
67
+ [Zolli Labs](https://github.com/Zolli-Labs):
68
+
69
+ - **[flashnode](https://github.com/Zolli-Labs/flashnode)** — open host agent
70
+ installed by resource contributors.
71
+ - **flashruntime** (this repo) — the open workload protocol and execution
72
+ layer. Self-hostable: useful without the cloud.
73
+ - **flashml-cloud** (private) — the managed control plane, marketplace, and
74
+ dashboard.
75
+
76
+ Read [`docs/SYSTEM_OVERVIEW.md`](docs/SYSTEM_OVERVIEW.md) for the full product
77
+ architecture, and [`AGENTS.md`](AGENTS.md) if you are an AI coding agent working
78
+ in this repo.
79
+
80
+ ## Install
81
+
82
+ The core is deliberately tiny — `pip install flashruntime` brings **only
83
+ pydantic**. Every core module (the `flash.submit()` SDK, planner, leases,
84
+ checkpoints, recovery) works with zero infrastructure:
85
+
86
+ ```bash
87
+ pip install flashruntime
88
+ ```
89
+
90
+ Torch is **not** a dependency. FlashRuntime *launches* PyTorch; it never imports
91
+ it. Install it yourself to run the DDP example — a CPU-only build is enough,
92
+ since DDP works over the `gloo` backend with no GPU:
93
+
94
+ ```bash
95
+ pip install torch # CPU build is fine
96
+ ```
97
+
98
+ ## 60-second run
99
+
100
+ `flash.submit()` operates any command. The one convention your script owes
101
+ FlashRuntime is to write a `metrics.json` (a flat JSON object) into its working
102
+ directory; FlashRuntime collects it and records it as a trial:
103
+
104
+ ```python
105
+ import flashruntime as flash
106
+
107
+ run = flash.submit(
108
+ flash.CommandWorkload(
109
+ command="python train.py --epochs 5",
110
+ source=flash.Source(path="~/my-project"),
111
+ outputs=flash.OutputSpec(collect=["metrics.json"]),
112
+ ),
113
+ max_restarts=2, # a crashed attempt relaunches from the last VALID
114
+ # checkpoint — up to twice (see below)
115
+ watch=True, # opens the live run page and prints its URL
116
+ )
117
+
118
+ print(run.state.value) # "SUCCEEDED" (or "FAILED")
119
+ print(run.trials) # [{'accuracy': 0.91, ...}] — parsed from metrics.json
120
+ print(run.artifacts) # [PosixPath('.../metrics.json'), ...]
121
+ print(run.viewer_url) # http://127.0.0.1:<port> — the live page
122
+ ```
123
+
124
+ `command` is `shlex`-split (there is no shell — for a pipe, pass
125
+ `command="bash -c '...'"`), and `source` is a `flash.Source`, so `~` is expanded
126
+ for you. The same thing from the shell:
127
+
128
+ ```bash
129
+ flashruntime submit "python train.py --epochs 5" \
130
+ --source ~/my-project --max-restarts 2 --watch
131
+ ```
132
+
133
+ A script that already reads its hyperparameters from `argparse` and writes a
134
+ small JSON of results needs **zero FlashRuntime imports** to be operated. The
135
+ contract at the boundary is thin on purpose: arguments in, `metrics.json` out.
136
+
137
+ ## One import, fault tolerant
138
+
139
+ `max_restarts=N` is the automatic recovery budget. On a **FAILED** attempt
140
+ FlashRuntime turns the exit into failure signals, classifies them
141
+ (`recovery.classify`), and consults a *versioned, deterministic* policy
142
+ (`recovery.decide`): a deterministic application bug fails fast — a retry would
143
+ only re-hit it — while a transient failure relaunches the same spec from the
144
+ job-scoped checkpoint. Same failure + same policy version ⇒ same action, every
145
+ time. **No LLM in the loop.** Every decision is recorded on the run as
146
+ `FAILURE_CLASSIFIED` / `RECOVERY_ACTION_SELECTED` events.
147
+
148
+ Kill-and-resume is proven end to end. `tests/test_examples_e2e.py` kills a
149
+ 2-process DDP run mid-training and asserts the resumed run's final loss
150
+ matches a never-crashed baseline to 1e-6 — recovery restores only a verified,
151
+ topology-compatible checkpoint (parts-first / manifest-last: a half-written
152
+ checkpoint can never look valid).
153
+
154
+ For a script you *are* willing to touch, `import flashruntime.torch as ft` adds
155
+ `ft.prepare` / `ft.checkpoint` / `ft.log_metrics`: torch's own DDP wrapped under
156
+ the same checkpoint contract, with correct per-rank device placement, so a
157
+ killed run resumes with a loss curve indistinguishable from the uninterrupted
158
+ one (1e-6 in the e2e). It is optional sugar on the launch-only
159
+ contract, never required.
160
+
161
+ **GPU-validated (2026-07-23).** The CUDA paths — `nccl` DDP, per-rank device
162
+ placement, and a GPU kill-and-resume — are validated end to end on real
163
+ hardware: **2×NVIDIA GeForce RTX 4090** (RunPod community cloud), torch
164
+ 2.7.1+cu128, CUDA 12.8. Two runs, **$0.0725 total**. See `tests/test_gpu_e2e.py`
165
+ and the harness `scripts/runpod_gpu_e2e.py`.
166
+
167
+ ## Watch it run
168
+
169
+ Pass `watch=True` (the default at an interactive terminal, off in pipes/CI) and
170
+ `flash.submit()` opens a live run page in your browser and prints its URL. It
171
+ draws the run's topology, its loss curve, its verified checkpoints, and every
172
+ recovery decision, refreshing every couple of seconds — served entirely from a
173
+ loopback server with **no external assets**, so it renders with the network cut.
174
+ These docs are served from that same viewer at `/docs`.
175
+
176
+ > _screenshot: run any `flashruntime submit … --watch` to see the live page._
177
+
178
+ ## Documentation
179
+
180
+ A PyTorch-style docs site is built from [`docs/site/`](docs/site/) by
181
+ `scripts/build_docs.py` (every byte inline — no CDN, web font, or remote image).
182
+ It is served offline by the viewer at `/docs`, and deploys to GitHub Pages on
183
+ release:
184
+
185
+ - **<https://zolli-labs.github.io/flashruntime/>** — _live after the first
186
+ Pages deploy._
187
+
188
+ Start with [Get started](docs/site/get-started.md) (install → first job → first
189
+ DDP run on CPU). The strategy planner's code walkthrough lives in
190
+ [`docs/planner/README.md`](docs/planner/README.md).
191
+
192
+ ## Benchmarks
193
+
194
+ An honest benchmark suite (`python -m benchmarks run`) records every result with
195
+ the host that produced it and reports missing comparators as skipped rows, never
196
+ faked. A measured baseline is committed under
197
+ [`benchmarks/results/`](benchmarks/results/). On **Apple M4 (CPU)**:
198
+
199
+ | Scenario | Measured | Read it honestly |
200
+ |---|---|---|
201
+ | Launch overhead | **+0.04 s** | `flash.submit` vs bare `torchrun`, paired. A one-time fixed cost, not per-step. |
202
+ | Checkpoint cost | **−1.2 ms / checkpoint** (p90 6.2 ms) | A few-KB state dict; the median delta is *below* the run-to-run noise floor at this size (it goes slightly negative), so checkpoints are effectively free here — the p90 bounds it. A larger model surfaces a real positive cost. |
203
+ | Auto-resume | **40 steps not recomputed** | The size-*independent* guarantee: resume never re-does work past the last valid checkpoint. |
204
+ | Adoption | **7 lines** | To make a vanilla script framework-ready (difflib vs each project's own docs). |
205
+ | Resilience | **5/5 faults handled · 20/20 integrity under `kill -9`** | Every fault type routed to the right typed recovery; a checkpoint survives a mid-write `SIGKILL` on 20/20 kills (naive `torch.save`: 20/20 corrupted). 16-trial storm, half crash-armed: 16/16 completed, all 8 crashed trials auto-resumed, 0 manual. Dead-worker MTTD **3.05 s**, MTTR **~3.5 ms**. |
206
+
207
+ The *timing* rows are smoke-scale on one host: the models are tiny, so the
208
+ *wall-clock* saved by recovery is at the noise floor here. The figure that
209
+ scales is `steps_not_recomputed`, not seconds — the seconds-saved grows with the
210
+ compute between the last checkpoint and the crash (negligible at smoke size,
211
+ hours on a real job). The resilience counts are exact, not smoke-diluted: they
212
+ are *counted* from real fault injection (`kill -9` in the checkpoint-write
213
+ window, a storm of crash-armed trials, a killed lease worker), and the full
214
+ Resilience table with per-scenario method lives in
215
+ [the benchmarks page](docs/site/benchmarks.md). Full provenance and caveats live
216
+ in the result file.
217
+
218
+ ---
219
+
220
+ ## Also: plan a job (no cluster required)
221
+
222
+ Before you launch, `flash.plan()` turns model + hardware + budget + deadline
223
+ into a ranked, explained execution plan — deterministic, framework-import-free:
224
+
225
+ ```python
226
+ import flashruntime as flash
227
+
228
+ report = flash.plan(flash.PlanRequest(
229
+ workload=flash.TransformerFineTune(model="Qwen/Qwen2.5-7B", method="lora",
230
+ train_tokens_m=25),
231
+ resources=flash.Resources(gpus=4, gpu_type="RTX4090",
232
+ hourly_cost_usd_per_gpu=0.44),
233
+ objective=flash.Objective(mode="balanced", max_cost_usd=20,
234
+ deadline_minutes=240),
235
+ ))
236
+ print(flash.render(report))
237
+ # → SELECTED: ddp + lora(r=16), 4 workers, torchrun, 21.6 GB/GPU, ~96 min,
238
+ # $2.82 — plus every rejected candidate with its arithmetic
239
+ ```
240
+
241
+ The report names the distributed method, the libraries it is built from
242
+ (torchrun, DDP/FSDP2, transformers+peft, bitsandbytes, PyTorch DCP) and their
243
+ roles, the knobs, and *why* — including why the alternatives lost. Details:
244
+ [`docs/planner/README.md`](docs/planner/README.md).
245
+
246
+ ## Also: leases, checkpoints, recovery
247
+
248
+ The reliability core is an embeddable library, not a service you must run:
249
+
250
+ ```python
251
+ from flashruntime.leases import LeaseManager
252
+ from flashruntime.recovery import FailureSignals, classify, decide
253
+ from flashruntime.protocol.v1alpha1 import TaskSpec
254
+
255
+ mgr = LeaseManager()
256
+ mgr.add_task(TaskSpec(task_id="trial-01", job_id="sweep", commit_key="sweep/trial-01"))
257
+ lease = mgr.claim(node_id="laptop-1") # pull, never push
258
+ mgr.heartbeat(lease.lease_id) # prove liveness
259
+ mgr.complete(lease.lease_id, output_sha256="…") # first valid commit wins
260
+
261
+ decision = decide(classify(FailureSignals(heartbeat_lost=True)),
262
+ mode="independent_tasks") # → RETRY_TASK, cordon node
263
+ ```
264
+
265
+ The FlashRuntime service exposes these same objects over HTTP; FlashNode's
266
+ device executor is their remote client. The lease coordinator's durable
267
+ `SqliteLeaseStore` means in-flight leases survive a coordinator restart.
268
+
269
+ ## Testing
270
+
271
+ ```bash
272
+ pytest # unit tests — pure Python, no infrastructure
273
+ pytest -m integration # opt-in: Docker / Kubernetes / MinIO environments
274
+ ```
275
+
276
+ Integration tests live in [`tests/integration/`](tests/integration/README.md)
277
+ and skip themselves (with instructions) when their environment is absent. The
278
+ one GPU test (`tests/test_gpu_e2e.py`) is CUDA-gated and skips without a GPU. The
279
+ local kind + KubeRay + MinIO stack is owned by the workspace Makefile, not this
280
+ repo — the library stays `pip install`-clean.
281
+
282
+ ## Package layout
283
+
284
+ **Pure-Python core** — `pip install flashruntime` brings pydantic and nothing
285
+ else; every module below works with zero infrastructure:
286
+
287
+ ```
288
+ flashruntime/
289
+ ├── protocol/ # versioned public schemas (v1alpha1: JobSpec, Event, Lease,
290
+ │ # TaskAttempt, CheckpointManifest, RecoveryDecision, plans)
291
+ ├── sdk.py # flash.submit(CommandWorkload) → Run: compile→launch→wait→
292
+ │ # collect, with the automatic max_restarts recovery loop
293
+ ├── workloads/ # CommandWorkload: framework-neutral "run this command"
294
+ ├── integrations/# one-file adapters (sklearn / pytorch DDP / HF) — no framework
295
+ │ # imports at module level; convention is CLI-flags-in/JSON-out
296
+ ├── torch/ # optional in-script helper: ft.prepare / checkpoint / log_metrics
297
+ ├── planner/ # strategy planner: estimators, curated menus, explained selector
298
+ ├── leases/ # Mode A core: claim / heartbeat / expiry sweep / idempotent
299
+ │ # commit; InMemory + SQLite stores (restart-durable)
300
+ ├── checkpoint/ # manifest catalog: parts-first/manifest-last, validation ladder
301
+ ├── recovery/ # failure-signal classifier + versioned deterministic policy
302
+ └── viewer/ # stdlib live run page + the built docs site (zero external assets)
303
+ ```
304
+
305
+ **Infrastructure integrations** (opt-in extras, never core imports):
306
+
307
+ ```
308
+ ├── service/ # [service] the coordinator: job submission + task expansion,
309
+ │ # lease/node/checkpoint HTTP endpoints, local artifact hosting,
310
+ │ # join codes, SQLite ledger, dashboard at GET /, the CLI
311
+ ├── launchers/ # LocalProcessLauncher (the concrete local launcher today)
312
+ ├── strategies/ # CommandWorkload → LaunchSpec compilation
313
+ ├── recipes/ # coordinator-side WorkloadRecipe registry (command jobs)
314
+ ├── backends/ # [k8s] ExecutionBackend contract + KubeRay backend (Mode B)
315
+ └── artifacts/ # [artifacts]/[oss] MinIO/S3-compatible + Alibaba OSS stores
316
+
317
+ flashml_workloads/ # runnable task modules (pure stdlib/sklearn):
318
+ │ # sklearn_trial, kmeans_shard + kmeans_driver,
319
+ │ # sgd_trainer (checkpointable, bit-identical resume)
320
+ ```
321
+
322
+ Still to come (each lands with its vertical slice, no empty scaffolds):
323
+ multi-node DDP rendezvous (`nnodes > 1` raises `NotImplementedError` today);
324
+ FlashNode's argv runner so service-side command jobs execute remotely;
325
+ checkpoint-manifest persistence (catalog is in-memory; checkpoint *files* are
326
+ durable); Stage-8 ledger metrics (MTTD/MTTR/goodput); `flash.run(plan)` wiring;
327
+ the cloud stage (Postgres, SSE, HF Trainer + PEFT LoRA recipes).
328
+
329
+ Four orthogonal axes keep the architecture honest: **providers** get machines,
330
+ **launchers** start processes, **strategies** configure execution, **recipes**
331
+ integrate user code. Hugging Face lives in recipes — it is the workload layer,
332
+ not an execution backend. The planner never imports framework code; it emits a
333
+ backend-neutral `StrategyPlan` that strategy compilers translate.
334
+
335
+ ## What FlashRuntime builds on (and what it owns)
336
+
337
+ | Layer | Libraries reused | What FlashRuntime adds |
338
+ |---|---|---|
339
+ | ML math & models | PyTorch, HF Transformers + PEFT, bitsandbytes, scikit-learn | Nothing — untouched, operated by recipes |
340
+ | Distribution strategies | DDP, FSDP2 (`fully_shard`); DeepSpeed ZeRO later | The *choice* and its explanation, compiled from a StrategyPlan |
341
+ | Launching | torchrun / Torch Elastic; Ray Core on clusters | Checkpoint-aware restart around torchrun; the **lease/heartbeat protocol** for machines outside any cluster — the layer with no existing library |
342
+ | Checkpointing | PyTorch Distributed Checkpoint (parallel save/load, resharding) | The manifest **catalog**: written-last commit, validation status, compatibility, recovery-time selection |
343
+ | Infrastructure | Kubernetes + KubeRay today; SkyPilot / Slurm / provider APIs later | Provider adapters + one cross-environment job model |
344
+
345
+ Deliberately not built on: HF Accelerate (overlaps torchrun and strategy
346
+ config — user scripts that use it still work via the launch-only contract) and
347
+ Ray Train (mid V1→V2 migration).
348
+
349
+ ## The dependency rule
350
+
351
+ `flashruntime` owns the public protocol and imports **neither** application
352
+ repo. `flashnode` and `flashml-cloud` both depend on `flashruntime`; they never
353
+ import each other.
354
+
355
+ ## Contributing, security, changelog
356
+
357
+ - [`CONTRIBUTING.md`](CONTRIBUTING.md) — dev setup, the test/doc/audit gates, and
358
+ the Developer Certificate of Origin (`git commit -s`).
359
+ - [`SECURITY.md`](SECURITY.md) — how to report a vulnerability.
360
+ - [`CHANGELOG.md`](CHANGELOG.md) — release notes ([Keep a Changelog](https://keepachangelog.com/)).
361
+
362
+ ## License
363
+
364
+ [Apache-2.0](LICENSE). Contributions via Developer Certificate of Origin
365
+ (`git commit -s`).
@@ -0,0 +1,95 @@
1
+ flashml_workloads/__init__.py,sha256=pIep8hH1zxpzY1ePodLdVSzH1N0vIogEQyFPrjgk6d0,310
2
+ flashml_workloads/fedavg_driver.py,sha256=nsB9bZCKZQ1I7uY67wVFUPQ672bA7kBcyXLIi3W_YzA,25803
3
+ flashml_workloads/fedavg_weights.py,sha256=U_a7JzOhciZs0bRzKoZn63dnGgQQ6S79bhd5sbDXWVc,9537
4
+ flashml_workloads/fedavg_worker.py,sha256=jZhH5myvuOhE2QOAGlFpySxCsaUmoRs4eTdUJYRQowY,5853
5
+ flashml_workloads/kmeans_driver.py,sha256=F5krmQ1ExqbVO1ZWEy89Fk3mlCS57Hwe3ZD8oNN_srY,5270
6
+ flashml_workloads/kmeans_shard.py,sha256=ktDBDoC41eTqGYxxR-Cm4heq5Ou-LNhU5cgVvXwd_d0,2199
7
+ flashml_workloads/sgd_trainer.py,sha256=uTWWX-CGbz3JrlRzKSKUzoeUiQJ0wwmn9tkTbJ-RcLY,4234
8
+ flashml_workloads/sharded_kmeans.py,sha256=XTLWBEN4IA3EjadctxcHQxkiHOVWEOVnbpwcgxui-1k,12981
9
+ flashml_workloads/sklearn_trial.py,sha256=VsifaHcRK2sqkO5l5skfm3ddOpskLuU1dvt_zdJHev8,3170
10
+ flashruntime/__init__.py,sha256=nCUteqemDt70iptY7r22iSQPInrhQK-xR_VpVE_j7fs,4774
11
+ flashruntime/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
12
+ flashruntime/sdk.py,sha256=UsfBOwnhF-aLHvyGcLZTrL_n03V5Qg41XfRlDrQAVfE,19004
13
+ flashruntime/artifacts/__init__.py,sha256=wApVUFtJOwsfyevfeleTSTohJ5LHftORJ3ozjWOFBQc,684
14
+ flashruntime/artifacts/store.py,sha256=zCFFpQeljYVwBkAmpPfLeJO2z-9eyTogRtE5fObxIew,7491
15
+ flashruntime/backends/__init__.py,sha256=_kUmnFZQoJaVxjYb7USR4ulf-il0VxAeFuOr_ZG_CL8,813
16
+ flashruntime/backends/base.py,sha256=8bz-xiY1x_HbHQvS61Pcp5C_Kra6_6YvwKL7j66sapE,1928
17
+ flashruntime/backends/kuberay.py,sha256=jGE3WgzWno6D193csYh2Xe1ZfqAE15hCLqSvQmFs9Qo,18634
18
+ flashruntime/checkpoint/__init__.py,sha256=nos0gYooP8K4DePVdG8RXOyhzbPLq_UhNjHzAMwMYOA,932
19
+ flashruntime/checkpoint/catalog.py,sha256=Z0AuYQ3myQUawAPndJbQoHA_dqSDBzoMHIfjhs8MVBM,8042
20
+ flashruntime/checkpoint/local.py,sha256=oMbsvD-bPdHA8U0epHGkTDzowOEFFVNgwqGzpxdaiiM,3943
21
+ flashruntime/checkpoint/store.py,sha256=JeHpiiD9ysjvy3Rc6_2Woqzc0Xrk228ch6YTMuMy8M8,3808
22
+ flashruntime/integrations/__init__.py,sha256=dsp6-hIvN2NSr9MuFDoP5EoslB3CMgNvdvmx23NDydk,295
23
+ flashruntime/integrations/huggingface.py,sha256=FgeAM8R6Gc7SD9wIMDoUFDtZF9EkzuriY63hF95OxOE,2447
24
+ flashruntime/integrations/pytorch.py,sha256=0Rh5ENwFUevL05OxidO9aiB1o4djhQ35EmLD0s1ULHk,1774
25
+ flashruntime/integrations/sklearn.py,sha256=HZ1uV-R79rQCmY58KYcn9_LYOkdbTuGfjNyk058RN18,1418
26
+ flashruntime/launchers/__init__.py,sha256=LV0wgeRvwnXx08h_rbXFxv0bQNeifEbw_3nium9MvIw,5490
27
+ flashruntime/launchers/local.py,sha256=27W7BMhen_1oLSu9f2AFcruTIaPyJpyqsxtwTFDb2KE,4845
28
+ flashruntime/leases/__init__.py,sha256=ocBDQPQmyVJldENdT0_UCa57w8SAmrhMPG7FnSgTBkM,1075
29
+ flashruntime/leases/manager.py,sha256=YCvj0eNzuUX5w6J5xd7yu7X1KPHQV8hG3aZywziD-ss,14766
30
+ flashruntime/leases/sqlite_store.py,sha256=-OaunF91vaqJe3hzIsn2VEnvYgMFPE7gGhoNk6pB4D0,6845
31
+ flashruntime/leases/store.py,sha256=iMBcd00sTyrdjTOAvALamQyZ2AtOTioZVtkEFySzHbg,3780
32
+ flashruntime/monitor/__init__.py,sha256=1Uc4rh-ZLa0c30-doxYJj3yemcf2e-NE3xhX0T8uEcs,244
33
+ flashruntime/monitor/sampler.py,sha256=vBrF_ZMFu_CWcYoqdEUEScyBAiOyOssU45grSLQm4QI,9336
34
+ flashruntime/planner/__init__.py,sha256=jYn-AefGhGwUxWDliR-V5aJiinUahHBn5PvsiN-75oo,2206
35
+ flashruntime/planner/candidates.py,sha256=QF2fYZF9x5HH89KO-QXGRRPtfIkDPAjYOJ_gPXCrR0E,24249
36
+ flashruntime/planner/catalog.py,sha256=QvzENe6uiPVHdChhC82rcC-Yjjuxk9B8iXyIuTmCUHM,4308
37
+ flashruntime/planner/comm.py,sha256=UD_-Aiu2mOfcrPJLnOBEY9tUgDeGhmoiqHivIHq1Czw,3440
38
+ flashruntime/planner/explain.py,sha256=5vmnEKOgwhmlO5xkw2caxUc_En1iylVna6lmcmM1jy4,4527
39
+ flashruntime/planner/memory.py,sha256=6BZguaCzvxvCx6hBqvpXFZ6bSD2oZ8Fz802f2ioIEAw,6492
40
+ flashruntime/planner/resolve.py,sha256=KtMxPUUEz1LAEwnv7rFlobRG37pUAFCRBoQLvprCZPE,4212
41
+ flashruntime/planner/selector.py,sha256=b-VuacYmlqmHWKUmr6niwnYpYTo3YX-hTfaGbRaS-_k,6134
42
+ flashruntime/planner/timecost.py,sha256=nCAtGfBg5FnY1rSB0nGb5-WY1IMZBSDeZzZJ8GE3XRg,3254
43
+ flashruntime/profiling/__init__.py,sha256=ncysF1ZC9qgp5rOBCPpaPEhk_aFZE7FYvg9RQiDeueg,4933
44
+ flashruntime/protocol/__init__.py,sha256=ZmEFFgfushwzsTVz8yS59JUOTtHH6DsyPk0bM6fv0gw,713
45
+ flashruntime/protocol/plan_v1alpha1.py,sha256=uncKTD8_fWkTZfx8wXrzVb-MOiXQNIf6K2ZwJcNIoJ8,12039
46
+ flashruntime/protocol/v1alpha1.py,sha256=lav3N-GglYXe2YYYua0hu8-soKGwn6gupFl7zKC2k18,16621
47
+ flashruntime/providers/__init__.py,sha256=mLlpW7ybdMdAFfKJLnYPQougAEO2vWMdfyJccXniSRk,5686
48
+ flashruntime/recipes/__init__.py,sha256=OeAwksQgJCxQPtj6-WZYD-X5t_RBtt7PTgJwEwYa4Q0,5960
49
+ flashruntime/recipes/command.py,sha256=j4eklDwEjoZumtuiPTasFvJ3OM-E74vrRomoMv3qD4s,7533
50
+ flashruntime/recovery/__init__.py,sha256=HJNxkIPW5ntOJo0_hrMRtaEP1-xFddG8iLjGzP1OS0s,957
51
+ flashruntime/recovery/policy.py,sha256=djvixKDdSBH-acFNQcn9Djnif42E6ojyUCmLthuKj3E,8450
52
+ flashruntime/recovery/signals.py,sha256=RM2utaCxUYiZ3hVBHNhrRHgL250K2Qo09av1MFaUHFo,7742
53
+ flashruntime/recovery/taxonomy.py,sha256=f-szAtRPJJb5l9TIl-XJVqmdapVNwmDR6qSS-fEAq90,3399
54
+ flashruntime/scheduler/__init__.py,sha256=96CvNI_wmXZTOFGNWZfHwzc030h7I2tgyPr0gMWIMCE,8709
55
+ flashruntime/service/__init__.py,sha256=JZTxtllXIIdZz4znR3QIvdjy3Z_tBtYJ-vquqbNIG_A,219
56
+ flashruntime/service/app.py,sha256=-e0pJjTDUyBAJUBRfeQEA5g_KiofBJKRhgbLxjCv1tg,16538
57
+ flashruntime/service/auth.py,sha256=c61pLGkBBcPG2aQrp28N1FqTmjmqYZ4uCaXl7UST0J8,7072
58
+ flashruntime/service/checkpoints.py,sha256=kT_ghRouGpYbskQ78RdvINZQZ1qSevo0_7AiAn3vEvg,3481
59
+ flashruntime/service/cli.py,sha256=HHCNRlimAv0HOTE30KI7ynlKSXr8Rvifn_NzAfETJkw,6445
60
+ flashruntime/service/dashboard.py,sha256=Zy6afH_kpFlnyGrvuSup9gtERryzpRcxQLeMLcMftic,7896
61
+ flashruntime/service/ledger.py,sha256=CebBgh0ZNSPHlla-VMRDI2S6QzxO3HB61hOcc1YAqdA,3282
62
+ flashruntime/service/modea.py,sha256=y1mDzt8C1H71zOXFShkaLGCO0zOI5RBSF6iMyTibXxM,35327
63
+ flashruntime/strategies/__init__.py,sha256=mnBLEpgmvvgo2CrCptEvHuf39FQxIDMRp4N_-Wd8z7g,6470
64
+ flashruntime/strategies/command.py,sha256=Z_qUlvibjT5MJloAZsV0kWnRqJmE1PJcQQEaGvB8i-o,2294
65
+ flashruntime/torch/__init__.py,sha256=UX4Z8yhhHDpgJqt4KMyW0VOK9eA1RWr090ioZ-eSiTs,10801
66
+ flashruntime/viewer/__init__.py,sha256=xDroJ1Rwl-AVEmu01QFnvzAo49X0jUwFrO5w2YKErwQ,822
67
+ flashruntime/viewer/flowmap.py,sha256=1amqvoii9ovGptnL7rnTK3RdUDE3I0qi_o_iEgmk4pY,16095
68
+ flashruntime/viewer/page.py,sha256=vLL6p5rCOKTCbvc1Kc-1TQeYgu97dUJk6z7ikbeDYVk,23601
69
+ flashruntime/viewer/server.py,sha256=2Zk6AP7lWLzdGThoJUpbKpok3rlZJj6psFNYe_sWAPM,6179
70
+ flashruntime/viewer/state.py,sha256=r0aa_J2QxYSqRrNUYE0UMf3TCTLwcGby_O7SrCGAdmA,11446
71
+ flashruntime/viewer/_docs/benchmarks.html,sha256=e4wZuXSh6cf2jU1p1_wCDr9XgngRH21voWsB7FctzeE,33119
72
+ flashruntime/viewer/_docs/get-started.html,sha256=O8ZdzZgkgZs0PNTEmg_kFCTkf-K5mEOujzzGSh3QwNA,15791
73
+ flashruntime/viewer/_docs/index.html,sha256=KpUt-dpTFjKfZCI1k6Wk3xxZgwz8Y7Sm3lmTgFw746k,14278
74
+ flashruntime/viewer/_docs/search-index.json,sha256=fSKcAEEM0bN20deayeTe29QQ0BwTFhuBYYVY2e-Gr9E,106694
75
+ flashruntime/viewer/_docs/concepts/architecture.html,sha256=CPZ9k7lj8HNUjBAQ5KUAqZth97B6OubunXUDp22F6nI,19568
76
+ flashruntime/viewer/_docs/guides/federated-averaging.html,sha256=h_bO--34hN_fS81SdmQYrDUaxJc1Y0SqxNMkd7x5kgQ,22524
77
+ flashruntime/viewer/_docs/guides/huggingface.html,sha256=ZYKRK9AERD4b2MYMa3sBMc1BHytRgvmFRtK-desvz-4,12890
78
+ flashruntime/viewer/_docs/guides/jobspec-and-isolation.html,sha256=U-1QtU-QOMNjEqoKxV-vZMTVYrKXymHvIRxQu_7GDyE,15531
79
+ flashruntime/viewer/_docs/guides/pytorch.html,sha256=-qtojwYaiTVf6gt-G4YWwhVs9X2J4hN8i6RyaSyh4lk,18264
80
+ flashruntime/viewer/_docs/guides/sklearn.html,sha256=32BnmGP_eBQgQNnGe_ieZl--IyYUwpqVaXAzeXUZ6ZQ,13498
81
+ flashruntime/viewer/_docs/reference/cli.html,sha256=4urAUM13nVQKyLLtPWpYpJEZ_s0gdvUWZB92At-MqAk,14341
82
+ flashruntime/viewer/_docs/reference/integrations.html,sha256=I5eZZk_rd9Tw7F6YtgTvsgPzHx088JvubttChVJvS3M,13676
83
+ flashruntime/viewer/_docs/reference/sdk.html,sha256=6in4enErqFmO0OaSPw9UA7ek_fXrTEW2MiQ5DYrxbVM,19183
84
+ flashruntime/viewer/_docs/reference/torch-helper.html,sha256=hrWHYc9Op1NwLrk37Ox2MhuLVWwW8UeZFL7vn6-2vMo,14435
85
+ flashruntime/viewer/_docs/tutorials/convnet.html,sha256=5cupZBXjjOcnHPoTR2NdgXPacnTLQKCfzbE_tKpG1sA,30784
86
+ flashruntime/viewer/_docs/tutorials/fault-tolerance.html,sha256=s32yBtDHWZF7pAHbXoses1bVmtyWwWKEyNTMzhQvqxw,20674
87
+ flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html,sha256=1c0ooeZHqgl8YSUJ_Cgxg1nf9VkIeYSkRtrj2UUdsbc,15813
88
+ flashruntime/workloads/__init__.py,sha256=R24mJHLQnhU45-MTPCpLkyTh7LAXFuJhLxb9AwMfgHY,307
89
+ flashruntime/workloads/command.py,sha256=Zbi5be3EI46orn7EWCdoG1PnmXuvK_c0uSxCCwB_AEw,4943
90
+ flashruntime-0.3.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
91
+ flashruntime-0.3.0.dist-info/METADATA,sha256=cSOH0tjvezgdJdlW4UtV-FQNzbwverwyCPG1DqCr2uo,18154
92
+ flashruntime-0.3.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
93
+ flashruntime-0.3.0.dist-info/entry_points.txt,sha256=EfbCXQlEqL3_n-UJAO5i_vU6A-rUJmjrOaiQ1TifE3c,63
94
+ flashruntime-0.3.0.dist-info/top_level.txt,sha256=H7CA1FVVVRcCiNDylGQnaAUuGiP6hv82AsXhXN0EsxE,31
95
+ flashruntime-0.3.0.dist-info/RECORD,,
@@ -0,0 +1,5 @@
1
+ Wheel-Version: 1.0
2
+ Generator: setuptools (83.0.0)
3
+ Root-Is-Purelib: true
4
+ Tag: py3-none-any
5
+
@@ -0,0 +1,2 @@
1
+ [console_scripts]
2
+ flashruntime = flashruntime.service.cli:main