flashruntime 0.3.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- flashml_workloads/__init__.py +7 -0
- flashml_workloads/fedavg_driver.py +569 -0
- flashml_workloads/fedavg_weights.py +223 -0
- flashml_workloads/fedavg_worker.py +166 -0
- flashml_workloads/kmeans_driver.py +134 -0
- flashml_workloads/kmeans_shard.py +69 -0
- flashml_workloads/sgd_trainer.py +127 -0
- flashml_workloads/sharded_kmeans.py +323 -0
- flashml_workloads/sklearn_trial.py +89 -0
- flashruntime/__init__.py +125 -0
- flashruntime/artifacts/__init__.py +25 -0
- flashruntime/artifacts/store.py +228 -0
- flashruntime/backends/__init__.py +26 -0
- flashruntime/backends/base.py +63 -0
- flashruntime/backends/kuberay.py +465 -0
- flashruntime/checkpoint/__init__.py +20 -0
- flashruntime/checkpoint/catalog.py +198 -0
- flashruntime/checkpoint/local.py +109 -0
- flashruntime/checkpoint/store.py +86 -0
- flashruntime/integrations/__init__.py +5 -0
- flashruntime/integrations/huggingface.py +59 -0
- flashruntime/integrations/pytorch.py +52 -0
- flashruntime/integrations/sklearn.py +42 -0
- flashruntime/launchers/__init__.py +130 -0
- flashruntime/launchers/local.py +126 -0
- flashruntime/leases/__init__.py +27 -0
- flashruntime/leases/manager.py +365 -0
- flashruntime/leases/sqlite_store.py +169 -0
- flashruntime/leases/store.py +103 -0
- flashruntime/monitor/__init__.py +7 -0
- flashruntime/monitor/sampler.py +232 -0
- flashruntime/planner/__init__.py +56 -0
- flashruntime/planner/candidates.py +597 -0
- flashruntime/planner/catalog.py +129 -0
- flashruntime/planner/comm.py +95 -0
- flashruntime/planner/explain.py +109 -0
- flashruntime/planner/memory.py +166 -0
- flashruntime/planner/resolve.py +120 -0
- flashruntime/planner/selector.py +169 -0
- flashruntime/planner/timecost.py +81 -0
- flashruntime/profiling/__init__.py +113 -0
- flashruntime/protocol/__init__.py +18 -0
- flashruntime/protocol/plan_v1alpha1.py +320 -0
- flashruntime/protocol/v1alpha1.py +465 -0
- flashruntime/providers/__init__.py +138 -0
- flashruntime/py.typed +0 -0
- flashruntime/recipes/__init__.py +135 -0
- flashruntime/recipes/command.py +166 -0
- flashruntime/recovery/__init__.py +21 -0
- flashruntime/recovery/policy.py +170 -0
- flashruntime/recovery/signals.py +135 -0
- flashruntime/recovery/taxonomy.py +91 -0
- flashruntime/scheduler/__init__.py +170 -0
- flashruntime/sdk.py +402 -0
- flashruntime/service/__init__.py +3 -0
- flashruntime/service/app.py +391 -0
- flashruntime/service/auth.py +180 -0
- flashruntime/service/checkpoints.py +90 -0
- flashruntime/service/cli.py +167 -0
- flashruntime/service/dashboard.py +193 -0
- flashruntime/service/ledger.py +101 -0
- flashruntime/service/modea.py +821 -0
- flashruntime/strategies/__init__.py +156 -0
- flashruntime/strategies/command.py +56 -0
- flashruntime/torch/__init__.py +274 -0
- flashruntime/viewer/__init__.py +20 -0
- flashruntime/viewer/_docs/benchmarks.html +771 -0
- flashruntime/viewer/_docs/concepts/architecture.html +302 -0
- flashruntime/viewer/_docs/get-started.html +263 -0
- flashruntime/viewer/_docs/guides/federated-averaging.html +363 -0
- flashruntime/viewer/_docs/guides/huggingface.html +223 -0
- flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +271 -0
- flashruntime/viewer/_docs/guides/pytorch.html +313 -0
- flashruntime/viewer/_docs/guides/sklearn.html +232 -0
- flashruntime/viewer/_docs/index.html +251 -0
- flashruntime/viewer/_docs/reference/cli.html +254 -0
- flashruntime/viewer/_docs/reference/integrations.html +240 -0
- flashruntime/viewer/_docs/reference/sdk.html +341 -0
- flashruntime/viewer/_docs/reference/torch-helper.html +244 -0
- flashruntime/viewer/_docs/search-index.json +1 -0
- flashruntime/viewer/_docs/tutorials/convnet.html +571 -0
- flashruntime/viewer/_docs/tutorials/fault-tolerance.html +375 -0
- flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +278 -0
- flashruntime/viewer/flowmap.py +307 -0
- flashruntime/viewer/page.py +594 -0
- flashruntime/viewer/server.py +134 -0
- flashruntime/viewer/state.py +250 -0
- flashruntime/workloads/__init__.py +6 -0
- flashruntime/workloads/command.py +127 -0
- flashruntime-0.3.0.dist-info/METADATA +365 -0
- flashruntime-0.3.0.dist-info/RECORD +95 -0
- flashruntime-0.3.0.dist-info/WHEEL +5 -0
- flashruntime-0.3.0.dist-info/entry_points.txt +2 -0
- flashruntime-0.3.0.dist-info/licenses/LICENSE +202 -0
- flashruntime-0.3.0.dist-info/top_level.txt +2 -0
|
@@ -0,0 +1,365 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: flashruntime
|
|
3
|
+
Version: 0.3.0
|
|
4
|
+
Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
|
|
5
|
+
Author: Zolli Labs
|
|
6
|
+
License-Expression: Apache-2.0
|
|
7
|
+
Project-URL: Homepage, https://github.com/Zolli-Labs/flashruntime
|
|
8
|
+
Project-URL: Documentation, https://zolli-labs.github.io/flashruntime/
|
|
9
|
+
Project-URL: Repository, https://github.com/Zolli-Labs/flashruntime
|
|
10
|
+
Project-URL: Issues, https://github.com/Zolli-Labs/flashruntime/issues
|
|
11
|
+
Project-URL: Changelog, https://github.com/Zolli-Labs/flashruntime/blob/main/CHANGELOG.md
|
|
12
|
+
Keywords: machine-learning,distributed-training,fault-tolerance,checkpointing,pytorch,ray,mlops,reliability,hyperparameter-search
|
|
13
|
+
Classifier: Development Status :: 4 - Beta
|
|
14
|
+
Classifier: Intended Audience :: Developers
|
|
15
|
+
Classifier: Intended Audience :: Science/Research
|
|
16
|
+
Classifier: Operating System :: OS Independent
|
|
17
|
+
Classifier: Programming Language :: Python :: 3
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
20
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
21
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
22
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
23
|
+
Classifier: Topic :: System :: Distributed Computing
|
|
24
|
+
Classifier: Typing :: Typed
|
|
25
|
+
Requires-Python: >=3.10
|
|
26
|
+
Description-Content-Type: text/markdown
|
|
27
|
+
License-File: LICENSE
|
|
28
|
+
Requires-Dist: pydantic>=2.7
|
|
29
|
+
Provides-Extra: sklearn
|
|
30
|
+
Requires-Dist: numpy>=1.26; extra == "sklearn"
|
|
31
|
+
Requires-Dist: scikit-learn>=1.3; extra == "sklearn"
|
|
32
|
+
Provides-Extra: k8s
|
|
33
|
+
Requires-Dist: kubernetes>=29; extra == "k8s"
|
|
34
|
+
Provides-Extra: artifacts
|
|
35
|
+
Requires-Dist: minio>=7.2; extra == "artifacts"
|
|
36
|
+
Provides-Extra: oss
|
|
37
|
+
Requires-Dist: oss2>=2.18; extra == "oss"
|
|
38
|
+
Provides-Extra: monitor
|
|
39
|
+
Requires-Dist: psutil>=5.9; extra == "monitor"
|
|
40
|
+
Provides-Extra: service
|
|
41
|
+
Requires-Dist: fastapi>=0.111; extra == "service"
|
|
42
|
+
Requires-Dist: uvicorn>=0.30; extra == "service"
|
|
43
|
+
Requires-Dist: httpx>=0.27; extra == "service"
|
|
44
|
+
Requires-Dist: kubernetes>=29; extra == "service"
|
|
45
|
+
Requires-Dist: minio>=7.2; extra == "service"
|
|
46
|
+
Requires-Dist: pyyaml>=6; extra == "service"
|
|
47
|
+
Provides-Extra: dev
|
|
48
|
+
Requires-Dist: pytest; extra == "dev"
|
|
49
|
+
Requires-Dist: pytest-asyncio>=0.23; extra == "dev"
|
|
50
|
+
Requires-Dist: numpy>=1.26; extra == "dev"
|
|
51
|
+
Requires-Dist: pyyaml>=6; extra == "dev"
|
|
52
|
+
Requires-Dist: markdown>=3.6; extra == "dev"
|
|
53
|
+
Dynamic: license-file
|
|
54
|
+
|
|
55
|
+
# FlashRuntime
|
|
56
|
+
|
|
57
|
+
> **The open fault-tolerant distributed ML runtime.** FlashRuntime *operates*
|
|
58
|
+
> your training job — it never rewrites it. You keep the model, the training
|
|
59
|
+
> loop, the loss, the data, and the framework you already have. FlashRuntime
|
|
60
|
+
> wraps the reliability layer around them: it launches your command, injects
|
|
61
|
+
> the environment it promised, tracks your metrics, validates your checkpoints,
|
|
62
|
+
> **recovers from crashes in one call**, and collects your artifacts. The
|
|
63
|
+
> distributed computation itself is always done by established libraries
|
|
64
|
+
> (PyTorch DDP/FSDP2/torchrun, Ray, Hugging Face, scikit-learn).
|
|
65
|
+
|
|
66
|
+
FlashRuntime is one of three components in the FlashML system by
|
|
67
|
+
[Zolli Labs](https://github.com/Zolli-Labs):
|
|
68
|
+
|
|
69
|
+
- **[flashnode](https://github.com/Zolli-Labs/flashnode)** — open host agent
|
|
70
|
+
installed by resource contributors.
|
|
71
|
+
- **flashruntime** (this repo) — the open workload protocol and execution
|
|
72
|
+
layer. Self-hostable: useful without the cloud.
|
|
73
|
+
- **flashml-cloud** (private) — the managed control plane, marketplace, and
|
|
74
|
+
dashboard.
|
|
75
|
+
|
|
76
|
+
Read [`docs/SYSTEM_OVERVIEW.md`](docs/SYSTEM_OVERVIEW.md) for the full product
|
|
77
|
+
architecture, and [`AGENTS.md`](AGENTS.md) if you are an AI coding agent working
|
|
78
|
+
in this repo.
|
|
79
|
+
|
|
80
|
+
## Install
|
|
81
|
+
|
|
82
|
+
The core is deliberately tiny — `pip install flashruntime` brings **only
|
|
83
|
+
pydantic**. Every core module (the `flash.submit()` SDK, planner, leases,
|
|
84
|
+
checkpoints, recovery) works with zero infrastructure:
|
|
85
|
+
|
|
86
|
+
```bash
|
|
87
|
+
pip install flashruntime
|
|
88
|
+
```
|
|
89
|
+
|
|
90
|
+
Torch is **not** a dependency. FlashRuntime *launches* PyTorch; it never imports
|
|
91
|
+
it. Install it yourself to run the DDP example — a CPU-only build is enough,
|
|
92
|
+
since DDP works over the `gloo` backend with no GPU:
|
|
93
|
+
|
|
94
|
+
```bash
|
|
95
|
+
pip install torch # CPU build is fine
|
|
96
|
+
```
|
|
97
|
+
|
|
98
|
+
## 60-second run
|
|
99
|
+
|
|
100
|
+
`flash.submit()` operates any command. The one convention your script owes
|
|
101
|
+
FlashRuntime is to write a `metrics.json` (a flat JSON object) into its working
|
|
102
|
+
directory; FlashRuntime collects it and records it as a trial:
|
|
103
|
+
|
|
104
|
+
```python
|
|
105
|
+
import flashruntime as flash
|
|
106
|
+
|
|
107
|
+
run = flash.submit(
|
|
108
|
+
flash.CommandWorkload(
|
|
109
|
+
command="python train.py --epochs 5",
|
|
110
|
+
source=flash.Source(path="~/my-project"),
|
|
111
|
+
outputs=flash.OutputSpec(collect=["metrics.json"]),
|
|
112
|
+
),
|
|
113
|
+
max_restarts=2, # a crashed attempt relaunches from the last VALID
|
|
114
|
+
# checkpoint — up to twice (see below)
|
|
115
|
+
watch=True, # opens the live run page and prints its URL
|
|
116
|
+
)
|
|
117
|
+
|
|
118
|
+
print(run.state.value) # "SUCCEEDED" (or "FAILED")
|
|
119
|
+
print(run.trials) # [{'accuracy': 0.91, ...}] — parsed from metrics.json
|
|
120
|
+
print(run.artifacts) # [PosixPath('.../metrics.json'), ...]
|
|
121
|
+
print(run.viewer_url) # http://127.0.0.1:<port> — the live page
|
|
122
|
+
```
|
|
123
|
+
|
|
124
|
+
`command` is `shlex`-split (there is no shell — for a pipe, pass
|
|
125
|
+
`command="bash -c '...'"`), and `source` is a `flash.Source`, so `~` is expanded
|
|
126
|
+
for you. The same thing from the shell:
|
|
127
|
+
|
|
128
|
+
```bash
|
|
129
|
+
flashruntime submit "python train.py --epochs 5" \
|
|
130
|
+
--source ~/my-project --max-restarts 2 --watch
|
|
131
|
+
```
|
|
132
|
+
|
|
133
|
+
A script that already reads its hyperparameters from `argparse` and writes a
|
|
134
|
+
small JSON of results needs **zero FlashRuntime imports** to be operated. The
|
|
135
|
+
contract at the boundary is thin on purpose: arguments in, `metrics.json` out.
|
|
136
|
+
|
|
137
|
+
## One import, fault tolerant
|
|
138
|
+
|
|
139
|
+
`max_restarts=N` is the automatic recovery budget. On a **FAILED** attempt
|
|
140
|
+
FlashRuntime turns the exit into failure signals, classifies them
|
|
141
|
+
(`recovery.classify`), and consults a *versioned, deterministic* policy
|
|
142
|
+
(`recovery.decide`): a deterministic application bug fails fast — a retry would
|
|
143
|
+
only re-hit it — while a transient failure relaunches the same spec from the
|
|
144
|
+
job-scoped checkpoint. Same failure + same policy version ⇒ same action, every
|
|
145
|
+
time. **No LLM in the loop.** Every decision is recorded on the run as
|
|
146
|
+
`FAILURE_CLASSIFIED` / `RECOVERY_ACTION_SELECTED` events.
|
|
147
|
+
|
|
148
|
+
Kill-and-resume is proven end to end. `tests/test_examples_e2e.py` kills a
|
|
149
|
+
2-process DDP run mid-training and asserts the resumed run's final loss
|
|
150
|
+
matches a never-crashed baseline to 1e-6 — recovery restores only a verified,
|
|
151
|
+
topology-compatible checkpoint (parts-first / manifest-last: a half-written
|
|
152
|
+
checkpoint can never look valid).
|
|
153
|
+
|
|
154
|
+
For a script you *are* willing to touch, `import flashruntime.torch as ft` adds
|
|
155
|
+
`ft.prepare` / `ft.checkpoint` / `ft.log_metrics`: torch's own DDP wrapped under
|
|
156
|
+
the same checkpoint contract, with correct per-rank device placement, so a
|
|
157
|
+
killed run resumes with a loss curve indistinguishable from the uninterrupted
|
|
158
|
+
one (1e-6 in the e2e). It is optional sugar on the launch-only
|
|
159
|
+
contract, never required.
|
|
160
|
+
|
|
161
|
+
**GPU-validated (2026-07-23).** The CUDA paths — `nccl` DDP, per-rank device
|
|
162
|
+
placement, and a GPU kill-and-resume — are validated end to end on real
|
|
163
|
+
hardware: **2×NVIDIA GeForce RTX 4090** (RunPod community cloud), torch
|
|
164
|
+
2.7.1+cu128, CUDA 12.8. Two runs, **$0.0725 total**. See `tests/test_gpu_e2e.py`
|
|
165
|
+
and the harness `scripts/runpod_gpu_e2e.py`.
|
|
166
|
+
|
|
167
|
+
## Watch it run
|
|
168
|
+
|
|
169
|
+
Pass `watch=True` (the default at an interactive terminal, off in pipes/CI) and
|
|
170
|
+
`flash.submit()` opens a live run page in your browser and prints its URL. It
|
|
171
|
+
draws the run's topology, its loss curve, its verified checkpoints, and every
|
|
172
|
+
recovery decision, refreshing every couple of seconds — served entirely from a
|
|
173
|
+
loopback server with **no external assets**, so it renders with the network cut.
|
|
174
|
+
These docs are served from that same viewer at `/docs`.
|
|
175
|
+
|
|
176
|
+
> _screenshot: run any `flashruntime submit … --watch` to see the live page._
|
|
177
|
+
|
|
178
|
+
## Documentation
|
|
179
|
+
|
|
180
|
+
A PyTorch-style docs site is built from [`docs/site/`](docs/site/) by
|
|
181
|
+
`scripts/build_docs.py` (every byte inline — no CDN, web font, or remote image).
|
|
182
|
+
It is served offline by the viewer at `/docs`, and deploys to GitHub Pages on
|
|
183
|
+
release:
|
|
184
|
+
|
|
185
|
+
- **<https://zolli-labs.github.io/flashruntime/>** — _live after the first
|
|
186
|
+
Pages deploy._
|
|
187
|
+
|
|
188
|
+
Start with [Get started](docs/site/get-started.md) (install → first job → first
|
|
189
|
+
DDP run on CPU). The strategy planner's code walkthrough lives in
|
|
190
|
+
[`docs/planner/README.md`](docs/planner/README.md).
|
|
191
|
+
|
|
192
|
+
## Benchmarks
|
|
193
|
+
|
|
194
|
+
An honest benchmark suite (`python -m benchmarks run`) records every result with
|
|
195
|
+
the host that produced it and reports missing comparators as skipped rows, never
|
|
196
|
+
faked. A measured baseline is committed under
|
|
197
|
+
[`benchmarks/results/`](benchmarks/results/). On **Apple M4 (CPU)**:
|
|
198
|
+
|
|
199
|
+
| Scenario | Measured | Read it honestly |
|
|
200
|
+
|---|---|---|
|
|
201
|
+
| Launch overhead | **+0.04 s** | `flash.submit` vs bare `torchrun`, paired. A one-time fixed cost, not per-step. |
|
|
202
|
+
| Checkpoint cost | **−1.2 ms / checkpoint** (p90 6.2 ms) | A few-KB state dict; the median delta is *below* the run-to-run noise floor at this size (it goes slightly negative), so checkpoints are effectively free here — the p90 bounds it. A larger model surfaces a real positive cost. |
|
|
203
|
+
| Auto-resume | **40 steps not recomputed** | The size-*independent* guarantee: resume never re-does work past the last valid checkpoint. |
|
|
204
|
+
| Adoption | **7 lines** | To make a vanilla script framework-ready (difflib vs each project's own docs). |
|
|
205
|
+
| Resilience | **5/5 faults handled · 20/20 integrity under `kill -9`** | Every fault type routed to the right typed recovery; a checkpoint survives a mid-write `SIGKILL` on 20/20 kills (naive `torch.save`: 20/20 corrupted). 16-trial storm, half crash-armed: 16/16 completed, all 8 crashed trials auto-resumed, 0 manual. Dead-worker MTTD **3.05 s**, MTTR **~3.5 ms**. |
|
|
206
|
+
|
|
207
|
+
The *timing* rows are smoke-scale on one host: the models are tiny, so the
|
|
208
|
+
*wall-clock* saved by recovery is at the noise floor here. The figure that
|
|
209
|
+
scales is `steps_not_recomputed`, not seconds — the seconds-saved grows with the
|
|
210
|
+
compute between the last checkpoint and the crash (negligible at smoke size,
|
|
211
|
+
hours on a real job). The resilience counts are exact, not smoke-diluted: they
|
|
212
|
+
are *counted* from real fault injection (`kill -9` in the checkpoint-write
|
|
213
|
+
window, a storm of crash-armed trials, a killed lease worker), and the full
|
|
214
|
+
Resilience table with per-scenario method lives in
|
|
215
|
+
[the benchmarks page](docs/site/benchmarks.md). Full provenance and caveats live
|
|
216
|
+
in the result file.
|
|
217
|
+
|
|
218
|
+
---
|
|
219
|
+
|
|
220
|
+
## Also: plan a job (no cluster required)
|
|
221
|
+
|
|
222
|
+
Before you launch, `flash.plan()` turns model + hardware + budget + deadline
|
|
223
|
+
into a ranked, explained execution plan — deterministic, framework-import-free:
|
|
224
|
+
|
|
225
|
+
```python
|
|
226
|
+
import flashruntime as flash
|
|
227
|
+
|
|
228
|
+
report = flash.plan(flash.PlanRequest(
|
|
229
|
+
workload=flash.TransformerFineTune(model="Qwen/Qwen2.5-7B", method="lora",
|
|
230
|
+
train_tokens_m=25),
|
|
231
|
+
resources=flash.Resources(gpus=4, gpu_type="RTX4090",
|
|
232
|
+
hourly_cost_usd_per_gpu=0.44),
|
|
233
|
+
objective=flash.Objective(mode="balanced", max_cost_usd=20,
|
|
234
|
+
deadline_minutes=240),
|
|
235
|
+
))
|
|
236
|
+
print(flash.render(report))
|
|
237
|
+
# → SELECTED: ddp + lora(r=16), 4 workers, torchrun, 21.6 GB/GPU, ~96 min,
|
|
238
|
+
# $2.82 — plus every rejected candidate with its arithmetic
|
|
239
|
+
```
|
|
240
|
+
|
|
241
|
+
The report names the distributed method, the libraries it is built from
|
|
242
|
+
(torchrun, DDP/FSDP2, transformers+peft, bitsandbytes, PyTorch DCP) and their
|
|
243
|
+
roles, the knobs, and *why* — including why the alternatives lost. Details:
|
|
244
|
+
[`docs/planner/README.md`](docs/planner/README.md).
|
|
245
|
+
|
|
246
|
+
## Also: leases, checkpoints, recovery
|
|
247
|
+
|
|
248
|
+
The reliability core is an embeddable library, not a service you must run:
|
|
249
|
+
|
|
250
|
+
```python
|
|
251
|
+
from flashruntime.leases import LeaseManager
|
|
252
|
+
from flashruntime.recovery import FailureSignals, classify, decide
|
|
253
|
+
from flashruntime.protocol.v1alpha1 import TaskSpec
|
|
254
|
+
|
|
255
|
+
mgr = LeaseManager()
|
|
256
|
+
mgr.add_task(TaskSpec(task_id="trial-01", job_id="sweep", commit_key="sweep/trial-01"))
|
|
257
|
+
lease = mgr.claim(node_id="laptop-1") # pull, never push
|
|
258
|
+
mgr.heartbeat(lease.lease_id) # prove liveness
|
|
259
|
+
mgr.complete(lease.lease_id, output_sha256="…") # first valid commit wins
|
|
260
|
+
|
|
261
|
+
decision = decide(classify(FailureSignals(heartbeat_lost=True)),
|
|
262
|
+
mode="independent_tasks") # → RETRY_TASK, cordon node
|
|
263
|
+
```
|
|
264
|
+
|
|
265
|
+
The FlashRuntime service exposes these same objects over HTTP; FlashNode's
|
|
266
|
+
device executor is their remote client. The lease coordinator's durable
|
|
267
|
+
`SqliteLeaseStore` means in-flight leases survive a coordinator restart.
|
|
268
|
+
|
|
269
|
+
## Testing
|
|
270
|
+
|
|
271
|
+
```bash
|
|
272
|
+
pytest # unit tests — pure Python, no infrastructure
|
|
273
|
+
pytest -m integration # opt-in: Docker / Kubernetes / MinIO environments
|
|
274
|
+
```
|
|
275
|
+
|
|
276
|
+
Integration tests live in [`tests/integration/`](tests/integration/README.md)
|
|
277
|
+
and skip themselves (with instructions) when their environment is absent. The
|
|
278
|
+
one GPU test (`tests/test_gpu_e2e.py`) is CUDA-gated and skips without a GPU. The
|
|
279
|
+
local kind + KubeRay + MinIO stack is owned by the workspace Makefile, not this
|
|
280
|
+
repo — the library stays `pip install`-clean.
|
|
281
|
+
|
|
282
|
+
## Package layout
|
|
283
|
+
|
|
284
|
+
**Pure-Python core** — `pip install flashruntime` brings pydantic and nothing
|
|
285
|
+
else; every module below works with zero infrastructure:
|
|
286
|
+
|
|
287
|
+
```
|
|
288
|
+
flashruntime/
|
|
289
|
+
├── protocol/ # versioned public schemas (v1alpha1: JobSpec, Event, Lease,
|
|
290
|
+
│ # TaskAttempt, CheckpointManifest, RecoveryDecision, plans)
|
|
291
|
+
├── sdk.py # flash.submit(CommandWorkload) → Run: compile→launch→wait→
|
|
292
|
+
│ # collect, with the automatic max_restarts recovery loop
|
|
293
|
+
├── workloads/ # CommandWorkload: framework-neutral "run this command"
|
|
294
|
+
├── integrations/# one-file adapters (sklearn / pytorch DDP / HF) — no framework
|
|
295
|
+
│ # imports at module level; convention is CLI-flags-in/JSON-out
|
|
296
|
+
├── torch/ # optional in-script helper: ft.prepare / checkpoint / log_metrics
|
|
297
|
+
├── planner/ # strategy planner: estimators, curated menus, explained selector
|
|
298
|
+
├── leases/ # Mode A core: claim / heartbeat / expiry sweep / idempotent
|
|
299
|
+
│ # commit; InMemory + SQLite stores (restart-durable)
|
|
300
|
+
├── checkpoint/ # manifest catalog: parts-first/manifest-last, validation ladder
|
|
301
|
+
├── recovery/ # failure-signal classifier + versioned deterministic policy
|
|
302
|
+
└── viewer/ # stdlib live run page + the built docs site (zero external assets)
|
|
303
|
+
```
|
|
304
|
+
|
|
305
|
+
**Infrastructure integrations** (opt-in extras, never core imports):
|
|
306
|
+
|
|
307
|
+
```
|
|
308
|
+
├── service/ # [service] the coordinator: job submission + task expansion,
|
|
309
|
+
│ # lease/node/checkpoint HTTP endpoints, local artifact hosting,
|
|
310
|
+
│ # join codes, SQLite ledger, dashboard at GET /, the CLI
|
|
311
|
+
├── launchers/ # LocalProcessLauncher (the concrete local launcher today)
|
|
312
|
+
├── strategies/ # CommandWorkload → LaunchSpec compilation
|
|
313
|
+
├── recipes/ # coordinator-side WorkloadRecipe registry (command jobs)
|
|
314
|
+
├── backends/ # [k8s] ExecutionBackend contract + KubeRay backend (Mode B)
|
|
315
|
+
└── artifacts/ # [artifacts]/[oss] MinIO/S3-compatible + Alibaba OSS stores
|
|
316
|
+
|
|
317
|
+
flashml_workloads/ # runnable task modules (pure stdlib/sklearn):
|
|
318
|
+
│ # sklearn_trial, kmeans_shard + kmeans_driver,
|
|
319
|
+
│ # sgd_trainer (checkpointable, bit-identical resume)
|
|
320
|
+
```
|
|
321
|
+
|
|
322
|
+
Still to come (each lands with its vertical slice, no empty scaffolds):
|
|
323
|
+
multi-node DDP rendezvous (`nnodes > 1` raises `NotImplementedError` today);
|
|
324
|
+
FlashNode's argv runner so service-side command jobs execute remotely;
|
|
325
|
+
checkpoint-manifest persistence (catalog is in-memory; checkpoint *files* are
|
|
326
|
+
durable); Stage-8 ledger metrics (MTTD/MTTR/goodput); `flash.run(plan)` wiring;
|
|
327
|
+
the cloud stage (Postgres, SSE, HF Trainer + PEFT LoRA recipes).
|
|
328
|
+
|
|
329
|
+
Four orthogonal axes keep the architecture honest: **providers** get machines,
|
|
330
|
+
**launchers** start processes, **strategies** configure execution, **recipes**
|
|
331
|
+
integrate user code. Hugging Face lives in recipes — it is the workload layer,
|
|
332
|
+
not an execution backend. The planner never imports framework code; it emits a
|
|
333
|
+
backend-neutral `StrategyPlan` that strategy compilers translate.
|
|
334
|
+
|
|
335
|
+
## What FlashRuntime builds on (and what it owns)
|
|
336
|
+
|
|
337
|
+
| Layer | Libraries reused | What FlashRuntime adds |
|
|
338
|
+
|---|---|---|
|
|
339
|
+
| ML math & models | PyTorch, HF Transformers + PEFT, bitsandbytes, scikit-learn | Nothing — untouched, operated by recipes |
|
|
340
|
+
| Distribution strategies | DDP, FSDP2 (`fully_shard`); DeepSpeed ZeRO later | The *choice* and its explanation, compiled from a StrategyPlan |
|
|
341
|
+
| Launching | torchrun / Torch Elastic; Ray Core on clusters | Checkpoint-aware restart around torchrun; the **lease/heartbeat protocol** for machines outside any cluster — the layer with no existing library |
|
|
342
|
+
| Checkpointing | PyTorch Distributed Checkpoint (parallel save/load, resharding) | The manifest **catalog**: written-last commit, validation status, compatibility, recovery-time selection |
|
|
343
|
+
| Infrastructure | Kubernetes + KubeRay today; SkyPilot / Slurm / provider APIs later | Provider adapters + one cross-environment job model |
|
|
344
|
+
|
|
345
|
+
Deliberately not built on: HF Accelerate (overlaps torchrun and strategy
|
|
346
|
+
config — user scripts that use it still work via the launch-only contract) and
|
|
347
|
+
Ray Train (mid V1→V2 migration).
|
|
348
|
+
|
|
349
|
+
## The dependency rule
|
|
350
|
+
|
|
351
|
+
`flashruntime` owns the public protocol and imports **neither** application
|
|
352
|
+
repo. `flashnode` and `flashml-cloud` both depend on `flashruntime`; they never
|
|
353
|
+
import each other.
|
|
354
|
+
|
|
355
|
+
## Contributing, security, changelog
|
|
356
|
+
|
|
357
|
+
- [`CONTRIBUTING.md`](CONTRIBUTING.md) — dev setup, the test/doc/audit gates, and
|
|
358
|
+
the Developer Certificate of Origin (`git commit -s`).
|
|
359
|
+
- [`SECURITY.md`](SECURITY.md) — how to report a vulnerability.
|
|
360
|
+
- [`CHANGELOG.md`](CHANGELOG.md) — release notes ([Keep a Changelog](https://keepachangelog.com/)).
|
|
361
|
+
|
|
362
|
+
## License
|
|
363
|
+
|
|
364
|
+
[Apache-2.0](LICENSE). Contributions via Developer Certificate of Origin
|
|
365
|
+
(`git commit -s`).
|
|
@@ -0,0 +1,95 @@
|
|
|
1
|
+
flashml_workloads/__init__.py,sha256=pIep8hH1zxpzY1ePodLdVSzH1N0vIogEQyFPrjgk6d0,310
|
|
2
|
+
flashml_workloads/fedavg_driver.py,sha256=nsB9bZCKZQ1I7uY67wVFUPQ672bA7kBcyXLIi3W_YzA,25803
|
|
3
|
+
flashml_workloads/fedavg_weights.py,sha256=U_a7JzOhciZs0bRzKoZn63dnGgQQ6S79bhd5sbDXWVc,9537
|
|
4
|
+
flashml_workloads/fedavg_worker.py,sha256=jZhH5myvuOhE2QOAGlFpySxCsaUmoRs4eTdUJYRQowY,5853
|
|
5
|
+
flashml_workloads/kmeans_driver.py,sha256=F5krmQ1ExqbVO1ZWEy89Fk3mlCS57Hwe3ZD8oNN_srY,5270
|
|
6
|
+
flashml_workloads/kmeans_shard.py,sha256=ktDBDoC41eTqGYxxR-Cm4heq5Ou-LNhU5cgVvXwd_d0,2199
|
|
7
|
+
flashml_workloads/sgd_trainer.py,sha256=uTWWX-CGbz3JrlRzKSKUzoeUiQJ0wwmn9tkTbJ-RcLY,4234
|
|
8
|
+
flashml_workloads/sharded_kmeans.py,sha256=XTLWBEN4IA3EjadctxcHQxkiHOVWEOVnbpwcgxui-1k,12981
|
|
9
|
+
flashml_workloads/sklearn_trial.py,sha256=VsifaHcRK2sqkO5l5skfm3ddOpskLuU1dvt_zdJHev8,3170
|
|
10
|
+
flashruntime/__init__.py,sha256=nCUteqemDt70iptY7r22iSQPInrhQK-xR_VpVE_j7fs,4774
|
|
11
|
+
flashruntime/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0
|
|
12
|
+
flashruntime/sdk.py,sha256=UsfBOwnhF-aLHvyGcLZTrL_n03V5Qg41XfRlDrQAVfE,19004
|
|
13
|
+
flashruntime/artifacts/__init__.py,sha256=wApVUFtJOwsfyevfeleTSTohJ5LHftORJ3ozjWOFBQc,684
|
|
14
|
+
flashruntime/artifacts/store.py,sha256=zCFFpQeljYVwBkAmpPfLeJO2z-9eyTogRtE5fObxIew,7491
|
|
15
|
+
flashruntime/backends/__init__.py,sha256=_kUmnFZQoJaVxjYb7USR4ulf-il0VxAeFuOr_ZG_CL8,813
|
|
16
|
+
flashruntime/backends/base.py,sha256=8bz-xiY1x_HbHQvS61Pcp5C_Kra6_6YvwKL7j66sapE,1928
|
|
17
|
+
flashruntime/backends/kuberay.py,sha256=jGE3WgzWno6D193csYh2Xe1ZfqAE15hCLqSvQmFs9Qo,18634
|
|
18
|
+
flashruntime/checkpoint/__init__.py,sha256=nos0gYooP8K4DePVdG8RXOyhzbPLq_UhNjHzAMwMYOA,932
|
|
19
|
+
flashruntime/checkpoint/catalog.py,sha256=Z0AuYQ3myQUawAPndJbQoHA_dqSDBzoMHIfjhs8MVBM,8042
|
|
20
|
+
flashruntime/checkpoint/local.py,sha256=oMbsvD-bPdHA8U0epHGkTDzowOEFFVNgwqGzpxdaiiM,3943
|
|
21
|
+
flashruntime/checkpoint/store.py,sha256=JeHpiiD9ysjvy3Rc6_2Woqzc0Xrk228ch6YTMuMy8M8,3808
|
|
22
|
+
flashruntime/integrations/__init__.py,sha256=dsp6-hIvN2NSr9MuFDoP5EoslB3CMgNvdvmx23NDydk,295
|
|
23
|
+
flashruntime/integrations/huggingface.py,sha256=FgeAM8R6Gc7SD9wIMDoUFDtZF9EkzuriY63hF95OxOE,2447
|
|
24
|
+
flashruntime/integrations/pytorch.py,sha256=0Rh5ENwFUevL05OxidO9aiB1o4djhQ35EmLD0s1ULHk,1774
|
|
25
|
+
flashruntime/integrations/sklearn.py,sha256=HZ1uV-R79rQCmY58KYcn9_LYOkdbTuGfjNyk058RN18,1418
|
|
26
|
+
flashruntime/launchers/__init__.py,sha256=LV0wgeRvwnXx08h_rbXFxv0bQNeifEbw_3nium9MvIw,5490
|
|
27
|
+
flashruntime/launchers/local.py,sha256=27W7BMhen_1oLSu9f2AFcruTIaPyJpyqsxtwTFDb2KE,4845
|
|
28
|
+
flashruntime/leases/__init__.py,sha256=ocBDQPQmyVJldENdT0_UCa57w8SAmrhMPG7FnSgTBkM,1075
|
|
29
|
+
flashruntime/leases/manager.py,sha256=YCvj0eNzuUX5w6J5xd7yu7X1KPHQV8hG3aZywziD-ss,14766
|
|
30
|
+
flashruntime/leases/sqlite_store.py,sha256=-OaunF91vaqJe3hzIsn2VEnvYgMFPE7gGhoNk6pB4D0,6845
|
|
31
|
+
flashruntime/leases/store.py,sha256=iMBcd00sTyrdjTOAvALamQyZ2AtOTioZVtkEFySzHbg,3780
|
|
32
|
+
flashruntime/monitor/__init__.py,sha256=1Uc4rh-ZLa0c30-doxYJj3yemcf2e-NE3xhX0T8uEcs,244
|
|
33
|
+
flashruntime/monitor/sampler.py,sha256=vBrF_ZMFu_CWcYoqdEUEScyBAiOyOssU45grSLQm4QI,9336
|
|
34
|
+
flashruntime/planner/__init__.py,sha256=jYn-AefGhGwUxWDliR-V5aJiinUahHBn5PvsiN-75oo,2206
|
|
35
|
+
flashruntime/planner/candidates.py,sha256=QF2fYZF9x5HH89KO-QXGRRPtfIkDPAjYOJ_gPXCrR0E,24249
|
|
36
|
+
flashruntime/planner/catalog.py,sha256=QvzENe6uiPVHdChhC82rcC-Yjjuxk9B8iXyIuTmCUHM,4308
|
|
37
|
+
flashruntime/planner/comm.py,sha256=UD_-Aiu2mOfcrPJLnOBEY9tUgDeGhmoiqHivIHq1Czw,3440
|
|
38
|
+
flashruntime/planner/explain.py,sha256=5vmnEKOgwhmlO5xkw2caxUc_En1iylVna6lmcmM1jy4,4527
|
|
39
|
+
flashruntime/planner/memory.py,sha256=6BZguaCzvxvCx6hBqvpXFZ6bSD2oZ8Fz802f2ioIEAw,6492
|
|
40
|
+
flashruntime/planner/resolve.py,sha256=KtMxPUUEz1LAEwnv7rFlobRG37pUAFCRBoQLvprCZPE,4212
|
|
41
|
+
flashruntime/planner/selector.py,sha256=b-VuacYmlqmHWKUmr6niwnYpYTo3YX-hTfaGbRaS-_k,6134
|
|
42
|
+
flashruntime/planner/timecost.py,sha256=nCAtGfBg5FnY1rSB0nGb5-WY1IMZBSDeZzZJ8GE3XRg,3254
|
|
43
|
+
flashruntime/profiling/__init__.py,sha256=ncysF1ZC9qgp5rOBCPpaPEhk_aFZE7FYvg9RQiDeueg,4933
|
|
44
|
+
flashruntime/protocol/__init__.py,sha256=ZmEFFgfushwzsTVz8yS59JUOTtHH6DsyPk0bM6fv0gw,713
|
|
45
|
+
flashruntime/protocol/plan_v1alpha1.py,sha256=uncKTD8_fWkTZfx8wXrzVb-MOiXQNIf6K2ZwJcNIoJ8,12039
|
|
46
|
+
flashruntime/protocol/v1alpha1.py,sha256=lav3N-GglYXe2YYYua0hu8-soKGwn6gupFl7zKC2k18,16621
|
|
47
|
+
flashruntime/providers/__init__.py,sha256=mLlpW7ybdMdAFfKJLnYPQougAEO2vWMdfyJccXniSRk,5686
|
|
48
|
+
flashruntime/recipes/__init__.py,sha256=OeAwksQgJCxQPtj6-WZYD-X5t_RBtt7PTgJwEwYa4Q0,5960
|
|
49
|
+
flashruntime/recipes/command.py,sha256=j4eklDwEjoZumtuiPTasFvJ3OM-E74vrRomoMv3qD4s,7533
|
|
50
|
+
flashruntime/recovery/__init__.py,sha256=HJNxkIPW5ntOJo0_hrMRtaEP1-xFddG8iLjGzP1OS0s,957
|
|
51
|
+
flashruntime/recovery/policy.py,sha256=djvixKDdSBH-acFNQcn9Djnif42E6ojyUCmLthuKj3E,8450
|
|
52
|
+
flashruntime/recovery/signals.py,sha256=RM2utaCxUYiZ3hVBHNhrRHgL250K2Qo09av1MFaUHFo,7742
|
|
53
|
+
flashruntime/recovery/taxonomy.py,sha256=f-szAtRPJJb5l9TIl-XJVqmdapVNwmDR6qSS-fEAq90,3399
|
|
54
|
+
flashruntime/scheduler/__init__.py,sha256=96CvNI_wmXZTOFGNWZfHwzc030h7I2tgyPr0gMWIMCE,8709
|
|
55
|
+
flashruntime/service/__init__.py,sha256=JZTxtllXIIdZz4znR3QIvdjy3Z_tBtYJ-vquqbNIG_A,219
|
|
56
|
+
flashruntime/service/app.py,sha256=-e0pJjTDUyBAJUBRfeQEA5g_KiofBJKRhgbLxjCv1tg,16538
|
|
57
|
+
flashruntime/service/auth.py,sha256=c61pLGkBBcPG2aQrp28N1FqTmjmqYZ4uCaXl7UST0J8,7072
|
|
58
|
+
flashruntime/service/checkpoints.py,sha256=kT_ghRouGpYbskQ78RdvINZQZ1qSevo0_7AiAn3vEvg,3481
|
|
59
|
+
flashruntime/service/cli.py,sha256=HHCNRlimAv0HOTE30KI7ynlKSXr8Rvifn_NzAfETJkw,6445
|
|
60
|
+
flashruntime/service/dashboard.py,sha256=Zy6afH_kpFlnyGrvuSup9gtERryzpRcxQLeMLcMftic,7896
|
|
61
|
+
flashruntime/service/ledger.py,sha256=CebBgh0ZNSPHlla-VMRDI2S6QzxO3HB61hOcc1YAqdA,3282
|
|
62
|
+
flashruntime/service/modea.py,sha256=y1mDzt8C1H71zOXFShkaLGCO0zOI5RBSF6iMyTibXxM,35327
|
|
63
|
+
flashruntime/strategies/__init__.py,sha256=mnBLEpgmvvgo2CrCptEvHuf39FQxIDMRp4N_-Wd8z7g,6470
|
|
64
|
+
flashruntime/strategies/command.py,sha256=Z_qUlvibjT5MJloAZsV0kWnRqJmE1PJcQQEaGvB8i-o,2294
|
|
65
|
+
flashruntime/torch/__init__.py,sha256=UX4Z8yhhHDpgJqt4KMyW0VOK9eA1RWr090ioZ-eSiTs,10801
|
|
66
|
+
flashruntime/viewer/__init__.py,sha256=xDroJ1Rwl-AVEmu01QFnvzAo49X0jUwFrO5w2YKErwQ,822
|
|
67
|
+
flashruntime/viewer/flowmap.py,sha256=1amqvoii9ovGptnL7rnTK3RdUDE3I0qi_o_iEgmk4pY,16095
|
|
68
|
+
flashruntime/viewer/page.py,sha256=vLL6p5rCOKTCbvc1Kc-1TQeYgu97dUJk6z7ikbeDYVk,23601
|
|
69
|
+
flashruntime/viewer/server.py,sha256=2Zk6AP7lWLzdGThoJUpbKpok3rlZJj6psFNYe_sWAPM,6179
|
|
70
|
+
flashruntime/viewer/state.py,sha256=r0aa_J2QxYSqRrNUYE0UMf3TCTLwcGby_O7SrCGAdmA,11446
|
|
71
|
+
flashruntime/viewer/_docs/benchmarks.html,sha256=e4wZuXSh6cf2jU1p1_wCDr9XgngRH21voWsB7FctzeE,33119
|
|
72
|
+
flashruntime/viewer/_docs/get-started.html,sha256=O8ZdzZgkgZs0PNTEmg_kFCTkf-K5mEOujzzGSh3QwNA,15791
|
|
73
|
+
flashruntime/viewer/_docs/index.html,sha256=KpUt-dpTFjKfZCI1k6Wk3xxZgwz8Y7Sm3lmTgFw746k,14278
|
|
74
|
+
flashruntime/viewer/_docs/search-index.json,sha256=fSKcAEEM0bN20deayeTe29QQ0BwTFhuBYYVY2e-Gr9E,106694
|
|
75
|
+
flashruntime/viewer/_docs/concepts/architecture.html,sha256=CPZ9k7lj8HNUjBAQ5KUAqZth97B6OubunXUDp22F6nI,19568
|
|
76
|
+
flashruntime/viewer/_docs/guides/federated-averaging.html,sha256=h_bO--34hN_fS81SdmQYrDUaxJc1Y0SqxNMkd7x5kgQ,22524
|
|
77
|
+
flashruntime/viewer/_docs/guides/huggingface.html,sha256=ZYKRK9AERD4b2MYMa3sBMc1BHytRgvmFRtK-desvz-4,12890
|
|
78
|
+
flashruntime/viewer/_docs/guides/jobspec-and-isolation.html,sha256=U-1QtU-QOMNjEqoKxV-vZMTVYrKXymHvIRxQu_7GDyE,15531
|
|
79
|
+
flashruntime/viewer/_docs/guides/pytorch.html,sha256=-qtojwYaiTVf6gt-G4YWwhVs9X2J4hN8i6RyaSyh4lk,18264
|
|
80
|
+
flashruntime/viewer/_docs/guides/sklearn.html,sha256=32BnmGP_eBQgQNnGe_ieZl--IyYUwpqVaXAzeXUZ6ZQ,13498
|
|
81
|
+
flashruntime/viewer/_docs/reference/cli.html,sha256=4urAUM13nVQKyLLtPWpYpJEZ_s0gdvUWZB92At-MqAk,14341
|
|
82
|
+
flashruntime/viewer/_docs/reference/integrations.html,sha256=I5eZZk_rd9Tw7F6YtgTvsgPzHx088JvubttChVJvS3M,13676
|
|
83
|
+
flashruntime/viewer/_docs/reference/sdk.html,sha256=6in4enErqFmO0OaSPw9UA7ek_fXrTEW2MiQ5DYrxbVM,19183
|
|
84
|
+
flashruntime/viewer/_docs/reference/torch-helper.html,sha256=hrWHYc9Op1NwLrk37Ox2MhuLVWwW8UeZFL7vn6-2vMo,14435
|
|
85
|
+
flashruntime/viewer/_docs/tutorials/convnet.html,sha256=5cupZBXjjOcnHPoTR2NdgXPacnTLQKCfzbE_tKpG1sA,30784
|
|
86
|
+
flashruntime/viewer/_docs/tutorials/fault-tolerance.html,sha256=s32yBtDHWZF7pAHbXoses1bVmtyWwWKEyNTMzhQvqxw,20674
|
|
87
|
+
flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html,sha256=1c0ooeZHqgl8YSUJ_Cgxg1nf9VkIeYSkRtrj2UUdsbc,15813
|
|
88
|
+
flashruntime/workloads/__init__.py,sha256=R24mJHLQnhU45-MTPCpLkyTh7LAXFuJhLxb9AwMfgHY,307
|
|
89
|
+
flashruntime/workloads/command.py,sha256=Zbi5be3EI46orn7EWCdoG1PnmXuvK_c0uSxCCwB_AEw,4943
|
|
90
|
+
flashruntime-0.3.0.dist-info/licenses/LICENSE,sha256=z8d0m5b2O9McPEK1xHG_dWgUBT6EfBDz6wA0F7xSPTA,11358
|
|
91
|
+
flashruntime-0.3.0.dist-info/METADATA,sha256=cSOH0tjvezgdJdlW4UtV-FQNzbwverwyCPG1DqCr2uo,18154
|
|
92
|
+
flashruntime-0.3.0.dist-info/WHEEL,sha256=K260EYznzXsJYBQGqmI8VTxEdiZYNvDZwW9cBh9-_MA,91
|
|
93
|
+
flashruntime-0.3.0.dist-info/entry_points.txt,sha256=EfbCXQlEqL3_n-UJAO5i_vU6A-rUJmjrOaiQ1TifE3c,63
|
|
94
|
+
flashruntime-0.3.0.dist-info/top_level.txt,sha256=H7CA1FVVVRcCiNDylGQnaAUuGiP6hv82AsXhXN0EsxE,31
|
|
95
|
+
flashruntime-0.3.0.dist-info/RECORD,,
|