flashruntime 0.4.0__tar.gz → 0.4.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {flashruntime-0.4.0/flashruntime.egg-info → flashruntime-0.4.1}/PKG-INFO +1 -1
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/protocol/v1alpha1.py +37 -1
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recipes/command.py +20 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/scheduler/__init__.py +59 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1/flashruntime.egg-info}/PKG-INFO +1 -1
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/SOURCES.txt +3 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/pyproject.toml +17 -6
- flashruntime-0.4.1/tests/test_placement_gpu.py +301 -0
- flashruntime-0.4.1/tests/test_protocol_gpu.py +152 -0
- flashruntime-0.4.1/tests/test_recipe_gpu.py +129 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/LICENSE +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/README.md +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/fedavg_driver.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/fedavg_weights.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/fedavg_worker.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/kmeans_driver.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/kmeans_shard.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/sgd_trainer.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/sharded_kmeans.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/sklearn_trial.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/artifacts/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/artifacts/store.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/backends/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/backends/base.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/backends/kuberay.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/catalog.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/local.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/store.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/huggingface.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/pytorch.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/sklearn.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/launchers/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/launchers/local.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/manager.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/sqlite_store.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/store.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/monitor/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/monitor/sampler.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/candidates.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/catalog.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/comm.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/explain.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/memory.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/resolve.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/selector.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/timecost.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/profiling/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/protocol/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/protocol/plan_v1alpha1.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/providers/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/py.typed +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recipes/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/policy.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/signals.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/taxonomy.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/sdk.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/app.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/auth.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/checkpoints.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/cli.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/dashboard.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/ledger.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/modea.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/strategies/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/strategies/command.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/torch/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/benchmarks.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/concepts/architecture.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/get-started.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/federated-averaging.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/huggingface.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/pytorch.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/sklearn.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/index.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/cli.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/integrations.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/sdk.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/torch-helper.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/search-index.json +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/tutorials/convnet.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/tutorials/fault-tolerance.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/flowmap.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/page.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/server.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/state.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/workloads/__init__.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/workloads/command.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/dependency_links.txt +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/entry_points.txt +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/requires.txt +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/top_level.txt +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/setup.cfg +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_auto_recovery.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_benchmarks.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_checkpoint.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_checkpoint_local.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_cli_submit.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_documentation.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_examples_e2e.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_convergence.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_driver.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_weights.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_worker.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_gpu_e2e.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_integrations.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_interfaces.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_kmeans_leases.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_kmeans_workload.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_kuberay_backend.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_launcher_local.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_leases.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_leases_scope.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_leases_sqlite.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_monitor_sampler.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_placement_local_data.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_planner.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_protocol_local_datasets.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_protocol_v1alpha1.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_recipe_local_inputs.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_recipes_command.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_recovery.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_run_json.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_scheduler_isolation.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_sdk_submit.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_auth.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_auth_startup.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_checkpoints.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_command_recipe.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_delegation.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_fedavg.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_modea.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_write_scope.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_sgd_trainer.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_strategy_command.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_torch_helper.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_flowmap.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_page.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_server.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_state.py +0 -0
- {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_workloads_command.py +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashruntime
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
|
|
5
5
|
Author: Zolli Labs
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -78,6 +78,13 @@ class ResourcesSpec(BaseModel):
|
|
|
78
78
|
maximumWorkers: int = Field(ge=1, default=3)
|
|
79
79
|
cpuPerTask: float = Field(gt=0, default=1)
|
|
80
80
|
memoryPerTask: str = "512Mi"
|
|
81
|
+
#: GPUs each task needs. 0 — the default, and every job that exists
|
|
82
|
+
#: today — means "no GPU required" and places anywhere. A non-zero value
|
|
83
|
+
#: is a hardware *requirement*, enforced fail-closed at placement
|
|
84
|
+
#: (`scheduler.IsolationAwarePlacement`): a CUDA task on a CPU-only host
|
|
85
|
+
#: does not politely requeue, it crashes or silently runs on the CPU two
|
|
86
|
+
#: orders of magnitude slower while reporting success.
|
|
87
|
+
gpuPerTask: int = Field(ge=0, default=0)
|
|
81
88
|
|
|
82
89
|
@field_validator("maximumWorkers")
|
|
83
90
|
@classmethod
|
|
@@ -269,10 +276,39 @@ class NodeEnvironment(str, Enum):
|
|
|
269
276
|
EDGE = "edge"
|
|
270
277
|
|
|
271
278
|
|
|
279
|
+
class GpuInfo(BaseModel):
|
|
280
|
+
"""One GPU as the host's driver reports it.
|
|
281
|
+
|
|
282
|
+
Every field but `index` is optional: a probe that cannot read a value says
|
|
283
|
+
nothing rather than guessing, so a partial reading is still reportable and
|
|
284
|
+
an unreadable one is still `[]` rather than a fabricated device.
|
|
285
|
+
|
|
286
|
+
Typed now although placement matches only on COUNT, because the wire
|
|
287
|
+
format is the expensive thing to change later — these agents run on
|
|
288
|
+
machines we cannot reach, and a field added to an existing model is cheap
|
|
289
|
+
where turning a `dict` into a model is not. `memory_total_mb` and
|
|
290
|
+
`compute_capability` are collected from the start so the data is already
|
|
291
|
+
flowing when matching rules arrive (spec §10.1).
|
|
292
|
+
|
|
293
|
+
No `schema_version`: like `CheckpointPart`, this is a nested element of a
|
|
294
|
+
versioned message (`NodeRegistration`), not a wire message of its own.
|
|
295
|
+
"""
|
|
296
|
+
|
|
297
|
+
index: int
|
|
298
|
+
name: str = ""
|
|
299
|
+
memory_total_mb: int | None = None
|
|
300
|
+
driver_version: str = ""
|
|
301
|
+
compute_capability: str = ""
|
|
302
|
+
|
|
303
|
+
|
|
272
304
|
class NodeCapabilities(BaseModel):
|
|
273
305
|
cpu_cores: float | None = None
|
|
274
306
|
memory_bytes: int | None = None
|
|
275
|
-
|
|
307
|
+
#: Devices this host's driver reports, one entry per GPU. Empty is the
|
|
308
|
+
#: honest answer for a host with no driver, no `nvidia-smi`, or output
|
|
309
|
+
#: this runtime cannot parse — never a guess. Placement reads the LENGTH
|
|
310
|
+
#: of this list and nothing else in v1.
|
|
311
|
+
gpus: list[GpuInfo] = Field(default_factory=list)
|
|
276
312
|
os: str = ""
|
|
277
313
|
architecture: str = ""
|
|
278
314
|
|
|
@@ -160,6 +160,26 @@ class CommandRecipe(WorkloadRecipe):
|
|
|
160
160
|
#
|
|
161
161
|
# `list(...)` so a payload never aliases the caller's spec.
|
|
162
162
|
payload["local_inputs"] = list(p["local_inputs"])
|
|
163
|
+
gpus = spec.spec.resources.gpuPerTask
|
|
164
|
+
if gpus:
|
|
165
|
+
# The GPU requirement is the ONLY payload key sourced from
|
|
166
|
+
# `spec.spec.resources` rather than `workload.parameters` —
|
|
167
|
+
# a branch of the spec nothing else in this method reads.
|
|
168
|
+
# That makes it the easiest forward in the file to leave out
|
|
169
|
+
# and the hardest to notice missing: there is no unrecognised
|
|
170
|
+
# parameter sitting in `p` to go looking for.
|
|
171
|
+
#
|
|
172
|
+
# Dropping it does NOT fail closed. `IsolationAwarePlacement`
|
|
173
|
+
# reads `task.payload["gpus"]`; with the key absent it sees a
|
|
174
|
+
# task requiring nothing, places it on any node, and a CUDA
|
|
175
|
+
# job lands on a CPU-only volunteer. Both ends of this hop
|
|
176
|
+
# have tests that pass while it is broken, because each
|
|
177
|
+
# constructs the payload directly.
|
|
178
|
+
#
|
|
179
|
+
# Absent stays absent, never 0 — the no-GPU path must keep
|
|
180
|
+
# exercising the key-missing branch, as unpack_inputs and
|
|
181
|
+
# local_inputs do.
|
|
182
|
+
payload["gpus"] = int(gpus)
|
|
163
183
|
tasks.append(
|
|
164
184
|
TaskSpec(
|
|
165
185
|
task_id=task_id,
|
|
@@ -172,6 +172,43 @@ class IsolationAwarePlacement(PlacementPolicy):
|
|
|
172
172
|
is not in its map) — failing closed here makes it fail before anything
|
|
173
173
|
touches the data, and before an attempt is burned.
|
|
174
174
|
|
|
175
|
+
A fifth gate applies to tasks whose payload asks for `gpus: N`: the
|
|
176
|
+
claiming node's `capabilities.gpus` must be a list of at least N entries.
|
|
177
|
+
It takes the argv/local-data polarity — **fail closed** — and deliberately
|
|
178
|
+
NOT the module gate's fail-open one, even though "the node cannot run
|
|
179
|
+
this" sounds like the same availability concern:
|
|
180
|
+
|
|
181
|
+
- A misplaced module task wastes retry attempts and nothing else. A CUDA
|
|
182
|
+
job on a CPU-only box does not politely fail and requeue. It either
|
|
183
|
+
crashes on `torch.cuda.is_available()` or, worse, silently falls back
|
|
184
|
+
to the CPU and runs two orders of magnitude slower while reporting
|
|
185
|
+
success. The second outcome is not a failure anything here can detect;
|
|
186
|
+
it is a bill.
|
|
187
|
+
- The capability must be a genuine *list*, because its LENGTH is the
|
|
188
|
+
whole matching rule in v1. Absent, `None`, a bare string, a dict, or a
|
|
189
|
+
bare `int` count as NO GPUs. The bare `int` deserves naming: `1` reads
|
|
190
|
+
like "one GPU" and is exactly what a hand-written node view would put
|
|
191
|
+
there, but accepting it would mean a second, looser matching rule
|
|
192
|
+
beside the one `NodeCapabilities.gpus` actually feeds.
|
|
193
|
+
- `capabilities` itself may be absent or type-confused; that is read as
|
|
194
|
+
no GPUs rather than allowed to raise. `(node.get("capabilities") or
|
|
195
|
+
{}).get(...)` is NOT sufficient for this — a string capabilities value
|
|
196
|
+
has no `.get` and would crash the predicate.
|
|
197
|
+
- The requirement itself must be a non-negative `int`. Anything else
|
|
198
|
+
(`"1"`, `-1`, `1.5`, a list) makes the task ineligible everywhere
|
|
199
|
+
rather than crashing, exactly as a non-list `local_inputs` does.
|
|
200
|
+
**`bool` is a subclass of `int`**, so `True` is excluded explicitly: a
|
|
201
|
+
`gpus: true` typo must not silently mean "one GPU" and place real work.
|
|
202
|
+
- `gpus: 0` requires nothing and runs anywhere, exactly like tier
|
|
203
|
+
`standard` and an empty `local_inputs`.
|
|
204
|
+
|
|
205
|
+
The gate is ONE-DIRECTIONAL. A node with GPUs still receives CPU work;
|
|
206
|
+
reserving GPU hosts for GPU jobs is a scheduling optimisation and a
|
|
207
|
+
separate decision, and making it a gate here would idle the scarcest
|
|
208
|
+
hardware on the network. `allowFallback` does not waive this gate either
|
|
209
|
+
— hardware either exists on a host or it does not, and the submitter's
|
|
210
|
+
isolation posture has nothing to say about it.
|
|
211
|
+
|
|
175
212
|
Everything genuinely standard keeps the fail-open placement default."""
|
|
176
213
|
|
|
177
214
|
def eligible(self, task: TaskSpec, node: NodeView) -> bool:
|
|
@@ -199,6 +236,28 @@ class IsolationAwarePlacement(PlacementPolicy):
|
|
|
199
236
|
return False # absent/None/type-confused capability ⇒ not capable
|
|
200
237
|
if any(name not in advertised for name in local_inputs):
|
|
201
238
|
return False
|
|
239
|
+
# Fail-closed like the argv and local-data gates, and checked before
|
|
240
|
+
# the allowFallback waiver below for the same reason: a submitter
|
|
241
|
+
# cannot waive their way onto hardware a host does not have.
|
|
242
|
+
required_gpus = task.payload.get("gpus")
|
|
243
|
+
if required_gpus is not None:
|
|
244
|
+
# `bool` is a subclass of `int`: without the explicit exclusion,
|
|
245
|
+
# a `gpus: true` typo would read as "1 GPU" and place real work.
|
|
246
|
+
if (
|
|
247
|
+
not isinstance(required_gpus, int)
|
|
248
|
+
or isinstance(required_gpus, bool)
|
|
249
|
+
or required_gpus < 0
|
|
250
|
+
):
|
|
251
|
+
return False # type-confused requirement ⇒ fail closed, no crash
|
|
252
|
+
if required_gpus > 0:
|
|
253
|
+
capabilities = node.get("capabilities")
|
|
254
|
+
# isinstance, not `or {}` — a string capabilities value has no
|
|
255
|
+
# `.get` and must fail closed rather than crash the predicate.
|
|
256
|
+
advertised = (
|
|
257
|
+
capabilities.get("gpus") if isinstance(capabilities, dict) else None
|
|
258
|
+
)
|
|
259
|
+
if not isinstance(advertised, list) or len(advertised) < required_gpus:
|
|
260
|
+
return False # absent/short/type-confused ⇒ not capable
|
|
202
261
|
isolation = task.payload.get("isolation")
|
|
203
262
|
if isolation is None:
|
|
204
263
|
return True # no isolation payload ⇒ standard, runs anywhere
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.4
|
|
2
2
|
Name: flashruntime
|
|
3
|
-
Version: 0.4.
|
|
3
|
+
Version: 0.4.1
|
|
4
4
|
Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
|
|
5
5
|
Author: Zolli Labs
|
|
6
6
|
License-Expression: Apache-2.0
|
|
@@ -118,10 +118,13 @@ tests/test_leases.py
|
|
|
118
118
|
tests/test_leases_scope.py
|
|
119
119
|
tests/test_leases_sqlite.py
|
|
120
120
|
tests/test_monitor_sampler.py
|
|
121
|
+
tests/test_placement_gpu.py
|
|
121
122
|
tests/test_placement_local_data.py
|
|
122
123
|
tests/test_planner.py
|
|
124
|
+
tests/test_protocol_gpu.py
|
|
123
125
|
tests/test_protocol_local_datasets.py
|
|
124
126
|
tests/test_protocol_v1alpha1.py
|
|
127
|
+
tests/test_recipe_gpu.py
|
|
125
128
|
tests/test_recipe_local_inputs.py
|
|
126
129
|
tests/test_recipes_command.py
|
|
127
130
|
tests/test_recovery.py
|
|
@@ -5,12 +5,23 @@ build-backend = "setuptools.build_meta"
|
|
|
5
5
|
|
|
6
6
|
[project]
|
|
7
7
|
name = "flashruntime"
|
|
8
|
-
# 0.4.
|
|
9
|
-
#
|
|
10
|
-
#
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
|
|
8
|
+
# 0.4.1 adds GpuInfo and ResourcesSpec.gpuPerTask. Both are additive wire
|
|
9
|
+
# fields, so the compatibility range does not move — but the PATCH bump is
|
|
10
|
+
# load-bearing, not bookkeeping. The `flashruntime-v0.4.0` tag was cut BEFORE
|
|
11
|
+
# the protocol commit that added them, and this tree went on calling itself
|
|
12
|
+
# 0.4.0 afterwards: one version string named two different protocols, which is
|
|
13
|
+
# exactly the drift the monorepo consolidation removed. Bump on any
|
|
14
|
+
# wire-visible change, even an additive one.
|
|
15
|
+
#
|
|
16
|
+
# flashnode’s floor moves to >=0.4.1,<0.5 in the same change — it imports
|
|
17
|
+
# GpuInfo at module scope, so for the agent this floor is an import-time hard
|
|
18
|
+
# requirement, not a silent-degradation one.
|
|
19
|
+
#
|
|
20
|
+
# 0.4.0 added NodeRegistration.local_datasets — a new wire field, so consumers
|
|
21
|
+
# must pin a range that includes it. (0.3.0 was the first published release;
|
|
22
|
+
# 0.1.0 and 0.2.0 were declared here but never reached PyPI.) Released from
|
|
23
|
+
# the Zolli-Labs/flashml monorepo by pushing a `flashruntime-vX.Y.Z` tag.
|
|
24
|
+
version = "0.4.1"
|
|
14
25
|
description = "Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute."
|
|
15
26
|
readme = "README.md"
|
|
16
27
|
# PEP 639 SPDX expression (string form) — replaces the deprecated
|
|
@@ -0,0 +1,301 @@
|
|
|
1
|
+
"""The fifth fail-closed placement gate: a task whose payload asks for `gpus: N`
|
|
2
|
+
may only be leased to a node whose `capabilities.gpus` is a list of at least N
|
|
3
|
+
entries (AGENTS.md rule 3).
|
|
4
|
+
|
|
5
|
+
Polarity follows `argv_capable` / `local_datasets`, not `module_capable`. A
|
|
6
|
+
misplaced module task only wastes retry attempts; a CUDA job on a CPU-only box
|
|
7
|
+
either crashes on `torch.cuda.is_available()` or silently falls back to the CPU
|
|
8
|
+
and runs two orders of magnitude slower while reporting success. Neither is
|
|
9
|
+
something to discover from a bill.
|
|
10
|
+
|
|
11
|
+
The gate is one-directional: a GPU node still receives CPU work.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import pytest
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
def _gpu_task(gpus=1, **payload_extra):
|
|
20
|
+
"""A task requiring GPUs. `gpus` is passed through RAW so tests can poison
|
|
21
|
+
it with type-confused values the protocol would never emit (the same trick
|
|
22
|
+
`_local_task` plays on `local_inputs`)."""
|
|
23
|
+
from flashruntime.protocol.v1alpha1 import TaskSpec
|
|
24
|
+
|
|
25
|
+
payload = {"module": "flashml_workloads.sklearn_trial"}
|
|
26
|
+
if gpus is not None:
|
|
27
|
+
payload["gpus"] = gpus
|
|
28
|
+
payload.update(payload_extra)
|
|
29
|
+
return TaskSpec(
|
|
30
|
+
task_id="task-000", job_id="job-a", commit_key="job-a/task-000/m.json",
|
|
31
|
+
payload=payload,
|
|
32
|
+
)
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def _node(gpu_count=0, **extra):
|
|
36
|
+
"""A node view shaped like the claim endpoint's: the GPU list lives under
|
|
37
|
+
`capabilities`, which is `NodeCapabilities.model_dump()` — plain dicts by
|
|
38
|
+
the time placement sees them, not GpuInfo instances."""
|
|
39
|
+
node = {
|
|
40
|
+
"node_id": "n1",
|
|
41
|
+
"capabilities": {
|
|
42
|
+
"cpu_cores": 8,
|
|
43
|
+
"gpus": [{"index": i, "name": "NVIDIA A10G"} for i in range(gpu_count)],
|
|
44
|
+
},
|
|
45
|
+
}
|
|
46
|
+
node.update(extra)
|
|
47
|
+
return node
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
# ---------------------------------------------------------------------------
|
|
51
|
+
# DoD 3–6: the four placement outcomes
|
|
52
|
+
# ---------------------------------------------------------------------------
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_one_gpu_job_is_refused_on_a_node_advertising_none():
|
|
56
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
57
|
+
|
|
58
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), _node(0)) is False
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
def test_one_gpu_job_is_placed_on_a_node_advertising_one():
|
|
62
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
63
|
+
|
|
64
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), _node(1)) is True
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def test_two_gpu_job_is_refused_on_a_one_gpu_node():
|
|
68
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
69
|
+
|
|
70
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(2), _node(1)) is False
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_two_gpu_job_is_placed_on_a_two_gpu_node():
|
|
74
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
75
|
+
|
|
76
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(2), _node(2)) is True
|
|
77
|
+
|
|
78
|
+
|
|
79
|
+
def test_a_job_with_no_gpu_requirement_is_placed_on_both():
|
|
80
|
+
"""The key must be ABSENT, not 0 — this is the branch every job that
|
|
81
|
+
exists today takes."""
|
|
82
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
83
|
+
|
|
84
|
+
policy = IsolationAwarePlacement()
|
|
85
|
+
task = _gpu_task(None) # no "gpus" key in the payload at all
|
|
86
|
+
assert policy.eligible(task, _node(0)) is True
|
|
87
|
+
assert policy.eligible(task, _node(2)) is True
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
def test_gpus_zero_requires_nothing_and_runs_anywhere():
|
|
91
|
+
"""`gpus: 0` is what an explicit opt-out compiles to. It demands nothing,
|
|
92
|
+
like tier "standard" and an empty `local_inputs`."""
|
|
93
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
94
|
+
|
|
95
|
+
policy = IsolationAwarePlacement()
|
|
96
|
+
assert policy.eligible(_gpu_task(0), _node(0)) is True
|
|
97
|
+
assert policy.eligible(_gpu_task(0), {"node_id": "n1"}) is True
|
|
98
|
+
|
|
99
|
+
|
|
100
|
+
def test_a_gpu_node_still_receives_cpu_work():
|
|
101
|
+
"""The gate is one-directional. Reserving GPU hosts for GPU jobs is a
|
|
102
|
+
separate scheduling decision; making it a gate here would idle the
|
|
103
|
+
scarcest hardware on the network."""
|
|
104
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
105
|
+
|
|
106
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(None), _node(4)) is True
|
|
107
|
+
|
|
108
|
+
|
|
109
|
+
# ---------------------------------------------------------------------------
|
|
110
|
+
# Fail-closed: the advertised capability
|
|
111
|
+
# ---------------------------------------------------------------------------
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
@pytest.mark.parametrize(
|
|
115
|
+
"advertised",
|
|
116
|
+
[
|
|
117
|
+
pytest.param([], id="empty-list"),
|
|
118
|
+
pytest.param(None, id="explicit-none"),
|
|
119
|
+
pytest.param("gpu", id="bare-string-type-confusion"),
|
|
120
|
+
pytest.param({}, id="empty-dict-type-confusion"),
|
|
121
|
+
pytest.param({"0": "NVIDIA A10G"}, id="dict-type-confusion"),
|
|
122
|
+
pytest.param(({"index": 0},), id="tuple-type-confusion"),
|
|
123
|
+
pytest.param(True, id="truthy-boolean"),
|
|
124
|
+
pytest.param(1, id="bare-count-int"),
|
|
125
|
+
],
|
|
126
|
+
)
|
|
127
|
+
def test_ineligible_unless_the_capability_is_a_list_long_enough(advertised):
|
|
128
|
+
"""Fail closed on absence AND on type confusion. A bare `1` is the most
|
|
129
|
+
tempting wrong answer — it reads like "one GPU" and has no length, so
|
|
130
|
+
accepting it would mean writing a second, looser matching rule beside the
|
|
131
|
+
one the probe actually feeds."""
|
|
132
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
133
|
+
|
|
134
|
+
node = {"node_id": "n1", "capabilities": {"gpus": advertised}}
|
|
135
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def test_node_with_no_capabilities_key_at_all_is_ineligible():
|
|
139
|
+
"""An already-deployed agent whose registration predates GPU probing
|
|
140
|
+
advertises nothing — absent must read as "has none", never "unknown, try
|
|
141
|
+
it and see"."""
|
|
142
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
143
|
+
|
|
144
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), {"node_id": "n1"}) is False
|
|
145
|
+
|
|
146
|
+
|
|
147
|
+
def test_node_with_capabilities_none_is_ineligible():
|
|
148
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
149
|
+
|
|
150
|
+
node = {"node_id": "n1", "capabilities": None}
|
|
151
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
|
|
152
|
+
|
|
153
|
+
|
|
154
|
+
def test_node_with_type_confused_capabilities_is_ineligible_without_crashing():
|
|
155
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
156
|
+
|
|
157
|
+
node = {"node_id": "n1", "capabilities": "8 cores, 1 gpu"}
|
|
158
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
|
|
159
|
+
|
|
160
|
+
|
|
161
|
+
def test_node_capabilities_without_a_gpus_key_is_ineligible():
|
|
162
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
163
|
+
|
|
164
|
+
node = {"node_id": "n1", "capabilities": {"cpu_cores": 8}}
|
|
165
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
|
|
166
|
+
|
|
167
|
+
|
|
168
|
+
# ---------------------------------------------------------------------------
|
|
169
|
+
# Fail-closed: the requirement itself
|
|
170
|
+
# ---------------------------------------------------------------------------
|
|
171
|
+
|
|
172
|
+
|
|
173
|
+
@pytest.mark.parametrize(
|
|
174
|
+
"required",
|
|
175
|
+
[
|
|
176
|
+
pytest.param("1", id="numeric-string"),
|
|
177
|
+
pytest.param(-1, id="negative"),
|
|
178
|
+
pytest.param(1.5, id="fractional"),
|
|
179
|
+
pytest.param(1.0, id="whole-float"),
|
|
180
|
+
pytest.param([1], id="list"),
|
|
181
|
+
pytest.param({"count": 1}, id="dict"),
|
|
182
|
+
pytest.param(True, id="boolean-true"),
|
|
183
|
+
],
|
|
184
|
+
)
|
|
185
|
+
def test_type_confused_requirement_is_ineligible_everywhere(required):
|
|
186
|
+
"""A poisoned requirement must fail closed on EVERY node — including one
|
|
187
|
+
with plenty of GPUs — and must never crash the predicate.
|
|
188
|
+
|
|
189
|
+
`True` is the one that needs saying out loud: `bool` is a subclass of
|
|
190
|
+
`int` in Python, so `isinstance(True, int)` is True and `True >= 1`.
|
|
191
|
+
Without the explicit bool exclusion, `gpus: true` would silently mean
|
|
192
|
+
"one GPU" — a JSON typo that placed real work.
|
|
193
|
+
"""
|
|
194
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
195
|
+
|
|
196
|
+
policy = IsolationAwarePlacement()
|
|
197
|
+
assert policy.eligible(_gpu_task(required), _node(0)) is False
|
|
198
|
+
assert policy.eligible(_gpu_task(required), _node(8)) is False
|
|
199
|
+
|
|
200
|
+
|
|
201
|
+
def test_boolean_false_is_not_read_as_zero_gpus():
|
|
202
|
+
"""The mirror of the above: `False` is not "no GPU required" either. It is
|
|
203
|
+
a type-confused requirement and fails closed like every other."""
|
|
204
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
205
|
+
|
|
206
|
+
assert IsolationAwarePlacement().eligible(_gpu_task(False), _node(8)) is False
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# ---------------------------------------------------------------------------
|
|
210
|
+
# Interaction with the existing gates
|
|
211
|
+
# ---------------------------------------------------------------------------
|
|
212
|
+
|
|
213
|
+
|
|
214
|
+
def test_allow_fallback_cannot_bypass_the_gpu_gate():
|
|
215
|
+
"""allowFallback waives the sandbox-tier requirement only. It is the
|
|
216
|
+
submitter's statement about their own isolation posture, and has nothing
|
|
217
|
+
to say about hardware that either exists or does not."""
|
|
218
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
219
|
+
|
|
220
|
+
task = _gpu_task(1, isolation={"tier": "sandboxed", "allowFallback": True})
|
|
221
|
+
node = _node(0, sandbox_capable=True)
|
|
222
|
+
assert IsolationAwarePlacement().eligible(task, node) is False
|
|
223
|
+
|
|
224
|
+
|
|
225
|
+
def test_a_gpu_node_still_fails_the_argv_gate_it_does_not_pass():
|
|
226
|
+
"""The GPU gate adds a requirement; it never satisfies another one."""
|
|
227
|
+
from flashruntime.protocol.v1alpha1 import TaskSpec
|
|
228
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
229
|
+
|
|
230
|
+
task = TaskSpec(
|
|
231
|
+
task_id="task-000", job_id="job-a", commit_key="job-a/task-000/m.json",
|
|
232
|
+
payload={"argv": ["python", "train.py"], "gpus": 1},
|
|
233
|
+
)
|
|
234
|
+
assert IsolationAwarePlacement().eligible(task, _node(1)) is False
|
|
235
|
+
assert IsolationAwarePlacement().eligible(task, _node(1, argv_capable=True)) is True
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
def test_claim_over_a_gpu_queue_serves_the_clean_task():
|
|
239
|
+
"""A GPU task at the head of the queue must not block a CPU-only node —
|
|
240
|
+
the next clean task is still leased, and the GPU task is never
|
|
241
|
+
mis-leased."""
|
|
242
|
+
from flashruntime.leases import LeaseManager
|
|
243
|
+
from flashruntime.protocol.v1alpha1 import TaskSpec
|
|
244
|
+
from flashruntime.scheduler import IsolationAwarePlacement
|
|
245
|
+
|
|
246
|
+
mgr = LeaseManager()
|
|
247
|
+
mgr.add_task(_gpu_task(1)) # head of queue, needs a GPU
|
|
248
|
+
mgr.add_task(
|
|
249
|
+
TaskSpec(task_id="task-001", job_id="job-a", commit_key="job-a/task-001/m.json")
|
|
250
|
+
)
|
|
251
|
+
policy = IsolationAwarePlacement()
|
|
252
|
+
|
|
253
|
+
lease = mgr.claim("n1", policy=policy, node=_node(0))
|
|
254
|
+
assert lease is not None
|
|
255
|
+
assert lease.task_id == "task-001"
|
|
256
|
+
|
|
257
|
+
lease2 = mgr.claim("n2", policy=policy, node=_node(1, node_id="n2"))
|
|
258
|
+
assert lease2 is not None
|
|
259
|
+
assert lease2.task_id == "task-000"
|
|
260
|
+
|
|
261
|
+
|
|
262
|
+
def test_claim_endpoint_leases_gpu_work_only_to_a_gpu_host():
|
|
263
|
+
"""End to end over the claim endpoint: the registry's claim-time node view
|
|
264
|
+
must forward `capabilities.gpus`, or the gate reads an absent capability on
|
|
265
|
+
EVERY node and a GPU task becomes unplaceable everywhere.
|
|
266
|
+
|
|
267
|
+
This pins hop 1 of the three that broke the `local_datasets` work — the
|
|
268
|
+
node view. It is believed correct today (`capabilities` is forwarded whole
|
|
269
|
+
as `model_dump()`), and this test is what keeps it that way.
|
|
270
|
+
"""
|
|
271
|
+
import pathlib
|
|
272
|
+
|
|
273
|
+
import fastapi
|
|
274
|
+
from fastapi.testclient import TestClient
|
|
275
|
+
|
|
276
|
+
from flashruntime.leases import LeaseManager
|
|
277
|
+
from flashruntime.service.modea import ModeAState, build_router
|
|
278
|
+
|
|
279
|
+
state = ModeAState(LeaseManager(), artifacts_dir=pathlib.Path("/tmp"))
|
|
280
|
+
app = fastapi.FastAPI()
|
|
281
|
+
app.include_router(build_router(state))
|
|
282
|
+
client = TestClient(app)
|
|
283
|
+
state.manager.add_task(_gpu_task(1))
|
|
284
|
+
|
|
285
|
+
def register(node_id: str, gpus: list[dict]):
|
|
286
|
+
r = client.post(
|
|
287
|
+
"/v1alpha1/nodes/register",
|
|
288
|
+
json={
|
|
289
|
+
"node_id": node_id, "kubernetes_node": "", "hostname": node_id,
|
|
290
|
+
"capabilities": {"cpu_cores": 8, "gpus": gpus},
|
|
291
|
+
},
|
|
292
|
+
)
|
|
293
|
+
assert r.status_code == 200
|
|
294
|
+
|
|
295
|
+
register("cpu-node", [])
|
|
296
|
+
register("gpu-node", [{"index": 0, "name": "NVIDIA A10G", "memory_total_mb": 22731}])
|
|
297
|
+
|
|
298
|
+
assert client.post("/v1alpha1/leases/claim", json={"node_id": "cpu-node"}).status_code == 204
|
|
299
|
+
r = client.post("/v1alpha1/leases/claim", json={"node_id": "gpu-node"})
|
|
300
|
+
assert r.status_code == 200
|
|
301
|
+
assert r.json()["task_id"] == "task-000"
|
|
@@ -0,0 +1,152 @@
|
|
|
1
|
+
"""`GpuInfo` and `ResourcesSpec.gpuPerTask` — the two wire additions GPU
|
|
2
|
+
placement needs.
|
|
3
|
+
|
|
4
|
+
These are protocol tests, not scheduler tests: they pin the *wire* behaviour
|
|
5
|
+
the fifth placement gate is later allowed to trust, exactly as
|
|
6
|
+
`test_protocol_local_datasets.py` does for `local_datasets`.
|
|
7
|
+
|
|
8
|
+
The typing change matters more than it looks. `NodeCapabilities.gpus` was
|
|
9
|
+
`list[dict[str, Any]]`; it becomes `list[GpuInfo]`. Agents already deployed on
|
|
10
|
+
machines we cannot reach send plain JSON objects, so a raw dict must still
|
|
11
|
+
coerce — a model that only accepted `GpuInfo` instances would be a breaking
|
|
12
|
+
change dressed up as a type annotation.
|
|
13
|
+
"""
|
|
14
|
+
|
|
15
|
+
from __future__ import annotations
|
|
16
|
+
|
|
17
|
+
import pytest
|
|
18
|
+
from pydantic import ValidationError
|
|
19
|
+
|
|
20
|
+
from flashruntime.protocol.v1alpha1 import GpuInfo, NodeCapabilities, ResourcesSpec
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
# ---------------------------------------------------------------------------
|
|
24
|
+
# GpuInfo
|
|
25
|
+
# ---------------------------------------------------------------------------
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def test_gpu_info_round_trips_with_only_index_set():
|
|
29
|
+
"""Every field but `index` is optional: a probe that cannot read a value
|
|
30
|
+
says nothing rather than guessing, and the omission must survive JSON."""
|
|
31
|
+
gpu = GpuInfo(index=0)
|
|
32
|
+
restored = GpuInfo.model_validate_json(gpu.model_dump_json())
|
|
33
|
+
|
|
34
|
+
assert restored.index == 0
|
|
35
|
+
assert restored.name == ""
|
|
36
|
+
assert restored.memory_total_mb is None
|
|
37
|
+
assert restored.driver_version == ""
|
|
38
|
+
assert restored.compute_capability == ""
|
|
39
|
+
|
|
40
|
+
|
|
41
|
+
def test_gpu_info_round_trips_a_fully_populated_device():
|
|
42
|
+
gpu = GpuInfo(
|
|
43
|
+
index=1,
|
|
44
|
+
name="NVIDIA GeForce RTX 4090",
|
|
45
|
+
memory_total_mb=24564,
|
|
46
|
+
driver_version="550.54.14",
|
|
47
|
+
compute_capability="8.9",
|
|
48
|
+
)
|
|
49
|
+
restored = GpuInfo.model_validate_json(gpu.model_dump_json())
|
|
50
|
+
|
|
51
|
+
assert restored == gpu
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def test_gpu_info_requires_an_index():
|
|
55
|
+
"""The one field a probe can always report. Without it there is no way to
|
|
56
|
+
tell two devices apart, so it is not defaulted."""
|
|
57
|
+
with pytest.raises(ValidationError):
|
|
58
|
+
GpuInfo.model_validate({"name": "NVIDIA GeForce RTX 4090"})
|
|
59
|
+
|
|
60
|
+
|
|
61
|
+
# ---------------------------------------------------------------------------
|
|
62
|
+
# NodeCapabilities.gpus
|
|
63
|
+
# ---------------------------------------------------------------------------
|
|
64
|
+
|
|
65
|
+
|
|
66
|
+
def test_node_capabilities_accepts_gpu_info_instances():
|
|
67
|
+
caps = NodeCapabilities(gpus=[GpuInfo(index=0)])
|
|
68
|
+
|
|
69
|
+
assert len(caps.gpus) == 1
|
|
70
|
+
assert caps.gpus[0].index == 0
|
|
71
|
+
|
|
72
|
+
|
|
73
|
+
def test_node_capabilities_coerces_raw_dicts_from_the_wire():
|
|
74
|
+
"""Deployed agents send JSON objects, not `GpuInfo` instances. The typing
|
|
75
|
+
change must be additive on the wire or every already-enrolled machine
|
|
76
|
+
fails to register the moment the coordinator upgrades."""
|
|
77
|
+
caps = NodeCapabilities.model_validate(
|
|
78
|
+
{
|
|
79
|
+
"cpu_cores": 8,
|
|
80
|
+
"gpus": [
|
|
81
|
+
{"index": 0, "name": "NVIDIA A10G", "memory_total_mb": 22731},
|
|
82
|
+
{"index": 1},
|
|
83
|
+
],
|
|
84
|
+
}
|
|
85
|
+
)
|
|
86
|
+
|
|
87
|
+
assert [g.index for g in caps.gpus] == [0, 1]
|
|
88
|
+
assert isinstance(caps.gpus[0], GpuInfo)
|
|
89
|
+
assert caps.gpus[0].name == "NVIDIA A10G"
|
|
90
|
+
assert caps.gpus[1].name == ""
|
|
91
|
+
|
|
92
|
+
|
|
93
|
+
def test_node_capabilities_ignores_fields_a_newer_probe_might_add():
|
|
94
|
+
"""Forward compatibility in the other direction: a newer agent reporting a
|
|
95
|
+
field this coordinator does not know must not fail registration."""
|
|
96
|
+
caps = NodeCapabilities.model_validate(
|
|
97
|
+
{"gpus": [{"index": 0, "some_future_field": "ignored"}]}
|
|
98
|
+
)
|
|
99
|
+
|
|
100
|
+
assert caps.gpus[0].index == 0
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def test_node_capabilities_defaults_to_no_gpus():
|
|
104
|
+
"""Absent means "no GPU", never "unknown, assume yes" — the gate reads
|
|
105
|
+
this list and its length is the whole matching rule in v1."""
|
|
106
|
+
assert NodeCapabilities().gpus == []
|
|
107
|
+
assert NodeCapabilities.model_validate({"cpu_cores": 4}).gpus == []
|
|
108
|
+
|
|
109
|
+
|
|
110
|
+
def test_node_capabilities_gpu_default_is_not_shared_between_instances():
|
|
111
|
+
# `default_factory` and not a bare `[]`: one shared list would let a
|
|
112
|
+
# single node's devices leak into every other node's capabilities.
|
|
113
|
+
first = NodeCapabilities()
|
|
114
|
+
second = NodeCapabilities()
|
|
115
|
+
|
|
116
|
+
assert first.gpus is not second.gpus
|
|
117
|
+
first.gpus.append(GpuInfo(index=0))
|
|
118
|
+
assert second.gpus == []
|
|
119
|
+
|
|
120
|
+
|
|
121
|
+
# ---------------------------------------------------------------------------
|
|
122
|
+
# ResourcesSpec.gpuPerTask
|
|
123
|
+
# ---------------------------------------------------------------------------
|
|
124
|
+
|
|
125
|
+
|
|
126
|
+
def test_gpu_per_task_defaults_to_zero():
|
|
127
|
+
"""0 means "no GPU required", which is every job that exists today."""
|
|
128
|
+
assert ResourcesSpec().gpuPerTask == 0
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def test_gpu_per_task_accepts_a_positive_count():
|
|
132
|
+
assert ResourcesSpec(gpuPerTask=2).gpuPerTask == 2
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
@pytest.mark.parametrize(
|
|
136
|
+
"value",
|
|
137
|
+
[
|
|
138
|
+
pytest.param(-1, id="negative"),
|
|
139
|
+
pytest.param(1.5, id="fractional"),
|
|
140
|
+
pytest.param("one", id="not-a-number"),
|
|
141
|
+
],
|
|
142
|
+
)
|
|
143
|
+
def test_gpu_per_task_rejects_values_that_are_not_a_device_count(value):
|
|
144
|
+
with pytest.raises(ValidationError):
|
|
145
|
+
ResourcesSpec(gpuPerTask=value)
|
|
146
|
+
|
|
147
|
+
|
|
148
|
+
def test_gpu_per_task_round_trips_through_json():
|
|
149
|
+
spec = ResourcesSpec(gpuPerTask=1)
|
|
150
|
+
restored = ResourcesSpec.model_validate_json(spec.model_dump_json())
|
|
151
|
+
|
|
152
|
+
assert restored.gpuPerTask == 1
|