flashruntime 0.4.0__tar.gz → 0.4.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (151) hide show
  1. {flashruntime-0.4.0/flashruntime.egg-info → flashruntime-0.4.1}/PKG-INFO +1 -1
  2. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/protocol/v1alpha1.py +37 -1
  3. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recipes/command.py +20 -0
  4. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/scheduler/__init__.py +59 -0
  5. {flashruntime-0.4.0 → flashruntime-0.4.1/flashruntime.egg-info}/PKG-INFO +1 -1
  6. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/SOURCES.txt +3 -0
  7. {flashruntime-0.4.0 → flashruntime-0.4.1}/pyproject.toml +17 -6
  8. flashruntime-0.4.1/tests/test_placement_gpu.py +301 -0
  9. flashruntime-0.4.1/tests/test_protocol_gpu.py +152 -0
  10. flashruntime-0.4.1/tests/test_recipe_gpu.py +129 -0
  11. {flashruntime-0.4.0 → flashruntime-0.4.1}/LICENSE +0 -0
  12. {flashruntime-0.4.0 → flashruntime-0.4.1}/README.md +0 -0
  13. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/__init__.py +0 -0
  14. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/fedavg_driver.py +0 -0
  15. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/fedavg_weights.py +0 -0
  16. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/fedavg_worker.py +0 -0
  17. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/kmeans_driver.py +0 -0
  18. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/kmeans_shard.py +0 -0
  19. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/sgd_trainer.py +0 -0
  20. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/sharded_kmeans.py +0 -0
  21. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashml_workloads/sklearn_trial.py +0 -0
  22. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/__init__.py +0 -0
  23. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/artifacts/__init__.py +0 -0
  24. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/artifacts/store.py +0 -0
  25. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/backends/__init__.py +0 -0
  26. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/backends/base.py +0 -0
  27. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/backends/kuberay.py +0 -0
  28. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/__init__.py +0 -0
  29. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/catalog.py +0 -0
  30. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/local.py +0 -0
  31. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/checkpoint/store.py +0 -0
  32. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/__init__.py +0 -0
  33. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/huggingface.py +0 -0
  34. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/pytorch.py +0 -0
  35. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/integrations/sklearn.py +0 -0
  36. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/launchers/__init__.py +0 -0
  37. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/launchers/local.py +0 -0
  38. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/__init__.py +0 -0
  39. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/manager.py +0 -0
  40. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/sqlite_store.py +0 -0
  41. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/leases/store.py +0 -0
  42. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/monitor/__init__.py +0 -0
  43. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/monitor/sampler.py +0 -0
  44. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/__init__.py +0 -0
  45. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/candidates.py +0 -0
  46. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/catalog.py +0 -0
  47. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/comm.py +0 -0
  48. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/explain.py +0 -0
  49. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/memory.py +0 -0
  50. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/resolve.py +0 -0
  51. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/selector.py +0 -0
  52. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/planner/timecost.py +0 -0
  53. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/profiling/__init__.py +0 -0
  54. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/protocol/__init__.py +0 -0
  55. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/protocol/plan_v1alpha1.py +0 -0
  56. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/providers/__init__.py +0 -0
  57. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/py.typed +0 -0
  58. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recipes/__init__.py +0 -0
  59. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/__init__.py +0 -0
  60. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/policy.py +0 -0
  61. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/signals.py +0 -0
  62. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/recovery/taxonomy.py +0 -0
  63. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/sdk.py +0 -0
  64. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/__init__.py +0 -0
  65. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/app.py +0 -0
  66. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/auth.py +0 -0
  67. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/checkpoints.py +0 -0
  68. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/cli.py +0 -0
  69. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/dashboard.py +0 -0
  70. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/ledger.py +0 -0
  71. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/service/modea.py +0 -0
  72. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/strategies/__init__.py +0 -0
  73. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/strategies/command.py +0 -0
  74. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/torch/__init__.py +0 -0
  75. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/__init__.py +0 -0
  76. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/benchmarks.html +0 -0
  77. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/concepts/architecture.html +0 -0
  78. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/get-started.html +0 -0
  79. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/federated-averaging.html +0 -0
  80. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/huggingface.html +0 -0
  81. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/jobspec-and-isolation.html +0 -0
  82. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/pytorch.html +0 -0
  83. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/guides/sklearn.html +0 -0
  84. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/index.html +0 -0
  85. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/cli.html +0 -0
  86. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/integrations.html +0 -0
  87. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/sdk.html +0 -0
  88. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/reference/torch-helper.html +0 -0
  89. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/search-index.json +0 -0
  90. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/tutorials/convnet.html +0 -0
  91. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/tutorials/fault-tolerance.html +0 -0
  92. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/_docs/tutorials/sklearn-sweeps.html +0 -0
  93. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/flowmap.py +0 -0
  94. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/page.py +0 -0
  95. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/server.py +0 -0
  96. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/viewer/state.py +0 -0
  97. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/workloads/__init__.py +0 -0
  98. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime/workloads/command.py +0 -0
  99. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/dependency_links.txt +0 -0
  100. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/entry_points.txt +0 -0
  101. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/requires.txt +0 -0
  102. {flashruntime-0.4.0 → flashruntime-0.4.1}/flashruntime.egg-info/top_level.txt +0 -0
  103. {flashruntime-0.4.0 → flashruntime-0.4.1}/setup.cfg +0 -0
  104. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_auto_recovery.py +0 -0
  105. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_benchmarks.py +0 -0
  106. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_checkpoint.py +0 -0
  107. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_checkpoint_local.py +0 -0
  108. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_cli_submit.py +0 -0
  109. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_documentation.py +0 -0
  110. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_examples_e2e.py +0 -0
  111. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_convergence.py +0 -0
  112. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_driver.py +0 -0
  113. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_weights.py +0 -0
  114. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_fedavg_worker.py +0 -0
  115. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_gpu_e2e.py +0 -0
  116. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_integrations.py +0 -0
  117. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_interfaces.py +0 -0
  118. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_kmeans_leases.py +0 -0
  119. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_kmeans_workload.py +0 -0
  120. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_kuberay_backend.py +0 -0
  121. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_launcher_local.py +0 -0
  122. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_leases.py +0 -0
  123. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_leases_scope.py +0 -0
  124. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_leases_sqlite.py +0 -0
  125. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_monitor_sampler.py +0 -0
  126. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_placement_local_data.py +0 -0
  127. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_planner.py +0 -0
  128. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_protocol_local_datasets.py +0 -0
  129. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_protocol_v1alpha1.py +0 -0
  130. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_recipe_local_inputs.py +0 -0
  131. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_recipes_command.py +0 -0
  132. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_recovery.py +0 -0
  133. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_run_json.py +0 -0
  134. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_scheduler_isolation.py +0 -0
  135. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_sdk_submit.py +0 -0
  136. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_auth.py +0 -0
  137. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_auth_startup.py +0 -0
  138. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_checkpoints.py +0 -0
  139. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_command_recipe.py +0 -0
  140. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_delegation.py +0 -0
  141. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_fedavg.py +0 -0
  142. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_modea.py +0 -0
  143. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_service_write_scope.py +0 -0
  144. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_sgd_trainer.py +0 -0
  145. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_strategy_command.py +0 -0
  146. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_torch_helper.py +0 -0
  147. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_flowmap.py +0 -0
  148. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_page.py +0 -0
  149. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_server.py +0 -0
  150. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_viewer_state.py +0 -0
  151. {flashruntime-0.4.0 → flashruntime-0.4.1}/tests/test_workloads_command.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flashruntime
3
- Version: 0.4.0
3
+ Version: 0.4.1
4
4
  Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
5
5
  Author: Zolli Labs
6
6
  License-Expression: Apache-2.0
@@ -78,6 +78,13 @@ class ResourcesSpec(BaseModel):
78
78
  maximumWorkers: int = Field(ge=1, default=3)
79
79
  cpuPerTask: float = Field(gt=0, default=1)
80
80
  memoryPerTask: str = "512Mi"
81
+ #: GPUs each task needs. 0 — the default, and every job that exists
82
+ #: today — means "no GPU required" and places anywhere. A non-zero value
83
+ #: is a hardware *requirement*, enforced fail-closed at placement
84
+ #: (`scheduler.IsolationAwarePlacement`): a CUDA task on a CPU-only host
85
+ #: does not politely requeue, it crashes or silently runs on the CPU two
86
+ #: orders of magnitude slower while reporting success.
87
+ gpuPerTask: int = Field(ge=0, default=0)
81
88
 
82
89
  @field_validator("maximumWorkers")
83
90
  @classmethod
@@ -269,10 +276,39 @@ class NodeEnvironment(str, Enum):
269
276
  EDGE = "edge"
270
277
 
271
278
 
279
+ class GpuInfo(BaseModel):
280
+ """One GPU as the host's driver reports it.
281
+
282
+ Every field but `index` is optional: a probe that cannot read a value says
283
+ nothing rather than guessing, so a partial reading is still reportable and
284
+ an unreadable one is still `[]` rather than a fabricated device.
285
+
286
+ Typed now although placement matches only on COUNT, because the wire
287
+ format is the expensive thing to change later — these agents run on
288
+ machines we cannot reach, and a field added to an existing model is cheap
289
+ where turning a `dict` into a model is not. `memory_total_mb` and
290
+ `compute_capability` are collected from the start so the data is already
291
+ flowing when matching rules arrive (spec §10.1).
292
+
293
+ No `schema_version`: like `CheckpointPart`, this is a nested element of a
294
+ versioned message (`NodeRegistration`), not a wire message of its own.
295
+ """
296
+
297
+ index: int
298
+ name: str = ""
299
+ memory_total_mb: int | None = None
300
+ driver_version: str = ""
301
+ compute_capability: str = ""
302
+
303
+
272
304
  class NodeCapabilities(BaseModel):
273
305
  cpu_cores: float | None = None
274
306
  memory_bytes: int | None = None
275
- gpus: list[dict[str, Any]] = Field(default_factory=list)
307
+ #: Devices this host's driver reports, one entry per GPU. Empty is the
308
+ #: honest answer for a host with no driver, no `nvidia-smi`, or output
309
+ #: this runtime cannot parse — never a guess. Placement reads the LENGTH
310
+ #: of this list and nothing else in v1.
311
+ gpus: list[GpuInfo] = Field(default_factory=list)
276
312
  os: str = ""
277
313
  architecture: str = ""
278
314
 
@@ -160,6 +160,26 @@ class CommandRecipe(WorkloadRecipe):
160
160
  #
161
161
  # `list(...)` so a payload never aliases the caller's spec.
162
162
  payload["local_inputs"] = list(p["local_inputs"])
163
+ gpus = spec.spec.resources.gpuPerTask
164
+ if gpus:
165
+ # The GPU requirement is the ONLY payload key sourced from
166
+ # `spec.spec.resources` rather than `workload.parameters` —
167
+ # a branch of the spec nothing else in this method reads.
168
+ # That makes it the easiest forward in the file to leave out
169
+ # and the hardest to notice missing: there is no unrecognised
170
+ # parameter sitting in `p` to go looking for.
171
+ #
172
+ # Dropping it does NOT fail closed. `IsolationAwarePlacement`
173
+ # reads `task.payload["gpus"]`; with the key absent it sees a
174
+ # task requiring nothing, places it on any node, and a CUDA
175
+ # job lands on a CPU-only volunteer. Both ends of this hop
176
+ # have tests that pass while it is broken, because each
177
+ # constructs the payload directly.
178
+ #
179
+ # Absent stays absent, never 0 — the no-GPU path must keep
180
+ # exercising the key-missing branch, as unpack_inputs and
181
+ # local_inputs do.
182
+ payload["gpus"] = int(gpus)
163
183
  tasks.append(
164
184
  TaskSpec(
165
185
  task_id=task_id,
@@ -172,6 +172,43 @@ class IsolationAwarePlacement(PlacementPolicy):
172
172
  is not in its map) — failing closed here makes it fail before anything
173
173
  touches the data, and before an attempt is burned.
174
174
 
175
+ A fifth gate applies to tasks whose payload asks for `gpus: N`: the
176
+ claiming node's `capabilities.gpus` must be a list of at least N entries.
177
+ It takes the argv/local-data polarity — **fail closed** — and deliberately
178
+ NOT the module gate's fail-open one, even though "the node cannot run
179
+ this" sounds like the same availability concern:
180
+
181
+ - A misplaced module task wastes retry attempts and nothing else. A CUDA
182
+ job on a CPU-only box does not politely fail and requeue. It either
183
+ crashes on `torch.cuda.is_available()` or, worse, silently falls back
184
+ to the CPU and runs two orders of magnitude slower while reporting
185
+ success. The second outcome is not a failure anything here can detect;
186
+ it is a bill.
187
+ - The capability must be a genuine *list*, because its LENGTH is the
188
+ whole matching rule in v1. Absent, `None`, a bare string, a dict, or a
189
+ bare `int` count as NO GPUs. The bare `int` deserves naming: `1` reads
190
+ like "one GPU" and is exactly what a hand-written node view would put
191
+ there, but accepting it would mean a second, looser matching rule
192
+ beside the one `NodeCapabilities.gpus` actually feeds.
193
+ - `capabilities` itself may be absent or type-confused; that is read as
194
+ no GPUs rather than allowed to raise. `(node.get("capabilities") or
195
+ {}).get(...)` is NOT sufficient for this — a string capabilities value
196
+ has no `.get` and would crash the predicate.
197
+ - The requirement itself must be a non-negative `int`. Anything else
198
+ (`"1"`, `-1`, `1.5`, a list) makes the task ineligible everywhere
199
+ rather than crashing, exactly as a non-list `local_inputs` does.
200
+ **`bool` is a subclass of `int`**, so `True` is excluded explicitly: a
201
+ `gpus: true` typo must not silently mean "one GPU" and place real work.
202
+ - `gpus: 0` requires nothing and runs anywhere, exactly like tier
203
+ `standard` and an empty `local_inputs`.
204
+
205
+ The gate is ONE-DIRECTIONAL. A node with GPUs still receives CPU work;
206
+ reserving GPU hosts for GPU jobs is a scheduling optimisation and a
207
+ separate decision, and making it a gate here would idle the scarcest
208
+ hardware on the network. `allowFallback` does not waive this gate either
209
+ — hardware either exists on a host or it does not, and the submitter's
210
+ isolation posture has nothing to say about it.
211
+
175
212
  Everything genuinely standard keeps the fail-open placement default."""
176
213
 
177
214
  def eligible(self, task: TaskSpec, node: NodeView) -> bool:
@@ -199,6 +236,28 @@ class IsolationAwarePlacement(PlacementPolicy):
199
236
  return False # absent/None/type-confused capability ⇒ not capable
200
237
  if any(name not in advertised for name in local_inputs):
201
238
  return False
239
+ # Fail-closed like the argv and local-data gates, and checked before
240
+ # the allowFallback waiver below for the same reason: a submitter
241
+ # cannot waive their way onto hardware a host does not have.
242
+ required_gpus = task.payload.get("gpus")
243
+ if required_gpus is not None:
244
+ # `bool` is a subclass of `int`: without the explicit exclusion,
245
+ # a `gpus: true` typo would read as "1 GPU" and place real work.
246
+ if (
247
+ not isinstance(required_gpus, int)
248
+ or isinstance(required_gpus, bool)
249
+ or required_gpus < 0
250
+ ):
251
+ return False # type-confused requirement ⇒ fail closed, no crash
252
+ if required_gpus > 0:
253
+ capabilities = node.get("capabilities")
254
+ # isinstance, not `or {}` — a string capabilities value has no
255
+ # `.get` and must fail closed rather than crash the predicate.
256
+ advertised = (
257
+ capabilities.get("gpus") if isinstance(capabilities, dict) else None
258
+ )
259
+ if not isinstance(advertised, list) or len(advertised) < required_gpus:
260
+ return False # absent/short/type-confused ⇒ not capable
202
261
  isolation = task.payload.get("isolation")
203
262
  if isolation is None:
204
263
  return True # no isolation payload ⇒ standard, runs anywhere
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: flashruntime
3
- Version: 0.4.0
3
+ Version: 0.4.1
4
4
  Summary: Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute.
5
5
  Author: Zolli Labs
6
6
  License-Expression: Apache-2.0
@@ -118,10 +118,13 @@ tests/test_leases.py
118
118
  tests/test_leases_scope.py
119
119
  tests/test_leases_sqlite.py
120
120
  tests/test_monitor_sampler.py
121
+ tests/test_placement_gpu.py
121
122
  tests/test_placement_local_data.py
122
123
  tests/test_planner.py
124
+ tests/test_protocol_gpu.py
123
125
  tests/test_protocol_local_datasets.py
124
126
  tests/test_protocol_v1alpha1.py
127
+ tests/test_recipe_gpu.py
125
128
  tests/test_recipe_local_inputs.py
126
129
  tests/test_recipes_command.py
127
130
  tests/test_recovery.py
@@ -5,12 +5,23 @@ build-backend = "setuptools.build_meta"
5
5
 
6
6
  [project]
7
7
  name = "flashruntime"
8
- # 0.4.0 adds NodeRegistration.local_datasets a new wire field, so consumers
9
- # must pin a range that includes it. flashnode’s floor moves to >=0.4,<0.5.
10
- # (0.3.0 was the first published release; 0.1.0 and 0.2.0 were declared here
11
- # but never reached PyPI.) Released from the Zolli-Labs/flashml monorepo by
12
- # pushing a `flashruntime-v0.4.0` tag.
13
- version = "0.4.0"
8
+ # 0.4.1 adds GpuInfo and ResourcesSpec.gpuPerTask. Both are additive wire
9
+ # fields, so the compatibility range does not move but the PATCH bump is
10
+ # load-bearing, not bookkeeping. The `flashruntime-v0.4.0` tag was cut BEFORE
11
+ # the protocol commit that added them, and this tree went on calling itself
12
+ # 0.4.0 afterwards: one version string named two different protocols, which is
13
+ # exactly the drift the monorepo consolidation removed. Bump on any
14
+ # wire-visible change, even an additive one.
15
+ #
16
+ # flashnode’s floor moves to >=0.4.1,<0.5 in the same change — it imports
17
+ # GpuInfo at module scope, so for the agent this floor is an import-time hard
18
+ # requirement, not a silent-degradation one.
19
+ #
20
+ # 0.4.0 added NodeRegistration.local_datasets — a new wire field, so consumers
21
+ # must pin a range that includes it. (0.3.0 was the first published release;
22
+ # 0.1.0 and 0.2.0 were declared here but never reached PyPI.) Released from
23
+ # the Zolli-Labs/flashml monorepo by pushing a `flashruntime-vX.Y.Z` tag.
24
+ version = "0.4.1"
14
25
  description = "Open fault-tolerant distributed ML runtime: strategy planning, job protocol, leases, checkpointing, and recovery across heterogeneous compute."
15
26
  readme = "README.md"
16
27
  # PEP 639 SPDX expression (string form) — replaces the deprecated
@@ -0,0 +1,301 @@
1
+ """The fifth fail-closed placement gate: a task whose payload asks for `gpus: N`
2
+ may only be leased to a node whose `capabilities.gpus` is a list of at least N
3
+ entries (AGENTS.md rule 3).
4
+
5
+ Polarity follows `argv_capable` / `local_datasets`, not `module_capable`. A
6
+ misplaced module task only wastes retry attempts; a CUDA job on a CPU-only box
7
+ either crashes on `torch.cuda.is_available()` or silently falls back to the CPU
8
+ and runs two orders of magnitude slower while reporting success. Neither is
9
+ something to discover from a bill.
10
+
11
+ The gate is one-directional: a GPU node still receives CPU work.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import pytest
17
+
18
+
19
+ def _gpu_task(gpus=1, **payload_extra):
20
+ """A task requiring GPUs. `gpus` is passed through RAW so tests can poison
21
+ it with type-confused values the protocol would never emit (the same trick
22
+ `_local_task` plays on `local_inputs`)."""
23
+ from flashruntime.protocol.v1alpha1 import TaskSpec
24
+
25
+ payload = {"module": "flashml_workloads.sklearn_trial"}
26
+ if gpus is not None:
27
+ payload["gpus"] = gpus
28
+ payload.update(payload_extra)
29
+ return TaskSpec(
30
+ task_id="task-000", job_id="job-a", commit_key="job-a/task-000/m.json",
31
+ payload=payload,
32
+ )
33
+
34
+
35
+ def _node(gpu_count=0, **extra):
36
+ """A node view shaped like the claim endpoint's: the GPU list lives under
37
+ `capabilities`, which is `NodeCapabilities.model_dump()` — plain dicts by
38
+ the time placement sees them, not GpuInfo instances."""
39
+ node = {
40
+ "node_id": "n1",
41
+ "capabilities": {
42
+ "cpu_cores": 8,
43
+ "gpus": [{"index": i, "name": "NVIDIA A10G"} for i in range(gpu_count)],
44
+ },
45
+ }
46
+ node.update(extra)
47
+ return node
48
+
49
+
50
+ # ---------------------------------------------------------------------------
51
+ # DoD 3–6: the four placement outcomes
52
+ # ---------------------------------------------------------------------------
53
+
54
+
55
+ def test_one_gpu_job_is_refused_on_a_node_advertising_none():
56
+ from flashruntime.scheduler import IsolationAwarePlacement
57
+
58
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), _node(0)) is False
59
+
60
+
61
+ def test_one_gpu_job_is_placed_on_a_node_advertising_one():
62
+ from flashruntime.scheduler import IsolationAwarePlacement
63
+
64
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), _node(1)) is True
65
+
66
+
67
+ def test_two_gpu_job_is_refused_on_a_one_gpu_node():
68
+ from flashruntime.scheduler import IsolationAwarePlacement
69
+
70
+ assert IsolationAwarePlacement().eligible(_gpu_task(2), _node(1)) is False
71
+
72
+
73
+ def test_two_gpu_job_is_placed_on_a_two_gpu_node():
74
+ from flashruntime.scheduler import IsolationAwarePlacement
75
+
76
+ assert IsolationAwarePlacement().eligible(_gpu_task(2), _node(2)) is True
77
+
78
+
79
+ def test_a_job_with_no_gpu_requirement_is_placed_on_both():
80
+ """The key must be ABSENT, not 0 — this is the branch every job that
81
+ exists today takes."""
82
+ from flashruntime.scheduler import IsolationAwarePlacement
83
+
84
+ policy = IsolationAwarePlacement()
85
+ task = _gpu_task(None) # no "gpus" key in the payload at all
86
+ assert policy.eligible(task, _node(0)) is True
87
+ assert policy.eligible(task, _node(2)) is True
88
+
89
+
90
+ def test_gpus_zero_requires_nothing_and_runs_anywhere():
91
+ """`gpus: 0` is what an explicit opt-out compiles to. It demands nothing,
92
+ like tier "standard" and an empty `local_inputs`."""
93
+ from flashruntime.scheduler import IsolationAwarePlacement
94
+
95
+ policy = IsolationAwarePlacement()
96
+ assert policy.eligible(_gpu_task(0), _node(0)) is True
97
+ assert policy.eligible(_gpu_task(0), {"node_id": "n1"}) is True
98
+
99
+
100
+ def test_a_gpu_node_still_receives_cpu_work():
101
+ """The gate is one-directional. Reserving GPU hosts for GPU jobs is a
102
+ separate scheduling decision; making it a gate here would idle the
103
+ scarcest hardware on the network."""
104
+ from flashruntime.scheduler import IsolationAwarePlacement
105
+
106
+ assert IsolationAwarePlacement().eligible(_gpu_task(None), _node(4)) is True
107
+
108
+
109
+ # ---------------------------------------------------------------------------
110
+ # Fail-closed: the advertised capability
111
+ # ---------------------------------------------------------------------------
112
+
113
+
114
+ @pytest.mark.parametrize(
115
+ "advertised",
116
+ [
117
+ pytest.param([], id="empty-list"),
118
+ pytest.param(None, id="explicit-none"),
119
+ pytest.param("gpu", id="bare-string-type-confusion"),
120
+ pytest.param({}, id="empty-dict-type-confusion"),
121
+ pytest.param({"0": "NVIDIA A10G"}, id="dict-type-confusion"),
122
+ pytest.param(({"index": 0},), id="tuple-type-confusion"),
123
+ pytest.param(True, id="truthy-boolean"),
124
+ pytest.param(1, id="bare-count-int"),
125
+ ],
126
+ )
127
+ def test_ineligible_unless_the_capability_is_a_list_long_enough(advertised):
128
+ """Fail closed on absence AND on type confusion. A bare `1` is the most
129
+ tempting wrong answer — it reads like "one GPU" and has no length, so
130
+ accepting it would mean writing a second, looser matching rule beside the
131
+ one the probe actually feeds."""
132
+ from flashruntime.scheduler import IsolationAwarePlacement
133
+
134
+ node = {"node_id": "n1", "capabilities": {"gpus": advertised}}
135
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
136
+
137
+
138
+ def test_node_with_no_capabilities_key_at_all_is_ineligible():
139
+ """An already-deployed agent whose registration predates GPU probing
140
+ advertises nothing — absent must read as "has none", never "unknown, try
141
+ it and see"."""
142
+ from flashruntime.scheduler import IsolationAwarePlacement
143
+
144
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), {"node_id": "n1"}) is False
145
+
146
+
147
+ def test_node_with_capabilities_none_is_ineligible():
148
+ from flashruntime.scheduler import IsolationAwarePlacement
149
+
150
+ node = {"node_id": "n1", "capabilities": None}
151
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
152
+
153
+
154
+ def test_node_with_type_confused_capabilities_is_ineligible_without_crashing():
155
+ from flashruntime.scheduler import IsolationAwarePlacement
156
+
157
+ node = {"node_id": "n1", "capabilities": "8 cores, 1 gpu"}
158
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
159
+
160
+
161
+ def test_node_capabilities_without_a_gpus_key_is_ineligible():
162
+ from flashruntime.scheduler import IsolationAwarePlacement
163
+
164
+ node = {"node_id": "n1", "capabilities": {"cpu_cores": 8}}
165
+ assert IsolationAwarePlacement().eligible(_gpu_task(1), node) is False
166
+
167
+
168
+ # ---------------------------------------------------------------------------
169
+ # Fail-closed: the requirement itself
170
+ # ---------------------------------------------------------------------------
171
+
172
+
173
+ @pytest.mark.parametrize(
174
+ "required",
175
+ [
176
+ pytest.param("1", id="numeric-string"),
177
+ pytest.param(-1, id="negative"),
178
+ pytest.param(1.5, id="fractional"),
179
+ pytest.param(1.0, id="whole-float"),
180
+ pytest.param([1], id="list"),
181
+ pytest.param({"count": 1}, id="dict"),
182
+ pytest.param(True, id="boolean-true"),
183
+ ],
184
+ )
185
+ def test_type_confused_requirement_is_ineligible_everywhere(required):
186
+ """A poisoned requirement must fail closed on EVERY node — including one
187
+ with plenty of GPUs — and must never crash the predicate.
188
+
189
+ `True` is the one that needs saying out loud: `bool` is a subclass of
190
+ `int` in Python, so `isinstance(True, int)` is True and `True >= 1`.
191
+ Without the explicit bool exclusion, `gpus: true` would silently mean
192
+ "one GPU" — a JSON typo that placed real work.
193
+ """
194
+ from flashruntime.scheduler import IsolationAwarePlacement
195
+
196
+ policy = IsolationAwarePlacement()
197
+ assert policy.eligible(_gpu_task(required), _node(0)) is False
198
+ assert policy.eligible(_gpu_task(required), _node(8)) is False
199
+
200
+
201
+ def test_boolean_false_is_not_read_as_zero_gpus():
202
+ """The mirror of the above: `False` is not "no GPU required" either. It is
203
+ a type-confused requirement and fails closed like every other."""
204
+ from flashruntime.scheduler import IsolationAwarePlacement
205
+
206
+ assert IsolationAwarePlacement().eligible(_gpu_task(False), _node(8)) is False
207
+
208
+
209
+ # ---------------------------------------------------------------------------
210
+ # Interaction with the existing gates
211
+ # ---------------------------------------------------------------------------
212
+
213
+
214
+ def test_allow_fallback_cannot_bypass_the_gpu_gate():
215
+ """allowFallback waives the sandbox-tier requirement only. It is the
216
+ submitter's statement about their own isolation posture, and has nothing
217
+ to say about hardware that either exists or does not."""
218
+ from flashruntime.scheduler import IsolationAwarePlacement
219
+
220
+ task = _gpu_task(1, isolation={"tier": "sandboxed", "allowFallback": True})
221
+ node = _node(0, sandbox_capable=True)
222
+ assert IsolationAwarePlacement().eligible(task, node) is False
223
+
224
+
225
+ def test_a_gpu_node_still_fails_the_argv_gate_it_does_not_pass():
226
+ """The GPU gate adds a requirement; it never satisfies another one."""
227
+ from flashruntime.protocol.v1alpha1 import TaskSpec
228
+ from flashruntime.scheduler import IsolationAwarePlacement
229
+
230
+ task = TaskSpec(
231
+ task_id="task-000", job_id="job-a", commit_key="job-a/task-000/m.json",
232
+ payload={"argv": ["python", "train.py"], "gpus": 1},
233
+ )
234
+ assert IsolationAwarePlacement().eligible(task, _node(1)) is False
235
+ assert IsolationAwarePlacement().eligible(task, _node(1, argv_capable=True)) is True
236
+
237
+
238
+ def test_claim_over_a_gpu_queue_serves_the_clean_task():
239
+ """A GPU task at the head of the queue must not block a CPU-only node —
240
+ the next clean task is still leased, and the GPU task is never
241
+ mis-leased."""
242
+ from flashruntime.leases import LeaseManager
243
+ from flashruntime.protocol.v1alpha1 import TaskSpec
244
+ from flashruntime.scheduler import IsolationAwarePlacement
245
+
246
+ mgr = LeaseManager()
247
+ mgr.add_task(_gpu_task(1)) # head of queue, needs a GPU
248
+ mgr.add_task(
249
+ TaskSpec(task_id="task-001", job_id="job-a", commit_key="job-a/task-001/m.json")
250
+ )
251
+ policy = IsolationAwarePlacement()
252
+
253
+ lease = mgr.claim("n1", policy=policy, node=_node(0))
254
+ assert lease is not None
255
+ assert lease.task_id == "task-001"
256
+
257
+ lease2 = mgr.claim("n2", policy=policy, node=_node(1, node_id="n2"))
258
+ assert lease2 is not None
259
+ assert lease2.task_id == "task-000"
260
+
261
+
262
+ def test_claim_endpoint_leases_gpu_work_only_to_a_gpu_host():
263
+ """End to end over the claim endpoint: the registry's claim-time node view
264
+ must forward `capabilities.gpus`, or the gate reads an absent capability on
265
+ EVERY node and a GPU task becomes unplaceable everywhere.
266
+
267
+ This pins hop 1 of the three that broke the `local_datasets` work — the
268
+ node view. It is believed correct today (`capabilities` is forwarded whole
269
+ as `model_dump()`), and this test is what keeps it that way.
270
+ """
271
+ import pathlib
272
+
273
+ import fastapi
274
+ from fastapi.testclient import TestClient
275
+
276
+ from flashruntime.leases import LeaseManager
277
+ from flashruntime.service.modea import ModeAState, build_router
278
+
279
+ state = ModeAState(LeaseManager(), artifacts_dir=pathlib.Path("/tmp"))
280
+ app = fastapi.FastAPI()
281
+ app.include_router(build_router(state))
282
+ client = TestClient(app)
283
+ state.manager.add_task(_gpu_task(1))
284
+
285
+ def register(node_id: str, gpus: list[dict]):
286
+ r = client.post(
287
+ "/v1alpha1/nodes/register",
288
+ json={
289
+ "node_id": node_id, "kubernetes_node": "", "hostname": node_id,
290
+ "capabilities": {"cpu_cores": 8, "gpus": gpus},
291
+ },
292
+ )
293
+ assert r.status_code == 200
294
+
295
+ register("cpu-node", [])
296
+ register("gpu-node", [{"index": 0, "name": "NVIDIA A10G", "memory_total_mb": 22731}])
297
+
298
+ assert client.post("/v1alpha1/leases/claim", json={"node_id": "cpu-node"}).status_code == 204
299
+ r = client.post("/v1alpha1/leases/claim", json={"node_id": "gpu-node"})
300
+ assert r.status_code == 200
301
+ assert r.json()["task_id"] == "task-000"
@@ -0,0 +1,152 @@
1
+ """`GpuInfo` and `ResourcesSpec.gpuPerTask` — the two wire additions GPU
2
+ placement needs.
3
+
4
+ These are protocol tests, not scheduler tests: they pin the *wire* behaviour
5
+ the fifth placement gate is later allowed to trust, exactly as
6
+ `test_protocol_local_datasets.py` does for `local_datasets`.
7
+
8
+ The typing change matters more than it looks. `NodeCapabilities.gpus` was
9
+ `list[dict[str, Any]]`; it becomes `list[GpuInfo]`. Agents already deployed on
10
+ machines we cannot reach send plain JSON objects, so a raw dict must still
11
+ coerce — a model that only accepted `GpuInfo` instances would be a breaking
12
+ change dressed up as a type annotation.
13
+ """
14
+
15
+ from __future__ import annotations
16
+
17
+ import pytest
18
+ from pydantic import ValidationError
19
+
20
+ from flashruntime.protocol.v1alpha1 import GpuInfo, NodeCapabilities, ResourcesSpec
21
+
22
+
23
+ # ---------------------------------------------------------------------------
24
+ # GpuInfo
25
+ # ---------------------------------------------------------------------------
26
+
27
+
28
+ def test_gpu_info_round_trips_with_only_index_set():
29
+ """Every field but `index` is optional: a probe that cannot read a value
30
+ says nothing rather than guessing, and the omission must survive JSON."""
31
+ gpu = GpuInfo(index=0)
32
+ restored = GpuInfo.model_validate_json(gpu.model_dump_json())
33
+
34
+ assert restored.index == 0
35
+ assert restored.name == ""
36
+ assert restored.memory_total_mb is None
37
+ assert restored.driver_version == ""
38
+ assert restored.compute_capability == ""
39
+
40
+
41
+ def test_gpu_info_round_trips_a_fully_populated_device():
42
+ gpu = GpuInfo(
43
+ index=1,
44
+ name="NVIDIA GeForce RTX 4090",
45
+ memory_total_mb=24564,
46
+ driver_version="550.54.14",
47
+ compute_capability="8.9",
48
+ )
49
+ restored = GpuInfo.model_validate_json(gpu.model_dump_json())
50
+
51
+ assert restored == gpu
52
+
53
+
54
+ def test_gpu_info_requires_an_index():
55
+ """The one field a probe can always report. Without it there is no way to
56
+ tell two devices apart, so it is not defaulted."""
57
+ with pytest.raises(ValidationError):
58
+ GpuInfo.model_validate({"name": "NVIDIA GeForce RTX 4090"})
59
+
60
+
61
+ # ---------------------------------------------------------------------------
62
+ # NodeCapabilities.gpus
63
+ # ---------------------------------------------------------------------------
64
+
65
+
66
+ def test_node_capabilities_accepts_gpu_info_instances():
67
+ caps = NodeCapabilities(gpus=[GpuInfo(index=0)])
68
+
69
+ assert len(caps.gpus) == 1
70
+ assert caps.gpus[0].index == 0
71
+
72
+
73
+ def test_node_capabilities_coerces_raw_dicts_from_the_wire():
74
+ """Deployed agents send JSON objects, not `GpuInfo` instances. The typing
75
+ change must be additive on the wire or every already-enrolled machine
76
+ fails to register the moment the coordinator upgrades."""
77
+ caps = NodeCapabilities.model_validate(
78
+ {
79
+ "cpu_cores": 8,
80
+ "gpus": [
81
+ {"index": 0, "name": "NVIDIA A10G", "memory_total_mb": 22731},
82
+ {"index": 1},
83
+ ],
84
+ }
85
+ )
86
+
87
+ assert [g.index for g in caps.gpus] == [0, 1]
88
+ assert isinstance(caps.gpus[0], GpuInfo)
89
+ assert caps.gpus[0].name == "NVIDIA A10G"
90
+ assert caps.gpus[1].name == ""
91
+
92
+
93
+ def test_node_capabilities_ignores_fields_a_newer_probe_might_add():
94
+ """Forward compatibility in the other direction: a newer agent reporting a
95
+ field this coordinator does not know must not fail registration."""
96
+ caps = NodeCapabilities.model_validate(
97
+ {"gpus": [{"index": 0, "some_future_field": "ignored"}]}
98
+ )
99
+
100
+ assert caps.gpus[0].index == 0
101
+
102
+
103
+ def test_node_capabilities_defaults_to_no_gpus():
104
+ """Absent means "no GPU", never "unknown, assume yes" — the gate reads
105
+ this list and its length is the whole matching rule in v1."""
106
+ assert NodeCapabilities().gpus == []
107
+ assert NodeCapabilities.model_validate({"cpu_cores": 4}).gpus == []
108
+
109
+
110
+ def test_node_capabilities_gpu_default_is_not_shared_between_instances():
111
+ # `default_factory` and not a bare `[]`: one shared list would let a
112
+ # single node's devices leak into every other node's capabilities.
113
+ first = NodeCapabilities()
114
+ second = NodeCapabilities()
115
+
116
+ assert first.gpus is not second.gpus
117
+ first.gpus.append(GpuInfo(index=0))
118
+ assert second.gpus == []
119
+
120
+
121
+ # ---------------------------------------------------------------------------
122
+ # ResourcesSpec.gpuPerTask
123
+ # ---------------------------------------------------------------------------
124
+
125
+
126
+ def test_gpu_per_task_defaults_to_zero():
127
+ """0 means "no GPU required", which is every job that exists today."""
128
+ assert ResourcesSpec().gpuPerTask == 0
129
+
130
+
131
+ def test_gpu_per_task_accepts_a_positive_count():
132
+ assert ResourcesSpec(gpuPerTask=2).gpuPerTask == 2
133
+
134
+
135
+ @pytest.mark.parametrize(
136
+ "value",
137
+ [
138
+ pytest.param(-1, id="negative"),
139
+ pytest.param(1.5, id="fractional"),
140
+ pytest.param("one", id="not-a-number"),
141
+ ],
142
+ )
143
+ def test_gpu_per_task_rejects_values_that_are_not_a_device_count(value):
144
+ with pytest.raises(ValidationError):
145
+ ResourcesSpec(gpuPerTask=value)
146
+
147
+
148
+ def test_gpu_per_task_round_trips_through_json():
149
+ spec = ResourcesSpec(gpuPerTask=1)
150
+ restored = ResourcesSpec.model_validate_json(spec.model_dump_json())
151
+
152
+ assert restored.gpuPerTask == 1