freesolo-flash-dev 1.1.0__tar.gz → 1.1.2__tar.gz

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: freesolo-flash-dev
-Version: 1.1.0
+Version: 1.1.2
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/flash/runner/__init__.py

@@ -2080,18 +2080,24 @@ def _persist_metrics(spec: JobSpec, metrics: dict) -> float:
     # case _gpu_rate prices off whichever configured provider offers the class.
     provider = str(metrics.get("allocated_provider") or "")
     rate = _gpu_rate(gpu_type, provider)
+    # `hourly_rate` is per CARD, so a sharded run costs the wall times the rate times the number of
+    # cards it actually occupied. `allocated_gpu_count` is worker/lifecycle-stamped for the same
+    # reason `allocated_gpu` is: the spec's gpu.count is only a ceiling and allocation may pick
+    # fewer. Absent on records predating the stamp, where one card is the correct reading.
+    gpu_count = max(1, int(metrics.get("allocated_gpu_count") or 1))
     cost = metrics.get("cost_usd")
     if cost:
         cost = float(cost or 0.0)
     else:
         wall = float(metrics.get("wall_seconds") or 0.0)
-        cost = wall / 3600.0 * rate
+        cost = wall / 3600.0 * rate * gpu_count
         metrics = {**metrics, "cost_usd": cost}
         metrics.setdefault("notes", {})
         if isinstance(metrics["notes"], dict):
             metrics["notes"]["provider"] = provider or "unknown"
             metrics["notes"]["gpu_rate_usd_hr"] = rate
             metrics["notes"]["gpu"] = gpu_type
+            metrics["notes"]["gpu_count"] = gpu_count
     with open(os.path.join(dest, "metrics.json"), "w") as f:
         json.dump(metrics, f, indent=2)
     with contextlib.suppress(Exception):

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/flash/runner/deploy.py

@@ -1110,6 +1110,7 @@ def attach_run(run_id: str, log_stream=None) -> RunStatus:
             raise ValueError("persisted attempt identity is missing or invalid")
         next_attempt = recovered_attempt + 1
         allocated_gpu = remote.pop("allocated_gpu", None)
+        allocated_gpu_count = remote.pop("allocated_gpu_count", None)
         handle = JobHandle.from_dict(remote)
         try:
             poll_spec = _spec_with_remaining_wall(worker_spec, require_provider_minimum=False)
@@ -1256,6 +1257,8 @@ def attach_run(run_id: str, log_stream=None) -> RunStatus:
             return status_for_return()
         if allocated_gpu and isinstance(res.metrics, dict):
             res.metrics.setdefault("allocated_gpu", allocated_gpu)
+        if allocated_gpu_count and isinstance(res.metrics, dict):
+            res.metrics.setdefault("allocated_gpu_count", int(allocated_gpu_count))
         if not _adopt_completed_attempt(
             run_id,
             worker_spec,

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/flash/runner/lifecycle.py

@@ -316,6 +316,9 @@ def _runpod_completed_metrics(handle, *, deadline_at: float | None = None) -> di
         allocated_gpu = original.get("allocated_gpu")
         if allocated_gpu:
             metrics.setdefault("allocated_gpu", allocated_gpu)
+        allocated_count = original.get("allocated_gpu_count")
+        if allocated_count:
+            metrics.setdefault("allocated_gpu_count", int(allocated_count))
         return metrics
     except _CompletedAttemptPending:
         raise
@@ -684,6 +687,11 @@ def _submit_seed_supervised(
                 **canonical_handle,
                 "seed": int(seed),
                 "allocated_gpu": current_gpu.get("name"),
+                # carried beside the gpu name for the same reason and by the same route: the
+                # canonical provider handle drops unknown keys, so a recovering process can only
+                # learn the shape from what was persisted here. without the count a run adopted
+                # after a control-plane restart prices its wall as a single card.
+                "allocated_gpu_count": current_gpu.get("count"),
                 "on_last_gpu": bool(current_on_last_gpu["value"]),
                 "code_prefix": code_prefix,
             }
@@ -742,6 +750,11 @@ def _submit_seed_supervised(
             metrics.setdefault("allocated_gpu", current_gpu["name"])
         if current_gpu.get("provider"):
             metrics.setdefault("allocated_provider", current_gpu["provider"])
+        # the runpod serverless route returns here rather than through the `res.ok` stamp below, so
+        # the card count has to be recorded on both or a sharded serverless run is still priced as
+        # one card. same source either way: the candidate allocation actually chose.
+        if current_gpu.get("count"):
+            metrics.setdefault("allocated_gpu_count", int(current_gpu["count"]))
         return metrics
 
     max_retries = int(spec.gpu.max_retries)
@@ -984,6 +997,7 @@ def _submit_seed_supervised(
                 raise
             current_gpu["name"] = chosen.gpu
             current_gpu["provider"] = chosen.provider
+            current_gpu["count"] = int(getattr(chosen, "gpu_count", 1) or 1)
             current_attempt["value"] = attempt
             retry_delay = 0
             submission_lock = _deploy_lock(spec.run_id)
@@ -1113,6 +1127,11 @@ def _submit_seed_supervised(
                 # the provider that actually billed this run, so cost attribution prices the
                 # class on ITS substrate rather than assuming RunPod's table.
                 res.metrics.setdefault("allocated_provider", chosen.provider)
+                # and how many cards of it. `hourly_rate(gpu_type)` is per CARD, so a fallback that
+                # prices the wall once records a 2x/4x run at half or a quarter of its real spend.
+                # the spec's own gpu.count cannot stand in: it is a ceiling, and allocation
+                # routinely picks fewer (see TRAINING.md, "a ceiling, not an exact count").
+                res.metrics.setdefault("allocated_gpu_count", int(getattr(chosen, "gpu_count", 1)))
             return res.metrics
         # cancel wins over any retry-shaped failure.
         try:

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.1.0"
+version = "1.1.2"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -121,7 +121,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.1.0"
+version = "1.1.2"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/tests/test_opd_resume_safety.py

@@ -656,6 +656,8 @@ def test_opd_automatic_retry_after_teardown_requires_all_markers_absent(monkeypa
         # stamped alongside the gpu so cost attribution prices the class on the substrate
         # that actually billed it.
         "allocated_provider": "runpod",
+        # and the card count, since the hourly rate is per card.
+        "allocated_gpu_count": 1,
     }
     assert provider.attempts == [0, 1]
     retry_submit = provider.events.index(("submit", 1))
@@ -744,6 +746,8 @@ def test_opd_retry_passes_gate_revision_and_overwrites_spoofed_value(monkeypatch
         # stamped alongside the gpu so cost attribution prices the class on the substrate
         # that actually billed it.
         "allocated_provider": "runpod",
+        # and the card count, since the hourly rate is per card.
+        "allocated_gpu_count": 1,
     }
     broker_transport = {
         "FLASH_CONTROL_PANEL_URL": "https://broker.example",

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/tests/test_provider_routing.py

@@ -796,6 +796,57 @@ def test_runpod_cost_projection_flows_into_run_status(orch, monkeypatch):
     assert cost == pytest.approx(0.345)  # 0.5 hr x $0.69/hr (RTX 4090)
 
 
+def test_a_multi_card_run_is_costed_for_every_card_it_occupied(orch):
+    """`hourly_rate` is per CARD, so the measured cost must multiply by the allocated count.
+
+    without it a 4-card run records a quarter of its real spend, and the cost analytics that compare
+    estimates against actuals are corrupted by exactly that factor. the count has to come from the
+    metrics stamp rather than `spec.gpu.count`, which is only a ceiling -- allocation routinely picks
+    fewer cards than authored.
+    """
+    spec = _spec()
+    _seed_status(orch, spec)
+    base = {"train_tokens": 4096, "wall_seconds": 1800, "allocated_gpu": "RTX 4090"}
+
+    four_card = orch._persist_metrics(spec, {**base, "allocated_gpu_count": 4})
+
+    assert four_card == pytest.approx(0.345 * 4), "a 4-card run was priced as one card"
+    # a record predating the stamp still reads as one card rather than zero or a crash
+    assert orch._persist_metrics(spec, dict(base)) == pytest.approx(0.345)
+
+
+def test_the_allocated_card_count_reaches_the_metrics_the_cost_is_read_from(orch, monkeypatch):
+    """The multiply above is inert unless allocation actually stamps the count it chose.
+
+    this drives the real submit path rather than handing `_persist_metrics` a literal, because the
+    two halves fail independently: pricing can multiply correctly by a count that is never recorded.
+    """
+    from flash.providers import allocator
+    from flash.providers.base import Candidate, PollResult
+    from flash.providers.runpod import jobs as rp_jobs
+
+    monkeypatch.setattr(
+        allocator,
+        "allocate",
+        lambda *a, **k: _alloc(candidates=(Candidate("runpod", "RTX 4090", 0.69, 24, 4),)),
+    )
+
+    def fake_runpod_submit(run_spec, seed, log=None, on_handle=None, attempt=0, **_):
+        if on_handle:
+            on_handle(_runpod_handle(attempt=attempt))
+        return PollResult(True, metrics={"train_tokens": 4096, "wall_seconds": 1800})
+
+    monkeypatch.setattr(rp_jobs, "submit_run", fake_runpod_submit)
+    spec = _spec(provider="runpod", type="RTX 4090", count=4)
+    _seed_status(orch, spec)
+
+    metrics = orch._submit_seed_supervised(spec, spec.seed, io.StringIO())
+
+    assert metrics["allocated_gpu_count"] == 4
+    # and the stamp is what pricing then reads, so the two halves compose
+    assert orch._persist_metrics(spec, metrics) == pytest.approx(0.345 * 4)
+
+
 def test_infra_retry_walks_to_next_runpod_class_and_deletes_endpoint(orch, monkeypatch):
     from flash.providers import allocator
     from flash.providers.base import Candidate, PollResult

{freesolo_flash_dev-1.1.0 → freesolo_flash_dev-1.1.2}/uv.lock

@@ -1025,7 +1025,7 @@ wheels = [
 
 [[package]]
 name = "freesolo-flash"
-version = "1.1.0"
+version = "1.1.2"
 source = { editable = "." }
 
 [package.optional-dependencies]