freesolo-flash-dev 1.2.23__tar.gz → 1.2.28__tar.gz

{freesolo_flash_dev-1.2.23 → freesolo_flash_dev-1.2.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.23
+Version: 1.2.28
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.23 → freesolo_flash_dev-1.2.28}/flash/providers/runpod/job_execution.py

@@ -513,7 +513,21 @@ def _classify_queue_state(
             # between unhealthy and empty cannot make a broken box look like starvation.
             state.ever_saw_worker = True
         if any(workers.get(k) for k in ("throttled", "unhealthy", "initializing")) or not usable:
-            context.say(f"queued; workers: {workers}")
+            message = f"queued; workers: {workers}"
+            if (
+                not usable
+                and not recovering
+                and not state.ever_saw_worker
+                and state.queued_timer.since is not None
+            ):
+                elapsed_s = max(0, int(now - state.queued_timer.since))
+                budget = (
+                    f"{context.queue_grace_s:g}s"
+                    if _jobs.math.isfinite(context.queue_grace_s)
+                    else "unbounded"
+                )
+                message += f"; waited {elapsed_s}s of {budget} capacity grace"
+            context.say(message)
         if state.unhealthy_timer.expired(
             workers.get("unhealthy") and not usable and not recovering,
             now,

{freesolo_flash_dev-1.2.23 → freesolo_flash_dev-1.2.28}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.23"
+version = "1.2.28"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.23"
+version = "1.2.28"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.2.23 → freesolo_flash_dev-1.2.28}/tests/test_jobs.py

@@ -617,7 +617,7 @@ def test_no_capacity_detail_never_predicts_the_retry_disposition(monkeypatch):
     assert "next-best" not in res.detail
 
 
-def _queued_forever(monkeypatch, health, *, step=20.0):
+def _queued_forever(monkeypatch, health, *, step=20.0, queue_grace_s=900.0, log=None):
     """Poll a permanently queued job with controlled endpoint health.
 
     The clock step preserves the 90-second probe cadence and 300-second worker-coming-up TTL.
@@ -634,12 +634,158 @@ def _queued_forever(monkeypatch, health, *, step=20.0):
     monkeypatch.setattr(jobs.time, "time", lambda: next(clock))
     return jobs.poll_job(
         _runpod_handle(jobs),
+        log=log,
         interval_s=0,
         heartbeat_reader=lambda: None,
         setup_grace_s=3000.0,
-        queue_grace_s=900.0,
+        queue_grace_s=queue_grace_s,
+    )
+
+
+def test_empty_workers_log_elapsed_capacity_grace_before_no_capacity(monkeypatch):
+    from itertools import pairwise
+
+    logs = io.StringIO()
+    res = _queued_forever(
+        monkeypatch,
+        lambda eid, _fp, **_kw: {"workers": {}},
+        queue_grace_s=700.0,
+        log=logs,
+    )
+
+    assert not res.ok
+    assert res.failure == "no_capacity"
+    assert "no RunPod capacity" in res.detail
+    capacity_lines = [line for line in logs.getvalue().splitlines() if "capacity grace" in line]
+    elapsed_values = []
+    for line in capacity_lines:
+        match = re.search(r"; waited (\d+)s of 700s capacity grace$", line)
+        assert match
+        elapsed_values.append(int(match.group(1)))
+    assert len(elapsed_values) >= 2
+    assert all(later > earlier for earlier, later in pairwise(elapsed_values))
+
+
+def test_unbounded_capacity_grace_keeps_throttled_worker_failure(monkeypatch):
+    import itertools
+
+    from flash.providers.runpod import api as runpod_api
+    from flash.providers.runpod import jobs
+
+    statuses = iter(
+        [
+            *itertools.repeat({"status": "IN_QUEUE"}, 8),
+            {"status": "FAILED", "error": "throttled timer was skipped"},
+        ]
+    )
+    monkeypatch.setattr(runpod_api, "job_status", lambda eid, jid, **_kw: next(statuses))
+    monkeypatch.setattr(
+        runpod_api,
+        "endpoint_health_for_fingerprint",
+        lambda eid, _fingerprint, **_kw: {"workers": {"throttled": 1}},
+    )
+    monkeypatch.setattr(jobs.time, "sleep", lambda s: None)
+    clock = itertools.count(start=0, step=100.0)
+    monkeypatch.setattr(jobs.time, "time", lambda: next(clock))
+    logs = io.StringIO()
+
+    res = jobs.poll_job(
+        _runpod_handle(jobs),
+        log=logs,
+        interval_s=0,
+        heartbeat_reader=lambda **_kw: None,
+        setup_grace_s=100000.0,
+        unhealthy_grace_s=100000.0,
+        throttled_grace_s=300.0,
+        queue_grace_s=float("inf"),
     )
 
+    assert res.failure == "no_capacity"
+    assert "worker stuck THROTTLED" in res.detail
+    capacity_lines = [line for line in logs.getvalue().splitlines() if "capacity grace" in line]
+    assert capacity_lines
+    assert all("of unbounded capacity grace" in line for line in capacity_lines)
+
+
+def test_four_card_last_gpu_log_names_only_scaled_capacity_grace(monkeypatch):
+    from flash.providers.runpod import jobs
+
+    logs = io.StringIO()
+    grace = jobs.stall_kwargs(on_last_gpu=True, gpu_count=4)["queue_grace_s"]
+    res = _queued_forever(
+        monkeypatch,
+        lambda eid, _fp, **_kw: {"workers": {}},
+        queue_grace_s=grace,
+        log=logs,
+    )
+
+    assert res.failure == "no_capacity"
+    capacity_output = "\n".join(
+        line for line in logs.getvalue().splitlines() if "capacity grace" in line
+    )
+    assert "of 3600s capacity grace" in capacity_output
+    assert "pinned" not in capacity_output.lower()
+    assert "escalation" not in capacity_output.lower()
+    assert "retry" not in capacity_output.lower()
+
+
+@pytest.mark.parametrize(
+    "workers",
+    [
+        {"initializing": 1},
+        {"ready": 1, "unhealthy": 1},
+    ],
+)
+def test_allocated_worker_log_suppresses_capacity_grace(monkeypatch, workers):
+    logs = io.StringIO()
+    res = _queued_forever(
+        monkeypatch,
+        lambda eid, _fp, **_kw: {"workers": workers},
+        log=logs,
+    )
+
+    assert res.failure == "stalled"
+    queued_lines = [line for line in logs.getvalue().splitlines() if "queued; workers:" in line]
+    assert queued_lines
+    assert all("capacity grace" not in line for line in queued_lines)
+
+
+def test_unhealthy_worker_log_suppresses_capacity_grace(monkeypatch):
+    logs = io.StringIO()
+    res = _queued_forever(
+        monkeypatch,
+        lambda eid, _fp, **_kw: {"workers": {"unhealthy": 1}},
+        log=logs,
+    )
+
+    assert not res.ok
+    assert res.failure == "stalled"
+    assert "worker stuck unhealthy" in res.detail
+    queued_lines = [line for line in logs.getvalue().splitlines() if "queued; workers:" in line]
+    assert queued_lines
+    assert all("capacity grace" not in line for line in queued_lines)
+
+
+@pytest.mark.parametrize("granted_workers", [{"initializing": 1}, {"ready": 1}])
+def test_empty_health_after_worker_grant_suppresses_capacity_grace(monkeypatch, granted_workers):
+    logs = io.StringIO()
+    probes = {"count": 0}
+
+    def health(eid, _fp, **_kw):
+        probes["count"] += 1
+        return {"workers": granted_workers if probes["count"] == 1 else {}}
+
+    res = _queued_forever(monkeypatch, health, queue_grace_s=5000.0, log=logs)
+
+    assert probes["count"] >= 2
+    assert not res.ok
+    assert res.failure == "stalled"
+    assert "setup (pre-training)" in res.detail
+    queued_lines = [line for line in logs.getvalue().splitlines() if "queued; workers:" in line]
+    assert queued_lines
+    assert any("workers: {}" in line for line in queued_lines)
+    assert all("capacity grace" not in line for line in queued_lines)
+
 
 def test_slow_image_pull_is_not_reported_as_missing_capacity(monkeypatch):
     # IN_QUEUE includes worker cold start. once health shows a worker, the setup grace must govern;

{freesolo_flash_dev-1.2.23 → freesolo_flash_dev-1.2.28}/uv.lock

@@ -1052,7 +1052,7 @@ wheels = [
 
 [[package]]
 name = "freesolo-flash"
-version = "1.2.23"
+version = "1.2.28"
 source = { editable = "." }
 
 [package.optional-dependencies]