freesolo-flash-dev 1.2.100__tar.gz → 1.2.102__tar.gz

{freesolo_flash_dev-1.2.100 → freesolo_flash_dev-1.2.102}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.100
+Version: 1.2.102
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

freesolo_flash_dev-1.2.102/flash/cli/commands/worker_output.py

@@ -0,0 +1,139 @@
+"""Printing the worker-uploaded text artifacts that `flash runs log` appends to a run log.
+
+These sections are NOT part of the chronological log. `cmd_log` prints the control-plane log first
+and appends these afterwards, so they always land at the tail no matter how old they are -- and the
+plane returns the highest UPLOADED attempt, which during a retry is the attempt that just died, not
+the one now running. Read without a label, the last thing on screen is a traceback from a worker
+that has already been replaced.
+
+So the job here is provenance: say which attempt produced each section, and say when that attempt is
+not the live one. The old output is kept rather than hidden because it carries the failure that
+explains why a retry exists at all.
+"""
+
+from __future__ import annotations
+
+import re
+from typing import TYPE_CHECKING
+
+from flash.cli.ui import render
+from flash.client import ClientError
+
+if TYPE_CHECKING:
+    from collections.abc import Mapping
+
+    from flash.client import ApiClient
+
+
+# stands in for the live attempt during the teardown window, when the plane has PROVEN there is no
+# live worker (an explicitly null ``remote``). distinct from ``None``, which means UNKNOWN.
+_NO_LIVE_WORKER = "no-live-worker"
+
+
+def _artifact_attempt(name: str) -> int | None:
+    """Return the bounded attempt encoded by a worker artifact name."""
+    from flash.providers._lifecycle.poll import _attempt_int
+
+    match = re.search(r"_attempt(\d+)\.txt$", name)
+    return _attempt_int(int(match.group(1))) if match else None
+
+
+def _worker_section_name(name: str, current_attempt: int | str | None) -> str:
+    """Label an attempt-scoped artifact and identify superseded output."""
+    attempt = _artifact_attempt(name)
+    if attempt is None:
+        return name
+    if current_attempt == _NO_LIVE_WORKER:
+        return f"{name} (attempt={attempt}, worker torn down; no live attempt)"
+    if current_attempt is None:
+        return f"{name} (attempt={attempt})"
+    if attempt != current_attempt:
+        return f"{name} (attempt={attempt}, previous attempt; current attempt={current_attempt})"
+    return f"{name} (attempt={attempt}, current attempt)"
+
+
+_HEARTBEAT_LINE_RE = re.compile(r"(?m)^(HEARTBEAT )(?!\[superseded)")
+
+
+def _mark_superseded_heartbeats(text: str, attempt: int, current_attempt: int | str) -> str:
+    """Tag every HEARTBEAT line in a dead attempt's console dump.
+
+    The section heading already says which attempt produced this text, but a heading is only read
+    by a human looking at the whole screen. The monitoring idiom is
+    ``runs log | grep HEARTBEAT | tail -1``, which sees ONE line and no heading -- and because the
+    plane appends the highest uploaded attempt after the chronological log, that one line is the
+    dead attempt's last heartbeat.
+
+    That is not merely stale, it inverts the reading: the reporter's monitor showed
+    ``step 0, 0 completions, device H200`` for twenty minutes while the run was in fact live on
+    B200, because H200 was the card that had already OOMed and been torn down.
+
+    Tagging the line itself means the provenance survives the pipe. A consumer that filters on the
+    tag gets only live heartbeats; one that does not at least sees the attempt in the line it
+    printed rather than silently trusting a dead worker's last words.
+    """
+    current = (
+        "worker torn down"
+        if current_attempt == _NO_LIVE_WORKER
+        else f"current attempt={current_attempt}"
+    )
+    return _HEARTBEAT_LINE_RE.sub(f"\\1[superseded attempt={attempt}; {current}] ", text)
+
+
+def _worker_sections(client: ApiClient, run_id: str) -> dict[str, str]:
+    """Fetch only worker artifacts with printable text."""
+    return {name: text for name, text in (client.get_worker_output(run_id) or {}).items() if text}
+
+
+def _snapshot_live_attempt(client: ApiClient, run_id: str) -> int | str | None:
+    """The live attempt, ``_NO_LIVE_WORKER`` during teardown, or ``None`` when unknown.
+
+    ``live_attempt`` answers ``None`` for two opposite situations: an explicitly null ``remote``,
+    which is PROOF the attached heartbeat belongs to a worker that is already gone, and a run whose
+    shape carries no attempt at all. Collapsing both to ``None`` disabled heartbeat tagging exactly
+    during the retry window the tagging exists for -- the dead attempt's heartbeat went back to
+    reaching ``grep HEARTBEAT | tail -1`` unmarked while replacement capacity was still being
+    acquired. Keep the two apart so teardown can be labelled as what it is.
+
+    A lookup failure stays ``None``: not knowing the live attempt must not make this claim there
+    is no live worker.
+    """
+    try:
+        run = client.get_run(run_id) or {}
+    except ClientError:
+        return None
+    attempt = render.live_attempt(run)
+    if attempt is None and run.get("remote", False) is None:
+        return _NO_LIVE_WORKER
+    return attempt
+
+
+def _print_worker_output(
+    sections: Mapping[str, str],
+    *,
+    printed_any: bool = False,
+    current_attempt: int | str | None = None,
+) -> bool:
+    """Print worker artifacts under headings naming the attempt that produced them."""
+    for name, text in sections.items():
+        # label provenance from the filename rather than letting the final section, which is merely
+        # the last one appended, read as the live attempt's output.
+        section_name = _worker_section_name(name, current_attempt)
+        # the heading is invisible to `grep HEARTBEAT | tail -1`, so a superseded attempt's
+        # heartbeats carry their own provenance into the pipe. `_NO_LIVE_WORKER` tags every
+        # attempt: during teardown the plane has PROVEN no worker is live, so every heartbeat on
+        # screen is a dead one and none of them may reach a monitor unmarked.
+        attempt = _artifact_attempt(name)
+        supersedes = current_attempt is not None and (
+            current_attempt == _NO_LIVE_WORKER or attempt != current_attempt
+        )
+        if attempt is not None and supersedes:
+            text = _mark_superseded_heartbeats(text, attempt, current_attempt)
+        sep = "\n" if printed_any else ""
+        if render.styled():
+            print(f"{sep}{render.log_section(section_name)}")
+        else:
+            print(f"{sep}----- {section_name} -----")
+        print(text, end="" if text.endswith("\n") else "\n")
+        printed_any = True
+    return printed_any

{freesolo_flash_dev-1.2.100 → freesolo_flash_dev-1.2.102}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.100"
+version = "1.2.102"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.100"
+version = "1.2.102"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.2.100 → freesolo_flash_dev-1.2.102}/tests/test_cli_commands.py

@@ -1606,7 +1606,10 @@ def test_log_labels_previous_attempt_artifacts_after_the_live_attempt_log(
         in out
     )
     assert out.index(live_line) < out.index(previous_header)
-    assert 'HEARTBEAT {"stage":"rl_step","step":0,"attempt":0' in out
+    # the dead attempt's heartbeat is still printed -- it explains why a retry exists -- but it
+    # carries its provenance inline, because a section heading does not survive
+    # `grep HEARTBEAT | tail -1`.
+    assert 'HEARTBEAT [superseded attempt=0; current attempt=1] {"stage":"rl_step"' in out
 
 
 def test_log_still_prints_artifacts_when_the_attempt_lookup_fails(

{freesolo_flash_dev-1.2.100 → freesolo_flash_dev-1.2.102}/tests/test_cli_commands_coverage.py

@@ -365,3 +365,126 @@ def test_chat_keeps_stdout_empty_when_a_styled_stream_carries_no_text(monkeypatc
     assert result == 1
     assert captured.out == ""
     assert "no response text from flash-1" in captured.err
+
+
+def test_superseded_attempt_heartbeats_are_tagged_in_the_log_stream(capsys) -> None:
+    """`runs log | grep HEARTBEAT | tail -1` must not silently return a dead attempt's heartbeat.
+
+    The plane appends the highest UPLOADED attempt after the chronological log, so during a retry
+    the last heartbeat on screen belongs to the attempt that just died. The section heading says
+    so, but a heading does not survive a pipe: the reporter's monitor read
+    ``step 0, 0 completions, device H200`` for twenty minutes while the run was live on B200,
+    because H200 was the card that had already OOMed and been torn down.
+    """
+    commands._print_worker_output(
+        {
+            "console_rl_attempt0.txt": (
+                'HEARTBEAT {"stage":"rl_step","step":76,"attempt":0,"device_name":"H200"}\n'
+                "Traceback (most recent call last):\n"
+            )
+        },
+        printed_any=True,
+        current_attempt=1,
+    )
+
+    out = capsys.readouterr().out
+    lines = [line for line in out.splitlines() if "HEARTBEAT" in line]
+    assert lines, "the dead attempt's console dump must still be printed"
+    # the stale line carries its own provenance, so it cannot be mistaken for the live attempt
+    assert "[superseded attempt=0; current attempt=1]" in lines[-1]
+    # and the failure dump itself is preserved -- it is why the retry exists
+    assert "Traceback (most recent call last):" in out
+
+
+def test_filtering_superseded_heartbeats_leaves_the_live_attempt_last(capsys) -> None:
+    """A consumer that drops tagged lines gets the LIVE attempt's heartbeat, not the dead one."""
+    print('HEARTBEAT {"stage":"rl_step","step":1,"attempt":1,"device_name":"B200"}')
+    commands._print_worker_output(
+        {
+            "console_rl_attempt0.txt": (
+                'HEARTBEAT {"stage":"rl_step","step":76,"attempt":0,"device_name":"H200"}\n'
+            )
+        },
+        printed_any=True,
+        current_attempt=1,
+    )
+
+    heartbeats = [line for line in capsys.readouterr().out.splitlines() if "HEARTBEAT" in line]
+    live = [line for line in heartbeats if "[superseded" not in line]
+    assert live[-1].endswith('{"stage":"rl_step","step":1,"attempt":1,"device_name":"B200"}')
+
+
+def test_current_attempt_heartbeats_are_left_untouched(capsys) -> None:
+    """Only a SUPERSEDED attempt is tagged; the live attempt's own artifact must read normally."""
+    commands._print_worker_output(
+        {"console_rl_attempt1.txt": 'HEARTBEAT {"stage":"rl_step","step":4,"attempt":1}\n'},
+        printed_any=True,
+        current_attempt=1,
+    )
+
+    out = capsys.readouterr().out
+    assert "[superseded" not in out
+
+
+def test_teardown_window_still_tags_the_dead_workers_heartbeat(capsys) -> None:
+    """An explicitly null ``remote`` is PROOF no worker is live, so every heartbeat is a dead one.
+
+    ``live_attempt`` answers ``None`` both for "the worker is torn down" and for "no attempt is
+    known", and collapsing the two disabled tagging exactly during the retry window the tagging
+    exists for -- the dead attempt's heartbeat reached ``grep HEARTBEAT | tail -1`` unmarked while
+    replacement capacity was still being acquired.
+    """
+    from flash.cli.commands.worker_output import _NO_LIVE_WORKER
+
+    commands._print_worker_output(
+        {
+            "console_rl_attempt0.txt": (
+                'HEARTBEAT {"stage":"rl_step","step":76,"attempt":0,"device_name":"H200"}\n'
+            )
+        },
+        printed_any=True,
+        current_attempt=_NO_LIVE_WORKER,
+    )
+
+    out = capsys.readouterr().out
+    lines = [line for line in out.splitlines() if "HEARTBEAT" in line]
+    assert lines, "the dead attempt's console dump must still be printed"
+    assert "[superseded attempt=0; worker torn down]" in lines[-1], (
+        f"a torn-down worker's heartbeat reached the pipe unmarked: {lines[-1]}"
+    )
+    assert "worker torn down; no live attempt" in out, "the heading must say so too"
+
+
+def test_snapshot_distinguishes_teardown_from_an_unknown_attempt() -> None:
+    """``remote: null`` means torn down; a failed lookup means unknown. They are not the same."""
+    from flash.cli.commands.worker_output import _NO_LIVE_WORKER, _snapshot_live_attempt
+    from flash.client import ClientError
+
+    class _Run:
+        def __init__(self, payload):
+            self._payload = payload
+
+        def get_run(self, _run_id):
+            if isinstance(self._payload, Exception):
+                raise self._payload
+            return self._payload
+
+    torn_down = {"state": "running", "remote": None, "last_heartbeat": {"attempt": 0}}
+    assert _snapshot_live_attempt(_Run(torn_down), "flash-1") == _NO_LIVE_WORKER
+    live = {"state": "running", "remote": {"attempt": 1}, "last_heartbeat": {"attempt": 0}}
+    assert _snapshot_live_attempt(_Run(live), "flash-1") == 1
+    # a lookup failure must NOT claim there is no live worker
+    assert _snapshot_live_attempt(_Run(ClientError("boom")), "flash-1") is None
+    # nor must a payload that simply carries no remote key
+    no_remote = {"state": "running", "last_heartbeat": {"attempt": 0}}
+    assert _snapshot_live_attempt(_Run(no_remote), "flash-1") == 0
+
+
+def test_heartbeat_tagging_is_idempotent() -> None:
+    """Re-printing an already-tagged dump must not stack a second marker onto the line."""
+    from flash.cli.commands.worker_output import _mark_superseded_heartbeats
+
+    once = _mark_superseded_heartbeats('HEARTBEAT {"step":1}\n', 0, 1)
+    twice = _mark_superseded_heartbeats(once, 0, 1)
+    assert once == twice
+    assert once.count("[superseded") == 1

{freesolo_flash_dev-1.2.100 → freesolo_flash_dev-1.2.102}/uv.lock

@@ -1052,7 +1052,7 @@ wheels = [
 
 [[package]]
 name = "freesolo-flash"
-version = "1.2.100"
+version = "1.2.102"
 source = { editable = "." }
 
 [package.optional-dependencies]

freesolo_flash_dev-1.2.100/flash/cli/commands/worker_output.py

@@ -1,76 +0,0 @@
-"""Printing the worker-uploaded text artifacts that `flash runs log` appends to a run log.
-
-These sections are NOT part of the chronological log. `cmd_log` prints the control-plane log first
-and appends these afterwards, so they always land at the tail no matter how old they are -- and the
-plane returns the highest UPLOADED attempt, which during a retry is the attempt that just died, not
-the one now running. Read without a label, the last thing on screen is a traceback from a worker
-that has already been replaced.
-
-So the job here is provenance: say which attempt produced each section, and say when that attempt is
-not the live one. The old output is kept rather than hidden because it carries the failure that
-explains why a retry exists at all.
-"""
-
-from __future__ import annotations
-
-import re
-from typing import TYPE_CHECKING
-
-from flash.cli.ui import render
-from flash.client import ClientError
-
-if TYPE_CHECKING:
-    from collections.abc import Mapping
-
-    from flash.client import ApiClient
-
-
-def _artifact_attempt(name: str) -> int | None:
-    """Return the bounded attempt encoded by a worker artifact name."""
-    from flash.providers._lifecycle.poll import _attempt_int
-
-    match = re.search(r"_attempt(\d+)\.txt$", name)
-    return _attempt_int(int(match.group(1))) if match else None
-
-
-def _worker_section_name(name: str, current_attempt: int | None) -> str:
-    """Label an attempt-scoped artifact and identify superseded output."""
-    attempt = _artifact_attempt(name)
-    if attempt is None:
-        return name
-    if current_attempt is None:
-        return f"{name} (attempt={attempt})"
-    if attempt != current_attempt:
-        return f"{name} (attempt={attempt}, previous attempt; current attempt={current_attempt})"
-    return f"{name} (attempt={attempt}, current attempt)"
-
-
-def _worker_sections(client: ApiClient, run_id: str) -> dict[str, str]:
-    """Fetch only worker artifacts with printable text."""
-    return {name: text for name, text in (client.get_worker_output(run_id) or {}).items() if text}
-
-
-def _snapshot_live_attempt(client: ApiClient, run_id: str) -> int | None:
-    """Read the live attempt without letting a heading lookup hide diagnostic artifacts."""
-    try:
-        return render.live_attempt(client.get_run(run_id) or {})
-    except ClientError:
-        return None
-
-
-def _print_worker_output(
-    sections: Mapping[str, str], *, printed_any: bool = False, current_attempt: int | None = None
-) -> bool:
-    """Print worker artifacts under headings naming the attempt that produced them."""
-    for name, text in sections.items():
-        # label provenance from the filename rather than letting the final section, which is merely
-        # the last one appended, read as the live attempt's output.
-        section_name = _worker_section_name(name, current_attempt)
-        sep = "\n" if printed_any else ""
-        if render.styled():
-            print(f"{sep}{render.log_section(section_name)}")
-        else:
-            print(f"{sep}----- {section_name} -----")
-        print(text, end="" if text.endswith("\n") else "\n")
-        printed_any = True
-    return printed_any