freesolo-flash-dev 1.2.22__tar.gz → 1.2.28__tar.gz

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.22
+Version: 1.2.28
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/flash/cli/ui/heartbeat.py

@@ -332,6 +332,60 @@ def _superseded_hint(
     )
 
 
+def _finite_positive(value: object) -> float | None:
+    """return a finite positive number from an untrusted heartbeat field."""
+    if isinstance(value, bool) or not isinstance(value, (int, float)):
+        return None
+    try:
+        number = float(value)
+    except OverflowError:
+        return None
+    return number if number > 0 and math.isfinite(number) else None
+
+
+def _humanize_duration(seconds: float) -> str:
+    if seconds < 90:
+        return f"{seconds:.0f}s"
+    if seconds < 5400:
+        return f"{seconds / 60:.0f}m"
+    return f"{seconds / 3600:.1f}h"
+
+
+def _humanize_step_duration(seconds: float) -> str:
+    if seconds < 10:
+        return f"{seconds:.1f}s"
+    if seconds < 600:
+        return f"{seconds:.0f}s"
+    return f"{seconds / 60:.1f}m"
+
+
+def _step_timing_pairs(
+    heartbeat: dict,
+    *,
+    running: bool,
+    current_attempt: bool,
+) -> list[tuple[str, str]]:
+    """render measured RL pace only for the live running attempt."""
+    if not running or not current_attempt or heartbeat.get("stage") != "rl_step":
+        return []
+    step_duration_s = _finite_positive(heartbeat.get("step_duration_s"))
+    if step_duration_s is None:
+        return []
+
+    projected_remaining_s = _finite_positive(heartbeat.get("projected_remaining_s"))
+    pairs = [("median pace", f"{_humanize_step_duration(step_duration_s)}/step")]
+    if projected_remaining_s is not None:
+        pairs.append(("mean ETA", f"~{_humanize_duration(projected_remaining_s)} left"))
+    if heartbeat.get("wall_deadline_at_risk") is True and projected_remaining_s is not None:
+        pairs.append(
+            (
+                "warning",
+                "mean-based remaining-time projection exceeds the run's wall time left",
+            )
+        )
+    return pairs
+
+
 def _heartbeat_pairs(obj: dict) -> list[tuple[str, str]]:
     """Worker heartbeat rows for the status panel: stage, step, age, and a quiet-is-normal hint."""
     hb = obj.get("last_heartbeat")
@@ -363,6 +417,13 @@ def _heartbeat_pairs(obj: dict) -> list[tuple[str, str]]:
     if datacenter:
         label = "datacenter" if from_current_attempt else "datacenter (previous attempt)"
         pairs.append((label, str(datacenter)[:64]))
+    pairs.extend(
+        _step_timing_pairs(
+            hb,
+            running=running,
+            current_attempt=from_current_attempt,
+        )
+    )
     stale_step = _stale_step_hint(
         hb,
         heartbeat_age_seconds,

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/flash/engine/worker/io/heartbeat.py

@@ -139,7 +139,13 @@ def _console_heartbeat_snapshot(payload: dict, payload_committed: bool = True) -
 
 
 def heartbeat(
-    stage: str, *, liveness: bool = False, force: bool = False, initial: bool = False, **kw
+    stage: str,
+    *,
+    liveness: bool = False,
+    force: bool = False,
+    initial: bool = False,
+    first_timing: bool = False,
+    **kw,
 ):
     global _HB_CLAIM_SEQ
     genuine_progress = not liveness
@@ -204,18 +210,19 @@ def heartbeat(
             # requires a step advance, but a sample-bearing payload may match the committed step because
             # the liveness daemon can commit that step first without the samples. the per-force floor still
             # coalesces fast bursts to protect the hf commit cap, while an unrelated liveness commit does
-            # not arm the floor and therefore cannot suppress the first sample-bearing payload.
+            # not arm the floor and therefore cannot suppress the first sample-bearing payload. the first
+            # timing payload gets one dedicated floor bypass because the forced metrics-only payload just
+            # before it armed that clock; its caller retries this flag until the timing upload succeeds.
             if force and not upload_due:
                 fstep = kw.get("step")
                 has_samples = bool(kw.get("sampled_completions"))
+                first_timing_due = first_timing and "step_duration_s" in kw
                 force_step_due = isinstance(fstep, (int, float)) and (
                     fstep > _w._HB_LAST_COMMITTED_STEP
                     or (has_samples and fstep == _w._HB_LAST_COMMITTED_STEP)
                 )
-                if (
-                    force_step_due
-                    and (now - _w._HB_LAST_FORCED_UPLOAD) >= _w._HB_FORCE_MIN_INTERVAL_S
-                ):
+                force_floor_due = (now - _w._HB_LAST_FORCED_UPLOAD) >= _w._HB_FORCE_MIN_INTERVAL_S
+                if force_step_due and (first_timing_due or force_floor_due):
                     upload_due = True
         # the initial training snapshot must land before the shared throttle can hide it.
         if initial:

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/flash/engine/worker/opd_train_runner.py

@@ -10,7 +10,6 @@ import os
 import random
 import shutil
 import time
-from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
 
@@ -19,6 +18,14 @@ from flash.engine.worker import opd_train as _opd_train
 from flash.engine.worker.train.opd.reporting import (
     _build_train_note_sections as _build_train_note_sections,
 )
+from flash.engine.worker.train.opd.state import (
+    _ChildCallbacks,
+    _ChildResult,
+    _OpdRequest,
+    _PromptState,
+    _RuntimeState,
+    _WorkloadState,
+)
 from flash.engine.worker.verl.child_io import (
     LORA_ROLLOUT_GUARD_SHIM,
     render_lora_rollout_guard_fragment,
@@ -29,92 +36,6 @@ from flash.engine.worker.verl.parallelism import (
 )
 
 
-@dataclass(frozen=True)
-class _OpdRequest:
-    spec: Any
-    env: Any
-    multi_turn: bool
-    max_turns: int
-    knobs: Any
-    model_id: str
-    model_revision: str
-    structured_outputs: Any = None
-    model_vocab_size: int | None = None
-
-
-@dataclass(frozen=True)
-class _PromptState:
-    teacher: Any
-    tokenizer: Any
-    thinking_prefill: str
-    max_model_len: int
-    prompt_budget: int
-    prompts: list[Any]
-    dropped_long: int
-
-
-@dataclass(frozen=True)
-class _WorkloadState:
-    prompts_per_step: int
-    update_horizon: int
-    prompt_pool_fingerprint: str
-    workdir: str
-    shim_dir: str
-    local_dir: str
-    export_root: str
-    mutation_failure_path: str
-    score_delivery_failure_path: str
-    abandonment_failure_path: str
-    resample_failure_path: str
-    cycle_commit_failure_path: str
-    train_file: str
-    val_file: str
-    lora_rank: int
-    lora_alpha: int
-    target_modules: Any
-    warmstart_adapter: str | None
-
-
-@dataclass(frozen=True)
-class _RuntimeState:
-    python_bin: str
-    model_path: str
-    gpu_count: int
-    save_freq: int
-    loggers: list[str]
-    project_name: str
-    experiment_name: str
-    gdn_reset_arch: str | None
-    entry_path: str
-    reward_path: str
-    resume_step: int
-    resume_state: dict[str, Any] | None
-    bridge: Any
-
-
-@dataclass(frozen=True)
-class _ChildCallbacks:
-    on_line: Any
-    on_step: Any
-    child_heartbeat: Any
-    liveness_fields: Any
-    progress: dict[str, Any]
-    wandb_link: dict[str, str | None]
-    child_tail: Any
-
-
-@dataclass(frozen=True)
-class _ChildResult:
-    final_accounting: dict[str, Any]
-    actor_dir: str
-    final_step: int
-    train_wall: float
-    peak_gpu_gb: float
-    train_started_at: float
-    wandb_url: str | None
-    wandb_id: str | None
-
-
 def _prepare_request(spec: Any) -> _OpdRequest:
     env = _opd_train._w.require_active_env()
     if getattr(env, "is_tool_env", False):

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/flash/engine/worker/rl_train.py

@@ -387,8 +387,11 @@ def run_rl_train():
             return state.progress["step"]
 
         def _reward_observability() -> dict:
-            """return reward metrics and sampled completions for one heartbeat."""
-            return reward_runtime.observability.heartbeat_fields()
+            """return reward observability and measured pace for one heartbeat."""
+            return {
+                **reward_runtime.observability.heartbeat_fields(),
+                **_step_timing_fields(inp, state),
+            }
 
         with liveness_heartbeat(
             "rl_step",
@@ -491,6 +494,7 @@ from flash.engine.worker.rl_train_runner import (  # noqa: E402,F401
     _RewardRuntime,
     _start_resume_uploader,
     _start_reward_runtime,
+    _step_timing_fields,
     _StepMetricState,
     _validate_rl_child,
     _write_rl_shim,

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/flash/engine/worker/rl_train_runner.py

@@ -39,6 +39,7 @@ from flash.engine.worker.io.heartbeat import (
 )
 from flash.engine.worker.perf import gpu_diagnostics, wait_for_gpu
 from flash.engine.worker.runtime.pkg_proxy import W as _w
+from flash.engine.worker.train.core.step_timing import StepTiming
 from flash.engine.worker.train.rl.multi_turn import (
     MultiTurnBridge,
     copy_multi_turn_child_modules,
@@ -80,9 +81,10 @@ class _StepMetricState:
     loss_curve: list[float] = field(default_factory=list)
     adv_spread_history: list[float] = field(default_factory=list)
     last_dump_step: list[int] = field(default_factory=lambda: [-1])
-    step_line_times: list[float] = field(default_factory=list)
     metrics_last: list[dict] = field(default_factory=list)
+    step_timing: StepTiming = field(default_factory=StepTiming)
     sent_first_metrics: bool = False
+    sent_first_timing: bool = False
 
 
 @dataclass
@@ -455,16 +457,23 @@ def _build_rl_child_env(inp, files, loggers, reward_url):
     return env_for_verl
 
 
+def _step_timing_fields(inp, state: _StepMetricState) -> dict:
+    return state.step_timing.heartbeat_fields(
+        current_step=state.progress["step"],
+        total_steps=int(inp["steps"]),
+        remaining_wall_s=_w._remaining_worker_wall_seconds(),
+    )
+
+
 def _ingest_step_metrics(
     line: str,
     inp,
     state: _StepMetricState,
     _reward_observability: Callable[[], dict],
 ) -> None:
-    sent_first_metrics = state.sent_first_metrics
     step_metrics = parse_verl_step_metrics(line)
     if step_metrics is not None:
-        state.step_line_times.append(time.time())
+        state.step_timing.record_duration(parse_verl_metric(line, "timing_s/step"))
         # a run constant rather than a verl metric, so it is stamped here from
         # the resolved run config.
         step_metrics["max_completion_tokens"] = inp["max_completion"]
@@ -472,19 +481,25 @@ def _ingest_step_metrics(
         # the worker's error path reads this global, so a run that dies mid-training
         # still reports the steps it did complete (worker/__init__.py:_err_metrics).
         LATEST_GRPO_METRICS_LAST[:] = state.metrics_last
-        # rl_train_start arms a 900s throttle, so force the first backlog through like
-        # heartbeat.py force_first_samples and retry until committed. later emissions remain
-        # throttled to protect the hf commit cap.
-        if not sent_first_metrics:
-            sent_first_metrics = _w.heartbeat(
+        heartbeat_fields = _reward_observability()
+        has_step_timing = "step_duration_s" in heartbeat_fields
+        # rl_train_start arms a 900s throttle, so force until both the first backlog and the first
+        # usable timing payload commit. the backlog commit also arms the force floor, so mark the first
+        # timing attempt for the wrapper's dedicated floor bypass until that upload succeeds.
+        if not state.sent_first_metrics or (has_step_timing and not state.sent_first_timing):
+            heartbeat_committed = _w.heartbeat(
                 "rl_step",
                 force=True,
+                first_timing=has_step_timing and not state.sent_first_timing,
                 step=step_metrics["step"],
                 metrics_last=list(state.metrics_last),
-                **_reward_observability(),
+                **heartbeat_fields,
                 gpu=gpu_diagnostics(include_torch=False),
             )
-            state.sent_first_metrics = sent_first_metrics
+            if heartbeat_committed:
+                state.sent_first_metrics = True
+                if has_step_timing:
+                    state.sent_first_timing = True
         # per-step series for train_meta observability parity. these live on the same
         # line as everything else: verl's only console metric sink is LocalLogger,
         # which always prints "step:N - ..." (verl/utils/logger/aggregate_logger.py),

freesolo_flash_dev-1.2.28/flash/engine/worker/train/core/step_timing.py

@@ -0,0 +1,62 @@
+"""RL step pace derived from Verl's native per-step duration metric."""
+
+from __future__ import annotations
+
+import math
+import statistics
+from dataclasses import dataclass, field
+
+_RETAINED_DURATION_SAMPLES = 64
+
+
+@dataclass
+class StepTiming:
+    """Retain recent steady-state RL durations for heartbeat projections."""
+
+    _warmup_seen: bool = False
+    _durations: list[float] = field(default_factory=list)
+
+    def record_duration(self, duration: float | None) -> None:
+        """record one positive native duration from a parsed Verl training step."""
+        if duration is None or duration <= 0:
+            return
+        if not self._warmup_seen:
+            self._warmup_seen = True
+            return
+        self._durations = [*self._durations, duration][-_RETAINED_DURATION_SAMPLES:]
+
+    def heartbeat_fields(
+        self,
+        *,
+        current_step: int,
+        total_steps: int,
+        remaining_wall_s: float | None,
+    ) -> dict[str, float | bool]:
+        """Return measured pace and a best-effort remaining-training projection."""
+        durations = self._durations
+        if not durations:
+            return {}
+
+        remaining_steps = max(0, int(total_steps) - int(current_step))
+        try:
+            median_duration_s = statistics.median(durations)
+            mean_duration_s = statistics.fmean(durations)
+            projected_remaining_s = mean_duration_s * remaining_steps
+        except OverflowError:
+            return {}
+        if not all(
+            math.isfinite(value)
+            for value in (median_duration_s, mean_duration_s, projected_remaining_s)
+        ):
+            return {}
+        fields: dict[str, float | bool] = {
+            "step_duration_s": median_duration_s,
+            "projected_remaining_s": projected_remaining_s,
+        }
+        if (
+            remaining_wall_s is not None
+            and remaining_wall_s >= 0
+            and projected_remaining_s > remaining_wall_s
+        ):
+            fields["wall_deadline_at_risk"] = True
+        return fields

freesolo_flash_dev-1.2.28/flash/engine/worker/train/opd/state.py

@@ -0,0 +1,92 @@
+"""immutable state carriers for OPD parent-runner stages."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+
+@dataclass(frozen=True)
+class _OpdRequest:
+    spec: Any
+    env: Any
+    multi_turn: bool
+    max_turns: int
+    knobs: Any
+    model_id: str
+    model_revision: str
+    structured_outputs: Any = None
+    model_vocab_size: int | None = None
+
+
+@dataclass(frozen=True)
+class _PromptState:
+    teacher: Any
+    tokenizer: Any
+    thinking_prefill: str
+    max_model_len: int
+    prompt_budget: int
+    prompts: list[Any]
+    dropped_long: int
+
+
+@dataclass(frozen=True)
+class _WorkloadState:
+    prompts_per_step: int
+    update_horizon: int
+    prompt_pool_fingerprint: str
+    workdir: str
+    shim_dir: str
+    local_dir: str
+    export_root: str
+    mutation_failure_path: str
+    score_delivery_failure_path: str
+    abandonment_failure_path: str
+    resample_failure_path: str
+    cycle_commit_failure_path: str
+    train_file: str
+    val_file: str
+    lora_rank: int
+    lora_alpha: int
+    target_modules: Any
+    warmstart_adapter: str | None
+
+
+@dataclass(frozen=True)
+class _RuntimeState:
+    python_bin: str
+    model_path: str
+    gpu_count: int
+    save_freq: int
+    loggers: list[str]
+    project_name: str
+    experiment_name: str
+    gdn_reset_arch: str | None
+    entry_path: str
+    reward_path: str
+    resume_step: int
+    resume_state: dict[str, Any] | None
+    bridge: Any
+
+
+@dataclass(frozen=True)
+class _ChildCallbacks:
+    on_line: Any
+    on_step: Any
+    child_heartbeat: Any
+    liveness_fields: Any
+    progress: dict[str, Any]
+    wandb_link: dict[str, str | None]
+    child_tail: Any
+
+
+@dataclass(frozen=True)
+class _ChildResult:
+    final_accounting: dict[str, Any]
+    actor_dir: str
+    final_step: int
+    train_wall: float
+    peak_gpu_gb: float
+    train_started_at: float
+    wandb_url: str | None
+    wandb_id: str | None

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/flash/providers/runpod/job_execution.py

@@ -513,7 +513,21 @@ def _classify_queue_state(
             # between unhealthy and empty cannot make a broken box look like starvation.
             state.ever_saw_worker = True
         if any(workers.get(k) for k in ("throttled", "unhealthy", "initializing")) or not usable:
-            context.say(f"queued; workers: {workers}")
+            message = f"queued; workers: {workers}"
+            if (
+                not usable
+                and not recovering
+                and not state.ever_saw_worker
+                and state.queued_timer.since is not None
+            ):
+                elapsed_s = max(0, int(now - state.queued_timer.since))
+                budget = (
+                    f"{context.queue_grace_s:g}s"
+                    if _jobs.math.isfinite(context.queue_grace_s)
+                    else "unbounded"
+                )
+                message += f"; waited {elapsed_s}s of {budget} capacity grace"
+            context.say(message)
         if state.unhealthy_timer.expired(
             workers.get("unhealthy") and not usable and not recovering,
             now,

{freesolo_flash_dev-1.2.22 → freesolo_flash_dev-1.2.28}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.22"
+version = "1.2.28"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.22"
+version = "1.2.28"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]