freesolo-flash-dev 1.2.14__tar.gz → 1.2.22__tar.gz

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.14
+Version: 1.2.22
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/flash/cli/ui/heartbeat.py

@@ -13,6 +13,7 @@ CLI tests reach them.
 
 from __future__ import annotations
 
+import math
 import time
 
 from flash.cli.ui.render import _dim
@@ -114,10 +115,13 @@ _QUIET_HEARTBEAT_HINT = (
 # than on 900s -- the incident that motivated this reported 559s and 687s, squarely inside that
 # window, where a 900s gate would stay silent and leave only the dead-end quiet hint.
 _STALE_STEP_AFTER_S = _HB_QUIET_HINT_AFTER_S
-# only the stages the worker actually holds on the 900s upload throttle. opd_step is excluded: its
-# post-update ping is force=True, so it re-commits at the 60s forced floor and an opd_step older
-# than 900s means a long step, failed uploads, or a real stall -- not reporting lag.
-_TRAINING_STEP_STAGES = frozenset({"rl_step", "sft_step"})
+# progress_age_s is the age of latest known progress at payload build time. combine it with upload age
+# for a conservative current bound, and speak once that bound reaches the throttle without claiming
+# either health or a stall: newer progress may be uncommitted, or the worker may be silent.
+_UPLOAD_THROTTLE_S = 900.0
+# only the stages the worker actually holds on the 900s upload throttle. opd_step emits ordinary
+# post-step and liveness heartbeats, so its uploaded step can lag just like rl_step and sft_step.
+_TRAINING_STEP_STAGES = frozenset({"opd_step", "rl_step", "sft_step"})
 
 # the setup stages that hold a liveness thread on the 240s tight cadence (_HB_SETUP_LIVENESS_INTERVAL_S).
 # these are the ones where age is genuinely informative: the worker pings every 240s while alive, so
@@ -255,8 +259,6 @@ def _stale_step_hint(
     # throttled progress hides that the replacement has published nothing.
     if not current_attempt:
         return None
-    if heartbeat_age_seconds <= _STALE_STEP_AFTER_S:
-        return None
     if str(heartbeat.get("stage") or "") not in _TRAINING_STEP_STAGES:
         return None
     # step 0 is the cold, still-running first step: no optimizer update has landed, so there is no
@@ -266,9 +268,32 @@ def _stale_step_hint(
 
     if not is_training_heartbeat(heartbeat.get("stage"), heartbeat.get("step")):
         return None
-    # do not suggest `runs log -f`: it shows control-plane logs, and worker output arrives only after
-    # termination (flash/cli/commands/__init__.py cmd_log) on a 3600s upload interval. use heartbeat age against
-    # the 900s throttle, with w&b as the optional live signal.
+    progress_age_s = heartbeat.get("progress_age_s")
+    progress_age: float | None = None
+    if not isinstance(progress_age_s, bool):
+        try:
+            candidate = float(progress_age_s)
+        except (OverflowError, TypeError, ValueError):
+            pass
+        else:
+            if candidate >= 0 and math.isfinite(candidate):
+                progress_age = candidate
+    if progress_age is not None:
+        progress_age_bound_s = heartbeat_age_seconds + progress_age
+        if progress_age_bound_s >= _UPLOAD_THROTTLE_S:
+            return (
+                "the step above is the last one UPLOADED; "
+                f"the last known progress can be as old as {progress_age_bound_s:.1f}s. upload "
+                "throttling no longer explains the gap, but newer progress may be uncommitted and "
+                "a long step may still be running; this signal does not show recent progress"
+            )
+        # below the 900s hold, throttling still explains the visible lag. fall through to the exact
+        # pre-existing reading so new workers never lose guidance in the incident's 300-900s band.
+    if heartbeat_age_seconds <= _STALE_STEP_AFTER_S:
+        return None
+    # old workers do not publish progress_age_s. keep their existing throttle-only reading exactly so
+    # content-addressed in-flight runs do not change interpretation when the CLI upgrades. invalid
+    # progress ages use the same fallback because they provide no trustworthy worker-side reading.
     return (
         "the step above is the last one UPLOADED, not necessarily the one training is on; "
         "a throttled worker can hold it for many minutes while the trainer advances normally. "

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/flash/content/multimodal.py

@@ -697,8 +697,8 @@ def _read_json_records(path: Path) -> list[dict]:
     return value
 
 
-def preflight_validate_image_opd(spec) -> None:
-    """Validate statically discoverable image OPD datasets before GPU allocation."""
+def preflight_validate_image_opd(spec, *, scan_packaged_environment: bool = True) -> None:
+    """validate statically discoverable image opd datasets before gpu allocation."""
     if getattr(spec, "algorithm", "") != "opd":
         return
     environment = getattr(spec, "environment", None)
@@ -708,6 +708,8 @@ def preflight_validate_image_opd(spec) -> None:
         if not isinstance(records, list) or not all(isinstance(row, dict) for row in records):
             return
     else:
+        if not scan_packaged_environment:
+            return
         from flash.envs.loader import (
             _packaged_dataset_file,
             _resolve_environment_reference,

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/flash/cost/analytical.py

@@ -87,6 +87,49 @@ def step_floor_seconds(gpu: str, completions: int) -> float:
 # by matched multi-card measurements.
 STEP_FLOOR_SHARDED_FRACTION = 0.799
 
+# --- sft training-side startup block + per-step floor -------------------------------------------
+# the flops term alone quoted sft at 28.6x UNDER the realized train wall (geometric, 17 measured
+# runpod arms, 0/17 inside the 0.70-1.43x band). it is the same structural gap the rollout floor
+# above closed for grpo, and it is NOT the cold start in `setup_seconds`: `setup_seconds` and
+# `train_wall` are DISJOINT intervals in the worker (sft_train.py stamps setup before launching the
+# training subprocess, and train_wall wraps `run_verl_training` plus the upload drain), so verl
+# startup, model load, lora wrap and fsdp init sit INSIDE the billed train wall and were priced at
+# zero.
+#
+# the block scales with model size -- measured median implied block 82s at 0.8B against 248s at 4B --
+# so a size-blind constant cannot cover both: at the same in-band count it leaves a 3.75x worst arm
+# and log-sd 0.50, against 2.13x and 0.32 for the size-scaled form below.
+#
+# fitted over 17 arms (2 cards, 2 model sizes, step counts 2/32/64/128/256, including the replicate
+# groups) and selected by holding out an ENTIRE step-count class at a time, the same methodology
+# that chose the rollout floor: a random holdout cannot test whether a form extrapolates to a shape
+# it never saw. least squares lands at (58.3, 44.3, 0.948); the shipped triple is the rounded
+# neighbour that survives holdout best. scored: 28.596x -> 1.013x geometric, 0/17 -> 14/17 in band,
+# worst arm 1135x -> 2.13x, and it holds under a model holdout in both directions (0.8B 1.005x,
+# 4B 1.033x), so the size term is not an artifact of the size that dominates the sample.
+#
+# KNOWN RESIDUAL, and the reason this is not tuned further: the same config on a different runpod
+# HOST CLASS (nvidia driver build) spans 1.56x-1.88x, while replicates within one build agree to
+# 1.01x-1.03x. the quote cannot observe which build it will be handed, so roughly half the remaining
+# per-arm error is hardware the estimator has no input for. re-fitting below that resolution would
+# be fitting the confounder. evidence: /home/azureuser/benchmark/cost-calib-20260801/ISSUES.md.
+SFT_STARTUP_BASE_SECONDS = 60.0
+SFT_STARTUP_SECONDS_PER_PARAM_B = 40.0
+SFT_STEP_FLOOR_SECONDS = 0.8
+
+
+def sft_overhead_seconds(config: RunConfig, steps: int) -> float:
+    """Training-wall seconds an sft run pays outside its flops term.
+
+    A one-time startup block that grows with checkpoint size, plus a per-step floor. Neither is
+    divided by card count: the block is model load and framework init that every rank pays, and the
+    per-step floor is dominated by the same non-shardable publish/sync work as the rollout floor.
+    """
+    n = config.normalized()
+    params_b = total_params_b(n.model_id, n.model_revision)
+    block = SFT_STARTUP_BASE_SECONDS + SFT_STARTUP_SECONDS_PER_PARAM_B * params_b
+    return block + SFT_STEP_FLOOR_SECONDS * max(0, steps)
+
 
 def _step_floor_seconds_for(config: RunConfig, gpu: str) -> float:
     """Step-floor seconds ``config`` pays on ``gpu``; 0 for sft, which runs no rollout."""
@@ -814,12 +857,16 @@ def estimate_cost(
     # training GPU time, so it belongs in the (billed) train term, not setup. Required saves are also
     # synchronous fixed overhead; neither is divided by the number of cards.
     compile_s = compile_seconds(config, gpu)
-    raw_train = compile_s + config.steps * sps + required_save_s
+    # sft pays a startup block and per-step floor inside its BILLED train wall that the flops term
+    # does not model (see sft_overhead_seconds); grpo/opd carry the equivalent in their step floor.
+    sft_overhead_s = sft_overhead_seconds(config, config.steps) if config.method == "sft" else 0.0
+    raw_train = compile_s + config.steps * sps + required_save_s + sft_overhead_s
     if not config.is_grpo and config.train_tokens is not None:
         raw_train = (
             compile_s
             + sft_seconds_for_tokens(config, gpu, config.train_tokens) / speedup
             + required_save_s
+            + sft_overhead_s
         )
     sps = raw_train / config.steps
 

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/flash/engine/worker/io/heartbeat.py

@@ -142,20 +142,20 @@ def heartbeat(
     stage: str, *, liveness: bool = False, force: bool = False, initial: bool = False, **kw
 ):
     global _HB_CLAIM_SEQ
-    ts = time.time()
-    # liveness pings don't count as progress; provider stall detection skips them.
-    if not liveness:
-        _w._HB_LAST_PROGRESS_TS = ts
+    genuine_progress = not liveness
     with _HB_LOCK:
-        if not liveness:
+        ts = time.time()
+        if genuine_progress:
+            _w._HB_LAST_PROGRESS_TS = ts
             _w._HB_PROGRESS_SEQ += 1
         elif _w._HB_PROGRESS_SEQ > _w._HB_PROGRESS_UPLOADED_SEQ:
             # progress-carry: a real heartbeat since the last committed snapshot never reached HF
             # (throttled away or its upload failed). upgrade this ping to a real heartbeat so the
             # control plane's stall clock sees that progress instead of killing a healthy run.
-            # deliberately after the _HB_LAST_PROGRESS_TS bump above: carried progress is not NEW
-            # progress, so the worker's own stall-dump timer keeps its original reference point.
+            # carried progress is not new progress, so do not advance _HB_LAST_PROGRESS_TS: the
+            # worker's own stall-dump timer and the published age keep the original reference point.
             liveness = False
+        latest_progress_ts = float(_w._HB_LAST_PROGRESS_TS or 0.0)
         my_progress_seq = _w._HB_PROGRESS_SEQ
     payload = {
         "stage": stage,
@@ -167,6 +167,12 @@ def heartbeat(
         **({"liveness": True} if liveness else {}),
         **kw,
     }
+    if genuine_progress:
+        payload["progress_age_s"] = 0.0
+    elif latest_progress_ts:
+        payload["progress_age_s"] = round(max(0.0, ts - latest_progress_ts), 1)
+    else:
+        payload.pop("progress_age_s", None)
     _dc = os.environ.get("RUNPOD_DC_ID") or ""
     if _dc:
         payload.setdefault("dc", _dc)

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/flash/engine/worker/opd_train_runner.py

@@ -16,6 +16,9 @@ from typing import Any
 
 from flash.engine.plan.steps import rl_data_parallel_cards
 from flash.engine.worker import opd_train as _opd_train
+from flash.engine.worker.train.opd.reporting import (
+    _build_train_note_sections as _build_train_note_sections,
+)
 from flash.engine.worker.verl.child_io import (
     LORA_ROLLOUT_GUARD_SHIM,
     render_lora_rollout_guard_fragment,
@@ -921,80 +924,3 @@ def _export_and_upload_adapter(
     )
     _opd_train._w.hf_upload_folder(adapter_dir, "adapter", required=True)
     return adapter_dir
-
-
-def _build_train_note_sections(
-    request: _OpdRequest,
-    prompt_state: _PromptState,
-    workload: _WorkloadState,
-    runtime: _RuntimeState,
-    result: _ChildResult,
-    download_seconds: float,
-) -> tuple[
-    dict[str, Any],
-    dict[str, Any],
-    dict[str, Any],
-    tuple[dict[str, Any], dict[str, Any]],
-]:
-    final_accounting = result.final_accounting
-    knobs = request.knobs
-    initial = {
-        "epochs": knobs.epochs,
-        "retained_prompts": len(prompt_state.prompts),
-        "dropped_long_prompts": prompt_state.dropped_long,
-        "method": "gkd",
-        "init_from_adapter": request.spec.train.init_from_adapter or None,
-        "teacher_model": knobs.teacher_model,
-        "download_seconds": download_seconds,
-        "thinking": _opd_train._w.THINKING,
-        "loss_curve": final_accounting["loss_curve"],
-        "mean_coverage": (
-            float(final_accounting["coverage_sum"]) / int(final_accounting["aligned_sequences"])
-            if final_accounting["aligned_sequences"]
-            else 0.0
-        ),
-    }
-    accounting = {
-        "truncated_rollouts": int(final_accounting["truncated_rollouts"]),
-        "forced_tokens": int(final_accounting["forced_tokens"]),
-        "dropped_forced_groups": int(final_accounting["dropped_forced_groups"]),
-        "teacher_input_tokens": int(final_accounting["teacher_input_tokens"]),
-        "teacher_output_tokens": int(final_accounting["teacher_output_tokens"]),
-        "aligned_sequences": int(final_accounting["aligned_sequences"]),
-        "empty_alignments": int(final_accounting["empty_alignments"]),
-        "teacher_ok": int(final_accounting["teacher_ok"]),
-    }
-    training = {
-        "temperature": knobs.temperature,
-        "group_size": knobs.group_size,
-        "prompts_per_step": workload.prompts_per_step,
-        "max_completion_len": knobs.max_completion,
-        "multi_turn": request.multi_turn,
-        "max_turns": request.max_turns if request.multi_turn else None,
-        "episodes": int(final_accounting["episodes_seen"]) if request.multi_turn else None,
-        "mean_turns_per_episode": (
-            int(final_accounting["mt_turn_records"]) / int(final_accounting["episodes_seen"])
-            if request.multi_turn and final_accounting["episodes_seen"]
-            else None
-        ),
-    }
-    backend = (
-        {
-            "rollout_backend": "verl_vllm",
-            "verl_version": "0.8.0",
-            "verl_backend": "fsdp",
-            # report the EXECUTED width, not the allocation: the card count here would claim a
-            # sequence-parallel run that did not happen. token-balanced batching makes every
-            # allocated rank a dp rank, so unlike sft the executed dp width IS the card count.
-            "ulysses_sequence_parallel_size": ULYSSES_SEQUENCE_PARALLEL_SIZE,
-            "data_parallel_size": runtime.gpu_count,
-        },
-        {
-            "peak_gpu_gb": result.peak_gpu_gb,
-            "warm_started": bool(workload.warmstart_adapter),
-            "resumed": bool(runtime.resume_step),
-            "wandb_project": runtime.project_name if "wandb" in runtime.loggers else None,
-            "wandb_run_name": runtime.experiment_name if "wandb" in runtime.loggers else None,
-        },
-    )
-    return initial, accounting, training, backend

{freesolo_flash_dev-1.2.14 → freesolo_flash_dev-1.2.22}/flash/engine/worker/rl_train_runner.py

@@ -46,11 +46,13 @@ from flash.engine.worker.train.rl.multi_turn import (
     start_reward_server,
 )
 from flash.engine.worker.train.rl.shims import (
+    render_deferred_patch_runtime,
     render_entropy_quantile_shim,
     render_exact_save_steps_shim,
     render_image_pad_ban_shim,
     render_kl_ref_adapter_shim,
     render_per_turn_credit_shim,
+    render_rank_device_assert_shim,
     render_reentrant_checkpointing_shim,
     render_reward_module,
     render_stop_sequences_shim,
@@ -178,6 +180,12 @@ def _prepare_rl_files(inp, prompts):
     shim_markers = shim_marker_file(shim_dir)
     if os.path.exists(shim_markers):
         os.remove(shim_markers)
+    # where each rank records the gpu uuid it opened, so a collision is caught before nccl init.
+    # removed for the same reason as the marker file: a stale one from a prior attempt would show
+    # ranks claiming devices this attempt never touched, and read as a collision that is not there.
+    rank_device_claims = os.path.join(shim_dir, "rank_device_claims.txt")
+    if os.path.exists(rank_device_claims):
+        os.remove(rank_device_claims)
     return {
         "rollout_examples": rollout_examples,
         "message_prompts": message_prompts,
@@ -190,6 +198,7 @@ def _prepare_rl_files(inp, prompts):
         "shim_dir": shim_dir,
         "shim_py": os.path.join(shim_dir, "sitecustomize.py"),
         "shim_markers": shim_markers,
+        "rank_device_claims": rank_device_claims,
     }
 
 
@@ -204,6 +213,9 @@ def _write_rl_shim(inp, files) -> list[str]:
     # loaded. each feature renderer returns "" when its feature is off; the tf32 fragment is
     # unconditional, so this source is never empty.
     required_fragments = [
+        # first, so a collapsed rank->device map is reported before any other patch runs and long
+        # before the model load that currently hides it. inert at one card.
+        ("rank-device-assert", render_rank_device_assert_shim(int(inp["dp_cards"]))),
         (
             "reentrant-checkpointing",
             render_reentrant_checkpointing_shim(
@@ -238,7 +250,20 @@ def _write_rl_shim(inp, files) -> list[str]:
             # into this process there is nothing left to fix.
             render_tilelang_cudart_shim(),
             render_shim_marker_prologue(files["shim_markers"]),
-            *(wrap_shim_fragment(name, source) for name, source in required_fragments),
+            # above every wrapped fragment: each one registers with this registry rather than
+            # importing verl at startup, which is what keeps ray free to pin each rank to its own
+            # card (see train/rl/shims.render_deferred_patch_runtime). unwrapped -- it only defines
+            # the registry and touches no cuda, so there is no patch here that could fail open.
+            render_deferred_patch_runtime(),
+            # every fragment above defers its patch to the import of the module it targets, so this
+            # wrapper now spans only the registration. a marker written here would prove a callback
+            # was queued, not that the patch ran, and the parent's verify_applied_shim_markers would
+            # pass for a child training unpatched. each deferred body records its own name once the
+            # patch is actually installed.
+            *(
+                wrap_shim_fragment(name, source, record_immediately=False)
+                for name, source in required_fragments
+            ),
             # unconditional: every flash rollout is a lora rollout, and a base-model fallback is
             # indistinguishable from a working run in the metrics.
             render_lora_rollout_guard_fragment(),
@@ -405,6 +430,10 @@ def _build_rl_child_env(inp, files, loggers, reward_url):
         shim_dir=files["shim_dir"], wandb_enabled="wandb" in loggers
     )
     env_for_verl["FLASH_VERL_REWARD_URL"] = reward_url
+    # where each rank records the gpu it opened. ray fans this env to every actor, which is what
+    # makes the file a rendezvous point: the check needs to compare ranks that have no process
+    # group yet (see render_rank_device_assert_shim).
+    env_for_verl["FLASH_RANK_DEVICE_CLAIMS"] = files["rank_device_claims"]
     # the model is prefetched above; keep the subprocess off hf's rate-limited api.
     env_for_verl["HF_HUB_OFFLINE"] = "1"
     env_for_verl["TRANSFORMERS_OFFLINE"] = "1"

freesolo_flash_dev-1.2.22/flash/engine/worker/train/opd/reporting.py

@@ -0,0 +1,94 @@
+"""training metadata reporting helpers for opd."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from flash.engine.worker.runtime.pkg_proxy import W as _w
+from flash.engine.worker.verl.parallelism import ULYSSES_SEQUENCE_PARALLEL_SIZE
+
+if TYPE_CHECKING:
+    from flash.engine.worker.opd_train_runner import (
+        _ChildResult,
+        _OpdRequest,
+        _PromptState,
+        _RuntimeState,
+        _WorkloadState,
+    )
+
+
+def _build_train_note_sections(
+    request: _OpdRequest,
+    prompt_state: _PromptState,
+    workload: _WorkloadState,
+    runtime: _RuntimeState,
+    result: _ChildResult,
+    download_seconds: float,
+) -> tuple[
+    dict[str, Any],
+    dict[str, Any],
+    dict[str, Any],
+    tuple[dict[str, Any], dict[str, Any]],
+]:
+    final_accounting = result.final_accounting
+    knobs = request.knobs
+    initial = {
+        "epochs": knobs.epochs,
+        "retained_prompts": len(prompt_state.prompts),
+        "dropped_long_prompts": prompt_state.dropped_long,
+        "method": "gkd",
+        "init_from_adapter": request.spec.train.init_from_adapter or None,
+        "teacher_model": knobs.teacher_model,
+        "download_seconds": download_seconds,
+        "thinking": _w.THINKING,
+        "loss_curve": final_accounting["loss_curve"],
+        "mean_coverage": (
+            float(final_accounting["coverage_sum"]) / int(final_accounting["aligned_sequences"])
+            if final_accounting["aligned_sequences"]
+            else 0.0
+        ),
+    }
+    accounting = {
+        "truncated_rollouts": int(final_accounting["truncated_rollouts"]),
+        "forced_tokens": int(final_accounting["forced_tokens"]),
+        "dropped_forced_groups": int(final_accounting["dropped_forced_groups"]),
+        "teacher_input_tokens": int(final_accounting["teacher_input_tokens"]),
+        "teacher_output_tokens": int(final_accounting["teacher_output_tokens"]),
+        "aligned_sequences": int(final_accounting["aligned_sequences"]),
+        "empty_alignments": int(final_accounting["empty_alignments"]),
+        "teacher_ok": int(final_accounting["teacher_ok"]),
+    }
+    training = {
+        "temperature": knobs.temperature,
+        "group_size": knobs.group_size,
+        "prompts_per_step": workload.prompts_per_step,
+        "max_completion_len": knobs.max_completion,
+        "multi_turn": request.multi_turn,
+        "max_turns": request.max_turns if request.multi_turn else None,
+        "episodes": int(final_accounting["episodes_seen"]) if request.multi_turn else None,
+        "mean_turns_per_episode": (
+            int(final_accounting["mt_turn_records"]) / int(final_accounting["episodes_seen"])
+            if request.multi_turn and final_accounting["episodes_seen"]
+            else None
+        ),
+    }
+    backend = (
+        {
+            "rollout_backend": "verl_vllm",
+            "verl_version": "0.8.0",
+            "verl_backend": "fsdp",
+            # report the executed width, not the allocation: the card count here would claim a
+            # sequence-parallel run that did not happen. token-balanced batching makes every
+            # allocated rank a dp rank, so unlike sft the executed dp width is the card count.
+            "ulysses_sequence_parallel_size": ULYSSES_SEQUENCE_PARALLEL_SIZE,
+            "data_parallel_size": runtime.gpu_count,
+        },
+        {
+            "peak_gpu_gb": result.peak_gpu_gb,
+            "warm_started": bool(workload.warmstart_adapter),
+            "resumed": bool(runtime.resume_step),
+            "wandb_project": runtime.project_name if "wandb" in runtime.loggers else None,
+            "wandb_run_name": runtime.experiment_name if "wandb" in runtime.loggers else None,
+        },
+    )
+    return initial, accounting, training, backend