freesolo-flash-dev 1.2.36__tar.gz → 1.2.39__tar.gz

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.36
+Version: 1.2.39
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/cli/commands/__init__.py

@@ -399,6 +399,7 @@ def cmd_train(args) -> int:
         # equally not a verification -- so treat anything but an explicit True as unverified.
         affordability_verified = status.pop("affordability_verified", None) is True
         cost = "and cost" if affordability_verified else "but NOT cost"
+        budget_validated = prompt_budget_validation_suffix(status)
         environment = (
             "it did NOT import or run your environment.py. packaged input/output fields and the "
             "statically readable training contract were tokenized together; tokens, retention, "
@@ -411,8 +412,8 @@ def cmd_train(args) -> int:
         )
         print(
             "dry-run validated: config/schema, model+algorithm compatibility, lora rank, "
-            f"runtime-secret presence, warm-start source, serving context cap, {cost}. "
-            f"{environment}",
+            f"runtime-secret presence, warm-start source, serving context cap{budget_validated}, "
+            f"{cost}. {environment}",
             file=sys.stderr,
         )
         if not affordability_verified:
@@ -430,8 +431,10 @@ def cmd_train(args) -> int:
         else:
             print(json.dumps(status, indent=2))
         _print_unpacked_batch_warning(status, spec)  # after the payload, so stdout stays parseable
+        print_status_prompt_budget_warning(status)
         _print_reasoning_loss_warning(status)
         return 0
+    local_budget = warn_before_paid_submit(client, spec)
     status = client.create_run(
         payload,
         runtime_secrets=runtime_secrets,
@@ -440,6 +443,7 @@ def cmd_train(args) -> int:
     run_id = status["run_id"]
     _print_unpacked_batch_warning(status, spec)  # a real submit overrides batch_size the same way
     _print_reasoning_loss_warning(status)  # and trains on whatever reasoning the template kept
+    print_warmstart_context_supplement(local_budget, status)
     logger.info(
         "submitted run %s: model=%s algorithm=%s gpu=%s",
         run_id,
@@ -963,9 +967,13 @@ from flash.cli.commands.deploy import (  # noqa: E402,F401
     cmd_undeploy,
 )
 
-# re-exported at the bottom rather than imported at the top: `train_cost` resolves names back
-# through this package, so a top import would be circular. `cmd_train` below calls these, and the
-# estimate tests import `_warn_if_wandb_requested_without_key` from this package by name.
+# imported at the bottom because the train helpers resolve patched command-package names lazily.
+from flash.cli.commands.prompt_budget import (  # noqa: E402
+    print_status_prompt_budget_warning,
+    print_warmstart_context_supplement,
+    prompt_budget_validation_suffix,
+    warn_before_paid_submit,
+)
 from flash.cli.commands.train_cost import (  # noqa: E402,F401
     _client_train_schema,
     _cmd_train_cost,

freesolo_flash_dev-1.2.39/flash/cli/commands/prompt_budget.py

@@ -0,0 +1,146 @@
+"""prompt-budget warnings and warm-start context lookup for training commands."""
+
+from __future__ import annotations
+
+import sys
+from typing import TypeGuard, cast
+
+from flash import __version__
+from flash.cli.ui import render
+from flash.engine.plan.prompt_budget import PromptBudget, rl_prompt_budget
+
+
+def _positive_int(value: object) -> TypeGuard[int]:
+    return isinstance(value, int) and not isinstance(value, bool) and value > 0
+
+
+def _budget_from_status(status: object) -> PromptBudget | None:
+    budget = status.get("prompt_budget") if isinstance(status, dict) else None
+    if not isinstance(budget, dict):
+        return None
+    if budget.get("algorithm") not in {"grpo", "opd", "rl"}:
+        return None
+    if budget.get("context_source") not in {"authored", "recipe_default"}:
+        return None
+    if budget.get("prompt_budget_is_upper_bound") is not True:
+        return None
+    engine_len = budget.get("engine_len")
+    max_completion = budget.get("max_completion")
+    prompt_budget = budget.get("prompt_budget")
+    if not _positive_int(engine_len):
+        return None
+    if not _positive_int(max_completion):
+        return None
+    if not _positive_int(prompt_budget):
+        return None
+    if prompt_budget != engine_len - max_completion:
+        return None
+    source_context = budget.get("warm_start_context")
+    if source_context is not None and not _positive_int(source_context):
+        return None
+    return cast("PromptBudget", budget)
+
+
+def prompt_budget_validation_suffix(status: object) -> str:
+    """qualify a dry-run summary only when the server returned a budget."""
+    return ", prompt budget (upper bound)" if _budget_from_status(status) is not None else ""
+
+
+def warmstart_source_context(client, spec) -> int | None:
+    """read a warm-start source's authored context through the cli's authenticated client."""
+    from flash.core.catalog import samples_on_policy
+    from flash.schema import parse_checkpoint_ref
+
+    ref = getattr(spec.train, "init_from_adapter", "")
+    if not ref or not samples_on_policy(spec.algorithm):
+        return None
+    parsed = parse_checkpoint_ref(ref)
+    if parsed is None:
+        return None
+    try:
+        source = client.get_run(parsed[0])
+    # reporting is best-effort and must never block a valid submission
+    except Exception:
+        return None
+    train = (source or {}).get("spec", {}).get("train", {}) if isinstance(source, dict) else {}
+    context = train.get("max_context_tokens") if isinstance(train, dict) else None
+    if isinstance(context, bool) or not isinstance(context, int) or context <= 0:
+        return None
+    return context
+
+
+def _warmstart_context_sentence(context: object) -> str | None:
+    if isinstance(context, bool) or not isinstance(context, int) or context <= 0:
+        return None
+    return (
+        f"The warm-start source was configured with max_context_tokens={context}; "
+        "it is NOT inherited."
+    )
+
+
+def prompt_budget_warning(
+    budget: object,
+    *,
+    derived_locally: bool = False,
+    include_warm_start_context: bool = True,
+) -> str | None:
+    """render the defaulted-budget warning for one prompt-budget descriptor."""
+    if not isinstance(budget, dict) or budget.get("context_source") != "recipe_default":
+        return None
+    try:
+        prompt_budget = int(budget["prompt_budget"])
+        max_completion = int(budget["max_completion"])
+    except (KeyError, TypeError, ValueError):
+        return None
+    algorithm = str(budget.get("algorithm") or "").upper() or "RL"
+    message = (
+        f"train.max_context_tokens is unset, so {algorithm} derives a prompt budget of at most "
+        f"{prompt_budget} tokens from the recipe default (engine {budget.get('engine_len')} minus "
+        f"max_completion_tokens {max_completion}), not from the model's context. The worker clamps "
+        "the engine length to the model architecture first, so its budget can be smaller than "
+        "this. Prompts over the budget are DROPPED, not truncated, so the run trains on a "
+        "shorter, biased subset."
+    )
+    if derived_locally:
+        message += (
+            f" (Derived locally by this CLI {__version__}; if the control plane is on a "
+            "different release its recipe default may differ.)"
+        )
+    if include_warm_start_context:
+        source = _warmstart_context_sentence(budget.get("warm_start_context"))
+        if source:
+            message += f" {source}"
+    return message + " Set train.max_context_tokens explicitly to choose the budget."
+
+
+def _print_warning(message: str | None) -> None:
+    if message:
+        print(render.warn(message) if render.styled() else f"warning: {message}", file=sys.stderr)
+
+
+def print_status_prompt_budget_warning(status: object) -> None:
+    """print the server-derived warning carried by a submit or dry-run response."""
+    _print_warning(prompt_budget_warning(_budget_from_status(status)))
+
+
+def warn_before_paid_submit(client, spec) -> PromptBudget | None:
+    """print the locally derived warning before create_run can start provisioning."""
+    budget = rl_prompt_budget(spec, warm_start_context=warmstart_source_context(client, spec))
+    _print_warning(prompt_budget_warning(budget, derived_locally=True))
+    return budget
+
+
+def print_warmstart_context_supplement(local_budget: PromptBudget | None, status: object) -> None:
+    """print only context the server resolved after the generic pre-submit warning."""
+    served_budget = _budget_from_status(status)
+    if served_budget is None or (
+        local_budget is not None and local_budget.get("warm_start_context")
+    ):
+        return
+    _print_warning(_warmstart_context_sentence(served_budget.get("warm_start_context")))
+
+
+def warn_cost_prompt_budget(spec) -> None:
+    """print the local prompt-budget warning for the offline ``--cost`` path."""
+    budget = rl_prompt_budget(spec)
+    _print_warning(prompt_budget_warning(budget, derived_locally=True))

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/cli/commands/train_cost.py

@@ -91,6 +91,9 @@ def _cmd_train_cost_offline(spec) -> int:
         print(render.cost_panel(estimate))
     else:
         print(estimate.breakdown())
+    from flash.cli.commands.prompt_budget import warn_cost_prompt_budget
+
+    warn_cost_prompt_budget(spec)
     return 0
 
 

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/core/catalog.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, replace
+from math import ceil
 from typing import Any
 
 ALGORITHMS = ("sft", "grpo", "opd")
@@ -622,6 +623,10 @@ def resolve_vocab_size(model_id: str, revision: str = "") -> int:
     return vocab_size_for(model_id)
 
 
+def _with_merge_disk_floor(info: ModelInfo) -> ModelInfo:
+    return replace(info, min_disk_gb=max(info.min_disk_gb, ceil(info.params_b * 2) + 64))
+
+
 def resolve_model(model_id: str, algorithm: str, model_revision: str = "") -> ModelInfo:
     """Resolve a curated model, validated for ``algorithm``; anything uncataloged is rejected.
 
@@ -643,9 +648,8 @@ def resolve_model(model_id: str, algorithm: str, model_revision: str = "") -> Mo
             params_b=params_b,
             params=f"{params_b:.1f}B",
             vocab_size=vocab_size,
-            min_disk_gb=max(info.min_disk_gb, int(params_b * 2) + 64),
         )
-    return info
+    return _with_merge_disk_floor(info)
 
 
 def validate_model_for_algorithm(model_id: str, algorithm: str) -> ModelInfo:
@@ -660,4 +664,4 @@ def validate_model_for_algorithm(model_id: str, algorithm: str) -> ModelInfo:
 
 
 def public_model_rows() -> list[dict[str, Any]]:
-    return [m.to_dict() for m in list_models()]
+    return [_with_merge_disk_floor(model).to_dict() for model in list_models()]

freesolo_flash_dev-1.2.39/flash/engine/plan/prompt_budget.py

@@ -0,0 +1,63 @@
+"""derived grpo/opd prompt-budget data reported before paid training."""
+
+from __future__ import annotations
+
+from typing import Literal, NotRequired, TypedDict
+
+from flash.engine.plan.vram import (
+    grpo_completion_len,
+    grpo_rollout_seq_len,
+    opd_completion_len,
+    opd_rollout_seq_len,
+)
+
+
+class PromptBudget(TypedDict):
+    """serialized prompt-budget descriptor carried by prepared and persisted runs."""
+
+    algorithm: Literal["grpo", "opd", "rl"]
+    engine_len: int
+    max_completion: int
+    prompt_budget: int
+    context_source: Literal["authored", "recipe_default"]
+    prompt_budget_is_upper_bound: Literal[True]
+    warm_start_context: NotRequired[int]
+
+
+def rl_prompt_budget(spec, *, warm_start_context: int | None = None) -> PromptBudget | None:
+    """derive the pre-clamp prompt budget used by a grpo/opd worker.
+
+    the worker clamps ``engine_len`` to the model architecture before subtracting the completion
+    allowance, so this control-plane value is an upper bound. sft returns ``None`` because it
+    truncates over-long rows and reports that behavior through its workload profile instead.
+    """
+    from typing import cast
+
+    from flash.core.catalog import samples_on_policy
+
+    raw_algorithm = getattr(spec, "algorithm", "")
+    if not samples_on_policy(raw_algorithm):
+        return None
+    algorithm = cast("Literal['grpo', 'opd', 'rl']", raw_algorithm)
+    train = spec.train
+    authored = int(getattr(train, "max_context_tokens", 0) or 0)
+    max_tokens = getattr(train, "max_completion_tokens", None)
+    thinking = bool(getattr(spec, "thinking", False))
+    if algorithm == "opd":
+        max_completion = opd_completion_len(max_tokens, thinking)
+        engine_len = opd_rollout_seq_len(authored, max_tokens, thinking)
+    else:
+        max_completion = grpo_completion_len(max_tokens, thinking)
+        engine_len = grpo_rollout_seq_len(authored, max_tokens, thinking)
+    budget: PromptBudget = {
+        "algorithm": algorithm,
+        "engine_len": int(engine_len),
+        "max_completion": int(max_completion),
+        "prompt_budget": int(engine_len) - int(max_completion),
+        "context_source": "authored" if authored else "recipe_default",
+        "prompt_budget_is_upper_bound": True,
+    }
+    source_context = int(warm_start_context or 0)
+    if source_context > 0:
+        budget["warm_start_context"] = source_context
+    return budget

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/engine/worker/backend_common.py

@@ -471,29 +471,14 @@ def ray_num_cpus(gpu_count: int = 1, *, cap: int = 16) -> int:
     return max(floor, min(affinity, cap))
 
 
-def run_verl_training(
+def _run_streaming_verl_subprocess(
     cmd: list[str],
     *,
     env: dict[str, str],
-    on_step: Callable[[int], None] | None = None,
-    on_line: Callable[[str], None] | None = None,
-    heartbeat: Callable[[], None] | None = None,
-    step_pattern: str = r"step:\s*(\d+)",
-    heartbeat_interval_s: float = 20.0,
-    tail: ChildOutputTail | None = None,
+    on_line: Callable[[str], None],
+    errors: str | None = None,
 ) -> int:
-    """run a verl trainer subprocess, streaming stdout and surfacing step progress.
-
-    returns the process exit code. stdout+stderr are merged and scanned line by line: ``on_line``
-    receives every line, ``on_step`` receives each parsed training step, and ``heartbeat`` is called
-    at most once per ``heartbeat_interval_s``. callback failures terminate the child before they are
-    re-raised so a failed required checkpoint upload cannot leave paid training running unattended.
-    """
-    step_re = re.compile(step_pattern)
-    child_tail = tail if tail is not None else ChildOutputTail()
-    # before the child exists, so any grandchild it orphans reparents here and can actually be
-    # reaped. this process is not pid 1 -- the runpod handler is -- so without it every wait below
-    # answers ChildProcessError for a zombie nobody will collect.
+    """stream a verl subprocess under the shared process-group lifecycle supervisor."""
     adopt_orphaned_descendants()
     proc = subprocess.Popen(
         cmd,
@@ -501,87 +486,88 @@ def run_verl_training(
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         text=True,
+        errors=errors,
         bufsize=1,
         start_new_session=True,
     )
-    # start_new_session makes the leader pid the stable group id even after that leader is reaped.
+    # start_new_session keeps the group addressable after the leader is reaped.
     process_group_id = proc.pid
-    last_hb = 0.0
     try:
-        # the child's exit is watched independently of pipe EOF. a grandchild holding the inherited
-        # pipe open after the trainer dies would otherwise keep the loop below running forever, on a
-        # paid gpu, having already lost the process whose output it is waiting for.
         with _ChildExitWatchdog(
             proc, process_group_id=process_group_id, grace_s=_ORPHANED_PIPE_GRACE_S
         ) as watchdog:
             assert proc.stdout is not None
             for line in proc.stdout:
-                # held across the callbacks, not just the arrival: on_step uploads a checkpoint, and
-                # a reader inside one is working, not stuck.
                 with watchdog.handling_line():
-                    print(line, end="", flush=True)
-                    child_tail.record(line)
-                    if on_line is not None:
-                        on_line(line)
-                    m = step_re.search(line)
-                    if m and on_step is not None:
-                        on_step(int(m.group(1)))
-                    if heartbeat is not None:
-                        now = time.monotonic()
-                        if now - last_hb >= heartbeat_interval_s:
-                            heartbeat()
-                            last_hb = now
+                    on_line(line)
     except BaseException:
         kill_process_group(proc, process_group_id=process_group_id)
         raise
     finally:
         if proc.poll() is None:
-            # BOUNDED. EOF is not the child's exit, so a child that outlives its own stdout would
-            # park the attempt here. one that has not exited within the grace after its stdout closed
-            # is not going to, so tear the group down and let the wait inside that call collect it.
             try:
                 proc.wait(timeout=_TEARDOWN_GRACE_S)
             except subprocess.TimeoutExpired:
                 kill_process_group(proc, process_group_id=process_group_id)
-        # every job boundary, not just the failing ones. a straggler an earlier teardown SIGKILLed
-        # but could not drain in time exits shortly after, and `kill_process_group` -- the only
-        # other caller of this -- runs on exceptions alone: a worker whose later jobs all succeed
-        # would hold that zombie for its whole life, as pid 1 with nothing else to reap it.
         reap_stragglers()
-    collected = proc.returncode
-    if collected is None:
-        # the bounded wait above can end with nothing collected, which the unbounded one it replaced
-        # could not: a group member wedged in uninterruptible io outlives even the SIGKILL inside
-        # teardown. `int(None)` would replace that diagnosis with a TypeError, so name the survivor
-        # instead -- callers already handle a RuntimeError from here.
+    if proc.returncode is None:
         raise RuntimeError(
             f"verl subprocess {proc.pid} did not exit after teardown; its process group is still "
             "holding the gpu"
         )
-    return_code = int(collected)
+    return_code = int(proc.returncode)
     if watchdog.tore_down and return_code == 0:
-        # the child exited 0 but a descendant held the pipe open past the grace, so the group was
-        # killed to release it. reporting success here would upload whatever partial artifacts exist
-        # as a completed run: the trainer's own exit status says nothing about the descendant that
-        # was still running, and that descendant is why the pipe never closed.
         raise RuntimeError(
             f"verl subprocess {proc.pid} exited 0 but a descendant held its output pipe open for "
             f"{_ORPHANED_PIPE_GRACE_S:.0f}s; the process group was torn down to release the gpu"
         )
-    try:
-        raise_for_classified_verl_exit(return_code, child_tail)
-    except BaseException:
-        kill_process_group(proc, process_group_id=process_group_id)
-        raise
     if return_code != 0:
-        # a nonzero exit that carried no recognized signature RETURNS from the classifier rather
-        # than raising, so this is the one failing path that reached neither teardown above. the
-        # direct child is gone but its group need not be: a surviving EngineCore keeps its cuda
-        # context, and a reusable worker then hands the next attempt an occupied gpu.
         kill_process_group(proc, process_group_id=process_group_id)
     return return_code
 
 
+def run_verl_training(
+    cmd: list[str],
+    *,
+    env: dict[str, str],
+    on_step: Callable[[int], None] | None = None,
+    on_line: Callable[[str], None] | None = None,
+    heartbeat: Callable[[], None] | None = None,
+    step_pattern: str = r"step:\s*(\d+)",
+    heartbeat_interval_s: float = 20.0,
+    tail: ChildOutputTail | None = None,
+) -> int:
+    """run a verl trainer subprocess, streaming stdout and surfacing step progress.
+
+    returns the process exit code. stdout+stderr are merged and scanned line by line: ``on_line``
+    receives every line, ``on_step`` receives each parsed training step, and ``heartbeat`` is called
+    at most once per ``heartbeat_interval_s``. callback failures terminate the process group before
+    they are re-raised.
+    """
+    step_re = re.compile(step_pattern)
+    child_tail = tail if tail is not None else ChildOutputTail()
+    last_hb = 0.0
+
+    def handle_line(line: str) -> None:
+        nonlocal last_hb
+        print(line, end="", flush=True)
+        child_tail.record(line)
+        if on_line is not None:
+            on_line(line)
+        match = step_re.search(line)
+        if match and on_step is not None:
+            on_step(int(match.group(1)))
+        if heartbeat is not None:
+            now = time.monotonic()
+            if now - last_hb >= heartbeat_interval_s:
+                heartbeat()
+                last_hb = now
+
+    return_code = _run_streaming_verl_subprocess(cmd, env=env, on_line=handle_line)
+    raise_for_classified_verl_exit(return_code, child_tail)
+    return return_code
+
+
 _TEARDOWN_GRACE_S = 10.0
 
 # grace for a descendant holding stdout open after the trainer exits. observe child exit separately

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/engine/worker/opd_train_runner.py

@@ -157,8 +157,12 @@ def _prepare_prompts(
     if tokenizer.pad_token is None:
         tokenizer.pad_token = tokenizer.eos_token
     thinking_prefill = _opd_train._thinking_prefill_text(tokenizer)
-    requested_len = request.knobs.max_length or (
-        _opd_train.RECIPE.opd.max_prompt_len + request.knobs.max_completion
+    from flash.engine.plan.vram import opd_rollout_seq_len
+
+    requested_len = opd_rollout_seq_len(
+        request.knobs.max_length,
+        request.knobs.max_completion,
+        bool(_opd_train._w.THINKING),
     )
     # clamp to the architecture BEFORE deriving the prompt budget, so every downstream length agrees.
     # clamping only the engine would admit prompts sized against the unclamped budget and then fail

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/opd/failures.py

@@ -291,6 +291,10 @@ class _OpdVerlCheckpointWatcher(_VerlCheckpointWatcher):
     def _should_publish(self, step: int) -> bool:
         return True
 
+    def _publishable(self, pending: list[tuple[int, str]]) -> list[tuple[int, str]]:
+        """keep every pending retry state because each checkpoint carries distinct accounting."""
+        return pending
+
     def _publish(self, step: int, checkpoint_dir: str) -> None:
         actor_dir = os.path.join(checkpoint_dir, "actor")
         adapter_dir = os.path.join(self.export_root, f"step-{step}")

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/rl/checkpoints.py

@@ -23,7 +23,11 @@ from flash.engine.worker.backend_common import (
 )
 from flash.engine.worker.io.heartbeat import join_while_draining
 from flash.engine.worker.runtime.pkg_proxy import W as _w
-from flash.engine.worker.verl.checkpoints import resume_checkpoint_is_loadable
+from flash.engine.worker.verl.checkpoints import (
+    MergeDiskExhaustedError,
+    MergeDiskHeadroomError,
+    resume_checkpoint_is_loadable,
+)
 
 
 def _rl_train():
@@ -121,6 +125,8 @@ class _VerlResumeUploader:
         called only after training finished cleanly: on a crash or cancel the missing steps are a
         symptom of the real failure, and raising here would mask it.
         """
+        if isinstance(self._error, (MergeDiskHeadroomError, MergeDiskExhaustedError)):
+            raise self._error
         if self._error is not None:
             raise RuntimeError("verl resume uploader failed") from self._error
         missing = sorted(self.required_steps - self.published_steps)

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/rl/inputs.py

@@ -208,15 +208,15 @@ def _grpo_is_multimodal(train, message_prompts):
 
 
 def _resolve_sequence_lengths(model_id, model_revision, train_spec, rl, gcfg, tok, multi_turn):
-    # same resolver the spec-parse prompt-budget guard uses, so a run rejected at submit is exactly
-    # the run this would have failed here after the gpu was already paid for.
-    from flash.engine.plan.vram import grpo_completion_len
+    from flash.engine.plan.vram import grpo_completion_len, grpo_rollout_seq_len
 
-    max_completion = grpo_completion_len(gcfg.get("max_tokens"), bool(_w.THINKING))
+    thinking = bool(_w.THINKING)
+    max_tokens = gcfg.get("max_tokens")
+    max_completion = grpo_completion_len(max_tokens, thinking)
     train_ctx = (
         train_spec.max_context_tokens if (train_spec and train_spec.max_context_tokens) else 0
     )
-    requested_len = int(train_ctx or max(1024, rl.max_prompt_len + max_completion))
+    requested_len = grpo_rollout_seq_len(train_ctx, max_tokens, thinking)
     # clamp to the architecture BEFORE deriving the prompt budget, so every downstream length agrees.
     # clamping only the engine would admit prompts up to the unclamped budget and then fail them at
     # rollout, and would let the token budget pack more than the one-sequence memory floor intends.

{freesolo_flash_dev-1.2.36 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/sft/checkpoints.py

@@ -23,6 +23,7 @@ from flash.engine.worker.backend_common import (
 )
 from flash.engine.worker.io.heartbeat import join_while_draining
 from flash.engine.worker.runtime.pkg_proxy import W as _w
+from flash.engine.worker.verl.checkpoints import MergeDiskExhaustedError, MergeDiskHeadroomError
 
 
 def _sft_train():
@@ -113,6 +114,8 @@ class _VerlCheckpointWatcher:
         self._thread.start()
 
     def raise_if_failed(self) -> None:
+        if isinstance(self._error, (MergeDiskHeadroomError, MergeDiskExhaustedError)):
+            raise self._error
         if self._error is not None:
             raise RuntimeError("verl checkpoint watcher failed") from self._error
 
@@ -141,6 +144,20 @@ class _VerlCheckpointWatcher:
     def _should_publish(self, step: int) -> bool:
         return not self.required_steps or step in self.required_steps
 
+    def _publishable(self, pending: list[tuple[int, str]]) -> list[tuple[int, str]]:
+        """coalesce only an optional multi-checkpoint backlog to its newest save."""
+        if self.required_steps or len(pending) <= 1:
+            return pending
+        superseded = pending[:-1]
+        self.processed_steps.update(step for step, _ in superseded)
+        print(
+            f"[ckpt] publishing step {pending[-1][0]} and skipping superseded periodic "
+            f"checkpoint(s) {', '.join(str(step) for step, _ in superseded)}: the publisher is "
+            "behind training, and each export writes a full model copy to the same disk",
+            flush=True,
+        )
+        return pending[-1:]
+
     def _staged_source(self, step: int, checkpoint_dir: str) -> str:
         """hardlink a completed checkpoint before verl retention can prune it.
 
@@ -220,7 +237,7 @@ class _VerlCheckpointWatcher:
     def _run(self) -> None:
         try:
             while True:
-                for step, checkpoint_dir in self._pending():
+                for step, checkpoint_dir in self._publishable(self._pending()):
                     self._publish(step, checkpoint_dir)
                 # re-read rather than reusing the sweep above: verl advances the tracker right up to
                 # the moment the child exits, so a step can become visible during that sweep.