freesolo-flash-dev 1.2.37__tar.gz → 1.2.39__tar.gz

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.37
+Version: 1.2.39
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/core/catalog.py

@@ -3,6 +3,7 @@
 from __future__ import annotations
 
 from dataclasses import asdict, dataclass, replace
+from math import ceil
 from typing import Any
 
 ALGORITHMS = ("sft", "grpo", "opd")
@@ -622,6 +623,10 @@ def resolve_vocab_size(model_id: str, revision: str = "") -> int:
     return vocab_size_for(model_id)
 
 
+def _with_merge_disk_floor(info: ModelInfo) -> ModelInfo:
+    return replace(info, min_disk_gb=max(info.min_disk_gb, ceil(info.params_b * 2) + 64))
+
+
 def resolve_model(model_id: str, algorithm: str, model_revision: str = "") -> ModelInfo:
     """Resolve a curated model, validated for ``algorithm``; anything uncataloged is rejected.
 
@@ -643,9 +648,8 @@ def resolve_model(model_id: str, algorithm: str, model_revision: str = "") -> Mo
             params_b=params_b,
             params=f"{params_b:.1f}B",
             vocab_size=vocab_size,
-            min_disk_gb=max(info.min_disk_gb, int(params_b * 2) + 64),
         )
-    return info
+    return _with_merge_disk_floor(info)
 
 
 def validate_model_for_algorithm(model_id: str, algorithm: str) -> ModelInfo:
@@ -660,4 +664,4 @@ def validate_model_for_algorithm(model_id: str, algorithm: str) -> ModelInfo:
 
 
 def public_model_rows() -> list[dict[str, Any]]:
-    return [m.to_dict() for m in list_models()]
+    return [_with_merge_disk_floor(model).to_dict() for model in list_models()]

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/engine/worker/backend_common.py

@@ -471,29 +471,14 @@ def ray_num_cpus(gpu_count: int = 1, *, cap: int = 16) -> int:
     return max(floor, min(affinity, cap))
 
 
-def run_verl_training(
+def _run_streaming_verl_subprocess(
     cmd: list[str],
     *,
     env: dict[str, str],
-    on_step: Callable[[int], None] | None = None,
-    on_line: Callable[[str], None] | None = None,
-    heartbeat: Callable[[], None] | None = None,
-    step_pattern: str = r"step:\s*(\d+)",
-    heartbeat_interval_s: float = 20.0,
-    tail: ChildOutputTail | None = None,
+    on_line: Callable[[str], None],
+    errors: str | None = None,
 ) -> int:
-    """run a verl trainer subprocess, streaming stdout and surfacing step progress.
-
-    returns the process exit code. stdout+stderr are merged and scanned line by line: ``on_line``
-    receives every line, ``on_step`` receives each parsed training step, and ``heartbeat`` is called
-    at most once per ``heartbeat_interval_s``. callback failures terminate the child before they are
-    re-raised so a failed required checkpoint upload cannot leave paid training running unattended.
-    """
-    step_re = re.compile(step_pattern)
-    child_tail = tail if tail is not None else ChildOutputTail()
-    # before the child exists, so any grandchild it orphans reparents here and can actually be
-    # reaped. this process is not pid 1 -- the runpod handler is -- so without it every wait below
-    # answers ChildProcessError for a zombie nobody will collect.
+    """stream a verl subprocess under the shared process-group lifecycle supervisor."""
     adopt_orphaned_descendants()
     proc = subprocess.Popen(
         cmd,
@@ -501,87 +486,88 @@ def run_verl_training(
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
         text=True,
+        errors=errors,
         bufsize=1,
         start_new_session=True,
     )
-    # start_new_session makes the leader pid the stable group id even after that leader is reaped.
+    # start_new_session keeps the group addressable after the leader is reaped.
     process_group_id = proc.pid
-    last_hb = 0.0
     try:
-        # the child's exit is watched independently of pipe EOF. a grandchild holding the inherited
-        # pipe open after the trainer dies would otherwise keep the loop below running forever, on a
-        # paid gpu, having already lost the process whose output it is waiting for.
         with _ChildExitWatchdog(
             proc, process_group_id=process_group_id, grace_s=_ORPHANED_PIPE_GRACE_S
         ) as watchdog:
             assert proc.stdout is not None
             for line in proc.stdout:
-                # held across the callbacks, not just the arrival: on_step uploads a checkpoint, and
-                # a reader inside one is working, not stuck.
                 with watchdog.handling_line():
-                    print(line, end="", flush=True)
-                    child_tail.record(line)
-                    if on_line is not None:
-                        on_line(line)
-                    m = step_re.search(line)
-                    if m and on_step is not None:
-                        on_step(int(m.group(1)))
-                    if heartbeat is not None:
-                        now = time.monotonic()
-                        if now - last_hb >= heartbeat_interval_s:
-                            heartbeat()
-                            last_hb = now
+                    on_line(line)
     except BaseException:
         kill_process_group(proc, process_group_id=process_group_id)
         raise
     finally:
         if proc.poll() is None:
-            # BOUNDED. EOF is not the child's exit, so a child that outlives its own stdout would
-            # park the attempt here. one that has not exited within the grace after its stdout closed
-            # is not going to, so tear the group down and let the wait inside that call collect it.
             try:
                 proc.wait(timeout=_TEARDOWN_GRACE_S)
             except subprocess.TimeoutExpired:
                 kill_process_group(proc, process_group_id=process_group_id)
-        # every job boundary, not just the failing ones. a straggler an earlier teardown SIGKILLed
-        # but could not drain in time exits shortly after, and `kill_process_group` -- the only
-        # other caller of this -- runs on exceptions alone: a worker whose later jobs all succeed
-        # would hold that zombie for its whole life, as pid 1 with nothing else to reap it.
         reap_stragglers()
-    collected = proc.returncode
-    if collected is None:
-        # the bounded wait above can end with nothing collected, which the unbounded one it replaced
-        # could not: a group member wedged in uninterruptible io outlives even the SIGKILL inside
-        # teardown. `int(None)` would replace that diagnosis with a TypeError, so name the survivor
-        # instead -- callers already handle a RuntimeError from here.
+    if proc.returncode is None:
         raise RuntimeError(
             f"verl subprocess {proc.pid} did not exit after teardown; its process group is still "
             "holding the gpu"
         )
-    return_code = int(collected)
+    return_code = int(proc.returncode)
     if watchdog.tore_down and return_code == 0:
-        # the child exited 0 but a descendant held the pipe open past the grace, so the group was
-        # killed to release it. reporting success here would upload whatever partial artifacts exist
-        # as a completed run: the trainer's own exit status says nothing about the descendant that
-        # was still running, and that descendant is why the pipe never closed.
         raise RuntimeError(
             f"verl subprocess {proc.pid} exited 0 but a descendant held its output pipe open for "
             f"{_ORPHANED_PIPE_GRACE_S:.0f}s; the process group was torn down to release the gpu"
         )
-    try:
-        raise_for_classified_verl_exit(return_code, child_tail)
-    except BaseException:
-        kill_process_group(proc, process_group_id=process_group_id)
-        raise
     if return_code != 0:
-        # a nonzero exit that carried no recognized signature RETURNS from the classifier rather
-        # than raising, so this is the one failing path that reached neither teardown above. the
-        # direct child is gone but its group need not be: a surviving EngineCore keeps its cuda
-        # context, and a reusable worker then hands the next attempt an occupied gpu.
         kill_process_group(proc, process_group_id=process_group_id)
     return return_code
 
 
+def run_verl_training(
+    cmd: list[str],
+    *,
+    env: dict[str, str],
+    on_step: Callable[[int], None] | None = None,
+    on_line: Callable[[str], None] | None = None,
+    heartbeat: Callable[[], None] | None = None,
+    step_pattern: str = r"step:\s*(\d+)",
+    heartbeat_interval_s: float = 20.0,
+    tail: ChildOutputTail | None = None,
+) -> int:
+    """run a verl trainer subprocess, streaming stdout and surfacing step progress.
+
+    returns the process exit code. stdout+stderr are merged and scanned line by line: ``on_line``
+    receives every line, ``on_step`` receives each parsed training step, and ``heartbeat`` is called
+    at most once per ``heartbeat_interval_s``. callback failures terminate the process group before
+    they are re-raised.
+    """
+    step_re = re.compile(step_pattern)
+    child_tail = tail if tail is not None else ChildOutputTail()
+    last_hb = 0.0
+
+    def handle_line(line: str) -> None:
+        nonlocal last_hb
+        print(line, end="", flush=True)
+        child_tail.record(line)
+        if on_line is not None:
+            on_line(line)
+        match = step_re.search(line)
+        if match and on_step is not None:
+            on_step(int(match.group(1)))
+        if heartbeat is not None:
+            now = time.monotonic()
+            if now - last_hb >= heartbeat_interval_s:
+                heartbeat()
+                last_hb = now
+
+    return_code = _run_streaming_verl_subprocess(cmd, env=env, on_line=handle_line)
+    raise_for_classified_verl_exit(return_code, child_tail)
+    return return_code
+
+
 _TEARDOWN_GRACE_S = 10.0
 
 # grace for a descendant holding stdout open after the trainer exits. observe child exit separately

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/opd/failures.py

@@ -291,6 +291,10 @@ class _OpdVerlCheckpointWatcher(_VerlCheckpointWatcher):
     def _should_publish(self, step: int) -> bool:
         return True
 
+    def _publishable(self, pending: list[tuple[int, str]]) -> list[tuple[int, str]]:
+        """keep every pending retry state because each checkpoint carries distinct accounting."""
+        return pending
+
     def _publish(self, step: int, checkpoint_dir: str) -> None:
         actor_dir = os.path.join(checkpoint_dir, "actor")
         adapter_dir = os.path.join(self.export_root, f"step-{step}")

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/rl/checkpoints.py

@@ -23,7 +23,11 @@ from flash.engine.worker.backend_common import (
 )
 from flash.engine.worker.io.heartbeat import join_while_draining
 from flash.engine.worker.runtime.pkg_proxy import W as _w
-from flash.engine.worker.verl.checkpoints import resume_checkpoint_is_loadable
+from flash.engine.worker.verl.checkpoints import (
+    MergeDiskExhaustedError,
+    MergeDiskHeadroomError,
+    resume_checkpoint_is_loadable,
+)
 
 
 def _rl_train():
@@ -121,6 +125,8 @@ class _VerlResumeUploader:
         called only after training finished cleanly: on a crash or cancel the missing steps are a
         symptom of the real failure, and raising here would mask it.
         """
+        if isinstance(self._error, (MergeDiskHeadroomError, MergeDiskExhaustedError)):
+            raise self._error
         if self._error is not None:
             raise RuntimeError("verl resume uploader failed") from self._error
         missing = sorted(self.required_steps - self.published_steps)

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/engine/worker/train/sft/checkpoints.py

@@ -23,6 +23,7 @@ from flash.engine.worker.backend_common import (
 )
 from flash.engine.worker.io.heartbeat import join_while_draining
 from flash.engine.worker.runtime.pkg_proxy import W as _w
+from flash.engine.worker.verl.checkpoints import MergeDiskExhaustedError, MergeDiskHeadroomError
 
 
 def _sft_train():
@@ -113,6 +114,8 @@ class _VerlCheckpointWatcher:
         self._thread.start()
 
     def raise_if_failed(self) -> None:
+        if isinstance(self._error, (MergeDiskHeadroomError, MergeDiskExhaustedError)):
+            raise self._error
         if self._error is not None:
             raise RuntimeError("verl checkpoint watcher failed") from self._error
 
@@ -141,6 +144,20 @@ class _VerlCheckpointWatcher:
     def _should_publish(self, step: int) -> bool:
         return not self.required_steps or step in self.required_steps
 
+    def _publishable(self, pending: list[tuple[int, str]]) -> list[tuple[int, str]]:
+        """coalesce only an optional multi-checkpoint backlog to its newest save."""
+        if self.required_steps or len(pending) <= 1:
+            return pending
+        superseded = pending[:-1]
+        self.processed_steps.update(step for step, _ in superseded)
+        print(
+            f"[ckpt] publishing step {pending[-1][0]} and skipping superseded periodic "
+            f"checkpoint(s) {', '.join(str(step) for step, _ in superseded)}: the publisher is "
+            "behind training, and each export writes a full model copy to the same disk",
+            flush=True,
+        )
+        return pending[-1:]
+
     def _staged_source(self, step: int, checkpoint_dir: str) -> str:
         """hardlink a completed checkpoint before verl retention can prune it.
 
@@ -220,7 +237,7 @@ class _VerlCheckpointWatcher:
     def _run(self) -> None:
         try:
             while True:
-                for step, checkpoint_dir in self._pending():
+                for step, checkpoint_dir in self._publishable(self._pending()):
                     self._publish(step, checkpoint_dir)
                 # re-read rather than reusing the sweep above: verl advances the tracker right up to
                 # the moment the child exits, so a step can become visible during that sweep.

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/engine/worker/verl/checkpoints.py

@@ -9,6 +9,7 @@ Split out of `flash.engine.worker.backend_common` to keep that module under the
 
 from __future__ import annotations
 
+import errno
 import json
 import os
 import re
@@ -20,24 +21,37 @@ class MergeDiskHeadroomError(RuntimeError):
     """the merged model this export must write does not fit beside the checkpoint it reads."""
 
 
+class MergeDiskExhaustedError(RuntimeError):
+    """the merge started with room and then ran the disk out partway through writing its output."""
+
+
+# a genuine short write leaves almost no free space; keep the fallback absolute so a normal partial
+# merge is not misclassified merely because its output is large.
+_MERGE_DISK_EXHAUSTED_FREE_BYTES = 64 * 1024 * 1024
+
+# quota exhaustion can leave the underlying filesystem reporting free space, so preserve both errno
+# and text forms emitted by torch and safetensors.
+_DISK_EXHAUSTED_ERRNOS = (errno.ENOSPC, errno.EDQUOT)
+_DISK_EXHAUSTED_MARKERS = (
+    "no space left on device",
+    "errno 28",
+    "os error 28",
+    "disk quota exceeded",
+    "errno 122",
+    "os error 122",
+)
+_SHORT_WRITE_MARKERS = ("unexpected pos",)
+
+# disk size is platform-managed, so checkpoint frequency is the reachable remedy.
+_FEWER_CHECKPOINTS_ADVICE = (
+    "publish fewer checkpoints: raise train.save_every (save_every == the run's total steps "
+    "publishes only the final adapter), or drop entries from train.save_at_steps if the run sets "
+    "them -- exact steps override save_every and are always published."
+)
+
+
 def _model_shard_bytes(path: str) -> int:
-    """total size of the fsdp model shards directly in `path`.
-
-    only `model_world_size_*_rank_*.pt` is measured, because only those files become merge output:
-    `_load_and_merge_state_dicts` loads exactly those names, and the merged dict is what
-    `save_pretrained` writes back out. the `optim_*` and `extra_state_*` files sitting beside them
-    are read by resume, never by the merger, so including them would inflate the requirement by the
-    whole optimizer state -- ~7.6 GB of adam moments on a 35b rank-32 run -- and refuse merges that
-    would have fit.
-
-    a flat scan, not a walk, because the merger's read is flat: it opens
-    `Path(local_dir) / f"model_world_size_{W}_rank_{r}.pt"` per rank and never recurses. walking
-    would add up nested files it will never read, which is the same over-estimate in a new form.
-
-    shares `_FSDP_SHARD_RE` with `checkpoint_world_size` rather than matching the name a second way:
-    one definition of what a shard filename is, so a future change to verl's layout cannot leave the
-    two disagreeing about which files a merge reads.
-    """
+    """total size of the top-level fsdp model shards the merger reads."""
     try:
         names = os.listdir(path)
     except OSError:
@@ -54,37 +68,12 @@ def _model_shard_bytes(path: str) -> int:
 
 
 def require_merge_headroom(ckpt_actor_dir: str, merge_out: str) -> None:
-    """Fail before `verl.model_merger` runs if its output cannot fit on the container disk.
-
-    `model_merger merge` does NOT write only the small lora adapter. `save_hf_model_and_tokenizer`
-    materializes the full base model and calls `model.save_pretrained(target_dir, state_dict=...)`
-    (verl `model_merger/base_model_merger.py`), so exporting one 35b checkpoint writes ~70 GB into
-    `<adapter>_merge` beside the ~60 GB checkpoint it is reading. That is the single largest
-    transient on the disk, and it recurs on EVERY publish, not just at finalization.
-
-    Without this guard the merger dies partway through `save_pretrained` with ENOSPC, and the run
-    fails after training has already succeeded. Checking first turns a silent late disk death into
-    an actionable error naming the shortfall, while the checkpoint is still intact and resumable.
-
-    The requirement is derived from what the merger actually moves rather than from the checkpoint's
-    total size: it loads the `model_world_size_*_rank_*.pt` shards, casts every tensor to bf16, and
-    writes that same state back out as base model plus adapter (`fsdp_model_merger.py`,
-    `base_model_merger.py`). Shard bytes in, shard bytes out. Sizing the whole directory instead
-    would add the `optim_*` and `extra_state_*` files, which resume reads and the merger never
-    materializes -- on a 35b rank-32 run that is ~7.6 GB of adam moments, enough to refuse a merge
-    that had room.
-
-    The two error directions are not symmetric, which is why no safety margin is added on top.
-    Underestimating leaves the merger to hit ENOSPC exactly as it does today, so the guard is merely
-    absent. Overestimating fails a run that would have completed, which is a regression this guard
-    would have introduced. When in doubt, let the merge proceed.
-    """
+    """fail before the merger when its full-bf16 output cannot fit beside its input shards."""
     need = _model_shard_bytes(ckpt_actor_dir)
     if need <= 0:
         return
-    try:
-        free = shutil.disk_usage(os.path.dirname(merge_out.rstrip("/")) or ".").free
-    except OSError:
+    free = _free_bytes(merge_out)
+    if free is None:
         # an unreadable mount is not evidence of exhaustion; let the merger run and report for real.
         return
     if free >= need:
@@ -92,10 +81,99 @@ def require_merge_headroom(ckpt_actor_dir: str, merge_out: str) -> None:
     raise MergeDiskHeadroomError(
         f"cannot export the adapter: merging {ckpt_actor_dir} needs about "
         f"{need / 1e9:.1f} GB beside it but only {free / 1e9:.1f} GB is free. "
-        "raise [gpu] disk_gb, or save fewer checkpoints with a larger save_every."
+        + _FEWER_CHECKPOINTS_ADVICE
+    )
+
+
+def _run_merger(cmd: list[str], env: dict[str, str]) -> None:
+    """stream merger output and prefer direct disk evidence over a short-write marker."""
+    from flash.engine.worker import backend_common
+
+    disk_line = ""
+    short_write_line = ""
+
+    def handle_line(line: str) -> None:
+        nonlocal disk_line, short_write_line
+        print(line, end="", flush=True)
+        lowered = line.lower()
+        if not disk_line and any(marker in lowered for marker in _DISK_EXHAUSTED_MARKERS):
+            disk_line = line.strip()
+        elif not short_write_line and any(marker in lowered for marker in _SHORT_WRITE_MARKERS):
+            short_write_line = line.strip()
+
+    merger_env = {**env, "PYTHONUNBUFFERED": "1"}
+    return_code = backend_common._run_streaming_verl_subprocess(
+        cmd, env=merger_env, on_line=handle_line, errors="replace"
+    )
+    if return_code != 0:
+        raise subprocess.CalledProcessError(
+            return_code, cmd, output=disk_line or short_write_line or None
+        )
+
+
+def _free_bytes(path: str) -> int | None:
+    """free bytes on the filesystem holding ``path``, or none when it cannot be read."""
+    try:
+        return shutil.disk_usage(os.path.dirname(path.rstrip("/")) or ".").free
+    except OSError:
+        return None
+
+
+def _error_chain_matches(
+    error: BaseException, *, errnos: tuple[int, ...] = (), markers: tuple[str, ...] = ()
+) -> bool:
+    """whether an error chain contains any requested errno or text marker."""
+    seen: set[int] = set()
+    current: BaseException | None = error
+    while current is not None and id(current) not in seen:
+        seen.add(id(current))
+        if isinstance(current, OSError) and current.errno in errnos:
+            return True
+        for evidence in (
+            current,
+            getattr(current, "output", None),
+            getattr(current, "stderr", None),
+        ):
+            if isinstance(evidence, bytes):
+                evidence = evidence.decode("utf-8", "replace")
+            if evidence is not None and any(marker in str(evidence).lower() for marker in markers):
+                return True
+        current = current.__cause__ or current.__context__
+    return False
+
+
+def _disk_exhausted_error(error: BaseException) -> bool:
+    """whether an error chain contains direct volume or quota exhaustion evidence."""
+    return _error_chain_matches(
+        error, errnos=_DISK_EXHAUSTED_ERRNOS, markers=_DISK_EXHAUSTED_MARKERS
     )
 
 
+def raise_for_merge_disk_exhaustion(
+    error: BaseException, ckpt_actor_dir: str, merge_out: str, *, merger_succeeded: bool = False
+) -> None:
+    """classify direct disk evidence or a low-space short write before cleanup."""
+    direct_evidence = _disk_exhausted_error(error)
+    if not direct_evidence and isinstance(error, OSError):
+        return
+    if merger_succeeded and not direct_evidence:
+        return
+    short_write_evidence = _error_chain_matches(error, markers=_SHORT_WRITE_MARKERS)
+    if not direct_evidence and not short_write_evidence:
+        return
+    free = _free_bytes(merge_out)
+    if not direct_evidence and (free is None or free > _MERGE_DISK_EXHAUSTED_FREE_BYTES):
+        return
+    free_text = "unknown" if free is None else f"{free / 1e9:.2f} GB"
+    raise MergeDiskExhaustedError(
+        f"ran out of disk while merging {ckpt_actor_dir} into {merge_out}: "
+        f"{free_text} free on that filesystem. publishing a checkpoint materializes the FULL base "
+        "model beside the checkpoint it reads, so a small lora adapter still needs room for a whole "
+        "model copy. the underlying error is a short write, not a corrupt checkpoint. "
+        + _FEWER_CHECKPOINTS_ADVICE
+    ) from error
+
+
 def resolve_checkpoint_actor_dir(step_dir: str) -> str:
     """return the directory inside ``global_step_N`` that holds the saved model + ``huggingface/``.
 
@@ -302,12 +380,10 @@ def export_peft_adapter(
     merge_env["HF_HUB_OFFLINE"] = "1"
     merge_env["TRANSFORMERS_OFFLINE"] = "1"
     merge_env["HF_HUB_DISABLE_XET"] = "1"
-    # the merge tree is this function's to clean up for its whole lifetime, not just on success.
-    # it holds the full merged model -- the largest transient on the disk -- so leaving it behind
-    # when the merger dies or writes an unexpected layout would strand tens of gb on the exact
-    # disk this guard exists to protect, and the next save would inherit less room than this one.
+    # after a successful merge, low free space is expected and cannot classify placement failures.
+    merger_succeeded = False
     try:
-        subprocess.run(
+        _run_merger(
             [
                 python_bin,
                 "-m",
@@ -320,21 +396,25 @@ def export_peft_adapter(
                 "--target_dir",
                 merge_out,
             ],
-            check=True,
-            env=merge_env,
+            merge_env,
         )
+        merger_succeeded = True
         lora_dir = os.path.join(merge_out, "lora_adapter")
         if not os.path.exists(os.path.join(lora_dir, "adapter_config.json")):
             raise RuntimeError(
                 f"verl model_merger did not produce a peft adapter at {lora_dir} (no adapter_config.json); "
                 "the merger output layout must be adjusted for this verl version."
             )
-        # rename rather than copy: every caller builds `out_adapter_dir` and `merge_out` as siblings
-        # in one run workdir, so the two are on one filesystem and `os.replace` is a metadata
-        # operation that moves no bytes. copying would hold a second copy of the adapter beside the
-        # still-undeleted merge tree, a peak this function's own headroom check does not budget for.
+        # move within the shared filesystem so placement does not create another adapter copy.
         for name in os.listdir(lora_dir):
             os.replace(os.path.join(lora_dir, name), os.path.join(out_adapter_dir, name))
+    except Exception as error:
+        # classify before cleanup frees the merge tree and its low-space evidence.
+        # cancellation remains a base exception and is never relabelled as disk exhaustion.
+        raise_for_merge_disk_exhaustion(
+            error, ckpt_actor_dir, merge_out, merger_succeeded=merger_succeeded
+        )
+        raise
     finally:
         shutil.rmtree(merge_out, ignore_errors=True)
 

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/flash/runner/preparation.py

@@ -92,7 +92,7 @@ def _inherit_warmstart_revision(
     """Adopt a warm-start source's pin BEFORE the spec is sized against it.
 
     Sizing reads the revision: ``resolve_model`` re-derives params/vocab/disk from the pinned
-    commit's geometry, and ``min_disk_gb`` becomes ``params_b * 2 + 64``, which for half of today's
+    commit's geometry, and ``min_disk_gb`` becomes ``ceil(2 * params_b) + 64``, which for half of today's
     catalog is strictly larger than the catalog default. Adopting the pin only inside
     ``_prepare_init_from_adapter`` -- which runs after ``resolve_model``, ``_with_model_disk``, and
     ``_assign_weight_cache_volume`` -- would provision the child as if unpinned while training it

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "freesolo-flash-dev"
-version = "1.2.37"
+version = "1.2.39"
 description ="Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI"
 readme = "README.md"
 # RunPod Flash supports Python 3.11-3.12 (not 3.13 yet).
@@ -137,7 +137,7 @@ artifacts = ["flash/py.typed"]
 # auto-published by .github/workflows/publish-dev.yml: a push to `dev` whose version isn't yet on
 # PyPI publishes it; pushes that don't change it find it already published and no-op. So cutting a
 # dev-channel release = bump this (and [project].version) and merge to `dev`.
-version = "1.2.37"
+version = "1.2.39"
 
 [tool.pytest.ini_options]
 pythonpath = ["."]

{freesolo_flash_dev-1.2.37 → freesolo_flash_dev-1.2.39}/tests/test_backend_common.py

@@ -3711,13 +3711,13 @@ def test_without_the_claim_the_orphan_reparents_past_this_process():
 
 
 @_needs_process_teardown
-def test_the_claim_is_made_before_the_trainer_is_spawned():
+def test_the_claim_is_made_before_each_verl_process_is_spawned():
     """Order matters: the kernel only reparents to a process that was ALREADY a subreaper.
 
     Claiming after the child exists leaves any grandchild it has already orphaned parented
     elsewhere, so the fix would work only for the second job onward on a reused worker.
     """
-    for fn in (vc.run_verl_training, rl_train._execute_rl_child):
+    for fn in (vc._run_streaming_verl_subprocess, rl_train._execute_rl_child):
         src = " ".join(inspect.getsource(fn).split())
         assert "adopt_orphaned_descendants()" in src, f"{fn.__name__} never claims its orphans"
         assert src.index("adopt_orphaned_descendants()") < src.index("subprocess.Popen("), (