freesolo-flash-dev 1.2.115__tar.gz → 1.2.116__tar.gz

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: freesolo-flash-dev
-Version: 1.2.115
+Version: 1.2.116
 Summary: Flash — managed LoRA post-training (SFT/GRPO/OPD) for Freesolo environments, driven by the `flash` CLI
 Project-URL: Homepage, https://github.com/freesolo-co/flash
 Project-URL: Repository, https://github.com/freesolo-co/flash

freesolo_flash_dev-1.2.116/flash/_internal/openai_sse.py

@@ -0,0 +1,167 @@
+"""pure-stdlib decoding for openai-compatible server-sent events."""
+
+from __future__ import annotations
+
+import json
+from collections.abc import Iterable, Iterator
+from dataclasses import dataclass
+
+
+class OpenAISSEError(ValueError):
+    """one decoded event stream violated the openai sse contract."""
+
+
+@dataclass(frozen=True, slots=True)
+class DeltaEvent:
+    reasoning_content: str | None
+    content: str | None
+
+
+@dataclass(frozen=True, slots=True)
+class ErrorEvent:
+    message: str
+
+
+@dataclass(frozen=True, slots=True)
+class DoneEvent:
+    pass
+
+
+OpenAISSEEvent = DeltaEvent | ErrorEvent | DoneEvent
+
+_UTF8_BOM = chr(0xFEFF)
+
+
+def _next_sse_line(buffered: str, *, final: bool = False) -> tuple[str, str] | None:
+    for index, character in enumerate(buffered):
+        if character == "\n":
+            return buffered[:index], buffered[index + 1 :]
+        if character != "\r":
+            continue
+        if index + 1 == len(buffered) and not final:
+            return None
+        delimiter_length = 2 if buffered[index + 1 : index + 2] == "\n" else 1
+        return buffered[:index], buffered[index + delimiter_length :]
+    return None
+
+
+def iter_openai_sse_events(chunks: Iterable[str]) -> Iterator[OpenAISSEEvent]:
+    """decode arbitrary text chunks and require one terminal event."""
+
+    buffered = ""
+    frame_lines: list[str] = []
+    first_line = True
+    for chunk in chunks:
+        buffered += chunk
+        while parsed := _next_sse_line(buffered):
+            line, buffered = parsed
+            if first_line:
+                line = line.removeprefix(_UTF8_BOM)
+                first_line = False
+            if line:
+                frame_lines.append(line)
+                continue
+            for event in _events_from_frame(frame_lines):
+                yield event
+                if isinstance(event, DoneEvent | ErrorEvent):
+                    return
+            frame_lines = []
+    while parsed := _next_sse_line(buffered, final=True):
+        line, buffered = parsed
+        if first_line:
+            line = line.removeprefix(_UTF8_BOM)
+            first_line = False
+        if line:
+            frame_lines.append(line)
+            continue
+        for event in _events_from_frame(frame_lines):
+            yield event
+            if isinstance(event, DoneEvent | ErrorEvent):
+                return
+        frame_lines = []
+    if buffered or frame_lines:
+        raise OpenAISSEError("chat stream ended with an incomplete server-sent event frame")
+    raise OpenAISSEError("chat stream ended before the terminal [DONE] event")
+
+
+def sse_data_is_terminal(data: bytes) -> bool:
+    """return whether one complete raw sse frame is terminal."""
+
+    normalized = data.replace(b"\r\n", b"\n").replace(b"\r", b"\n")
+    data_lines = [
+        line.removeprefix(b"data:").strip()
+        for line in normalized.split(b"\n")
+        if line.startswith(b"data:")
+    ]
+    if not data_lines:
+        return False
+    raw = b"\n".join(data_lines)
+    if raw == b"[DONE]":
+        return True
+    try:
+        payload = json.loads(raw)
+    except (UnicodeDecodeError, json.JSONDecodeError):
+        return False
+    if not isinstance(payload, dict):
+        return False
+    if isinstance(payload.get("error"), dict):
+        return True
+    choices = payload.get("choices") or []
+    return isinstance(choices, list) and any(
+        isinstance(choice, dict) and choice.get("finish_reason") == "error" for choice in choices
+    )
+
+
+def _events_from_frame(lines: list[str]) -> tuple[OpenAISSEEvent, ...]:
+    data_lines = []
+    for line in lines:
+        if not line.startswith("data:"):
+            continue
+        data = line.removeprefix("data:")
+        data_lines.append(data.removeprefix(" "))
+    if not data_lines:
+        return ()
+    data = "\n".join(data_lines)
+    if not data:
+        return ()
+    if data == "[DONE]":
+        return (DoneEvent(),)
+    try:
+        payload = json.loads(data)
+    except json.JSONDecodeError as exc:
+        raise OpenAISSEError("chat stream contained invalid openai sse json") from exc
+    if not isinstance(payload, dict):
+        raise OpenAISSEError("chat stream contained a non-object openai sse payload")
+    if "error" in payload:
+        error = payload["error"]
+        if not isinstance(error, dict):
+            raise OpenAISSEError("chat stream error must be an object")
+        return (ErrorEvent(str(error.get("message") or "chat stream ended with an error")),)
+    choices = payload.get("choices", [])
+    if not isinstance(choices, list):
+        raise OpenAISSEError("chat stream choices must be an array")
+    events: list[DeltaEvent | ErrorEvent] = []
+    for choice in choices:
+        if not isinstance(choice, dict):
+            raise OpenAISSEError("chat stream choice must be an object")
+        if choice.get("finish_reason") == "error":
+            events.append(ErrorEvent("chat stream ended with an engine error"))
+            continue
+        delta = choice.get("delta", {})
+        if not isinstance(delta, dict):
+            raise OpenAISSEError("chat stream delta must be an object")
+        reasoning = delta.get("reasoning_content")
+        content = delta.get("content")
+        if reasoning is not None and not isinstance(reasoning, str):
+            raise OpenAISSEError("chat stream reasoning_content must be a string or null")
+        if content is not None and not isinstance(content, str):
+            raise OpenAISSEError("chat stream content must be a string or null")
+        normalized_reasoning = reasoning if isinstance(reasoning, str) else None
+        normalized_content = content if isinstance(content, str) else None
+        if normalized_reasoning is not None or normalized_content is not None:
+            events.append(DeltaEvent(normalized_reasoning, normalized_content))
+    if not events:
+        return ()
+    if len(events) > 1:
+        raise OpenAISSEError("chat stream contained multiple text choices")
+    return (events[0],)

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/flash/client/http.py

@@ -15,6 +15,12 @@ from collections.abc import Iterator, Mapping
 from typing import Any
 
 from flash._internal.channel import CLI_NAME
+from flash._internal.openai_sse import (
+    DeltaEvent,
+    ErrorEvent,
+    OpenAISSEError,
+    iter_openai_sse_events,
+)
 from flash.client.config import load_credentials_with_source
 from flash.client.shapes import RequireSpec, matches_require
 from flash.client.streaming import (
@@ -232,6 +238,42 @@ def _prepare_chat_request(
     return base_run_id, body
 
 
+def _openai_sse_text(chunks: Iterator[str]) -> Iterator[str]:
+    """render normalized openai events for cli and environment evaluation."""
+
+    reasoning_open = False
+    reasoning_done = False
+    try:
+        for event in iter_openai_sse_events(chunks):
+            if isinstance(event, ErrorEvent):
+                if reasoning_open:
+                    reasoning_open = False
+                    yield "</think>"
+                raise ClientError(event.message)
+            if not isinstance(event, DeltaEvent):
+                continue
+            reasoning = event.reasoning_content
+            if reasoning is not None:
+                if not reasoning_open and not (reasoning_done and not reasoning):
+                    reasoning_open = True
+                    yield "<think>"
+                if reasoning:
+                    yield reasoning
+            content = event.content
+            if content:
+                if reasoning_open:
+                    reasoning_open = False
+                    reasoning_done = True
+                    yield "</think>"
+                yield content
+    except OpenAISSEError as exc:
+        if reasoning_open:
+            yield "</think>"
+        raise ClientError(str(exc)) from exc
+    if reasoning_open:
+        yield "</think>"
+
+
 def _parse_adapter_target(target: str) -> tuple[str, int | None]:
     from flash.schema import parse_checkpoint_ref
 
@@ -815,7 +857,8 @@ class ApiClient:
             urllib.request.urlopen(req, timeout=30 * 60) as resp,
         ):
             content_type = resp.headers.get("Content-Type", "")
-            if "application/json" in content_type:
+            media_type = content_type.partition(";")[0].strip().lower()
+            if media_type == "application/json":
                 payload = self._decode_response(
                     f"/v1/runs/{base_run_id}/chat",
                     resp.read(),
@@ -829,7 +872,8 @@ class ApiClient:
             read1 = getattr(resp, "read1", None)
             read = read1 if read1 is not None else resp.read
             read_size = 4096 if read1 is not None else 1
-            try:
+
+            def decoded_chunks() -> Iterator[str]:
                 while raw := read(read_size):
                     state = decoder.getstate()
                     try:
@@ -844,6 +888,13 @@ class ApiClient:
                         raise exc
                     yield from decoded
                 yield from decoder.decode(b"", final=True)
+
+            try:
+                chunks = decoded_chunks()
+                if media_type == "text/event-stream":
+                    yield from _openai_sse_text(chunks)
+                else:
+                    yield from chunks
             except (http.client.IncompleteRead, ConnectionError) as exc:
                 # the server aborts the chunked response when the serving backend fails
                 # mid-generation; urllib reports the missing terminating chunk (or reset) here.

freesolo_flash_dev-1.2.116/flash/engine/verl_checkpoint.py

@@ -0,0 +1,142 @@
+"""Dependency-light validation for native Verl FSDP checkpoint manifests."""
+
+from __future__ import annotations
+
+import json
+import re
+from collections.abc import Iterable
+from dataclasses import dataclass
+
+from flash._internal.fileio import reject_duplicate_keys
+from flash.engine.verl_policy import FsdpGeneration
+
+VERL_FSDP_CONFIG_FILE = "fsdp_config.json"
+_MAX_NATIVE_FSDP_WORLD_SIZE = 8
+_REQUIRED_SHARD_CLASSES = ("model", "optim", "extra_state")
+_SHARD_RE = re.compile(r"(model|optim|extra_state)_world_size_(\d+)_rank_(\d+)\.pt")
+_REJECT_DUPLICATE_CONFIG_KEYS = reject_duplicate_keys(
+    lambda key: ValueError(f"duplicate fsdp config key: {key}")
+)
+
+
+@dataclass(frozen=True)
+class FsdpManifestFile:
+    """One direct actor-manifest member and its known size, if available."""
+
+    name: str
+    size: int | None
+
+
+@dataclass(frozen=True)
+class FsdpCheckpointInspection:
+    """One parsed native checkpoint verdict shared by selection and diagnostics."""
+
+    loadable: bool
+    generation: int | None
+    width: int | None
+    rejection_reason: str | None = None
+    missing_path: str | None = None
+
+    def diagnostic(self) -> str:
+        detail = self.rejection_reason or "loadable native checkpoint"
+        return f"{detail}: {self.missing_path}" if self.missing_path else detail
+
+
+def _positive_int(value: object) -> int | None:
+    if isinstance(value, int) and not isinstance(value, bool) and value >= 1:
+        return value
+    return None
+
+
+def parse_fsdp_stamp(raw: bytes | str | None) -> tuple[int, int] | None:
+    """Parse Verl's strict generation and width stamp without worker dependencies."""
+    if raw is None:
+        return None
+    try:
+        text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
+        decoded = json.loads(text, object_pairs_hook=_REJECT_DUPLICATE_CONFIG_KEYS)
+    except (UnicodeDecodeError, ValueError, TypeError):
+        return None
+    if not isinstance(decoded, dict):
+        return None
+    generation = _positive_int(decoded.get("FSDP_version"))
+    width = _positive_int(decoded.get("world_size"))
+    if generation is None or width is None or width > _MAX_NATIVE_FSDP_WORLD_SIZE:
+        return None
+    return generation, width
+
+
+def inspect_fsdp_checkpoint_manifest(
+    stamp_raw: bytes | str | None,
+    files: Iterable[FsdpManifestFile],
+    *,
+    expected_generation: FsdpGeneration,
+    expected_world_size: int | None,
+) -> FsdpCheckpointInspection:
+    """Validate the exact canonical native shard set for one stamped actor directory."""
+    stamp = parse_fsdp_stamp(stamp_raw)
+    if stamp is None:
+        return FsdpCheckpointInspection(False, None, None, "invalid or missing fsdp stamp")
+    generation, width = stamp
+    if generation != expected_generation:
+        return FsdpCheckpointInspection(
+            False,
+            generation,
+            width,
+            f"fsdp generation mismatch (stamped {generation}, expected {expected_generation})",
+        )
+    if expected_world_size is not None and width != expected_world_size:
+        return FsdpCheckpointInspection(
+            False,
+            generation,
+            width,
+            f"fsdp world-size mismatch (stamped {width}, expected {expected_world_size})",
+        )
+
+    actual: dict[str, int | None] = {}
+    for item in files:
+        name = item.name
+        if name in actual:
+            return FsdpCheckpointInspection(False, generation, width, "malformed shard name", name)
+        match = _SHARD_RE.fullmatch(name)
+        if match is None:
+            if any(name.startswith(f"{kind}_world_size_") for kind in _REQUIRED_SHARD_CLASSES):
+                return FsdpCheckpointInspection(
+                    False, generation, width, "malformed shard name", name
+                )
+            continue
+        _kind, width_text, rank_text = match.groups()
+        parsed_width = int(width_text)
+        parsed_rank = int(rank_text)
+        if width_text != str(parsed_width) or rank_text != str(parsed_rank):
+            return FsdpCheckpointInspection(False, generation, width, "malformed shard name", name)
+        actual[name] = item.size
+
+    expected = tuple(
+        f"{kind}_world_size_{width}_rank_{rank}.pt"
+        for kind in _REQUIRED_SHARD_CLASSES
+        for rank in range(width)
+    )
+    for name in expected:
+        if name not in actual:
+            return FsdpCheckpointInspection(False, generation, width, "missing shard", name)
+    extras = sorted(set(actual) - set(expected))
+    if extras:
+        return FsdpCheckpointInspection(
+            False,
+            generation,
+            width,
+            "malformed shard name",
+            extras[0],
+        )
+    for name in expected:
+        size = actual[name]
+        if size is not None and (isinstance(size, bool) or not isinstance(size, int) or size <= 0):
+            return FsdpCheckpointInspection(
+                False,
+                generation,
+                width,
+                "empty or unreadable shard",
+                name,
+            )
+    return FsdpCheckpointInspection(True, generation, width)

freesolo_flash_dev-1.2.116/flash/engine/verl_policy.py

@@ -0,0 +1,20 @@
+"""Private Verl strategy policy shared by control-plane and worker paths."""
+
+from __future__ import annotations
+
+from collections.abc import Sequence
+from typing import Literal
+
+FsdpGeneration = Literal[1, 2]
+
+
+def _resolve_fsdp_generation(
+    algorithm: str,
+    target_parameters: Sequence[str] | None,
+) -> FsdpGeneration:
+    """Return the only validated FSDP generation for one resolved workload."""
+    if algorithm == "sft" or algorithm == "grpo":
+        return 2
+    if algorithm == "opd":
+        return 2 if target_parameters else 1
+    raise ValueError(f"unsupported Verl algorithm for FSDP policy: {algorithm!r}")

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/flash/engine/worker/opd_train_runner.py

@@ -14,6 +14,7 @@ from typing import Any
 from flash.adapters.targets import resolve_lora_targeting
 from flash.core.catalog import get_model
 from flash.engine.plan.steps import rl_data_parallel_cards
+from flash.engine.verl_policy import _resolve_fsdp_generation
 from flash.engine.worker import opd_train as _opd_train
 from flash.engine.worker.train.core.child.runtime import TEXT_LORA_TARGET_SHIM
 from flash.engine.worker.train.core.prompt_rows import canonical_prompt_messages
@@ -171,6 +172,8 @@ def _prepare_workload(
     target_modules = targeting.target_modules
     if isinstance(target_modules, set | frozenset):
         target_modules = sorted(target_modules)
+    target_parameters = tuple(targeting.target_parameters) if targeting.target_parameters else None
+    fsdp_generation = _resolve_fsdp_generation("opd", target_parameters)
     lora_rank = int(lora_config.r)
     return _WorkloadState(
         prompts_per_step,
@@ -191,6 +194,8 @@ def _prepare_workload(
         lora_rank,
         int(lora_config.lora_alpha),
         target_modules,
+        target_parameters,
+        fsdp_generation,
         targeting.exclude_modules,
         _opd_train._warmstart_adapter_path(
             request.model_id,
@@ -238,9 +243,11 @@ def _materialize_child_files(
         workload.local_dir,
         prompt_pool_fingerprint=workload.prompt_pool_fingerprint,
         update_horizon=workload.update_horizon,
-        # the same count this attempt hands verl as n_gpus_per_node, which is the DATA-parallel
+        # the same count this attempt hands verl as n_gpus_per_node, which is the data-parallel
         # width: ulysses is pinned to 1, so every rank is a dp rank.
         world_size=gpu_count,
+        # native state is generation-specific, so use the same resolved policy as the child config.
+        expected_fsdp_generation=workload.fsdp_generation,
     )
     bridge = _opd_train._TeacherAlignmentBridge(
         prompts=prompt_state.prompts,
@@ -384,7 +391,8 @@ def _build_base_config(
         "lora_alpha": workload.lora_alpha,
         "target_modules": workload.target_modules,
         "exclude_modules": None,
-        "target_parameters": _opd_train._w.lora_target_parameters(request.model_id),
+        "target_parameters": workload.target_parameters,
+        "fsdp_generation": workload.fsdp_generation,
         "lora_adapter_path": workload.warmstart_adapter,
         "learning_rate": knobs.learning_rate,
         "local_dir": workload.local_dir,

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/flash/engine/worker/rl_train_runner.py

@@ -148,7 +148,11 @@ def _prepare_rl_files(inp, prompts):
     # width this attempt actually launches verl at. NOT the rented card count: the clamp that bounds
     # the launch by the step's sequences can make those differ, and a mismatch here silently discards
     # a loadable checkpoint and restarts the run from step 0.
-    resume_step = _rl_train()._restore_verl_resume(local_dir, world_size=int(inp["dp_cards"]))
+    resume_step = _rl_train()._restore_verl_resume(
+        local_dir,
+        world_size=int(inp["dp_cards"]),
+        expected_fsdp_generation=inp["fsdp_generation"],
+    )
     train_pq = os.path.join(workdir, "train.parquet")
     val_pq = os.path.join(workdir, "val.parquet")
     reward_py = os.path.join(workdir, "reward.py")

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/flash/engine/worker/sft_train.py

@@ -16,9 +16,11 @@ import time
 
 from flash._internal.diagnostics import SECRET_ENV_KEYS_ENV
 from flash.adapters.lora_rank import alpha_from_adapter_config, rank_from_adapter_config
+from flash.engine.verl_checkpoint import FsdpCheckpointInspection
+from flash.engine.verl_policy import FsdpGeneration
 from flash.engine.worker.runtime.kernel_warmup import KERNEL_CACHE_ENV_SUBDIRS
 from flash.engine.worker.runtime.pkg_proxy import W as _w
-from flash.engine.worker.verl.checkpoints import resume_checkpoint_is_loadable
+from flash.engine.worker.verl.checkpoints import inspect_resume_checkpoint
 from flash.engine.worker.verl.parallelism import ULYSSES_SEQUENCE_PARALLEL_SIZE
 
 # todo: run the two-gpu sft smoke on the exact runpod image and command assembled below.
@@ -126,7 +128,12 @@ def _verl_image_message_content(content) -> str:
     return "".join(parts)
 
 
-def _restore_verl_resume(local_dir: str, *, world_size: int) -> int:
+def _restore_verl_resume(
+    local_dir: str,
+    *,
+    world_size: int,
+    expected_fsdp_generation: FsdpGeneration,
+) -> int:
     """stage this run's streamed resume checkpoint; 0 when there is nothing usable to resume from.
 
     ``world_size`` is the rank count this attempt launches verl at; a checkpoint written at a
@@ -134,12 +141,30 @@ def _restore_verl_resume(local_dir: str, *, world_size: int) -> int:
     prefers a lower loadable checkpoint over a higher incompatible one, so a repeated discard cannot
     starve a compatible checkpoint uploaded after the one this attempt already rejected.
     """
-    resume = _w.hf_resume_checkpoint(
-        prefer=lambda path: resume_checkpoint_is_loadable(path, world_size=world_size)
-    )
+    inspections: dict[str, FsdpCheckpointInspection] = {}
+
+    def prefer(path: str) -> bool:
+        inspection = inspections.get(path)
+        if inspection is None:
+            inspection = inspect_resume_checkpoint(
+                path,
+                world_size=world_size,
+                expected_fsdp_generation=expected_fsdp_generation,
+            )
+            inspections[path] = inspection
+        return inspection.loadable
+
+    resume = _w.hf_resume_checkpoint(prefer=prefer)
     if not resume:
         return 0
-    return stage_verl_resume(resume, local_dir, job_label="SFT", world_size=world_size)
+    return stage_verl_resume(
+        resume,
+        local_dir,
+        job_label="SFT",
+        world_size=world_size,
+        expected_fsdp_generation=expected_fsdp_generation,
+        inspection=inspections.get(resume),
+    )
 
 
 def _durable_required_save_steps(required_steps: tuple[int, ...], resume_step: int) -> set[int]:

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/flash/engine/worker/sft_train_runner.py

@@ -17,6 +17,7 @@ from math import gcd
 from flash.adapters.targets import resolve_lora_targeting
 from flash.core.catalog import get_model
 from flash.engine.plan.steps import sft_data_parallel_cards, widest_usable_dp_width
+from flash.engine.verl_policy import _resolve_fsdp_generation
 from flash.engine.worker import sft_train as _sft_train
 from flash.engine.worker.train.core.child.runtime import TEXT_LORA_TARGET_SHIM
 from flash.engine.worker.verl.parallelism import ULYSSES_SEQUENCE_PARALLEL_SIZE
@@ -620,6 +621,8 @@ def _prepare_sft_child(
     # probe is part of the setup silence that wrap exists to cover, and because a packed run must
     # take the RAISING gate there rather than the soft form.
     world_size, micro_batch = _resolve_sft_width_and_micro_batch(options, data, model)
+    target_parameters = _w.lora_target_parameters(options.model_id)
+    fsdp_generation = _resolve_fsdp_generation("sft", target_parameters)
     config = {
         "train_files": data.train_file,
         "train_batch_size": model.train_batch_size,
@@ -632,7 +635,8 @@ def _prepare_sft_child(
         "lora_alpha": model.lora_alpha,
         "target_modules": model.target_modules,
         "exclude_modules": None,
-        "target_parameters": _w.lora_target_parameters(options.model_id),
+        "target_parameters": target_parameters,
+        "fsdp_generation": fsdp_generation,
         "lora_adapter_path": model.warmstart_adapter,
         "ulysses_sp_size": ULYSSES_SEQUENCE_PARALLEL_SIZE,
         "lr": options.learning_rate,
@@ -683,7 +687,11 @@ def _prepare_sft_child(
     # shards by data and the width is bounded by the batch and the row count, so the two differ
     # whenever either fails to divide the allocation -- passing gpu_count here would discard a
     # checkpoint that matches the run about to start, and keep one that does not.
-    resume_step = _sft_train._restore_verl_resume(options.paths.local_dir, world_size=world_size)
+    resume_step = _sft_train._restore_verl_resume(
+        options.paths.local_dir,
+        world_size=world_size,
+        expected_fsdp_generation=fsdp_generation,
+    )
     watcher = _sft_train._VerlCheckpointWatcher(
         local_dir=options.paths.local_dir,
         export_root=options.paths.export_root,

{freesolo_flash_dev-1.2.115 → freesolo_flash_dev-1.2.116}/flash/engine/worker/train/opd/failures.py

@@ -18,6 +18,7 @@ import shutil
 from dataclasses import dataclass
 from pathlib import Path
 
+from flash.engine.verl_policy import FsdpGeneration
 from flash.engine.worker.runtime.pkg_proxy import W as _w
 from flash.engine.worker.sft_train import (
     SHIM_FRAGMENT_FAILED_EXIT_CODE,
@@ -27,8 +28,7 @@ from flash.engine.worker.sft_train import (
 from flash.engine.worker.train.opd.bridge import _TeacherAlignmentBridge
 from flash.engine.worker.train.opd.child.bridge import _render_rollout_failure
 from flash.engine.worker.verl.checkpoints import (
-    checkpoint_world_size,
-    resume_checkpoint_is_loadable,
+    inspect_resume_checkpoint,
     resume_topology_matches,
     resume_upload_unavailable,
 )
@@ -364,6 +364,7 @@ def _restore_verl_resume(
     prompt_pool_fingerprint: str,
     update_horizon: int,
     world_size: int,
+    expected_fsdp_generation: FsdpGeneration,
 ) -> tuple[int, dict | None]:
     revision = _w.OPD_RESUME_REVISION or None
     # no `prefer`: opd pins an exact commit via OPD_RESUME_REVISION with fail_closed, and
@@ -382,19 +383,27 @@ def _restore_verl_resume(
         # a pinned revision means a prior attempt crossed optimizer.step(): the control-plane
         # gate (verify_opd_replacement_safe) authorized this replacement only to continue from
         # exactly this checkpoint. discarding it and restarting from step 0 would repeat
-        # already-billed teacher work and optimizer steps outside what the gate approved, so a
-        # topology mismatch fails closed here instead of falling through to a fresh run.
-        if not resume_checkpoint_is_loadable(resume, world_size=world_size):
-            written = checkpoint_world_size(resume)
+        # already-billed teacher work and optimizer steps outside what the gate approved, so an
+        # incompatible native checkpoint fails closed here instead of starting fresh.
+        inspection = inspect_resume_checkpoint(
+            resume,
+            world_size=world_size,
+            expected_fsdp_generation=expected_fsdp_generation,
+        )
+        if not inspection.loadable:
             raise RuntimeError(
                 f"permanent OPD resume failure: pinned resume revision {revision!r} names "
-                f"checkpoint {os.path.basename(resume)}, whose fsdp shards were written at "
-                f"world size {written if written is not None else 'unknown'} while this "
-                f"attempt runs at world size {world_size}. restarting from step 0 would "
-                "violate the pinned-resume contract, so this attempt refuses to train; "
-                "relaunch the retry at the checkpoint's gpu count."
+                f"checkpoint {os.path.basename(resume)}, rejected because "
+                f"{inspection.diagnostic()}. restarting from step 0 would violate the pinned-resume "
+                "contract, so this attempt refuses to train; relaunch from a compatible complete "
+                f"fsdp{expected_fsdp_generation} checkpoint."
             )
-    elif not resume_topology_matches(resume, world_size=world_size, job_label="OPD"):
+    elif not resume_topology_matches(
+        resume,
+        world_size=world_size,
+        expected_fsdp_generation=expected_fsdp_generation,
+        job_label="OPD",
+    ):
         return 0, None
     with open(os.path.join(resume, "opd_state.json"), encoding="utf-8") as file:
         state = validate_opd_resume_state_metadata(